testlib: Clarify that our XUnit reporter is actually a JUnit reporter

The reporter was probably named 'xunit' based on the historical use of
xUnit to refer to testing frameworks derived from Smalltalk's SUnit.
These frameworks typically added their own prefix, e.g. JUnit for Java,
RUnit for R, etc.

The most popular of these was the JUnit framework, and the corresponding
XML output produced by the Ant built tool became somewhat of a de facto
standard, which is probably why we chose to model our reporter after it.

Nowadays however, naming it 'xunit' is problematic as there is actually
a testing famework named xUnit.net, typically shortened to, you guessed
it: xunit.

Test report consumers will typically have a junit mode, and an xunit
mode, and the latter could easily be mistaken for what testlib outputs,
unless we clarify this.

The clarification also allows us to safely extend our support for the
JUnit XML format to incorporate some elements that are nowadays common,
but where we are lagging behind the standard.

[ChangeLog][QTestLib] The formerly named 'xunitxml' test reporter has
been renamed to what it actually is: a JUnit test reporter, and is now
triggered by passing -o junitxml to the test binary.

Change-Id: Ieb20d3d2b5905c74e55b98174948cc70870c0ef9
Reviewed-by: Friedemann Kleint <Friedemann.Kleint@qt.io>
Reviewed-by: Simon Hausmann <simon.hausmann@qt.io>
bb10
Tor Arne Vestbø 2020-01-22 16:43:40 +01:00
parent 75ad13d2bc
commit 27db9e458c
61 changed files with 91 additions and 84 deletions

View File

@ -237,7 +237,7 @@
\list
\li \c -o \e{filename,format} \br
Writes output to the specified file, in the specified format (one of
\c txt, \c xml, \c lightxml, \c xunitxml or \c tap). The special filename \c -
\c txt, \c xml, \c lightxml, \c junitxml or \c tap). The special filename \c -
may be used to log to standard output.
\li \c -o \e filename \br
Writes output to the specified file.
@ -247,8 +247,8 @@
Outputs results as an XML document.
\li \c -lightxml \br
Outputs results as a stream of XML tags.
\li \c -xunitxml \br
Outputs results as an Xunit XML document.
\li \c -junitxml \br
Outputs results as an JUnit XML document.
\li \c -csv \br
Outputs results as comma-separated values (CSV). This mode is only suitable for
benchmarks, since it suppresses normal pass/fail messages.
@ -264,7 +264,7 @@
If the first version of the \c -o option is used, neither the second version
of the \c -o option nor the \c -txt, \c -xml, \c -lightxml, \c -teamcity,
\c -xunitxml or \c -tap options should be used.
\c -junitxml or \c -tap options should be used.
If neither version of the \c -o option is used, test results will be logged to
standard output. If no format option is used, test results will be logged in

View File

@ -37,9 +37,9 @@
**
****************************************************************************/
#include <QtTest/private/qxunittestlogger_p.h>
#include <QtTest/private/qjunittestlogger_p.h>
#include <QtTest/private/qtestelement_p.h>
#include <QtTest/private/qtestxunitstreamer_p.h>
#include <QtTest/private/qtestjunitstreamer_p.h>
#include <QtTest/qtestcase.h>
#include <QtTest/private/qtestresult_p.h>
#include <QtTest/private/qbenchmark_p.h>
@ -57,27 +57,27 @@
QT_BEGIN_NAMESPACE
QXunitTestLogger::QXunitTestLogger(const char *filename)
QJUnitTestLogger::QJUnitTestLogger(const char *filename)
: QAbstractTestLogger(filename)
{
}
QXunitTestLogger::~QXunitTestLogger()
QJUnitTestLogger::~QJUnitTestLogger()
{
delete currentLogElement;
delete logFormatter;
}
void QXunitTestLogger::startLogging()
void QJUnitTestLogger::startLogging()
{
QAbstractTestLogger::startLogging();
logFormatter = new QTestXunitStreamer(this);
logFormatter = new QTestJUnitStreamer(this);
delete errorLogElement;
errorLogElement = new QTestElement(QTest::LET_SystemError);
}
void QXunitTestLogger::stopLogging()
void QJUnitTestLogger::stopLogging()
{
QTestElement *iterator = listOfTestcases;
@ -132,7 +132,7 @@ void QXunitTestLogger::stopLogging()
QAbstractTestLogger::stopLogging();
}
void QXunitTestLogger::enterTestFunction(const char *function)
void QJUnitTestLogger::enterTestFunction(const char *function)
{
currentLogElement = new QTestElement(QTest::LET_TestCase);
currentLogElement->addAttribute(QTest::AI_Name, function);
@ -141,11 +141,11 @@ void QXunitTestLogger::enterTestFunction(const char *function)
++testCounter;
}
void QXunitTestLogger::leaveTestFunction()
void QJUnitTestLogger::leaveTestFunction()
{
}
void QXunitTestLogger::addIncident(IncidentTypes type, const char *description,
void QJUnitTestLogger::addIncident(IncidentTypes type, const char *description,
const char *file, int line)
{
const char *typeBuf = nullptr;
@ -242,15 +242,15 @@ void QXunitTestLogger::addIncident(IncidentTypes type, const char *description,
currentLogElement->addAttribute(QTest::AI_Line, buf);
/*
Since XFAIL does not add a failure to the testlog in xunitxml, add a message, so we still
Since XFAIL does not add a failure to the testlog in junitxml, add a message, so we still
have some information about the expected failure.
*/
if (type == QAbstractTestLogger::XFail) {
QXunitTestLogger::addMessage(QAbstractTestLogger::Info, QString::fromUtf8(description), file, line);
QJUnitTestLogger::addMessage(QAbstractTestLogger::Info, QString::fromUtf8(description), file, line);
}
}
void QXunitTestLogger::addBenchmarkResult(const QBenchmarkResult &result)
void QJUnitTestLogger::addBenchmarkResult(const QBenchmarkResult &result)
{
QTestElement *benchmarkElement = new QTestElement(QTest::LET_Benchmark);
@ -268,7 +268,7 @@ void QXunitTestLogger::addBenchmarkResult(const QBenchmarkResult &result)
currentLogElement->addLogElement(benchmarkElement);
}
void QXunitTestLogger::addTag(QTestElement* element)
void QJUnitTestLogger::addTag(QTestElement* element)
{
const char *tag = QTestResult::currentDataTag();
const char *gtag = QTestResult::currentGlobalDataTag();
@ -289,7 +289,7 @@ void QXunitTestLogger::addTag(QTestElement* element)
element->addAttribute(QTest::AI_Tag, buf.constData());
}
void QXunitTestLogger::addMessage(MessageTypes type, const QString &message, const char *file, int line)
void QJUnitTestLogger::addMessage(MessageTypes type, const QString &message, const char *file, int line)
{
QTestElement *errorElement = new QTestElement(QTest::LET_Error);
const char *typeBuf = nullptr;

View File

@ -37,8 +37,8 @@
**
****************************************************************************/
#ifndef QXUNITTESTLOGGER_P_H
#define QXUNITTESTLOGGER_P_H
#ifndef QJUNITTESTLOGGER_P_H
#define QJUNITTESTLOGGER_P_H
//
// W A R N I N G
@ -55,14 +55,14 @@
QT_BEGIN_NAMESPACE
class QTestXunitStreamer;
class QTestJUnitStreamer;
class QTestElement;
class QXunitTestLogger : public QAbstractTestLogger
class QJUnitTestLogger : public QAbstractTestLogger
{
public:
QXunitTestLogger(const char *filename);
~QXunitTestLogger();
QJUnitTestLogger(const char *filename);
~QJUnitTestLogger();
void startLogging() override;
void stopLogging() override;
@ -82,7 +82,7 @@ class QXunitTestLogger : public QAbstractTestLogger
QTestElement *listOfTestcases = nullptr;
QTestElement *currentLogElement = nullptr;
QTestElement *errorLogElement = nullptr;
QTestXunitStreamer *logFormatter = nullptr;
QTestJUnitStreamer *logFormatter = nullptr;
int testCounter = 0;
int failureCounter = 0;
@ -91,4 +91,4 @@ class QXunitTestLogger : public QAbstractTestLogger
QT_END_NAMESPACE
#endif // QXUNITTESTLOGGER_P_H
#endif // QJUNITTESTLOGGER_P_H

View File

@ -541,7 +541,7 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, const char *const argv[], bool
" Valid formats are:\n"
" txt : Plain text\n"
" csv : CSV format (suitable for benchmarks)\n"
" xunitxml : XML XUnit document\n"
" junitxml : XML JUnit document\n"
" xml : XML document\n"
" lightxml : A stream of XML tags\n"
" teamcity : TeamCity format\n"
@ -553,7 +553,7 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, const char *const argv[], bool
" -o filename : Write the output into file\n"
" -txt : Output results in Plain Text\n"
" -csv : Output results in a CSV format (suitable for benchmarks)\n"
" -xunitxml : Output results as XML XUnit document\n"
" -junitxml : Output results as XML JUnit document\n"
" -xml : Output results as XML document\n"
" -lightxml : Output results as stream of XML tags\n"
" -teamcity : Output results in TeamCity format\n"
@ -637,8 +637,8 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, const char *const argv[], bool
logFormat = QTestLog::Plain;
} else if (strcmp(argv[i], "-csv") == 0) {
logFormat = QTestLog::CSV;
} else if (strcmp(argv[i], "-xunitxml") == 0) {
logFormat = QTestLog::XunitXML;
} else if (strcmp(argv[i], "-junitxml") == 0 || strcmp(argv[i], "-xunitxml") == 0) {
logFormat = QTestLog::JUnitXML;
} else if (strcmp(argv[i], "-xml") == 0) {
logFormat = QTestLog::XML;
} else if (strcmp(argv[i], "-lightxml") == 0) {
@ -677,14 +677,14 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, const char *const argv[], bool
logFormat = QTestLog::LightXML;
else if (strcmp(format, "xml") == 0)
logFormat = QTestLog::XML;
else if (strcmp(format, "xunitxml") == 0)
logFormat = QTestLog::XunitXML;
else if (strcmp(format, "junitxml") == 0 || strcmp(format, "xunitxml") == 0)
logFormat = QTestLog::JUnitXML;
else if (strcmp(format, "teamcity") == 0)
logFormat = QTestLog::TeamCity;
else if (strcmp(format, "tap") == 0)
logFormat = QTestLog::TAP;
else {
fprintf(stderr, "output format must be one of txt, csv, lightxml, xml, tap, teamcity or xunitxml\n");
fprintf(stderr, "output format must be one of txt, csv, lightxml, xml, tap, teamcity or junitxml\n");
exit(1);
}
if (strcmp(filename, "-") == 0 && QTestLog::loggerUsingStdout()) {

View File

@ -37,8 +37,8 @@
**
****************************************************************************/
#include <QtTest/private/qtestxunitstreamer_p.h>
#include <QtTest/private/qxunittestlogger_p.h>
#include <QtTest/private/qtestjunitstreamer_p.h>
#include <QtTest/private/qjunittestlogger_p.h>
#include <QtTest/private/qtestelement_p.h>
#include <QtTest/private/qtestelementattribute_p.h>
#include <QtTest/qtestassert.h>
@ -48,15 +48,15 @@
QT_BEGIN_NAMESPACE
QTestXunitStreamer::QTestXunitStreamer(QXunitTestLogger *logger)
QTestJUnitStreamer::QTestJUnitStreamer(QJUnitTestLogger *logger)
: testLogger(logger)
{
QTEST_ASSERT(testLogger);
}
QTestXunitStreamer::~QTestXunitStreamer() = default;
QTestJUnitStreamer::~QTestJUnitStreamer() = default;
void QTestXunitStreamer::indentForElement(const QTestElement* element, char* buf, int size)
void QTestJUnitStreamer::indentForElement(const QTestElement* element, char* buf, int size)
{
if (size == 0) return;
@ -74,7 +74,7 @@ void QTestXunitStreamer::indentForElement(const QTestElement* element, char* buf
}
}
void QTestXunitStreamer::formatStart(const QTestElement *element, QTestCharBuffer *formatted) const
void QTestJUnitStreamer::formatStart(const QTestElement *element, QTestCharBuffer *formatted) const
{
if (!element || !formatted )
return;
@ -95,7 +95,7 @@ void QTestXunitStreamer::formatStart(const QTestElement *element, QTestCharBuffe
QTest::qt_asprintf(formatted, "%s<%s", indent, element->elementName());
}
void QTestXunitStreamer::formatEnd(const QTestElement *element, QTestCharBuffer *formatted) const
void QTestJUnitStreamer::formatEnd(const QTestElement *element, QTestCharBuffer *formatted) const
{
if (!element || !formatted )
return;
@ -111,7 +111,7 @@ void QTestXunitStreamer::formatEnd(const QTestElement *element, QTestCharBuffer
QTest::qt_asprintf(formatted, "%s</%s>\n", indent, element->elementName());
}
void QTestXunitStreamer::formatAttributes(const QTestElement* element, const QTestElementAttribute *attribute, QTestCharBuffer *formatted) const
void QTestJUnitStreamer::formatAttributes(const QTestElement* element, const QTestElementAttribute *attribute, QTestCharBuffer *formatted) const
{
if (!attribute || !formatted )
return;
@ -143,7 +143,7 @@ void QTestXunitStreamer::formatAttributes(const QTestElement* element, const QTe
}
}
void QTestXunitStreamer::formatAfterAttributes(const QTestElement *element, QTestCharBuffer *formatted) const
void QTestJUnitStreamer::formatAfterAttributes(const QTestElement *element, QTestCharBuffer *formatted) const
{
if (!element || !formatted )
return;
@ -164,7 +164,7 @@ void QTestXunitStreamer::formatAfterAttributes(const QTestElement *element, QTes
QTest::qt_asprintf(formatted, ">\n");
}
void QTestXunitStreamer::output(QTestElement *element) const
void QTestJUnitStreamer::output(QTestElement *element) const
{
QTEST_ASSERT(element);
@ -172,7 +172,7 @@ void QTestXunitStreamer::output(QTestElement *element) const
outputElements(element);
}
void QTestXunitStreamer::outputElements(QTestElement *element, bool) const
void QTestJUnitStreamer::outputElements(QTestElement *element, bool) const
{
QTestCharBuffer buf;
bool hasChildren;
@ -205,7 +205,7 @@ void QTestXunitStreamer::outputElements(QTestElement *element, bool) const
}
}
void QTestXunitStreamer::outputElementAttributes(const QTestElement* element, QTestElementAttribute *attribute) const
void QTestJUnitStreamer::outputElementAttributes(const QTestElement* element, QTestElementAttribute *attribute) const
{
QTestCharBuffer buf;
while (attribute) {
@ -215,7 +215,7 @@ void QTestXunitStreamer::outputElementAttributes(const QTestElement* element, QT
}
}
void QTestXunitStreamer::outputString(const char *msg) const
void QTestJUnitStreamer::outputString(const char *msg) const
{
testLogger->outputString(msg);
}

View File

@ -37,8 +37,8 @@
**
****************************************************************************/
#ifndef QTESTXUNITSTREAMER_P_H
#define QTESTXUNITSTREAMER_P_H
#ifndef QTESTJUNITSTREAMER_P_H
#define QTESTJUNITSTREAMER_P_H
//
// W A R N I N G
@ -58,14 +58,14 @@ QT_BEGIN_NAMESPACE
class QTestElement;
class QTestElementAttribute;
class QXunitTestLogger;
class QJUnitTestLogger;
struct QTestCharBuffer;
class QTestXunitStreamer
class QTestJUnitStreamer
{
public:
QTestXunitStreamer(QXunitTestLogger *logger);
~QTestXunitStreamer();
QTestJUnitStreamer(QJUnitTestLogger *logger);
~QTestJUnitStreamer();
void formatStart(const QTestElement *element, QTestCharBuffer *formatted) const;
void formatEnd(const QTestElement *element, QTestCharBuffer *formatted) const;
@ -78,10 +78,9 @@ class QTestXunitStreamer
void outputString(const char *msg) const;
private:
void displayXunitXmlHeader() const;
static void indentForElement(const QTestElement* element, char* buf, int size);
QXunitTestLogger *testLogger;
QJUnitTestLogger *testLogger;
};
QT_END_NAMESPACE

View File

@ -44,7 +44,7 @@
#include <QtTest/private/qabstracttestlogger_p.h>
#include <QtTest/private/qplaintestlogger_p.h>
#include <QtTest/private/qcsvbenchmarklogger_p.h>
#include <QtTest/private/qxunittestlogger_p.h>
#include <QtTest/private/qjunittestlogger_p.h>
#include <QtTest/private/qxmltestlogger_p.h>
#include <QtTest/private/qteamcitylogger_p.h>
#include <QtTest/private/qtaptestlogger_p.h>
@ -456,8 +456,8 @@ void QTestLog::addLogger(LogMode mode, const char *filename)
case QTestLog::LightXML:
logger = new QXmlTestLogger(QXmlTestLogger::Light, filename);
break;
case QTestLog::XunitXML:
logger = new QXunitTestLogger(filename);
case QTestLog::JUnitXML:
logger = new QJUnitTestLogger(filename);
break;
case QTestLog::TeamCity:
logger = new QTeamCityLogger(filename);

View File

@ -71,7 +71,7 @@ public:
Q_DISABLE_COPY_MOVE(QTestLog)
enum LogMode {
Plain = 0, XML, LightXML, XunitXML, CSV, TeamCity, TAP
Plain = 0, XML, LightXML, JUnitXML, CSV, TeamCity, TAP
#if defined(QT_USE_APPLE_UNIFIED_LOGGING)
, Apple
#endif

View File

@ -51,10 +51,10 @@ HEADERS = \
qtestblacklist_p.h \
qtesthelpers_p.h \
qttestglobal.h \
qtestxunitstreamer_p.h \
qtestjunitstreamer_p.h \
qtaptestlogger_p.h \
qxmltestlogger_p.h \
qxunittestlogger_p.h
qjunittestlogger_p.h
SOURCES = \
qtestcase.cpp \
@ -77,8 +77,8 @@ SOURCES = \
qtestelement.cpp \
qtestelementattribute.cpp \
qtestmouse.cpp \
qtestxunitstreamer.cpp \
qxunittestlogger.cpp \
qtestjunitstreamer.cpp \
qjunittestlogger.cpp \
qtestblacklist.cpp \
qtaptestlogger.cpp

View File

@ -86,7 +86,7 @@ class Cleaner (object):
# Build details:
(r'(Config: Using QtTest library).*', r'\1'), # txt
(r'( *<QtBuild)>[^<]+</QtBuild>', r'\1/>'), # xml, lightxml
(r'(<property value=")[^"]+(" name="QtBuild"/>)', r'\1\2'), # xunitxml
(r'(<property value=")[^"]+(" name="QtBuild"/>)', r'\1\2'), # junitxml
# Line numbers in source files:
(r'(ASSERT: ".*" in file .*, line) \d+', r'\1 0'), # lightxml
(r'(Loc: \[[^[\]()]+)\(\d+\)', r'\1(0)'), # txt
@ -295,7 +295,7 @@ def testEnv(testname,
return data
def generateTestData(testname, clean,
formats = ('xml', 'txt', 'xunitxml', 'lightxml', 'teamcity', 'tap'),
formats = ('xml', 'txt', 'junitxml', 'lightxml', 'teamcity', 'tap'),
# Make sure this matches tst_Selftests::runSubTest_data():
extraArgs = {
"commandlinedata": "fiveTablePasses fiveTablePasses:fiveTablePasses_data1 -v2",

View File

@ -280,13 +280,17 @@ QList<LoggerSet> tst_Selftests::allLoggerSets() const
QStringList() << "xml",
QStringList() << "-xml" << "-o" << logName("xml")
)
<< LoggerSet("old stdout xunitxml",
QStringList() << "stdout xunitxml",
QStringList() << "-xunitxml"
<< LoggerSet("old stdout junitxml",
QStringList() << "stdout junitxml",
QStringList() << "-junitxml"
)
<< LoggerSet("old xunitxml",
QStringList() << "xunitxml",
QStringList() << "-xunitxml" << "-o" << logName("xunitxml")
<< LoggerSet("old junitxml",
QStringList() << "junitxml",
QStringList() << "-junitxml" << "-o" << logName("junitxml")
)
<< LoggerSet("old xunitxml compatibility",
QStringList() << "junitxml",
QStringList() << "-xunitxml" << "-o" << logName("junitxml")
)
<< LoggerSet("old stdout lightxml",
QStringList() << "stdout lightxml",
@ -335,13 +339,17 @@ QList<LoggerSet> tst_Selftests::allLoggerSets() const
QStringList() << "xml",
QStringList() << "-o" << logName("xml")+",xml"
)
<< LoggerSet("new stdout xunitxml",
QStringList() << "stdout xunitxml",
<< LoggerSet("new stdout junitxml",
QStringList() << "stdout junitxml",
QStringList() << "-o" << "-,junitxml"
)
<< LoggerSet("new stdout xunitxml compatibility",
QStringList() << "stdout junitxml",
QStringList() << "-o" << "-,xunitxml"
)
<< LoggerSet("new xunitxml",
QStringList() << "xunitxml",
QStringList() << "-o" << logName("xunitxml")+",xunitxml"
<< LoggerSet("new junitxml",
QStringList() << "junitxml",
QStringList() << "-o" << logName("junitxml")+",junitxml"
)
<< LoggerSet("new stdout lightxml",
QStringList() << "stdout lightxml",
@ -384,24 +392,24 @@ QList<LoggerSet> tst_Selftests::allLoggerSets() const
QStringList() << "-o" << logName("xml")+",xml"
<< "-o" << "-,txt"
)
<< LoggerSet("txt + xunitxml",
QStringList() << "txt" << "xunitxml",
<< LoggerSet("txt + junitxml",
QStringList() << "txt" << "junitxml",
QStringList() << "-o" << logName("txt")+",txt"
<< "-o" << logName("xunitxml")+",xunitxml"
<< "-o" << logName("junitxml")+",junitxml"
)
<< LoggerSet("lightxml + stdout xunitxml",
QStringList() << "lightxml" << "stdout xunitxml",
<< LoggerSet("lightxml + stdout junitxml",
QStringList() << "lightxml" << "stdout junitxml",
QStringList() << "-o" << logName("lightxml")+",lightxml"
<< "-o" << "-,xunitxml"
<< "-o" << "-,junitxml"
)
// All loggers at the same time (except csv)
<< LoggerSet("all loggers",
QStringList() << "txt" << "xml" << "lightxml" << "stdout txt" << "xunitxml" << "tap",
QStringList() << "txt" << "xml" << "lightxml" << "stdout txt" << "junitxml" << "tap",
QStringList() << "-o" << logName("txt")+",txt"
<< "-o" << logName("xml")+",xml"
<< "-o" << logName("lightxml")+",lightxml"
<< "-o" << "-,txt"
<< "-o" << logName("xunitxml")+",xunitxml"
<< "-o" << logName("junitxml")+",junitxml"
<< "-o" << logName("teamcity")+",teamcity"
<< "-o" << logName("tap")+",tap"
)