From 973e74399e807fb6f351a7d330c8c8b85b66bc5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tor=20Arne=20Vestb=C3=B8?= Date: Thu, 29 Jul 2021 09:59:59 +0200 Subject: [PATCH] testlib: Improve JUnit XML conformance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The JUnit test framework did not initially have any XML reporting facilities built in. Instead, the XML report was generated by the Apache Ant JUnit task: https://github.com/apache/ant/search?q=filename%3AXMLJUnitResultFormatter.java Many users interacted with these reports via the Jenkins JUnit plugin, which provided graphical visualization of the test results: https://plugins.jenkins.io/junit/ Due to the lack of an official XML schema for the Apache Ant JUnit report there was some confusion about what the actual format was. People started documenting the de-facto format, both as produced by Ant, and as consumed by Jenkins: https://github.com/windyroad/JUnit-Schema/blob/master/JUnit.xsd https://github.com/junit-team/junit5/search?q=filename%3Ajenkins-junit.xsd The XML produced by the Qt Test JUnit reporter was far from these schemas, causing issues when importing results into tools such as Jenkins, Allure2, or Test Center. The following changes have been made to improve conformance: - The 'timestamp' attribute on is is now in ISO 8601 local time, without any time zone specified - The 'hostname' attribute on is now included - The 'classname' attribute on is now included - The non-standard 'result' attribute on has been removed - The non-standard 'result' attribute on has been renamed to 'type' - The element on is always included, even when empty - The non-standard 'tag' attribute on has been removed. Data-driven tests are now represented as individual elements, e.g.: The resulting XML validates against both the de-facto Apache Ant 'JUnit 4' schema and the Jenkins JUnit plugin schema. Task-number: QTBUG-95424 Change-Id: I6fc9abedbfb319f2545b99b37d059b18c16776ff Reviewed-by: Mårten Nordheim --- src/testlib/qjunittestlogger.cpp | 116 ++++--- src/testlib/qjunittestlogger_p.h | 7 +- src/testlib/qtestelementattribute.cpp | 4 +- src/testlib/qtestelementattribute_p.h | 4 +- .../selftests/expected_assert.junitxml | 11 +- .../selftests/expected_badxml.junitxml | 68 ++-- .../expected_benchlibcallgrind.junitxml | 8 +- .../expected_benchlibcounting.junitxml | 14 +- .../expected_benchlibeventcounter.junitxml | 15 +- .../expected_benchliboptions.junitxml | 27 +- .../expected_benchlibtickcounter.junitxml | 9 +- .../expected_benchlibwalltime.junitxml | 13 +- .../selftests/expected_blacklisted.junitxml | 16 +- .../selftests/expected_cmptest.junitxml | 292 +++++++++++------- .../expected_commandlinedata.junitxml | 31 +- .../selftests/expected_counting.junitxml | 89 ++++-- .../selftests/expected_datatable.junitxml | 81 +++-- .../selftests/expected_datetime.junitxml | 27 +- .../expected_exceptionthrow.junitxml | 9 +- .../selftests/expected_expectfail.junitxml | 68 ++-- .../selftests/expected_failcleanup.junitxml | 11 +- .../selftests/expected_faildatatype.junitxml | 8 +- .../selftests/expected_failfetchtype.junitxml | 11 +- .../selftests/expected_failinit.junitxml | 9 +- .../selftests/expected_failinitdata.junitxml | 7 +- .../selftests/expected_fetchbogus.junitxml | 11 +- .../selftests/expected_findtestdata.junitxml | 9 +- .../selftests/expected_globaldata.junitxml | 114 ++++--- .../testlib/selftests/expected_junit.junitxml | 26 +- .../selftests/expected_keyboard.junitxml | 9 +- .../selftests/expected_longstring.junitxml | 11 +- .../selftests/expected_maxwarnings.junitxml | 8 +- .../selftests/expected_multiexec.junitxml | 45 +-- .../expected_pairdiagnostics.junitxml | 15 +- .../testlib/selftests/expected_pass.junitxml | 13 +- .../expected_qexecstringlist.junitxml | 15 +- .../selftests/expected_signaldumper.junitxml | 262 ++++++++-------- .../selftests/expected_silent.junitxml | 22 +- .../selftests/expected_singleskip.junitxml | 8 +- .../testlib/selftests/expected_skip.junitxml | 18 +- .../selftests/expected_skipcleanup.junitxml | 8 +- .../selftests/expected_skipinit.junitxml | 6 +- .../selftests/expected_skipinitdata.junitxml | 4 +- .../testlib/selftests/expected_sleep.junitxml | 11 +- .../selftests/expected_strcmp.junitxml | 28 +- .../selftests/expected_subtest.junitxml | 76 +++-- .../selftests/expected_testlib.junitxml | 20 +- .../expected_tuplediagnostics.junitxml | 17 +- .../selftests/expected_verbose1.junitxml | 89 ++++-- .../selftests/expected_verbose2.junitxml | 130 ++++---- .../expected_verifyexceptionthrown.junitxml | 37 +-- .../selftests/expected_warnings.junitxml | 32 +- .../selftests/expected_watchdog.junitxml | 9 +- .../selftests/generate_expected_output.py | 1 + .../auto/testlib/selftests/tst_selftests.cpp | 2 + 55 files changed, 1154 insertions(+), 857 deletions(-) diff --git a/src/testlib/qjunittestlogger.cpp b/src/testlib/qjunittestlogger.cpp index 6374eb0f9d..a34b431060 100644 --- a/src/testlib/qjunittestlogger.cpp +++ b/src/testlib/qjunittestlogger.cpp @@ -69,6 +69,14 @@ QJUnitTestLogger::~QJUnitTestLogger() delete logFormatter; } +// We track test timing per test case, so we +// need to maintain our own elapsed timer. +static QElapsedTimer elapsedTestcaseTime; +static qreal elapsedTestCaseSeconds() +{ + return elapsedTestcaseTime.nsecsElapsed() / 1e9; +} + void QJUnitTestLogger::startLogging() { QAbstractTestLogger::startLogging(); @@ -84,9 +92,11 @@ void QJUnitTestLogger::startLogging() currentTestSuite->addAttribute(QTest::AI_Name, QTestResult::currentTestObjectName()); auto localTime = QDateTime::currentDateTime(); - auto localTimeWithUtcOffset = localTime.toOffsetFromUtc(localTime.offsetFromUtc()); currentTestSuite->addAttribute(QTest::AI_Timestamp, - localTimeWithUtcOffset.toString(Qt::ISODate).toUtf8().constData()); + localTime.toString(Qt::ISODate).toUtf8().constData()); + + currentTestSuite->addAttribute(QTest::AI_Hostname, + QSysInfo::machineHostName().toUtf8().constData()); QTestElement *property; QTestElement *properties = new QTestElement(QTest::LET_Properties); @@ -107,6 +117,8 @@ void QJUnitTestLogger::startLogging() properties->addLogElement(property); currentTestSuite->addLogElement(properties); + + elapsedTestcaseTime.start(); } void QJUnitTestLogger::stopLogging() @@ -134,8 +146,7 @@ void QJUnitTestLogger::stopLogging() testcase = testcase->nextElement(); } - if (systemOutputElement->childElements()) - currentTestSuite->addLogElement(systemOutputElement); + currentTestSuite->addLogElement(systemOutputElement); currentTestSuite->addLogElement(systemErrorElement); logFormatter->output(currentTestSuite); @@ -147,20 +158,54 @@ void QJUnitTestLogger::stopLogging() } void QJUnitTestLogger::enterTestFunction(const char *function) +{ + enterTestCase(function); +} + +void QJUnitTestLogger::enterTestCase(const char *name) { currentLogElement = new QTestElement(QTest::LET_TestCase); - currentLogElement->addAttribute(QTest::AI_Name, function); + currentLogElement->addAttribute(QTest::AI_Name, name); + currentLogElement->addAttribute(QTest::AI_Classname, QTestResult::currentTestObjectName()); currentLogElement->addToList(&listOfTestcases); // The element will be deleted when the suite is deleted ++testCounter; + + elapsedTestcaseTime.restart(); +} + +void QJUnitTestLogger::enterTestData(QTestData *) +{ + QTestCharBuffer testIdentifier; + QTestPrivate::generateTestIdentifier(&testIdentifier, + QTestPrivate::TestFunction | QTestPrivate::TestDataTag); + + static const char *lastTestFunction = nullptr; + if (QTestResult::currentTestFunction() != lastTestFunction) { + // Adopt existing testcase for the initial test data + auto *name = const_cast( + currentLogElement->attribute(QTest::AI_Name)); + name->setPair(QTest::AI_Name, testIdentifier.data()); + lastTestFunction = QTestResult::currentTestFunction(); + elapsedTestcaseTime.restart(); + } else { + // Create new test cases for remaining test data + leaveTestCase(); + enterTestCase(testIdentifier.data()); + } } void QJUnitTestLogger::leaveTestFunction() +{ + leaveTestCase(); +} + +void QJUnitTestLogger::leaveTestCase() { currentLogElement->addAttribute(QTest::AI_Time, - QByteArray::number(QTestLog::msecsFunctionTime() / 1000, 'f').constData()); + QByteArray::number(elapsedTestCaseSeconds(), 'f').constData()); } void QJUnitTestLogger::addIncident(IncidentTypes type, const char *description, @@ -204,46 +249,11 @@ void QJUnitTestLogger::addIncident(IncidentTypes type, const char *description, if (type == QAbstractTestLogger::Fail || type == QAbstractTestLogger::XPass) { QTestElement *failureElement = new QTestElement(QTest::LET_Failure); - failureElement->addAttribute(QTest::AI_Result, typeBuf); + failureElement->addAttribute(QTest::AI_Type, typeBuf); failureElement->addAttribute(QTest::AI_Message, description); - addTag(failureElement); currentLogElement->addLogElement(failureElement); } - /* - Only one result can be shown for the whole testfunction. - Check if we currently have a result, and if so, overwrite it - iff the new result is worse. - */ - QTestElementAttribute* resultAttr = - const_cast(currentLogElement->attribute(QTest::AI_Result)); - if (resultAttr) { - const char* oldResult = resultAttr->value(); - bool overwrite = false; - if (!strcmp(oldResult, "pass")) { - overwrite = true; - } - else if (!strcmp(oldResult, "bpass") || !strcmp(oldResult, "bxfail")) { - overwrite = (type == QAbstractTestLogger::XPass || type == QAbstractTestLogger::Fail) || (type == QAbstractTestLogger::XFail) - || (type == QAbstractTestLogger::BlacklistedFail) || (type == QAbstractTestLogger::BlacklistedXPass); - } - else if (!strcmp(oldResult, "bfail") || !strcmp(oldResult, "bxpass")) { - overwrite = (type == QAbstractTestLogger::XPass || type == QAbstractTestLogger::Fail) || (type == QAbstractTestLogger::XFail); - } - else if (!strcmp(oldResult, "xfail")) { - overwrite = (type == QAbstractTestLogger::XPass || type == QAbstractTestLogger::Fail); - } - else if (!strcmp(oldResult, "xpass")) { - overwrite = (type == QAbstractTestLogger::Fail); - } - if (overwrite) { - resultAttr->setPair(QTest::AI_Result, typeBuf); - } - } - else { - currentLogElement->addAttribute(QTest::AI_Result, typeBuf); - } - /* Since XFAIL does not add a failure to the testlog in junitxml, add a message, so we still have some information about the expected failure. @@ -253,27 +263,6 @@ void QJUnitTestLogger::addIncident(IncidentTypes type, const char *description, } } -void QJUnitTestLogger::addTag(QTestElement* element) -{ - const char *tag = QTestResult::currentDataTag(); - const char *gtag = QTestResult::currentGlobalDataTag(); - const char *filler = (tag && gtag) ? ":" : ""; - if ((!tag || !tag[0]) && (!gtag || !gtag[0])) { - return; - } - - if (!tag) { - tag = ""; - } - if (!gtag) { - gtag = ""; - } - - QTestCharBuffer buf; - QTest::qt_asprintf(&buf, "%s%s%s", gtag, filler, tag); - element->addAttribute(QTest::AI_Tag, buf.constData()); -} - void QJUnitTestLogger::addMessage(MessageTypes type, const QString &message, const char *file, int line) { Q_UNUSED(file); @@ -318,7 +307,6 @@ void QJUnitTestLogger::addMessage(MessageTypes type, const QString &message, con messageElement->addAttribute(QTest::AI_Type, typeBuf); messageElement->addAttribute(QTest::AI_Message, message.toUtf8().constData()); - addTag(messageElement); currentLogElement->addLogElement(messageElement); ++errorCounter; diff --git a/src/testlib/qjunittestlogger_p.h b/src/testlib/qjunittestlogger_p.h index 30de0b2eb0..4c5f05e885 100644 --- a/src/testlib/qjunittestlogger_p.h +++ b/src/testlib/qjunittestlogger_p.h @@ -70,16 +70,19 @@ class QJUnitTestLogger : public QAbstractTestLogger void enterTestFunction(const char *function) override; void leaveTestFunction() override; + void enterTestData(QTestData *) override; + void addIncident(IncidentTypes type, const char *description, const char *file = nullptr, int line = 0) override; - void addTag(QTestElement* element); - void addMessage(MessageTypes type, const QString &message, const char *file = nullptr, int line = 0) override; void addBenchmarkResult(const QBenchmarkResult &) override {} private: + void enterTestCase(const char *name); + void leaveTestCase(); + QTestElement *currentTestSuite = nullptr; QTestElement *listOfTestcases = nullptr; QTestElement *currentLogElement = nullptr; diff --git a/src/testlib/qtestelementattribute.cpp b/src/testlib/qtestelementattribute.cpp index 265f126e5a..d091804f8c 100644 --- a/src/testlib/qtestelementattribute.cpp +++ b/src/testlib/qtestelementattribute.cpp @@ -143,7 +143,9 @@ const char *QTestElementAttribute::name() const "value", "iterations", "time", - "timestamp" + "timestamp", + "hostname", + "classname" }; if (attributeIndex != QTest::AI_Undefined) diff --git a/src/testlib/qtestelementattribute_p.h b/src/testlib/qtestelementattribute_p.h index a09e19dccb..973d43ace8 100644 --- a/src/testlib/qtestelementattribute_p.h +++ b/src/testlib/qtestelementattribute_p.h @@ -78,7 +78,9 @@ namespace QTest { AI_Value = 14, AI_Iterations = 15, AI_Time = 16, - AI_Timestamp = 17 + AI_Timestamp = 17, + AI_Hostname = 18, + AI_Classname = 19, }; enum LogElementType diff --git a/tests/auto/testlib/selftests/expected_assert.junitxml b/tests/auto/testlib/selftests/expected_assert.junitxml index 4905ec73c5..dbe35c0f7f 100644 --- a/tests/auto/testlib/selftests/expected_assert.junitxml +++ b/tests/auto/testlib/selftests/expected_assert.junitxml @@ -1,16 +1,17 @@ - + - - - + + + - + + diff --git a/tests/auto/testlib/selftests/expected_badxml.junitxml b/tests/auto/testlib/selftests/expected_badxml.junitxml index 75a08ead6c..31317ecc17 100644 --- a/tests/auto/testlib/selftests/expected_badxml.junitxml +++ b/tests/auto/testlib/selftests/expected_badxml.junitxml @@ -1,38 +1,58 @@ - + - - - - - - - - - - - - - - + + + + - - - - - + + - - + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + diff --git a/tests/auto/testlib/selftests/expected_benchlibcallgrind.junitxml b/tests/auto/testlib/selftests/expected_benchlibcallgrind.junitxml index a2e67447d5..ed546cbb94 100644 --- a/tests/auto/testlib/selftests/expected_benchlibcallgrind.junitxml +++ b/tests/auto/testlib/selftests/expected_benchlibcallgrind.junitxml @@ -1,15 +1,15 @@ - + - - + + - + diff --git a/tests/auto/testlib/selftests/expected_benchlibcounting.junitxml b/tests/auto/testlib/selftests/expected_benchlibcounting.junitxml index 166a924c9b..c5e4641903 100644 --- a/tests/auto/testlib/selftests/expected_benchlibcounting.junitxml +++ b/tests/auto/testlib/selftests/expected_benchlibcounting.junitxml @@ -1,19 +1,19 @@ - + - - - + + + - - + + - + diff --git a/tests/auto/testlib/selftests/expected_benchlibeventcounter.junitxml b/tests/auto/testlib/selftests/expected_benchlibeventcounter.junitxml index 68d2ec129e..059299b017 100644 --- a/tests/auto/testlib/selftests/expected_benchlibeventcounter.junitxml +++ b/tests/auto/testlib/selftests/expected_benchlibeventcounter.junitxml @@ -1,12 +1,19 @@ - + - - - + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_benchliboptions.junitxml b/tests/auto/testlib/selftests/expected_benchliboptions.junitxml index 1744c63973..b2546057da 100644 --- a/tests/auto/testlib/selftests/expected_benchliboptions.junitxml +++ b/tests/auto/testlib/selftests/expected_benchliboptions.junitxml @@ -1,36 +1,39 @@ - + - - - + + + + - + - - - + + + + - + - - - + + + + diff --git a/tests/auto/testlib/selftests/expected_benchlibtickcounter.junitxml b/tests/auto/testlib/selftests/expected_benchlibtickcounter.junitxml index 59240506bd..7ad245c7ee 100644 --- a/tests/auto/testlib/selftests/expected_benchlibtickcounter.junitxml +++ b/tests/auto/testlib/selftests/expected_benchlibtickcounter.junitxml @@ -1,12 +1,13 @@ - + - - - + + + + diff --git a/tests/auto/testlib/selftests/expected_benchlibwalltime.junitxml b/tests/auto/testlib/selftests/expected_benchlibwalltime.junitxml index 0267b52ce6..1c525faf8d 100644 --- a/tests/auto/testlib/selftests/expected_benchlibwalltime.junitxml +++ b/tests/auto/testlib/selftests/expected_benchlibwalltime.junitxml @@ -1,14 +1,15 @@ - + - - - - - + + + + + + diff --git a/tests/auto/testlib/selftests/expected_blacklisted.junitxml b/tests/auto/testlib/selftests/expected_blacklisted.junitxml index 9d6131403e..34b3e37966 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.junitxml +++ b/tests/auto/testlib/selftests/expected_blacklisted.junitxml @@ -1,19 +1,19 @@ - + - - - + + + - - - - + + + + diff --git a/tests/auto/testlib/selftests/expected_cmptest.junitxml b/tests/auto/testlib/selftests/expected_cmptest.junitxml index 1ce3271dcd..be8a0f36fe 100644 --- a/tests/auto/testlib/selftests/expected_cmptest.junitxml +++ b/tests/auto/testlib/selftests/expected_cmptest.junitxml @@ -1,187 +1,249 @@ - + - - - + + - - + - - + - - + + - + + + + Expected (expectedWindowFlags): Window|FramelessWindowHint"/> - - + + - + + + + Expected (expectedFlags): 0x5"/> - - - - - + + + + - - - + Expected (expected): QVariant(QString,hi)"/> - - + + + + + + + + + + + - - + + + - + + + - + + + - + + + - + + + + Expected (opB) size: 12"/> - - + + - + Expected (expected) size: 3"/> - - + + + + + - + Expected (expected) size: 3"/> - - + + + + + - + Expected (ARG({1, 2, 3})) size: 3"/> - - + + + + - - + + - + + + + Expected (colorB): #ffff0000"/> - - + + - + + + - + + + + - - + + + + + + + Expected (opB): 2"/> - - + + - + + + - + + + + - + + + - - + + + + + + + Expected (opB): 2"/> - - + + - + Expected (rB): QRegion(null)"/> - - + + + + + - - + - - + - - + + - - + + - - + + - - + + - - + + + diff --git a/tests/auto/testlib/selftests/expected_commandlinedata.junitxml b/tests/auto/testlib/selftests/expected_commandlinedata.junitxml index 170a461075..202a9eaa27 100644 --- a/tests/auto/testlib/selftests/expected_commandlinedata.junitxml +++ b/tests/auto/testlib/selftests/expected_commandlinedata.junitxml @@ -1,22 +1,31 @@ - + - - - - - - - + + + - - + + - + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_counting.junitxml b/tests/auto/testlib/selftests/expected_counting.junitxml index a9278ad0a4..5575d0805b 100644 --- a/tests/auto/testlib/selftests/expected_counting.junitxml +++ b/tests/auto/testlib/selftests/expected_counting.junitxml @@ -1,55 +1,76 @@ - + - - - - + + + + + + - - + + + - - + + - - - + + + - - - + + - - + + - - - + + - - - + + - - + + + - - - + + - - + + - - - + + - + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_datatable.junitxml b/tests/auto/testlib/selftests/expected_datatable.junitxml index c3db9c584d..179309d840 100644 --- a/tests/auto/testlib/selftests/expected_datatable.junitxml +++ b/tests/auto/testlib/selftests/expected_datatable.junitxml @@ -1,37 +1,70 @@ - + - - - - - - - - - - + + + + + + + + + + - - + + - - + + - - + + - - - - - - + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_datetime.junitxml b/tests/auto/testlib/selftests/expected_datetime.junitxml index 607fa86122..4d69961f0d 100644 --- a/tests/auto/testlib/selftests/expected_datetime.junitxml +++ b/tests/auto/testlib/selftests/expected_datetime.junitxml @@ -1,24 +1,29 @@ - + - - - + + - - + + - + Expected (operandB): Invalid URL: "/> - + + + + + + diff --git a/tests/auto/testlib/selftests/expected_exceptionthrow.junitxml b/tests/auto/testlib/selftests/expected_exceptionthrow.junitxml index d92e9dcd50..fba0ecdbd3 100644 --- a/tests/auto/testlib/selftests/expected_exceptionthrow.junitxml +++ b/tests/auto/testlib/selftests/expected_exceptionthrow.junitxml @@ -1,13 +1,14 @@ - + - - - + + + + diff --git a/tests/auto/testlib/selftests/expected_expectfail.junitxml b/tests/auto/testlib/selftests/expected_expectfail.junitxml index 6b70eab354..c26b9cfa8d 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.junitxml +++ b/tests/auto/testlib/selftests/expected_expectfail.junitxml @@ -1,54 +1,68 @@ - + - - + + - + - - + + - + - - - + + + + - - - + + - - - - + + + + - - - + + - - + + + - - + + - - + + - + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_failcleanup.junitxml b/tests/auto/testlib/selftests/expected_failcleanup.junitxml index 36b4703670..2234e999fb 100644 --- a/tests/auto/testlib/selftests/expected_failcleanup.junitxml +++ b/tests/auto/testlib/selftests/expected_failcleanup.junitxml @@ -1,14 +1,15 @@ - + - - - - + + + + + diff --git a/tests/auto/testlib/selftests/expected_faildatatype.junitxml b/tests/auto/testlib/selftests/expected_faildatatype.junitxml index 9ca9a69030..0f7d4856cd 100644 --- a/tests/auto/testlib/selftests/expected_faildatatype.junitxml +++ b/tests/auto/testlib/selftests/expected_faildatatype.junitxml @@ -1,15 +1,15 @@ - + - - + + - + diff --git a/tests/auto/testlib/selftests/expected_failfetchtype.junitxml b/tests/auto/testlib/selftests/expected_failfetchtype.junitxml index a8263b8963..dd3a1076e1 100644 --- a/tests/auto/testlib/selftests/expected_failfetchtype.junitxml +++ b/tests/auto/testlib/selftests/expected_failfetchtype.junitxml @@ -1,15 +1,16 @@ - + - - - - + + + + + diff --git a/tests/auto/testlib/selftests/expected_failinit.junitxml b/tests/auto/testlib/selftests/expected_failinit.junitxml index 04ccad6d2b..3540f8e2a4 100644 --- a/tests/auto/testlib/selftests/expected_failinit.junitxml +++ b/tests/auto/testlib/selftests/expected_failinit.junitxml @@ -1,13 +1,14 @@ - + - - + + - + + diff --git a/tests/auto/testlib/selftests/expected_failinitdata.junitxml b/tests/auto/testlib/selftests/expected_failinitdata.junitxml index c0ad679d1b..c99a7d493f 100644 --- a/tests/auto/testlib/selftests/expected_failinitdata.junitxml +++ b/tests/auto/testlib/selftests/expected_failinitdata.junitxml @@ -1,12 +1,13 @@ - + - - + + + diff --git a/tests/auto/testlib/selftests/expected_fetchbogus.junitxml b/tests/auto/testlib/selftests/expected_fetchbogus.junitxml index a967cc8b3d..db7d4e8fc4 100644 --- a/tests/auto/testlib/selftests/expected_fetchbogus.junitxml +++ b/tests/auto/testlib/selftests/expected_fetchbogus.junitxml @@ -1,15 +1,16 @@ - + - - - - + + + + + diff --git a/tests/auto/testlib/selftests/expected_findtestdata.junitxml b/tests/auto/testlib/selftests/expected_findtestdata.junitxml index 1a0eebca8e..78dbc575c2 100644 --- a/tests/auto/testlib/selftests/expected_findtestdata.junitxml +++ b/tests/auto/testlib/selftests/expected_findtestdata.junitxml @@ -1,15 +1,16 @@ - + - - + + - + + diff --git a/tests/auto/testlib/selftests/expected_globaldata.junitxml b/tests/auto/testlib/selftests/expected_globaldata.junitxml index d371a5affd..b1d1794680 100644 --- a/tests/auto/testlib/selftests/expected_globaldata.junitxml +++ b/tests/auto/testlib/selftests/expected_globaldata.junitxml @@ -1,63 +1,81 @@ - + - + - - - - - - - - - - - - - - - - - + + + + + - - + + + + + - - - - - - - - - - - - - + + + + + - - - - - - - - - - - - - + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_junit.junitxml b/tests/auto/testlib/selftests/expected_junit.junitxml index 787c422251..57e449238c 100644 --- a/tests/auto/testlib/selftests/expected_junit.junitxml +++ b/tests/auto/testlib/selftests/expected_junit.junitxml @@ -1,36 +1,36 @@ - + - - + + - + - - + - - + + - + - + - - + + - + ]]> diff --git a/tests/auto/testlib/selftests/expected_keyboard.junitxml b/tests/auto/testlib/selftests/expected_keyboard.junitxml index 22e41dff3a..f3f15f8ea3 100644 --- a/tests/auto/testlib/selftests/expected_keyboard.junitxml +++ b/tests/auto/testlib/selftests/expected_keyboard.junitxml @@ -1,12 +1,13 @@ - + - - - + + + + diff --git a/tests/auto/testlib/selftests/expected_longstring.junitxml b/tests/auto/testlib/selftests/expected_longstring.junitxml index c84eaddd6a..cdb953a861 100644 --- a/tests/auto/testlib/selftests/expected_longstring.junitxml +++ b/tests/auto/testlib/selftests/expected_longstring.junitxml @@ -1,13 +1,13 @@ - + - - - + + - + + diff --git a/tests/auto/testlib/selftests/expected_maxwarnings.junitxml b/tests/auto/testlib/selftests/expected_maxwarnings.junitxml index 02d84648dc..7f01c69eba 100644 --- a/tests/auto/testlib/selftests/expected_maxwarnings.junitxml +++ b/tests/auto/testlib/selftests/expected_maxwarnings.junitxml @@ -1,12 +1,12 @@ - + - - + + @@ -2010,7 +2010,7 @@ - + diff --git a/tests/auto/testlib/selftests/expected_multiexec.junitxml b/tests/auto/testlib/selftests/expected_multiexec.junitxml index de4e05c5d8..dfc8e2afb3 100644 --- a/tests/auto/testlib/selftests/expected_multiexec.junitxml +++ b/tests/auto/testlib/selftests/expected_multiexec.junitxml @@ -1,60 +1,65 @@ - + - - - + + + + - + - - - + + + + - + - - - + + + + - + - - - + + + + - + - - - + + + + diff --git a/tests/auto/testlib/selftests/expected_pairdiagnostics.junitxml b/tests/auto/testlib/selftests/expected_pairdiagnostics.junitxml index 1f6ce0ba06..f8d90cc14c 100644 --- a/tests/auto/testlib/selftests/expected_pairdiagnostics.junitxml +++ b/tests/auto/testlib/selftests/expected_pairdiagnostics.junitxml @@ -1,21 +1,22 @@ - + - - - + + - - + - + + diff --git a/tests/auto/testlib/selftests/expected_pass.junitxml b/tests/auto/testlib/selftests/expected_pass.junitxml index d35c798499..406e1c1f0d 100644 --- a/tests/auto/testlib/selftests/expected_pass.junitxml +++ b/tests/auto/testlib/selftests/expected_pass.junitxml @@ -1,14 +1,15 @@ - + - - - - - + + + + + + diff --git a/tests/auto/testlib/selftests/expected_qexecstringlist.junitxml b/tests/auto/testlib/selftests/expected_qexecstringlist.junitxml index 441562fa5a..a066bf6a5f 100644 --- a/tests/auto/testlib/selftests/expected_qexecstringlist.junitxml +++ b/tests/auto/testlib/selftests/expected_qexecstringlist.junitxml @@ -1,15 +1,18 @@ - + - - - - - + + + + + + + + ********* Start testing of tst_QExecStringList ********* diff --git a/tests/auto/testlib/selftests/expected_signaldumper.junitxml b/tests/auto/testlib/selftests/expected_signaldumper.junitxml index b1aa68cc67..37e4b38844 100644 --- a/tests/auto/testlib/selftests/expected_signaldumper.junitxml +++ b/tests/auto/testlib/selftests/expected_signaldumper.junitxml @@ -1,141 +1,157 @@ - + - - + + - - - - - - - - - - - - - + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + - - - - - - - - - - - - - + + + + + + + + + + - - - - - - - - - - - - - - - - - - - + + + + + + + - - - - - - - - - - - - - - - - - - - + + + + + + + - - - - - - - - - - - - - - - - - - - - - - + + + + + + + - - - - - - - + + + + + + + + + + + + + - - - - - - - - + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -149,10 +165,10 @@ - + - + diff --git a/tests/auto/testlib/selftests/expected_silent.junitxml b/tests/auto/testlib/selftests/expected_silent.junitxml index c39fee5bf7..0130b4700e 100644 --- a/tests/auto/testlib/selftests/expected_silent.junitxml +++ b/tests/auto/testlib/selftests/expected_silent.junitxml @@ -1,25 +1,25 @@ - + - - - + + + - - + + - + - - + + - + @@ -27,7 +27,7 @@ - + diff --git a/tests/auto/testlib/selftests/expected_singleskip.junitxml b/tests/auto/testlib/selftests/expected_singleskip.junitxml index 2949553671..c596de7560 100644 --- a/tests/auto/testlib/selftests/expected_singleskip.junitxml +++ b/tests/auto/testlib/selftests/expected_singleskip.junitxml @@ -1,15 +1,15 @@ - + - - + + - + diff --git a/tests/auto/testlib/selftests/expected_skip.junitxml b/tests/auto/testlib/selftests/expected_skip.junitxml index f4213a723f..f99358d10d 100644 --- a/tests/auto/testlib/selftests/expected_skip.junitxml +++ b/tests/auto/testlib/selftests/expected_skip.junitxml @@ -1,22 +1,24 @@ - + - - + + - + - - - + + - + + + + diff --git a/tests/auto/testlib/selftests/expected_skipcleanup.junitxml b/tests/auto/testlib/selftests/expected_skipcleanup.junitxml index a7d7fd0bb4..c576d43f2e 100644 --- a/tests/auto/testlib/selftests/expected_skipcleanup.junitxml +++ b/tests/auto/testlib/selftests/expected_skipcleanup.junitxml @@ -1,13 +1,13 @@ - + - - - + + + diff --git a/tests/auto/testlib/selftests/expected_skipinit.junitxml b/tests/auto/testlib/selftests/expected_skipinit.junitxml index 2213d11cfe..a4a028f172 100644 --- a/tests/auto/testlib/selftests/expected_skipinit.junitxml +++ b/tests/auto/testlib/selftests/expected_skipinit.junitxml @@ -1,14 +1,14 @@ - + - + - + diff --git a/tests/auto/testlib/selftests/expected_skipinitdata.junitxml b/tests/auto/testlib/selftests/expected_skipinitdata.junitxml index 4ae6af2399..f2c8e92b45 100644 --- a/tests/auto/testlib/selftests/expected_skipinitdata.junitxml +++ b/tests/auto/testlib/selftests/expected_skipinitdata.junitxml @@ -1,11 +1,11 @@ - + - + diff --git a/tests/auto/testlib/selftests/expected_sleep.junitxml b/tests/auto/testlib/selftests/expected_sleep.junitxml index 871fdf32c0..1997144f21 100644 --- a/tests/auto/testlib/selftests/expected_sleep.junitxml +++ b/tests/auto/testlib/selftests/expected_sleep.junitxml @@ -1,13 +1,14 @@ - + - - - - + + + + + diff --git a/tests/auto/testlib/selftests/expected_strcmp.junitxml b/tests/auto/testlib/selftests/expected_strcmp.junitxml index 6d87553356..fb753a34ea 100644 --- a/tests/auto/testlib/selftests/expected_strcmp.junitxml +++ b/tests/auto/testlib/selftests/expected_strcmp.junitxml @@ -1,41 +1,41 @@ - + - - - + + + - - - + - - + - - + - - + - + diff --git a/tests/auto/testlib/selftests/expected_subtest.junitxml b/tests/auto/testlib/selftests/expected_subtest.junitxml index 36a919d5b2..8d0cfe001b 100644 --- a/tests/auto/testlib/selftests/expected_subtest.junitxml +++ b/tests/auto/testlib/selftests/expected_subtest.junitxml @@ -1,55 +1,63 @@ - + - + - + - + - - - - - - - - - - - - + + + + - + + + + + + + + + + + + + - - - - - - - - - - - - + + + + - + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_testlib.junitxml b/tests/auto/testlib/selftests/expected_testlib.junitxml index 0172787ed1..c26fbe54e1 100644 --- a/tests/auto/testlib/selftests/expected_testlib.junitxml +++ b/tests/auto/testlib/selftests/expected_testlib.junitxml @@ -1,16 +1,22 @@ - + - - - + + + - - - + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_tuplediagnostics.junitxml b/tests/auto/testlib/selftests/expected_tuplediagnostics.junitxml index cc56990dee..23ea603c37 100644 --- a/tests/auto/testlib/selftests/expected_tuplediagnostics.junitxml +++ b/tests/auto/testlib/selftests/expected_tuplediagnostics.junitxml @@ -1,22 +1,23 @@ - + - - - - + + + - - + - + + diff --git a/tests/auto/testlib/selftests/expected_verbose1.junitxml b/tests/auto/testlib/selftests/expected_verbose1.junitxml index a9278ad0a4..5575d0805b 100644 --- a/tests/auto/testlib/selftests/expected_verbose1.junitxml +++ b/tests/auto/testlib/selftests/expected_verbose1.junitxml @@ -1,55 +1,76 @@ - + - - - - + + + + + + - - + + + - - + + - - - + + + - - - + + - - + + - - - + + - - - + + - - + + + - - - + + - - + + - - - + + - + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_verbose2.junitxml b/tests/auto/testlib/selftests/expected_verbose2.junitxml index 408c71425c..0c7a46c8ec 100644 --- a/tests/auto/testlib/selftests/expected_verbose2.junitxml +++ b/tests/auto/testlib/selftests/expected_verbose2.junitxml @@ -1,74 +1,100 @@ - + - - - - - - + + + + - - - - + + + - - - - - + + + - - - - + + - - - + + + - - - - + + + - - - - - + + - - - - + + + - - - - - + + - - + + - - - + + - - + + + - - - + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_verifyexceptionthrown.junitxml b/tests/auto/testlib/selftests/expected_verifyexceptionthrown.junitxml index 2de587b4c6..1ad839080d 100644 --- a/tests/auto/testlib/selftests/expected_verifyexceptionthrown.junitxml +++ b/tests/auto/testlib/selftests/expected_verifyexceptionthrown.junitxml @@ -1,32 +1,33 @@ - + - - - - - - + + + + + + - - + + - - + + - - + + - - + + - - + + - + + diff --git a/tests/auto/testlib/selftests/expected_warnings.junitxml b/tests/auto/testlib/selftests/expected_warnings.junitxml index 8d49705972..328e2fab06 100644 --- a/tests/auto/testlib/selftests/expected_warnings.junitxml +++ b/tests/auto/testlib/selftests/expected_warnings.junitxml @@ -1,12 +1,12 @@ - + - - + + @@ -18,24 +18,26 @@ - + - + - + - + - - - - - - - + + + + - + + + + + + diff --git a/tests/auto/testlib/selftests/expected_watchdog.junitxml b/tests/auto/testlib/selftests/expected_watchdog.junitxml index 8fc2fd4873..e862d4c358 100644 --- a/tests/auto/testlib/selftests/expected_watchdog.junitxml +++ b/tests/auto/testlib/selftests/expected_watchdog.junitxml @@ -1,15 +1,16 @@ - + - - + + - + + diff --git a/tests/auto/testlib/selftests/generate_expected_output.py b/tests/auto/testlib/selftests/generate_expected_output.py index fb1b7ee96c..7dd9aa4576 100755 --- a/tests/auto/testlib/selftests/generate_expected_output.py +++ b/tests/auto/testlib/selftests/generate_expected_output.py @@ -116,6 +116,7 @@ class Cleaner (object): (r'(Config: Using QtTest library).*', r'\1'), # txt (r'( *[^<]+', r'\1/>'), # xml, lightxml (r'()', r'\1@HOSTNAME@\2'), # junit # Line numbers in source files: (r'(ASSERT: ("|").*("|") in file .*, line) \d+', r'\1 0'), # lightxml (r'(Loc: \[[^[\]()]+)\(\d+\)', r'\1(0)'), # txt diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp index 3643f4d4d3..c5ae337629 100644 --- a/tests/auto/testlib/selftests/tst_selftests.cpp +++ b/tests/auto/testlib/selftests/tst_selftests.cpp @@ -236,6 +236,8 @@ bool compareOutput(const QString &logger, const QString &subdir, actualLine.replace(timestampRegex, "timestamp=\"@TEST_START_TIME@\""); static QRegularExpression timeRegex("time=\".*?\""); actualLine.replace(timeRegex, "time=\"@TEST_DURATION@\""); + static QRegularExpression hostnameRegex("hostname=\".*?\""); + actualLine.replace(hostnameRegex, "hostname=\"@HOSTNAME@\""); } // Special handling for ignoring _FILE_ and _LINE_ if logger is teamcity