From 539553a57216c4ece292ff54fb623807c334cdb6 Mon Sep 17 00:00:00 2001 From: Friedemann Kleint Date: Wed, 13 Jan 2021 11:21:01 +0100 Subject: [PATCH] QTestlib/JUnit XML: Log most messages to standard output instead of error output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an enumeration for system-out and alog element for it. Redirect the messages types that are not warnings/errors to this element. For compatibility, write it out only if it is not empty. Rename enumerations and members accordingly. [ChangeLog][QtTestLib] In JUnit XML, output that is not a warning/error is now logged under instead of . Fixes: QTBUG-86540 Change-Id: I55598eafa7dafa486ac5a8221029c332ff47413b Reviewed-by: Tor Arne Vestbø --- src/testlib/qjunittestlogger.cpp | 40 +++++++++++-------- src/testlib/qjunittestlogger_p.h | 3 +- src/testlib/qtestcoreelement_p.h | 3 +- src/testlib/qtestelementattribute_p.h | 5 ++- src/testlib/qtestjunitstreamer.cpp | 34 +++++++++++----- .../selftests/expected_badxml.junitxml | 5 ++- .../expected_benchlibcounting.junitxml | 5 ++- .../selftests/expected_blacklisted.junitxml | 8 ++-- .../expected_commandlinedata.junitxml | 5 ++- .../selftests/expected_counting.junitxml | 5 ++- .../selftests/expected_expectfail.junitxml | 5 ++- .../selftests/expected_faildatatype.junitxml | 4 +- .../testlib/selftests/expected_float.junitxml | 7 +++- .../selftests/expected_globaldata.junitxml | 5 ++- .../selftests/expected_maxwarnings.junitxml | 4 +- .../selftests/expected_signaldumper.junitxml | 5 ++- .../selftests/expected_silent.junitxml | 8 ++-- .../selftests/expected_singleskip.junitxml | 5 ++- .../testlib/selftests/expected_skip.junitxml | 5 ++- .../selftests/expected_skipcleanup.junitxml | 5 ++- .../selftests/expected_skipinit.junitxml | 5 ++- .../selftests/expected_skipinitdata.junitxml | 5 ++- .../selftests/expected_strcmp.junitxml | 5 ++- .../selftests/expected_subtest.junitxml | 5 ++- .../selftests/expected_verbose1.junitxml | 5 ++- .../selftests/expected_verbose2.junitxml | 5 ++- .../selftests/expected_warnings.junitxml | 10 +++-- .../testlib/selftests/expected_xunit.junitxml | 6 ++- 28 files changed, 133 insertions(+), 79 deletions(-) diff --git a/src/testlib/qjunittestlogger.cpp b/src/testlib/qjunittestlogger.cpp index bba1c777c5..147777f77c 100644 --- a/src/testlib/qjunittestlogger.cpp +++ b/src/testlib/qjunittestlogger.cpp @@ -74,8 +74,10 @@ void QJUnitTestLogger::startLogging() QAbstractTestLogger::startLogging(); logFormatter = new QTestJUnitStreamer(this); - delete errorLogElement; - errorLogElement = new QTestElement(QTest::LET_SystemError); + delete systemOutputElement; + systemOutputElement = new QTestElement(QTest::LET_SystemOutput); + delete systemErrorElement; + systemErrorElement = new QTestElement(QTest::LET_SystemError); Q_ASSERT(!currentTestSuite); currentTestSuite = new QTestElement(QTest::LET_TestSuite); @@ -132,7 +134,9 @@ void QJUnitTestLogger::stopLogging() testcase = testcase->nextElement(); } - currentTestSuite->addLogElement(errorLogElement); + if (systemOutputElement->childElements()) + currentTestSuite->addLogElement(systemOutputElement); + currentTestSuite->addLogElement(systemErrorElement); logFormatter->output(currentTestSuite); @@ -305,11 +309,13 @@ void QJUnitTestLogger::addTag(QTestElement* element) void QJUnitTestLogger::addMessage(MessageTypes type, const QString &message, const char *file, int line) { - QTestElement *errorElement = new QTestElement(QTest::LET_Error); + auto messageElement = new QTestElement(QTest::LET_Message); + auto systemLogElement = systemOutputElement; const char *typeBuf = nullptr; switch (type) { case QAbstractTestLogger::Warn: + systemLogElement = systemErrorElement; typeBuf = "warn"; break; case QAbstractTestLogger::QSystem: @@ -322,9 +328,11 @@ void QJUnitTestLogger::addMessage(MessageTypes type, const QString &message, con typeBuf = "qinfo"; break; case QAbstractTestLogger::QWarning: + systemLogElement = systemErrorElement; typeBuf = "qwarn"; break; case QAbstractTestLogger::QFatal: + systemLogElement = systemErrorElement; typeBuf = "qfatal"; break; case QAbstractTestLogger::Skip: @@ -338,27 +346,27 @@ void QJUnitTestLogger::addMessage(MessageTypes type, const QString &message, con break; } - errorElement->addAttribute(QTest::AI_Type, typeBuf); - errorElement->addAttribute(QTest::AI_Description, message.toUtf8().constData()); - addTag(errorElement); + messageElement->addAttribute(QTest::AI_Type, typeBuf); + messageElement->addAttribute(QTest::AI_Description, message.toUtf8().constData()); + addTag(messageElement); if (file) - errorElement->addAttribute(QTest::AI_File, file); + messageElement->addAttribute(QTest::AI_File, file); else - errorElement->addAttribute(QTest::AI_File, ""); + messageElement->addAttribute(QTest::AI_File, ""); char buf[100]; qsnprintf(buf, sizeof(buf), "%i", line); - errorElement->addAttribute(QTest::AI_Line, buf); + messageElement->addAttribute(QTest::AI_Line, buf); - currentLogElement->addLogElement(errorElement); + currentLogElement->addLogElement(messageElement); ++errorCounter; - // Also add the message to the system error log (i.e. stderr), if one exists - if (errorLogElement) { - QTestElement *systemErrorElement = new QTestElement(QTest::LET_Error); - systemErrorElement->addAttribute(QTest::AI_Description, message.toUtf8().constData()); - errorLogElement->addLogElement(systemErrorElement); + // Also add the message to the system log (stdout/stderr), if one exists + if (systemLogElement) { + auto messageElement = new QTestElement(QTest::LET_Message); + messageElement->addAttribute(QTest::AI_Description, message.toUtf8().constData()); + systemLogElement->addLogElement(messageElement); } } diff --git a/src/testlib/qjunittestlogger_p.h b/src/testlib/qjunittestlogger_p.h index 2784f00dd4..0be9e8aeb0 100644 --- a/src/testlib/qjunittestlogger_p.h +++ b/src/testlib/qjunittestlogger_p.h @@ -82,7 +82,8 @@ class QJUnitTestLogger : public QAbstractTestLogger QTestElement *currentTestSuite = nullptr; QTestElement *listOfTestcases = nullptr; QTestElement *currentLogElement = nullptr; - QTestElement *errorLogElement = nullptr; + QTestElement *systemOutputElement = nullptr; + QTestElement *systemErrorElement = nullptr; QTestJUnitStreamer *logFormatter = nullptr; int testCounter = 0; diff --git a/src/testlib/qtestcoreelement_p.h b/src/testlib/qtestcoreelement_p.h index a101ab5ea3..4eb7361a5c 100644 --- a/src/testlib/qtestcoreelement_p.h +++ b/src/testlib/qtestcoreelement_p.h @@ -139,7 +139,8 @@ const char *QTestCoreElement::elementName() const "testcase", "testsuite", "benchmark", - "system-err" + "system-err", + "system-out" }; if (type != QTest::LET_Undefined) diff --git a/src/testlib/qtestelementattribute_p.h b/src/testlib/qtestelementattribute_p.h index 4f899c3a80..523dd7435e 100644 --- a/src/testlib/qtestelementattribute_p.h +++ b/src/testlib/qtestelementattribute_p.h @@ -87,11 +87,12 @@ namespace QTest { LET_Property = 0, LET_Properties = 1, LET_Failure = 2, - LET_Error = 3, + LET_Message = 3, LET_TestCase = 4, LET_TestSuite = 5, LET_Benchmark = 6, - LET_SystemError = 7 + LET_SystemError = 7, + LET_SystemOutput = 8 }; } diff --git a/src/testlib/qtestjunitstreamer.cpp b/src/testlib/qtestjunitstreamer.cpp index 844f80f90c..28879c6e06 100644 --- a/src/testlib/qtestjunitstreamer.cpp +++ b/src/testlib/qtestjunitstreamer.cpp @@ -82,12 +82,17 @@ void QTestJUnitStreamer::formatStart(const QTestElement *element, QTestCharBuffe char indent[20]; indentForElement(element, indent, sizeof(indent)); - // Errors are written as CDATA within system-err, comments elsewhere - if (element->elementType() == QTest::LET_Error) { - if (element->parentElement()->elementType() == QTest::LET_SystemError) { + // Messages/errors are written as CDATA within system-out, system-err, + // respectively, comments elsewhere + if (element->elementType() == QTest::LET_Message) { + switch (element->parentElement()->elementType()) { + case QTest::LET_SystemOutput: + case QTest::LET_SystemError: QTest::qt_asprintf(formatted, "index(); - // For errors within system-err, we only want to output `message' - if (element && element->elementType() == QTest::LET_Error - && element->parentElement()->elementType() == QTest::LET_SystemError) { + // For messages/errors within system-out, system-err, respectively, + // we only want to output `message' + if (element && element->elementType() == QTest::LET_Message + && (element->parentElement()->elementType() == QTest::LET_SystemOutput + || element->parentElement()->elementType() == QTest::LET_SystemError)) { if (attrindex != QTest::AI_Description) return; @@ -148,12 +155,17 @@ void QTestJUnitStreamer::formatAfterAttributes(const QTestElement *element, QTes if (!element || !formatted ) return; - // Errors are written as CDATA within system-err, comments elsewhere - if (element->elementType() == QTest::LET_Error) { - if (element->parentElement()->elementType() == QTest::LET_SystemError) { + // Messages/errors are written as CDATA within system-out, system-err, + // respectively, comments elsewhere + if (element->elementType() == QTest::LET_Message) { + switch (element->parentElement()->elementType()) { + case QTest::LET_SystemOutput: + case QTest::LET_SystemError: QTest::qt_asprintf(formatted, "]]>\n"); - } else { + break; + default: QTest::qt_asprintf(formatted, " -->\n"); + break; } return; } diff --git a/tests/auto/testlib/selftests/expected_badxml.junitxml b/tests/auto/testlib/selftests/expected_badxml.junitxml index 0781f1c92e..75a08ead6c 100644 --- a/tests/auto/testlib/selftests/expected_badxml.junitxml +++ b/tests/auto/testlib/selftests/expected_badxml.junitxml @@ -33,7 +33,7 @@ - + @@ -47,5 +47,6 @@ open < tags < text]]> " mixed ]]]> up > " in < the ]]]> hopes < of triggering "< ]]]> bugs]]> - + + diff --git a/tests/auto/testlib/selftests/expected_benchlibcounting.junitxml b/tests/auto/testlib/selftests/expected_benchlibcounting.junitxml index 09e3de7ef0..1766bbd5b2 100644 --- a/tests/auto/testlib/selftests/expected_benchlibcounting.junitxml +++ b/tests/auto/testlib/selftests/expected_benchlibcounting.junitxml @@ -15,7 +15,8 @@ - + - + + diff --git a/tests/auto/testlib/selftests/expected_blacklisted.junitxml b/tests/auto/testlib/selftests/expected_blacklisted.junitxml index a9779140fb..9d6131403e 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.junitxml +++ b/tests/auto/testlib/selftests/expected_blacklisted.junitxml @@ -22,14 +22,16 @@ - + - - + + + + diff --git a/tests/auto/testlib/selftests/expected_commandlinedata.junitxml b/tests/auto/testlib/selftests/expected_commandlinedata.junitxml index e5cd575d34..170a461075 100644 --- a/tests/auto/testlib/selftests/expected_commandlinedata.junitxml +++ b/tests/auto/testlib/selftests/expected_commandlinedata.junitxml @@ -17,12 +17,13 @@ - + - + + diff --git a/tests/auto/testlib/selftests/expected_counting.junitxml b/tests/auto/testlib/selftests/expected_counting.junitxml index 49632731d0..a9278ad0a4 100644 --- a/tests/auto/testlib/selftests/expected_counting.junitxml +++ b/tests/auto/testlib/selftests/expected_counting.junitxml @@ -50,7 +50,7 @@ - + @@ -61,5 +61,6 @@ - + + diff --git a/tests/auto/testlib/selftests/expected_expectfail.junitxml b/tests/auto/testlib/selftests/expected_expectfail.junitxml index e38933a3df..6b70eab354 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.junitxml +++ b/tests/auto/testlib/selftests/expected_expectfail.junitxml @@ -49,7 +49,7 @@ - + @@ -63,5 +63,6 @@ - + + diff --git a/tests/auto/testlib/selftests/expected_faildatatype.junitxml b/tests/auto/testlib/selftests/expected_faildatatype.junitxml index e654fa4441..9ca9a69030 100644 --- a/tests/auto/testlib/selftests/expected_faildatatype.junitxml +++ b/tests/auto/testlib/selftests/expected_faildatatype.junitxml @@ -11,8 +11,10 @@ - + + + diff --git a/tests/auto/testlib/selftests/expected_float.junitxml b/tests/auto/testlib/selftests/expected_float.junitxml index 831f8fefd8..c298974967 100644 --- a/tests/auto/testlib/selftests/expected_float.junitxml +++ b/tests/auto/testlib/selftests/expected_float.junitxml @@ -1,5 +1,5 @@ - + @@ -201,12 +201,15 @@ Actual (operandLeft) : 1 Expected (operandRight): 3" tag="should FAIL 1"/> + - + @@ -104,5 +104,6 @@ - + + diff --git a/tests/auto/testlib/selftests/expected_maxwarnings.junitxml b/tests/auto/testlib/selftests/expected_maxwarnings.junitxml index def32cdb16..02d84648dc 100644 --- a/tests/auto/testlib/selftests/expected_maxwarnings.junitxml +++ b/tests/auto/testlib/selftests/expected_maxwarnings.junitxml @@ -2011,6 +2011,9 @@ + + + @@ -4013,6 +4016,5 @@ - diff --git a/tests/auto/testlib/selftests/expected_signaldumper.junitxml b/tests/auto/testlib/selftests/expected_signaldumper.junitxml index a6f34414f8..b1aa68cc67 100644 --- a/tests/auto/testlib/selftests/expected_signaldumper.junitxml +++ b/tests/auto/testlib/selftests/expected_signaldumper.junitxml @@ -153,7 +153,7 @@ - + @@ -278,5 +278,6 @@ - + + diff --git a/tests/auto/testlib/selftests/expected_silent.junitxml b/tests/auto/testlib/selftests/expected_silent.junitxml index 10d8151d2e..c39fee5bf7 100644 --- a/tests/auto/testlib/selftests/expected_silent.junitxml +++ b/tests/auto/testlib/selftests/expected_silent.junitxml @@ -29,15 +29,17 @@ - + - - + + + + diff --git a/tests/auto/testlib/selftests/expected_singleskip.junitxml b/tests/auto/testlib/selftests/expected_singleskip.junitxml index a2568ca553..2949553671 100644 --- a/tests/auto/testlib/selftests/expected_singleskip.junitxml +++ b/tests/auto/testlib/selftests/expected_singleskip.junitxml @@ -10,7 +10,8 @@ - + - + + diff --git a/tests/auto/testlib/selftests/expected_skip.junitxml b/tests/auto/testlib/selftests/expected_skip.junitxml index b82b176ecc..f4213a723f 100644 --- a/tests/auto/testlib/selftests/expected_skip.junitxml +++ b/tests/auto/testlib/selftests/expected_skip.junitxml @@ -17,10 +17,11 @@ - + - + + diff --git a/tests/auto/testlib/selftests/expected_skipcleanup.junitxml b/tests/auto/testlib/selftests/expected_skipcleanup.junitxml index cb45485b4b..a7d7fd0bb4 100644 --- a/tests/auto/testlib/selftests/expected_skipcleanup.junitxml +++ b/tests/auto/testlib/selftests/expected_skipcleanup.junitxml @@ -10,7 +10,8 @@ - + - + + diff --git a/tests/auto/testlib/selftests/expected_skipinit.junitxml b/tests/auto/testlib/selftests/expected_skipinit.junitxml index 9bdb22230a..2213d11cfe 100644 --- a/tests/auto/testlib/selftests/expected_skipinit.junitxml +++ b/tests/auto/testlib/selftests/expected_skipinit.junitxml @@ -9,7 +9,8 @@ - + - + + diff --git a/tests/auto/testlib/selftests/expected_skipinitdata.junitxml b/tests/auto/testlib/selftests/expected_skipinitdata.junitxml index 56ffbcba86..4ae6af2399 100644 --- a/tests/auto/testlib/selftests/expected_skipinitdata.junitxml +++ b/tests/auto/testlib/selftests/expected_skipinitdata.junitxml @@ -8,7 +8,8 @@ - + - + + diff --git a/tests/auto/testlib/selftests/expected_strcmp.junitxml b/tests/auto/testlib/selftests/expected_strcmp.junitxml index 9d51829f4f..6d87553356 100644 --- a/tests/auto/testlib/selftests/expected_strcmp.junitxml +++ b/tests/auto/testlib/selftests/expected_strcmp.junitxml @@ -36,9 +36,10 @@ Expected (QByteArray("7")): "7""/> - + - + + diff --git a/tests/auto/testlib/selftests/expected_subtest.junitxml b/tests/auto/testlib/selftests/expected_subtest.junitxml index 27996a4992..36a919d5b2 100644 --- a/tests/auto/testlib/selftests/expected_subtest.junitxml +++ b/tests/auto/testlib/selftests/expected_subtest.junitxml @@ -52,7 +52,7 @@ - + @@ -84,5 +84,6 @@ - + + diff --git a/tests/auto/testlib/selftests/expected_verbose1.junitxml b/tests/auto/testlib/selftests/expected_verbose1.junitxml index 49632731d0..a9278ad0a4 100644 --- a/tests/auto/testlib/selftests/expected_verbose1.junitxml +++ b/tests/auto/testlib/selftests/expected_verbose1.junitxml @@ -50,7 +50,7 @@ - + @@ -61,5 +61,6 @@ - + + diff --git a/tests/auto/testlib/selftests/expected_verbose2.junitxml b/tests/auto/testlib/selftests/expected_verbose2.junitxml index 88dd82f620..408c71425c 100644 --- a/tests/auto/testlib/selftests/expected_verbose2.junitxml +++ b/tests/auto/testlib/selftests/expected_verbose2.junitxml @@ -69,7 +69,7 @@ - + @@ -98,5 +98,6 @@ - + + diff --git a/tests/auto/testlib/selftests/expected_warnings.junitxml b/tests/auto/testlib/selftests/expected_warnings.junitxml index 0a0a8aec26..8d49705972 100644 --- a/tests/auto/testlib/selftests/expected_warnings.junitxml +++ b/tests/auto/testlib/selftests/expected_warnings.junitxml @@ -36,9 +36,7 @@ - - - + @@ -46,7 +44,6 @@ - @@ -54,5 +51,10 @@ + + + + + diff --git a/tests/auto/testlib/selftests/expected_xunit.junitxml b/tests/auto/testlib/selftests/expected_xunit.junitxml index 817f85e7f0..e1b21760f2 100644 --- a/tests/auto/testlib/selftests/expected_xunit.junitxml +++ b/tests/auto/testlib/selftests/expected_xunit.junitxml @@ -31,11 +31,13 @@ - - + ]]> + + +