testlib: Produce <error> elements on fatal errors in JUnit reporter

Test errors represents unanticipated problems, e.g., an unhandled exception, or a problem with the implementation of the test. Pick-to: 6.2 Change-Id: I87219e7ffdea56862278f005de44526ad97545f0 Reviewed-by: Mårten Nordheim <marten.nordheim@qt.io>
2021-08-02 15:14:53 +02:00 · 2021-08-02 15:14:53 +02:00 · 825e4291cd
parent dfabb5b677
commit 825e4291cd
8 changed files with 73 additions and 89 deletions
--- a/src/testlib/qjunittestlogger.cpp
+++ b/src/testlib/qjunittestlogger.cpp
@ -219,66 +219,58 @@ void QJUnitTestLogger::leaveTestCase()
 void QJUnitTestLogger::addIncident(IncidentTypes type, const char *description,
                                   const char *file, int line)
 {
-    const char *typeBuf = nullptr;
-
-    switch (type) {
-    case QAbstractTestLogger::XPass:
-        ++failureCounter;
-        typeBuf = "xpass";
-        break;
-    case QAbstractTestLogger::Pass:
-        typeBuf = "pass";
-        break;
-    case QAbstractTestLogger::XFail:
-        typeBuf = "xfail";
-        break;
-    case QAbstractTestLogger::Fail:
-        ++failureCounter;
-        typeBuf = "fail";
-        break;
-    case QAbstractTestLogger::BlacklistedPass:
-        typeBuf = "bpass";
-        break;
-    case QAbstractTestLogger::BlacklistedFail:
-        typeBuf = "bfail";
-        break;
-    case QAbstractTestLogger::BlacklistedXPass:
-        typeBuf = "bxpass";
-        break;
-    case QAbstractTestLogger::BlacklistedXFail:
-        typeBuf = "bxfail";
-        break;
-    default:
-        typeBuf = "??????";
-        break;
-    }
-
    if (type == QAbstractTestLogger::Fail || type == QAbstractTestLogger::XPass) {
-        QTestElement *failureElement = new QTestElement(QTest::LET_Failure);
-        failureElement->addAttribute(QTest::AI_Type, typeBuf);
+        auto failureType = [&]() {
+            switch (type) {
+            case QAbstractTestLogger::Fail: return "fail";
+            case QAbstractTestLogger::XPass: return "xpass";
+            default: Q_UNREACHABLE();
+            }
+        }();

-        // Assume the first line is the message, and the remainder are details
-        QString descriptionString = QString::fromUtf8(description);
-        QString message = descriptionString.section(QLatin1Char('\n'), 0, 0);
-        QString details = descriptionString.section(QLatin1Char('\n'), 1);
+        addFailure(QTest::LET_Failure, failureType, QString::fromUtf8(description));
+    } else if (type == QAbstractTestLogger::XFail) {
+        // Since XFAIL does not add a failure to the testlog in JUnit XML we add a
+        // message, so we still have some information about the expected failure.
+        addMessage(QAbstractTestLogger::Info, QString::fromUtf8(description), file, line);
+    }
+}

-        failureElement->addAttribute(QTest::AI_Message, message.toUtf8().constData());
-
-        if (!details.isEmpty()) {
-            auto messageElement = new QTestElement(QTest::LET_Message);
-            messageElement->addAttribute(QTest::AI_Message, details.toUtf8().constData());
-            failureElement->addLogElement(messageElement);
+void QJUnitTestLogger::addFailure(QTest::LogElementType elementType,
+    const char *failureType, const QString &failureDescription)
+{
+    if (elementType == QTest::LET_Failure) {
+        // Make sure we're not adding failure when we already have error,
+        // or adding additional failures when we already have a failure.
+        for (auto *childElement = currentTestCase->childElements();
+                   childElement; childElement = childElement->nextElement()) {
+            if (childElement->elementType() == QTest::LET_Error ||
+                childElement->elementType() == QTest::LET_Failure)
+                return;
        }
-
-        currentTestCase->addLogElement(failureElement);
    }

-    /*
-        Since XFAIL does not add a failure to the testlog in junitxml, add a message, so we still
-        have some information about the expected failure.
-    */
-    if (type == QAbstractTestLogger::XFail) {
-        QJUnitTestLogger::addMessage(QAbstractTestLogger::Info, QString::fromUtf8(description), file, line);
+    QTestElement *failureElement = new QTestElement(elementType);
+    failureElement->addAttribute(QTest::AI_Type, failureType);
+
+    // Assume the first line is the message, and the remainder are details
+    QString message = failureDescription.section(QLatin1Char('\n'), 0, 0);
+    QString details = failureDescription.section(QLatin1Char('\n'), 1);
+
+    failureElement->addAttribute(QTest::AI_Message, message.toUtf8().constData());
+
+    if (!details.isEmpty()) {
+        auto messageElement = new QTestElement(QTest::LET_Message);
+        messageElement->addAttribute(QTest::AI_Message, details.toUtf8().constData());
+        failureElement->addLogElement(messageElement);
+    }
+
+    currentTestCase->addLogElement(failureElement);
+
+    switch (elementType) {
+    case QTest::LET_Failure: ++failureCounter; break;
+    case QTest::LET_Error: ++errorCounter; break;
+    default: Q_UNREACHABLE();
    }
 }

@ -294,7 +286,6 @@ void QJUnitTestLogger::addMessage(MessageTypes type, const QString &message, con
        return;
    }

-    auto messageElement = new QTestElement(QTest::LET_Message);
    auto systemLogElement = systemOutputElement;
    const char *typeBuf = nullptr;

@ -331,6 +322,12 @@ void QJUnitTestLogger::addMessage(MessageTypes type, const QString &message, con
        break;
    }

+    if (type == QAbstractTestLogger::QFatal) {
+        addFailure(QTest::LET_Error, typeBuf, message);
+        return;
+    }
+
+    auto messageElement = new QTestElement(QTest::LET_Message);
    messageElement->addAttribute(QTest::AI_Type, typeBuf);
    messageElement->addAttribute(QTest::AI_Message, message.toUtf8().constData());

--- a/src/testlib/qjunittestlogger_p.h
+++ b/src/testlib/qjunittestlogger_p.h
@ -52,6 +52,7 @@
 //

 #include <QtTest/private/qabstracttestlogger_p.h>
+#include <QtTest/private/qtestelementattribute_p.h>

 QT_BEGIN_NAMESPACE

@ -83,6 +84,9 @@ class QJUnitTestLogger : public QAbstractTestLogger
        void enterTestCase(const char *name);
        void leaveTestCase();

+        void addFailure(QTest::LogElementType elementType,
+            const char *failureType, const QString &failureDescription);
+
        QTestElement *currentTestSuite = nullptr;
        QTestElement *listOfTestcases = nullptr;
        QTestElement *currentTestCase = nullptr;
--- a/tests/auto/testlib/selftests/expected_assert.junitxml
+++ b/tests/auto/testlib/selftests/expected_assert.junitxml
@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<testsuite name="tst_Assert" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="3" failures="1" errors="0" skipped="0" time="@TEST_DURATION@">
+<testsuite name="tst_Assert" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="3" failures="0" errors="1" skipped="0" time="@TEST_DURATION@">
  <properties>
    <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
    <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
@ -8,11 +8,8 @@
  <testcase name="initTestCase" classname="tst_Assert" time="@TEST_DURATION@"/>
  <testcase name="testNumber1" classname="tst_Assert" time="@TEST_DURATION@"/>
  <testcase name="testNumber2" classname="tst_Assert" time="@TEST_DURATION@">
-    <!-- type="qfatal" message="ASSERT: &quot;false&quot; in file qtbase/tests/auto/testlib/selftests/assert/tst_assert.cpp, line 0" -->
-    <failure type="fail" message="Received a fatal error."/>
+    <error type="qfatal" message="ASSERT: &quot;false&quot; in file qtbase/tests/auto/testlib/selftests/assert/tst_assert.cpp, line 0"/>
  </testcase>
  <system-out/>
-  <system-err>
-    <![CDATA[ASSERT: "false" in file qtbase/tests/auto/testlib/selftests/assert/tst_assert.cpp, line 0]]>
-  </system-err>
+  <system-err/>
 </testsuite>
--- a/tests/auto/testlib/selftests/expected_faildatatype.junitxml
+++ b/tests/auto/testlib/selftests/expected_faildatatype.junitxml
@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<testsuite name="tst_FailDataType" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="2" failures="1" errors="0" skipped="0" time="@TEST_DURATION@">
+<testsuite name="tst_FailDataType" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="2" failures="0" errors="1" skipped="0" time="@TEST_DURATION@">
  <properties>
    <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
    <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
@ -8,13 +8,10 @@
  <testcase name="initTestCase" classname="tst_FailDataType" time="@TEST_DURATION@"/>
  <testcase name="value" classname="tst_FailDataType" time="@TEST_DURATION@">
    <!-- type="qdebug" message="expected data of type &apos;QString&apos;, got &apos;bool&apos; for element 0 of data with tag &apos;bool&#x002D;as&#x002D;string&apos;" -->
-    <!-- type="qfatal" message="ASSERT: &quot;false&quot; in file qtbase/src/testlib/qtestdata.cpp, line 0" -->
-    <failure type="fail" message="Received a fatal error."/>
+    <error type="qfatal" message="ASSERT: &quot;false&quot; in file qtbase/src/testlib/qtestdata.cpp, line 0"/>
  </testcase>
  <system-out>
    <![CDATA[expected data of type 'QString', got 'bool' for element 0 of data with tag 'bool-as-string']]>
  </system-out>
-  <system-err>
-    <![CDATA[ASSERT: "false" in file qtbase/src/testlib/qtestdata.cpp, line 0]]>
-  </system-err>
+  <system-err/>
 </testsuite>
--- a/tests/auto/testlib/selftests/expected_failfetchtype.junitxml
+++ b/tests/auto/testlib/selftests/expected_failfetchtype.junitxml
@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<testsuite name="tst_FailFetchType" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="2" failures="1" errors="0" skipped="0" time="@TEST_DURATION@">
+<testsuite name="tst_FailFetchType" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="2" failures="0" errors="1" skipped="0" time="@TEST_DURATION@">
  <properties>
    <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
    <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
@ -7,11 +7,8 @@
  </properties>
  <testcase name="initTestCase" classname="tst_FailFetchType" time="@TEST_DURATION@"/>
  <testcase name="fetch(bool)" classname="tst_FailFetchType" time="@TEST_DURATION@">
-    <!-- type="qfatal" message="Requested type &apos;QString&apos; does not match available type &apos;bool&apos;." -->
-    <failure type="fail" message="Received a fatal error."/>
+    <error type="qfatal" message="Requested type &apos;QString&apos; does not match available type &apos;bool&apos;."/>
  </testcase>
  <system-out/>
-  <system-err>
-    <![CDATA[Requested type 'QString' does not match available type 'bool'.]]>
-  </system-err>
+  <system-err/>
 </testsuite>
--- a/tests/auto/testlib/selftests/expected_fetchbogus.junitxml
+++ b/tests/auto/testlib/selftests/expected_fetchbogus.junitxml
@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<testsuite name="tst_FetchBogus" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="2" failures="1" errors="0" skipped="0" time="@TEST_DURATION@">
+<testsuite name="tst_FetchBogus" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="2" failures="0" errors="1" skipped="0" time="@TEST_DURATION@">
  <properties>
    <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
    <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
@ -7,11 +7,8 @@
  </properties>
  <testcase name="initTestCase" classname="tst_FetchBogus" time="@TEST_DURATION@"/>
  <testcase name="fetchBogus(foo)" classname="tst_FetchBogus" time="@TEST_DURATION@">
-    <!-- type="qfatal" message="QFETCH: Requested testdata &apos;bubu&apos; not available, check your _data function." -->
-    <failure type="fail" message="Received a fatal error."/>
+    <error type="qfatal" message="QFETCH: Requested testdata &apos;bubu&apos; not available, check your _data function."/>
  </testcase>
  <system-out/>
-  <system-err>
-    <![CDATA[QFETCH: Requested testdata 'bubu' not available, check your _data function.]]>
-  </system-err>
+  <system-err/>
 </testsuite>
--- a/tests/auto/testlib/selftests/expected_silent.junitxml
+++ b/tests/auto/testlib/selftests/expected_silent.junitxml
@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<testsuite name="tst_Silent" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="7" failures="3" errors="0" skipped="1" time="@TEST_DURATION@">
+<testsuite name="tst_Silent" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="7" failures="2" errors="1" skipped="1" time="@TEST_DURATION@">
  <properties>
    <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
    <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
@ -26,8 +26,7 @@
    <!-- type="system" message="This is a critical message that should not appear in silent test output" -->
    <!-- type="qinfo" message="This is an info message that should not appear in silent test output" -->
    <!-- type="info" message="This is an internal testlib info message that should not appear in silent test output" -->
-    <!-- type="qfatal" message="This is a fatal error message that should still appear in silent test output" -->
-    <failure type="fail" message="Received a fatal error."/>
+    <error type="qfatal" message="This is a fatal error message that should still appear in silent test output"/>
  </testcase>
  <system-out>
    <![CDATA[This test should XFAIL]]>
@ -39,6 +38,5 @@
  <system-err>
    <![CDATA[This is a warning that should not appear in silent test output]]>
    <![CDATA[This is an internal testlib warning that should not appear in silent test output]]>
-    <![CDATA[This is a fatal error message that should still appear in silent test output]]>
  </system-err>
 </testsuite>
--- a/tests/auto/testlib/selftests/expected_watchdog.junitxml
+++ b/tests/auto/testlib/selftests/expected_watchdog.junitxml
@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<testsuite name="tst_Watchdog" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="2" failures="1" errors="0" skipped="0" time="@TEST_DURATION@">
+<testsuite name="tst_Watchdog" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="2" failures="0" errors="1" skipped="0" time="@TEST_DURATION@">
  <properties>
    <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
    <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
@ -7,11 +7,8 @@
  </properties>
  <testcase name="initTestCase" classname="tst_Watchdog" time="@TEST_DURATION@"/>
  <testcase name="delay" classname="tst_Watchdog" time="@TEST_DURATION@">
-    <!-- type="qfatal" message="Test function timed out" -->
-    <failure type="fail" message="Received a fatal error."/>
+    <error type="qfatal" message="Test function timed out"/>
  </testcase>
  <system-out/>
-  <system-err>
-    <![CDATA[Test function timed out]]>
-  </system-err>
+  <system-err/>
 </testsuite>