From edba9cddbb7f2e68436ee256d4471909e33d288f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tor=20Arne=20Vestb=C3=B8?= Date: Wed, 28 Jul 2021 00:03:00 +0200 Subject: [PATCH] testlib: Accurately name JUnit test, and only run for JUnitXML reporter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The JUnit reporter was initially named xunit, but the naming was inaccurate and the reporter was renamed in 27db9e458cef512fca3a6b5c9ebbcda7a8172428. The corresponding test has now been renamed as well, and as an added bonus we only run it for that reporter. Pick-to: 6.2 Change-Id: I59cb7d949514cdf46a0199a53a7a3e39f833207c Reviewed-by: Mårten Nordheim Reviewed-by: Friedemann Kleint --- .gitignore | 2 +- tests/auto/testlib/selftests/CMakeLists.txt | 2 +- ...xunit.junitxml => expected_junit.junitxml} | 2 +- .../testlib/selftests/expected_xunit.lightxml | 64 ------------------ .../auto/testlib/selftests/expected_xunit.tap | 44 ------------ .../testlib/selftests/expected_xunit.teamcity | 26 ------- .../auto/testlib/selftests/expected_xunit.txt | 26 ------- .../auto/testlib/selftests/expected_xunit.xml | 67 ------------------- .../selftests/generate_expected_output.py | 4 +- .../testlib/selftests/junit/CMakeLists.txt | 11 +++ .../tst_xunit.cpp => junit/tst_junit.cpp} | 24 +++---- .../auto/testlib/selftests/tst_selftests.cpp | 5 +- .../testlib/selftests/xunit/CMakeLists.txt | 21 ------ 13 files changed, 33 insertions(+), 265 deletions(-) rename tests/auto/testlib/selftests/{expected_xunit.junitxml => expected_junit.junitxml} (96%) delete mode 100644 tests/auto/testlib/selftests/expected_xunit.lightxml delete mode 100644 tests/auto/testlib/selftests/expected_xunit.tap delete mode 100644 tests/auto/testlib/selftests/expected_xunit.teamcity delete mode 100644 tests/auto/testlib/selftests/expected_xunit.txt delete mode 100644 tests/auto/testlib/selftests/expected_xunit.xml create mode 100644 tests/auto/testlib/selftests/junit/CMakeLists.txt rename tests/auto/testlib/selftests/{xunit/tst_xunit.cpp => junit/tst_junit.cpp} (87%) delete mode 100644 tests/auto/testlib/selftests/xunit/CMakeLists.txt diff --git a/.gitignore b/.gitignore index b809298028..6126a58ddf 100644 --- a/.gitignore +++ b/.gitignore @@ -191,7 +191,7 @@ __pycache__ /tests/auto/testlib/selftests/verifyexceptionthrown/verifyexceptionthrown /tests/auto/testlib/selftests/waitwithoutgui/waitwithoutgui /tests/auto/testlib/selftests/warnings/warnings -/tests/auto/testlib/selftests/xunit/xunit +/tests/auto/testlib/selftests/junit/junit /tests/auto/widgets/kernel/qapplication/modal/modal QObject.log diff --git a/tests/auto/testlib/selftests/CMakeLists.txt b/tests/auto/testlib/selftests/CMakeLists.txt index 3a2def6de2..0564fdaaf7 100644 --- a/tests/auto/testlib/selftests/CMakeLists.txt +++ b/tests/auto/testlib/selftests/CMakeLists.txt @@ -91,6 +91,7 @@ set(subprograms findtestdata float globaldata + junit longstring maxwarnings multiexec @@ -116,7 +117,6 @@ set(subprograms verifyexceptionthrown warnings watchdog - xunit ) if(TARGET Qt::Gui) diff --git a/tests/auto/testlib/selftests/expected_xunit.junitxml b/tests/auto/testlib/selftests/expected_junit.junitxml similarity index 96% rename from tests/auto/testlib/selftests/expected_xunit.junitxml rename to tests/auto/testlib/selftests/expected_junit.junitxml index e1b21760f2..b3591a6868 100644 --- a/tests/auto/testlib/selftests/expected_xunit.junitxml +++ b/tests/auto/testlib/selftests/expected_junit.junitxml @@ -1,5 +1,5 @@ - + diff --git a/tests/auto/testlib/selftests/expected_xunit.lightxml b/tests/auto/testlib/selftests/expected_xunit.lightxml deleted file mode 100644 index 863501bddf..0000000000 --- a/tests/auto/testlib/selftests/expected_xunit.lightxml +++ /dev/null @@ -1,64 +0,0 @@ - - @INSERT_QT_VERSION_HERE@ - - @INSERT_QT_VERSION_HERE@ - - - - - - - - - - - - - - - ]]> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/tests/auto/testlib/selftests/expected_xunit.tap b/tests/auto/testlib/selftests/expected_xunit.tap deleted file mode 100644 index e71a10853d..0000000000 --- a/tests/auto/testlib/selftests/expected_xunit.tap +++ /dev/null @@ -1,44 +0,0 @@ -TAP version 13 -# tst_Xunit -ok 1 - initTestCase() -# just a QWARN() ! -ok 2 - testFunc1() -# a qDebug() call with comment-ending stuff --> -not ok 3 - testFunc2() - --- - type: QCOMPARE - message: Compared values are not the same - wanted: 3 (3) - found: 2 (2) - expected: 3 (3) - actual: 2 (2) - at: tst_Xunit::testFunc2() (qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp:0) - file: qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp - line: 0 - ... -ok 4 - testFunc3() # SKIP skipping this function! -not ok 5 - testFunc4() - --- - # a forced failure! - at: tst_Xunit::testFunc4() (qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp:0) - file: qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp - line: 0 - ... -not ok 6 - testFunc5() # TODO this failure is expected - --- - at: tst_Xunit::testFunc5() (qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp:0) - file: qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp - line: 0 - ... -not ok 7 - testFunc6() # TODO this failure is also expected - --- - at: tst_Xunit::testFunc6() (qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp:0) - file: qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp - line: 0 - ... -ok 8 - testFunc7() # TODO 'true' returned TRUE unexpectedly. () -ok 9 - cleanupTestCase() -1..9 -# tests 9 -# pass 5 -# fail 3 diff --git a/tests/auto/testlib/selftests/expected_xunit.teamcity b/tests/auto/testlib/selftests/expected_xunit.teamcity deleted file mode 100644 index 9969624413..0000000000 --- a/tests/auto/testlib/selftests/expected_xunit.teamcity +++ /dev/null @@ -1,26 +0,0 @@ -##teamcity[testSuiteStarted name='tst_Xunit' flowId='tst_Xunit'] -##teamcity[testStarted name='initTestCase()' flowId='tst_Xunit'] -##teamcity[testFinished name='initTestCase()' flowId='tst_Xunit'] -##teamcity[testStarted name='testFunc1()' flowId='tst_Xunit'] -##teamcity[testStdOut name='testFunc1()' out='WARNING |[Loc: qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)|]: just a QWARN() !' flowId='tst_Xunit'] -##teamcity[testFinished name='testFunc1()' flowId='tst_Xunit'] -##teamcity[testStarted name='testFunc2()' flowId='tst_Xunit'] -##teamcity[testFailed name='testFunc2()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)|]' details='Compared values are not the same|n Actual (2): 2|n Expected (3): 3' flowId='tst_Xunit'] -##teamcity[testStdOut name='testFunc2()' out='QDEBUG: a qDebug() call with comment-ending stuff -->' flowId='tst_Xunit'] -##teamcity[testFinished name='testFunc2()' flowId='tst_Xunit'] -##teamcity[testIgnored name='testFunc3()' message='skipping this function! |[Loc: qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)|]' flowId='tst_Xunit'] -##teamcity[testStarted name='testFunc4()' flowId='tst_Xunit'] -##teamcity[testFailed name='testFunc4()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)|]' details='a forced failure!' flowId='tst_Xunit'] -##teamcity[testFinished name='testFunc4()' flowId='tst_Xunit'] -##teamcity[testStarted name='testFunc5()' flowId='tst_Xunit'] -##teamcity[testStdOut name='testFunc5()' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)|]: this failure is expected' flowId='tst_Xunit'] -##teamcity[testFinished name='testFunc5()' flowId='tst_Xunit'] -##teamcity[testStarted name='testFunc6()' flowId='tst_Xunit'] -##teamcity[testStdOut name='testFunc6()' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)|]: this failure is also expected' flowId='tst_Xunit'] -##teamcity[testFinished name='testFunc6()' flowId='tst_Xunit'] -##teamcity[testStarted name='testFunc7()' flowId='tst_Xunit'] -##teamcity[testFailed name='testFunc7()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_Xunit'] -##teamcity[testFinished name='testFunc7()' flowId='tst_Xunit'] -##teamcity[testStarted name='cleanupTestCase()' flowId='tst_Xunit'] -##teamcity[testFinished name='cleanupTestCase()' flowId='tst_Xunit'] -##teamcity[testSuiteFinished name='tst_Xunit' flowId='tst_Xunit'] diff --git a/tests/auto/testlib/selftests/expected_xunit.txt b/tests/auto/testlib/selftests/expected_xunit.txt deleted file mode 100644 index bc6f50a58f..0000000000 --- a/tests/auto/testlib/selftests/expected_xunit.txt +++ /dev/null @@ -1,26 +0,0 @@ -********* Start testing of tst_Xunit ********* -Config: Using QtTest library -PASS : tst_Xunit::initTestCase() -WARNING: tst_Xunit::testFunc1() just a QWARN() ! - Loc: [qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)] -PASS : tst_Xunit::testFunc1() -QDEBUG : tst_Xunit::testFunc2() a qDebug() call with comment-ending stuff --> -FAIL! : tst_Xunit::testFunc2() Compared values are not the same - Actual (2): 2 - Expected (3): 3 - Loc: [qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)] -SKIP : tst_Xunit::testFunc3() skipping this function! - Loc: [qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)] -FAIL! : tst_Xunit::testFunc4() a forced failure! - Loc: [qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)] -XFAIL : tst_Xunit::testFunc5() this failure is expected - Loc: [qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)] -PASS : tst_Xunit::testFunc5() -XFAIL : tst_Xunit::testFunc6() this failure is also expected - Loc: [qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)] -PASS : tst_Xunit::testFunc6() -XPASS : tst_Xunit::testFunc7() 'true' returned TRUE unexpectedly. () - Loc: [qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)] -PASS : tst_Xunit::cleanupTestCase() -Totals: 5 passed, 3 failed, 1 skipped, 0 blacklisted, 0ms -********* Finished testing of tst_Xunit ********* diff --git a/tests/auto/testlib/selftests/expected_xunit.xml b/tests/auto/testlib/selftests/expected_xunit.xml deleted file mode 100644 index 723ae6c914..0000000000 --- a/tests/auto/testlib/selftests/expected_xunit.xml +++ /dev/null @@ -1,67 +0,0 @@ - - - - @INSERT_QT_VERSION_HERE@ - - @INSERT_QT_VERSION_HERE@ - - - - - - - - - - - - - - - ]]> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/tests/auto/testlib/selftests/generate_expected_output.py b/tests/auto/testlib/selftests/generate_expected_output.py index bb224e2a4f..fb1b7ee96c 100755 --- a/tests/auto/testlib/selftests/generate_expected_output.py +++ b/tests/auto/testlib/selftests/generate_expected_output.py @@ -64,7 +64,7 @@ TESTS = ['assert', 'badxml', 'benchlibcallgrind', 'benchlibcounting', 'signaldumper', 'silent', 'singleskip', 'skip', 'skipcleanup', 'skipinit', 'skipinitdata', 'sleep', 'strcmp', 'subtest', 'testlib', 'tuplediagnostics', 'verbose1', 'verbose2', 'verifyexceptionthrown', - 'warnings', 'watchdog', 'xunit', 'keyboard'] + 'warnings', 'watchdog', 'junit', 'keyboard'] class Fail (Exception): pass @@ -306,6 +306,8 @@ def generateTestData(test_path, expected_path, clean, formats): env = testEnv(testname) for format in formats: + if testname == "junit" and not format == "junitxml": + continue print(f' running {testname}/{format}') cmd = [path, f'-{format}'] expected_file = f'expected_{testname}.{format}' diff --git a/tests/auto/testlib/selftests/junit/CMakeLists.txt b/tests/auto/testlib/selftests/junit/CMakeLists.txt new file mode 100644 index 0000000000..94138b6577 --- /dev/null +++ b/tests/auto/testlib/selftests/junit/CMakeLists.txt @@ -0,0 +1,11 @@ + +qt_internal_add_executable(junit + NO_INSTALL # special case + OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} # special case + SOURCES + tst_junit.cpp + PUBLIC_LIBRARIES + Qt::Test +) + +qt_internal_apply_testlib_coverage_options(junit) diff --git a/tests/auto/testlib/selftests/xunit/tst_xunit.cpp b/tests/auto/testlib/selftests/junit/tst_junit.cpp similarity index 87% rename from tests/auto/testlib/selftests/xunit/tst_xunit.cpp rename to tests/auto/testlib/selftests/junit/tst_junit.cpp index d620784e41..77f369b779 100644 --- a/tests/auto/testlib/selftests/xunit/tst_xunit.cpp +++ b/tests/auto/testlib/selftests/junit/tst_junit.cpp @@ -28,12 +28,12 @@ #include -class tst_Xunit : public QObject +class tst_JUnit : public QObject { Q_OBJECT public: - tst_Xunit(); + tst_JUnit(); private slots: void testFunc1(); @@ -45,28 +45,28 @@ private slots: void testFunc7(); }; -tst_Xunit::tst_Xunit() +tst_JUnit::tst_JUnit() { } -void tst_Xunit::testFunc1() +void tst_JUnit::testFunc1() { QWARN("just a QWARN() !"); QCOMPARE(1,1); } -void tst_Xunit::testFunc2() +void tst_JUnit::testFunc2() { qDebug("a qDebug() call with comment-ending stuff -->"); QCOMPARE(2, 3); } -void tst_Xunit::testFunc3() +void tst_JUnit::testFunc3() { QSKIP("skipping this function!"); } -void tst_Xunit::testFunc4() +void tst_JUnit::testFunc4() { QFAIL("a forced failure!"); } @@ -79,24 +79,24 @@ void tst_Xunit::testFunc4() count. */ -void tst_Xunit::testFunc5() +void tst_JUnit::testFunc5() { QEXPECT_FAIL("", "this failure is expected", Abort); QVERIFY(false); } -void tst_Xunit::testFunc6() +void tst_JUnit::testFunc6() { QEXPECT_FAIL("", "this failure is also expected", Abort); QFAIL("This is a deliberate failure"); } -void tst_Xunit::testFunc7() +void tst_JUnit::testFunc7() { QEXPECT_FAIL("", "this pass is unexpected", Abort); QVERIFY(true); } -QTEST_APPLESS_MAIN(tst_Xunit) -#include "tst_xunit.moc" +QTEST_APPLESS_MAIN(tst_JUnit) +#include "tst_junit.moc" diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp index 1b1ad50e9b..786a79a2f3 100644 --- a/tests/auto/testlib/selftests/tst_selftests.cpp +++ b/tests/auto/testlib/selftests/tst_selftests.cpp @@ -748,6 +748,9 @@ bool TestLogger::shouldIgnoreTest(const QString &test) const if (logger == QTestLog::TeamCity && test.startsWith("benchlib")) return true; // Skip benchmark for TeamCity logger + if (logger != QTestLog::JUnitXML && test == "junit") + return true; + return false; } @@ -775,7 +778,7 @@ void checkErrorOutput(const QString &test, const QByteArray &errorOutput) || test == "cmptest" // QImage comparison requires QGuiApplication || test == "fetchbogus" || test == "watchdog" - || test == "xunit" + || test == "junit" || test == "benchlibcallgrind") return; diff --git a/tests/auto/testlib/selftests/xunit/CMakeLists.txt b/tests/auto/testlib/selftests/xunit/CMakeLists.txt deleted file mode 100644 index c153d085d0..0000000000 --- a/tests/auto/testlib/selftests/xunit/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# Generated from xunit.pro. - -##################################################################### -## xunit Binary: -##################################################################### - -qt_internal_add_executable(xunit - NO_INSTALL # special case - OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} # special case - SOURCES - tst_xunit.cpp - PUBLIC_LIBRARIES - Qt::Test -) - -## Scopes: -##################################################################### - -# special case begin -qt_internal_apply_testlib_coverage_options(xunit) -# special case end