whytofly
/
qt6-bb10
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

testlib: Accurately name JUnit test, and only run for JUnitXML reporter 

The JUnit reporter was initially named xunit, but the naming was inaccurate
and the reporter was renamed in 27db9e458c.

The corresponding test has now been renamed as well, and as an added bonus
we only run it for that reporter.

Pick-to: 6.2
Change-Id: I59cb7d949514cdf46a0199a53a7a3e39f833207c
Reviewed-by: Mårten Nordheim <marten.nordheim@qt.io>
Reviewed-by: Friedemann Kleint <Friedemann.Kleint@qt.io>
bb10
Tor Arne Vestbø 2021-07-28 00:03:00 +02:00
parent 08a1bcfa9b
commit edba9cddbb
13 changed files with 33 additions and 265 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -191,7 +191,7 @@ __pycache__
/tests/auto/testlib/selftests/verifyexceptionthrown/verifyexceptionthrown
/tests/auto/testlib/selftests/waitwithoutgui/waitwithoutgui
/tests/auto/testlib/selftests/warnings/warnings
/tests/auto/testlib/selftests/xunit/xunit
/tests/auto/testlib/selftests/junit/junit
/tests/auto/widgets/kernel/qapplication/modal/modal
QObject.log

2
tests/auto/testlib/selftests/CMakeLists.txt
View File

@ -91,6 +91,7 @@ set(subprograms
findtestdata
float
globaldata
junit
longstring
maxwarnings
multiexec
@ -116,7 +117,6 @@ set(subprograms
verifyexceptionthrown
warnings
watchdog
xunit
)
if(TARGET Qt::Gui)

2
tests/auto/testlib/selftests/expected_xunit.junitxml → tests/auto/testlib/selftests/expected_junit.junitxml
View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8" ?>
<testsuite name="tst_Xunit" timestamp="@TEST_START_TIME@" tests="9" failures="3" errors="5" time="@TEST_DURATION@">
<testsuite name="tst_JUnit" timestamp="@TEST_START_TIME@" tests="9" failures="3" errors="5" time="@TEST_DURATION@">
<properties>
<property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
<property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>

64
tests/auto/testlib/selftests/expected_xunit.lightxml
View File

@ -1,64 +0,0 @@
<Environment>
<QtVersion>@INSERT_QT_VERSION_HERE@</QtVersion>
<QtBuild/>
<QTestVersion>@INSERT_QT_VERSION_HERE@</QTestVersion>
</Environment>
<TestFunction name="initTestCase">
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFunc1">
<Message type="warn" file="qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp" line="0">
<Description><![CDATA[just a QWARN() !]]></Description>
</Message>
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFunc2">
<Message type="qdebug" file="" line="0">
<Description><![CDATA[a qDebug() call with comment-ending stuff -->]]></Description>
</Message>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp" line="0">
<Description><![CDATA[Compared values are not the same
Actual (2): 2
Expected (3): 3]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFunc3">
<Message type="skip" file="qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp" line="0">
<Description><![CDATA[skipping this function!]]></Description>
</Message>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFunc4">
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp" line="0">
<Description><![CDATA[a forced failure!]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFunc5">
<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp" line="0">
<Description><![CDATA[this failure is expected]]></Description>
</Incident>
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFunc6">
<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp" line="0">
<Description><![CDATA[this failure is also expected]]></Description>
</Incident>
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFunc7">
<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp" line="0">
<Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="cleanupTestCase">
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<Duration msecs="0"/>

44
tests/auto/testlib/selftests/expected_xunit.tap
View File

@ -1,44 +0,0 @@
TAP version 13
# tst_Xunit
ok 1 - initTestCase()
# just a QWARN() !
ok 2 - testFunc1()
# a qDebug() call with comment-ending stuff -->
not ok 3 - testFunc2()
---
type: QCOMPARE
message: Compared values are not the same
wanted: 3 (3)
found: 2 (2)
expected: 3 (3)
actual: 2 (2)
at: tst_Xunit::testFunc2() (qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp:0)
file: qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp
line: 0
...
ok 4 - testFunc3() # SKIP skipping this function!
not ok 5 - testFunc4()
---
# a forced failure!
at: tst_Xunit::testFunc4() (qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp:0)
file: qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp
line: 0
...
not ok 6 - testFunc5() # TODO this failure is expected
---
at: tst_Xunit::testFunc5() (qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp:0)
file: qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp
line: 0
...
not ok 7 - testFunc6() # TODO this failure is also expected
---
at: tst_Xunit::testFunc6() (qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp:0)
file: qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp
line: 0
...
ok 8 - testFunc7() # TODO 'true' returned TRUE unexpectedly. ()
ok 9 - cleanupTestCase()
1..9
# tests 9
# pass 5
# fail 3

26
tests/auto/testlib/selftests/expected_xunit.teamcity
View File

@ -1,26 +0,0 @@
##teamcity[testSuiteStarted name='tst_Xunit' flowId='tst_Xunit']
##teamcity[testStarted name='initTestCase()' flowId='tst_Xunit']
##teamcity[testFinished name='initTestCase()' flowId='tst_Xunit']
##teamcity[testStarted name='testFunc1()' flowId='tst_Xunit']
##teamcity[testStdOut name='testFunc1()' out='WARNING |[Loc: qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)|]: just a QWARN() !' flowId='tst_Xunit']
##teamcity[testFinished name='testFunc1()' flowId='tst_Xunit']
##teamcity[testStarted name='testFunc2()' flowId='tst_Xunit']
##teamcity[testFailed name='testFunc2()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)|]' details='Compared values are not the same|n Actual (2): 2|n Expected (3): 3' flowId='tst_Xunit']
##teamcity[testStdOut name='testFunc2()' out='QDEBUG: a qDebug() call with comment-ending stuff -->' flowId='tst_Xunit']
##teamcity[testFinished name='testFunc2()' flowId='tst_Xunit']
##teamcity[testIgnored name='testFunc3()' message='skipping this function! |[Loc: qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)|]' flowId='tst_Xunit']
##teamcity[testStarted name='testFunc4()' flowId='tst_Xunit']
##teamcity[testFailed name='testFunc4()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)|]' details='a forced failure!' flowId='tst_Xunit']
##teamcity[testFinished name='testFunc4()' flowId='tst_Xunit']
##teamcity[testStarted name='testFunc5()' flowId='tst_Xunit']
##teamcity[testStdOut name='testFunc5()' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)|]: this failure is expected' flowId='tst_Xunit']
##teamcity[testFinished name='testFunc5()' flowId='tst_Xunit']
##teamcity[testStarted name='testFunc6()' flowId='tst_Xunit']
##teamcity[testStdOut name='testFunc6()' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)|]: this failure is also expected' flowId='tst_Xunit']
##teamcity[testFinished name='testFunc6()' flowId='tst_Xunit']
##teamcity[testStarted name='testFunc7()' flowId='tst_Xunit']
##teamcity[testFailed name='testFunc7()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_Xunit']
##teamcity[testFinished name='testFunc7()' flowId='tst_Xunit']
##teamcity[testStarted name='cleanupTestCase()' flowId='tst_Xunit']
##teamcity[testFinished name='cleanupTestCase()' flowId='tst_Xunit']
##teamcity[testSuiteFinished name='tst_Xunit' flowId='tst_Xunit']

26
tests/auto/testlib/selftests/expected_xunit.txt
View File

@ -1,26 +0,0 @@
********* Start testing of tst_Xunit *********
Config: Using QtTest library
PASS : tst_Xunit::initTestCase()
WARNING: tst_Xunit::testFunc1() just a QWARN() !
Loc: [qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)]
PASS : tst_Xunit::testFunc1()
QDEBUG : tst_Xunit::testFunc2() a qDebug() call with comment-ending stuff -->
FAIL! : tst_Xunit::testFunc2() Compared values are not the same
Actual (2): 2
Expected (3): 3
Loc: [qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)]
SKIP : tst_Xunit::testFunc3() skipping this function!
Loc: [qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)]
FAIL! : tst_Xunit::testFunc4() a forced failure!
Loc: [qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)]
XFAIL : tst_Xunit::testFunc5() this failure is expected
Loc: [qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)]
PASS : tst_Xunit::testFunc5()
XFAIL : tst_Xunit::testFunc6() this failure is also expected
Loc: [qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)]
PASS : tst_Xunit::testFunc6()
XPASS : tst_Xunit::testFunc7() 'true' returned TRUE unexpectedly. ()
Loc: [qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp(0)]
PASS : tst_Xunit::cleanupTestCase()
Totals: 5 passed, 3 failed, 1 skipped, 0 blacklisted, 0ms
********* Finished testing of tst_Xunit *********

67
tests/auto/testlib/selftests/expected_xunit.xml
View File

@ -1,67 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<TestCase name="tst_Xunit">
<Environment>
<QtVersion>@INSERT_QT_VERSION_HERE@</QtVersion>
<QtBuild/>
<QTestVersion>@INSERT_QT_VERSION_HERE@</QTestVersion>
</Environment>
<TestFunction name="initTestCase">
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFunc1">
<Message type="warn" file="qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp" line="0">
<Description><![CDATA[just a QWARN() !]]></Description>
</Message>
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFunc2">
<Message type="qdebug" file="" line="0">
<Description><![CDATA[a qDebug() call with comment-ending stuff -->]]></Description>
</Message>
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp" line="0">
<Description><![CDATA[Compared values are not the same
Actual (2): 2
Expected (3): 3]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFunc3">
<Message type="skip" file="qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp" line="0">
<Description><![CDATA[skipping this function!]]></Description>
</Message>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFunc4">
<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp" line="0">
<Description><![CDATA[a forced failure!]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFunc5">
<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp" line="0">
<Description><![CDATA[this failure is expected]]></Description>
</Incident>
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFunc6">
<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp" line="0">
<Description><![CDATA[this failure is also expected]]></Description>
</Incident>
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="testFunc7">
<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/xunit/tst_xunit.cpp" line="0">
<Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description>
</Incident>
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="cleanupTestCase">
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
<Duration msecs="0"/>
</TestCase>

4
tests/auto/testlib/selftests/generate_expected_output.py
View File

@ -64,7 +64,7 @@ TESTS = ['assert', 'badxml', 'benchlibcallgrind', 'benchlibcounting',
'signaldumper', 'silent', 'singleskip', 'skip', 'skipcleanup',
'skipinit', 'skipinitdata', 'sleep', 'strcmp', 'subtest', 'testlib',
'tuplediagnostics', 'verbose1', 'verbose2', 'verifyexceptionthrown',
'warnings', 'watchdog', 'xunit', 'keyboard']
'warnings', 'watchdog', 'junit', 'keyboard']
class Fail (Exception): pass
@ -306,6 +306,8 @@ def generateTestData(test_path, expected_path, clean, formats):
env = testEnv(testname)
for format in formats:
if testname == "junit" and not format == "junitxml":
continue
print(f' running {testname}/{format}')
cmd = [path, f'-{format}']
expected_file = f'expected_{testname}.{format}'

11
tests/auto/testlib/selftests/junit/CMakeLists.txt Normal file
View File

@ -0,0 +1,11 @@
qt_internal_add_executable(junit
NO_INSTALL # special case
OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} # special case
SOURCES
tst_junit.cpp
PUBLIC_LIBRARIES
Qt::Test
)
qt_internal_apply_testlib_coverage_options(junit)

24
tests/auto/testlib/selftests/xunit/tst_xunit.cpp → tests/auto/testlib/selftests/junit/tst_junit.cpp
View File

@ -28,12 +28,12 @@
#include <QTest>
class tst_Xunit : public QObject
class tst_JUnit : public QObject
{
Q_OBJECT
public:
tst_Xunit();
tst_JUnit();
private slots:
void testFunc1();
@ -45,28 +45,28 @@ private slots:
void testFunc7();
};
tst_Xunit::tst_Xunit()
tst_JUnit::tst_JUnit()
{
}
void tst_Xunit::testFunc1()
void tst_JUnit::testFunc1()
{
QWARN("just a QWARN() !");
QCOMPARE(1,1);
}
void tst_Xunit::testFunc2()
void tst_JUnit::testFunc2()
{
qDebug("a qDebug() call with comment-ending stuff -->");
QCOMPARE(2, 3);
}
void tst_Xunit::testFunc3()
void tst_JUnit::testFunc3()
{
QSKIP("skipping this function!");
}
void tst_Xunit::testFunc4()
void tst_JUnit::testFunc4()
{
QFAIL("a forced failure!");
}
@ -79,24 +79,24 @@ void tst_Xunit::testFunc4()
count.
*/
void tst_Xunit::testFunc5()
void tst_JUnit::testFunc5()
{
QEXPECT_FAIL("", "this failure is expected", Abort);
QVERIFY(false);
}
void tst_Xunit::testFunc6()
void tst_JUnit::testFunc6()
{
QEXPECT_FAIL("", "this failure is also expected", Abort);
QFAIL("This is a deliberate failure");
}
void tst_Xunit::testFunc7()
void tst_JUnit::testFunc7()
{
QEXPECT_FAIL("", "this pass is unexpected", Abort);
QVERIFY(true);
}
QTEST_APPLESS_MAIN(tst_Xunit)
#include "tst_xunit.moc"
QTEST_APPLESS_MAIN(tst_JUnit)
#include "tst_junit.moc"

5
tests/auto/testlib/selftests/tst_selftests.cpp
View File

@ -748,6 +748,9 @@ bool TestLogger::shouldIgnoreTest(const QString &test) const
if (logger == QTestLog::TeamCity && test.startsWith("benchlib"))
return true; // Skip benchmark for TeamCity logger
if (logger != QTestLog::JUnitXML && test == "junit")
return true;
return false;
}
@ -775,7 +778,7 @@ void checkErrorOutput(const QString &test, const QByteArray &errorOutput)
|| test == "cmptest" // QImage comparison requires QGuiApplication
|| test == "fetchbogus"
|| test == "watchdog"
|| test == "xunit"
|| test == "junit"
|| test == "benchlibcallgrind")
return;

21
tests/auto/testlib/selftests/xunit/CMakeLists.txt
View File

@ -1,21 +0,0 @@
# Generated from xunit.pro.
#####################################################################
## xunit Binary:
#####################################################################
qt_internal_add_executable(xunit
NO_INSTALL # special case
OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} # special case
SOURCES
tst_xunit.cpp
PUBLIC_LIBRARIES
Qt::Test
)
## Scopes:
#####################################################################
# special case begin
qt_internal_apply_testlib_coverage_options(xunit)
# special case end