From 8d0b5270fa57f9825215ffdb29ad450457ddb05b Mon Sep 17 00:00:00 2001 From: Edward Welbourne Date: Mon, 27 Sep 2021 16:05:46 +0200 Subject: [PATCH] TAP test logger: skip XFail results as well as pass after an XFail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rationale for skipping pass after XFail is "to emit a single test point for" the test; emitting several XFails violates that aim. Task-number: QTBUG-96844 Change-Id: Ia8626dfc2dded234b3aa530fc2dc2324f1e28400 Reviewed-by: Tor Arne Vestbø Reviewed-by: Thiago Macieira --- src/testlib/qtaptestlogger.cpp | 3 ++- tests/auto/testlib/selftests/expected_expectfail.tap | 12 ------------ tests/auto/testlib/selftests/expected_strcmp.tap | 12 ------------ 3 files changed, 2 insertions(+), 25 deletions(-) diff --git a/src/testlib/qtaptestlogger.cpp b/src/testlib/qtaptestlogger.cpp index 840624fa3b..30712c274a 100644 --- a/src/testlib/qtaptestlogger.cpp +++ b/src/testlib/qtaptestlogger.cpp @@ -115,7 +115,8 @@ void QTapTestLogger::outputTestLine(bool ok, int testNumber, QTestCharBuffer &di void QTapTestLogger::addIncident(IncidentTypes type, const char *description, const char *file, int line) { - if (m_wasExpectedFail && (type == Pass || type == BlacklistedPass)) { + if (m_wasExpectedFail && (type == Pass || type == BlacklistedPass + || type == XFail || type == BlacklistedXFail)) { // XFail comes with a corresponding Pass incident, but we only want // to emit a single test point for it, so skip the this pass. return; diff --git a/tests/auto/testlib/selftests/expected_expectfail.tap b/tests/auto/testlib/selftests/expected_expectfail.tap index ef464e41b0..c520bc25c0 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.tap +++ b/tests/auto/testlib/selftests/expected_expectfail.tap @@ -70,12 +70,6 @@ not ok 11 - xfailWithQString() # TODO A string file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp line: 0 ... -not ok 11 - xfailWithQString() # TODO Bug 5 (The message) - --- - at: tst_ExpectFail::xfailWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) - file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp - line: 0 - ... ok 12 - xfailDataDrivenWithQString(Pass Abort) # SKIP Each Continue or Pass reports this and increments skip-count ok 13 - xfailDataDrivenWithQString(Pass Continue) # SKIP Each Continue or Pass reports this and increments skip-count not ok 14 - xfailDataDrivenWithQString(Fail Abort) # TODO A string @@ -90,12 +84,6 @@ not ok 15 - xfailDataDrivenWithQString(Fail Continue) # TODO A string file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp line: 0 ... -not ok 15 - xfailDataDrivenWithQString(Fail Continue) # TODO Bug 5 (The message) - --- - at: tst_ExpectFail::xfailDataDrivenWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) - file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp - line: 0 - ... ok 15 - xfailDataDrivenWithQString(Fail Continue) # SKIP Each Continue or Pass reports this and increments skip-count ok 16 - xfailDataDrivenWithQVerify(Pass Abort) ok 17 - xfailDataDrivenWithQVerify(Pass Continue) diff --git a/tests/auto/testlib/selftests/expected_strcmp.tap b/tests/auto/testlib/selftests/expected_strcmp.tap index c516df367d..0ff00d9b67 100644 --- a/tests/auto/testlib/selftests/expected_strcmp.tap +++ b/tests/auto/testlib/selftests/expected_strcmp.tap @@ -2,18 +2,6 @@ TAP version 13 # tst_StrCmp ok 1 - initTestCase() ok 2 - compareCharStars() -not ok 3 - compareByteArray() # TODO Next test should fail - --- - at: tst_StrCmp::compareByteArray() (qtbase/tests/auto/testlib/selftests/strcmp/tst_strcmp.cpp:0) - file: qtbase/tests/auto/testlib/selftests/strcmp/tst_strcmp.cpp - line: 0 - ... -not ok 3 - compareByteArray() # TODO Next test should fail - --- - at: tst_StrCmp::compareByteArray() (qtbase/tests/auto/testlib/selftests/strcmp/tst_strcmp.cpp:0) - file: qtbase/tests/auto/testlib/selftests/strcmp/tst_strcmp.cpp - line: 0 - ... not ok 3 - compareByteArray() # TODO Next test should fail --- at: tst_StrCmp::compareByteArray() (qtbase/tests/auto/testlib/selftests/strcmp/tst_strcmp.cpp:0)