From fde57300ab51c7deda168f6a4515dcf7b1340618 Mon Sep 17 00:00:00 2001 From: Marc Mutz Date: Thu, 25 Jan 2024 18:37:26 +0100 Subject: [PATCH] QTest: add -[no]throwon{fail,skip} command line arguments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ... to complement QTEST_THROW_ON_FAIl/SKIP environment variables. This allows to conveniently test both modes by running each test twice. Task-number: QTBUG-66320 Change-Id: I8b2810e8345061c98472d846017de910a11e0657 Reviewed-by: Tor Arne Vestbø Reviewed-by: Qt CI Bot --- src/testlib/qtestcase.cpp | 12 ++++++++++++ .../auto/testlib/selftests/tst_selftests.cpp | 19 +++++++++++++++---- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/testlib/qtestcase.cpp b/src/testlib/qtestcase.cpp index 33b1d06398..b553c02908 100644 --- a/src/testlib/qtestcase.cpp +++ b/src/testlib/qtestcase.cpp @@ -972,6 +972,10 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, const char *const argv[], bool " repeated forever. This is intended as a developer tool, and\n" " is only supported with the plain text logger.\n" " -skipblacklisted : Skip blacklisted tests. Useful for measuring test coverage.\n" + " -[no]throwonfail : Enables/disables throwing on QCOMPARE()/QVERIFY()/etc.\n" + " Default: off, unless QTEST_THROW_ON_FAIL is set." + " -[no]throwonskip : Enables/disables throwing on QSKIP().\n" + " Default: off, unless QTEST_THROW_ON_SKIP is set." "\n" " Benchmarking options:\n" #if QT_CONFIG(valgrind) @@ -1133,6 +1137,14 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, const char *const argv[], bool QTest::noCrashHandler = true; } else if (strcmp(argv[i], "-skipblacklisted") == 0) { QTest::skipBlacklisted = true; + } else if (strcmp(argv[i], "-throwonfail") == 0) { + QTest::setThrowOnFail(true); + } else if (strcmp(argv[i], "-nothrowonfail") == 0) { + QTest::setThrowOnFail(false); + } else if (strcmp(argv[i], "-throwonskip") == 0) { + QTest::setThrowOnSkip(true); + } else if (strcmp(argv[i], "-nothrowonskip") == 0) { + QTest::setThrowOnSkip(false); #if QT_CONFIG(valgrind) } else if (strcmp(argv[i], "-callgrind") == 0) { if (!QBenchmarkValgrindUtils::haveValgrind()) { diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp index 9dcce2bf13..7309b12523 100644 --- a/tests/auto/testlib/selftests/tst_selftests.cpp +++ b/tests/auto/testlib/selftests/tst_selftests.cpp @@ -1014,10 +1014,12 @@ TestProcessResult runTestProcess(const QString &test, const QStringList &argumen return { process.exitCode(), standardOutput, standardError }; } +enum class Throw { OnFail = 1 }; + /* Runs a single test and verifies the output against the expected results. */ -void runTest(const QString &test, const TestLoggers &requestedLoggers) +void runTest(const QString &test, const TestLoggers &requestedLoggers, Throw throwing = {}) { TestLoggers loggers; for (auto logger : requestedLoggers) { @@ -1031,6 +1033,10 @@ void runTest(const QString &test, const TestLoggers &requestedLoggers) QStringList arguments; for (auto logger : loggers) arguments += logger.arguments(test); + if (throwing == Throw::OnFail) // don't distinguish between throwonfail/throwonskip + arguments += {"-throwonfail", "-throwonskip"}; + else + arguments += {"-nothrowonfail", "-nothrowonskip"}; CAPTURE(test); CAPTURE(arguments); @@ -1057,9 +1063,9 @@ void runTest(const QString &test, const TestLoggers &requestedLoggers) /* Runs a single test and verifies the output against the expected result. */ -void runTest(const QString &test, const TestLogger &logger) +void runTest(const QString &test, const TestLogger &logger, Throw t = {}) { - runTest(test, TestLoggers{logger}); + runTest(test, TestLoggers{logger}, t); } // ----------------------- Catch helpers ----------------------- @@ -1202,7 +1208,12 @@ SCENARIO("Test output of the loggers is as expected") GIVEN("The " << logger << " logger") { for (QString test : tests) { AND_GIVEN("The " << test << " subtest") { - runTest(test, TestLogger(logger, StdoutOutput)); + WHEN("Throwing on failure or skip") { + runTest(test, TestLogger(logger, StdoutOutput), Throw::OnFail); + } + WHEN("Returning on failure or skip") { + runTest(test, TestLogger(logger, StdoutOutput)); + } } } }