Improve readability of expectfail selftest.

Rename some of the test functions to improve consistency, and change the
order so that all the xfail tests are together (some new xpass tests
will be added later).

Change-Id: Ice1ac5bf61bc26109bb0d08f72eb74e1a0424101
Reviewed-by: Rohan McGovern <rohan.mcgovern@nokia.com>
bb10
Jason McDonald 2012-02-01 14:33:42 +10:00 committed by Qt by Nokia
parent 58c617c798
commit 4d1dcaef19
5 changed files with 92 additions and 92 deletions

View File

@ -5,7 +5,7 @@
<TestFunction name="initTestCase">
<Incident type="pass" file="" line="0" />
</TestFunction>
<TestFunction name="expectAndContinue">
<TestFunction name="xfailAndContinue">
<Message type="qdebug" file="" line="0">
<Description><![CDATA[begin]]></Description>
</Message>
@ -17,7 +17,7 @@
</Message>
<Incident type="pass" file="" line="0" />
</TestFunction>
<TestFunction name="expectAndAbort">
<TestFunction name="xfailAndAbort">
<Message type="qdebug" file="" line="0">
<Description><![CDATA[begin]]></Description>
</Message>
@ -26,8 +26,8 @@
</Incident>
<Incident type="pass" file="" line="0" />
</TestFunction>
<TestFunction name="expectTwice">
<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="87">
<TestFunction name="xfailTwice">
<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="88">
<Description><![CDATA[Already expecting a fail]]></Description>
</Incident>
</TestFunction>
@ -40,36 +40,36 @@
</Incident>
<Incident type="pass" file="" line="0" />
</TestFunction>
<TestFunction name="xpass">
<Incident type="xpass" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="108">
<Description><![CDATA['true' returned FALSE. ()]]></Description>
</Incident>
</TestFunction>
<TestFunction name="dataDrivenTest">
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="141">
<TestFunction name="xfailDataDriven">
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="131">
<DataTag><![CDATA[Abort]]></DataTag>
<Description><![CDATA[This test should xfail]]></Description>
</Incident>
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="141">
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="131">
<DataTag><![CDATA[Continue]]></DataTag>
<Description><![CDATA[This test should xfail]]></Description>
</Incident>
<Incident type="pass" file="" line="0" />
</TestFunction>
<TestFunction name="expectOnWrongRow">
<TestFunction name="xfailOnWrongRow">
<Incident type="pass" file="" line="0" />
</TestFunction>
<TestFunction name="expectOnAnyRow">
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="176">
<TestFunction name="xfailOnAnyRow">
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="166">
<DataTag><![CDATA[first row]]></DataTag>
<Description><![CDATA[This test should xfail]]></Description>
</Incident>
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="176">
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="166">
<DataTag><![CDATA[second row]]></DataTag>
<Description><![CDATA[This test should xfail]]></Description>
</Incident>
<Incident type="pass" file="" line="0" />
</TestFunction>
<TestFunction name="xpass">
<Incident type="xpass" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="172">
<Description><![CDATA['true' returned FALSE. ()]]></Description>
</Incident>
</TestFunction>
<TestFunction name="cleanupTestCase">
<Incident type="pass" file="" line="0" />
</TestFunction>

View File

@ -1,35 +1,35 @@
********* Start testing of tst_ExpectFail *********
Config: Using QTest library @INSERT_QT_VERSION_HERE@, Qt @INSERT_QT_VERSION_HERE@
PASS : tst_ExpectFail::initTestCase()
QDEBUG : tst_ExpectFail::expectAndContinue() begin
XFAIL : tst_ExpectFail::expectAndContinue() This should xfail
QDEBUG : tst_ExpectFail::xfailAndContinue() begin
XFAIL : tst_ExpectFail::xfailAndContinue() This should xfail
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(70)]
QDEBUG : tst_ExpectFail::expectAndContinue() after
PASS : tst_ExpectFail::expectAndContinue()
QDEBUG : tst_ExpectFail::expectAndAbort() begin
XFAIL : tst_ExpectFail::expectAndAbort() This should xfail
QDEBUG : tst_ExpectFail::xfailAndContinue() after
PASS : tst_ExpectFail::xfailAndContinue()
QDEBUG : tst_ExpectFail::xfailAndAbort() begin
XFAIL : tst_ExpectFail::xfailAndAbort() This should xfail
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(78)]
PASS : tst_ExpectFail::expectAndAbort()
FAIL! : tst_ExpectFail::expectTwice() Already expecting a fail
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(87)]
PASS : tst_ExpectFail::xfailAndAbort()
FAIL! : tst_ExpectFail::xfailTwice() Already expecting a fail
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(88)]
XFAIL : tst_ExpectFail::xfailWithQString() A string
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(97)]
XFAIL : tst_ExpectFail::xfailWithQString() Bug 5 (The message)
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(102)]
PASS : tst_ExpectFail::xfailWithQString()
XFAIL : tst_ExpectFail::xfailDataDriven(Abort) This test should xfail
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(131)]
XFAIL : tst_ExpectFail::xfailDataDriven(Continue) This test should xfail
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(131)]
PASS : tst_ExpectFail::xfailDataDriven()
PASS : tst_ExpectFail::xfailOnWrongRow()
XFAIL : tst_ExpectFail::xfailOnAnyRow(first row) This test should xfail
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(166)]
XFAIL : tst_ExpectFail::xfailOnAnyRow(second row) This test should xfail
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(166)]
PASS : tst_ExpectFail::xfailOnAnyRow()
XPASS : tst_ExpectFail::xpass() 'true' returned FALSE. ()
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(108)]
XFAIL : tst_ExpectFail::dataDrivenTest(Abort) This test should xfail
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(141)]
XFAIL : tst_ExpectFail::dataDrivenTest(Continue) This test should xfail
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(141)]
PASS : tst_ExpectFail::dataDrivenTest()
PASS : tst_ExpectFail::expectOnWrongRow()
XFAIL : tst_ExpectFail::expectOnAnyRow(first row) This test should xfail
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(176)]
XFAIL : tst_ExpectFail::expectOnAnyRow(second row) This test should xfail
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(176)]
PASS : tst_ExpectFail::expectOnAnyRow()
Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(172)]
PASS : tst_ExpectFail::cleanupTestCase()
Totals: 8 passed, 2 failed, 0 skipped
********* Finished testing of tst_ExpectFail *********

View File

@ -7,7 +7,7 @@
<TestFunction name="initTestCase">
<Incident type="pass" file="" line="0" />
</TestFunction>
<TestFunction name="expectAndContinue">
<TestFunction name="xfailAndContinue">
<Message type="qdebug" file="" line="0">
<Description><![CDATA[begin]]></Description>
</Message>
@ -19,7 +19,7 @@
</Message>
<Incident type="pass" file="" line="0" />
</TestFunction>
<TestFunction name="expectAndAbort">
<TestFunction name="xfailAndAbort">
<Message type="qdebug" file="" line="0">
<Description><![CDATA[begin]]></Description>
</Message>
@ -28,8 +28,8 @@
</Incident>
<Incident type="pass" file="" line="0" />
</TestFunction>
<TestFunction name="expectTwice">
<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="87">
<TestFunction name="xfailTwice">
<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="88">
<Description><![CDATA[Already expecting a fail]]></Description>
</Incident>
</TestFunction>
@ -42,36 +42,36 @@
</Incident>
<Incident type="pass" file="" line="0" />
</TestFunction>
<TestFunction name="xpass">
<Incident type="xpass" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="108">
<Description><![CDATA['true' returned FALSE. ()]]></Description>
</Incident>
</TestFunction>
<TestFunction name="dataDrivenTest">
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="141">
<TestFunction name="xfailDataDriven">
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="131">
<DataTag><![CDATA[Abort]]></DataTag>
<Description><![CDATA[This test should xfail]]></Description>
</Incident>
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="141">
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="131">
<DataTag><![CDATA[Continue]]></DataTag>
<Description><![CDATA[This test should xfail]]></Description>
</Incident>
<Incident type="pass" file="" line="0" />
</TestFunction>
<TestFunction name="expectOnWrongRow">
<TestFunction name="xfailOnWrongRow">
<Incident type="pass" file="" line="0" />
</TestFunction>
<TestFunction name="expectOnAnyRow">
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="176">
<TestFunction name="xfailOnAnyRow">
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="166">
<DataTag><![CDATA[first row]]></DataTag>
<Description><![CDATA[This test should xfail]]></Description>
</Incident>
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="176">
<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="166">
<DataTag><![CDATA[second row]]></DataTag>
<Description><![CDATA[This test should xfail]]></Description>
</Incident>
<Incident type="pass" file="" line="0" />
</TestFunction>
<TestFunction name="xpass">
<Incident type="xpass" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="172">
<Description><![CDATA['true' returned FALSE. ()]]></Description>
</Incident>
</TestFunction>
<TestFunction name="cleanupTestCase">
<Incident type="pass" file="" line="0" />
</TestFunction>

View File

@ -1,38 +1,38 @@
<?xml version="1.0" encoding="UTF-8" ?>
<testsuite errors="11" failures="2" tests="10" name="tst_ExpectFail">
<properties>
<property value="@INSERT_QT_VERSION_HERE@" name="QTestVersion"/>
<property value="@INSERT_QT_VERSION_HERE@" name="QtVersion"/>
<property value="5.0.0" name="QTestVersion"/>
<property value="5.0.0" name="QtVersion"/>
</properties>
<testcase result="pass" name="initTestCase"/>
<testcase result="xfail" name="expectAndContinue">
<testcase result="xfail" name="xfailAndContinue">
<!-- message="begin" type="qdebug" -->
<!-- message="This should xfail" type="info" -->
<!-- message="after" type="qdebug" -->
</testcase>
<testcase result="xfail" name="expectAndAbort">
<testcase result="xfail" name="xfailAndAbort">
<!-- message="begin" type="qdebug" -->
<!-- message="This should xfail" type="info" -->
</testcase>
<testcase result="fail" name="expectTwice">
<testcase result="fail" name="xfailTwice">
<failure message="Already expecting a fail" result="fail"/>
</testcase>
<testcase result="xfail" name="xfailWithQString">
<!-- message="A string" type="info" -->
<!-- message="Bug 5 (The message)" type="info" -->
</testcase>
<testcase result="xpass" name="xpass">
<failure message="&apos;true&apos; returned FALSE. ()" result="xpass"/>
</testcase>
<testcase result="xfail" name="dataDrivenTest">
<testcase result="xfail" name="xfailDataDriven">
<!-- tag="Abort" message="This test should xfail" type="info" -->
<!-- tag="Continue" message="This test should xfail" type="info" -->
</testcase>
<testcase result="pass" name="expectOnWrongRow"/>
<testcase result="xfail" name="expectOnAnyRow">
<testcase result="pass" name="xfailOnWrongRow"/>
<testcase result="xfail" name="xfailOnAnyRow">
<!-- tag="first row" message="This test should xfail" type="info" -->
<!-- tag="second row" message="This test should xfail" type="info" -->
</testcase>
<testcase result="xpass" name="xpass">
<failure message="&apos;true&apos; returned FALSE. ()" result="xpass"/>
</testcase>
<testcase result="pass" name="cleanupTestCase"/>
<system-err>
<![CDATA[begin]]>

View File

@ -50,20 +50,20 @@ class tst_ExpectFail: public QObject
Q_OBJECT
private slots:
void expectAndContinue() const;
void expectAndAbort() const;
void expectTwice() const;
void xfailAndContinue() const;
void xfailAndAbort() const;
void xfailTwice() const;
void xfailWithQString() const;
void xfailDataDriven_data() const;
void xfailDataDriven() const;
void xfailOnWrongRow_data() const;
void xfailOnWrongRow() const;
void xfailOnAnyRow_data() const;
void xfailOnAnyRow() const;
void xpass() const;
void dataDrivenTest_data() const;
void dataDrivenTest() const;
void expectOnWrongRow_data() const;
void expectOnWrongRow() const;
void expectOnAnyRow_data() const;
void expectOnAnyRow() const;
};
void tst_ExpectFail::expectAndContinue() const
void tst_ExpectFail::xfailAndContinue() const
{
qDebug("begin");
QEXPECT_FAIL("", "This should xfail", Continue);
@ -71,7 +71,7 @@ void tst_ExpectFail::expectAndContinue() const
qDebug("after");
}
void tst_ExpectFail::expectAndAbort() const
void tst_ExpectFail::xfailAndAbort() const
{
qDebug("begin");
QEXPECT_FAIL("", "This should xfail", Abort);
@ -81,7 +81,7 @@ void tst_ExpectFail::expectAndAbort() const
QVERIFY2(false, "This should not be reached");
}
void tst_ExpectFail::expectTwice() const
void tst_ExpectFail::xfailTwice() const
{
QEXPECT_FAIL("", "Calling QEXPECT_FAIL once is fine", Abort);
QEXPECT_FAIL("", "Calling QEXPECT_FAIL when already expecting a failure is "
@ -102,17 +102,7 @@ void tst_ExpectFail::xfailWithQString() const
QVERIFY(false);
}
void tst_ExpectFail::xpass() const
{
QEXPECT_FAIL("", "This test should xpass", Abort);
QVERIFY(true);
// If we get here the test did not correctly abort on the previous
// unexpected pass.
QVERIFY2(false, "This should not be reached");
}
void tst_ExpectFail::dataDrivenTest_data() const
void tst_ExpectFail::xfailDataDriven_data() const
{
QTest::addColumn<bool>("shouldPass");
QTest::addColumn<QTest::TestFailMode>("failMode");
@ -123,7 +113,7 @@ void tst_ExpectFail::dataDrivenTest_data() const
QTest::newRow("Continue") << false << QTest::Continue;
}
void tst_ExpectFail::dataDrivenTest() const
void tst_ExpectFail::xfailDataDriven() const
{
QFETCH(bool, shouldPass);
QFETCH(QTest::TestFailMode, failMode);
@ -146,21 +136,21 @@ void tst_ExpectFail::dataDrivenTest() const
QCOMPARE(failMode, QTest::Continue);
}
void tst_ExpectFail::expectOnWrongRow_data() const
void tst_ExpectFail::xfailOnWrongRow_data() const
{
QTest::addColumn<int>("dummy");
QTest::newRow("right row") << 0;
}
void tst_ExpectFail::expectOnWrongRow() const
void tst_ExpectFail::xfailOnWrongRow() const
{
// QEXPECT_FAIL for a row that is not the current row should be ignored.
QEXPECT_FAIL("wrong row", "This xfail should be ignored", Abort);
QVERIFY(true);
}
void tst_ExpectFail::expectOnAnyRow_data() const
void tst_ExpectFail::xfailOnAnyRow_data() const
{
QTest::addColumn<int>("dummy");
@ -168,7 +158,7 @@ void tst_ExpectFail::expectOnAnyRow_data() const
QTest::newRow("second row") << 1;
}
void tst_ExpectFail::expectOnAnyRow() const
void tst_ExpectFail::xfailOnAnyRow() const
{
// In a data-driven test, passing an empty first parameter to QEXPECT_FAIL
// should mean that the failure is expected for all data rows.
@ -176,5 +166,15 @@ void tst_ExpectFail::expectOnAnyRow() const
QVERIFY(false);
}
void tst_ExpectFail::xpass() const
{
QEXPECT_FAIL("", "This test should xpass", Abort);
QVERIFY(true);
// If we get here the test did not correctly abort on the previous
// unexpected pass.
QVERIFY2(false, "This should not be reached");
}
QTEST_MAIN(tst_ExpectFail)
#include "tst_expectfail.moc"