Test that mismatches of datatype in test data columns are rejected
One test for bad data for the column, another for a bad QFETCH. Incidentally extend blacklist testing by blacklisting them. Reorganise a QEMU condition that needed extended as part of this. Task-number: QTPM-1385 Change-Id: Iac72ada19760321c5c9264ddfff7740d1fdd0700 Reviewed-by: Jędrzej Nowacki <jedrzej.nowacki@qt.io>bb10
Edward Welbourne
parent
e61028281f
commit
dfcb7fce35
|
|
@ -0,0 +1,22 @@
|
|||
<Environment>
|
||||
<QtVersion>@INSERT_QT_VERSION_HERE@</QtVersion>
|
||||
<QtBuild/>
|
||||
<QTestVersion>@INSERT_QT_VERSION_HERE@</QTestVersion>
|
||||
</Environment>
|
||||
<TestFunction name="initTestCase">
|
||||
<Incident type="pass" file="" line="0" />
|
||||
<Duration msecs="0"/>
|
||||
</TestFunction>
|
||||
<TestFunction name="value">
|
||||
<Message type="qdebug" file="" line="0">
|
||||
<Description><![CDATA[expected data of type 'QString', got 'bool' for element 0 of data with tag 'bool-as-string']]></Description>
|
||||
</Message>
|
||||
<Message type="qfatal" file="" line="0">
|
||||
<Description><![CDATA[ASSERT: "false" in file qtbase/src/testlib/qtestdata.cpp, line 0]]></Description>
|
||||
</Message>
|
||||
<Incident type="fail" file="Unknown file" line="0">
|
||||
<Description><![CDATA[Received a fatal error.]]></Description>
|
||||
</Incident>
|
||||
<Duration msecs="0"/>
|
||||
</TestFunction>
|
||||
<Duration msecs="0"/>
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
TAP version 13
|
||||
# tst_FailDataType
|
||||
ok 1 - initTestCase()
|
||||
# expected data of type 'QString', got 'bool' for element 0 of data with tag 'bool-as-string'
|
||||
# ASSERT: "false" in file qtbase/src/testlib/qtestdata.cpp, line 0
|
||||
not ok 2 - value()
|
||||
---
|
||||
# Received a fatal error.
|
||||
at: tst_FailDataType::value() (Unknown file:0)
|
||||
file: Unknown file
|
||||
line: 0
|
||||
...
|
||||
1..2
|
||||
# tests 2
|
||||
# pass 1
|
||||
# fail 1
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
##teamcity[testSuiteStarted name='tst_FailDataType' flowId='tst_FailDataType']
|
||||
##teamcity[testStarted name='initTestCase()' flowId='tst_FailDataType']
|
||||
##teamcity[testFinished name='initTestCase()' flowId='tst_FailDataType']
|
||||
##teamcity[testStarted name='value()' flowId='tst_FailDataType']
|
||||
##teamcity[testFailed name='value()' message='Failure! |[Loc: Unknown file(0)|]' details='Received a fatal error.' flowId='tst_FailDataType']
|
||||
##teamcity[testStdOut name='value()' out='QDEBUG: expected data of type |'QString|', got |'bool|' for element 0 of data with tag |'bool-as-string|'|nQFATAL: ASSERT: "false" in file qtbase/src/testlib/qtestdata.cpp, line 0' flowId='tst_FailDataType']
|
||||
##teamcity[testFinished name='value()' flowId='tst_FailDataType']
|
||||
##teamcity[testSuiteFinished name='tst_FailDataType' flowId='tst_FailDataType']
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
********* Start testing of tst_FailDataType *********
|
||||
Config: Using QtTest library
|
||||
PASS : tst_FailDataType::initTestCase()
|
||||
QDEBUG : tst_FailDataType::value() expected data of type 'QString', got 'bool' for element 0 of data with tag 'bool-as-string'
|
||||
QFATAL : tst_FailDataType::value() ASSERT: "false" in file qtbase/src/testlib/qtestdata.cpp, line 0
|
||||
FAIL! : tst_FailDataType::value() Received a fatal error.
|
||||
Loc: [Unknown file(0)]
|
||||
Totals: 1 passed, 1 failed, 0 skipped, 0 blacklisted, 0ms
|
||||
********* Finished testing of tst_FailDataType *********
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<TestCase name="tst_FailDataType">
|
||||
<Environment>
|
||||
<QtVersion>@INSERT_QT_VERSION_HERE@</QtVersion>
|
||||
<QtBuild/>
|
||||
<QTestVersion>@INSERT_QT_VERSION_HERE@</QTestVersion>
|
||||
</Environment>
|
||||
<TestFunction name="initTestCase">
|
||||
<Incident type="pass" file="" line="0" />
|
||||
<Duration msecs="0"/>
|
||||
</TestFunction>
|
||||
<TestFunction name="value">
|
||||
<Message type="qdebug" file="" line="0">
|
||||
<Description><![CDATA[expected data of type 'QString', got 'bool' for element 0 of data with tag 'bool-as-string']]></Description>
|
||||
</Message>
|
||||
<Message type="qfatal" file="" line="0">
|
||||
<Description><![CDATA[ASSERT: "false" in file qtbase/src/testlib/qtestdata.cpp, line 0]]></Description>
|
||||
</Message>
|
||||
<Incident type="fail" file="Unknown file" line="0">
|
||||
<Description><![CDATA[Received a fatal error.]]></Description>
|
||||
</Incident>
|
||||
<Duration msecs="0"/>
|
||||
</TestFunction>
|
||||
<Duration msecs="0"/>
|
||||
</TestCase>
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
<?xml version="1.0" encoding="UTF-8" ?>
|
||||
<testsuite errors="2" failures="1" tests="2" name="tst_FailDataType">
|
||||
<properties>
|
||||
<property value="@INSERT_QT_VERSION_HERE@" name="QTestVersion"/>
|
||||
<property value="@INSERT_QT_VERSION_HERE@" name="QtVersion"/>
|
||||
<property value="" name="QtBuild"/>
|
||||
</properties>
|
||||
<testcase result="pass" name="initTestCase"/>
|
||||
<testcase result="fail" name="value">
|
||||
<!-- message="expected data of type 'QString', got 'bool' for element 0 of data with tag 'bool-as-string'" type="qdebug" -->
|
||||
<!-- message="ASSERT: "false" in file qtbase/src/testlib/qtestdata.cpp, line 92" type="qfatal" -->
|
||||
<failure message="Received a fatal error." result="fail"/>
|
||||
</testcase>
|
||||
<system-err>
|
||||
<![CDATA[expected data of type 'QString', got 'bool' for element 0 of data with tag 'bool-as-string']]>
|
||||
<![CDATA[ASSERT: "false" in file qtbase/src/testlib/qtestdata.cpp, line 0]]>
|
||||
</system-err>
|
||||
</testsuite>
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
<Environment>
|
||||
<QtVersion>@INSERT_QT_VERSION_HERE@</QtVersion>
|
||||
<QtBuild/>
|
||||
<QTestVersion>@INSERT_QT_VERSION_HERE@</QTestVersion>
|
||||
</Environment>
|
||||
<TestFunction name="initTestCase">
|
||||
<Incident type="pass" file="" line="0" />
|
||||
<Duration msecs="0"/>
|
||||
</TestFunction>
|
||||
<TestFunction name="fetch">
|
||||
<Message type="qfatal" file="" line="0">
|
||||
<DataTag><![CDATA[bool]]></DataTag>
|
||||
<Description><![CDATA[Requested type 'QString' does not match available type 'bool'.]]></Description>
|
||||
</Message>
|
||||
<Incident type="fail" file="Unknown file" line="0">
|
||||
<DataTag><![CDATA[bool]]></DataTag>
|
||||
<Description><![CDATA[Received a fatal error.]]></Description>
|
||||
</Incident>
|
||||
<Duration msecs="0"/>
|
||||
</TestFunction>
|
||||
<Duration msecs="0"/>
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
TAP version 13
|
||||
# tst_FailFetchType
|
||||
ok 1 - initTestCase()
|
||||
# Requested type 'QString' does not match available type 'bool'.
|
||||
not ok 2 - fetch(bool)
|
||||
---
|
||||
# Received a fatal error.
|
||||
at: tst_FailFetchType::fetch() (Unknown file:0)
|
||||
file: Unknown file
|
||||
line: 0
|
||||
...
|
||||
1..2
|
||||
# tests 2
|
||||
# pass 1
|
||||
# fail 1
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
##teamcity[testSuiteStarted name='tst_FailFetchType' flowId='tst_FailFetchType']
|
||||
##teamcity[testStarted name='initTestCase()' flowId='tst_FailFetchType']
|
||||
##teamcity[testFinished name='initTestCase()' flowId='tst_FailFetchType']
|
||||
##teamcity[testStarted name='fetch(bool)' flowId='tst_FailFetchType']
|
||||
##teamcity[testFailed name='fetch(bool)' message='Failure! |[Loc: Unknown file(0)|]' details='Received a fatal error.' flowId='tst_FailFetchType']
|
||||
##teamcity[testStdOut name='fetch(bool)' out='QFATAL: Requested type |'QString|' does not match available type |'bool|'.' flowId='tst_FailFetchType']
|
||||
##teamcity[testFinished name='fetch(bool)' flowId='tst_FailFetchType']
|
||||
##teamcity[testSuiteFinished name='tst_FailFetchType' flowId='tst_FailFetchType']
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
********* Start testing of tst_FailFetchType *********
|
||||
Config: Using QtTest library
|
||||
PASS : tst_FailFetchType::initTestCase()
|
||||
QFATAL : tst_FailFetchType::fetch(bool) Requested type 'QString' does not match available type 'bool'.
|
||||
FAIL! : tst_FailFetchType::fetch(bool) Received a fatal error.
|
||||
Loc: [Unknown file(0)]
|
||||
Totals: 1 passed, 1 failed, 0 skipped, 0 blacklisted, 0ms
|
||||
********* Finished testing of tst_FailFetchType *********
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<TestCase name="tst_FailFetchType">
|
||||
<Environment>
|
||||
<QtVersion>@INSERT_QT_VERSION_HERE@</QtVersion>
|
||||
<QtBuild/>
|
||||
<QTestVersion>@INSERT_QT_VERSION_HERE@</QTestVersion>
|
||||
</Environment>
|
||||
<TestFunction name="initTestCase">
|
||||
<Incident type="pass" file="" line="0" />
|
||||
<Duration msecs="0"/>
|
||||
</TestFunction>
|
||||
<TestFunction name="fetch">
|
||||
<Message type="qfatal" file="" line="0">
|
||||
<DataTag><![CDATA[bool]]></DataTag>
|
||||
<Description><![CDATA[Requested type 'QString' does not match available type 'bool'.]]></Description>
|
||||
</Message>
|
||||
<Incident type="fail" file="Unknown file" line="0">
|
||||
<DataTag><![CDATA[bool]]></DataTag>
|
||||
<Description><![CDATA[Received a fatal error.]]></Description>
|
||||
</Incident>
|
||||
<Duration msecs="0"/>
|
||||
</TestFunction>
|
||||
<Duration msecs="0"/>
|
||||
</TestCase>
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
<?xml version="1.0" encoding="UTF-8" ?>
|
||||
<testsuite errors="1" failures="1" tests="2" name="tst_FailFetchType">
|
||||
<properties>
|
||||
<property value="@INSERT_QT_VERSION_HERE@" name="QTestVersion"/>
|
||||
<property value="@INSERT_QT_VERSION_HERE@" name="QtVersion"/>
|
||||
<property value="" name="QtBuild"/>
|
||||
</properties>
|
||||
<testcase result="pass" name="initTestCase"/>
|
||||
<testcase result="fail" name="fetch">
|
||||
<!-- tag="bool" message="Requested type 'QString' does not match available type 'bool'." type="qfatal" -->
|
||||
<failure tag="bool" message="Received a fatal error." result="fail"/>
|
||||
</testcase>
|
||||
<system-err>
|
||||
<![CDATA[Requested type 'QString' does not match available type 'bool'.]]>
|
||||
</system-err>
|
||||
</testsuite>
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
# See qtbase/src/testlib/qtestblacklist.cpp for format
|
||||
# Incidental test: exercise more of the blacklisting code
|
||||
[value:bool-as-string]
|
||||
*
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
SOURCES += tst_faildatatype.cpp
|
||||
QT = core testlib
|
||||
|
||||
darwin: CONFIG -= app_bundle
|
||||
CONFIG -= debug_and_release_target
|
||||
|
||||
TARGET = faildatatype
|
||||
|
||||
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
/****************************************************************************
|
||||
**
|
||||
** Copyright (C) 2018 The Qt Company Ltd.
|
||||
** Contact: https://www.qt.io/licensing/
|
||||
**
|
||||
** This file is part of the test suite of the Qt Toolkit.
|
||||
**
|
||||
** $QT_BEGIN_LICENSE:GPL-EXCEPT$
|
||||
** Commercial License Usage
|
||||
** Licensees holding valid commercial Qt licenses may use this file in
|
||||
** accordance with the commercial license agreement provided with the
|
||||
** Software or, alternatively, in accordance with the terms contained in
|
||||
** a written agreement between you and The Qt Company. For licensing terms
|
||||
** and conditions see https://www.qt.io/terms-conditions. For further
|
||||
** information use the contact form at https://www.qt.io/contact-us.
|
||||
**
|
||||
** GNU General Public License Usage
|
||||
** Alternatively, this file may be used under the terms of the GNU
|
||||
** General Public License version 3 as published by the Free Software
|
||||
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
|
||||
** included in the packaging of this file. Please review the following
|
||||
** information to ensure the GNU General Public License requirements will
|
||||
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
|
||||
**
|
||||
** $QT_END_LICENSE$
|
||||
**
|
||||
****************************************************************************/
|
||||
|
||||
|
||||
#include <QtTest/QtTest>
|
||||
|
||||
class tst_FailDataType: public QObject
|
||||
{
|
||||
Q_OBJECT
|
||||
private slots:
|
||||
void value_data() const;
|
||||
void value() const;
|
||||
};
|
||||
|
||||
void tst_FailDataType::value_data() const
|
||||
{
|
||||
QTest::addColumn<QString>("value");
|
||||
|
||||
QTest::newRow("bool-as-string") << true; // assertion should fail here
|
||||
}
|
||||
|
||||
/*! \internal
|
||||
This function should never be run because its _data() fails.
|
||||
*/
|
||||
void tst_FailDataType::value() const
|
||||
{
|
||||
QFAIL("ERROR: this function is NOT supposed to be run.");
|
||||
}
|
||||
|
||||
QTEST_APPLESS_MAIN(tst_FailDataType)
|
||||
|
||||
#include "tst_faildatatype.moc"
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
# See qtbase/src/testlib/qtestblacklist.cpp for format
|
||||
# Incidental test: exercise more of the blacklisting code
|
||||
[fetch:no-such-dataset]
|
||||
*
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
SOURCES += tst_failfetchtype.cpp
|
||||
QT = core testlib
|
||||
|
||||
darwin: CONFIG -= app_bundle
|
||||
CONFIG -= debug_and_release_target
|
||||
|
||||
TARGET = failfetchtype
|
||||
|
||||
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
/****************************************************************************
|
||||
**
|
||||
** Copyright (C) 2018 The Qt Company Ltd.
|
||||
** Contact: https://www.qt.io/licensing/
|
||||
**
|
||||
** This file is part of the test suite of the Qt Toolkit.
|
||||
**
|
||||
** $QT_BEGIN_LICENSE:GPL-EXCEPT$
|
||||
** Commercial License Usage
|
||||
** Licensees holding valid commercial Qt licenses may use this file in
|
||||
** accordance with the commercial license agreement provided with the
|
||||
** Software or, alternatively, in accordance with the terms contained in
|
||||
** a written agreement between you and The Qt Company. For licensing terms
|
||||
** and conditions see https://www.qt.io/terms-conditions. For further
|
||||
** information use the contact form at https://www.qt.io/contact-us.
|
||||
**
|
||||
** GNU General Public License Usage
|
||||
** Alternatively, this file may be used under the terms of the GNU
|
||||
** General Public License version 3 as published by the Free Software
|
||||
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
|
||||
** included in the packaging of this file. Please review the following
|
||||
** information to ensure the GNU General Public License requirements will
|
||||
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
|
||||
**
|
||||
** $QT_END_LICENSE$
|
||||
**
|
||||
****************************************************************************/
|
||||
|
||||
|
||||
#include <QtTest/QtTest>
|
||||
|
||||
class tst_FailFetchType: public QObject
|
||||
{
|
||||
Q_OBJECT
|
||||
private slots:
|
||||
void fetch_data() const;
|
||||
void fetch() const;
|
||||
};
|
||||
|
||||
void tst_FailFetchType::fetch_data() const
|
||||
{
|
||||
QTest::addColumn<bool>("value");
|
||||
|
||||
QTest::newRow("bool") << true;
|
||||
}
|
||||
|
||||
void tst_FailFetchType::fetch() const
|
||||
{
|
||||
QFETCH(QString, value); // assertion should fail here
|
||||
QFAIL("ERROR: this function is NOT supposed to be run.");
|
||||
}
|
||||
|
||||
QTEST_APPLESS_MAIN(tst_FailFetchType)
|
||||
|
||||
#include "tst_failfetchtype.moc"
|
||||
|
|
@ -244,8 +244,8 @@ def generateTestData(testname, clean,
|
|||
},
|
||||
# These are actually *other* crashers, beside those in extraEnv;
|
||||
# must match tst_Selftests::runSubTest_data():
|
||||
crashers = ("assert", "blacklisted", "crashedterminate",
|
||||
"exceptionthrow", "fetchbogus", "silent")):
|
||||
crashers = ("assert", "blacklisted", "crashedterminate", "exceptionthrow",
|
||||
"faildatatype", "failfetchtype", "fetchbogus", "silent")):
|
||||
"""Run one test and save its cleaned results.
|
||||
|
||||
Required arguments are the name of the test directory (the binary
|
||||
|
|
|
|||
|
|
@ -21,6 +21,8 @@ SUBPROGRAMS = \
|
|||
exceptionthrow \
|
||||
expectfail \
|
||||
failcleanup \
|
||||
faildatatype \
|
||||
failfetchtype \
|
||||
failinit \
|
||||
failinitdata \
|
||||
fetchbogus \
|
||||
|
|
|
|||
|
|
@ -468,10 +468,13 @@ void tst_Selftests::runSubTest_data()
|
|||
#endif
|
||||
<< "expectfail"
|
||||
<< "failcleanup"
|
||||
#ifndef Q_OS_WIN // these assert, by design; so same problem as "assert"
|
||||
<< "faildatatype"
|
||||
<< "failfetchtype"
|
||||
#endif
|
||||
<< "failinit"
|
||||
<< "failinitdata"
|
||||
#if !defined(Q_OS_WIN)
|
||||
// Disable this test on Windows, as the run-time will popup dialogs with warnings
|
||||
#ifndef Q_OS_WIN // asserts, by design; so same problem as "assert"
|
||||
<< "fetchbogus"
|
||||
#endif
|
||||
<< "findtestdata"
|
||||
|
|
@ -616,6 +619,7 @@ void tst_Selftests::runSubTest_data()
|
|||
// Keep in sync with generateTestData()'s crashers in generate_expected_output.py:
|
||||
const bool crashes = subtest == QLatin1String("assert") || subtest == QLatin1String("exceptionthrow")
|
||||
|| subtest == QLatin1String("fetchbogus") || subtest == QLatin1String("crashedterminate")
|
||||
|| subtest == QLatin1String("faildatatype") || subtest == QLatin1String("failfetchtype")
|
||||
|| subtest == QLatin1String("crashes") || subtest == QLatin1String("silent")
|
||||
|| subtest == QLatin1String("blacklisted");
|
||||
QTest::newRow(qPrintable(QString("%1 %2").arg(subtest).arg(loggerSet.name)))
|
||||
|
|
@ -745,11 +749,13 @@ void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& logge
|
|||
#endif
|
||||
#ifdef Q_OS_LINUX
|
||||
// QEMU outputs to stderr about uncaught signals
|
||||
&& (!EmulationDetector::isRunningArmOnX86() ||
|
||||
(subdir != QLatin1String("blacklisted")
|
||||
&& subdir != QLatin1String("silent")
|
||||
&& subdir != QLatin1String("assert")
|
||||
&& subdir != QLatin1String("crashes")
|
||||
&& !(EmulationDetector::isRunningArmOnX86() &&
|
||||
(subdir == QLatin1String("assert")
|
||||
|| subdir == QLatin1String("blacklisted")
|
||||
|| subdir == QLatin1String("crashes")
|
||||
|| subdir == QLatin1String("faildatatype")
|
||||
|| subdir == QLatin1String("failfetchtype")
|
||||
|| subdir == QLatin1String("silent")
|
||||
)
|
||||
)
|
||||
#endif
|
||||
|
|
@ -902,7 +908,8 @@ bool tst_Selftests::compareLine(const QString &logger, const QString &subdir,
|
|||
if (actualLine == expectedLine)
|
||||
return true;
|
||||
|
||||
if (subdir == QLatin1String("assert")
|
||||
if ((subdir == QLatin1String("assert")
|
||||
|| subdir == QLatin1String("faildatatype") || subdir == QLatin1String("failfetchtype"))
|
||||
&& actualLine.contains(QLatin1String("ASSERT: "))
|
||||
&& expectedLine.contains(QLatin1String("ASSERT: "))) {
|
||||
// Q_ASSERT uses __FILE__, the exact contents of which are
|
||||
|
|
|
|||
Loading…
Reference in New Issue