Rewrite Qt Testlib selftest to not rely on Qt Testlib itself
We use the Catch2 testing framework to test Qt Testlib, which also opens up the possibility of using it for other internal testing once it's made available through the build system. The test now has a --rebase mode which will write out the actual results as new expected files. Once we add the required post-processing to the results to remove timestamps and other testrun-specific data we can remove the standalone python script generate_expected_output.py that today has to be kept in sync with the test itself. No attempt has been made to clean up the comparison-functions, but these could all benefit from moving their logic from the comparison to the sanitization step. This will both make the expected files more generic, and will reduce the diff once a failure occurs, since we're not seeing all the hunks that the comparison-functions ignored. Change-Id: I1769d42e7958d56d1ad5da958db0e8fe3a2a3c23 Reviewed-by: Lars Knoll <lars.knoll@qt.io>bb10
Tor Arne Vestbø
parent
ef8640596c
commit
24e83de8d1
|
|
@ -0,0 +1,23 @@
|
|||
Boost Software License - Version 1.0 - August 17th, 2003
|
||||
|
||||
Permission is hereby granted, free of charge, to any person or organization
|
||||
obtaining a copy of the software and accompanying documentation covered by
|
||||
this license (the "Software") to use, reproduce, display, distribute,
|
||||
execute, and transmit the Software, and to prepare derivative works of the
|
||||
Software, and to permit third-parties to whom the Software is furnished to
|
||||
do so, all subject to the following:
|
||||
|
||||
The copyright notices in the Software and this entire statement, including
|
||||
the above license grant, this restriction and the following disclaimer,
|
||||
must be included in all copies of the Software, in whole or in part, and
|
||||
all derivative works of the Software, unless such copies or derivative
|
||||
works are solely in the form of machine-executable object code generated by
|
||||
a source language processor.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
|
||||
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
|
||||
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
|
||||
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
DEALINGS IN THE SOFTWARE.
|
||||
|
|
@ -7,9 +7,11 @@ include(../../../../src/testlib/selfcover.cmake)
|
|||
|
||||
qt_add_test(tst_selftests
|
||||
OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/"
|
||||
EXCEPTIONS
|
||||
SOURCES
|
||||
../../../shared/emulationdetector.h
|
||||
tst_selftests.cpp
|
||||
catch.cpp
|
||||
INCLUDE_DIRECTORIES
|
||||
../../../shared
|
||||
PUBLIC_LIBRARIES
|
||||
|
|
@ -60,69 +62,85 @@ qt_apply_testlib_coverage_options(tst_selftests)
|
|||
|
||||
# ------------- Sub tests -------------
|
||||
|
||||
# special case begin
|
||||
if(NOT WINRT)
|
||||
add_subdirectory(assert)
|
||||
add_subdirectory(badxml)
|
||||
add_subdirectory(benchlibcallgrind)
|
||||
add_subdirectory(benchlibcounting)
|
||||
add_subdirectory(benchlibeventcounter)
|
||||
add_subdirectory(benchliboptions)
|
||||
add_subdirectory(benchlibtickcounter)
|
||||
add_subdirectory(benchlibwalltime)
|
||||
add_subdirectory(blacklisted)
|
||||
add_subdirectory(cmptest)
|
||||
add_subdirectory(commandlinedata)
|
||||
add_subdirectory(counting)
|
||||
add_subdirectory(crashes)
|
||||
add_subdirectory(datatable)
|
||||
add_subdirectory(datetime)
|
||||
add_subdirectory(deleteLater)
|
||||
add_subdirectory(deleteLater_noApp)
|
||||
add_subdirectory(differentexec)
|
||||
add_subdirectory(exceptionthrow)
|
||||
add_subdirectory(expectfail)
|
||||
add_subdirectory(failcleanup)
|
||||
add_subdirectory(faildatatype)
|
||||
add_subdirectory(failfetchtype)
|
||||
add_subdirectory(failinit)
|
||||
add_subdirectory(failinitdata)
|
||||
add_subdirectory(fetchbogus)
|
||||
add_subdirectory(findtestdata)
|
||||
add_subdirectory(float)
|
||||
add_subdirectory(globaldata)
|
||||
add_subdirectory(longstring)
|
||||
add_subdirectory(maxwarnings)
|
||||
add_subdirectory(multiexec)
|
||||
add_subdirectory(pass)
|
||||
add_subdirectory(pairdiagnostics)
|
||||
add_subdirectory(printdatatags)
|
||||
add_subdirectory(printdatatagswithglobaltags)
|
||||
add_subdirectory(qexecstringlist)
|
||||
add_subdirectory(silent)
|
||||
add_subdirectory(signaldumper)
|
||||
add_subdirectory(singleskip)
|
||||
add_subdirectory(skip)
|
||||
add_subdirectory(skipcleanup)
|
||||
add_subdirectory(skipinit)
|
||||
add_subdirectory(skipinitdata)
|
||||
add_subdirectory(sleep)
|
||||
add_subdirectory(strcmp)
|
||||
add_subdirectory(subtest)
|
||||
add_subdirectory(testlib)
|
||||
add_subdirectory(tuplediagnostics)
|
||||
add_subdirectory(verbose1)
|
||||
add_subdirectory(verbose2)
|
||||
add_subdirectory(verifyexceptionthrown)
|
||||
add_subdirectory(warnings)
|
||||
add_subdirectory(watchdog)
|
||||
add_subdirectory(xunit)
|
||||
set(subprograms
|
||||
assert
|
||||
badxml
|
||||
benchlibcallgrind
|
||||
benchlibcounting
|
||||
benchlibeventcounter
|
||||
benchliboptions
|
||||
benchlibtickcounter
|
||||
benchlibwalltime
|
||||
blacklisted
|
||||
cmptest
|
||||
commandlinedata
|
||||
counting
|
||||
crashes
|
||||
datatable
|
||||
datetime
|
||||
deleteLater
|
||||
deleteLater_noApp
|
||||
differentexec
|
||||
exceptionthrow
|
||||
expectfail
|
||||
failcleanup
|
||||
faildatatype
|
||||
failfetchtype
|
||||
failinit
|
||||
failinitdata
|
||||
fetchbogus
|
||||
findtestdata
|
||||
float
|
||||
globaldata
|
||||
longstring
|
||||
maxwarnings
|
||||
multiexec
|
||||
pass
|
||||
pairdiagnostics
|
||||
printdatatags
|
||||
printdatatagswithglobaltags
|
||||
qexecstringlist
|
||||
silent
|
||||
signaldumper
|
||||
singleskip
|
||||
skip
|
||||
skipcleanup
|
||||
skipinit
|
||||
skipinitdata
|
||||
sleep
|
||||
strcmp
|
||||
subtest
|
||||
testlib
|
||||
tuplediagnostics
|
||||
verbose1
|
||||
verbose2
|
||||
verifyexceptionthrown
|
||||
warnings
|
||||
watchdog
|
||||
xunit
|
||||
)
|
||||
|
||||
if(TARGET Qt::Gui)
|
||||
add_subdirectory(keyboard)
|
||||
add_subdirectory(mouse)
|
||||
list(APPEND subprograms
|
||||
keyboard
|
||||
mouse
|
||||
)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
foreach(subprogram IN LISTS subprograms)
|
||||
add_subdirectory(${subprogram})
|
||||
endforeach()
|
||||
|
||||
list(JOIN subprograms " " subprograms)
|
||||
qt_extend_target(tst_selftests
|
||||
DEFINES
|
||||
SUBPROGRAMS=${subprograms}
|
||||
)
|
||||
# special case end
|
||||
|
||||
#### Keys ignored in scope 1:.:.:test.pro:<TRUE>:
|
||||
# CONFIG = "testcase"
|
||||
# expected_files.base = "$$PWD/.."
|
||||
|
|
|
|||
|
|
@ -0,0 +1,66 @@
|
|||
/****************************************************************************
|
||||
**
|
||||
** Copyright (C) 2020 The Qt Company Ltd.
|
||||
** Contact: https://www.qt.io/licensing/
|
||||
**
|
||||
** This file is part of the QtTest module of the Qt Toolkit.
|
||||
**
|
||||
** $QT_BEGIN_LICENSE:LGPL$
|
||||
** Commercial License Usage
|
||||
** Licensees holding valid commercial Qt licenses may use this file in
|
||||
** accordance with the commercial license agreement provided with the
|
||||
** Software or, alternatively, in accordance with the terms contained in
|
||||
** a written agreement between you and The Qt Company. For licensing terms
|
||||
** and conditions see https://www.qt.io/terms-conditions. For further
|
||||
** information use the contact form at https://www.qt.io/contact-us.
|
||||
**
|
||||
** GNU Lesser General Public License Usage
|
||||
** Alternatively, this file may be used under the terms of the GNU Lesser
|
||||
** General Public License version 3 as published by the Free Software
|
||||
** Foundation and appearing in the file LICENSE.LGPL3 included in the
|
||||
** packaging of this file. Please review the following information to
|
||||
** ensure the GNU Lesser General Public License version 3 requirements
|
||||
** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
|
||||
**
|
||||
** GNU General Public License Usage
|
||||
** Alternatively, this file may be used under the terms of the GNU
|
||||
** General Public License version 2.0 or (at your option) the GNU General
|
||||
** Public license version 3 or any later version approved by the KDE Free
|
||||
** Qt Foundation. The licenses are as published by the Free Software
|
||||
** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
|
||||
** included in the packaging of this file. Please review the following
|
||||
** information to ensure the GNU General Public License requirements will
|
||||
** be met: https://www.gnu.org/licenses/gpl-2.0.html and
|
||||
** https://www.gnu.org/licenses/gpl-3.0.html.
|
||||
**
|
||||
** $QT_END_LICENSE$
|
||||
**
|
||||
****************************************************************************/
|
||||
|
||||
#define CATCH_CONFIG_RUNNER
|
||||
#define CATCH_CLARA_CONFIG_CONSOLE_WIDTH 1000
|
||||
|
||||
#if defined(QT_NO_EXCEPTIONS)
|
||||
#define CATCH_CONFIG_DISABLE_EXCEPTIONS
|
||||
#endif
|
||||
|
||||
#include "catch_p.h"
|
||||
|
||||
QT_BEGIN_NAMESPACE
|
||||
|
||||
namespace QTestPrivate {
|
||||
|
||||
int catchMain(int argc, char **argv)
|
||||
{
|
||||
Catch::Session session;
|
||||
|
||||
if (int returnCode = session.applyCommandLine(argc, argv))
|
||||
return returnCode; // Command line error
|
||||
|
||||
return session.run();
|
||||
}
|
||||
|
||||
} // namespace QTestPrivate
|
||||
|
||||
QT_END_NAMESPACE
|
||||
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
/****************************************************************************
|
||||
**
|
||||
** Copyright (C) 2020 The Qt Company Ltd.
|
||||
** Contact: https://www.qt.io/licensing/
|
||||
**
|
||||
** This file is part of the QtTest module of the Qt Toolkit.
|
||||
**
|
||||
** $QT_BEGIN_LICENSE:LGPL$
|
||||
** Commercial License Usage
|
||||
** Licensees holding valid commercial Qt licenses may use this file in
|
||||
** accordance with the commercial license agreement provided with the
|
||||
** Software or, alternatively, in accordance with the terms contained in
|
||||
** a written agreement between you and The Qt Company. For licensing terms
|
||||
** and conditions see https://www.qt.io/terms-conditions. For further
|
||||
** information use the contact form at https://www.qt.io/contact-us.
|
||||
**
|
||||
** GNU Lesser General Public License Usage
|
||||
** Alternatively, this file may be used under the terms of the GNU Lesser
|
||||
** General Public License version 3 as published by the Free Software
|
||||
** Foundation and appearing in the file LICENSE.LGPL3 included in the
|
||||
** packaging of this file. Please review the following information to
|
||||
** ensure the GNU Lesser General Public License version 3 requirements
|
||||
** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
|
||||
**
|
||||
** GNU General Public License Usage
|
||||
** Alternatively, this file may be used under the terms of the GNU
|
||||
** General Public License version 2.0 or (at your option) the GNU General
|
||||
** Public license version 3 or any later version approved by the KDE Free
|
||||
** Qt Foundation. The licenses are as published by the Free Software
|
||||
** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
|
||||
** included in the packaging of this file. Please review the following
|
||||
** information to ensure the GNU General Public License requirements will
|
||||
** be met: https://www.gnu.org/licenses/gpl-2.0.html and
|
||||
** https://www.gnu.org/licenses/gpl-3.0.html.
|
||||
**
|
||||
** $QT_END_LICENSE$
|
||||
**
|
||||
****************************************************************************/
|
||||
|
||||
#ifndef QCATCH_P_H
|
||||
#define QCATCH_P_H
|
||||
|
||||
//
|
||||
// W A R N I N G
|
||||
// -------------
|
||||
//
|
||||
// This file is not part of the Qt API. It exists purely as an
|
||||
// implementation detail. This header file may change from version to
|
||||
// version without notice, or even be removed.
|
||||
//
|
||||
// We mean it.
|
||||
//
|
||||
|
||||
#include <QtCore/qglobal.h>
|
||||
|
||||
#include "catch_p_p.h"
|
||||
|
||||
QT_BEGIN_NAMESPACE
|
||||
|
||||
namespace QTestPrivate {
|
||||
int catchMain(int argc, char* argv[]);
|
||||
}
|
||||
|
||||
QT_END_NAMESPACE
|
||||
|
||||
#endif // QCATCH_P_H
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,17 @@
|
|||
[
|
||||
{
|
||||
"Id": "catch2",
|
||||
"Name": "Catch2",
|
||||
"QDocModule": "qttestlib",
|
||||
"QtUsage": "Used for testing of the Qt Test module.",
|
||||
"Files": "catch_p_p.h",
|
||||
|
||||
"Description": "Catch2 is a multi-paradigm test framework for C++.",
|
||||
"Homepage": "https://github.com/catchorg/Catch2",
|
||||
"Version": "2.11.1",
|
||||
"License": "Boost Software License 1.0",
|
||||
"LicenseId": "BSL-1.0",
|
||||
"LicenseFile": "CATCH_LICENSE.txt",
|
||||
"Copyright": "Copyright (c) 2019 Two Blue Cubes Ltd. All rights reserved."
|
||||
}
|
||||
]
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
CONFIG += testcase
|
||||
SOURCES += tst_selftests.cpp
|
||||
SOURCES += tst_selftests.cpp catch.cpp
|
||||
QT = core testlib-private
|
||||
|
||||
TARGET = tst_selftests
|
||||
|
|
@ -17,6 +17,7 @@ expected_files.base = $$PWD
|
|||
RESOURCES += expected_files
|
||||
|
||||
include(selftests.pri)
|
||||
DEFINES += SUBPROGRAMS=$$shell_quote($$SUBPROGRAMS)
|
||||
!android:!winrt: for(file, SUBPROGRAMS): TEST_HELPER_INSTALLS += "$${file}/$${file}"
|
||||
|
||||
include($$QT_SOURCE_TREE/src/testlib/selfcover.pri)
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue