TestCase: Also check main source path when looking for test data

This is for example useful when looking for a possible BLACKLIST
file while doing a shadow build.

Add autotests for blacklist.

Change-Id: I41d3939d31d21d10187fefcb82604736d911b6ad
Reviewed-by: Friedemann Kleint <Friedemann.Kleint@theqtcompany.com>
Reviewed-by: Simon Hausmann <simon.hausmann@theqtcompany.com>

src/testlib/qtest.h

@@ -274,10 +274,17 @@ inline bool qCompare(quint32 const &t1, quint64 const &t2, const char *actual,
 }
 QT_END_NAMESPACE
 
+#ifdef QT_TESTCASE_BUILDDIR
+#  define QTEST_SET_MAIN_SOURCE_PATH  QTest::setMainSourcePath(__FILE__, QT_TESTCASE_BUILDDIR);
+#else
+#  define QTEST_SET_MAIN_SOURCE_PATH  QTest::setMainSourcePath(__FILE__);
+#endif
+
 #define QTEST_APPLESS_MAIN(TestObject) \
 int main(int argc, char *argv[]) \
 { \
     TestObject tc; \
+    QTEST_SET_MAIN_SOURCE_PATH \
     return QTest::qExec(&tc, argc, argv); \
 }
 
@@ -300,6 +307,7 @@ int main(int argc, char *argv[]) \
     app.setAttribute(Qt::AA_Use96Dpi, true); \
     QTEST_DISABLE_KEYPAD_NAVIGATION \
     TestObject tc; \
+    QTEST_SET_MAIN_SOURCE_PATH \
     return QTest::qExec(&tc, argc, argv); \
 }
 
@@ -313,6 +321,7 @@ int main(int argc, char *argv[]) \
     QGuiApplication app(argc, argv); \
     app.setAttribute(Qt::AA_Use96Dpi, true); \
     TestObject tc; \
+    QTEST_SET_MAIN_SOURCE_PATH \
     return QTest::qExec(&tc, argc, argv); \
 }
 
@@ -324,6 +333,7 @@ int main(int argc, char *argv[]) \
     QCoreApplication app(argc, argv); \
     app.setAttribute(Qt::AA_Use96Dpi, true); \
     TestObject tc; \
+    QTEST_SET_MAIN_SOURCE_PATH \
     return QTest::qExec(&tc, argc, argv); \
 }
 
@@ -335,6 +345,7 @@ int main(int argc, char *argv[]) \
     QCoreApplication app(argc, argv); \
     app.setAttribute(Qt::AA_Use96Dpi, true); \
     TestObject tc; \
+    QTEST_SET_MAIN_SOURCE_PATH \
     return QTest::qExec(&tc, argc, argv); \
 }
 

src/testlib/qtestcase.cpp

@@ -1332,6 +1332,7 @@ static bool installCoverageTool(const char * appname, const char * testname)
 namespace QTest
 {
     static QObject *currentTestObject = 0;
+    static QString mainSourcePath;
 
     class TestFunction {
     public:
@@ -2932,6 +2933,13 @@ QString QTest::qFindTestData(const QString& base, const char *file, int line, co
             found = candidate;
     }
 
+    // 6. Try main source directory
+    if (found.isEmpty()) {
+        QString candidate = QTest::mainSourcePath % QLatin1Char('/') % base;
+        if (QFileInfo(candidate).exists())
+            found = candidate;
+    }
+
     if (found.isEmpty()) {
         QTest::qWarn(qPrintable(
             QString::fromLatin1("testdata %1 could not be located!").arg(base)),
@@ -3118,6 +3126,19 @@ QObject *QTest::testObject()
     return currentTestObject;
 }
 
+/*! \internal
+ */
+void QTest::setMainSourcePath(const char *file, const char *builddir)
+{
+    QString mainSourceFile = QFile::decodeName(file);
+    QFileInfo fi;
+    if (builddir)
+        fi.setFile(QDir(QFile::decodeName(builddir)), mainSourceFile);
+    else
+        fi.setFile(mainSourceFile);
+    QTest::mainSourcePath = fi.absolutePath();
+}
+
 /*! \internal
     This function is called by various specializations of QTest::qCompare
     to decide whether to report a failure and to produce verbose test output.

src/testlib/qtestcase.h

@@ -236,6 +236,8 @@ namespace QTest
     Q_TESTLIB_EXPORT int qExec(QObject *testObject, int argc = 0, char **argv = 0);
     Q_TESTLIB_EXPORT int qExec(QObject *testObject, const QStringList &arguments);
 
+    Q_TESTLIB_EXPORT void setMainSourcePath(const char *file, const char *builddir = 0);
+
     Q_TESTLIB_EXPORT bool qVerify(bool statement, const char *statementStr, const char *description,
                                  const char *file, int line);
     Q_TESTLIB_EXPORT void qFail(const char *statementStr, const char *file, int line);

tests/auto/testlib/selftests/blacklisted/BLACKLIST

@@ -0,0 +1,12 @@
+[pass]
+*
+[skip]
+*
+[fail]
+*
+[xpass]
+*
+[xfail]
+*
+[messages]
+*

tests/auto/testlib/selftests/blacklisted/blacklisted.pro

@@ -0,0 +1,7 @@
+SOURCES += tst_blacklisted.cpp
+QT = core testlib-private
+
+mac: CONFIG -= app_bundle
+CONFIG -= debug_and_release_target
+
+TARGET = blacklisted

tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp

@@ -0,0 +1,94 @@
+/****************************************************************************
+**
+** Copyright (C) 2015 The Qt Company Ltd.
+** Contact: http://www.qt.io/licensing/
+**
+** This file is part of the test suite of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL21$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see http://www.qt.io/terms-conditions. For further
+** information use the contact form at http://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 or version 3 as published by the Free
+** Software Foundation and appearing in the file LICENSE.LGPLv21 and
+** LICENSE.LGPLv3 included in the packaging of this file. Please review the
+** following information to ensure the GNU Lesser General Public License
+** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** As a special exception, The Qt Company gives you certain additional
+** rights. These rights are described in The Qt Company LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include <QtCore/QCoreApplication>
+#include <QtTest/QtTest>
+#include <private/qtestlog_p.h>
+
+class tst_Blacklisted : public QObject
+{
+    Q_OBJECT
+
+private slots:
+    void pass();
+    void skip();
+    void fail();
+    void xfail();
+    void xpass();
+
+    // This test function must be last, as it calls qFatal().
+    void messages();
+};
+
+// All the tests below have been blacklisted in blacklisted/BLACKLIST
+
+void tst_Blacklisted::pass()
+{
+    QVERIFY(true);
+}
+
+void tst_Blacklisted::skip()
+{
+    QSKIP("This test should SKIP");
+}
+
+void tst_Blacklisted::fail()
+{
+    QVERIFY2(false, "This test should BFAIL");
+}
+
+void tst_Blacklisted::xfail()
+{
+    QEXPECT_FAIL("", "This test should XFAIL then BFAIL", Abort);
+    QVERIFY(false);
+}
+
+void tst_Blacklisted::xpass()
+{
+    QEXPECT_FAIL("", "This test should XPASS", Abort);
+    QVERIFY2(true, "This test should XPASS, blacklist ignored for XPASS");
+}
+
+void tst_Blacklisted::messages()
+{
+    qWarning("This is a warning that should not appear in silent test output");
+    QWARN("This is an internal testlib warning that should not appear in silent test output");
+    qDebug("This is a debug message that should not appear in silent test output");
+    qCritical("This is a critical message that should not appear in silent test output");
+    qInfo("This is an info message that should not appear in silent test output");
+    QTestLog::info("This is an internal testlib info message that should not appear in silent test output", __FILE__, __LINE__);
+    qFatal("This is a fatal error message that should still appear in silent test output");
+}
+
+QTEST_MAIN(tst_Blacklisted)
+#include "tst_blacklisted.moc"

tests/auto/testlib/selftests/expected_blacklisted.txt

@@ -0,0 +1,26 @@
+********* Start testing of tst_Blacklisted *********
+Config: Using QtTest library @INSERT_QT_VERSION_HERE@, Qt @INSERT_QT_VERSION_HERE@
+PASS   : tst_Blacklisted::initTestCase()
+BPASS  : tst_Blacklisted::pass()
+SKIP   : tst_Blacklisted::skip() This test should SKIP
+   Loc: [tst_blacklisted.cpp(62)]
+BFAIL  : tst_Blacklisted::fail() 'false' returned FALSE. (This test should BFAIL)
+   Loc: [tst_blacklisted.cpp(67)]
+XFAIL  : tst_Blacklisted::xfail() This test should XFAIL then BFAIL
+   Loc: [tst_blacklisted.cpp(73)]
+BPASS  : tst_Blacklisted::xfail()
+XPASS  : tst_Blacklisted::xpass() 'true' returned TRUE unexpectedly. (This test should XPASS, blacklist ignored for XPASS)
+   Loc: [tst_blacklisted.cpp(79)]
+QWARN  : tst_Blacklisted::messages() This is a warning that should not appear in silent test output
+WARNING: tst_Blacklisted::messages() This is an internal testlib warning that should not appear in silent test output
+   Loc: [tst_blacklisted.cpp(85)]
+QDEBUG : tst_Blacklisted::messages() This is a debug message that should not appear in silent test output
+QSYSTEM: tst_Blacklisted::messages() This is a critical message that should not appear in silent test output
+QINFO  : tst_Blacklisted::messages() This is an info message that should not appear in silent test output
+INFO   : tst_Blacklisted::messages() This is an internal testlib info message that should not appear in silent test output
+   Loc: [tst_blacklisted.cpp(89)]
+QFATAL : tst_Blacklisted::messages() This is a fatal error message that should still appear in silent test output
+BFAIL  : tst_Blacklisted::messages() Received a fatal error.
+   Loc: [Unknown file(0)]
+Totals: 1 passed, 1 failed, 1 skipped, 4 blacklisted
+********* Finished testing of tst_Blacklisted *********

tests/auto/testlib/selftests/selftests.pri

@@ -8,6 +8,7 @@ SUBPROGRAMS = \
      benchliboptions \
      benchlibtickcounter \
      benchlibwalltime \
+     blacklisted \
      cmptest \
      commandlinedata \
      counting \

tests/auto/testlib/selftests/selftests.qrc

@@ -151,5 +151,6 @@
         <file>expected_xunit.txt</file>
         <file>expected_xunit.xml</file>
         <file>expected_xunit.xunitxml</file>
+        <file>expected_blacklisted.txt</file>
     </qresource>
 </RCC>

tests/auto/testlib/selftests/tst_selftests.cpp

@@ -352,6 +352,7 @@ void tst_Selftests::runSubTest_data()
         << "benchlibcounting"
         << "benchlibeventcounter"
         << "benchliboptions"
+        << "blacklisted"
         << "cmptest"
         << "commandlinedata"
         << "counting"
@@ -470,6 +471,9 @@ void tst_Selftests::runSubTest_data()
                 if (subtest == "benchliboptions") {
                     continue;
                 }
+                if (subtest == "blacklisted") {
+                    continue;
+                }
                 if (subtest == "printdatatags") {
                     continue;
                 }
@@ -503,7 +507,8 @@ void tst_Selftests::runSubTest_data()
 
             const bool crashes = subtest == QLatin1String("assert") || subtest == QLatin1String("exceptionthrow")
                 || subtest == QLatin1String("fetchbogus") || subtest == QLatin1String("crashedterminate")
-                || subtest == QLatin1String("crashes") || subtest == QLatin1String("silent");
+                || subtest == QLatin1String("crashes") || subtest == QLatin1String("silent")
+                || subtest == QLatin1String("blacklisted");
             QTest::newRow(qPrintable(QString("%1 %2").arg(subtest).arg(loggerSet.name)))
                 << subtest
                 << loggers
@@ -610,6 +615,7 @@ void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& logge
         && subdir != QLatin1String("fetchbogus")
         && subdir != QLatin1String("xunit")
 #ifdef Q_CC_MINGW
+        && subdir != QLatin1String("blacklisted") // calls qFatal()
         && subdir != QLatin1String("silent") // calls qFatal()
 #endif
         && subdir != QLatin1String("benchlibcallgrind"))