qt6-bb10/src/testlib/qtestlog.cpp

718 lines
19 KiB
C++

// Copyright (C) 2021 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
#include <QtTest/qtestassert.h>
#include <QtTest/private/qtestlog_p.h>
#include <QtTest/private/qtestresult_p.h>
#include <QtTest/private/qabstracttestlogger_p.h>
#include <QtTest/private/qplaintestlogger_p.h>
#include <QtTest/private/qcsvbenchmarklogger_p.h>
#include <QtTest/private/qjunittestlogger_p.h>
#include <QtTest/private/qxmltestlogger_p.h>
#include <QtTest/private/qteamcitylogger_p.h>
#include <QtTest/private/qtaptestlogger_p.h>
#if defined(HAVE_XCTEST)
#include <QtTest/private/qxctestlogger_p.h>
#endif
#if defined(Q_OS_DARWIN)
#include <QtTest/private/qappletestlogger_p.h>
#endif
#include <QtCore/qatomic.h>
#include <QtCore/qbytearray.h>
#include <QtCore/qelapsedtimer.h>
#include <QtCore/qlist.h>
#include <QtCore/qmutex.h>
#include <QtCore/qvariant.h>
#if QT_CONFIG(regularexpression)
#include <QtCore/QRegularExpression>
#endif
#include <stdlib.h>
#include <string.h>
#include <limits.h>
#include <vector>
#include <vector>
#include <memory>
QT_BEGIN_NAMESPACE
using namespace Qt::StringLiterals;
static void saveCoverageTool(const char * appname, bool testfailed, bool installedTestCoverage)
{
#ifdef __COVERAGESCANNER__
# if QT_CONFIG(testlib_selfcover)
__coveragescanner_teststate(QTestLog::failCount() > 0 ? "FAILED" :
QTestLog::passCount() > 0 ? "PASSED" : "SKIPPED");
# else
if (!installedTestCoverage)
return;
// install again to make sure the filename is correct.
// without this, a plugin or similar may have changed the filename.
__coveragescanner_install(appname);
__coveragescanner_teststate(testfailed ? "FAILED" : "PASSED");
__coveragescanner_save();
__coveragescanner_testname("");
__coveragescanner_clear();
unsetenv("QT_TESTCOCOON_ACTIVE");
# endif // testlib_selfcover
#else
Q_UNUSED(appname);
Q_UNUSED(testfailed);
Q_UNUSED(installedTestCoverage);
#endif
}
Q_CONSTINIT static QElapsedTimer elapsedFunctionTime;
Q_CONSTINIT static QElapsedTimer elapsedTotalTime;
#define FOREACH_TEST_LOGGER for (const auto &logger : std::as_const(*QTest::loggers()))
namespace QTest {
int fails = 0;
int passes = 0;
int skips = 0;
int blacklists = 0;
enum { Unresolved, Passed, Skipped, Suppressed, Failed } currentTestState;
struct IgnoreResultList
{
inline IgnoreResultList(QtMsgType tp, const QVariant &patternIn)
: type(tp), pattern(patternIn) {}
static inline void clearList(IgnoreResultList *&list)
{
while (list) {
IgnoreResultList *current = list;
list = list->next;
delete current;
}
}
static void append(IgnoreResultList *&list, QtMsgType type, const QVariant &patternIn)
{
QTest::IgnoreResultList *item = new QTest::IgnoreResultList(type, patternIn);
if (!list) {
list = item;
return;
}
IgnoreResultList *last = list;
for ( ; last->next; last = last->next) ;
last->next = item;
}
static bool stringsMatch(const QString &expected, const QString &actual)
{
if (expected == actual)
return true;
// ignore an optional whitespace at the end of str
// (the space was added automatically by ~QDebug() until Qt 5.3,
// so autotests still might expect it)
if (expected.endsWith(u' '))
return actual == QStringView{expected}.left(expected.size() - 1);
return false;
}
inline bool matches(QtMsgType tp, const QString &message) const
{
return tp == type
&& (pattern.userType() == QMetaType::QString ?
stringsMatch(pattern.toString(), message) :
#if QT_CONFIG(regularexpression)
pattern.toRegularExpression().match(message).hasMatch());
#else
false);
#endif
}
QtMsgType type;
QVariant pattern;
IgnoreResultList *next = nullptr;
};
static IgnoreResultList *ignoreResultList = nullptr;
Q_CONSTINIT static QBasicMutex mutex;
static std::vector<QVariant> failOnWarningList;
Q_GLOBAL_STATIC(std::vector<std::unique_ptr<QAbstractTestLogger>>, loggers)
static int verbosity = 0;
static int maxWarnings = 2002;
static bool installedTestCoverage = true;
static QtMessageHandler oldMessageHandler;
static bool handleIgnoredMessage(QtMsgType type, const QString &message)
{
const QMutexLocker mutexLocker(&QTest::mutex);
if (!ignoreResultList)
return false;
IgnoreResultList *last = nullptr;
IgnoreResultList *list = ignoreResultList;
while (list) {
if (list->matches(type, message)) {
// remove the item from the list
if (last)
last->next = list->next;
else
ignoreResultList = list->next;
delete list;
return true;
}
last = list;
list = list->next;
}
return false;
}
static bool handleFailOnWarning(const QMessageLogContext &context, const QString &message)
{
// failOnWarning can be called multiple times per test function, so let
// each call cause a failure if required.
for (const auto &pattern : failOnWarningList) {
if (pattern.metaType() == QMetaType::fromType<QString>()) {
if (message != pattern.toString())
continue;
}
#if QT_CONFIG(regularexpression)
else if (pattern.metaType() == QMetaType::fromType<QRegularExpression>()) {
if (!message.contains(pattern.toRegularExpression()))
continue;
}
#endif
const size_t maxMsgLen = 1024;
char msg[maxMsgLen] = {'\0'};
qsnprintf(msg, maxMsgLen, "Received a warning that resulted in a failure:\n%s",
qPrintable(message));
QTestResult::addFailure(msg, context.file, context.line);
return true;
}
return false;
}
static void messageHandler(QtMsgType type, const QMessageLogContext & context, const QString &message)
{
static QBasicAtomicInt counter = Q_BASIC_ATOMIC_INITIALIZER(QTest::maxWarnings);
if (!QTestLog::hasLoggers()) {
// if this goes wrong, something is seriously broken.
qInstallMessageHandler(oldMessageHandler);
QTEST_ASSERT(QTestLog::hasLoggers());
}
if (handleIgnoredMessage(type, message)) {
// the message is expected, so just swallow it.
return;
}
if (type == QtWarningMsg && handleFailOnWarning(context, message))
return;
if (type != QtFatalMsg) {
if (counter.loadRelaxed() <= 0)
return;
if (!counter.deref()) {
FOREACH_TEST_LOGGER {
logger->addMessage(QAbstractTestLogger::Warn,
QStringLiteral("Maximum amount of warnings exceeded. Use -maxwarnings to override."));
}
return;
}
}
FOREACH_TEST_LOGGER
logger->addMessage(type, context, message);
if (type == QtFatalMsg) {
/* Right now, we're inside the custom message handler and we're
* being qt_message_output in qglobal.cpp. After we return from
* this function, it will proceed with calling exit() and abort()
* and hence crash. Therefore, we call these logging functions such
* that we wrap up nicely, and in particular produce well-formed XML. */
QTestResult::addFailure("Received a fatal error.", context.file, context.line);
QTestLog::leaveTestFunction();
QTestLog::stopLogging();
}
}
}
void QTestLog::enterTestFunction(const char* function)
{
elapsedFunctionTime.restart();
if (printAvailableTags)
return;
QTEST_ASSERT(function);
FOREACH_TEST_LOGGER
logger->enterTestFunction(function);
}
void QTestLog::enterTestData(QTestData *data)
{
QTEST_ASSERT(data);
FOREACH_TEST_LOGGER
logger->enterTestData(data);
}
int QTestLog::unhandledIgnoreMessages()
{
const QMutexLocker mutexLocker(&QTest::mutex);
int i = 0;
QTest::IgnoreResultList *list = QTest::ignoreResultList;
while (list) {
++i;
list = list->next;
}
return i;
}
void QTestLog::leaveTestFunction()
{
if (printAvailableTags)
return;
FOREACH_TEST_LOGGER
logger->leaveTestFunction();
}
void QTestLog::printUnhandledIgnoreMessages()
{
const QMutexLocker mutexLocker(&QTest::mutex);
QString message;
QTest::IgnoreResultList *list = QTest::ignoreResultList;
while (list) {
if (list->pattern.userType() == QMetaType::QString) {
message = "Did not receive message: \"%1\""_L1.arg(list->pattern.toString());
} else {
#if QT_CONFIG(regularexpression)
message = "Did not receive any message matching: \"%1\""_L1.arg(
list->pattern.toRegularExpression().pattern());
#endif
}
FOREACH_TEST_LOGGER
logger->addMessage(QAbstractTestLogger::Info, message);
list = list->next;
}
}
void QTestLog::clearIgnoreMessages()
{
const QMutexLocker mutexLocker(&QTest::mutex);
QTest::IgnoreResultList::clearList(QTest::ignoreResultList);
}
void QTestLog::clearFailOnWarnings()
{
QTest::failOnWarningList.clear();
}
void QTestLog::clearCurrentTestState()
{
clearIgnoreMessages();
clearFailOnWarnings();
QTest::currentTestState = QTest::Unresolved;
}
void QTestLog::addPass(const char *msg)
{
if (printAvailableTags)
return;
QTEST_ASSERT(msg);
Q_ASSERT(QTest::currentTestState == QTest::Unresolved);
++QTest::passes;
QTest::currentTestState = QTest::Passed;
FOREACH_TEST_LOGGER
logger->addIncident(QAbstractTestLogger::Pass, msg);
}
void QTestLog::addFail(const char *msg, const char *file, int line)
{
QTEST_ASSERT(msg);
if (QTest::currentTestState == QTest::Unresolved) {
++QTest::fails;
} else {
// After an XPASS/Continue, or fail or skip in a function the test
// calls, we can subsequently fail.
Q_ASSERT(QTest::currentTestState == QTest::Failed
|| QTest::currentTestState == QTest::Skipped);
}
// It is up to particular loggers to decide whether to report such
// subsequent failures; they may carry useful information.
QTest::currentTestState = QTest::Failed;
FOREACH_TEST_LOGGER
logger->addIncident(QAbstractTestLogger::Fail, msg, file, line);
}
void QTestLog::addXFail(const char *msg, const char *file, int line)
{
QTEST_ASSERT(msg);
// Will be counted in addPass() if we get there.
FOREACH_TEST_LOGGER
logger->addIncident(QAbstractTestLogger::XFail, msg, file, line);
}
void QTestLog::addXPass(const char *msg, const char *file, int line)
{
QTEST_ASSERT(msg);
if (QTest::currentTestState == QTest::Unresolved) {
++QTest::fails;
} else {
// After an XPASS/Continue, we can subsequently XPASS again.
// Likewise after a fail or skip in a function called by the test.
Q_ASSERT(QTest::currentTestState == QTest::Failed
|| QTest::currentTestState == QTest::Skipped);
}
QTest::currentTestState = QTest::Failed;
FOREACH_TEST_LOGGER
logger->addIncident(QAbstractTestLogger::XPass, msg, file, line);
}
void QTestLog::addBPass(const char *msg)
{
QTEST_ASSERT(msg);
Q_ASSERT(QTest::currentTestState == QTest::Unresolved);
++QTest::blacklists; // Not passes ?
QTest::currentTestState = QTest::Suppressed;
FOREACH_TEST_LOGGER
logger->addIncident(QAbstractTestLogger::BlacklistedPass, msg);
}
void QTestLog::addBFail(const char *msg, const char *file, int line)
{
QTEST_ASSERT(msg);
if (QTest::currentTestState == QTest::Unresolved) {
++QTest::blacklists;
} else {
// After a BXPASS/Continue, we can subsequently fail.
// Likewise after a fail or skip in a function called by a test.
Q_ASSERT(QTest::currentTestState == QTest::Suppressed
|| QTest::currentTestState == QTest::Skipped);
}
QTest::currentTestState = QTest::Suppressed;
FOREACH_TEST_LOGGER
logger->addIncident(QAbstractTestLogger::BlacklistedFail, msg, file, line);
}
void QTestLog::addBXPass(const char *msg, const char *file, int line)
{
QTEST_ASSERT(msg);
if (QTest::currentTestState == QTest::Unresolved) {
++QTest::blacklists;
} else {
// After a BXPASS/Continue, we may BXPASS again.
// Likewise after a fail or skip in a function called by a test.
Q_ASSERT(QTest::currentTestState == QTest::Suppressed
|| QTest::currentTestState == QTest::Skipped);
}
QTest::currentTestState = QTest::Suppressed;
FOREACH_TEST_LOGGER
logger->addIncident(QAbstractTestLogger::BlacklistedXPass, msg, file, line);
}
void QTestLog::addBXFail(const char *msg, const char *file, int line)
{
QTEST_ASSERT(msg);
// Will be counted in addBPass() if we get there.
FOREACH_TEST_LOGGER
logger->addIncident(QAbstractTestLogger::BlacklistedXFail, msg, file, line);
}
void QTestLog::addSkip(const char *msg, const char *file, int line)
{
QTEST_ASSERT(msg);
if (QTest::currentTestState == QTest::Unresolved) {
++QTest::skips;
QTest::currentTestState = QTest::Skipped;
} else {
// After an B?XPASS/Continue, we might subsequently skip.
// Likewise after a skip in a function called by a test.
Q_ASSERT(QTest::currentTestState == QTest::Suppressed
|| QTest::currentTestState == QTest::Failed
|| QTest::currentTestState == QTest::Skipped);
}
// It is up to particular loggers to decide whether to report such
// subsequent skips; they may carry useful information.
FOREACH_TEST_LOGGER
logger->addIncident(QAbstractTestLogger::Skip, msg, file, line);
}
void QTestLog::addBenchmarkResults(const QList<QBenchmarkResult> &results)
{
FOREACH_TEST_LOGGER
logger->addBenchmarkResults(results);
}
void QTestLog::startLogging()
{
elapsedTotalTime.start();
elapsedFunctionTime.start();
FOREACH_TEST_LOGGER
logger->startLogging();
QTest::oldMessageHandler = qInstallMessageHandler(QTest::messageHandler);
}
void QTestLog::stopLogging()
{
qInstallMessageHandler(QTest::oldMessageHandler);
FOREACH_TEST_LOGGER {
logger->stopLogging();
}
QTest::loggers()->clear();
saveCoverageTool(QTestResult::currentAppName(), failCount() != 0, QTestLog::installedTestCoverage());
}
void QTestLog::addLogger(LogMode mode, const char *filename)
{
if (filename && strcmp(filename, "-") == 0)
filename = nullptr;
QAbstractTestLogger *logger = nullptr;
switch (mode) {
case QTestLog::Plain:
logger = new QPlainTestLogger(filename);
break;
case QTestLog::CSV:
logger = new QCsvBenchmarkLogger(filename);
break;
case QTestLog::XML:
logger = new QXmlTestLogger(QXmlTestLogger::Complete, filename);
break;
case QTestLog::LightXML:
logger = new QXmlTestLogger(QXmlTestLogger::Light, filename);
break;
case QTestLog::JUnitXML:
logger = new QJUnitTestLogger(filename);
break;
case QTestLog::TeamCity:
logger = new QTeamCityLogger(filename);
break;
case QTestLog::TAP:
logger = new QTapTestLogger(filename);
break;
#if defined(QT_USE_APPLE_UNIFIED_LOGGING)
case QTestLog::Apple:
logger = new QAppleTestLogger;
break;
#endif
#if defined(HAVE_XCTEST)
case QTestLog::XCTest:
logger = new QXcodeTestLogger;
break;
#endif
}
QTEST_ASSERT(logger);
addLogger(logger);
}
/*!
\internal
Adds a new logger to the set of loggers that will be used
to report incidents and messages during testing.
The function takes ownership of the logger.
*/
void QTestLog::addLogger(QAbstractTestLogger *logger)
{
QTEST_ASSERT(logger);
QTest::loggers()->emplace_back(logger);
}
bool QTestLog::hasLoggers()
{
return !QTest::loggers()->empty();
}
/*!
\internal
Returns true if all loggers support repeated test runs
*/
bool QTestLog::isRepeatSupported()
{
FOREACH_TEST_LOGGER {
if (!logger->isRepeatSupported())
return false;
}
return true;
}
bool QTestLog::loggerUsingStdout()
{
FOREACH_TEST_LOGGER {
if (logger->isLoggingToStdout())
return true;
}
return false;
}
void QTestLog::warn(const char *msg, const char *file, int line)
{
QTEST_ASSERT(msg);
FOREACH_TEST_LOGGER
logger->addMessage(QAbstractTestLogger::Warn, QString::fromUtf8(msg), file, line);
}
void QTestLog::info(const char *msg, const char *file, int line)
{
QTEST_ASSERT(msg);
FOREACH_TEST_LOGGER
logger->addMessage(QAbstractTestLogger::Info, QString::fromUtf8(msg), file, line);
}
void QTestLog::setVerboseLevel(int level)
{
QTest::verbosity = level;
}
int QTestLog::verboseLevel()
{
return QTest::verbosity;
}
void QTestLog::ignoreMessage(QtMsgType type, const char *msg)
{
QTEST_ASSERT(msg);
const QMutexLocker mutexLocker(&QTest::mutex);
QTest::IgnoreResultList::append(QTest::ignoreResultList, type, QString::fromUtf8(msg));
}
#if QT_CONFIG(regularexpression)
void QTestLog::ignoreMessage(QtMsgType type, const QRegularExpression &expression)
{
QTEST_ASSERT(expression.isValid());
const QMutexLocker mutexLocker(&QTest::mutex);
QTest::IgnoreResultList::append(QTest::ignoreResultList, type, QVariant(expression));
}
#endif // QT_CONFIG(regularexpression)
void QTestLog::failOnWarning()
{
QTest::failOnWarningList.push_back({});
}
void QTestLog::failOnWarning(const char *msg)
{
QTest::failOnWarningList.push_back(QString::fromUtf8(msg));
}
#if QT_CONFIG(regularexpression)
void QTestLog::failOnWarning(const QRegularExpression &expression)
{
QTEST_ASSERT(expression.isValid());
QTest::failOnWarningList.push_back(QVariant::fromValue(expression));
}
#endif // QT_CONFIG(regularexpression)
void QTestLog::setMaxWarnings(int m)
{
QTest::maxWarnings = m <= 0 ? INT_MAX : m + 2;
}
bool QTestLog::printAvailableTags = false;
void QTestLog::setPrintAvailableTagsMode()
{
printAvailableTags = true;
}
int QTestLog::passCount()
{
return QTest::passes;
}
int QTestLog::failCount()
{
return QTest::fails;
}
int QTestLog::skipCount()
{
return QTest::skips;
}
int QTestLog::blacklistCount()
{
return QTest::blacklists;
}
int QTestLog::totalCount()
{
return passCount() + failCount() + skipCount() + blacklistCount();
}
void QTestLog::resetCounters()
{
QTest::passes = 0;
QTest::fails = 0;
QTest::skips = 0;
}
void QTestLog::setInstalledTestCoverage(bool installed)
{
QTest::installedTestCoverage = installed;
}
bool QTestLog::installedTestCoverage()
{
return QTest::installedTestCoverage;
}
qint64 QTestLog::nsecsTotalTime()
{
return elapsedTotalTime.nsecsElapsed();
}
qint64 QTestLog::nsecsFunctionTime()
{
return elapsedFunctionTime.nsecsElapsed();
}
QT_END_NAMESPACE
#include "moc_qtestlog_p.cpp"