266 lines
9.8 KiB
C++
266 lines
9.8 KiB
C++
/****************************************************************************
|
|
**
|
|
** Copyright (C) 2018 The Qt Company Ltd.
|
|
** Contact: https://www.qt.io/licensing/
|
|
**
|
|
** This file is part of the QtTest module of the Qt Toolkit.
|
|
**
|
|
** $QT_BEGIN_LICENSE:LGPL$
|
|
** Commercial License Usage
|
|
** Licensees holding valid commercial Qt licenses may use this file in
|
|
** accordance with the commercial license agreement provided with the
|
|
** Software or, alternatively, in accordance with the terms contained in
|
|
** a written agreement between you and The Qt Company. For licensing terms
|
|
** and conditions see https://www.qt.io/terms-conditions. For further
|
|
** information use the contact form at https://www.qt.io/contact-us.
|
|
**
|
|
** GNU Lesser General Public License Usage
|
|
** Alternatively, this file may be used under the terms of the GNU Lesser
|
|
** General Public License version 3 as published by the Free Software
|
|
** Foundation and appearing in the file LICENSE.LGPL3 included in the
|
|
** packaging of this file. Please review the following information to
|
|
** ensure the GNU Lesser General Public License version 3 requirements
|
|
** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
|
|
**
|
|
** GNU General Public License Usage
|
|
** Alternatively, this file may be used under the terms of the GNU
|
|
** General Public License version 2.0 or (at your option) the GNU General
|
|
** Public license version 3 or any later version approved by the KDE Free
|
|
** Qt Foundation. The licenses are as published by the Free Software
|
|
** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
|
|
** included in the packaging of this file. Please review the following
|
|
** information to ensure the GNU General Public License requirements will
|
|
** be met: https://www.gnu.org/licenses/gpl-2.0.html and
|
|
** https://www.gnu.org/licenses/gpl-3.0.html.
|
|
**
|
|
** $QT_END_LICENSE$
|
|
**
|
|
****************************************************************************/
|
|
|
|
#include "qtaptestlogger_p.h"
|
|
|
|
#include "qtestlog_p.h"
|
|
#include "qtestresult_p.h"
|
|
#include "qtestassert.h"
|
|
|
|
#if QT_CONFIG(regularexpression)
|
|
# include <QtCore/qregularexpression.h>
|
|
#endif
|
|
|
|
QT_BEGIN_NAMESPACE
|
|
|
|
QTapTestLogger::QTapTestLogger(const char *filename)
|
|
: QAbstractTestLogger(filename)
|
|
, m_wasExpectedFail(false)
|
|
{
|
|
}
|
|
|
|
QTapTestLogger::~QTapTestLogger()
|
|
{
|
|
}
|
|
|
|
void QTapTestLogger::startLogging()
|
|
{
|
|
QAbstractTestLogger::startLogging();
|
|
|
|
QTestCharBuffer preamble;
|
|
QTest::qt_asprintf(&preamble, "TAP version 13\n"
|
|
// By convention, test suite names are output as diagnostics lines
|
|
// This is a pretty poor convention, as consumers will then treat
|
|
// actual diagnostics, e.g. qDebug, as test suite names o_O
|
|
"# %s\n", QTestResult::currentTestObjectName());
|
|
outputString(preamble.data());
|
|
}
|
|
|
|
void QTapTestLogger::stopLogging()
|
|
{
|
|
const int total = QTestLog::totalCount();
|
|
|
|
QTestCharBuffer testPlanAndStats;
|
|
QTest::qt_asprintf(&testPlanAndStats,
|
|
"1..%d\n"
|
|
"# tests %d\n"
|
|
"# pass %d\n"
|
|
"# fail %d\n",
|
|
total, total, QTestLog::passCount(), QTestLog::failCount());
|
|
outputString(testPlanAndStats.data());
|
|
|
|
QAbstractTestLogger::stopLogging();
|
|
}
|
|
|
|
void QTapTestLogger::enterTestFunction(const char *function)
|
|
{
|
|
Q_UNUSED(function);
|
|
m_wasExpectedFail = false;
|
|
}
|
|
|
|
void QTapTestLogger::enterTestData(QTestData *data)
|
|
{
|
|
Q_UNUSED(data);
|
|
m_wasExpectedFail = false;
|
|
}
|
|
|
|
using namespace QTestPrivate;
|
|
|
|
void QTapTestLogger::outputTestLine(bool ok, int testNumber, QTestCharBuffer &directive)
|
|
{
|
|
QTestCharBuffer testIdentifier;
|
|
QTestPrivate::generateTestIdentifier(&testIdentifier, TestFunction | TestDataTag);
|
|
|
|
QTestCharBuffer testLine;
|
|
QTest::qt_asprintf(&testLine, "%s %d - %s%s\n",
|
|
ok ? "ok" : "not ok", testNumber, testIdentifier.data(), directive.data());
|
|
|
|
outputString(testLine.data());
|
|
}
|
|
|
|
void QTapTestLogger::addIncident(IncidentTypes type, const char *description,
|
|
const char *file, int line)
|
|
{
|
|
if (m_wasExpectedFail && type == Pass) {
|
|
// XFail comes with a corresponding Pass incident, but we only want
|
|
// to emit a single test point for it, so skip the this pass.
|
|
return;
|
|
}
|
|
|
|
bool ok = type == Pass || type == XPass || type == BlacklistedPass || type == BlacklistedXPass;
|
|
|
|
QTestCharBuffer directive;
|
|
if (type == XFail || type == XPass || type == BlacklistedFail || type == BlacklistedPass
|
|
|| type == BlacklistedXFail || type == BlacklistedXPass) {
|
|
// We treat expected or blacklisted failures/passes as TODO-failures/passes,
|
|
// which should be treated as soft issues by consumers. Not all do though :/
|
|
QTest::qt_asprintf(&directive, " # TODO %s", description);
|
|
}
|
|
|
|
int testNumber = QTestLog::totalCount();
|
|
if (type == XFail) {
|
|
// The global test counter hasn't been updated yet for XFail
|
|
testNumber += 1;
|
|
}
|
|
|
|
outputTestLine(ok, testNumber, directive);
|
|
|
|
if (!ok) {
|
|
// All failures need a diagnostics sections to not confuse consumers
|
|
|
|
// The indent needs to be two spaces for maximum compatibility
|
|
#define YAML_INDENT " "
|
|
|
|
outputString(YAML_INDENT "---\n");
|
|
|
|
if (type != XFail) {
|
|
#if QT_CONFIG(regularexpression)
|
|
// This is fragile, but unfortunately testlib doesn't plumb
|
|
// the expected and actual values to the loggers (yet).
|
|
static QRegularExpression verifyRegex(
|
|
QLatin1String("^'(?<actualexpression>.*)' returned (?<actual>\\w+).+\\((?<message>.*)\\)$"));
|
|
|
|
static QRegularExpression comparRegex(
|
|
QLatin1String("^(?<message>.*)\n"
|
|
"\\s*Actual\\s+\\((?<actualexpression>.*)\\)\\s*: (?<actual>.*)\n"
|
|
"\\s*Expected\\s+\\((?<expectedexpresssion>.*)\\)\\s*: (?<expected>.*)$"));
|
|
|
|
QString descriptionString = QString::fromUtf8(description);
|
|
QRegularExpressionMatch match = verifyRegex.match(descriptionString);
|
|
if (!match.hasMatch())
|
|
match = comparRegex.match(descriptionString);
|
|
|
|
if (match.hasMatch()) {
|
|
bool isVerify = match.regularExpression() == verifyRegex;
|
|
QString message = match.captured(QLatin1String("message"));
|
|
QString expected;
|
|
QString actual;
|
|
|
|
if (isVerify) {
|
|
QString expression = QLatin1String(" (")
|
|
% match.captured(QLatin1String("actualexpression")) % QLatin1Char(')') ;
|
|
actual = match.captured(QLatin1String("actual")).toLower() % expression;
|
|
expected = (actual.startsWith(QLatin1String("true")) ? QLatin1String("false") : QLatin1String("true")) % expression;
|
|
if (message.isEmpty())
|
|
message = QLatin1String("Verification failed");
|
|
} else {
|
|
expected = match.captured(QLatin1String("expected"))
|
|
% QLatin1String(" (") % match.captured(QLatin1String("expectedexpresssion")) % QLatin1Char(')');
|
|
actual = match.captured(QLatin1String("actual"))
|
|
% QLatin1String(" (") % match.captured(QLatin1String("actualexpression")) % QLatin1Char(')');
|
|
}
|
|
|
|
QTestCharBuffer diagnosticsYamlish;
|
|
QTest::qt_asprintf(&diagnosticsYamlish,
|
|
YAML_INDENT "type: %s\n"
|
|
YAML_INDENT "message: %s\n"
|
|
|
|
// Some consumers understand 'wanted/found', while others need
|
|
// 'expected/actual', so we do both for maximum compatibility.
|
|
YAML_INDENT "wanted: %s\n"
|
|
YAML_INDENT "found: %s\n"
|
|
YAML_INDENT "expected: %s\n"
|
|
YAML_INDENT "actual: %s\n",
|
|
|
|
isVerify ? "QVERIFY" : "QCOMPARE",
|
|
qPrintable(message),
|
|
qPrintable(expected), qPrintable(actual),
|
|
qPrintable(expected), qPrintable(actual)
|
|
);
|
|
|
|
outputString(diagnosticsYamlish.data());
|
|
} else {
|
|
QTestCharBuffer unparsableDescription;
|
|
QTest::qt_asprintf(&unparsableDescription,
|
|
YAML_INDENT "# %s\n", description);
|
|
outputString(unparsableDescription.data());
|
|
}
|
|
#else
|
|
QTestCharBuffer unparsableDescription;
|
|
QTest::qt_asprintf(&unparsableDescription,
|
|
YAML_INDENT "# %s\n", description);
|
|
outputString(unparsableDescription.data());
|
|
#endif
|
|
}
|
|
|
|
if (file) {
|
|
QTestCharBuffer location;
|
|
QTest::qt_asprintf(&location,
|
|
// The generic 'at' key is understood by most consumers.
|
|
YAML_INDENT "at: %s::%s() (%s:%d)\n"
|
|
|
|
// The file and line keys are for consumers that are able
|
|
// to read more granular location info.
|
|
YAML_INDENT "file: %s\n"
|
|
YAML_INDENT "line: %d\n",
|
|
|
|
QTestResult::currentTestObjectName(),
|
|
QTestResult::currentTestFunction(),
|
|
file, line, file, line
|
|
);
|
|
outputString(location.data());
|
|
}
|
|
|
|
outputString(YAML_INDENT "...\n");
|
|
}
|
|
|
|
m_wasExpectedFail = type == XFail;
|
|
}
|
|
|
|
void QTapTestLogger::addMessage(MessageTypes type, const QString &message,
|
|
const char *file, int line)
|
|
{
|
|
Q_UNUSED(file);
|
|
Q_UNUSED(line);
|
|
|
|
if (type == Skip) {
|
|
QTestCharBuffer directive;
|
|
QTest::qt_asprintf(&directive, " # SKIP %s", message.toUtf8().constData());
|
|
outputTestLine(/* ok = */ true, QTestLog::totalCount(), directive);
|
|
return;
|
|
}
|
|
|
|
QTestCharBuffer diagnostics;
|
|
QTest::qt_asprintf(&diagnostics, "# %s\n", qPrintable(message));
|
|
outputString(diagnostics.data());
|
|
}
|
|
|
|
QT_END_NAMESPACE
|
|
|