whytofly
/
qt6-bb10
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Start work on a new API to replace QTextCodec 

The new QStringEncoder and QStringDecoder classes
(with a common QStringConverter base class) are
there to replace QTextCodec in Qt 6.

It currently uses a trivial wrapper around the utf
encoding functionality.

Added some autotests, mostly copied from the text codec
tests.

Change-Id: Ib6eeee55fba918b9424be244cbda9dfd5096f7eb
Reviewed-by: Thiago Macieira <thiago.macieira@intel.com>
bb10
Lars Knoll 2020-04-30 10:50:44 +02:00
parent f437c8c5f9
commit f64a6bd638
16 changed files with 2001 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
src/corelib/codecs/qtextcodec.cpp
View File

@ -338,26 +338,6 @@ static void setup() {}
Constructs a ConverterState object initialized with the given \a flags.
*/
/*!
Destroys the ConverterState object.
*/
QTextCodec::ConverterState::~ConverterState()
{
clear();
}
void QTextCodec::ConverterState::clear()
{
if (clearFn)
clearFn(this);
remainingChars = 0;
invalidChars = 0;
state_data[0] = 0;
state_data[1] = 0;
state_data[2] = 0;
state_data[3] = 0;
}
/*!
\class QTextCodec
\inmodule QtCore

33
src/corelib/codecs/qtextcodec.h
View File

@ -40,7 +40,7 @@
#ifndef QTEXTCODEC_H
#define QTEXTCODEC_H
#include <QtCore/qstring.h>
#include <QtCore/qstringconverter.h>
#include <QtCore/qlist.h>
QT_REQUIRE_CONFIG(textcodec);
@ -53,10 +53,13 @@ class QIODevice;
class QTextDecoder;
class QTextEncoder;
class Q_CORE_EXPORT QTextCodec
class Q_CORE_EXPORT QTextCodec : public QStringConverterBase
{
Q_DISABLE_COPY(QTextCodec)
public:
using ConversionFlags = QStringConverterBase::Flags;
using ConverterState = QStringConverterBase::State;
static QTextCodec* codecForName(const QByteArray &name);
static QTextCodec* codecForName(const char *name) { return codecForName(QByteArray(name)); }
static QTextCodec* codecForMib(int mib);
@ -89,31 +92,6 @@ public:
QByteArray fromUnicode(const QString& uc) const;
#endif
QByteArray fromUnicode(QStringView uc) const;
enum ConversionFlag {
DefaultConversion,
ConvertInvalidToNull = 0x80000000,
IgnoreHeader = 0x1
};
Q_DECLARE_FLAGS(ConversionFlags, ConversionFlag)
struct Q_CORE_EXPORT ConverterState {
ConverterState(ConversionFlags f = DefaultConversion)
: flags(f), state_data{0, 0, 0, 0} {}
~ConverterState();
ConversionFlags flags;
int remainingChars = 0;
int invalidChars = 0;
union {
uint state_data[4];
void *d[2];
};
void clear();
using ClearDataFn = void (*)(ConverterState *);
ClearDataFn clearFn = nullptr;
private:
Q_DISABLE_COPY(ConverterState)
};
QString toUnicode(const char *in, int length, ConverterState *state = nullptr) const
{ return convertToUnicode(in, length, state); }
@ -137,7 +115,6 @@ protected:
private:
friend struct QCoreGlobalData;
};
Q_DECLARE_OPERATORS_FOR_FLAGS(QTextCodec::ConversionFlags)
class Q_CORE_EXPORT QTextEncoder {
Q_DISABLE_COPY(QTextEncoder)

2
src/corelib/serialization/qtextstream.cpp
View File

@ -355,7 +355,7 @@ QTextStreamPrivate::~QTextStreamPrivate()
#if QT_CONFIG(textcodec)
static void resetCodecConverterStateHelper(QTextCodec::ConverterState *state)
{
state->~ConverterState();
state->~State();
new (state) QTextCodec::ConverterState;
}

186
src/corelib/text/qstringconverter.cpp Normal file
View File

@ -0,0 +1,186 @@
/****************************************************************************
**
** Copyright (C) 2020 The Qt Company Ltd.
** Copyright (C) 2018 Intel Corporation.
** Contact: https://www.qt.io/licensing/
**
** This file is part of the QtCore module of the Qt Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 3 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL3 included in the
** packaging of this file. Please review the following information to
** ensure the GNU Lesser General Public License version 3 requirements
** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 2.0 or (at your option) the GNU General
** Public license version 3 or any later version approved by the KDE Free
** Qt Foundation. The licenses are as published by the Free Software
** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-2.0.html and
** https://www.gnu.org/licenses/gpl-3.0.html.
**
** $QT_END_LICENSE$
**
****************************************************************************/
#include <qstringconverter.h>
#include <private/qutfcodec_p.h>
QT_BEGIN_NAMESPACE
/*!
\enum QStringConverter::Flag
\value DefaultConversion No flag is set.
\value ConvertInvalidToNull If this flag is set, each invalid input
character is output as a null character.
\value IgnoreHeader Ignore any Unicode byte-order mark and don't generate any.
\value Stateless Ignore possible converter states between different function calls
to encode or decode strings.
*/
void QStringConverter::State::clear()
{
if (clearFn)
clearFn(this);
state_data[0] = state_data[1] = state_data[2] = state_data[3] = 0;
remainingChars = 0;
invalidChars = 0;
}
static QChar *fromUtf8(QChar *out, const char *in, qsizetype length, QStringConverter::State *state)
{
QString s = QUtf8::convertToUnicode(in, length, state);
memcpy(out, s.constData(), s.length()*sizeof(QChar));
return out + s.length();
}
static char *toUtf8(char *out, QStringView in, QStringConverter::State *state)
{
QByteArray s = QUtf8::convertFromUnicode(in.data(), in.length(), state);
memcpy(out, s.constData(), s.length());
return out + s.length();
}
static QChar *fromUtf16(QChar *out, const char *in, qsizetype length, QStringConverter::State *state)
{
QString s = QUtf16::convertToUnicode(in, length, state);
memcpy(out, s.constData(), s.length()*sizeof(QChar));
return out + s.length();
}
static char *toUtf16(char *out, QStringView in, QStringConverter::State *state)
{
QByteArray s = QUtf16::convertFromUnicode(in.data(), in.length(), state);
memcpy(out, s.constData(), s.length());
return out + s.length();
}
static QChar *fromUtf16BE(QChar *out, const char *in, qsizetype length, QStringConverter::State *state)
{
QString s = QUtf16::convertToUnicode(in, length, state, BigEndianness);
memcpy(out, s.constData(), s.length()*sizeof(QChar));
return out + s.length();
}
static char *toUtf16BE(char *out, QStringView in, QStringConverter::State *state)
{
QByteArray s = QUtf16::convertFromUnicode(in.data(), in.length(), state, BigEndianness);
memcpy(out, s.constData(), s.length());
return out + s.length();
}
static QChar *fromUtf16LE(QChar *out, const char *in, qsizetype length, QStringConverter::State *state)
{
QString s = QUtf16::convertToUnicode(in, length, state, LittleEndianness);
memcpy(out, s.constData(), s.length()*sizeof(QChar));
return out + s.length();
}
static char *toUtf16LE(char *out, QStringView in, QStringConverter::State *state)
{
QByteArray s = QUtf16::convertFromUnicode(in.data(), in.length(), state, LittleEndianness);
memcpy(out, s.constData(), s.length());
return out + s.length();
}
static QChar *fromUtf32(QChar *out, const char *in, qsizetype length, QStringConverter::State *state)
{
QString s = QUtf32::convertToUnicode(in, length, state);
memcpy(out, s.constData(), s.length()*sizeof(QChar));
return out + s.length();
}
static char *toUtf32(char *out, QStringView in, QStringConverter::State *state)
{
QByteArray s = QUtf32::convertFromUnicode(in.data(), in.length(), state);
memcpy(out, s.constData(), s.length());
return out + s.length();
}
static QChar *fromUtf32BE(QChar *out, const char *in, qsizetype length, QStringConverter::State *state)
{
QString s = QUtf32::convertToUnicode(in, length, state, BigEndianness);
memcpy(out, s.constData(), s.length()*sizeof(QChar));
return out + s.length();
}
static char *toUtf32BE(char *out, QStringView in, QStringConverter::State *state)
{
QByteArray s = QUtf32::convertFromUnicode(in.data(), in.length(), state, BigEndianness);
memcpy(out, s.constData(), s.length());
return out + s.length();
}
static QChar *fromUtf32LE(QChar *out, const char *in, qsizetype length, QStringConverter::State *state)
{
QString s = QUtf32::convertToUnicode(in, length, state, LittleEndianness);
memcpy(out, s.constData(), s.length()*sizeof(QChar));
return out + s.length();
}
static char *toUtf32LE(char *out, QStringView in, QStringConverter::State *state)
{
QByteArray s = QUtf32::convertFromUnicode(in.data(), in.length(), state, LittleEndianness);
memcpy(out, s.constData(), s.length());
return out + s.length();
}
static qsizetype fromUtf8Len(qsizetype l) { return l + 1; }
static qsizetype toUtf8Len(qsizetype l) { return 3*(l + 1); }
static qsizetype fromUtf16Len(qsizetype l) { return l/2 + 2; }
static qsizetype toUtf16Len(qsizetype l) { return 2*(l + 1); }
static qsizetype fromUtf32Len(qsizetype l) { return l + 1; }
static qsizetype toUtf32Len(qsizetype l) { return 4*(l + 1); }
const QStringConverter::Interface QStringConverter::encodingInterfaces[QStringConverter::LastEncoding + 1] =
{
{ fromUtf8, fromUtf8Len, toUtf8, toUtf8Len },
{ fromUtf16, fromUtf16Len, toUtf16, toUtf16Len },
{ fromUtf16LE, fromUtf16Len, toUtf16LE, toUtf16Len },
{ fromUtf16BE, fromUtf16Len, toUtf16BE, toUtf16Len },
{ fromUtf32, fromUtf32Len, toUtf32, toUtf32Len },
{ fromUtf32LE, fromUtf32Len, toUtf32LE, toUtf32Len },
{ fromUtf32BE, fromUtf32Len, toUtf32BE, toUtf32Len }
};
QT_END_NAMESPACE

195
src/corelib/text/qstringconverter.h Normal file
View File

@ -0,0 +1,195 @@
/****************************************************************************
**
** Copyright (C) 2020 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of the QtCore module of the Qt Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 3 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL3 included in the
** packaging of this file. Please review the following information to
** ensure the GNU Lesser General Public License version 3 requirements
** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 2.0 or (at your option) the GNU General
** Public license version 3 or any later version approved by the KDE Free
** Qt Foundation. The licenses are as published by the Free Software
** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-2.0.html and
** https://www.gnu.org/licenses/gpl-3.0.html.
**
** $QT_END_LICENSE$
**
****************************************************************************/
#ifndef QSTRINGCONVERTER_H
#define QSTRINGCONVERTER_H
#include <QtCore/qstring.h>
QT_BEGIN_NAMESPACE
// work around a compiler bug in GCC 7
#if defined(Q_CC_GNU) && __GNUC__ == 7
#define QSTRINGCONVERTER_CONSTEXPR
#else
#define QSTRINGCONVERTER_CONSTEXPR constexpr
#endif
class QStringConverterBase
{
public:
enum Flag {
DefaultConversion,
ConvertInvalidToNull = 0x1,
IgnoreHeader = 0x2,
Stateless = 0x4
};
Q_DECLARE_FLAGS(Flags, Flag)
struct State {
constexpr State(Flags f = DefaultConversion)
: flags(f), state_data{0, 0, 0, 0} {}
~State() { clear(); }
Q_CORE_EXPORT void clear();
Flags flags;
qsizetype remainingChars = 0;
qsizetype invalidChars = 0;
union {
uint state_data[4];
void *d[2];
};
using ClearDataFn = void (*)(State *);
ClearDataFn clearFn = nullptr;
private:
Q_DISABLE_COPY(State)
};
};
Q_DECLARE_OPERATORS_FOR_FLAGS(QStringConverterBase::Flags)
class QStringConverter : public QStringConverterBase
{
public:
enum Encoding {
Utf8,
Utf16,
Utf16LE,
Utf16BE,
Utf32,
Utf32LE,
Utf32BE,
LastEncoding = Utf32BE
};
protected:
struct Interface
{
// ### FIXME: need a QByteArrayView
using DecoderFn = QChar * (*)(QChar *out, const char *in, qsizetype length, State *state);
using LengthFn = qsizetype (*)(qsizetype inLength);
using EncoderFn = char * (*)(char *out, QStringView in, State *state);
DecoderFn toUtf16 = nullptr;
LengthFn toUtf16Len = nullptr;
EncoderFn fromUtf16 = nullptr;
LengthFn fromUtf16Len = nullptr;
};
QSTRINGCONVERTER_CONSTEXPR QStringConverter(Encoding encoding, Flags f)
: iface(&encodingInterfaces[int(encoding)]), state(f)
{}
QSTRINGCONVERTER_CONSTEXPR QStringConverter(const Interface *i)
: iface(i)
{}
public:
bool isValid() const { return iface != nullptr; }
void resetState()
{
state.clear();
}
bool hasError() const { return state.invalidChars != 0; }
protected:
const Interface *iface;
State state;
private:
Q_CORE_EXPORT static const Interface encodingInterfaces[Encoding::LastEncoding + 1];
};
class QStringEncoder : public QStringConverter
{
protected:
QSTRINGCONVERTER_CONSTEXPR QStringEncoder(const Interface *i)
: QStringConverter(i)
{}
public:
// ### We shouldn't write a BOM by default. Need to resolve this
// while keeping compat with QTextCodec
QSTRINGCONVERTER_CONSTEXPR QStringEncoder(Encoding encoding, Flags flags = IgnoreHeader)
: QStringConverter(encoding, flags)
{}
QByteArray operator()(const QChar *in, qsizetype length)
{ return (*this)(QStringView(in, length)); }
QByteArray operator()(QStringView in)
{
QByteArray result(iface->fromUtf16Len(in.size()), Qt::Uninitialized);
char *out = result.data();
// ### Fixme: needs to be moved into the conversion methods to honor the other flags
out = iface->fromUtf16(out, in, state.flags & Stateless ? nullptr : &state);
result.truncate(out - result.constData());
return result;
}
};
class QStringDecoder : public QStringConverter
{
protected:
QSTRINGCONVERTER_CONSTEXPR QStringDecoder(const Interface *i)
: QStringConverter(i)
{}
public:
QSTRINGCONVERTER_CONSTEXPR QStringDecoder(Encoding encoding, Flags flags = DefaultConversion)
: QStringConverter(encoding, flags)
{}
QString operator()(const char *in, qsizetype length)
{
QString result(iface->toUtf16Len(length), Qt::Uninitialized);
QChar *out = result.data();
// ### Fixme: needs to be moved into the conversion methods
out = iface->toUtf16(out, in, length, state.flags & Stateless ? nullptr : &state);
result.truncate(out - result.constData());
return result;
}
QString operator()(const QByteArray &ba)
{ return (*this)(ba.constData(), ba.size()); }
QString operator()(const char *chars)
{ return (*this)(chars, strlen(chars)); }
};
QT_END_NAMESPACE
#undef QSTRINGCONVERTER_CONSTEXPR
#endif

2
src/corelib/text/text.pri
View File

@ -19,6 +19,7 @@ HEADERS += \
text/qstringalgorithms.h \
text/qstringalgorithms_p.h \
text/qstringbuilder.h \
text/qstringconverter.h \
text/qstringiterator_p.h \
text/qstringlist.h \
text/qstringliteral.h \
@ -39,6 +40,7 @@ SOURCES += \
text/qregexp.cpp \
text/qstring.cpp \
text/qstringbuilder.cpp \
text/qstringconverter.cpp \
text/qstringlist.cpp \
text/qstringview.cpp \
text/qtextboundaryfinder.cpp \

1
src/tools/bootstrap/.prev_CMakeLists.txt
View File

@ -107,6 +107,7 @@ qt_add_module(Bootstrap
../../corelib/text/qstring.cpp
../../corelib/text/qstring_compat.cpp
../../corelib/text/qstringbuilder.cpp
../../corelib/text/qstringconverter.cpp
../../corelib/text/qstringlist.cpp
../../corelib/text/qstringview.cpp
../../corelib/text/qvsnprintf.cpp

1
src/tools/bootstrap/CMakeLists.txt
View File

@ -108,6 +108,7 @@ qt_extend_target(Bootstrap
../../corelib/text/qstring.cpp
../../corelib/text/qstring_compat.cpp
../../corelib/text/qstringbuilder.cpp
../../corelib/text/qstringconverter.cpp
../../corelib/text/qstringlist.cpp
../../corelib/text/qstringview.cpp
../../corelib/text/qvsnprintf.cpp

1
src/tools/bootstrap/bootstrap.pro
View File

@ -93,6 +93,7 @@ SOURCES += \
../../corelib/text/qregularexpression.cpp \
../../corelib/text/qstring.cpp \
../../corelib/text/qstringbuilder.cpp \
../../corelib/text/qstringconverter.cpp \
../../corelib/text/qstring_compat.cpp \
../../corelib/text/qstringlist.cpp \
../../corelib/text/qstringview.cpp \

1
tests/auto/corelib/codecs/qtextcodec/tst_qtextcodec.cpp
View File

@ -1686,7 +1686,6 @@ void tst_QTextCodec::utf8stateful()
QVERIFY(utf8codec);
QTextCodec::ConverterState state;
memset(&state, 0, sizeof state);
QString decoded1 = utf8codec->toUnicode(buffer1, buffer1.size(), &state);
if (result.isNull()) {

1
tests/auto/corelib/text/CMakeLists.txt
View File

@ -14,6 +14,7 @@ add_subdirectory(qstring)
add_subdirectory(qstring_no_cast_from_bytearray)
add_subdirectory(qstringapisymmetry)
add_subdirectory(qstringbuilder)
add_subdirectory(qstringconverter)
add_subdirectory(qstringiterator)
add_subdirectory(qstringlist)
add_subdirectory(qstringmatcher)

10
tests/auto/corelib/text/qstringconverter/CMakeLists.txt Normal file
View File

@ -0,0 +1,10 @@
# Generated from qstringconverter.pro.
#####################################################################
## tst_qstringconverter Test:
#####################################################################
qt_add_test(tst_qstringconverter
SOURCES
tst_qstringconverter.cpp
)

5
tests/auto/corelib/text/qstringconverter/qstringconverter.pro Normal file
View File

@ -0,0 +1,5 @@
CONFIG += testcase
QT = core testlib
SOURCES = tst_qstringconverter.cpp
TARGET = tst_qstringconverter

1591
tests/auto/corelib/text/qstringconverter/tst_qstringconverter.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

1
tests/auto/corelib/text/qstringconverter/utf8.txt Normal file
View File

@ -0,0 +1 @@
<doc>𐀀􏿽</doc>

1
tests/auto/corelib/text/text.pro
View File

@ -15,6 +15,7 @@ SUBDIRS = \
qstring_no_cast_from_bytearray \
qstringapisymmetry \
qstringbuilder \
qstringconverter \
qstringiterator \
qstringlist \
qstringmatcher \