whytofly
/
qt6-bb10
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add QStringConverter::encodingForData() 

Add method that tries to determine the encoding of the data
from an initial byte order mark.

Change-Id: I348c51a3d4db9b434af53359b739a7e17acfc760
Reviewed-by: Thiago Macieira <thiago.macieira@intel.com>
bb10
Lars Knoll 2020-04-30 10:30:40 +02:00
parent a639bcda1e
commit 13af1312f7
3 changed files with 99 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
src/corelib/text/qstringconverter.cpp
View File

@ -1491,6 +1491,46 @@ std::optional<QStringConverter::Encoding> QStringConverter::encodingForName(cons
return std::nullopt;
}
std::optional<QStringConverter::Encoding> QStringConverter::encodingForData(const char *buf, qsizetype arraySize, char16_t expectedFirstCharacter)
{
if (arraySize > 3) {
uint uc = qFromUnaligned<uint>(buf);
if (uc == qToBigEndian(uint(QChar::ByteOrderMark)))
return QStringConverter::Utf32BE;
if (uc == qToLittleEndian(uint(QChar::ByteOrderMark)))
return QStringConverter::Utf32LE;
if (expectedFirstCharacter) {
// catch also anything starting with the expected character
if (qToLittleEndian(uc) == expectedFirstCharacter)
return QStringConverter::Utf32LE;
else if (qToBigEndian(uc) == expectedFirstCharacter)
return QStringConverter::Utf32BE;
}
}
if (arraySize > 2) {
static const char utf8bom[] = "\xef\xbb\xbf";
if (memcmp(buf, utf8bom, sizeof(utf8bom) - 1) == 0)
return QStringConverter::Utf8;
}
if (arraySize > 1) {
ushort uc = qFromUnaligned<ushort>(buf);
if (uc == qToBigEndian(ushort(QChar::ByteOrderMark)))
return QStringConverter::Utf16BE;
if (uc == qToLittleEndian(ushort(QChar::ByteOrderMark)))
return QStringConverter::Utf16LE;
if (expectedFirstCharacter) {
// catch also anything starting with the expected character
if (qToLittleEndian(uc) == expectedFirstCharacter)
return QStringConverter::Utf16LE;
else if (qToBigEndian(uc) == expectedFirstCharacter)
return QStringConverter::Utf16BE;
}
}
return std::nullopt;
}
const char *QStringConverter::nameForEncoding(QStringConverter::Encoding e)
{
return encodingInterfaces[int(e)].name;

1
src/corelib/text/qstringconverter.h
View File

@ -167,6 +167,7 @@ public:
Q_CORE_EXPORT static std::optional<Encoding> encodingForName(const char *name);
Q_CORE_EXPORT static const char *nameForEncoding(Encoding e);
Q_CORE_EXPORT static std::optional<Encoding> encodingForData(const char *buf, qsizetype arraySize, char16_t expectedFirstCharacter = 0);
protected:
const Interface *iface;

58
tests/auto/corelib/text/qstringconverter/tst_qstringconverter.cpp
View File

@ -65,6 +65,9 @@ private slots:
void nameForEncoding_data();
void nameForEncoding();
void encodingForData_data();
void encodingForData();
};
void tst_QStringConverter::constructByName()
@ -1664,6 +1667,61 @@ void tst_QStringConverter::nameForEncoding()
QCOMPARE(n, name);
}
void tst_QStringConverter::encodingForData_data()
{
QTest::addColumn<QByteArray>("encoded");
QTest::addColumn<std::optional<QStringConverter::Encoding>>("encoding");
QTest::newRow("utf8 bom")
<< QByteArray("\xef\xbb\xbfhello")
<< std::optional<QStringConverter::Encoding>(QStringConverter::Utf8);
QTest::newRow("utf8 nobom")
<< QByteArray("hello")
<< std::optional<QStringConverter::Encoding>();
QTest::newRow("utf16 bom be")
<< QByteArray("\xfe\xff\0h\0e\0l", 8)
<< std::optional<QStringConverter::Encoding>(QStringConverter::Utf16BE);
QTest::newRow("utf16 bom le")
<< QByteArray("\xff\xfeh\0e\0l\0", 8)
<< std::optional<QStringConverter::Encoding>(QStringConverter::Utf16LE);
QTest::newRow("utf16 nobom be")
<< QByteArray("\0<\0e\0l", 6)
<< std::optional<QStringConverter::Encoding>(QStringConverter::Utf16BE);
QTest::newRow("utf16 nobom le")
<< QByteArray("<\0e\0l\0", 6)
<< std::optional<QStringConverter::Encoding>(QStringConverter::Utf16LE);
QTest::newRow("utf16 nobom no match")
<< QByteArray("h\0e\0l\0", 6)
<< std::optional<QStringConverter::Encoding>();
QTest::newRow("utf32 bom be")
<< QByteArray("\0\0\xfe\xff\0\0\0h\0\0\0e\0\0\0l", 16)
<< std::optional<QStringConverter::Encoding>(QStringConverter::Utf32BE);
QTest::newRow("utf32 bom le")
<< QByteArray("\xff\xfe\0\0h\0\0\0e\0\0\0l\0\0\0", 16)
<< std::optional<QStringConverter::Encoding>(QStringConverter::Utf32LE);
QTest::newRow("utf32 nobom be")
<< QByteArray("\0\0\0<\0\0\0e\0\0\0l", 12)
<< std::optional<QStringConverter::Encoding>(QStringConverter::Utf32BE);
QTest::newRow("utf32 nobom")
<< QByteArray("<\0\0\0e\0\0\0l\0\0\0", 12)
<< std::optional<QStringConverter::Encoding>(QStringConverter::Utf32LE);
QTest::newRow("utf32 nobom no match")
<< QByteArray("\0\0\0h\0\0\0e\0\0\0l", 12)
<< std::optional<QStringConverter::Encoding>();
}
void tst_QStringConverter::encodingForData()
{
QFETCH(QByteArray, encoded);
QFETCH(std::optional<QStringConverter::Encoding>, encoding);
auto e = QStringConverter::encodingForData(encoded.constData(), encoded.size(), char16_t('<'));
QCOMPARE(e, encoding);
}
class LoadAndConvert: public QRunnable
{
public: