QStringConverter: use the QUtf8 codec when Windows is using UTF-8

The QLocal8Bit implementation assumes that there's at most one
continuation byte -- that is, that all codecs are either Single or
Double Byte Character Sets (SBCS or DBCS). It appears to be the case for
all Windows default codepages, except for CP_UTF8, which is an opt-in
anyway.

Instead of fixing our codec, let's just use the optimized UTF-8
implementation.

[ChangeLog][Windows] Fixed support for using Qt applications with UTF-8
as the system codepage or by enabling that in the application's
manifest.

Discussed-on: https://lists.qt-project.org/pipermail/interest/2022-May/038241.html
Pick-to: 6.2 6.3
Change-Id: I77c8221eb2824c369feffffd16f0912550a98049
Reviewed-by: Lars Knoll <lars.knoll@qt.io>

src/corelib/text/qstringconverter.cpp

@@ -1173,6 +1173,11 @@ QChar *QUtf32::convertToUnicode(QChar *out, QByteArrayView in, QStringConverter:
 }
 
 #if defined(Q_OS_WIN) && !defined(QT_BOOTSTRAPPED)
+int QLocal8Bit::checkUtf8()
+{
+    return GetACP() == CP_UTF8 ? 1 : -1;
+}
+
 static QString convertToUnicodeCharByChar(QByteArrayView in, QStringConverter::State *state)
 {
     qsizetype length = in.size();
@@ -1226,7 +1231,7 @@ static QString convertToUnicodeCharByChar(QByteArrayView in, QStringConverter::S
 }
 
 
-QString QLocal8Bit::convertToUnicode(QByteArrayView in, QStringConverter::State *state)
+QString QLocal8Bit::convertToUnicode_sys(QByteArrayView in, QStringConverter::State *state)
 {
     qsizetype length = in.size();
 
@@ -1314,7 +1319,7 @@ QString QLocal8Bit::convertToUnicode(QByteArrayView in, QStringConverter::State
     return s;
 }
 
-QByteArray QLocal8Bit::convertFromUnicode(QStringView in, QStringConverter::State *state)
+QByteArray QLocal8Bit::convertFromUnicode_sys(QStringView in, QStringConverter::State *state)
 {
     const QChar *ch = in.data();
     qsizetype uclen = in.size();

src/corelib/text/qstringconverter_p.h

@@ -301,8 +301,31 @@ struct Q_CORE_EXPORT QLocal8Bit
     static QByteArray convertFromUnicode(QStringView in, QStringConverter::State *state)
     { return QUtf8::convertFromUnicode(in, state); }
 #else
-    static QString convertToUnicode(QByteArrayView, QStringConverter::State *);
-    static QByteArray convertFromUnicode(QStringView, QStringConverter::State *);
+    static int checkUtf8();
+    static bool isUtf8()
+    {
+        static QBasicAtomicInteger<qint8> result = { 0 };
+        int r = result.loadRelaxed();
+        if (r == 0) {
+            r = checkUtf8();
+            result.storeRelaxed(r);
+        }
+        return r > 0;
+    }
+    static QString convertToUnicode_sys(QByteArrayView, QStringConverter::State *);
+    static QString convertToUnicode(QByteArrayView in, QStringConverter::State *state)
+    {
+        if (isUtf8())
+            return QUtf8::convertToUnicode(in, state);
+        return convertToUnicode_sys(in, state);
+    }
+    static QByteArray convertFromUnicode_sys(QStringView, QStringConverter::State *);
+    static QByteArray convertFromUnicode(QStringView in, QStringConverter::State *state)
+    {
+        if (isUtf8())
+            return QUtf8::convertFromUnicode(in, state);
+        return convertFromUnicode_sys(in, state);
+    }
 #endif
 };
 

tests/auto/corelib/text/qstringconverter/tst_qstringconverter.cpp

@@ -7,13 +7,54 @@
 #include <qstringconverter.h>
 #include <qthreadpool.h>
 
+#include <array>
+
+enum CodecLimitation {
+    AsciiOnly,
+    Latin1Only,
+    FullUnicode
+};
+
+#ifdef Q_OS_WIN
+#  include <qt_windows.h>
+static bool localeIsUtf8()
+{
+    return GetACP() == CP_UTF8;
+}
+#else
+static constexpr bool localeIsUtf8()
+{
+    return true;
+}
+#endif
+
 using namespace Qt::StringLiterals;
 
+struct Codec
+{
+    const char name[12];
+    QStringConverter::Encoding code;
+    CodecLimitation limitation = FullUnicode;
+};
+static const std::array codes = {
+    Codec{ "UTF-8", QStringConverter::Utf8 },
+    Codec{ "UTF-16", QStringConverter::Utf16 },
+    Codec{ "UTF-16-le", QStringConverter::Utf16LE },
+    Codec{ "UTF-16-be", QStringConverter::Utf16BE },
+    Codec{ "UTF-32", QStringConverter::Utf32 },
+    Codec{ "UTF-32-le", QStringConverter::Utf32LE },
+    Codec{ "UTF-32-be", QStringConverter::Utf32BE },
+    Codec{ "Latin-1", QStringConverter::Latin1, Latin1Only },
+    Codec{ "System", QStringConverter::System, localeIsUtf8() ? FullUnicode : AsciiOnly }
+};
+
 class tst_QStringConverter : public QObject
 {
     Q_OBJECT
 
 private slots:
+    void initTestCase();
+
     void threadSafety();
 
     void constructByName();
@@ -132,24 +173,11 @@ void tst_QStringConverter::roundtrip_data()
     QTest::addColumn<QString>("utf16");
     QTest::addColumn<QStringConverter::Encoding>("code");
 
-    const struct {
-        QStringConverter::Encoding code;
-        const char *name;
-    } codes[] = {
-        { QStringConverter::Utf8, "UTF-8" },
-        { QStringConverter::Utf16, "UTF-16" },
-        { QStringConverter::Utf16LE, "UTF-16-le" },
-        { QStringConverter::Utf16BE, "UTF-16-be" },
-        { QStringConverter::Utf32, "UTF-32" },
-        { QStringConverter::Utf32LE, "UTF-32-le" },
-        { QStringConverter::Utf32BE, "UTF-32-be" },
-        // Latin1, System: not guaranteed to be able to represent arbitrary Unicode.
-    };
     // TODO: include flag variations, too.
 
     for (const auto code : codes) {
         QTest::addRow("empty-%s", code.name) << u""_s << code.code;
-        {
+        if (code.limitation == FullUnicode) {
             const char32_t zeroVal = 0x11136; // Unicode's representation of Chakma zero
             const QChar data[] = {
                 QChar::highSurrogate(zeroVal), QChar::lowSurrogate(zeroVal),
@@ -1870,6 +1898,14 @@ public:
     }
 };
 
+void tst_QStringConverter::initTestCase()
+{
+    if (localeIsUtf8())
+        qInfo("System locale is UTF-8");
+    else
+        qInfo("System locale is not UTF-8");
+}
+
 void tst_QStringConverter::threadSafety()
 {
     QThreadPool::globalInstance()->setMaxThreadCount(12);