From 8acec4dbe6f9faac2c48a8be67f73e0d2ec1185b Mon Sep 17 00:00:00 2001
From: Marc Mutz <marc.mutz@qt.io>
Date: Mon, 31 Oct 2022 17:27:15 +0100
Subject: [PATCH] Long live QUtf8::convertFromLatin1()!
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

With the introduction of QAnyStringView, overloading based on UTF-8
and Latin-1 is becoming more common. Often, the two overloads can
share the processing backend, because we're only interested in the
US-ASCII subset of each.

But if they can't, we need a faster way to convert L1 into UTF-8 than
going via UTF-16. This is where the new private API comes in.

Eventually, we should have the converse operation, too, to complete
the set of direct conversions between the possible three
QAnyStringView encodings L1/U8/U16, but this direction is easier to
code (there are no error cases) and more immediately useful, so
provide L1->U8 alone for now.

Change-Id: I3f7e1a9c89979d0eb604cb9e42dedf3d514fca2c
Reviewed-by: Edward Welbourne <edward.welbourne@qt.io>
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Mårten Nordheim <marten.nordheim@qt.io>
Reviewed-by: Thiago Macieira <thiago.macieira@intel.com>
---
 src/corelib/text/qstringconverter.cpp            | 15 +++++++++++++++
 src/corelib/text/qstringconverter_p.h            |  1 +
 .../qstringconverter/tst_qstringconverter.cpp    | 16 ++++++++++++++++
 3 files changed, 32 insertions(+)

diff --git a/src/corelib/text/qstringconverter.cpp b/src/corelib/text/qstringconverter.cpp
index 329e1990da..a0e0173cbb 100644
--- a/src/corelib/text/qstringconverter.cpp
+++ b/src/corelib/text/qstringconverter.cpp
@@ -571,6 +571,21 @@ char *QUtf8::convertFromUnicode(char *out, QStringView in, QStringConverter::Sta
     return reinterpret_cast<char *>(cursor);
 }
 
+char *QUtf8::convertFromLatin1(char *out, QLatin1StringView in)
+{
+    // ### SIMD-optimize:
+    for (uchar ch : in) {
+        if (ch < 128) {
+            *out++ = ch;
+        } else {
+            // as per https://en.wikipedia.org/wiki/UTF-8#Encoding, 2nd row
+            *out++ = 0b110'0'0000u | (ch >> 6);
+            *out++ = 0b10'00'0000u | (ch & 0b0011'1111);
+        }
+    }
+    return out;
+}
+
 QString QUtf8::convertToUnicode(QByteArrayView in)
 {
     // UTF-8 to UTF-16 always needs the exact same number of words or less:
diff --git a/src/corelib/text/qstringconverter_p.h b/src/corelib/text/qstringconverter_p.h
index e801272646..26be8b713c 100644
--- a/src/corelib/text/qstringconverter_p.h
+++ b/src/corelib/text/qstringconverter_p.h
@@ -270,6 +270,7 @@ struct QUtf8
     Q_CORE_EXPORT static QByteArray convertFromUnicode(QStringView in);
     Q_CORE_EXPORT static QByteArray convertFromUnicode(QStringView in, QStringConverterBase::State *state);
     static char *convertFromUnicode(char *out, QStringView in, QStringConverter::State *state);
+    Q_CORE_EXPORT static char *convertFromLatin1(char *out, QLatin1StringView in);
     struct ValidUtf8Result {
         bool isValidUtf8;
         bool isValidAscii;
diff --git a/tests/auto/corelib/text/qstringconverter/tst_qstringconverter.cpp b/tests/auto/corelib/text/qstringconverter/tst_qstringconverter.cpp
index a346615e39..a60478d542 100644
--- a/tests/auto/corelib/text/qstringconverter/tst_qstringconverter.cpp
+++ b/tests/auto/corelib/text/qstringconverter/tst_qstringconverter.cpp
@@ -6,9 +6,11 @@
 
 #include <QtCore/private/qglobal_p.h>
 #include <qstringconverter.h>
+#include <private/qstringconverter_p.h>
 #include <qthreadpool.h>
 
 #include <array>
+#include <numeric>
 
 using namespace Qt::StringLiterals;
 
@@ -130,6 +132,8 @@ private slots:
     void roundtrip_data();
     void roundtrip();
 
+    void convertL1U8();
+
 #if QT_CONFIG(icu)
     void roundtripIcu_data();
     void roundtripIcu();
@@ -427,6 +431,18 @@ void tst_QStringConverter::roundtrip()
     QCOMPARE(decoded, uniString);
 }
 
+void tst_QStringConverter::convertL1U8()
+{
+    {
+        std::array<char, 256> latin1;
+        std::iota(latin1.data(), latin1.data() + latin1.size(), uchar(0));
+        std::array<char, 512> utf8;
+        auto out = QUtf8::convertFromLatin1(utf8.data(), QLatin1StringView{latin1.data(), latin1.size()});
+        QCOMPARE(QString::fromLatin1(latin1.data(), latin1.size()),
+                 QString::fromUtf8(utf8.data(), out - utf8.data()));
+    }
+}
+
 #if QT_CONFIG(icu)
 
 void tst_QStringConverter::roundtripIcu_data()