QSettings: support reading UTF-8 keys in INI files

[ChangeLog][QtCore][QSettings] The INI file reader now supports
keys encoded with UTF-8, as well as the %-encoded format. Writing
the keys back to the INI file is still done using %-encoded format.
This change does not touch the way the *values* are handled - they
are both read and written in UTF-8.

Drive-by: remove misleading comments from the reading algorithm.

Task-number: QTBUG-99401
Change-Id: I6a83cbf24d919a499540403688615f93cb195e93
Reviewed-by: Edward Welbourne <edward.welbourne@qt.io>

src/corelib/io/qsettings.cpp

@@ -545,11 +545,13 @@ void QSettingsPrivate::iniEscapedKey(const QString &key, QByteArray &result)
 
 bool QSettingsPrivate::iniUnescapedKey(const QByteArray &key, int from, int to, QString &result)
 {
+    const QString decoded = QString::fromUtf8(QByteArrayView(key).sliced(from, to - from));
+    const qsizetype size = decoded.size();
+    result.reserve(result.length() + size);
+    qsizetype i = 0;
     bool lowercaseOnly = true;
-    int i = from;
-    result.reserve(result.length() + (to - from));
-    while (i < to) {
-        char16_t ch = (uchar)key.at(i);
+    while (i < size) {
+        char16_t ch = decoded.at(i).unicode();
 
         if (ch == '\\') {
             result += QLatin1Char('/');
@@ -557,10 +559,11 @@ bool QSettingsPrivate::iniUnescapedKey(const QByteArray &key, int from, int to,
             continue;
         }
 
-        if (ch != '%' || i == to - 1) {
-            if (uint(ch - 'A') <= 'Z' - 'A') // only for ASCII
+        if (ch != '%' || i == size - 1) {
+            QChar qch(ch);
+            if (qch.isUpper())
                 lowercaseOnly = false;
-            result += ch;
+            result += qch;
             ++i;
             continue;
         }
@@ -568,24 +571,22 @@ bool QSettingsPrivate::iniUnescapedKey(const QByteArray &key, int from, int to,
         int numDigits = 2;
         int firstDigitPos = i + 1;
 
-        ch = key.at(i + 1);
+        ch = decoded.at(i + 1).unicode();
         if (ch == 'U') {
             ++firstDigitPos;
             numDigits = 4;
         }
 
-        if (firstDigitPos + numDigits > to) {
+        if (firstDigitPos + numDigits > size) {
             result += QLatin1Char('%');
-            // ### missing U
             ++i;
             continue;
         }
 
         bool ok;
-        ch = key.mid(firstDigitPos, numDigits).toUShort(&ok, 16);
+        ch = QStringView(decoded).sliced(firstDigitPos, numDigits).toUShort(&ok, 16);
         if (!ok) {
             result += QLatin1Char('%');
-            // ### missing U
             ++i;
             continue;
         }
@@ -2427,9 +2428,12 @@ void QConfFileSettingsPrivate::ensureSectionParsed(QConfFile *confFile,
         such as "General/someKey", the key will be located in the
         "%General" section, \e not in the "General" section.
 
-    \li In line with most implementations today, QSettings will
-        assume the INI file is utf-8 encoded. This means that keys and values
+    \li In line with most implementations today, QSettings will assume that
+        \e values in the INI file are utf-8 encoded. This means that \e values
         will be decoded as utf-8 encoded entries and written back as utf-8.
+        To retain backward compatibility with older Qt versions, \e keys in the
+        INI file are written in %-encoded format, but can be read in both
+        %-encoded and utf-8 formats.
 
     \endlist
 

tests/auto/corelib/io/qsettings/CMakeLists.txt

@@ -15,6 +15,8 @@ set(qsettings_resource_files
     "resourcefile6.plist"
     "withcomments.ini"
     "float.ini"
+    "qt5settings.ini"
+    "utf8settings.ini"
 )
 
 qt_internal_add_test(tst_qsettings

tests/auto/corelib/io/qsettings/qt5settings.ini

@@ -0,0 +1,11 @@
+[General]
+.%2C%27%25U%U0430%U0431%U0432%U0433%22%09=".,'%!@#$"
+%U265F=\x2658\x265a
+%UD83C%UDF0D=\xd83c\xdf10
+
+[Test]
+BAR=BAR
+B%C4R=B\xc4R
+OST=OST
+%D6SE=\xd6SE
+%U042D%U0442%U043E\%U0442%U0435%U0441%U0442=\x42d\x442\x43e \x442\x435\x441\x442

tests/auto/corelib/io/qsettings/tst_qsettings.cpp

@@ -198,6 +198,10 @@ private slots:
     void floatAsQVariant();
 
     void testXdg();
+
+    void testReadKeys_data();
+    void testReadKeys();
+
 private:
     void cleanupTestFiles();
 
@@ -3663,5 +3667,39 @@ void tst_QSettings::testXdg()
 #endif
 }
 
+void tst_QSettings::testReadKeys_data()
+{
+    QTest::addColumn<QString>("filepath");
+
+    QTest::newRow("escaped") << ":/qt5settings.ini";
+    QTest::newRow("utf-8") << ":/utf8settings.ini";
+}
+
+void tst_QSettings::testReadKeys()
+{
+    QFETCH(QString, filepath);
+
+    QSettings settings(filepath, QSettings::IniFormat);
+    QCOMPARE(settings.status(), QSettings::NoError);
+
+    QVariantMap expectedValues;
+    expectedValues.insert("Test/BAR", "BAR");
+    expectedValues.insert("Test/OST", "OST");
+    expectedValues.insert("Test/B\xC3\x84R", "B\xC3\x84R"); // BÄR
+    expectedValues.insert("Test/\xC3\x96SE", "\xC3\x96SE"); // ÖSE
+    expectedValues.insert(
+            "Test/\xD0\xAD\xD1\x82\xD0\xBE/\xD1\x82\xD0\xB5\xD1\x81\xD1\x82", // Это/тест
+            "\xD0\xAD\xD1\x82\xD0\xBE \xD1\x82\xD0\xB5\xD1\x81\xD1\x82"); // Это тест
+    expectedValues.insert(".,'%U\xD0\xB0\xD0\xB1\xD0\xB2\xD0\xB3\"\t", ".,'%!@#$");
+    expectedValues.insert("\xE2\x99\x9F", "\xE2\x99\x98\xE2\x99\x9A"); // ♟︎ ♘♚
+    expectedValues.insert("\xF0\x9F\x8C\x8D", "\xF0\x9F\x8C\x90"); // 🌍 🌐
+
+    QVariantMap readValues;
+    for (const auto &key : settings.allKeys())
+        readValues.insert(key, settings.value(key));
+
+    QCOMPARE(readValues, expectedValues);
+}
+
 QTEST_MAIN(tst_QSettings)
 #include "tst_qsettings.moc"

tests/auto/corelib/io/qsettings/utf8settings.ini

@@ -0,0 +1,11 @@
+[General]
+.%2C%27%25Uабвг%22%09=".,'%!@#$"
+♟=♘♚
+🌍=🌐
+
+[Test]
+BAR=BAR
+BÄR=BÄR
+OST=OST
+ÖSE=ÖSE
+Это\тест=Это тест