QCoreApplication: force the process locale codec to UTF-8

As discussed in the mailing list and in the Qt Contributor Summit 2019.

Tested on Linux, macOS and FreeBSD, showing these fallbacks:
OS        environment         setlocale() call
FreeBSD   empty               "C.UTF-8"
FreeBSD   LC_ALL=C            "C.UTF-8"
Linux     empty               "C.UTF-8"
Linux     LC_ALL=C            "C.UTF-8"
Linux     LANG=en_US          "en_US.UTF-8"
Linux     LANG=de_DE@euro     "de_DE.UTF-8"
Linux     LANG=en_GB.iso885915 "en_GB.UTF-8"
Linux     LANG=hy_AM.armscii8 "hy_AM.UTF-8"
Linux     LANG=ja_JP.sjis     "ja_JP.UTF-8"
Linux     LANG=ru_RU.koi8r    "ru_RU.UTF-8"
macOS     empty               "UTF-8"
macOS     LC_ALL=C            "UTF-8"

Versions tested: FreeBSD 12.1, Linux w/ glibc 2.30, macOS 10.14.2.

See
* https://wiki.qt.io/Qt_Contributor_Summit_2019_-_QtCore
* https://lists.qt-project.org/pipermail/development/2019-October/037791.html

Change-Id: Ia2aa807ffa8a4c798425fffd15d97ddb4f35b0ae
Reviewed-by: Thiago Macieira <thiago.macieira@intel.com>
Reviewed-by: Lars Knoll <lars.knoll@qt.io>

src/corelib/kernel/qcoreapplication.cpp

@@ -112,6 +112,7 @@
 
 #ifdef Q_OS_UNIX
 #  include <locale.h>
+#  include <langinfo.h>
 #  include <unistd.h>
 #  include <sys/types.h>
 #endif
@@ -338,9 +339,6 @@ void Q_CORE_EXPORT qt_call_post_routines()
 }
 
 
-// initialized in qcoreapplication and in qtextstream autotest when setlocale is called.
-static bool qt_locale_initialized = false;
-
 #ifndef QT_NO_QOBJECT
 
 // app starting up if false
@@ -590,11 +588,45 @@ QString qAppName()
 
 void QCoreApplicationPrivate::initLocale()
 {
+#if defined(Q_OS_UNIX) && !defined(QT_BOOTSTRAPPED)
+    static bool qt_locale_initialized = false;
     if (qt_locale_initialized)
         return;
     qt_locale_initialized = true;
-#if defined(Q_OS_UNIX) && !defined(QT_BOOTSTRAPPED)
-    setlocale(LC_ALL, "");
+
+#ifdef Q_OS_ANDROID
+    // Android's Bionic didn't get nl_langinfo until NDK 15 (Android 8.0),
+    // which is too new for Qt, so we just assume it's always UTF-8.
+    auto nl_langinfo = [](int) { return "UTF-8"; };
+#endif
+
+    const char *locale = setlocale(LC_ALL, "");
+    const char *codec = nl_langinfo(CODESET);
+    if (Q_UNLIKELY(strcmp(codec, "UTF-8") != 0 && strcmp(codec, "utf8") != 0)) {
+        QByteArray oldLocale = locale;
+        QByteArray newLocale = setlocale(LC_CTYPE, nullptr);
+        if (int dot = newLocale.indexOf('.'); dot != -1)
+            newLocale.truncate(dot);    // remove encoding, if any
+        if (int at = newLocale.indexOf('@'); at != -1)
+            newLocale.truncate(at);     // remove variant, as the old de_DE@euro
+        newLocale += ".UTF-8";
+        newLocale = setlocale(LC_CTYPE, newLocale);
+
+        // if locale doesn't exist, try some fallbacks
+#  ifdef Q_OS_DARWIN
+        if (newLocale.isEmpty())
+            newLocale = setlocale(LC_CTYPE, "UTF-8");
+#  endif
+        if (newLocale.isEmpty())
+            newLocale = setlocale(LC_CTYPE, "C.UTF-8");
+        if (newLocale.isEmpty())
+            newLocale = setlocale(LC_CTYPE, "C.utf8");
+
+        qWarning("Detected system locale encoding (%s, locale \"%s\") is not UTF-8.\n"
+                 "Qt shall use a UTF-8 locale (\"%s\") instead. If this causes problems,\n"
+                 "reconfigure your locale. See the locale(1) manual for more information.",
+                 codec, oldLocale.constData(), newLocale.constData());
+    }
 #endif
 }
 

tests/auto/testlib/selftests/generate_expected_output.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 #############################################################################
 ##
-## Copyright (C) 2016 The Qt Company Ltd.
+## Copyright (C) 2020 The Qt Company Ltd.
 ## Contact: https://www.qt.io/licensing/
 ##
 ## This file is part of the release tools of the Qt Toolkit.
@@ -263,7 +263,7 @@ def baseEnv(platname=None,
             keep += preserveLib
 
         cached = dict(
-            LC_ALL = 'C', # Use standard locale
+            LC_ALL = 'en-US.UTF-8', # Use standard locale
             # Avoid interference from any qtlogging.ini files, e.g. in
             # /etc/xdg/QtProject/, (must match tst_selftests.cpp's
             # processEnvironment()'s value):

tests/auto/testlib/selftests/tst_selftests.cpp

@@ -632,6 +632,11 @@ static QProcessEnvironment processEnvironment()
         result.insert(QStringLiteral("QT_LOGGING_RULES"),
                       // Must match generate_expected_output.py's main()'s value:
                       QStringLiteral("*.debug=true;qt.*=false"));
+
+#if defined(Q_OS_UNIX)
+        // avoid the warning from QCoreApplication
+        result.insert(QStringLiteral("LC_ALL"), QStringLiteral("en_US.UTF-8"));
+#endif
     }
     return result;
 }