Linux: Bypass glibc's broken open() implementation

Glibc 2.21 and earlier have a bug that leaves the mode parameter unset
when O_TMPFILE is used. So we bypass the glibc implementation and go
directly for the syscall. We do this only for 32-bit x86, since of the
current, modern platforms, it's the only one that passes parameters on
the stack. Technically speaking, the glibc bug applies to all platforms,
but it turns out that on all others, it appears to work.

By doing this, we have two minor differences:
 1) open() is a cancellation point, but syscall() isn't
 2) if anyone tries to intercept open() calls via LD_PRELOAD, they're
    not going to catch Qt's.

[ChangeLog][QtCore][QTemporaryFile] Worked around a bug in the GNU C
Library versions 2.21 and earlier (used on Linux) that caused temporary
files to be created with permissions 000.

Task-number: QTBUG-69436
Change-Id: I20fd00e600264ff98c6afffd1540dceea89fa91f
Reviewed-by: Simon Hausmann <simon.hausmann@qt.io>

src/corelib/kernel/qcore_unix.cpp

@@ -44,6 +44,12 @@
 
 #include <stdlib.h>
 
+#ifdef __GLIBC__
+#  include <sys/syscall.h>
+#  include <pthread.h>
+#  include <unistd.h>
+#endif
+
 #ifdef Q_OS_MAC
 #include <mach/mach_time.h>
 #endif
@@ -79,6 +85,20 @@ QByteArray qt_readlink(const char *path)
     return buf;
 }
 
+#if defined(Q_PROCESSOR_X86_32) && defined(__GLIBC__)
+#  if !__GLIBC_PREREQ(2, 22)
+// glibc prior to release 2.22 had a bug that suppresses the third argument to
+// open() / open64() / openat(), causing file creation with O_TMPFILE to have
+// the wrong permissions. So we bypass the glibc implementation and go straight
+// for the syscall. See
+// https://sourceware.org/git/?p=glibc.git;a=commit;h=65f6f938cd562a614a68e15d0581a34b177ec29d
+int qt_open64(const char *pathname, int flags, mode_t mode)
+{
+    return syscall(SYS_open, pathname, flags | O_LARGEFILE, mode);
+}
+#  endif
+#endif
+
 #ifndef QT_BOOTSTRAPPED
 
 #if QT_CONFIG(poll_pollts)

src/corelib/kernel/qcore_unix_p.h

@@ -176,6 +176,14 @@ inline void qt_ignore_sigpipe()
     }
 }
 
+#if defined(Q_PROCESSOR_X86_32) && defined(__GLIBC__)
+#  if !__GLIBC_PREREQ(2, 22)
+int qt_open64(const char *pathname, int flags, mode_t);
+#    undef QT_OPEN
+#    define QT_OPEN qt_open64
+#  endif
+#endif
+
 // don't call QT_OPEN or ::open
 // call qt_safe_open
 static inline int qt_safe_open(const char *pathname, int flags, mode_t mode = 0777)