moc: Treat number + characters as Identifier, not Number

While <DIGIT>+<Character> is not a valid identifier by itself in the C++
language, it might become one when using it with the token pasting
operator.

This risks confusing some number literals with suffix as Identifiers,
but those are currently not supported anyway, so this shouldn't break
anything that is currently working.

Fixes: QTBUG-87219
Fixes: QTBUG-124288
Pick-to: 6.7
Change-Id: If73255cc0e6649bc90c52b1d177aac8ff975ae69
Reviewed-by: Ivan Solovev <ivan.solovev@qt.io>
Reviewed-by: Ulf Hermann <ulf.hermann@qt.io>

src/tools/moc/preprocessor.cpp

@@ -214,7 +214,9 @@ Symbols Preprocessor::tokenize(const QByteArray& input, int lineNum, Preprocesso
                     data -= 2;
                     break;
                 case DIGIT:
-                    while (isAsciiDigit(*data) || *data == '\'')
+                    {
+                    bool hasSeenTokenSeparator = false;;
+                    while (isAsciiDigit(*data) || (hasSeenTokenSeparator = *data == '\''))
                         ++data;
                     if (!*data || *data != '.') {
                         token = INTEGER_LITERAL;
@@ -223,15 +225,22 @@ Symbols Preprocessor::tokenize(const QByteArray& input, int lineNum, Preprocesso
                              || *data == 'b' || *data == 'B')
                             && *lexem == '0') {
                             ++data;
-                            while (isHexDigit(*data) || *data == '\'')
+                            while (isHexDigit(*data) || (hasSeenTokenSeparator = *data == '\''))
                                 ++data;
                         } else if (*data == 'L') // TODO: handle other suffixes
                             ++data;
+                        if (!hasSeenTokenSeparator) {
+                            while (is_ident_char(*data)) {
+                                ++data;
+                                token = IDENTIFIER;
+                            }
+                        }
                         break;
                     }
                     token = FLOATING_LITERAL;
                     ++data;
                     Q_FALLTHROUGH();
+                    }
                 case FLOATING_LITERAL:
                     while (isAsciiDigit(*data) || *data == '\'')
                         ++data;

tests/auto/tools/moc/tst_moc.cpp

@@ -85,6 +85,26 @@ namespace A {
 }
 #endif
 
+
+namespace TokenStartingWithNumber
+{
+Q_NAMESPACE
+
+#define FOR_EACH_ITEM( CALL ) \
+  CALL( EXAMPLE ) \
+  CALL( 123_EXAMPLE ) \
+  CALL( OTHER_EXAMPLE )
+
+enum FooItems
+{
+
+#define ENUM_ITEM(NAME, ...) FOO ## NAME,
+  FOR_EACH_ITEM( ENUM_ITEM )
+};
+
+Q_ENUM_NS(FooItems)
+}
+
 Q_DECLARE_METATYPE(const QMetaObject*);
 
 #define TESTEXPORTMACRO Q_DECL_EXPORT
@@ -852,6 +872,7 @@ private slots:
     void readWriteThroughBindable();
     void invokableCtors();
     void virtualInlineTaggedSlot();
+    void tokenStartingWithNumber();
 
 signals:
     void sigWithUnsignedArg(unsigned foo);
@@ -4673,6 +4694,15 @@ void tst_Moc::virtualInlineTaggedSlot()
     QCOMPARE(method.returnMetaType(), QMetaType::fromType<int>());
 }
 
+void tst_Moc::tokenStartingWithNumber()
+{
+    auto *mo  = &TokenStartingWithNumber::staticMetaObject;
+    int index = mo->indexOfEnumerator("FooItems");
+    QMetaEnum metaEnum = mo->enumerator(index);
+    QVERIFY(metaEnum.isValid());
+    QCOMPARE(metaEnum.keyCount(), 3);
+}
+
 QTEST_MAIN(tst_Moc)
 
 // the generated code must compile with QT_NO_KEYWORDS