Proper macro replacment and branch evaluation

Correclty replace macros according to the C++ standard.
Use the correct replacement method also to evaluate

With this moc correctly processes boost headers.

Task-number: QTBUG-27546

Change-Id: I001b3054c5fcdc34d46cfa53d1387bd19436f361
Reviewed-by: Olivier Goffart <ogoffart@woboq.com>

src/tools/moc/preprocessor.cpp

@@ -536,50 +536,84 @@ static Symbols tokenize(const QByteArray &input, int lineNum = 1, TokenizeMode m
     return symbols;
 }
 
-void Preprocessor::macroExpandSymbols(int lineNum, const Symbols &symbolList, Symbols &expanded, MacroSafeSet safeset)
+Symbols Preprocessor::macroExpand(Preprocessor *that, Symbols &toExpand, int &index, int lineNum, bool one)
 {
-    Symbols saveSymbols = symbols;
-    int saveIndex = index;
-    symbols = symbolList;
-    index = 0;
+    SymbolStack symbols;
+    SafeSymbols sf;
+    sf.symbols = toExpand;
+    sf.index = index;
+    symbols.push(sf);
 
-    while (hasNext()) {
-        next();
-        macroExpandIdentifier(lineNum, expanded, safeset);
+    Symbols result;
+    if (toExpand.isEmpty())
+        return result;
+
+    for (;;) {
+        QByteArray macro;
+        Symbols newSyms = macroExpandIdentifier(that, symbols, lineNum, &macro);
+
+        if (macro.isEmpty()) {
+            result += newSyms;
+        } else {
+            SafeSymbols sf;
+            sf.symbols = newSyms;
+            sf.index = 0;
+            sf.expandedMacro = macro;
+            symbols.push(sf);
+        }
+        if (!symbols.hasNext() || (one && symbols.size() == 1))
+                break;
+        symbols.next();
     }
 
-    symbols = saveSymbols;
-    index = saveIndex;
+    if (symbols.size())
+        index = symbols.top().index;
+    else
+        index = toExpand.size();
+
+    return result;
 }
 
-void Preprocessor::macroExpandIdentifier(int lineNum, Symbols &preprocessed, MacroSafeSet safeset)
+
+Symbols Preprocessor::macroExpandIdentifier(Preprocessor *that, SymbolStack &symbols, int lineNum, QByteArray *macroName)
 {
-    const Symbol &s = symbol();
+    Symbol s = symbols.symbol();
+
     // not a macro
-    if (s.token != PP_IDENTIFIER || !macros.contains(s) || safeset.contains(s)) {
-        preprocessed += s;
-        preprocessed.last().lineNum = lineNum;
-        return;
+    if (s.token != PP_IDENTIFIER || !that->macros.contains(s) || symbols.dontReplaceSymbol(s.lexem())) {
+        Symbols syms;
+        syms += s;
+        syms.last().lineNum = lineNum;
+        return syms;
     }
 
-    const Macro &macro = macros.value(s);
-    safeset += s;
+    const Macro &macro = that->macros.value(s);
+    *macroName = s.lexem();
 
-    if (macro.isFunction) {
-        while (test(PP_WHITESPACE)) {}
-        if (!test(PP_LPAREN)) {
-            preprocessed += s;
-            return;
+    Symbols expansion;
+    if (!macro.isFunction) {
+        expansion = macro.symbols;
+    } else {
+        bool haveSpace = false;
+        while (symbols.test(PP_WHITESPACE)) { haveSpace = true; }
+        if (!symbols.test(PP_LPAREN)) {
+            *macroName = QByteArray();
+            Symbols syms;
+            if (haveSpace)
+                syms += Symbol(lineNum, PP_WHITESPACE);
+            syms += s;
+            syms.last().lineNum = lineNum;
+            return syms;
         }
         QList<Symbols> arguments;
-        while (hasNext()) {
+        while (symbols.hasNext()) {
             Symbols argument;
             // strip leading space
-            while (test(PP_WHITESPACE)) {}
+            while (symbols.test(PP_WHITESPACE)) {}
             int nesting = 0;
             bool vararg = macro.isVariadic && (arguments.size() == macro.arguments.size() - 1);
-            while (hasNext()) {
-                Token t = next();
+            while (symbols.hasNext()) {
+                Token t = symbols.next();
                 if (t == PP_LPAREN) {
                     ++nesting;
                 } else if (t == PP_RPAREN) {
@@ -590,7 +624,7 @@ void Preprocessor::macroExpandIdentifier(int lineNum, Symbols &preprocessed, Mac
                     if (!vararg)
                         break;
                 }
-                argument += symbol();
+                argument += symbols.symbol();
             }
             arguments += argument;
 
@@ -606,11 +640,9 @@ void Preprocessor::macroExpandIdentifier(int lineNum, Symbols &preprocessed, Mac
             // 0 argument macros are a bit special. They are ok if the
             // argument is pure whitespace or empty
             (macro.arguments.size() != 0 || arguments.size() != 1 || !arguments.at(0).isEmpty()))
-            error("Macro argument mismatch.");
+            that->error("Macro argument mismatch.");
 
         // now replace the macro arguments with the expanded arguments
-
-        Symbols expansion;
         enum Mode {
             Normal,
             Hash,
@@ -627,10 +659,10 @@ void Preprocessor::macroExpandIdentifier(int lineNum, Symbols &preprocessed, Mac
             if (mode == Normal) {
                 if (index >= 0) {
                     // each argument undoergoes macro expansion if it's not used as part of a # or ##
-                    if (i < macro.symbols.size() - 1 && macro.symbols.at(i + 1).token != PP_HASHHASH) {
-                        Symbols expanded;
-                        macroExpandSymbols(lineNum, arguments.at(index), expanded, safeset);
-                        expansion += expanded;
+                    if (i == macro.symbols.size() - 1 || macro.symbols.at(i + 1).token != PP_HASHHASH) {
+                        Symbols arg = arguments.at(index);
+                        int idx = 1;
+                        expansion += macroExpand(that, arg, idx, lineNum, false);
                     } else {
                         expansion += arguments.at(index);
                     }
@@ -639,7 +671,7 @@ void Preprocessor::macroExpandIdentifier(int lineNum, Symbols &preprocessed, Mac
                 }
             } else if (mode == Hash) {
                 if (index < 0)
-                    error("'#' is not followed by a macro parameter");
+                    that->error("'#' is not followed by a macro parameter");
 
                 const Symbols &arg = arguments.at(index);
                 QByteArray stringified;
@@ -656,11 +688,6 @@ void Preprocessor::macroExpandIdentifier(int lineNum, Symbols &preprocessed, Mac
 
                 while (expansion.size() && expansion.last().token == PP_WHITESPACE)
                     expansion.pop_back();
-                if (!expansion.size())
-                    error("'##' can't appear first in macro argument");
-
-                Symbol last = expansion.last();
-                expansion.pop_back();
 
                 Symbol next = s;
                 if (index >= 0) {
@@ -672,14 +699,16 @@ void Preprocessor::macroExpandIdentifier(int lineNum, Symbols &preprocessed, Mac
                     next = arg.at(0);
                 }
 
-                if (last.token == STRING_LITERAL || s.token == STRING_LITERAL)
-                    error("Can't concatenate non identifier tokens");
+                if (!expansion.isEmpty() && expansion.last().token == s.token) {
+                    Symbol last = expansion.last();
+                    expansion.pop_back();
+
+                    if (last.token == STRING_LITERAL || s.token == STRING_LITERAL)
+                        that->error("Can't concatenate non identifier tokens");
 
-                if (last.token == s.token) {
                     QByteArray lexem = last.lexem() + next.lexem();
                     expansion += Symbol(lineNum, last.token, lexem);
                 } else {
-                    expansion += last;
                     expansion += next;
                 }
 
@@ -692,42 +721,19 @@ void Preprocessor::macroExpandIdentifier(int lineNum, Symbols &preprocessed, Mac
             mode = Normal;
         }
         if (mode != Normal)
-            error("'#' or '##' found at the end of a macro argument");
+            that->error("'#' or '##' found at the end of a macro argument");
 
-        macroExpandSymbols(lineNum, expansion, preprocessed, safeset);
-    } else {
-        macroExpandSymbols(lineNum, macro.symbols, preprocessed, safeset);
     }
+
+    return expansion;
 }
 
-
-void Preprocessor::substituteMacro(const MacroName &macro, Symbols &substituted, MacroSafeSet safeset)
-{
-    Symbols saveSymbols = symbols;
-    int saveIndex = index;
-
-    symbols = macros.value(macro).symbols;
-    index = 0;
-
-    safeset += macro;
-    substituteUntilNewline(substituted, safeset);
-
-    symbols = saveSymbols;
-    index = saveIndex;
-}
-
-
-
-void Preprocessor::substituteUntilNewline(Symbols &substituted, MacroSafeSet safeset)
+void Preprocessor::substituteUntilNewline(Symbols &substituted)
 {
     while (hasNext()) {
         Token token = next();
         if (token == PP_IDENTIFIER) {
-            MacroName macro = symbol();
-            if (macros.contains(macro) && !safeset.contains(macro)) {
-                substituteMacro(macro, substituted, safeset);
-                continue;
-            }
+            substituted += macroExpand(this, symbols, index, symbol().lineNum, true);
         } else if (token == PP_DEFINED) {
             bool braces = test(PP_LPAREN);
             next(PP_IDENTIFIER);
@@ -740,8 +746,9 @@ void Preprocessor::substituteUntilNewline(Symbols &substituted, MacroSafeSet saf
         } else if (token == PP_NEWLINE) {
             substituted += symbol();
             break;
+        } else {
+            substituted += symbol();
         }
-        substituted += symbol();
     }
 }
 
@@ -1092,6 +1099,12 @@ void Preprocessor::preprocess(const QByteArray &filename, Symbols &preprocessed)
                    (macro.symbols.last().token == PP_WHITESPACE || macro.symbols.last().token == WHITESPACE))
                 macro.symbols.pop_back();
 
+            if (macro.symbols.first().token == PP_HASHHASH ||
+                macro.symbols.last().token == PP_HASHHASH)
+                error("'##' cannot appear at either end of a macro expansion");
+            if (macro.symbols.last().token == HASH ||
+                macro.symbols.last().token == PP_HASH)
+                error("'#' is not followed by a macro parameter");
             macros.insert(name, macro);
             continue;
         }
@@ -1104,7 +1117,7 @@ void Preprocessor::preprocess(const QByteArray &filename, Symbols &preprocessed)
         }
         case PP_IDENTIFIER: {
             // substitute macros
-            macroExpandIdentifier(symbol().lineNum, preprocessed);
+            preprocessed += macroExpand(this, symbols, index, symbol().lineNum, true);
             continue;
         }
         case PP_HASH:

src/tools/moc/preprocessor.h

@@ -64,7 +64,6 @@ typedef QByteArray MacroName;
 typedef SubArray MacroName;
 #endif
 typedef QHash<MacroName, Macro> Macros;
-typedef QVector<MacroName> MacroSafeSet;
 
 class QIODevice;
 
@@ -84,10 +83,9 @@ public:
     void skipUntilEndif();
     bool skipBranch();
 
-    void substituteMacro(const MacroName &macro, Symbols &substituted, MacroSafeSet safeset = MacroSafeSet());
-    void substituteUntilNewline(Symbols &substituted, MacroSafeSet safeset = MacroSafeSet());
-    void macroExpandIdentifier(int lineNum, Symbols &preprocessed, MacroSafeSet safeset = MacroSafeSet());
-    void macroExpandSymbols(int lineNum, const Symbols &symbols, Symbols &expanded, MacroSafeSet safeset);
+    void substituteUntilNewline(Symbols &substituted);
+    static Symbols macroExpandIdentifier(Preprocessor *that, SymbolStack &symbols, int lineNum, QByteArray *macroName);
+    static Symbols macroExpand(Preprocessor *that, Symbols &toExpand, int &index, int lineNum, bool one);
 
     int evaluateCondition();
 

src/tools/moc/symbols.h

@@ -46,6 +46,7 @@
 #include <qstring.h>
 #include <qhash.h>
 #include <qvector.h>
+#include <qstack.h>
 #include <qdebug.h>
 
 QT_BEGIN_NAMESPACE
@@ -128,9 +129,61 @@ struct Symbol
 };
 Q_DECLARE_TYPEINFO(Symbol, Q_MOVABLE_TYPE);
 
-
 typedef QVector<Symbol> Symbols;
 
+struct SafeSymbols {
+    Symbols symbols;
+    QByteArray expandedMacro;
+    int index;
+};
+
+class SymbolStack : public QStack<SafeSymbols>
+{
+public:
+    inline bool hasNext() {
+        while (!isEmpty() && top().index >= top().symbols.size())
+            pop();
+        return !isEmpty();
+    }
+    inline Token next() {
+        while (!isEmpty() && top().index >= top().symbols.size())
+            pop();
+        if (isEmpty())
+            return NOTOKEN;
+        return top().symbols.at(top().index++).token;
+    }
+    bool test(Token);
+    inline const Symbol &symbol() const { return top().symbols.at(top().index-1); }
+    inline Token token() { return symbol().token; }
+    inline QByteArray lexem() const { return symbol().lexem(); }
+    inline QByteArray unquotedLexem() { return symbol().unquotedLexem(); }
+
+    bool dontReplaceSymbol(const QByteArray &name);
+};
+
+inline bool SymbolStack::test(Token token)
+{
+    int stackPos = size() - 1;
+    while (stackPos >= 0 && at(stackPos).index >= at(stackPos).symbols.size())
+        --stackPos;
+    if (stackPos < 0)
+        return false;
+    if (at(stackPos).symbols.at(at(stackPos).index).token == token) {
+        next();
+        return true;
+    }
+    return false;
+}
+
+inline bool SymbolStack::dontReplaceSymbol(const QByteArray &name)
+{
+    for (int i = 0; i < size(); ++i) {
+        if (name == at(i).expandedMacro)
+            return true;
+    }
+    return false;
+}
+
 QT_END_NAMESPACE
 
 #endif // SYMBOLS_H

tests/auto/tools/moc/parse-defines.h

@@ -52,6 +52,7 @@
 #define PD_CLASSNAME ParseDefine
 
 #define PD_STRINGIFY(a) #a
+#define PD_XSTRINGIFY(a) PD_STRINGIFY(a)
 #define PD_SCOPED_STRING(a, b) PD_STRINGIFY(a) "::" PD_STRINGIFY(b)
 #define PD_DEFINE1(a,b) a##b
 #define PD_DEFINE2(a,b) a comb##b
@@ -81,7 +82,8 @@ class PD_CLASSNAME : public QObject
 {
     Q_OBJECT
     Q_CLASSINFO("TestString", PD_STRINGIFY(PD_CLASSNAME))
-    PD_CLASSINFO("TestString2", "TestValue")
+    Q_CLASSINFO("TestString2", PD_XSTRINGIFY(PD_CLASSNAME))
+    PD_CLASSINFO("TestString3", "TestValue")
 public:
     PD_CLASSNAME() {}
 
@@ -108,6 +110,24 @@ public slots:
     PD_VARARGEXT(void vararg5, int) {}
     PD_VARARGEXT(void vararg6, int, int) {}
 #endif
+
+#define OUTERFUNCTION(x) x
+#define INNERFUNCTION(x) OUTERFUNCTION(x)
+#define INNER INNERFUNCTION
+
+    void INNERFUNCTION(INNERFUNCTION)(int) {}
+    void OUTERFUNCTION(INNERFUNCTION)(inner_expanded(int)) {}
+    void expanded_method OUTERFUNCTION(INNER)((int)) {}
+
+#undef INNERFUNCTION
+
+#define cond1() 0x1
+#define cond2() 0x2
+
+#if !(cond1() & cond2())
+    void conditionSlot() {}
+#endif
+
 };
 
 #undef QString

tests/auto/tools/moc/tst_moc.cpp

@@ -2752,19 +2752,34 @@ void tst_Moc::parseDefines()
     QVERIFY(index != -1);
 #endif
 
+    index = mo->indexOfSlot("INNERFUNCTION(int)");
+    QVERIFY(index != -1);
+    index = mo->indexOfSlot("inner_expanded(int)");
+    QVERIFY(index != -1);
+    index = mo->indexOfSlot("expanded_method(int)");
+    QVERIFY(index != -1);
+
+    index = mo->indexOfSlot("conditionSlot()");
+    QVERIFY(index != -1);
+
     int count = 0;
     for (int i = 0; i < mo->classInfoCount(); ++i) {
         QMetaClassInfo mci = mo->classInfo(i);
         if (!qstrcmp(mci.name(), "TestString")) {
             ++count;
-            QVERIFY(!qstrcmp(mci.value(), "ParseDefine"));
+            QVERIFY(!qstrcmp(mci.value(), "PD_CLASSNAME"));
         }
         if (!qstrcmp(mci.name(), "TestString2")) {
+            ++count;
+            qDebug() << mci.value();
+            QVERIFY(!qstrcmp(mci.value(), "ParseDefine"));
+        }
+        if (!qstrcmp(mci.name(), "TestString3")) {
             ++count;
             QVERIFY(!qstrcmp(mci.value(), "TestValue"));
         }
     }
-    QVERIFY(count == 2);
+    QVERIFY(count == 3);
 }
 
 void tst_Moc::preprocessorOnly()