Fix moc preprocessor-only mode with input that contains seemingly invalid identifiers

In WebKit we use moc -E to pre-process various files before throwing at
further build creation tools. The pre-processing is used to filter out
code depending in #ifdef'fed features.

The latest addition to the family of pre-processed files is the CSS grammar,
which is written in Bison. It contains rule lines like

   $$ = parser->createFoo()

and when pre-processing this moc stumbles over the dollar sign. Instead
of ignoring un-tokenizable input we should add it to the current token
if we're in preprocessor-only mode, otherwise the $$ gets eaten and we
produce data-loss by printing out less characters than.

Change-Id: Ib32e7c04b38dd2ba3726201e76f27405f7ea6c0d
Reviewed-by: Olivier Goffart <ogoffart@woboq.com>

src/tools/moc/preprocessor.cpp

@@ -193,9 +193,12 @@ static Symbols tokenize(const QByteArray &input, int lineNum = 1, TokenizeMode m
                 token = keywords[state].ident;
 
             if (token == NOTOKEN) {
-                // an error really
                 ++data;
-                continue;
+                // an error really, but let's ignore this input
+                // to not confuse moc later. However in pre-processor
+                // only mode let's continue.
+                if (!Preprocessor::preprocessOnly)
+                    continue;
             }
 
             ++column;

tests/auto/tools/moc/pp-dollar-signs.h

@@ -0,0 +1,42 @@
+/****************************************************************************
+**
+** Copyright (C) 2012 Digia Plc and/or its subsidiary(-ies).
+** Contact: http://www.qt-project.org/legal
+**
+** This file is part of the test suite of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and Digia.  For licensing terms and
+** conditions see http://qt.digia.com/licensing.  For further information
+** use the contact form at http://qt.digia.com/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Digia gives you certain additional
+** rights.  These rights are described in the Digia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3.0 as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU General Public License version 3.0 requirements will be
+** met: http://www.gnu.org/copyleft/gpl.html.
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+$$ = parser->createFoo()

tests/auto/tools/moc/tst_moc.cpp

@@ -554,6 +554,7 @@ private slots:
     void autoPropertyMetaTypeRegistration();
     void autoMethodArgumentMetaTypeRegistration();
     void parseDefines();
+    void preprocessorOnly();
 
 signals:
     void sigWithUnsignedArg(unsigned foo);
@@ -2760,6 +2761,26 @@ void tst_Moc::parseDefines()
     QVERIFY(count == 2);
 }
 
+void tst_Moc::preprocessorOnly()
+{
+#ifdef MOC_CROSS_COMPILED
+    QSKIP("Not tested when cross-compiled");
+#endif
+#if defined(Q_OS_LINUX) && defined(Q_CC_GNU) && !defined(QT_NO_PROCESS)
+    QProcess proc;
+    proc.start("moc", QStringList() << "-E" << srcify("/pp-dollar-signs.h"));
+    QVERIFY(proc.waitForFinished());
+    QCOMPARE(proc.exitCode(), 0);
+    QByteArray mocOut = proc.readAllStandardOutput();
+    QVERIFY(!mocOut.isEmpty());
+    QCOMPARE(proc.readAllStandardError(), QByteArray());
+
+    QVERIFY(mocOut.contains("$$ = parser->createFoo()"));
+#else
+    QSKIP("Only tested on linux/gcc");
+#endif
+}
+
 QTEST_MAIN(tst_Moc)
 
 #include "tst_moc.moc"