diff --git a/tests/tests.pro b/tests/tests.pro
index 75ca12094f..7a4af15fe2 100644
--- a/tests/tests.pro
+++ b/tests/tests.pro
@@ -1,3 +1,5 @@
 TEMPLATE = subdirs
-SUBDIRS = auto \
-        benchmarks
+SUBDIRS = auto
+
+# benchmarks in debug mode is rarely sensible
+contains(QT_CONFIG,release):SUBDIRS += benchmarks