CMake: Add support for enforcing passing CI tests per repository

Create a new test instructions yaml template that does not ignore the exit code of running tests. This template can be included by repositories where tests pass in all configurations tested by the CI. The template should be included in module_config.yaml via Test: - !include "{{qt/qtbase}}/coin_module_test_template_v3.yaml" This allows us to opt into enforcing test runs per repository, rather than waiting to fix all tests in all qt5.git repositories. Implementation notes Try to extract the common parts of the instructions as much as possible to avoid duplication. Unfortunately some duplication still remains due to the restricted yaml language supported by Coin. Add a short README.md file that describes the differences between the existing templates. The v2 non-enforcing test template should probably be removed in the future. Task-number: QTBUG-84886 Task-number: QTBUG-85364 Change-Id: I718fc3cb44d6aefdbebc2fd2088a910a8095a375 Reviewed-by: Joerg Bornemann <joerg.bornemann@qt.io>
2020-07-01 18:14:41 +02:00 · 2020-07-01 18:14:41 +02:00 · 0423814a2f
parent 939f5112a1
commit 0423814a2f
9 changed files with 94 additions and 23 deletions
--- a/coin/instructions/README.md
+++ b/coin/instructions/README.md
@ -0,0 +1,20 @@
+# Information about Coin instruction templates
+
+## Build templates
+
+* ``coin_qtbase_build_template_v2.yaml`` did not exist. The build instructions were directly embedded into
+  ``module_config.yaml`` and did not support repos outside of qtbase, also no cross-compilation.
+* ``coin_qtbase_build_template_v2`` introduced support for building other repos, and also enabled
+  build cross-compiling targets like ``Android`` and ``iOS``.
+  A bit later the template gained the ability to build ``qemu`` cross-compiling configurations.
+  The counterpart to qtbase to build other repositories is ``coin_module_build_template_v2``
+
+## Test templates
+
+* ``coin_module_test_template_v1`` did not exist. The test instructions were directly embedded into
+  ``module_config.yaml`` and did not support repos outside of qtbase, also no cross-compilation.
+* ``coin_module_test_template_v2`` introduced support for building tests for other repos, and made
+  sure not to build and run tests on cross-compiling configuration. A bit later the template gained
+  the ability to build and run tests for ``qemu`` cross-compiling configurations.
+* ``coin_module_test_template_v3`` changed the run test instructions to not ignore the exit code
+  and thus enforce that tests pass in the CI.
--- a/coin/instructions/cmake_regular_test_instructions.yaml
+++ b/coin/instructions/cmake_regular_test_instructions.yaml
@ -1,23 +1,7 @@
 type: Group
 instructions:
-  - type: MakeDirectory
-    directory: "{{.SourceDir}}_standalone_tests"
-  - type: InstallTestBinaryArchive
-    relativeStoragePath: "{{.Env.MODULE_ARTIFACTS_RELATIVE_STORAGE_PATH}}/tests.tar.gz"
-    directory: "{{.SourceDir}}_standalone_tests"
-    maxTimeInSeconds: 1200
-    maxTimeBetweenOutput: 1200
-    userMessageOnFailure: >
-      Failed to install tests archive.
-  - type: ChangeDirectory
-    directory: "{{.SourceDir}}_standalone_tests"
-  - type: ExecuteCommand
-    command: "{{.Env.TESTS_ENV_PREFIX}} ctest -V --rerun-failed --force-new-ctest-process"
-    ignoreExitCode: true
-    maxTimeInSeconds: 7200
-    maxTimeBetweenOutput: 900
-    userMessageOnFailure: >
-      Failed to run tests.
+  - !include "{{qt/qtbase}}/cmake_regular_test_instructions_common.yaml"
+  - !include "{{qt/qtbase}}/cmake_run_ctest_ignore_exit_code.yaml"
 disable_if:
  condition: property
  property: configureArgs
--- a/coin/instructions/cmake_regular_test_instructions_common.yaml
+++ b/coin/instructions/cmake_regular_test_instructions_common.yaml
@ -0,0 +1,13 @@
+type: Group
+instructions:
+  - type: MakeDirectory
+    directory: "{{.SourceDir}}_standalone_tests"
+  - type: InstallTestBinaryArchive
+    relativeStoragePath: "{{.Env.MODULE_ARTIFACTS_RELATIVE_STORAGE_PATH}}/tests.tar.gz"
+    directory: "{{.SourceDir}}_standalone_tests"
+    maxTimeInSeconds: 1200
+    maxTimeBetweenOutput: 1200
+    userMessageOnFailure: >
+      Failed to install tests archive.
+  - type: ChangeDirectory
+    directory: "{{.SourceDir}}_standalone_tests"
--- a/coin/instructions/cmake_regular_test_instructions_enforced.yaml
+++ b/coin/instructions/cmake_regular_test_instructions_enforced.yaml
@ -0,0 +1,8 @@
+type: Group
+instructions:
+  - !include "{{qt/qtbase}}/cmake_regular_test_instructions_common.yaml"
+  - !include "{{qt/qtbase}}/cmake_run_ctest_enforce_exit_code.yaml"
+disable_if:
+  condition: property
+  property: configureArgs
+  contains_value: "-DBUILD_SHARED_LIBS=OFF"
--- a/coin/instructions/cmake_run_ctest_enforce_exit_code.yaml
+++ b/coin/instructions/cmake_run_ctest_enforce_exit_code.yaml
@ -0,0 +1,9 @@
+type: Group
+instructions:
+  - type: ExecuteCommand
+    command: "{{.Env.TESTS_ENV_PREFIX}} ctest -V --rerun-failed --force-new-ctest-process"
+    ignoreExitCode: false
+    maxTimeInSeconds: 7200
+    maxTimeBetweenOutput: 900
+    userMessageOnFailure: >
+      Failed to run tests.
--- a/coin/instructions/cmake_run_ctest_ignore_exit_code.yaml
+++ b/coin/instructions/cmake_run_ctest_ignore_exit_code.yaml
@ -0,0 +1,9 @@
+type: Group
+instructions:
+  - type: ExecuteCommand
+    command: "{{.Env.TESTS_ENV_PREFIX}} ctest -V --rerun-failed --force-new-ctest-process"
+    ignoreExitCode: true
+    maxTimeInSeconds: 7200
+    maxTimeBetweenOutput: 900
+    userMessageOnFailure: >
+      Failed to run tests.
--- a/coin/instructions/coin_module_test_template_common.yaml
+++ b/coin/instructions/coin_module_test_template_common.yaml
@ -0,0 +1,7 @@
+type: Group
+instructions:
+  # The build env is needed on MSVC so that tst_qmake can properly build apps / libraries.
+  - !include "{{qt/qtbase}}/prepare_building_env.yaml"
+  # The test env vars are needed to pick the proper prefix.bat file.
+  - !include "{{qt/qtbase}}/cmake_setup_running_tests_env_vars.yaml"
+  - !include "{{qt/qtbase}}/coin_module_test_qemu_env_vars.yaml"
--- a/coin/instructions/coin_module_test_template_v2.yaml
+++ b/coin/instructions/coin_module_test_template_v2.yaml
@ -2,11 +2,7 @@ type: Group
 instructions:
  - type: Group
    instructions:
-      # The build env is needed on MSVC so that tst_qmake can properly build apps / libraries.
-      - !include "{{qt/qtbase}}/prepare_building_env.yaml"
-      # The test env vars are needed to pick the proper prefix.bat file.
-      - !include "{{qt/qtbase}}/cmake_setup_running_tests_env_vars.yaml"
-      - !include "{{qt/qtbase}}/coin_module_test_qemu_env_vars.yaml"
+      - !include "{{qt/qtbase}}/coin_module_test_template_common.yaml"
      - type: Group
        instructions:
          - !include "{{qt/qtbase}}/cmake_regular_test_instructions.yaml"
--- a/coin/instructions/coin_module_test_template_v3.yaml
+++ b/coin/instructions/coin_module_test_template_v3.yaml
@ -0,0 +1,25 @@
+type: Group
+instructions:
+  - type: Group
+    instructions:
+      - !include "{{qt/qtbase}}/coin_module_test_template_common.yaml"
+      - type: Group
+        instructions:
+          - !include "{{qt/qtbase}}/cmake_regular_test_instructions_enforced.yaml"
+        enable_if:
+          condition: or
+          conditions:
+            - condition: property
+              property: host.os
+              equals_property: target.os
+            - condition: property
+              property: target.osVersion
+              equals_value: QEMU
+    disable_if:
+      condition: property
+      property: features
+      contains_value: DisableTests
+
+  - type: EnvironmentVariable
+    variableName: Dummy
+    variableValue: dummy