Merge pull request #366 from shym/dune-onetest

Teach `dune` to focus on one particular test
ocaml-multicore · Jul 6, 2023 · 9cb4f47 · 9cb4f47
2 parents 7262577 + dc39965
commit 9cb4f47
Show file tree

Hide file tree

Showing 4 changed files with 103 additions and 89 deletions.
diff --git a/.github/workflows/common.yml b/.github/workflows/common.yml
@@ -28,26 +28,10 @@ on:
         description: 'OCAMLRUNPARAM to use'
         type: string
         default: ''
-      only_test:
-        description: 'Only test to run (eg “src/array/lin_tests.exe”); whole suite is run if empty'
+      dune_alias:
+        description: 'dune alias that should be built in the main step'
         type: string
-        default: ''
-      seed:
-        description: 'Seed for the only test'
-        type: string
-        default: ''
-      repeats:
-        description: 'Number of test attempts'
-        type: string
-        default: '10'
-      repeats_failfast:
-        description: 'When repeating a test, stop as soon as one test fails'
-        type: boolean
-        default: false
-      subsuite:
-        description: 'Directories which should be taken as part of the test suite'
-        type: string
-        default: 'src'
+        default: 'runtest'
       compiler:
         description: 'Compiler to use'
         type: string
@@ -83,19 +67,15 @@ on:
     outputs:
       skippart2:
         description: 'Whether Part 2 (Cygwin workflow) should be skipped (because we are running only one test)'
-        value: ${{ jobs.test.outputs.skippart2 }}
+        value: ${{ jobs.test.outputs.skipnextjob }}
 
 jobs:
   test:
     env:
       QCHECK_MSG_INTERVAL:      '60'
       DUNE_PROFILE:             ${{ inputs.dune_profile }}
       OCAMLRUNPARAM:            ${{ inputs.runparam }}
-      ONLY_TEST:                ${{ inputs.only_test }}
-      SEED:                     ${{ inputs.seed }}
-      REPEATS:                  ${{ inputs.repeats }}
-      REPEATS_FAILFAST:         ${{ inputs.repeats_failfast }}
-      SUBSUITE:                 ${{ inputs.subsuite }}
+      DUNE_CI_ALIAS:            ${{ inputs.dune_alias }}
       COMPILER:                 ${{ inputs.compiler }}
       OCAML_COMPILER_GIT_REF:   ${{ inputs.compiler_git_ref }}
       CUSTOM_COMPILER_VERSION:  ${{ inputs.custom_compiler_version }}
@@ -113,7 +93,7 @@ jobs:
     timeout-minutes: ${{ inputs.timeout }}
 
     outputs:
-      skippart2: ${{ steps.winonlyone.outputs.skippart2 }}
+      skipnextjob: ${{ steps.mainstep.outputs.skipnextjob }}
 
     steps:
       - name: Configure environment (Cygwin)
@@ -298,66 +278,10 @@ jobs:
 
       - name: Build the test suite
         run: opam exec -- dune build
-        if: env.ONLY_TEST == ''
 
       - name: Run the internal package tests
         run: opam exec -- dune runtest -j1 --no-buffer --display=quiet --cache=disabled --error-reporting=twice test/
-        if: env.ONLY_TEST == ''
 
-      - name: Run the multicore test suite (Linux / macOS)
-        run: opam exec -- dune runtest -j1 --no-buffer --display=quiet --cache=disabled --error-reporting=twice $SUBSUITE
-        if: "runner.os != 'Windows' && env.ONLY_TEST == ''"
-
-      - name: Run the multicore test suite (Windows / Cygwin)
-        run: opam exec -- dune runtest -j1 --no-buffer --display=quiet --cache=disabled --error-reporting=twice @(-Split $Env:SUBSUITE)
-        if: "runner.os == 'Windows' && env.ONLY_TEST == ''"
-
-      - name: Run only one test (Linux / macOS)
-        run: |
-          failures=0
-          for i in `seq "$REPEATS"`; do
-            echo
-            echo "Starting $i-th run"
-            if [ -n "$SEED" ]; then
-              if ! opam exec -- dune exec "$ONLY_TEST" -- -v -s "$SEED"; then
-                if [ "$REPEATS_FAILFAST" = "true" ]; then
-                  exit 1
-                else
-                  failures=$((failures + 1))
-                fi
-              fi
-            else
-              if ! opam exec -- dune exec "$ONLY_TEST" -- -v; then
-                if [ "$REPEATS_FAILFAST" = "true" ]; then
-                  exit 1
-                else
-                  failures=$((failures + 1))
-                fi
-              fi
-            fi
-          done
-          echo "Test failed $failures times"
-          [ "$failures" = 0 ]
-        if: env.ONLY_TEST != '' && runner.os != 'Windows'
-
-      - name: Run only one test (Windows)
-        id: winonlyone
-        run: |
-          echo "skippart2=true" >> "${env:GITHUB_OUTPUT}"
-          if("${env:REPEATS_FAILFAST}" -eq "false") {
-            $ErrorActionPreference = 'Continue'
-          }
-          $failures = 0
-          for($i = 1; $i -le "${env:REPEATS}"; $i++) {
-            echo ""
-            echo "Starting $i-th run"
-            if("${env:SEED}" -eq "") {
-              opam exec -- dune exec "${env:ONLY_TEST}" -- -v
-              if($? -eq 0) { $failures += 1 }
-            } else {
-              opam exec -- dune exec "${env:ONLY_TEST}" -- -v -s "${env:SEED}"
-              if($? -eq 0) { $failures += 1 }
-            }
-          }
-          echo "Test failed $failures times"
-        if: env.ONLY_TEST != '' && runner.os == 'Windows'
+      - name: Run the multicore test suite
+        id: mainstep
+        run: opam exec -- dune build "@ci" -j1 --no-buffer --display=quiet --cache=disabled --error-reporting=twice
diff --git a/.github/workflows/cygwin-510.yml b/.github/workflows/cygwin-510.yml
@@ -10,7 +10,7 @@ jobs:
       compiler: ocaml-variants.5.1.0~beta1+options+win
       cygwin: true
       timeout: 360
-      subsuite: src/array src/atomic src/bigarray src/buffer src/bytes src/domain src/dynlink src/ephemeron src/floatarray src/hashtbl src/io
+      dune_alias: 'ci1'
 
   part2:
     needs: part1
@@ -21,4 +21,4 @@ jobs:
       compiler: ocaml-variants.5.1.0~beta1+options+win
       cygwin: true
       timeout: 360
-      subsuite: src/lazy src/neg_tests src/queue src/semaphore src/stack src/statistics src/sys src/thread src/threadomain src/weak
+      dune_alias: 'ci2'
diff --git a/.github/workflows/cygwin-520-trunk.yml b/.github/workflows/cygwin-520-trunk.yml
@@ -11,7 +11,7 @@ jobs:
       cygwin: true
       compiler_git_ref: refs/heads/trunk
       timeout: 360
-      subsuite: src/array src/atomic src/bigarray src/buffer src/bytes src/domain src/dynlink src/ephemeron src/floatarray src/hashtbl src/io
+      dune_alias: 'ci1'
 
   part2:
     needs: part1
@@ -23,4 +23,4 @@ jobs:
       cygwin: true
       compiler_git_ref: refs/heads/trunk
       timeout: 360
-      subsuite: src/lazy src/neg_tests src/queue src/semaphore src/stack src/statistics src/sys src/thread src/threadomain src/weak
+      dune_alias: 'ci2'
diff --git a/dune b/dune
@@ -7,3 +7,93 @@
  (name default)
  (package multicoretests)
  (deps (alias src/default)))
+
+; The aliases to control what is run in CI
+; It can either be the full test suite, or focus on a single test
+; Aliases ci1 and ci2 are useful to split the test suite into
+; subparts, for platforms such as Cygwin which are too slow
+(alias
+ (name ci)
+ (package multicoretests)
+ (deps
+  (alias_rec %{env:DUNE_CI_ALIAS=runtest})))
+  ; (alias_rec focusedtest)))
+
+(alias
+ (name ci1)
+ (package multicoretests)
+ (deps
+  (alias_rec src/array/runtest)
+  (alias_rec src/atomic/runtest)
+  (alias_rec src/bigarray/runtest)
+  (alias_rec src/buffer/runtest)
+  (alias_rec src/bytes/runtest)
+  (alias_rec src/domain/runtest)
+  (alias_rec src/dynlink/runtest)
+  (alias_rec src/ephemeron/runtest)
+  (alias_rec src/floatarray/runtest)
+  (alias_rec src/hashtbl/runtest)
+  (alias_rec src/io/runtest)))
+
+(alias
+ (name ci2)
+ (package multicoretests)
+ (deps
+  (alias_rec src/lazy/runtest)
+  (alias_rec src/neg_tests/runtest)
+  (alias_rec src/queue/runtest)
+  (alias_rec src/semaphore/runtest)
+  (alias_rec src/stack/runtest)
+  (alias_rec src/sys/runtest)
+  (alias_rec src/thread/runtest)
+  (alias_rec src/threadomain/runtest)
+  (alias_rec src/weak/runtest)))
+
+; @focusedtest
+; repeat a single test a couple of times
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+; A rule to repeat the test executable given as dependency a couple of
+; times and report at the end whether this worked
+
+; To change the test to repeat, change the source of the `copy`:
+(rule
+ (copy src/io/lin_tests_dsl_domain.exe focusedtest.exe))
+
+(rule
+ (alias focusedtest)
+ (package multicoretests)
+ (deps focusedtest.exe)
+ (enabled_if (<> %{os_type} Win32))
+ (action
+  (no-infer
+   (progn
+    (system "echo skipnextjob=true >> \"%{env:GITHUB_OUTPUT=dummy_file}\"")
+     ; ^ this is to ensure only one job is run in multi-job GitHub
+     ; workflows (only Cygwin at the moment)
+
+    (write-file hoped "")
+    (write-file failed-runs "")
+    (bash
+     "for i in `seq 20`; do echo Starting $i-th run; if ! ./focusedtest.exe -v ; then echo $i >> failed-runs; fi; done")
+    ; edit the previous line to focus on a particular seed
+    (diff failed-runs hoped)))))
+
+(rule
+ (alias focusedtest)
+ (package multicoretests)
+ (deps focusedtest.exe)
+ (enabled_if (= %{os_type} Win32))
+ (action
+  (no-infer
+   (progn
+    (system "echo skipnextjob=true >> %{env:GITHUB_OUTPUT=dummy_file}")
+     ; ^ this is to ensure only one job is run in multi-job GitHub
+     ; workflows (only Cygwin at the moment)
+
+    (write-file hoped "")
+    (write-file failed-runs "")
+    (run cmd /q /c
+     "for %G in (1,2,3,4, 5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20) do (echo Starting %G-th run && focusedtest.exe -v || echo %G >> failed-runs)")
+    ; edit the previous line to focus on a particular seed
+    (diff failed-runs hoped)))))