Merge pull request ocaml#11183 from Engil/refactor_github_actions

Refactor GitHub actions (cherry picked from commit 3ad1567)
dra27 · Jul 22, 2022 · 62ad584 · 62ad584
1 parent 4a58209
commit 62ad584
Show file tree

Hide file tree

Showing 7 changed files with 115 additions and 55 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -2,30 +2,88 @@ name: Build
 
 on: [push, pull_request]
 
-jobs:
+# List of test directories for the debug-s4096 and linux-O0 jobs.
+# These directories are selected because of their tendencies to reach corner
+# cases in the runtime system.
+env:
+  PARALLEL_TESTS: parallel callback gc-roots weak-ephe-final
 
+jobs:
+# This job will do the initial build of the compiler (on linux), with flambda on.
+# We then upload the compiler tree as a build artifact to enable re-use in
+# subsequent jobs.
   build:
-    name: 'linux'
     runs-on: 'ubuntu-latest'
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-      - name: configure tree
+      - name: Configure tree
         run: |
-          MAKE_ARG=-j XARCH=x64 bash -xe tools/ci/actions/runner.sh configure
+          MAKE_ARG=-j XARCH=x64 CONFIG_ARG='--enable-flambda --enable-cmm-invariants --enable-dependency-generation --enable-native-toplevel' OCAMLRUNPARAM=b,v=0 bash -xe tools/ci/actions/runner.sh configure
       - name: Build
         run: |
           MAKE_ARG=-j bash -xe tools/ci/actions/runner.sh build
       - name: Prepare Artifact
         run: |
+          git config --local --unset http.https://github.com/.extraheader
           tar --zstd -cf /tmp/sources.tar.zstd .
-      - uses: actions/upload-artifact@v2
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v2
         with:
           name: compiler
           path: /tmp/sources.tar.zstd
           retention-days: 1
 
-  build-misc:
+# Full testsuite run, and other sanity checks
+  normal:
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download Artifact
+        uses: actions/download-artifact@v2
+        with:
+          name: compiler
+      - name: Unpack Artifact
+        run: |
+          tar --zstd -xf sources.tar.zstd
+          rm -f sources.tar.zstd
+      - name: Packages
+        run: |
+          sudo apt-get update -y && sudo apt-get install -y texlive-latex-extra texlive-fonts-recommended hevea sass
+      - name: Run the testsuite
+        run: |
+          MAKE_ARG=-j OCAMLRUNPARAM=b,v=0 bash -xe tools/ci/actions/runner.sh test
+      - name: Build API Documentation
+        run: |
+          MAKE_ARG=-j OCAMLRUNPARAM=b,v=0 bash -xe tools/ci/actions/runner.sh api-docs
+      - name: Install
+        run: |
+         MAKE_ARG=-j OCAMLRUNPARAM=b,v=0 bash -xe tools/ci/actions/runner.sh install
+      - name: Check for manual changes
+        id: manual
+        run: >-
+         tools/ci/actions/check-manual-modified.sh
+         '${{ github.ref }}'
+         '${{ github.event_name }}'
+         '${{ github.event.pull_request.base.ref }}'
+         '${{ github.event.pull_request.base.sha }}'
+         '${{ github.event.pull_request.head.ref }}'
+         '${{ github.event.pull_request.head.sha }}'
+         '${{ github.event.ref }}'
+         '${{ github.event.before }}'
+         '${{ github.event.ref }}'
+         '${{ github.event.after }}'
+         '${{ github.event.repository.full_name }}'
+      - name: Build the manual
+        run: |
+          MAKE_ARG=-j OCAMLRUNPARAM=b,v=0 bash -xe tools/ci/actions/runner.sh manual
+        if: steps.manual.outputs.changed == 'true'
+      - name: Other checks
+        run: |
+          MAKE_ARG=-j OCAMLRUNPARAM=b,v=0 bash -xe tools/ci/actions/runner.sh other-checks
+
+# MacOS build+testsuite run, and Linux O0 run.
+  others:
     name: ${{ matrix.name }}
     runs-on: ${{ matrix.os }}
     strategy:
@@ -34,9 +92,6 @@ jobs:
           - name: linux-O0
             os: ubuntu-latest
             config_arg: CFLAGS='-O0'
-          - name: linux-debug
-            os: ubuntu-latest
-            env: OCAMLRUNPARAM=v=0,V=1 USE_RUNTIME=d
           - name: macos
             os: macos-latest
     steps:
@@ -55,52 +110,51 @@ jobs:
         if: ${{ matrix.name != 'linux-O0' }}
         run: |
           bash -c 'SHOW_TIMINGS=1 tools/ci/actions/runner.sh test'
-      - name: Run the testsuite (linux-O0, parallel)
+      - name: Run the testsuite (linux-O0)
         if: ${{ matrix.name == 'linux-O0' }}
         env:
           OCAMLRUNPARAM: v=0,V=1
           USE_RUNTIME: d
         run: |
-          bash -xe tools/ci/actions/runner.sh test_multicore 1 "parallel" "lib-threads" "lib-systhreads"
+          for dir in $PARALLEL_TESTS; do \
+           bash -cxe "SHOW_TIMINGS=1 tools/ci/actions/runner.sh test_prefix $dir"; \
+          done
 
-  testsuite:
+# "extra" testsuite runs, reusing the previously built compiler tree.
+# debug: running the full testsuite with the
+#        debug runtime and minor heap verification.
+# debug-s4086: select testsuite run with the debug runtime and a small
+#              minor heap.
+  extra:
     needs: build
-    # https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idneeds    strategy:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         id:
+          - debug
           - debug-s4096
-          - taskset
-          - normal
-          - super
     steps:
-      - uses: actions/download-artifact@v2
+      - name: Download artifact
+        uses: actions/download-artifact@v2
         with:
           name: compiler
       - name: Unpack Artifact
         run: |
           tar --zstd -xf sources.tar.zstd
-      - name: Run the testsuite
-        if: ${{ matrix.id == 'normal' }}
-        run: |
-          bash -xe tools/ci/actions/runner.sh test
-      - name: Run the testsuite (Super Charged)
-        if: ${{ matrix.id == 'super' }}
+          rm -f sources.tar.zstd
+      - name: Run the testsuite (debug runtime)
+        if: ${{ matrix.id == 'debug' }}
+        env:
+          OCAMLRUNPARAM: v=0,V=1
+          USE_RUNTIME: d
         run: |
-          bash -xe tools/ci/actions/runner.sh test_multicore 3 "parallel" \
-          "callback" "gc-roots" "lib-threads" "lib-systhreads" \
-          "weak-ephe-final"
+          bash -cxe "SHOW_TIMINGS=1 tools/ci/actions/runner.sh test"
       - name: Run the testsuite (s=4096, debug runtime)
+        if: ${{ matrix.id == 'debug-s4096' }}
         env:
           OCAMLRUNPARAM: s=4096,v=0
           USE_RUNTIME: d
-        if: ${{ matrix.id == 'debug-s4096' }}
-        run: |
-          bash -xe tools/ci/actions/runner.sh test_multicore 1 "parallel" \
-          "lib-threads" "lib-systhreads" "weak-ephe-final"
-      - name: Run the testsuite (taskset -c 0)
-        if: ${{ matrix.id == 'taskset' }}
         run: |
-          bash -xe tools/ci/actions/runner.sh test_multicore 1 "parallel" \
-          "lib-threads" "lib-systhreads" "weak-ephe-final"
+          for dir in $PARALLEL_TESTS; do \
+            bash -cxe "SHOW_TIMINGS=1 tools/ci/actions/runner.sh test_prefix $dir"; \
+          done
diff --git a/manual/src/html_processing/.gitignore b/manual/src/html_processing/.gitignore
@@ -1,3 +1,4 @@
+camlp-streams
 dune
 markup.ml
 uchar

diff --git a/manual/src/html_processing/Makefile b/manual/src/html_processing/Makefile
@@ -88,14 +88,15 @@ distclean::
 	rm -rf .sass-cache
 
 # We need Dune and Lambda Soup; Markup.ml and Uutf are dependencies
-DUNE_TAG = 2.6.2
-LAMBDASOUP_TAG = 0.7.1
-MARKUP_TAG = 0.8.2
-UUTF_TAG = v1.0.2
-RE_TAG = 1.9.0
+DUNE_TAG = 3.4.0
+LAMBDASOUP_TAG = 500
+MARKUP_TAG = 1.0.3
+UUTF_TAG = v1.0.3
+RE_TAG = 1.10.4
+CAMLP_STREAMS_TAG = v5.0.1
 
 # Duniverse rules - set-up dune and the dependencies in-tree for CI
-duniverse: dune/dune.exe re markup.ml uutf lambdasoup
+duniverse: dune/dune.exe re markup.ml uutf lambdasoup camlp-streams
 
 dune/dune.exe: dune
 	cd dune; ocaml bootstrap.ml
@@ -112,12 +113,13 @@ distclean::
 re:
 	git clone https://github.com/ocaml/ocaml-re.git -n -o upstream
 	cd ocaml-re; $(GIT_CHECKOUT) $(RE_TAG)
+	sed -i.bak -e '/(libraries seq)/d' ocaml-re/lib/dune
 
 distclean::
 	rm -rf ocaml-re
 
 lambdasoup:
-	git clone https://github.com/aantron/lambdasoup.git -n -o upstream
+	git clone https://github.com/dra27/lambdasoup.git -n -o dra27
 	cd lambdasoup; $(GIT_CHECKOUT) $(LAMBDASOUP_TAG)
 
 distclean::
@@ -142,4 +144,11 @@ uutf:
 distclean::
 	rm -rf uutf
 
+camlp-streams:
+	git clone https://github.com/ocaml/camlp-streams.git -n -o upstream
+	cd camlp-streams; $(GIT_CHECKOUT) $(CAMLP_STREAMS_TAG)
+
+distclean::
+	rm -rf camlp-streams
+
 .PHONY: css js img duniverse
diff --git a/testsuite/tests/misc/pr7168.ml b/testsuite/tests/misc/pr7168.ml
@@ -1,6 +1,6 @@
 (* TEST
 
-ocamlrunparam = "l=100000"
+ocamlrunparam += "l=100000"
 *)
 
 let rec f x =

diff --git a/testsuite/tests/runtime-errors/stackoverflow.ml b/testsuite/tests/runtime-errors/stackoverflow.ml
@@ -1,6 +1,6 @@
 (* TEST
 flags = "-w -a"
-ocamlrunparam = "l=100000"
+ocamlrunparam += "l=100000"
 *)
 
 let rec f x =

diff --git a/tools/ci/actions/check-manual-modified.sh b/tools/ci/actions/check-manual-modified.sh
@@ -31,4 +31,5 @@ else
   fi
 fi
 
+echo "Manual altered: $result"
 echo "::set-output name=changed::$result"
diff --git a/tools/ci/actions/runner.sh b/tools/ci/actions/runner.sh
@@ -71,17 +71,12 @@ Test () {
   cd ..
 }
 
-TestLoop () {
-  echo Running testsuite for "$@"
-  rm -f to_test.txt
-  for test in "$@"
-  do
-      echo tests/$test >> to_test.txt
-  done
-  for it in {1..$2}
-  do
-      $MAKE -C testsuite one LIST=../to_test.txt || exit 1
-  done || exit 1
+# By default, TestPrefix will attempt to run the tests
+# in the given directory in parallel.
+TestPrefix () {
+  TO_RUN=parallel-"$1"
+  echo Running single testsuite directory with $TO_RUN
+  $MAKE -C testsuite $TO_RUN
   cd ..
 }
 
@@ -170,7 +165,7 @@ case $1 in
 configure) Configure;;
 build) Build;;
 test) Test;;
-test_multicore) TestLoop "${@:3}";;
+test_prefix) TestPrefix $2;;
 api-docs) API_Docs;;
 install) Install;;
 manual) BuildManual;;