From 62ad5842d15b9f4eded2ae81755288c047dcc46a Mon Sep 17 00:00:00 2001 From: Enguerrand Decorne Date: Tue, 15 Mar 2022 17:10:10 +0100 Subject: [PATCH] Merge pull request #11183 from Engil/refactor_github_actions Refactor GitHub actions (cherry picked from commit 3ad1567958d76aa1da45a4b12163cbd15950ee65) --- .github/workflows/build.yml | 122 +++++++++++++----- manual/src/html_processing/.gitignore | 1 + manual/src/html_processing/Makefile | 23 +++- testsuite/tests/misc/pr7168.ml | 2 +- .../tests/runtime-errors/stackoverflow.ml | 2 +- tools/ci/actions/check-manual-modified.sh | 1 + tools/ci/actions/runner.sh | 19 +-- 7 files changed, 115 insertions(+), 55 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 09d0f3f59577..423c553cc972 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,30 +2,88 @@ name: Build on: [push, pull_request] -jobs: +# List of test directories for the debug-s4096 and linux-O0 jobs. +# These directories are selected because of their tendencies to reach corner +# cases in the runtime system. +env: + PARALLEL_TESTS: parallel callback gc-roots weak-ephe-final +jobs: +# This job will do the initial build of the compiler (on linux), with flambda on. +# We then upload the compiler tree as a build artifact to enable re-use in +# subsequent jobs. build: - name: 'linux' runs-on: 'ubuntu-latest' steps: - name: Checkout uses: actions/checkout@v2 - - name: configure tree + - name: Configure tree run: | - MAKE_ARG=-j XARCH=x64 bash -xe tools/ci/actions/runner.sh configure + MAKE_ARG=-j XARCH=x64 CONFIG_ARG='--enable-flambda --enable-cmm-invariants --enable-dependency-generation --enable-native-toplevel' OCAMLRUNPARAM=b,v=0 bash -xe tools/ci/actions/runner.sh configure - name: Build run: | MAKE_ARG=-j bash -xe tools/ci/actions/runner.sh build - name: Prepare Artifact run: | + git config --local --unset http.https://github.com/.extraheader tar --zstd -cf /tmp/sources.tar.zstd . - - uses: actions/upload-artifact@v2 + - name: Upload Artifact + uses: actions/upload-artifact@v2 with: name: compiler path: /tmp/sources.tar.zstd retention-days: 1 - build-misc: +# Full testsuite run, and other sanity checks + normal: + needs: build + runs-on: ubuntu-latest + steps: + - name: Download Artifact + uses: actions/download-artifact@v2 + with: + name: compiler + - name: Unpack Artifact + run: | + tar --zstd -xf sources.tar.zstd + rm -f sources.tar.zstd + - name: Packages + run: | + sudo apt-get update -y && sudo apt-get install -y texlive-latex-extra texlive-fonts-recommended hevea sass + - name: Run the testsuite + run: | + MAKE_ARG=-j OCAMLRUNPARAM=b,v=0 bash -xe tools/ci/actions/runner.sh test + - name: Build API Documentation + run: | + MAKE_ARG=-j OCAMLRUNPARAM=b,v=0 bash -xe tools/ci/actions/runner.sh api-docs + - name: Install + run: | + MAKE_ARG=-j OCAMLRUNPARAM=b,v=0 bash -xe tools/ci/actions/runner.sh install + - name: Check for manual changes + id: manual + run: >- + tools/ci/actions/check-manual-modified.sh + '${{ github.ref }}' + '${{ github.event_name }}' + '${{ github.event.pull_request.base.ref }}' + '${{ github.event.pull_request.base.sha }}' + '${{ github.event.pull_request.head.ref }}' + '${{ github.event.pull_request.head.sha }}' + '${{ github.event.ref }}' + '${{ github.event.before }}' + '${{ github.event.ref }}' + '${{ github.event.after }}' + '${{ github.event.repository.full_name }}' + - name: Build the manual + run: | + MAKE_ARG=-j OCAMLRUNPARAM=b,v=0 bash -xe tools/ci/actions/runner.sh manual + if: steps.manual.outputs.changed == 'true' + - name: Other checks + run: | + MAKE_ARG=-j OCAMLRUNPARAM=b,v=0 bash -xe tools/ci/actions/runner.sh other-checks + +# MacOS build+testsuite run, and Linux O0 run. + others: name: ${{ matrix.name }} runs-on: ${{ matrix.os }} strategy: @@ -34,9 +92,6 @@ jobs: - name: linux-O0 os: ubuntu-latest config_arg: CFLAGS='-O0' - - name: linux-debug - os: ubuntu-latest - env: OCAMLRUNPARAM=v=0,V=1 USE_RUNTIME=d - name: macos os: macos-latest steps: @@ -55,52 +110,51 @@ jobs: if: ${{ matrix.name != 'linux-O0' }} run: | bash -c 'SHOW_TIMINGS=1 tools/ci/actions/runner.sh test' - - name: Run the testsuite (linux-O0, parallel) + - name: Run the testsuite (linux-O0) if: ${{ matrix.name == 'linux-O0' }} env: OCAMLRUNPARAM: v=0,V=1 USE_RUNTIME: d run: | - bash -xe tools/ci/actions/runner.sh test_multicore 1 "parallel" "lib-threads" "lib-systhreads" + for dir in $PARALLEL_TESTS; do \ + bash -cxe "SHOW_TIMINGS=1 tools/ci/actions/runner.sh test_prefix $dir"; \ + done - testsuite: +# "extra" testsuite runs, reusing the previously built compiler tree. +# debug: running the full testsuite with the +# debug runtime and minor heap verification. +# debug-s4086: select testsuite run with the debug runtime and a small +# minor heap. + extra: needs: build - # https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idneeds strategy: runs-on: ubuntu-latest strategy: matrix: id: + - debug - debug-s4096 - - taskset - - normal - - super steps: - - uses: actions/download-artifact@v2 + - name: Download artifact + uses: actions/download-artifact@v2 with: name: compiler - name: Unpack Artifact run: | tar --zstd -xf sources.tar.zstd - - name: Run the testsuite - if: ${{ matrix.id == 'normal' }} - run: | - bash -xe tools/ci/actions/runner.sh test - - name: Run the testsuite (Super Charged) - if: ${{ matrix.id == 'super' }} + rm -f sources.tar.zstd + - name: Run the testsuite (debug runtime) + if: ${{ matrix.id == 'debug' }} + env: + OCAMLRUNPARAM: v=0,V=1 + USE_RUNTIME: d run: | - bash -xe tools/ci/actions/runner.sh test_multicore 3 "parallel" \ - "callback" "gc-roots" "lib-threads" "lib-systhreads" \ - "weak-ephe-final" + bash -cxe "SHOW_TIMINGS=1 tools/ci/actions/runner.sh test" - name: Run the testsuite (s=4096, debug runtime) + if: ${{ matrix.id == 'debug-s4096' }} env: OCAMLRUNPARAM: s=4096,v=0 USE_RUNTIME: d - if: ${{ matrix.id == 'debug-s4096' }} - run: | - bash -xe tools/ci/actions/runner.sh test_multicore 1 "parallel" \ - "lib-threads" "lib-systhreads" "weak-ephe-final" - - name: Run the testsuite (taskset -c 0) - if: ${{ matrix.id == 'taskset' }} run: | - bash -xe tools/ci/actions/runner.sh test_multicore 1 "parallel" \ - "lib-threads" "lib-systhreads" "weak-ephe-final" + for dir in $PARALLEL_TESTS; do \ + bash -cxe "SHOW_TIMINGS=1 tools/ci/actions/runner.sh test_prefix $dir"; \ + done diff --git a/manual/src/html_processing/.gitignore b/manual/src/html_processing/.gitignore index fcd498c1d897..bd7098d0bc96 100644 --- a/manual/src/html_processing/.gitignore +++ b/manual/src/html_processing/.gitignore @@ -1,3 +1,4 @@ +camlp-streams dune markup.ml uchar diff --git a/manual/src/html_processing/Makefile b/manual/src/html_processing/Makefile index 4fa6abe30291..f4e03afa6f49 100644 --- a/manual/src/html_processing/Makefile +++ b/manual/src/html_processing/Makefile @@ -88,14 +88,15 @@ distclean:: rm -rf .sass-cache # We need Dune and Lambda Soup; Markup.ml and Uutf are dependencies -DUNE_TAG = 2.6.2 -LAMBDASOUP_TAG = 0.7.1 -MARKUP_TAG = 0.8.2 -UUTF_TAG = v1.0.2 -RE_TAG = 1.9.0 +DUNE_TAG = 3.4.0 +LAMBDASOUP_TAG = 500 +MARKUP_TAG = 1.0.3 +UUTF_TAG = v1.0.3 +RE_TAG = 1.10.4 +CAMLP_STREAMS_TAG = v5.0.1 # Duniverse rules - set-up dune and the dependencies in-tree for CI -duniverse: dune/dune.exe re markup.ml uutf lambdasoup +duniverse: dune/dune.exe re markup.ml uutf lambdasoup camlp-streams dune/dune.exe: dune cd dune; ocaml bootstrap.ml @@ -112,12 +113,13 @@ distclean:: re: git clone https://github.com/ocaml/ocaml-re.git -n -o upstream cd ocaml-re; $(GIT_CHECKOUT) $(RE_TAG) + sed -i.bak -e '/(libraries seq)/d' ocaml-re/lib/dune distclean:: rm -rf ocaml-re lambdasoup: - git clone https://github.com/aantron/lambdasoup.git -n -o upstream + git clone https://github.com/dra27/lambdasoup.git -n -o dra27 cd lambdasoup; $(GIT_CHECKOUT) $(LAMBDASOUP_TAG) distclean:: @@ -142,4 +144,11 @@ uutf: distclean:: rm -rf uutf +camlp-streams: + git clone https://github.com/ocaml/camlp-streams.git -n -o upstream + cd camlp-streams; $(GIT_CHECKOUT) $(CAMLP_STREAMS_TAG) + +distclean:: + rm -rf camlp-streams + .PHONY: css js img duniverse diff --git a/testsuite/tests/misc/pr7168.ml b/testsuite/tests/misc/pr7168.ml index bc3698dcca23..dcd56eed1308 100644 --- a/testsuite/tests/misc/pr7168.ml +++ b/testsuite/tests/misc/pr7168.ml @@ -1,6 +1,6 @@ (* TEST -ocamlrunparam = "l=100000" +ocamlrunparam += "l=100000" *) let rec f x = diff --git a/testsuite/tests/runtime-errors/stackoverflow.ml b/testsuite/tests/runtime-errors/stackoverflow.ml index 43fb8aa57f5a..b617e64cdca0 100644 --- a/testsuite/tests/runtime-errors/stackoverflow.ml +++ b/testsuite/tests/runtime-errors/stackoverflow.ml @@ -1,6 +1,6 @@ (* TEST flags = "-w -a" -ocamlrunparam = "l=100000" +ocamlrunparam += "l=100000" *) let rec f x = diff --git a/tools/ci/actions/check-manual-modified.sh b/tools/ci/actions/check-manual-modified.sh index 782b5dee475f..57cdbe50d692 100755 --- a/tools/ci/actions/check-manual-modified.sh +++ b/tools/ci/actions/check-manual-modified.sh @@ -31,4 +31,5 @@ else fi fi +echo "Manual altered: $result" echo "::set-output name=changed::$result" diff --git a/tools/ci/actions/runner.sh b/tools/ci/actions/runner.sh index 7cae9a190a38..ede27f710a17 100755 --- a/tools/ci/actions/runner.sh +++ b/tools/ci/actions/runner.sh @@ -71,17 +71,12 @@ Test () { cd .. } -TestLoop () { - echo Running testsuite for "$@" - rm -f to_test.txt - for test in "$@" - do - echo tests/$test >> to_test.txt - done - for it in {1..$2} - do - $MAKE -C testsuite one LIST=../to_test.txt || exit 1 - done || exit 1 +# By default, TestPrefix will attempt to run the tests +# in the given directory in parallel. +TestPrefix () { + TO_RUN=parallel-"$1" + echo Running single testsuite directory with $TO_RUN + $MAKE -C testsuite $TO_RUN cd .. } @@ -170,7 +165,7 @@ case $1 in configure) Configure;; build) Build;; test) Test;; -test_multicore) TestLoop "${@:3}";; +test_prefix) TestPrefix $2;; api-docs) API_Docs;; install) Install;; manual) BuildManual;;