ci: Allow for multiple icount benchmarks in the same run

.github/workflows/main.yaml

@@ -195,8 +195,14 @@ jobs:
 
   benchmarks:
     name: Benchmarks
-    runs-on: ubuntu-24.04
     timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - target: x86_64-unknown-linux-gnu
+          os: ubuntu-24.04
+    runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@master
       with:
@@ -215,12 +221,14 @@ jobs:
         cargo binstall -y iai-callgrind-runner --version "$iai_version"
         sudo apt-get install valgrind
     - uses: Swatinem/rust-cache@v2
+      with:
+        key: ${{ matrix.target }}
 
     - name: Run icount benchmarks
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         PR_NUMBER: ${{ github.event.pull_request.number }}
-      run: ./ci/bench-icount.sh
+      run: ./ci/bench-icount.sh ${{ matrix.target }}
 
     - name: Upload the benchmark baseline
       uses: actions/upload-artifact@v4

ci/bench-icount.sh

@@ -2,10 +2,21 @@
 
 set -eux
 
+target="${1:-}"
+
+if [ -z "$target" ]; then
+    host_target=$(rustc -vV | awk '/^host/ { print $2 }')
+    echo "Defaulted to host target $host_target"
+    target="$host_target"
+fi
+
 iai_home="iai-home"
 
+# Use the arch as a tag to disambiguate artifacts
+tag="$(echo "$target" | cut -d'-' -f1)"
+
 # Download the baseline from master
-./ci/ci-util.py locate-baseline --download --extract
+./ci/ci-util.py locate-baseline --download --extract --tag "$tag"
 
 # Run benchmarks once
 function run_icount_benchmarks() {
@@ -44,6 +55,7 @@ function run_icount_benchmarks() {
         # If this is for a pull request, ignore regressions if specified.
         ./ci/ci-util.py check-regressions --home "$iai_home" --allow-pr-override "$PR_NUMBER"
     else
+        # Disregard regressions after merge
         ./ci/ci-util.py check-regressions --home "$iai_home" || true
     fi
 }
@@ -53,6 +65,6 @@ run_icount_benchmarks --features force-soft-floats -- --save-baseline=softfloat
 run_icount_benchmarks -- --save-baseline=hardfloat
 
 # Name and tar the new baseline
-name="baseline-icount-$(date -u +'%Y%m%d%H%M')-${GITHUB_SHA:0:12}"
+name="baseline-icount-$tag-$(date -u +'%Y%m%d%H%M')-${GITHUB_SHA:0:12}"
 echo "BASELINE_NAME=$name" >>"$GITHUB_ENV"
 tar cJf "$name.tar.xz" "$iai_home"

ci/ci-util.py

@@ -28,11 +28,14 @@
             Calculate a matrix of which functions had source change, print that as
             a JSON object.
 
-        locate-baseline [--download] [--extract]
+        locate-baseline [--download] [--extract] [--tag TAG]
             Locate the most recent benchmark baseline available in CI and, if flags
             specify, download and extract it. Never exits with nonzero status if
             downloading fails.
 
+            `--tag` can be specified to look for artifacts with a specific tag, such as
+            for a specific architecture.
+
             Note that `--extract` will overwrite files in `iai-home`.
 
         check-regressions [--home iai-home] [--allow-pr-override pr_number]
@@ -50,7 +53,7 @@
 GIT = ["git", "-C", REPO_ROOT]
 DEFAULT_BRANCH = "master"
 WORKFLOW_NAME = "CI"  # Workflow that generates the benchmark artifacts
-ARTIFACT_GLOB = "baseline-icount*"
+ARTIFACT_PREFIX = "baseline-icount*"
 # Place this in a PR body to skip regression checks (must be at the start of a line).
 REGRESSION_DIRECTIVE = "ci: allow-regressions"
 # Place this in a PR body to skip extensive tests
@@ -278,13 +281,17 @@ def locate_baseline(flags: list[str]) -> None:
 
     download = False
     extract = False
+    tag = ""
 
     while len(flags) > 0:
         match flags[0]:
             case "--download":
                 download = True
             case "--extract":
                 extract = True
+            case "--tag":
+                tag = flags[1]
+                flags = flags[1:]
             case _:
                 eprint(USAGE)
                 exit(1)
@@ -333,8 +340,10 @@ def locate_baseline(flags: list[str]) -> None:
         eprint("skipping download step")
         return
 
+    artifact_glob = f"{ARTIFACT_PREFIX}{f"-{tag}" if tag else ""}*"
+
     sp.run(
-        ["gh", "run", "download", str(job_id), f"--pattern={ARTIFACT_GLOB}"],
+        ["gh", "run", "download", str(job_id), f"--pattern={artifact_glob}"],
         check=False,
     )
 
@@ -344,7 +353,7 @@ def locate_baseline(flags: list[str]) -> None:
 
     # Find the baseline with the most recent timestamp. GH downloads the files to e.g.
     # `some-dirname/some-dirname.tar.xz`, so just glob the whole thing together.
-    candidate_baselines = glob(f"{ARTIFACT_GLOB}/{ARTIFACT_GLOB}")
+    candidate_baselines = glob(f"{artifact_glob}/{artifact_glob}")
     if len(candidate_baselines) == 0:
         eprint("no possible baseline directories found")
         return