Auto merge of #124366 - Kobzol:remove-yaml-expansion, r=pietroalbini

CI: remove `expand-yaml-anchors`

This PR unifies all CI outcome jobs in a single job, and then removes the `expand-yaml-anchors` tool, since it is no longer needed after this change.

I have tested try builds for both situations with the new `outcome` job (note that these two workflow runs use a different step structure in the outcome job, I have simplified it since):
- [Success](https://github.com/rust-lang-ci/rust/actions/runs/8831529677/job/24251135366)
- [Failure](https://github.com/rust-lang-ci/rust/actions/runs/8833052319/job/24251628792)

r? `@ghost`

.github/workflows/ci.yml

@@ -1,21 +1,14 @@
-#############################################################
-#   WARNING: automatically generated file, DO NOT CHANGE!   #
-#############################################################
-
-# This file was automatically generated by the expand-yaml-anchors tool. The
-# source file that generated this one is:
-#
-#   src/ci/github-actions/ci.yml
-#
-# Once you make changes to that file you need to run:
+# This file defines our primary CI workflow that runs on pull requests
+# and also on pushes to special branches (auto, try).
 #
-#   ./x.py run src/tools/expand-yaml-anchors/
-#
-# The CI build will fail if the tool is not run after changes to this file.
+# The actual definition of the executed jobs is calculated by a Python
+# script located at src/ci/github-actions/calculate-job-matrix.py, which
+# uses job definition data from src/ci/github-actions/jobs.yml.
+# You should primarily modify the `jobs.yml` file if you want to modify
+# what jobs are executed in CI.
 
----
 name: CI
-"on":
+on:
   push:
     branches:
       - auto
@@ -25,176 +18,232 @@ name: CI
   pull_request:
     branches:
       - "**"
+
 permissions:
   contents: read
   packages: write
+
 defaults:
   run:
+    # On Linux, macOS, and Windows, use the system-provided bash as the default
+    # shell. (This should only make a difference on Windows, where the default
+    # shell is PowerShell.)
     shell: bash
+
 concurrency:
-  group: "${{ github.workflow }}-${{ ((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.sha) || github.ref }}"
+  # For a given workflow, if we push to the same branch, cancel all previous builds on that branch.
+  # We add an exception for try builds (try branch) and unrolled rollup builds (try-perf), which
+  # are all triggered on the same branch, but which should be able to run concurrently.
+  group: ${{ github.workflow }}-${{ ((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.sha) || github.ref }}
   cancel-in-progress: true
 env:
   TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate"
 jobs:
+  # The job matrix for `calculate_matrix` is defined in src/ci/github-actions/jobs.yml.
+  # It calculates which jobs should be executed, based on the data of the ${{ github }} context.
+  # If you want to modify CI jobs, take a look at src/ci/github-actions/jobs.yml.
   calculate_matrix:
     name: Calculate job matrix
     runs-on: ubuntu-latest
     outputs:
-      jobs: "${{ steps.jobs.outputs.jobs }}"
+      jobs: ${{ steps.jobs.outputs.jobs }}
+      run_type: ${{ steps.jobs.outputs.run_type }}
     steps:
       - name: Checkout the source code
         uses: actions/checkout@v4
       - name: Calculate the CI job matrix
         run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT
         id: jobs
   job:
-    name: "${{ matrix.name }}"
-    needs:
-      - calculate_matrix
+    name: ${{ matrix.name }}
+    needs: [ calculate_matrix ]
     runs-on: "${{ matrix.os }}"
     defaults:
       run:
-        shell: "${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}"
+        shell: ${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}
     timeout-minutes: 600
     env:
-      CI_JOB_NAME: "${{ matrix.image }}"
+      CI_JOB_NAME: ${{ matrix.image }}
       CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
-      HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}"
-      DOCKER_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+      # commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs.
+      HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+      DOCKER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       SCCACHE_BUCKET: rust-lang-ci-sccache2
       CACHE_DOMAIN: ci-caches.rust-lang.org
-    continue-on-error: "${{ matrix.continue_on_error || false }}"
+    continue-on-error: ${{ matrix.continue_on_error || false }}
     strategy:
       matrix:
-        include: "${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}"
-    if: "fromJSON(needs.calculate_matrix.outputs.jobs)[0] != null"
+        # Check the `calculate_matrix` job to see how is the matrix defined.
+        include: ${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}
+    # GitHub Actions fails the workflow if an empty list of jobs is provided to
+    # the workflow, so we need to skip this job if nothing was produced by
+    # the Python script.
+    #
+    # Unfortunately checking whether a list is empty is not possible in a nice
+    # way due to GitHub Actions expressions limits.
+    # This hack is taken from https://github.com/ferrocene/ferrocene/blob/d43edc6b7697cf1719ec1c17c54904ab94825763/.github/workflows/release.yml#L75-L82
+    if: fromJSON(needs.calculate_matrix.outputs.jobs)[0] != null
     steps:
-      - if: "contains(matrix.os, 'windows')"
+      - if: contains(matrix.os, 'windows')
         uses: msys2/setup-msys2@v2.22.0
         with:
-          msystem: "${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }}"
+          # i686 jobs use mingw32. x86_64 and cross-compile jobs use mingw64.
+          msystem: ${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }}
+          # don't try to download updates for already installed packages
           update: false
+          # don't try to use the msys that comes built-in to the github runner,
+          # so we can control what is installed (i.e. not python)
           release: true
+          # Inherit the full path from the Windows environment, with MSYS2's */bin/
+          # dirs placed in front. This lets us run Windows-native Python etc.
           path-type: inherit
-          install: "make dos2unix diffutils\n"
+          install: >
+            make
+            dos2unix
+            diffutils
+
       - name: disable git crlf conversion
         run: git config --global core.autocrlf false
+
       - name: checkout the source code
         uses: actions/checkout@v4
         with:
           fetch-depth: 2
+
+      # Rust Log Analyzer can't currently detect the PR number of a GitHub
+      # Actions build on its own, so a hint in the log message is needed to
+      # point it in the right direction.
       - name: configure the PR in which the error message will be posted
-        run: "echo \"[CI_PR_NUMBER=$num]\""
+        run: echo "[CI_PR_NUMBER=$num]"
         env:
-          num: "${{ github.event.number }}"
-        if: "success() && github.event_name == 'pull_request'"
+          num: ${{ github.event.number }}
+        if: needs.calculate_matrix.outputs.run_type == 'pr'
+
       - name: add extra environment variables
         run: src/ci/scripts/setup-environment.sh
         env:
-          EXTRA_VARIABLES: "${{ toJson(matrix.env) }}"
+          # Since it's not possible to merge `${{ matrix.env }}` with the other
+          # variables in `job.<name>.env`, the variables defined in the matrix
+          # are passed to the `setup-environment.sh` script encoded in JSON,
+          # which then uses log commands to actually set them.
+          EXTRA_VARIABLES: ${{ toJson(matrix.env) }}
+
       - name: ensure the channel matches the target branch
         run: src/ci/scripts/verify-channel.sh
+
       - name: collect CPU statistics
         run: src/ci/scripts/collect-cpu-stats.sh
+
       - name: show the current environment
         run: src/ci/scripts/dump-environment.sh
+
       - name: install awscli
         run: src/ci/scripts/install-awscli.sh
+
       - name: install sccache
         run: src/ci/scripts/install-sccache.sh
+
       - name: select Xcode
         run: src/ci/scripts/select-xcode.sh
+
       - name: install clang
         run: src/ci/scripts/install-clang.sh
+
       - name: install tidy
         run: src/ci/scripts/install-tidy.sh
+
       - name: install WIX
         run: src/ci/scripts/install-wix.sh
+
       - name: disable git crlf conversion
         run: src/ci/scripts/disable-git-crlf-conversion.sh
+
       - name: checkout submodules
         run: src/ci/scripts/checkout-submodules.sh
+
       - name: install MSYS2
         run: src/ci/scripts/install-msys2.sh
+
       - name: install MinGW
         run: src/ci/scripts/install-mingw.sh
+
       - name: install ninja
         run: src/ci/scripts/install-ninja.sh
+
       - name: enable ipv6 on Docker
         run: src/ci/scripts/enable-docker-ipv6.sh
+
+      # Disable automatic line ending conversion (again). On Windows, when we're
+      # installing dependencies, something switches the git configuration directory or
+      # re-enables autocrlf. We've not tracked down the exact cause -- and there may
+      # be multiple -- but this should ensure submodules are checked out with the
+      # appropriate line endings.
       - name: disable git crlf conversion
         run: src/ci/scripts/disable-git-crlf-conversion.sh
+
       - name: ensure line endings are correct
         run: src/ci/scripts/verify-line-endings.sh
+
       - name: ensure backported commits are in upstream branches
         run: src/ci/scripts/verify-backported-commits.sh
+
       - name: ensure the stable version number is correct
         run: src/ci/scripts/verify-stable-version-number.sh
+
       - name: run the build
+        # Redirect stderr to stdout to avoid reordering the two streams in the GHA logs.
         run: src/ci/scripts/run-build-from-ci.sh 2>&1
         env:
-          AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}"
-          AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}"
-          TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}"
+          AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}
+          TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
+
       - name: create github artifacts
         run: src/ci/scripts/create-doc-artifacts.sh
+
       - name: upload artifacts to github
         uses: actions/upload-artifact@v4
         with:
-          name: "${{ env.DOC_ARTIFACT_NAME }}"
+          # name is set in previous step
+          name: ${{ env.DOC_ARTIFACT_NAME }}
           path: obj/artifacts/doc
           if-no-files-found: ignore
           retention-days: 5
+
       - name: upload artifacts to S3
         run: src/ci/scripts/upload-artifacts.sh
         env:
-          AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}"
-          AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}"
-        if: "success() && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')"
-  try-success:
-    needs:
-      - job
-    if: "success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'"
-    steps:
-      - name: mark the job as a success
-        run: exit 0
-        shell: bash
-    name: bors build finished
-    runs-on: ubuntu-latest
-  try-failure:
-    needs:
-      - job
-    if: "!success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'"
-    steps:
-      - name: mark the job as a failure
-        run: exit 1
-        shell: bash
+          AWS_ACCESS_KEY_ID: ${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}
+        # Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy
+        # builders *should* have the AWS credentials available. Still, explicitly
+        # adding the condition is helpful as this way CI will not silently skip
+        # deploying artifacts from a dist builder if the variables are misconfigured,
+        # erroring about invalid credentials instead.
+        if: github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1'
+
+  # This job isused to tell bors the final status of the build, as there is no practical way to detect
+  # when a workflow is successful listening to webhooks only in our current bors implementation (homu).
+  outcome:
     name: bors build finished
     runs-on: ubuntu-latest
-  auto-success:
-    needs:
-      - job
-    if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
+    needs: [ calculate_matrix, job ]
+    # !cancelled() executes the job regardless of whether the previous jobs passed or failed
+    if: ${{ !cancelled() && contains(fromJSON('["auto", "try"]'), needs.calculate_matrix.outputs.run_type) }}
     steps:
       - name: checkout the source code
         uses: actions/checkout@v4
         with:
           fetch-depth: 2
+      # Calculate the exit status of the whole CI workflow.
+      # If all dependent jobs were successful, this exits with 0 (and the outcome job continues successfully).
+      # If a some dependent job has failed, this exits with 1.
+      - name: calculate the correct exit status
+        run: jq --exit-status 'all(.result == "success" or .result == "skipped")' <<< '${{ toJson(needs) }}'
+      # Publish the toolstate if an auto build succeeds (just before push to master)
       - name: publish toolstate
         run: src/ci/publish_toolstate.sh
         shell: bash
+        if: needs.calculate_matrix.outputs.run_type == 'auto'
         env:
-          TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}"
-    name: bors build finished
-    runs-on: ubuntu-latest
-  auto-failure:
-    needs:
-      - job
-    if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
-    steps:
-      - name: mark the job as a failure
-        run: exit 1
-        shell: bash
-    name: bors build finished
-    runs-on: ubuntu-latest
+          TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}

Cargo.lock

@@ -1254,14 +1254,6 @@ dependencies = [
  "mdbook",
 ]
 
-[[package]]
-name = "expand-yaml-anchors"
-version = "0.1.0"
-dependencies = [
- "yaml-merge-keys",
- "yaml-rust",
-]
-
 [[package]]
 name = "expect-test"
 version = "1.5.0"
@@ -2234,12 +2226,6 @@ dependencies = [
  "regex",
 ]
 
-[[package]]
-name = "linked-hash-map"
-version = "0.5.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
-
 [[package]]
 name = "lint-docs"
 version = "0.1.0"
@@ -6540,26 +6526,6 @@ dependencies = [
  "lzma-sys",
 ]
 
-[[package]]
-name = "yaml-merge-keys"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd236a7dc9bb598f349fe4a8754f49181fee50284daa15cd1ba652d722280004"
-dependencies = [
- "lazy_static",
- "thiserror",
- "yaml-rust",
-]
-
-[[package]]
-name = "yaml-rust"
-version = "0.4.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85"
-dependencies = [
- "linked-hash-map",
-]
-
 [[package]]
 name = "yansi-term"
 version = "0.1.2"

Cargo.toml

@@ -31,7 +31,6 @@ members = [
   "src/tools/miri/cargo-miri",
   "src/tools/rustdoc-themes",
   "src/tools/unicode-table-generator",
-  "src/tools/expand-yaml-anchors",
   "src/tools/jsondocck",
   "src/tools/jsondoclint",
   "src/tools/llvm-bitcode-linker",