From 9fec43ddf12118f6bfa66aedb17bca91618f153b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Fri, 19 Apr 2024 21:41:49 +0200 Subject: [PATCH 1/8] Move calculate-job-matrix.py to `ci/github-actions` --- .github/workflows/ci.yml | 2 +- src/ci/{scripts => github-actions}/calculate-job-matrix.py | 2 +- src/ci/github-actions/ci.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename src/ci/{scripts => github-actions}/calculate-job-matrix.py (84%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 90269f56c52a4..e01eec949a21b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,7 +45,7 @@ jobs: - name: Checkout the source code uses: actions/checkout@v4 - name: Calculate the CI job matrix - run: python3 src/ci/scripts/calculate-job-matrix.py >> $GITHUB_OUTPUT + run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT id: jobs pr: name: "PR - ${{ matrix.name }}" diff --git a/src/ci/scripts/calculate-job-matrix.py b/src/ci/github-actions/calculate-job-matrix.py similarity index 84% rename from src/ci/scripts/calculate-job-matrix.py rename to src/ci/github-actions/calculate-job-matrix.py index 9b1e74c23c324..ba40f4da84752 100755 --- a/src/ci/scripts/calculate-job-matrix.py +++ b/src/ci/github-actions/calculate-job-matrix.py @@ -15,7 +15,7 @@ import yaml -JOBS_YAML_PATH = Path(__file__).absolute().parent.parent / "github-actions" / "jobs.yml" +JOBS_YAML_PATH = Path(__file__).absolute().parent / "jobs.yml" if __name__ == "__main__": diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index de71b9f874f8d..9d1c3284be20c 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -351,7 +351,7 @@ jobs: - name: Checkout the source code uses: actions/checkout@v4 - name: Calculate the CI job matrix - run: python3 src/ci/scripts/calculate-job-matrix.py >> $GITHUB_OUTPUT + run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT id: jobs pr: <<: *base-ci-job From 7a90679e28382ebd48790eaab80cb2381704223e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Sat, 20 Apr 2024 09:07:11 +0200 Subject: [PATCH 2/8] Perform PR and try builds dynamically --- .github/workflows/ci.yml | 155 +----------------- src/ci/github-actions/calculate-job-matrix.py | 42 ++++- src/ci/github-actions/ci.yml | 41 ++--- src/ci/github-actions/jobs.yml | 45 ++++- 4 files changed, 102 insertions(+), 181 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e01eec949a21b..25be8607edd7d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,23 +45,23 @@ jobs: - name: Checkout the source code uses: actions/checkout@v4 - name: Calculate the CI job matrix + env: + GITHUB_CTX: "${{ toJSON(github) }}" run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT id: jobs - pr: - name: "PR - ${{ matrix.name }}" + job: + name: "${{ matrix.name }}" needs: - calculate_matrix env: - PR_CI_JOB: 1 - CI_JOB_NAME: "${{ matrix.name }}" + CI_JOB_NAME: "${{ matrix.image }}" CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}" DOCKER_TOKEN: "${{ secrets.GITHUB_TOKEN }}" SCCACHE_BUCKET: rust-lang-ci-sccache2 TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate" CACHE_DOMAIN: ci-caches.rust-lang.org - if: "github.event_name == 'pull_request'" - continue-on-error: "${{ matrix.name == 'mingw-check-tidy' }}" + continue-on-error: "${{ matrix.continue_on_error || false }}" strategy: matrix: include: "${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}" @@ -589,145 +589,6 @@ jobs: AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}" AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}" if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')" - try: - name: "try - ${{ matrix.name }}" - env: - DIST_TRY_BUILD: 1 - CI_JOB_NAME: "${{ matrix.name }}" - CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse - HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}" - DOCKER_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - SCCACHE_BUCKET: rust-lang-ci-sccache2 - DEPLOY_BUCKET: rust-lang-ci2 - TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate" - TOOLSTATE_ISSUES_API_URL: "https://api.github.com/repos/rust-lang/rust/issues" - TOOLSTATE_PUBLISH: 1 - CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL - ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55 - AWS_REGION: us-west-1 - CACHE_DOMAIN: ci-caches.rust-lang.org - if: "github.event_name == 'push' && (((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust') || ((github.ref == 'refs/heads/automation/bors/try') && github.repository == 'rust-lang/rust'))" - strategy: - matrix: - include: - - name: dist-x86_64-linux - env: - CODEGEN_BACKENDS: "llvm,cranelift" - os: ubuntu-20.04-16core-64gb - defaults: - run: - shell: "${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}" - timeout-minutes: 600 - runs-on: "${{ matrix.os }}" - steps: - - if: "contains(matrix.os, 'windows')" - uses: msys2/setup-msys2@v2.22.0 - with: - msystem: "${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }}" - update: false - release: true - path-type: inherit - install: "make dos2unix diffutils\n" - - name: disable git crlf conversion - run: git config --global core.autocrlf false - - name: checkout the source code - uses: actions/checkout@v4 - with: - fetch-depth: 2 - - name: configure the PR in which the error message will be posted - run: "echo \"[CI_PR_NUMBER=$num]\"" - env: - num: "${{ github.event.number }}" - if: "success() && !env.SKIP_JOB && github.event_name == 'pull_request'" - - name: add extra environment variables - run: src/ci/scripts/setup-environment.sh - env: - EXTRA_VARIABLES: "${{ toJson(matrix.env) }}" - if: success() && !env.SKIP_JOB - - name: decide whether to skip this job - run: src/ci/scripts/should-skip-this.sh - if: success() && !env.SKIP_JOB - - name: ensure the channel matches the target branch - run: src/ci/scripts/verify-channel.sh - if: success() && !env.SKIP_JOB - - name: collect CPU statistics - run: src/ci/scripts/collect-cpu-stats.sh - if: success() && !env.SKIP_JOB - - name: show the current environment - run: src/ci/scripts/dump-environment.sh - if: success() && !env.SKIP_JOB - - name: install awscli - run: src/ci/scripts/install-awscli.sh - if: success() && !env.SKIP_JOB - - name: install sccache - run: src/ci/scripts/install-sccache.sh - if: success() && !env.SKIP_JOB - - name: select Xcode - run: src/ci/scripts/select-xcode.sh - if: success() && !env.SKIP_JOB - - name: install clang - run: src/ci/scripts/install-clang.sh - if: success() && !env.SKIP_JOB - - name: install tidy - run: src/ci/scripts/install-tidy.sh - if: success() && !env.SKIP_JOB - - name: install WIX - run: src/ci/scripts/install-wix.sh - if: success() && !env.SKIP_JOB - - name: disable git crlf conversion - run: src/ci/scripts/disable-git-crlf-conversion.sh - if: success() && !env.SKIP_JOB - - name: checkout submodules - run: src/ci/scripts/checkout-submodules.sh - if: success() && !env.SKIP_JOB - - name: install MSYS2 - run: src/ci/scripts/install-msys2.sh - if: success() && !env.SKIP_JOB - - name: install MinGW - run: src/ci/scripts/install-mingw.sh - if: success() && !env.SKIP_JOB - - name: install ninja - run: src/ci/scripts/install-ninja.sh - if: success() && !env.SKIP_JOB - - name: enable ipv6 on Docker - run: src/ci/scripts/enable-docker-ipv6.sh - if: success() && !env.SKIP_JOB - - name: disable git crlf conversion - run: src/ci/scripts/disable-git-crlf-conversion.sh - if: success() && !env.SKIP_JOB - - name: ensure line endings are correct - run: src/ci/scripts/verify-line-endings.sh - if: success() && !env.SKIP_JOB - - name: ensure backported commits are in upstream branches - run: src/ci/scripts/verify-backported-commits.sh - if: success() && !env.SKIP_JOB - - name: ensure the stable version number is correct - run: src/ci/scripts/verify-stable-version-number.sh - if: success() && !env.SKIP_JOB - - name: run the build - run: src/ci/scripts/run-build-from-ci.sh 2>&1 - env: - AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}" - AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}" - TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}" - if: success() && !env.SKIP_JOB - - name: create github artifacts - run: src/ci/scripts/create-doc-artifacts.sh - if: success() && !env.SKIP_JOB - - name: upload artifacts to github - uses: actions/upload-artifact@v4 - with: - name: "${{ env.DOC_ARTIFACT_NAME }}" - path: obj/artifacts/doc - if-no-files-found: ignore - retention-days: 5 - if: success() && !env.SKIP_JOB - - name: upload artifacts to S3 - run: src/ci/scripts/upload-artifacts.sh - env: - AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}" - AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}" - if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')" master: name: master runs-on: ubuntu-latest @@ -755,7 +616,7 @@ jobs: if: success() && !env.SKIP_JOB try-success: needs: - - try + - job if: "success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'" steps: - name: mark the job as a success @@ -765,7 +626,7 @@ jobs: runs-on: ubuntu-latest try-failure: needs: - - try + - job if: "!success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'" steps: - name: mark the job as a failure diff --git a/src/ci/github-actions/calculate-job-matrix.py b/src/ci/github-actions/calculate-job-matrix.py index ba40f4da84752..59c191cb45db4 100755 --- a/src/ci/github-actions/calculate-job-matrix.py +++ b/src/ci/github-actions/calculate-job-matrix.py @@ -7,19 +7,53 @@ It reads job definitions from `src/ci/github-actions/jobs.yml` and filters them based on the event that happened on CI. -Currently, it only supports PR builds. +Currently, it only supports PR and try builds. """ import json +import os +import sys from pathlib import Path +from typing import List, Dict import yaml JOBS_YAML_PATH = Path(__file__).absolute().parent / "jobs.yml" +def name_jobs(jobs: List[Dict], prefix: str) -> List[Dict]: + for job in jobs: + job["name"] = f"{prefix} - {job['image']}" + return jobs + + if __name__ == "__main__": + github_ctx = json.loads(os.environ["GITHUB_CTX"]) + with open(JOBS_YAML_PATH) as f: - jobs = yaml.safe_load(f) - job_output = jobs["pr"] - print(f"jobs={json.dumps(job_output)}") + data = yaml.safe_load(f) + + event_name = github_ctx["event_name"] + ref = github_ctx["ref"] + repository = github_ctx["repository"] + + old_bors_try_build = ( + ref in ("refs/heads/try", "refs/heads/try-perf") and + repository == "rust-lang-ci/rust" + ) + new_bors_try_build = ( + ref == "refs/heads/automation/bors/try" and + repository == "rust-lang/rust" + ) + try_build = old_bors_try_build or new_bors_try_build + + jobs = [] + # Pull request CI jobs. Their name is 'PR - ' + if event_name == "pull_request": + jobs = name_jobs(data["pr"], "PR") + # Try builds + elif event_name == "push" and try_build: + jobs = name_jobs(data["try"], "try") + + print(f"Output:\n{json.dumps(jobs, indent=4)}", file=sys.stderr) + print(f"jobs={json.dumps(jobs)}") diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 9d1c3284be20c..e7847a57e9741 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -351,17 +351,25 @@ jobs: - name: Checkout the source code uses: actions/checkout@v4 - name: Calculate the CI job matrix + env: + GITHUB_CTX: ${{ toJSON(github) }} run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT id: jobs - pr: + # This is either a PR or a try job, decided by calculate-job-matrix.py + job: <<: *base-ci-job - name: PR - ${{ matrix.name }} + name: ${{ matrix.name }} needs: [ calculate_matrix ] env: - <<: [*shared-ci-variables, *public-variables] - PR_CI_JOB: 1 - if: github.event_name == 'pull_request' - continue-on-error: ${{ matrix.name == 'mingw-check-tidy' }} + CI_JOB_NAME: ${{ matrix.image }} + CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse + # commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs. + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + DOCKER_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SCCACHE_BUCKET: rust-lang-ci-sccache2 + TOOLSTATE_REPO: https://github.com/rust-lang-nursery/rust-toolstate + CACHE_DOMAIN: ci-caches.rust-lang.org + continue-on-error: ${{ matrix.continue_on_error || false }} strategy: matrix: # Check the `calculate_matrix` job to see how is the matrix defined. @@ -751,23 +759,6 @@ jobs: SCRIPT: python x.py dist bootstrap --include-default-paths <<: *job-windows-8c - try: - <<: *base-ci-job - name: try - ${{ matrix.name }} - env: - DIST_TRY_BUILD: 1 - <<: [*shared-ci-variables, *prod-variables] - if: github.event_name == 'push' && (((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust') || ((github.ref == 'refs/heads/automation/bors/try') && github.repository == 'rust-lang/rust')) - strategy: - matrix: - include: - - &dist-x86_64-linux - name: dist-x86_64-linux - env: - CODEGEN_BACKENDS: llvm,cranelift - <<: *job-linux-16c - - master: name: master runs-on: ubuntu-latest @@ -791,11 +782,11 @@ jobs: # build completed, as there is no practical way to detect when a workflow is # successful listening to webhooks only. try-success: - needs: [try] + needs: [ job ] if: "success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'" <<: *base-success-job try-failure: - needs: [try] + needs: [ job ] if: "!success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'" <<: *base-failure-job auto-success: diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml index 7e89eef2670e4..2aea6ce4fa7d8 100644 --- a/src/ci/github-actions/jobs.yml +++ b/src/ci/github-actions/jobs.yml @@ -2,7 +2,7 @@ # dynamically in CI from ci.yml. # You *do not* need to re-run `src/tools/expand-yaml-anchors` when you # modify this file. -shared_defs: +runners: - &base-job env: { } @@ -37,14 +37,49 @@ shared_defs: - &job-aarch64-linux os: [ self-hosted, ARM64, linux ] +envs: + - &prod-variables + DEPLOY_BUCKET: rust-lang-ci2 + TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/rust-lang/rust/issues + TOOLSTATE_PUBLISH: 1 + # AWS_SECRET_ACCESS_KEYs are stored in GitHub's secrets storage, named + # AWS_SECRET_ACCESS_KEY_. Including the key id in the name allows to + # rotate them in a single branch while keeping the old key in another + # branch, which wouldn't be possible if the key was named with the kind + # (caches, artifacts...). + CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL + ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55 + AWS_REGION: us-west-1 + + - &pr-env + PR_CI_JOB: 1 + +# Pull request jobs pr: - - name: mingw-check + - image: mingw-check + env: + <<: *pr-env <<: *job-linux-4c - - name: mingw-check-tidy + - image: mingw-check-tidy + continue_on_error: true + env: + <<: *pr-env <<: *job-linux-4c - - name: x86_64-gnu-llvm-17 + - image: x86_64-gnu-llvm-17 env: ENABLE_GCC_CODEGEN: "1" + <<: *pr-env <<: *job-linux-16c - - name: x86_64-gnu-tools + - image: x86_64-gnu-tools + env: + <<: *pr-env + <<: *job-linux-16c + +# Try build jobs +try: + - image: dist-x86_64-linux + env: + DIST_TRY_BUILD: 1 + CODEGEN_BACKENDS: llvm,cranelift + <<: *prod-variables <<: *job-linux-16c From 6f0ff0b03a291382ceb51fc12ea6882605684a5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Sat, 20 Apr 2024 09:16:16 +0200 Subject: [PATCH 3/8] Refactor `calculate-job-matrix.py` --- src/ci/github-actions/calculate-job-matrix.py | 71 +++++++++++++------ 1 file changed, 48 insertions(+), 23 deletions(-) diff --git a/src/ci/github-actions/calculate-job-matrix.py b/src/ci/github-actions/calculate-job-matrix.py index 59c191cb45db4..15a835e260cce 100755 --- a/src/ci/github-actions/calculate-job-matrix.py +++ b/src/ci/github-actions/calculate-job-matrix.py @@ -9,12 +9,12 @@ Currently, it only supports PR and try builds. """ - +import enum import json +import logging import os -import sys from pathlib import Path -from typing import List, Dict +from typing import List, Dict, Any, Optional import yaml @@ -27,33 +27,58 @@ def name_jobs(jobs: List[Dict], prefix: str) -> List[Dict]: return jobs -if __name__ == "__main__": - github_ctx = json.loads(os.environ["GITHUB_CTX"]) +class JobType(enum.Enum): + PR = enum.auto() + Try = enum.auto() - with open(JOBS_YAML_PATH) as f: - data = yaml.safe_load(f) +def find_job_type(github_ctx: Dict[str, Any]) -> Optional[JobType]: event_name = github_ctx["event_name"] ref = github_ctx["ref"] repository = github_ctx["repository"] - old_bors_try_build = ( - ref in ("refs/heads/try", "refs/heads/try-perf") and - repository == "rust-lang-ci/rust" - ) - new_bors_try_build = ( - ref == "refs/heads/automation/bors/try" and - repository == "rust-lang/rust" - ) - try_build = old_bors_try_build or new_bors_try_build + if event_name == "pull_request": + return JobType.PR + elif event_name == "push": + old_bors_try_build = ( + ref in ("refs/heads/try", "refs/heads/try-perf") and + repository == "rust-lang-ci/rust" + ) + new_bors_try_build = ( + ref == "refs/heads/automation/bors/try" and + repository == "rust-lang/rust" + ) + try_build = old_bors_try_build or new_bors_try_build + + if try_build: + return JobType.Try + + return None + + +def calculate_jobs(job_type: JobType, job_data: Dict[str, Any]) -> List[Dict[str, Any]]: + if job_type == JobType.PR: + return name_jobs(job_data["pr"], "PR") + elif job_type == JobType.Try: + return name_jobs(job_data["try"], "try") + + return [] + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + + github_ctx = json.loads(os.environ["GITHUB_CTX"]) + + with open(JOBS_YAML_PATH) as f: + data = yaml.safe_load(f) + + job_type = find_job_type(github_ctx) + logging.info(f"Job type: {job_type}") jobs = [] - # Pull request CI jobs. Their name is 'PR - ' - if event_name == "pull_request": - jobs = name_jobs(data["pr"], "PR") - # Try builds - elif event_name == "push" and try_build: - jobs = name_jobs(data["try"], "try") + if job_type is not None: + jobs = calculate_jobs(job_type, data) - print(f"Output:\n{json.dumps(jobs, indent=4)}", file=sys.stderr) + logging.info(f"Output:\n{yaml.dump(jobs, indent=4)}") print(f"jobs={json.dumps(jobs)}") From d2059aef270fc60f65273f816467761a4eeeb638 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Sat, 20 Apr 2024 09:42:23 +0200 Subject: [PATCH 4/8] Move `auto` jobs to calculate-job-matrix.py --- .github/workflows/ci.yml | 414 +----------------- src/ci/github-actions/calculate-job-matrix.py | 31 +- src/ci/github-actions/ci.yml | 391 +---------------- src/ci/github-actions/jobs.yml | 398 ++++++++++++++++- 4 files changed, 417 insertions(+), 817 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 25be8607edd7d..010872c980f95 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -179,416 +179,6 @@ jobs: AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}" AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}" if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')" - auto: - name: "auto - ${{ matrix.name }}" - env: - CI_JOB_NAME: "${{ matrix.name }}" - CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse - HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}" - DOCKER_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - SCCACHE_BUCKET: rust-lang-ci-sccache2 - DEPLOY_BUCKET: rust-lang-ci2 - TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate" - TOOLSTATE_ISSUES_API_URL: "https://api.github.com/repos/rust-lang/rust/issues" - TOOLSTATE_PUBLISH: 1 - CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL - ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55 - AWS_REGION: us-west-1 - CACHE_DOMAIN: ci-caches.rust-lang.org - if: "github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'" - strategy: - matrix: - include: - - name: aarch64-gnu - os: - - self-hosted - - ARM64 - - linux - - name: arm-android - os: ubuntu-20.04-8core-32gb - env: {} - - name: armhf-gnu - os: ubuntu-20.04-8core-32gb - env: {} - - name: dist-aarch64-linux - env: - CODEGEN_BACKENDS: "llvm,cranelift" - os: ubuntu-20.04-8core-32gb - - name: dist-android - os: ubuntu-20.04-8core-32gb - env: {} - - name: dist-arm-linux - os: ubuntu-20.04-16core-64gb - env: {} - - name: dist-armhf-linux - os: ubuntu-20.04-8core-32gb - env: {} - - name: dist-armv7-linux - os: ubuntu-20.04-8core-32gb - env: {} - - name: dist-i586-gnu-i586-i686-musl - os: ubuntu-20.04-8core-32gb - env: {} - - name: dist-i686-linux - os: ubuntu-20.04-8core-32gb - env: {} - - name: dist-loongarch64-linux - os: ubuntu-20.04-8core-32gb - env: {} - - name: dist-ohos - os: ubuntu-20.04-8core-32gb - env: {} - - name: dist-powerpc-linux - os: ubuntu-20.04-8core-32gb - env: {} - - name: dist-powerpc64-linux - os: ubuntu-20.04-8core-32gb - env: {} - - name: dist-powerpc64le-linux - os: ubuntu-20.04-8core-32gb - env: {} - - name: dist-riscv64-linux - os: ubuntu-20.04-8core-32gb - env: {} - - name: dist-s390x-linux - os: ubuntu-20.04-8core-32gb - env: {} - - name: dist-various-1 - os: ubuntu-20.04-8core-32gb - env: {} - - name: dist-various-2 - os: ubuntu-20.04-8core-32gb - env: {} - - name: dist-x86_64-freebsd - os: ubuntu-20.04-8core-32gb - env: {} - - name: dist-x86_64-illumos - os: ubuntu-20.04-8core-32gb - env: {} - - name: dist-x86_64-linux - env: - CODEGEN_BACKENDS: "llvm,cranelift" - os: ubuntu-20.04-16core-64gb - - name: dist-x86_64-linux-alt - env: - IMAGE: dist-x86_64-linux - CODEGEN_BACKENDS: "llvm,cranelift" - os: ubuntu-20.04-16core-64gb - - name: dist-x86_64-musl - env: - CODEGEN_BACKENDS: "llvm,cranelift" - os: ubuntu-20.04-8core-32gb - - name: dist-x86_64-netbsd - os: ubuntu-20.04-8core-32gb - env: {} - - name: i686-gnu - os: ubuntu-20.04-8core-32gb - env: {} - - name: i686-gnu-nopt - os: ubuntu-20.04-8core-32gb - env: {} - - name: mingw-check - os: ubuntu-20.04-4core-16gb - env: {} - - name: test-various - os: ubuntu-20.04-8core-32gb - env: {} - - name: x86_64-gnu - os: ubuntu-20.04-4core-16gb - env: {} - - name: x86_64-gnu-stable - env: - IMAGE: x86_64-gnu - RUST_CI_OVERRIDE_RELEASE_CHANNEL: stable - CI_ONLY_WHEN_CHANNEL: nightly - os: ubuntu-20.04-4core-16gb - - name: x86_64-gnu-aux - os: ubuntu-20.04-4core-16gb - env: {} - - name: x86_64-gnu-integration - env: - CI_ONLY_WHEN_CHANNEL: nightly - os: ubuntu-20.04-8core-32gb - - name: x86_64-gnu-debug - os: ubuntu-20.04-8core-32gb - env: {} - - name: x86_64-gnu-distcheck - os: ubuntu-20.04-8core-32gb - env: {} - - name: x86_64-gnu-llvm-18 - env: - RUST_BACKTRACE: 1 - os: ubuntu-20.04-8core-32gb - - name: x86_64-gnu-llvm-17 - env: - RUST_BACKTRACE: 1 - os: ubuntu-20.04-8core-32gb - - name: x86_64-gnu-nopt - os: ubuntu-20.04-4core-16gb - env: {} - - name: x86_64-gnu-tools - env: - DEPLOY_TOOLSTATES_JSON: toolstates-linux.json - os: ubuntu-20.04-8core-32gb - - name: dist-x86_64-apple - env: - SCRIPT: "./x.py dist bootstrap --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin" - RUST_CONFIGURE_ARGS: "--enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set rust.lto=thin --set rust.codegen-units=1" - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.12 - SELECT_XCODE: /Applications/Xcode_14.3.1.app - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - NO_OVERFLOW_CHECKS: 1 - DIST_REQUIRE_ALL_TOOLS: 1 - CODEGEN_BACKENDS: "llvm,cranelift" - os: macos-13 - - name: dist-apple-various - env: - SCRIPT: "./x.py dist bootstrap --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim" - RUST_CONFIGURE_ARGS: "--enable-sanitizers --enable-profiler --set rust.jemalloc" - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.12 - SELECT_XCODE: /Applications/Xcode_14.3.1.app - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - NO_OVERFLOW_CHECKS: 1 - os: macos-13 - - name: x86_64-apple-1 - env: - SCRIPT: "./x.py --stage 2 test --skip tests/ui --skip tests/rustdoc --skip tests/run-make-fulldeps" - RUST_CONFIGURE_ARGS: "--build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc" - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.12 - MACOSX_STD_DEPLOYMENT_TARGET: 10.12 - SELECT_XCODE: /Applications/Xcode_14.3.1.app - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - NO_OVERFLOW_CHECKS: 1 - os: macos-13 - - name: x86_64-apple-2 - env: - SCRIPT: "./x.py --stage 2 test tests/ui tests/rustdoc tests/run-make-fulldeps" - RUST_CONFIGURE_ARGS: "--build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc" - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.12 - MACOSX_STD_DEPLOYMENT_TARGET: 10.12 - SELECT_XCODE: /Applications/Xcode_14.3.1.app - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - NO_OVERFLOW_CHECKS: 1 - os: macos-13 - - name: dist-aarch64-apple - env: - SCRIPT: "./x.py dist bootstrap --include-default-paths --host=aarch64-apple-darwin --target=aarch64-apple-darwin" - RUST_CONFIGURE_ARGS: "--enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false --set rust.lto=thin" - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - SELECT_XCODE: /Applications/Xcode_14.3.1.app - USE_XCODE_CLANG: 1 - MACOSX_DEPLOYMENT_TARGET: 11.0 - MACOSX_STD_DEPLOYMENT_TARGET: 11.0 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - NO_OVERFLOW_CHECKS: 1 - DIST_REQUIRE_ALL_TOOLS: 1 - os: macos-14 - - name: aarch64-apple - env: - SCRIPT: "./x.py --stage 2 test --host=aarch64-apple-darwin --target=aarch64-apple-darwin" - RUST_CONFIGURE_ARGS: "--enable-sanitizers --enable-profiler --set rust.jemalloc" - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - SELECT_XCODE: /Applications/Xcode_14.3.1.app - USE_XCODE_CLANG: 1 - MACOSX_DEPLOYMENT_TARGET: 11.0 - MACOSX_STD_DEPLOYMENT_TARGET: 11.0 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - NO_OVERFLOW_CHECKS: 1 - os: macos-14 - - name: x86_64-msvc - env: - RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-profiler" - SCRIPT: make ci-msvc - os: windows-2019-8core-32gb - - name: i686-msvc - env: - RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-msvc" - SCRIPT: make ci-msvc - os: windows-2019-8core-32gb - - name: x86_64-msvc-ext - env: - SCRIPT: python x.py --stage 2 test src/tools/cargotest src/tools/cargo && src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows - HOST_TARGET: x86_64-pc-windows-msvc - RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-lld --save-toolstates=/tmp/toolstate/toolstates.json" - DEPLOY_TOOLSTATES_JSON: toolstates-windows.json - os: windows-2019-8core-32gb - - name: i686-mingw - env: - RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu" - SCRIPT: make ci-mingw - NO_DOWNLOAD_CI_LLVM: 1 - CUSTOM_MINGW: 1 - os: windows-2019-8core-32gb - - name: x86_64-mingw - env: - SCRIPT: make ci-mingw - RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-profiler" - NO_DOWNLOAD_CI_LLVM: 1 - CUSTOM_MINGW: 1 - os: windows-2019-8core-32gb - - name: dist-x86_64-msvc - env: - RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=x86_64-pc-windows-msvc --target=x86_64-pc-windows-msvc --enable-full-tools --enable-profiler --set rust.codegen-units=1" - SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths - DIST_REQUIRE_ALL_TOOLS: 1 - os: windows-2019-8core-32gb - - name: dist-i686-msvc - env: - RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-msvc --host=i686-pc-windows-msvc --target=i686-pc-windows-msvc,i586-pc-windows-msvc --enable-full-tools --enable-profiler" - SCRIPT: python x.py dist bootstrap --include-default-paths - DIST_REQUIRE_ALL_TOOLS: 1 - os: windows-2019-8core-32gb - - name: dist-aarch64-msvc - env: - RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=aarch64-pc-windows-msvc --enable-full-tools --enable-profiler" - SCRIPT: python x.py dist bootstrap --include-default-paths - DIST_REQUIRE_ALL_TOOLS: 1 - os: windows-2019-8core-32gb - - name: dist-i686-mingw - env: - RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu --enable-full-tools --enable-profiler" - NO_DOWNLOAD_CI_LLVM: 1 - SCRIPT: python x.py dist bootstrap --include-default-paths - CUSTOM_MINGW: 1 - DIST_REQUIRE_ALL_TOOLS: 1 - os: windows-2019-8core-32gb - - name: dist-x86_64-mingw - env: - SCRIPT: python x.py dist bootstrap --include-default-paths - RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler" - NO_DOWNLOAD_CI_LLVM: 1 - CUSTOM_MINGW: 1 - DIST_REQUIRE_ALL_TOOLS: 1 - os: windows-2019-8core-32gb - - name: dist-x86_64-msvc-alt - env: - RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-extended --enable-profiler" - SCRIPT: python x.py dist bootstrap --include-default-paths - os: windows-2019-8core-32gb - defaults: - run: - shell: "${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}" - timeout-minutes: 600 - runs-on: "${{ matrix.os }}" - steps: - - if: "contains(matrix.os, 'windows')" - uses: msys2/setup-msys2@v2.22.0 - with: - msystem: "${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }}" - update: false - release: true - path-type: inherit - install: "make dos2unix diffutils\n" - - name: disable git crlf conversion - run: git config --global core.autocrlf false - - name: checkout the source code - uses: actions/checkout@v4 - with: - fetch-depth: 2 - - name: configure the PR in which the error message will be posted - run: "echo \"[CI_PR_NUMBER=$num]\"" - env: - num: "${{ github.event.number }}" - if: "success() && !env.SKIP_JOB && github.event_name == 'pull_request'" - - name: add extra environment variables - run: src/ci/scripts/setup-environment.sh - env: - EXTRA_VARIABLES: "${{ toJson(matrix.env) }}" - if: success() && !env.SKIP_JOB - - name: decide whether to skip this job - run: src/ci/scripts/should-skip-this.sh - if: success() && !env.SKIP_JOB - - name: ensure the channel matches the target branch - run: src/ci/scripts/verify-channel.sh - if: success() && !env.SKIP_JOB - - name: collect CPU statistics - run: src/ci/scripts/collect-cpu-stats.sh - if: success() && !env.SKIP_JOB - - name: show the current environment - run: src/ci/scripts/dump-environment.sh - if: success() && !env.SKIP_JOB - - name: install awscli - run: src/ci/scripts/install-awscli.sh - if: success() && !env.SKIP_JOB - - name: install sccache - run: src/ci/scripts/install-sccache.sh - if: success() && !env.SKIP_JOB - - name: select Xcode - run: src/ci/scripts/select-xcode.sh - if: success() && !env.SKIP_JOB - - name: install clang - run: src/ci/scripts/install-clang.sh - if: success() && !env.SKIP_JOB - - name: install tidy - run: src/ci/scripts/install-tidy.sh - if: success() && !env.SKIP_JOB - - name: install WIX - run: src/ci/scripts/install-wix.sh - if: success() && !env.SKIP_JOB - - name: disable git crlf conversion - run: src/ci/scripts/disable-git-crlf-conversion.sh - if: success() && !env.SKIP_JOB - - name: checkout submodules - run: src/ci/scripts/checkout-submodules.sh - if: success() && !env.SKIP_JOB - - name: install MSYS2 - run: src/ci/scripts/install-msys2.sh - if: success() && !env.SKIP_JOB - - name: install MinGW - run: src/ci/scripts/install-mingw.sh - if: success() && !env.SKIP_JOB - - name: install ninja - run: src/ci/scripts/install-ninja.sh - if: success() && !env.SKIP_JOB - - name: enable ipv6 on Docker - run: src/ci/scripts/enable-docker-ipv6.sh - if: success() && !env.SKIP_JOB - - name: disable git crlf conversion - run: src/ci/scripts/disable-git-crlf-conversion.sh - if: success() && !env.SKIP_JOB - - name: ensure line endings are correct - run: src/ci/scripts/verify-line-endings.sh - if: success() && !env.SKIP_JOB - - name: ensure backported commits are in upstream branches - run: src/ci/scripts/verify-backported-commits.sh - if: success() && !env.SKIP_JOB - - name: ensure the stable version number is correct - run: src/ci/scripts/verify-stable-version-number.sh - if: success() && !env.SKIP_JOB - - name: run the build - run: src/ci/scripts/run-build-from-ci.sh 2>&1 - env: - AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}" - AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}" - TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}" - if: success() && !env.SKIP_JOB - - name: create github artifacts - run: src/ci/scripts/create-doc-artifacts.sh - if: success() && !env.SKIP_JOB - - name: upload artifacts to github - uses: actions/upload-artifact@v4 - with: - name: "${{ env.DOC_ARTIFACT_NAME }}" - path: obj/artifacts/doc - if-no-files-found: ignore - retention-days: 5 - if: success() && !env.SKIP_JOB - - name: upload artifacts to S3 - run: src/ci/scripts/upload-artifacts.sh - env: - AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}" - AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}" - if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')" master: name: master runs-on: ubuntu-latest @@ -636,7 +226,7 @@ jobs: runs-on: ubuntu-latest auto-success: needs: - - auto + - job if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'" steps: - name: mark the job as a success @@ -646,7 +236,7 @@ jobs: runs-on: ubuntu-latest auto-failure: needs: - - auto + - job if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'" steps: - name: mark the job as a failure diff --git a/src/ci/github-actions/calculate-job-matrix.py b/src/ci/github-actions/calculate-job-matrix.py index 15a835e260cce..935b1bb56c374 100755 --- a/src/ci/github-actions/calculate-job-matrix.py +++ b/src/ci/github-actions/calculate-job-matrix.py @@ -6,8 +6,6 @@ It reads job definitions from `src/ci/github-actions/jobs.yml` and filters them based on the event that happened on CI. - -Currently, it only supports PR and try builds. """ import enum import json @@ -22,14 +20,30 @@ def name_jobs(jobs: List[Dict], prefix: str) -> List[Dict]: + """ + Add a `name` attribute to each job, based on its image and the given `prefix`. + """ for job in jobs: job["name"] = f"{prefix} - {job['image']}" return jobs +def add_base_env(jobs: List[Dict], environment: Dict[str, str]) -> List[Dict]: + """ + Prepends `environment` to the `env` attribute of each job. + The `env` of each job has higher precedence than `environment`. + """ + for job in jobs: + env = environment.copy() + env.update(job.get("env", {})) + job["env"] = env + return jobs + + class JobType(enum.Enum): PR = enum.auto() Try = enum.auto() + Auto = enum.auto() def find_job_type(github_ctx: Dict[str, Any]) -> Optional[JobType]: @@ -53,14 +67,19 @@ def find_job_type(github_ctx: Dict[str, Any]) -> Optional[JobType]: if try_build: return JobType.Try + if ref == "refs/heads/auto" and repository == "rust-lang-ci/rust": + return JobType.Auto + return None def calculate_jobs(job_type: JobType, job_data: Dict[str, Any]) -> List[Dict[str, Any]]: if job_type == JobType.PR: - return name_jobs(job_data["pr"], "PR") + return add_base_env(name_jobs(job_data["pr"], "PR"), job_data["envs"]["pr"]) elif job_type == JobType.Try: - return name_jobs(job_data["try"], "try") + return add_base_env(name_jobs(job_data["try"], "try"), job_data["envs"]["production"]) + elif job_type == JobType.Auto: + return add_base_env(name_jobs(job_data["auto"], "auto"), job_data["envs"]["production"]) return [] @@ -68,11 +87,11 @@ def calculate_jobs(job_type: JobType, job_data: Dict[str, Any]) -> List[Dict[str if __name__ == "__main__": logging.basicConfig(level=logging.INFO) - github_ctx = json.loads(os.environ["GITHUB_CTX"]) - with open(JOBS_YAML_PATH) as f: data = yaml.safe_load(f) + github_ctx = json.loads(os.environ["GITHUB_CTX"]) + job_type = find_job_type(github_ctx) logging.info(f"Job type: {job_type}") diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index e7847a57e9741..2a271b43a8a58 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -342,6 +342,8 @@ concurrency: jobs: # The job matrix for `calculate_matrix` is defined in src/ci/github-actions/jobs.yml. + # It calculates which jobs should be executed, based on the data of the ${{ github }} context. + # If you want to modify CI jobs, take a look at src/ci/github-actions/jobs.yml. calculate_matrix: name: Calculate job matrix runs-on: ubuntu-latest @@ -355,7 +357,6 @@ jobs: GITHUB_CTX: ${{ toJSON(github) }} run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT id: jobs - # This is either a PR or a try job, decided by calculate-job-matrix.py job: <<: *base-ci-job name: ${{ matrix.name }} @@ -375,390 +376,6 @@ jobs: # Check the `calculate_matrix` job to see how is the matrix defined. include: ${{ fromJSON(needs.calculate_matrix.outputs.jobs) }} - auto: - <<: *base-ci-job - name: auto - ${{ matrix.name }} - env: - <<: [*shared-ci-variables, *prod-variables] - if: github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust' - strategy: - matrix: - include: - ############################# - # Linux/Docker builders # - ############################# - - - name: aarch64-gnu - <<: *job-aarch64-linux - - - name: arm-android - <<: *job-linux-8c - - - name: armhf-gnu - <<: *job-linux-8c - - - name: dist-aarch64-linux - env: - CODEGEN_BACKENDS: llvm,cranelift - <<: *job-linux-8c - - - name: dist-android - <<: *job-linux-8c - - - name: dist-arm-linux - <<: *job-linux-16c - - - name: dist-armhf-linux - <<: *job-linux-8c - - - name: dist-armv7-linux - <<: *job-linux-8c - - - name: dist-i586-gnu-i586-i686-musl - <<: *job-linux-8c - - - name: dist-i686-linux - <<: *job-linux-8c - - - name: dist-loongarch64-linux - <<: *job-linux-8c - - - name: dist-ohos - <<: *job-linux-8c - - - name: dist-powerpc-linux - <<: *job-linux-8c - - - name: dist-powerpc64-linux - <<: *job-linux-8c - - - name: dist-powerpc64le-linux - <<: *job-linux-8c - - - name: dist-riscv64-linux - <<: *job-linux-8c - - - name: dist-s390x-linux - <<: *job-linux-8c - - - name: dist-various-1 - <<: *job-linux-8c - - - name: dist-various-2 - <<: *job-linux-8c - - - name: dist-x86_64-freebsd - <<: *job-linux-8c - - - name: dist-x86_64-illumos - <<: *job-linux-8c - - - &dist-x86_64-linux - name: dist-x86_64-linux - env: - CODEGEN_BACKENDS: llvm,cranelift - <<: *job-linux-16c - - - name: dist-x86_64-linux-alt - env: - IMAGE: dist-x86_64-linux - CODEGEN_BACKENDS: llvm,cranelift - <<: *job-linux-16c - - - name: dist-x86_64-musl - env: - CODEGEN_BACKENDS: llvm,cranelift - <<: *job-linux-8c - - - name: dist-x86_64-netbsd - <<: *job-linux-8c - - - name: i686-gnu - <<: *job-linux-8c - - - name: i686-gnu-nopt - <<: *job-linux-8c - - - name: mingw-check - <<: *job-linux-4c - - - name: test-various - <<: *job-linux-8c - - - name: x86_64-gnu - <<: *job-linux-4c - - # This job ensures commits landing on nightly still pass the full - # test suite on the stable channel. There are some UI tests that - # depend on the channel being built (for example if they include the - # channel name on the output), and this builder prevents landing - # changes that would result in broken builds after a promotion. - - name: x86_64-gnu-stable - env: - IMAGE: x86_64-gnu - RUST_CI_OVERRIDE_RELEASE_CHANNEL: stable - # Only run this job on the nightly channel. Running this on beta - # could cause failures when `dev: 1` in `stage0.txt`, and running - # this on stable is useless. - CI_ONLY_WHEN_CHANNEL: nightly - <<: *job-linux-4c - - - name: x86_64-gnu-aux - <<: *job-linux-4c - - - name: x86_64-gnu-integration - env: - # Only run this job on the nightly channel. Fuchsia requires - # nightly features to compile, and this job would fail if - # executed on beta and stable. - CI_ONLY_WHEN_CHANNEL: nightly - <<: *job-linux-8c - - - name: x86_64-gnu-debug - <<: *job-linux-8c - - - name: x86_64-gnu-distcheck - <<: *job-linux-8c - - - name: x86_64-gnu-llvm-18 - env: - RUST_BACKTRACE: 1 - <<: *job-linux-8c - - - name: x86_64-gnu-llvm-17 - env: - RUST_BACKTRACE: 1 - <<: *job-linux-8c - - - name: x86_64-gnu-nopt - <<: *job-linux-4c - - - name: x86_64-gnu-tools - env: - DEPLOY_TOOLSTATES_JSON: toolstates-linux.json - <<: *job-linux-8c - - #################### - # macOS Builders # - #################### - - - name: dist-x86_64-apple - env: - SCRIPT: ./x.py dist bootstrap --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin - RUST_CONFIGURE_ARGS: --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set rust.lto=thin --set rust.codegen-units=1 - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.12 - SELECT_XCODE: /Applications/Xcode_14.3.1.app - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - NO_OVERFLOW_CHECKS: 1 - DIST_REQUIRE_ALL_TOOLS: 1 - CODEGEN_BACKENDS: llvm,cranelift - <<: *job-macos-xl - - - name: dist-apple-various - env: - SCRIPT: ./x.py dist bootstrap --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim - RUST_CONFIGURE_ARGS: --enable-sanitizers --enable-profiler --set rust.jemalloc - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.12 - SELECT_XCODE: /Applications/Xcode_14.3.1.app - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - NO_OVERFLOW_CHECKS: 1 - <<: *job-macos-xl - - - name: x86_64-apple-1 - env: &env-x86_64-apple-tests - SCRIPT: ./x.py --stage 2 test --skip tests/ui --skip tests/rustdoc --skip tests/run-make-fulldeps - RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.12 - MACOSX_STD_DEPLOYMENT_TARGET: 10.12 - SELECT_XCODE: /Applications/Xcode_14.3.1.app - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - NO_OVERFLOW_CHECKS: 1 - <<: *job-macos-xl - - - name: x86_64-apple-2 - env: - SCRIPT: ./x.py --stage 2 test tests/ui tests/rustdoc tests/run-make-fulldeps - <<: *env-x86_64-apple-tests - <<: *job-macos-xl - - # This target only needs to support 11.0 and up as nothing else supports the hardware - - name: dist-aarch64-apple - env: - SCRIPT: ./x.py dist bootstrap --include-default-paths --host=aarch64-apple-darwin --target=aarch64-apple-darwin - RUST_CONFIGURE_ARGS: >- - --enable-full-tools - --enable-sanitizers - --enable-profiler - --set rust.jemalloc - --set llvm.ninja=false - --set rust.lto=thin - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - SELECT_XCODE: /Applications/Xcode_14.3.1.app - USE_XCODE_CLANG: 1 - MACOSX_DEPLOYMENT_TARGET: 11.0 - MACOSX_STD_DEPLOYMENT_TARGET: 11.0 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - NO_OVERFLOW_CHECKS: 1 - DIST_REQUIRE_ALL_TOOLS: 1 - <<: *job-macos-m1 - - # This target only needs to support 11.0 and up as nothing else supports the hardware - - name: aarch64-apple - env: - SCRIPT: ./x.py --stage 2 test --host=aarch64-apple-darwin --target=aarch64-apple-darwin - RUST_CONFIGURE_ARGS: >- - --enable-sanitizers - --enable-profiler - --set rust.jemalloc - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - SELECT_XCODE: /Applications/Xcode_14.3.1.app - USE_XCODE_CLANG: 1 - MACOSX_DEPLOYMENT_TARGET: 11.0 - MACOSX_STD_DEPLOYMENT_TARGET: 11.0 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - NO_OVERFLOW_CHECKS: 1 - <<: *job-macos-m1 - - ###################### - # Windows Builders # - ###################### - - - name: x86_64-msvc - env: - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler - SCRIPT: make ci-msvc - <<: *job-windows-8c - - - name: i686-msvc - env: - RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc - SCRIPT: make ci-msvc - <<: *job-windows-8c - - - name: x86_64-msvc-ext - env: - SCRIPT: python x.py --stage 2 test src/tools/cargotest src/tools/cargo && src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows - HOST_TARGET: x86_64-pc-windows-msvc - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-lld --save-toolstates=/tmp/toolstate/toolstates.json - DEPLOY_TOOLSTATES_JSON: toolstates-windows.json - <<: *job-windows-8c - - # 32/64-bit MinGW builds. - # - # We are using MinGW with POSIX threads since LLVM requires - # C++'s std::thread which is disabled in libstdc++ with win32 threads. - # FIXME: Libc++ doesn't have this limitation so we can avoid - # winpthreads if we switch to it. - # - # Instead of relying on the MinGW version installed on CI we download - # and install one ourselves so we won't be surprised by changes to CI's - # build image. - # - # Finally, note that the downloads below are all in the `rust-lang-ci` S3 - # bucket, but they clearly didn't originate there! The downloads originally - # came from the mingw-w64 SourceForge download site. Unfortunately - # SourceForge is notoriously flaky, so we mirror it on our own infrastructure. - - - name: i686-mingw - env: - RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu - SCRIPT: make ci-mingw - # We are intentionally allowing an old toolchain on this builder (and that's - # incompatible with LLVM downloads today). - NO_DOWNLOAD_CI_LLVM: 1 - CUSTOM_MINGW: 1 - <<: *job-windows-8c - - - name: x86_64-mingw - env: - SCRIPT: make ci-mingw - RUST_CONFIGURE_ARGS: >- - --build=x86_64-pc-windows-gnu - --enable-profiler - # We are intentionally allowing an old toolchain on this builder (and that's - # incompatible with LLVM downloads today). - NO_DOWNLOAD_CI_LLVM: 1 - CUSTOM_MINGW: 1 - <<: *job-windows-8c - - - name: dist-x86_64-msvc - env: - RUST_CONFIGURE_ARGS: >- - --build=x86_64-pc-windows-msvc - --host=x86_64-pc-windows-msvc - --target=x86_64-pc-windows-msvc - --enable-full-tools - --enable-profiler - --set rust.codegen-units=1 - SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths - DIST_REQUIRE_ALL_TOOLS: 1 - <<: *job-windows-8c - - - name: dist-i686-msvc - env: - RUST_CONFIGURE_ARGS: >- - --build=i686-pc-windows-msvc - --host=i686-pc-windows-msvc - --target=i686-pc-windows-msvc,i586-pc-windows-msvc - --enable-full-tools - --enable-profiler - SCRIPT: python x.py dist bootstrap --include-default-paths - DIST_REQUIRE_ALL_TOOLS: 1 - <<: *job-windows-8c - - - name: dist-aarch64-msvc - env: - RUST_CONFIGURE_ARGS: >- - --build=x86_64-pc-windows-msvc - --host=aarch64-pc-windows-msvc - --enable-full-tools - --enable-profiler - SCRIPT: python x.py dist bootstrap --include-default-paths - DIST_REQUIRE_ALL_TOOLS: 1 - <<: *job-windows-8c - - - name: dist-i686-mingw - env: - RUST_CONFIGURE_ARGS: >- - --build=i686-pc-windows-gnu - --enable-full-tools - --enable-profiler - # We are intentionally allowing an old toolchain on this builder (and that's - # incompatible with LLVM downloads today). - NO_DOWNLOAD_CI_LLVM: 1 - SCRIPT: python x.py dist bootstrap --include-default-paths - CUSTOM_MINGW: 1 - DIST_REQUIRE_ALL_TOOLS: 1 - <<: *job-windows-8c - - - name: dist-x86_64-mingw - env: - SCRIPT: python x.py dist bootstrap --include-default-paths - RUST_CONFIGURE_ARGS: >- - --build=x86_64-pc-windows-gnu - --enable-full-tools - --enable-profiler - # We are intentionally allowing an old toolchain on this builder (and that's - # incompatible with LLVM downloads today). - NO_DOWNLOAD_CI_LLVM: 1 - CUSTOM_MINGW: 1 - DIST_REQUIRE_ALL_TOOLS: 1 - <<: *job-windows-8c - - - name: dist-x86_64-msvc-alt - env: - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler - SCRIPT: python x.py dist bootstrap --include-default-paths - <<: *job-windows-8c - master: name: master runs-on: ubuntu-latest @@ -790,10 +407,10 @@ jobs: if: "!success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'" <<: *base-failure-job auto-success: - needs: [auto] + needs: [ job ] if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'" <<: *base-success-job auto-failure: - needs: [auto] + needs: [ job ] if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'" <<: *base-failure-job diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml index 2aea6ce4fa7d8..cfa0732cf2d63 100644 --- a/src/ci/github-actions/jobs.yml +++ b/src/ci/github-actions/jobs.yml @@ -38,7 +38,7 @@ runners: os: [ self-hosted, ARM64, linux ] envs: - - &prod-variables + production: DEPLOY_BUCKET: rust-lang-ci2 TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/rust-lang/rust/issues TOOLSTATE_PUBLISH: 1 @@ -51,35 +51,409 @@ envs: ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55 AWS_REGION: us-west-1 - - &pr-env + pr: PR_CI_JOB: 1 -# Pull request jobs +# Jobs that run on each push to a pull request (PR) +# These jobs automatically inherit envs.pr, to avoid repeating +# it in each job definition. pr: - image: mingw-check - env: - <<: *pr-env <<: *job-linux-4c - image: mingw-check-tidy continue_on_error: true - env: - <<: *pr-env <<: *job-linux-4c - image: x86_64-gnu-llvm-17 env: ENABLE_GCC_CODEGEN: "1" - <<: *pr-env <<: *job-linux-16c - image: x86_64-gnu-tools - env: - <<: *pr-env <<: *job-linux-16c -# Try build jobs +# Jobs that run when you perform a try build (@bors try) +# These jobs automatically inherit envs.production, to avoid repeating +# it in each job definition. try: - image: dist-x86_64-linux env: DIST_TRY_BUILD: 1 CODEGEN_BACKENDS: llvm,cranelift - <<: *prod-variables <<: *job-linux-16c + +# Main CI jobs that have to be green to merge a commit into master +# These jobs automatically inherit envs.production, to avoid repeating +# it in each job definition. +auto: + ############################# + # Linux/Docker builders # + ############################# + + - name: aarch64-gnu + <<: *job-aarch64-linux + + - name: arm-android + <<: *job-linux-8c + + - name: armhf-gnu + <<: *job-linux-8c + + - name: dist-aarch64-linux + env: + CODEGEN_BACKENDS: llvm,cranelift + <<: *job-linux-8c + + - name: dist-android + <<: *job-linux-8c + + - name: dist-arm-linux + <<: *job-linux-16c + + - name: dist-armhf-linux + <<: *job-linux-8c + + - name: dist-armv7-linux + <<: *job-linux-8c + + - name: dist-i586-gnu-i586-i686-musl + <<: *job-linux-8c + + - name: dist-i686-linux + <<: *job-linux-8c + + - name: dist-loongarch64-linux + <<: *job-linux-8c + + - name: dist-ohos + <<: *job-linux-8c + + - name: dist-powerpc-linux + <<: *job-linux-8c + + - name: dist-powerpc64-linux + <<: *job-linux-8c + + - name: dist-powerpc64le-linux + <<: *job-linux-8c + + - name: dist-riscv64-linux + <<: *job-linux-8c + + - name: dist-s390x-linux + <<: *job-linux-8c + + - name: dist-various-1 + <<: *job-linux-8c + + - name: dist-various-2 + <<: *job-linux-8c + + - name: dist-x86_64-freebsd + <<: *job-linux-8c + + - name: dist-x86_64-illumos + <<: *job-linux-8c + + - name: dist-x86_64-linux + env: + CODEGEN_BACKENDS: llvm,cranelift + <<: *job-linux-16c + + - name: dist-x86_64-linux-alt + env: + IMAGE: dist-x86_64-linux + CODEGEN_BACKENDS: llvm,cranelift + <<: *job-linux-16c + + - name: dist-x86_64-musl + env: + CODEGEN_BACKENDS: llvm,cranelift + <<: *job-linux-8c + + - name: dist-x86_64-netbsd + <<: *job-linux-8c + + - name: i686-gnu + <<: *job-linux-8c + + - name: i686-gnu-nopt + <<: *job-linux-8c + + - name: mingw-check + <<: *job-linux-4c + + - name: test-various + <<: *job-linux-8c + + - name: x86_64-gnu + <<: *job-linux-4c + + # This job ensures commits landing on nightly still pass the full + # test suite on the stable channel. There are some UI tests that + # depend on the channel being built (for example if they include the + # channel name on the output), and this builder prevents landing + # changes that would result in broken builds after a promotion. + - name: x86_64-gnu-stable + env: + IMAGE: x86_64-gnu + RUST_CI_OVERRIDE_RELEASE_CHANNEL: stable + # Only run this job on the nightly channel. Running this on beta + # could cause failures when `dev: 1` in `stage0.txt`, and running + # this on stable is useless. + CI_ONLY_WHEN_CHANNEL: nightly + <<: *job-linux-4c + + - name: x86_64-gnu-aux + <<: *job-linux-4c + + - name: x86_64-gnu-integration + env: + # Only run this job on the nightly channel. Fuchsia requires + # nightly features to compile, and this job would fail if + # executed on beta and stable. + CI_ONLY_WHEN_CHANNEL: nightly + <<: *job-linux-8c + + - name: x86_64-gnu-debug + <<: *job-linux-8c + + - name: x86_64-gnu-distcheck + <<: *job-linux-8c + + - name: x86_64-gnu-llvm-18 + env: + RUST_BACKTRACE: 1 + <<: *job-linux-8c + + - name: x86_64-gnu-llvm-17 + env: + RUST_BACKTRACE: 1 + <<: *job-linux-8c + + - name: x86_64-gnu-nopt + <<: *job-linux-4c + + - name: x86_64-gnu-tools + env: + DEPLOY_TOOLSTATES_JSON: toolstates-linux.json + <<: *job-linux-8c + + #################### + # macOS Builders # + #################### + + - name: dist-x86_64-apple + env: + SCRIPT: ./x.py dist bootstrap --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin + RUST_CONFIGURE_ARGS: --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set rust.lto=thin --set rust.codegen-units=1 + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.12 + SELECT_XCODE: /Applications/Xcode_14.3.1.app + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + NO_OVERFLOW_CHECKS: 1 + DIST_REQUIRE_ALL_TOOLS: 1 + CODEGEN_BACKENDS: llvm,cranelift + <<: *job-macos-xl + + - name: dist-apple-various + env: + SCRIPT: ./x.py dist bootstrap --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim + RUST_CONFIGURE_ARGS: --enable-sanitizers --enable-profiler --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.12 + SELECT_XCODE: /Applications/Xcode_14.3.1.app + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + NO_OVERFLOW_CHECKS: 1 + <<: *job-macos-xl + + - name: x86_64-apple-1 + env: &env-x86_64-apple-tests + SCRIPT: ./x.py --stage 2 test --skip tests/ui --skip tests/rustdoc --skip tests/run-make-fulldeps + RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.12 + MACOSX_STD_DEPLOYMENT_TARGET: 10.12 + SELECT_XCODE: /Applications/Xcode_14.3.1.app + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + NO_OVERFLOW_CHECKS: 1 + <<: *job-macos-xl + + - name: x86_64-apple-2 + env: + SCRIPT: ./x.py --stage 2 test tests/ui tests/rustdoc tests/run-make-fulldeps + <<: *env-x86_64-apple-tests + <<: *job-macos-xl + + # This target only needs to support 11.0 and up as nothing else supports the hardware + - name: dist-aarch64-apple + env: + SCRIPT: ./x.py dist bootstrap --include-default-paths --host=aarch64-apple-darwin --target=aarch64-apple-darwin + RUST_CONFIGURE_ARGS: >- + --enable-full-tools + --enable-sanitizers + --enable-profiler + --set rust.jemalloc + --set llvm.ninja=false + --set rust.lto=thin + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + SELECT_XCODE: /Applications/Xcode_14.3.1.app + USE_XCODE_CLANG: 1 + MACOSX_DEPLOYMENT_TARGET: 11.0 + MACOSX_STD_DEPLOYMENT_TARGET: 11.0 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + NO_OVERFLOW_CHECKS: 1 + DIST_REQUIRE_ALL_TOOLS: 1 + <<: *job-macos-m1 + + # This target only needs to support 11.0 and up as nothing else supports the hardware + - name: aarch64-apple + env: + SCRIPT: ./x.py --stage 2 test --host=aarch64-apple-darwin --target=aarch64-apple-darwin + RUST_CONFIGURE_ARGS: >- + --enable-sanitizers + --enable-profiler + --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + SELECT_XCODE: /Applications/Xcode_14.3.1.app + USE_XCODE_CLANG: 1 + MACOSX_DEPLOYMENT_TARGET: 11.0 + MACOSX_STD_DEPLOYMENT_TARGET: 11.0 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + NO_OVERFLOW_CHECKS: 1 + <<: *job-macos-m1 + + ###################### + # Windows Builders # + ###################### + + - name: x86_64-msvc + env: + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler + SCRIPT: make ci-msvc + <<: *job-windows-8c + + - name: i686-msvc + env: + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc + SCRIPT: make ci-msvc + <<: *job-windows-8c + + - name: x86_64-msvc-ext + env: + SCRIPT: python x.py --stage 2 test src/tools/cargotest src/tools/cargo && src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows + HOST_TARGET: x86_64-pc-windows-msvc + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-lld --save-toolstates=/tmp/toolstate/toolstates.json + DEPLOY_TOOLSTATES_JSON: toolstates-windows.json + <<: *job-windows-8c + + # 32/64-bit MinGW builds. + # + # We are using MinGW with POSIX threads since LLVM requires + # C++'s std::thread which is disabled in libstdc++ with win32 threads. + # FIXME: Libc++ doesn't have this limitation so we can avoid + # winpthreads if we switch to it. + # + # Instead of relying on the MinGW version installed on CI we download + # and install one ourselves so we won't be surprised by changes to CI's + # build image. + # + # Finally, note that the downloads below are all in the `rust-lang-ci` S3 + # bucket, but they clearly didn't originate there! The downloads originally + # came from the mingw-w64 SourceForge download site. Unfortunately + # SourceForge is notoriously flaky, so we mirror it on our own infrastructure. + + - name: i686-mingw + env: + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu + SCRIPT: make ci-mingw + # We are intentionally allowing an old toolchain on this builder (and that's + # incompatible with LLVM downloads today). + NO_DOWNLOAD_CI_LLVM: 1 + CUSTOM_MINGW: 1 + <<: *job-windows-8c + + - name: x86_64-mingw + env: + SCRIPT: make ci-mingw + RUST_CONFIGURE_ARGS: >- + --build=x86_64-pc-windows-gnu + --enable-profiler + # We are intentionally allowing an old toolchain on this builder (and that's + # incompatible with LLVM downloads today). + NO_DOWNLOAD_CI_LLVM: 1 + CUSTOM_MINGW: 1 + <<: *job-windows-8c + + - name: dist-x86_64-msvc + env: + RUST_CONFIGURE_ARGS: >- + --build=x86_64-pc-windows-msvc + --host=x86_64-pc-windows-msvc + --target=x86_64-pc-windows-msvc + --enable-full-tools + --enable-profiler + --set rust.codegen-units=1 + SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths + DIST_REQUIRE_ALL_TOOLS: 1 + <<: *job-windows-8c + + - name: dist-i686-msvc + env: + RUST_CONFIGURE_ARGS: >- + --build=i686-pc-windows-msvc + --host=i686-pc-windows-msvc + --target=i686-pc-windows-msvc,i586-pc-windows-msvc + --enable-full-tools + --enable-profiler + SCRIPT: python x.py dist bootstrap --include-default-paths + DIST_REQUIRE_ALL_TOOLS: 1 + <<: *job-windows-8c + + - name: dist-aarch64-msvc + env: + RUST_CONFIGURE_ARGS: >- + --build=x86_64-pc-windows-msvc + --host=aarch64-pc-windows-msvc + --enable-full-tools + --enable-profiler + SCRIPT: python x.py dist bootstrap --include-default-paths + DIST_REQUIRE_ALL_TOOLS: 1 + <<: *job-windows-8c + + - name: dist-i686-mingw + env: + RUST_CONFIGURE_ARGS: >- + --build=i686-pc-windows-gnu + --enable-full-tools + --enable-profiler + # We are intentionally allowing an old toolchain on this builder (and that's + # incompatible with LLVM downloads today). + NO_DOWNLOAD_CI_LLVM: 1 + SCRIPT: python x.py dist bootstrap --include-default-paths + CUSTOM_MINGW: 1 + DIST_REQUIRE_ALL_TOOLS: 1 + <<: *job-windows-8c + + - name: dist-x86_64-mingw + env: + SCRIPT: python x.py dist bootstrap --include-default-paths + RUST_CONFIGURE_ARGS: >- + --build=x86_64-pc-windows-gnu + --enable-full-tools + --enable-profiler + # We are intentionally allowing an old toolchain on this builder (and that's + # incompatible with LLVM downloads today). + NO_DOWNLOAD_CI_LLVM: 1 + CUSTOM_MINGW: 1 + DIST_REQUIRE_ALL_TOOLS: 1 + <<: *job-windows-8c + + - name: dist-x86_64-msvc-alt + env: + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler + SCRIPT: python x.py dist bootstrap --include-default-paths + <<: *job-windows-8c From c251abc7ec1425bc36941d22af4600ea5c8ea505 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 23 Apr 2024 08:52:28 +0200 Subject: [PATCH 5/8] Load GitHub context from environment variables --- .github/workflows/ci.yml | 2 -- src/ci/github-actions/calculate-job-matrix.py | 36 ++++++++++++------- src/ci/github-actions/ci.yml | 2 -- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 010872c980f95..670a2854fed41 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,8 +45,6 @@ jobs: - name: Checkout the source code uses: actions/checkout@v4 - name: Calculate the CI job matrix - env: - GITHUB_CTX: "${{ toJSON(github) }}" run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT id: jobs job: diff --git a/src/ci/github-actions/calculate-job-matrix.py b/src/ci/github-actions/calculate-job-matrix.py index 935b1bb56c374..a844b201c16d9 100755 --- a/src/ci/github-actions/calculate-job-matrix.py +++ b/src/ci/github-actions/calculate-job-matrix.py @@ -7,6 +7,7 @@ It reads job definitions from `src/ci/github-actions/jobs.yml` and filters them based on the event that happened on CI. """ +import dataclasses import enum import json import logging @@ -46,28 +47,31 @@ class JobType(enum.Enum): Auto = enum.auto() -def find_job_type(github_ctx: Dict[str, Any]) -> Optional[JobType]: - event_name = github_ctx["event_name"] - ref = github_ctx["ref"] - repository = github_ctx["repository"] +@dataclasses.dataclass +class GitHubCtx: + event_name: str + ref: str + repository: str - if event_name == "pull_request": + +def find_job_type(ctx: GitHubCtx) -> Optional[JobType]: + if ctx.event_name == "pull_request": return JobType.PR - elif event_name == "push": + elif ctx.event_name == "push": old_bors_try_build = ( - ref in ("refs/heads/try", "refs/heads/try-perf") and - repository == "rust-lang-ci/rust" + ctx.ref in ("refs/heads/try", "refs/heads/try-perf") and + ctx.repository == "rust-lang-ci/rust" ) new_bors_try_build = ( - ref == "refs/heads/automation/bors/try" and - repository == "rust-lang/rust" + ctx.ref == "refs/heads/automation/bors/try" and + ctx.repository == "rust-lang/rust" ) try_build = old_bors_try_build or new_bors_try_build if try_build: return JobType.Try - if ref == "refs/heads/auto" and repository == "rust-lang-ci/rust": + if ctx.ref == "refs/heads/auto" and ctx.repository == "rust-lang-ci/rust": return JobType.Auto return None @@ -84,13 +88,21 @@ def calculate_jobs(job_type: JobType, job_data: Dict[str, Any]) -> List[Dict[str return [] +def get_github_ctx() -> GitHubCtx: + return GitHubCtx( + event_name=os.environ["GITHUB_EVENT_NAME"], + ref=os.environ["GITHUB_REF"], + repository=os.environ["GITHUB_REPOSITORY"] + ) + + if __name__ == "__main__": logging.basicConfig(level=logging.INFO) with open(JOBS_YAML_PATH) as f: data = yaml.safe_load(f) - github_ctx = json.loads(os.environ["GITHUB_CTX"]) + github_ctx = get_github_ctx() job_type = find_job_type(github_ctx) logging.info(f"Job type: {job_type}") diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 2a271b43a8a58..1efdd7e22e90a 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -353,8 +353,6 @@ jobs: - name: Checkout the source code uses: actions/checkout@v4 - name: Calculate the CI job matrix - env: - GITHUB_CTX: ${{ toJSON(github) }} run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT id: jobs job: From 2f2d59dd06dfe88878e7873cb87d4bc39fe6b188 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 23 Apr 2024 08:53:07 +0200 Subject: [PATCH 6/8] Fix image field in auto jobs --- src/ci/github-actions/jobs.yml | 112 ++++++++++++++++----------------- 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml index cfa0732cf2d63..a287a4383e37c 100644 --- a/src/ci/github-actions/jobs.yml +++ b/src/ci/github-actions/jobs.yml @@ -88,103 +88,103 @@ auto: # Linux/Docker builders # ############################# - - name: aarch64-gnu + - image: aarch64-gnu <<: *job-aarch64-linux - - name: arm-android + - image: arm-android <<: *job-linux-8c - - name: armhf-gnu + - image: armhf-gnu <<: *job-linux-8c - - name: dist-aarch64-linux + - image: dist-aarch64-linux env: CODEGEN_BACKENDS: llvm,cranelift <<: *job-linux-8c - - name: dist-android + - image: dist-android <<: *job-linux-8c - - name: dist-arm-linux + - image: dist-arm-linux <<: *job-linux-16c - - name: dist-armhf-linux + - image: dist-armhf-linux <<: *job-linux-8c - - name: dist-armv7-linux + - image: dist-armv7-linux <<: *job-linux-8c - - name: dist-i586-gnu-i586-i686-musl + - image: dist-i586-gnu-i586-i686-musl <<: *job-linux-8c - - name: dist-i686-linux + - image: dist-i686-linux <<: *job-linux-8c - - name: dist-loongarch64-linux + - image: dist-loongarch64-linux <<: *job-linux-8c - - name: dist-ohos + - image: dist-ohos <<: *job-linux-8c - - name: dist-powerpc-linux + - image: dist-powerpc-linux <<: *job-linux-8c - - name: dist-powerpc64-linux + - image: dist-powerpc64-linux <<: *job-linux-8c - - name: dist-powerpc64le-linux + - image: dist-powerpc64le-linux <<: *job-linux-8c - - name: dist-riscv64-linux + - image: dist-riscv64-linux <<: *job-linux-8c - - name: dist-s390x-linux + - image: dist-s390x-linux <<: *job-linux-8c - - name: dist-various-1 + - image: dist-various-1 <<: *job-linux-8c - - name: dist-various-2 + - image: dist-various-2 <<: *job-linux-8c - - name: dist-x86_64-freebsd + - image: dist-x86_64-freebsd <<: *job-linux-8c - - name: dist-x86_64-illumos + - image: dist-x86_64-illumos <<: *job-linux-8c - - name: dist-x86_64-linux + - image: dist-x86_64-linux env: CODEGEN_BACKENDS: llvm,cranelift <<: *job-linux-16c - - name: dist-x86_64-linux-alt + - image: dist-x86_64-linux-alt env: IMAGE: dist-x86_64-linux CODEGEN_BACKENDS: llvm,cranelift <<: *job-linux-16c - - name: dist-x86_64-musl + - image: dist-x86_64-musl env: CODEGEN_BACKENDS: llvm,cranelift <<: *job-linux-8c - - name: dist-x86_64-netbsd + - image: dist-x86_64-netbsd <<: *job-linux-8c - - name: i686-gnu + - image: i686-gnu <<: *job-linux-8c - - name: i686-gnu-nopt + - image: i686-gnu-nopt <<: *job-linux-8c - - name: mingw-check + - image: mingw-check <<: *job-linux-4c - - name: test-various + - image: test-various <<: *job-linux-8c - - name: x86_64-gnu + - image: x86_64-gnu <<: *job-linux-4c # This job ensures commits landing on nightly still pass the full @@ -192,7 +192,7 @@ auto: # depend on the channel being built (for example if they include the # channel name on the output), and this builder prevents landing # changes that would result in broken builds after a promotion. - - name: x86_64-gnu-stable + - image: x86_64-gnu-stable env: IMAGE: x86_64-gnu RUST_CI_OVERRIDE_RELEASE_CHANNEL: stable @@ -202,10 +202,10 @@ auto: CI_ONLY_WHEN_CHANNEL: nightly <<: *job-linux-4c - - name: x86_64-gnu-aux + - image: x86_64-gnu-aux <<: *job-linux-4c - - name: x86_64-gnu-integration + - image: x86_64-gnu-integration env: # Only run this job on the nightly channel. Fuchsia requires # nightly features to compile, and this job would fail if @@ -213,26 +213,26 @@ auto: CI_ONLY_WHEN_CHANNEL: nightly <<: *job-linux-8c - - name: x86_64-gnu-debug + - image: x86_64-gnu-debug <<: *job-linux-8c - - name: x86_64-gnu-distcheck + - image: x86_64-gnu-distcheck <<: *job-linux-8c - - name: x86_64-gnu-llvm-18 + - image: x86_64-gnu-llvm-18 env: RUST_BACKTRACE: 1 <<: *job-linux-8c - - name: x86_64-gnu-llvm-17 + - image: x86_64-gnu-llvm-17 env: RUST_BACKTRACE: 1 <<: *job-linux-8c - - name: x86_64-gnu-nopt + - image: x86_64-gnu-nopt <<: *job-linux-4c - - name: x86_64-gnu-tools + - image: x86_64-gnu-tools env: DEPLOY_TOOLSTATES_JSON: toolstates-linux.json <<: *job-linux-8c @@ -241,7 +241,7 @@ auto: # macOS Builders # #################### - - name: dist-x86_64-apple + - image: dist-x86_64-apple env: SCRIPT: ./x.py dist bootstrap --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin RUST_CONFIGURE_ARGS: --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set rust.lto=thin --set rust.codegen-units=1 @@ -255,7 +255,7 @@ auto: CODEGEN_BACKENDS: llvm,cranelift <<: *job-macos-xl - - name: dist-apple-various + - image: dist-apple-various env: SCRIPT: ./x.py dist bootstrap --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim RUST_CONFIGURE_ARGS: --enable-sanitizers --enable-profiler --set rust.jemalloc @@ -267,7 +267,7 @@ auto: NO_OVERFLOW_CHECKS: 1 <<: *job-macos-xl - - name: x86_64-apple-1 + - image: x86_64-apple-1 env: &env-x86_64-apple-tests SCRIPT: ./x.py --stage 2 test --skip tests/ui --skip tests/rustdoc --skip tests/run-make-fulldeps RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc @@ -280,14 +280,14 @@ auto: NO_OVERFLOW_CHECKS: 1 <<: *job-macos-xl - - name: x86_64-apple-2 + - image: x86_64-apple-2 env: SCRIPT: ./x.py --stage 2 test tests/ui tests/rustdoc tests/run-make-fulldeps <<: *env-x86_64-apple-tests <<: *job-macos-xl # This target only needs to support 11.0 and up as nothing else supports the hardware - - name: dist-aarch64-apple + - image: dist-aarch64-apple env: SCRIPT: ./x.py dist bootstrap --include-default-paths --host=aarch64-apple-darwin --target=aarch64-apple-darwin RUST_CONFIGURE_ARGS: >- @@ -309,7 +309,7 @@ auto: <<: *job-macos-m1 # This target only needs to support 11.0 and up as nothing else supports the hardware - - name: aarch64-apple + - image: aarch64-apple env: SCRIPT: ./x.py --stage 2 test --host=aarch64-apple-darwin --target=aarch64-apple-darwin RUST_CONFIGURE_ARGS: >- @@ -330,19 +330,19 @@ auto: # Windows Builders # ###################### - - name: x86_64-msvc + - image: x86_64-msvc env: RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler SCRIPT: make ci-msvc <<: *job-windows-8c - - name: i686-msvc + - image: i686-msvc env: RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc SCRIPT: make ci-msvc <<: *job-windows-8c - - name: x86_64-msvc-ext + - image: x86_64-msvc-ext env: SCRIPT: python x.py --stage 2 test src/tools/cargotest src/tools/cargo && src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows HOST_TARGET: x86_64-pc-windows-msvc @@ -366,7 +366,7 @@ auto: # came from the mingw-w64 SourceForge download site. Unfortunately # SourceForge is notoriously flaky, so we mirror it on our own infrastructure. - - name: i686-mingw + - image: i686-mingw env: RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu SCRIPT: make ci-mingw @@ -376,7 +376,7 @@ auto: CUSTOM_MINGW: 1 <<: *job-windows-8c - - name: x86_64-mingw + - image: x86_64-mingw env: SCRIPT: make ci-mingw RUST_CONFIGURE_ARGS: >- @@ -388,7 +388,7 @@ auto: CUSTOM_MINGW: 1 <<: *job-windows-8c - - name: dist-x86_64-msvc + - image: dist-x86_64-msvc env: RUST_CONFIGURE_ARGS: >- --build=x86_64-pc-windows-msvc @@ -401,7 +401,7 @@ auto: DIST_REQUIRE_ALL_TOOLS: 1 <<: *job-windows-8c - - name: dist-i686-msvc + - image: dist-i686-msvc env: RUST_CONFIGURE_ARGS: >- --build=i686-pc-windows-msvc @@ -413,7 +413,7 @@ auto: DIST_REQUIRE_ALL_TOOLS: 1 <<: *job-windows-8c - - name: dist-aarch64-msvc + - image: dist-aarch64-msvc env: RUST_CONFIGURE_ARGS: >- --build=x86_64-pc-windows-msvc @@ -424,7 +424,7 @@ auto: DIST_REQUIRE_ALL_TOOLS: 1 <<: *job-windows-8c - - name: dist-i686-mingw + - image: dist-i686-mingw env: RUST_CONFIGURE_ARGS: >- --build=i686-pc-windows-gnu @@ -438,7 +438,7 @@ auto: DIST_REQUIRE_ALL_TOOLS: 1 <<: *job-windows-8c - - name: dist-x86_64-mingw + - image: dist-x86_64-mingw env: SCRIPT: python x.py dist bootstrap --include-default-paths RUST_CONFIGURE_ARGS: >- @@ -452,7 +452,7 @@ auto: DIST_REQUIRE_ALL_TOOLS: 1 <<: *job-windows-8c - - name: dist-x86_64-msvc-alt + - image: dist-x86_64-msvc-alt env: RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler SCRIPT: python x.py dist bootstrap --include-default-paths From 162892b030b95aa8ad422212a82457d940b15a4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 23 Apr 2024 08:55:35 +0200 Subject: [PATCH 7/8] Make separate envs for try and auto builds --- src/ci/github-actions/calculate-job-matrix.py | 4 ++-- src/ci/github-actions/jobs.yml | 9 ++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/ci/github-actions/calculate-job-matrix.py b/src/ci/github-actions/calculate-job-matrix.py index a844b201c16d9..c24cefa8d8962 100755 --- a/src/ci/github-actions/calculate-job-matrix.py +++ b/src/ci/github-actions/calculate-job-matrix.py @@ -81,9 +81,9 @@ def calculate_jobs(job_type: JobType, job_data: Dict[str, Any]) -> List[Dict[str if job_type == JobType.PR: return add_base_env(name_jobs(job_data["pr"], "PR"), job_data["envs"]["pr"]) elif job_type == JobType.Try: - return add_base_env(name_jobs(job_data["try"], "try"), job_data["envs"]["production"]) + return add_base_env(name_jobs(job_data["try"], "try"), job_data["envs"]["try"]) elif job_type == JobType.Auto: - return add_base_env(name_jobs(job_data["auto"], "auto"), job_data["envs"]["production"]) + return add_base_env(name_jobs(job_data["auto"], "auto"), job_data["envs"]["auto"]) return [] diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml index a287a4383e37c..ec58bd0924e86 100644 --- a/src/ci/github-actions/jobs.yml +++ b/src/ci/github-actions/jobs.yml @@ -39,6 +39,7 @@ runners: envs: production: + &production DEPLOY_BUCKET: rust-lang-ci2 TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/rust-lang/rust/issues TOOLSTATE_PUBLISH: 1 @@ -51,6 +52,13 @@ envs: ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55 AWS_REGION: us-west-1 + try: + <<: *production + DIST_TRY_BUILD: 1 + + auto: + <<: *production + pr: PR_CI_JOB: 1 @@ -76,7 +84,6 @@ pr: try: - image: dist-x86_64-linux env: - DIST_TRY_BUILD: 1 CODEGEN_BACKENDS: llvm,cranelift <<: *job-linux-16c From 2632c6139d9b1a7ca7be4da1924e7a78edfa452c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 23 Apr 2024 08:59:45 +0200 Subject: [PATCH 8/8] Fix empty job matrix --- .github/workflows/ci.yml | 1 + src/ci/github-actions/ci.yml | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 670a2854fed41..872b671b031e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,6 +63,7 @@ jobs: strategy: matrix: include: "${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}" + if: "fromJSON(needs.calculate_matrix.outputs.jobs)[0] != null" defaults: run: shell: "${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}" diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 1efdd7e22e90a..19d6b517552f4 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -373,6 +373,14 @@ jobs: matrix: # Check the `calculate_matrix` job to see how is the matrix defined. include: ${{ fromJSON(needs.calculate_matrix.outputs.jobs) }} + # GitHub Actions fails the workflow if an empty list of jobs is provided to + # the workflow, so we need to skip this job if nothing was produced by + # the Python script. + # + # Unfortunately checking whether a list is empty is not possible in a nice + # way due to GitHub Actions expressions limits. + # This hack is taken from https://github.com/ferrocene/ferrocene/blob/d43edc6b7697cf1719ec1c17c54904ab94825763/.github/workflows/release.yml#L75-L82 + if: fromJSON(needs.calculate_matrix.outputs.jobs)[0] != null master: name: master