ci.yml

######################################################
#   WARNING! Action needed when changing this file   #
######################################################

# Due to GitHub Actions limitations, we can't use YAML Anchors directly in the
# CI configuration stored on the repository. To work around that this file is
# expanded by a tool in the repository, and the expansion is committed as well.
#
# After you make any change to the file you'll need to run this command:
#
#   ./x.py run src/tools/expand-yaml-anchors
#
# ...and commit the file it updated in addition to this one. If you forget this
# step CI will fail.

---

###############################
#   YAML Anchors Definition   #
###############################

# This key contains most of the YAML anchors that will be used later in the
# document. YAML anchors allows us to greatly reduce duplication inside the CI
# configuration by reusing parts of the configuration.
#
# YAML anchors work by defining an anchor with `&anchor-name` and reusing its
# content in another place with `*anchor-name`. The special `<<` map key merges
# the content of the map with the content of the anchor (or list of anchors).
#
# The expand-yaml-anchors tool will automatically remove this block from the
# output YAML file.
x--expand-yaml-anchors--remove:

  - &shared-ci-variables
    CI_JOB_NAME: ${{ matrix.name }}

  - &public-variables
    SCCACHE_BUCKET: rust-lang-ci-sccache2
    TOOLSTATE_REPO: https://github.com/rust-lang-nursery/rust-toolstate
    CACHE_DOMAIN: ci-caches.rust-lang.org

  - &prod-variables
    SCCACHE_BUCKET: rust-lang-ci-sccache2
    DEPLOY_BUCKET: rust-lang-ci2
    TOOLSTATE_REPO: https://github.com/rust-lang-nursery/rust-toolstate
    TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/rust-lang/rust/issues
    TOOLSTATE_PUBLISH: 1
    # AWS_SECRET_ACCESS_KEYs are stored in GitHub's secrets storage, named
    # AWS_SECRET_ACCESS_KEY_<keyid>. Including the key id in the name allows to
    # rotate them in a single branch while keeping the old key in another
    # branch, which wouldn't be possible if the key was named with the kind
    # (caches, artifacts...).
    CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL
    ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55
    CACHE_DOMAIN: ci-caches.rust-lang.org

  - &dummy-variables
    SCCACHE_BUCKET: rust-lang-gha-caches
    DEPLOY_BUCKET: rust-lang-gha
    TOOLSTATE_REPO: https://github.com/pietroalbini/rust-toolstate
    TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/pietroalbini/rust-toolstate/issues
    TOOLSTATE_PUBLISH: 1
    # AWS_SECRET_ACCESS_KEYs are stored in GitHub's secrets storage, named
    # AWS_SECRET_ACCESS_KEY_<keyid>. Including the key id in the name allows to
    # rotate them in a single branch while keeping the old key in another
    # branch, which wouldn't be possible if the key was named with the kind
    # (caches, artifacts...).
    CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZOMUQATD5
    ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZH5AYXDVF
    CACHE_DOMAIN: ci-caches-gha.rust-lang.org

  - &base-job
    env: {}

  - &job-linux-xl
    os: ubuntu-latest-xl
    <<: *base-job

  - &job-macos-xl
    os: macos-latest  # We don't have an XL builder for this
    <<: *base-job

  - &job-windows-xl
    os: windows-latest-xl
    <<: *base-job

  - &job-aarch64-linux
    os: [self-hosted, ARM64, linux]

  - &step
    if: success() && !env.SKIP_JOB

  - &base-ci-job
    timeout-minutes: 600
    runs-on: "${{ matrix.os }}"
    env: *shared-ci-variables
    steps:
      - name: disable git crlf conversion
        run: git config --global core.autocrlf false

      - name: checkout the source code
        uses: actions/checkout@v1
        with:
          fetch-depth: 2

      # Rust Log Analyzer can't currently detect the PR number of a GitHub
      # Actions build on its own, so a hint in the log message is needed to
      # point it in the right direction.
      - name: configure the PR in which the error message will be posted
        run: echo "[CI_PR_NUMBER=$num]"
        env:
          num: ${{ github.event.number }}
        if: success() && !env.SKIP_JOBS && github.event_name == 'pull_request'

      - name: add extra environment variables
        run: src/ci/scripts/setup-environment.sh
        env:
          # Since it's not possible to merge `${{ matrix.env }}` with the other
          # variables in `job.<name>.env`, the variables defined in the matrix
          # are passed to the `setup-environment.sh` script encoded in JSON,
          # which then uses log commands to actually set them.
          EXTRA_VARIABLES: ${{ toJson(matrix.env) }}
        <<: *step

      - name: decide whether to skip this job
        run: src/ci/scripts/should-skip-this.sh
        <<: *step

      - name: configure GitHub Actions to kill the build when outdated
        uses: rust-lang/simpleinfra/github-actions/cancel-outdated-builds@master
        with:
          github_token: "${{ secrets.github_token }}"
        if: success() && !env.SKIP_JOB && github.ref != 'refs/heads/try'
        <<: *step

      - name: collect CPU statistics
        run: src/ci/scripts/collect-cpu-stats.sh
        <<: *step

      - name: show the current environment
        run: src/ci/scripts/dump-environment.sh
        <<: *step

      - name: install awscli
        run: src/ci/scripts/install-awscli.sh
        <<: *step

      - name: install sccache
        run: src/ci/scripts/install-sccache.sh
        <<: *step

      - name: install clang
        run: src/ci/scripts/install-clang.sh
        <<: *step

      - name: install WIX
        run: src/ci/scripts/install-wix.sh
        <<: *step

      - name: ensure the build happens on a partition with enough space
        run: src/ci/scripts/symlink-build-dir.sh
        <<: *step

      - name: disable git crlf conversion
        run: src/ci/scripts/disable-git-crlf-conversion.sh
        <<: *step

      - name: install MSYS2
        run: src/ci/scripts/install-msys2.sh
        <<: *step

      - name: install MinGW
        run: src/ci/scripts/install-mingw.sh
        <<: *step

      - name: install ninja
        run: src/ci/scripts/install-ninja.sh
        <<: *step

      - name: enable ipv6 on Docker
        run: src/ci/scripts/enable-docker-ipv6.sh
        <<: *step

      # Disable automatic line ending conversion (again). On Windows, when we're
      # installing dependencies, something switches the git configuration directory or
      # re-enables autocrlf. We've not tracked down the exact cause -- and there may
      # be multiple -- but this should ensure submodules are checked out with the
      # appropriate line endings.
      - name: disable git crlf conversion
        run: src/ci/scripts/disable-git-crlf-conversion.sh
        <<: *step

      - name: checkout submodules
        run: src/ci/scripts/checkout-submodules.sh
        <<: *step

      - name: ensure line endings are correct
        run: src/ci/scripts/verify-line-endings.sh
        <<: *step

      - name: run the build
        run: src/ci/scripts/run-build-from-ci.sh
        env:
          AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}
          TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
        <<: *step

      - name: upload artifacts to S3
        run: src/ci/scripts/upload-artifacts.sh
        env:
          AWS_ACCESS_KEY_ID: ${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}
        # Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy
        # builders *should* have the AWS credentials available. Still, explicitly
        # adding the condition is helpful as this way CI will not silently skip
        # deploying artifacts from a dist builder if the variables are misconfigured,
        # erroring about invalid credentials instead.
        if: success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')
        <<: *step

  # These snippets are used by the try-success, try-failure, auto-success and auto-failure jobs.
  # Check out their documentation for more information on why they're needed.

  - &base-outcome-job
    name: bors build finished
    runs-on: ubuntu-latest

  - &base-success-job
    steps:
      - name: mark the job as a success
        run: exit 0
        shell: bash
    <<: *base-outcome-job

  - &base-failure-job
    steps:
      - name: mark the job as a failure
        run: exit 1
        shell: bash
    <<: *base-outcome-job

###########################
#   Builders definition   #
###########################

name: CI
on:
  push:
    branches:
      - auto
      - try
      - try-perf
      - master
  pull_request:
    branches:
      - "**"

defaults:
  run:
    # On Linux, macOS, and Windows, use the system-provided bash as the default
    # shell. (This should only make a difference on Windows, where the default
    # shell is PowerShell.)
    shell: bash

jobs:
  pr:
    <<: *base-ci-job
    name: PR
    env:
      <<: [*shared-ci-variables, *public-variables]
    if: github.event_name == 'pull_request'
    strategy:
      matrix:
        include:
          - name: mingw-check
            <<: *job-linux-xl

          - name: x86_64-gnu-llvm-8
            <<: *job-linux-xl

          - name: x86_64-gnu-tools
            env:
              CI_ONLY_WHEN_SUBMODULES_CHANGED: 1
            <<: *job-linux-xl

  try:
    <<: *base-ci-job
    name: try
    env:
      <<: [*shared-ci-variables, *prod-variables]
    if: github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'
    strategy:
      matrix:
        include:
          - name: dist-x86_64-linux
            <<: *job-linux-xl

  auto:
    <<: *base-ci-job
    name: auto
    env:
      <<: [*shared-ci-variables, *prod-variables]
    if: github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'
    strategy:
      matrix:
        include:
          #############################
          #   Linux/Docker builders   #
          #############################

          - name: arm-android
            <<: *job-linux-xl

          - name: armhf-gnu
            <<: *job-linux-xl

          - name: dist-aarch64-linux
            <<: *job-linux-xl

          - name: dist-android
            <<: *job-linux-xl

          - name: dist-arm-linux
            <<: *job-linux-xl

          - name: dist-armhf-linux
            <<: *job-linux-xl

          - name: dist-armv7-linux
            <<: *job-linux-xl

          - name: dist-i586-gnu-i586-i686-musl
            <<: *job-linux-xl

          - name: dist-i686-freebsd
            <<: *job-linux-xl

          - name: dist-i686-linux
            <<: *job-linux-xl

          - name: dist-mips-linux
            <<: *job-linux-xl

          - name: dist-mips64-linux
            <<: *job-linux-xl

          - name: dist-mips64el-linux
            <<: *job-linux-xl

          - name: dist-mipsel-linux
            <<: *job-linux-xl

          - name: dist-powerpc-linux
            <<: *job-linux-xl

          - name: dist-powerpc64-linux
            <<: *job-linux-xl

          - name: dist-powerpc64le-linux
            <<: *job-linux-xl

          - name: dist-riscv64-linux
            <<: *job-linux-xl

          - name: dist-s390x-linux
            <<: *job-linux-xl

          - name: dist-various-1
            <<: *job-linux-xl

          - name: dist-various-2
            <<: *job-linux-xl

          - name: dist-x86_64-freebsd
            <<: *job-linux-xl

          - name: dist-x86_64-illumos
            <<: *job-linux-xl

          - name: dist-x86_64-linux
            <<: *job-linux-xl

          - name: dist-x86_64-linux-alt
            env:
              IMAGE: dist-x86_64-linux
            <<: *job-linux-xl

          - name: dist-x86_64-musl
            <<: *job-linux-xl

          - name: dist-x86_64-netbsd
            <<: *job-linux-xl

          - name: i686-gnu
            <<: *job-linux-xl

          - name: i686-gnu-nopt
            <<: *job-linux-xl

          - name: mingw-check
            <<: *job-linux-xl

          - name: test-various
            <<: *job-linux-xl

          - name: wasm32
            <<: *job-linux-xl

          - name: x86_64-gnu
            <<: *job-linux-xl

          - name: x86_64-gnu-aux
            <<: *job-linux-xl

          - name: x86_64-gnu-debug
            <<: *job-linux-xl

          - name: x86_64-gnu-distcheck
            <<: *job-linux-xl

          - name: x86_64-gnu-llvm-8
            env:
              RUST_BACKTRACE: 1
            <<: *job-linux-xl

          - name: x86_64-gnu-nopt
            <<: *job-linux-xl

          - name: x86_64-gnu-tools
            env:
              DEPLOY_TOOLSTATES_JSON: toolstates-linux.json
            <<: *job-linux-xl

          ####################
          #  macOS Builders  #
          ####################

          - name: dist-x86_64-apple
            env:
              SCRIPT: ./x.py dist
              RUST_CONFIGURE_ARGS: --host=x86_64-apple-darwin --target=x86_64-apple-darwin,aarch64-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false
              RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
              MACOSX_DEPLOYMENT_TARGET: 10.7
              NO_LLVM_ASSERTIONS: 1
              NO_DEBUG_ASSERTIONS: 1
              DIST_REQUIRE_ALL_TOOLS: 1
            <<: *job-macos-xl

          - name: dist-x86_64-apple-alt
            env:
              SCRIPT: ./x.py dist
              RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --set rust.jemalloc --set llvm.ninja=false
              RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
              MACOSX_DEPLOYMENT_TARGET: 10.7
              NO_LLVM_ASSERTIONS: 1
              NO_DEBUG_ASSERTIONS: 1
            <<: *job-macos-xl

          - name: x86_64-apple
            env:
              SCRIPT: ./x.py --stage 2 test
              RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false
              RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
              MACOSX_DEPLOYMENT_TARGET: 10.8
              MACOSX_STD_DEPLOYMENT_TARGET: 10.7
              NO_LLVM_ASSERTIONS: 1
              NO_DEBUG_ASSERTIONS: 1
            <<: *job-macos-xl

          ######################
          #  Windows Builders  #
          ######################

          - name: x86_64-msvc-1
            env:
              RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
              SCRIPT: make ci-subset-1
              # FIXME(#59637)
              NO_DEBUG_ASSERTIONS: 1
              NO_LLVM_ASSERTIONS: 1
            <<: *job-windows-xl

          - name: x86_64-msvc-2
            env:
              RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
              SCRIPT: make ci-subset-2
            <<: *job-windows-xl

          - name: i686-msvc-1
            env:
              RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
              SCRIPT: make ci-subset-1
              # FIXME(#59637)
              NO_DEBUG_ASSERTIONS: 1
              NO_LLVM_ASSERTIONS: 1
            <<: *job-windows-xl

          - name: i686-msvc-2
            env:
              RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
              SCRIPT: make ci-subset-2
              # FIXME(#59637)
              NO_DEBUG_ASSERTIONS: 1
              NO_LLVM_ASSERTIONS: 1
            <<: *job-windows-xl

          - name: x86_64-msvc-cargo
            env:
              SCRIPT: python x.py --stage 2 test src/tools/cargotest src/tools/cargo
              RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-lld
              VCVARS_BAT: vcvars64.bat
              # FIXME(#59637)
              NO_DEBUG_ASSERTIONS: 1
              NO_LLVM_ASSERTIONS: 1
            <<: *job-windows-xl

          - name: x86_64-msvc-tools
            env:
              SCRIPT: src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows
              RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstate/toolstates.json
            <<: *job-windows-xl

          # 32/64-bit MinGW builds.
          #
          # We are using MinGW with POSIX threads since LLVM requires
          # C++'s std::thread which is disabled in libstdc++ with win32 threads.
          # FIXME: Libc++ doesn't have this limitation so we can avoid
          # winpthreads if we switch to it.
          #
          # Instead of relying on the MinGW version installed on CI we download
          # and install one ourselves so we won't be surprised by changes to CI's
          # build image.
          #
          # Finally, note that the downloads below are all in the `rust-lang-ci` S3
          # bucket, but they clearly didn't originate there! The downloads originally
          # came from the mingw-w64 SourceForge download site. Unfortunately
          # SourceForge is notoriously flaky, so we mirror it on our own infrastructure.

          - name: i686-mingw-1
            env:
              RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
              SCRIPT: make ci-mingw-subset-1
              CUSTOM_MINGW: 1
              # FIXME(#59637)
              NO_DEBUG_ASSERTIONS: 1
              NO_LLVM_ASSERTIONS: 1
            <<: *job-windows-xl

          - name: i686-mingw-2
            env:
              RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
              SCRIPT: make ci-mingw-subset-2
              CUSTOM_MINGW: 1
            <<: *job-windows-xl

          - name: x86_64-mingw-1
            env:
              SCRIPT: make ci-mingw-subset-1
              RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-profiler
              CUSTOM_MINGW: 1
              # FIXME(#59637)
              NO_DEBUG_ASSERTIONS: 1
              NO_LLVM_ASSERTIONS: 1
            <<: *job-windows-xl

          - name: x86_64-mingw-2
            env:
              SCRIPT: make ci-mingw-subset-2
              RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-profiler
              CUSTOM_MINGW: 1
            <<: *job-windows-xl

          - name: dist-x86_64-msvc
            env:
              RUST_CONFIGURE_ARGS: >-
                --build=x86_64-pc-windows-msvc
                --host=x86_64-pc-windows-msvc
                --target=x86_64-pc-windows-msvc,aarch64-pc-windows-msvc
                --enable-full-tools
                --enable-profiler
              SCRIPT: python x.py dist
              DIST_REQUIRE_ALL_TOOLS: 1
            <<: *job-windows-xl

          - name: dist-i686-msvc
            env:
              RUST_CONFIGURE_ARGS: >-
                --build=i686-pc-windows-msvc
                --host=i686-pc-windows-msvc
                --target=i686-pc-windows-msvc,i586-pc-windows-msvc
                --enable-full-tools
                --enable-profiler
              SCRIPT: python x.py dist
              DIST_REQUIRE_ALL_TOOLS: 1
            <<: *job-windows-xl

          - name: dist-i686-mingw
            env:
              RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools --enable-profiler
              SCRIPT: python x.py dist
              CUSTOM_MINGW: 1
              DIST_REQUIRE_ALL_TOOLS: 1
            <<: *job-windows-xl

          - name: dist-x86_64-mingw
            env:
              SCRIPT: python x.py dist
              RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler
              CUSTOM_MINGW: 1
              DIST_REQUIRE_ALL_TOOLS: 1
            <<: *job-windows-xl

          - name: dist-x86_64-msvc-alt
            env:
              RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler
              SCRIPT: python x.py dist
            <<: *job-windows-xl

  auto-fallible:
    <<: *base-ci-job
    name: auto-fallible
    env:
      <<: [*shared-ci-variables, *dummy-variables]
    if: github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'
    strategy:
      fail-fast: false
      matrix:
        include:
          #############################
          #   Linux/Docker builders   #
          #############################

          - name: aarch64-gnu
            <<: *job-aarch64-linux

  master:
    name: master
    runs-on: ubuntu-latest
    env:
      <<: [*prod-variables]
    if: github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'rust-lang-ci/rust'
    steps:
      - name: checkout the source code
        uses: actions/checkout@v1
        with:
          fetch-depth: 2

      - name: publish toolstate
        run: src/ci/publish_toolstate.sh
        shell: bash
        env:
          TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
        <<: *step

  # These jobs don't actually test anything, but they're used to tell bors the
  # build completed, as there is no practical way to detect when a workflow is
  # successful listening to webhooks only.
  try-success:
    needs: [try]
    if: "success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'"
    <<: *base-success-job
  try-failure:
    needs: [try]
    if: "!success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'"
    <<: *base-failure-job
  auto-success:
    needs: [auto]
    if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
    <<: *base-success-job
  auto-failure:
    needs: [auto]
    if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
    <<: *base-failure-job