diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000000..275823e9dbcf8 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,781 @@ +############################################################# +# WARNING: automatically generated file, DO NOT CHANGE! # +############################################################# + +# This file was automatically generated by the expand-yaml-anchors tool. The +# source file that generated this one is: +# +# src/ci/github-actions/ci.yml +# +# Once you make changes to that file you need to run: +# +# ./x.py run src/tools/expand-yaml-anchors/ +# +# The CI build will fail if the tool is not run after changes to this file. + +--- +name: CI +"on": + push: + branches: + - auto + - try + - master + pull_request: + branches: + - "**" +jobs: + pr: + name: PR + env: + CI_JOB_NAME: "${{ matrix.name }}" + SCCACHE_BUCKET: rust-lang-gha-caches + TOOLSTATE_REPO: "https://github.com/pietroalbini/rust-toolstate" + if: "github.event_name == 'pull_request'" + strategy: + matrix: + name: + - mingw-check + - x86_64-gnu-llvm-7 + - x86_64-gnu-tools + include: + - name: mingw-check + os: ubuntu-latest-xl + env: {} + - name: x86_64-gnu-llvm-7 + os: ubuntu-latest-xl + env: {} + - name: x86_64-gnu-tools + env: + CI_ONLY_WHEN_SUBMODULES_CHANGED: 1 + os: ubuntu-latest-xl + timeout-minutes: 600 + runs-on: "${{ matrix.os }}" + steps: + - name: disable git crlf conversion + run: git config --global core.autocrlf false + shell: bash + - name: checkout the source code + uses: actions/checkout@v1 + with: + fetch-depth: 2 + - name: configure GitHub Actions to kill the build when outdated + uses: rust-lang/simpleinfra/github-actions/cancel-outdated-builds@master + with: + github_token: "${{ secrets.github_token }}" + if: success() && !env.SKIP_JOB + - name: add extra environment variables + run: src/ci/scripts/setup-environment.sh + env: + EXTRA_VARIABLES: "${{ toJson(matrix.env) }}" + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: decide whether to skip this job + run: src/ci/scripts/should-skip-this.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: collect CPU statistics + run: src/ci/scripts/collect-cpu-stats.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: show the current environment + run: src/ci/scripts/dump-environment.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install awscli + run: src/ci/scripts/install-awscli.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install sccache + run: src/ci/scripts/install-sccache.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install clang + run: src/ci/scripts/install-clang.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install WIX + run: src/ci/scripts/install-wix.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install InnoSetup + run: src/ci/scripts/install-innosetup.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: ensure the build happens on a partition with enough space + run: src/ci/scripts/symlink-build-dir.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: disable git crlf conversion + run: src/ci/scripts/disable-git-crlf-conversion.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install MSYS2 + run: src/ci/scripts/install-msys2.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install MSYS2 packages + run: src/ci/scripts/install-msys2-packages.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install MinGW + run: src/ci/scripts/install-mingw.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install ninja + run: src/ci/scripts/install-ninja.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: enable ipv6 on Docker + run: src/ci/scripts/enable-docker-ipv6.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: disable git crlf conversion + run: src/ci/scripts/disable-git-crlf-conversion.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: checkout submodules + run: src/ci/scripts/checkout-submodules.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: ensure line endings are correct + run: src/ci/scripts/verify-line-endings.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: run the build + run: src/ci/scripts/run-build-from-ci.sh + env: + AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}" + AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}" + TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}" + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: upload artifacts to S3 + run: src/ci/scripts/upload-artifacts.sh + env: + AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}" + AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}" + if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')" + shell: "python src/ci/exec-with-shell.py {0}" + try: + name: try + env: + CI_JOB_NAME: "${{ matrix.name }}" + SCCACHE_BUCKET: rust-lang-gha-caches + DEPLOY_BUCKET: rust-lang-gha + TOOLSTATE_REPO: "https://github.com/pietroalbini/rust-toolstate" + TOOLSTATE_ISSUES_API_URL: "https://api.github.com/repos/pietroalbini/rust-toolstate/issues" + TOOLSTATE_PUBLISH: 1 + CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZOMUQATD5 + ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZH5AYXDVF + if: "github.event_name == 'push' && github.ref == 'refs/heads/try'" + strategy: + matrix: + name: + - dist-x86_64-linux + - dist-x86_64-linux-alt + include: + - name: dist-x86_64-linux + os: ubuntu-latest-xl + env: {} + - name: dist-x86_64-linux-alt + env: + IMAGE: dist-x86_64-linux + os: ubuntu-latest-xl + timeout-minutes: 600 + runs-on: "${{ matrix.os }}" + steps: + - name: disable git crlf conversion + run: git config --global core.autocrlf false + shell: bash + - name: checkout the source code + uses: actions/checkout@v1 + with: + fetch-depth: 2 + - name: configure GitHub Actions to kill the build when outdated + uses: rust-lang/simpleinfra/github-actions/cancel-outdated-builds@master + with: + github_token: "${{ secrets.github_token }}" + if: success() && !env.SKIP_JOB + - name: add extra environment variables + run: src/ci/scripts/setup-environment.sh + env: + EXTRA_VARIABLES: "${{ toJson(matrix.env) }}" + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: decide whether to skip this job + run: src/ci/scripts/should-skip-this.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: collect CPU statistics + run: src/ci/scripts/collect-cpu-stats.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: show the current environment + run: src/ci/scripts/dump-environment.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install awscli + run: src/ci/scripts/install-awscli.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install sccache + run: src/ci/scripts/install-sccache.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install clang + run: src/ci/scripts/install-clang.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install WIX + run: src/ci/scripts/install-wix.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install InnoSetup + run: src/ci/scripts/install-innosetup.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: ensure the build happens on a partition with enough space + run: src/ci/scripts/symlink-build-dir.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: disable git crlf conversion + run: src/ci/scripts/disable-git-crlf-conversion.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install MSYS2 + run: src/ci/scripts/install-msys2.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install MSYS2 packages + run: src/ci/scripts/install-msys2-packages.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install MinGW + run: src/ci/scripts/install-mingw.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install ninja + run: src/ci/scripts/install-ninja.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: enable ipv6 on Docker + run: src/ci/scripts/enable-docker-ipv6.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: disable git crlf conversion + run: src/ci/scripts/disable-git-crlf-conversion.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: checkout submodules + run: src/ci/scripts/checkout-submodules.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: ensure line endings are correct + run: src/ci/scripts/verify-line-endings.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: run the build + run: src/ci/scripts/run-build-from-ci.sh + env: + AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}" + AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}" + TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}" + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: upload artifacts to S3 + run: src/ci/scripts/upload-artifacts.sh + env: + AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}" + AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}" + if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')" + shell: "python src/ci/exec-with-shell.py {0}" + auto: + name: auto + env: + CI_JOB_NAME: "${{ matrix.name }}" + SCCACHE_BUCKET: rust-lang-gha-caches + DEPLOY_BUCKET: rust-lang-gha + TOOLSTATE_REPO: "https://github.com/pietroalbini/rust-toolstate" + TOOLSTATE_ISSUES_API_URL: "https://api.github.com/repos/pietroalbini/rust-toolstate/issues" + TOOLSTATE_PUBLISH: 1 + CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZOMUQATD5 + ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZH5AYXDVF + if: "github.event_name == 'push' && github.ref == 'refs/heads/auto'" + strategy: + matrix: + name: + - arm-android + - armhf-gnu + - dist-aarch64-linux + - dist-android + - dist-arm-linux + - dist-armhf-linux + - dist-armv7-linux + - dist-i586-gnu-i586-i686-musl + - dist-i686-freebsd + - dist-i686-linux + - dist-i686-mingw + - dist-i686-msvc + - dist-mips-linux + - dist-mips64-linux + - dist-mips64el-linux + - dist-mipsel-linux + - dist-powerpc-linux + - dist-powerpc64-linux + - dist-powerpc64le-linux + - dist-s390x-linux + - dist-various-1 + - dist-various-2 + - dist-x86_64-apple + - dist-x86_64-apple-alt + - dist-x86_64-freebsd + - dist-x86_64-linux + - dist-x86_64-linux-alt + - dist-x86_64-mingw + - dist-x86_64-msvc + - dist-x86_64-msvc-alt + - dist-x86_64-musl + - dist-x86_64-netbsd + - i686-gnu + - i686-gnu-nopt + - i686-mingw-1 + - i686-mingw-2 + - i686-msvc-1 + - i686-msvc-2 + - mingw-check + - test-various + - wasm32 + - x86_64-apple + - x86_64-gnu + - x86_64-gnu-aux + - x86_64-gnu-debug + - x86_64-gnu-distcheck + - x86_64-gnu-full-bootstrap + - x86_64-gnu-llvm-7 + - x86_64-gnu-nopt + - x86_64-gnu-tools + - x86_64-mingw-1 + - x86_64-mingw-2 + - x86_64-msvc-1 + - x86_64-msvc-2 + - x86_64-msvc-aux + - x86_64-msvc-cargo + - x86_64-msvc-tools + include: + - name: arm-android + os: ubuntu-latest-xl + env: {} + - name: armhf-gnu + os: ubuntu-latest-xl + env: {} + - name: dist-aarch64-linux + os: ubuntu-latest-xl + env: {} + - name: dist-android + os: ubuntu-latest-xl + env: {} + - name: dist-arm-linux + os: ubuntu-latest-xl + env: {} + - name: dist-armhf-linux + os: ubuntu-latest-xl + env: {} + - name: dist-armv7-linux + os: ubuntu-latest-xl + env: {} + - name: dist-i586-gnu-i586-i686-musl + os: ubuntu-latest-xl + env: {} + - name: dist-i686-freebsd + os: ubuntu-latest-xl + env: {} + - name: dist-i686-linux + os: ubuntu-latest-xl + env: {} + - name: dist-mips-linux + os: ubuntu-latest-xl + env: {} + - name: dist-mips64-linux + os: ubuntu-latest-xl + env: {} + - name: dist-mips64el-linux + os: ubuntu-latest-xl + env: {} + - name: dist-mipsel-linux + os: ubuntu-latest-xl + env: {} + - name: dist-powerpc-linux + os: ubuntu-latest-xl + env: {} + - name: dist-powerpc64-linux + os: ubuntu-latest-xl + env: {} + - name: dist-powerpc64le-linux + os: ubuntu-latest-xl + env: {} + - name: dist-s390x-linux + os: ubuntu-latest-xl + env: {} + - name: dist-various-1 + os: ubuntu-latest-xl + env: {} + - name: dist-various-2 + os: ubuntu-latest-xl + env: {} + - name: dist-x86_64-freebsd + os: ubuntu-latest-xl + env: {} + - name: dist-x86_64-linux + os: ubuntu-latest-xl + env: {} + - name: dist-x86_64-linux-alt + env: + IMAGE: dist-x86_64-linux + os: ubuntu-latest-xl + - name: dist-x86_64-musl + os: ubuntu-latest-xl + env: {} + - name: dist-x86_64-netbsd + os: ubuntu-latest-xl + env: {} + - name: i686-gnu + os: ubuntu-latest-xl + env: {} + - name: i686-gnu-nopt + os: ubuntu-latest-xl + env: {} + - name: mingw-check + os: ubuntu-latest-xl + env: {} + - name: test-various + os: ubuntu-latest-xl + env: {} + - name: wasm32 + os: ubuntu-latest-xl + env: {} + - name: x86_64-gnu + os: ubuntu-latest-xl + env: {} + - name: x86_64-gnu-aux + os: ubuntu-latest-xl + env: {} + - name: x86_64-gnu-debug + os: ubuntu-latest-xl + env: {} + - name: x86_64-gnu-distcheck + os: ubuntu-latest-xl + env: {} + - name: x86_64-gnu-full-bootstrap + os: ubuntu-latest-xl + env: {} + - name: x86_64-gnu-llvm-7 + env: + RUST_BACKTRACE: 1 + os: ubuntu-latest-xl + - name: x86_64-gnu-nopt + os: ubuntu-latest-xl + env: {} + - name: x86_64-gnu-tools + env: + DEPLOY_TOOLSTATES_JSON: toolstates-linux.json + os: ubuntu-latest-xl + - name: dist-x86_64-apple + env: + SCRIPT: "./x.py dist" + RUST_CONFIGURE_ARGS: "--target=aarch64-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc" + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + DIST_REQUIRE_ALL_TOOLS: 1 + os: macos-latest + - name: dist-x86_64-apple-alt + env: + SCRIPT: "./x.py dist" + RUST_CONFIGURE_ARGS: "--enable-extended --enable-profiler --set rust.jemalloc" + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + os: macos-latest + - name: x86_64-apple + env: + SCRIPT: "./x.py test" + RUST_CONFIGURE_ARGS: "--build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc" + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.8 + MACOSX_STD_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + os: macos-latest + - name: x86_64-msvc-1 + env: + RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-profiler" + SCRIPT: make ci-subset-1 + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + os: windows-latest-xl + - name: x86_64-msvc-2 + env: + RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-profiler" + SCRIPT: make ci-subset-2 + os: windows-latest-xl + - name: i686-msvc-1 + env: + RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-msvc" + SCRIPT: make ci-subset-1 + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + os: windows-latest-xl + - name: i686-msvc-2 + env: + RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-msvc" + SCRIPT: make ci-subset-2 + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + os: windows-latest-xl + - name: x86_64-msvc-aux + env: + RUST_CHECK_TARGET: check-aux EXCLUDE_CARGO=1 + RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc" + os: windows-latest-xl + - name: x86_64-msvc-cargo + env: + SCRIPT: python x.py test src/tools/cargotest src/tools/cargo + RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc" + VCVARS_BAT: vcvars64.bat + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + os: windows-latest-xl + - name: x86_64-msvc-tools + env: + SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows + RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstate/toolstates.json" + os: windows-latest-xl + - name: i686-mingw-1 + env: + RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu" + SCRIPT: make ci-mingw-subset-1 + CUSTOM_MINGW: 1 + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + os: windows-latest-xl + - name: i686-mingw-2 + env: + RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu" + SCRIPT: make ci-mingw-subset-2 + CUSTOM_MINGW: 1 + os: windows-latest-xl + - name: x86_64-mingw-1 + env: + SCRIPT: make ci-mingw-subset-1 + RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu" + CUSTOM_MINGW: 1 + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + os: windows-latest-xl + - name: x86_64-mingw-2 + env: + SCRIPT: make ci-mingw-subset-2 + RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu" + CUSTOM_MINGW: 1 + os: windows-latest-xl + - name: dist-x86_64-msvc + env: + RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --target=x86_64-pc-windows-msvc,aarch64-pc-windows-msvc --enable-full-tools --enable-profiler" + SCRIPT: python x.py dist + DIST_REQUIRE_ALL_TOOLS: 1 + os: windows-latest-xl + - name: dist-i686-msvc + env: + RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-msvc --target=i586-pc-windows-msvc --enable-full-tools --enable-profiler" + SCRIPT: python x.py dist + DIST_REQUIRE_ALL_TOOLS: 1 + os: windows-latest-xl + - name: dist-i686-mingw + env: + RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu --enable-full-tools --enable-profiler" + SCRIPT: python x.py dist + CUSTOM_MINGW: 1 + DIST_REQUIRE_ALL_TOOLS: 1 + os: windows-latest-xl + - name: dist-x86_64-mingw + env: + SCRIPT: python x.py dist + RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler" + CUSTOM_MINGW: 1 + DIST_REQUIRE_ALL_TOOLS: 1 + os: windows-latest-xl + - name: dist-x86_64-msvc-alt + env: + RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-extended --enable-profiler" + SCRIPT: python x.py dist + os: windows-latest-xl + timeout-minutes: 600 + runs-on: "${{ matrix.os }}" + steps: + - name: disable git crlf conversion + run: git config --global core.autocrlf false + shell: bash + - name: checkout the source code + uses: actions/checkout@v1 + with: + fetch-depth: 2 + - name: configure GitHub Actions to kill the build when outdated + uses: rust-lang/simpleinfra/github-actions/cancel-outdated-builds@master + with: + github_token: "${{ secrets.github_token }}" + if: success() && !env.SKIP_JOB + - name: add extra environment variables + run: src/ci/scripts/setup-environment.sh + env: + EXTRA_VARIABLES: "${{ toJson(matrix.env) }}" + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: decide whether to skip this job + run: src/ci/scripts/should-skip-this.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: collect CPU statistics + run: src/ci/scripts/collect-cpu-stats.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: show the current environment + run: src/ci/scripts/dump-environment.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install awscli + run: src/ci/scripts/install-awscli.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install sccache + run: src/ci/scripts/install-sccache.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install clang + run: src/ci/scripts/install-clang.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install WIX + run: src/ci/scripts/install-wix.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install InnoSetup + run: src/ci/scripts/install-innosetup.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: ensure the build happens on a partition with enough space + run: src/ci/scripts/symlink-build-dir.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: disable git crlf conversion + run: src/ci/scripts/disable-git-crlf-conversion.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install MSYS2 + run: src/ci/scripts/install-msys2.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install MSYS2 packages + run: src/ci/scripts/install-msys2-packages.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install MinGW + run: src/ci/scripts/install-mingw.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install ninja + run: src/ci/scripts/install-ninja.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: enable ipv6 on Docker + run: src/ci/scripts/enable-docker-ipv6.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: disable git crlf conversion + run: src/ci/scripts/disable-git-crlf-conversion.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: checkout submodules + run: src/ci/scripts/checkout-submodules.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: ensure line endings are correct + run: src/ci/scripts/verify-line-endings.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: run the build + run: src/ci/scripts/run-build-from-ci.sh + env: + AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}" + AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}" + TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}" + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: upload artifacts to S3 + run: src/ci/scripts/upload-artifacts.sh + env: + AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}" + AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}" + if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')" + shell: "python src/ci/exec-with-shell.py {0}" + master: + name: master + runs-on: ubuntu-latest + env: + SCCACHE_BUCKET: rust-lang-gha-caches + DEPLOY_BUCKET: rust-lang-gha + TOOLSTATE_REPO: "https://github.com/pietroalbini/rust-toolstate" + TOOLSTATE_ISSUES_API_URL: "https://api.github.com/repos/pietroalbini/rust-toolstate/issues" + TOOLSTATE_PUBLISH: 1 + CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZOMUQATD5 + ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZH5AYXDVF + if: "github.event_name == 'push' && github.ref == 'refs/heads/master'" + steps: + - name: checkout the source code + uses: actions/checkout@v1 + with: + fetch-depth: 2 + - name: publish toolstate + run: src/ci/publish_toolstate.sh + env: + TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}" + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + try-success: + needs: + - try + if: success() + steps: + - name: mark the job as a success + run: exit 0 + name: bors build finished + runs-on: ubuntu-latest + try-failure: + needs: + - try + if: "!success()" + steps: + - name: mark the job as a failure + run: exit 1 + name: bors build finished + runs-on: ubuntu-latest + auto-success: + needs: + - auto + if: success() + steps: + - name: mark the job as a success + run: exit 0 + name: bors build finished + runs-on: ubuntu-latest + auto-failure: + needs: + - auto + if: "!success()" + steps: + - name: mark the job as a failure + run: exit 1 + name: bors build finished + runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index 22a06151353ba..f45637e647998 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -443,7 +443,7 @@ dependencies = [ "textwrap", "unicode-width", "vec_map", - "yaml-rust", + "yaml-rust 0.3.5", ] [[package]] @@ -1026,6 +1026,14 @@ dependencies = [ "walkdir", ] +[[package]] +name = "expand-yaml-anchors" +version = "0.1.0" +dependencies = [ + "yaml-merge-keys", + "yaml-rust 0.4.3", +] + [[package]] name = "failure" version = "0.1.5" @@ -1830,6 +1838,12 @@ dependencies = [ name = "linkchecker" version = "0.1.0" +[[package]] +name = "linked-hash-map" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae91b68aebc4ddb91978b11a1b02ddd8602a05ec19002801c5666000e05e0f83" + [[package]] name = "lock_api" version = "0.3.1" @@ -4854,6 +4868,26 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "thiserror" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9fb62ff737e573b1e677459bea6fd023cd5d6e868c3242d3cdf3ef2f0554824" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24069c0ba08aab54289d6a25f5036d94afc61e1538bbc42ae5501df141c9027d" +dependencies = [ + "proc-macro2 1.0.3", + "quote 1.0.2", + "syn 1.0.11", +] + [[package]] name = "thread_local" version = "0.3.6" @@ -5536,8 +5570,28 @@ dependencies = [ "lzma-sys", ] +[[package]] +name = "yaml-merge-keys" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59893318ba3ad2b704498c7761214a10eaf42c5f838bce9fc0145bf2ba658cfa" +dependencies = [ + "lazy_static 1.4.0", + "thiserror", + "yaml-rust 0.4.3", +] + [[package]] name = "yaml-rust" version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e66366e18dc58b46801afbf2ca7661a9f59cc8c5962c29892b6039b4f86fa992" + +[[package]] +name = "yaml-rust" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65923dd1784f44da1d2c3dbbc5e822045628c590ba72123e1c73d3c230c4434d" +dependencies = [ + "linked-hash-map", +] diff --git a/Cargo.toml b/Cargo.toml index 2f5a708e8dc6c..7b5e0fa1c2817 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ members = [ "src/tools/miri", "src/tools/rustdoc-themes", "src/tools/unicode-table-generator", + "src/tools/expand-yaml-anchors", ] exclude = [ "build", diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index dd519506d42a0..243cd3fa19998 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -21,6 +21,7 @@ use crate::doc; use crate::flags::Subcommand; use crate::install; use crate::native; +use crate::run; use crate::test; use crate::tool; use crate::util::{self, add_dylib_path, add_link_lib_path, exe, libdir}; @@ -313,6 +314,7 @@ pub enum Kind { Dist, Doc, Install, + Run, } impl<'a> Builder<'a> { @@ -353,6 +355,7 @@ impl<'a> Builder<'a> { } Kind::Test => describe!( crate::toolstate::ToolStateCheck, + test::ExpandYamlAnchors, test::Tidy, test::Ui, test::CompileFail, @@ -454,6 +457,7 @@ impl<'a> Builder<'a> { install::Src, install::Rustc ), + Kind::Run => describe!(run::ExpandYamlAnchors,), } } @@ -507,6 +511,7 @@ impl<'a> Builder<'a> { Subcommand::Bench { ref paths, .. } => (Kind::Bench, &paths[..]), Subcommand::Dist { ref paths } => (Kind::Dist, &paths[..]), Subcommand::Install { ref paths } => (Kind::Install, &paths[..]), + Subcommand::Run { ref paths } => (Kind::Run, &paths[..]), Subcommand::Format { .. } | Subcommand::Clean { .. } => panic!(), }; diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs index d8831c6d9e566..eda26f7df1f1c 100644 --- a/src/bootstrap/flags.rs +++ b/src/bootstrap/flags.rs @@ -86,6 +86,9 @@ pub enum Subcommand { Install { paths: Vec, }, + Run { + paths: Vec, + }, } impl Default for Subcommand { @@ -113,6 +116,7 @@ Subcommands: clean Clean out build directories dist Build distribution artifacts install Install distribution artifacts + run Run tools contained in this repository To learn more about a subcommand, run `./x.py -h`", ); @@ -188,6 +192,7 @@ To learn more about a subcommand, run `./x.py -h`", || (s == "clean") || (s == "dist") || (s == "install") + || (s == "run") }); let subcommand = match subcommand { Some(s) => s, @@ -400,6 +405,18 @@ Arguments: ./x.py doc --stage 1", ); } + "run" => { + subcommand_help.push_str( + "\n +Arguments: + This subcommand accepts a number of paths to tools to build and run. For + example: + + ./x.py run src/tool/expand-yaml-anchors + + At least a tool needs to be called.", + ); + } _ => {} }; // Get any optional paths which occur after the subcommand @@ -468,6 +485,13 @@ Arguments: "fmt" => Subcommand::Format { check: matches.opt_present("check") }, "dist" => Subcommand::Dist { paths }, "install" => Subcommand::Install { paths }, + "run" => { + if paths.is_empty() { + println!("\nrun requires at least a path!\n"); + usage(1, &opts, &subcommand_help, &extra_help); + } + Subcommand::Run { paths } + } _ => { usage(1, &opts, &subcommand_help, &extra_help); } diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index a476d25f10214..6436fa756558b 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -140,6 +140,7 @@ mod format; mod install; mod metadata; mod native; +mod run; mod sanity; mod test; mod tool; diff --git a/src/bootstrap/run.rs b/src/bootstrap/run.rs new file mode 100644 index 0000000000000..22f88b516cd0c --- /dev/null +++ b/src/bootstrap/run.rs @@ -0,0 +1,45 @@ +use crate::builder::{Builder, RunConfig, ShouldRun, Step}; +use crate::tool::Tool; +use std::process::Command; + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct ExpandYamlAnchors; + +impl Step for ExpandYamlAnchors { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + /// Runs the `expand-yaml_anchors` tool. + /// + /// This tool in `src/tools` read the CI configuration files written in YAML and expands the + /// anchors in them, since GitHub Actions doesn't support them. + fn run(self, builder: &Builder<'_>) { + builder.info("Expanding YAML anchors in the GitHub Actions configuration"); + try_run( + builder, + &mut builder.tool_cmd(Tool::ExpandYamlAnchors).arg("generate").arg(&builder.src), + ); + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/expand-yaml-anchors") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(ExpandYamlAnchors); + } +} + +fn try_run(builder: &Builder<'_>, cmd: &mut Command) -> bool { + if !builder.fail_fast { + if !builder.try_run(cmd) { + let mut failures = builder.delayed_failures.borrow_mut(); + failures.push(format!("{:?}", cmd)); + return false; + } + } else { + builder.run(cmd); + } + true +} diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index aa1d1b7c42413..e75868f7ebd08 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -750,6 +750,35 @@ impl Step for Tidy { } } +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct ExpandYamlAnchors; + +impl Step for ExpandYamlAnchors { + type Output = (); + const ONLY_HOSTS: bool = true; + + /// Ensure the `generate-ci-config` tool was run locally. + /// + /// The tool in `src/tools` reads the CI definition in `src/ci/builders.yml` and generates the + /// appropriate configuration for all our CI providers. This step ensures the tool was called + /// by the user before committing CI changes. + fn run(self, builder: &Builder<'_>) { + builder.info("Ensuring the YAML anchors in the GitHub Actions config were expanded"); + try_run( + builder, + &mut builder.tool_cmd(Tool::ExpandYamlAnchors).arg("check").arg(&builder.src), + ); + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/expand-yaml-anchors") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(ExpandYamlAnchors); + } +} + fn testdir(builder: &Builder<'_>, host: Interned) -> PathBuf { builder.out.join(host).join("test") } diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs index c8ccba467e509..52f750f448e20 100644 --- a/src/bootstrap/tool.rs +++ b/src/bootstrap/tool.rs @@ -378,6 +378,7 @@ bootstrap_tool!( RemoteTestClient, "src/tools/remote-test-client", "remote-test-client"; RustInstaller, "src/tools/rust-installer", "fabricate", is_external_tool = true; RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes"; + ExpandYamlAnchors, "src/tools/expand-yaml-anchors", "expand-yaml-anchors"; ); #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] diff --git a/src/ci/docker/mingw-check/Dockerfile b/src/ci/docker/mingw-check/Dockerfile index e973ba2e33c64..1293717f975aa 100644 --- a/src/ci/docker/mingw-check/Dockerfile +++ b/src/ci/docker/mingw-check/Dockerfile @@ -22,7 +22,8 @@ RUN sh /scripts/sccache.sh COPY mingw-check/validate-toolstate.sh /scripts/ ENV RUN_CHECK_WITH_PARALLEL_QUERIES 1 -ENV SCRIPT python2.7 ../x.py check --target=i686-pc-windows-gnu --host=i686-pc-windows-gnu && \ +ENV SCRIPT python2.7 ../x.py test src/tools/expand-yaml-anchors && \ + python2.7 ../x.py check --target=i686-pc-windows-gnu --host=i686-pc-windows-gnu && \ python2.7 ../x.py build --stage 0 src/tools/build-manifest && \ python2.7 ../x.py test --stage 0 src/tools/compiletest && \ python2.7 ../x.py test src/tools/tidy && \ diff --git a/src/ci/exec-with-shell.py b/src/ci/exec-with-shell.py new file mode 100755 index 0000000000000..26ce69e33d9c3 --- /dev/null +++ b/src/ci/exec-with-shell.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python +# A simple wrapper that forwards the arguments to bash, unless the +# CI_OVERRIDE_SHELL environment variable is present: in that case the content +# of that environment variable is used as the shell path. + +import os +import sys +import subprocess + +try: + shell = os.environ["CI_OVERRIDE_SHELL"] +except KeyError: + shell = "bash" + +res = subprocess.call([shell] + sys.argv[1:]) +sys.exit(res) diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml new file mode 100644 index 0000000000000..33573a4de9f68 --- /dev/null +++ b/src/ci/github-actions/ci.yml @@ -0,0 +1,707 @@ +###################################################### +# WARNING! Action needed when changing this file # +###################################################### + +# Due to GitHub Actions limitations, we can't use YAML Anchors directly in the +# CI configuration stored on the repository. To work around that this file is +# expanded by a tool in the repository, and the expansion is committed as well. +# +# After you make any change to the file you'll need to run this command: +# +# ./x.py run src/tools/expand-yaml-anchors +# +# ...and commit the file it updated in addition to this one. If you forget this +# step CI will fail. + +--- + +############################### +# YAML Anchors Definition # +############################### + +# This key contains most of the YAML anchors that will be used later in the +# document. YAML anchors allows us to greatly reduce duplication inside the CI +# configuration by reusing parts of the configuration. +# +# YAML anchors work by defining an anchor with `&anchor-name` and reusing its +# content in another place with `*anchor-name`. The special `<<` map key merges +# the content of the map with the content of the anchor (or list of anchors). +# +# The expand-yaml-anchors tool will automatically remove this block from the +# output YAML file. +x--expand-yaml-anchors--remove: + + - &shared-ci-variables + CI_JOB_NAME: ${{ matrix.name }} + + - &public-variables + SCCACHE_BUCKET: rust-lang-gha-caches + TOOLSTATE_REPO: https://github.com/pietroalbini/rust-toolstate + + - &prod-variables + SCCACHE_BUCKET: rust-lang-gha-caches + DEPLOY_BUCKET: rust-lang-gha + TOOLSTATE_REPO: https://github.com/pietroalbini/rust-toolstate + TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/pietroalbini/rust-toolstate/issues + TOOLSTATE_PUBLISH: 1 + # AWS_SECRET_ACCESS_KEYs are stored in GitHub's secrets storage, named + # AWS_SECRET_ACCESS_KEY_. Including the key id in the name allows to + # rotate them in a single branch while keeping the old key in another + # branch, which wouldn't be possible if the key was named with the kind + # (caches, artifacts...). + CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZOMUQATD5 + ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZH5AYXDVF + + - &base-job + env: {} + + - &job-linux-xl + os: ubuntu-latest-xl + <<: *base-job + + - &job-macos-xl + os: macos-latest # We don't have an XL builder for this + <<: *base-job + + - &job-windows-xl + os: windows-latest-xl + <<: *base-job + + - &step + if: success() && !env.SKIP_JOB + + - &step-run + <<: *step + # While on Linux and macOS builders it just forwards the arguments to the + # system bash, this wrapper allows switching from the host's bash.exe to + # the one we install along with MSYS2 mid-build on Windows. + # + # Once the step to install MSYS2 is executed, the CI_OVERRIDE_SHELL + # environment variable is set pointing to our MSYS2's bash.exe. From that + # moment the host's bash.exe will not be called anymore. + # + # This is needed because we can't launch our own bash.exe from the host + # bash.exe, as that would load two different cygwin1.dll in memory, causing + # "cygwin heap mismatch" errors. + shell: python src/ci/exec-with-shell.py {0} + + - &base-ci-job + timeout-minutes: 600 + runs-on: "${{ matrix.os }}" + env: *shared-ci-variables + steps: + - name: disable git crlf conversion + run: git config --global core.autocrlf false + shell: bash + + - name: checkout the source code + uses: actions/checkout@v1 + with: + fetch-depth: 2 + + - name: configure GitHub Actions to kill the build when outdated + uses: rust-lang/simpleinfra/github-actions/cancel-outdated-builds@master + with: + github_token: "${{ secrets.github_token }}" + <<: *step + + - name: add extra environment variables + run: src/ci/scripts/setup-environment.sh + env: + # Since it's not possible to merge `${{ matrix.env }}` with the other + # variables in `job..env`, the variables defined in the matrix + # are passed to the `setup-environment.sh` script encoded in JSON, + # which then uses log commands to actually set them. + EXTRA_VARIABLES: ${{ toJson(matrix.env) }} + <<: *step-run + + - name: decide whether to skip this job + run: src/ci/scripts/should-skip-this.sh + <<: *step-run + + - name: collect CPU statistics + run: src/ci/scripts/collect-cpu-stats.sh + <<: *step-run + + - name: show the current environment + run: src/ci/scripts/dump-environment.sh + <<: *step-run + + - name: install awscli + run: src/ci/scripts/install-awscli.sh + <<: *step-run + + - name: install sccache + run: src/ci/scripts/install-sccache.sh + <<: *step-run + + - name: install clang + run: src/ci/scripts/install-clang.sh + <<: *step-run + + - name: install WIX + run: src/ci/scripts/install-wix.sh + <<: *step-run + + - name: install InnoSetup + run: src/ci/scripts/install-innosetup.sh + <<: *step-run + + - name: ensure the build happens on a partition with enough space + run: src/ci/scripts/symlink-build-dir.sh + <<: *step-run + + - name: disable git crlf conversion + run: src/ci/scripts/disable-git-crlf-conversion.sh + <<: *step-run + + - name: install MSYS2 + run: src/ci/scripts/install-msys2.sh + <<: *step-run + + - name: install MSYS2 packages + run: src/ci/scripts/install-msys2-packages.sh + <<: *step-run + + - name: install MinGW + run: src/ci/scripts/install-mingw.sh + <<: *step-run + + - name: install ninja + run: src/ci/scripts/install-ninja.sh + <<: *step-run + + - name: enable ipv6 on Docker + run: src/ci/scripts/enable-docker-ipv6.sh + <<: *step-run + + # Disable automatic line ending conversion (again). On Windows, when we're + # installing dependencies, something switches the git configuration directory or + # re-enables autocrlf. We've not tracked down the exact cause -- and there may + # be multiple -- but this should ensure submodules are checked out with the + # appropriate line endings. + - name: disable git crlf conversion + run: src/ci/scripts/disable-git-crlf-conversion.sh + <<: *step-run + + - name: checkout submodules + run: src/ci/scripts/checkout-submodules.sh + <<: *step-run + + - name: ensure line endings are correct + run: src/ci/scripts/verify-line-endings.sh + <<: *step-run + + - name: run the build + run: src/ci/scripts/run-build-from-ci.sh + env: + AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }} + TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }} + <<: *step-run + + - name: upload artifacts to S3 + run: src/ci/scripts/upload-artifacts.sh + env: + AWS_ACCESS_KEY_ID: ${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }} + # Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy + # builders *should* have the AWS credentials available. Still, explicitly + # adding the condition is helpful as this way CI will not silently skip + # deploying artifacts from a dist builder if the variables are misconfigured, + # erroring about invalid credentials instead. + if: success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1') + <<: *step-run + + # These snippets are used by the try-success, try-failure, auto-success and auto-failure jobs. + # Check out their documentation for more information on why they're needed. + + - &base-outcome-job + name: bors build finished + runs-on: ubuntu-latest + + - &base-success-job + if: success() + steps: + - name: mark the job as a success + run: exit 0 + <<: *base-outcome-job + + - &base-failure-job + if: "!success()" + steps: + - name: mark the job as a failure + run: exit 1 + <<: *base-outcome-job + +########################### +# Builders definition # +########################### + +name: CI +on: + push: + branches: + - auto + - try + - master + pull_request: + branches: + - "**" + +jobs: + pr: + <<: *base-ci-job + name: PR + env: + <<: [*shared-ci-variables, *public-variables] + if: github.event_name == 'pull_request' + strategy: + matrix: + name: + - mingw-check + - x86_64-gnu-llvm-7 + - x86_64-gnu-tools + include: + - name: mingw-check + <<: *job-linux-xl + + - name: x86_64-gnu-llvm-7 + <<: *job-linux-xl + + - name: x86_64-gnu-tools + env: + CI_ONLY_WHEN_SUBMODULES_CHANGED: 1 + <<: *job-linux-xl + + try: + <<: *base-ci-job + name: try + env: + <<: [*shared-ci-variables, *prod-variables] + if: github.event_name == 'push' && github.ref == 'refs/heads/try' + strategy: + matrix: + name: + - dist-x86_64-linux + - dist-x86_64-linux-alt + include: + - name: dist-x86_64-linux + <<: *job-linux-xl + + - name: dist-x86_64-linux-alt + env: + IMAGE: dist-x86_64-linux + <<: *job-linux-xl + + auto: + <<: *base-ci-job + name: auto + env: + <<: [*shared-ci-variables, *prod-variables] + if: github.event_name == 'push' && github.ref == 'refs/heads/auto' + strategy: + matrix: + name: + - arm-android + - armhf-gnu + - dist-aarch64-linux + - dist-android + - dist-arm-linux + - dist-armhf-linux + - dist-armv7-linux + - dist-i586-gnu-i586-i686-musl + - dist-i686-freebsd + - dist-i686-linux + - dist-i686-mingw + - dist-i686-msvc + - dist-mips-linux + - dist-mips64-linux + - dist-mips64el-linux + - dist-mipsel-linux + - dist-powerpc-linux + - dist-powerpc64-linux + - dist-powerpc64le-linux + - dist-s390x-linux + - dist-various-1 + - dist-various-2 + - dist-x86_64-apple + - dist-x86_64-apple-alt + - dist-x86_64-freebsd + - dist-x86_64-linux + - dist-x86_64-linux-alt + - dist-x86_64-mingw + - dist-x86_64-msvc + - dist-x86_64-msvc-alt + - dist-x86_64-musl + - dist-x86_64-netbsd + - i686-gnu + - i686-gnu-nopt + - i686-mingw-1 + - i686-mingw-2 + - i686-msvc-1 + - i686-msvc-2 + - mingw-check + - test-various + - wasm32 + - x86_64-apple + - x86_64-gnu + - x86_64-gnu-aux + - x86_64-gnu-debug + - x86_64-gnu-distcheck + - x86_64-gnu-full-bootstrap + - x86_64-gnu-llvm-7 + - x86_64-gnu-nopt + - x86_64-gnu-tools + - x86_64-mingw-1 + - x86_64-mingw-2 + - x86_64-msvc-1 + - x86_64-msvc-2 + - x86_64-msvc-aux + - x86_64-msvc-cargo + - x86_64-msvc-tools + include: + ############################# + # Linux/Docker builders # + ############################# + + - name: arm-android + <<: *job-linux-xl + + - name: armhf-gnu + <<: *job-linux-xl + + - name: dist-aarch64-linux + <<: *job-linux-xl + + - name: dist-android + <<: *job-linux-xl + + - name: dist-arm-linux + <<: *job-linux-xl + + - name: dist-armhf-linux + <<: *job-linux-xl + + - name: dist-armv7-linux + <<: *job-linux-xl + + - name: dist-i586-gnu-i586-i686-musl + <<: *job-linux-xl + + - name: dist-i686-freebsd + <<: *job-linux-xl + + - name: dist-i686-linux + <<: *job-linux-xl + + - name: dist-mips-linux + <<: *job-linux-xl + + - name: dist-mips64-linux + <<: *job-linux-xl + + - name: dist-mips64el-linux + <<: *job-linux-xl + + - name: dist-mipsel-linux + <<: *job-linux-xl + + - name: dist-powerpc-linux + <<: *job-linux-xl + + - name: dist-powerpc64-linux + <<: *job-linux-xl + + - name: dist-powerpc64le-linux + <<: *job-linux-xl + + - name: dist-s390x-linux + <<: *job-linux-xl + + - name: dist-various-1 + <<: *job-linux-xl + + - name: dist-various-2 + <<: *job-linux-xl + + - name: dist-x86_64-freebsd + <<: *job-linux-xl + + - name: dist-x86_64-linux + <<: *job-linux-xl + + - name: dist-x86_64-linux-alt + env: + IMAGE: dist-x86_64-linux + <<: *job-linux-xl + + - name: dist-x86_64-musl + <<: *job-linux-xl + + - name: dist-x86_64-netbsd + <<: *job-linux-xl + + - name: i686-gnu + <<: *job-linux-xl + + - name: i686-gnu-nopt + <<: *job-linux-xl + + - name: mingw-check + <<: *job-linux-xl + + - name: test-various + <<: *job-linux-xl + + - name: wasm32 + <<: *job-linux-xl + + - name: x86_64-gnu + <<: *job-linux-xl + + - name: x86_64-gnu-aux + <<: *job-linux-xl + + - name: x86_64-gnu-debug + <<: *job-linux-xl + + - name: x86_64-gnu-distcheck + <<: *job-linux-xl + + - name: x86_64-gnu-full-bootstrap + <<: *job-linux-xl + + - name: x86_64-gnu-llvm-7 + env: + RUST_BACKTRACE: 1 + <<: *job-linux-xl + + - name: x86_64-gnu-nopt + <<: *job-linux-xl + + - name: x86_64-gnu-tools + env: + DEPLOY_TOOLSTATES_JSON: toolstates-linux.json + <<: *job-linux-xl + + #################### + # macOS Builders # + #################### + + - name: dist-x86_64-apple + env: + SCRIPT: ./x.py dist + RUST_CONFIGURE_ARGS: --target=aarch64-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + DIST_REQUIRE_ALL_TOOLS: 1 + <<: *job-macos-xl + + - name: dist-x86_64-apple-alt + env: + SCRIPT: ./x.py dist + RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + <<: *job-macos-xl + + - name: x86_64-apple + env: + SCRIPT: ./x.py test + RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.8 + MACOSX_STD_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + <<: *job-macos-xl + + ###################### + # Windows Builders # + ###################### + + - name: x86_64-msvc-1 + env: + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler + SCRIPT: make ci-subset-1 + # FIXME(#59637) + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + <<: *job-windows-xl + + - name: x86_64-msvc-2 + env: + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler + SCRIPT: make ci-subset-2 + <<: *job-windows-xl + + - name: i686-msvc-1 + env: + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc + SCRIPT: make ci-subset-1 + # FIXME(#59637) + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + <<: *job-windows-xl + + - name: i686-msvc-2 + env: + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc + SCRIPT: make ci-subset-2 + # FIXME(#59637) + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + <<: *job-windows-xl + + - name: x86_64-msvc-aux + env: + RUST_CHECK_TARGET: check-aux EXCLUDE_CARGO=1 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc + <<: *job-windows-xl + + - name: x86_64-msvc-cargo + env: + SCRIPT: python x.py test src/tools/cargotest src/tools/cargo + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc + VCVARS_BAT: vcvars64.bat + # FIXME(#59637) + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + <<: *job-windows-xl + + - name: x86_64-msvc-tools + env: + SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstate/toolstates.json + <<: *job-windows-xl + + # 32/64-bit MinGW builds. + # + # We are using MinGW with posix threads since LLVM does not compile with + # the win32 threads version due to missing support for C++'s std::thread. + # + # Instead of relying on the MinGW version installed on appveryor we download + # and install one ourselves so we won't be surprised by changes to appveyor's + # build image. + # + # Finally, note that the downloads below are all in the `rust-lang-ci` S3 + # bucket, but they cleraly didn't originate there! The downloads originally + # came from the mingw-w64 SourceForge download site. Unfortunately + # SourceForge is notoriously flaky, so we mirror it on our own infrastructure. + + - name: i686-mingw-1 + env: + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu + SCRIPT: make ci-mingw-subset-1 + CUSTOM_MINGW: 1 + # FIXME(#59637) + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + <<: *job-windows-xl + + - name: i686-mingw-2 + env: + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu + SCRIPT: make ci-mingw-subset-2 + CUSTOM_MINGW: 1 + <<: *job-windows-xl + + - name: x86_64-mingw-1 + env: + SCRIPT: make ci-mingw-subset-1 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu + CUSTOM_MINGW: 1 + # FIXME(#59637) + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + <<: *job-windows-xl + + - name: x86_64-mingw-2 + env: + SCRIPT: make ci-mingw-subset-2 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu + CUSTOM_MINGW: 1 + <<: *job-windows-xl + + - name: dist-x86_64-msvc + env: + RUST_CONFIGURE_ARGS: >- + --build=x86_64-pc-windows-msvc + --target=x86_64-pc-windows-msvc,aarch64-pc-windows-msvc + --enable-full-tools + --enable-profiler + SCRIPT: python x.py dist + DIST_REQUIRE_ALL_TOOLS: 1 + <<: *job-windows-xl + + - name: dist-i686-msvc + env: + RUST_CONFIGURE_ARGS: >- + --build=i686-pc-windows-msvc + --target=i586-pc-windows-msvc + --enable-full-tools + --enable-profiler + SCRIPT: python x.py dist + DIST_REQUIRE_ALL_TOOLS: 1 + <<: *job-windows-xl + + - name: dist-i686-mingw + env: + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools --enable-profiler + SCRIPT: python x.py dist + CUSTOM_MINGW: 1 + DIST_REQUIRE_ALL_TOOLS: 1 + <<: *job-windows-xl + + - name: dist-x86_64-mingw + env: + SCRIPT: python x.py dist + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler + CUSTOM_MINGW: 1 + DIST_REQUIRE_ALL_TOOLS: 1 + <<: *job-windows-xl + + - name: dist-x86_64-msvc-alt + env: + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler + SCRIPT: python x.py dist + <<: *job-windows-xl + + master: + name: master + runs-on: ubuntu-latest + env: + <<: [*prod-variables] + if: github.event_name == 'push' && github.ref == 'refs/heads/master' + steps: + - name: checkout the source code + uses: actions/checkout@v1 + with: + fetch-depth: 2 + + - name: publish toolstate + run: src/ci/publish_toolstate.sh + env: + TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }} + <<: *step-run + + # These jobs don't actually test anything, but they're used to tell bors the + # build completed, as there is no practical way to detect when a workflow is + # successful listening to webhooks only. + try-success: + needs: [try] + <<: *base-success-job + try-failure: + needs: [try] + <<: *base-failure-job + auto-success: + needs: [auto] + <<: *base-success-job + auto-failure: + needs: [auto] + <<: *base-failure-job diff --git a/src/ci/scripts/install-mingw.sh b/src/ci/scripts/install-mingw.sh index 98373df7fce50..78728dd7d005c 100755 --- a/src/ci/scripts/install-mingw.sh +++ b/src/ci/scripts/install-mingw.sh @@ -50,8 +50,8 @@ if isWindows; then esac if [[ "${CUSTOM_MINGW-0}" -ne 1 ]]; then - pacman -S --noconfirm --needed mingw-w64-$arch-toolchain mingw-w64-$arch-cmake \ - mingw-w64-$arch-gcc mingw-w64-$arch-python2 + pacman -S --noconfirm --needed mingw-w64-$arch-toolchain \ + mingw-w64-$arch-cmake mingw-w64-$arch-gcc mingw-w64-$arch-python2 ciCommandAddPath "$(ciCheckoutPath)/msys2/mingw${bits}/bin" else mingw_dir="mingw${bits}" diff --git a/src/ci/scripts/install-msys2.sh b/src/ci/scripts/install-msys2.sh index 9e899ba9d8947..3c3b5007f8697 100755 --- a/src/ci/scripts/install-msys2.sh +++ b/src/ci/scripts/install-msys2.sh @@ -22,4 +22,7 @@ if isWindows; then rm msys2.nupkg chocolatey-core.extension.nupkg mkdir -p "$(ciCheckoutPath)/msys2/home/${USERNAME}" ciCommandAddPath "$(ciCheckoutPath)/msys2/usr/bin" + + echo "switching shell to use our own bash" + ciCommandSetEnv CI_OVERRIDE_SHELL "$(ciCheckoutPath)/msys2/usr/bin/bash.exe" fi diff --git a/src/ci/scripts/setup-environment.sh b/src/ci/scripts/setup-environment.sh index d134fcd47baf1..411ef6f9b2822 100755 --- a/src/ci/scripts/setup-environment.sh +++ b/src/ci/scripts/setup-environment.sh @@ -11,16 +11,34 @@ source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" # Since matrix variables are readonly in Azure Pipelines, we take # INITIAL_RUST_CONFIGURE_ARGS and establish RUST_CONFIGURE_ARGS # which downstream steps can alter -# macOS ships with Bash 3.16, so we cannot use [[ -v FOO ]], -# which was introduced in Bash 4.2 -if [[ -z "${INITIAL_RUST_CONFIGURE_ARGS+x}" ]]; then - INITIAL_RUST_CONFIG="" - echo "No initial Rust configure args set" -else - INITIAL_RUST_CONFIG="${INITIAL_RUST_CONFIGURE_ARGS}" - ciCommandSetEnv RUST_CONFIGURE_ARGS "${INITIAL_RUST_CONFIG}" +if isAzurePipelines; then + # macOS ships with Bash 3.16, so we cannot use [[ -v FOO ]], + # which was introduced in Bash 4.2 + if [[ -z "${INITIAL_RUST_CONFIGURE_ARGS+x}" ]]; then + INITIAL_RUST_CONFIG="" + echo "No initial Rust configure args set" + else + INITIAL_RUST_CONFIG="${INITIAL_RUST_CONFIGURE_ARGS}" + ciCommandSetEnv RUST_CONFIGURE_ARGS "${INITIAL_RUST_CONFIG}" + fi fi +# Load extra environment variables +vars="${EXTRA_VARIABLES-}" +echo "${vars}" | jq '' >/dev/null # Validate JSON and exit on errors +for key in $(echo "${vars}" | jq "keys[]" -r); do + # On Windows, for whatever reason, $key contains the BOM character in it, + # and that messes up `jq ".${key}"`. This line strips the BOM from the key. + # + # https://unix.stackexchange.com/a/381263 + key="$(echo "${key}" | sed '1s/^\xEF\xBB\xBF//')" + + echo "adding extra environment variable ${key}" + value="$(echo "${vars}" | jq ".${key}" -r)" + export "${key}"="${value}" + ciCommandSetEnv "${key}" "${value}" +done + # Builders starting with `dist-` are dist builders, but if they also end with # `-alt` they are alternate dist builders. if [[ "${CI_JOB_NAME}" = dist-* ]]; then diff --git a/src/tools/expand-yaml-anchors/Cargo.toml b/src/tools/expand-yaml-anchors/Cargo.toml new file mode 100644 index 0000000000000..2c63e28b693da --- /dev/null +++ b/src/tools/expand-yaml-anchors/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "expand-yaml-anchors" +version = "0.1.0" +authors = ["Pietro Albini "] +edition = "2018" + +[dependencies] +yaml-rust = "0.4.3" +yaml-merge-keys = "0.4.0" diff --git a/src/tools/expand-yaml-anchors/src/main.rs b/src/tools/expand-yaml-anchors/src/main.rs new file mode 100644 index 0000000000000..f2ed8aa409a36 --- /dev/null +++ b/src/tools/expand-yaml-anchors/src/main.rs @@ -0,0 +1,202 @@ +use std::error::Error; +use std::path::{Path, PathBuf}; +use yaml_rust::{Yaml, YamlEmitter, YamlLoader}; + +/// List of directories containing files to expand. The first tuple element is the source +/// directory, while the second tuple element is the destination directory. +#[rustfmt::skip] +static TO_EXPAND: &[(&str, &str)] = &[ + ("src/ci/github-actions", ".github/workflows"), +]; + +/// Name of a special key that will be removed from all the maps in expanded configuration files. +/// This key can then be used to contain shared anchors. +static REMOVE_MAP_KEY: &str = "x--expand-yaml-anchors--remove"; + +/// Message that will be included at the top of all the expanded files. {source} will be replaced +/// with the source filename relative to the base path. +static HEADER_MESSAGE: &str = "\ +############################################################# +# WARNING: automatically generated file, DO NOT CHANGE! # +############################################################# + +# This file was automatically generated by the expand-yaml-anchors tool. The +# source file that generated this one is: +# +# {source} +# +# Once you make changes to that file you need to run: +# +# ./x.py run src/tools/expand-yaml-anchors/ +# +# The CI build will fail if the tool is not run after changes to this file. + +"; + +enum Mode { + Check, + Generate, +} + +struct App { + mode: Mode, + base: PathBuf, +} + +impl App { + fn from_args() -> Result> { + // Parse CLI arguments + let args = std::env::args().skip(1).collect::>(); + let (mode, base) = match args.iter().map(|s| s.as_str()).collect::>().as_slice() { + &["generate", ref base] => (Mode::Generate, PathBuf::from(base)), + &["check", ref base] => (Mode::Check, PathBuf::from(base)), + _ => { + eprintln!("usage: expand-yaml-anchors "); + std::process::exit(1); + } + }; + + Ok(App { mode, base }) + } + + fn run(&self) -> Result<(), Box> { + for (source, dest) in TO_EXPAND { + let source = self.base.join(source); + let dest = self.base.join(dest); + for entry in std::fs::read_dir(&source)? { + let path = entry?.path(); + if !path.is_file() || path.extension().and_then(|e| e.to_str()) != Some("yml") { + continue; + } + + let dest_path = dest.join(path.file_name().unwrap()); + self.expand(&path, &dest_path).with_context(|| match self.mode { + Mode::Generate => format!( + "failed to expand {} into {}", + self.path(&path), + self.path(&dest_path) + ), + Mode::Check => format!("{} is not up to date", self.path(&dest_path)), + })?; + } + } + Ok(()) + } + + fn expand(&self, source: &Path, dest: &Path) -> Result<(), Box> { + let content = std::fs::read_to_string(source) + .with_context(|| format!("failed to read {}", self.path(source)))?; + + let mut buf = HEADER_MESSAGE.replace("{source}", &self.path(source).to_string()); + + let documents = YamlLoader::load_from_str(&content) + .with_context(|| format!("failed to parse {}", self.path(source)))?; + for mut document in documents.into_iter() { + document = yaml_merge_keys::merge_keys(document) + .with_context(|| format!("failed to expand {}", self.path(source)))?; + document = filter_document(document); + + YamlEmitter::new(&mut buf).dump(&document).map_err(|err| WithContext { + context: "failed to serialize the expanded yaml".into(), + source: Box::new(err), + })?; + buf.push('\n'); + } + + match self.mode { + Mode::Check => { + let old = std::fs::read_to_string(dest) + .with_context(|| format!("failed to read {}", self.path(dest)))?; + if old != buf { + return Err(Box::new(StrError(format!( + "{} and {} are different", + self.path(source), + self.path(dest), + )))); + } + } + Mode::Generate => { + std::fs::write(dest, buf.as_bytes()) + .with_context(|| format!("failed to write to {}", self.path(dest)))?; + } + } + Ok(()) + } + + fn path<'a>(&self, path: &'a Path) -> impl std::fmt::Display + 'a { + path.strip_prefix(&self.base).unwrap_or(path).display() + } +} + +fn filter_document(document: Yaml) -> Yaml { + match document { + Yaml::Hash(map) => Yaml::Hash( + map.into_iter() + .filter(|(key, _)| { + if let Yaml::String(string) = &key { string != REMOVE_MAP_KEY } else { true } + }) + .map(|(key, value)| (filter_document(key), filter_document(value))) + .collect(), + ), + Yaml::Array(vec) => { + Yaml::Array(vec.into_iter().map(|item| filter_document(item)).collect()) + } + other => other, + } +} + +fn main() { + if let Err(err) = App::from_args().and_then(|app| app.run()) { + eprintln!("error: {}", err); + + let mut source = err.as_ref() as &dyn Error; + while let Some(err) = source.source() { + eprintln!("caused by: {}", err); + source = err; + } + + std::process::exit(1); + } +} + +#[derive(Debug)] +struct StrError(String); + +impl Error for StrError {} + +impl std::fmt::Display for StrError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + std::fmt::Display::fmt(&self.0, f) + } +} + +#[derive(Debug)] +struct WithContext { + context: String, + source: Box, +} + +impl std::fmt::Display for WithContext { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}", self.context) + } +} + +impl Error for WithContext { + fn source(&self) -> Option<&(dyn Error + 'static)> { + Some(self.source.as_ref()) + } +} + +pub(crate) trait ResultExt { + fn with_context String>(self, f: F) -> Result>; +} + +impl>> ResultExt for Result { + fn with_context String>(self, f: F) -> Result> { + match self { + Ok(ok) => Ok(ok), + Err(err) => Err(WithContext { source: err.into(), context: f() }.into()), + } + } +}