-
Notifications
You must be signed in to change notification settings - Fork 12.1k
249 lines (217 loc) · 10.1 KB
/
ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
# This file defines our primary CI workflow that runs on pull requests
# and also on pushes to special branches (auto, try).
#
# The actual definition of the executed jobs is calculated by a Python
# script located at src/ci/github-actions/calculate-job-matrix.py, which
# uses job definition data from src/ci/github-actions/jobs.yml.
# You should primarily modify the `jobs.yml` file if you want to modify
# what jobs are executed in CI.
name: CI
on:
push:
branches:
- auto
- try
- try-perf
- automation/bors/try
pull_request:
branches:
- "**"
permissions:
contents: read
packages: write
defaults:
run:
# On Linux, macOS, and Windows, use the system-provided bash as the default
# shell. (This should only make a difference on Windows, where the default
# shell is PowerShell.)
shell: bash
concurrency:
# For a given workflow, if we push to the same branch, cancel all previous builds on that branch.
# We add an exception for try builds (try branch) and unrolled rollup builds (try-perf), which
# are all triggered on the same branch, but which should be able to run concurrently.
group: ${{ github.workflow }}-${{ ((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.sha) || github.ref }}
cancel-in-progress: true
env:
TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate"
jobs:
# The job matrix for `calculate_matrix` is defined in src/ci/github-actions/jobs.yml.
# It calculates which jobs should be executed, based on the data of the ${{ github }} context.
# If you want to modify CI jobs, take a look at src/ci/github-actions/jobs.yml.
calculate_matrix:
name: Calculate job matrix
runs-on: ubuntu-latest
outputs:
jobs: ${{ steps.jobs.outputs.jobs }}
run_type: ${{ steps.jobs.outputs.run_type }}
steps:
- name: Checkout the source code
uses: actions/checkout@v4
- name: Calculate the CI job matrix
run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT
id: jobs
job:
name: ${{ matrix.name }}
needs: [ calculate_matrix ]
runs-on: "${{ matrix.os }}"
defaults:
run:
shell: ${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}
timeout-minutes: 600
env:
CI_JOB_NAME: ${{ matrix.image }}
CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
# commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs.
HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
DOCKER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SCCACHE_BUCKET: rust-lang-ci-sccache2
CACHE_DOMAIN: ci-caches.rust-lang.org
continue-on-error: ${{ matrix.continue_on_error || false }}
strategy:
matrix:
# Check the `calculate_matrix` job to see how is the matrix defined.
include: ${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}
# GitHub Actions fails the workflow if an empty list of jobs is provided to
# the workflow, so we need to skip this job if nothing was produced by
# the Python script.
#
# Unfortunately checking whether a list is empty is not possible in a nice
# way due to GitHub Actions expressions limits.
# This hack is taken from https://github.com/ferrocene/ferrocene/blob/d43edc6b7697cf1719ec1c17c54904ab94825763/.github/workflows/release.yml#L75-L82
if: fromJSON(needs.calculate_matrix.outputs.jobs)[0] != null
steps:
- if: contains(matrix.os, 'windows')
uses: msys2/setup-msys2@v2.22.0
with:
# i686 jobs use mingw32. x86_64 and cross-compile jobs use mingw64.
msystem: ${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }}
# don't try to download updates for already installed packages
update: false
# don't try to use the msys that comes built-in to the github runner,
# so we can control what is installed (i.e. not python)
release: true
# Inherit the full path from the Windows environment, with MSYS2's */bin/
# dirs placed in front. This lets us run Windows-native Python etc.
path-type: inherit
install: >
make
dos2unix
diffutils
- name: disable git crlf conversion
run: git config --global core.autocrlf false
- name: checkout the source code
uses: actions/checkout@v4
with:
fetch-depth: 2
# Rust Log Analyzer can't currently detect the PR number of a GitHub
# Actions build on its own, so a hint in the log message is needed to
# point it in the right direction.
- name: configure the PR in which the error message will be posted
run: echo "[CI_PR_NUMBER=$num]"
env:
num: ${{ github.event.number }}
if: needs.calculate_matrix.outputs.run_type == 'pr'
- name: add extra environment variables
run: src/ci/scripts/setup-environment.sh
env:
# Since it's not possible to merge `${{ matrix.env }}` with the other
# variables in `job.<name>.env`, the variables defined in the matrix
# are passed to the `setup-environment.sh` script encoded in JSON,
# which then uses log commands to actually set them.
EXTRA_VARIABLES: ${{ toJson(matrix.env) }}
- name: ensure the channel matches the target branch
run: src/ci/scripts/verify-channel.sh
- name: collect CPU statistics
run: src/ci/scripts/collect-cpu-stats.sh
- name: show the current environment
run: src/ci/scripts/dump-environment.sh
- name: install awscli
run: src/ci/scripts/install-awscli.sh
- name: install sccache
run: src/ci/scripts/install-sccache.sh
- name: select Xcode
run: src/ci/scripts/select-xcode.sh
- name: install clang
run: src/ci/scripts/install-clang.sh
- name: install tidy
run: src/ci/scripts/install-tidy.sh
- name: install WIX
run: src/ci/scripts/install-wix.sh
- name: disable git crlf conversion
run: src/ci/scripts/disable-git-crlf-conversion.sh
- name: checkout submodules
run: src/ci/scripts/checkout-submodules.sh
- name: install MSYS2
run: src/ci/scripts/install-msys2.sh
- name: install MinGW
run: src/ci/scripts/install-mingw.sh
- name: install ninja
run: src/ci/scripts/install-ninja.sh
- name: enable ipv6 on Docker
run: src/ci/scripts/enable-docker-ipv6.sh
# Disable automatic line ending conversion (again). On Windows, when we're
# installing dependencies, something switches the git configuration directory or
# re-enables autocrlf. We've not tracked down the exact cause -- and there may
# be multiple -- but this should ensure submodules are checked out with the
# appropriate line endings.
- name: disable git crlf conversion
run: src/ci/scripts/disable-git-crlf-conversion.sh
- name: ensure line endings are correct
run: src/ci/scripts/verify-line-endings.sh
- name: ensure backported commits are in upstream branches
run: src/ci/scripts/verify-backported-commits.sh
- name: ensure the stable version number is correct
run: src/ci/scripts/verify-stable-version-number.sh
- name: run the build
# Redirect stderr to stdout to avoid reordering the two streams in the GHA logs.
run: src/ci/scripts/run-build-from-ci.sh 2>&1
env:
AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}
TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
- name: create github artifacts
run: src/ci/scripts/create-doc-artifacts.sh
- name: upload artifacts to github
uses: actions/upload-artifact@v4
with:
# name is set in previous step
name: ${{ env.DOC_ARTIFACT_NAME }}
path: obj/artifacts/doc
if-no-files-found: ignore
retention-days: 5
- name: upload artifacts to S3
run: src/ci/scripts/upload-artifacts.sh
env:
AWS_ACCESS_KEY_ID: ${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}
# Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy
# builders *should* have the AWS credentials available. Still, explicitly
# adding the condition is helpful as this way CI will not silently skip
# deploying artifacts from a dist builder if the variables are misconfigured,
# erroring about invalid credentials instead.
if: github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1'
# This job isused to tell bors the final status of the build, as there is no practical way to detect
# when a workflow is successful listening to webhooks only in our current bors implementation (homu).
outcome:
name: bors build finished
runs-on: ubuntu-latest
needs: [ calculate_matrix, job ]
# !cancelled() executes the job regardless of whether the previous jobs passed or failed
if: ${{ !cancelled() && contains(fromJSON('["auto", "try"]'), needs.calculate_matrix.outputs.run_type) }}
steps:
- name: checkout the source code
uses: actions/checkout@v4
with:
fetch-depth: 2
# Calculate the exit status of the whole CI workflow.
# If all dependent jobs were successful, this exits with 0 (and the outcome job continues successfully).
# If a some dependent job has failed, this exits with 1.
- name: calculate the correct exit status
run: jq --exit-status 'all(.result == "success" or .result == "skipped")' <<< '${{ toJson(needs) }}'
# Publish the toolstate if an auto build succeeds (just before push to master)
- name: publish toolstate
run: src/ci/publish_toolstate.sh
shell: bash
if: needs.calculate_matrix.outputs.run_type == 'auto'
env:
TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}