-
Notifications
You must be signed in to change notification settings - Fork 21.9k
266 lines (240 loc) · 10.3 KB
/
lint.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
name: Lint
on:
pull_request:
branches-ignore:
- nightly
push:
branches:
- main
- release/*
- landchecks/*
workflow_dispatch:
permissions: read-all
# The names of steps that actually test the code should be suffixed with `(nonretryable)`.
# When any other step fails, it's job will be retried once by retryBot.
jobs:
lintrunner-clang:
uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
with:
timeout: 120
runner: linux.2xlarge
docker-image: pytorch-linux-jammy-cuda11.8-cudnn8-py3.9-linter
# NB: A shallow checkout won't work here because calculate-docker-image requires a full checkout
# to run git rev-parse HEAD~:.ci/docker when a new image is needed
fetch-depth: 0
submodules: true
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
script: |
export ADDITIONAL_LINTRUNNER_ARGS="--take CLANGTIDY,CLANGFORMAT"
export CLANG=1
.github/scripts/lintrunner.sh
lintrunner-noclang:
uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
with:
timeout: 120
runner: linux.2xlarge
docker-image: pytorch-linux-jammy-cuda11.8-cudnn8-py3.9-linter
# NB: A shallow checkout won't work here because calculate-docker-image requires a full checkout
# to run git rev-parse HEAD~:.ci/docker when a new image is needed
fetch-depth: 0
submodules: true
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
script: |
export ADDITIONAL_LINTRUNNER_ARGS="--skip CLANGTIDY,CLANGFORMAT"
.github/scripts/lintrunner.sh
quick-checks:
uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
with:
runner: linux.2xlarge
docker-image: pytorch-linux-focal-linter
fetch-depth: 0
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
script: |
# The generic Linux job chooses to use base env, not the one setup by the image
CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
conda activate "${CONDA_ENV}"
# Ensure no non-breaking spaces
# NB: We use 'printf' below rather than '\u000a' since bash pre-4.2
# does not support the '\u000a' syntax (which is relevant for local linters)
(! git --no-pager grep -In "$(printf '\xC2\xA0')" -- . || (echo "The above lines have non-breaking spaces (U+00A0); please convert them to spaces (U+0020)"; false))
# Ensure cross-OS compatible file names
(! git ls-files | grep -E '([<>:"|?*]|[ .]$)' || (echo "The above file names are not valid across all operating systems. Please ensure they don't contain the characters '<>:""|?*' and don't end with a white space or a '.' "; false))
# Ensure no versionless Python shebangs
(! git --no-pager grep -In '#!.*python$' -- . || (echo "The above lines have versionless Python shebangs; please specify either python2 or python3"; false))
# Ensure ciflow tags mentioned in config
python3 .github/scripts/collect_ciflow_labels.py --validate-tags
# C++ docs check
pushd docs/cpp/source
./check-doxygen.sh
popd
# CUDA kernel launch check
set -eux
python3 torch/testing/_internal/check_kernel_launches.py |& tee cuda_kernel_launch_checks.txt
pr-sanity-checks:
name: pr-sanity-checks
runs-on: [self-hosted, linux.large]
# Only run this on pull requests. This check is simple enough to be done without a Docker image
if: github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'skip-pr-sanity-checks')
steps:
- name: Checkout PyTorch
uses: pytorch/pytorch/.github/actions/checkout-pytorch@main
with:
submodules: false
fetch-depth: -1
- name: PR size check (nonretryable)
env:
BASE: ${{ github.event.pull_request.base.sha }}
HEAD: ${{ github.event.pull_request.head.sha }}
run: |
bash .github/scripts/pr-sanity-check.sh
workflow-checks:
uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
with:
runner: linux.2xlarge
docker-image: pytorch-linux-focal-linter
fetch-depth: -1
submodules: true
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
script: |
# The generic Linux job chooses to use base env, not the one setup by the image
CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
conda activate "${CONDA_ENV}"
# Regenerate workflows
.github/scripts/generate_ci_workflows.py
RC=0
# Assert that regenerating the workflows didn't change them
if ! .github/scripts/report_git_status.sh .github/workflows; then
echo
echo 'As shown by the above diff, the committed .github/workflows'
echo 'are not up to date according to .github/templates.'
echo 'Please run this command, commit, and push again to your PR:'
echo
echo ' .github/scripts/generate_ci_workflows.py'
echo
echo 'If running that command does nothing, you may need to rebase'
echo 'onto a more recent commit from the PyTorch main branch.'
RC=1
fi
# Check that jobs will be cancelled
.github/scripts/ensure_actions_will_cancel.py
exit $RC
toc:
uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
with:
runner: linux.2xlarge
docker-image: pytorch-linux-focal-linter
fetch-depth: 0
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
script: |
# The generic Linux job chooses to use base env, not the one setup by the image
CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
conda activate "${CONDA_ENV}"
# Regenerate ToCs and check that they didn't change
set -eu
export PATH=~/.npm-global/bin:"$PATH"
for FILE in $(git grep -Il '<!-- toc -->' -- '**.md'); do
markdown-toc --bullets='-' -i "$FILE"
done
if ! .github/scripts/report_git_status.sh .; then
echo
echo 'As shown by the above diff, the table of contents in one or'
echo 'more Markdown files is not up to date with the file contents.'
echo 'You can either apply that Git diff directly to correct the'
echo 'table of contents, or if you have npm installed, you can'
echo 'install the npm package markdown-toc and run the following'
# shellcheck disable=SC2016
echo 'command (replacing $FILE with the filename for which you want'
echo 'to regenerate the table of contents):'
echo
# shellcheck disable=SC2016
echo " markdown-toc --bullets='-' -i \"\$FILE\""
false
fi
test-tools:
name: Test tools
if: ${{ github.repository == 'pytorch/pytorch' }}
uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
with:
runner: linux.2xlarge
docker-image: pytorch-linux-focal-linter
fetch-depth: 0
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
script: |
# The generic Linux job chooses to use base env, not the one setup by the image
CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
conda activate "${CONDA_ENV}"
# Test tools
python3 -m unittest discover -vs tools/test -p 'test_*.py'
python3 -m unittest discover -vs .github/scripts -p 'test_*.py'
test_run_test:
name: Test `run_test.py` is usable without boto3/rockset
if: ${{ github.repository == 'pytorch/pytorch' }}
runs-on: linux.20_04.4x
steps:
- name: Checkout PyTorch
uses: pytorch/pytorch/.github/actions/checkout-pytorch@main
with:
submodules: false
fetch-depth: 1
- name: Setup Python 3.8
uses: actions/setup-python@v4
with:
python-version: '3.8'
architecture: x64
cache: pip
- name: Install dependencies
run: |
pip install pytest-rerunfailures==11.1.* pytest-flakefinder==1.1.* pytest-xdist==3.3.* expecttest==0.1.* numpy==1.24.*
pip install torch --pre --index-url https://download.pytorch.org/whl/nightly/cpu/
- name: Run run_test.py (nonretryable)
run: |
# Run test_vulkan, which is a fast noop on Linux
python3 test/run_test.py --include test_vulkan --verbose
test_collect_env:
if: ${{ github.repository == 'pytorch/pytorch' }}
name: Test collect_env
runs-on: linux.20_04.4x
strategy:
matrix:
test_type: [with_torch, without_torch, older_python_version]
steps:
# [see note: pytorch repo ref]
# deep clone (fetch-depth 0) required, to allow us to use git log
- name: Checkout PyTorch
uses: pytorch/pytorch/.github/actions/checkout-pytorch@main
with:
submodules: false
fetch-depth: 1
- name: Setup Python 3.6
if: matrix.test_type == 'older_python_version'
uses: actions/setup-python@v4
with:
python-version: '3.6'
architecture: x64
check-latest: false
cache: pip
cache-dependency-path: |
**/requirements.txt
- name: Setup Python 3.8
if: matrix.test_type != 'older_python_version'
uses: actions/setup-python@v4
with:
python-version: '3.8'
architecture: x64
check-latest: false
cache: pip
cache-dependency-path: |
**/requirements.txt
- name: Install torch
if: matrix.test_type == 'with_torch'
run: |
pip install -r requirements.txt
# Doesn't really matter what torch version, we just need ANY torch installed
pip install 'torch==1.*'
- name: Run collect_env.py (nonretryable)
run: |
# All we need to see is that it passes
python3 torch/utils/collect_env.py
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
cancel-in-progress: true