-
Notifications
You must be signed in to change notification settings - Fork 24
342 lines (325 loc) · 12.1 KB
/
ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
name: CI Tests
on: push
jobs:
cancel-previous-runs:
name: Cancel previous runs
runs-on: ubuntu-20.04
steps:
- name: Cancel previous runs
if: ${{ github.ref != 'refs/heads/main' }}
uses: styfle/cancel-workflow-action@0.6.0
with:
access_token: ${{ github.token }}
black:
name: Black
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
- name: Python Setup
uses: ./.github/actions/python-setup
- name: Black
run: pipenv run black . --check
check-if-pipenv-locked:
name: Check if Pipenv is locked and synced
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
- name: Python Setup
uses: ./.github/actions/python-setup
- name: Check pipenv is locked
run: pipenv run python -m recidiviz.tools.is_pipenv_locked
- name: Check pipenv is synced
run: pipenv run recidiviz/tools/diff_pipenv.sh
mypy:
name: Mypy
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
- name: Python Setup
uses: ./.github/actions/python-setup
- name: Mypy check
run: pipenv run mypy recidiviz
# Verify that all YAML files are properly formatted, according to prettier
prettier:
name: Prettier [yaml]
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
- uses: actions/setup-node@v2
- run: npm install prettier
- run: npx prettier --check "**/*.{yaml,yml}"
pylint:
name: Pylint
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
with:
# This clones the repo and checks out the SHA that triggered this action.
# We set fetch-depth 0 to fetch all branches and history so that merge-base
# is guaranteed to be able to find the common ancestor with the base branch.
# See https://github.com/actions/checkout#readme
fetch-depth: 0
- name: Python Setup
uses: ./.github/actions/python-setup
- name: Install Pylint
run: pipenv run pip install pylint
- name: Pylint
run: pipenv run ./recidiviz/tools/lint/run_pylint.sh
terraform-validation:
name: Terraform Validation
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
- uses: hashicorp/setup-terraform@v1
with:
# Note: this verison number should be kept in sync with the ones in Dockerfile,
# .devcontainer/devcontainer.json, recidiviz/tools/deploy/terraform/terraform.tf, and
# recidiviz/tools/deploy/deploy_helpers.sh
terraform_version: ~1.7.0
- name: Validate terraform
run: |
terraform -chdir=recidiviz/tools/deploy/terraform init -backend=false
terraform -chdir=recidiviz/tools/deploy/terraform validate
# TODO(#6431): pre-commit was flaky in CI, so we turned it off. It should be
# re-enabled soon.
# - name: pre-commit
# # If pre-commit proves useful, we can remove redundant checks (e.g. black).
# uses: pre-commit/action@v2.0.0
# # This can also create a new commit with changes, but it merges in latest
# # main and can't re-trigger CI, so we don't do so at this time.
# with:
# # Pass extra_args to only run on changed files. The default is '--all-files'.
# # TODO(#6234): Remove once PA files can be generated again.
# extra_args: --from-ref ${{ env.merge_base }} --to-ref HEAD
validate-source-visibility:
name: Validate source visibility
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
- name: Python Setup
uses: ./.github/actions/python-setup
- name: Validate source visibility
run: pipenv run python -m recidiviz.tools.validate_source_visibility
frontend-tests:
name: Frontend tests
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
with:
# This clones the repo and checks out the SHA that triggered this action.
# We set fetch-depth 0 to fetch all branches and history so that merge-base
# is guaranteed to be able to find the common ancestor with the base branch.
# See https://github.com/actions/checkout#readme
fetch-depth: 0
- name: Yarn caching
uses: actions/cache@v2
id: yarn-caching
with:
path: "**/node_modules"
key: yarn-v2-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
yarn-v2-${{ hashFiles('**/yarn.lock') }}
yarn-v2-
yarn-
- name: Yarn install (admin panel)
if: steps.yarn-caching.outputs.cache-hit != 'true'
working-directory: ./frontends/admin-panel/
run: yarn install --frozen-lockfile
- name: Yarn lint (admin panel)
working-directory: ./frontends/admin-panel/
run: yarn lint --max-warnings=0
- name: Yarn test (admin panel)
working-directory: ./frontends/admin-panel/
run: yarn test
asset-generation-tests:
name: Asset Generation Service tests
runs-on: ubuntu-20.04
defaults:
run:
working-directory: ./nodejs/asset-generation/
steps:
- uses: actions/checkout@v2
- uses: actions/setup-node@v3
with:
node-version: "18"
cache: "yarn"
cache-dependency-path: nodejs/asset-generation/yarn.lock
- name: Install fonts
run: |
cp src/fonts/* /usr/share/fonts/
fc-cache -fv
- name: Yarn install
run: yarn install --frozen-lockfile
- name: Lint
run: yarn lint --max-warnings=0
- name: Test
run: yarn test
unit-tests:
name: Base unit tests
runs-on: ubuntu-20.04
strategy:
fail-fast: false
matrix:
suite: [1, 2, 3]
suite-count: [3]
steps:
- uses: actions/checkout@v2
- name: Python Setup
uses: ./.github/actions/python-setup
- name: Run unit tests in parallel
run: |
pipenv run pytest recidiviz/tests \
-n logical \
-m 'not uses_db and not isolated and not uses_bq_emulator' \
--suite-count ${{matrix.suite-count}} \
--suite=${{ matrix.suite }} \
--durations=25
# These unit tests need to be run separately because they depend on a different Python
# environment defined in the Pipfile in the recidiviz/airflow package.
airflow-tests:
name: Airflow unit tests
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
- name: Airflow Python Setup
uses: ./.github/actions/airflow-python-setup
- name: Cache postgres
uses: actions/cache@v2
id: cache-postgres
with:
path: "~/postgres"
key: ${{ runner.os }}-postgres-13
- name: Install postgres
env:
CACHE_HIT: ${{steps.cache-postgres.outputs.cache-hit}}
shell: bash
run: |
if [[ "$CACHE_HIT" == 'true' ]]; then
sudo cp --force --recursive ~/postgres/* /
else
sudo apt-get update
sudo apt-get -y upgrade
wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" | sudo tee /etc/apt/sources.list.d/postgresql-pgdg.list > /dev/null
sudo apt-get update
sudo apt-get install postgresql-13
mkdir -p ~/postgres
for dep in postgresql-13; do
dpkg -L $dep | while IFS= read -r f; do if test -f $f; then echo $f; fi; done | xargs cp --parents --target-directory ~/postgres/
done
fi
echo "/usr/lib/postgresql/13/bin" >> $GITHUB_PATH
sudo chown -R $USER /var/run/postgresql
- name: Run airflow tests in parallel
working-directory: ./recidiviz/airflow
run: |
pipenv run pytest tests
db-tests:
name: Database tests
runs-on: ubuntu-20.04
strategy:
fail-fast: false
matrix:
suite: [1, 2, 3]
suite-count: [3]
steps:
- uses: actions/checkout@v2
- name: Python Setup
uses: ./.github/actions/python-setup
# Postgres caching strategy adapted from https://raymii.org/s/articles/Github_Actions_cpp_boost_cmake_speedup.html
- name: Cache postgres
uses: actions/cache@v2
id: cache-postgres
with:
path: "~/postgres"
key: ${{ runner.os }}-postgres-13
- name: Install postgres
env:
CACHE_HIT: ${{steps.cache-postgres.outputs.cache-hit}}
shell: bash
run: |
if [[ "$CACHE_HIT" == 'true' ]]; then
sudo cp --force --recursive ~/postgres/* /
else
sudo apt-get update
sudo apt-get -y upgrade
wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" | sudo tee /etc/apt/sources.list.d/postgresql-pgdg.list > /dev/null
sudo apt-get update
sudo apt-get install postgresql-13
mkdir -p ~/postgres
for dep in postgresql-13; do
dpkg -L $dep | while IFS= read -r f; do if test -f $f; then echo $f; fi; done | xargs cp --parents --target-directory ~/postgres/
done
fi
echo "/usr/lib/postgresql/13/bin" >> $GITHUB_PATH
sudo chown -R $USER /var/run/postgresql
- name: Run db tests
run: |
pipenv run pytest recidiviz/tests \
-n logical \
-m 'uses_db' \
--suite-count ${{matrix.suite-count}} \
--suite=${{ matrix.suite }} \
--durations=25
big-query-emulator-tests:
name: BigQuery Emulator Tests
runs-on: ubuntu-20.04
strategy:
fail-fast: false
matrix:
suite: [1, 2, 3]
suite-count: [3]
steps:
- uses: actions/checkout@v2
- name: Python Setup
uses: ./.github/actions/python-setup
# TODO(#20786): This step can be removed when moving back to the upstream bigquery emulator.
- name: Docker auth with ghcr
run: |
echo ${{ secrets.github_token }} | docker login ghcr.io -u ${{ github.actor }} --password-stdin
- name: Pull emulator Docker image
run: pipenv run pull-bq-emulator
# TODO(#20786): Replace ghcr.io/recidiviz with ghcr.io/goccy to move back to the
# upstream bigquery emulator.
# Start two emulator processes for each of the running pytest processes.
# We must run one emulator per process specified in `pytest -n` below.
- name: Start emulator (1)
run: |
docker run -d -p 9050:9050 \
--name=recidiviz-bq-emulator-1 \
--cpus 1 \
--cpuset-cpus 0 \
ghcr.io/recidiviz/bigquery-emulator:0.4.4-recidiviz.8 \
/bin/bigquery-emulator \
--port=9050 \
--project=recidiviz-bq-emulator-project \
--log-level=info
- name: Start emulator (2)
run: |
docker run -d -p 9051:9051 \
--name=recidiviz-bq-emulator-2 \
--cpus 1 \
--cpuset-cpus 1 \
ghcr.io/recidiviz/bigquery-emulator:0.4.4-recidiviz.8 \
/bin/bigquery-emulator \
--port=9051 \
--project=recidiviz-bq-emulator-project \
--log-level=info
- name: Run unittests
run: |
pipenv run pytest recidiviz/tests \
-m 'uses_bq_emulator' \
-n 2 \
--suite-count ${{matrix.suite-count}} \
--suite=${{ matrix.suite }} \
--durations=25
- name: Show logs (1)
if: ${{ failure() || cancelled() }}
run: docker logs recidiviz-bq-emulator-1
- name: Show logs (2)
if: ${{ failure() || cancelled() }}
run: docker logs recidiviz-bq-emulator-2
- name: Stop the container
if: ${{ always() }}
run: docker stop recidiviz-bq-emulator-1 && docker rm recidiviz-bq-emulator-1 && docker stop recidiviz-bq-emulator-2 && docker rm recidiviz-bq-emulator-2