-
Notifications
You must be signed in to change notification settings - Fork 1.3k
385 lines (339 loc) · 14.4 KB
/
selenium-lab-tests.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
name: Selenium Lab Tests
on:
workflow_dispatch:
# Allows for manual triggering on PRs. They should be reviewed first, to
# avoid malicious code executing in the lab.
inputs:
pr:
description: "A PR number to build and test in the lab. If empty, will build and test from main."
required: false
test_filter:
description: "A regex filter to run a subset of the tests. If empty, all tests will run."
required: false
browser_filter:
description: "A list of browsers to run the tests. If empty, all browsers will run."
required: false
workflow_call:
# Allows for reuse from other workflows, such as "Update All Screenshots"
# workflow.
inputs:
pr:
description: "A PR number to build and test in the lab. If empty, will build and test from main."
required: false
type: string
test_filter:
description: "A regex filter to run a subset of the tests. If empty, all tests will run."
required: false
type: string
browser_filter:
description: "A list of browsers to run the tests. If empty, all browsers will run."
required: false
type: string
ignore_test_status:
description: "If true, ignore test success or failure, and always upload screenshots."
required: false
type: boolean
skip_commit_status:
description: "If true, skip the commit status."
required: false
type: boolean
job_name_prefix:
description: "A prefix added to the job name when setting commit status, needed to correctly link to each job. Use when skip_commit_status is false, and set to the name of the parent job, plus space-slash-space."
required: false
type: string
schedule:
# Runs every night at 2am PST / 10am UTC, testing against the main branch.
- cron: '0 10 * * *'
# Only one run of this workflow is allowed at a time, since it uses physical
# resources in our lab.
concurrency: selenium-lab
jobs:
compute-ref:
name: Compute ref
runs-on: ubuntu-latest
outputs:
REF: ${{ steps.compute.outputs.REF }}
steps:
- name: Compute ref
id: compute
run: |
if [[ "${{ inputs.pr }}" != "" ]]; then
LAB_TEST_REF="refs/pull/${{ inputs.pr }}/head"
else
LAB_TEST_REF="main"
fi
echo "REF=$LAB_TEST_REF" | tee -a $GITHUB_OUTPUT
# Configure the build matrix based on our grid's YAML config.
# The matrix contents will be computed by this first job and deserialized
# into the second job's config.
matrix-config:
name: Matrix config
needs: compute-ref
runs-on: ubuntu-latest
outputs:
INCLUDE: ${{ steps.configure.outputs.INCLUDE }}
steps:
- uses: actions/checkout@v4
with:
ref: ${{ needs.compute-ref.outputs.REF }}
- name: Install dependencies
run: npm ci
- name: Configure build matrix
id: configure
shell: node {0}
run: |
const fs = require('fs');
const yaml = require(
'${{ github.workspace }}/node_modules/js-yaml/index.js');
// Convert the input "browser_filter" into a set of strings. Take
// care to filter so that the empty string turns into an empty set.
const browserFilter = new Set( "${{ inputs.browser_filter }}"
.split(/\s+/)
.map(x => x.toLowerCase())
.filter(x => !!x)
);
const gridBrowserYaml =
fs.readFileSync('build/shaka-lab.yaml', 'utf8');
const gridBrowserMetadata = yaml.load(gridBrowserYaml);
const include = [];
for (const name in gridBrowserMetadata) {
if (name == 'vars') {
// Skip variable defs in the YAML file
continue;
}
const thisBrowserRequested = browserFilter.has(name.toLowerCase());
const nothingRequested = browserFilter.size == 0;
const thisBrowserOnByDefault = !gridBrowserMetadata[name].disabled;
// A browser is enabled if it's requested, or if it's on by default
// and nothing was requested.
const enabled = thisBrowserRequested ||
(nothingRequested == 0 && thisBrowserOnByDefault);
if (enabled) {
include.push({browser: name});
}
}
// Output JSON object consumed by the build matrix below.
fs.appendFileSync(
process.env['GITHUB_OUTPUT'],
`INCLUDE=${ JSON.stringify(include) }\n`);
// Log the output, for the sake of debugging this script.
console.log({include});
# Build Shaka Player once, then distribute that build to the runners in the
# build matrix. For N runners, runs N times faster (since all the
# self-hosted Selenium jobs are run in containers on one machine).
build-shaka:
name: Pre-build Player
needs: compute-ref
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
ref: ${{ needs.compute-ref.outputs.REF }}
- name: Set commit status to pending
if: ${{ inputs.skip_test_status == false }}
uses: ./.github/workflows/custom-actions/set-commit-status
with:
context: Selenium / Build
job_name: "${{ inputs.job_name_prefix }}Pre-build Player"
state: pending
token: ${{ secrets.GITHUB_TOKEN }}
- name: Build Player
run: python3 build/all.py
- name: Preprocess with Babel
run: |
# Run the test preprocessor without running the actual tests.
# This lets us cache Babel's output and run it only once.
# Ignore the exit code, since you get an error code if the filter
# excludes all tests.
./build/test.py \
--use-xvfb --browsers Chrome \
--filter ThisFilterMatchesNoTests || true
- name: Cache dependencies
uses: actions/cache@v4
id: npm-cache
with:
path: node_modules/
key: node-${{ hashFiles('package-lock.json') }}
- name: Cache Babel output
uses: actions/cache@v4
id: babel-cache
with:
path: .babel-cache
key: babel-${{ hashFiles('*.js', 'demo/**.js', 'lib/**.js', 'ui/**.js', 'test/**.js', 'third_party/**.js') }}
- name: Store Player build
uses: actions/upload-artifact@v4
with:
name: shaka-player
path: dist/
retention-days: 1
- name: Report final commit status
# Will run on success or failure, but not if the workflow is cancelled
# or if we were asked to ignore the test status.
if: ${{ (success() || failure()) && inputs.skip_commit_status == false }}
uses: ./.github/workflows/custom-actions/set-commit-status
with:
context: Selenium / Build
job_name: "${{ inputs.job_name_prefix }}Pre-build Player"
state: ${{ job.status }}
token: ${{ secrets.GITHUB_TOKEN }}
lab-tests:
# This is a self-hosted runner in a Docker container, with access to our
# lab's Selenium grid on port 4444.
runs-on: self-hosted-selenium
needs: [compute-ref, build-shaka, matrix-config]
strategy:
fail-fast: false
matrix:
include: ${{ fromJSON(needs.matrix-config.outputs.INCLUDE) }}
name: ${{ matrix.browser }}
steps:
- uses: actions/checkout@v4
with:
ref: ${{ needs.compute-ref.outputs.REF }}
- name: Set commit status to pending
if: ${{ inputs.skip_commit_status == false }}
uses: ./.github/workflows/custom-actions/set-commit-status
with:
context: Selenium / ${{ matrix.browser }}
job_name: "${{ inputs.job_name_prefix }}${{ matrix.browser }}"
state: pending
token: ${{ secrets.GITHUB_TOKEN }}
- uses: actions/setup-node@v4
with:
node-version: 16
registry-url: 'https://registry.npmjs.org'
# The Docker image for this self-hosted runner doesn't contain java.
- uses: actions/setup-java@v4
with:
distribution: zulu
java-version: 11
- name: Cache dependencies
uses: actions/cache@v4
id: npm-cache
with:
path: node_modules/
key: node-${{ hashFiles('package-lock.json') }}
fail-on-cache-miss: true # Cached by the build-shaka job above
enableCrossOsArchive: true # Share archives from Linux to Windows
- name: Cache Babel output
uses: actions/cache@v4
id: babel-cache
with:
path: .babel-cache
key: babel-${{ hashFiles('*.js', 'demo/**.js', 'lib/**.js', 'ui/**.js', 'test/**.js', 'third_party/**.js') }}
fail-on-cache-miss: true # Cached by the build-shaka job above
enableCrossOsArchive: true # Share archives from Linux to Windows
- name: Install dependencies
if: steps.npm-cache.outputs.cache-hit != 'true'
run: npm ci
# Instead of building Shaka N times, build it once and fetch the build to
# each Selenium runner in the matrix.
- name: Fetch Player build
uses: actions/download-artifact@v4
with:
name: shaka-player
path: dist/
# Run tests on the Selenium grid in our lab. This uses a private
# hostname and TLS cert to get EME tests working on all platforms
# (since EME only works on https or localhost). The variable
# ALLOCATED_PORT must be defined by the self-hosted runner, and mapped
# from the host to the container.
- name: Test Player
run: |
# Use of an array keeps elements intact, and allows an element to
# contain spaces without being expanded into multiple arguments in a
# shell command.
extra_flags=()
# Generate a coverage report from uncompiled code on ChromeLinux.
# It should be the uncompiled build, or else we won't execute any
# coverage instrumentation on full-stack player integration tests.
if [[ "${{ matrix.browser }}" == "Edge" ]]; then
extra_flags+=(--html-coverage-report --uncompiled)
fi
if [[ "${{ inputs.test_filter }}" != "" ]]; then
echo "Adding filter: ${{ inputs.test_filter }}"
extra_flags+=(--filter "${{ inputs.test_filter }}")
fi
# Do not automatically fail when a command fails. This allows us to
# implement the ignore_test_status input by capturing the exit code
# and examining it.
set +e
# Run the tests with any extra flags.
python3 build/test.py \
--no-build \
--reporters spec --spec-hide-passed \
--tls-key /etc/letsencrypt/live/karma.shakalab.rocks/privkey.pem \
--tls-cert /etc/letsencrypt/live/karma.shakalab.rocks/fullchain.pem \
--hostname karma.shakalab.rocks \
--port $ALLOCATED_PORT \
--grid-config build/shaka-lab.yaml \
--grid-address selenium-grid.lab.shaka:4444 \
--browsers ${{ matrix.browser }} \
"${extra_flags[@]}"
# Capture the test exit code immediately after running the tests.
# There cannot be any other command between test.py and here.
exit_code=$?
# If ignoring test status, treat this as an exit code of 0 (success).
if [[ "${{ inputs.ignore_test_status }}" == "true" ]]; then
exit_code=0
fi
# Report the captured (and possibly overridden) exit status.
exit $exit_code
- name: Find coverage report (Edge only)
id: coverage
# Run even if an earlier step fails, but only on Edge.
if: ${{ always() && matrix.browser == 'Edge' }}
shell: bash
run: |
# Find the path to the coverage report specifically for Chrome on
# Linux. It includes the exact browser version in the path, so it
# will vary. Having a single path will make the artifact zip
# simpler, whereas using a wildcard in the upload step will result
# in a zip file with internal directories.
coverage_report="$( (ls coverage/Edge*/coverage.json || true) | head -1 )"
# Show what's there, for debugging purposes.
ls -l coverage/
if [ -f "$coverage_report" ]; then
echo "Found coverage report: $coverage_report"
echo "coverage_report=$coverage_report" >> $GITHUB_OUTPUT
else
echo "Could not locate coverage report!"
exit 1
fi
- name: Upload coverage report (Edge only)
uses: actions/upload-artifact@v4
# If there's a coverage report, upload it, even if a previous step
# failed.
if: ${{ always() && steps.coverage.outputs.coverage_report }}
with:
# This will create a download called coverage.zip containing only
# coverage.json.
path: ${{ steps.coverage.outputs.coverage_report }}
name: coverage
# Since we've already filtered this step for instances where there is
# an environment variable set for this, the file should definitely be
# there.
if-no-files-found: error
# Upload new screenshots and diffs on failure; ignore if missing
- name: Upload screenshots
uses: actions/upload-artifact@v4
if: ${{ failure() || inputs.ignore_test_status }}
with:
# In this workflow, "browser" is the selenium node name, which can
# contain both browser and OS, such as "ChromeLinux".
name: screenshots-${{ matrix.browser }}
path: |
test/test/assets/screenshots/*/*.png-new
test/test/assets/screenshots/*/*.png-diff
if-no-files-found: ignore
retention-days: 5
- name: Report final commit status
# Will run on success or failure, but not if the workflow is cancelled
# or if we were asked to ignore the test status.
if: ${{ (success() || failure()) && inputs.skip_commit_status == false }}
uses: ./.github/workflows/custom-actions/set-commit-status
with:
context: Selenium / ${{ matrix.browser }}
job_name: "${{ inputs.job_name_prefix }}${{ matrix.browser }}"
state: ${{ job.status }}
token: ${{ secrets.GITHUB_TOKEN }}