Selenium Lab Tests #583
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Selenium Lab Tests | |
on: | |
workflow_dispatch: | |
# Allows for manual triggering on PRs. They should be reviewed first, to | |
# avoid malicious code executing in the lab. | |
inputs: | |
pr: | |
description: "A PR number to build and test in the lab. If empty, will build and test from main." | |
required: false | |
test_filter: | |
description: "A regex filter to run a subset of the tests. If empty, all tests will run." | |
required: false | |
browser_filter: | |
description: "A list of browsers to run the tests. If empty, all browsers will run." | |
required: false | |
workflow_call: | |
# Allows for reuse from other workflows, such as "Update All Screenshots" | |
# workflow. | |
inputs: | |
pr: | |
description: "A PR number to build and test in the lab. If empty, will build and test from main." | |
required: false | |
type: string | |
test_filter: | |
description: "A regex filter to run a subset of the tests. If empty, all tests will run." | |
required: false | |
type: string | |
browser_filter: | |
description: "A list of browsers to run the tests. If empty, all browsers will run." | |
required: false | |
type: string | |
ignore_test_status: | |
description: "If true, ignore test success or failure, never set the commit status, and always upload screenshots." | |
required: false | |
type: boolean | |
schedule: | |
# Runs every night at 2am PST / 10am UTC, testing against the main branch. | |
- cron: '0 10 * * *' | |
# Only one run of this workflow is allowed at a time, since it uses physical | |
# resources in our lab. | |
concurrency: selenium-lab | |
jobs: | |
compute-ref: | |
name: Compute ref | |
runs-on: ubuntu-latest | |
outputs: | |
REF: ${{ steps.compute.outputs.REF }} | |
steps: | |
- name: Compute ref | |
id: compute | |
run: | | |
if [[ "${{ inputs.pr }}" != "" ]]; then | |
LAB_TEST_REF="refs/pull/${{ inputs.pr }}/head" | |
else | |
LAB_TEST_REF="main" | |
fi | |
echo "REF=$LAB_TEST_REF" | tee -a $GITHUB_OUTPUT | |
# Configure the build matrix based on our grid's YAML config. | |
# The matrix contents will be computed by this first job and deserialized | |
# into the second job's config. | |
matrix-config: | |
name: Matrix config | |
needs: compute-ref | |
runs-on: ubuntu-latest | |
outputs: | |
INCLUDE: ${{ steps.configure.outputs.INCLUDE }} | |
steps: | |
- uses: actions/checkout@v3 | |
with: | |
ref: ${{ needs.compute-ref.outputs.REF }} | |
- name: Install dependencies | |
run: npm ci | |
- name: Configure build matrix | |
id: configure | |
shell: node {0} | |
run: | | |
const fs = require('fs'); | |
const yaml = require( | |
'${{ github.workspace }}/node_modules/js-yaml/index.js'); | |
// Convert the input "browser_filter" into a set of strings. Take | |
// care to filter so that the empty string turns into an empty set. | |
const browserFilter = new Set( "${{ inputs.browser_filter }}" | |
.split(/\s+/) | |
.map(x => x.toLowerCase()) | |
.filter(x => !!x) | |
); | |
const gridBrowserYaml = | |
fs.readFileSync('build/shaka-lab.yaml', 'utf8'); | |
const gridBrowserMetadata = yaml.load(gridBrowserYaml); | |
const include = []; | |
for (const name in gridBrowserMetadata) { | |
if (name == 'vars') { | |
// Skip variable defs in the YAML file | |
continue; | |
} | |
// A browser is enabled if it's not disabled and (either the browser | |
// filter is empty or it contains the browser name). | |
const enabled = !gridBrowserMetadata[name].disabled && | |
(browserFilter.size == 0 || | |
browserFilter.has(name.toLowerCase())); | |
if (enabled) { | |
include.push({browser: name}); | |
} | |
} | |
// Output JSON object consumed by the build matrix below. | |
fs.appendFileSync( | |
process.env['GITHUB_OUTPUT'], | |
`INCLUDE=${ JSON.stringify(include) }\n`); | |
// Log the output, for the sake of debugging this script. | |
console.log({include}); | |
# Build Shaka Player once, then distribute that build to the runners in the | |
# build matrix. For N runners, runs N times faster (since all the | |
# self-hosted Selenium jobs are run in containers on one machine). | |
build-shaka: | |
name: Pre-build Player | |
needs: compute-ref | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v3 | |
with: | |
ref: ${{ needs.compute-ref.outputs.REF }} | |
- name: Set commit status to pending | |
if: ${{ inputs.ignore_test_status == false }} | |
uses: ./.github/workflows/custom-actions/set-commit-status | |
with: | |
context: Selenium / Build | |
state: pending | |
token: ${{ secrets.GITHUB_TOKEN }} | |
- name: Build Player | |
run: python3 build/all.py | |
- name: Store Player build | |
uses: actions/upload-artifact@v3 | |
with: | |
name: shaka-player | |
path: dist/ | |
retention-days: 1 | |
- name: Report final commit status | |
# Will run on success or failure, but not if the workflow is cancelled | |
# or if we were asked to ignore the test status. | |
if: ${{ (success() || failure()) && inputs.ignore_test_status == false }} | |
uses: ./.github/workflows/custom-actions/set-commit-status | |
with: | |
context: Selenium / Build | |
state: ${{ job.status }} | |
token: ${{ secrets.GITHUB_TOKEN }} | |
lab-tests: | |
# This is a self-hosted runner in a Docker container, with access to our | |
# lab's Selenium grid on port 4444. | |
runs-on: self-hosted-selenium | |
needs: [compute-ref, build-shaka, matrix-config] | |
strategy: | |
fail-fast: false | |
matrix: | |
include: ${{ fromJSON(needs.matrix-config.outputs.INCLUDE) }} | |
name: ${{ matrix.browser }} | |
steps: | |
- uses: actions/checkout@v3 | |
with: | |
ref: ${{ needs.compute-ref.outputs.REF }} | |
- name: Set commit status to pending | |
if: ${{ inputs.ignore_test_status == false }} | |
uses: ./.github/workflows/custom-actions/set-commit-status | |
with: | |
context: Selenium / ${{ matrix.browser }} | |
state: pending | |
token: ${{ secrets.GITHUB_TOKEN }} | |
- uses: actions/setup-node@v3 | |
with: | |
node-version: 16 | |
registry-url: 'https://registry.npmjs.org' | |
# The Docker image for this self-hosted runner doesn't contain java. | |
- uses: actions/setup-java@v3 | |
with: | |
distribution: zulu | |
java-version: 11 | |
- name: Cache dependencies | |
uses: actions/cache@v3 | |
id: npm-cache | |
with: | |
path: node_modules/ | |
key: node-${{ hashFiles('package-lock.json') }} | |
- name: Install dependencies | |
if: steps.npm-cache.outputs.cache-hit != 'true' | |
run: npm ci | |
# Instead of building Shaka N times, build it once and fetch the build to | |
# each Selenium runner in the matrix. | |
- name: Fetch Player build | |
uses: actions/download-artifact@v3 | |
with: | |
name: shaka-player | |
path: dist/ | |
# Run tests on the Selenium grid in our lab. This uses a private | |
# hostname and TLS cert to get EME tests working on all platforms | |
# (since EME only works on https or localhost). The variable KARMA_PORT | |
# must be defined by the self-hosted runner, and mapped from the host to | |
# the container. | |
- name: Test Player | |
run: | | |
# Use of an array keeps elements intact, and allows an element to | |
# contain spaces without being expanded into multiple arguments in a | |
# shell command. | |
extra_flags=() | |
# Generate a coverage report from uncompiled code on ChromeLinux. | |
# It should be the uncompiled build, or else we won't execute any | |
# coverage instrumentation on full-stack player integration tests. | |
if [[ "${{ matrix.browser }}" == "ChromeLinux" ]]; then | |
extra_flags+=(--html-coverage-report --uncompiled) | |
fi | |
if [[ "${{ inputs.test_filter }}" != "" ]]; then | |
echo "Adding filter: ${{ inputs.test_filter }}" | |
extra_flags+=(--filter "${{ inputs.test_filter }}") | |
fi | |
# Do not automatically fail when a command fails. This allows us to | |
# implement the ignore_test_status input by capturing the exit code | |
# and examining it. | |
set +e | |
# Run the tests with any extra flags. | |
python3 build/test.py \ | |
--no-build \ | |
--reporters spec --spec-hide-passed \ | |
--lets-encrypt-folder /etc/shakalab.rocks \ | |
--hostname karma.shakalab.rocks \ | |
--port $KARMA_PORT \ | |
--grid-config build/shaka-lab.yaml \ | |
--grid-address selenium-grid.lab:4444 \ | |
--browsers ${{ matrix.browser }} \ | |
"${extra_flags[@]}" | |
# Capture the test exit code immediately after running the tests. | |
# There cannot be any other command between test.py and here. | |
exit_code=$? | |
# If ignoring test status, treat this as an exit code of 0 (success). | |
if [[ "${{ inputs.ignore_test_status }}" == "true" ]]; then | |
exit_code=0 | |
fi | |
# Report the captured (and possibly overridden) exit status. | |
exit $exit_code | |
- name: Find coverage report (ChromeLinux only) | |
id: coverage | |
# Run even if an earlier step fails, but only on ChromeLinux. | |
if: ${{ always() && matrix.browser == 'ChromeLinux' }} | |
shell: bash | |
run: | | |
# Find the path to the coverage report specifically for Chrome on | |
# Linux. It includes the exact browser version in the path, so it | |
# will vary. Having a single path will make the artifact zip | |
# simpler, whereas using a wildcard in the upload step will result | |
# in a zip file with internal directories. | |
coverage_report="$( (ls coverage/Chrome*Linux*/coverage.json || true) | head -1 )" | |
# Show what's there, for debugging purposes. | |
ls -l coverage/ | |
if [ -f "$coverage_report" ]; then | |
echo "Found coverage report: $coverage_report" | |
echo "coverage_report=$coverage_report" >> $GITHUB_OUTPUT | |
else | |
echo "Could not locate coverage report!" | |
exit 1 | |
fi | |
- name: Upload coverage report (ChromeLinux only) | |
uses: actions/upload-artifact@v3 | |
# If there's a coverage report, upload it, even if a previous step | |
# failed. | |
if: ${{ always() && steps.coverage.outputs.coverage_report }} | |
with: | |
# This will create a download called coverage.zip containing only | |
# coverage.json. | |
path: ${{ steps.coverage.outputs.coverage_report }} | |
name: coverage | |
# Since we've already filtered this step for instances where there is | |
# an environment variable set for this, the file should definitely be | |
# there. | |
if-no-files-found: error | |
# Upload new screenshots and diffs on failure; ignore if missing | |
- name: Upload screenshots | |
uses: actions/upload-artifact@v3 | |
if: ${{ failure() || inputs.ignore_test_status }} | |
with: | |
# In this workflow, "browser" is the selenium node name, which can | |
# contain both browser and OS, such as "ChromeLinux". | |
name: screenshots-${{ matrix.browser }} | |
path: | | |
test/test/assets/screenshots/*/*.png-new | |
test/test/assets/screenshots/*/*.png-diff | |
if-no-files-found: ignore | |
retention-days: 5 | |
- name: Report final commit status | |
# Will run on success or failure, but not if the workflow is cancelled | |
# or if we were asked to ignore the test status. | |
if: ${{ (success() || failure()) && inputs.ignore_test_status == false }} | |
uses: ./.github/workflows/custom-actions/set-commit-status | |
with: | |
context: Selenium / ${{ matrix.browser }} | |
state: ${{ job.status }} | |
token: ${{ secrets.GITHUB_TOKEN }} |