From 94b14dcc854cd7ce731ff4ae4a67a986db622b75 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 12 Aug 2022 10:56:16 -0700 Subject: [PATCH] test: Upload coverage report for PR and nightly tests (#4390) This will attach a code coverage report to PRs and to nightly tests, both specifically from Chrome on Linux. Reports will be attached to workflow runs whether or not the tests pass. --- .github/workflows/build-and-test.yaml | 80 +++++++++++++++++++---- .github/workflows/selenium-lab-tests.yaml | 45 ++++++++++++- karma.conf.js | 9 +-- 3 files changed, 113 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml index 3ba88957ce..9cb36c3df9 100644 --- a/.github/workflows/build-and-test.yaml +++ b/.github/workflows/build-and-test.yaml @@ -38,23 +38,34 @@ jobs: needs: lint strategy: matrix: - os: ["ubuntu-latest", "macos-latest", "windows-latest"] - browser: ["Chrome", "Firefox", "Edge", "Safari", "Safari-14"] - exclude: - - os: ubuntu-latest - browser: Edge - - os: windows-latest - browser: Safari - - os: windows-latest - browser: Safari-14 - - os: ubuntu-latest - browser: Safari - - os: ubuntu-latest - browser: Safari-14 include: # Run Linux browsers with xvfb, so they're in a headless X session. + # Additionally, generate a code coverage report from Linux Chrome. + - os: ubuntu-latest + browser: Chrome + extra_flags: "--use-xvfb --html-coverage-report" - os: ubuntu-latest + browser: Firefox extra_flags: "--use-xvfb" + + - os: macos-latest + browser: Chrome + - os: macos-latest + browser: Firefox + - os: macos-latest + browser: Edge + - os: macos-latest + browser: Safari + - os: macos-latest + browser: Safari-14 + + - os: windows-latest + browser: Chrome + - os: windows-latest + browser: Firefox + - os: windows-latest + browser: Edge + # Disable fail-fast so that one matrix-job failing doesn't make the other # ones end early. fail-fast: false @@ -114,6 +125,49 @@ jobs: --reporters spec --spec-hide-passed \ ${{ matrix.extra_flags }} + - name: Find coverage report + id: coverage + if: always() # Even on failure of an earlier step. + shell: bash + run: | + # If the directory exists... + if [ -d coverage ]; then + # Find the path to the coverage report. It includes the exact + # browser version in the path, so it will vary. Having a single + # path will make the artifact zip simpler, whereas using a wildcard + # in the upload step will result in a zip file with internal + # directories. In case there are multiple folders (there shouldn't + # be), this shell script will extract just a single path. + coverage_report="$( (ls coverage/*/coverage.json || true) | head -1 )" + + # Show what's there, for debugging purposes. + ls -l coverage/ + + if [ -f "$coverage_report" ]; then + echo "Found coverage report: $coverage_report" + echo "::set-output name=coverage_report::$coverage_report" + else + echo "Could not locate coverage report!" + exit 1 + fi + else + echo "No coverage report generated." + fi + + - uses: actions/upload-artifact@v3 + # If there's a coverage report, upload it, even if a previous step + # failed. + if: ${{ always() && steps.coverage.outputs.coverage_report }} + with: + # This will create a download called coverage.zip containing only + # coverage.json. + path: ${{ steps.coverage.outputs.coverage_report }} + name: coverage + # Since we've already filtered this step for instances where there is + # an environment variable set for this, the file should definitely be + # there. + if-no-files-found: error + build_in_docker: # Don't waste time doing a full matrix of test runs when there was an # obvious linter error. diff --git a/.github/workflows/selenium-lab-tests.yaml b/.github/workflows/selenium-lab-tests.yaml index 88567a3d0f..294c2f0155 100644 --- a/.github/workflows/selenium-lab-tests.yaml +++ b/.github/workflows/selenium-lab-tests.yaml @@ -80,7 +80,50 @@ jobs: --hostname karma.shakalab.rocks \ --port 61731 \ --grid-config build/shaka-lab.yaml \ - --grid-address selenium-grid.lab:4444 + --grid-address selenium-grid.lab:4444 \ + --html-coverage-report + + - name: Find coverage report + id: coverage + if: always() # Even on failure of an earlier step. + shell: bash + run: | + # If the directory exists... + if [ -d coverage ]; then + # Find the path to the coverage report specifically for Chrome on + # Linux. It includes the exact browser version in the path, so it + # will vary. Having a single path will make the artifact zip + # simpler, whereas using a wildcard in the upload step will result + # in a zip file with internal directories. + coverage_report="$( (ls coverage/Chrome*Linux*/coverage.json || true) | head -1 )" + + # Show what's there, for debugging purposes. + ls -l coverage/ + + if [ -f "$coverage_report" ]; then + echo "Found coverage report: $coverage_report" + echo "::set-output name=coverage_report::$coverage_report" + else + echo "Could not locate coverage report!" + exit 1 + fi + else + echo "No coverage report generated." + fi + + - uses: actions/upload-artifact@v3 + # If there's a coverage report, upload it, even if a previous step + # failed. + if: ${{ always() && steps.coverage.outputs.coverage_report }} + with: + # This will create a download called coverage.zip containing only + # coverage.json. + path: ${{ steps.coverage.outputs.coverage_report }} + name: coverage + # Since we've already filtered this step for instances where there is + # an environment variable set for this, the file should definitely be + # there. + if-no-files-found: error - name: Report Final Commit Status # Will run on success or failure, but not if the workflow is cancelled. diff --git a/karma.conf.js b/karma.conf.js index 2c3b7902af..ebc9217ec4 100644 --- a/karma.conf.js +++ b/karma.conf.js @@ -302,13 +302,6 @@ module.exports = (config) => { // Force failure when running empty test-suites. failOnEmptyTestSuite: true, - coverageReporter: { - includeAllSources: true, - reporters: [ - {type: 'text'}, - ], - }, - specReporter: { suppressSkipped: true, showBrowser: true, @@ -401,9 +394,11 @@ module.exports = (config) => { config.set({ coverageReporter: { + includeAllSources: true, reporters: [ {type: 'html', dir: 'coverage'}, {type: 'cobertura', dir: 'coverage', file: 'coverage.xml'}, + {type: 'json-summary', dir: 'coverage', file: 'coverage.json'}, ], }, });