chore(ci): e2e tests are run on CI

.github/workflows/actions/build-core/action.yml

@@ -26,4 +26,9 @@ runs:
       with:
         name: ionicons-build
         output: IoniconsBuild.zip
-        paths: dist components icons
+        paths: dist components icons www
+    - uses: ./.github/workflows/actions/upload-archive
+      with:
+        name: ionicons-src
+        output: IoniconsSrc.zip
+        paths: src

.github/workflows/actions/test-e2e/action.yml

@@ -0,0 +1,86 @@
+name: 'Test E2E'
+description: 'Test E2E'
+inputs:
+  shard:
+    description: 'Playwright Test Shard (ex: 2)'
+  totalShards:
+    description: 'Playwright total number of test shards (ex: 4)'
+  update:
+    description: 'Whether or not to update the reference snapshots'
+runs:
+  using: 'composite'
+  steps:
+    - uses: actions/setup-node@v3
+
+    - name: Cache Node Modules
+      uses: actions/cache@v3
+      env:
+        cache-name: node-modules
+      with:
+        path: ./node_modules
+        key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('./package-lock.json') }}-v1
+    - uses: ./.github/workflows/actions/download-archive
+      with:
+        name: ionicons-build
+        path: .
+        filename: IoniconsBuild.zip
+    - uses: ./.github/workflows/actions/download-archive
+      with:
+        name: ionicons-src
+        path: .
+        filename: IoniconsSrc.zip
+    - name: Install Playwright Dependencies
+      run: npx playwright install && npx playwright install-deps
+      shell: bash
+    - name: Test
+      if: inputs.update != 'true'
+      run: npx playwright test --shard=${{ inputs.shard }}/${{ inputs.totalShards }}
+      shell: bash
+    - name: Test and Update
+      id: test-and-update
+      if: inputs.update == 'true'
+      # Keep track of the files that were
+      # changed so they can be correctly restored
+      # in the combine step.
+      # To do this, we move only the changed files
+      # to a separate directory, while preserving the
+      # directory structure of the source.
+      # When, we create and archive of these results
+      # so that the combine step can simply
+      # unzip and move the changed files into place.
+      # We have extra logic added so that job runners
+      # that do not have any new screenshots do not create
+      # an unnecessary .zip.
+      run: |
+        npx playwright test --shard=${{ inputs.shard }}/${{ inputs.totalShards }} --update-snapshots
+        git add src/\*.png --force
+        mkdir updated-screenshots
+        echo 'made dir'
+        rsync -R --progress $(git diff --name-only --cached) updated-screenshots
+        echo 'copied images'
+        if [ -d updated-screenshots ]; then
+          echo "::set-output name=hasUpdatedScreenshots::$(echo 'true')"
+          cd updated-screenshots
+          ls
+          zip -q -r ../UpdatedScreenshots-${{ inputs.shard }}-${{ inputs.totalShards }}.zip ./
+        fi
+      shell: bash
+    - name: Archive Updated Screenshots
+      if: inputs.update == 'true' && steps.test-and-update.outputs.hasUpdatedScreenshots == 'true'
+      uses: actions/upload-artifact@v3
+      with:
+        name: updated-screenshots-${{ inputs.shard }}-${{ inputs.totalShards }}
+        path: UpdatedScreenshots-${{ inputs.shard }}-${{ inputs.totalShards }}.zip
+    - name: Archive Test Results
+      # The always() ensures that this step
+      # runs even if the previous step fails.
+      # We want the test results to be archived
+      # even if the test fails in the previous
+      # step, otherwise there would be no way
+      # to debug these tests.
+      if: always()
+      uses: ./.github/workflows/actions/upload-archive
+      with:
+        name: test-results-${{ inputs.shard }}-${{ inputs.totalShards }}
+        output: TestResults-${{ inputs.shard }}-${{ inputs.totalShards }}.zip
+        paths: playwright-report

.github/workflows/actions/update-reference-screenshots/action.yml

@@ -0,0 +1,38 @@
+name: 'Update Reference Screenshots'
+description: 'Update Reference Screenshots'
+
+on:
+  workflow_dispatch:
+
+runs:
+  using: 'composite'
+  steps:
+    - uses: actions/setup-node@v3
+    - uses: actions/download-artifact@v3
+      with:
+        path: ./artifacts
+    - name: Extract Archives
+      # This finds all .zip files in the ./artifacts
+      # directory, including nested directories.
+      # It then unzips every .zip to the root directory
+      run: |
+        find . -type f -name 'UpdatedScreenshots-*.zip' -exec unzip -q -o -d ../ {} \;
+      shell: bash
+      working-directory: ./artifacts
+    - name: Push Screenshots
+      # Configure user as Ionitron
+      # and push only the changed .png snapshots
+      # to the remote branch.
+      # Screenshots are in .gitignore
+      # to prevent local screenshots from getting
+      # pushed to Git. As a result, we need --force
+      # here so that CI generated screenshots can
+      # get added to git. Screenshot ground truths
+      # should only be added via this CI process.
+      run: |
+        git config user.name ionitron
+        git config user.email hi@ionicframework.com
+        git add src/\*.png --force
+        git commit -m "chore(): add updated snapshots"
+        git push
+      shell: bash

.github/workflows/build.yml

@@ -26,4 +26,46 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: ./.github/workflows/actions/test-spec
+      - uses: ./.github/workflows/actions/test-spec
+
+  test-e2e:
+    strategy:
+      # This ensures that all screenshot shard
+      # failures are reported so the dev can
+      # review everything at once.
+      fail-fast: false
+      matrix:
+        # Divide the tests into n buckets
+        # and run those buckets in parallel.
+        # To increase the number of shards,
+        # add new items to the shard array
+        # and change the value of totalShards
+        # to be the length of the shard array.
+        shard: [1, 2, 3]
+        totalShards: [3]
+    needs: [build-core]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/workflows/actions/test-e2e
+        with:
+          shard: ${{ matrix.shard }}
+          totalShards: ${{ matrix.totalShards }}
+
+  # Screenshots are required to pass
+  # in order for the branch to be merge
+  # eligible. However, the screenshot tests
+  # are run on n runners where n can change
+  # over time. The verify-screenshots step allows
+  # us to have a required status check for screenshot
+  # results without having to manually add each
+  # matrix run in the branch protection rules
+  # Source: https://github.community/t/status-check-for-a-matrix-jobs/127354
+  verify-e2e:
+    if: ${{ always() }}
+    needs: test-e2e
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check build matrix status
+        if: ${{ needs.test-e2e.result != 'success' }}
+        run: exit 1

.github/workflows/update-screenshots.yml

@@ -1,11 +1,52 @@
-name: 'Update Screenshot References'
+name: 'Update Reference Screenshots'
 
 on:
   workflow_dispatch:
 
 jobs:
-  stub:
+  build-core:
+    runs-on: ubuntu-latest
     steps:
-      - name: Stub
-        run: echo 'This is a stub'
-        shell: bash
+      - uses: actions/checkout@v3
+      - uses: ./.github/workflows/actions/build-core
+
+  test-e2e:
+    strategy:
+      # This ensures that all screenshot shard
+      # failures are reported so the dev can
+      # review everything at once.
+      fail-fast: false
+      matrix:
+        # Divide the tests into n buckets
+        # and run those buckets in parallel.
+        # To increase the number of shards,
+        # add new items to the shard array
+        # and change the value of totalShards
+        # to be the length of the shard array.
+        shard: [1, 2, 3]
+        totalShards: [3]
+    needs: [build-core]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/workflows/actions/test-e2e
+        with:
+          shard: ${{ matrix.shard }}
+          totalShards: ${{ matrix.totalShards }}
+          update: true
+
+  update-reference-screenshots:
+    runs-on: ubuntu-latest
+    needs: [test-e2e]
+    steps:
+      - uses: actions/checkout@v2
+        # Normally, we could just push with the
+        # default GITHUB_TOKEN, but that will
+        # not cause the build workflow
+        # to re-run. We use Ionitron's
+        # Personal Access Token instead
+        # to allow for this build workflow
+        # to run when the screenshots are pushed.
+        with:
+          token: ${{ secrets.IONITRON_TOKEN }}
+      - uses: ./.github/workflows/actions/update-reference-screenshots

playwright.config.ts

@@ -88,7 +88,7 @@ const config: PlaywrightTestConfig = {
   /* Configure projects for major browsers */
   projects: projects,
   webServer: {
-    command: 'serve -p 3333',
+    command: 'serve www -p 3333',
     port: 3333,
     reuseExistingServer: !process.env.CI
   }

src/components/icon/test/icon.e2e.ts

@@ -4,6 +4,9 @@ import { test } from '@utils/test/playwright';
 test.describe('icon: basic', () => {
   test('should not have visual regressions', async ({ page }) => {
     await page.goto(`/`);
+
+    // Wait for all SVGs to be lazily loaded before taking screenshots
+    await page.waitForLoadState('networkidle');
 
     expect(await page.screenshot({ fullPage: true })).toMatchSnapshot(`icon-diff.png`);
   });