diff --git a/.github/workflows/actions/build-core/action.yml b/.github/workflows/actions/build-core/action.yml index e45f1e107..f48a273fa 100644 --- a/.github/workflows/actions/build-core/action.yml +++ b/.github/workflows/actions/build-core/action.yml @@ -26,4 +26,9 @@ runs: with: name: ionicons-build output: IoniconsBuild.zip - paths: dist components icons \ No newline at end of file + paths: dist components icons www + - uses: ./.github/workflows/actions/upload-archive + with: + name: ionicons-src + output: IoniconsSrc.zip + paths: src \ No newline at end of file diff --git a/.github/workflows/actions/test-e2e/action.yml b/.github/workflows/actions/test-e2e/action.yml new file mode 100644 index 000000000..447c61eee --- /dev/null +++ b/.github/workflows/actions/test-e2e/action.yml @@ -0,0 +1,86 @@ +name: 'Test E2E' +description: 'Test E2E' +inputs: + shard: + description: 'Playwright Test Shard (ex: 2)' + totalShards: + description: 'Playwright total number of test shards (ex: 4)' + update: + description: 'Whether or not to update the reference snapshots' +runs: + using: 'composite' + steps: + - uses: actions/setup-node@v3 + + - name: Cache Node Modules + uses: actions/cache@v3 + env: + cache-name: node-modules + with: + path: ./node_modules + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('./package-lock.json') }}-v1 + - uses: ./.github/workflows/actions/download-archive + with: + name: ionicons-build + path: . + filename: IoniconsBuild.zip + - uses: ./.github/workflows/actions/download-archive + with: + name: ionicons-src + path: . + filename: IoniconsSrc.zip + - name: Install Playwright Dependencies + run: npx playwright install && npx playwright install-deps + shell: bash + - name: Test + if: inputs.update != 'true' + run: npx playwright test --shard=${{ inputs.shard }}/${{ inputs.totalShards }} + shell: bash + - name: Test and Update + id: test-and-update + if: inputs.update == 'true' + # Keep track of the files that were + # changed so they can be correctly restored + # in the combine step. + # To do this, we move only the changed files + # to a separate directory, while preserving the + # directory structure of the source. + # When, we create and archive of these results + # so that the combine step can simply + # unzip and move the changed files into place. + # We have extra logic added so that job runners + # that do not have any new screenshots do not create + # an unnecessary .zip. + run: | + npx playwright test --shard=${{ inputs.shard }}/${{ inputs.totalShards }} --update-snapshots + git add src/\*.png --force + mkdir updated-screenshots + echo 'made dir' + rsync -R --progress $(git diff --name-only --cached) updated-screenshots + echo 'copied images' + if [ -d updated-screenshots ]; then + echo "::set-output name=hasUpdatedScreenshots::$(echo 'true')" + cd updated-screenshots + ls + zip -q -r ../UpdatedScreenshots-${{ inputs.shard }}-${{ inputs.totalShards }}.zip ./ + fi + shell: bash + - name: Archive Updated Screenshots + if: inputs.update == 'true' && steps.test-and-update.outputs.hasUpdatedScreenshots == 'true' + uses: actions/upload-artifact@v3 + with: + name: updated-screenshots-${{ inputs.shard }}-${{ inputs.totalShards }} + path: UpdatedScreenshots-${{ inputs.shard }}-${{ inputs.totalShards }}.zip + - name: Archive Test Results + # The always() ensures that this step + # runs even if the previous step fails. + # We want the test results to be archived + # even if the test fails in the previous + # step, otherwise there would be no way + # to debug these tests. + if: always() + uses: ./.github/workflows/actions/upload-archive + with: + name: test-results-${{ inputs.shard }}-${{ inputs.totalShards }} + output: TestResults-${{ inputs.shard }}-${{ inputs.totalShards }}.zip + paths: playwright-report \ No newline at end of file diff --git a/.github/workflows/actions/update-reference-screenshots/action.yml b/.github/workflows/actions/update-reference-screenshots/action.yml new file mode 100644 index 000000000..447f15dc5 --- /dev/null +++ b/.github/workflows/actions/update-reference-screenshots/action.yml @@ -0,0 +1,38 @@ +name: 'Update Reference Screenshots' +description: 'Update Reference Screenshots' + +on: + workflow_dispatch: + +runs: + using: 'composite' + steps: + - uses: actions/setup-node@v3 + - uses: actions/download-artifact@v3 + with: + path: ./artifacts + - name: Extract Archives + # This finds all .zip files in the ./artifacts + # directory, including nested directories. + # It then unzips every .zip to the root directory + run: | + find . -type f -name 'UpdatedScreenshots-*.zip' -exec unzip -q -o -d ../ {} \; + shell: bash + working-directory: ./artifacts + - name: Push Screenshots + # Configure user as Ionitron + # and push only the changed .png snapshots + # to the remote branch. + # Screenshots are in .gitignore + # to prevent local screenshots from getting + # pushed to Git. As a result, we need --force + # here so that CI generated screenshots can + # get added to git. Screenshot ground truths + # should only be added via this CI process. + run: | + git config user.name ionitron + git config user.email hi@ionicframework.com + git add src/\*.png --force + git commit -m "chore(): add updated snapshots" + git push + shell: bash \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0d83d4a2d..f386a61a1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,4 +26,46 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: ./.github/workflows/actions/test-spec \ No newline at end of file + - uses: ./.github/workflows/actions/test-spec + + test-e2e: + strategy: + # This ensures that all screenshot shard + # failures are reported so the dev can + # review everything at once. + fail-fast: false + matrix: + # Divide the tests into n buckets + # and run those buckets in parallel. + # To increase the number of shards, + # add new items to the shard array + # and change the value of totalShards + # to be the length of the shard array. + shard: [1, 2, 3] + totalShards: [3] + needs: [build-core] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: ./.github/workflows/actions/test-e2e + with: + shard: ${{ matrix.shard }} + totalShards: ${{ matrix.totalShards }} + + # Screenshots are required to pass + # in order for the branch to be merge + # eligible. However, the screenshot tests + # are run on n runners where n can change + # over time. The verify-screenshots step allows + # us to have a required status check for screenshot + # results without having to manually add each + # matrix run in the branch protection rules + # Source: https://github.community/t/status-check-for-a-matrix-jobs/127354 + verify-e2e: + if: ${{ always() }} + needs: test-e2e + runs-on: ubuntu-latest + steps: + - name: Check build matrix status + if: ${{ needs.test-e2e.result != 'success' }} + run: exit 1 \ No newline at end of file diff --git a/.github/workflows/update-screenshots.yml b/.github/workflows/update-screenshots.yml index a44c47436..cf52f88e6 100644 --- a/.github/workflows/update-screenshots.yml +++ b/.github/workflows/update-screenshots.yml @@ -1,11 +1,52 @@ -name: 'Update Screenshot References' +name: 'Update Reference Screenshots' on: workflow_dispatch: jobs: - stub: + build-core: + runs-on: ubuntu-latest steps: - - name: Stub - run: echo 'This is a stub' - shell: bash \ No newline at end of file + - uses: actions/checkout@v3 + - uses: ./.github/workflows/actions/build-core + + test-e2e: + strategy: + # This ensures that all screenshot shard + # failures are reported so the dev can + # review everything at once. + fail-fast: false + matrix: + # Divide the tests into n buckets + # and run those buckets in parallel. + # To increase the number of shards, + # add new items to the shard array + # and change the value of totalShards + # to be the length of the shard array. + shard: [1, 2, 3] + totalShards: [3] + needs: [build-core] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: ./.github/workflows/actions/test-e2e + with: + shard: ${{ matrix.shard }} + totalShards: ${{ matrix.totalShards }} + update: true + + update-reference-screenshots: + runs-on: ubuntu-latest + needs: [test-e2e] + steps: + - uses: actions/checkout@v2 + # Normally, we could just push with the + # default GITHUB_TOKEN, but that will + # not cause the build workflow + # to re-run. We use Ionitron's + # Personal Access Token instead + # to allow for this build workflow + # to run when the screenshots are pushed. + with: + token: ${{ secrets.IONITRON_TOKEN }} + - uses: ./.github/workflows/actions/update-reference-screenshots \ No newline at end of file diff --git a/playwright.config.ts b/playwright.config.ts index 0294b5848..0fd6b2391 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -88,7 +88,7 @@ const config: PlaywrightTestConfig = { /* Configure projects for major browsers */ projects: projects, webServer: { - command: 'serve -p 3333', + command: 'serve www -p 3333', port: 3333, reuseExistingServer: !process.env.CI } diff --git a/src/components/icon/test/icon.e2e.ts b/src/components/icon/test/icon.e2e.ts index b3001897d..9f39ba950 100644 --- a/src/components/icon/test/icon.e2e.ts +++ b/src/components/icon/test/icon.e2e.ts @@ -4,6 +4,9 @@ import { test } from '@utils/test/playwright'; test.describe('icon: basic', () => { test('should not have visual regressions', async ({ page }) => { await page.goto(`/`); + + // Wait for all SVGs to be lazily loaded before taking screenshots + await page.waitForLoadState('networkidle'); expect(await page.screenshot({ fullPage: true })).toMatchSnapshot(`icon-diff.png`); }); diff --git a/src/components/icon/test/icon.e2e.ts-snapshots/icon-diff-Mobile-Chrome-linux.png b/src/components/icon/test/icon.e2e.ts-snapshots/icon-diff-Mobile-Chrome-linux.png new file mode 100644 index 000000000..0bf4bea85 Binary files /dev/null and b/src/components/icon/test/icon.e2e.ts-snapshots/icon-diff-Mobile-Chrome-linux.png differ diff --git a/src/components/icon/test/icon.e2e.ts-snapshots/icon-diff-Mobile-Firefox-linux.png b/src/components/icon/test/icon.e2e.ts-snapshots/icon-diff-Mobile-Firefox-linux.png new file mode 100644 index 000000000..b47c37950 Binary files /dev/null and b/src/components/icon/test/icon.e2e.ts-snapshots/icon-diff-Mobile-Firefox-linux.png differ diff --git a/src/components/icon/test/icon.e2e.ts-snapshots/icon-diff-Mobile-Safari-linux.png b/src/components/icon/test/icon.e2e.ts-snapshots/icon-diff-Mobile-Safari-linux.png new file mode 100644 index 000000000..b1a97673c Binary files /dev/null and b/src/components/icon/test/icon.e2e.ts-snapshots/icon-diff-Mobile-Safari-linux.png differ