Setup visual and a11y tests (#21)

.github/workflows/tests.yml

@@ -11,6 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Cancel previous runs
+        if: github.ref != 'refs/heads/main'
         uses: styfle/cancel-workflow-action@0.9.0
         with:
           access_token: ${{ github.token }}
@@ -41,6 +42,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Cancel previous runs
+        if: github.ref != 'refs/heads/main'
         uses: styfle/cancel-workflow-action@0.9.0
         with:
           access_token: ${{ github.token }}
@@ -67,19 +69,61 @@ jobs:
         if: failure()
         uses: actions/upload-artifact@v2
         with:
-          name: failure-screenshots-trace
+          name: failure-screenshots-trace-e2e
+          path: test-results
+          retention-days: 10
+
+  visual_tests:
+    name: Visual Regression tests
+    runs-on: ubuntu-latest
+    env:
+      HAPPO_API_KEY: ${{ secrets.HAPPO_API_KEY }}
+      HAPPO_API_SECRET: ${{ secrets.HAPPO_API_SECRET }}
+    steps:
+      - name: Cancel previous runs
+        if: github.ref != 'refs/heads/main'
+        uses: styfle/cancel-workflow-action@0.9.0
+        with:
+          access_token: ${{ github.token }}
+
+      - uses: actions/checkout@v3
+
+      - name: Setup
+        uses: ./.github/actions/setup
+
+      - name: Run tests
+        run: yarn test:visual:ci
+        env:
+          FORCE_COLOR: 2
+
+      - name: Upload Allure report
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: allure-results-visual
+          path: allure-results
+          retention-days: 10
+
+      - name: Save screenshots and trace in case of failure
+        if: failure()
+        uses: actions/upload-artifact@v2
+        with:
+          name: failure-screenshots-trace-visual
           path: test-results
           retention-days: 10
 
   deploy_report:
     name: Deploy report to GH Pages
-    needs: [api_tests, e2e_tests]
+    needs: [api_tests, e2e_tests, visual_tests]
     if: always() && github.ref == 'refs/heads/main'
     runs-on: ubuntu-latest
     permissions:
       contents: read
       pages: write
       id-token: write
+    concurrency:
+      group: pages
+      cancel-in-progress: true
     environment:
       name: github-pages
       url: ${{ steps.deployment.outputs.page_url }}
@@ -96,6 +140,12 @@ jobs:
           name: allure-results-e2e
           path: allure-results
 
+      - name: Download Allure reports from Visual Regression tests
+        uses: actions/download-artifact@v3
+        with:
+          name: allure-results-visual
+          path: allure-results
+
       - name: Get Allure history
         uses: actions/checkout@v3
         if: always()
@@ -124,7 +174,9 @@ jobs:
           name: |
             allure-results-api
             allure-results-e2e
-            failure-screenshots-trace
+            allure-results-visual
+            failure-screenshots-trace-e2e
+            failure-screenshots-trace-visual
           failOnError: false
 
       - name: Upload pages artifact

.happo.js

@@ -0,0 +1,26 @@
+const { RemoteBrowserTarget } = require('happo.io');
+
+module.exports = {
+  apiKey: process.env.HAPPO_API_KEY,
+  apiSecret: process.env.HAPPO_API_SECRET,
+  compareThreshold: 0.005,
+  project: 'demo-playwright-test',
+  targets: {
+    chrome: new RemoteBrowserTarget('chrome', {
+      viewport: '1440x900'
+    }),
+    firefox: new RemoteBrowserTarget('firefox', {
+      viewport: '1440x900'
+    }),
+    safari: new RemoteBrowserTarget('safari', {
+      viewport: '1440x900',
+      scrollStitch: true
+    }),
+    iosSafari: new RemoteBrowserTarget('ios-safari', {
+      viewport: '320x568'
+    }),
+    ipadSafari: new RemoteBrowserTarget('ipad-safari', {
+      viewport: '1080x810'
+    })
+  }
+}

README.md

@@ -31,6 +31,8 @@ demo-playwright-test/
  │         ├── login.api.test.js            # API tests
  │    ├── e2e
  │         ├── create-user.e2e.test.js      # End-to-end tests
+ │    ├── visual
+ │         ├── login.visual.test.js         # Visual regression tests
  ├── playwright.config.js                   # Playwright configuration file
 ```
 
@@ -42,6 +44,15 @@ demo-playwright-test/
 - Start Serverest server: `yarn api:start`
 - Run API tests: `yarn test:api`
 - Run End-to-end tests: `yarn test:e2e`
+- Run Visual Regression tests: `yarn test:visual` (you will need a Happo account and `HAPPO_API_KEY`/`HAPPO_API_SECRET` environment variables set)
+
+### Debugging
+
+To run Playwright in debug mode, pass the `PWDEBUG=1` environment variable in the command, for example: `PWDEBUG=1 yarn test:e2e`
+
+When a test fails, the project is configured to save screenshots and a trace file, inside `test-reports` folder. You can run [Playwright's Trace Viewer](https://playwright.dev/docs/trace-viewer) with `show-trace` command: `yarn playwright show-trace test-results/some-test-path/trace.zip`
+
+Please refer to [Playwright's Debugging docs](https://playwright.dev/docs/debug) for further information on debugging features.
 
 ### Tips
 
@@ -59,3 +70,9 @@ Test reports can be generated with [Allure reports](https://github.com/allure-fr
 ## CI
 
 The project uses [GitHub Actions](https://docs.github.com/en/actions) and tests are run automatically on PRs and on merge to `main` branch.
+
+---
+
+## Contributing
+
+We have a [Kanban board](https://github.com/stefanteixeira/demo-playwright-test/projects/1) with a backlog of tasks to work on. If you are interested in contributing to the project, please reach out to @stefanteixeira to become a collaborator and get a task assigned to you.

lib/a11y.js

@@ -0,0 +1,14 @@
+const { injectAxe, checkA11y } = require('axe-playwright')
+
+const testA11y = async (page, context = undefined) => {
+  await injectAxe(page)
+
+  await checkA11y(page, context, {
+    detailedReport: true,
+    detailedReportOptions: { html: true }
+  })
+}
+
+module.exports = {
+  testA11y
+}

lib/helpers.js

@@ -56,6 +56,12 @@ const createUser = async page => {
   await page.click('data-testid=cadastrar')
 }
 
+const openHomepage = async page => {
+  await page.goto('/cadastrarusuarios')
+  await createUser(page)
+  await page.waitForNavigation()
+}
+
 const getAuthToken = async request => {
   const body = getUserBody()
   const loginBody = {
@@ -77,5 +83,6 @@ module.exports = {
   getCartBody,
   getCartId,
   createUser,
+  openHomepage,
   getAuthToken
 }

package.json

@@ -11,9 +11,13 @@
     "@playwright/test": "^1.22.1",
     "allure-commandline": "^2.17.2",
     "allure-playwright": "^2.0.0-beta.16",
+    "axe-playwright": "^1.1.11",
     "eslint": "^8.15.0",
     "eslint-plugin-playwright": "^0.9.0",
     "expect-playwright": "^0.8.0",
+    "happo-e2e": "^1.2.0",
+    "happo-playwright": "^1.1.0",
+    "happo.io": "^7.2.1",
     "playwright": "^1.22.1",
     "wait-on": "^6.0.1"
   },
@@ -26,6 +30,8 @@
     "test:api:ci:wait": "wait-on -i 1000 -t 30000 -l http://localhost:3000",
     "test:api:ci": "yarn test:api:ci:wait && yarn test:api --workers 2",
     "test:e2e": "yarn playwright test --project e2e --config playwright.config.js",
-    "test:e2e:ci": "yarn test:e2e --workers 2"
+    "test:e2e:ci": "yarn test:e2e --workers 2",
+    "test:visual": "HAPPO_DOWNLOAD_ALL=true yarn happo-e2e -- -- yarn playwright test --project visual --config playwright.config.js",
+    "test:visual:ci": "yarn test:visual --workers 2"
   }
 }

playwright.config.js

@@ -38,6 +38,13 @@ module.exports = {
           headless: true
         }
       }
-    }
+    },
+    {
+      name: 'visual',
+      testMatch: '**/*.visual.test.js',
+      use: {
+        baseURL: 'https://front.serverest.dev'
+      }
+    },
   ]
 }

tests/e2e/create-user.e2e.test.js

@@ -1,6 +1,7 @@
 const { test, expect } = require('@playwright/test')
 
 const { createUser } = require('../../lib/helpers')
+const { testA11y } = require('../../lib/a11y')
 
 test.describe.parallel('Create user', () => {
   test.beforeEach(async ({ page }) => {
@@ -20,4 +21,9 @@ test.describe.parallel('Create user', () => {
     await page.waitForNavigation()
     await expect(page).toHaveURL('/admin/home')
   })
+
+  // eslint-disable-next-line playwright/no-skipped-test
+  test.skip('has no a11y issues in the form', async ({ page }) => {
+    await testA11y(page, '.form')
+  })
 })

tests/visual/login.visual.test.js

@@ -0,0 +1,27 @@
+const { test } = require('@playwright/test')
+const happoPlaywright = require('happo-playwright')
+
+test.describe('Login page', () => {
+  let context, page
+
+  test.beforeAll(async ({ browser }) => {
+    context = await browser.newContext()
+    page = await context.newPage()
+    await happoPlaywright.init(context)
+  })
+
+  test.afterAll(async () => {
+    await happoPlaywright.finish()
+  })
+
+  test('has correct form layout', async () => {
+    await page.goto('/login')
+
+    const form = page.locator('form')
+
+    await happoPlaywright.screenshot(page, form, {
+      component: 'Login Form',
+      variant: 'Default'
+    })
+  })
+})

tests/visual/navbar.visual.test.js

@@ -0,0 +1,29 @@
+const { test } = require('@playwright/test')
+const happoPlaywright = require('happo-playwright')
+
+const { openHomepage } = require('../../lib/helpers')
+
+test.describe('Navbar', () => {
+  let context, page
+
+  test.beforeAll(async ({ browser }) => {
+    context = await browser.newContext()
+    page = await context.newPage()
+    await happoPlaywright.init(context)
+
+    await openHomepage(page)
+  })
+
+  test.afterAll(async () => {
+    await happoPlaywright.finish()
+  })
+
+  test('has correct layout', async () => {
+    const navbar = page.locator('.navbar')
+
+    await happoPlaywright.screenshot(page, navbar, {
+      component: 'Navbar',
+      variant: 'Default'
+    })
+  })
+})