Playwright test stability and speed improvements (#2299)

* Prevent permanent hang of test if post install page is already open.

* Shard playwright tests across 4 workers

* Merged reports from shards

* Use html reporter in CI

* Use github reporter instead

.github/workflows/checks.yml

@@ -46,6 +46,11 @@ jobs:
   playwright-tests:
     timeout-minutes: 14
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        test: [playwright, playwright-mv3]
+        shard: [1/4, 2/4, 3/4, 4/4]
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v3
@@ -56,31 +61,10 @@ jobs:
       - name: Install Playwright Browsers
         run: npx playwright install --with-deps chromium
       - name: Run tests
-        run: xvfb-run --auto-servernum -- npm run playwright
-      - uses: actions/upload-artifact@v3
-        if: always()
-        with:
-          name: playwright-report
-          path: playwright-report/
-          retention-days: 30
-
-  playwright-tests-mv3:
-    timeout-minutes: 14
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-node@v3
-        with:
-          node-version: 18
-          cache: 'npm'
-      - run: npm run install-ci
-      - name: Install Playwright Browsers
-        run: npx playwright install --with-deps chromium
-      - name: Run tests
-        run: xvfb-run --auto-servernum -- npm run playwright-mv3
+        run: xvfb-run --auto-servernum -- npm run ${{ matrix.test }} -- --shard ${{ matrix.shard }}
       - uses: actions/upload-artifact@v3
         if: always()
         with:
-          name: playwright-report-mv3
+          name: ${{ matrix.test }}-report
           path: playwright-report/
-          retention-days: 30
+          retention-days: 1

integration-test/helpers/backgroundWait.js

@@ -70,14 +70,21 @@ export async function forAllConfiguration (bgPage) {
     })
 }
 
+/**
+ * @param {import('@playwright/test').BrowserContext} context
+ */
 export async function forExtensionLoaded (context) {
     return /** @type {Promise<string>} */(new Promise((resolve) => {
+        const postInstallPage = 'https://duckduckgo.com/extension-success'
         const listenForPostinstall = (page) => {
-            if (page.url().startsWith('https://duckduckgo.com/extension-success')) {
+            if (page.url().startsWith(postInstallPage)) {
                 resolve(page.url())
                 context.off('page', listenForPostinstall)
             }
         }
+        if (context.pages().find(p => p.url().startsWith(postInstallPage))) {
+            return resolve()
+        }
         context.on('page', listenForPostinstall)
     }))
 }

playwright.config.js

@@ -29,7 +29,7 @@ export default defineConfig({
     /* Opt out of parallel tests on CI. */
     workers: undefined,
     /* Reporter to use. See https://playwright.dev/docs/test-reporters */
-    reporter: process.env.CI ? 'line' : 'html',
+    reporter: process.env.CI ? 'github' : 'html',
     /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
     use: {
     /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */