perf(vrt): parallelize visual regression tests across N browser pages

.github/workflows/tests.yml

@@ -45,7 +45,7 @@ jobs:
         run: cd visual-regression && pnpm exec playwright install chromium
 
       - name: Run Visual Regression Tests
-        run: pnpm run test:visual --color
+        run: pnpm run test:visual --color -w 4
         env:
           RUNTIME_ENV: ci
 

examples/index.ts

@@ -102,7 +102,11 @@ const defaultPhysicalPixelRatio = 1;
     return;
   }
   assertTruthy(automation);
-  await runAutomation(renderMode, test, logFps);
+  // Optional shard string in the form "i/N" — when present, this page only
+  // runs the subset of tests where `index % N === i`. Used by the VRT runner
+  // to parallelize across multiple browser pages.
+  const shardParam = urlParams.get('shard');
+  await runAutomation(renderMode, test, logFps, shardParam);
 })().catch((err) => {
   console.error(err);
 });
@@ -348,7 +352,17 @@ async function runAutomation(
   renderMode: string,
   filter: string | null,
   logFps: boolean,
+  shard: string | null,
 ) {
+  let shardIndex = 0;
+  let shardTotal = 1;
+  if (shard) {
+    const match = /^(\d+)\/(\d+)$/.exec(shard);
+    if (match) {
+      shardIndex = Number(match[1]);
+      shardTotal = Number(match[2]);
+    }
+  }
   const logicalPixelRatio = defaultResolution / appHeight;
   const { renderer, appElement } = await initRenderer(
     renderMode,
@@ -359,14 +373,17 @@ async function runAutomation(
     false, // enableInspector
   );
 
-  // Iterate through all test modules
-  for (const testPath in testModules) {
+  // Iterate through all test modules. Sort so sharding is deterministic
+  // across pages, and apply the filter up front so the shard step indexes
+  // only the tests that will actually run.
+  const orderedPaths = Object.keys(testModules)
+    .sort()
+    .filter((p) => !filter || wildcardMatch(getTestName(p), filter));
+  for (let i = 0; i < orderedPaths.length; i++) {
+    if (i % shardTotal !== shardIndex) continue;
+    const testPath = orderedPaths[i]!;
     const testModule = testModules[testPath];
     const testName = getTestName(testPath);
-    // Skip tests that don't match the filter (if provided)
-    if (filter && !wildcardMatch(testName, filter)) {
-      continue;
-    }
     assertTruthy(testModule);
 
     // Setup Math.random to use a seeded random number generator for consistent

visual-regression/src/index.ts

@@ -93,6 +93,13 @@ const argv = yargs(hideBin(process.argv))
       default: '*',
       description: 'Tests to run ("*" wildcard pattern)',
     },
+    workers: {
+      type: 'number',
+      alias: 'w',
+      default: 1,
+      description:
+        'Number of parallel browser pages used to run tests (sharded round-robin)',
+    },
   })
   .parseSync();
 
@@ -129,6 +136,7 @@ async function dockerCiMode(): Promise<number> {
     argv.skipBuild ? '--skipBuild' : '',
     argv.port ? `--port ${argv.port}` : '',
     argv.filter ? `--filter "${argv.filter}"` : '',
+    argv.workers > 1 ? `--workers ${argv.workers}` : '',
   ].join(' ');
 
   // Get the directory of the current file
@@ -252,7 +260,9 @@ async function runTest(browserType: 'chromium') {
     await fs.emptyDir(failedResultsDir);
   }
 
-  // Launch browser and create page
+  // Launch the browser once; each worker runs in its own Page sharing the
+  // same browser process. Pages run in parallel, each handling a round-robin
+  // shard of the test list (see runAutomation in examples/index.ts).
   const browser = await browsers[browserType].launch({
     args: [
       '--disable-font-subpixel-positioning',
@@ -262,22 +272,14 @@ async function runTest(browserType: 'chromium') {
     ],
   });
 
-  const page = await browser.newPage();
-
-  // If verbose, log out console messages from the browser
-  if (argv.verbose) {
-    page.on('console', (msg) => console.log(`console: ${msg.text()}`));
-  }
-
-  /**
-   * Keeps track of the latest snapshot index for each test
-   */
-  const testCounters: Record<string, number> = {};
+  const workerCount = Math.max(1, argv.workers);
 
-  // Expose the `snapshot()` function to the browser
-  await page.exposeFunction(
-    'snapshot',
-    async (test: string, options: SnapshotOptions) => {
+  // Each test's first snapshot is index 1, second is index 2, etc. With
+  // sharding, a given test only runs in one worker, so per-worker counters
+  // are sufficient. Snapshot filenames are still globally unique.
+  const makeSnapshotHandler = (page: import('playwright').Page) => {
+    const testCounters: Record<string, number> = {};
+    return async (test: string, options: SnapshotOptions) => {
       snapshotsTested++;
 
       // Ensure clip dimensions are integers (matches Playwright's clip shape
@@ -345,101 +347,94 @@ async function runTest(browserType: 'chromium') {
           ),
         );
       }
-    },
-  );
+    };
+  };
 
-  /**
-   * Resolve function for the donePromise below
-   */
-  let resolveDonePromise: (exitCode: number) => void;
-  /**
-   * Promise that resolves when all tests are done
-   */
-  const donePromise = new Promise<number>((resolve) => {
-    resolveDonePromise = resolve;
-  });
+  const runWorker = async (shardIndex: number) => {
+    const page = await browser.newPage();
 
-  // Expose the `doneTests()` function to the browser
-  // which will close the browser, calculate/print results and resolve the donePromise
-  await page.exposeFunction('doneTests', async () => {
-    await browser.close();
-
-    // Summarize results
-
-    const passPerc: string = (
-      (snapshotsPassed / snapshotsTested) *
-      100
-    ).toFixed(1);
-    const failPerc: string = (
-      (snapshotsFailed / snapshotsTested) *
-      100
-    ).toFixed(1);
-    const skipPerc: string = (
-      (snapshotsSkipped / snapshotsTested) *
-      100
-    ).toFixed(1);
-
-    if (argv.capture) {
-      console.log(
-        chalk.white.underline(`\nVisual Regression Test Capture Completed:`),
+    if (argv.verbose) {
+      page.on('console', (msg) =>
+        console.log(`console[${shardIndex}]: ${msg.text()}`),
       );
+    }
 
-      if (snapshotsPassed > 0) {
-        console.log(
-          chalk.green(
-            `   ${snapshotsPassed} snapshots captured (${passPerc}%)`,
-          ),
-        );
-      }
+    await page.exposeFunction('snapshot', makeSnapshotHandler(page));
 
-      if (snapshotsSkipped > 0) {
-        console.log(
-          chalk.yellow(
-            `   ${snapshotsSkipped} snapshots skipped (${skipPerc}%)`,
-          ),
-        );
-      }
+    const donePromise = new Promise<void>((resolve) => {
+      void page.exposeFunction('doneTests', () => {
+        resolve();
+      });
+    });
 
-      console.log(chalk.gray(`   ${snapshotsTested} snapshots detected`));
-    } else {
-      console.log(
-        chalk.white.underline(`\nVisual Regression Tests Completed:`),
-      );
+    const shardParam =
+      workerCount > 1 ? `&shard=${shardIndex}/${workerCount}` : '';
+    await page.goto(
+      `http://localhost:${argv.port}/?automation=true&test=${argv.filter}${shardParam}`,
+    );
 
-      if (snapshotsFailed > 0) {
-        console.log(
-          chalk.red(`   ${snapshotsFailed} snapshots failed (${failPerc}%)`),
-        );
-        console.log(
-          chalk.gray(
-            `      (See \`${failedResultsDir}\` directory for failed results)`,
-          ),
-        );
-      }
+    await donePromise;
+    await page.close();
+  };
 
-      if (snapshotsPassed > 0) {
-        console.log(
-          chalk.green(`   ${snapshotsPassed} snapshots passed (${passPerc}%)`),
-        );
-      }
+  await Promise.all(
+    Array.from({ length: workerCount }, (_, i) => runWorker(i)),
+  );
+  await browser.close();
+
+  // Summarize results
+  const passPerc: string = ((snapshotsPassed / snapshotsTested) * 100).toFixed(
+    1,
+  );
+  const failPerc: string = ((snapshotsFailed / snapshotsTested) * 100).toFixed(
+    1,
+  );
+  const skipPerc: string = ((snapshotsSkipped / snapshotsTested) * 100).toFixed(
+    1,
+  );
 
-      console.log(chalk.gray(`   ${snapshotsTested} snapshots tested`));
+  if (argv.capture) {
+    console.log(
+      chalk.white.underline(`\nVisual Regression Test Capture Completed:`),
+    );
+
+    if (snapshotsPassed > 0) {
+      console.log(
+        chalk.green(`   ${snapshotsPassed} snapshots captured (${passPerc}%)`),
+      );
+    }
+
+    if (snapshotsSkipped > 0) {
+      console.log(
+        chalk.yellow(`   ${snapshotsSkipped} snapshots skipped (${skipPerc}%)`),
+      );
     }
 
-    // Extra new line
-    console.log(chalk.reset(''));
+    console.log(chalk.gray(`   ${snapshotsTested} snapshots detected`));
+  } else {
+    console.log(chalk.white.underline(`\nVisual Regression Tests Completed:`));
 
     if (snapshotsFailed > 0) {
-      resolveDonePromise(1);
-    } else {
-      resolveDonePromise(0);
+      console.log(
+        chalk.red(`   ${snapshotsFailed} snapshots failed (${failPerc}%)`),
+      );
+      console.log(
+        chalk.gray(
+          `      (See \`${failedResultsDir}\` directory for failed results)`,
+        ),
+      );
     }
-  });
 
-  // Go to the examples page
-  await page.goto(
-    `http://localhost:${argv.port}/?automation=true&test=${argv.filter}`,
-  );
+    if (snapshotsPassed > 0) {
+      console.log(
+        chalk.green(`   ${snapshotsPassed} snapshots passed (${passPerc}%)`),
+      );
+    }
+
+    console.log(chalk.gray(`   ${snapshotsTested} snapshots tested`));
+  }
+
+  console.log(chalk.reset(''));
 
-  return donePromise;
+  return snapshotsFailed > 0 ? 1 : 0;
 }