feat(test runner): --only-changed option

docs/src/ci-intro.md

@@ -383,6 +383,43 @@ jobs:
         PLAYWRIGHT_TEST_BASE_URL: ${{ github.event.deployment_status.target_url }}
 ```
 
+### Fail-Fast
+* langs: js
+
+Even with sharding enabled, large test suites can take very long to execute. Running changed tests first on PRs will give you a faster feedback loop and use less CI resources.
+
+```yml js title=".github/workflows/playwright.yml"
+name: Playwright Tests
+on:
+  push:
+    branches: [ main, master ]
+  pull_request:
+    branches: [ main, master ]
+jobs:
+  test:
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-node@v4
+      with:
+        node-version: 18
+    - name: Install dependencies
+      run: npm ci
+    - name: Install Playwright Browsers
+      run: npx playwright install --with-deps
+    - name: Run changed Playwright tests
+      run: npx playwright test --only-changed=$GITHUB_BASE_REF
+      if: github.event_name == 'pull_request'
+    - name: Run Playwright tests
+      run: npx playwright test
+    - uses: actions/upload-artifact@v4
+      if: ${{ !cancelled() }}
+      with:
+        name: playwright-report
+        path: playwright-report/
+        retention-days: 30
+```
 
 ## Create a Repo and Push to GitHub
 

docs/src/test-cli-js.md

@@ -93,6 +93,7 @@ Complete set of Playwright Test options is available in the [configuration file]
 | `--max-failures <N>` or `-x`| Stop after the first `N` test failures. Passing `-x` stops after the first failure.|
 | `--no-deps` | Ignore the dependencies between projects and behave as if they were not specified. |
 | `--output <dir>` | Directory for artifacts produced by tests, defaults to `test-results`. |
+| `--only-changed [ref]` | Only run tests that have been changed between working tree and "ref". Defaults to running all uncommitted changes with ref=HEAD. Only supports Git. |
 | `--pass-with-no-tests` | Allows the test suite to pass when no files are found. |
 | `--project <name>` | Only run tests from the specified [projects](./test-projects.md), supports '*' wildcard. Defaults to running all projects defined in the configuration file.|
 | `--quiet` | Whether to suppress stdout and stderr from the tests. |

packages/playwright-ct-core/src/vitePlugin.ts

@@ -69,6 +69,10 @@ export function createPlugin(): TestRunnerPlugin {
       if (stoppableServer)
         await new Promise(f => stoppableServer.stop(f));
     },
+
+    populateDependencies: async () => {
+      await buildBundle(config, configDir);
+    },
   };
 }
 
@@ -157,7 +161,7 @@ export async function buildBundle(config: FullConfig, configDir: string): Promis
   const viteConfig = await createConfig(dirs, config, frameworkPluginFactory, jsxInJS);
 
   if (sourcesDirty) {
-    // Only add out own plugin when we actually build / transform.
+    // Only add our own plugin when we actually build / transform.
     log('build');
     const depsCollector = new Map<string, string[]>();
     const buildConfig = mergeConfig(viteConfig, {

packages/playwright/src/common/config.ts

@@ -49,6 +49,7 @@ export class FullConfigInternal {
   cliArgs: string[] = [];
   cliGrep: string | undefined;
   cliGrepInvert: string | undefined;
+  cliOnlyChanged: string | undefined;
   cliProjectFilter?: string[];
   cliListOnly = false;
   cliPassWithNoTests?: boolean;

packages/playwright/src/plugins/index.ts

@@ -20,6 +20,7 @@ import type { ReporterV2 } from '../reporters/reporterV2';
 export interface TestRunnerPlugin {
   name: string;
   setup?(config: FullConfig, configDir: string, reporter: ReporterV2): Promise<void>;
+  populateDependencies?(): Promise<void>;
   begin?(suite: Suite): Promise<void>;
   end?(): Promise<void>;
   teardown?(): Promise<void>;

packages/playwright/src/program.ts

@@ -153,12 +153,14 @@ Examples:
   $ npx playwright merge-reports playwright-report`);
 }
 
-
 async function runTests(args: string[], opts: { [key: string]: any }) {
   await startProfiling();
   const cliOverrides = overridesFromOptions(opts);
 
   if (opts.ui || opts.uiHost || opts.uiPort) {
+    if (opts.onlyChanged)
+      throw new Error(`--only-changed is not supported in UI mode. If you'd like that to change, see https://github.com/microsoft/playwright/issues/15075 for more details.`);
+
     const status = await testServer.runUIMode(opts.config, {
       host: opts.uiHost,
       port: opts.uiPort ? +opts.uiPort : undefined,
@@ -192,6 +194,7 @@ async function runTests(args: string[], opts: { [key: string]: any }) {
 
   config.cliArgs = args;
   config.cliGrep = opts.grep as string | undefined;
+  config.cliOnlyChanged = opts.onlyChanged === true ? 'HEAD' : opts.onlyChanged;
   config.cliGrepInvert = opts.grepInvert as string | undefined;
   config.cliListOnly = !!opts.list;
   config.cliProjectFilter = opts.project || undefined;
@@ -352,6 +355,7 @@ const testOptions: [string, string][] = [
   ['--max-failures <N>', `Stop after the first N failures`],
   ['--no-deps', 'Do not run project dependencies'],
   ['--output <dir>', `Folder for output artifacts (default: "test-results")`],
+  ['--only-changed [ref]', `Only run tests that have been changed between 'HEAD' and 'ref'. Defaults to running all uncommitted changes. Only supports Git.`],
   ['--pass-with-no-tests', `Makes test run succeed even if no tests were found`],
   ['--project <project-name...>', `Only run tests from the specified list of projects, supports '*' wildcard (default: run all projects)`],
   ['--quiet', `Suppress stdio`],

packages/playwright/src/runner/loadUtils.ts

@@ -32,6 +32,7 @@ import { dependenciesForTestFile } from '../transform/compilationCache';
 import { sourceMapSupport } from '../utilsBundle';
 import type { RawSourceMap } from 'source-map';
 
+
 export async function collectProjectsAndTestFiles(testRun: TestRun, doNotRunTestsOutsideProjectFilter: boolean, additionalFileMatcher?: Matcher) {
   const config = testRun.config;
   const fsCache = new Map();
@@ -118,7 +119,7 @@ export async function loadFileSuites(testRun: TestRun, mode: 'out-of-process' |
   }
 }
 
-export async function createRootSuite(testRun: TestRun, errors: TestError[], shouldFilterOnly: boolean): Promise<Suite> {
+export async function createRootSuite(testRun: TestRun, errors: TestError[], shouldFilterOnly: boolean, additionalFileMatcher?: Matcher): Promise<Suite> {
   const config = testRun.config;
   // Create root suite, where each child will be a project suite with cloned file suites inside it.
   const rootSuite = new Suite('', 'root');
@@ -135,7 +136,8 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho
 
     // Filter file suites for all projects.
     for (const [project, fileSuites] of testRun.projectSuites) {
-      const projectSuite = createProjectSuite(project, fileSuites);
+      const filteredFileSuites = additionalFileMatcher ? fileSuites.filter(fileSuite => additionalFileMatcher(fileSuite.location!.file)) : fileSuites;
+      const projectSuite = createProjectSuite(project, filteredFileSuites);
       projectSuites.set(project, projectSuite);
       const filteredProjectSuite = filterProjectSuite(projectSuite, { cliFileFilters, cliTitleMatcher, testIdMatcher: config.testIdMatcher });
       filteredProjectSuites.set(project, filteredProjectSuite);

packages/playwright/src/runner/reporters.ts

@@ -107,6 +107,8 @@ function computeCommandHash(config: FullConfigInternal) {
     command.cliGrep = config.cliGrep;
   if (config.cliGrepInvert)
     command.cliGrepInvert = config.cliGrepInvert;
+  if (config.cliOnlyChanged)
+    command.cliOnlyChanged = config.cliOnlyChanged;
   if (Object.keys(command).length)
     parts.push(calculateSha1(JSON.stringify(command)).substring(0, 7));
   return parts.join('-');

packages/playwright/src/runner/tasks.ts

@@ -31,6 +31,7 @@ import type { Matcher } from '../util';
 import { Suite } from '../common/test';
 import { buildDependentProjects, buildTeardownToSetupsMap, filterProjects } from './projectUtils';
 import { FailureTracker } from './failureTracker';
+import { detectChangedTests } from './vcs';
 
 const readDirAsync = promisify(fs.readdir);
 
@@ -64,7 +65,7 @@ export class TestRun {
 export function createTaskRunner(config: FullConfigInternal, reporter: ReporterV2): TaskRunner<TestRun> {
   const taskRunner = new TaskRunner<TestRun>(reporter, config.config.globalTimeout);
   addGlobalSetupTasks(taskRunner, config);
-  taskRunner.addTask('load tests', createLoadTask('in-process', { filterOnly: true, failOnLoadErrors: true }));
+  taskRunner.addTask('load tests', createLoadTask('in-process', { filterOnly: true, filterOnlyChanged: true, failOnLoadErrors: true }));
   addRunTasks(taskRunner, config);
   return taskRunner;
 }
@@ -77,14 +78,14 @@ export function createTaskRunnerForWatchSetup(config: FullConfigInternal, report
 
 export function createTaskRunnerForWatch(config: FullConfigInternal, reporter: ReporterV2, additionalFileMatcher?: Matcher): TaskRunner<TestRun> {
   const taskRunner = new TaskRunner<TestRun>(reporter, 0);
-  taskRunner.addTask('load tests', createLoadTask('out-of-process', { filterOnly: true, failOnLoadErrors: false, doNotRunDepsOutsideProjectFilter: true, additionalFileMatcher }));
+  taskRunner.addTask('load tests', createLoadTask('out-of-process', { filterOnly: true, filterOnlyChanged: true, failOnLoadErrors: false, doNotRunDepsOutsideProjectFilter: true, additionalFileMatcher }));
   addRunTasks(taskRunner, config);
   return taskRunner;
 }
 
 export function createTaskRunnerForTestServer(config: FullConfigInternal, reporter: ReporterV2): TaskRunner<TestRun> {
   const taskRunner = new TaskRunner<TestRun>(reporter, 0);
-  taskRunner.addTask('load tests', createLoadTask('out-of-process', { filterOnly: true, failOnLoadErrors: false, doNotRunDepsOutsideProjectFilter: true }));
+  taskRunner.addTask('load tests', createLoadTask('out-of-process', { filterOnly: true, filterOnlyChanged: true, failOnLoadErrors: false, doNotRunDepsOutsideProjectFilter: true }));
   addRunTasks(taskRunner, config);
   return taskRunner;
 }
@@ -109,7 +110,7 @@ function addRunTasks(taskRunner: TaskRunner<TestRun>, config: FullConfigInternal
 
 export function createTaskRunnerForList(config: FullConfigInternal, reporter: ReporterV2, mode: 'in-process' | 'out-of-process', options: { failOnLoadErrors: boolean }): TaskRunner<TestRun> {
   const taskRunner = new TaskRunner<TestRun>(reporter, config.config.globalTimeout);
-  taskRunner.addTask('load tests', createLoadTask(mode, { ...options, filterOnly: false }));
+  taskRunner.addTask('load tests', createLoadTask(mode, { ...options, filterOnly: false, filterOnlyChanged: false }));
   taskRunner.addTask('report begin', createReportBeginTask());
   return taskRunner;
 }
@@ -223,12 +224,21 @@ function createListFilesTask(): Task<TestRun> {
   };
 }
 
-function createLoadTask(mode: 'out-of-process' | 'in-process', options: { filterOnly: boolean, failOnLoadErrors: boolean, doNotRunDepsOutsideProjectFilter?: boolean, additionalFileMatcher?: Matcher }): Task<TestRun> {
+function createLoadTask(mode: 'out-of-process' | 'in-process', options: { filterOnly: boolean, filterOnlyChanged: boolean, failOnLoadErrors: boolean, doNotRunDepsOutsideProjectFilter?: boolean, additionalFileMatcher?: Matcher }): Task<TestRun> {
   return {
     setup: async (testRun, errors, softErrors) => {
       await collectProjectsAndTestFiles(testRun, !!options.doNotRunDepsOutsideProjectFilter, options.additionalFileMatcher);
       await loadFileSuites(testRun, mode, options.failOnLoadErrors ? errors : softErrors);
-      testRun.rootSuite = await createRootSuite(testRun, options.failOnLoadErrors ? errors : softErrors, !!options.filterOnly);
+
+      let cliOnlyChangedMatcher: Matcher | undefined = undefined;
+      if (testRun.config.cliOnlyChanged && options.filterOnlyChanged) {
+        for (const plugin of testRun.config.plugins)
+          await plugin.instance?.populateDependencies?.();
+        const changedFiles = await detectChangedTests(testRun.config.cliOnlyChanged, testRun.config.configDir);
+        cliOnlyChangedMatcher = file => changedFiles.has(file);
+      }
+
+      testRun.rootSuite = await createRootSuite(testRun, options.failOnLoadErrors ? errors : softErrors, !!options.filterOnly, cliOnlyChangedMatcher);
       testRun.failureTracker.onRootSuite(testRun.rootSuite);
       // Fail when no tests.
       if (options.failOnLoadErrors && !testRun.rootSuite.allTests().length && !testRun.config.cliPassWithNoTests && !testRun.config.config.shard) {

packages/playwright/src/runner/vcs.ts

@@ -0,0 +1,45 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import childProcess from 'child_process';
+import { affectedTestFiles } from '../transform/compilationCache';
+import path from 'path';
+
+export async function detectChangedTests(baseCommit: string, configDir: string): Promise<Set<string>> {
+  function gitFileList(command: string) {
+    try {
+      return childProcess.execSync(
+          `git ${command}`,
+          { encoding: 'utf-8', stdio: 'pipe', cwd: configDir }
+      ).split('\n').filter(Boolean);
+    } catch (_error) {
+      const error = _error as childProcess.SpawnSyncReturns<string>;
+      throw new Error([
+        `Cannot detect changed files for --only-changed mode:`,
+        `git ${command}`,
+        '',
+        ...error.output,
+      ].join('\n'));
+    }
+  }
+
+  const untrackedFiles = gitFileList(`ls-files --others --exclude-standard`).map(file => path.join(configDir, file));
+
+  const [gitRoot] = gitFileList('rev-parse --show-toplevel');
+  const trackedFilesWithChanges = gitFileList(`diff ${baseCommit} --name-only`).map(file => path.join(gitRoot, file));
+
+  return new Set(affectedTestFiles([...untrackedFiles, ...trackedFilesWithChanges]));
+}