feat: --debug=end

docs/src/test-cli-js.md

@@ -80,7 +80,7 @@ npx playwright test --ui
 | :--- | :--- |
 | Non-option arguments | Each argument is treated as a regular expression matched against the full test file path. Only tests from files matching the pattern will be executed. Special symbols like `$` or `*` should be escaped with `\`. In many shells/terminals you may need to quote the arguments. |
 | `-c <file>` or `--config <file>` | Configuration file, or a test directory with optional "playwright.config.&#123;m,c&#125;?&#123;js,ts&#125;". Defaults to `playwright.config.ts` or `playwright.config.js` in the current directory. |
-| `--debug` | Run tests with Playwright Inspector. Shortcut for `PWDEBUG=1` environment variable and `--timeout=0 --max-failures=1 --headed --workers=1` options. |
+| `--debug` | Run tests with Playwright Inspector. See TODO for more info. |
 | `--fail-on-flaky-tests` | Fail if any test is flagged as flaky (default: false). |
 | `--forbid-only` | Fail if `test.only` is called (default: false). Useful on CI. |
 | `--fully-parallel` | Run all tests in parallel (default: false). |

docs/src/test-reporter-api/class-reporter.md

@@ -262,6 +262,22 @@ Test that has been started.
 
 Result of the test run, this object gets populated while the test runs.
 
+## optional method: Reporter.onTestPaused
+* since: v1.54
+
+Called after a test paused.
+
+### param: Reporter.onTestPaused.test
+* since: v1.54
+- `test` <[TestCase]>
+
+Test that has been paused.
+
+### param: Reporter.onTestPaused.result
+* since: v1.54
+- `result` <[TestResult]>
+
+Result of the test run up to this point.
 
 ## optional method: Reporter.onTestEnd
 * since: v1.10

packages/playwright/src/common/config.ts

@@ -97,7 +97,7 @@ export class FullConfigInternal {
       globalTimeout: takeFirst(configCLIOverrides.debug ? 0 : undefined, configCLIOverrides.globalTimeout, userConfig.globalTimeout, 0),
       grep: takeFirst(userConfig.grep, defaultGrep),
       grepInvert: takeFirst(userConfig.grepInvert, null),
-      maxFailures: takeFirst(configCLIOverrides.debug ? 1 : undefined, configCLIOverrides.maxFailures, userConfig.maxFailures, 0),
+      maxFailures: takeFirst(configCLIOverrides.debug === 'begin' ? 1 : undefined, configCLIOverrides.maxFailures, userConfig.maxFailures, 0),
       metadata: metadata ?? userConfig.metadata,
       preserveOutput: takeFirst(userConfig.preserveOutput, 'always'),
       reporter: takeFirst(configCLIOverrides.reporter, resolveReporters(userConfig.reporter, configDir), [[defaultReporter]]),

packages/playwright/src/common/ipc.ts

@@ -23,7 +23,7 @@ import type { ReporterDescription, TestInfoError, TestStatus } from '../../types
 import type { SerializedCompilationCache  } from '../transform/compilationCache';
 
 export type ConfigCLIOverrides = {
-  debug?: boolean;
+  debug?: 'begin' | 'end';
   failOnFlakyTests?: boolean;
   forbidOnly?: boolean;
   fullyParallel?: boolean;
@@ -117,6 +117,7 @@ export type StepEndPayload = {
 export type TestEntry = {
   testId: string;
   retry: number;
+  shouldPauseAtEnd: boolean | 'if-failure';
 };
 
 export type RunPayload = {

packages/playwright/src/index.ts

@@ -23,6 +23,7 @@ import { setBoxedStackPrefixes, createGuid, currentZone, debugMode, jsonStringif
 import { currentTestInfo } from './common/globals';
 import { rootTestType } from './common/testType';
 import { stepTitle } from './util';
+import { Disposable } from './isomorphic/events';
 
 import type { Fixtures, PlaywrightTestArgs, PlaywrightTestOptions, PlaywrightWorkerArgs, PlaywrightWorkerOptions, ScreenshotMode, TestInfo, TestType, VideoMode } from '../types/test';
 import type { ContextReuseMode } from './common/config';
@@ -420,17 +421,25 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures> = ({
     await (browser as any)._stopPendingOperations(closeReason);
   },
 
-  page: async ({ context, _reuseContext }, use) => {
-    if (!_reuseContext) {
-      await use(await context.newPage());
-      return;
+  page: async ({ context, _reuseContext }, use, testInfo) => {
+    const testInfoImpl = testInfo as TestInfoImpl;
+    const existingPage = _reuseContext ? context.pages()[0] : undefined;
+    const page = existingPage ?? await context.newPage();
+
+    const disposables: Disposable[] = [];
+    if (testInfoImpl._configInternal.configCLIOverrides.debug === 'end') {
+      testInfoImpl._onPaused(location => {
+        // TODO: pass location to debugger
+        const resumeClicked = page.pause();
+        void resumeClicked.then(() => {
+          testInfoImpl._resume();
+        });
+      }, disposables);
     }
 
-    // First time we are reusing the context, we should create the page.
-    let [page] = context.pages();
-    if (!page)
-      page = await context.newPage();
     await use(page);
+
+    Disposable.disposeAll(disposables);
   },
 
   request: async ({ playwright }, use) => {

packages/playwright/src/isomorphic/teleReceiver.ts

@@ -178,6 +178,10 @@ export class TeleReporterReceiver {
       this._onTestBegin(params.testId, params.result);
       return;
     }
+    if (method === 'onTestPaused') {
+      this._onTestPaused(params.test, params.result);
+      return;
+    }
     if (method === 'onTestEnd') {
       this._onTestEnd(params.test, params.result);
       return;
@@ -242,6 +246,22 @@ export class TeleReporterReceiver {
     this._reporter.onTestBegin?.(test, testResult);
   }
 
+  private _onTestPaused(testEndPayload: JsonTestEnd, payload: JsonTestResultEnd) {
+    const test = this._tests.get(testEndPayload.testId)!;
+    test.timeout = testEndPayload.timeout;
+    test.expectedStatus = testEndPayload.expectedStatus;
+    const result = test.results.find(r => r._id === payload.id)!;
+    result.duration = payload.duration;
+    result.status = payload.status;
+    result.errors = payload.errors;
+    result.error = result.errors?.[0];
+    if (payload.annotations) {
+      result.annotations = payload.annotations;
+      test.annotations = result.annotations;
+    }
+    this._reporter.onTestPaused?.(test, result);
+  }
+
   private _onTestEnd(testEndPayload: JsonTestEnd, payload: JsonTestResultEnd) {
     const test = this._tests.get(testEndPayload.testId)!;
     test.timeout = testEndPayload.timeout;

packages/playwright/src/program.ts

@@ -173,6 +173,8 @@ async function runTests(args: string[], opts: { [key: string]: any }) {
   if (opts.ui || opts.uiHost || opts.uiPort) {
     if (opts.onlyChanged)
       throw new Error(`--only-changed is not supported in UI mode. If you'd like that to change, see https://github.com/microsoft/playwright/issues/15075 for more details.`);
+    if (cliOverrides.debug === 'end')
+      throw new Error(`--debug=end is not supported in UI mode. If you'd like that to change, file an issue and let us know about your use case for it.`);
 
     const status = await testServer.runUIMode(opts.config, cliOverrides, {
       host: opts.uiHost,
@@ -191,7 +193,9 @@ async function runTests(args: string[], opts: { [key: string]: any }) {
 
   if (process.env.PWTEST_WATCH) {
     if (opts.onlyChanged)
-      throw new Error(`--only-changed is not supported in watch mode. If you'd like that to change, file an issue and let us know about your usecase for it.`);
+      throw new Error(`--only-changed is not supported in watch mode. If you'd like that to change, file an issue and let us know about your use case for it.`);
+    if (cliOverrides.debug === 'end')
+      throw new Error(`--debug=end is not supported in watch mode. If you'd like that to change, file an issue and let us know about your use case for it.`);
 
     const status = await runWatchModeLoop(
         resolveConfigLocation(opts.config),
@@ -207,6 +211,9 @@ async function runTests(args: string[], opts: { [key: string]: any }) {
     return;
   }
 
+  if (cliOverrides.debug === 'end' && config.config.maxFailures > 0)
+    throw new Error(`--debug=end is not supported with --max-failures. If you'd like that to change, file an issue and let us know about your use case for it.`);
+
   const runner = new Runner(config);
   const status = await runner.runAllTests();
   await stopProfiling('runner');
@@ -311,8 +318,16 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrid
   if (options.headed || options.debug)
     overrides.use = { headless: false };
   if (!options.ui && options.debug) {
-    overrides.debug = true;
-    process.env.PWDEBUG = '1';
+    if (options.debug === true)
+      options.debug = 'begin';
+
+    if (!['begin', 'end'].includes(options.debug))
+      throw new Error(`Unsupported debug mode "${options.debug}", must be one of "begin" or "end"`);
+    overrides.debug = options.debug;
+    if (overrides.debug === 'begin')
+      process.env.PWDEBUG = '1';
+    if (overrides.debug === 'end' && options.updateSnapshots)
+      throw new Error(`--debug=end is not supported with --update-snapshots. If you'd like that to change, file an issue and let us know about your use case for it.`);
   }
   if (!options.ui && options.trace) {
     if (!kTraceModes.includes(options.trace))
@@ -376,7 +391,7 @@ const kTraceModes: TraceMode[] = ['on', 'off', 'on-first-retry', 'on-all-retries
 const testOptions: [string, string][] = [
   /* deprecated */ ['--browser <browser>', `Browser to use for tests, one of "all", "chromium", "firefox" or "webkit" (default: "chromium")`],
   ['-c, --config <file>', `Configuration file, or a test directory with optional "playwright.config.{m,c}?{js,ts}"`],
-  ['--debug', `Run tests with Playwright Inspector. Shortcut for "PWDEBUG=1" environment variable and "--timeout=0 --max-failures=1 --headed --workers=1" options`],
+  ['--debug [mode]', `Run tests with Playwright Inspector. See docs for more info.`],
   ['--fail-on-flaky-tests', `Fail if any test is flagged as flaky (default: false)`],
   ['--forbid-only', `Fail if test.only is called (default: false)`],
   ['--fully-parallel', `Run all tests in parallel (default: false)`],

packages/playwright/src/reporters/base.ts

@@ -119,6 +119,7 @@ export class TerminalReporter implements ReporterV2 {
   private _omitFailures: boolean;
   private _fatalErrors: TestError[] = [];
   private _failureCount: number = 0;
+  private _skipEpilogue = false;
 
   constructor(options: { omitFailures?: boolean } = {}) {
     this._omitFailures = options.omitFailures || false;
@@ -152,6 +153,23 @@ export class TerminalReporter implements ReporterV2 {
     (result as any)[kOutputSymbol].push(output);
   }
 
+  /**
+   * --debug=end makes the test runner wait at the end of the very last test, but before its teardown.
+   * At this point, both the test result and the full result are unlikely to change,
+   * so for simplicity we pretend both are done.
+   * TODO: check if things changed in real onTestEnd and onEnd.
+   */
+  onTestPaused(test: TestCase, result: TestResult) {
+    this.onTestEnd(test, result);
+    const fakeEnd: FullResult = { duration: -1, status: 'passed', startTime: new Date() };
+    void this.onEnd(fakeEnd).then(() => {
+      this._skipEpilogue = true;
+      console.log();
+      console.log(this.screen.colors.yellow(`    Keeping test environment up for debugging. Press ${this.screen.colors.cyan('Ctrl+C')} to continue.`));
+      console.log();
+    });
+  }
+
   onTestEnd(test: TestCase, result: TestResult) {
     if (result.status !== 'skipped' && result.status !== test.expectedStatus)
       ++this._failureCount;
@@ -271,6 +289,9 @@ export class TerminalReporter implements ReporterV2 {
   }
 
   epilogue(full: boolean) {
+    if (this._skipEpilogue)
+      return;
+
     const summary = this.generateSummary();
     const summaryMessage = this.generateSummaryMessage(summary);
     if (full && summary.failuresToPrint.length && !this._omitFailures)

packages/playwright/src/reporters/internalReporter.ts

@@ -66,6 +66,10 @@ export class InternalReporter implements ReporterV2 {
     this._reporter.onStdErr?.(chunk, test, result);
   }
 
+  onTestPaused(test: TestCase, result: TestResult) {
+    this._reporter.onTestPaused?.(test, result);
+  }
+
   onTestEnd(test: TestCase, result: TestResult) {
     this._addSnippetToTestErrors(test, result);
     this._reporter.onTestEnd?.(test, result);

packages/playwright/src/reporters/multiplexer.ts

@@ -54,6 +54,11 @@ export class Multiplexer implements ReporterV2 {
       wrap(() => reporter.onStdErr?.(chunk, test, result));
   }
 
+  onTestPaused(test: TestCase, result: TestResult) {
+    for (const reporter of this._reporters)
+      wrap(() => reporter.onTestPaused?.(test, result));
+  }
+
   onTestEnd(test: TestCase, result: TestResult) {
     for (const reporter of this._reporters)
       wrap(() => reporter.onTestEnd?.(test, result));

packages/playwright/src/reporters/reporterV2.ts

@@ -22,6 +22,7 @@ export interface ReporterV2 {
   onTestBegin?(test: TestCase, result: TestResult): void;
   onStdOut?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void;
   onStdErr?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void;
+  onTestPaused?(test: TestCase, result: TestResult): void;
   onTestEnd?(test: TestCase, result: TestResult): void;
   onEnd?(result: FullResult): Promise<{ status?: FullResult['status'] } | undefined | void> | void;
   onExit?(): void | Promise<void>;

packages/playwright/src/reporters/teleEmitter.ts

@@ -70,6 +70,23 @@ export class TeleReporterEmitter implements ReporterV2 {
     });
   }
 
+  onTestPaused(test: reporterTypes.TestCase, result: reporterTypes.TestResult): void {
+    const testEnd: teleReceiver.JsonTestEnd = {
+      testId: test.id,
+      expectedStatus: test.expectedStatus,
+      timeout: test.timeout,
+      annotations: []
+    };
+    this._sendNewAttachments(result, test.id);
+    this._messageSink({
+      method: 'onTestPaused',
+      params: {
+        test: testEnd,
+        result: this._serializeResultEnd(result),
+      }
+    });
+  }
+
   onTestEnd(test: reporterTypes.TestCase, result: reporterTypes.TestResult): void {
     const testEnd: teleReceiver.JsonTestEnd = {
       testId: test.id,

packages/playwright/src/runner/dispatcher.ts

@@ -46,6 +46,7 @@ export class Dispatcher {
   private _config: FullConfigInternal;
   private _reporter: ReporterV2;
   private _failureTracker: FailureTracker;
+  private _shouldPauseAtEnd = false;
 
   private _extraEnvByProjectId: EnvByProjectId = new Map();
   private _producedEnvByProjectId: EnvByProjectId = new Map();
@@ -147,7 +148,7 @@ export class Dispatcher {
     if (startError)
       jobDispatcher.onExit(startError);
     else
-      jobDispatcher.runInWorker(worker);
+      jobDispatcher.runInWorker(worker, this._shouldPauseAtEnd && this._queue.length === 0);
     const result = await jobDispatcher.jobResult;
     this._updateCounterForWorkerHash(job.workerHash, -1);
 
@@ -193,8 +194,9 @@ export class Dispatcher {
     this._queuedOrRunningHashCount.set(hash, delta + (this._queuedOrRunningHashCount.get(hash) || 0));
   }
 
-  async run(testGroups: TestGroup[], extraEnvByProjectId: EnvByProjectId) {
+  async run(testGroups: TestGroup[], extraEnvByProjectId: EnvByProjectId, shouldPauseAtEnd: boolean) {
     this._extraEnvByProjectId = extraEnvByProjectId;
+    this._shouldPauseAtEnd = shouldPauseAtEnd;
     this._queue = testGroups;
     for (const group of testGroups)
       this._updateCounterForWorkerHash(group.workerHash, +1);
@@ -306,21 +308,22 @@ class JobDispatcher {
     this._currentlyRunning = { test, result };
   }
 
-  private _onTestEnd(params: TestEndPayload) {
+  private _onTestPaused(params: TestEndPayload) {
+    const data = this._dataByTestId.get(params.testId);
+    if (!data)
+      return;
+    const { test, result } = data;
+    this._updateTest(params, data.test, data.result);
+    this._reporter.onTestPaused?.(test, result);
+  }
+
+  private _updateTest(params: TestEndPayload, test: TestCase, result: TestResult) {
     if (this._failureTracker.hasReachedMaxFailures()) {
       // Do not show more than one error to avoid confusion, but report
       // as interrupted to indicate that we did actually start the test.
       params.status = 'interrupted';
       params.errors = [];
     }
-    const data = this._dataByTestId.get(params.testId);
-    if (!data) {
-      // TODO: this should never be the case, report an internal error?
-      return;
-    }
-    this._dataByTestId.delete(params.testId);
-    this._remainingByTestId.delete(params.testId);
-    const { result, test } = data;
     result.duration = params.duration;
     result.errors = params.errors;
     result.error = result.errors[0];
@@ -329,6 +332,16 @@ class JobDispatcher {
     test.annotations = [...params.annotations]; // last test result wins
     test.expectedStatus = params.expectedStatus;
     test.timeout = params.timeout;
+  }
+
+  private _onTestEnd(params: TestEndPayload) {
+    const data = this._dataByTestId.get(params.testId);
+    if (!data)
+      return;
+    this._dataByTestId.delete(params.testId);
+    this._remainingByTestId.delete(params.testId);
+    const { test, result } = data;
+    this._updateTest(params, test, result);
     const isFailure = result.status !== 'skipped' && result.status !== test.expectedStatus;
     if (isFailure)
       this._failedTests.add(test);
@@ -549,20 +562,25 @@ class JobDispatcher {
     this.jobResult.resolve(result);
   }
 
-  runInWorker(worker: WorkerHost) {
+  runInWorker(worker: WorkerHost, shouldPauseAtEnd: boolean | 'if-failure') {
     this._parallelIndex = worker.parallelIndex;
     this._workerIndex = worker.workerIndex;
 
     const runPayload: RunPayload = {
       file: this.job.requireFile,
-      entries: this.job.tests.map(test => {
-        return { testId: test.id, retry: test.results.length };
+      entries: this.job.tests.map((test, index) => {
+        return {
+          testId: test.id,
+          retry: test.results.length,
+          shouldPauseAtEnd: index === this.job.tests.length - 1 ? shouldPauseAtEnd : false,
+        };
       }),
     };
     worker.runTestGroup(runPayload);
 
     this._listeners = [
       eventsHelper.addEventListener(worker, 'testBegin', this._onTestBegin.bind(this)),
+      eventsHelper.addEventListener(worker, 'testPaused', this._onTestPaused.bind(this)),
       eventsHelper.addEventListener(worker, 'testEnd', this._onTestEnd.bind(this)),
       eventsHelper.addEventListener(worker, 'stepBegin', this._onStepBegin.bind(this)),
       eventsHelper.addEventListener(worker, 'stepEnd', this._onStepEnd.bind(this)),