From 79114c042e361d8ffc0afbf4d0025539c0410309 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Thu, 14 May 2026 09:15:16 -0700 Subject: [PATCH] feat(trace): add retain-all-failures trace mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Records a trace for every attempt and retains it only for attempts that failed, regardless of the final test outcome. Useful for debugging flaky tests that eventually pass on retry — existing retain-on-failure discards the failed-attempt traces in that case. Fixes https://github.com/microsoft/playwright/issues/40832 --- docs/src/test-api/class-testoptions.md | 5 ++- docs/src/test-cli-js.md | 2 +- packages/playwright/src/program.ts | 2 +- packages/playwright/src/worker/testTracing.ts | 5 +++ packages/playwright/types/test.d.ts | 4 +- .../playwright-test/playwright.trace.spec.ts | 44 ++++++++++++++++++- utils/generate_types/overrides-test.d.ts | 2 +- 7 files changed, 57 insertions(+), 7 deletions(-) diff --git a/docs/src/test-api/class-testoptions.md b/docs/src/test-api/class-testoptions.md index fdd5b13cb73d3..5573cd0a2c0ee 100644 --- a/docs/src/test-api/class-testoptions.md +++ b/docs/src/test-api/class-testoptions.md @@ -571,8 +571,8 @@ export default defineConfig({ ## property: TestOptions.trace * since: v1.10 -- type: <[Object]|[TraceMode]<"off"|"on"|"retain-on-failure"|"on-first-retry"|"retain-on-first-failure"|"retain-on-failure-and-retries">> - - `mode` <[TraceMode]<"off"|"on"|"retain-on-failure"|"on-first-retry"|"on-all-retries"|"retain-on-first-failure"|"retain-on-failure-and-retries">> Trace recording mode. +- type: <[Object]|[TraceMode]<"off"|"on"|"retain-on-failure"|"on-first-retry"|"retain-on-first-failure"|"retain-on-failure-and-retries"|"retain-all-failures">> + - `mode` <[TraceMode]<"off"|"on"|"retain-on-failure"|"on-first-retry"|"on-all-retries"|"retain-on-first-failure"|"retain-on-failure-and-retries"|"retain-all-failures">> Trace recording mode. - `attachments` ?<[boolean]> Whether to include test attachments. Defaults to true. Optional. - `screenshots` ?<[boolean]> Whether to capture screenshots during tracing. Screenshots are used to build a timeline preview. Defaults to true. Optional. - `snapshots` ?<[boolean]> Whether to capture DOM snapshot on every action. Defaults to true. Optional. @@ -586,6 +586,7 @@ Whether to record trace for each test. Defaults to `'off'`. * `'retain-on-failure'`: Record trace for each test. When test run passes, remove the recorded trace. * `'retain-on-first-failure'`: Record trace for the first run of each test, but not for retries. When test run passes, remove the recorded trace. * `'retain-on-failure-and-retries'`: Record trace for each test run. Retains all traces when an attempt fails. +* `'retain-all-failures'`: Record trace for each test run. Retains the trace only for attempts that failed, regardless of the final test outcome. For more control, pass an object that specifies `mode` and trace features to enable. diff --git a/docs/src/test-cli-js.md b/docs/src/test-cli-js.md index fabc0f62fe8d6..979fd99b177d6 100644 --- a/docs/src/test-cli-js.md +++ b/docs/src/test-cli-js.md @@ -106,7 +106,7 @@ npx playwright test --ui | `--test-list ` | Path to a file containing a list of tests to run. See [test list](#test-list) for details. | | `--test-list-invert ` | Path to a file containing a list of tests to skip. See [test list](#test-list) for details. | | `--timeout ` | Specify test timeout threshold in milliseconds, zero for unlimited (default: 30 seconds). | -| `--trace ` | Force tracing mode, can be `on`, `off`, `on-first-retry`, `on-all-retries`, `retain-on-failure`, `retain-on-first-failure`, `retain-on-failure-and-retries`. | +| `--trace ` | Force tracing mode, can be `on`, `off`, `on-first-retry`, `on-all-retries`, `retain-on-failure`, `retain-on-first-failure`, `retain-on-failure-and-retries`, `retain-all-failures`. | | `--tsconfig ` | Path to a single tsconfig applicable to all imported files (default: look up tsconfig for each imported file separately). | | `--ui` | Run tests in interactive UI mode. | | `--ui-host ` | Host to serve UI on; specifying this option opens UI in a browser tab. | diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts index f3ad17431ae25..ea447ad948c44 100644 --- a/packages/playwright/src/program.ts +++ b/packages/playwright/src/program.ts @@ -175,7 +175,7 @@ function addInitAgentsCommand(program: Command) { }); } -const kTraceModes: TraceMode[] = ['on', 'off', 'on-first-retry', 'on-all-retries', 'retain-on-failure', 'retain-on-first-failure', 'retain-on-failure-and-retries']; +const kTraceModes: TraceMode[] = ['on', 'off', 'on-first-retry', 'on-all-retries', 'retain-on-failure', 'retain-on-first-failure', 'retain-on-failure-and-retries', 'retain-all-failures']; // Note: update docs/src/test-cli-js.md when you update this, program is the source of truth. diff --git a/packages/playwright/src/worker/testTracing.ts b/packages/playwright/src/worker/testTracing.ts index ccf66ee5fb61e..593c67d3b9569 100644 --- a/packages/playwright/src/worker/testTracing.ts +++ b/packages/playwright/src/worker/testTracing.ts @@ -90,6 +90,9 @@ export class TestTracing { if (this._options?.mode === 'retain-on-failure-and-retries') return true; + if (this._options?.mode === 'retain-all-failures') + return true; + return false; } @@ -168,6 +171,8 @@ export class TestTracing { const testFailed = this._testInfo.status !== this._testInfo.expectedStatus; if (this._options.mode === 'retain-on-failure-and-retries') return !testFailed && this._testInfo.retry === 0; + if (this._options.mode === 'retain-all-failures') + return !testFailed; return !testFailed && (this._options.mode === 'retain-on-failure' || this._options.mode === 'retain-on-first-failure'); } diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index 1aa13bee4194d..7b45706f9b3ca 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -6937,6 +6937,8 @@ export interface PlaywrightWorkerOptions { * - `'retain-on-first-failure'`: Record trace for the first run of each test, but not for retries. When test run * passes, remove the recorded trace. * - `'retain-on-failure-and-retries'`: Record trace for each test run. Retains all traces when an attempt fails. + * - `'retain-all-failures'`: Record trace for each test run. Retains the trace only for attempts that failed, + * regardless of the final test outcome. * * For more control, pass an object that specifies `mode` and trace features to enable. * @@ -6991,7 +6993,7 @@ export interface PlaywrightWorkerOptions { } export type ScreenshotMode = 'off' | 'on' | 'only-on-failure' | 'on-first-failure'; -export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure' | 'retain-on-failure-and-retries'; +export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure' | 'retain-on-failure-and-retries' | 'retain-all-failures'; export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry'; /** * Playwright Test provides many options to configure test environment, diff --git a/tests/playwright-test/playwright.trace.spec.ts b/tests/playwright-test/playwright.trace.spec.ts index 78ebd785be167..c85fe3bd9d083 100644 --- a/tests/playwright-test/playwright.trace.spec.ts +++ b/tests/playwright-test/playwright.trace.spec.ts @@ -386,7 +386,7 @@ test('should respect --trace', async ({ runInlineTest }, testInfo) => { expect(fs.existsSync(testInfo.outputPath('test-results', 'a-test-1', 'trace.zip'))).toBeTruthy(); }); -for (const mode of ['off', 'retain-on-failure', 'on-first-retry', 'on-all-retries', 'retain-on-first-failure', 'retain-on-failure-and-retries']) { +for (const mode of ['off', 'retain-on-failure', 'on-first-retry', 'on-all-retries', 'retain-on-first-failure', 'retain-on-failure-and-retries', 'retain-all-failures']) { test(`trace:${mode} should not create trace zip artifact if page test passed`, async ({ runInlineTest }) => { const result = await runInlineTest({ 'a.spec.ts': ` @@ -1196,6 +1196,48 @@ test('trace:retain-on-failure-and-retries should keep all traces when test fails expect(fs.existsSync(retryTracePath)).toBeTruthy(); }); +test('trace:retain-all-failures should keep traces only for failed attempts when test is flaky', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('flaky', async ({ page }) => { + await page.goto('about:blank'); + expect(test.info().retry).toBe(2); + }); + `, + }, { trace: 'retain-all-failures', retries: 2 }); + + expect(result.exitCode).toBe(0); + expect(result.flaky).toBe(1); + + const firstRunTracePath = testInfo.outputPath('test-results', 'a-flaky', 'trace.zip'); + expect(fs.existsSync(firstRunTracePath)).toBeTruthy(); + const retry1TracePath = testInfo.outputPath('test-results', 'a-flaky-retry1', 'trace.zip'); + expect(fs.existsSync(retry1TracePath)).toBeTruthy(); + const retry2TracePath = testInfo.outputPath('test-results', 'a-flaky-retry2', 'trace.zip'); + expect(fs.existsSync(retry2TracePath)).toBeFalsy(); +}); + +test('trace:retain-all-failures should keep all traces when test fails on every attempt', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('fail', async ({ page }) => { + await page.goto('about:blank'); + expect(true).toBe(false); + }); + `, + }, { trace: 'retain-all-failures', retries: 1 }); + + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); + + const firstRunTracePath = testInfo.outputPath('test-results', 'a-fail', 'trace.zip'); + expect(fs.existsSync(firstRunTracePath)).toBeTruthy(); + const retryTracePath = testInfo.outputPath('test-results', 'a-fail-retry1', 'trace.zip'); + expect(fs.existsSync(retryTracePath)).toBeTruthy(); +}); + test('should not corrupt actions when no library trace is present', async ({ runInlineTest }) => { const result = await runInlineTest({ 'a.spec.ts': ` diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 5e302cb820c39..0c43f4ddc0a8c 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -266,7 +266,7 @@ export interface PlaywrightWorkerOptions { } export type ScreenshotMode = 'off' | 'on' | 'only-on-failure' | 'on-first-failure'; -export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure' | 'retain-on-failure-and-retries'; +export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure' | 'retain-on-failure-and-retries' | 'retain-all-failures'; export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry'; export interface PlaywrightTestOptions { acceptDownloads: boolean;