diff --git a/docs/src/test-api/class-fullconfig.md b/docs/src/test-api/class-fullconfig.md index 627218db7b55c..e3097cb16fa8b 100644 --- a/docs/src/test-api/class-fullconfig.md +++ b/docs/src/test-api/class-fullconfig.md @@ -21,6 +21,12 @@ any argument-parsing library. Path to the configuration file used to run the tests. The value is an empty string if no config file was used. +## property: FullConfig.failOnFlakyTests +* since: v1.61 +- type: <[boolean]> + +See [`property: TestConfig.failOnFlakyTests`]. + ## property: FullConfig.forbidOnly * since: v1.10 - type: <[boolean]> diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts index 475386f853c54..2b8ae1a60f75b 100644 --- a/packages/playwright/src/common/config.ts +++ b/packages/playwright/src/common/config.ts @@ -48,7 +48,6 @@ export class FullConfigInternal { readonly projects: FullProjectInternal[] = []; readonly singleTSConfigPath?: string; readonly captureGitInfo: Config['captureGitInfo']; - readonly failOnFlakyTests: boolean; defineConfigWasUsed = false; globalSetups: string[] = []; @@ -68,7 +67,6 @@ export class FullConfigInternal { this.plugins = (privateConfiguration?.plugins || []).map((p: any) => ({ factory: p })); this.singleTSConfigPath = pathResolve(configDir, userConfig.tsconfig); this.captureGitInfo = userConfig.captureGitInfo; - this.failOnFlakyTests = takeFirst(configCLIOverrides.failOnFlakyTests, userConfig.failOnFlakyTests, false); this.globalSetups = (Array.isArray(userConfig.globalSetup) ? userConfig.globalSetup : [userConfig.globalSetup]).map(s => resolveScript(s, configDir)).filter(script => script !== undefined); this.globalTeardowns = (Array.isArray(userConfig.globalTeardown) ? userConfig.globalTeardown : [userConfig.globalTeardown]).map(s => resolveScript(s, configDir)).filter(script => script !== undefined); @@ -87,6 +85,7 @@ export class FullConfigInternal { argv: configCLIOverrides.argv ?? [], configFile: resolvedConfigFile, rootDir: pathResolve(configDir, userConfig.testDir) || configDir, + failOnFlakyTests: takeFirst(configCLIOverrides.failOnFlakyTests, userConfig.failOnFlakyTests, false), forbidOnly: takeFirst(configCLIOverrides.forbidOnly, userConfig.forbidOnly, false), fullyParallel: takeFirst(configCLIOverrides.fullyParallel, userConfig.fullyParallel, false), globalSetup: this.globalSetups[0] ?? null, diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts index bd534c9216f49..bfde074d8d1c1 100644 --- a/packages/playwright/src/isomorphic/teleReceiver.ts +++ b/packages/playwright/src/isomorphic/teleReceiver.ts @@ -29,6 +29,7 @@ export type JsonConfig = Pick !test.ok()); const hasFlakyTests = this.rootSuite?.allTests().some(test => test.outcome() === 'flaky'); - return this.hasWorkerErrors || this.hasReachedMaxFailures() || hasFailedTests || (this.config.failOnFlakyTests && hasFlakyTests) ? 'failed' : 'passed'; + return this.hasWorkerErrors || this.hasReachedMaxFailures() || hasFailedTests || (this.config.config.failOnFlakyTests && hasFlakyTests) ? 'failed' : 'passed'; } } diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index 52e56752ee97d..1e292f153a35a 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -2079,6 +2079,12 @@ export interface FullConfig { */ configFile?: string; + /** + * See + * [testConfig.failOnFlakyTests](https://playwright.dev/docs/api/class-testconfig#test-config-fail-on-flaky-tests). + */ + failOnFlakyTests: boolean; + /** * See [testConfig.forbidOnly](https://playwright.dev/docs/api/class-testconfig#test-config-forbid-only). */ diff --git a/tests/playwright-test/config.spec.ts b/tests/playwright-test/config.spec.ts index fc83a7eb52b37..ab04bf8970c03 100644 --- a/tests/playwright-test/config.spec.ts +++ b/tests/playwright-test/config.spec.ts @@ -77,9 +77,17 @@ test('should support failOnFlakyTests config option', async ({ runInlineTest }) 'playwright.config.ts': ` module.exports = { failOnFlakyTests: true, - retries: 1 + retries: 1, + reporter: [['line'], ['./reporter.js']], }; `, + 'reporter.js': ` + module.exports = class Reporter { + onBegin(config) { + console.log('reporter.failOnFlakyTests:', config.failOnFlakyTests); + } + }; + `, 'a.test.js': ` import { test, expect } from '@playwright/test'; test('flake', async ({}, testInfo) => { @@ -89,6 +97,7 @@ test('should support failOnFlakyTests config option', async ({ runInlineTest }) }, { 'retries': 1 }); expect(result.exitCode).not.toBe(0); expect(result.flaky).toBe(1); + expect(result.output).toContain('reporter.failOnFlakyTests: true'); }); test('should read config from --config, resolve relative testDir', async ({ runInlineTest }) => { diff --git a/tests/playwright-test/reporter-blob.spec.ts b/tests/playwright-test/reporter-blob.spec.ts index 7c620d85f5f3a..9ded1e17cc099 100644 --- a/tests/playwright-test/reporter-blob.spec.ts +++ b/tests/playwright-test/reporter-blob.spec.ts @@ -972,6 +972,7 @@ test('preserve config fields', async ({ runInlineTest, mergeReports }) => { const config: PlaywrightTestConfig = { // Runner options: globalTimeout: 202300, + failOnFlakyTests: true, maxFailures: 3, metadata: { 'a': 'b', @@ -1043,6 +1044,7 @@ test('preserve config fields', async ({ runInlineTest, mergeReports }) => { expect(json.rootDir).toBe(test.info().outputDir); expect(json.globalTimeout).toBe(config.globalTimeout); expect(json.maxFailures).toBe(config.maxFailures); + expect(json.failOnFlakyTests).toBe(config.failOnFlakyTests); expect(json.metadata).toEqual(expect.objectContaining(config.metadata)); expect(json.workers).toBe(2); expect(json.version).toBeTruthy();