feat(reporter): expose failOnFlakyTests on FullConfig

docs/src/test-api/class-fullconfig.md

@@ -21,6 +21,12 @@ any argument-parsing library.
 
 Path to the configuration file used to run the tests. The value is an empty string if no config file was used.
 
+## property: FullConfig.failOnFlakyTests
+* since: v1.61
+- type: <[boolean]>
+
+See [`property: TestConfig.failOnFlakyTests`].
+
 ## property: FullConfig.forbidOnly
 * since: v1.10
 - type: <[boolean]>

packages/playwright/src/common/config.ts

@@ -48,7 +48,6 @@ export class FullConfigInternal {
   readonly projects: FullProjectInternal[] = [];
   readonly singleTSConfigPath?: string;
   readonly captureGitInfo: Config['captureGitInfo'];
-  readonly failOnFlakyTests: boolean;
   defineConfigWasUsed = false;
 
   globalSetups: string[] = [];
@@ -68,7 +67,6 @@ export class FullConfigInternal {
     this.plugins = (privateConfiguration?.plugins || []).map((p: any) => ({ factory: p }));
     this.singleTSConfigPath = pathResolve(configDir, userConfig.tsconfig);
     this.captureGitInfo = userConfig.captureGitInfo;
-    this.failOnFlakyTests = takeFirst(configCLIOverrides.failOnFlakyTests, userConfig.failOnFlakyTests, false);
 
     this.globalSetups = (Array.isArray(userConfig.globalSetup) ? userConfig.globalSetup : [userConfig.globalSetup]).map(s => resolveScript(s, configDir)).filter(script => script !== undefined);
     this.globalTeardowns = (Array.isArray(userConfig.globalTeardown) ? userConfig.globalTeardown : [userConfig.globalTeardown]).map(s => resolveScript(s, configDir)).filter(script => script !== undefined);
@@ -87,6 +85,7 @@ export class FullConfigInternal {
       argv: configCLIOverrides.argv ?? [],
       configFile: resolvedConfigFile,
       rootDir: pathResolve(configDir, userConfig.testDir) || configDir,
+      failOnFlakyTests: takeFirst(configCLIOverrides.failOnFlakyTests, userConfig.failOnFlakyTests, false),
       forbidOnly: takeFirst(configCLIOverrides.forbidOnly, userConfig.forbidOnly, false),
       fullyParallel: takeFirst(configCLIOverrides.fullyParallel, userConfig.fullyParallel, false),
       globalSetup: this.globalSetups[0] ?? null,

packages/playwright/src/isomorphic/teleReceiver.ts

@@ -29,6 +29,7 @@ export type JsonConfig = Pick<reporterTypes.FullConfig, 'configFile' | 'globalTi
   // optional for backwards compatibility
   tags?: reporterTypes.FullConfig['tags'],
   webServer?: reporterTypes.FullConfig['webServer'],
+  failOnFlakyTests?: reporterTypes.FullConfig['failOnFlakyTests'],
 };
 
 export type JsonPattern = {
@@ -780,6 +781,7 @@ export type TeleFullProject = reporterTypes.FullProject;
 
 export const baseFullConfig: reporterTypes.FullConfig = {
   argv: [],
+  failOnFlakyTests: false,
   forbidOnly: false,
   fullyParallel: false,
   globalSetup: null,

packages/playwright/src/reporters/teleEmitter.ts

@@ -200,6 +200,7 @@ export class TeleReporterEmitter implements ReporterV2 {
       globalTeardown: config.globalTeardown,
       tags: config.tags,
       webServer: config.webServer,
+      failOnFlakyTests: config.failOnFlakyTests,
     };
   }
 

packages/playwright/src/runner/tasks.ts

@@ -120,7 +120,7 @@ export class TestRun {
   result(): 'failed' | 'passed' {
     const hasFailedTests = this.rootSuite?.allTests().some(test => !test.ok());
     const hasFlakyTests = this.rootSuite?.allTests().some(test => test.outcome() === 'flaky');
-    return this.hasWorkerErrors || this.hasReachedMaxFailures() || hasFailedTests || (this.config.failOnFlakyTests && hasFlakyTests) ? 'failed' : 'passed';
+    return this.hasWorkerErrors || this.hasReachedMaxFailures() || hasFailedTests || (this.config.config.failOnFlakyTests && hasFlakyTests) ? 'failed' : 'passed';
   }
 }
 

packages/playwright/types/test.d.ts

@@ -2079,6 +2079,12 @@ export interface FullConfig<TestArgs = {}, WorkerArgs = {}> {
    */
   configFile?: string;
 
+  /**
+   * See
+   * [testConfig.failOnFlakyTests](https://playwright.dev/docs/api/class-testconfig#test-config-fail-on-flaky-tests).
+   */
+  failOnFlakyTests: boolean;
+
   /**
    * See [testConfig.forbidOnly](https://playwright.dev/docs/api/class-testconfig#test-config-forbid-only).
    */

tests/playwright-test/config.spec.ts

@@ -77,9 +77,17 @@ test('should support failOnFlakyTests config option', async ({ runInlineTest })
     'playwright.config.ts': `
         module.exports = {
           failOnFlakyTests: true,
-          retries: 1
+          retries: 1,
+          reporter: [['line'], ['./reporter.js']],
         };
     `,
+    'reporter.js': `
+      module.exports = class Reporter {
+        onBegin(config) {
+          console.log('reporter.failOnFlakyTests:', config.failOnFlakyTests);
+        }
+      };
+    `,
     'a.test.js': `
       import { test, expect } from '@playwright/test';
       test('flake', async ({}, testInfo) => {
@@ -89,6 +97,7 @@ test('should support failOnFlakyTests config option', async ({ runInlineTest })
   }, { 'retries': 1 });
   expect(result.exitCode).not.toBe(0);
   expect(result.flaky).toBe(1);
+  expect(result.output).toContain('reporter.failOnFlakyTests: true');
 });
 
 test('should read config from --config, resolve relative testDir', async ({ runInlineTest }) => {

tests/playwright-test/reporter-blob.spec.ts

@@ -972,6 +972,7 @@ test('preserve config fields', async ({ runInlineTest, mergeReports }) => {
   const config: PlaywrightTestConfig = {
     // Runner options:
     globalTimeout: 202300,
+    failOnFlakyTests: true,
     maxFailures: 3,
     metadata: {
       'a': 'b',
@@ -1043,6 +1044,7 @@ test('preserve config fields', async ({ runInlineTest, mergeReports }) => {
   expect(json.rootDir).toBe(test.info().outputDir);
   expect(json.globalTimeout).toBe(config.globalTimeout);
   expect(json.maxFailures).toBe(config.maxFailures);
+  expect(json.failOnFlakyTests).toBe(config.failOnFlakyTests);
   expect(json.metadata).toEqual(expect.objectContaining(config.metadata));
   expect(json.workers).toBe(2);
   expect(json.version).toBeTruthy();