feat(config): failOnFlakyTests option

This patch adds a configuration option to make the test runner
exit with a non-zero status when any test is flaky. The config
option does not take precedence over the CLI flag.

Fixes #34397

docs/src/test-api/class-fullconfig.md

@@ -10,6 +10,12 @@ Resolved configuration which is accessible via [`property: TestInfo.config`] and
 
 Path to the configuration file used to run the tests. The value is an empty string if no config file was used.
 
+## property: FullConfig.failOnFlakyTests
+* since: v1.51
+- type: <[boolean]>
+
+See [`property: TestConfig.failOnFlakyTests`].
+
 ## property: FullConfig.forbidOnly
 * since: v1.10
 - type: <[boolean]>

docs/src/test-api/class-testconfig.md

@@ -76,6 +76,24 @@ export default defineConfig({
 });
 ```
 
+## property: TestConfig.failOnFlakyTests
+* since: v1.51
+- type: ?<[boolean]>
+
+Whether to exit with an error if any tests are marked as flaky. Useful on CI.
+
+Also available in the [command line](../test-cli.md) with the `--fail-on-flaky-tests` option.
+
+**Usage**
+
+```js title="playwright.config.ts"
+import { defineConfig } from '@playwright/test';
+
+export default defineConfig({
+  failOnFlakyTests: process.env.CI ? true : false,
+});
+```
+
 ## property: TestConfig.forbidOnly
 * since: v1.10
 - type: ?<[boolean]>

packages/playwright/src/common/config.ts

@@ -90,6 +90,7 @@ export class FullConfigInternal {
     this.config = {
       configFile: resolvedConfigFile,
       rootDir: pathResolve(configDir, userConfig.testDir) || configDir,
+      failOnFlakyTests: takeFirst(configCLIOverrides.failOnFlakyTests, userConfig.failOnFlakyTests, false),
       forbidOnly: takeFirst(configCLIOverrides.forbidOnly, userConfig.forbidOnly, false),
       fullyParallel: takeFirst(configCLIOverrides.fullyParallel, userConfig.fullyParallel, false),
       globalSetup: this.globalSetups[0] ?? null,

packages/playwright/src/common/ipc.ts

@@ -25,6 +25,7 @@ import type { SerializedCompilationCache  } from '../transform/compilationCache'
 
 export type ConfigCLIOverrides = {
   debug?: boolean;
+  failOnFlakyTests?: boolean;
   forbidOnly?: boolean;
   fullyParallel?: boolean;
   globalTimeout?: number;

packages/playwright/src/isomorphic/teleReceiver.ts

@@ -592,6 +592,7 @@ export class TeleTestResult implements reporterTypes.TestResult {
 export type TeleFullProject = reporterTypes.FullProject;
 
 export const baseFullConfig: reporterTypes.FullConfig = {
+  failOnFlakyTests: false,
   forbidOnly: false,
   fullyParallel: false,
   globalSetup: null,

packages/playwright/src/runner/failureTracker.ts

@@ -49,7 +49,7 @@ export class FailureTracker {
   }
 
   result(): 'failed' | 'passed' {
-    return this._hasWorkerErrors || this.hasReachedMaxFailures() || this.hasFailedTests() || (this._config.cliFailOnFlakyTests && this.hasFlakyTests()) ? 'failed' : 'passed';
+    return this._hasWorkerErrors || this.hasReachedMaxFailures() || this.hasFailedTests() || (this.failOnFlakyTests() && this.hasFlakyTests()) ? 'failed' : 'passed';
   }
 
   hasFailedTests() {
@@ -63,4 +63,8 @@ export class FailureTracker {
   maxFailures() {
     return this._config.config.maxFailures;
   }
+
+  failOnFlakyTests() {
+    return this._config.config.failOnFlakyTests || this._config.cliFailOnFlakyTests;
+  }
 }

packages/playwright/types/test.d.ts

@@ -1096,6 +1096,25 @@ interface TestConfig<TestArgs = {}, WorkerArgs = {}> {
     };
   };
 
+  /**
+   * Whether to exit with an error if any tests are marked as flaky. Useful on CI.
+   *
+   * Also available in the [command line](https://playwright.dev/docs/test-cli) with the `--fail-on-flaky-tests` option.
+   *
+   * **Usage**
+   *
+   * ```js
+   * // playwright.config.ts
+   * import { defineConfig } from '@playwright/test';
+   *
+   * export default defineConfig({
+   *   failOnFlakyTests: process.env.CI ? true : false,
+   * });
+   * ```
+   *
+   */
+  failOnFlakyTests?: boolean;
+
   /**
    * Whether to exit with an error if any tests or groups are marked as
    * [test.only(title[, details, body])](https://playwright.dev/docs/api/class-test#test-only) or
@@ -1855,6 +1874,12 @@ export interface FullConfig<TestArgs = {}, WorkerArgs = {}> {
    */
   configFile?: string;
 
+  /**
+   * See
+   * [testConfig.failOnFlakyTests](https://playwright.dev/docs/api/class-testconfig#test-config-fail-on-flaky-tests).
+   */
+  failOnFlakyTests: boolean;
+
   /**
    * See [testConfig.forbidOnly](https://playwright.dev/docs/api/class-testconfig#test-config-forbid-only).
    */

tests/playwright-test/config.spec.ts

@@ -72,6 +72,92 @@ test('should prioritize command line timeout over project timeout', async ({ run
   expect(result.output).toContain('Test timeout of 500ms exceeded.');
 });
 
+test('should default to failOnFlakyTests false', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      import { test, expect } from '@playwright/test';
+      test('flake', async ({}, testInfo) => {
+        expect(testInfo.retry).toBe(1);
+      });
+    `,
+  });
+  expect(result.exitCode).not.toBe(0);
+  expect(result.flaky).toBe(0);
+});
+
+test('should prioritize command line --fail-on-flaky-tests flag over config failOnFlakyTests', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = {
+        failOnFlakyTests: false
+      };
+    `,
+    'a.test.js': `
+      import { test, expect } from '@playwright/test';
+      test('flake', async ({}, testInfo) => {
+        expect(testInfo.retry).toBe(1);
+      });
+    `,
+  }, { 'retries': 1, 'fail-on-flaky-tests': true });
+  expect(result.exitCode).not.toBe(0);
+  expect(result.flaky).toBe(1);
+});
+
+test('should support failOnFlakyTests config option', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+        module.exports = {
+          failOnFlakyTests: true,
+          retries: 1
+        };
+    `,
+    'a.test.js': `
+      import { test, expect } from '@playwright/test';
+      test('flake', async ({}, testInfo) => {
+        expect(testInfo.retry).toBe(1);
+      });
+    `,
+  }, { 'retries': 1 });
+  expect(result.exitCode).not.toBe(0);
+  expect(result.flaky).toBe(1);
+});
+
+test('should support failOnFlakyTests config option + retries CLI flag', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+        module.exports = {
+          failOnFlakyTests: true,
+        };
+    `,
+    'a.test.js': `
+      import { test, expect } from '@playwright/test';
+      test('flake', async ({}, testInfo) => {
+        expect(testInfo.retry).toBe(1);
+      });
+    `,
+  }, { 'retries': 1 });
+  expect(result.exitCode).not.toBe(0);
+  expect(result.flaky).toBe(1);
+});
+
+test('should support fail-on-flaky-tests CLI flag + retries config option', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+        module.exports = {
+          retries: 1,
+        };
+    `,
+    'a.test.js': `
+      import { test, expect } from '@playwright/test';
+      test('flake', async ({}, testInfo) => {
+        expect(testInfo.retry).toBe(1);
+      });
+    `,
+  }, { 'fail-on-flaky-tests': true });
+  expect(result.exitCode).not.toBe(0);
+  expect(result.flaky).toBe(1);
+});
+
 test('should read config from --config, resolve relative testDir', async ({ runInlineTest }) => {
   const result = await runInlineTest({
     'my.config.ts': `