feat: Adding trace option 'on-all-retries' (#21985)

Fixes: #21860

docs/src/test-api/class-testoptions.md

@@ -172,7 +172,7 @@ Custom attribute to be used in [`method: Page.getByTestId`]. `data-testid` is us
 ## property: TestOptions.trace
 * since: v1.10
 - type: <[Object]|[TraceMode]<"off"|"on"|"retain-on-failure"|"on-first-retry">>
-  - `mode` <[TraceMode]<"off"|"on"|"retain-on-failure"|"on-first-retry">> Trace recording mode.
+  - `mode` <[TraceMode]<"off"|"on"|"retain-on-failure"|"on-first-retry"|"on-all-retries">> Trace recording mode.
   - `screenshots` ?<[boolean]> Whether to capture screenshots during tracing. Screenshots are used to build a timeline preview. Defaults to true. Optional.
   - `snapshots` ?<[boolean]> Whether to capture DOM snapshot on every action. Defaults to true. Optional.
   - `sources` ?<[boolean]> Whether to include source files for trace actions. Defaults to true. Optional.
@@ -182,6 +182,7 @@ Whether to record trace for each test. Defaults to `'off'`.
 * `'on'`: Record trace for each test.
 * `'retain-on-failure'`: Record trace for each test, but remove all traces from successful test runs.
 * `'on-first-retry'`: Record trace only when retrying a test for the first time.
+* `'on-all-retries'`: Record traces only when retrying for all retries.
 
 For more control, pass an object that specifies `mode` and trace features to enable.
 

docs/src/test-cli-js.md

@@ -59,7 +59,7 @@ Here are the most common options available in the command line.
   ```bash
   npx playwright test --debug
   ```
-  
+
   -- Run tests in interactive UI mode, with a built-in watch mode (Preview)
   ```bash
   npx playwright test --ui
@@ -96,7 +96,7 @@ Complete set of Playwright Test options is available in the [configuration file]
 | `--retries <number>` | The maximum number of [retries](./test-retries.md#retries) for flaky tests, defaults to zero (no retries). |
 | `--shard <shard>` | [Shard](./test-parallel.md#shard-tests-between-multiple-machines) tests and execute only selected shard, specified in the form `current/all`, 1-based, for example `3/5`.|
 | `--timeout <number>` | Maximum timeout in milliseconds for each test, defaults to 30 seconds. Learn more about [various timeouts](./test-timeouts.md).|
-| `--trace <mode>` | Force tracing mode, can be `on`, `off`, `on-first-retry`, `retain-on-failure` |
+| `--trace <mode>` | Force tracing mode, can be `on`, `off`, `on-first-retry`, `on-all-retries`, `retain-on-failure` |
 | `--ignore-snapshots` | Whether to ignore [snapshots](./test-snapshots.md). Use this when snapshot expectations are known to be different, e.g. running tests on Linux against Windows screenshots. |
 | `--update-snapshots` or `-u` | Whether to update [snapshots](./test-snapshots.md) with actual results instead of comparing them. Use this when snapshot expectations have changed.|
 | `--workers <number>` or `-j <number>`| The maximum number of concurrent worker processes that run in [parallel](./test-parallel.md). |

docs/src/trace-viewer.md

@@ -97,7 +97,7 @@ Here is what the typical Action snapshot looks like:
 
 Notice how it highlights both, the DOM Node as well as the exact click position.
 
-## Call 
+## Call
 
 See what action was called, the time and duration as well as parameters, return value and log.
 
@@ -126,7 +126,7 @@ See the source code for your entire test.
 ## Recording a trace locally
 * langs: js
 
-To record a trace during development mode set the `--trace` flag to `on` when running your tests. 
+To record a trace during development mode set the `--trace` flag to `on` when running your tests.
 
 ```bash
 npx playwright test --trace on
@@ -168,6 +168,7 @@ await context.tracing.stop({ path: 'trace.zip' });
 
 Available options to record a trace:
 - `'on-first-retry'` - Record a trace only when retrying a test for the first time.
+- `'on-all-retries'` - Record traces for all test retries.
 - `'off'` - Do not record a trace.
 - `'on'` - Record a trace for each test. (not recommended as it's performance heavy)
 - `'retain-on-failure'` - Record a trace for each test, but remove it from successful test runs.

packages/playwright-test/src/cli.ts

@@ -232,7 +232,7 @@ function restartWithExperimentalTsEsm(configFile: string | null): boolean {
   return true;
 }
 
-const kTraceModes: TraceMode[] = ['on', 'off', 'on-first-retry', 'retain-on-failure'];
+const kTraceModes: TraceMode[] = ['on', 'off', 'on-first-retry', 'on-all-retries', 'retain-on-failure'];
 
 const testOptions: [string, string][] = [
   ['--browser <browser>', `Browser to use for tests, one of "all", "chromium", "firefox" or "webkit" (default: "chromium")`],

packages/playwright-test/src/index.ts

@@ -400,7 +400,7 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures> = ({
 
     // 3. Determine whether we need the artifacts.
     const testFailed = testInfo.status !== testInfo.expectedStatus;
-    const preserveTrace = captureTrace && (traceMode === 'on' || (testFailed && traceMode === 'retain-on-failure') || (traceMode === 'on-first-retry' && testInfo.retry === 1));
+    const preserveTrace = captureTrace && (traceMode === 'on' || (testFailed && traceMode === 'retain-on-failure') || (traceMode === 'on-first-retry' && testInfo.retry === 1) || (traceMode === 'on-all-retries' && testInfo.retry > 0));
     const captureScreenshots = screenshotMode === 'on' || (screenshotMode === 'only-on-failure' && testFailed);
 
     const screenshotAttachments: string[] = [];
@@ -626,7 +626,7 @@ function normalizeTraceMode(trace: TraceMode | 'retry-with-trace' | { mode: Trac
 }
 
 function shouldCaptureTrace(traceMode: TraceMode, testInfo: TestInfo) {
-  return traceMode === 'on' || traceMode === 'retain-on-failure' || (traceMode === 'on-first-retry' && testInfo.retry === 1);
+  return traceMode === 'on' || traceMode === 'retain-on-failure' || (traceMode === 'on-first-retry' && testInfo.retry === 1) || (traceMode === 'on-all-retries' && testInfo.retry > 0);
 }
 
 function normalizeScreenshotMode(screenshot: PlaywrightWorkerOptions['screenshot'] | undefined): ScreenshotMode {

packages/playwright-test/types/test.d.ts

@@ -3402,6 +3402,7 @@ export interface PlaywrightWorkerOptions {
    * - `'on'`: Record trace for each test.
    * - `'retain-on-failure'`: Record trace for each test, but remove all traces from successful test runs.
    * - `'on-first-retry'`: Record trace only when retrying a test for the first time.
+   * - `'on-all-retries'`: Record traces only when retrying for all retries.
    *
    * For more control, pass an object that specifies `mode` and trace features to enable.
    *
@@ -3426,7 +3427,7 @@ export interface PlaywrightWorkerOptions {
 }
 
 export type ScreenshotMode = 'off' | 'on' | 'only-on-failure';
-export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry';
+export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries';
 export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry';
 
 /**

tests/page/pageTestApi.ts

@@ -27,7 +27,7 @@ export type PageWorkerFixtures = {
   headless: boolean;
   channel: string;
   screenshot: ScreenshotMode | { mode: ScreenshotMode } & Pick<PageScreenshotOptions, 'fullPage' | 'omitBackground'>;
-  trace: 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | /** deprecated */ 'retry-with-trace';
+  trace: 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | /** deprecated */ 'retry-with-trace';
   video: VideoMode | { mode: VideoMode, size: ViewportSize };
   browserName: 'chromium' | 'firefox' | 'webkit';
   browserVersion: string;

tests/playwright-test/playwright.artifacts.spec.ts

@@ -301,6 +301,41 @@ test('should work with trace: on-first-retry', async ({ runInlineTest }, testInf
   ]);
 });
 
+test('should work with trace: on-all-retries', async ({ runInlineTest }, testInfo) => {
+  const result = await runInlineTest({
+    ...testFiles,
+    'playwright.config.ts': `
+      module.exports = { use: { trace: 'on-all-retries' } };
+    `,
+  }, { workers: 1, retries: 2 });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.passed).toBe(5);
+  expect(result.failed).toBe(5);
+  expect(listFiles(testInfo.outputPath('test-results'))).toEqual([
+    'artifacts-failing-retry1',
+    '  trace.zip',
+    'artifacts-failing-retry2',
+    '  trace.zip',
+    'artifacts-own-context-failing-retry1',
+    '  trace.zip',
+    'artifacts-own-context-failing-retry2',
+    '  trace.zip',
+    'artifacts-persistent-failing-retry1',
+    '  trace.zip',
+    'artifacts-persistent-failing-retry2',
+    '  trace.zip',
+    'artifacts-shared-shared-failing-retry1',
+    '  trace.zip',
+    'artifacts-shared-shared-failing-retry2',
+    '  trace.zip',
+    'artifacts-two-contexts-failing-retry1',
+    '  trace.zip',
+    'artifacts-two-contexts-failing-retry2',
+    '  trace.zip',
+  ]);
+});
+
 test('should take screenshot when page is closed in afterEach', async ({ runInlineTest }, testInfo) => {
   const result = await runInlineTest({
     'playwright.config.ts': `

utils/generate_types/overrides-test.d.ts

@@ -210,7 +210,7 @@ export interface PlaywrightWorkerOptions {
 }
 
 export type ScreenshotMode = 'off' | 'on' | 'only-on-failure';
-export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry';
+export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries';
 export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry';
 
 export interface PlaywrightTestOptions {