diff --git a/docs/src/test-api/class-testoptions.md b/docs/src/test-api/class-testoptions.md index 68381ae478..1068ef86cd 100644 --- a/docs/src/test-api/class-testoptions.md +++ b/docs/src/test-api/class-testoptions.md @@ -172,7 +172,7 @@ Custom attribute to be used in [`method: Page.getByTestId`]. `data-testid` is us ## property: TestOptions.trace * since: v1.10 - type: <[Object]|[TraceMode]<"off"|"on"|"retain-on-failure"|"on-first-retry">> - - `mode` <[TraceMode]<"off"|"on"|"retain-on-failure"|"on-first-retry">> Trace recording mode. + - `mode` <[TraceMode]<"off"|"on"|"retain-on-failure"|"on-first-retry"|"on-all-retries">> Trace recording mode. - `screenshots` ?<[boolean]> Whether to capture screenshots during tracing. Screenshots are used to build a timeline preview. Defaults to true. Optional. - `snapshots` ?<[boolean]> Whether to capture DOM snapshot on every action. Defaults to true. Optional. - `sources` ?<[boolean]> Whether to include source files for trace actions. Defaults to true. Optional. @@ -182,6 +182,7 @@ Whether to record trace for each test. Defaults to `'off'`. * `'on'`: Record trace for each test. * `'retain-on-failure'`: Record trace for each test, but remove all traces from successful test runs. * `'on-first-retry'`: Record trace only when retrying a test for the first time. +* `'on-all-retries'`: Record traces only when retrying for all retries. For more control, pass an object that specifies `mode` and trace features to enable. diff --git a/docs/src/test-cli-js.md b/docs/src/test-cli-js.md index 64dcbe1f34..5bb0beb343 100644 --- a/docs/src/test-cli-js.md +++ b/docs/src/test-cli-js.md @@ -59,7 +59,7 @@ Here are the most common options available in the command line. ```bash npx playwright test --debug ``` - + -- Run tests in interactive UI mode, with a built-in watch mode (Preview) ```bash npx playwright test --ui @@ -96,7 +96,7 @@ Complete set of Playwright Test options is available in the [configuration file] | `--retries ` | The maximum number of [retries](./test-retries.md#retries) for flaky tests, defaults to zero (no retries). | | `--shard ` | [Shard](./test-parallel.md#shard-tests-between-multiple-machines) tests and execute only selected shard, specified in the form `current/all`, 1-based, for example `3/5`.| | `--timeout ` | Maximum timeout in milliseconds for each test, defaults to 30 seconds. Learn more about [various timeouts](./test-timeouts.md).| -| `--trace ` | Force tracing mode, can be `on`, `off`, `on-first-retry`, `retain-on-failure` | +| `--trace ` | Force tracing mode, can be `on`, `off`, `on-first-retry`, `on-all-retries`, `retain-on-failure` | | `--ignore-snapshots` | Whether to ignore [snapshots](./test-snapshots.md). Use this when snapshot expectations are known to be different, e.g. running tests on Linux against Windows screenshots. | | `--update-snapshots` or `-u` | Whether to update [snapshots](./test-snapshots.md) with actual results instead of comparing them. Use this when snapshot expectations have changed.| | `--workers ` or `-j `| The maximum number of concurrent worker processes that run in [parallel](./test-parallel.md). | diff --git a/docs/src/trace-viewer.md b/docs/src/trace-viewer.md index ad6075e8c3..079f336f69 100644 --- a/docs/src/trace-viewer.md +++ b/docs/src/trace-viewer.md @@ -97,7 +97,7 @@ Here is what the typical Action snapshot looks like: Notice how it highlights both, the DOM Node as well as the exact click position. -## Call +## Call See what action was called, the time and duration as well as parameters, return value and log. @@ -126,7 +126,7 @@ See the source code for your entire test. ## Recording a trace locally * langs: js -To record a trace during development mode set the `--trace` flag to `on` when running your tests. +To record a trace during development mode set the `--trace` flag to `on` when running your tests. ```bash npx playwright test --trace on @@ -168,6 +168,7 @@ await context.tracing.stop({ path: 'trace.zip' }); Available options to record a trace: - `'on-first-retry'` - Record a trace only when retrying a test for the first time. +- `'on-all-retries'` - Record traces for all test retries. - `'off'` - Do not record a trace. - `'on'` - Record a trace for each test. (not recommended as it's performance heavy) - `'retain-on-failure'` - Record a trace for each test, but remove it from successful test runs. diff --git a/packages/playwright-test/src/cli.ts b/packages/playwright-test/src/cli.ts index f5f8d39809..46c577547f 100644 --- a/packages/playwright-test/src/cli.ts +++ b/packages/playwright-test/src/cli.ts @@ -232,7 +232,7 @@ function restartWithExperimentalTsEsm(configFile: string | null): boolean { return true; } -const kTraceModes: TraceMode[] = ['on', 'off', 'on-first-retry', 'retain-on-failure']; +const kTraceModes: TraceMode[] = ['on', 'off', 'on-first-retry', 'on-all-retries', 'retain-on-failure']; const testOptions: [string, string][] = [ ['--browser ', `Browser to use for tests, one of "all", "chromium", "firefox" or "webkit" (default: "chromium")`], diff --git a/packages/playwright-test/src/index.ts b/packages/playwright-test/src/index.ts index 5ec95e1ad7..a65e7cf3ff 100644 --- a/packages/playwright-test/src/index.ts +++ b/packages/playwright-test/src/index.ts @@ -400,7 +400,7 @@ const playwrightFixtures: Fixtures = ({ // 3. Determine whether we need the artifacts. const testFailed = testInfo.status !== testInfo.expectedStatus; - const preserveTrace = captureTrace && (traceMode === 'on' || (testFailed && traceMode === 'retain-on-failure') || (traceMode === 'on-first-retry' && testInfo.retry === 1)); + const preserveTrace = captureTrace && (traceMode === 'on' || (testFailed && traceMode === 'retain-on-failure') || (traceMode === 'on-first-retry' && testInfo.retry === 1) || (traceMode === 'on-all-retries' && testInfo.retry > 0)); const captureScreenshots = screenshotMode === 'on' || (screenshotMode === 'only-on-failure' && testFailed); const screenshotAttachments: string[] = []; @@ -626,7 +626,7 @@ function normalizeTraceMode(trace: TraceMode | 'retry-with-trace' | { mode: Trac } function shouldCaptureTrace(traceMode: TraceMode, testInfo: TestInfo) { - return traceMode === 'on' || traceMode === 'retain-on-failure' || (traceMode === 'on-first-retry' && testInfo.retry === 1); + return traceMode === 'on' || traceMode === 'retain-on-failure' || (traceMode === 'on-first-retry' && testInfo.retry === 1) || (traceMode === 'on-all-retries' && testInfo.retry > 0); } function normalizeScreenshotMode(screenshot: PlaywrightWorkerOptions['screenshot'] | undefined): ScreenshotMode { diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index 01e4a1a917..6d804e0502 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -3402,6 +3402,7 @@ export interface PlaywrightWorkerOptions { * - `'on'`: Record trace for each test. * - `'retain-on-failure'`: Record trace for each test, but remove all traces from successful test runs. * - `'on-first-retry'`: Record trace only when retrying a test for the first time. + * - `'on-all-retries'`: Record traces only when retrying for all retries. * * For more control, pass an object that specifies `mode` and trace features to enable. * @@ -3426,7 +3427,7 @@ export interface PlaywrightWorkerOptions { } export type ScreenshotMode = 'off' | 'on' | 'only-on-failure'; -export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry'; +export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries'; export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry'; /** diff --git a/tests/page/pageTestApi.ts b/tests/page/pageTestApi.ts index 5f315658fe..77340dfb2c 100644 --- a/tests/page/pageTestApi.ts +++ b/tests/page/pageTestApi.ts @@ -27,7 +27,7 @@ export type PageWorkerFixtures = { headless: boolean; channel: string; screenshot: ScreenshotMode | { mode: ScreenshotMode } & Pick; - trace: 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | /** deprecated */ 'retry-with-trace'; + trace: 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | /** deprecated */ 'retry-with-trace'; video: VideoMode | { mode: VideoMode, size: ViewportSize }; browserName: 'chromium' | 'firefox' | 'webkit'; browserVersion: string; diff --git a/tests/playwright-test/playwright.artifacts.spec.ts b/tests/playwright-test/playwright.artifacts.spec.ts index 0b1fffb1c0..7490532010 100644 --- a/tests/playwright-test/playwright.artifacts.spec.ts +++ b/tests/playwright-test/playwright.artifacts.spec.ts @@ -301,6 +301,41 @@ test('should work with trace: on-first-retry', async ({ runInlineTest }, testInf ]); }); +test('should work with trace: on-all-retries', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + ...testFiles, + 'playwright.config.ts': ` + module.exports = { use: { trace: 'on-all-retries' } }; + `, + }, { workers: 1, retries: 2 }); + + expect(result.exitCode).toBe(1); + expect(result.passed).toBe(5); + expect(result.failed).toBe(5); + expect(listFiles(testInfo.outputPath('test-results'))).toEqual([ + 'artifacts-failing-retry1', + ' trace.zip', + 'artifacts-failing-retry2', + ' trace.zip', + 'artifacts-own-context-failing-retry1', + ' trace.zip', + 'artifacts-own-context-failing-retry2', + ' trace.zip', + 'artifacts-persistent-failing-retry1', + ' trace.zip', + 'artifacts-persistent-failing-retry2', + ' trace.zip', + 'artifacts-shared-shared-failing-retry1', + ' trace.zip', + 'artifacts-shared-shared-failing-retry2', + ' trace.zip', + 'artifacts-two-contexts-failing-retry1', + ' trace.zip', + 'artifacts-two-contexts-failing-retry2', + ' trace.zip', + ]); +}); + test('should take screenshot when page is closed in afterEach', async ({ runInlineTest }, testInfo) => { const result = await runInlineTest({ 'playwright.config.ts': ` diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 81fa59f62d..519d361962 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -210,7 +210,7 @@ export interface PlaywrightWorkerOptions { } export type ScreenshotMode = 'off' | 'on' | 'only-on-failure'; -export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry'; +export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries'; export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry'; export interface PlaywrightTestOptions {