diff --git a/docs/src/test-api/class-testoptions.md b/docs/src/test-api/class-testoptions.md index 946171a935..0fba476d73 100644 --- a/docs/src/test-api/class-testoptions.md +++ b/docs/src/test-api/class-testoptions.md @@ -546,8 +546,8 @@ export default defineConfig({ ## property: TestOptions.trace * since: v1.10 -- type: <[Object]|[TraceMode]<"off"|"on"|"retain-on-failure"|"on-first-retry">> - - `mode` <[TraceMode]<"off"|"on"|"retain-on-failure"|"on-first-retry"|"on-all-retries">> Trace recording mode. +- type: <[Object]|[TraceMode]<"off"|"on"|"retain-on-failure"|"on-first-retry"|"retain-on-first-failure">> + - `mode` <[TraceMode]<"off"|"on"|"retain-on-failure"|"on-first-retry"|"on-all-retries"|"retain-on-first-failure">> Trace recording mode. - `attachments` ?<[boolean]> Whether to include test attachments. Defaults to true. Optional. - `screenshots` ?<[boolean]> Whether to capture screenshots during tracing. Screenshots are used to build a timeline preview. Defaults to true. Optional. - `snapshots` ?<[boolean]> Whether to capture DOM snapshot on every action. Defaults to true. Optional. @@ -559,6 +559,7 @@ Whether to record trace for each test. Defaults to `'off'`. * `'retain-on-failure'`: Record trace for each test, but remove all traces from successful test runs. * `'on-first-retry'`: Record trace only when retrying a test for the first time. * `'on-all-retries'`: Record traces only when retrying for all retries. +* `'retain-on-first-failure'`: Record traces only when the test fails for the first time. For more control, pass an object that specifies `mode` and trace features to enable. diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts index d24add96d3..fc67d5353e 100644 --- a/packages/playwright/src/program.ts +++ b/packages/playwright/src/program.ts @@ -290,7 +290,7 @@ function resolveReporter(id: string) { return require.resolve(id, { paths: [process.cwd()] }); } -const kTraceModes: TraceMode[] = ['on', 'off', 'on-first-retry', 'on-all-retries', 'retain-on-failure']; +const kTraceModes: TraceMode[] = ['on', 'off', 'on-first-retry', 'on-all-retries', 'retain-on-failure', 'retain-on-first-failure']; const testOptions: [string, string][] = [ ['--browser ', `Browser to use for tests, one of "all", "chromium", "firefox" or "webkit" (default: "chromium")`], diff --git a/packages/playwright/src/worker/testTracing.ts b/packages/playwright/src/worker/testTracing.ts index 579bf8bc87..82c84000e7 100644 --- a/packages/playwright/src/worker/testTracing.ts +++ b/packages/playwright/src/worker/testTracing.ts @@ -48,8 +48,31 @@ export class TestTracing { this._tracesDir = path.join(this._artifactsDir, 'traces'); } + private _shouldCaptureTrace() { + if (process.env.PW_TEST_DISABLE_TRACING) + return false; + + if (this._options?.mode === 'on') + return true; + + if (this._options?.mode === 'retain-on-failure') + return true; + + if (this._options?.mode === 'on-first-retry' && this._testInfo.retry === 1) + return true; + + if (this._options?.mode === 'on-all-retries' && this._testInfo.retry > 0) + return true; + + if (this._options?.mode === 'retain-on-first-failure' && this._testInfo.retry === 0) + return true; + + return false; + } + async startIfNeeded(value: TraceFixtureValue) { const defaultTraceOptions: TraceOptions = { screenshots: true, snapshots: true, sources: true, attachments: true, _live: false, mode: 'off' }; + if (!value) { this._options = defaultTraceOptions; } else if (typeof value === 'string') { @@ -59,9 +82,7 @@ export class TestTracing { this._options = { ...defaultTraceOptions, ...value, mode: (mode as string) === 'retry-with-trace' ? 'on-first-retry' : mode }; } - let shouldCaptureTrace = this._options.mode === 'on' || this._options.mode === 'retain-on-failure' || (this._options.mode === 'on-first-retry' && this._testInfo.retry === 1) || (this._options.mode === 'on-all-retries' && this._testInfo.retry > 0); - shouldCaptureTrace = shouldCaptureTrace && !process.env.PW_TEST_DISABLE_TRACING; - if (!shouldCaptureTrace) { + if (!this._shouldCaptureTrace()) { this._options = undefined; return; } @@ -110,7 +131,8 @@ export class TestTracing { return; const testFailed = this._testInfo.status !== this._testInfo.expectedStatus; - const shouldAbandonTrace = !testFailed && this._options.mode === 'retain-on-failure'; + const shouldAbandonTrace = !testFailed && (this._options.mode === 'retain-on-failure' || this._options.mode === 'retain-on-first-failure'); + if (shouldAbandonTrace) { for (const file of this._temporaryTraceFiles) await fs.promises.unlink(file).catch(() => {}); diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index 7e329ae595..11ceb7153a 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -5586,6 +5586,7 @@ export interface PlaywrightWorkerOptions { * - `'retain-on-failure'`: Record trace for each test, but remove all traces from successful test runs. * - `'on-first-retry'`: Record trace only when retrying a test for the first time. * - `'on-all-retries'`: Record traces only when retrying for all retries. + * - `'retain-on-first-failure'`: Record traces only when the test fails for the first time. * * For more control, pass an object that specifies `mode` and trace features to enable. * @@ -5636,7 +5637,7 @@ export interface PlaywrightWorkerOptions { } export type ScreenshotMode = 'off' | 'on' | 'only-on-failure'; -export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries'; +export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure'; export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry'; /** @@ -7099,7 +7100,8 @@ type MergedExpect = Expect>; export function mergeExpects(...expects: List): MergedExpect; // This is required to not export everything by default. See https://github.com/Microsoft/TypeScript/issues/19545#issuecomment-340490459 -export {}; +export { }; + /** diff --git a/tests/page/pageTestApi.ts b/tests/page/pageTestApi.ts index 77340dfb2c..a7b124a84b 100644 --- a/tests/page/pageTestApi.ts +++ b/tests/page/pageTestApi.ts @@ -27,7 +27,7 @@ export type PageWorkerFixtures = { headless: boolean; channel: string; screenshot: ScreenshotMode | { mode: ScreenshotMode } & Pick; - trace: 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | /** deprecated */ 'retry-with-trace'; + trace: 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'retain-on-first-failure' | 'on-all-retries' | /** deprecated */ 'retry-with-trace'; video: VideoMode | { mode: VideoMode, size: ViewportSize }; browserName: 'chromium' | 'firefox' | 'webkit'; browserVersion: string; diff --git a/tests/playwright-test/playwright.artifacts.spec.ts b/tests/playwright-test/playwright.artifacts.spec.ts index 5a54d8f981..8a6db7fd2e 100644 --- a/tests/playwright-test/playwright.artifacts.spec.ts +++ b/tests/playwright-test/playwright.artifacts.spec.ts @@ -338,6 +338,31 @@ test('should work with trace: on-all-retries', async ({ runInlineTest }, testInf ]); }); +test('should work with trace: retain-on-first-failure', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + ...testFiles, + 'playwright.config.ts': ` + module.exports = { use: { trace: 'retain-on-first-failure' } }; + `, + }, { workers: 1, retries: 2 }); + + expect(result.exitCode).toBe(1); + expect(result.passed).toBe(5); + expect(result.failed).toBe(5); + expect(listFiles(testInfo.outputPath('test-results'))).toEqual([ + 'artifacts-failing', + ' trace.zip', + 'artifacts-own-context-failing', + ' trace.zip', + 'artifacts-persistent-failing', + ' trace.zip', + 'artifacts-shared-shared-failing', + ' trace.zip', + 'artifacts-two-contexts-failing', + ' trace.zip', + ]); +}); + test('should take screenshot when page is closed in afterEach', async ({ runInlineTest }, testInfo) => { const result = await runInlineTest({ 'playwright.config.ts': ` diff --git a/tests/playwright-test/playwright.trace.spec.ts b/tests/playwright-test/playwright.trace.spec.ts index 69f1912752..41ce802500 100644 --- a/tests/playwright-test/playwright.trace.spec.ts +++ b/tests/playwright-test/playwright.trace.spec.ts @@ -133,7 +133,6 @@ test('should record api trace', async ({ runInlineTest, server }, testInfo) => { ]); }); - test('should not throw with trace: on-first-retry and two retries in the same worker', async ({ runInlineTest }, testInfo) => { const files = {}; for (let i = 0; i < 6; i++) { @@ -402,7 +401,7 @@ test('should respect PW_TEST_DISABLE_TRACING', async ({ runInlineTest }, testInf expect(fs.existsSync(testInfo.outputPath('test-results', 'a-test-1', 'trace.zip'))).toBe(false); }); -for (const mode of ['off', 'retain-on-failure', 'on-first-retry', 'on-all-retries']) { +for (const mode of ['off', 'retain-on-failure', 'on-first-retry', 'on-all-retries', 'retain-on-first-failure']) { test(`trace:${mode} should not create trace zip artifact if page test passed`, async ({ runInlineTest }) => { const result = await runInlineTest({ 'a.spec.ts': ` @@ -1034,3 +1033,77 @@ test('should attribute worker fixture teardown to the right test', async ({ runI ' step in foo teardown', ]); }); + +test('trace:retain-on-first-failure should create trace but only on first failure', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('fail', async ({ page }) => { + await page.goto('about:blank'); + expect(true).toBe(false); + }); + `, + }, { trace: 'retain-on-first-failure', retries: 1 }); + + const retryTracePath = test.info().outputPath('test-results', 'a-fail-retry1', 'trace.zip'); + const retryTraceExists = fs.existsSync(retryTracePath); + expect(retryTraceExists).toBe(false); + + const tracePath = test.info().outputPath('test-results', 'a-fail', 'trace.zip'); + const trace = await parseTrace(tracePath); + expect(trace.apiNames).toContain('page.goto'); + expect(result.failed).toBe(1); +}); + +test('trace:retain-on-first-failure should create trace if context is closed before failure in the test', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('fail', async ({ page, context }) => { + await page.goto('about:blank'); + await context.close(); + expect(1).toBe(2); + }); + `, + }, { trace: 'retain-on-first-failure' }); + const tracePath = test.info().outputPath('test-results', 'a-fail', 'trace.zip'); + const trace = await parseTrace(tracePath); + expect(trace.apiNames).toContain('page.goto'); + expect(result.failed).toBe(1); +}); + +test('trace:retain-on-first-failure should create trace if context is closed before failure in afterEach', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('fail', async ({ page, context }) => { + }); + test.afterEach(async ({ page, context }) => { + await page.goto('about:blank'); + await context.close(); + expect(1).toBe(2); + }); + `, + }, { trace: 'retain-on-first-failure' }); + const tracePath = test.info().outputPath('test-results', 'a-fail', 'trace.zip'); + const trace = await parseTrace(tracePath); + expect(trace.apiNames).toContain('page.goto'); + expect(result.failed).toBe(1); +}); + +test('trace:retain-on-first-failure should create trace if request context is disposed before failure', async ({ runInlineTest, server }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('fail', async ({ request }) => { + expect(await request.get('${server.EMPTY_PAGE}')).toBeOK(); + await request.dispose(); + expect(1).toBe(2); + }); + `, + }, { trace: 'retain-on-first-failure' }); + const tracePath = test.info().outputPath('test-results', 'a-fail', 'trace.zip'); + const trace = await parseTrace(tracePath); + expect(trace.apiNames).toContain('apiRequestContext.get'); + expect(result.failed).toBe(1); +}); diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index fd88e84dca..eed775d21f 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -248,7 +248,7 @@ export interface PlaywrightWorkerOptions { } export type ScreenshotMode = 'off' | 'on' | 'only-on-failure'; -export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries'; +export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure'; export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry'; export interface PlaywrightTestOptions { @@ -484,4 +484,5 @@ type MergedExpect = Expect>; export function mergeExpects(...expects: List): MergedExpect; // This is required to not export everything by default. See https://github.com/Microsoft/TypeScript/issues/19545#issuecomment-340490459 -export {}; +export { }; +