From 7b64161a372a9e057df79c4159fb3fef6756a000 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Mon, 8 Nov 2021 15:39:58 -0800 Subject: [PATCH] feat(test-runner): allow specifying fine-grained trace options (#10147) --- docs/src/test-api/class-testoptions.md | 18 +++--- packages/playwright-test/src/index.ts | 14 +++-- packages/playwright-test/types/test.d.ts | 17 +++--- .../playwright-test/playwright.trace.spec.ts | 57 ++++++++++++++++++- utils/generate_types/overrides-test.d.ts | 7 ++- 5 files changed, 89 insertions(+), 24 deletions(-) diff --git a/docs/src/test-api/class-testoptions.md b/docs/src/test-api/class-testoptions.md index 76f150a7c3..5a9fd78d37 100644 --- a/docs/src/test-api/class-testoptions.md +++ b/docs/src/test-api/class-testoptions.md @@ -166,13 +166,17 @@ Learn more about [automatic screenshots](./test-configuration.md#automatic-scree ## property: TestOptions.timezoneId = %%-context-option-timezoneid-%% ## property: TestOptions.trace -- type: <[Screenshot]<"off"|"on"|"retain-on-failure"|"on-first-retry">> +- type: <[Object]|[TraceMode]<"off"|"on"|"retain-on-failure"|"on-first-retry">> + - `mode` <[TraceMode]<"off"|"on"|"retain-on-failure"|"on-first-retry">> Trace recording mode. + - `screenshots` <[boolean]> Whether to capture screenshots during tracing. Screenshots are used to build a timeline preview. Defaults to true. Optional. + - `snapshots` <[boolean]> Whether to capture DOM snapshot on every action. Defaults to true. Optional. + - `sources` <[boolean]> Whether to include source files for trace actions. Defaults to true. Optional. -Whether to record a trace for each test. Defaults to `'off'`. -* `'off'`: Do not record a trace. -* `'on'`: Record a trace for each test. -* `'retain-on-failure'`: Record a trace for each test, but remove it from successful test runs. -* `'on-first-retry'`: Record a trace only when retrying a test for the first time. +Whether to record trace for each test. Defaults to `'off'`. +* `'off'`: Do not record trace. +* `'on'`: Record trace for each test. +* `'retain-on-failure'`: Record trace for each test, but remove all traces from successful test runs. +* `'on-first-retry'`: Record trace only when retrying a test for the first time. Learn more about [recording trace](./test-configuration.md#record-test-trace). @@ -181,7 +185,7 @@ Learn more about [recording trace](./test-configuration.md#record-test-trace). ## property: TestOptions.video - type: <[Object]|[VideoMode]<"off"|"on"|"retain-on-failure"|"on-first-retry">> - `mode` <[VideoMode]<"off"|"on"|"retain-on-failure"|"on-first-retry">> Video recording mode. - - `size` <[Object]> Size of the recorded video. + - `size` <[Object]> Size of the recorded video. Optional. - `width` <[int]> - `height` <[int]> diff --git a/packages/playwright-test/src/index.ts b/packages/playwright-test/src/index.ts index a31ceb2dfa..13b421be7a 100644 --- a/packages/playwright-test/src/index.ts +++ b/packages/playwright-test/src/index.ts @@ -176,9 +176,13 @@ export const test = _baseTest.extend({ if (process.env.PWDEBUG) testInfo.setTimeout(0); - if (trace === 'retry-with-trace') - trace = 'on-first-retry'; - const captureTrace = (trace === 'on' || trace === 'retain-on-failure' || (trace === 'on-first-retry' && testInfo.retry === 1)); + let traceMode = typeof trace === 'string' ? trace : trace.mode; + if (traceMode as any === 'retry-with-trace') + traceMode = 'on-first-retry'; + const defaultTraceOptions = { screenshots: true, snapshots: true, sources: true }; + const traceOptions = typeof trace === 'string' ? defaultTraceOptions : { ...defaultTraceOptions, ...trace, mode: undefined }; + + const captureTrace = (traceMode === 'on' || traceMode === 'retain-on-failure' || (traceMode === 'on-first-retry' && testInfo.retry === 1)); const temporaryTraceFiles: string[] = []; const temporaryScreenshots: string[] = []; @@ -188,7 +192,7 @@ export const test = _baseTest.extend({ if (captureTrace) { const title = [path.relative(testInfo.project.testDir, testInfo.file) + ':' + testInfo.line, ...testInfo.titlePath.slice(1)].join(' › '); if (!(context.tracing as any)[kTracingStarted]) { - await context.tracing.start({ screenshots: true, snapshots: true, sources: true, title }); + await context.tracing.start({ ...traceOptions, title }); (context.tracing as any)[kTracingStarted] = true; } else { await context.tracing.startChunk({ title }); @@ -253,7 +257,7 @@ export const test = _baseTest.extend({ // 3. Determine whether we need the artifacts. const testFailed = testInfo.status !== testInfo.expectedStatus; const isHook = !!hookType(testInfo); - const preserveTrace = captureTrace && !isHook && (trace === 'on' || (testFailed && trace === 'retain-on-failure') || (trace === 'on-first-retry' && testInfo.retry === 1)); + const preserveTrace = captureTrace && !isHook && (traceMode === 'on' || (testFailed && traceMode === 'retain-on-failure') || (traceMode === 'on-first-retry' && testInfo.retry === 1)); const captureScreenshots = !isHook && (screenshot === 'on' || (screenshot === 'only-on-failure' && testFailed)); const traceAttachments: string[] = []; diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index d5e722237f..9e88be609f 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -2686,15 +2686,15 @@ export interface PlaywrightWorkerOptions { */ screenshot: 'off' | 'on' | 'only-on-failure'; /** - * Whether to record a trace for each test. Defaults to `'off'`. - * - `'off'`: Do not record a trace. - * - `'on'`: Record a trace for each test. - * - `'retain-on-failure'`: Record a trace for each test, but remove it from successful test runs. - * - `'on-first-retry'`: Record a trace only when retrying a test for the first time. + * Whether to record trace for each test. Defaults to `'off'`. + * - `'off'`: Do not record trace. + * - `'on'`: Record trace for each test. + * - `'retain-on-failure'`: Record trace for each test, but remove all traces from successful test runs. + * - `'on-first-retry'`: Record trace only when retrying a test for the first time. * * Learn more about [recording trace](https://playwright.dev/docs/test-configuration#record-test-trace). */ - trace: 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | /** deprecated */ 'retry-with-trace'; + trace: TraceMode | /** deprecated */ 'retry-with-trace' | { mode: TraceMode, snapshots?: boolean, screenshots?: boolean, sources?: boolean }; /** * Whether to record video for each test. Defaults to `'off'`. * - `'off'`: Do not record video. @@ -2704,10 +2704,11 @@ export interface PlaywrightWorkerOptions { * * Learn more about [recording video](https://playwright.dev/docs/test-configuration#record-video). */ - video: VideoMode | { mode: VideoMode, size: ViewportSize }; + video: VideoMode | /** deprecated */ 'retry-with-video' | { mode: VideoMode, size?: ViewportSize }; } -export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | /** deprecated */ 'retry-with-video'; +export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry'; +export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry'; /** * Playwright Test provides many options to configure test environment, [Browser], [BrowserContext] and more. diff --git a/tests/playwright-test/playwright.trace.spec.ts b/tests/playwright-test/playwright.trace.spec.ts index 61f7bce694..4206991472 100644 --- a/tests/playwright-test/playwright.trace.spec.ts +++ b/tests/playwright-test/playwright.trace.spec.ts @@ -15,6 +15,7 @@ */ import { test, expect, stripAscii } from './playwright-test-fixtures'; +import { ZipFileSystem } from '../../packages/playwright-core/lib/utils/vfs'; import fs from 'fs'; test('should stop tracing with trace: on-first-retry, when not retrying', async ({ runInlineTest }, testInfo) => { @@ -92,7 +93,7 @@ test('should not throw with trace: on-first-retry and two retries in the same wo test('should not throw with trace and timeouts', async ({ runInlineTest }, testInfo) => { const result = await runInlineTest({ 'playwright.config.ts': ` - module.exports = { timeout: 2000, repeatEach: 20, use: { trace: 'on' } }; + module.exports = { timeout: 2000, repeatEach: 10, use: { trace: 'on' } }; `, 'a.spec.ts': ` const { test } = pwt; @@ -110,3 +111,57 @@ test('should not throw with trace and timeouts', async ({ runInlineTest }, testI expect(stripAscii(result.output)).not.toContain('tracing.stopChunk:'); expect(stripAscii(result.output)).not.toContain('tracing.stop:'); }); + +test('should save sources when requested', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + use: { + trace: 'on', + } + }; + `, + 'a.spec.ts': ` + const { test } = pwt; + test('pass', async ({ page }) => { + await page.evaluate(2 + 2); + }); + `, + }, { workers: 1 }); + expect(result.exitCode).toEqual(0); + const resources = await parseTrace(testInfo.outputPath('test-results', 'a-pass', 'trace.zip')); + expect([...resources.keys()].filter(f => f.includes('src@'))).toHaveLength(1); +}); + +test('should not save sources when not requested', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + use: { + trace: { + mode: 'on', + sources: false, + } + } + }; + `, + 'a.spec.ts': ` + const { test } = pwt; + test('pass', async ({ page }) => { + await page.evaluate(2 + 2); + }); + `, + }, { workers: 1 }); + expect(result.exitCode).toEqual(0); + const resources = await parseTrace(testInfo.outputPath('test-results', 'a-pass', 'trace.zip')); + expect([...resources.keys()].filter(f => f.includes('src@'))).toHaveLength(0); +}); + +async function parseTrace(file: string): Promise> { + const zipFS = new ZipFileSystem(file); + const resources = new Map(); + for (const entry of await zipFS.entries()) + resources.set(entry, await zipFS.read(entry)); + zipFS.close(); + return resources; +} diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 610a1493a1..3d9f0c8389 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -300,11 +300,12 @@ export interface PlaywrightWorkerOptions { channel: BrowserChannel | undefined; launchOptions: LaunchOptions; screenshot: 'off' | 'on' | 'only-on-failure'; - trace: 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | /** deprecated */ 'retry-with-trace'; - video: VideoMode | { mode: VideoMode, size: ViewportSize }; + trace: TraceMode | /** deprecated */ 'retry-with-trace' | { mode: TraceMode, snapshots?: boolean, screenshots?: boolean, sources?: boolean }; + video: VideoMode | /** deprecated */ 'retry-with-video' | { mode: VideoMode, size?: ViewportSize }; } -export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | /** deprecated */ 'retry-with-video'; +export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry'; +export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry'; export interface PlaywrightTestOptions { acceptDownloads: boolean | undefined;