From fd24521f2e11d0672852a7fe867bc3d01b8bcf94 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Fri, 7 Feb 2025 09:02:20 +0100 Subject: [PATCH] chore: add page snapshot on test end (#34573) Signed-off-by: Simon Knott Co-authored-by: Yury Semikhatsky --- packages/playwright/src/index.ts | 219 ++++++++++-------- .../src/isomorphic/testServerInterface.ts | 1 + packages/playwright/src/runner/testServer.ts | 1 + .../playwright.artifacts.spec.ts | 68 ++++++ 4 files changed, 198 insertions(+), 91 deletions(-) diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.ts index 83913c18dc..f956d8d8ef 100644 --- a/packages/playwright/src/index.ts +++ b/packages/playwright/src/index.ts @@ -54,6 +54,7 @@ type WorkerFixtures = PlaywrightWorkerArgs & PlaywrightWorkerOptions & { _optionContextReuseMode: ContextReuseMode, _optionConnectOptions: PlaywrightWorkerOptions['connectOptions'], _reuseContext: boolean, + _pageSnapshot: PageSnapshotOption, }; const playwrightFixtures: Fixtures = ({ @@ -71,6 +72,7 @@ const playwrightFixtures: Fixtures = ({ screenshot: ['off', { scope: 'worker', option: true }], video: ['off', { scope: 'worker', option: true }], trace: ['off', { scope: 'worker', option: true }], + _pageSnapshot: ['off', { scope: 'worker', option: true }], _browserOptions: [async ({ playwright, headless, channel, launchOptions }, use) => { const options: LaunchOptions = { @@ -247,13 +249,13 @@ const playwrightFixtures: Fixtures = ({ } }, { auto: 'all-hooks-included', title: 'context configuration', box: true } as any], - _setupArtifacts: [async ({ playwright, screenshot }, use, testInfo) => { + _setupArtifacts: [async ({ playwright, screenshot, _pageSnapshot }, use, testInfo) => { // This fixture has a separate zero-timeout slot to ensure that artifact collection // happens even after some fixtures or hooks time out. // Now that default test timeout is known, we can replace zero with an actual value. testInfo.setTimeout(testInfo.project.timeout); - const artifactsRecorder = new ArtifactsRecorder(playwright, tracing().artifactsDir(), screenshot); + const artifactsRecorder = new ArtifactsRecorder(playwright, tracing().artifactsDir(), screenshot, _pageSnapshot); await artifactsRecorder.willStartTest(testInfo as TestInfoImpl); const tracingGroupSteps: TestStepInternal[] = []; @@ -452,6 +454,7 @@ const playwrightFixtures: Fixtures = ({ type ScreenshotOption = PlaywrightWorkerOptions['screenshot'] | undefined; type Playwright = PlaywrightWorkerArgs['playwright']; +type PageSnapshotOption = 'off' | 'on' | 'only-on-failure'; function normalizeVideoMode(video: VideoMode | 'retry-with-video' | { mode: VideoMode } | undefined): VideoMode { if (!video) @@ -523,41 +526,132 @@ function connectOptionsFromEnv() { }; } -class ArtifactsRecorder { - private _testInfo!: TestInfoImpl; - private _playwright: Playwright; - private _artifactsDir: string; - private _screenshotMode: ScreenshotMode; - private _screenshotOptions: { mode: ScreenshotMode } & Pick | undefined; - private _temporaryScreenshots: string[] = []; - private _temporaryArtifacts: string[] = []; - private _reusedContexts = new Set(); - private _screenshotOrdinal = 0; - private _screenshottedSymbol: symbol; - private _startedCollectingArtifacts: symbol; +class SnapshotRecorder { + private _ordinal = 0; + private _temporary: string[] = []; + private _snapshottedSymbol = Symbol('snapshotted'); - constructor(playwright: Playwright, artifactsDir: string, screenshot: ScreenshotOption) { - this._playwright = playwright; - this._artifactsDir = artifactsDir; - this._screenshotMode = normalizeScreenshotMode(screenshot); - this._screenshotOptions = typeof screenshot === 'string' ? undefined : screenshot; - this._screenshottedSymbol = Symbol('screenshotted'); - this._startedCollectingArtifacts = Symbol('startedCollectingArtifacts'); + constructor( + private _artifactsRecorder: ArtifactsRecorder, + private _mode: ScreenshotMode | PageSnapshotOption, + private _name: string, + private _contentType: string, + private _extension: string, + private _doSnapshot: (page: Page, path: string) => Promise) { + } + + fixOrdinal() { + // Since beforeAll(s), test and afterAll(s) reuse the same TestInfo, make sure we do not + // overwrite previous screenshots. + this._ordinal = this.testInfo.attachments.filter(a => a.name === this._name).length; + } + + private shouldCaptureUponFinish() { + return this._mode === 'on' || + (this._mode === 'only-on-failure' && this.testInfo._isFailure()) || + (this._mode === 'on-first-failure' && this.testInfo._isFailure() && this.testInfo.retry === 0); + } + + async maybeCapture() { + if (!this.shouldCaptureUponFinish()) + return; + + const contexts: BrowserContext[] = []; + const playwright = this._artifactsRecorder._playwright; + for (const browserType of [playwright.chromium, playwright.firefox, playwright.webkit]) + contexts.push(...(browserType as any)._contexts); + + await Promise.all(contexts.flatMap(context => context.pages().map(page => this._snapshotPage(page, false)))); + } + + async persistTemporary() { + if (this.shouldCaptureUponFinish()) { + await Promise.all(this._temporary.map(async file => { + try { + const path = this._createAttachmentPath(); + await fs.promises.rename(file, path); + this._attach(path); + } catch { + } + })); + } + } + + async captureTemporary(context: BrowserContext) { + if (this._mode === 'on' || this._mode === 'only-on-failure' || (this._mode === 'on-first-failure' && this.testInfo.retry === 0)) + await Promise.all(context.pages().map(page => this._snapshotPage(page, true))); + } + + private _attach(screenshotPath: string) { + this.testInfo.attachments.push({ name: this._name, path: screenshotPath, contentType: this._contentType }); + } + + private _createAttachmentPath() { + const testFailed = this.testInfo._isFailure(); + const index = this._ordinal + 1; + ++this._ordinal; + const path = this.testInfo.outputPath(`test-${testFailed ? 'failed' : 'finished'}-${index}${this._extension}`); + return path; } private _createTemporaryArtifact(...name: string[]) { - const file = path.join(this._artifactsDir, ...name); - this._temporaryArtifacts.push(file); + const file = path.join(this._artifactsRecorder._artifactsDir, ...name); return file; } + private async _snapshotPage(page: Page, temporary: boolean) { + if ((page as any)[this._snapshottedSymbol]) + return; + (page as any)[this._snapshottedSymbol] = true; + try { + const path = temporary ? this._createTemporaryArtifact(createGuid() + this._extension) : this._createAttachmentPath(); + await this._doSnapshot(page, path); + if (temporary) + this._temporary.push(path); + else + this._attach(path); + } catch { + // snapshot may fail, just ignore. + } + } + + private get testInfo(): TestInfoImpl { + return this._artifactsRecorder._testInfo; + } +} + +class ArtifactsRecorder { + _testInfo!: TestInfoImpl; + _playwright: Playwright; + _artifactsDir: string; + private _reusedContexts = new Set(); + private _startedCollectingArtifacts: symbol; + + private _pageSnapshotRecorder: SnapshotRecorder; + private _screenshotRecorder: SnapshotRecorder; + + constructor(playwright: Playwright, artifactsDir: string, screenshot: ScreenshotOption, pageSnapshot: PageSnapshotOption) { + this._playwright = playwright; + this._artifactsDir = artifactsDir; + const screenshotOptions = typeof screenshot === 'string' ? undefined : screenshot; + this._startedCollectingArtifacts = Symbol('startedCollectingArtifacts'); + + this._screenshotRecorder = new SnapshotRecorder(this, normalizeScreenshotMode(screenshot), 'screenshot', 'image/png', '.png', async (page, path) => { + await page.screenshot({ ...screenshotOptions, timeout: 5000, path, caret: 'initial' }); + }); + + this._pageSnapshotRecorder = new SnapshotRecorder(this, pageSnapshot, 'pageSnapshot', 'text/plain', '.ariasnapshot', async (page, path) => { + const ariaSnapshot = await page.locator('body').ariaSnapshot(); + await fs.promises.writeFile(path, ariaSnapshot); + }); + } + async willStartTest(testInfo: TestInfoImpl) { this._testInfo = testInfo; testInfo._onDidFinishTestFunction = () => this.didFinishTestFunction(); - // Since beforeAll(s), test and afterAll(s) reuse the same TestInfo, make sure we do not - // overwrite previous screenshots. - this._screenshotOrdinal = testInfo.attachments.filter(a => a.name === 'screenshot').length; + this._screenshotRecorder.fixOrdinal(); + this._pageSnapshotRecorder.fixOrdinal(); // Process existing contexts. for (const browserType of [this._playwright.chromium, this._playwright.firefox, this._playwright.webkit]) { @@ -587,11 +681,9 @@ class ArtifactsRecorder { if (this._reusedContexts.has(context)) return; await this._stopTracing(context.tracing); - if (this._screenshotMode === 'on' || this._screenshotMode === 'only-on-failure' || (this._screenshotMode === 'on-first-failure' && this._testInfo.retry === 0)) { - // Capture screenshot for now. We'll know whether we have to preserve them - // after the test finishes. - await Promise.all(context.pages().map(page => this._screenshotPage(page, true))); - } + + await this._screenshotRecorder.captureTemporary(context); + await this._pageSnapshotRecorder.captureTemporary(context); } async didCreateRequestContext(context: APIRequestContext) { @@ -604,21 +696,13 @@ class ArtifactsRecorder { await this._stopTracing(tracing); } - private _shouldCaptureScreenshotUponFinish() { - return this._screenshotMode === 'on' || - (this._screenshotMode === 'only-on-failure' && this._testInfo._isFailure()) || - (this._screenshotMode === 'on-first-failure' && this._testInfo._isFailure() && this._testInfo.retry === 0); - } - async didFinishTestFunction() { - if (this._shouldCaptureScreenshotUponFinish()) - await this._screenshotOnTestFailure(); + await this._screenshotRecorder.maybeCapture(); + await this._pageSnapshotRecorder.maybeCapture(); } async didFinishTest() { - const captureScreenshots = this._shouldCaptureScreenshotUponFinish(); - if (captureScreenshots) - await this._screenshotOnTestFailure(); + await this.didFinishTestFunction(); let leftoverContexts: BrowserContext[] = []; for (const browserType of [this._playwright.chromium, this._playwright.firefox, this._playwright.webkit]) @@ -634,55 +718,8 @@ class ArtifactsRecorder { await this._stopTracing(tracing); }))); - // Attach temporary screenshots for contexts closed before collecting the test trace. - if (captureScreenshots) { - for (const file of this._temporaryScreenshots) { - try { - const screenshotPath = this._createScreenshotAttachmentPath(); - await fs.promises.rename(file, screenshotPath); - this._attachScreenshot(screenshotPath); - } catch { - } - } - } - } - - private _createScreenshotAttachmentPath() { - const testFailed = this._testInfo._isFailure(); - const index = this._screenshotOrdinal + 1; - ++this._screenshotOrdinal; - const screenshotPath = this._testInfo.outputPath(`test-${testFailed ? 'failed' : 'finished'}-${index}.png`); - return screenshotPath; - } - - private async _screenshotPage(page: Page, temporary: boolean) { - if ((page as any)[this._screenshottedSymbol]) - return; - (page as any)[this._screenshottedSymbol] = true; - try { - const screenshotPath = temporary ? this._createTemporaryArtifact(createGuid() + '.png') : this._createScreenshotAttachmentPath(); - // Pass caret=initial to avoid any evaluations that might slow down the screenshot - // and let the page modify itself from the problematic state it had at the moment of failure. - await page.screenshot({ ...this._screenshotOptions, timeout: 5000, path: screenshotPath, caret: 'initial' }); - if (temporary) - this._temporaryScreenshots.push(screenshotPath); - else - this._attachScreenshot(screenshotPath); - } catch { - // Screenshot may fail, just ignore. - } - } - - private _attachScreenshot(screenshotPath: string) { - this._testInfo.attachments.push({ name: 'screenshot', path: screenshotPath, contentType: 'image/png' }); - } - - private async _screenshotOnTestFailure() { - const contexts: BrowserContext[] = []; - for (const browserType of [this._playwright.chromium, this._playwright.firefox, this._playwright.webkit]) - contexts.push(...(browserType as any)._contexts); - const pages = contexts.map(ctx => ctx.pages()).flat(); - await Promise.all(pages.map(page => this._screenshotPage(page, false))); + await this._screenshotRecorder.persistTemporary(); + await this._pageSnapshotRecorder.persistTemporary(); } private async _startTraceChunkOnContextCreation(tracing: Tracing) { diff --git a/packages/playwright/src/isomorphic/testServerInterface.ts b/packages/playwright/src/isomorphic/testServerInterface.ts index 694610ecdd..e4faa682ac 100644 --- a/packages/playwright/src/isomorphic/testServerInterface.ts +++ b/packages/playwright/src/isomorphic/testServerInterface.ts @@ -96,6 +96,7 @@ export interface TestServerInterface { workers?: number | string; updateSnapshots?: 'all' | 'changed' | 'missing' | 'none'; updateSourceMethod?: 'overwrite' | 'patch' | '3way'; + pageSnapshot?: 'off' | 'on' | 'only-on-failure'; reporters?: string[], trace?: 'on' | 'off'; video?: 'on' | 'off'; diff --git a/packages/playwright/src/runner/testServer.ts b/packages/playwright/src/runner/testServer.ts index e4ff3a7a2f..db080728e9 100644 --- a/packages/playwright/src/runner/testServer.ts +++ b/packages/playwright/src/runner/testServer.ts @@ -311,6 +311,7 @@ export class TestServerDispatcher implements TestServerInterface { ...(params.headed !== undefined ? { headless: !params.headed } : {}), _optionContextReuseMode: params.reuseContext ? 'when-possible' : undefined, _optionConnectOptions: params.connectWsEndpoint ? { wsEndpoint: params.connectWsEndpoint } : undefined, + _pageSnapshot: params.pageSnapshot, }, ...(params.updateSnapshots ? { updateSnapshots: params.updateSnapshots } : {}), ...(params.updateSourceMethod ? { updateSourceMethod: params.updateSourceMethod } : {}), diff --git a/tests/playwright-test/playwright.artifacts.spec.ts b/tests/playwright-test/playwright.artifacts.spec.ts index 2e3d99766b..0360ca7bce 100644 --- a/tests/playwright-test/playwright.artifacts.spec.ts +++ b/tests/playwright-test/playwright.artifacts.spec.ts @@ -420,3 +420,71 @@ test('should take screenshot when page is closed in afterEach', async ({ runInli expect(result.failed).toBe(1); expect(fs.existsSync(testInfo.outputPath('test-results', 'a-fails', 'test-failed-1.png'))).toBeTruthy(); }); + +test('should work with _pageSnapshot: on', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + ...testFiles, + 'playwright.config.ts': ` + module.exports = { use: { _pageSnapshot: 'on' } }; + `, + }, { workers: 1 }); + + expect(result.exitCode).toBe(1); + expect(result.passed).toBe(5); + expect(result.failed).toBe(5); + expect(listFiles(testInfo.outputPath('test-results'))).toEqual([ + '.last-run.json', + 'artifacts-failing', + ' test-failed-1.ariasnapshot', + 'artifacts-own-context-failing', + ' test-failed-1.ariasnapshot', + 'artifacts-own-context-passing', + ' test-finished-1.ariasnapshot', + 'artifacts-passing', + ' test-finished-1.ariasnapshot', + 'artifacts-persistent-failing', + ' test-failed-1.ariasnapshot', + 'artifacts-persistent-passing', + ' test-finished-1.ariasnapshot', + 'artifacts-shared-shared-failing', + ' test-failed-1.ariasnapshot', + ' test-failed-2.ariasnapshot', + 'artifacts-shared-shared-passing', + ' test-finished-1.ariasnapshot', + ' test-finished-2.ariasnapshot', + 'artifacts-two-contexts', + ' test-finished-1.ariasnapshot', + ' test-finished-2.ariasnapshot', + 'artifacts-two-contexts-failing', + ' test-failed-1.ariasnapshot', + ' test-failed-2.ariasnapshot', + ]); +}); + +test('should work with _pageSnapshot: only-on-failure', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + ...testFiles, + 'playwright.config.ts': ` + module.exports = { use: { _pageSnapshot: 'only-on-failure' } }; + `, + }, { workers: 1 }); + + expect(result.exitCode).toBe(1); + expect(result.passed).toBe(5); + expect(result.failed).toBe(5); + expect(listFiles(testInfo.outputPath('test-results'))).toEqual([ + '.last-run.json', + 'artifacts-failing', + ' test-failed-1.ariasnapshot', + 'artifacts-own-context-failing', + ' test-failed-1.ariasnapshot', + 'artifacts-persistent-failing', + ' test-failed-1.ariasnapshot', + 'artifacts-shared-shared-failing', + ' test-failed-1.ariasnapshot', + ' test-failed-2.ariasnapshot', + 'artifacts-two-contexts-failing', + ' test-failed-1.ariasnapshot', + ' test-failed-2.ariasnapshot', + ]); +});