From ef84051c919577fcb4b0f4084de24e16176fb9cf Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Tue, 22 Oct 2024 14:12:25 +0200 Subject: [PATCH] feat(tracing): clip `canvas` contents from screenshots (#33119) --- .../server/trace/test/inMemorySnapshotter.ts | 2 +- packages/trace-viewer/src/sw/main.ts | 7 ++ .../trace-viewer/src/sw/snapshotRenderer.ts | 99 ++++++++++++++++++- .../trace-viewer/src/sw/snapshotServer.ts | 9 ++ .../trace-viewer/src/sw/snapshotStorage.ts | 5 +- .../trace-viewer/src/sw/traceModernizer.ts | 2 +- packages/trace-viewer/src/ui/snapshotTab.tsx | 4 + tests/assets/screenshots/canvas.html | 3 + tests/library/trace-viewer.spec.ts | 27 +++++ 9 files changed, 152 insertions(+), 6 deletions(-) diff --git a/packages/playwright-core/src/server/trace/test/inMemorySnapshotter.ts b/packages/playwright-core/src/server/trace/test/inMemorySnapshotter.ts index f28c5ef03b..8de07c2aad 100644 --- a/packages/playwright-core/src/server/trace/test/inMemorySnapshotter.ts +++ b/packages/playwright-core/src/server/trace/test/inMemorySnapshotter.ts @@ -85,7 +85,7 @@ export class InMemorySnapshotter implements SnapshotterDelegate, HarTracerDelega onFrameSnapshot(snapshot: FrameSnapshot): void { ++this._snapshotCount; - const renderer = this._storage.addFrameSnapshot(snapshot); + const renderer = this._storage.addFrameSnapshot(snapshot, []); this._snapshotReadyPromises.get(snapshot.snapshotName || '')?.resolve(renderer); } diff --git a/packages/trace-viewer/src/sw/main.ts b/packages/trace-viewer/src/sw/main.ts index 7888aa6a30..6773b9fb76 100644 --- a/packages/trace-viewer/src/sw/main.ts +++ b/packages/trace-viewer/src/sw/main.ts @@ -129,6 +129,13 @@ async function doFetch(event: FetchEvent): Promise { return response; } + if (relativePath.startsWith('/closest-screenshot/')) { + const { snapshotServer } = loadedTraces.get(traceUrl!) || {}; + if (!snapshotServer) + return new Response(null, { status: 404 }); + return snapshotServer.serveClosestScreenshot(relativePath, url.searchParams); + } + if (relativePath.startsWith('/sha1/')) { // Sha1 for sources is based on the file path, can't load it of a random model. const sha1 = relativePath.slice('/sha1/'.length); diff --git a/packages/trace-viewer/src/sw/snapshotRenderer.ts b/packages/trace-viewer/src/sw/snapshotRenderer.ts index 0d2f2af502..438683ed84 100644 --- a/packages/trace-viewer/src/sw/snapshotRenderer.ts +++ b/packages/trace-viewer/src/sw/snapshotRenderer.ts @@ -16,6 +16,16 @@ import { escapeHTMLAttribute, escapeHTML } from '@isomorphic/stringUtils'; import type { FrameSnapshot, NodeNameAttributesChildNodesSnapshot, NodeSnapshot, RenderedFrameSnapshot, ResourceSnapshot, SubtreeReferenceSnapshot } from '@trace/snapshot'; +import type { PageEntry } from '../types/entries'; + +function findClosest(items: T[], metric: (v: T) => number, target: number) { + return items.find((item, index) => { + if (index === items.length - 1) + return true; + const next = items[index + 1]; + return Math.abs(metric(item) - target) < Math.abs(metric(next) - target); + }); +} function isNodeNameAttributesChildNodesSnapshot(n: NodeSnapshot): n is NodeNameAttributesChildNodesSnapshot { return Array.isArray(n) && typeof n[0] === 'string'; @@ -60,13 +70,15 @@ export class SnapshotRenderer { private _resources: ResourceSnapshot[]; private _snapshot: FrameSnapshot; private _callId: string; + private _screencastFrames: PageEntry['screencastFrames']; - constructor(resources: ResourceSnapshot[], snapshots: FrameSnapshot[], index: number) { + constructor(resources: ResourceSnapshot[], snapshots: FrameSnapshot[], screencastFrames: PageEntry['screencastFrames'], index: number) { this._resources = resources; this._snapshots = snapshots; this._index = index; this._snapshot = snapshots[index]; this._callId = snapshots[index].callId; + this._screencastFrames = screencastFrames; this.snapshotName = snapshots[index].snapshotName; } @@ -78,6 +90,14 @@ export class SnapshotRenderer { return this._snapshots[this._index].viewport; } + closestScreenshot(): string | undefined { + const { wallTime, timestamp } = this.snapshot(); + const closestFrame = (wallTime && this._screencastFrames[0]?.frameSwapWallTime) + ? findClosest(this._screencastFrames, frame => frame.frameSwapWallTime!, wallTime) + : findClosest(this._screencastFrames, frame => frame.timestamp, timestamp); + return closestFrame?.sha1; + } + render(): RenderedFrameSnapshot { const result: string[] = []; const visit = (n: NodeSnapshot, snapshotIndex: number, parentTag: string | undefined, parentAttrs: [string, string][] | undefined) => { @@ -244,6 +264,8 @@ function snapshotNodes(snapshot: FrameSnapshot): NodeSnapshot[] { function snapshotScript(...targetIds: (string | undefined)[]) { function applyPlaywrightAttributes(unwrapPopoutUrl: (url: string) => string, ...targetIds: (string | undefined)[]) { + const isUnderTest = new URLSearchParams(location.search).has('isUnderTest'); + const kPointerWarningTitle = 'Recorded click position in absolute coordinates did not' + ' match the center of the clicked element. This is likely due to a difference between' + ' the test runner and the trace viewer operating systems.'; @@ -251,6 +273,7 @@ function snapshotScript(...targetIds: (string | undefined)[]) { const scrollTops: Element[] = []; const scrollLefts: Element[] = []; const targetElements: Element[] = []; + const canvasElements: HTMLCanvasElement[] = []; const visit = (root: Document | ShadowRoot) => { // Collect all scrolled elements for later use. @@ -326,6 +349,8 @@ function snapshotScript(...targetIds: (string | undefined)[]) { } (root as any).adoptedStyleSheets = adoptedSheets; } + + canvasElements.push(...root.querySelectorAll('canvas')); }; const onLoad = () => { @@ -342,12 +367,12 @@ function snapshotScript(...targetIds: (string | undefined)[]) { document.styleSheets[0].disabled = true; const search = new URL(window.location.href).searchParams; + const isTopFrame = window.location.pathname.match(/\/page@[a-z0-9]+$/); if (search.get('pointX') && search.get('pointY')) { const pointX = +search.get('pointX')!; const pointY = +search.get('pointY')!; const hasInputTarget = search.has('hasInputTarget'); - const isTopFrame = window.location.pathname.match(/\/page@[a-z0-9]+$/); const hasTargetElements = targetElements.length > 0; const roots = document.documentElement ? [document.documentElement] : []; for (const target of (hasTargetElements ? targetElements : roots)) { @@ -393,6 +418,76 @@ function snapshotScript(...targetIds: (string | undefined)[]) { } } } + + if (canvasElements.length > 0) { + function drawCheckerboard(context: CanvasRenderingContext2D, canvas: HTMLCanvasElement) { + function createCheckerboardPattern() { + const pattern = document.createElement('canvas'); + pattern.width = pattern.width / Math.floor(pattern.width / 24); + pattern.height = pattern.height / Math.floor(pattern.height / 24); + const context = pattern.getContext('2d')!; + context.fillStyle = 'lightgray'; + context.fillRect(0, 0, pattern.width, pattern.height); + context.fillStyle = 'white'; + context.fillRect(0, 0, pattern.width / 2, pattern.height / 2); + context.fillRect(pattern.width / 2, pattern.height / 2, pattern.width, pattern.height); + return context.createPattern(pattern, 'repeat')!; + } + + context.fillStyle = createCheckerboardPattern(); + context.fillRect(0, 0, canvas.width, canvas.height); + } + + + if (!isTopFrame) { + for (const canvas of canvasElements) { + const context = canvas.getContext('2d')!; + drawCheckerboard(context, canvas); + canvas.title = `Playwright displays canvas contents on a best-effort basis. It doesn't support canvas elements inside an iframe yet. If this impacts your workflow, please open an issue so we can prioritize.`; + } + return; + } + + const img = new Image(); + img.onload = () => { + for (const canvas of canvasElements) { + const context = canvas.getContext('2d')!; + + const boundingRect = canvas.getBoundingClientRect(); + const xStart = boundingRect.left / window.innerWidth; + const yStart = boundingRect.top / window.innerHeight; + const xEnd = boundingRect.right / window.innerWidth; + const yEnd = boundingRect.bottom / window.innerHeight; + + const partiallyUncaptured = xEnd > 1 || yEnd > 1; + const fullyUncaptured = xStart > 1 || yStart > 1; + if (fullyUncaptured) { + canvas.title = `Playwright couldn't capture canvas contents because it's located outside the viewport.`; + continue; + } + + drawCheckerboard(context, canvas); + + context.drawImage(img, xStart * img.width, yStart * img.height, (xEnd - xStart) * img.width, (yEnd - yStart) * img.height, 0, 0, canvas.width, canvas.height); + if (isUnderTest) + // eslint-disable-next-line no-console + console.log(`canvas drawn:`, JSON.stringify([xStart, yStart, xEnd, yEnd].map(v => Math.floor(v * 100)))); + + if (partiallyUncaptured) + canvas.title = `Playwright couldn't capture full canvas contents because it's located partially outside the viewport.`; + else + canvas.title = `Canvas contents are displayed on a best-effort basis based on viewport screenshots taken during test execution.`; + } + }; + img.onerror = () => { + for (const canvas of canvasElements) { + const context = canvas.getContext('2d')!; + drawCheckerboard(context, canvas); + canvas.title = `Playwright couldn't show canvas contents because the screenshot failed to load.`; + } + }; + img.src = location.href.replace('/snapshot', '/closest-screenshot'); + } }; const onDOMContentLoaded = () => visit(document); diff --git a/packages/trace-viewer/src/sw/snapshotServer.ts b/packages/trace-viewer/src/sw/snapshotServer.ts index 4b61104d33..e1978c79b6 100644 --- a/packages/trace-viewer/src/sw/snapshotServer.ts +++ b/packages/trace-viewer/src/sw/snapshotServer.ts @@ -35,11 +35,20 @@ export class SnapshotServer { const snapshot = this._snapshot(pathname.substring('/snapshot'.length), searchParams); if (!snapshot) return new Response(null, { status: 404 }); + const renderedSnapshot = snapshot.render(); this._snapshotIds.set(snapshotUrl, snapshot); return new Response(renderedSnapshot.html, { status: 200, headers: { 'Content-Type': 'text/html; charset=utf-8' } }); } + async serveClosestScreenshot(pathname: string, searchParams: URLSearchParams): Promise { + const snapshot = this._snapshot(pathname.substring('/closest-screenshot'.length), searchParams); + const sha1 = snapshot?.closestScreenshot(); + if (!sha1) + return new Response(null, { status: 404 }); + return new Response(await this._resourceLoader(sha1)); + } + serveSnapshotInfo(pathname: string, searchParams: URLSearchParams): Response { const snapshot = this._snapshot(pathname.substring('/snapshotInfo'.length), searchParams); return this._respondWithJson(snapshot ? { diff --git a/packages/trace-viewer/src/sw/snapshotStorage.ts b/packages/trace-viewer/src/sw/snapshotStorage.ts index 9f4aea60c2..5e10fc97fb 100644 --- a/packages/trace-viewer/src/sw/snapshotStorage.ts +++ b/packages/trace-viewer/src/sw/snapshotStorage.ts @@ -16,6 +16,7 @@ import type { FrameSnapshot, ResourceSnapshot } from '@trace/snapshot'; import { rewriteURLForCustomProtocol, SnapshotRenderer } from './snapshotRenderer'; +import type { PageEntry } from '../types/entries'; export class SnapshotStorage { private _resources: ResourceSnapshot[] = []; @@ -29,7 +30,7 @@ export class SnapshotStorage { this._resources.push(resource); } - addFrameSnapshot(snapshot: FrameSnapshot) { + addFrameSnapshot(snapshot: FrameSnapshot, screencastFrames: PageEntry['screencastFrames']) { for (const override of snapshot.resourceOverrides) override.url = rewriteURLForCustomProtocol(override.url); let frameSnapshots = this._frameSnapshots.get(snapshot.frameId); @@ -43,7 +44,7 @@ export class SnapshotStorage { this._frameSnapshots.set(snapshot.pageId, frameSnapshots); } frameSnapshots.raw.push(snapshot); - const renderer = new SnapshotRenderer(this._resources, frameSnapshots.raw, frameSnapshots.raw.length - 1); + const renderer = new SnapshotRenderer(this._resources, frameSnapshots.raw, screencastFrames, frameSnapshots.raw.length - 1); frameSnapshots.renderers.push(renderer); return renderer; } diff --git a/packages/trace-viewer/src/sw/traceModernizer.ts b/packages/trace-viewer/src/sw/traceModernizer.ts index 69d7f965dc..80f98762db 100644 --- a/packages/trace-viewer/src/sw/traceModernizer.ts +++ b/packages/trace-viewer/src/sw/traceModernizer.ts @@ -159,7 +159,7 @@ export class TraceModernizer { contextEntry.resources.push(event.snapshot); break; case 'frame-snapshot': - this._snapshotStorage.addFrameSnapshot(event.snapshot); + this._snapshotStorage.addFrameSnapshot(event.snapshot, this._pageEntry(event.snapshot.pageId).screencastFrames); break; } // Make sure there is a page entry for each page, even without screencast frames, diff --git a/packages/trace-viewer/src/ui/snapshotTab.tsx b/packages/trace-viewer/src/ui/snapshotTab.tsx index 82e4282c6f..f8bcc3a5a3 100644 --- a/packages/trace-viewer/src/ui/snapshotTab.tsx +++ b/packages/trace-viewer/src/ui/snapshotTab.tsx @@ -369,10 +369,14 @@ export function collectSnapshots(action: ActionTraceEvent | undefined): Snapshot return { action: actionSnapshot, before: beforeSnapshot, after: afterSnapshot }; } +const isUnderTest = new URLSearchParams(window.location.search).has('isUnderTest'); + export function extendSnapshot(snapshot: Snapshot): SnapshotUrls { const params = new URLSearchParams(); params.set('trace', context(snapshot.action).traceUrl); params.set('name', snapshot.snapshotName); + if (isUnderTest) + params.set('isUnderTest', 'true'); if (snapshot.point) { params.set('pointX', String(snapshot.point.x)); params.set('pointY', String(snapshot.point.y)); diff --git a/tests/assets/screenshots/canvas.html b/tests/assets/screenshots/canvas.html index 011148c5ff..a0ebc55829 100644 --- a/tests/assets/screenshots/canvas.html +++ b/tests/assets/screenshots/canvas.html @@ -7,4 +7,7 @@ ctx.fillRect(25, 25, 100, 100); ctx.clearRect(45, 45, 60, 60); ctx.strokeRect(50, 50, 50, 50); + + if (location.hash.includes('canvas-on-edge')) + canvas.style.marginTop = '90vh'; diff --git a/tests/library/trace-viewer.spec.ts b/tests/library/trace-viewer.spec.ts index aa794eb716..0ae5716e2f 100644 --- a/tests/library/trace-viewer.spec.ts +++ b/tests/library/trace-viewer.spec.ts @@ -21,6 +21,7 @@ import path from 'path'; import { pathToFileURL } from 'url'; import { expect, playwrightTest } from '../config/browserTest'; import type { FrameLocator } from '@playwright/test'; +import { rafraf } from 'tests/page/pageTest'; const test = playwrightTest.extend(traceViewerFixtures); @@ -1439,6 +1440,32 @@ test.skip('should allow showing screenshots instead of snapshots', async ({ runA await expect(screenshot).toBeVisible(); }); +test('canvas clipping', async ({ runAndTrace, page, server }) => { + const traceViewer = await runAndTrace(async () => { + await page.goto(server.PREFIX + '/screenshots/canvas.html#canvas-on-edge'); + await rafraf(page, 5); + }); + + const msg = await traceViewer.page.waitForEvent('console', { predicate: msg => msg.text().startsWith('canvas drawn:') }); + expect(msg.text()).toEqual('canvas drawn: [0,91,12,111]'); + + const snapshot = await traceViewer.snapshotFrame('page.goto'); + await expect(snapshot.locator('canvas')).toHaveAttribute('title', `Playwright couldn't capture full canvas contents because it's located partially outside the viewport.`); +}); + +test('canvas clipping in iframe', async ({ runAndTrace, page, server }) => { + const traceViewer = await runAndTrace(async () => { + await page.setContent(` + + `); + await rafraf(page, 5); + }); + + const snapshot = await traceViewer.snapshotFrame('page.evaluate'); + const canvas = snapshot.locator('iframe').contentFrame().locator('canvas'); + await expect(canvas).toHaveAttribute('title', `Playwright displays canvas contents on a best-effort basis. It doesn't support canvas elements inside an iframe yet. If this impacts your workflow, please open an issue so we can prioritize.`); +}); + test.skip('should handle case where neither snapshots nor screenshots exist', async ({ runAndTrace, page, server }) => { const traceViewer = await runAndTrace(async () => { await page.goto(server.PREFIX + '/one-style.html');