From 1bedf56705b3be60372a1ec039e06872f88ba5fd Mon Sep 17 00:00:00 2001 From: Rui Figueira Date: Thu, 28 Nov 2024 23:41:04 +0000 Subject: [PATCH] feat(trace-viewer): render iframe canvas in trace viewer Closes: #33779 --- .../trace/recorder/snapshotterInjected.ts | 12 +-- .../trace-viewer/src/sw/snapshotRenderer.ts | 85 +++++++++++++++---- tests/library/trace-viewer.spec.ts | 6 +- 3 files changed, 81 insertions(+), 22 deletions(-) diff --git a/packages/playwright-core/src/server/trace/recorder/snapshotterInjected.ts b/packages/playwright-core/src/server/trace/recorder/snapshotterInjected.ts index 4d575ef0e9..4730c3e5dd 100644 --- a/packages/playwright-core/src/server/trace/recorder/snapshotterInjected.ts +++ b/packages/playwright-core/src/server/trace/recorder/snapshotterInjected.ts @@ -438,13 +438,13 @@ export function frameSnapshotStreamer(snapshotStreamer: string, removeNoScript: expectValue(value); attrs[kSelectedAttribute] = value; } - if (nodeName === 'CANVAS') { - const boundingRect = (element as HTMLCanvasElement).getBoundingClientRect(); + if (nodeName === 'CANVAS' || nodeName === 'IFRAME' || nodeName === 'FRAME') { + const boundingRect = (element as HTMLElement).getBoundingClientRect(); const value = JSON.stringify({ - left: boundingRect.left / window.innerWidth, - top: boundingRect.top / window.innerHeight, - right: boundingRect.right / window.innerWidth, - bottom: boundingRect.bottom / window.innerHeight + left: boundingRect.left, + top: boundingRect.top, + right: boundingRect.right, + bottom: boundingRect.bottom }); expectValue(kBoundingRectAttribute); expectValue(value); diff --git a/packages/trace-viewer/src/sw/snapshotRenderer.ts b/packages/trace-viewer/src/sw/snapshotRenderer.ts index 93363878ab..2dd2ced436 100644 --- a/packages/trace-viewer/src/sw/snapshotRenderer.ts +++ b/packages/trace-viewer/src/sw/snapshotRenderer.ts @@ -152,7 +152,7 @@ export class SnapshotRenderer { const html = prefix + [ // Hide the document in order to prevent flickering. We will unhide once script has processed shadow. '', - `` + `` ].join('') + result.join(''); return { value: html, size: html.length }; }); @@ -236,10 +236,33 @@ function snapshotNodes(snapshot: FrameSnapshot): NodeSnapshot[] { return (snapshot as any)._nodes; } -function snapshotScript(...targetIds: (string | undefined)[]) { - function applyPlaywrightAttributes(unwrapPopoutUrl: (url: string) => string, ...targetIds: (string | undefined)[]) { +type ViewportSize = { width: number, height: number }; +type BoundingRect = { left: number, top: number, right: number, bottom: number }; +type CanvasRenderInfo = { + viewport: ViewportSize; + frames: WeakMap; +}; + +declare global { + interface Window { + __playwright_canvas_render_info__: CanvasRenderInfo; + } +} + +function snapshotScript(viewport: ViewportSize, ...targetIds: (string | undefined)[]) { + function applyPlaywrightAttributes(unwrapPopoutUrl: (url: string) => string, viewport: ViewportSize, ...targetIds: (string | undefined)[]) { const isUnderTest = new URLSearchParams(location.search).has('isUnderTest'); + const canvasRenderInfo = { + viewport, + frames: new WeakMap(), + }; + window['__playwright_canvas_render_info__'] = canvasRenderInfo; + const kPointerWarningTitle = 'Recorded click position in absolute coordinates did not' + ' match the center of the clicked element. This is likely due to a difference between' + ' the test runner and the trace viewer operating systems.'; @@ -249,6 +272,10 @@ function snapshotScript(...targetIds: (string | undefined)[]) { const targetElements: Element[] = []; const canvasElements: HTMLCanvasElement[] = []; + let topFrameWindow: Window = window; + while (topFrameWindow !== topFrameWindow.parent && !topFrameWindow.location.pathname.match(/\/page@[a-z0-9]+$/)) + topFrameWindow = topFrameWindow.parent; + const visit = (root: Document | ShadowRoot) => { // Collect all scrolled elements for later use. for (const e of root.querySelectorAll(`[__playwright_scroll_top_]`)) @@ -288,6 +315,10 @@ function snapshotScript(...targetIds: (string | undefined)[]) { } for (const iframe of root.querySelectorAll('iframe, frame')) { + const boundingRectJson = iframe.getAttribute('__playwright_bounding_rect__'); + iframe.removeAttribute('__playwright_bounding_rect__'); + const boundingRect = boundingRectJson ? JSON.parse(boundingRectJson) : undefined; + canvasRenderInfo.frames.set(iframe, { boundingRect, scrollLeft: 0, scrollTop: 0 }); const src = iframe.getAttribute('__playwright_src__'); if (!src) { iframe.setAttribute('src', 'data:text/html,'); @@ -339,16 +370,20 @@ function snapshotScript(...targetIds: (string | undefined)[]) { for (const element of scrollTops) { element.scrollTop = +element.getAttribute('__playwright_scroll_top_')!; element.removeAttribute('__playwright_scroll_top_'); + if (canvasRenderInfo.frames.has(element)) + canvasRenderInfo.frames.get(element)!.scrollTop = element.scrollTop; } for (const element of scrollLefts) { element.scrollLeft = +element.getAttribute('__playwright_scroll_left_')!; element.removeAttribute('__playwright_scroll_left_'); + if (canvasRenderInfo.frames.has(element)) + canvasRenderInfo.frames.get(element)!.scrollLeft = element.scrollTop; } document.styleSheets[0].disabled = true; const search = new URL(window.location.href).searchParams; - const isTopFrame = window.location.pathname.match(/\/page@[a-z0-9]+$/); + const isTopFrame = window === topFrameWindow; if (search.get('pointX') && search.get('pointY')) { const pointX = +search.get('pointX')!; @@ -419,16 +454,6 @@ function snapshotScript(...targetIds: (string | undefined)[]) { context.fillRect(0, 0, canvas.width, canvas.height); } - - if (!isTopFrame) { - for (const canvas of canvasElements) { - const context = canvas.getContext('2d')!; - drawCheckerboard(context, canvas); - canvas.title = `Playwright displays canvas contents on a best-effort basis. It doesn't support canvas elements inside an iframe yet. If this impacts your workflow, please open an issue so we can prioritize.`; - } - return; - } - const img = new Image(); img.onload = () => { for (const canvas of canvasElements) { @@ -446,6 +471,36 @@ function snapshotScript(...targetIds: (string | undefined)[]) { continue; } + let currWindow: Window = window; + while (currWindow !== topFrameWindow) { + const iframe = currWindow.frameElement!; + currWindow = currWindow.parent; + + const currCanvasRenderInfo = currWindow['__playwright_canvas_render_info__']; + const iframeRenderInfo = currCanvasRenderInfo.frames.get(iframe); + + // eslint-disable-next-line no-console + console.log('frame canvas info', iframe, currCanvasRenderInfo); + + if (!iframeRenderInfo?.boundingRect) + break; + + const leftOffset = iframeRenderInfo.boundingRect.left - iframeRenderInfo.scrollLeft; + const topOffset = iframeRenderInfo.boundingRect.top - iframeRenderInfo.scrollTop; + + boundingRect.left += leftOffset; + boundingRect.top += topOffset; + boundingRect.right += leftOffset; + boundingRect.bottom += topOffset; + } + + const { width, height } = topFrameWindow['__playwright_canvas_render_info__'].viewport; + + boundingRect.left = boundingRect.left / width; + boundingRect.top = boundingRect.top / height; + boundingRect.right = boundingRect.right / width; + boundingRect.bottom = boundingRect.bottom / height; + const partiallyUncaptured = boundingRect.right > 1 || boundingRect.bottom > 1; const fullyUncaptured = boundingRect.left > 1 || boundingRect.top > 1; if (fullyUncaptured) { @@ -483,7 +538,7 @@ function snapshotScript(...targetIds: (string | undefined)[]) { window.addEventListener('DOMContentLoaded', onDOMContentLoaded); } - return `\n(${applyPlaywrightAttributes.toString()})(${unwrapPopoutUrl.toString()}${targetIds.map(id => `, "${id}"`).join('')})`; + return `\n(${applyPlaywrightAttributes.toString()})(${unwrapPopoutUrl.toString()}, ${JSON.stringify(viewport)}${targetIds.map(id => `, "${id}"`).join('')})`; } diff --git a/tests/library/trace-viewer.spec.ts b/tests/library/trace-viewer.spec.ts index 2bd7b85888..3ffba28582 100644 --- a/tests/library/trace-viewer.spec.ts +++ b/tests/library/trace-viewer.spec.ts @@ -1539,12 +1539,16 @@ test('canvas clipping in iframe', async ({ runAndTrace, page, server }) => { await page.setContent(` `); + await page.locator('iframe').contentFrame().locator('canvas').scrollIntoViewIfNeeded(); await rafraf(page, 5); }); + const msg = await traceViewer.page.waitForEvent('console', { predicate: msg => msg.text().startsWith('canvas drawn:') }); + expect(msg.text()).toEqual('canvas drawn: [1,1,11,20]'); + const snapshot = await traceViewer.snapshotFrame('page.evaluate'); const canvas = snapshot.locator('iframe').contentFrame().locator('canvas'); - await expect(canvas).toHaveAttribute('title', `Playwright displays canvas contents on a best-effort basis. It doesn't support canvas elements inside an iframe yet. If this impacts your workflow, please open an issue so we can prioritize.`); + await expect(canvas).toHaveAttribute('title', 'Canvas contents are displayed on a best-effort basis based on viewport screenshots taken during test execution.'); }); test('should show only one pointer with multilevel iframes', async ({ page, runAndTrace, server, browserName }) => {