feat(trace-viewer): render iframe canvas in trace viewer (#33809)

packages/playwright-core/src/server/trace/recorder/snapshotterInjected.ts

@@ -438,13 +438,13 @@ export function frameSnapshotStreamer(snapshotStreamer: string, removeNoScript:
             expectValue(value);
             attrs[kSelectedAttribute] = value;
           }
-          if (nodeName === 'CANVAS') {
+          if (nodeName === 'CANVAS' || nodeName === 'IFRAME' || nodeName === 'FRAME') {
-            const boundingRect = (element as HTMLCanvasElement).getBoundingClientRect();
+            const boundingRect = (element as HTMLElement).getBoundingClientRect();
             const value = JSON.stringify({
-              left: boundingRect.left / window.innerWidth,
+              left: boundingRect.left,
-              top: boundingRect.top / window.innerHeight,
+              top: boundingRect.top,
-              right: boundingRect.right / window.innerWidth,
+              right: boundingRect.right,
-              bottom: boundingRect.bottom / window.innerHeight
+              bottom: boundingRect.bottom
             });
             expectValue(kBoundingRectAttribute);
             expectValue(value);

packages/trace-viewer/src/sw/snapshotRenderer.ts

@@ -152,7 +152,7 @@ export class SnapshotRenderer {
       const html = prefix + [
         // Hide the document in order to prevent flickering. We will unhide once script has processed shadow.
         '<style>*,*::before,*::after { visibility: hidden }</style>',
-        `<script>${snapshotScript(this._callId, this.snapshotName)}</script>`
+        `<script>${snapshotScript(this.viewport(), this._callId, this.snapshotName)}</script>`
       ].join('') + result.join('');
       return { value: html, size: html.length };
     });
@@ -236,10 +236,39 @@ function snapshotNodes(snapshot: FrameSnapshot): NodeSnapshot[] {
   return (snapshot as any)._nodes;
 }
 
-function snapshotScript(...targetIds: (string | undefined)[]) {
+type ViewportSize = { width: number, height: number };
-  function applyPlaywrightAttributes(unwrapPopoutUrl: (url: string) => string, ...targetIds: (string | undefined)[]) {
+type BoundingRect = { left: number, top: number, right: number, bottom: number };
+type FrameBoundingRectsInfo = {
+  viewport: ViewportSize;
+  frames: WeakMap<Element, {
+    boundingRect: BoundingRect;
+    scrollLeft: number;
+    scrollTop: number;
+  }>;
+};
+
+declare global {
+  interface Window {
+    __playwright_frame_bounding_rects__: FrameBoundingRectsInfo;
+  }
+}
+
+function snapshotScript(viewport: ViewportSize, ...targetIds: (string | undefined)[]) {
+  function applyPlaywrightAttributes(unwrapPopoutUrl: (url: string) => string, viewport: ViewportSize, ...targetIds: (string | undefined)[]) {
     const isUnderTest = new URLSearchParams(location.search).has('isUnderTest');
 
+    // info to recursively compute canvas position relative to the top snapshot frame.
+    // Before rendering each iframe, its parent extracts the '__playwright_canvas_render_info__' attribute
+    // value and keeps in this variable. It can then remove the attribute and render the element,
+    // which will eventually trigger the same process inside the iframe recursively.
+    // When there's a canvas to render, we iterate over its ancestor frames to compute
+    // its position relative to the top snapshot frame.
+    const frameBoundingRectsInfo = {
+      viewport,
+      frames: new WeakMap(),
+    };
+    window['__playwright_frame_bounding_rects__'] = frameBoundingRectsInfo;
+
     const kPointerWarningTitle = 'Recorded click position in absolute coordinates did not' +
         ' match the center of the clicked element. This is likely due to a difference between' +
         ' the test runner and the trace viewer operating systems.';
@@ -249,6 +278,10 @@ function snapshotScript(...targetIds: (string | undefined)[]) {
     const targetElements: Element[] = [];
     const canvasElements: HTMLCanvasElement[] = [];
 
+    let topSnapshotWindow: Window = window;
+    while (topSnapshotWindow !== topSnapshotWindow.parent && !topSnapshotWindow.location.pathname.match(/\/page@[a-z0-9]+$/))
+      topSnapshotWindow = topSnapshotWindow.parent;
+
     const visit = (root: Document | ShadowRoot) => {
       // Collect all scrolled elements for later use.
       for (const e of root.querySelectorAll(`[__playwright_scroll_top_]`))
@@ -288,6 +321,11 @@ function snapshotScript(...targetIds: (string | undefined)[]) {
       }
 
       for (const iframe of root.querySelectorAll('iframe, frame')) {
+        const boundingRectJson = iframe.getAttribute('__playwright_bounding_rect__');
+        iframe.removeAttribute('__playwright_bounding_rect__');
+        const boundingRect = boundingRectJson ? JSON.parse(boundingRectJson) : undefined;
+        if (boundingRect)
+          frameBoundingRectsInfo.frames.set(iframe, { boundingRect, scrollLeft: 0, scrollTop: 0 });
         const src = iframe.getAttribute('__playwright_src__');
         if (!src) {
           iframe.setAttribute('src', 'data:text/html,<body style="background: #ddd"></body>');
@@ -339,16 +377,20 @@ function snapshotScript(...targetIds: (string | undefined)[]) {
       for (const element of scrollTops) {
         element.scrollTop = +element.getAttribute('__playwright_scroll_top_')!;
         element.removeAttribute('__playwright_scroll_top_');
+        if (frameBoundingRectsInfo.frames.has(element))
+          frameBoundingRectsInfo.frames.get(element)!.scrollTop = element.scrollTop;
       }
       for (const element of scrollLefts) {
         element.scrollLeft = +element.getAttribute('__playwright_scroll_left_')!;
         element.removeAttribute('__playwright_scroll_left_');
+        if (frameBoundingRectsInfo.frames.has(element))
+          frameBoundingRectsInfo.frames.get(element)!.scrollLeft = element.scrollTop;
       }
 
       document.styleSheets[0].disabled = true;
 
       const search = new URL(window.location.href).searchParams;
-      const isTopFrame = window.location.pathname.match(/\/page@[a-z0-9]+$/);
+      const isTopFrame = window === topSnapshotWindow;
 
       if (search.get('pointX') && search.get('pointY')) {
         const pointX = +search.get('pointX')!;
@@ -419,16 +461,6 @@ function snapshotScript(...targetIds: (string | undefined)[]) {
           context.fillRect(0, 0, canvas.width, canvas.height);
         }
 
-
-        if (!isTopFrame) {
-          for (const canvas of canvasElements) {
-            const context = canvas.getContext('2d')!;
-            drawCheckerboard(context, canvas);
-            canvas.title = `Playwright displays canvas contents on a best-effort basis. It doesn't support canvas elements inside an iframe yet. If this impacts your workflow, please open an issue so we can prioritize.`;
-          }
-          return;
-        }
-
         const img = new Image();
         img.onload = () => {
           for (const canvas of canvasElements) {
@@ -446,6 +478,31 @@ function snapshotScript(...targetIds: (string | undefined)[]) {
               continue;
             }
 
+            let currWindow: Window = window;
+            while (currWindow !== topSnapshotWindow) {
+              const iframe = currWindow.frameElement!;
+              currWindow = currWindow.parent;
+
+              const iframeInfo = currWindow['__playwright_frame_bounding_rects__']?.frames.get(iframe);
+              if (!iframeInfo?.boundingRect)
+                break;
+
+              const leftOffset = iframeInfo.boundingRect.left - iframeInfo.scrollLeft;
+              const topOffset = iframeInfo.boundingRect.top - iframeInfo.scrollTop;
+
+              boundingRect.left += leftOffset;
+              boundingRect.top += topOffset;
+              boundingRect.right += leftOffset;
+              boundingRect.bottom += topOffset;
+            }
+
+            const { width, height } = topSnapshotWindow['__playwright_frame_bounding_rects__'].viewport;
+
+            boundingRect.left = boundingRect.left / width;
+            boundingRect.top = boundingRect.top / height;
+            boundingRect.right = boundingRect.right / width;
+            boundingRect.bottom = boundingRect.bottom / height;
+
             const partiallyUncaptured = boundingRect.right > 1 || boundingRect.bottom > 1;
             const fullyUncaptured = boundingRect.left > 1 || boundingRect.top > 1;
             if (fullyUncaptured) {
@@ -483,7 +540,7 @@ function snapshotScript(...targetIds: (string | undefined)[]) {
     window.addEventListener('DOMContentLoaded', onDOMContentLoaded);
   }
 
-  return `\n(${applyPlaywrightAttributes.toString()})(${unwrapPopoutUrl.toString()}${targetIds.map(id => `, "${id}"`).join('')})`;
+  return `\n(${applyPlaywrightAttributes.toString()})(${unwrapPopoutUrl.toString()}, ${JSON.stringify(viewport)}${targetIds.map(id => `, "${id}"`).join('')})`;
 }
 
 

tests/library/snapshotter.spec.ts

@@ -268,10 +268,12 @@ it.describe('snapshots', () => {
   });
 });
 
-function distillSnapshot(snapshot, distillTarget = true) {
+function distillSnapshot(snapshot, options: { distillTarget: boolean, distillBoundingRect: boolean } = { distillTarget: true, distillBoundingRect: true }) {
   let { html } = snapshot.render();
-  if (distillTarget)
+  if (options.distillTarget)
     html = html.replace(/\s__playwright_target__="[^"]+"/g, '');
+  if (options.distillBoundingRect)
+    html = html.replace(/\s__playwright_bounding_rect__="[^"]+"/g, '');
   return html
       .replace(/<style>\*,\*::before,\*::after { visibility: hidden }<\/style>/, '')
       .replace(/<script>[.\s\S]+<\/script>/, '')

tests/library/trace-viewer.spec.ts

@@ -1539,12 +1539,16 @@ test('canvas clipping in iframe', async ({ runAndTrace, page, server }) => {
     await page.setContent(`
       <iframe src="${server.PREFIX}/screenshots/canvas.html#canvas-on-edge"></iframe>
     `);
+    await page.locator('iframe').contentFrame().locator('canvas').scrollIntoViewIfNeeded();
     await rafraf(page, 5);
   });
 
+  const msg = await traceViewer.page.waitForEvent('console', { predicate: msg => msg.text().startsWith('canvas drawn:') });
+  expect(msg.text()).toEqual('canvas drawn: [1,1,11,20]');
+
   const snapshot = await traceViewer.snapshotFrame('page.evaluate');
   const canvas = snapshot.locator('iframe').contentFrame().locator('canvas');
-  await expect(canvas).toHaveAttribute('title', `Playwright displays canvas contents on a best-effort basis. It doesn't support canvas elements inside an iframe yet. If this impacts your workflow, please open an issue so we can prioritize.`);
+  await expect(canvas).toHaveAttribute('title', 'Canvas contents are displayed on a best-effort basis based on viewport screenshots taken during test execution.');
 });
 
 test('should show only one pointer with multilevel iframes', async ({ page, runAndTrace, server, browserName }) => {