feat(screencast): scale fixes (#6475)

browsers.json

@@ -8,17 +8,17 @@
     },
     {
       "name": "firefox",
-      "revision": "1257",
+      "revision": "1258",
       "installByDefault": true
     },
     {
       "name": "firefox-stable",
-      "revision": "1247",
+      "revision": "1248",
       "installByDefault": false
     },
     {
       "name": "webkit",
-      "revision": "1472",
+      "revision": "1476",
       "installByDefault": true,
       "revisionOverrides": {
         "mac10.14": "1443"

src/server/chromium/crPage.ts

@@ -289,13 +289,13 @@ export class CRPage implements PageDelegate {
     return this._sessionForHandle(handle)._scrollRectIntoViewIfNeeded(handle, rect);
   }
 
-  async setScreencastEnabled(enabled: boolean): Promise<void> {
-    if (enabled) {
+  async setScreencastOptions(options: { width: number, height: number, quality: number } | null): Promise<void> {
+    if (options) {
       await this._mainFrameSession._startScreencast(this, {
         format: 'jpeg',
-        quality: 90,
-        maxWidth: 800,
-        maxHeight: 600,
+        quality: options.quality,
+        maxWidth: options.width,
+        maxHeight: options.height
       });
     } else {
       await this._mainFrameSession._stopScreencast(this);

src/server/firefox/ffPage.ts

@@ -479,9 +479,9 @@ export class FFPage implements PageDelegate {
     });
   }
 
-  async setScreencastEnabled(enabled: boolean): Promise<void> {
-    if (enabled) {
-      const { screencastId } = await this._session.send('Page.startScreencast', { width: 800, height: 600, quality: 70 });
+  async setScreencastOptions(options: { width: number, height: number, quality: number } | null): Promise<void> {
+    if (options) {
+      const { screencastId } = await this._session.send('Page.startScreencast', options);
       this._screencastId = screencastId;
     } else {
       await this._session.send('Page.stopScreencast');

src/server/page.ts

@@ -70,7 +70,7 @@ export interface PageDelegate {
   getBoundingBox(handle: dom.ElementHandle): Promise<types.Rect | null>;
   getFrameElement(frame: frames.Frame): Promise<dom.ElementHandle>;
   scrollRectIntoViewIfNeeded(handle: dom.ElementHandle, rect?: types.Rect): Promise<'error:notvisible' | 'error:notconnected' | 'done'>;
-  setScreencastEnabled(enabled: boolean): Promise<void>;
+  setScreencastOptions(options: { width: number, height: number, quality: number } | null): Promise<void>;
 
   getAccessibilityTree(needle?: dom.ElementHandle): Promise<{tree: accessibility.AXNode, needle: accessibility.AXNode | null}>;
   pdf?: (options?: types.PDFOptions) => Promise<Buffer>;
@@ -501,8 +501,8 @@ export class Page extends SdkObject {
     return this._pageBindings.get(identifier) || this._browserContext._pageBindings.get(identifier);
   }
 
-  setScreencastEnabled(enabled: boolean) {
-    this._delegate.setScreencastEnabled(enabled).catch(e => debugLogger.log('error', e));
+  setScreencastOptions(options: { width: number, height: number, quality: number } | null) {
+    this._delegate.setScreencastOptions(options).catch(e => debugLogger.log('error', e));
   }
 }
 

src/server/trace/recorder/tracing.ts

@@ -102,7 +102,7 @@ export class Tracing implements InstrumentationListener {
     for (const { sdkObject, metadata } of this._pendingCalls.values())
       await this.onAfterCall(sdkObject, metadata);
     for (const page of this._context.pages())
-      page.setScreencastEnabled(false);
+      page.setScreencastOptions(null);
 
     // Ensure all writes are finished.
     await this._appendEventChain;
@@ -185,7 +185,7 @@ export class Tracing implements InstrumentationListener {
     };
     this._appendTraceEvent(event);
     if (screenshots)
-      page.setScreencastEnabled(true);
+      page.setScreencastOptions({ width: 800, height: 600, quality: 90 });
 
     this._eventListeners.push(
         helper.addEventListener(page, Page.Events.Dialog, (dialog: Dialog) => {

src/server/webkit/protocol.ts

@@ -8050,6 +8050,7 @@ the top of the viewport and Y increases as it proceeds towards the bottom of the
       file: string;
       width: number;
       height: number;
+      toolbarHeight: number;
       scale?: number;
     }
     export type startVideoReturnValue = {
@@ -8071,6 +8072,7 @@ the top of the viewport and Y increases as it proceeds towards the bottom of the
     export type startScreencastParameters = {
       width: number;
       height: number;
+      toolbarHeight: number;
       quality: number;
     }
     export type startScreencastReturnValue = {

src/server/webkit/wkPage.ts

@@ -739,6 +739,12 @@ export class WKPage implements PageDelegate {
     await this._session.send('Page.setDefaultBackgroundColorOverride', { color });
   }
 
+  private _toolbarHeight(): number {
+    if (this._page._browserContext._browser?.options.headful)
+      return hostPlatform.startsWith('10.15') ? 55 : 59;
+    return 0;
+  }
+
   private async _startVideo(options: types.PageScreencastOptions): Promise<void> {
     assert(!this._recordingVideoFile);
     const START_VIDEO_PROTOCOL_COMMAND = hostPlatform === 'mac10.14' ? 'Screencast.start' : 'Screencast.startVideo';
@@ -746,6 +752,7 @@ export class WKPage implements PageDelegate {
       file: options.outputFile,
       width: options.width,
       height: options.height,
+      toolbarHeight: this._toolbarHeight()
     });
     this._recordingVideoFile = options.outputFile;
     this._browserContext._browser._videoStarted(this._browserContext, screencastId, options.outputFile, this.pageOrError());
@@ -827,9 +834,10 @@ export class WKPage implements PageDelegate {
     });
   }
 
-  async setScreencastEnabled(enabled: boolean): Promise<void> {
-    if (enabled) {
-      const { generation } = await this._pageProxySession.send('Screencast.startScreencast', { width: 800, height: 600, quality: 70 });
+  async setScreencastOptions(options: { width: number, height: number, quality: number } | null): Promise<void> {
+    if (options) {
+      const so = { ...options, toolbarHeight: this._toolbarHeight() };
+      const { generation } = await this._pageProxySession.send('Screencast.startScreencast', so);
       this._screencastGeneration = generation;
     } else {
       await this._pageProxySession.send('Screencast.stopScreencast');

tests/tracing.spec.ts

@@ -15,14 +15,16 @@
  */
 
 import path from 'path';
-import { expect, contextTest as test } from './config/browserTest';
+import { expect, contextTest as test, browserTest } from './config/browserTest';
 import yauzl from 'yauzl';
 import removeFolder from 'rimraf';
+import jpeg from 'jpeg-js';
 
 const traceDir = path.join(__dirname, '..', 'test-results', 'trace-' + process.env.FOLIO_WORKER_INDEX);
 test.useOptions({ traceDir });
 
-test.beforeEach(async () => {
+test.beforeEach(async ({ browserName, headful }) => {
+  test.fixme(browserName === 'chromium' && headful, 'Chromium screencast on headful has a min width issue');
   await new Promise(f => removeFolder(traceDir, f));
 });
 
@@ -99,6 +101,61 @@ test('should collect two traces', async ({ context, page, server }, testInfo) =>
   }
 });
 
+for (const params of [
+  {
+    id: 'fit',
+    width: 500,
+    height: 500,
+  }, {
+    id: 'crop',
+    width: 400, // Less than 450 to test firefox
+    height: 800,
+  }, {
+    id: 'scale',
+    width: 1024,
+    height: 768,
+  }]) {
+  browserTest(`should produce screencast frames ${params.id}`, async ({ video, contextFactory, browserName }, testInfo) => {
+    browserTest.fixme(browserName === 'chromium' && video, 'Same screencast resolution conflicts');
+    const scale = Math.min(800 / params.width, 600 / params.height, 1);
+    const previewWidth = params.width * scale;
+    const previewHeight = params.height * scale;
+
+    const context = await contextFactory({ viewport: { width: params.width, height: params.height }});
+    await (context as any)._tracing.start({ name: 'test', screenshots: true, snapshots: true });
+    const page = await context.newPage();
+    await page.setContent('<body style="box-sizing: border-box; width: 100%; height: 100%; margin:0; background: red; border: 50px solid blue"></body>');
+    // Make sure we have a chance to paint.
+    for (let i = 0; i < 10; ++i)
+      await page.evaluate(() => new Promise(requestAnimationFrame));
+    await (context as any)._tracing.stop();
+    await (context as any)._tracing.export(testInfo.outputPath('trace.zip'));
+
+    const { events, resources } = await parseTrace(testInfo.outputPath('trace.zip'));
+    const frames = events.filter(e => e.type === 'screencast-frame');
+    // Check all frame sizes
+    for (const frame of frames) {
+      expect(frame.width).toBe(params.width);
+      expect(frame.height).toBe(params.height);
+      const buffer = resources.get('resources/' + frame.sha1);
+      const image = jpeg.decode(buffer);
+      expect(image.width).toBe(previewWidth);
+      expect(image.height).toBe(previewHeight);
+    }
+
+    // Check last frame content.
+    const frame = frames[frames.length - 1]; // pick last frame.
+    const buffer = resources.get('resources/' + frame.sha1);
+    const image = jpeg.decode(buffer);
+    expect(image.data.byteLength).toBe(previewWidth * previewHeight * 4);
+    expectRed(image.data, previewWidth * previewHeight * 4 / 2 + previewWidth * 4 / 2); // center is red
+    expectBlue(image.data, previewWidth * 5 * 4 + previewWidth * 4 / 2); // top
+    expectBlue(image.data, previewWidth * (previewHeight - 5) * 4 + previewWidth * 4 / 2); // bottom
+    expectBlue(image.data, previewWidth * previewHeight * 4 / 2 + 5 * 4); // left
+    expectBlue(image.data, previewWidth * previewHeight * 4 / 2 + (previewWidth - 5) * 4); // right
+  });
+}
+
 async function parseTrace(file: string): Promise<{ events: any[], resources: Map<string, Buffer> }> {
   const entries = await new Promise<any[]>(f => {
     const entries: Promise<any>[] = [];
@@ -128,4 +185,26 @@ async function parseTrace(file: string): Promise<{ events: any[], resources: Map
     events,
     resources,
   };
-}
+}
+
+function expectRed(pixels: Buffer, offset: number) {
+  const r = pixels.readUInt8(offset);
+  const g = pixels.readUInt8(offset + 1);
+  const b = pixels.readUInt8(offset + 2);
+  const a = pixels.readUInt8(offset + 3);
+  expect(r).toBeGreaterThan(200);
+  expect(g).toBeLessThan(70);
+  expect(b).toBeLessThan(70);
+  expect(a).toBe(255);
+}
+
+function expectBlue(pixels: Buffer, offset: number) {
+  const r = pixels.readUInt8(offset);
+  const g = pixels.readUInt8(offset + 1);
+  const b = pixels.readUInt8(offset + 2);
+  const a = pixels.readUInt8(offset + 3);
+  expect(r).toBeLessThan(70);
+  expect(g).toBeLessThan(70);
+  expect(b).toBeGreaterThan(200);
+  expect(a).toBe(255);
+}