From 8ec55e1fb24e063a78f354f0fe8e2521594bccc9 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Mon, 31 Aug 2020 08:43:14 -0700 Subject: [PATCH] feat(screencast): use ffmpeg to produce webm in chromium (#3668) --- package-lock.json | 56 ++++++++++++ package.json | 1 + src/server/chromium/crPage.ts | 35 +++++++- src/server/chromium/videoRecorder.ts | 128 +++++++++++++++++++++++++++ src/server/processLauncher.ts | 3 + test/page-goto.spec.ts | 4 +- test/screencast.spec.ts | 74 +++++++++------- utils/testserver/index.d.ts | 3 +- utils/testserver/index.js | 16 ++-- 9 files changed, 274 insertions(+), 46 deletions(-) create mode 100644 src/server/chromium/videoRecorder.ts diff --git a/package-lock.json b/package-lock.json index 07d28e6a8c..9e10af068d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1081,6 +1081,62 @@ "sumchecker": "^3.0.1" } }, + "@ffmpeg-installer/darwin-x64": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@ffmpeg-installer/darwin-x64/-/darwin-x64-4.1.0.tgz", + "integrity": "sha512-Z4EyG3cIFjdhlY8wI9aLUXuH8nVt7E9SlMVZtWvSPnm2sm37/yC2CwjUzyCQbJbySnef1tQwGG2Sx+uWhd9IAw==", + "optional": true + }, + "@ffmpeg-installer/ffmpeg": { + "version": "1.0.20", + "resolved": "https://registry.npmjs.org/@ffmpeg-installer/ffmpeg/-/ffmpeg-1.0.20.tgz", + "integrity": "sha512-wbgd//6OdwbFXYgV68ZyKrIcozEQpUKlvV66XHaqO2h3sFbX0jYLzx62Q0v8UcFWN21LoxT98NU2P+K0OWsKNA==", + "requires": { + "@ffmpeg-installer/darwin-x64": "4.1.0", + "@ffmpeg-installer/linux-arm": "4.1.3", + "@ffmpeg-installer/linux-arm64": "4.1.4", + "@ffmpeg-installer/linux-ia32": "4.1.0", + "@ffmpeg-installer/linux-x64": "4.1.0", + "@ffmpeg-installer/win32-ia32": "4.1.0", + "@ffmpeg-installer/win32-x64": "4.1.0" + } + }, + "@ffmpeg-installer/linux-arm": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/@ffmpeg-installer/linux-arm/-/linux-arm-4.1.3.tgz", + "integrity": "sha512-NDf5V6l8AfzZ8WzUGZ5mV8O/xMzRag2ETR6+TlGIsMHp81agx51cqpPItXPib/nAZYmo55Bl2L6/WOMI3A5YRg==", + "optional": true + }, + "@ffmpeg-installer/linux-arm64": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/@ffmpeg-installer/linux-arm64/-/linux-arm64-4.1.4.tgz", + "integrity": "sha512-dljEqAOD0oIM6O6DxBW9US/FkvqvQwgJ2lGHOwHDDwu/pX8+V0YsDL1xqHbj1DMX/+nP9rxw7G7gcUvGspSoKg==", + "optional": true + }, + "@ffmpeg-installer/linux-ia32": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@ffmpeg-installer/linux-ia32/-/linux-ia32-4.1.0.tgz", + "integrity": "sha512-0LWyFQnPf+Ij9GQGD034hS6A90URNu9HCtQ5cTqo5MxOEc7Rd8gLXrJvn++UmxhU0J5RyRE9KRYstdCVUjkNOQ==", + "optional": true + }, + "@ffmpeg-installer/linux-x64": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@ffmpeg-installer/linux-x64/-/linux-x64-4.1.0.tgz", + "integrity": "sha512-Y5BWhGLU/WpQjOArNIgXD3z5mxxdV8c41C+U15nsE5yF8tVcdCGet5zPs5Zy3Ta6bU7haGpIzryutqCGQA/W8A==", + "optional": true + }, + "@ffmpeg-installer/win32-ia32": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@ffmpeg-installer/win32-ia32/-/win32-ia32-4.1.0.tgz", + "integrity": "sha512-FV2D7RlaZv/lrtdhaQ4oETwoFUsUjlUiasiZLDxhEUPdNDWcH1OU9K1xTvqz+OXLdsmYelUDuBS/zkMOTtlUAw==", + "optional": true + }, + "@ffmpeg-installer/win32-x64": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@ffmpeg-installer/win32-x64/-/win32-x64-4.1.0.tgz", + "integrity": "sha512-Drt5u2vzDnIONf4ZEkKtFlbvwj6rI3kxw1Ck9fpudmtgaZIHD4ucsWB2lCZBXRxJgXR+2IMSti+4rtM4C4rXgg==", + "optional": true + }, "@jest/types": { "version": "26.3.0", "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.3.0.tgz", diff --git a/package.json b/package.json index db3955c545..584c0bb80e 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ }, "license": "Apache-2.0", "dependencies": { + "@ffmpeg-installer/ffmpeg": "^1.0.20", "debug": "^4.1.1", "extract-zip": "^2.0.1", "https-proxy-agent": "^5.0.0", diff --git a/src/server/chromium/crPage.ts b/src/server/chromium/crPage.ts index a7f92f6cbd..e196e83b60 100644 --- a/src/server/chromium/crPage.ts +++ b/src/server/chromium/crPage.ts @@ -37,6 +37,7 @@ import { ConsoleMessage } from '../console'; import * as sourceMap from '../../utils/sourceMap'; import { rewriteErrorMessage } from '../../utils/stackTrace'; import { assert, headersArrayToObject } from '../../utils/utils'; +import { VideoRecorder } from './videoRecorder'; const UTILITY_WORLD_NAME = '__playwright_utility_world__'; @@ -209,11 +210,11 @@ export class CRPage implements PageDelegate { } async startScreencast(options: types.PageScreencastOptions): Promise { - throw new Error('Not implemented'); + await this._mainFrameSession._startScreencast(options); } async stopScreencast(): Promise { - throw new Error('Not implemented'); + await this._mainFrameSession._stopScreencast(); } async takeScreenshot(format: 'png' | 'jpeg', documentRect: types.Rect | undefined, viewportRect: types.Rect | undefined, quality: number | undefined): Promise { @@ -324,6 +325,7 @@ class FrameSession { // Marks the oopif session that remote -> local transition has happened in the parent. // See Target.detachedFromTarget handler for details. private _swappedIn = false; + private _videoRecorder: VideoRecorder | null = null; constructor(crPage: CRPage, client: CRSession, targetId: string, parentSession: FrameSession | null) { this._client = client; @@ -358,6 +360,7 @@ class FrameSession { helper.addEventListener(this._client, 'Page.navigatedWithinDocument', event => this._onFrameNavigatedWithinDocument(event.frameId, event.url)), helper.addEventListener(this._client, 'Page.downloadWillBegin', event => this._onDownloadWillBegin(event)), helper.addEventListener(this._client, 'Page.downloadProgress', event => this._onDownloadProgress(event)), + helper.addEventListener(this._client, 'Page.screencastFrame', event => this._onScreencastFrame(event)), helper.addEventListener(this._client, 'Runtime.bindingCalled', event => this._onBindingCalled(event)), helper.addEventListener(this._client, 'Runtime.consoleAPICalled', event => this._onConsoleAPI(event)), helper.addEventListener(this._client, 'Runtime.exceptionThrown', exception => this._handleException(exception.exceptionDetails)), @@ -724,6 +727,34 @@ class FrameSession { this._crPage._browserContext._browser._downloadFinished(payload.guid, 'canceled'); } + _onScreencastFrame(payload: Protocol.Page.screencastFramePayload) { + if (!this._videoRecorder) + return; + const buffer = Buffer.from(payload.data, 'base64'); + this._videoRecorder.writeFrame(buffer, payload.metadata.timestamp!); + this._client.send('Page.screencastFrameAck', {sessionId: payload.sessionId}); + } + + async _startScreencast(options: types.PageScreencastOptions): Promise { + assert(!this._videoRecorder, 'Already started'); + this._videoRecorder = await VideoRecorder.launch(options); + await this._client.send('Page.startScreencast', { + format: 'jpeg', + quality: 90, + maxWidth: options.width, + maxHeight: options.height, + }); + } + + async _stopScreencast(): Promise { + if (!this._videoRecorder) + return; + const recorder = this._videoRecorder; + this._videoRecorder = null; + await this._client.send('Page.stopScreencast'); + await recorder.stop(); + } + async _updateExtraHTTPHeaders(): Promise { const headers = network.mergeHeaders([ this._crPage._browserContext._options.extraHTTPHeaders, diff --git a/src/server/chromium/videoRecorder.ts b/src/server/chromium/videoRecorder.ts new file mode 100644 index 0000000000..da50437879 --- /dev/null +++ b/src/server/chromium/videoRecorder.ts @@ -0,0 +1,128 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const ffmpegPath = require('@ffmpeg-installer/ffmpeg').path; +import { launchProcess } from '../processLauncher'; +import { ChildProcess } from 'child_process'; +import { Progress, runAbortableTask } from '../progress'; +import * as types from '../types'; +import { assert } from '../../utils/utils'; + +const fps = 25; + +export class VideoRecorder { + private _process: ChildProcess | null = null; + private _gracefullyClose: (() => Promise) | null = null; + private _lastWritePromise: Promise; + private _lastFrameTimestamp: number = 0; + private _lastFrameBuffer: Buffer | null = null; + private _lastWriteTimestamp: number = 0; + private readonly _progress: Progress; + + static async launch(options: types.PageScreencastOptions): Promise { + if (!options.outputFile.endsWith('.webm')) + throw new Error('File must have .webm extension'); + + return await runAbortableTask(async progress => { + const recorder = new VideoRecorder(progress); + await recorder._launch(options); + return recorder; + }, 0, 'browser'); + } + + private constructor(progress: Progress) { + this._progress = progress; + this._lastWritePromise = Promise.resolve(); + } + + private async _launch(options: types.PageScreencastOptions) { + assert(!this._isRunning()); + const w = options.width; + const h = options.height; + const args = `-f image2pipe -c:v mjpeg -i - -y -an -r ${fps} -c:v vp8 -vf pad=${w}:${h}:0:0:gray,crop=${w}:${h}:0:0`.split(' '); + args.push(options.outputFile); + const progress = this._progress; + const { launchedProcess, gracefullyClose } = await launchProcess({ + executablePath: ffmpegPath, + args, + pipeStdin: true, + progress, + tempDirectories: [], + attemptToGracefullyClose: async () => { + progress.log('Closing stdin...'); + launchedProcess.stdin.end(); + }, + onExit: (exitCode, signal) => { + progress.log(`ffmpeg onkill exitCode=${exitCode} signal=${signal}`); + }, + }); + launchedProcess.stdin.on('finish', () => { + progress.log('ffmpeg finished input.'); + }); + launchedProcess.stdin.on('error', () => { + progress.log('ffmpeg error.'); + }); + this._process = launchedProcess; + this._gracefullyClose = gracefullyClose; + } + + async writeFrame(frame: Buffer, timestamp: number) { + assert(this._process); + if (!this._isRunning()) + return; + const duration = this._lastFrameTimestamp ? Math.max(1, Math.round(25 * (timestamp - this._lastFrameTimestamp))) : 1; + this._progress.log(`writing ${duration} frame(s)`); + this._lastFrameBuffer = frame; + this._lastFrameTimestamp = timestamp; + this._lastWriteTimestamp = Date.now(); + + const previousWrites = this._lastWritePromise; + let finishedWriting: () => void; + this._lastWritePromise = new Promise(fulfill => finishedWriting = fulfill); + const writePromise = this._lastWritePromise; + await previousWrites; + for (let i = 0; i < duration; i++) { + const callFinish = i === (duration - 1); + this._process.stdin.write(frame, (error: Error | null | undefined) => { + if (error) + this._progress.log(`ffmpeg failed to write: ${error}`); + if (callFinish) + finishedWriting(); + }); + } + return writePromise; + } + + async stop() { + if (!this._gracefullyClose) + return; + + if (this._lastWriteTimestamp) { + const durationSec = (Date.now() - this._lastWriteTimestamp) / 1000; + if (durationSec > 1 / fps) + this.writeFrame(this._lastFrameBuffer!, this._lastFrameTimestamp + durationSec); + } + + const close = this._gracefullyClose; + this._gracefullyClose = null; + await this._lastWritePromise; + await close(); + } + + private _isRunning(): boolean { + return !!this._gracefullyClose; + } +} \ No newline at end of file diff --git a/src/server/processLauncher.ts b/src/server/processLauncher.ts index acae3fa412..1de89f376f 100644 --- a/src/server/processLauncher.ts +++ b/src/server/processLauncher.ts @@ -35,6 +35,7 @@ export type LaunchProcessOptions = { handleSIGTERM?: boolean, handleSIGHUP?: boolean, pipe?: boolean, + pipeStdin?: boolean, tempDirectories: string[], cwd?: string, @@ -62,6 +63,8 @@ export async function launchProcess(options: LaunchProcessOptions): Promise ${options.executablePath} ${options.args.join(' ')}`); const spawnedProcess = childProcess.spawn( options.executablePath, diff --git a/test/page-goto.spec.ts b/test/page-goto.spec.ts index dab8b9d207..055579487a 100644 --- a/test/page-goto.spec.ts +++ b/test/page-goto.spec.ts @@ -450,7 +450,7 @@ it('should reject referer option when setExtraHTTPHeaders provides referer', asy it('should override referrer-policy', async ({page, server}) => { server.setRoute('/grid.html', (req, res) => { res.setHeader('Referrer-Policy', 'no-referrer'); - server.serveFile(req, res, '/grid.html'); + server.serveFile(req, res); }); const [request1, request2] = await Promise.all([ server.waitForRequest('/grid.html'), @@ -482,7 +482,7 @@ it('extraHttpHeaders should be pushed to provisional page', test => { const pagePath = '/one-style.html'; server.setRoute(pagePath, async (req, res) => { page.setExtraHTTPHeaders({ foo: 'bar' }); - server.serveFile(req, res, pagePath); + server.serveFile(req, res); }); const [htmlReq, cssReq] = await Promise.all([ server.waitForRequest(pagePath), diff --git a/test/screencast.spec.ts b/test/screencast.spec.ts index 8a06fbdded..69c5fa6c29 100644 --- a/test/screencast.spec.ts +++ b/test/screencast.spec.ts @@ -20,7 +20,7 @@ import type { Page } from '..'; import fs from 'fs'; import path from 'path'; -import url from 'url'; +import { TestServer } from '../utils/testserver'; declare global { @@ -29,7 +29,7 @@ declare global { } } -registerFixture('videoPlayer', async ({playwright, context}, test) => { +registerFixture('videoPlayer', async ({playwright, context, server}, test) => { let firefox; if (options.WEBKIT && !LINUX) { // WebKit on Mac & Windows cannot replay webm/vp8 video, so we launch Firefox. @@ -38,7 +38,7 @@ registerFixture('videoPlayer', async ({playwright, context}, test) => { } const page = await context.newPage(); - const player = new VideoPlayer(page); + const player = new VideoPlayer(page, server); await test(player); if (firefox) await firefox.close(); @@ -90,12 +90,19 @@ function expectAll(pixels, rgbaPredicate) { class VideoPlayer { private readonly _page: Page; - constructor(page: Page) { + private readonly _server: TestServer; + constructor(page: Page, server: TestServer) { this._page = page; + this._server = server; } - async load(videoFile) { - await this._page.goto(url.pathToFileURL(videoFile).href); + async load(videoFile: string) { + const servertPath = '/v.webm'; + this._server.setRoute(servertPath, (req, response) => { + this._server.serveFile(req, response, videoFile); + }); + + await this._page.goto(this._server.PREFIX + servertPath); await this._page.$eval('video', (v: HTMLVideoElement) => { return new Promise(fulfil => { // In case video playback autostarts. @@ -172,36 +179,35 @@ class VideoPlayer { } } +it('should capture static page', test => { + test.skip(options.WIRE); +}, async ({page, tmpDir, videoPlayer, toImpl}) => { + const videoFile = path.join(tmpDir, 'v.webm'); + await page.evaluate(() => document.body.style.backgroundColor = 'red'); + await toImpl(page)._delegate.startScreencast({outputFile: videoFile, width: 640, height: 480}); + // TODO: in WebKit figure out why video size is not reported correctly for + // static pictures. + if (options.HEADLESS && options.WEBKIT) + await page.setViewportSize({width: 1270, height: 950}); + await new Promise(r => setTimeout(r, 300)); + await toImpl(page)._delegate.stopScreencast(); + expect(fs.existsSync(videoFile)).toBe(true); + + await videoPlayer.load(videoFile); + const duration = await videoPlayer.duration(); + expect(duration).toBeGreaterThan(0); + + expect(await videoPlayer.videoWidth()).toBe(640); + expect(await videoPlayer.videoHeight()).toBe(480); + + await videoPlayer.seekLastNonEmptyFrame(); + const pixels = await videoPlayer.pixels(); + expectAll(pixels, almostRed); +}); + describe('screencast', suite => { - suite.skip(options.WIRE); - suite.fixme(options.CHROMIUM); + suite.skip(options.WIRE || options.CHROMIUM); }, () => { - it('should capture static page', test => { - test.fixme(); - }, async ({page, tmpDir, videoPlayer, toImpl}) => { - const videoFile = path.join(tmpDir, 'v.webm'); - await page.evaluate(() => document.body.style.backgroundColor = 'red'); - await toImpl(page)._delegate.startScreencast({outputFile: videoFile, width: 640, height: 480}); - // TODO: in WebKit figure out why video size is not reported correctly for - // static pictures. - if (options.HEADLESS && options.WEBKIT) - await page.setViewportSize({width: 1270, height: 950}); - await new Promise(r => setTimeout(r, 300)); - await toImpl(page)._delegate.stopScreencast(); - expect(fs.existsSync(videoFile)).toBe(true); - - await videoPlayer.load(videoFile); - const duration = await videoPlayer.duration(); - expect(duration).toBeGreaterThan(0); - - expect(await videoPlayer.videoWidth()).toBe(640); - expect(await videoPlayer.videoHeight()).toBe(480); - - await videoPlayer.seekLastNonEmptyFrame(); - const pixels = await videoPlayer.pixels(); - expectAll(pixels, almostRed); - }); - it('should capture navigation', test => { test.flaky(options.WEBKIT); test.flaky(options.FIREFOX); diff --git a/utils/testserver/index.d.ts b/utils/testserver/index.d.ts index 639949547b..a243956071 100644 --- a/utils/testserver/index.d.ts +++ b/utils/testserver/index.d.ts @@ -29,7 +29,8 @@ export class TestServer { setRedirect(from: string, to: string); waitForRequest(path: string): Promise; reset(); - serveFile(request: IncomingMessage, response: ServerResponse, pathName: string); + serveFile(request: IncomingMessage, response: ServerResponse); + serveFile(request: IncomingMessage, response: ServerResponse, filePath: string); PORT: number; PREFIX: string; diff --git a/utils/testserver/index.js b/utils/testserver/index.js index b43bbff636..3f5773b1e1 100644 --- a/utils/testserver/index.js +++ b/utils/testserver/index.js @@ -228,20 +228,22 @@ class TestServer { if (handler) { handler.call(null, request, response); } else { - const pathName = url.parse(request.url).path; - this.serveFile(request, response, pathName); + this.serveFile(request, response); } } /** * @param {!http.IncomingMessage} request * @param {!http.ServerResponse} response - * @param {string} pathName + * @param {string|undefined} filePath */ - serveFile(request, response, pathName) { - if (pathName === '/') - pathName = '/index.html'; - const filePath = path.join(this._dirPath, pathName.substring(1)); + serveFile(request, response, filePath) { + let pathName = url.parse(request.url).path; + if (!filePath) { + if (pathName === '/') + pathName = '/index.html'; + filePath = path.join(this._dirPath, pathName.substring(1)); + } if (this._cachedPathPrefix !== null && filePath.startsWith(this._cachedPathPrefix)) { if (request.headers['if-modified-since']) {