cherry-pick(release-1.12): allow specifying video size (#7306)

- feat(test runner): add tests for playwright-specific fixtures (#6952)
- PR #7158 SHA 184f2c2

Co-authored-by: Dmitry Gozman <dgozman@gmail.com>

.github/workflows/tests_primary.yml

@@ -63,4 +63,6 @@ jobs:
       env:
         DEBUG: pw:install
     - run: npm run build
+    - run: node lib/cli/cli install-deps
+    - run: node lib/cli/cli install
     - run: npm run ttest

src/test/cli.ts

@@ -28,10 +28,10 @@ const builtinReporters = ['list', 'line', 'dot', 'json', 'junit', 'null'];
 const tsConfig = 'playwright.config.ts';
 const jsConfig = 'playwright.config.js';
 const defaultConfig: Config = {
-  preserveOutput: process.env.CI ? 'failures-only' : 'always',
+  preserveOutput: 'failures-only',
   reporter: [ [defaultReporter] ],
   timeout: defaultTimeout,
-  updateSnapshots: process.env.CI ? 'none' : 'missing',
+  updateSnapshots: 'missing',
   workers: Math.ceil(require('os').cpus().length / 2),
 };
 

src/test/index.ts

@@ -17,7 +17,7 @@
 import * as fs from 'fs';
 import * as path from 'path';
 import * as os from 'os';
-import type { LaunchOptions, BrowserContextOptions, Page } from '../../types/types';
+import type { LaunchOptions, BrowserContextOptions, Page, ViewportSize } from '../../types/types';
 import type { TestType, PlaywrightTestArgs, PlaywrightTestOptions, PlaywrightWorkerArgs, PlaywrightWorkerOptions } from '../../types/test';
 import { rootTestType } from './testType';
 import { createGuid, removeFolders } from '../utils/utils';
@@ -84,16 +84,18 @@ export const test = _baseTest.extend<PlaywrightTestArgs & PlaywrightTestOptions,
     if (process.env.PWDEBUG)
       testInfo.setTimeout(0);
 
+    const videoMode = typeof video === 'string' ? video : video.mode;
     let recordVideoDir: string | null = null;
-    if (video === 'on' || (video === 'retry-with-video' && !!testInfo.retry))
+    const recordVideoSize = typeof video === 'string' ? undefined : video.size;
+    if (videoMode === 'on' || (videoMode === 'retry-with-video' && !!testInfo.retry))
       recordVideoDir = testInfo.outputPath('');
-    if (video === 'retain-on-failure') {
+    if (videoMode === 'retain-on-failure') {
       await fs.promises.mkdir(artifactsFolder, { recursive: true });
       recordVideoDir = artifactsFolder;
     }
 
     const options: BrowserContextOptions = {
-      recordVideo: recordVideoDir ? { dir: recordVideoDir } : undefined,
+      recordVideo: recordVideoDir ? { dir: recordVideoDir, size: recordVideoSize } : undefined,
       ...contextOptions,
     };
     if (acceptDownloads !== undefined)
@@ -167,15 +169,15 @@ export const test = _baseTest.extend<PlaywrightTestArgs & PlaywrightTestOptions,
     }
     await context.close();
 
-    if (video === 'retain-on-failure' && testFailed) {
+    if (videoMode === 'retain-on-failure' && testFailed) {
       await Promise.all(allPages.map(async page => {
-        const video = page.video();
-        if (!video)
+        const v = page.video();
+        if (!v)
           return;
         try {
-          const videoPath = await video.path();
+          const videoPath = await v.path();
           const fileName = path.basename(videoPath);
-          await video.saveAs(testInfo.outputPath(fileName));
+          await v.saveAs(testInfo.outputPath(fileName));
         } catch (e) {
           // Silent catch empty videos.
         }

tests/playwright-test/config.spec.ts

@@ -330,7 +330,7 @@ test('should work with undefined values and base', async ({ runInlineTest }) =>
     'a.test.ts': `
       const { test } = pwt;
       test('pass', async ({}, testInfo) => {
-        expect(testInfo.config.updateSnapshots).toBe('none');
+        expect(testInfo.config.updateSnapshots).toBe('missing');
       });
     `
   }, {}, { CI: '1' });

tests/playwright-test/golden.spec.ts

@@ -80,20 +80,6 @@ test('should write missing expectations locally', async ({runInlineTest}, testIn
   expect(data.toString()).toBe('Hello world');
 });
 
-test('should not write missing expectations on CI', async ({runInlineTest}, testInfo) => {
-  const result = await runInlineTest({
-    'a.spec.js': `
-      const { test } = pwt;
-      test('is a test', ({}) => {
-        expect('Hello world').toMatchSnapshot('snapshot.txt');
-      });
-    `
-  }, {}, { CI: '1' });
-  expect(result.exitCode).toBe(1);
-  expect(result.output).toContain('snapshot.txt is missing in snapshots');
-  expect(fs.existsSync(testInfo.outputPath('a.spec.js-snapshots/snapshot.txt'))).toBe(false);
-});
-
 test('should update expectations', async ({runInlineTest}, testInfo) => {
   const result = await runInlineTest({
     'a.spec.js-snapshots/snapshot.txt': `Hello world`,

tests/playwright-test/playwright-test.config.ts

@@ -19,7 +19,7 @@ import { Config } from '../config/test-runner';
 const config: Config = {
   testDir: __dirname,
   testIgnore: 'assets/**',
-  timeout: 20000,
+  timeout: 30000,
   forbidOnly: !!process.env.CI,
   projects: [
     { name: 'playwright-test' },

tests/playwright-test/playwright.spec.ts

@@ -0,0 +1,242 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './playwright-test-fixtures';
+import fs from 'fs';
+import path from 'path';
+import { spawnSync } from 'child_process';
+import { Registry } from '../../src/utils/registry';
+
+const registry = new Registry(path.join(__dirname, '..', '..'));
+const ffmpeg = registry.executablePath('ffmpeg') || '';
+
+export class VideoPlayer {
+  videoWidth: number;
+  videoHeight: number;
+
+  constructor(fileName: string) {
+    var output = spawnSync(ffmpeg, ['-i', fileName, '-r', '25', `${fileName}-%03d.png`]).stderr.toString();
+    const lines = output.split('\n');
+    const streamLine = lines.find(l => l.trim().startsWith('Stream #0:0'));
+    const resolutionMatch = streamLine.match(/, (\d+)x(\d+),/);
+    this.videoWidth = parseInt(resolutionMatch![1], 10);
+    this.videoHeight = parseInt(resolutionMatch![2], 10);
+  }
+}
+
+test('should respect viewport option', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { use: { viewport: { width: 800, height: 800 } } };
+    `,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('pass', async ({ page }) => {
+        expect(page.viewportSize()).toEqual({ width: 800, height: 800 });
+      });
+    `,
+    'b.test.ts': `
+      const { test } = pwt;
+      test.use({ viewport: { width: 600, height: 600 } });
+      test('pass', async ({ page }) => {
+        expect(page.viewportSize()).toEqual({ width: 600, height: 600 });
+      });
+    `,
+  }, { workers: 1 });
+
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(2);
+});
+
+test('should run in three browsers with --browser', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { use: { viewport: { width: 800, height: 800 } } };
+    `,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('pass', async ({ page, browserName }) => {
+        expect(page.viewportSize()).toEqual({ width: 800, height: 800 });
+        console.log('\\n%%browser=' + browserName);
+      });
+    `,
+  }, { browser: 'all', workers: 1 });
+
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(3);
+  expect(result.output.split('\n').filter(line => line.startsWith('%%')).sort()).toEqual([
+    '%%browser=chromium',
+    '%%browser=firefox',
+    '%%browser=webkit',
+  ]);
+});
+
+test('should run in one browser with --browser', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { use: { viewport: { width: 800, height: 800 } } };
+    `,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('pass', async ({ page, browserName }) => {
+        expect(page.viewportSize()).toEqual({ width: 800, height: 800 });
+        console.log('\\n%%browser=' + browserName);
+      });
+    `,
+  }, { browser: 'webkit', workers: 1 });
+
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+  expect(result.output.split('\n').filter(line => line.startsWith('%%')).sort()).toEqual([
+    '%%browser=webkit',
+  ]);
+});
+
+test('should complain with projects and --browser', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { projects: [ {} ] };
+    `,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('pass', async ({ page }) => {
+      });
+    `,
+  }, { browser: 'webkit', workers: 1 });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.passed).toBe(0);
+  expect(result.output).toContain('Cannot use --browser option when configuration file defines projects');
+});
+
+test('should work with screenshot: only-on-failure', async ({ runInlineTest }, testInfo) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { use: { screenshot: 'only-on-failure' } };
+    `,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('pass', async ({ page }) => {
+        await page.setContent('<div>PASS</div>');
+        test.expect(1 + 1).toBe(2);
+      });
+      test('fail', async ({ page }) => {
+        await page.setContent('<div>FAIL</div>');
+        test.expect(1 + 1).toBe(1);
+      });
+    `,
+  }, { workers: 1 });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.passed).toBe(1);
+  expect(result.failed).toBe(1);
+  const screenshotPass = testInfo.outputPath('test-results', 'a-pass-chromium', 'test-failed-1.png');
+  const screenshotFail = testInfo.outputPath('test-results', 'a-fail-chromium', 'test-failed-1.png');
+  expect(fs.existsSync(screenshotPass)).toBe(false);
+  expect(fs.existsSync(screenshotFail)).toBe(true);
+});
+
+test('should work with video: retain-on-failure', async ({ runInlineTest }, testInfo) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { use: { video: 'retain-on-failure' } };
+    `,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('pass', async ({ page }) => {
+        await page.setContent('<div>PASS</div>');
+        await page.waitForTimeout(3000);
+        test.expect(1 + 1).toBe(2);
+      });
+      test('fail', async ({ page }) => {
+        await page.setContent('<div>FAIL</div>');
+        await page.waitForTimeout(3000);
+        test.expect(1 + 1).toBe(1);
+      });
+    `,
+  }, { workers: 1 });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.passed).toBe(1);
+  expect(result.failed).toBe(1);
+
+  const dirPass = testInfo.outputPath('test-results', 'a-pass-chromium');
+  const videoPass = fs.existsSync(dirPass) ? fs.readdirSync(dirPass).find(file => file.endsWith('webm')) : undefined;
+  expect(videoPass).toBeFalsy();
+
+  const videoFail = fs.readdirSync(testInfo.outputPath('test-results', 'a-fail-chromium')).find(file => file.endsWith('webm'));
+  expect(videoFail).toBeTruthy();
+});
+
+test('should work with video: retry-with-video', async ({ runInlineTest }, testInfo) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { use: { video: 'retry-with-video' }, retries: 1 };
+    `,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('pass', async ({ page }) => {
+        await page.setContent('<div>PASS</div>');
+        await page.waitForTimeout(3000);
+        test.expect(1 + 1).toBe(2);
+      });
+      test('fail', async ({ page }) => {
+        await page.setContent('<div>FAIL</div>');
+        await page.waitForTimeout(3000);
+        test.expect(1 + 1).toBe(1);
+      });
+    `,
+  }, { workers: 1 });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.passed).toBe(1);
+  expect(result.failed).toBe(1);
+
+  const dirPass = testInfo.outputPath('test-results', 'a-pass-chromium');
+  expect(fs.existsSync(dirPass)).toBeFalsy();
+
+  const dirFail = testInfo.outputPath('test-results', 'a-fail-chromium');
+  expect(fs.existsSync(dirFail)).toBeFalsy();
+
+  const videoFailRetry = fs.readdirSync(testInfo.outputPath('test-results', 'a-fail-chromium-retry1')).find(file => file.endsWith('webm'));
+  expect(videoFailRetry).toBeTruthy();
+});
+
+test('should work with video size', async ({ runInlineTest, browserName }, testInfo) => {
+  const result = await runInlineTest({
+    'playwright.config.js': `
+      module.exports = {
+        use: { video: { mode: 'on', size: { width: 220, height: 110 } } },
+        preserveOutput: 'always',
+      };
+    `,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('pass', async ({ page }) => {
+        await page.setContent('<div>PASS</div>');
+        await page.waitForTimeout(3000);
+        test.expect(1 + 1).toBe(2);
+      });
+    `,
+  }, { workers: 1 });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+  const folder = testInfo.outputPath(`test-results/a-pass-${browserName}/`);
+  const [file] = fs.readdirSync(folder);
+  const videoPlayer = new VideoPlayer(path.join(folder, file));
+  expect(videoPlayer.videoWidth).toBe(220);
+  expect(videoPlayer.videoHeight).toBe(110);
+});

tests/playwright-test/test-output-dir.spec.ts

@@ -213,7 +213,7 @@ test('should remove folders with preserveOutput=never', async ({ runInlineTest }
   expect(fs.existsSync(testInfo.outputPath('test-results', 'dir-my-test-test-1-retry2'))).toBe(false);
 });
 
-test('should not remove folders on non-CI', async ({ runInlineTest }, testInfo) => {
+test('should preserve failed results', async ({ runInlineTest }, testInfo) => {
   const result = await runInlineTest({
     'dir/my-test.spec.js': `
       const { test } = pwt;
@@ -223,13 +223,12 @@ test('should not remove folders on non-CI', async ({ runInlineTest }, testInfo)
           throw new Error('Give me retries');
       });
     `,
-  }, { 'retries': 2 }, { CI: '' });
+  }, { 'retries': 2 });
   expect(result.exitCode).toBe(0);
   expect(result.results.length).toBe(3);
 
   expect(fs.existsSync(testInfo.outputPath('test-results', 'dir-my-test-test-1'))).toBe(true);
   expect(fs.existsSync(testInfo.outputPath('test-results', 'dir-my-test-test-1-retry1'))).toBe(true);
-  expect(fs.existsSync(testInfo.outputPath('test-results', 'dir-my-test-test-1-retry2'))).toBe(true);
 });
 
 

types/test.d.ts

@@ -953,6 +953,15 @@ export type PlaywrightWorkerOptions = {
   launchOptions: LaunchOptions;
 };
 
+/**
+ * Video recording mode:
+ * - `off`: Do not record video.
+ * - `on`: Record video for each test.
+ * - `retain-on-failure`: Record video for each test, but remove all videos from successful test runs.
+ * - `retry-with-video`: Record video only when retrying a test.
+ */
+export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'retry-with-video';
+
 /**
  * Options available to configure each test.
  *   - Set options in config:
@@ -985,13 +994,13 @@ export type PlaywrightTestOptions = {
   trace: 'off' | 'on' | 'retain-on-failure' | 'retry-with-trace';
 
   /**
-  * Whether to record video for each test, off by default.
-  * - `off`: Do not record video.
-  * - `on`: Record video for each test.
-  * - `retain-on-failure`: Record video for each test, but remove all videos from successful test runs.
-  * - `retry-with-video`: Record video only when retrying a test.
-  */
-  video: 'off' | 'on' | 'retain-on-failure' | 'retry-with-video';
+   * Whether to record video for each test, off by default.
+   * - `off`: Do not record video.
+   * - `on`: Record video for each test.
+   * - `retain-on-failure`: Record video for each test, but remove all videos from successful test runs.
+   * - `retry-with-video`: Record video only when retrying a test.
+   */
+  video: VideoMode | { mode: VideoMode, size: ViewportSize };
 
   /**
    * Whether to automatically download all the attachments. Takes priority over `contextOptions`.