feat(test-runner): allow setting pixel match threshold for project (#7123)

docs/src/test-snapshots.md

@@ -53,6 +53,26 @@ test('example test', async ({ page }) => {
 });
 ```
 
+If you'd like to share the default value among all the tests in the project, you can specify it in the playwright config, either globally or per project:
+
+```js js-flavor=js
+module.exports = {
+  expect: {
+    toMatchSnapshot: { threshold: 0.1 },
+  },
+};
+```
+
+```js js-flavor=ts
+import { PlaywrightTestConfig } from '@playwright/test';
+const config: PlaywrightTestConfig = {
+  expect: {
+    toMatchSnapshot: { threshold: 0.1 },
+  },
+};
+export default config;
+```
+
 Apart from screenshots, `expect(value).toMatchSnapshot(snapshotName)` can also be used to compare text, png and jpeg images, or arbitrary binary data. Playwright Test auto-detects the content type and uses the appropriate comparison algorithm.
 
 Here we compare text content against the reference.

src/test/expect.ts

@@ -33,6 +33,10 @@ function toMatchSnapshot(received: Buffer | string, nameOrOptions: string | { na
   if (!options.name)
     throw new Error(`toMatchSnapshot() requires a "name" parameter`);
 
+  const projectThreshold = testInfo.project.expect?.toMatchSnapshot?.threshold;
+  if (options.threshold === undefined && projectThreshold !== undefined)
+    options.threshold = projectThreshold;
+
   const { pass, message } = compare(received, options.name, testInfo.snapshotPath, testInfo.outputPath, testInfo.config.updateSnapshots, options);
   return { pass, message: () => message };
 }

src/test/loader.ts

@@ -190,6 +190,7 @@ export class Loader {
 
     const fullProject: FullProject = {
       define: takeFirst(this._configOverrides.define, projectConfig.define, this._config.define, []),
+      expect: takeFirst(this._configOverrides.expect, projectConfig.expect, this._config.expect, undefined),
       outputDir: takeFirst(this._configOverrides.outputDir, projectConfig.outputDir, this._config.outputDir, path.resolve(process.cwd(), 'test-results')),
       repeatEach: takeFirst(this._configOverrides.repeatEach, projectConfig.repeatEach, this._config.repeatEach, 1),
       retries: takeFirst(this._configOverrides.retries, projectConfig.retries, this._config.retries, 0),

tests/playwright-test/golden.spec.ts

@@ -254,3 +254,27 @@ test('should respect threshold', async ({runInlineTest}) => {
   });
   expect(result.exitCode).toBe(0);
 });
+
+test('should respect project threshold', async ({runInlineTest}) => {
+  const expected = fs.readFileSync(path.join(__dirname, 'assets/screenshot-canvas-expected.png'));
+  const actual = fs.readFileSync(path.join(__dirname, 'assets/screenshot-canvas-actual.png'));
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { projects: [
+        { expect: { toMatchSnapshot: { threshold: 0.2 } } },
+      ]};
+    `,
+    'a.spec.js-snapshots/snapshot.png': expected,
+    'a.spec.js-snapshots/snapshot2.png': expected,
+    'a.spec.js': `
+      const { test } = pwt;
+      test('is a test', ({}) => {
+        expect(Buffer.from('${actual.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png', { threshold: 0.3 });
+        expect(Buffer.from('${actual.toString('base64')}', 'base64')).not.toMatchSnapshot('snapshot.png');
+        expect(Buffer.from('${actual.toString('base64')}', 'base64')).toMatchSnapshot('snapshot2.png', { threshold: 0.3 });
+        expect(Buffer.from('${actual.toString('base64')}', 'base64')).toMatchSnapshot({ name: 'snapshot2.png', threshold: 0.3 });
+      });
+    `
+  });
+  expect(result.exitCode).toBe(0);
+});

types/test.d.ts

@@ -34,10 +34,22 @@ export type UpdateSnapshots = 'all' | 'none' | 'missing';
 
 type FixtureDefine<TestArgs extends KeyValue = {}, WorkerArgs extends KeyValue = {}> = { test: TestType<TestArgs, WorkerArgs>, fixtures: Fixtures<{}, {}, TestArgs, WorkerArgs> };
 
+type ExpectSettings = {
+  toMatchSnapshot?: {
+    // Pixel match threshold.
+    threshold?: number
+  }
+};
+
 /**
  * Test run configuration.
  */
 interface ProjectBase {
+  /**
+   * Expect matcher settings.
+   */
+  expect?: ExpectSettings;
+
   /**
    * Any JSON-serializable metadata that will be put directly to the test report.
    */
@@ -102,6 +114,7 @@ export interface Project<TestArgs = {}, WorkerArgs = {}> extends ProjectBase {
    */
   use?: Fixtures<{}, {}, TestArgs, WorkerArgs>;
 }
+
 export type FullProject<TestArgs = {}, WorkerArgs = {}> = Required<Project<TestArgs, WorkerArgs>>;
 
 /**