feat: bring back maxDiffPixels in toMatchSnapshot (#12627)

This patch reverts 2 commits that removed the feature from the method:
- "fix: explicitly ignore maxDiffPixels in toMatchSnapshot (#12570)"
  commit b8af8458d6.
- "chore: remove `maxDiffPixels` from toMatchSnapshot (#12539)"
  commit a3dff45974.

docs/src/api/class-screenshotassertions.md

@@ -34,4 +34,8 @@ Learn more about [visual comparisons](./test-snapshots.md).
 
 Snapshot name.
 
+### option: ScreenshotAssertions.toMatchSnapshot.maxDiffPixels = %%-assertions-max-diff-pixels-%%
+
+### option: ScreenshotAssertions.toMatchSnapshot.maxDiffPixelRatio = %%-assertions-max-diff-pixel-ratio-%%
+
 ### option: ScreenshotAssertions.toMatchSnapshot.threshold = %%-assertions-threshold-%%

docs/src/test-api/class-testconfig.md

@@ -42,6 +42,8 @@ export default config;
     - `maxDiffPixelRatio` <[float]> an acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default.
   - `toMatchSnapshot` <[Object]>
     - `threshold` <[float]> an acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between the same pixel in compared images, between zero (strict) and one (lax). Defaults to `0.2`.
+    - `maxDiffPixels` <[int]> an acceptable amount of pixels that could be different, unset by default.
+    - `maxDiffPixelRatio` <[float]> an acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default.
 
 Configuration for the `expect` assertion library. Learn more about [various timeouts](./test-timeouts.md).
 

docs/src/test-api/class-testproject.md

@@ -113,6 +113,8 @@ export default config;
     - `maxDiffPixelRatio` <[float]> an acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default.
   - `toMatchSnapshot` <[Object]>
     - `threshold` <[float]> an acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between the same pixel in compared images, between zero (strict) and one (lax). Defaults to `0.2`.
+    - `maxDiffPixels` <[int]> an acceptable amount of pixels that could be different, unset by default.
+    - `maxDiffPixelRatio` <[float]> an acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default.
 
 Configuration for the `expect` assertion library.
 

packages/playwright-test/src/matchers/toMatchSnapshot.ts

@@ -41,8 +41,8 @@ const SNAPSHOT_COUNTER = Symbol('noname-snapshot-counter');
 export function getSnapshotName(
   testInfo: TestInfoImpl,
   received: any,
-  nameOrOptions: NameOrSegments | { name?: NameOrSegments } & MatchSnapshotOptions = {},
+  nameOrOptions: NameOrSegments | { name?: NameOrSegments } = {},
-  optOptions: MatchSnapshotOptions = {}
+  optOptions: any = {}
 ) {
   const [
     anonymousSnapshotExtension,
@@ -232,13 +232,11 @@ class SnapshotHelper<T extends ImageComparatorOptions> {
   }
 }
 
-type MatchSnapshotOptions = Omit<ImageComparatorOptions, 'maxDiffPixels' | 'maxDiffPixelRatio'>;
-
 export function toMatchSnapshot(
   this: ReturnType<Expect['getState']>,
   received: Buffer | string,
-  nameOrOptions: NameOrSegments | { name?: NameOrSegments } & MatchSnapshotOptions = {},
+  nameOrOptions: NameOrSegments | { name?: NameOrSegments } & ImageComparatorOptions = {},
-  optOptions: MatchSnapshotOptions = {}
+  optOptions: ImageComparatorOptions = {}
 ): SyncExpectationResult {
   const testInfo = currentTestInfo();
   if (!testInfo)
@@ -262,11 +260,7 @@ export function toMatchSnapshot(
     return helper.handleMissing(received);
 
   const expected = fs.readFileSync(helper.snapshotPath);
-  const result = comparator(received, expected, {
+  const result = comparator(received, expected, helper.comparatorOptions);
-    ...helper.comparatorOptions,
-    maxDiffPixels: undefined,
-    maxDiffPixelRatio: undefined,
-  });
   if (!result)
     return helper.handleMatching();
 

packages/playwright-test/types/test.d.ts

@@ -60,6 +60,14 @@ type ExpectSettings = {
     /** An acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between pixels in compared images, between zero (strict) and one (lax). Defaults to `0.2`.
      */
     threshold?: number,
+    /**
+     * An acceptable amount of pixels that could be different, unset by default.
+     */
+    maxDiffPixels?: number,
+    /**
+     * An acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default.
+     */
+    maxDiffPixelRatio?: number,
   }
 };
 

tests/playwright-test/golden.spec.ts

@@ -17,7 +17,7 @@
 import colors from 'colors/safe';
 import * as fs from 'fs';
 import * as path from 'path';
-import { test, expect, stripAnsi } from './playwright-test-fixtures';
+import { test, expect, stripAnsi, createWhiteImage, paintBlackPixels } from './playwright-test-fixtures';
 
 const files = {
   'helper.ts': `
@@ -79,6 +79,8 @@ test('should compile with different option combinations', async ({ runTSC }) =>
       test('is a test', async ({ page }) => {
         expect('foo').toMatchSnapshot();
         expect('foo').toMatchSnapshot({ threshold: 0.2 });
+        expect('foo').toMatchSnapshot({ maxDiffPixelRatio: 0.2 });
+        expect('foo').toMatchSnapshot({ maxDiffPixels: 0.2 });
       });
     `
   });
@@ -394,6 +396,106 @@ test('should compare binary', async ({ runInlineTest }) => {
   expect(result.exitCode).toBe(0);
 });
 
+test('should respect maxDiffPixels option', async ({ runInlineTest }) => {
+  const width = 20, height = 20;
+  const BAD_PIXELS = 120;
+  const image1 = createWhiteImage(width, height);
+  const image2 = paintBlackPixels(image1, BAD_PIXELS);
+
+  await test.step('make sure default comparison fails', async () => {
+    const result = await runInlineTest({
+      ...files,
+      'a.spec.js-snapshots/snapshot.png': image1,
+      'a.spec.js': `
+        const { test } = require('./helper');
+        test('is a test', ({}) => {
+          expect(Buffer.from('${image2.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png');
+        });
+      `
+    });
+    expect(stripAnsi(result.output)).toContain('120 pixels');
+    expect(stripAnsi(result.output)).toContain('ratio 0.30');
+    expect(result.exitCode).toBe(1);
+  });
+
+  expect((await runInlineTest({
+    ...files,
+    'a.spec.js-snapshots/snapshot.png': image1,
+    'a.spec.js': `
+      const { test } = require('./helper');
+      test('is a test', ({}) => {
+        expect(Buffer.from('${image2.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png', {
+          maxDiffPixels: ${BAD_PIXELS}
+        });
+      });
+    `
+  })).exitCode, 'make sure maxDiffPixels option is respected').toBe(0);
+
+  expect((await runInlineTest({
+    ...files,
+    'playwright.config.ts': `
+      module.exports = { projects: [
+        { expect: { toMatchSnapshot: { maxDiffPixels: ${BAD_PIXELS} } } },
+      ]};
+    `,
+    'a.spec.js-snapshots/snapshot.png': image1,
+    'a.spec.js': `
+      const { test } = require('./helper');
+      test('is a test', ({}) => {
+        expect(Buffer.from('${image2.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png');
+      });
+    `
+  })).exitCode, 'make sure maxDiffPixels option in project config is respected').toBe(0);
+});
+
+test('should respect maxDiffPixelRatio option', async ({ runInlineTest }) => {
+  const width = 20, height = 20;
+  const BAD_RATIO = 0.25;
+  const BAD_PIXELS = Math.floor(width * height * BAD_RATIO);
+  const image1 = createWhiteImage(width, height);
+  const image2 = paintBlackPixels(image1, BAD_PIXELS);
+
+  expect((await runInlineTest({
+    ...files,
+    'a.spec.js-snapshots/snapshot.png': image1,
+    'a.spec.js': `
+      const { test } = require('./helper');
+      test('is a test', ({}) => {
+        expect(Buffer.from('${image2.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png');
+      });
+    `
+  })).exitCode, 'make sure default comparison fails').toBe(1);
+
+  expect((await runInlineTest({
+    ...files,
+    'a.spec.js-snapshots/snapshot.png': image1,
+    'a.spec.js': `
+      const { test } = require('./helper');
+      test('is a test', ({}) => {
+        expect(Buffer.from('${image2.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png', {
+          maxDiffPixelRatio: ${BAD_RATIO}
+        });
+      });
+    `
+  })).exitCode, 'make sure maxDiffPixelRatio option is respected').toBe(0);
+
+  expect((await runInlineTest({
+    ...files,
+    'playwright.config.ts': `
+      module.exports = { projects: [
+        { expect: { toMatchSnapshot: { maxDiffPixelRatio: ${BAD_RATIO} } } },
+      ]};
+    `,
+    'a.spec.js-snapshots/snapshot.png': image1,
+    'a.spec.js': `
+      const { test } = require('./helper');
+      test('is a test', ({}) => {
+        expect(Buffer.from('${image2.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png');
+      });
+    `
+  })).exitCode, 'make sure maxDiffPixels option in project config is respected').toBe(0);
+});
+
 test('should compare PNG images', async ({ runInlineTest }) => {
   const result = await runInlineTest({
     ...files,
@@ -417,11 +519,7 @@ test('should compare different PNG images', async ({ runInlineTest }, testInfo)
     'a.spec.js': `
       const { test } = require('./helper');
       test('is a test', ({}) => {
-        expect(Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVQYV2NgYAAAAAMAAWgmWQ0AAAAASUVORK5CYII==', 'base64')).toMatchSnapshot('snapshot.png', {
+        expect(Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVQYV2NgYAAAAAMAAWgmWQ0AAAAASUVORK5CYII==', 'base64')).toMatchSnapshot('snapshot.png');
-          // make sure maxDiffPixelRatio is *not* respected.
-          // See https://github.com/microsoft/playwright/issues/12564
-          maxDiffPixelRatio: 1.0,
-        });
       });
     `
   });

utils/generate_types/overrides-test.d.ts

@@ -59,6 +59,14 @@ type ExpectSettings = {
     /** An acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between pixels in compared images, between zero (strict) and one (lax). Defaults to `0.2`.
      */
     threshold?: number,
+    /**
+     * An acceptable amount of pixels that could be different, unset by default.
+     */
+    maxDiffPixels?: number,
+    /**
+     * An acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default.
+     */
+    maxDiffPixelRatio?: number,
   }
 };