From 4897967780d6c784912fec919cdb63830263797e Mon Sep 17 00:00:00 2001 From: Andrey Lushnikov Date: Fri, 11 Mar 2022 18:02:44 -0700 Subject: [PATCH] cherry-pick(#12627): bring back maxDiffPixels in toMatchSnapshot (#12701) This patch reverts 2 commits that removed the feature from the method: - "fix: explicitly ignore maxDiffPixels in toMatchSnapshot (#12570)" commit b8af8458d614dcc53ef324553d63522448e94d94. - "chore: remove `maxDiffPixels` from toMatchSnapshot (#12539)" commit a3dff45974d53087bd6ebea5164c75d5eb357e28. --- docs/src/api/class-screenshotassertions.md | 4 + docs/src/test-api/class-testconfig.md | 2 + docs/src/test-api/class-testproject.md | 2 + .../src/matchers/toMatchSnapshot.ts | 12 +- packages/playwright-test/types/test.d.ts | 8 ++ tests/playwright-test/golden.spec.ts | 110 +++++++++++++++++- utils/generate_types/overrides-test.d.ts | 8 ++ 7 files changed, 131 insertions(+), 15 deletions(-) diff --git a/docs/src/api/class-screenshotassertions.md b/docs/src/api/class-screenshotassertions.md index a0c4d0329e..3089adfbdd 100644 --- a/docs/src/api/class-screenshotassertions.md +++ b/docs/src/api/class-screenshotassertions.md @@ -34,4 +34,8 @@ Learn more about [visual comparisons](./test-snapshots.md). Snapshot name. +### option: ScreenshotAssertions.toMatchSnapshot.maxDiffPixels = %%-assertions-max-diff-pixels-%% + +### option: ScreenshotAssertions.toMatchSnapshot.maxDiffPixelRatio = %%-assertions-max-diff-pixel-ratio-%% + ### option: ScreenshotAssertions.toMatchSnapshot.threshold = %%-assertions-threshold-%% diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index 0cae21d625..1f53340a19 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -42,6 +42,8 @@ export default config; - `maxDiffPixelRatio` <[float]> an acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default. - `toMatchSnapshot` <[Object]> - `threshold` <[float]> an acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between the same pixel in compared images, between zero (strict) and one (lax). Defaults to `0.2`. + - `maxDiffPixels` <[int]> an acceptable amount of pixels that could be different, unset by default. + - `maxDiffPixelRatio` <[float]> an acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default. Configuration for the `expect` assertion library. Learn more about [various timeouts](./test-timeouts.md). diff --git a/docs/src/test-api/class-testproject.md b/docs/src/test-api/class-testproject.md index f3985c4d54..748ef27ebf 100644 --- a/docs/src/test-api/class-testproject.md +++ b/docs/src/test-api/class-testproject.md @@ -113,6 +113,8 @@ export default config; - `maxDiffPixelRatio` <[float]> an acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default. - `toMatchSnapshot` <[Object]> - `threshold` <[float]> an acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between the same pixel in compared images, between zero (strict) and one (lax). Defaults to `0.2`. + - `maxDiffPixels` <[int]> an acceptable amount of pixels that could be different, unset by default. + - `maxDiffPixelRatio` <[float]> an acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default. Configuration for the `expect` assertion library. diff --git a/packages/playwright-test/src/matchers/toMatchSnapshot.ts b/packages/playwright-test/src/matchers/toMatchSnapshot.ts index a5b9c6bc95..1e063f49e8 100644 --- a/packages/playwright-test/src/matchers/toMatchSnapshot.ts +++ b/packages/playwright-test/src/matchers/toMatchSnapshot.ts @@ -206,13 +206,11 @@ class SnapshotHelper { } } -type MatchSnapshotOptions = Omit; - export function toMatchSnapshot( this: ReturnType, received: Buffer | string, - nameOrOptions: NameOrSegments | { name?: NameOrSegments } & MatchSnapshotOptions = {}, - optOptions: MatchSnapshotOptions = {} + nameOrOptions: NameOrSegments | { name?: NameOrSegments } & ImageComparatorOptions = {}, + optOptions: ImageComparatorOptions = {} ): SyncExpectationResult { const testInfo = currentTestInfo(); if (!testInfo) @@ -236,11 +234,7 @@ export function toMatchSnapshot( return helper.handleMissing(received); const expected = fs.readFileSync(helper.snapshotPath); - const result = comparator(received, expected, { - ...helper.comparatorOptions, - maxDiffPixels: undefined, - maxDiffPixelRatio: undefined, - }); + const result = comparator(received, expected, helper.comparatorOptions); if (!result) return helper.handleMatching(); diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index 3591e63d4d..5322b9591d 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -60,6 +60,14 @@ type ExpectSettings = { /** An acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between pixels in compared images, between zero (strict) and one (lax). Defaults to `0.2`. */ threshold?: number, + /** + * An acceptable amount of pixels that could be different, unset by default. + */ + maxDiffPixels?: number, + /** + * An acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default. + */ + maxDiffPixelRatio?: number, } }; diff --git a/tests/playwright-test/golden.spec.ts b/tests/playwright-test/golden.spec.ts index ffccf1ba62..9952e6163b 100644 --- a/tests/playwright-test/golden.spec.ts +++ b/tests/playwright-test/golden.spec.ts @@ -17,7 +17,7 @@ import colors from 'colors/safe'; import * as fs from 'fs'; import * as path from 'path'; -import { test, expect, stripAnsi } from './playwright-test-fixtures'; +import { test, expect, stripAnsi, createWhiteImage, paintBlackPixels } from './playwright-test-fixtures'; const files = { 'helper.ts': ` @@ -79,6 +79,8 @@ test('should compile with different option combinations', async ({ runTSC }) => test('is a test', async ({ page }) => { expect('foo').toMatchSnapshot(); expect('foo').toMatchSnapshot({ threshold: 0.2 }); + expect('foo').toMatchSnapshot({ maxDiffPixelRatio: 0.2 }); + expect('foo').toMatchSnapshot({ maxDiffPixels: 0.2 }); }); ` }); @@ -394,6 +396,106 @@ test('should compare binary', async ({ runInlineTest }) => { expect(result.exitCode).toBe(0); }); +test('should respect maxDiffPixels option', async ({ runInlineTest }) => { + const width = 20, height = 20; + const BAD_PIXELS = 120; + const image1 = createWhiteImage(width, height); + const image2 = paintBlackPixels(image1, BAD_PIXELS); + + await test.step('make sure default comparison fails', async () => { + const result = await runInlineTest({ + ...files, + 'a.spec.js-snapshots/snapshot.png': image1, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', ({}) => { + expect(Buffer.from('${image2.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png'); + }); + ` + }); + expect(stripAnsi(result.output)).toContain('120 pixels'); + expect(stripAnsi(result.output)).toContain('ratio 0.30'); + expect(result.exitCode).toBe(1); + }); + + expect((await runInlineTest({ + ...files, + 'a.spec.js-snapshots/snapshot.png': image1, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', ({}) => { + expect(Buffer.from('${image2.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png', { + maxDiffPixels: ${BAD_PIXELS} + }); + }); + ` + })).exitCode, 'make sure maxDiffPixels option is respected').toBe(0); + + expect((await runInlineTest({ + ...files, + 'playwright.config.ts': ` + module.exports = { projects: [ + { expect: { toMatchSnapshot: { maxDiffPixels: ${BAD_PIXELS} } } }, + ]}; + `, + 'a.spec.js-snapshots/snapshot.png': image1, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', ({}) => { + expect(Buffer.from('${image2.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png'); + }); + ` + })).exitCode, 'make sure maxDiffPixels option in project config is respected').toBe(0); +}); + +test('should respect maxDiffPixelRatio option', async ({ runInlineTest }) => { + const width = 20, height = 20; + const BAD_RATIO = 0.25; + const BAD_PIXELS = Math.floor(width * height * BAD_RATIO); + const image1 = createWhiteImage(width, height); + const image2 = paintBlackPixels(image1, BAD_PIXELS); + + expect((await runInlineTest({ + ...files, + 'a.spec.js-snapshots/snapshot.png': image1, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', ({}) => { + expect(Buffer.from('${image2.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png'); + }); + ` + })).exitCode, 'make sure default comparison fails').toBe(1); + + expect((await runInlineTest({ + ...files, + 'a.spec.js-snapshots/snapshot.png': image1, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', ({}) => { + expect(Buffer.from('${image2.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png', { + maxDiffPixelRatio: ${BAD_RATIO} + }); + }); + ` + })).exitCode, 'make sure maxDiffPixelRatio option is respected').toBe(0); + + expect((await runInlineTest({ + ...files, + 'playwright.config.ts': ` + module.exports = { projects: [ + { expect: { toMatchSnapshot: { maxDiffPixelRatio: ${BAD_RATIO} } } }, + ]}; + `, + 'a.spec.js-snapshots/snapshot.png': image1, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', ({}) => { + expect(Buffer.from('${image2.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png'); + }); + ` + })).exitCode, 'make sure maxDiffPixels option in project config is respected').toBe(0); +}); + test('should compare PNG images', async ({ runInlineTest }) => { const result = await runInlineTest({ ...files, @@ -417,11 +519,7 @@ test('should compare different PNG images', async ({ runInlineTest }, testInfo) 'a.spec.js': ` const { test } = require('./helper'); test('is a test', ({}) => { - expect(Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVQYV2NgYAAAAAMAAWgmWQ0AAAAASUVORK5CYII==', 'base64')).toMatchSnapshot('snapshot.png', { - // make sure maxDiffPixelRatio is *not* respected. - // See https://github.com/microsoft/playwright/issues/12564 - maxDiffPixelRatio: 1.0, - }); + expect(Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVQYV2NgYAAAAAMAAWgmWQ0AAAAASUVORK5CYII==', 'base64')).toMatchSnapshot('snapshot.png'); }); ` }); diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 2fc439b831..52484b5511 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -59,6 +59,14 @@ type ExpectSettings = { /** An acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between pixels in compared images, between zero (strict) and one (lax). Defaults to `0.2`. */ threshold?: number, + /** + * An acceptable amount of pixels that could be different, unset by default. + */ + maxDiffPixels?: number, + /** + * An acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default. + */ + maxDiffPixelRatio?: number, } };