diff --git a/docs/src/api/class-screenshotassertions.md b/docs/src/api/class-screenshotassertions.md index 3089adfbdd..a0c4d0329e 100644 --- a/docs/src/api/class-screenshotassertions.md +++ b/docs/src/api/class-screenshotassertions.md @@ -34,8 +34,4 @@ Learn more about [visual comparisons](./test-snapshots.md). Snapshot name. -### option: ScreenshotAssertions.toMatchSnapshot.maxDiffPixels = %%-assertions-max-diff-pixels-%% - -### option: ScreenshotAssertions.toMatchSnapshot.maxDiffPixelRatio = %%-assertions-max-diff-pixel-ratio-%% - ### option: ScreenshotAssertions.toMatchSnapshot.threshold = %%-assertions-threshold-%% diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index 1f53340a19..0cae21d625 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -42,8 +42,6 @@ export default config; - `maxDiffPixelRatio` <[float]> an acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default. - `toMatchSnapshot` <[Object]> - `threshold` <[float]> an acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between the same pixel in compared images, between zero (strict) and one (lax). Defaults to `0.2`. - - `maxDiffPixels` <[int]> an acceptable amount of pixels that could be different, unset by default. - - `maxDiffPixelRatio` <[float]> an acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default. Configuration for the `expect` assertion library. Learn more about [various timeouts](./test-timeouts.md). diff --git a/docs/src/test-api/class-testproject.md b/docs/src/test-api/class-testproject.md index 748ef27ebf..f3985c4d54 100644 --- a/docs/src/test-api/class-testproject.md +++ b/docs/src/test-api/class-testproject.md @@ -113,8 +113,6 @@ export default config; - `maxDiffPixelRatio` <[float]> an acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default. - `toMatchSnapshot` <[Object]> - `threshold` <[float]> an acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between the same pixel in compared images, between zero (strict) and one (lax). Defaults to `0.2`. - - `maxDiffPixels` <[int]> an acceptable amount of pixels that could be different, unset by default. - - `maxDiffPixelRatio` <[float]> an acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default. Configuration for the `expect` assertion library. diff --git a/packages/playwright-test/src/matchers/toMatchSnapshot.ts b/packages/playwright-test/src/matchers/toMatchSnapshot.ts index 1e063f49e8..ca66a264aa 100644 --- a/packages/playwright-test/src/matchers/toMatchSnapshot.ts +++ b/packages/playwright-test/src/matchers/toMatchSnapshot.ts @@ -206,11 +206,13 @@ class SnapshotHelper { } } +type MatchSnapshotOptions = Omit; + export function toMatchSnapshot( this: ReturnType, received: Buffer | string, - nameOrOptions: NameOrSegments | { name?: NameOrSegments } & ImageComparatorOptions = {}, - optOptions: ImageComparatorOptions = {} + nameOrOptions: NameOrSegments | { name?: NameOrSegments } & MatchSnapshotOptions = {}, + optOptions: MatchSnapshotOptions = {} ): SyncExpectationResult { const testInfo = currentTestInfo(); if (!testInfo) diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index 5322b9591d..3591e63d4d 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -60,14 +60,6 @@ type ExpectSettings = { /** An acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between pixels in compared images, between zero (strict) and one (lax). Defaults to `0.2`. */ threshold?: number, - /** - * An acceptable amount of pixels that could be different, unset by default. - */ - maxDiffPixels?: number, - /** - * An acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default. - */ - maxDiffPixelRatio?: number, } }; diff --git a/tests/playwright-test/golden.spec.ts b/tests/playwright-test/golden.spec.ts index bfa73951f4..43703c72f2 100644 --- a/tests/playwright-test/golden.spec.ts +++ b/tests/playwright-test/golden.spec.ts @@ -17,7 +17,7 @@ import colors from 'colors/safe'; import * as fs from 'fs'; import * as path from 'path'; -import { test, expect, stripAnsi, createWhiteImage, paintBlackPixels } from './playwright-test-fixtures'; +import { test, expect, stripAnsi } from './playwright-test-fixtures'; const files = { 'helper.ts': ` @@ -79,8 +79,6 @@ test('should compile with different option combinations', async ({ runTSC }) => test('is a test', async ({ page }) => { expect('foo').toMatchSnapshot(); expect('foo').toMatchSnapshot({ threshold: 0.2 }); - expect('foo').toMatchSnapshot({ maxDiffPixelRatio: 0.2 }); - expect('foo').toMatchSnapshot({ maxDiffPixels: 0.2 }); }); ` }); @@ -396,190 +394,6 @@ test('should compare binary', async ({ runInlineTest }) => { expect(result.exitCode).toBe(0); }); -test('should throw for invalid maxDiffPixels values', async ({ runInlineTest }) => { - expect((await runInlineTest({ - ...files, - 'a.spec.js': ` - const { test } = require('./helper'); - test('is a test', ({}) => { - expect(Buffer.from([1,2,3,4])).toMatchSnapshot({ - maxDiffPixels: -1, - }); - }); - ` - })).exitCode).toBe(1); -}); - -test('should throw for invalid maxDiffPixelRatio values', async ({ runInlineTest }) => { - expect((await runInlineTest({ - ...files, - 'a.spec.js': ` - const { test } = require('./helper'); - test('is a test', ({}) => { - expect(Buffer.from([1,2,3,4])).toMatchSnapshot({ - maxDiffPixelRatio: 12, - }); - }); - ` - })).exitCode).toBe(1); -}); - -test('should respect maxDiffPixels option', async ({ runInlineTest }) => { - const width = 20, height = 20; - const BAD_PIXELS = 120; - const image1 = createWhiteImage(width, height); - const image2 = paintBlackPixels(image1, BAD_PIXELS); - - expect((await runInlineTest({ - ...files, - 'a.spec.js-snapshots/snapshot.png': image1, - 'a.spec.js': ` - const { test } = require('./helper'); - test('is a test', ({}) => { - expect(Buffer.from('${image2.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png'); - }); - ` - })).exitCode, 'make sure default comparison fails').toBe(1); - - expect((await runInlineTest({ - ...files, - 'a.spec.js-snapshots/snapshot.png': image1, - 'a.spec.js': ` - const { test } = require('./helper'); - test('is a test', ({}) => { - expect(Buffer.from('${image2.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png', { - maxDiffPixels: ${BAD_PIXELS} - }); - }); - ` - })).exitCode, 'make sure maxDiffPixels option is respected').toBe(0); - - expect((await runInlineTest({ - ...files, - 'playwright.config.ts': ` - module.exports = { projects: [ - { expect: { toMatchSnapshot: { maxDiffPixels: ${BAD_PIXELS} } } }, - ]}; - `, - 'a.spec.js-snapshots/snapshot.png': image1, - 'a.spec.js': ` - const { test } = require('./helper'); - test('is a test', ({}) => { - expect(Buffer.from('${image2.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png'); - }); - ` - })).exitCode, 'make sure maxDiffPixels option in project config is respected').toBe(0); -}); - -test('should respect maxDiffPixelRatio option', async ({ runInlineTest }) => { - const width = 20, height = 20; - const BAD_RATIO = 0.25; - const BAD_PIXELS = Math.floor(width * height * BAD_RATIO); - const image1 = createWhiteImage(width, height); - const image2 = paintBlackPixels(image1, BAD_PIXELS); - - expect((await runInlineTest({ - ...files, - 'a.spec.js-snapshots/snapshot.png': image1, - 'a.spec.js': ` - const { test } = require('./helper'); - test('is a test', ({}) => { - expect(Buffer.from('${image2.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png'); - }); - ` - })).exitCode, 'make sure default comparison fails').toBe(1); - - expect((await runInlineTest({ - ...files, - 'a.spec.js-snapshots/snapshot.png': image1, - 'a.spec.js': ` - const { test } = require('./helper'); - test('is a test', ({}) => { - expect(Buffer.from('${image2.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png', { - maxDiffPixelRatio: ${BAD_RATIO} - }); - }); - ` - })).exitCode, 'make sure maxDiffPixelRatio option is respected').toBe(0); - - expect((await runInlineTest({ - ...files, - 'playwright.config.ts': ` - module.exports = { projects: [ - { expect: { toMatchSnapshot: { maxDiffPixelRatio: ${BAD_RATIO} } } }, - ]}; - `, - 'a.spec.js-snapshots/snapshot.png': image1, - 'a.spec.js': ` - const { test } = require('./helper'); - test('is a test', ({}) => { - expect(Buffer.from('${image2.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png'); - }); - ` - })).exitCode, 'make sure maxDiffPixels option in project config is respected').toBe(0); -}); - -test('should satisfy both maxDiffPixelRatio and maxDiffPixels', async ({ runInlineTest }) => { - const width = 20, height = 20; - const BAD_RATIO = 0.25; - const BAD_COUNT = Math.floor(width * height * BAD_RATIO); - const image1 = createWhiteImage(width, height); - const image2 = paintBlackPixels(image1, BAD_COUNT); - - expect((await runInlineTest({ - ...files, - 'a.spec.js-snapshots/snapshot.png': image1, - 'a.spec.js': ` - const { test } = require('./helper'); - test('is a test', ({}) => { - expect(Buffer.from('${image2.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png'); - }); - ` - })).exitCode, 'make sure default comparison fails').toBe(1); - - expect((await runInlineTest({ - ...files, - 'a.spec.js-snapshots/snapshot.png': image1, - 'a.spec.js': ` - const { test } = require('./helper'); - test('is a test', ({}) => { - expect(Buffer.from('${image2.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png', { - maxDiffPixels: ${Math.floor(BAD_COUNT / 2)}, - maxDiffPixelRatio: ${BAD_RATIO}, - }); - }); - ` - })).exitCode, 'make sure it fails when maxDiffPixels < actualBadPixels < maxDiffPixelRatio').toBe(1); - - expect((await runInlineTest({ - ...files, - 'a.spec.js-snapshots/snapshot.png': image1, - 'a.spec.js': ` - const { test } = require('./helper'); - test('is a test', ({}) => { - expect(Buffer.from('${image2.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png', { - maxDiffPixels: ${BAD_COUNT}, - maxDiffPixelRatio: ${BAD_RATIO / 2}, - }); - }); - ` - })).exitCode, 'make sure it fails when maxDiffPixelRatio < actualBadPixels < maxDiffPixels').toBe(1); - - expect((await runInlineTest({ - ...files, - 'a.spec.js-snapshots/snapshot.png': image1, - 'a.spec.js': ` - const { test } = require('./helper'); - test('is a test', ({}) => { - expect(Buffer.from('${image2.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png', { - maxDiffPixels: ${BAD_COUNT}, - maxDiffPixelRatio: ${BAD_RATIO}, - }); - }); - ` - })).exitCode, 'make sure it passes when actualBadPixels < maxDiffPixelRatio && actualBadPixels < maxDiffPixels').toBe(0); -}); - test('should compare PNG images', async ({ runInlineTest }) => { const result = await runInlineTest({ ...files, diff --git a/tests/playwright-test/to-have-screenshot.spec.ts b/tests/playwright-test/to-have-screenshot.spec.ts index 7bc3ad66a4..ef78a1cfa2 100644 --- a/tests/playwright-test/to-have-screenshot.spec.ts +++ b/tests/playwright-test/to-have-screenshot.spec.ts @@ -520,6 +520,67 @@ test('should respect maxDiffPixels option', async ({ runInlineTest }) => { })).exitCode, 'make sure maxDiffPixels option in project config is respected').toBe(0); }); +test('should satisfy both maxDiffPixelRatio and maxDiffPixels', async ({ runInlineTest }) => { + const BAD_RATIO = 0.25; + const BAD_COUNT = Math.floor(IMG_WIDTH * IMG_HEIGHT * BAD_RATIO); + const EXPECTED_SNAPSHOT = paintBlackPixels(whiteImage, BAD_COUNT); + + expect((await runInlineTest({ + ...files, + 'a.spec.js-snapshots/snapshot.png': EXPECTED_SNAPSHOT, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await expect(page).toHaveScreenshot('snapshot.png', { timeout: 2000 }); + }); + ` + })).exitCode, 'make sure default comparison fails').toBe(1); + + expect((await runInlineTest({ + ...files, + 'a.spec.js-snapshots/snapshot.png': EXPECTED_SNAPSHOT, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await expect(page).toHaveScreenshot('snapshot.png', { + maxDiffPixels: ${Math.floor(BAD_COUNT / 2)}, + maxDiffPixelRatio: ${BAD_RATIO}, + timeout: 2000, + }); + }); + ` + })).exitCode, 'make sure it fails when maxDiffPixels < actualBadPixels < maxDiffPixelRatio').toBe(1); + + expect((await runInlineTest({ + ...files, + 'a.spec.js-snapshots/snapshot.png': EXPECTED_SNAPSHOT, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await expect(page).toHaveScreenshot('snapshot.png', { + maxDiffPixels: ${BAD_COUNT}, + maxDiffPixelRatio: ${BAD_RATIO / 2}, + timeout: 2000, + }); + }); + ` + })).exitCode, 'make sure it fails when maxDiffPixelRatio < actualBadPixels < maxDiffPixels').toBe(1); + + expect((await runInlineTest({ + ...files, + 'a.spec.js-snapshots/snapshot.png': EXPECTED_SNAPSHOT, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await expect(page).toHaveScreenshot('snapshot.png', { + maxDiffPixels: ${BAD_COUNT}, + maxDiffPixelRatio: ${BAD_RATIO}, + }); + }); + ` + })).exitCode, 'make sure it passes when actualBadPixels < maxDiffPixelRatio && actualBadPixels < maxDiffPixels').toBe(0); +}); + test('should respect maxDiffPixelRatio option', async ({ runInlineTest }) => { const BAD_RATIO = 0.25; const BAD_PIXELS = IMG_WIDTH * IMG_HEIGHT * BAD_RATIO; @@ -566,6 +627,35 @@ test('should respect maxDiffPixelRatio option', async ({ runInlineTest }) => { })).exitCode, 'make sure maxDiffPixels option in project config is respected').toBe(0); }); +test('should throw for invalid maxDiffPixels values', async ({ runInlineTest }) => { + expect((await runInlineTest({ + ...files, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await expect(page).toHaveScreenshot({ + maxDiffPixels: -1, + }); + }); + ` + })).exitCode).toBe(1); +}); + +test('should throw for invalid maxDiffPixelRatio values', async ({ runInlineTest }) => { + expect((await runInlineTest({ + ...files, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await expect(page).toHaveScreenshot({ + maxDiffPixelRatio: 12, + }); + }); + ` + })).exitCode).toBe(1); +}); + + test('should attach expected/actual and no diff when sizes are different', async ({ runInlineTest }, testInfo) => { const result = await runInlineTest({ ...files, diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 52484b5511..2fc439b831 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -59,14 +59,6 @@ type ExpectSettings = { /** An acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between pixels in compared images, between zero (strict) and one (lax). Defaults to `0.2`. */ threshold?: number, - /** - * An acceptable amount of pixels that could be different, unset by default. - */ - maxDiffPixels?: number, - /** - * An acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default. - */ - maxDiffPixelRatio?: number, } };