diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index a169d21673..f58262d48f 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -37,7 +37,9 @@ export default config; - type: <[Object]> - `timeout` <[int]> Default timeout for async expect matchers in milliseconds, defaults to 5000ms. - `toMatchSnapshot` <[Object]> - - `threshold` <[float]> Image matching threshold between zero (strict) and one (lax). + - `threshold` <[float]> an acceptable percieved color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between pixels in compared images, between zero (strict) and one (lax). Defaults to `0.2`. + - `pixelCount` <[int]> an acceptable amount of pixels that could be different, unset by default. + - `pixelRatio` <[float]> an acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default. Configuration for the `expect` assertion library. Learn more about [various timeouts](./test-timeouts.md). diff --git a/docs/src/test-api/class-testproject.md b/docs/src/test-api/class-testproject.md index 3fc8c7c1ef..c9c4f4df95 100644 --- a/docs/src/test-api/class-testproject.md +++ b/docs/src/test-api/class-testproject.md @@ -108,7 +108,9 @@ export default config; - type: <[Object]> - `timeout` <[int]> Default timeout for async expect matchers in milliseconds, defaults to 5000ms. - `toMatchSnapshot` <[Object]> - - `threshold` <[float]> Image matching threshold between zero (strict) and one (lax). + - `threshold` <[float]> an acceptable percieved color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between pixels in compared images, between zero (strict) and one (lax). Defaults to `0.2`. + - `pixelCount` <[int]> an acceptable amount of pixels that could be different, unset by default. + - `pixelRatio` <[float]> an acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default. Configuration for the `expect` assertion library. diff --git a/docs/src/test-assertions-js.md b/docs/src/test-assertions-js.md index a4c2911705..9bc9e5e61e 100644 --- a/docs/src/test-assertions-js.md +++ b/docs/src/test-assertions-js.md @@ -347,7 +347,9 @@ await expect(page).toHaveURL(/.*checkout/); ## expect(value).toMatchSnapshot(name[, options]) - `name` <[string] | [Array]<[string]>> Snapshot name. - `options` - - `threshold` <[float]> Image matching threshold between zero (strict) and one (lax), default is configurable with [`property: TestConfig.expect`]. + - `threshold` <[float]> an acceptable percieved color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between pixels in compared images, between zero (strict) and one (lax), default is configurable with [`property: TestConfig.expect`]. Defaults to `0.2`. + - `pixelCount` <[int]> an acceptable amount of pixels that could be different, unset by default. + - `pixelRatio` <[float]> an acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default. Ensures that passed value, either a [string] or a [Buffer], matches the expected snapshot stored in the test snapshots directory. diff --git a/docs/src/test-snapshots-js.md b/docs/src/test-snapshots-js.md index bab2d1534d..1ca3c2c2ee 100644 --- a/docs/src/test-snapshots-js.md +++ b/docs/src/test-snapshots-js.md @@ -67,7 +67,7 @@ npx playwright test --update-snapshots > Note that `snapshotName` also accepts an array of path segments to the snapshot file such as `expect(value).toMatchSnapshot(['relative', 'path', 'to', 'snapshot.png'])`. > However, this path must stay within the snapshots directory for each test file (i.e. `a.spec.js-snapshots`), otherwise it will throw. -Playwright Test uses the [pixelmatch](https://github.com/mapbox/pixelmatch) library. You can pass comparison `threshold` as an option. +Playwright Test uses the [pixelmatch](https://github.com/mapbox/pixelmatch) library. You can [pass various options](./test-assertions#expectvaluetomatchsnapshotname-options) to modify its behavior: ```js js-flavor=js // example.spec.js @@ -75,7 +75,7 @@ const { test, expect } = require('@playwright/test'); test('example test', async ({ page }) => { await page.goto('https://playwright.dev'); - expect(await page.screenshot()).toMatchSnapshot('home.png', { threshold: 0.2 }); + expect(await page.screenshot()).toMatchSnapshot('home.png', { pixelCount: 100 }); }); ``` @@ -85,7 +85,7 @@ import { test, expect } from '@playwright/test'; test('example test', async ({ page }) => { await page.goto('https://playwright.dev'); - expect(await page.screenshot()).toMatchSnapshot('home.png', { threshold: 0.2 }); + expect(await page.screenshot()).toMatchSnapshot('home.png', { pixelCount: 100 }); }); ``` @@ -94,7 +94,7 @@ If you'd like to share the default value among all the tests in the project, you ```js js-flavor=js module.exports = { expect: { - toMatchSnapshot: { threshold: 0.1 }, + toMatchSnapshot: { pixelCount: 100 }, }, }; ``` @@ -103,7 +103,7 @@ module.exports = { import { PlaywrightTestConfig } from '@playwright/test'; const config: PlaywrightTestConfig = { expect: { - toMatchSnapshot: { threshold: 0.1 }, + toMatchSnapshot: { pixelCount: 100 }, }, }; export default config; diff --git a/packages/playwright-test/src/matchers/golden.ts b/packages/playwright-test/src/matchers/golden.ts index cc95e3c325..214c379c4d 100644 --- a/packages/playwright-test/src/matchers/golden.ts +++ b/packages/playwright-test/src/matchers/golden.ts @@ -57,7 +57,7 @@ function compareBuffersOrStrings(actualBuffer: Buffer | string, expectedBuffer: return null; } -function compareImages(actualBuffer: Buffer | string, expectedBuffer: Buffer, mimeType: string, options = {}): { diff?: Buffer; errorMessage?: string; } | null { +function compareImages(actualBuffer: Buffer | string, expectedBuffer: Buffer, mimeType: string, options: { threshold?: number, pixelCount?: number, pixelRatio?: number } = {}): { diff?: Buffer; errorMessage?: string; } | null { if (!actualBuffer || !(actualBuffer instanceof Buffer)) return { errorMessage: 'Actual result should be a Buffer.' }; @@ -79,7 +79,15 @@ function compareImages(actualBuffer: Buffer | string, expectedBuffer: Buffer, mi return result.code !== BlinkDiff.RESULT_IDENTICAL ? { diff: PNG.sync.write(diff._imageOutput.getImage()) } : null; } const count = pixelmatch(expected.data, actual.data, diff.data, expected.width, expected.height, thresholdOptions); - return count > 0 ? { diff: PNG.sync.write(diff) } : null; + + const pixelCount1 = options.pixelCount; + const pixelCount2 = options.pixelRatio !== undefined ? expected.width * expected.height * options.pixelRatio : undefined; + let pixelCount; + if (pixelCount1 !== undefined && pixelCount2 !== undefined) + pixelCount = Math.min(pixelCount1, pixelCount2); + else + pixelCount = pixelCount1 ?? pixelCount2 ?? 0; + return count > pixelCount ? { diff: PNG.sync.write(diff) } : null; } function compareText(actual: Buffer | string, expectedBuffer: Buffer): { diff?: Buffer; errorMessage?: string; diffExtension?: string; } | null { @@ -102,7 +110,7 @@ export function compare( testInfo: TestInfoImpl, updateSnapshots: UpdateSnapshots, withNegateComparison: boolean, - options?: { threshold?: number } + options?: { threshold?: number, pixelCount?: number, pixelRatio?: number } ): { pass: boolean; message?: string; expectedPath?: string, actualPath?: string, diffPath?: string, mimeType?: string } { const snapshotFile = testInfo.snapshotPath(...pathSegments); const outputFile = testInfo.outputPath(...pathSegments); diff --git a/packages/playwright-test/src/matchers/toMatchSnapshot.ts b/packages/playwright-test/src/matchers/toMatchSnapshot.ts index ab0ecdc5d3..27493318d3 100644 --- a/packages/playwright-test/src/matchers/toMatchSnapshot.ts +++ b/packages/playwright-test/src/matchers/toMatchSnapshot.ts @@ -27,8 +27,8 @@ type SyncExpectationResult = { type NameOrSegments = string | string[]; const SNAPSHOT_COUNTER = Symbol('noname-snapshot-counter'); -export function toMatchSnapshot(this: ReturnType, received: Buffer | string, nameOrOptions: NameOrSegments | { name: NameOrSegments, threshold?: number }, optOptions: { threshold?: number } = {}): SyncExpectationResult { - let options: { name: NameOrSegments, threshold?: number }; +export function toMatchSnapshot(this: ReturnType, received: Buffer | string, nameOrOptions: NameOrSegments | { name: NameOrSegments, threshold?: number }, optOptions: { threshold?: number, pixelCount?: number, pixelRatio?: number } = {}): SyncExpectationResult { + let options: { name: NameOrSegments, threshold?: number, pixelCount?: number, pixelRatio?: number }; const testInfo = currentTestInfo(); if (!testInfo) throw new Error(`toMatchSnapshot() must be called during the test`); @@ -45,9 +45,16 @@ export function toMatchSnapshot(this: ReturnType, received: options.name = sanitizeForFilePath(trimLongString(fullTitleWithoutSpec)) + determineFileExtension(received); } - const projectThreshold = testInfo.project.expect?.toMatchSnapshot?.threshold; - if (options.threshold === undefined && projectThreshold !== undefined) - options.threshold = projectThreshold; + options = { + ...(testInfo.project.expect?.toMatchSnapshot || {}), + ...options, + }; + + if (options.pixelCount !== undefined && options.pixelCount < 0) + throw new Error('`pixelCount` option value must be non-negative integer'); + + if (options.pixelRatio !== undefined && (options.pixelRatio < 0 || options.pixelRatio > 1)) + throw new Error('`pixelRatio` option value must be between 0 and 1'); // sanitizes path if string const pathSegments = Array.isArray(options.name) ? options.name : [addSuffixToFilePath(options.name, '', undefined, true)]; diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index acbcef581b..ba9a766457 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -39,11 +39,22 @@ export type UpdateSnapshots = 'all' | 'none' | 'missing'; type UseOptions = { [K in keyof WorkerArgs]?: WorkerArgs[K] } & { [K in keyof TestArgs]?: TestArgs[K] }; type ExpectSettings = { - // Default timeout for async expect matchers in milliseconds, defaults to 5000ms. + /** + * Default timeout for async expect matchers in milliseconds, defaults to 5000ms. + */ timeout?: number; toMatchSnapshot?: { - // Pixel match threshold. - threshold?: number + /** An acceptable percieved color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between pixels in compared images, between zero (strict) and one (lax). Defaults to `0.2`. + */ + threshold?: number, + /** + * An acceptable amount of pixels that could be different, unset by default. + */ + pixelCount?: number, + /** + * An acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default. + */ + pixelRatio?: number, } }; diff --git a/packages/playwright-test/types/testExpect.d.ts b/packages/playwright-test/types/testExpect.d.ts index dc55261a52..0c638176ec 100644 --- a/packages/playwright-test/types/testExpect.d.ts +++ b/packages/playwright-test/types/testExpect.d.ts @@ -79,7 +79,9 @@ declare global { */ toMatchSnapshot(options?: { name?: string | string[], - threshold?: number + threshold?: number, + pixelCount?: number, + pixelRatio?: number, }): R; /** * Match snapshot diff --git a/tests/playwright-test/golden.spec.ts b/tests/playwright-test/golden.spec.ts index f82e10cc9c..36ab8d640c 100644 --- a/tests/playwright-test/golden.spec.ts +++ b/tests/playwright-test/golden.spec.ts @@ -17,6 +17,7 @@ import colors from 'colors/safe'; import * as fs from 'fs'; import * as path from 'path'; +import { PNG } from 'pngjs'; import { test, expect, stripAnsi } from './playwright-test-fixtures'; const files = { @@ -72,12 +73,15 @@ test('should generate default name', async ({ runInlineTest }, testInfo) => { expect(fs.existsSync(testInfo.outputPath('a.spec.js-snapshots', 'is-a-test-5.bin'))).toBe(true); }); -test('should compile without name', async ({ runTSC }) => { +test('should compile with different option combinations', async ({ runTSC }) => { const result = await runTSC({ 'a.spec.js': ` const { test, expect } = pwt; test('is a test', async ({ page }) => { expect('foo').toMatchSnapshot(); + expect('foo').toMatchSnapshot({ threshold: 0.2 }); + expect('foo').toMatchSnapshot({ pixelRatio: 0.2 }); + expect('foo').toMatchSnapshot({ pixelCount: 0.2 }); }); ` }); @@ -408,6 +412,186 @@ test('should compare binary', async ({ runInlineTest }) => { expect(result.exitCode).toBe(0); }); +test('should throw for invalid pixelCount values', async ({ runInlineTest }) => { + expect((await runInlineTest({ + ...files, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', ({}) => { + expect(Buffer.from([1,2,3,4])).toMatchSnapshot({ + pixelCount: -1, + }); + }); + ` + })).exitCode).toBe(1); +}); + +test('should throw for invalid pixelRatio values', async ({ runInlineTest }) => { + expect((await runInlineTest({ + ...files, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', ({}) => { + expect(Buffer.from([1,2,3,4])).toMatchSnapshot({ + pixelRatio: 12, + }); + }); + ` + })).exitCode).toBe(1); +}); + +test('should respect pixelCount option', async ({ runInlineTest }) => { + const width = 20, height = 20; + const BAD_PIXELS = 120; + const [image1, image2] = createImagesWithDifferentPixels(width, height, BAD_PIXELS); + + expect((await runInlineTest({ + ...files, + 'a.spec.js-snapshots/snapshot.png': image1, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', ({}) => { + expect(Buffer.from('${image2.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png'); + }); + ` + })).exitCode, 'make sure default comparison fails').toBe(1); + + expect((await runInlineTest({ + ...files, + 'a.spec.js-snapshots/snapshot.png': image1, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', ({}) => { + expect(Buffer.from('${image2.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png', { + pixelCount: ${BAD_PIXELS} + }); + }); + ` + })).exitCode, 'make sure pixelCount option is respected').toBe(0); + + expect((await runInlineTest({ + ...files, + 'playwright.config.ts': ` + module.exports = { projects: [ + { expect: { toMatchSnapshot: { pixelCount: ${BAD_PIXELS} } } }, + ]}; + `, + 'a.spec.js-snapshots/snapshot.png': image1, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', ({}) => { + expect(Buffer.from('${image2.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png'); + }); + ` + })).exitCode, 'make sure pixelCount option in project config is respected').toBe(0); +}); + +test('should respect pixelRatio option', async ({ runInlineTest }) => { + const width = 20, height = 20; + const BAD_PERCENT = 0.25; + const [image1, image2] = createImagesWithDifferentPixels(width, height, width * height * BAD_PERCENT); + + expect((await runInlineTest({ + ...files, + 'a.spec.js-snapshots/snapshot.png': image1, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', ({}) => { + expect(Buffer.from('${image2.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png'); + }); + ` + })).exitCode, 'make sure default comparison fails').toBe(1); + + expect((await runInlineTest({ + ...files, + 'a.spec.js-snapshots/snapshot.png': image1, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', ({}) => { + expect(Buffer.from('${image2.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png', { + pixelRatio: ${BAD_PERCENT} + }); + }); + ` + })).exitCode, 'make sure pixelRatio option is respected').toBe(0); + + expect((await runInlineTest({ + ...files, + 'playwright.config.ts': ` + module.exports = { projects: [ + { expect: { toMatchSnapshot: { pixelRatio: ${BAD_PERCENT} } } }, + ]}; + `, + 'a.spec.js-snapshots/snapshot.png': image1, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', ({}) => { + expect(Buffer.from('${image2.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png'); + }); + ` + })).exitCode, 'make sure pixelCount option in project config is respected').toBe(0); +}); + +test('should satisfy both pixelRatio and pixelCount', async ({ runInlineTest }) => { + const width = 20, height = 20; + const BAD_PERCENT = 0.25; + const BAD_COUNT = Math.floor(width * height * BAD_PERCENT); + const [image1, image2] = createImagesWithDifferentPixels(width, height, BAD_COUNT); + + expect((await runInlineTest({ + ...files, + 'a.spec.js-snapshots/snapshot.png': image1, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', ({}) => { + expect(Buffer.from('${image2.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png'); + }); + ` + })).exitCode, 'make sure default comparison fails').toBe(1); + + expect((await runInlineTest({ + ...files, + 'a.spec.js-snapshots/snapshot.png': image1, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', ({}) => { + expect(Buffer.from('${image2.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png', { + pixelCount: ${Math.floor(BAD_COUNT / 2)}, + pixelRatio: ${BAD_PERCENT}, + }); + }); + ` + })).exitCode, 'make sure it fails when pixelCount < actualBadPixels < pixelRatio').toBe(1); + + expect((await runInlineTest({ + ...files, + 'a.spec.js-snapshots/snapshot.png': image1, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', ({}) => { + expect(Buffer.from('${image2.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png', { + pixelCount: ${BAD_COUNT}, + pixelRatio: ${BAD_PERCENT / 2}, + }); + }); + ` + })).exitCode, 'make sure it fails when pixelRatio < actualBadPixels < pixelCount').toBe(1); + + expect((await runInlineTest({ + ...files, + 'a.spec.js-snapshots/snapshot.png': image1, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', ({}) => { + expect(Buffer.from('${image2.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png', { + pixelCount: ${BAD_COUNT}, + pixelRatio: ${BAD_PERCENT}, + }); + }); + ` + })).exitCode, 'make sure it passes when actualBadPixels < pixelRatio && actualBadPixels < pixelCount').toBe(0); +}); + test('should compare PNG images', async ({ runInlineTest }) => { const result = await runInlineTest({ ...files, @@ -820,3 +1004,21 @@ test('should allow comparing text with text without file extension', async ({ ru }); expect(result.exitCode).toBe(0); }); + +function createImagesWithDifferentPixels(width: number, height: number, differentPixels: number): [Buffer, Buffer] { + const image1 = new PNG({ width, height }); + const image2 = new PNG({ width, height }); + // Make both images red. + for (let i = 0; i < width * height; ++i) { + image1.data[i * 4] = 255; // red + image1.data[i * 4 + 3] = 255; // opacity + image2.data[i * 4] = 255; // red + image2.data[i * 4 + 3] = 255; // opacity + } + // Color some pixels blue. + for (let i = 0; i < differentPixels; ++i) { + image1.data[i * 4] = 0; // red + image1.data[i * 4 + 2] = 255; // blue + } + return [PNG.sync.write(image1), PNG.sync.write(image2)]; +} diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 3f35f1b7b5..a5e33d76ca 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -38,11 +38,22 @@ export type UpdateSnapshots = 'all' | 'none' | 'missing'; type UseOptions = { [K in keyof WorkerArgs]?: WorkerArgs[K] } & { [K in keyof TestArgs]?: TestArgs[K] }; type ExpectSettings = { - // Default timeout for async expect matchers in milliseconds, defaults to 5000ms. + /** + * Default timeout for async expect matchers in milliseconds, defaults to 5000ms. + */ timeout?: number; toMatchSnapshot?: { - // Pixel match threshold. - threshold?: number + /** An acceptable percieved color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between pixels in compared images, between zero (strict) and one (lax). Defaults to `0.2`. + */ + threshold?: number, + /** + * An acceptable amount of pixels that could be different, unset by default. + */ + pixelCount?: number, + /** + * An acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default. + */ + pixelRatio?: number, } };