From 4953fc48457fc55a029df2c57c7c4299748eb3b4 Mon Sep 17 00:00:00 2001 From: Andrey Lushnikov Date: Fri, 4 Mar 2022 18:30:43 -0700 Subject: [PATCH] chore: introduce toHaveScreenshot configuration options (#12507) References https://github.com/microsoft/playwright/issues/12441 --- docs/src/api/params.md | 6 ++-- docs/src/test-api/class-testconfig.md | 16 ++++++---- docs/src/test-api/class-testinfo.md | 2 +- docs/src/test-api/class-testproject.md | 8 +++-- docs/src/test-snapshots-js.md | 31 ++++++++++--------- .../src/matchers/toMatchSnapshot.ts | 13 ++++++-- packages/playwright-test/types/test.d.ts | 31 ++++++++++++++----- .../to-have-screenshot.spec.ts | 4 +-- utils/generate_types/overrides-test.d.ts | 15 ++++++++- 9 files changed, 86 insertions(+), 40 deletions(-) diff --git a/docs/src/api/params.md b/docs/src/api/params.md index 057c3db7e6..242aa1d89a 100644 --- a/docs/src/api/params.md +++ b/docs/src/api/params.md @@ -719,19 +719,19 @@ Time to retry the assertion for. * langs: js - `maxDiffPixels` <[int]> -An acceptable amount of pixels that could be different, unset by default. +An acceptable amount of pixels that could be different, default is configurable with `TestConfig.expect`. Default is configurable with `TestConfig.expect`. Unset by default. ## assertions-max-diff-pixel-ratio * langs: js - `maxDiffPixelRatio` <[float]> -An acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1`, unset by default. +An acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1`. Default is configurable with `TestConfig.expect`. Unset by default. ## assertions-threshold * langs: js - `threshold` <[float]> -An acceptable percieved color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between the same pixel in compared images, between zero (strict) and one (lax), default is configurable with `TestConfig.expect`. Defaults to `0.2`. +An acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between the same pixel in compared images, between zero (strict) and one (lax), default is configurable with `TestConfig.expect`. Defaults to `0.2`. ## shared-context-params-list - %%-context-option-acceptdownloads-%% diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index 44dcabd0d1..1f53340a19 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -36,8 +36,12 @@ export default config; ## property: TestConfig.expect - type: <[Object]> - `timeout` <[int]> Default timeout for async expect matchers in milliseconds, defaults to 5000ms. + - `toHaveScreenshot` <[Object]> + - `threshold` <[float]> an acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between the same pixel in compared images, between zero (strict) and one (lax). Defaults to `0.2`. + - `maxDiffPixels` <[int]> an acceptable amount of pixels that could be different, unset by default. + - `maxDiffPixelRatio` <[float]> an acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default. - `toMatchSnapshot` <[Object]> - - `threshold` <[float]> an acceptable percieved color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between the same pixel in compared images, between zero (strict) and one (lax). Defaults to `0.2`. + - `threshold` <[float]> an acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between the same pixel in compared images, between zero (strict) and one (lax). Defaults to `0.2`. - `maxDiffPixels` <[int]> an acceptable amount of pixels that could be different, unset by default. - `maxDiffPixelRatio` <[float]> an acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default. @@ -51,8 +55,8 @@ Configuration for the `expect` assertion library. Learn more about [various time const config = { expect: { timeout: 10000, - toMatchSnapshot: { - threshold: 0.3, + toHaveScreenshot: { + maxDiffPixels: 10, }, }, }; @@ -67,8 +71,8 @@ import { PlaywrightTestConfig } from '@playwright/test'; const config: PlaywrightTestConfig = { expect: { timeout: 10000, - toMatchSnapshot: { - threshold: 0.3, + toHaveScreenshot: { + maxDiffPixels: 10, }, }, }; @@ -299,7 +303,7 @@ test('example test', async ({}, testInfo) => { ## property: TestConfig.snapshotDir - type: <[string]> -The base directory, relative to the config file, for snapshot files created with `toMatchSnapshot`. Defaults to [`property: TestConfig.testDir`]. +The base directory, relative to the config file, for snapshot files created with `toMatchSnapshot` and `toHaveScreenshot`. Defaults to [`property: TestConfig.testDir`]. The directory for each test can be accessed by [`property: TestInfo.snapshotDir`] and [`method: TestInfo.snapshotPath`]. diff --git a/docs/src/test-api/class-testinfo.md b/docs/src/test-api/class-testinfo.md index 0b087c6b7a..c55dffb3d3 100644 --- a/docs/src/test-api/class-testinfo.md +++ b/docs/src/test-api/class-testinfo.md @@ -382,7 +382,7 @@ The name of the snapshot or the path segments to define the snapshot file path. ## property: TestInfo.snapshotSuffix - type: <[string]> -Suffix used to differentiate snapshots between multiple test configurations. For example, if snapshots depend on the platform, you can set `testInfo.snapshotSuffix` equal to `process.platform`. In this case `expect(value).toMatchSnapshot(snapshotName)` will use different snapshots depending on the platform. Learn more about [snapshots](./test-snapshots.md). +Suffix used to differentiate snapshots between multiple test configurations. For example, if snapshots depend on the platform, you can set `testInfo.snapshotSuffix` equal to `process.platform`. In this case both `expect(value).toMatchSnapshot(snapshotName)` and `expect(page).toHaveScreenshot(snapshotName)` will use different snapshots depending on the platform. Learn more about [snapshots](./test-snapshots.md). ## property: TestInfo.status - type: <[void]|[TestStatus]<"passed"|"failed"|"timedOut"|"skipped">> diff --git a/docs/src/test-api/class-testproject.md b/docs/src/test-api/class-testproject.md index 9292a90913..748ef27ebf 100644 --- a/docs/src/test-api/class-testproject.md +++ b/docs/src/test-api/class-testproject.md @@ -107,8 +107,12 @@ export default config; ## property: TestProject.expect - type: <[Object]> - `timeout` <[int]> Default timeout for async expect matchers in milliseconds, defaults to 5000ms. + - `toHaveScreenshot` <[Object]> + - `threshold` <[float]> an acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between the same pixel in compared images, between zero (strict) and one (lax). Defaults to `0.2`. + - `maxDiffPixels` <[int]> an acceptable amount of pixels that could be different, unset by default. + - `maxDiffPixelRatio` <[float]> an acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default. - `toMatchSnapshot` <[Object]> - - `threshold` <[float]> an acceptable percieved color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between the same pixel in compared images, between zero (strict) and one (lax). Defaults to `0.2`. + - `threshold` <[float]> an acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between the same pixel in compared images, between zero (strict) and one (lax). Defaults to `0.2`. - `maxDiffPixels` <[int]> an acceptable amount of pixels that could be different, unset by default. - `maxDiffPixelRatio` <[float]> an acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default. @@ -151,7 +155,7 @@ Project name is visible in the report and during test execution. ## property: TestProject.snapshotDir - type: <[string]> -The base directory, relative to the config file, for snapshot files created with `toMatchSnapshot`. Defaults to [`property: TestProject.testDir`]. +The base directory, relative to the config file, for snapshot files created with `toMatchSnapshot` and `toHaveScreenshot`. Defaults to [`property: TestProject.testDir`]. The directory for each test can be accessed by [`property: TestInfo.snapshotDir`] and [`method: TestInfo.snapshotPath`]. diff --git a/docs/src/test-snapshots-js.md b/docs/src/test-snapshots-js.md index 9bbf6b9da6..10c45db94c 100644 --- a/docs/src/test-snapshots-js.md +++ b/docs/src/test-snapshots-js.md @@ -3,7 +3,7 @@ id: test-snapshots title: "Visual comparisons" --- -Playwright Test includes the ability to produce and visually compare screenshots using `expect(value).toMatchSnapshot()`. On first execution, Playwright test will generate reference screenshots. Subsequent runs will compare against the reference. +Playwright Test includes the ability to produce and visually compare screenshots using `await expect(pageOrLocator).toHaveScreenshot()`. On first execution, Playwright test will generate reference screenshots. Subsequent runs will compare against the reference. ```js js-flavor=js // example.spec.js @@ -11,7 +11,7 @@ const { test, expect } = require('@playwright/test'); test('example test', async ({ page }) => { await page.goto('https://playwright.dev'); - expect(await page.screenshot()).toMatchSnapshot(); + await expect(page).toHaveScreenshot(); }); ``` @@ -21,7 +21,7 @@ import { test, expect } from '@playwright/test'; test('example test', async ({ page }) => { await page.goto('https://playwright.dev'); - expect(await page.screenshot()).toMatchSnapshot(); + await expect(page).toHaveScreenshot(); }); ``` @@ -30,7 +30,10 @@ When you run above for the first time, test runner will say: Error: example.spec.ts-snapshots/example-test-1-chromium-darwin.png is missing in snapshots, writing actual. ``` -That's because there was no golden file yet. It is now created and is ready to be added to the repository. The name of the folder with the golden expectations starts with the name of your test file: +That's because there was no golden file yet. This method took a bunch of screenshots until two consecutive +screenshots matched, and saved the last screenshot to file system. It is now ready to be added to the repository. + +The name of the folder with the golden expectations starts with the name of your test file: ```bash drwxr-xr-x 5 user group 160 Jun 4 11:46 . @@ -42,10 +45,10 @@ drwxr-xr-x 3 user group 96 Jun 4 11:46 example.spec.ts-snapshots The snapshot name `example-test-1-chromium-darwin.png` consists of a few parts: - `example-test-1.png` - an auto-generated name of the snapshot. Alternatively you can specify snapshot name as the first argument of the `toMatchSnapshot()` method: ```js js-flavor=js - expect(await page.screenshot()).toMatchSnapshot('landing.png'); + await expect(page).toHaveScreenshot('landing.png'); ``` ```js js-flavor=ts - expect(await page.screenshot()).toMatchSnapshot('landing.png'); + await expect(page).toHaveScreenshot('landing.png'); ``` - `chromium-darwin` - the browser name and the platform. Screenshots differ between browsers and platforms due to different rendering, fonts and more, so you will need different snapshots for them. If you use multiple projects in your [configuration file](./test-configuration.md), project name will be used instead of `chromium`. @@ -53,7 +56,7 @@ The snapshot name `example-test-1-chromium-darwin.png` consists of a few parts: If you are not on the same operating system as your CI system, you can use Docker to generate/update the screenshots: ```bash -docker run --rm --network host -v $(pwd):/work/ -w /work/ -it mcr.microsoft.com/playwright:v1.12.3-focal /bin/bash +docker run --rm --network host -v $(pwd):/work/ -w /work/ -it mcr.microsoft.com/playwright:v1.20.0-focal /bin/bash npm install npx playwright test --update-snapshots ``` @@ -64,10 +67,10 @@ Sometimes you need to update the reference screenshot, for example when the page npx playwright test --update-snapshots ``` -> Note that `snapshotName` also accepts an array of path segments to the snapshot file such as `expect(value).toMatchSnapshot(['relative', 'path', 'to', 'snapshot.png'])`. +> Note that `snapshotName` also accepts an array of path segments to the snapshot file such as `await expect(page).toHaveScreenshot(['relative', 'path', 'to', 'snapshot.png'])`. > However, this path must stay within the snapshots directory for each test file (i.e. `a.spec.js-snapshots`), otherwise it will throw. -Playwright Test uses the [pixelmatch](https://github.com/mapbox/pixelmatch) library. You can [pass various options](./test-assertions#expectvaluetomatchsnapshotname-options) to modify its behavior: +Playwright Test uses the [pixelmatch](https://github.com/mapbox/pixelmatch) library. You can [pass various options](./test-assertions#expectpageorlocatortohavescreenshot-options) to modify its behavior: ```js js-flavor=js // example.spec.js @@ -75,7 +78,7 @@ const { test, expect } = require('@playwright/test'); test('example test', async ({ page }) => { await page.goto('https://playwright.dev'); - expect(await page.screenshot()).toMatchSnapshot('home.png', { maxDiffPixels: 100 }); + await expect(page).toHaveScreenshot({ maxDiffPixels: 100 }); }); ``` @@ -85,7 +88,7 @@ import { test, expect } from '@playwright/test'; test('example test', async ({ page }) => { await page.goto('https://playwright.dev'); - expect(await page.screenshot()).toMatchSnapshot('home.png', { maxDiffPixels: 100 }); + await expect(page).toHaveScreenshot({ maxDiffPixels: 100 }); }); ``` @@ -94,7 +97,7 @@ If you'd like to share the default value among all the tests in the project, you ```js js-flavor=js module.exports = { expect: { - toMatchSnapshot: { maxDiffPixels: 100 }, + toHaveScreenshot: { maxDiffPixels: 100 }, }, }; ``` @@ -103,13 +106,13 @@ module.exports = { import { PlaywrightTestConfig } from '@playwright/test'; const config: PlaywrightTestConfig = { expect: { - toMatchSnapshot: { maxDiffPixels: 100 }, + toHaveScreenshot: { maxDiffPixels: 100 }, }, }; export default config; ``` -Apart from screenshots, `expect(value).toMatchSnapshot(snapshotName)` can also be used to compare text, png and jpeg images, or arbitrary binary data. Playwright Test auto-detects the content type and uses the appropriate comparison algorithm. +Apart from screenshots, you can use `expect(value).toMatchSnapshot(snapshotName)` to compare text or arbitrary binary data. Playwright Test auto-detects the content type and uses the appropriate comparison algorithm. Here we compare text content against the reference. diff --git a/packages/playwright-test/src/matchers/toMatchSnapshot.ts b/packages/playwright-test/src/matchers/toMatchSnapshot.ts index ec6823323d..1e063f49e8 100644 --- a/packages/playwright-test/src/matchers/toMatchSnapshot.ts +++ b/packages/playwright-test/src/matchers/toMatchSnapshot.ts @@ -53,6 +53,7 @@ class SnapshotHelper { constructor( testInfo: TestInfoImpl, anonymousSnapshotExtension: string, + configOptions: ImageComparatorOptions, nameOrOptions: NameOrSegments | { name?: NameOrSegments } & T, optOptions: T, ) { @@ -76,7 +77,7 @@ class SnapshotHelper { } options = { - ...(testInfo.project.expect?.toMatchSnapshot || {}), + ...configOptions, ...options, }; @@ -214,7 +215,10 @@ export function toMatchSnapshot( const testInfo = currentTestInfo(); if (!testInfo) throw new Error(`toMatchSnapshot() must be called during the test`); - const helper = new SnapshotHelper(testInfo, determineFileExtension(received), nameOrOptions, optOptions); + const helper = new SnapshotHelper( + testInfo, determineFileExtension(received), + testInfo.project.expect?.toMatchSnapshot || {}, + nameOrOptions, optOptions); const comparator: Comparator = mimeTypeToComparator[helper.mimeType]; if (!comparator) throw new Error('Failed to find comparator with type ' + helper.mimeType + ': ' + helper.snapshotPath); @@ -255,7 +259,10 @@ export async function toHaveScreenshot( const testInfo = currentTestInfo(); if (!testInfo) throw new Error(`toHaveScreenshot() must be called during the test`); - const helper = new SnapshotHelper(testInfo, 'png', nameOrOptions, optOptions); + const helper = new SnapshotHelper( + testInfo, 'png', + testInfo.project.expect?.toHaveScreenshot || {}, + nameOrOptions, optOptions); const [page, locator] = pageOrLocator.constructor.name === 'Page' ? [(pageOrLocator as PageEx), undefined] : [(pageOrLocator as Locator).page() as PageEx, pageOrLocator as LocatorEx]; const screenshotOptions = { ...helper.allOptions, diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index 8bd850bc83..5322b9591d 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -43,8 +43,21 @@ type ExpectSettings = { * Default timeout for async expect matchers in milliseconds, defaults to 5000ms. */ timeout?: number; + toHaveScreenshot?: { + /** An acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between pixels in compared images, between zero (strict) and one (lax). Defaults to `0.2`. + */ + threshold?: number, + /** + * An acceptable amount of pixels that could be different, unset by default. + */ + maxDiffPixels?: number, + /** + * An acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default. + */ + maxDiffPixelRatio?: number, + } toMatchSnapshot?: { - /** An acceptable percieved color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between pixels in compared images, between zero (strict) and one (lax). Defaults to `0.2`. + /** An acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between pixels in compared images, between zero (strict) and one (lax). Defaults to `0.2`. */ threshold?: number, /** @@ -157,7 +170,8 @@ interface TestProject { */ name?: string; /** - * The base directory, relative to the config file, for snapshot files created with `toMatchSnapshot`. Defaults to + * The base directory, relative to the config file, for snapshot files created with `toMatchSnapshot` and + * `toHaveScreenshot`. Defaults to * [testProject.testDir](https://playwright.dev/docs/api/class-testproject#test-project-test-dir). * * The directory for each test can be accessed by @@ -695,8 +709,8 @@ interface TestConfig { * const config: PlaywrightTestConfig = { * expect: { * timeout: 10000, - * toMatchSnapshot: { - * threshold: 0.3, + * toHaveScreenshot: { + * maxDiffPixels: 10, * }, * }, * }; @@ -711,7 +725,8 @@ interface TestConfig { metadata?: any; name?: string; /** - * The base directory, relative to the config file, for snapshot files created with `toMatchSnapshot`. Defaults to + * The base directory, relative to the config file, for snapshot files created with `toMatchSnapshot` and + * `toHaveScreenshot`. Defaults to * [testConfig.testDir](https://playwright.dev/docs/api/class-testconfig#test-config-test-dir). * * The directory for each test can be accessed by @@ -1564,9 +1579,9 @@ export interface TestInfo { stderr: (string | Buffer)[]; /** * Suffix used to differentiate snapshots between multiple test configurations. For example, if snapshots depend on the - * platform, you can set `testInfo.snapshotSuffix` equal to `process.platform`. In this case - * `expect(value).toMatchSnapshot(snapshotName)` will use different snapshots depending on the platform. Learn more about - * [snapshots](https://playwright.dev/docs/test-snapshots). + * platform, you can set `testInfo.snapshotSuffix` equal to `process.platform`. In this case both + * `expect(value).toMatchSnapshot(snapshotName)` and `expect(page).toHaveScreenshot(snapshotName)` will use different + * snapshots depending on the platform. Learn more about [snapshots](https://playwright.dev/docs/test-snapshots). */ snapshotSuffix: string; /** diff --git a/tests/playwright-test/to-have-screenshot.spec.ts b/tests/playwright-test/to-have-screenshot.spec.ts index bc6918b12f..7bc3ad66a4 100644 --- a/tests/playwright-test/to-have-screenshot.spec.ts +++ b/tests/playwright-test/to-have-screenshot.spec.ts @@ -507,7 +507,7 @@ test('should respect maxDiffPixels option', async ({ runInlineTest }) => { ...files, 'playwright.config.ts': ` module.exports = { projects: [ - { expect: { toMatchSnapshot: { maxDiffPixels: ${BAD_PIXELS} } } }, + { expect: { toHaveScreenshot: { maxDiffPixels: ${BAD_PIXELS} } } }, ]}; `, 'a.spec.js-snapshots/snapshot.png': EXPECTED_SNAPSHOT, @@ -553,7 +553,7 @@ test('should respect maxDiffPixelRatio option', async ({ runInlineTest }) => { ...files, 'playwright.config.ts': ` module.exports = { projects: [ - { expect: { toMatchSnapshot: { maxDiffPixelRatio: ${BAD_RATIO} } } }, + { expect: { toHaveScreenshot: { maxDiffPixelRatio: ${BAD_RATIO} } } }, ]}; `, 'a.spec.js-snapshots/snapshot.png': EXPECTED_SNAPSHOT, diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 86fdabd5be..52484b5511 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -42,8 +42,21 @@ type ExpectSettings = { * Default timeout for async expect matchers in milliseconds, defaults to 5000ms. */ timeout?: number; + toHaveScreenshot?: { + /** An acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between pixels in compared images, between zero (strict) and one (lax). Defaults to `0.2`. + */ + threshold?: number, + /** + * An acceptable amount of pixels that could be different, unset by default. + */ + maxDiffPixels?: number, + /** + * An acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default. + */ + maxDiffPixelRatio?: number, + } toMatchSnapshot?: { - /** An acceptable percieved color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between pixels in compared images, between zero (strict) and one (lax). Defaults to `0.2`. + /** An acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between pixels in compared images, between zero (strict) and one (lax). Defaults to `0.2`. */ threshold?: number, /**