From 66d9f3acbe2e327d5c65f72bfb3aebcc6c46801f Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Fri, 22 Nov 2024 17:41:31 -0800 Subject: [PATCH] chore: introduce update-snapshots=changed (#33735) --- docs/src/test-api/class-fullconfig.md | 2 +- docs/src/test-api/class-testconfig.md | 7 +- docs/src/test-cli-js.md | 63 ++++++++------- .../playwright-core/src/utils/comparators.ts | 2 +- packages/playwright/src/common/ipc.ts | 2 +- .../src/matchers/toMatchAriaSnapshot.ts | 13 ++-- .../src/matchers/toMatchSnapshot.ts | 52 +++++++++---- packages/playwright/src/program.ts | 15 +++- packages/playwright/src/runner/rebase.ts | 16 ++-- packages/playwright/types/test.d.ts | 11 +-- .../to-have-screenshot.spec.ts | 77 +++++++++++++++++++ .../update-aria-snapshot.spec.ts | 66 ++++++++++++++++ 12 files changed, 255 insertions(+), 71 deletions(-) diff --git a/docs/src/test-api/class-fullconfig.md b/docs/src/test-api/class-fullconfig.md index e6437ab314..81e102436c 100644 --- a/docs/src/test-api/class-fullconfig.md +++ b/docs/src/test-api/class-fullconfig.md @@ -114,7 +114,7 @@ See [`property: TestConfig.shard`]. ## property: FullConfig.updateSnapshots * since: v1.10 -- type: <[UpdateSnapshots]<"all"|"none"|"missing">> +- type: <[UpdateSnapshots]<"all"|"changed"|"missing"|"none">> See [`property: TestConfig.updateSnapshots`]. diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index cd70b21b70..d9641128bc 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -570,12 +570,13 @@ export default defineConfig({ ## property: TestConfig.updateSnapshots * since: v1.10 -- type: ?<[UpdateSnapshots]<"all"|"none"|"missing">> +- type: ?<[UpdateSnapshots]<"all"|"changed"|"missing"|"none">> Whether to update expected snapshots with the actual results produced by the test run. Defaults to `'missing'`. -* `'all'` - All tests that are executed will update snapshots that did not match. Matching snapshots will not be updated. -* `'none'` - No snapshots are updated. +* `'all'` - All tests that are executed will update snapshots. +* `'changed'` - All tests that are executed will update snapshots that did not match. Matching snapshots will not be updated. * `'missing'` - Missing snapshots are created, for example when authoring a new test and running it for the first time. This is the default. +* `'none'` - No snapshots are updated. Learn more about [snapshots](../test-snapshots.md). diff --git a/docs/src/test-cli-js.md b/docs/src/test-cli-js.md index 005452138a..e91c091fb4 100644 --- a/docs/src/test-cli-js.md +++ b/docs/src/test-cli-js.md @@ -76,33 +76,40 @@ Here are the most common options available in the command line. Complete set of Playwright Test options is available in the [configuration file](./test-use-options.md). Following options can be passed to a command line and take priority over the configuration file: + + | Option | Description | | :- | :- | -| Non-option arguments | Each argument is treated as a regular expression matched against the full test file path. Only tests from the files matching the pattern will be executed. Special symbols like `$` or `*` should be escaped with `\`. In many shells/terminals you may need to quote the arguments. | -| `-c ` or `--config `| Configuration file. If not passed, defaults to `playwright.config.ts` or `playwright.config.js` in the current directory. | -| `--debug`| Run tests with Playwright Inspector. Shortcut for `PWDEBUG=1` environment variable and `--timeout=0 --max-failures=1 --headed --workers=1` options.| -| `--fail-on-flaky-tests` | Fails test runs that contain flaky tests. By default flaky tests count as successes. | -| `--forbid-only` | Whether to disallow `test.only`. Useful on CI.| -| `--global-timeout ` | Total timeout for the whole test run in milliseconds. By default, there is no global timeout. Learn more about [various timeouts](./test-timeouts.md).| -| `-g ` or `--grep ` | Only run tests matching this regular expression. For example, this will run `'should add to cart'` when passed `-g "add to cart"`. The regular expression will be tested against the string that consists of the project name, test file name, `test.describe` titles if any, test title and all test tags, separated by spaces, e.g. `chromium my-test.spec.ts my-suite my-test @smoke`. The filter does not apply to the tests from dependency projects, i.e. Playwright will still run all tests from [project dependencies](./test-projects.md#dependencies). | -| `--grep-invert ` | Only run tests **not** matching this regular expression. The opposite of `--grep`. The filter does not apply to the tests from dependency projects, i.e. Playwright will still run all tests from [project dependencies](./test-projects.md#dependencies).| -| `--headed` | Run tests in headed browsers. Useful for debugging. | -| `--ignore-snapshots` | Whether to ignore [snapshots](./test-snapshots.md). Use this when snapshot expectations are known to be different, e.g. running tests on Linux against Windows screenshots. | -| `--last-failed` | Only re-run the failures.| -| `--list` | list all the tests, but do not run them.| -| `--max-failures ` or `-x`| Stop after the first `N` test failures. Passing `-x` stops after the first failure.| -| `--no-deps` | Ignore the dependencies between projects and behave as if they were not specified. | -| `--output ` | Directory for artifacts produced by tests, defaults to `test-results`. | -| `--only-changed [ref]` | Only run test files that have been changed between working tree and "ref". Defaults to running all uncommitted changes with ref=HEAD. Only supports Git. | -| `--pass-with-no-tests` | Allows the test suite to pass when no files are found. | -| `--project ` | Only run tests from the specified [projects](./test-projects.md), supports '*' wildcard. Defaults to running all projects defined in the configuration file.| -| `--quiet` | Whether to suppress stdout and stderr from the tests. | -| `--repeat-each ` | Run each test `N` times, defaults to one. | -| `--reporter ` | Choose a reporter: minimalist `dot`, concise `line` or detailed `list`. See [reporters](./test-reporters.md) for more information. You can also pass a path to a [custom reporter](./test-reporters.md#custom-reporters) file. | -| `--retries ` | The maximum number of [retries](./test-retries.md#retries) for flaky tests, defaults to zero (no retries). | -| `--shard ` | [Shard](./test-parallel.md#shard-tests-between-multiple-machines) tests and execute only selected shard, specified in the form `current/all`, 1-based, for example `3/5`.| -| `--timeout ` | Maximum timeout in milliseconds for each test, defaults to 30 seconds. Learn more about [various timeouts](./test-timeouts.md).| -| `--trace ` | Force tracing mode, can be `on`, `off`, `on-first-retry`, `on-all-retries`, `retain-on-failure` | -| `--tsconfig ` | Path to a single tsconfig applicable to all imported files. See [tsconfig resolution](./test-typescript.md#tsconfig-resolution) for more details. | -| `--update-snapshots` or `-u` | Whether to update [snapshots](./test-snapshots.md) with actual results instead of comparing them. Use this when snapshot expectations have changed.| -| `--workers ` or `-j `| The maximum number of concurrent worker processes that run in [parallel](./test-parallel.md). | +| Non-option arguments | Each argument is treated as a regular expression matched against the full test file path. Only tests from files matching the pattern will be executed. Special symbols like `$` or `*` should be escaped with `\`. In many shells/terminals you may need to quote the arguments. | +| `-c ` or `--config ` | Configuration file, or a test directory with optional "playwright.config.{m,c}?{js,ts}". Defaults to `playwright.config.ts` or `playwright.config.js` in the current directory. | +| `--debug` | Run tests with Playwright Inspector. Shortcut for `PWDEBUG=1` environment variable and `--timeout=0 --max-failures=1 --headed --workers=1` options. | +| `--fail-on-flaky-tests` | Fail if any test is flagged as flaky (default: false). | +| `--forbid-only` | Fail if `test.only` is called (default: false). Useful on CI. | +| `--fully-parallel` | Run all tests in parallel (default: false). | +| `--global-timeout ` | Maximum time this test suite can run in milliseconds (default: unlimited). | +| `-g ` or `--grep ` | Only run tests matching this regular expression (default: ".*"). | +| `-gv ` or `--grep-invert ` | Only run tests that do not match this regular expression. | +| `--headed` | Run tests in headed browsers (default: headless). | +| `--ignore-snapshots` | Ignore screenshot and snapshot expectations. | +| `--last-failed` | Only re-run the failures. | +| `--list` | Collect all the tests and report them, but do not run. | +| `--max-failures ` or `-x` | Stop after the first `N` failures. Passing `-x` stops after the first failure. | +| `--no-deps` | Do not run project dependencies. | +| `--output ` | Folder for output artifacts (default: "test-results"). | +| `--only-changed [ref]` | Only run test files that have been changed between 'HEAD' and 'ref'. Defaults to running all uncommitted changes. Only supports Git. | +| `--pass-with-no-tests` | Makes test run succeed even if no tests were found. | +| `--project ` | Only run tests from the specified list of projects, supports '*' wildcard (default: run all projects). | +| `--quiet` | Suppress stdio. | +| `--repeat-each ` | Run each test `N` times (default: 1). | +| `--reporter ` | Reporter to use, comma-separated, can be "dot", "line", "list", or others (default: "list"). You can also pass a path to a custom reporter file. | +| `--retries ` | Maximum retry count for flaky tests, zero for no retries (default: no retries). | +| `--shard ` | Shard tests and execute only the selected shard, specified in the form "current/all", 1-based, e.g., "3/5". | +| `--timeout ` | Specify test timeout threshold in milliseconds, zero for unlimited (default: 30 seconds). | +| `--trace ` | Force tracing mode, can be "on", "off", "on-first-retry", "on-all-retries", "retain-on-failure", "retain-on-first-failure". | +| `--tsconfig ` | Path to a single tsconfig applicable to all imported files (default: look up tsconfig for each imported file separately). | +| `--ui` | Run tests in interactive UI mode. | +| `--ui-host ` | Host to serve UI on; specifying this option opens UI in a browser tab. | +| `--ui-port ` | Port to serve UI on, 0 for any free port; specifying this option opens UI in a browser tab. | +| `-u` or `--update-snapshots [mode]` | Update snapshots with actual results. Possible values are "all", "changed", "missing", and "none". Not passing defaults to "missing"; passing without a value defaults to "changed". | +| `-j ` or `--workers ` | Number of concurrent workers or percentage of logical CPU cores, use 1 to run in a single worker (default: 50%). | +| `-x` | Stop after the first failure. | diff --git a/packages/playwright-core/src/utils/comparators.ts b/packages/playwright-core/src/utils/comparators.ts index acf897af49..7f64df240c 100644 --- a/packages/playwright-core/src/utils/comparators.ts +++ b/packages/playwright-core/src/utils/comparators.ts @@ -38,7 +38,7 @@ export function getComparator(mimeType: string): Comparator { const JPEG_JS_MAX_BUFFER_SIZE_IN_MB = 5 * 1024; // ~5 GB -function compareBuffersOrStrings(actualBuffer: Buffer | string, expectedBuffer: Buffer): ComparatorResult { +export function compareBuffersOrStrings(actualBuffer: Buffer | string, expectedBuffer: Buffer): ComparatorResult { if (typeof actualBuffer === 'string') return compareText(actualBuffer, expectedBuffer); if (!actualBuffer || !(actualBuffer instanceof Buffer)) diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.ts index dcde2b28d4..2f08a87650 100644 --- a/packages/playwright/src/common/ipc.ts +++ b/packages/playwright/src/common/ipc.ts @@ -38,7 +38,7 @@ export type ConfigCLIOverrides = { timeout?: number; tsconfig?: string; ignoreSnapshots?: boolean; - updateSnapshots?: 'all'|'none'|'missing'; + updateSnapshots?: 'all'|'changed'|'missing'|'none'; workers?: number | string; projects?: { name: string, use?: any }[], use?: any; diff --git a/packages/playwright/src/matchers/toMatchAriaSnapshot.ts b/packages/playwright/src/matchers/toMatchAriaSnapshot.ts index 0bb600f7b0..552ad2e36d 100644 --- a/packages/playwright/src/matchers/toMatchAriaSnapshot.ts +++ b/packages/playwright/src/matchers/toMatchAriaSnapshot.ts @@ -57,8 +57,6 @@ export async function toMatchAriaSnapshot( } const generateMissingBaseline = updateSnapshots === 'missing' && !expected; - const generateNewBaseline = updateSnapshots === 'all' || generateMissingBaseline; - if (generateMissingBaseline) { if (this.isNot) { const message = `Matchers using ".not" can't generate new baselines`; @@ -100,10 +98,13 @@ export async function toMatchAriaSnapshot( } }; - if (!this.isNot && pass === this.isNot && generateNewBaseline) { - // Only rebaseline failed snapshots. - const suggestedRebaseline = `toMatchAriaSnapshot(\`\n${escapeTemplateString(indent(typedReceived.regex, '{indent} '))}\n{indent}\`)`; - return { pass: this.isNot, message: () => '', name: 'toMatchAriaSnapshot', suggestedRebaseline }; + if (!this.isNot) { + if ((updateSnapshots === 'all') || + (updateSnapshots === 'changed' && pass === this.isNot) || + generateMissingBaseline) { + const suggestedRebaseline = `toMatchAriaSnapshot(\`\n${escapeTemplateString(indent(typedReceived.regex, '{indent} '))}\n{indent}\`)`; + return { pass: this.isNot, message: () => '', name: 'toMatchAriaSnapshot', suggestedRebaseline }; + } } return { diff --git a/packages/playwright/src/matchers/toMatchSnapshot.ts b/packages/playwright/src/matchers/toMatchSnapshot.ts index 374fba3db5..8d16f11fc2 100644 --- a/packages/playwright/src/matchers/toMatchSnapshot.ts +++ b/packages/playwright/src/matchers/toMatchSnapshot.ts @@ -18,7 +18,7 @@ import type { Locator, Page } from 'playwright-core'; import type { ExpectScreenshotOptions, Page as PageEx } from 'playwright-core/lib/client/page'; import { currentTestInfo } from '../common/globals'; import type { ImageComparatorOptions, Comparator } from 'playwright-core/lib/utils'; -import { getComparator, isString, sanitizeForFilePath } from 'playwright-core/lib/utils'; +import { compareBuffersOrStrings, getComparator, isString, sanitizeForFilePath } from 'playwright-core/lib/utils'; import { addSuffixToFilePath, trimLongString, callLogText, @@ -83,7 +83,7 @@ class SnapshotHelper { readonly diffPath: string; readonly mimeType: string; readonly kind: 'Screenshot'|'Snapshot'; - readonly updateSnapshots: 'all' | 'none' | 'missing'; + readonly updateSnapshots: 'all' | 'changed' | 'missing' | 'none'; readonly comparator: Comparator; readonly options: Omit & { comparator?: string }; readonly matcherName: string; @@ -199,7 +199,7 @@ class SnapshotHelper { } handleMissingNegated(): ImageMatcherResult { - const isWriteMissingMode = this.updateSnapshots === 'all' || this.updateSnapshots === 'missing'; + const isWriteMissingMode = this.updateSnapshots !== 'none'; const message = `A snapshot doesn't exist at ${this.expectedPath}${isWriteMissingMode ? ', matchers using ".not" won\'t write them automatically.' : '.'}`; // NOTE: 'isNot' matcher implies inversed value. return this.createMatcherResult(message, true); @@ -221,14 +221,14 @@ class SnapshotHelper { } handleMissing(actual: Buffer | string): ImageMatcherResult { - const isWriteMissingMode = this.updateSnapshots === 'all' || this.updateSnapshots === 'missing'; + const isWriteMissingMode = this.updateSnapshots !== 'none'; if (isWriteMissingMode) writeFileSync(this.expectedPath, actual); this.testInfo.attachments.push({ name: addSuffixToFilePath(this.attachmentBaseName, '-expected'), contentType: this.mimeType, path: this.expectedPath }); writeFileSync(this.actualPath, actual); this.testInfo.attachments.push({ name: addSuffixToFilePath(this.attachmentBaseName, '-actual'), contentType: this.mimeType, path: this.actualPath }); const message = `A snapshot doesn't exist at ${this.expectedPath}${isWriteMissingMode ? ', writing actual.' : '.'}`; - if (this.updateSnapshots === 'all') { + if (this.updateSnapshots === 'all' || this.updateSnapshots === 'changed') { /* eslint-disable no-console */ console.log(message); return this.createMatcherResult(message, true); @@ -317,17 +317,30 @@ export function toMatchSnapshot( return helper.handleMissing(received); const expected = fs.readFileSync(helper.expectedPath); - const result = helper.comparator(received, expected, helper.options); - if (!result) - return helper.handleMatching(); if (helper.updateSnapshots === 'all') { + if (!compareBuffersOrStrings(received, expected)) + return helper.handleMatching(); + writeFileSync(helper.expectedPath, received); + /* eslint-disable no-console */ + console.log(helper.expectedPath + ' is not the same, writing actual.'); + return helper.createMatcherResult(helper.expectedPath + ' running with --update-snapshots, writing actual.', true); + } + + if (helper.updateSnapshots === 'changed') { + const result = helper.comparator(received, expected, helper.options); + if (!result) + return helper.handleMatching(); writeFileSync(helper.expectedPath, received); /* eslint-disable no-console */ console.log(helper.expectedPath + ' does not match, writing actual.'); return helper.createMatcherResult(helper.expectedPath + ' running with --update-snapshots, writing actual.', true); } + const result = helper.comparator(received, expected, helper.options); + if (!result) + return helper.handleMatching(); + const receiver = isString(received) ? 'string' : 'Buffer'; const header = matcherHint(this, undefined, 'toMatchSnapshot', receiver, undefined, undefined); return helper.handleDifferent(received, expected, undefined, result.diff, header, result.errorMessage, undefined); @@ -421,21 +434,30 @@ export async function toHaveScreenshot( // General case: // - snapshot exists // - regular matcher (i.e. not a `.not`) - // - perhaps an 'all' flag to update non-matching screenshots - expectScreenshotOptions.expected = await fs.promises.readFile(helper.expectedPath); + const expected = await fs.promises.readFile(helper.expectedPath); + expectScreenshotOptions.expected = helper.updateSnapshots === 'all' ? undefined : expected; + const { actual, previous, diff, errorMessage, log, timedOut } = await page._expectScreenshot(expectScreenshotOptions); - - if (!errorMessage) - return helper.handleMatching(); - - if (helper.updateSnapshots === 'all') { + const writeFiles = () => { writeFileSync(helper.expectedPath, actual!); writeFileSync(helper.actualPath, actual!); /* eslint-disable no-console */ console.log(helper.expectedPath + ' is re-generated, writing actual.'); return helper.createMatcherResult(helper.expectedPath + ' running with --update-snapshots, writing actual.', true); + }; + + if (!errorMessage) { + // Screenshot is matching, but is not necessarily the same as the expected. + if (helper.updateSnapshots === 'all' && actual && compareBuffersOrStrings(actual, expected)) { + console.log(helper.expectedPath + ' is re-generated, writing actual.'); + return writeFiles(); + } + return helper.handleMatching(); } + if (helper.updateSnapshots === 'changed' || helper.updateSnapshots === 'all') + return writeFiles(); + const header = matcherHint(this, undefined, 'toHaveScreenshot', receiver, undefined, undefined, timedOut ? timeout : undefined); return helper.handleDifferent(actual, expectScreenshotOptions.expected, previous, diff, header, errorMessage, log); } diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts index b4a12c0ea7..a827483bbd 100644 --- a/packages/playwright/src/program.ts +++ b/packages/playwright/src/program.ts @@ -281,6 +281,13 @@ async function mergeReports(reportDir: string | undefined, opts: { [key: string] function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrides { const shardPair = options.shard ? options.shard.split('/').map((t: string) => parseInt(t, 10)) : undefined; + + let updateSnapshots: 'all' | 'changed' | 'missing' | 'none'; + if (['all', 'changed', 'missing', 'none'].includes(options.updateSnapshots)) + updateSnapshots = options.updateSnapshots; + else + updateSnapshots = 'updateSnapshots' in options ? 'changed' : 'missing'; + const overrides: ConfigCLIOverrides = { forbidOnly: options.forbidOnly ? true : undefined, fullyParallel: options.fullyParallel ? true : undefined, @@ -295,7 +302,7 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrid timeout: options.timeout ? parseInt(options.timeout, 10) : undefined, tsconfig: options.tsconfig ? path.resolve(process.cwd(), options.tsconfig) : undefined, ignoreSnapshots: options.ignoreSnapshots ? !!options.ignoreSnapshots : undefined, - updateSnapshots: options.updateSnapshots ? 'all' as const : undefined, + updateSnapshots, workers: options.workers, }; @@ -344,8 +351,10 @@ function resolveReporter(id: string) { const kTraceModes: TraceMode[] = ['on', 'off', 'on-first-retry', 'on-all-retries', 'retain-on-failure', 'retain-on-first-failure']; +// Note: update docs/src/test-cli-js.md when you update this, program is the source of truth. + const testOptions: [string, string][] = [ - ['--browser ', `Browser to use for tests, one of "all", "chromium", "firefox" or "webkit" (default: "chromium")`], + /* deprecated */ ['--browser ', `Browser to use for tests, one of "all", "chromium", "firefox" or "webkit" (default: "chromium")`], ['-c, --config ', `Configuration file, or a test directory with optional "playwright.config.{m,c}?{js,ts}"`], ['--debug', `Run tests with Playwright Inspector. Shortcut for "PWDEBUG=1" environment variable and "--timeout=0 --max-failures=1 --headed --workers=1" options`], ['--fail-on-flaky-tests', `Fail if any test is flagged as flaky (default: false)`], @@ -375,7 +384,7 @@ const testOptions: [string, string][] = [ ['--ui', `Run tests in interactive UI mode`], ['--ui-host ', 'Host to serve UI on; specifying this option opens UI in a browser tab'], ['--ui-port ', 'Port to serve UI on, 0 for any free port; specifying this option opens UI in a browser tab'], - ['-u, --update-snapshots', `Update snapshots with actual results (default: only create missing snapshots)`], + ['-u, --update-snapshots [mode]', `Update snapshots with actual results. Possible values are 'all', 'changed', 'missing' and 'none'. Not passing defaults to 'missing', passing without value defaults to 'changed'`], ['-j, --workers ', `Number of concurrent workers or percentage of logical CPU cores, use 1 to run in a single worker (default: 50%)`], ['-x', `Stop after the first failure`], ]; diff --git a/packages/playwright/src/runner/rebase.ts b/packages/playwright/src/runner/rebase.ts index b412a4a775..e088b82742 100644 --- a/packages/playwright/src/runner/rebase.ts +++ b/packages/playwright/src/runner/rebase.ts @@ -44,7 +44,7 @@ export function addSuggestedRebaseline(location: Location, suggestedRebaseline: } export async function applySuggestedRebaselines(config: FullConfigInternal, reporter: InternalReporter) { - if (config.config.updateSnapshots !== 'all' && config.config.updateSnapshots !== 'missing') + if (config.config.updateSnapshots === 'none') return; if (!suggestedRebaselines.size) return; @@ -106,15 +106,15 @@ export async function applySuggestedRebaselines(config: FullConfigInternal, repo const relativeName = path.relative(gitFolder || process.cwd(), fileName); files.push(relativeName); patches.push(createPatch(relativeName, source, result)); + + const patchFile = path.join(project.project.outputDir, 'rebaselines.patch'); + await fs.promises.mkdir(path.dirname(patchFile), { recursive: true }); + await fs.promises.writeFile(patchFile, patches.join('\n')); + + const fileList = files.map(file => ' ' + colors.dim(file)).join('\n'); + reporter.onStdErr(`\nNew baselines created for:\n\n${fileList}\n\n ` + colors.cyan('git apply ' + path.relative(process.cwd(), patchFile)) + '\n'); } } - - const patchFile = path.join(project.project.outputDir, 'rebaselines.patch'); - await fs.promises.mkdir(path.dirname(patchFile), { recursive: true }); - await fs.promises.writeFile(patchFile, patches.join('\n')); - - const fileList = files.map(file => ' ' + colors.dim(file)).join('\n'); - reporter.onStdErr(`\nNew baselines created for:\n\n${fileList}\n\n ` + colors.cyan('git apply ' + path.relative(process.cwd(), patchFile)) + '\n'); } function createPatch(fileName: string, before: string, after: string) { diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index 84111533c9..c80329ca8a 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -1665,11 +1665,12 @@ interface TestConfig { /** * Whether to update expected snapshots with the actual results produced by the test run. Defaults to `'missing'`. - * - `'all'` - All tests that are executed will update snapshots that did not match. Matching snapshots will not be - * updated. - * - `'none'` - No snapshots are updated. + * - `'all'` - All tests that are executed will update snapshots. + * - `'changed'` - All tests that are executed will update snapshots that did not match. Matching snapshots will not + * be updated. * - `'missing'` - Missing snapshots are created, for example when authoring a new test and running it for the first * time. This is the default. + * - `'none'` - No snapshots are updated. * * Learn more about [snapshots](https://playwright.dev/docs/test-snapshots). * @@ -1685,7 +1686,7 @@ interface TestConfig { * ``` * */ - updateSnapshots?: "all"|"none"|"missing"; + updateSnapshots?: "all"|"changed"|"missing"|"none"; /** * The maximum number of concurrent worker processes to use for parallelizing tests. Can also be set as percentage of @@ -1834,7 +1835,7 @@ export interface FullConfig { /** * See [testConfig.updateSnapshots](https://playwright.dev/docs/api/class-testconfig#test-config-update-snapshots). */ - updateSnapshots: "all"|"none"|"missing"; + updateSnapshots: "all"|"changed"|"missing"|"none"; /** * Playwright version. diff --git a/tests/playwright-test/to-have-screenshot.spec.ts b/tests/playwright-test/to-have-screenshot.spec.ts index ee053cc130..83642bd19e 100644 --- a/tests/playwright-test/to-have-screenshot.spec.ts +++ b/tests/playwright-test/to-have-screenshot.spec.ts @@ -1393,3 +1393,80 @@ test('should trim+sanitize attachment names and paths', async ({ runInlineTest } ]); }); +test.describe('update-snapshots', () => { + test('should rebase non-matching image', async ({ runInlineTest }) => { + const BAD_PIXELS = 10; + const EXPECTED_SNAPSHOT = paintBlackPixels(whiteImage, BAD_PIXELS); + + const result = await runInlineTest({ + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + }), + '__screenshots__/a.spec.js/snapshot.png': EXPECTED_SNAPSHOT, + 'a.spec.js': ` + const { test, expect } = require('@playwright/test'); + test('is a test', async ({ page }) => { + await expect(page).toHaveScreenshot('snapshot.png', { timeout: 2000 }); + }); + ` + }, { 'update-snapshots': 'changed' }); + expect(result.exitCode).toBe(0); + const newBaseline = fs.readFileSync(test.info().outputPath('__screenshots__/a.spec.js/snapshot.png')); + expect(comparePNGs(newBaseline, whiteImage)).toBe(null); + expect(comparePNGs(newBaseline, EXPECTED_SNAPSHOT)).not.toBe(null); + }); + + test('should not rebase matching image', async ({ runInlineTest }) => { + const BAD_PIXELS = 10; + const EXPECTED_SNAPSHOT = paintBlackPixels(whiteImage, BAD_PIXELS); + + const result = await runInlineTest({ + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + expect: { + toHaveScreenshot: { + maxDiffPixels: BAD_PIXELS + } + } + }), + '__screenshots__/a.spec.js/snapshot.png': EXPECTED_SNAPSHOT, + 'a.spec.js': ` + const { test, expect } = require('@playwright/test'); + test('is a test', async ({ page }) => { + await expect(page).toHaveScreenshot('snapshot.png', { timeout: 2000 }); + }); + ` + }, { 'update-snapshots': 'changed' }); + expect(result.exitCode).toBe(0); + const newBaseline = fs.readFileSync(test.info().outputPath('__screenshots__/a.spec.js/snapshot.png')); + expect(comparePNGs(newBaseline, EXPECTED_SNAPSHOT)).toBe(null); + expect(comparePNGs(newBaseline, whiteImage)).not.toBe(null); + }); + + test('should rebase matching image with update-snapshots=all', async ({ runInlineTest }) => { + const BAD_PIXELS = 10; + const EXPECTED_SNAPSHOT = paintBlackPixels(whiteImage, BAD_PIXELS); + + const result = await runInlineTest({ + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + expect: { + toHaveScreenshot: { + maxDiffPixels: BAD_PIXELS + } + } + }), + '__screenshots__/a.spec.js/snapshot.png': EXPECTED_SNAPSHOT, + 'a.spec.js': ` + const { test, expect } = require('@playwright/test'); + test('is a test', async ({ page }) => { + await expect(page).toHaveScreenshot('snapshot.png', { timeout: 2000 }); + }); + ` + }, { 'update-snapshots': 'all' }); + expect(result.exitCode).toBe(0); + const newBaseline = fs.readFileSync(test.info().outputPath('__screenshots__/a.spec.js/snapshot.png')); + expect(comparePNGs(newBaseline, whiteImage)).toBe(null); + expect(comparePNGs(newBaseline, EXPECTED_SNAPSHOT)).not.toBe(null); + }); +}); diff --git a/tests/playwright-test/update-aria-snapshot.spec.ts b/tests/playwright-test/update-aria-snapshot.spec.ts index f7bc744561..3b07e8b93e 100644 --- a/tests/playwright-test/update-aria-snapshot.spec.ts +++ b/tests/playwright-test/update-aria-snapshot.spec.ts @@ -424,3 +424,69 @@ test('should not update snapshots when locator did not match', async ({ runInlin expect(result.output).toContain('Expected: "- heading"'); expect(result.output).toContain('Received: '); }); + +test.describe('update-snapshots none', () => { + test('should create new baseline for matching snapshot', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + '.git/marker': '', + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('test', async ({ page }) => { + await page.setContent(\`

hello

world

\`); + await expect(page.locator('body')).toMatchAriaSnapshot(\`\`); + }); + ` + }, { 'update-snapshots': 'none' }); + + expect(result.exitCode).toBe(1); + const patchPath = testInfo.outputPath('test-results/rebaselines.patch'); + expect(fs.existsSync(patchPath)).toBeFalsy(); + }); +}); + +test.describe('update-snapshots all', () => { + test('should create new baseline for matching snapshot', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + '.git/marker': '', + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('test', async ({ page }) => { + await page.setContent(\`

hello

world

\`); + await expect(page.locator('body')).toMatchAriaSnapshot(\` + - heading "hello" + \`); + }); + ` + }, { 'update-snapshots': 'all' }); + + expect(result.exitCode).toBe(0); + const patchPath = testInfo.outputPath('test-results/rebaselines.patch'); + const data = fs.readFileSync(patchPath, 'utf-8'); + expect(trimPatch(data)).toBe(`diff --git a/a.spec.ts b/a.spec.ts +--- a/a.spec.ts ++++ b/a.spec.ts +@@ -3,7 +3,8 @@ + test('test', async ({ page }) => { + await page.setContent(\`

hello

world

\`); + await expect(page.locator('body')).toMatchAriaSnapshot(\` +- - heading "hello" ++ - heading "hello" [level=1] ++ - heading "world" [level=1] + \`); + }); + +\\ No newline at end of file +`); + + expect(stripAnsi(result.output).replace(/\\/g, '/')).toContain(`New baselines created for: + + a.spec.ts + + git apply test-results/rebaselines.patch +`); + + execSync(`patch -p1 < ${patchPath}`, { cwd: testInfo.outputPath() }); + const result2 = await runInlineTest({}); + expect(result2.exitCode).toBe(0); + }); +});