diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index ddd673bcae..e1b68487b0 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -226,6 +226,11 @@ Filter to only run tests with a title **not** matching one of the patterns. This `grepInvert` option is also useful for [tagging tests](../test-annotations.md#tag-tests). +## property: TestConfig.ignoreSnapshots +* since: v1.26 +- type: ?<[boolean]> + +Whether to skip snapshot expectations, such as `expect(value).toMatchSnapshot()` and `await expect(page).toHaveScreenshot()`. ## property: TestConfig.maxFailures * since: v1.10 diff --git a/docs/src/test-cli-js.md b/docs/src/test-cli-js.md index 21b30dca34..a6571a2a6e 100644 --- a/docs/src/test-cli-js.md +++ b/docs/src/test-cli-js.md @@ -111,6 +111,8 @@ Complete set of Playwright Test options is available in the [configuration file] - `--timeout `: Maximum timeout in milliseconds for each test, defaults to 30 seconds. Learn more about [various timeouts](./test-timeouts.md). +- `--ignore-snapshots` or `-i`: Whether to ignore [snapshots](./test-snapshots.md). Use this when snapshot expectations are known to be different, e.g. running tests on Linux against Windows screenshots. + - `--update-snapshots` or `-u`: Whether to update [snapshots](./test-snapshots.md) with actual results instead of comparing them. Use this when snapshot expectations have changed. - `--workers ` or `-j `: The maximum number of concurrent worker processes that run in [parallel](./test-parallel.md). diff --git a/packages/playwright-test/src/cli.ts b/packages/playwright-test/src/cli.ts index 1ae5cf5542..a2cbcdf959 100644 --- a/packages/playwright-test/src/cli.ts +++ b/packages/playwright-test/src/cli.ts @@ -59,6 +59,7 @@ function addTestCommand(program: Command) { command.option('--project ', `Only run tests from the specified list of projects (default: run all projects)`); command.option('--timeout ', `Specify test timeout threshold in milliseconds, zero for unlimited (default: ${defaultTimeout})`); command.option('--trace ', `Force tracing mode, can be ${kTraceModes.map(mode => `"${mode}"`).join(', ')}`); + command.option('-i, --ignore-snapshots', `Ignore screenshot and snapshot expectations`); command.option('-u, --update-snapshots', `Update snapshots with actual results (default: only create missing snapshots)`); command.option('-x', `Stop after the first failure`); command.action(async (args, opts) => { @@ -216,6 +217,7 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrid reporter: (options.reporter && options.reporter.length) ? options.reporter.split(',').map((r: string) => [resolveReporter(r)]) : undefined, shard: shardPair ? { current: shardPair[0], total: shardPair[1] } : undefined, timeout: options.timeout ? parseInt(options.timeout, 10) : undefined, + ignoreSnapshots: options.ignoreSnapshots ? !!options.ignoreSnapshots : undefined, updateSnapshots: options.updateSnapshots ? 'all' as const : undefined, workers: options.workers ? parseInt(options.workers, 10) : undefined, }; diff --git a/packages/playwright-test/src/loader.ts b/packages/playwright-test/src/loader.ts index 22cdeda058..7beb4dcedf 100644 --- a/packages/playwright-test/src/loader.ts +++ b/packages/playwright-test/src/loader.ts @@ -94,6 +94,7 @@ export class Loader { config.shard = takeFirst(this._configCLIOverrides.shard, config.shard); config.timeout = takeFirst(this._configCLIOverrides.timeout, config.timeout); config.updateSnapshots = takeFirst(this._configCLIOverrides.updateSnapshots, config.updateSnapshots); + config.ignoreSnapshots = takeFirst(this._configCLIOverrides.ignoreSnapshots, config.ignoreSnapshots); if (this._configCLIOverrides.projects && config.projects) throw new Error(`Cannot use --browser option when configuration file defines projects. Specify browserName in the projects instead.`); config.projects = takeFirst(this._configCLIOverrides.projects, config.projects as any); @@ -139,6 +140,7 @@ export class Loader { this._fullConfig.reportSlowTests = takeFirst(config.reportSlowTests, baseFullConfig.reportSlowTests); this._fullConfig.quiet = takeFirst(config.quiet, baseFullConfig.quiet); this._fullConfig.shard = takeFirst(config.shard, baseFullConfig.shard); + this._fullConfig._ignoreSnapshots = takeFirst(config.ignoreSnapshots, baseFullConfig._ignoreSnapshots); this._fullConfig.updateSnapshots = takeFirst(config.updateSnapshots, baseFullConfig.updateSnapshots); this._fullConfig.workers = takeFirst(config.workers, baseFullConfig.workers); const webServers = takeFirst(config.webServer, baseFullConfig.webServer); @@ -553,6 +555,11 @@ function validateConfig(file: string, config: Config) { throw errorWithFile(file, `config.shard.current must be a positive number, not greater than config.shard.total`); } + if ('ignoreSnapshots' in config && config.ignoreSnapshots !== undefined) { + if (typeof config.ignoreSnapshots !== 'boolean') + throw errorWithFile(file, `config.ignoreSnapshots must be a boolean`); + } + if ('updateSnapshots' in config && config.updateSnapshots !== undefined) { if (typeof config.updateSnapshots !== 'string' || !['all', 'none', 'missing'].includes(config.updateSnapshots)) throw errorWithFile(file, `config.updateSnapshots must be one of "all", "none" or "missing"`); @@ -647,6 +654,7 @@ export const baseFullConfig: FullConfigInternal = { _globalOutputDir: path.resolve(process.cwd()), _configDir: '', _testGroupsCount: 0, + _ignoreSnapshots: false, _workerIsolation: 'isolate-pools', }; diff --git a/packages/playwright-test/src/matchers/toMatchSnapshot.ts b/packages/playwright-test/src/matchers/toMatchSnapshot.ts index 26ced3e18b..733bb8a5ce 100644 --- a/packages/playwright-test/src/matchers/toMatchSnapshot.ts +++ b/packages/playwright-test/src/matchers/toMatchSnapshot.ts @@ -251,6 +251,10 @@ export function toMatchSnapshot( throw new Error(`toMatchSnapshot() must be called during the test`); if (received instanceof Promise) throw new Error('An unresolved Promise was passed to toMatchSnapshot(), make sure to resolve it by adding await to it.'); + + if (testInfo.config._ignoreSnapshots) + return { pass: !this.isNot, message: () => '' }; + const helper = new SnapshotHelper( testInfo, testInfo.snapshotPath.bind(testInfo), determineFileExtension(received), testInfo.project._expect?.toMatchSnapshot || {}, @@ -292,6 +296,10 @@ export async function toHaveScreenshot( const testInfo = currentTestInfo(); if (!testInfo) throw new Error(`toHaveScreenshot() must be called during the test`); + + if (testInfo.config._ignoreSnapshots) + return { pass: !this.isNot, message: () => '' }; + const config = (testInfo.project._expect as any)?.toHaveScreenshot; const snapshotPathResolver = process.env.PWTEST_USE_SCREENSHOTS_DIR_FOR_TEST ? testInfo._screenshotPath.bind(testInfo) diff --git a/packages/playwright-test/src/runner.ts b/packages/playwright-test/src/runner.ts index 9287a068fa..c783afae9d 100644 --- a/packages/playwright-test/src/runner.ts +++ b/packages/playwright-test/src/runner.ts @@ -73,6 +73,7 @@ export type ConfigCLIOverrides = { reporter?: string; shard?: { current: number, total: number }; timeout?: number; + ignoreSnapshots?: boolean; updateSnapshots?: 'all'|'none'|'missing'; workers?: number; projects?: { name: string, use?: any }[], diff --git a/packages/playwright-test/src/types.ts b/packages/playwright-test/src/types.ts index cc70699d38..45de213264 100644 --- a/packages/playwright-test/src/types.ts +++ b/packages/playwright-test/src/types.ts @@ -46,6 +46,7 @@ export interface FullConfigInternal extends FullConfigPublic { _configDir: string; _testGroupsCount: number; _watchMode: boolean; + _ignoreSnapshots: boolean; _workerIsolation: WorkerIsolation; /** * If populated, this should also be the first/only entry in _webServers. Legacy singleton `webServer` as well as those provided via an array in the user-facing playwright.config.{ts,js} will be in `_webServers`. The legacy field (`webServer`) field additionally stores the backwards-compatible singleton `webServer` since it had been showing up in globalSetup to the user. diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index 6e7b3b6ae6..70fe146aee 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -678,6 +678,12 @@ interface TestConfig { */ grepInvert?: RegExp|Array; + /** + * Whether to skip snapshot expectations, such as `expect(value).toMatchSnapshot()` and `await + * expect(page).toHaveScreenshot()`. + */ + ignoreSnapshots?: boolean; + /** * The maximum number of test failures for the whole test suite run. After reaching this number, testing will stop and exit * with an error. Setting to zero (default) disables this behavior. diff --git a/tests/playwright-test/config.spec.ts b/tests/playwright-test/config.spec.ts index fe7ce22dec..a699168ee5 100644 --- a/tests/playwright-test/config.spec.ts +++ b/tests/playwright-test/config.spec.ts @@ -373,6 +373,26 @@ test('should inerhit use options in projects', async ({ runInlineTest }) => { expect(result.passed).toBe(1); }); +test('should support ignoreSnapshots config option', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + ignoreSnapshots: true, + }; + `, + 'a.test.ts': ` + const { test } = pwt; + test('pass', async ({}, testInfo) => { + expect('foo').toMatchSnapshot(); + expect('foo').not.toMatchSnapshot(); + }); + ` + }); + + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); +}); + test('should work with undefined values and base', async ({ runInlineTest }) => { const result = await runInlineTest({ 'playwright.config.ts': ` diff --git a/tests/playwright-test/golden.spec.ts b/tests/playwright-test/golden.spec.ts index e816ef9402..afd61657ac 100644 --- a/tests/playwright-test/golden.spec.ts +++ b/tests/playwright-test/golden.spec.ts @@ -268,6 +268,27 @@ test('should update snapshot with the update-snapshots flag', async ({ runInline expect(data.toString()).toBe(ACTUAL_SNAPSHOT); }); +test('should ignore text snapshot with the ignore-snapshots flag', async ({ runInlineTest }, testInfo) => { + const EXPECTED_SNAPSHOT = 'Hello world'; + const ACTUAL_SNAPSHOT = 'Hello world updated'; + const result = await runInlineTest({ + ...files, + 'a.spec.js-snapshots/snapshot.txt': EXPECTED_SNAPSHOT, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', ({}) => { + expect('${ACTUAL_SNAPSHOT}').toMatchSnapshot('snapshot.txt'); + }); + ` + }, { 'ignore-snapshots': true }); + + expect(result.exitCode).toBe(0); + const snapshotOutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.txt'); + expect(result.output).toContain(``); + const data = fs.readFileSync(snapshotOutputPath); + expect(data.toString()).toBe(EXPECTED_SNAPSHOT); +}); + test('shouldn\'t update snapshot with the update-snapshots flag for negated matcher', async ({ runInlineTest }, testInfo) => { const EXPECTED_SNAPSHOT = 'Hello world'; const ACTUAL_SNAPSHOT = 'Hello world updated'; diff --git a/tests/playwright-test/to-have-screenshot.spec.ts b/tests/playwright-test/to-have-screenshot.spec.ts index 8465c21826..8240567675 100644 --- a/tests/playwright-test/to-have-screenshot.spec.ts +++ b/tests/playwright-test/to-have-screenshot.spec.ts @@ -555,6 +555,20 @@ test('should fail on same snapshots with negate matcher', async ({ runInlineTest expect(result.output).toContain('Expected result should be different from the actual one.'); }); +test('should not fail if --ignore-snapshots is passed', async ({ runInlineTest }) => { + const result = await runInlineTest({ + ...playwrightConfig({ screenshotsDir: '__screenshots__' }), + '__screenshots__/a.spec.js/snapshot.png': redImage, + 'a.spec.js': ` + pwt.test('is a test', async ({ page }) => { + await expect(page).toHaveScreenshot('snapshot.png', { timeout: 2000 }); + }); + ` + }, { 'ignore-snapshots': true }); + + expect(result.exitCode).toBe(0); +}); + test('should write missing expectations locally twice and continue', async ({ runInlineTest }, testInfo) => { const result = await runInlineTest({ ...playwrightConfig({ screenshotsDir: '__screenshots__' }),