diff --git a/docs/src/test-api/class-testproject.md b/docs/src/test-api/class-testproject.md index 2cb6a77938..2ed68de275 100644 --- a/docs/src/test-api/class-testproject.md +++ b/docs/src/test-api/class-testproject.md @@ -135,6 +135,39 @@ Filter to only run tests with a title **not** matching one of the patterns. This `grepInvert` option is also useful for [tagging tests](../test-annotations.md#tag-tests). +## property: TestProject.ignoreSnapshots +* since: v1.44 +- type: ?<[boolean]> + +Whether to skip snapshot expectations, such as `expect(value).toMatchSnapshot()` and `await expect(page).toHaveScreenshot()`. + +**Usage** + +The following example will only perform screenshot assertions on Chromium. + +```js title="playwright.config.ts" +import { defineConfig } from '@playwright/test'; + +export default defineConfig({ + projects: [ + { + name: 'chromium', + use: devices['Desktop Chrome'], + }, + { + name: 'firefox', + use: devices['Desktop Firefox'], + ignoreSnapshots: true, + }, + { + name: 'webkit', + use: devices['Desktop Safari'], + ignoreSnapshots: true, + }, + ], +}); +``` + ## property: TestProject.metadata * since: v1.10 - type: ?<[Metadata]> diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts index 9b8be8e730..31819bdecc 100644 --- a/packages/playwright/src/common/config.ts +++ b/packages/playwright/src/common/config.ts @@ -44,7 +44,6 @@ export class FullConfigInternal { readonly globalOutputDir: string; readonly configDir: string; readonly configCLIOverrides: ConfigCLIOverrides; - readonly ignoreSnapshots: boolean; readonly preserveOutputDir: boolean; readonly webServers: NonNullable[]; readonly plugins: TestRunnerPluginRegistration[]; @@ -71,7 +70,6 @@ export class FullConfigInternal { this.configCLIOverrides = configCLIOverrides; this.globalOutputDir = takeFirst(configCLIOverrides.outputDir, pathResolve(configDir, userConfig.outputDir), throwawayArtifactsPath, path.resolve(process.cwd())); this.preserveOutputDir = configCLIOverrides.preserveOutputDir || false; - this.ignoreSnapshots = takeFirst(configCLIOverrides.ignoreSnapshots, userConfig.ignoreSnapshots, false); const privateConfiguration = (userConfig as any)['@playwright/test']; this.plugins = (privateConfiguration?.plugins || []).map((p: any) => ({ factory: p })); @@ -164,6 +162,7 @@ export class FullProjectInternal { readonly expect: Project['expect']; readonly respectGitIgnore: boolean; readonly snapshotPathTemplate: string; + readonly ignoreSnapshots: boolean; id = ''; deps: FullProjectInternal[] = []; teardown: FullProjectInternal | undefined; @@ -200,6 +199,7 @@ export class FullProjectInternal { this.expect.toHaveScreenshot.stylePath = stylePaths.map(stylePath => path.resolve(configDir, stylePath)); } this.respectGitIgnore = !projectConfig.testDir && !config.testDir; + this.ignoreSnapshots = takeFirst(configCLIOverrides.ignoreSnapshots, projectConfig.ignoreSnapshots, config.ignoreSnapshots, false); } } diff --git a/packages/playwright/src/common/configLoader.ts b/packages/playwright/src/common/configLoader.ts index de568114a1..81ea74e8e3 100644 --- a/packages/playwright/src/common/configLoader.ts +++ b/packages/playwright/src/common/configLoader.ts @@ -214,11 +214,6 @@ function validateConfig(file: string, config: Config) { throw errorWithFile(file, `config.shard.current must be a positive number, not greater than config.shard.total`); } - if ('ignoreSnapshots' in config && config.ignoreSnapshots !== undefined) { - if (typeof config.ignoreSnapshots !== 'boolean') - throw errorWithFile(file, `config.ignoreSnapshots must be a boolean`); - } - if ('updateSnapshots' in config && config.updateSnapshots !== undefined) { if (typeof config.updateSnapshots !== 'string' || !['all', 'none', 'missing'].includes(config.updateSnapshots)) throw errorWithFile(file, `config.updateSnapshots must be one of "all", "none" or "missing"`); @@ -284,6 +279,11 @@ function validateProject(file: string, project: Project, title: string) { if (!project.use || typeof project.use !== 'object') throw errorWithFile(file, `${title}.use must be an object`); } + + if ('ignoreSnapshots' in project && project.ignoreSnapshots !== undefined) { + if (typeof project.ignoreSnapshots !== 'boolean') + throw errorWithFile(file, `${title}.ignoreSnapshots must be a boolean`); + } } export function resolveConfigLocation(configFile: string | undefined): ConfigLocation { diff --git a/packages/playwright/src/matchers/toMatchSnapshot.ts b/packages/playwright/src/matchers/toMatchSnapshot.ts index b0114bd2f6..ca7b38dfc5 100644 --- a/packages/playwright/src/matchers/toMatchSnapshot.ts +++ b/packages/playwright/src/matchers/toMatchSnapshot.ts @@ -302,7 +302,7 @@ export function toMatchSnapshot( if (received instanceof Promise) throw new Error('An unresolved Promise was passed to toMatchSnapshot(), make sure to resolve it by adding await to it.'); - if (testInfo._configInternal.ignoreSnapshots) + if (testInfo._projectInternal.ignoreSnapshots) return { pass: !this.isNot, message: () => '', name: 'toMatchSnapshot', expected: nameOrOptions }; const configOptions = testInfo._projectInternal.expect?.toMatchSnapshot || {}; @@ -357,7 +357,7 @@ export async function toHaveScreenshot( if (!testInfo) throw new Error(`toHaveScreenshot() must be called during the test`); - if (testInfo._configInternal.ignoreSnapshots) + if (testInfo._projectInternal.ignoreSnapshots) return { pass: !this.isNot, message: () => '', name: 'toHaveScreenshot', expected: nameOrOptions }; expectTypes(pageOrLocator, ['Page', 'Locator'], 'toHaveScreenshot'); diff --git a/packages/playwright/src/runner/runner.ts b/packages/playwright/src/runner/runner.ts index af24c5f162..3e4b3c6a78 100644 --- a/packages/playwright/src/runner/runner.ts +++ b/packages/playwright/src/runner/runner.ts @@ -23,7 +23,6 @@ import { collectFilesForProject, filterProjects } from './projectUtils'; import { createReporters } from './reporters'; import { TestRun, createTaskRunner, createTaskRunnerForList } from './tasks'; import type { FullConfigInternal } from '../common/config'; -import { colors } from 'playwright-core/lib/utilsBundle'; import { runWatchModeLoop } from './watchMode'; import { InternalReporter } from '../reporters/internalReporter'; import { Multiplexer } from '../reporters/multiplexer'; @@ -86,15 +85,6 @@ export class Runner { const testRun = new TestRun(config, reporter); reporter.onConfigure(config.config); - if (!listOnly && config.ignoreSnapshots) { - reporter.onStdOut(colors.dim([ - 'NOTE: running with "ignoreSnapshots" option. All of the following asserts are silently ignored:', - '- expect().toMatchSnapshot()', - '- expect().toHaveScreenshot()', - '', - ].join('\n'))); - } - const taskStatus = await taskRunner.run(testRun, deadline); let status: FullResult['status'] = testRun.failureTracker.result(); if (status === 'passed' && taskStatus !== 'passed') diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index df4783a75d..01470c2bba 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -280,6 +280,41 @@ interface TestProject { */ grepInvert?: RegExp|Array; + /** + * Whether to skip snapshot expectations, such as `expect(value).toMatchSnapshot()` and `await + * expect(page).toHaveScreenshot()`. + * + * **Usage** + * + * The following example will only perform screenshot assertions on Chromium. + * + * ```js + * // playwright.config.ts + * import { defineConfig } from '@playwright/test'; + * + * export default defineConfig({ + * projects: [ + * { + * name: 'chromium', + * use: devices['Desktop Chrome'], + * }, + * { + * name: 'firefox', + * use: devices['Desktop Firefox'], + * ignoreSnapshots: true, + * }, + * { + * name: 'webkit', + * use: devices['Desktop Safari'], + * ignoreSnapshots: true, + * }, + * ], + * }); + * ``` + * + */ + ignoreSnapshots?: boolean; + /** * Metadata that will be put directly to the test report serialized as JSON. */ diff --git a/tests/playwright-test/config.spec.ts b/tests/playwright-test/config.spec.ts index 38fef69915..5550d84253 100644 --- a/tests/playwright-test/config.spec.ts +++ b/tests/playwright-test/config.spec.ts @@ -439,19 +439,26 @@ test('should support ignoreSnapshots config option', async ({ runInlineTest }) = 'playwright.config.ts': ` module.exports = { ignoreSnapshots: true, + projects: [ + { name: 'p1' }, + { name: 'p2', ignoreSnapshots: false }, + ] }; `, 'a.test.ts': ` import { test, expect } from '@playwright/test'; test('pass', async ({}, testInfo) => { - expect('foo').toMatchSnapshot(); - expect('foo').not.toMatchSnapshot(); + testInfo.snapshotSuffix = ''; + expect(testInfo.project.name).toMatchSnapshot(); }); ` }); - expect(result.exitCode).toBe(0); + expect(result.exitCode).toBe(1); expect(result.passed).toBe(1); + expect(result.failed).toBe(1); + expect(result.output).not.toContain(`pass-1-p1.txt, writing actual.`); + expect(result.output).toContain(`pass-1-p2.txt, writing actual.`); }); test('should validate workers option set to percent', async ({ runInlineTest }, testInfo) => { @@ -640,7 +647,7 @@ test('should merge ct configs', async ({ runInlineTest }) => { use: { foo: 1, bar: 2 }, grep: 'hi', '@playwright/test': expect.objectContaining({ - babelPlugins: [[expect.stringContaining('tsxTransform.js')]] + babelPlugins: [[expect.stringContaining('tsxTransform.js')]] }), '@playwright/experimental-ct-core': expect.objectContaining({ registerSourceFile: expect.stringContaining('registerSource'),