feat: TestProject.ignoreSnapshots (#30466)

docs/src/test-api/class-testproject.md

@@ -135,6 +135,39 @@ Filter to only run tests with a title **not** matching one of the patterns. This
 
 `grepInvert` option is also useful for [tagging tests](../test-annotations.md#tag-tests).
 
+## property: TestProject.ignoreSnapshots
+* since: v1.44
+- type: ?<[boolean]>
+
+Whether to skip snapshot expectations, such as `expect(value).toMatchSnapshot()` and `await expect(page).toHaveScreenshot()`.
+
+**Usage**
+
+The following example will only perform screenshot assertions on Chromium.
+
+```js title="playwright.config.ts"
+import { defineConfig } from '@playwright/test';
+
+export default defineConfig({
+  projects: [
+    {
+      name: 'chromium',
+      use: devices['Desktop Chrome'],
+    },
+    {
+      name: 'firefox',
+      use: devices['Desktop Firefox'],
+      ignoreSnapshots: true,
+    },
+    {
+      name: 'webkit',
+      use: devices['Desktop Safari'],
+      ignoreSnapshots: true,
+    },
+  ],
+});
+```
+
 ## property: TestProject.metadata
 * since: v1.10
 - type: ?<[Metadata]>

packages/playwright/src/common/config.ts

@@ -44,7 +44,6 @@ export class FullConfigInternal {
   readonly globalOutputDir: string;
   readonly configDir: string;
   readonly configCLIOverrides: ConfigCLIOverrides;
-  readonly ignoreSnapshots: boolean;
   readonly preserveOutputDir: boolean;
   readonly webServers: NonNullable<FullConfig['webServer']>[];
   readonly plugins: TestRunnerPluginRegistration[];
@@ -71,7 +70,6 @@ export class FullConfigInternal {
     this.configCLIOverrides = configCLIOverrides;
     this.globalOutputDir = takeFirst(configCLIOverrides.outputDir, pathResolve(configDir, userConfig.outputDir), throwawayArtifactsPath, path.resolve(process.cwd()));
     this.preserveOutputDir = configCLIOverrides.preserveOutputDir || false;
-    this.ignoreSnapshots = takeFirst(configCLIOverrides.ignoreSnapshots, userConfig.ignoreSnapshots, false);
     const privateConfiguration = (userConfig as any)['@playwright/test'];
     this.plugins = (privateConfiguration?.plugins || []).map((p: any) => ({ factory: p }));
 
@@ -164,6 +162,7 @@ export class FullProjectInternal {
   readonly expect: Project['expect'];
   readonly respectGitIgnore: boolean;
   readonly snapshotPathTemplate: string;
+  readonly ignoreSnapshots: boolean;
   id = '';
   deps: FullProjectInternal[] = [];
   teardown: FullProjectInternal | undefined;
@@ -200,6 +199,7 @@ export class FullProjectInternal {
       this.expect.toHaveScreenshot.stylePath = stylePaths.map(stylePath => path.resolve(configDir, stylePath));
     }
     this.respectGitIgnore = !projectConfig.testDir && !config.testDir;
+    this.ignoreSnapshots = takeFirst(configCLIOverrides.ignoreSnapshots,  projectConfig.ignoreSnapshots, config.ignoreSnapshots, false);
   }
 }
 

packages/playwright/src/common/configLoader.ts

@@ -214,11 +214,6 @@ function validateConfig(file: string, config: Config) {
       throw errorWithFile(file, `config.shard.current must be a positive number, not greater than config.shard.total`);
   }
 
-  if ('ignoreSnapshots' in config && config.ignoreSnapshots !== undefined) {
-    if (typeof config.ignoreSnapshots !== 'boolean')
-      throw errorWithFile(file, `config.ignoreSnapshots must be a boolean`);
-  }
-
   if ('updateSnapshots' in config && config.updateSnapshots !== undefined) {
     if (typeof config.updateSnapshots !== 'string' || !['all', 'none', 'missing'].includes(config.updateSnapshots))
       throw errorWithFile(file, `config.updateSnapshots must be one of "all", "none" or "missing"`);
@@ -284,6 +279,11 @@ function validateProject(file: string, project: Project, title: string) {
     if (!project.use || typeof project.use !== 'object')
       throw errorWithFile(file, `${title}.use must be an object`);
   }
+
+  if ('ignoreSnapshots' in project && project.ignoreSnapshots !== undefined) {
+    if (typeof project.ignoreSnapshots !== 'boolean')
+      throw errorWithFile(file, `${title}.ignoreSnapshots must be a boolean`);
+  }
 }
 
 export function resolveConfigLocation(configFile: string | undefined): ConfigLocation {

packages/playwright/src/matchers/toMatchSnapshot.ts

@@ -302,7 +302,7 @@ export function toMatchSnapshot(
   if (received instanceof Promise)
     throw new Error('An unresolved Promise was passed to toMatchSnapshot(), make sure to resolve it by adding await to it.');
 
-  if (testInfo._configInternal.ignoreSnapshots)
+  if (testInfo._projectInternal.ignoreSnapshots)
     return { pass: !this.isNot, message: () => '', name: 'toMatchSnapshot', expected: nameOrOptions };
 
   const configOptions = testInfo._projectInternal.expect?.toMatchSnapshot || {};
@@ -357,7 +357,7 @@ export async function toHaveScreenshot(
   if (!testInfo)
     throw new Error(`toHaveScreenshot() must be called during the test`);
 
-  if (testInfo._configInternal.ignoreSnapshots)
+  if (testInfo._projectInternal.ignoreSnapshots)
     return { pass: !this.isNot, message: () => '', name: 'toHaveScreenshot', expected: nameOrOptions };
 
   expectTypes(pageOrLocator, ['Page', 'Locator'], 'toHaveScreenshot');

packages/playwright/src/runner/runner.ts

@@ -23,7 +23,6 @@ import { collectFilesForProject, filterProjects } from './projectUtils';
 import { createReporters } from './reporters';
 import { TestRun, createTaskRunner, createTaskRunnerForList } from './tasks';
 import type { FullConfigInternal } from '../common/config';
-import { colors } from 'playwright-core/lib/utilsBundle';
 import { runWatchModeLoop } from './watchMode';
 import { InternalReporter } from '../reporters/internalReporter';
 import { Multiplexer } from '../reporters/multiplexer';
@@ -86,15 +85,6 @@ export class Runner {
     const testRun = new TestRun(config, reporter);
     reporter.onConfigure(config.config);
 
-    if (!listOnly && config.ignoreSnapshots) {
-      reporter.onStdOut(colors.dim([
-        'NOTE: running with "ignoreSnapshots" option. All of the following asserts are silently ignored:',
-        '- expect().toMatchSnapshot()',
-        '- expect().toHaveScreenshot()',
-        '',
-      ].join('\n')));
-    }
-
     const taskStatus = await taskRunner.run(testRun, deadline);
     let status: FullResult['status'] = testRun.failureTracker.result();
     if (status === 'passed' && taskStatus !== 'passed')

packages/playwright/types/test.d.ts

@@ -280,6 +280,41 @@ interface TestProject<TestArgs = {}, WorkerArgs = {}> {
    */
   grepInvert?: RegExp|Array<RegExp>;
 
+  /**
+   * Whether to skip snapshot expectations, such as `expect(value).toMatchSnapshot()` and `await
+   * expect(page).toHaveScreenshot()`.
+   *
+   * **Usage**
+   *
+   * The following example will only perform screenshot assertions on Chromium.
+   *
+   * ```js
+   * // playwright.config.ts
+   * import { defineConfig } from '@playwright/test';
+   *
+   * export default defineConfig({
+   *   projects: [
+   *     {
+   *       name: 'chromium',
+   *       use: devices['Desktop Chrome'],
+   *     },
+   *     {
+   *       name: 'firefox',
+   *       use: devices['Desktop Firefox'],
+   *       ignoreSnapshots: true,
+   *     },
+   *     {
+   *       name: 'webkit',
+   *       use: devices['Desktop Safari'],
+   *       ignoreSnapshots: true,
+   *     },
+   *   ],
+   * });
+   * ```
+   *
+   */
+  ignoreSnapshots?: boolean;
+
   /**
    * Metadata that will be put directly to the test report serialized as JSON.
    */

tests/playwright-test/config.spec.ts

@@ -439,19 +439,26 @@ test('should support ignoreSnapshots config option', async ({ runInlineTest }) =
     'playwright.config.ts': `
       module.exports = {
         ignoreSnapshots: true,
+        projects: [
+          { name: 'p1' },
+          { name: 'p2', ignoreSnapshots: false },
+        ]
       };
     `,
     'a.test.ts': `
       import { test, expect } from '@playwright/test';
       test('pass', async ({}, testInfo) => {
-        expect('foo').toMatchSnapshot();
+        testInfo.snapshotSuffix = '';
-        expect('foo').not.toMatchSnapshot();
+        expect(testInfo.project.name).toMatchSnapshot();
       });
     `
   });
 
-  expect(result.exitCode).toBe(0);
+  expect(result.exitCode).toBe(1);
   expect(result.passed).toBe(1);
+  expect(result.failed).toBe(1);
+  expect(result.output).not.toContain(`pass-1-p1.txt, writing actual.`);
+  expect(result.output).toContain(`pass-1-p2.txt, writing actual.`);
 });
 
 test('should validate workers option set to percent', async ({ runInlineTest }, testInfo) => {