feat(docker): use the __screenshots__ dir for snapshots by default (#17311)

This patch opts into the `__screnshots__` folder snapshot management
for docker.

With this patch, docker-originating snapshots will be stored in the
following folder:

```
{testDir}/__screenshots__/{projectName}/{testFilePath}/{snapshotName}
```

Where `{testFilePath}` is a test file path relative to `testDir`

Drive-by: introduce and document the `PLAYWRIGHT_DOCKER` environment
variable that enables docker integration.

docs/src/docker.md

@@ -193,3 +193,18 @@ Docker integration usage:
     ```bash js
     npx playwright docker stop
     ```
+
+Playwright Test sets `PLAYWRIGHT_DOCKER` environment variable when it uses Docker integration.
+You can use this variable to customize config or tests behavior, for example:
+
+```ts
+// playwright.config.ts
+import type { PlaywrightTestConfig } from '@playwright/test';
+
+const config: PlaywrightTestConfig = {
+  // Ignore all snapshot expectations when running outside
+  // of docker integration.
+  ignoreSnapshots: !process.env.PLAYWRIGHT_DOCKER,
+};
+export default config;
+```

packages/playwright-test/src/cli.ts

@@ -120,6 +120,7 @@ function addTestCommand(program: Command, isDocker: boolean) {
   command.option('-x', `Stop after the first failure`);
   command.action(async (args, opts) => {
     try {
+      isDocker = isDocker || !!process.env.PLAYWRIGHT_DOCKER;
       if (isDocker && !process.env.PW_TS_ESM_ON) {
         console.log(colors.dim('Using docker container to run browsers.'));
         await docker.ensureDockerEngineIsRunningOrDie();
@@ -139,7 +140,7 @@ function addTestCommand(program: Command, isDocker: boolean) {
         process.env.PW_TEST_CONNECT_HEADERS = JSON.stringify({
           'x-playwright-proxy': '*',
         });
-        process.env.PW_TEST_SNAPSHOT_SUFFIX = 'docker';
+        process.env.PLAYWRIGHT_DOCKER = '1';
       }
       await runTests(args, opts);
     } catch (e) {

packages/playwright-test/src/index.ts

@@ -223,7 +223,7 @@ export const test = _baseTest.extend<TestFixtures, WorkerFixtures>({
     });
   },
 
-  _snapshotSuffix: [process.env.PW_TEST_SNAPSHOT_SUFFIX ?? process.platform, { scope: 'worker' }],
+  _snapshotSuffix: [process.platform, { scope: 'worker' }],
 
   _setupContextOptionsAndArtifacts: [async ({ playwright, _snapshotSuffix, _combinedContextOptions, _browserOptions, _artifactsDir, trace, screenshot, actionTimeout, navigationTimeout }, use, testInfo) => {
     testInfo.snapshotSuffix = _snapshotSuffix;

packages/playwright-test/src/loader.ts

@@ -270,7 +270,12 @@ export class Loader {
     const outputDir = takeFirst(projectConfig.outputDir, config.outputDir, path.join(throwawayArtifactsPath, 'test-results'));
     const snapshotDir = takeFirst(projectConfig.snapshotDir, config.snapshotDir, testDir);
     const name = takeFirst(projectConfig.name, config.name, '');
-    const screenshotsDir = takeFirst((projectConfig as any).screenshotsDir, (config as any).screenshotsDir, path.join(testDir, '__screenshots__', process.platform, name));
+
+    let screenshotsDir = takeFirst((projectConfig as any).screenshotsDir, (config as any).screenshotsDir, path.join(testDir, '__screenshots__', process.platform, name));
+    if (process.env.PLAYWRIGHT_DOCKER) {
+      screenshotsDir = path.join(testDir, '__screenshots__', name);
+      process.env.PWTEST_USE_SCREENSHOTS_DIR = '1';
+    }
     return {
       _id: '',
       _fullConfig: fullConfig,

packages/playwright-test/src/matchers/toMatchSnapshot.ts

@@ -301,7 +301,7 @@ export async function toHaveScreenshot(
     return { pass: !this.isNot, message: () => '' };
 
   const config = (testInfo.project._expect as any)?.toHaveScreenshot;
-  const snapshotPathResolver = process.env.PWTEST_USE_SCREENSHOTS_DIR_FOR_TEST
+  const snapshotPathResolver = process.env.PWTEST_USE_SCREENSHOTS_DIR
     ? testInfo._screenshotPath.bind(testInfo)
     : testInfo.snapshotPath.bind(testInfo);
   const helper = new SnapshotHelper(
@@ -315,6 +315,7 @@ export async function toHaveScreenshot(
   if (!helper.snapshotPath.toLowerCase().endsWith('.png'))
     throw new Error(`Screenshot name "${path.basename(helper.snapshotPath)}" must have '.png' extension`);
   expectTypes(pageOrLocator, ['Page', 'Locator'], 'toHaveScreenshot');
+
   const [page, locator] = pageOrLocator.constructor.name === 'Page' ? [(pageOrLocator as PageEx), undefined] : [(pageOrLocator as Locator).page() as PageEx, pageOrLocator as LocatorEx];
   const screenshotOptions = {
     animations: config?.animations ?? 'disabled',

tests/installation/docker-integration.spec.ts

@@ -15,7 +15,6 @@
  */
 import { test, expect } from './npmTest';
 import * as path from 'path';
-import * as fs from 'fs';
 import { TestServer } from '../../utils/testserver';
 
 // Skipping docker tests on CI on non-linux since GHA does not have
@@ -78,6 +77,18 @@ test.describe('installed image', () => {
       expect(result).toContain('@firefox Linux');
     });
 
+    test('supports PLAYWRIGHT_DOCKER env variable', async ({ exec }) => {
+      await exec('npm i --foreground-scripts @playwright/test');
+      const result = await exec('npx playwright test docker.spec.js --grep platform --browser all', {
+        env: {
+          PLAYWRIGHT_DOCKER: '1',
+        },
+      });
+      expect(result).toContain('@chromium Linux');
+      expect(result).toContain('@webkit Linux');
+      expect(result).toContain('@firefox Linux');
+    });
+
     test('all browsers work headed', async ({ exec }) => {
       await exec('npm i --foreground-scripts @playwright/test');
       {
@@ -98,15 +109,14 @@ test.describe('installed image', () => {
       }
     });
 
-    test('screenshots have docker suffix', async ({ exec, tmpWorkspace }) => {
+    test('screenshots should use __screenshots__ folder', async ({ exec, tmpWorkspace }) => {
       await exec('npm i --foreground-scripts @playwright/test');
       await exec('npx playwright docker test docker.spec.js --grep screenshot --browser all', {
         expectToExitWithError: true,
       });
-      const files = await fs.promises.readdir(path.join(tmpWorkspace, 'docker.spec.js-snapshots'));
-      expect(files).toContain('img-chromium-docker.png');
-      expect(files).toContain('img-firefox-docker.png');
-      expect(files).toContain('img-webkit-docker.png');
+      await expect(path.join(tmpWorkspace, '__screenshots__', 'firefox', 'docker.spec.js', 'img.png')).toExistOnFS();
+      await expect(path.join(tmpWorkspace, '__screenshots__', 'chromium', 'docker.spec.js', 'img.png')).toExistOnFS();
+      await expect(path.join(tmpWorkspace, '__screenshots__', 'webkit', 'docker.spec.js', 'img.png')).toExistOnFS();
     });
 
     test('port forwarding works', async ({ exec, tmpWorkspace }) => {

tests/installation/expect.d.ts

@@ -20,6 +20,7 @@ declare global {
     namespace PlaywrightTest {
        interface Matchers<R, T> {
           toHaveLoggedSoftwareDownload(browsers: ("chromium" | "firefox" | "webkit" | "ffmpeg")[]): R;
+          toExistOnFS(): R;
        }
     }
 }

tests/installation/npmTest.ts

@@ -39,6 +39,17 @@ const debug = debugLogger('itest');
  */
 
 _expect.extend({
+  async toExistOnFS(received: any) {
+    if (typeof received !== 'string')
+      throw new Error(`Expected argument to be a string.`);
+    try {
+      await fs.promises.access(received);
+      return { pass: true };
+    } catch (e) {
+      return { pass: false, message: () => 'file does not exist' };
+    }
+  },
+
   toHaveLoggedSoftwareDownload(received: any, browsers: ('chromium' | 'firefox' | 'webkit' | 'ffmpeg')[]) {
     if (typeof received !== 'string')
       throw new Error(`Expected argument to be a string.`);

tests/playwright-test/reporter-html.spec.ts

@@ -182,7 +182,7 @@ test('should include multiple image diffs', async ({ runInlineTest, page, showRe
 
   const result = await runInlineTest({
     'playwright.config.ts': `
-      process.env.PWTEST_USE_SCREENSHOTS_DIR_FOR_TEST = '1';
+      process.env.PWTEST_USE_SCREENSHOTS_DIR = '1';
       module.exports = {
         screenshotsDir: '__screenshots__',
         use: { viewport: { width: ${IMG_WIDTH}, height: ${IMG_HEIGHT} }}

tests/playwright-test/to-have-screenshot.spec.ts

@@ -135,7 +135,7 @@ test('should fail with proper error when unsupported argument is given', async (
 
 test('should use match snapshot paths by default', async ({ runInlineTest }, testInfo) => {
   const result = await runInlineTest({
-    // The helper function `playwrightConfig` set PWTEST_USE_SCREENSHOTS_DIR_FOR_TEST env variable.
+    // The helper function `playwrightConfig` set PWTEST_USE_SCREENSHOTS env variable.
     // Provide default config manually instead.
     'playwright.config.js': `
       module.exports = {};
@@ -1017,7 +1017,7 @@ test('should update expectations with retries', async ({ runInlineTest }, testIn
 function playwrightConfig(obj: any) {
   return {
     'playwright.config.js': `
-      process.env.PWTEST_USE_SCREENSHOTS_DIR_FOR_TEST = '1';
+      process.env.PWTEST_USE_SCREENSHOTS_DIR = '1';
       module.exports = ${JSON.stringify(obj, null, 2)}
     `,
   };