diff --git a/docs/src/api/params.md b/docs/src/api/params.md index c64fb46cc6..127eaaea40 100644 --- a/docs/src/api/params.md +++ b/docs/src/api/params.md @@ -1350,3 +1350,89 @@ Allows locating elements by their title. For example, this method will find the ```html ``` + +## test-config-snapshot-template-path +- `type` ?<[string]> + +This configuration option allows to set a string with template values for precise control over snapshot path location. + +```js tab=js-ts +// playwright.config.ts +import type { PlaywrightTestConfig } from '@playwright/test'; + +const config: PlaywrightTestConfig = { + testDir: './tests', + snapshotPathTemplate: '{testDir}/__screenshots__/{testFilePath}/{arg}{ext}', +}; + +export default config; +``` + +The value might include some "tokens" that will be replaced with actual values during test execution. + +Consider the following file structure: + +``` +playwright.config.ts +tests/ +└── page/ + └── page-click.spec.ts +``` + +And the following `page-click.spec.ts` that uses `toHaveScreenshot()` call: + +```ts +// page-click.spec.ts +import { test, expect } from '@playwright/test'; + +test('should work', async ({ page }) => { + await expect(page).toHaveScreenshot(['foo', 'bar', 'baz.png']); +}); +``` + +The list of supported tokens: + +* `{testDir}` - Project's `testDir`. + * Example: `tests/` +* `{snapshotDir}` - Project's `snapshotDir`. + * Example: `tests/` (since `snapshotDir` is not provided in config, it defaults to `testDir`) +* `{platform}` - The value of `process.platform`. +* `{snapshotSuffix}` - The value of `testInfo.snapshotSuffix`. +* `{projectName}` - Project's sanitized name, if any. + * Example: `undefined`. +* `{testFileDir}` - Directories in relative path from `testDir` to **test file**. + * Example: `page/` +* `{testFileName}` - Test file name with extension. + * Example: `page-click.spec.ts` +* `{testFilePath}` - Relative path from `testDir` to **test file** + * Example: `page/page-click.spec.ts` +* `{arg}` - Relative snapshot path **without extension**. These come from the arguments passed to the `toHaveScreenshot()` and `toMatchSnapshot()` calls; if called without arguments, this will be an auto-generated snapshot name. + * Example: `foo/bar/baz` +* `{ext}` - snapshot extension (with dots) + * Example: `.png` + +Each token can be preceded with a single character that will be used **only if** this token has non-empty value. + +Consider the following config: + +```js tab=js-ts +// playwright.config.ts +import type { PlaywrightTestConfig } from '@playwright/test'; + +const config: PlaywrightTestConfig = { + snapshotPathTemplate: '__screenshots__{/projectName}/{testFilePath}/{arg}{ext}', + testMatch: 'example.spec.ts', + projects: [ + { use: { browserName: 'firefox' } }, + { name: 'chromium', use: { browserName: 'chromium' } }, + ], +}; +export default config; +``` + +In this config: +1. First project **does not** have a name, so its snapshots will be stored in `/__screenshots__/example.spec.ts/...`. +1. Second project **does** have a name, so its snapshots will be stored in `/__screenshots__/chromium/example.spec.ts/..`. +1. Since `snapshotPathTemplate` resolves to relative path, it will be resolved relative to `configDir`. +1. Forward slashes `"/"` can be used as path separators regarding of the platform and work everywhere. + diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index 6e98c8631f..772005e1f3 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -337,6 +337,9 @@ The directory for each test can be accessed by [`property: TestInfo.snapshotDir` This path will serve as the base directory for each test file snapshot directory. Setting `snapshotDir` to `'snapshots'`, the [`property: TestInfo.snapshotDir`] would resolve to `snapshots/a.spec.js-snapshots`. +## property: TestConfig.snapshotPathTemplate = %%-test-config-snapshot-template-path-%% +* since: v1.28 + ## property: TestConfig.preserveOutput * since: v1.10 - type: ?<[PreserveOutput]<"always"|"never"|"failures-only">> @@ -441,43 +444,6 @@ const config: PlaywrightTestConfig = { export default config; ``` -## property: TestConfig.screenshotsDir -* since: v1.10 -* experimental -- type: ?<[string]> - -The base directory, relative to the config file, for screenshot files created with [`method: PageAssertions.toHaveScreenshot#1`]. Defaults to - -``` -/__screenshots__// -``` - -This path will serve as the base directory for each test file screenshot directory. For example, the following test structure: - -``` -smoke-tests/ -└── basic.spec.ts -``` - -will result in the following screenshots folder structure: - -``` -__screenshots__/ -└── darwin/ - ├── Mobile Safari/ - │ └── smoke-tests/ - │ └── basic.spec.ts/ - │ └── screenshot-expectation.png - └── Desktop Chrome/ - └── smoke-tests/ - └── basic.spec.ts/ - └── screenshot-expectation.png -``` - -where: -* `darwin/` - a platform name folder -* `Mobile Safari` and `Desktop Chrome` - project names - ## property: TestConfig.shard * since: v1.10 - type: ?<[null]|[Object]> diff --git a/docs/src/test-api/class-testinfo.md b/docs/src/test-api/class-testinfo.md index b2c6c676a0..f2c3b33ded 100644 --- a/docs/src/test-api/class-testinfo.md +++ b/docs/src/test-api/class-testinfo.md @@ -260,6 +260,8 @@ Line number where the currently running test is declared. Absolute path to the snapshot output directory for this specific test. Each test suite gets its own directory so they cannot conflict. +This property does not account for the [`property: TestProject.snapshotPathTemplate`] configuration. + ## property: TestInfo.outputDir * since: v1.10 - type: <[string]> diff --git a/docs/src/test-api/class-testproject.md b/docs/src/test-api/class-testproject.md index 60a8e26796..8210f28d07 100644 --- a/docs/src/test-api/class-testproject.md +++ b/docs/src/test-api/class-testproject.md @@ -168,44 +168,6 @@ Project name is visible in the report and during test execution. Project setup files that would be executed before all tests in the project. If project setup fails the tests in this project will be skipped. All project setup files will run in every shard if the project is sharded. -## property: TestProject.screenshotsDir -* since: v1.10 -* experimental -- type: ?<[string]> - -The base directory, relative to the config file, for screenshot files created with `toHaveScreenshot`. Defaults to - -``` -/__screenshots__// -``` - -This path will serve as the base directory for each test file screenshot directory. For example, the following test structure: - -``` -smoke-tests/ -└── basic.spec.ts -``` - -will result in the following screenshots folder structure: - -``` -__screenshots__/ -└── darwin/ - ├── Mobile Safari/ - │ └── smoke-tests/ - │ └── basic.spec.ts/ - │ └── screenshot-expectation.png - └── Desktop Chrome/ - └── smoke-tests/ - └── basic.spec.ts/ - └── screenshot-expectation.png -``` - -where: -* `darwin/` - a platform name folder -* `Mobile Safari` and `Desktop Chrome` - project names - - ## property: TestProject.snapshotDir * since: v1.10 - type: ?<[string]> @@ -216,6 +178,9 @@ The directory for each test can be accessed by [`property: TestInfo.snapshotDir` This path will serve as the base directory for each test file snapshot directory. Setting `snapshotDir` to `'snapshots'`, the [`property: TestInfo.snapshotDir`] would resolve to `snapshots/a.spec.js-snapshots`. +## property: TestProject.snapshotPathTemplate = %%-test-config-snapshot-template-path-%% +* since: v1.28 + ## property: TestProject.outputDir * since: v1.10 - type: ?<[string]> diff --git a/packages/playwright-test/src/loader.ts b/packages/playwright-test/src/loader.ts index 6923b7d640..8526532b20 100644 --- a/packages/playwright-test/src/loader.ts +++ b/packages/playwright-test/src/loader.ts @@ -115,8 +115,6 @@ export class Loader { config.testDir = path.resolve(configDir, config.testDir); if (config.outputDir !== undefined) config.outputDir = path.resolve(configDir, config.outputDir); - if ((config as any).screenshotsDir !== undefined) - (config as any).screenshotsDir = path.resolve(configDir, (config as any).screenshotsDir); if (config.snapshotDir !== undefined) config.snapshotDir = path.resolve(configDir, config.snapshotDir); @@ -267,8 +265,6 @@ export class Loader { projectConfig.testDir = path.resolve(this._configDir, projectConfig.testDir); if (projectConfig.outputDir !== undefined) projectConfig.outputDir = path.resolve(this._configDir, projectConfig.outputDir); - if ((projectConfig as any).screenshotsDir !== undefined) - (projectConfig as any).screenshotsDir = path.resolve(this._configDir, (projectConfig as any).screenshotsDir); if (projectConfig.snapshotDir !== undefined) projectConfig.snapshotDir = path.resolve(this._configDir, projectConfig.snapshotDir); @@ -280,11 +276,8 @@ export class Loader { const name = takeFirst(projectConfig.name, config.name, ''); const _setup = takeFirst(projectConfig.setup, []); - let screenshotsDir = takeFirst((projectConfig as any).screenshotsDir, (config as any).screenshotsDir, path.join(testDir, '__screenshots__', process.platform, name)); - if (process.env.PLAYWRIGHT_DOCKER) { - screenshotsDir = path.join(testDir, '__screenshots__', name); - process.env.PWTEST_USE_SCREENSHOTS_DIR = '1'; - } + const defaultSnapshotPathTemplate = '{snapshotDir}/{testFileDir}/{testFileName}-snapshots/{arg}{-projectName}{-snapshotSuffix}{ext}'; + const snapshotPathTemplate = takeFirst((projectConfig as any).snapshotPathTemplate, (config as any).snapshotPathTemplate, defaultSnapshotPathTemplate); return { _id: '', _fullConfig: fullConfig, @@ -301,7 +294,7 @@ export class Loader { _setup, _respectGitIgnore: respectGitIgnore, snapshotDir, - _screenshotsDir: screenshotsDir, + snapshotPathTemplate: snapshotPathTemplate, testIgnore: takeFirst(projectConfig.testIgnore, config.testIgnore, []), testMatch: takeFirst(projectConfig.testMatch, config.testMatch, '**/?(*.)@(spec|test).*'), timeout: takeFirst(projectConfig.timeout, config.timeout, defaultTimeout), diff --git a/packages/playwright-test/src/matchers/toMatchSnapshot.ts b/packages/playwright-test/src/matchers/toMatchSnapshot.ts index 0de4f3212f..bd6e0eeba3 100644 --- a/packages/playwright-test/src/matchers/toMatchSnapshot.ts +++ b/packages/playwright-test/src/matchers/toMatchSnapshot.ts @@ -301,9 +301,7 @@ export async function toHaveScreenshot( return { pass: !this.isNot, message: () => '' }; const config = (testInfo.project._expect as any)?.toHaveScreenshot; - const snapshotPathResolver = process.env.PWTEST_USE_SCREENSHOTS_DIR - ? testInfo._screenshotPath.bind(testInfo) - : testInfo.snapshotPath.bind(testInfo); + const snapshotPathResolver = testInfo.snapshotPath.bind(testInfo); const helper = new SnapshotHelper( testInfo, snapshotPathResolver, 'png', { diff --git a/packages/playwright-test/src/testInfo.ts b/packages/playwright-test/src/testInfo.ts index 3deb50dfb8..c7af65b7df 100644 --- a/packages/playwright-test/src/testInfo.ts +++ b/packages/playwright-test/src/testInfo.ts @@ -23,7 +23,7 @@ import type { Loader } from './loader'; import type { TestCase } from './test'; import { TimeoutManager } from './timeoutManager'; import type { Annotation, FullConfigInternal, FullProjectInternal, TestStepInternal } from './types'; -import { addSuffixToFilePath, getContainedPath, normalizeAndSaveAttachment, sanitizeForFilePath, serializeError, trimLongString } from './util'; +import { getContainedPath, normalizeAndSaveAttachment, sanitizeForFilePath, serializeError, trimLongString } from './util'; export class TestInfoImpl implements TestInfo { private _addStepImpl: (data: Omit) => TestStepInternal; @@ -32,7 +32,6 @@ export class TestInfoImpl implements TestInfo { readonly _startTime: number; readonly _startWallTime: number; private _hasHardError: boolean = false; - readonly _screenshotsDir: string; readonly _onTestFailureImmediateCallbacks = new Map<() => Promise, string>(); // fn -> title _didTimeout = false; @@ -130,10 +129,6 @@ export class TestInfoImpl implements TestInfo { const relativeTestFilePath = path.relative(this.project.testDir, test._requireFile); return path.join(this.project.snapshotDir, relativeTestFilePath + '-snapshots'); })(); - this._screenshotsDir = (() => { - const relativeTestFilePath = path.relative(this.project.testDir, test._requireFile); - return path.join(this.project._screenshotsDir, relativeTestFilePath); - })(); } private _modifier(type: 'skip' | 'fail' | 'fixme' | 'slow', modifierArgs: [arg?: any, description?: string]) { @@ -240,25 +235,23 @@ export class TestInfoImpl implements TestInfo { } snapshotPath(...pathSegments: string[]) { - let suffix = ''; - const projectNamePathSegment = sanitizeForFilePath(this.project.name); - if (projectNamePathSegment) - suffix += '-' + projectNamePathSegment; - if (this.snapshotSuffix) - suffix += '-' + this.snapshotSuffix; - const subPath = addSuffixToFilePath(path.join(...pathSegments), suffix); - const snapshotPath = getContainedPath(this.snapshotDir, subPath); - if (snapshotPath) - return snapshotPath; - throw new Error(`The snapshotPath is not allowed outside of the parent directory. Please fix the defined path.\n\n\tsnapshotPath: ${subPath}`); - } - - _screenshotPath(...pathSegments: string[]) { const subPath = path.join(...pathSegments); - const screenshotPath = getContainedPath(this._screenshotsDir, subPath); - if (screenshotPath) - return screenshotPath; - throw new Error(`Screenshot name "${subPath}" should not point outside of the parent directory.`); + const parsedSubPath = path.parse(subPath); + const relativeTestFilePath = path.relative(this.project.testDir, this._test._requireFile); + const parsedRelativeTestFilePath = path.parse(relativeTestFilePath); + const projectNamePathSegment = sanitizeForFilePath(this.project.name); + const snapshotPath = path.resolve(this.config._configDir, this.project.snapshotPathTemplate + .replace(/\{(.)?testDir\}/g, '$1' + this.project.testDir) + .replace(/\{(.)?snapshotDir\}/g, '$1' + this.project.snapshotDir) + .replace(/\{(.)?snapshotSuffix\}/g, this.snapshotSuffix ? '$1' + this.snapshotSuffix : '')) + .replace(/\{(.)?platform\}/g, '$1' + process.platform) + .replace(/\{(.)?projectName\}/g, projectNamePathSegment ? '$1' + projectNamePathSegment : projectNamePathSegment) + .replace(/\{(.)?testFileDir\}/g, '$1' + parsedRelativeTestFilePath.dir) + .replace(/\{(.)?testFileName\}/g, '$1' + parsedRelativeTestFilePath.base) + .replace(/\{(.)?testFilePath\}/g, '$1' + relativeTestFilePath) + .replace(/\{(.)?arg\}/g, '$1' + path.join(parsedSubPath.dir, parsedSubPath.name)) + .replace(/\{(.)?ext\}/g, '$1' + parsedSubPath.ext); + return path.normalize(snapshotPath); } skip(...args: [arg?: any, description?: string]) { diff --git a/packages/playwright-test/src/types.ts b/packages/playwright-test/src/types.ts index 899c2ae567..e4e2ae7a2b 100644 --- a/packages/playwright-test/src/types.ts +++ b/packages/playwright-test/src/types.ts @@ -68,7 +68,6 @@ export interface FullProjectInternal extends FullProjectPublic { _fullConfig: FullConfigInternal; _fullyParallel: boolean; _expect: Project['expect']; - _screenshotsDir: string; _respectGitIgnore: boolean; _setup: string | RegExp | (string | RegExp)[]; } diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index 9a3aa471ca..7a87c0ef4f 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -215,6 +215,84 @@ export interface FullProject { * resolve to `snapshots/a.spec.js-snapshots`. */ snapshotDir: string; + /** + * This configuration option allows to set a string with template values for precise control over snapshot path location. + * + * ```js + * // playwright.config.ts + * import type { PlaywrightTestConfig } from '@playwright/test'; + * + * const config: PlaywrightTestConfig = { + * testDir: './tests', + * snapshotPathTemplate: '{testDir}/__screenshots__/{testFilePath}/{arg}{ext}', + * }; + * + * export default config; + * ``` + * + * The value might include some "tokens" that will be replaced with actual values during test execution. + * + * Consider the following file structure: + * + * ``` + * playwright.config.ts + * tests/ + * └── page/ + * └── page-click.spec.ts + * ``` + * + * And the following `page-click.spec.ts` that uses `toHaveScreenshot()` call: + * + * The list of supported tokens: + * - `{testDir}` - Project's `testDir`. + * - Example: `tests/` + * - `{snapshotDir}` - Project's `snapshotDir`. + * - Example: `tests/` (since `snapshotDir` is not provided in config, it defaults to `testDir`) + * - `{platform}` - The value of `process.platform`. + * - `{snapshotSuffix}` - The value of `testInfo.snapshotSuffix`. + * - `{projectName}` - Project's sanitized name, if any. + * - Example: `undefined`. + * - `{testFileDir}` - Directories in relative path from `testDir` to **test file**. + * - Example: `page/` + * - `{testFileName}` - Test file name with extension. + * - Example: `page-click.spec.ts` + * - `{testFilePath}` - Relative path from `testDir` to **test file** + * - Example: `page/page-click.spec.ts` + * - `{arg}` - Relative snapshot path **without extension**. These come from the arguments passed to the + * `toHaveScreenshot()` and `toMatchSnapshot()` calls; if called without arguments, this will be an auto-generated + * snapshot name. + * - Example: `foo/bar/baz` + * - `{ext}` - snapshot extension (with dots) + * - Example: `.png` + * + * Each token can be preceded with a single character that will be used **only if** this token has non-empty value. + * + * Consider the following config: + * + * ```js + * // playwright.config.ts + * import type { PlaywrightTestConfig } from '@playwright/test'; + * + * const config: PlaywrightTestConfig = { + * snapshotPathTemplate: '__screenshots__{/projectName}/{testFilePath}/{arg}{ext}', + * testMatch: 'example.spec.ts', + * projects: [ + * { use: { browserName: 'firefox' } }, + * { name: 'chromium', use: { browserName: 'chromium' } }, + * ], + * }; + * export default config; + * ``` + * + * In this config: + * 1. First project **does not** have a name, so its snapshots will be stored in + * `/__screenshots__/example.spec.ts/...`. + * 1. Second project **does** have a name, so its snapshots will be stored in + * `/__screenshots__/chromium/example.spec.ts/..`. + * 1. Since `snapshotPathTemplate` resolves to relative path, it will be resolved relative to `configDir`. + * 1. Forward slashes `"/"` can be used as path separators regarding of the platform and work everywhere. + */ + snapshotPathTemplate: string; /** * The output directory for files created during test execution. Defaults to `/test-results`. * @@ -769,6 +847,85 @@ interface TestConfig { */ snapshotDir?: string; + /** + * This configuration option allows to set a string with template values for precise control over snapshot path location. + * + * ```js + * // playwright.config.ts + * import type { PlaywrightTestConfig } from '@playwright/test'; + * + * const config: PlaywrightTestConfig = { + * testDir: './tests', + * snapshotPathTemplate: '{testDir}/__screenshots__/{testFilePath}/{arg}{ext}', + * }; + * + * export default config; + * ``` + * + * The value might include some "tokens" that will be replaced with actual values during test execution. + * + * Consider the following file structure: + * + * ``` + * playwright.config.ts + * tests/ + * └── page/ + * └── page-click.spec.ts + * ``` + * + * And the following `page-click.spec.ts` that uses `toHaveScreenshot()` call: + * + * The list of supported tokens: + * - `{testDir}` - Project's `testDir`. + * - Example: `tests/` + * - `{snapshotDir}` - Project's `snapshotDir`. + * - Example: `tests/` (since `snapshotDir` is not provided in config, it defaults to `testDir`) + * - `{platform}` - The value of `process.platform`. + * - `{snapshotSuffix}` - The value of `testInfo.snapshotSuffix`. + * - `{projectName}` - Project's sanitized name, if any. + * - Example: `undefined`. + * - `{testFileDir}` - Directories in relative path from `testDir` to **test file**. + * - Example: `page/` + * - `{testFileName}` - Test file name with extension. + * - Example: `page-click.spec.ts` + * - `{testFilePath}` - Relative path from `testDir` to **test file** + * - Example: `page/page-click.spec.ts` + * - `{arg}` - Relative snapshot path **without extension**. These come from the arguments passed to the + * `toHaveScreenshot()` and `toMatchSnapshot()` calls; if called without arguments, this will be an auto-generated + * snapshot name. + * - Example: `foo/bar/baz` + * - `{ext}` - snapshot extension (with dots) + * - Example: `.png` + * + * Each token can be preceded with a single character that will be used **only if** this token has non-empty value. + * + * Consider the following config: + * + * ```js + * // playwright.config.ts + * import type { PlaywrightTestConfig } from '@playwright/test'; + * + * const config: PlaywrightTestConfig = { + * snapshotPathTemplate: '__screenshots__{/projectName}/{testFilePath}/{arg}{ext}', + * testMatch: 'example.spec.ts', + * projects: [ + * { use: { browserName: 'firefox' } }, + * { name: 'chromium', use: { browserName: 'chromium' } }, + * ], + * }; + * export default config; + * ``` + * + * In this config: + * 1. First project **does not** have a name, so its snapshots will be stored in + * `/__screenshots__/example.spec.ts/...`. + * 1. Second project **does** have a name, so its snapshots will be stored in + * `/__screenshots__/chromium/example.spec.ts/..`. + * 1. Since `snapshotPathTemplate` resolves to relative path, it will be resolved relative to `configDir`. + * 1. Forward slashes `"/"` can be used as path separators regarding of the platform and work everywhere. + */ + snapshotPathTemplate?: string; + /** * Whether to preserve test output in the * [testConfig.outputDir](https://playwright.dev/docs/api/class-testconfig#test-config-output-dir). Defaults to `'always'`. @@ -1569,6 +1726,10 @@ export interface TestInfo { /** * Absolute path to the snapshot output directory for this specific test. Each test suite gets its own directory so they * cannot conflict. + * + * This property does not account for the + * [testProject.snapshotPathTemplate](https://playwright.dev/docs/api/class-testproject#test-project-snapshot-path-template) + * configuration. */ snapshotDir: string; @@ -4490,6 +4651,85 @@ interface TestProject { */ snapshotDir?: string; + /** + * This configuration option allows to set a string with template values for precise control over snapshot path location. + * + * ```js + * // playwright.config.ts + * import type { PlaywrightTestConfig } from '@playwright/test'; + * + * const config: PlaywrightTestConfig = { + * testDir: './tests', + * snapshotPathTemplate: '{testDir}/__screenshots__/{testFilePath}/{arg}{ext}', + * }; + * + * export default config; + * ``` + * + * The value might include some "tokens" that will be replaced with actual values during test execution. + * + * Consider the following file structure: + * + * ``` + * playwright.config.ts + * tests/ + * └── page/ + * └── page-click.spec.ts + * ``` + * + * And the following `page-click.spec.ts` that uses `toHaveScreenshot()` call: + * + * The list of supported tokens: + * - `{testDir}` - Project's `testDir`. + * - Example: `tests/` + * - `{snapshotDir}` - Project's `snapshotDir`. + * - Example: `tests/` (since `snapshotDir` is not provided in config, it defaults to `testDir`) + * - `{platform}` - The value of `process.platform`. + * - `{snapshotSuffix}` - The value of `testInfo.snapshotSuffix`. + * - `{projectName}` - Project's sanitized name, if any. + * - Example: `undefined`. + * - `{testFileDir}` - Directories in relative path from `testDir` to **test file**. + * - Example: `page/` + * - `{testFileName}` - Test file name with extension. + * - Example: `page-click.spec.ts` + * - `{testFilePath}` - Relative path from `testDir` to **test file** + * - Example: `page/page-click.spec.ts` + * - `{arg}` - Relative snapshot path **without extension**. These come from the arguments passed to the + * `toHaveScreenshot()` and `toMatchSnapshot()` calls; if called without arguments, this will be an auto-generated + * snapshot name. + * - Example: `foo/bar/baz` + * - `{ext}` - snapshot extension (with dots) + * - Example: `.png` + * + * Each token can be preceded with a single character that will be used **only if** this token has non-empty value. + * + * Consider the following config: + * + * ```js + * // playwright.config.ts + * import type { PlaywrightTestConfig } from '@playwright/test'; + * + * const config: PlaywrightTestConfig = { + * snapshotPathTemplate: '__screenshots__{/projectName}/{testFilePath}/{arg}{ext}', + * testMatch: 'example.spec.ts', + * projects: [ + * { use: { browserName: 'firefox' } }, + * { name: 'chromium', use: { browserName: 'chromium' } }, + * ], + * }; + * export default config; + * ``` + * + * In this config: + * 1. First project **does not** have a name, so its snapshots will be stored in + * `/__screenshots__/example.spec.ts/...`. + * 1. Second project **does** have a name, so its snapshots will be stored in + * `/__screenshots__/chromium/example.spec.ts/..`. + * 1. Since `snapshotPathTemplate` resolves to relative path, it will be resolved relative to `configDir`. + * 1. Forward slashes `"/"` can be used as path separators regarding of the platform and work everywhere. + */ + snapshotPathTemplate?: string; + /** * The output directory for files created during test execution. Defaults to `/test-results`. * diff --git a/tests/installation/docker-integration.spec.ts b/tests/installation/docker-integration.spec.ts index 7cb64cf40e..60165b4a7b 100755 --- a/tests/installation/docker-integration.spec.ts +++ b/tests/installation/docker-integration.spec.ts @@ -93,17 +93,6 @@ test.describe('installed image', () => { } }); - test('screenshots should use __screenshots__ folder', async ({ exec, tmpWorkspace }) => { - await exec('npm i --foreground-scripts @playwright/test'); - await exec('npx playwright test docker.spec.js --grep screenshot --browser all', { - expectToExitWithError: true, - env: { PLAYWRIGHT_DOCKER: '1' }, - }); - await expect(path.join(tmpWorkspace, '__screenshots__', 'firefox', 'docker.spec.js', 'img.png')).toExistOnFS(); - await expect(path.join(tmpWorkspace, '__screenshots__', 'chromium', 'docker.spec.js', 'img.png')).toExistOnFS(); - await expect(path.join(tmpWorkspace, '__screenshots__', 'webkit', 'docker.spec.js', 'img.png')).toExistOnFS(); - }); - test('port forwarding works', async ({ exec, tmpWorkspace }) => { await exec('npm i --foreground-scripts @playwright/test'); const TEST_PORT = 8425; diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts index 48c5d79e1c..157c7428f5 100644 --- a/tests/playwright-test/reporter-html.spec.ts +++ b/tests/playwright-test/reporter-html.spec.ts @@ -158,9 +158,8 @@ test('should include multiple image diffs', async ({ runInlineTest, page, showRe const result = await runInlineTest({ 'playwright.config.ts': ` - process.env.PWTEST_USE_SCREENSHOTS_DIR = '1'; module.exports = { - screenshotsDir: '__screenshots__', + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', use: { viewport: { width: ${IMG_WIDTH}, height: ${IMG_HEIGHT} }} }; `, diff --git a/tests/playwright-test/snapshot-path-template.spec.ts b/tests/playwright-test/snapshot-path-template.spec.ts new file mode 100644 index 0000000000..fc69fc36ea --- /dev/null +++ b/tests/playwright-test/snapshot-path-template.spec.ts @@ -0,0 +1,131 @@ +/** + * Copyright Microsoft Corporation. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import path from 'path'; +import fs from 'fs'; +import { test, expect, stripAnsi } from './playwright-test-fixtures'; + +async function getSnapshotPaths(runInlineTest, testInfo, playwrightConfig, pathArgs) { + const SEPARATOR = '==== 8< ---- '; + const result = await runInlineTest({ + 'playwright.config.js': ` + module.exports = ${JSON.stringify(playwrightConfig, null, 2)} + `, + 'a/b/c/d.spec.js': ` + pwt.test('test', async ({ page }, testInfo) => { + console.log([ + ${JSON.stringify(SEPARATOR)}, + testInfo.project.name, + ${JSON.stringify(SEPARATOR)}, + testInfo.snapshotPath(...${JSON.stringify(pathArgs)}), + ${JSON.stringify(SEPARATOR)}, + ].join('')); + }); + ` + }, { workers: 1 }); + expect(result.exitCode).toBe(0); + const allSegments = stripAnsi(result.output).split(SEPARATOR); + const projToSnapshot = {}; + for (let i = 1; i < allSegments.length; i += 3) + projToSnapshot[allSegments[i]] = path.relative(testInfo.outputDir, allSegments[i + 1]); + return projToSnapshot; +} + +test('tokens should expand property', async ({ runInlineTest }, testInfo) => { + const snapshotPath = await getSnapshotPaths(runInlineTest, testInfo, { + projects: [{ + name: 'proj1', + snapshotPathTemplate: '{projectName}', + }, { + name: 'proj 2', + snapshotPathTemplate: '{-projectName}', + }, { + name: 'proj3', + snapshotPathTemplate: 'foo{/projectName}', + }, { + snapshotPathTemplate: '{/projectName}', + }, { + name: 'platform', + snapshotPathTemplate: '{platform}', + }, { + name: 'extension', + snapshotPathTemplate: 'mysnapshot{ext}', + }, { + name: 'arg', + snapshotPathTemplate: 'bar/{arg}', + }, { + name: 'testFileDir', + snapshotPathTemplate: '{testFileDir}', + }, { + name: 'testFilePath', + snapshotPathTemplate: '{testFilePath}', + }, { + name: 'testFileName', + snapshotPathTemplate: '{testFileName}', + }, { + name: 'snapshotDir', + snapshotDir: './a-snapshot-dir', + snapshotPathTemplate: '{snapshotDir}.png', + }, { + name: 'snapshotSuffix', + snapshotPathTemplate: '{-snapshotSuffix}', + }], + }, ['foo.png']); + expect.soft(snapshotPath['proj1']).toBe('proj1'); + expect.soft(snapshotPath['proj 2']).toBe('-proj-2'); + expect.soft(snapshotPath['proj3']).toBe(path.join('foo', 'proj3')); + expect.soft(snapshotPath['']).toBe(''); + expect.soft(snapshotPath['platform']).toBe(process.platform); + expect.soft(snapshotPath['extension']).toBe('mysnapshot.png'); + expect.soft(snapshotPath['arg']).toBe(path.join('bar', 'foo')); + expect.soft(snapshotPath['testFileDir']).toBe(path.join('a', 'b', 'c')); + expect.soft(snapshotPath['testFilePath']).toBe(path.join('a', 'b', 'c', 'd.spec.js')); + expect.soft(snapshotPath['testFileName']).toBe('d.spec.js'); + expect.soft(snapshotPath['snapshotDir']).toBe('a-snapshot-dir.png'); + expect.soft(snapshotPath['snapshotSuffix']).toBe('-' + process.platform); +}); + +test('args array should work', async ({ runInlineTest }, testInfo) => { + const snapshotPath = await getSnapshotPaths(runInlineTest, testInfo, { + projects: [{ + name: 'proj', + snapshotPathTemplate: '{ext}{arg}', + }], + }, ['foo', 'bar', 'baz.jpeg']); + expect.soft(snapshotPath['proj']).toBe(path.join('.jpegfoo', 'bar', 'baz')); +}); + +test('arg should receive default arg', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'playwright.config.js': ` + module.exports = { + snapshotPathTemplate: '__screenshots__/{arg}{ext}', + } + `, + 'a.spec.js': ` + pwt.test('is a test', async ({ page }) => { + await expect(page).toHaveScreenshot(); + }); + ` + }, { 'update-snapshots': true }); + + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + const snapshotOutputPath = testInfo.outputPath('__screenshots__/is-a-test-1.png'); + expect(result.output).toContain(`${snapshotOutputPath} is missing in snapshots, writing actual`); + expect(fs.existsSync(snapshotOutputPath)).toBe(true); +}); + diff --git a/tests/playwright-test/test-output-dir.spec.ts b/tests/playwright-test/test-output-dir.spec.ts index 3749954284..84f07a58e5 100644 --- a/tests/playwright-test/test-output-dir.spec.ts +++ b/tests/playwright-test/test-output-dir.spec.ts @@ -244,36 +244,6 @@ test('should include path option in snapshot', async ({ runInlineTest }) => { expect(result.output).toContain('my-test.spec.js-snapshots/test/path/bar-foo-suffix.txt'); }); -test('should error if snapshotPath is resolved to outside of parent', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'helper.ts': ` - export const test = pwt.test.extend({ - auto: [ async ({}, run, testInfo) => { - testInfo.snapshotSuffix = 'suffix'; - await run(); - }, { auto: true } ] - }); - `, - 'playwright.config.ts': ` - module.exports = { projects: [ - { name: 'foo' }, - ] }; - `, - 'my-test.spec.js': ` - const { test } = require('./helper'); - test('test with parent path', async ({}, testInfo) => { - console.log(testInfo.snapshotPath('..', 'test', 'path', 'bar.txt').replace(/\\\\/g, '/')); - }); - `, - }); - - expect(result.exitCode).toBe(1); - expect(result.results[0].status).toBe('failed'); - expect(result.output).toContain('The snapshotPath is not allowed outside of the parent directory. Please fix the defined path.'); - const badPath = path.join('..', 'test', 'path', 'bar-foo-suffix.txt'); - expect(result.output).toContain(`snapshotPath: ${badPath}`); -}); - test('should error if outputPath is resolved to outside of parent', async ({ runInlineTest }) => { const result = await runInlineTest({ 'helper.ts': ` diff --git a/tests/playwright-test/to-have-screenshot.spec.ts b/tests/playwright-test/to-have-screenshot.spec.ts index 7896bba802..3536df2531 100644 --- a/tests/playwright-test/to-have-screenshot.spec.ts +++ b/tests/playwright-test/to-have-screenshot.spec.ts @@ -131,29 +131,11 @@ test('should fail with proper error when unsupported argument is given', async ( expect(stripAnsi(result.output)).toContain(`Expected options.clip.width not to be 0`); }); -test('should use match snapshot paths by default', async ({ runInlineTest }, testInfo) => { - const result = await runInlineTest({ - // The helper function `playwrightConfig` set PWTEST_USE_SCREENSHOTS env variable. - // Provide default config manually instead. - 'playwright.config.js': ` - module.exports = {}; - `, - 'a.spec.js': ` - pwt.test('is a test', async ({ page }, testInfo) => { - testInfo.snapshotSuffix = ''; - await expect(page).toHaveScreenshot('snapshot.png'); - }); - ` - }, { 'update-snapshots': true }); - expect(result.exitCode).toBe(0); - - const snapshotOutputPath = testInfo.outputPath('a.spec.js-snapshots', 'snapshot.png'); - expect(fs.existsSync(snapshotOutputPath)).toBe(true); -}); - test('should have scale:css by default', async ({ runInlineTest }, testInfo) => { const result = await runInlineTest({ - ...playwrightConfig({ screenshotsDir: '__screenshots__' }), + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + }), 'a.spec.js': ` pwt.test('is a test', async ({ browser }) => { const context = await browser.newContext({ @@ -175,7 +157,7 @@ test('should have scale:css by default', async ({ runInlineTest }, testInfo) => test('should ignore non-documented options in toHaveScreenshot config', async ({ runInlineTest }, testInfo) => { const result = await runInlineTest({ ...playwrightConfig({ - screenshotsDir: '__screenshots__', + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', expect: { toHaveScreenshot: { clip: { x: 0, y: 0, width: 10, height: 10 }, @@ -194,36 +176,6 @@ test('should ignore non-documented options in toHaveScreenshot config', async ({ expect(comparePNGs(fs.readFileSync(snapshotOutputPath), whiteImage)).toBe(null); }); -test('screenshotPath should include platform and project name by default', async ({ runInlineTest }, testInfo) => { - const PROJECT_NAME = 'woof-woof'; - const result = await runInlineTest({ - ...playwrightConfig({ - projects: [{ - name: PROJECT_NAME, - }], - }), - 'a.spec.js': ` - pwt.test('is a test', async ({ page }, testInfo) => { - await pwt.expect(page).toHaveScreenshot('snapshot.png'); - }); - `, - 'foo/b.spec.js': ` - pwt.test('is a test', async ({ page }, testInfo) => { - await pwt.expect(page).toHaveScreenshot('snapshot.png'); - }); - `, - 'foo/bar/baz/c.spec.js': ` - pwt.test('is a test', async ({ page }, testInfo) => { - await pwt.expect(page).toHaveScreenshot('snapshot.png'); - }); - `, - }, { 'update-snapshots': true }); - expect(result.exitCode).toBe(0); - expect(fs.existsSync(testInfo.outputPath('__screenshots__', process.platform, PROJECT_NAME, 'a.spec.js', 'snapshot.png'))).toBeTruthy(); - expect(fs.existsSync(testInfo.outputPath('__screenshots__', process.platform, PROJECT_NAME, 'foo', 'b.spec.js', 'snapshot.png'))).toBeTruthy(); - expect(fs.existsSync(testInfo.outputPath('__screenshots__', process.platform, PROJECT_NAME, 'foo', 'bar', 'baz', 'c.spec.js', 'snapshot.png'))).toBeTruthy(); -}); - test('should report toHaveScreenshot step with expectation name in title', async ({ runInlineTest }) => { const result = await runInlineTest({ 'reporter.ts': ` @@ -260,7 +212,7 @@ test('should not fail when racing with navigation', async ({ runInlineTest }, te const infiniteAnimationURL = pathToFileURL(path.join(__dirname, '../assets/rotate-z.html')); const result = await runInlineTest({ ...playwrightConfig({ - screenshotsDir: '__screenshots__', + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', }), '__screenshots__/a.spec.js/snapshot.png': createImage(10, 10, 255, 0, 0), 'a.spec.js': ` @@ -282,7 +234,9 @@ test('should not fail when racing with navigation', async ({ runInlineTest }, te test('should successfully screenshot a page with infinite animation with disableAnimation: true', async ({ runInlineTest }, testInfo) => { const infiniteAnimationURL = pathToFileURL(path.join(__dirname, '../assets/rotate-z.html')); const result = await runInlineTest({ - ...playwrightConfig({ screenshotsDir: '__screenshots__' }), + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + }), 'a.spec.js': ` pwt.test('is a test', async ({ page }) => { await page.goto('${infiniteAnimationURL}'); @@ -298,7 +252,9 @@ test('should successfully screenshot a page with infinite animation with disable test('should support clip option for page', async ({ runInlineTest }, testInfo) => { const result = await runInlineTest({ - ...playwrightConfig({ screenshotsDir: '__screenshots__' }), + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + }), '__screenshots__/a.spec.js/snapshot.png': createImage(50, 50, 255, 255, 255), 'a.spec.js': ` pwt.test('is a test', async ({ page }) => { @@ -314,7 +270,9 @@ test('should support clip option for page', async ({ runInlineTest }, testInfo) test('should support omitBackground option for locator', async ({ runInlineTest }, testInfo) => { const result = await runInlineTest({ - ...playwrightConfig({ screenshotsDir: '__screenshots__' }), + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + }), 'a.spec.js': ` pwt.test('is a test', async ({ page }) => { await page.evaluate(() => { @@ -344,7 +302,7 @@ test('should fail to screenshot an element with infinite animation', async ({ ru const infiniteAnimationURL = pathToFileURL(path.join(__dirname, '../assets/rotate-z.html')); const result = await runInlineTest({ ...playwrightConfig({ - screenshotsDir: '__screenshots__', + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', projects: [{ expect: { toHaveScreenshot: { @@ -374,7 +332,7 @@ test('should fail to screenshot an element that keeps moving', async ({ runInlin const infiniteAnimationURL = pathToFileURL(path.join(__dirname, '../assets/rotate-z.html')); const result = await runInlineTest({ ...playwrightConfig({ - screenshotsDir: '__screenshots__', + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', expect: { toHaveScreenshot: { animations: 'allow', @@ -399,7 +357,9 @@ test('should fail to screenshot an element that keeps moving', async ({ runInlin test('should generate default name', async ({ runInlineTest }, testInfo) => { const result = await runInlineTest({ - ...playwrightConfig({ screenshotsDir: '__screenshots__' }), + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + }), 'a.spec.js': ` pwt.test('is a test', async ({ page }) => { await expect(page).toHaveScreenshot(); @@ -457,7 +417,9 @@ test('should compile with different option combinations', async ({ runTSC }) => test('should fail when screenshot is different size', async ({ runInlineTest }) => { const result = await runInlineTest({ - ...playwrightConfig({ screenshotsDir: '__screenshots__' }), + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + }), '__screenshots__/a.spec.js/snapshot.png': createImage(22, 33), 'a.spec.js': ` pwt.test('is a test', async ({ page }) => { @@ -473,7 +435,9 @@ test('should fail when screenshot is different size', async ({ runInlineTest }) test('should fail when given non-png snapshot name', async ({ runInlineTest }) => { const result = await runInlineTest({ - ...playwrightConfig({}), + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + }), 'a.spec.js': ` pwt.test('is a test', async ({ page }) => { await expect(page).toHaveScreenshot('snapshot.jpeg'); @@ -499,7 +463,9 @@ test('should fail when given buffer', async ({ runInlineTest }) => { test('should fail when screenshot is different pixels', async ({ runInlineTest }) => { const result = await runInlineTest({ - ...playwrightConfig({ screenshotsDir: '__screenshots__' }), + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + }), '__screenshots__/a.spec.js/snapshot.png': paintBlackPixels(whiteImage, 12345), 'a.spec.js': ` pwt.test('is a test', async ({ page }) => { @@ -518,7 +484,9 @@ test('should fail when screenshot is different pixels', async ({ runInlineTest } test('doesn\'t create comparison artifacts in an output folder for passed negated snapshot matcher', async ({ runInlineTest }, testInfo) => { const result = await runInlineTest({ - ...playwrightConfig({ screenshotsDir: '__screenshots__' }), + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + }), '__screenshots__/a.spec.js/snapshot.png': blueImage, 'a.spec.js': ` pwt.test('is a test', async ({ page }) => { @@ -539,7 +507,9 @@ test('doesn\'t create comparison artifacts in an output folder for passed negate test('should fail on same snapshots with negate matcher', async ({ runInlineTest }) => { const result = await runInlineTest({ - ...playwrightConfig({ screenshotsDir: '__screenshots__' }), + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + }), '__screenshots__/a.spec.js/snapshot.png': whiteImage, 'a.spec.js': ` pwt.test('is a test', async ({ page }) => { @@ -555,7 +525,9 @@ test('should fail on same snapshots with negate matcher', async ({ runInlineTest test('should not fail if --ignore-snapshots is passed', async ({ runInlineTest }) => { const result = await runInlineTest({ - ...playwrightConfig({ screenshotsDir: '__screenshots__' }), + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + }), '__screenshots__/a.spec.js/snapshot.png': redImage, 'a.spec.js': ` pwt.test('is a test', async ({ page }) => { @@ -569,7 +541,9 @@ test('should not fail if --ignore-snapshots is passed', async ({ runInlineTest } test('should write missing expectations locally twice and continue', async ({ runInlineTest }, testInfo) => { const result = await runInlineTest({ - ...playwrightConfig({ screenshotsDir: '__screenshots__' }), + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + }), 'a.spec.js': ` pwt.test('is a test', async ({ page }) => { await expect(page).toHaveScreenshot('snapshot.png'); @@ -599,7 +573,9 @@ test('should write missing expectations locally twice and continue', async ({ ru test('shouldn\'t write missing expectations locally for negated matcher', async ({ runInlineTest }, testInfo) => { const result = await runInlineTest({ - ...playwrightConfig({ screenshotsDir: '__screenshots__' }), + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + }), 'a.spec.js': ` pwt.test('is a test', async ({ page }) => { await expect(page).not.toHaveScreenshot('snapshot.png'); @@ -615,7 +591,9 @@ test('shouldn\'t write missing expectations locally for negated matcher', async test('should update snapshot with the update-snapshots flag', async ({ runInlineTest }, testInfo) => { const result = await runInlineTest({ - ...playwrightConfig({ screenshotsDir: '__screenshots__' }), + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + }), '__screenshots__/a.spec.js/snapshot.png': blueImage, 'a.spec.js': ` pwt.test('is a test', async ({ page }) => { @@ -633,7 +611,9 @@ test('should update snapshot with the update-snapshots flag', async ({ runInline test('shouldn\'t update snapshot with the update-snapshots flag for negated matcher', async ({ runInlineTest }, testInfo) => { const EXPECTED_SNAPSHOT = blueImage; const result = await runInlineTest({ - ...playwrightConfig({ screenshotsDir: '__screenshots__' }), + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + }), '__screenshots__/a.spec.js/snapshot.png': EXPECTED_SNAPSHOT, 'a.spec.js': ` pwt.test('is a test', async ({ page }) => { @@ -649,7 +629,9 @@ test('shouldn\'t update snapshot with the update-snapshots flag for negated matc test('should silently write missing expectations locally with the update-snapshots flag', async ({ runInlineTest }, testInfo) => { const result = await runInlineTest({ - ...playwrightConfig({ screenshotsDir: '__screenshots__' }), + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + }), 'a.spec.js': ` pwt.test('is a test', async ({ page }) => { await expect(page).toHaveScreenshot('snapshot.png'); @@ -666,7 +648,9 @@ test('should silently write missing expectations locally with the update-snapsho test('should not write missing expectations locally with the update-snapshots flag for negated matcher', async ({ runInlineTest }, testInfo) => { const result = await runInlineTest({ - ...playwrightConfig({ screenshotsDir: '__screenshots__' }), + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + }), 'a.spec.js': ` pwt.test('is a test', async ({ page }) => { await expect(page).not.toHaveScreenshot('snapshot.png'); @@ -682,7 +666,9 @@ test('should not write missing expectations locally with the update-snapshots fl test('should match multiple snapshots', async ({ runInlineTest }) => { const result = await runInlineTest({ - ...playwrightConfig({ screenshotsDir: '__screenshots__' }), + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + }), '__screenshots__/a.spec.js/red.png': redImage, '__screenshots__/a.spec.js/green.png': greenImage, '__screenshots__/a.spec.js/blue.png': blueImage, @@ -708,7 +694,9 @@ test('should match multiple snapshots', async ({ runInlineTest }) => { test('should use provided name', async ({ runInlineTest }) => { const result = await runInlineTest({ - ...playwrightConfig({ screenshotsDir: '__screenshots__' }), + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + }), '__screenshots__/a.spec.js/provided.png': whiteImage, 'a.spec.js': ` pwt.test('is a test', async ({ page }) => { @@ -721,7 +709,9 @@ test('should use provided name', async ({ runInlineTest }) => { test('should use provided name via options', async ({ runInlineTest }) => { const result = await runInlineTest({ - ...playwrightConfig({ screenshotsDir: '__screenshots__' }), + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + }), '__screenshots__/a.spec.js/provided.png': whiteImage, 'a.spec.js': ` pwt.test('is a test', async ({ page }) => { @@ -737,7 +727,9 @@ test('should respect maxDiffPixels option', async ({ runInlineTest }) => { const EXPECTED_SNAPSHOT = paintBlackPixels(whiteImage, BAD_PIXELS); expect((await runInlineTest({ - ...playwrightConfig({ screenshotsDir: '__screenshots__' }), + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + }), '__screenshots__/a.spec.js/snapshot.png': EXPECTED_SNAPSHOT, 'a.spec.js': ` pwt.test('is a test', async ({ page }) => { @@ -747,7 +739,9 @@ test('should respect maxDiffPixels option', async ({ runInlineTest }) => { })).exitCode, 'make sure default comparison fails').toBe(1); expect((await runInlineTest({ - ...playwrightConfig({ screenshotsDir: '__screenshots__' }), + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + }), '__screenshots__/a.spec.js/snapshot.png': EXPECTED_SNAPSHOT, 'a.spec.js': ` pwt.test('is a test', async ({ page }) => { @@ -767,7 +761,7 @@ test('should respect maxDiffPixels option', async ({ runInlineTest }) => { }, projects: [ { - screenshotsDir: '__screenshots__', + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', }, ], }), @@ -785,7 +779,9 @@ test('should not update screenshot that matches with maxDiffPixels option when - const EXPECTED_SNAPSHOT = paintBlackPixels(whiteImage, BAD_PIXELS); const result = await runInlineTest({ - ...playwrightConfig({ screenshotsDir: '__screenshots__' }), + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + }), '__screenshots__/a.spec.js/snapshot.png': EXPECTED_SNAPSHOT, 'a.spec.js': ` pwt.test('is a test', async ({ page }) => { @@ -811,7 +807,9 @@ test('should satisfy both maxDiffPixelRatio and maxDiffPixels', async ({ runInli const EXPECTED_SNAPSHOT = paintBlackPixels(whiteImage, BAD_COUNT); expect((await runInlineTest({ - ...playwrightConfig({ screenshotsDir: '__screenshots__' }), + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + }), '__screenshots__/a.spec.js/snapshot.png': EXPECTED_SNAPSHOT, 'a.spec.js': ` pwt.test('is a test', async ({ page }) => { @@ -821,7 +819,9 @@ test('should satisfy both maxDiffPixelRatio and maxDiffPixels', async ({ runInli })).exitCode, 'make sure default comparison fails').toBe(1); expect((await runInlineTest({ - ...playwrightConfig({ screenshotsDir: '__screenshots__' }), + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + }), '__screenshots__/a.spec.js/snapshot.png': EXPECTED_SNAPSHOT, 'a.spec.js': ` pwt.test('is a test', async ({ page }) => { @@ -835,7 +835,9 @@ test('should satisfy both maxDiffPixelRatio and maxDiffPixels', async ({ runInli })).exitCode, 'make sure it fails when maxDiffPixels < actualBadPixels < maxDiffPixelRatio').toBe(1); expect((await runInlineTest({ - ...playwrightConfig({ screenshotsDir: '__screenshots__' }), + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + }), '__screenshots__/a.spec.js/snapshot.png': EXPECTED_SNAPSHOT, 'a.spec.js': ` pwt.test('is a test', async ({ page }) => { @@ -849,7 +851,9 @@ test('should satisfy both maxDiffPixelRatio and maxDiffPixels', async ({ runInli })).exitCode, 'make sure it fails when maxDiffPixelRatio < actualBadPixels < maxDiffPixels').toBe(1); expect((await runInlineTest({ - ...playwrightConfig({ screenshotsDir: '__screenshots__' }), + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + }), '__screenshots__/a.spec.js/snapshot.png': EXPECTED_SNAPSHOT, 'a.spec.js': ` pwt.test('is a test', async ({ page }) => { @@ -868,7 +872,9 @@ test('should respect maxDiffPixelRatio option', async ({ runInlineTest }) => { const EXPECTED_SNAPSHOT = paintBlackPixels(whiteImage, BAD_PIXELS); expect((await runInlineTest({ - ...playwrightConfig({ screenshotsDir: '__screenshots__' }), + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + }), '__screenshots__/a.spec.js/snapshot.png': EXPECTED_SNAPSHOT, 'a.spec.js': ` pwt.test('is a test', async ({ page }) => { @@ -878,7 +884,9 @@ test('should respect maxDiffPixelRatio option', async ({ runInlineTest }) => { })).exitCode, 'make sure default comparison fails').toBe(1); expect((await runInlineTest({ - ...playwrightConfig({ screenshotsDir: '__screenshots__' }), + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + }), '__screenshots__/a.spec.js/snapshot.png': EXPECTED_SNAPSHOT, 'a.spec.js': ` pwt.test('is a test', async ({ page }) => { @@ -897,7 +905,7 @@ test('should respect maxDiffPixelRatio option', async ({ runInlineTest }) => { }, }, projects: [{ - screenshotsDir: '__screenshots__', + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', }], }), '__screenshots__/a.spec.js/snapshot.png': EXPECTED_SNAPSHOT, @@ -938,7 +946,9 @@ test('should throw for invalid maxDiffPixelRatio values', async ({ runInlineTest test('should attach expected/actual and no diff when sizes are different', async ({ runInlineTest }, testInfo) => { const result = await runInlineTest({ - ...playwrightConfig({ screenshotsDir: '__screenshots__' }), + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + }), '__screenshots__/a.spec.js/snapshot.png': createImage(2, 2), 'a.spec.js': ` pwt.test.afterEach(async ({}, testInfo) => { @@ -974,7 +984,7 @@ test('should fail with missing expectations and retries', async ({ runInlineTest const result = await runInlineTest({ ...playwrightConfig({ retries: 1, - screenshotsDir: '__screenshots__' + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', }), 'a.spec.js': ` pwt.test('is a test', async ({ page }) => { @@ -995,7 +1005,7 @@ test('should update expectations with retries', async ({ runInlineTest }, testIn const result = await runInlineTest({ ...playwrightConfig({ retries: 1, - screenshotsDir: '__screenshots__' + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', }), 'a.spec.js': ` pwt.test('is a test', async ({ page }) => { @@ -1015,7 +1025,6 @@ test('should update expectations with retries', async ({ runInlineTest }, testIn function playwrightConfig(obj: any) { return { 'playwright.config.js': ` - process.env.PWTEST_USE_SCREENSHOTS_DIR = '1'; module.exports = ${JSON.stringify(obj, null, 2)} `, }; diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 68de70bc65..129cc0b37e 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -43,6 +43,7 @@ export interface FullProject { metadata: Metadata; name: string; snapshotDir: string; + snapshotPathTemplate: string; outputDir: string; repeatEach: number; retries: number;