From 66f709663e66c5e80b46a3ee94dae9e79908803f Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Fri, 22 Nov 2024 14:53:29 -0800 Subject: [PATCH 1/5] fix(webkit): do not auto play audio without user gesture (#33734) --- packages/playwright-core/browsers.json | 2 +- tests/library/capabilities.spec.ts | 35 ++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/packages/playwright-core/browsers.json b/packages/playwright-core/browsers.json index b1d8b9487e..e4b95c9a59 100644 --- a/packages/playwright-core/browsers.json +++ b/packages/playwright-core/browsers.json @@ -33,7 +33,7 @@ }, { "name": "webkit", - "revision": "2108", + "revision": "2110", "installByDefault": true, "revisionOverrides": { "debian11-x64": "2105", diff --git a/tests/library/capabilities.spec.ts b/tests/library/capabilities.spec.ts index 116784854f..7526359c20 100644 --- a/tests/library/capabilities.spec.ts +++ b/tests/library/capabilities.spec.ts @@ -428,3 +428,38 @@ it('should not crash when clicking a label with a ', { const fileChooser = await fileChooserPromise; expect(fileChooser.page()).toBe(page); }); + +it('should not auto play audio', { + annotation: { + type: 'issue', + description: 'https://github.com/microsoft/playwright/issues/33590' + } +}, async ({ page, browserName, isWindows }) => { + it.fixme(browserName === 'webkit' && isWindows); + await page.route('**/*', async route => { + await route.fulfill({ + status: 200, + contentType: 'text/html', + body: ` + + +
+ `, + }); + }); + await page.goto('http://127.0.0.1/audio.html'); + await expect(page.locator('#log')).toHaveText('State: suspended'); +}); From 66d9f3acbe2e327d5c65f72bfb3aebcc6c46801f Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Fri, 22 Nov 2024 17:41:31 -0800 Subject: [PATCH 2/5] chore: introduce update-snapshots=changed (#33735) --- docs/src/test-api/class-fullconfig.md | 2 +- docs/src/test-api/class-testconfig.md | 7 +- docs/src/test-cli-js.md | 63 ++++++++------- .../playwright-core/src/utils/comparators.ts | 2 +- packages/playwright/src/common/ipc.ts | 2 +- .../src/matchers/toMatchAriaSnapshot.ts | 13 ++-- .../src/matchers/toMatchSnapshot.ts | 52 +++++++++---- packages/playwright/src/program.ts | 15 +++- packages/playwright/src/runner/rebase.ts | 16 ++-- packages/playwright/types/test.d.ts | 11 +-- .../to-have-screenshot.spec.ts | 77 +++++++++++++++++++ .../update-aria-snapshot.spec.ts | 66 ++++++++++++++++ 12 files changed, 255 insertions(+), 71 deletions(-) diff --git a/docs/src/test-api/class-fullconfig.md b/docs/src/test-api/class-fullconfig.md index e6437ab314..81e102436c 100644 --- a/docs/src/test-api/class-fullconfig.md +++ b/docs/src/test-api/class-fullconfig.md @@ -114,7 +114,7 @@ See [`property: TestConfig.shard`]. ## property: FullConfig.updateSnapshots * since: v1.10 -- type: <[UpdateSnapshots]<"all"|"none"|"missing">> +- type: <[UpdateSnapshots]<"all"|"changed"|"missing"|"none">> See [`property: TestConfig.updateSnapshots`]. diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index cd70b21b70..d9641128bc 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -570,12 +570,13 @@ export default defineConfig({ ## property: TestConfig.updateSnapshots * since: v1.10 -- type: ?<[UpdateSnapshots]<"all"|"none"|"missing">> +- type: ?<[UpdateSnapshots]<"all"|"changed"|"missing"|"none">> Whether to update expected snapshots with the actual results produced by the test run. Defaults to `'missing'`. -* `'all'` - All tests that are executed will update snapshots that did not match. Matching snapshots will not be updated. -* `'none'` - No snapshots are updated. +* `'all'` - All tests that are executed will update snapshots. +* `'changed'` - All tests that are executed will update snapshots that did not match. Matching snapshots will not be updated. * `'missing'` - Missing snapshots are created, for example when authoring a new test and running it for the first time. This is the default. +* `'none'` - No snapshots are updated. Learn more about [snapshots](../test-snapshots.md). diff --git a/docs/src/test-cli-js.md b/docs/src/test-cli-js.md index 005452138a..e91c091fb4 100644 --- a/docs/src/test-cli-js.md +++ b/docs/src/test-cli-js.md @@ -76,33 +76,40 @@ Here are the most common options available in the command line. Complete set of Playwright Test options is available in the [configuration file](./test-use-options.md). Following options can be passed to a command line and take priority over the configuration file: + + | Option | Description | | :- | :- | -| Non-option arguments | Each argument is treated as a regular expression matched against the full test file path. Only tests from the files matching the pattern will be executed. Special symbols like `$` or `*` should be escaped with `\`. In many shells/terminals you may need to quote the arguments. | -| `-c ` or `--config `| Configuration file. If not passed, defaults to `playwright.config.ts` or `playwright.config.js` in the current directory. | -| `--debug`| Run tests with Playwright Inspector. Shortcut for `PWDEBUG=1` environment variable and `--timeout=0 --max-failures=1 --headed --workers=1` options.| -| `--fail-on-flaky-tests` | Fails test runs that contain flaky tests. By default flaky tests count as successes. | -| `--forbid-only` | Whether to disallow `test.only`. Useful on CI.| -| `--global-timeout ` | Total timeout for the whole test run in milliseconds. By default, there is no global timeout. Learn more about [various timeouts](./test-timeouts.md).| -| `-g ` or `--grep ` | Only run tests matching this regular expression. For example, this will run `'should add to cart'` when passed `-g "add to cart"`. The regular expression will be tested against the string that consists of the project name, test file name, `test.describe` titles if any, test title and all test tags, separated by spaces, e.g. `chromium my-test.spec.ts my-suite my-test @smoke`. The filter does not apply to the tests from dependency projects, i.e. Playwright will still run all tests from [project dependencies](./test-projects.md#dependencies). | -| `--grep-invert ` | Only run tests **not** matching this regular expression. The opposite of `--grep`. The filter does not apply to the tests from dependency projects, i.e. Playwright will still run all tests from [project dependencies](./test-projects.md#dependencies).| -| `--headed` | Run tests in headed browsers. Useful for debugging. | -| `--ignore-snapshots` | Whether to ignore [snapshots](./test-snapshots.md). Use this when snapshot expectations are known to be different, e.g. running tests on Linux against Windows screenshots. | -| `--last-failed` | Only re-run the failures.| -| `--list` | list all the tests, but do not run them.| -| `--max-failures ` or `-x`| Stop after the first `N` test failures. Passing `-x` stops after the first failure.| -| `--no-deps` | Ignore the dependencies between projects and behave as if they were not specified. | -| `--output ` | Directory for artifacts produced by tests, defaults to `test-results`. | -| `--only-changed [ref]` | Only run test files that have been changed between working tree and "ref". Defaults to running all uncommitted changes with ref=HEAD. Only supports Git. | -| `--pass-with-no-tests` | Allows the test suite to pass when no files are found. | -| `--project ` | Only run tests from the specified [projects](./test-projects.md), supports '*' wildcard. Defaults to running all projects defined in the configuration file.| -| `--quiet` | Whether to suppress stdout and stderr from the tests. | -| `--repeat-each ` | Run each test `N` times, defaults to one. | -| `--reporter ` | Choose a reporter: minimalist `dot`, concise `line` or detailed `list`. See [reporters](./test-reporters.md) for more information. You can also pass a path to a [custom reporter](./test-reporters.md#custom-reporters) file. | -| `--retries ` | The maximum number of [retries](./test-retries.md#retries) for flaky tests, defaults to zero (no retries). | -| `--shard ` | [Shard](./test-parallel.md#shard-tests-between-multiple-machines) tests and execute only selected shard, specified in the form `current/all`, 1-based, for example `3/5`.| -| `--timeout ` | Maximum timeout in milliseconds for each test, defaults to 30 seconds. Learn more about [various timeouts](./test-timeouts.md).| -| `--trace ` | Force tracing mode, can be `on`, `off`, `on-first-retry`, `on-all-retries`, `retain-on-failure` | -| `--tsconfig ` | Path to a single tsconfig applicable to all imported files. See [tsconfig resolution](./test-typescript.md#tsconfig-resolution) for more details. | -| `--update-snapshots` or `-u` | Whether to update [snapshots](./test-snapshots.md) with actual results instead of comparing them. Use this when snapshot expectations have changed.| -| `--workers ` or `-j `| The maximum number of concurrent worker processes that run in [parallel](./test-parallel.md). | +| Non-option arguments | Each argument is treated as a regular expression matched against the full test file path. Only tests from files matching the pattern will be executed. Special symbols like `$` or `*` should be escaped with `\`. In many shells/terminals you may need to quote the arguments. | +| `-c ` or `--config ` | Configuration file, or a test directory with optional "playwright.config.{m,c}?{js,ts}". Defaults to `playwright.config.ts` or `playwright.config.js` in the current directory. | +| `--debug` | Run tests with Playwright Inspector. Shortcut for `PWDEBUG=1` environment variable and `--timeout=0 --max-failures=1 --headed --workers=1` options. | +| `--fail-on-flaky-tests` | Fail if any test is flagged as flaky (default: false). | +| `--forbid-only` | Fail if `test.only` is called (default: false). Useful on CI. | +| `--fully-parallel` | Run all tests in parallel (default: false). | +| `--global-timeout ` | Maximum time this test suite can run in milliseconds (default: unlimited). | +| `-g ` or `--grep ` | Only run tests matching this regular expression (default: ".*"). | +| `-gv ` or `--grep-invert ` | Only run tests that do not match this regular expression. | +| `--headed` | Run tests in headed browsers (default: headless). | +| `--ignore-snapshots` | Ignore screenshot and snapshot expectations. | +| `--last-failed` | Only re-run the failures. | +| `--list` | Collect all the tests and report them, but do not run. | +| `--max-failures ` or `-x` | Stop after the first `N` failures. Passing `-x` stops after the first failure. | +| `--no-deps` | Do not run project dependencies. | +| `--output ` | Folder for output artifacts (default: "test-results"). | +| `--only-changed [ref]` | Only run test files that have been changed between 'HEAD' and 'ref'. Defaults to running all uncommitted changes. Only supports Git. | +| `--pass-with-no-tests` | Makes test run succeed even if no tests were found. | +| `--project ` | Only run tests from the specified list of projects, supports '*' wildcard (default: run all projects). | +| `--quiet` | Suppress stdio. | +| `--repeat-each ` | Run each test `N` times (default: 1). | +| `--reporter ` | Reporter to use, comma-separated, can be "dot", "line", "list", or others (default: "list"). You can also pass a path to a custom reporter file. | +| `--retries ` | Maximum retry count for flaky tests, zero for no retries (default: no retries). | +| `--shard ` | Shard tests and execute only the selected shard, specified in the form "current/all", 1-based, e.g., "3/5". | +| `--timeout ` | Specify test timeout threshold in milliseconds, zero for unlimited (default: 30 seconds). | +| `--trace ` | Force tracing mode, can be "on", "off", "on-first-retry", "on-all-retries", "retain-on-failure", "retain-on-first-failure". | +| `--tsconfig ` | Path to a single tsconfig applicable to all imported files (default: look up tsconfig for each imported file separately). | +| `--ui` | Run tests in interactive UI mode. | +| `--ui-host ` | Host to serve UI on; specifying this option opens UI in a browser tab. | +| `--ui-port ` | Port to serve UI on, 0 for any free port; specifying this option opens UI in a browser tab. | +| `-u` or `--update-snapshots [mode]` | Update snapshots with actual results. Possible values are "all", "changed", "missing", and "none". Not passing defaults to "missing"; passing without a value defaults to "changed". | +| `-j ` or `--workers ` | Number of concurrent workers or percentage of logical CPU cores, use 1 to run in a single worker (default: 50%). | +| `-x` | Stop after the first failure. | diff --git a/packages/playwright-core/src/utils/comparators.ts b/packages/playwright-core/src/utils/comparators.ts index acf897af49..7f64df240c 100644 --- a/packages/playwright-core/src/utils/comparators.ts +++ b/packages/playwright-core/src/utils/comparators.ts @@ -38,7 +38,7 @@ export function getComparator(mimeType: string): Comparator { const JPEG_JS_MAX_BUFFER_SIZE_IN_MB = 5 * 1024; // ~5 GB -function compareBuffersOrStrings(actualBuffer: Buffer | string, expectedBuffer: Buffer): ComparatorResult { +export function compareBuffersOrStrings(actualBuffer: Buffer | string, expectedBuffer: Buffer): ComparatorResult { if (typeof actualBuffer === 'string') return compareText(actualBuffer, expectedBuffer); if (!actualBuffer || !(actualBuffer instanceof Buffer)) diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.ts index dcde2b28d4..2f08a87650 100644 --- a/packages/playwright/src/common/ipc.ts +++ b/packages/playwright/src/common/ipc.ts @@ -38,7 +38,7 @@ export type ConfigCLIOverrides = { timeout?: number; tsconfig?: string; ignoreSnapshots?: boolean; - updateSnapshots?: 'all'|'none'|'missing'; + updateSnapshots?: 'all'|'changed'|'missing'|'none'; workers?: number | string; projects?: { name: string, use?: any }[], use?: any; diff --git a/packages/playwright/src/matchers/toMatchAriaSnapshot.ts b/packages/playwright/src/matchers/toMatchAriaSnapshot.ts index 0bb600f7b0..552ad2e36d 100644 --- a/packages/playwright/src/matchers/toMatchAriaSnapshot.ts +++ b/packages/playwright/src/matchers/toMatchAriaSnapshot.ts @@ -57,8 +57,6 @@ export async function toMatchAriaSnapshot( } const generateMissingBaseline = updateSnapshots === 'missing' && !expected; - const generateNewBaseline = updateSnapshots === 'all' || generateMissingBaseline; - if (generateMissingBaseline) { if (this.isNot) { const message = `Matchers using ".not" can't generate new baselines`; @@ -100,10 +98,13 @@ export async function toMatchAriaSnapshot( } }; - if (!this.isNot && pass === this.isNot && generateNewBaseline) { - // Only rebaseline failed snapshots. - const suggestedRebaseline = `toMatchAriaSnapshot(\`\n${escapeTemplateString(indent(typedReceived.regex, '{indent} '))}\n{indent}\`)`; - return { pass: this.isNot, message: () => '', name: 'toMatchAriaSnapshot', suggestedRebaseline }; + if (!this.isNot) { + if ((updateSnapshots === 'all') || + (updateSnapshots === 'changed' && pass === this.isNot) || + generateMissingBaseline) { + const suggestedRebaseline = `toMatchAriaSnapshot(\`\n${escapeTemplateString(indent(typedReceived.regex, '{indent} '))}\n{indent}\`)`; + return { pass: this.isNot, message: () => '', name: 'toMatchAriaSnapshot', suggestedRebaseline }; + } } return { diff --git a/packages/playwright/src/matchers/toMatchSnapshot.ts b/packages/playwright/src/matchers/toMatchSnapshot.ts index 374fba3db5..8d16f11fc2 100644 --- a/packages/playwright/src/matchers/toMatchSnapshot.ts +++ b/packages/playwright/src/matchers/toMatchSnapshot.ts @@ -18,7 +18,7 @@ import type { Locator, Page } from 'playwright-core'; import type { ExpectScreenshotOptions, Page as PageEx } from 'playwright-core/lib/client/page'; import { currentTestInfo } from '../common/globals'; import type { ImageComparatorOptions, Comparator } from 'playwright-core/lib/utils'; -import { getComparator, isString, sanitizeForFilePath } from 'playwright-core/lib/utils'; +import { compareBuffersOrStrings, getComparator, isString, sanitizeForFilePath } from 'playwright-core/lib/utils'; import { addSuffixToFilePath, trimLongString, callLogText, @@ -83,7 +83,7 @@ class SnapshotHelper { readonly diffPath: string; readonly mimeType: string; readonly kind: 'Screenshot'|'Snapshot'; - readonly updateSnapshots: 'all' | 'none' | 'missing'; + readonly updateSnapshots: 'all' | 'changed' | 'missing' | 'none'; readonly comparator: Comparator; readonly options: Omit & { comparator?: string }; readonly matcherName: string; @@ -199,7 +199,7 @@ class SnapshotHelper { } handleMissingNegated(): ImageMatcherResult { - const isWriteMissingMode = this.updateSnapshots === 'all' || this.updateSnapshots === 'missing'; + const isWriteMissingMode = this.updateSnapshots !== 'none'; const message = `A snapshot doesn't exist at ${this.expectedPath}${isWriteMissingMode ? ', matchers using ".not" won\'t write them automatically.' : '.'}`; // NOTE: 'isNot' matcher implies inversed value. return this.createMatcherResult(message, true); @@ -221,14 +221,14 @@ class SnapshotHelper { } handleMissing(actual: Buffer | string): ImageMatcherResult { - const isWriteMissingMode = this.updateSnapshots === 'all' || this.updateSnapshots === 'missing'; + const isWriteMissingMode = this.updateSnapshots !== 'none'; if (isWriteMissingMode) writeFileSync(this.expectedPath, actual); this.testInfo.attachments.push({ name: addSuffixToFilePath(this.attachmentBaseName, '-expected'), contentType: this.mimeType, path: this.expectedPath }); writeFileSync(this.actualPath, actual); this.testInfo.attachments.push({ name: addSuffixToFilePath(this.attachmentBaseName, '-actual'), contentType: this.mimeType, path: this.actualPath }); const message = `A snapshot doesn't exist at ${this.expectedPath}${isWriteMissingMode ? ', writing actual.' : '.'}`; - if (this.updateSnapshots === 'all') { + if (this.updateSnapshots === 'all' || this.updateSnapshots === 'changed') { /* eslint-disable no-console */ console.log(message); return this.createMatcherResult(message, true); @@ -317,17 +317,30 @@ export function toMatchSnapshot( return helper.handleMissing(received); const expected = fs.readFileSync(helper.expectedPath); - const result = helper.comparator(received, expected, helper.options); - if (!result) - return helper.handleMatching(); if (helper.updateSnapshots === 'all') { + if (!compareBuffersOrStrings(received, expected)) + return helper.handleMatching(); + writeFileSync(helper.expectedPath, received); + /* eslint-disable no-console */ + console.log(helper.expectedPath + ' is not the same, writing actual.'); + return helper.createMatcherResult(helper.expectedPath + ' running with --update-snapshots, writing actual.', true); + } + + if (helper.updateSnapshots === 'changed') { + const result = helper.comparator(received, expected, helper.options); + if (!result) + return helper.handleMatching(); writeFileSync(helper.expectedPath, received); /* eslint-disable no-console */ console.log(helper.expectedPath + ' does not match, writing actual.'); return helper.createMatcherResult(helper.expectedPath + ' running with --update-snapshots, writing actual.', true); } + const result = helper.comparator(received, expected, helper.options); + if (!result) + return helper.handleMatching(); + const receiver = isString(received) ? 'string' : 'Buffer'; const header = matcherHint(this, undefined, 'toMatchSnapshot', receiver, undefined, undefined); return helper.handleDifferent(received, expected, undefined, result.diff, header, result.errorMessage, undefined); @@ -421,21 +434,30 @@ export async function toHaveScreenshot( // General case: // - snapshot exists // - regular matcher (i.e. not a `.not`) - // - perhaps an 'all' flag to update non-matching screenshots - expectScreenshotOptions.expected = await fs.promises.readFile(helper.expectedPath); + const expected = await fs.promises.readFile(helper.expectedPath); + expectScreenshotOptions.expected = helper.updateSnapshots === 'all' ? undefined : expected; + const { actual, previous, diff, errorMessage, log, timedOut } = await page._expectScreenshot(expectScreenshotOptions); - - if (!errorMessage) - return helper.handleMatching(); - - if (helper.updateSnapshots === 'all') { + const writeFiles = () => { writeFileSync(helper.expectedPath, actual!); writeFileSync(helper.actualPath, actual!); /* eslint-disable no-console */ console.log(helper.expectedPath + ' is re-generated, writing actual.'); return helper.createMatcherResult(helper.expectedPath + ' running with --update-snapshots, writing actual.', true); + }; + + if (!errorMessage) { + // Screenshot is matching, but is not necessarily the same as the expected. + if (helper.updateSnapshots === 'all' && actual && compareBuffersOrStrings(actual, expected)) { + console.log(helper.expectedPath + ' is re-generated, writing actual.'); + return writeFiles(); + } + return helper.handleMatching(); } + if (helper.updateSnapshots === 'changed' || helper.updateSnapshots === 'all') + return writeFiles(); + const header = matcherHint(this, undefined, 'toHaveScreenshot', receiver, undefined, undefined, timedOut ? timeout : undefined); return helper.handleDifferent(actual, expectScreenshotOptions.expected, previous, diff, header, errorMessage, log); } diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts index b4a12c0ea7..a827483bbd 100644 --- a/packages/playwright/src/program.ts +++ b/packages/playwright/src/program.ts @@ -281,6 +281,13 @@ async function mergeReports(reportDir: string | undefined, opts: { [key: string] function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrides { const shardPair = options.shard ? options.shard.split('/').map((t: string) => parseInt(t, 10)) : undefined; + + let updateSnapshots: 'all' | 'changed' | 'missing' | 'none'; + if (['all', 'changed', 'missing', 'none'].includes(options.updateSnapshots)) + updateSnapshots = options.updateSnapshots; + else + updateSnapshots = 'updateSnapshots' in options ? 'changed' : 'missing'; + const overrides: ConfigCLIOverrides = { forbidOnly: options.forbidOnly ? true : undefined, fullyParallel: options.fullyParallel ? true : undefined, @@ -295,7 +302,7 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrid timeout: options.timeout ? parseInt(options.timeout, 10) : undefined, tsconfig: options.tsconfig ? path.resolve(process.cwd(), options.tsconfig) : undefined, ignoreSnapshots: options.ignoreSnapshots ? !!options.ignoreSnapshots : undefined, - updateSnapshots: options.updateSnapshots ? 'all' as const : undefined, + updateSnapshots, workers: options.workers, }; @@ -344,8 +351,10 @@ function resolveReporter(id: string) { const kTraceModes: TraceMode[] = ['on', 'off', 'on-first-retry', 'on-all-retries', 'retain-on-failure', 'retain-on-first-failure']; +// Note: update docs/src/test-cli-js.md when you update this, program is the source of truth. + const testOptions: [string, string][] = [ - ['--browser ', `Browser to use for tests, one of "all", "chromium", "firefox" or "webkit" (default: "chromium")`], + /* deprecated */ ['--browser ', `Browser to use for tests, one of "all", "chromium", "firefox" or "webkit" (default: "chromium")`], ['-c, --config ', `Configuration file, or a test directory with optional "playwright.config.{m,c}?{js,ts}"`], ['--debug', `Run tests with Playwright Inspector. Shortcut for "PWDEBUG=1" environment variable and "--timeout=0 --max-failures=1 --headed --workers=1" options`], ['--fail-on-flaky-tests', `Fail if any test is flagged as flaky (default: false)`], @@ -375,7 +384,7 @@ const testOptions: [string, string][] = [ ['--ui', `Run tests in interactive UI mode`], ['--ui-host ', 'Host to serve UI on; specifying this option opens UI in a browser tab'], ['--ui-port ', 'Port to serve UI on, 0 for any free port; specifying this option opens UI in a browser tab'], - ['-u, --update-snapshots', `Update snapshots with actual results (default: only create missing snapshots)`], + ['-u, --update-snapshots [mode]', `Update snapshots with actual results. Possible values are 'all', 'changed', 'missing' and 'none'. Not passing defaults to 'missing', passing without value defaults to 'changed'`], ['-j, --workers ', `Number of concurrent workers or percentage of logical CPU cores, use 1 to run in a single worker (default: 50%)`], ['-x', `Stop after the first failure`], ]; diff --git a/packages/playwright/src/runner/rebase.ts b/packages/playwright/src/runner/rebase.ts index b412a4a775..e088b82742 100644 --- a/packages/playwright/src/runner/rebase.ts +++ b/packages/playwright/src/runner/rebase.ts @@ -44,7 +44,7 @@ export function addSuggestedRebaseline(location: Location, suggestedRebaseline: } export async function applySuggestedRebaselines(config: FullConfigInternal, reporter: InternalReporter) { - if (config.config.updateSnapshots !== 'all' && config.config.updateSnapshots !== 'missing') + if (config.config.updateSnapshots === 'none') return; if (!suggestedRebaselines.size) return; @@ -106,15 +106,15 @@ export async function applySuggestedRebaselines(config: FullConfigInternal, repo const relativeName = path.relative(gitFolder || process.cwd(), fileName); files.push(relativeName); patches.push(createPatch(relativeName, source, result)); + + const patchFile = path.join(project.project.outputDir, 'rebaselines.patch'); + await fs.promises.mkdir(path.dirname(patchFile), { recursive: true }); + await fs.promises.writeFile(patchFile, patches.join('\n')); + + const fileList = files.map(file => ' ' + colors.dim(file)).join('\n'); + reporter.onStdErr(`\nNew baselines created for:\n\n${fileList}\n\n ` + colors.cyan('git apply ' + path.relative(process.cwd(), patchFile)) + '\n'); } } - - const patchFile = path.join(project.project.outputDir, 'rebaselines.patch'); - await fs.promises.mkdir(path.dirname(patchFile), { recursive: true }); - await fs.promises.writeFile(patchFile, patches.join('\n')); - - const fileList = files.map(file => ' ' + colors.dim(file)).join('\n'); - reporter.onStdErr(`\nNew baselines created for:\n\n${fileList}\n\n ` + colors.cyan('git apply ' + path.relative(process.cwd(), patchFile)) + '\n'); } function createPatch(fileName: string, before: string, after: string) { diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index 84111533c9..c80329ca8a 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -1665,11 +1665,12 @@ interface TestConfig { /** * Whether to update expected snapshots with the actual results produced by the test run. Defaults to `'missing'`. - * - `'all'` - All tests that are executed will update snapshots that did not match. Matching snapshots will not be - * updated. - * - `'none'` - No snapshots are updated. + * - `'all'` - All tests that are executed will update snapshots. + * - `'changed'` - All tests that are executed will update snapshots that did not match. Matching snapshots will not + * be updated. * - `'missing'` - Missing snapshots are created, for example when authoring a new test and running it for the first * time. This is the default. + * - `'none'` - No snapshots are updated. * * Learn more about [snapshots](https://playwright.dev/docs/test-snapshots). * @@ -1685,7 +1686,7 @@ interface TestConfig { * ``` * */ - updateSnapshots?: "all"|"none"|"missing"; + updateSnapshots?: "all"|"changed"|"missing"|"none"; /** * The maximum number of concurrent worker processes to use for parallelizing tests. Can also be set as percentage of @@ -1834,7 +1835,7 @@ export interface FullConfig { /** * See [testConfig.updateSnapshots](https://playwright.dev/docs/api/class-testconfig#test-config-update-snapshots). */ - updateSnapshots: "all"|"none"|"missing"; + updateSnapshots: "all"|"changed"|"missing"|"none"; /** * Playwright version. diff --git a/tests/playwright-test/to-have-screenshot.spec.ts b/tests/playwright-test/to-have-screenshot.spec.ts index ee053cc130..83642bd19e 100644 --- a/tests/playwright-test/to-have-screenshot.spec.ts +++ b/tests/playwright-test/to-have-screenshot.spec.ts @@ -1393,3 +1393,80 @@ test('should trim+sanitize attachment names and paths', async ({ runInlineTest } ]); }); +test.describe('update-snapshots', () => { + test('should rebase non-matching image', async ({ runInlineTest }) => { + const BAD_PIXELS = 10; + const EXPECTED_SNAPSHOT = paintBlackPixels(whiteImage, BAD_PIXELS); + + const result = await runInlineTest({ + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + }), + '__screenshots__/a.spec.js/snapshot.png': EXPECTED_SNAPSHOT, + 'a.spec.js': ` + const { test, expect } = require('@playwright/test'); + test('is a test', async ({ page }) => { + await expect(page).toHaveScreenshot('snapshot.png', { timeout: 2000 }); + }); + ` + }, { 'update-snapshots': 'changed' }); + expect(result.exitCode).toBe(0); + const newBaseline = fs.readFileSync(test.info().outputPath('__screenshots__/a.spec.js/snapshot.png')); + expect(comparePNGs(newBaseline, whiteImage)).toBe(null); + expect(comparePNGs(newBaseline, EXPECTED_SNAPSHOT)).not.toBe(null); + }); + + test('should not rebase matching image', async ({ runInlineTest }) => { + const BAD_PIXELS = 10; + const EXPECTED_SNAPSHOT = paintBlackPixels(whiteImage, BAD_PIXELS); + + const result = await runInlineTest({ + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + expect: { + toHaveScreenshot: { + maxDiffPixels: BAD_PIXELS + } + } + }), + '__screenshots__/a.spec.js/snapshot.png': EXPECTED_SNAPSHOT, + 'a.spec.js': ` + const { test, expect } = require('@playwright/test'); + test('is a test', async ({ page }) => { + await expect(page).toHaveScreenshot('snapshot.png', { timeout: 2000 }); + }); + ` + }, { 'update-snapshots': 'changed' }); + expect(result.exitCode).toBe(0); + const newBaseline = fs.readFileSync(test.info().outputPath('__screenshots__/a.spec.js/snapshot.png')); + expect(comparePNGs(newBaseline, EXPECTED_SNAPSHOT)).toBe(null); + expect(comparePNGs(newBaseline, whiteImage)).not.toBe(null); + }); + + test('should rebase matching image with update-snapshots=all', async ({ runInlineTest }) => { + const BAD_PIXELS = 10; + const EXPECTED_SNAPSHOT = paintBlackPixels(whiteImage, BAD_PIXELS); + + const result = await runInlineTest({ + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + expect: { + toHaveScreenshot: { + maxDiffPixels: BAD_PIXELS + } + } + }), + '__screenshots__/a.spec.js/snapshot.png': EXPECTED_SNAPSHOT, + 'a.spec.js': ` + const { test, expect } = require('@playwright/test'); + test('is a test', async ({ page }) => { + await expect(page).toHaveScreenshot('snapshot.png', { timeout: 2000 }); + }); + ` + }, { 'update-snapshots': 'all' }); + expect(result.exitCode).toBe(0); + const newBaseline = fs.readFileSync(test.info().outputPath('__screenshots__/a.spec.js/snapshot.png')); + expect(comparePNGs(newBaseline, whiteImage)).toBe(null); + expect(comparePNGs(newBaseline, EXPECTED_SNAPSHOT)).not.toBe(null); + }); +}); diff --git a/tests/playwright-test/update-aria-snapshot.spec.ts b/tests/playwright-test/update-aria-snapshot.spec.ts index f7bc744561..3b07e8b93e 100644 --- a/tests/playwright-test/update-aria-snapshot.spec.ts +++ b/tests/playwright-test/update-aria-snapshot.spec.ts @@ -424,3 +424,69 @@ test('should not update snapshots when locator did not match', async ({ runInlin expect(result.output).toContain('Expected: "- heading"'); expect(result.output).toContain('Received: '); }); + +test.describe('update-snapshots none', () => { + test('should create new baseline for matching snapshot', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + '.git/marker': '', + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('test', async ({ page }) => { + await page.setContent(\`

hello

world

\`); + await expect(page.locator('body')).toMatchAriaSnapshot(\`\`); + }); + ` + }, { 'update-snapshots': 'none' }); + + expect(result.exitCode).toBe(1); + const patchPath = testInfo.outputPath('test-results/rebaselines.patch'); + expect(fs.existsSync(patchPath)).toBeFalsy(); + }); +}); + +test.describe('update-snapshots all', () => { + test('should create new baseline for matching snapshot', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + '.git/marker': '', + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('test', async ({ page }) => { + await page.setContent(\`

hello

world

\`); + await expect(page.locator('body')).toMatchAriaSnapshot(\` + - heading "hello" + \`); + }); + ` + }, { 'update-snapshots': 'all' }); + + expect(result.exitCode).toBe(0); + const patchPath = testInfo.outputPath('test-results/rebaselines.patch'); + const data = fs.readFileSync(patchPath, 'utf-8'); + expect(trimPatch(data)).toBe(`diff --git a/a.spec.ts b/a.spec.ts +--- a/a.spec.ts ++++ b/a.spec.ts +@@ -3,7 +3,8 @@ + test('test', async ({ page }) => { + await page.setContent(\`

hello

world

\`); + await expect(page.locator('body')).toMatchAriaSnapshot(\` +- - heading "hello" ++ - heading "hello" [level=1] ++ - heading "world" [level=1] + \`); + }); + +\\ No newline at end of file +`); + + expect(stripAnsi(result.output).replace(/\\/g, '/')).toContain(`New baselines created for: + + a.spec.ts + + git apply test-results/rebaselines.patch +`); + + execSync(`patch -p1 < ${patchPath}`, { cwd: testInfo.outputPath() }); + const result2 = await runInlineTest({}); + expect(result2.exitCode).toBe(0); + }); +}); From 971b5da741670733b1b0a961d8016faf7f01e596 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Fri, 22 Nov 2024 18:30:35 -0800 Subject: [PATCH 3/5] chore: introduce update-source-method (#33738) --- docs/src/test-api/class-fullconfig.md | 6 ++ docs/src/test-api/class-testconfig.md | 9 +++ packages/playwright/src/common/config.ts | 1 + packages/playwright/src/common/ipc.ts | 1 + .../playwright/src/isomorphic/teleReceiver.ts | 1 + packages/playwright/src/program.ts | 2 + packages/playwright/src/runner/rebase.ts | 29 ++++--- packages/playwright/types/test.d.ts | 14 ++++ .../update-aria-snapshot.spec.ts | 79 +++++++++++++++++++ 9 files changed, 130 insertions(+), 12 deletions(-) diff --git a/docs/src/test-api/class-fullconfig.md b/docs/src/test-api/class-fullconfig.md index 81e102436c..923c9fa858 100644 --- a/docs/src/test-api/class-fullconfig.md +++ b/docs/src/test-api/class-fullconfig.md @@ -118,6 +118,12 @@ See [`property: TestConfig.shard`]. See [`property: TestConfig.updateSnapshots`]. +## property: FullConfig.updateSourceMethod +* since: v1.50 +- type: <[UpdateSourceMethod]<"overwrite"|"3way"|"patch">> + +See [`property: TestConfig.updateSourceMethod`]. + ## property: FullConfig.version * since: v1.20 - type: <[string]> diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index d9641128bc..37b9ca5f27 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -590,6 +590,15 @@ export default defineConfig({ }); ``` +## property: TestConfig.updateSourceMethod +* since: v1.50 +- type: ?<[UpdateSourceMethod]<"overwrite"|"3way"|"patch">> + +Defines how to update the source code snapshots. +* `'overwrite'` - Overwrite the source code snapshot with the actual result. +* `'3way'` - Use a three-way merge to update the source code snapshot. +* `'patch'` - Use a patch to update the source code snapshot. This is the default. + ## property: TestConfig.use * since: v1.10 - type: ?<[TestOptions]> diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts index 031a5215f2..0e8babce10 100644 --- a/packages/playwright/src/common/config.ts +++ b/packages/playwright/src/common/config.ts @@ -97,6 +97,7 @@ export class FullConfigInternal { projects: [], shard: takeFirst(configCLIOverrides.shard, userConfig.shard, null), updateSnapshots: takeFirst(configCLIOverrides.updateSnapshots, userConfig.updateSnapshots, 'missing'), + updateSourceMethod: takeFirst(configCLIOverrides.updateSourceMethod, userConfig.updateSourceMethod, 'patch'), version: require('../../package.json').version, workers: 0, webServer: null, diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.ts index 2f08a87650..ad0e91f5c3 100644 --- a/packages/playwright/src/common/ipc.ts +++ b/packages/playwright/src/common/ipc.ts @@ -39,6 +39,7 @@ export type ConfigCLIOverrides = { tsconfig?: string; ignoreSnapshots?: boolean; updateSnapshots?: 'all'|'changed'|'missing'|'none'; + updateSourceMethod?: 'overwrite'|'patch'|'3way'; workers?: number | string; projects?: { name: string, use?: any }[], use?: any; diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts index 0c4408096d..f96547d427 100644 --- a/packages/playwright/src/isomorphic/teleReceiver.ts +++ b/packages/playwright/src/isomorphic/teleReceiver.ts @@ -594,6 +594,7 @@ export const baseFullConfig: reporterTypes.FullConfig = { quiet: false, shard: null, updateSnapshots: 'missing', + updateSourceMethod: 'patch', version: '', workers: 0, webServer: null, diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts index a827483bbd..ea0a48fe6a 100644 --- a/packages/playwright/src/program.ts +++ b/packages/playwright/src/program.ts @@ -303,6 +303,7 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrid tsconfig: options.tsconfig ? path.resolve(process.cwd(), options.tsconfig) : undefined, ignoreSnapshots: options.ignoreSnapshots ? !!options.ignoreSnapshots : undefined, updateSnapshots, + updateSourceMethod: options.updateSourceMethod || 'patch', workers: options.workers, }; @@ -385,6 +386,7 @@ const testOptions: [string, string][] = [ ['--ui-host ', 'Host to serve UI on; specifying this option opens UI in a browser tab'], ['--ui-port ', 'Port to serve UI on, 0 for any free port; specifying this option opens UI in a browser tab'], ['-u, --update-snapshots [mode]', `Update snapshots with actual results. Possible values are 'all', 'changed', 'missing' and 'none'. Not passing defaults to 'missing', passing without value defaults to 'changed'`], + ['--update-source-method ', `Chooses the way source is updated. Possible values are 'overwrite', '3way' and 'patch'. Defaults to 'patch'`], ['-j, --workers ', `Number of concurrent workers or percentage of logical CPU cores, use 1 to run in a single worker (default: 50%)`], ['-x', `Stop after the first failure`], ]; diff --git a/packages/playwright/src/runner/rebase.ts b/packages/playwright/src/runner/rebase.ts index e088b82742..de18df465b 100644 --- a/packages/playwright/src/runner/rebase.ts +++ b/packages/playwright/src/runner/rebase.ts @@ -56,6 +56,8 @@ export async function applySuggestedRebaselines(config: FullConfigInternal, repo const files: string[] = []; const gitCache = new Map(); + const patchFile = path.join(project.project.outputDir, 'rebaselines.patch'); + for (const fileName of [...suggestedRebaselines.keys()].sort()) { const source = await fs.promises.readFile(fileName, 'utf8'); const lines = source.split('\n'); @@ -97,24 +99,27 @@ export async function applySuggestedRebaselines(config: FullConfigInternal, repo for (const range of ranges) result = result.substring(0, range.start) + range.newText + result.substring(range.end); - if (process.env.PWTEST_UPDATE_SNAPSHOTS === 'overwrite') { + const relativeName = path.relative(process.cwd(), fileName); + files.push(relativeName); + + if (config.config.updateSourceMethod === 'overwrite') { await fs.promises.writeFile(fileName, result); - } else if (process.env.PWTEST_UPDATE_SNAPSHOTS === 'manual') { + } else if (config.config.updateSourceMethod === '3way') { await fs.promises.writeFile(fileName, applyPatchWithConflictMarkers(source, result)); } else { const gitFolder = findGitRoot(path.dirname(fileName), gitCache); - const relativeName = path.relative(gitFolder || process.cwd(), fileName); - files.push(relativeName); - patches.push(createPatch(relativeName, source, result)); - - const patchFile = path.join(project.project.outputDir, 'rebaselines.patch'); - await fs.promises.mkdir(path.dirname(patchFile), { recursive: true }); - await fs.promises.writeFile(patchFile, patches.join('\n')); - - const fileList = files.map(file => ' ' + colors.dim(file)).join('\n'); - reporter.onStdErr(`\nNew baselines created for:\n\n${fileList}\n\n ` + colors.cyan('git apply ' + path.relative(process.cwd(), patchFile)) + '\n'); + const relativeToGit = path.relative(gitFolder || process.cwd(), fileName); + patches.push(createPatch(relativeToGit, source, result)); } } + + const fileList = files.map(file => ' ' + colors.dim(file)).join('\n'); + reporter.onStdErr(`\nNew baselines created for:\n\n${fileList}\n`); + if (config.config.updateSourceMethod === 'patch') { + await fs.promises.mkdir(path.dirname(patchFile), { recursive: true }); + await fs.promises.writeFile(patchFile, patches.join('\n')); + reporter.onStdErr(`\n ` + colors.cyan('git apply ' + path.relative(process.cwd(), patchFile)) + '\n'); + } } function createPatch(fileName: string, before: string, after: string) { diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index c80329ca8a..8d05bdafef 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -1688,6 +1688,14 @@ interface TestConfig { */ updateSnapshots?: "all"|"changed"|"missing"|"none"; + /** + * Defines how to update the source code snapshots. + * - `'overwrite'` - Overwrite the source code snapshot with the actual result. + * - `'3way'` - Use a three-way merge to update the source code snapshot. + * - `'patch'` - Use a patch to update the source code snapshot. This is the default. + */ + updateSourceMethod?: "overwrite"|"3way"|"patch"; + /** * The maximum number of concurrent worker processes to use for parallelizing tests. Can also be set as percentage of * logical CPU cores, e.g. `'50%'.` @@ -1837,6 +1845,12 @@ export interface FullConfig { */ updateSnapshots: "all"|"changed"|"missing"|"none"; + /** + * See + * [testConfig.updateSourceMethod](https://playwright.dev/docs/api/class-testconfig#test-config-update-source-method). + */ + updateSourceMethod: "overwrite"|"3way"|"patch"; + /** * Playwright version. */ diff --git a/tests/playwright-test/update-aria-snapshot.spec.ts b/tests/playwright-test/update-aria-snapshot.spec.ts index 3b07e8b93e..6a3d10eb43 100644 --- a/tests/playwright-test/update-aria-snapshot.spec.ts +++ b/tests/playwright-test/update-aria-snapshot.spec.ts @@ -490,3 +490,82 @@ test.describe('update-snapshots all', () => { expect(result2.exitCode).toBe(0); }); }); + +test.describe('update-source-method', () => { + test('should overwrite source', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + '.git/marker': '', + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('test', async ({ page }) => { + await page.setContent(\`

hello

\`); + await expect(page.locator('body')).toMatchAriaSnapshot(\` + - heading "world" + \`); + }); + ` + }, { 'update-snapshots': 'all', 'update-source-method': 'overwrite' }); + + expect(result.exitCode).toBe(0); + const patchPath = testInfo.outputPath('test-results/rebaselines.patch'); + expect(fs.existsSync(patchPath)).toBeFalsy(); + + const data = fs.readFileSync(testInfo.outputPath('a.spec.ts'), 'utf-8'); + expect(data).toBe(` + import { test, expect } from '@playwright/test'; + test('test', async ({ page }) => { + await page.setContent(\`

hello

\`); + await expect(page.locator('body')).toMatchAriaSnapshot(\` + - heading "hello" [level=1] + \`); + }); + `); + + expect(stripAnsi(result.output).replace(/\\/g, '/')).toContain(`New baselines created for: + + a.spec.ts +`); + + const result2 = await runInlineTest({}); + expect(result2.exitCode).toBe(0); + }); + + test('should 3way source', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + '.git/marker': '', + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('test', async ({ page }) => { + await page.setContent(\`

hello

\`); + await expect(page.locator('body')).toMatchAriaSnapshot(\` + - heading "world" + \`); + }); + ` + }, { 'update-snapshots': 'all', 'update-source-method': '3way' }); + + expect(result.exitCode).toBe(0); + const patchPath = testInfo.outputPath('test-results/rebaselines.patch'); + expect(fs.existsSync(patchPath)).toBeFalsy(); + + const data = fs.readFileSync(testInfo.outputPath('a.spec.ts'), 'utf-8'); + expect(data).toBe(` + import { test, expect } from '@playwright/test'; + test('test', async ({ page }) => { + await page.setContent(\`

hello

\`); + await expect(page.locator('body')).toMatchAriaSnapshot(\` +\<<<<<<< HEAD + - heading "world" +======= + - heading "hello" [level=1] +>>>>>>> SNAPSHOT + \`); + }); + `); + + expect(stripAnsi(result.output).replace(/\\/g, '/')).toContain(`New baselines created for: + + a.spec.ts +`); + }); +}); From 35dd3dd1048ae6bf69dd4bdebb6bdd8c225dc4db Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Sat, 23 Nov 2024 11:39:04 -0800 Subject: [PATCH 4/5] chore: use diff for snapshot delta (#33739) --- .../playwright-core/src/utils/comparators.ts | 33 +++++++++--------- tests/playwright-test/golden.spec.ts | 7 ++-- tests/playwright-test/reporter-html.spec.ts | 34 ++----------------- 3 files changed, 24 insertions(+), 50 deletions(-) diff --git a/packages/playwright-core/src/utils/comparators.ts b/packages/playwright-core/src/utils/comparators.ts index 7f64df240c..c848200d66 100644 --- a/packages/playwright-core/src/utils/comparators.ts +++ b/packages/playwright-core/src/utils/comparators.ts @@ -109,26 +109,27 @@ function validateBuffer(buffer: Buffer, mimeType: string): void { function compareText(actual: Buffer | string, expectedBuffer: Buffer): ComparatorResult { if (typeof actual !== 'string') return { errorMessage: 'Actual result should be a string' }; - const expected = expectedBuffer.toString('utf-8'); + let expected = expectedBuffer.toString('utf-8'); if (expected === actual) return null; - const diffs = diff.diffChars(expected, actual); - return { - errorMessage: diff_prettyTerminal(diffs), - }; -} + // Eliminate '\\ No newline at end of file' + if (!actual.endsWith('\n')) + actual += '\n'; + if (!expected.endsWith('\n')) + expected += '\n'; -function diff_prettyTerminal(diffs: Diff.Change[]): string { - const result = diffs.map(part => { - const text = part.value; - if (part.added) - return colors.green(text); - else if (part.removed) - return colors.reset(colors.strikethrough(colors.red(text))); - else - return text; + const lines = diff.createPatch('file', expected, actual, undefined, undefined, { context: 5 }).split('\n'); + const coloredLines = lines.slice(4).map(line => { + if (line.startsWith('-')) + return colors.red(line); + if (line.startsWith('+')) + return colors.green(line); + if (line.startsWith('@@')) + return colors.dim(line); + return line; }); - return result.join(''); + const errorMessage = coloredLines.join('\n'); + return { errorMessage }; } function resizeImage(image: ImageData, size: { width: number, height: number }): ImageData { diff --git a/tests/playwright-test/golden.spec.ts b/tests/playwright-test/golden.spec.ts index 340851e384..bf3dbb8880 100644 --- a/tests/playwright-test/golden.spec.ts +++ b/tests/playwright-test/golden.spec.ts @@ -58,7 +58,8 @@ test('should work with non-txt extensions', async ({ runInlineTest }) => { ` }); expect(result.exitCode).toBe(1); - expect(result.output).toContain(`1,2,34`); + expect(result.rawOutput).toContain(colors.red('-1,2,3')); + expect(result.rawOutput).toContain(colors.green('+1,2,4')); }); @@ -202,8 +203,8 @@ Line7`, }); expect(result.exitCode).toBe(1); expect(result.output).toContain('Line1'); - expect(result.rawOutput).toContain('Line2' + colors.green('2')); - expect(result.rawOutput).toContain('line' + colors.reset(colors.strikethrough(colors.red('1'))) + colors.green('2')); + expect(result.rawOutput).toContain(colors.red('-Line2')); + expect(result.rawOutput).toContain(colors.green('+Line22')); expect(result.output).toContain('Line3'); expect(result.output).toContain('Line5'); expect(result.output).toContain('Line7'); diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts index 63d4c34a28..4b294126b9 100644 --- a/tests/playwright-test/reporter-html.spec.ts +++ b/tests/playwright-test/reporter-html.spec.ts @@ -916,7 +916,7 @@ for (const useIntermediateMergeReport of [true, false] as const) { ])); }); - test('should strikethrough textual diff', async ({ runInlineTest, showReport, page }) => { + test('should highlight textual diff', async ({ runInlineTest, showReport, page }) => { const result = await runInlineTest({ 'helper.ts': ` import { test as base } from '@playwright/test'; @@ -940,36 +940,8 @@ for (const useIntermediateMergeReport of [true, false] as const) { await showReport(); await page.click('text="is a test"'); - await expect(page.locator('.test-error-view').getByText('old')).toHaveCSS('text-decoration', 'line-through solid rgb(205, 49, 49)'); - await expect(page.locator('.test-error-view').getByText('new', { exact: true })).toHaveCSS('text-decoration', 'none solid rgb(0, 188, 0)'); - }); - - test('should strikethrough textual diff with commonalities', async ({ runInlineTest, showReport, page }) => { - const result = await runInlineTest({ - 'helper.ts': ` - import { test as base } from '@playwright/test'; - export * from '@playwright/test'; - export const test = base.extend({ - auto: [ async ({}, run, testInfo) => { - testInfo.snapshotSuffix = ''; - await run(); - }, { auto: true } ] - }); - `, - 'a.spec.js-snapshots/snapshot.txt': `oldcommon`, - 'a.spec.js': ` - const { test, expect } = require('./helper'); - test('is a test', ({}) => { - expect('newcommon').toMatchSnapshot('snapshot.txt'); - }); - ` - }, { reporter: 'dot,html' }, { PLAYWRIGHT_HTML_OPEN: 'never' }); - expect(result.exitCode).toBe(1); - await showReport(); - await page.click('text="is a test"'); - await expect(page.locator('.test-error-view').getByText('old')).toHaveCSS('text-decoration', 'line-through solid rgb(205, 49, 49)'); - await expect(page.locator('.test-error-view').getByText('new', { exact: true })).toHaveCSS('text-decoration', 'none solid rgb(0, 188, 0)'); - await expect(page.locator('.test-error-view').getByText('common Expected:')).toHaveCSS('text-decoration', 'none solid rgb(36, 41, 47)'); + await expect(page.locator('.test-error-view').getByText('-old')).toHaveCSS('color', 'rgb(205, 49, 49)'); + await expect(page.locator('.test-error-view').getByText('+new', { exact: true })).toHaveCSS('color', 'rgb(0, 188, 0)'); }); test('should highlight inline textual diff in toHaveText', async ({ runInlineTest, showReport, page }) => { From 0d9bcd45d57385bddc3ccbeff3ffb2d8314f6be3 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Sat, 23 Nov 2024 11:48:34 -0800 Subject: [PATCH 5/5] chore: pin typescript while vue-tsc is broken (#33746) --- tests/components/ct-vue-vite/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/components/ct-vue-vite/package.json b/tests/components/ct-vue-vite/package.json index 608a128763..6fac022cb5 100644 --- a/tests/components/ct-vue-vite/package.json +++ b/tests/components/ct-vue-vite/package.json @@ -14,6 +14,7 @@ "devDependencies": { "@vitejs/plugin-vue": "^4.1.0", "@vue/tsconfig": "^0.5.1", + "typescript": "5.6.2", "vite": "^5.2.8", "vue-tsc": "^2.0.21" }