diff --git a/packages/playwright/src/matchers/expect.ts b/packages/playwright/src/matchers/expect.ts index a628af9caa..d2530cfa7b 100644 --- a/packages/playwright/src/matchers/expect.ts +++ b/packages/playwright/src/matchers/expect.ts @@ -59,12 +59,6 @@ export type { ExpectMatcherContext } from '../common/expectBundle'; import { zones } from 'playwright-core/lib/utils'; import { TestInfoImpl } from '../worker/testInfo'; -// from expect/build/types -export type SyncExpectationResult = { - pass: boolean; - message: () => string; -}; - // #region // Mirrored from https://github.com/facebook/jest/blob/f13abff8df9a0e1148baf3584bcde6d1b479edc7/packages/expect/src/print.ts /** diff --git a/packages/playwright/src/matchers/matcherHint.ts b/packages/playwright/src/matchers/matcherHint.ts index fbdb10ce07..3f58a32032 100644 --- a/packages/playwright/src/matchers/matcherHint.ts +++ b/packages/playwright/src/matchers/matcherHint.ts @@ -28,7 +28,7 @@ export function matcherHint(state: ExpectMatcherContext, locator: Locator | unde } export type MatcherResult = { - locator: Locator; + locator?: Locator; name: string; expected: E; message: () => string; diff --git a/packages/playwright/src/matchers/toMatchSnapshot.ts b/packages/playwright/src/matchers/toMatchSnapshot.ts index 95840c8ba7..d4a7d5895d 100644 --- a/packages/playwright/src/matchers/toMatchSnapshot.ts +++ b/packages/playwright/src/matchers/toMatchSnapshot.ts @@ -30,7 +30,8 @@ import fs from 'fs'; import path from 'path'; import { mime } from 'playwright-core/lib/utilsBundle'; import type { TestInfoImpl } from '../worker/testInfo'; -import type { ExpectMatcherContext, SyncExpectationResult } from './expect'; +import type { ExpectMatcherContext } from './expect'; +import type { MatcherResult } from './matcherHint'; type NameOrSegments = string | string[]; const snapshotNamesSymbol = Symbol('snapshotNames'); @@ -40,6 +41,8 @@ type SnapshotNames = { namedSnapshotIndex: { [key: string]: number }; }; +type ImageMatcherResult = MatcherResult & { diff?: string }; + class SnapshotHelper { readonly testInfo: TestInfoImpl; readonly snapshotName: string; @@ -54,9 +57,13 @@ class SnapshotHelper { readonly comparatorOptions: ImageComparatorOptions; readonly comparator: Comparator; readonly allOptions: T; + readonly matcherName: string; + readonly locator: Locator | undefined; constructor( testInfo: TestInfoImpl, + matcherName: string, + locator: Locator | undefined, snapshotPathResolver: (...pathSegments: string[]) => string, anonymousSnapshotExtension: string, configOptions: ImageComparatorOptions, @@ -130,6 +137,8 @@ class SnapshotHelper { this.previousPath = addSuffixToFilePath(outputFile, '-previous'); this.actualPath = addSuffixToFilePath(outputFile, '-actual'); this.diffPath = addSuffixToFilePath(outputFile, '-diff'); + this.matcherName = matcherName; + this.locator = locator; this.updateSnapshots = testInfo.config.updateSnapshots; if (this.updateSnapshots === 'missing' && testInfo.retry < testInfo.project.retries) @@ -148,32 +157,42 @@ class SnapshotHelper { this.kind = this.mimeType.startsWith('image/') ? 'Screenshot' : 'Snapshot'; } - handleMissingNegated() { - const isWriteMissingMode = this.updateSnapshots === 'all' || this.updateSnapshots === 'missing'; - const message = `A snapshot doesn't exist at ${this.snapshotPath}${isWriteMissingMode ? ', matchers using ".not" won\'t write them automatically.' : '.'}`; - return { - // NOTE: 'isNot' matcher implies inversed value. - pass: true, + createMatcherResult(message: string, pass: boolean): ImageMatcherResult { + const unfiltered: ImageMatcherResult = { + name: this.matcherName, + locator: this.locator, + expected: this.snapshotPath, + actual: this.actualPath, + diff: this.diffPath, + pass, message: () => message, }; + return Object.fromEntries(Object.entries(unfiltered).filter(([_, v]) => v !== undefined)) as ImageMatcherResult; } - handleDifferentNegated() { + handleMissingNegated(): ImageMatcherResult { + const isWriteMissingMode = this.updateSnapshots === 'all' || this.updateSnapshots === 'missing'; + const message = `A snapshot doesn't exist at ${this.snapshotPath}${isWriteMissingMode ? ', matchers using ".not" won\'t write them automatically.' : '.'}`; // NOTE: 'isNot' matcher implies inversed value. - return { pass: false, message: () => '' }; + return this.createMatcherResult(message, true); } - handleMatchingNegated() { + handleDifferentNegated(): ImageMatcherResult { + // NOTE: 'isNot' matcher implies inversed value. + return this.createMatcherResult('', false); + } + + handleMatchingNegated(): ImageMatcherResult { const message = [ colors.red(`${this.kind} comparison failed:`), '', indent('Expected result should be different from the actual one.', ' '), ].join('\n'); // NOTE: 'isNot' matcher implies inversed value. - return { pass: true, message: () => message }; + return this.createMatcherResult(message, true); } - handleMissing(actual: Buffer | string) { + handleMissing(actual: Buffer | string): ImageMatcherResult { const isWriteMissingMode = this.updateSnapshots === 'all' || this.updateSnapshots === 'missing'; if (isWriteMissingMode) { writeFileSync(this.snapshotPath, actual); @@ -184,13 +203,13 @@ class SnapshotHelper { if (this.updateSnapshots === 'all') { /* eslint-disable no-console */ console.log(message); - return { pass: true, message: () => message }; + return this.createMatcherResult(message, true); } if (this.updateSnapshots === 'missing') { this.testInfo._failWithError(serializeError(new Error(message)), false /* isHardError */); - return { pass: true, message: () => '' }; + return this.createMatcherResult('', true); } - return { pass: false, message: () => message }; + return this.createMatcherResult(message, false); } handleDifferent( @@ -200,24 +219,20 @@ class SnapshotHelper { diff: Buffer | string | undefined, diffError: string | undefined, log: string[] | undefined, - title = `${this.kind} comparison failed:`) { + title = `${this.kind} comparison failed:`): ImageMatcherResult { const output = [ colors.red(title), '', ]; if (diffError) output.push(indent(diffError, ' ')); - if (log?.length) - output.push(callLogText(log)); - else - output.push(''); if (expected !== undefined) { // Copy the expectation inside the `test-results/` folder for backwards compatibility, // so that one can upload `test-results/` directory and have all the data inside. writeFileSync(this.legacyExpectedPath, expected); this.testInfo.attachments.push({ name: addSuffixToFilePath(this.snapshotName, '-expected'), contentType: this.mimeType, path: this.snapshotPath }); - output.push(`Expected: ${colors.yellow(this.snapshotPath)}`); + output.push(`\nExpected: ${colors.yellow(this.snapshotPath)}`); } if (previous !== undefined) { writeFileSync(this.previousPath, previous); @@ -234,11 +249,17 @@ class SnapshotHelper { this.testInfo.attachments.push({ name: addSuffixToFilePath(this.snapshotName, '-diff'), contentType: this.mimeType, path: this.diffPath }); output.push(` Diff: ${colors.yellow(this.diffPath)}`); } - return { pass: false, message: () => output.join('\n'), }; + + if (log?.length) + output.push(callLogText(log)); + else + output.push(''); + + return this.createMatcherResult(output.join('\n'), false); } - handleMatching() { - return { pass: true, message: () => '' }; + handleMatching(): ImageMatcherResult { + return this.createMatcherResult('', true); } } @@ -247,7 +268,7 @@ export function toMatchSnapshot( received: Buffer | string, nameOrOptions: NameOrSegments | { name?: NameOrSegments } & ImageComparatorOptions = {}, optOptions: ImageComparatorOptions = {} -): SyncExpectationResult { +): MatcherResult { const testInfo = currentTestInfo(); if (!testInfo) throw new Error(`toMatchSnapshot() must be called during the test`); @@ -255,10 +276,10 @@ export function toMatchSnapshot( throw new Error('An unresolved Promise was passed to toMatchSnapshot(), make sure to resolve it by adding await to it.'); if (testInfo._configInternal.ignoreSnapshots) - return { pass: !this.isNot, message: () => '' }; + return { pass: !this.isNot, message: () => '', name: 'toMatchSnapshot', expected: nameOrOptions }; const helper = new SnapshotHelper( - testInfo, testInfo.snapshotPath.bind(testInfo), determineFileExtension(received), + testInfo, 'toMatchSnapshot', undefined, testInfo.snapshotPath.bind(testInfo), determineFileExtension(received), testInfo._projectInternal.expect?.toMatchSnapshot || {}, nameOrOptions, optOptions); @@ -281,7 +302,7 @@ export function toMatchSnapshot( writeFileSync(helper.snapshotPath, received); /* eslint-disable no-console */ console.log(helper.snapshotPath + ' does not match, writing actual.'); - return { pass: true, message: () => helper.snapshotPath + ' running with --update-snapshots, writing actual.' }; + return helper.createMatcherResult(helper.snapshotPath + ' running with --update-snapshots, writing actual.', true); } return helper.handleDifferent(received, expected, undefined, result.diff, result.errorMessage, undefined); @@ -306,18 +327,20 @@ export async function toHaveScreenshot( pageOrLocator: Page | Locator, nameOrOptions: NameOrSegments | { name?: NameOrSegments } & HaveScreenshotOptions = {}, optOptions: HaveScreenshotOptions = {} -): Promise { +): Promise> { const testInfo = currentTestInfo(); if (!testInfo) throw new Error(`toHaveScreenshot() must be called during the test`); if (testInfo._configInternal.ignoreSnapshots) - return { pass: !this.isNot, message: () => '' }; + return { pass: !this.isNot, message: () => '', name: 'toHaveScreenshot', expected: nameOrOptions }; + expectTypes(pageOrLocator, ['Page', 'Locator'], 'toHaveScreenshot'); + const [page, locator] = pageOrLocator.constructor.name === 'Page' ? [(pageOrLocator as PageEx), undefined] : [(pageOrLocator as Locator).page() as PageEx, pageOrLocator as LocatorEx]; const config = (testInfo._projectInternal.expect as any)?.toHaveScreenshot; const snapshotPathResolver = testInfo.snapshotPath.bind(testInfo); const helper = new SnapshotHelper( - testInfo, snapshotPathResolver, 'png', + testInfo, 'toHaveScreenshot', locator, snapshotPathResolver, 'png', { _comparator: config?._comparator, maxDiffPixels: config?.maxDiffPixels, @@ -329,7 +352,6 @@ export async function toHaveScreenshot( throw new Error(`Screenshot name "${path.basename(helper.snapshotPath)}" must have '.png' extension`); expectTypes(pageOrLocator, ['Page', 'Locator'], 'toHaveScreenshot'); - const [page, locator] = pageOrLocator.constructor.name === 'Page' ? [(pageOrLocator as PageEx), undefined] : [(pageOrLocator as Locator).page() as PageEx, pageOrLocator as LocatorEx]; const screenshotOptions = { animations: config?.animations ?? 'disabled', scale: config?.scale ?? 'css', @@ -367,7 +389,7 @@ export async function toHaveScreenshot( // Fast path: there's no screenshot and we don't intend to update it. if (helper.updateSnapshots === 'none' && !hasSnapshot) - return { pass: false, message: () => `A snapshot doesn't exist at ${helper.snapshotPath}.` }; + return helper.createMatcherResult(`A snapshot doesn't exist at ${helper.snapshotPath}.`, false); if (!hasSnapshot) { // Regenerate a new screenshot by waiting until two screenshots are the same. @@ -411,10 +433,7 @@ export async function toHaveScreenshot( writeFileSync(helper.actualPath, actual!); /* eslint-disable no-console */ console.log(helper.snapshotPath + ' is re-generated, writing actual.'); - return { - pass: true, - message: () => helper.snapshotPath + ' running with --update-snapshots, writing actual.' - }; + return helper.createMatcherResult(helper.snapshotPath + ' running with --update-snapshots, writing actual.', true); } return helper.handleDifferent(actual, expected, undefined, diff, errorMessage, log); diff --git a/tests/page/expect-matcher-result.spec.ts b/tests/page/expect-matcher-result.spec.ts index 1c2b599a92..ba11f690cb 100644 --- a/tests/page/expect-matcher-result.spec.ts +++ b/tests/page/expect-matcher-result.spec.ts @@ -245,3 +245,25 @@ Call log`); } }); + +test('toHaveScreenshot should populate matcherResult', async ({ page, server }) => { + await page.setViewportSize({ width: 500, height: 500 }); + await page.goto(server.EMPTY_PAGE); + const e = await expect(page).toHaveScreenshot('screenshot-sanity.png').catch(e => e); + e.matcherResult.message = stripAnsi(e.matcherResult.message); + + expect.soft(e.matcherResult).toEqual({ + actual: expect.stringContaining('screenshot-sanity-actual'), + expected: expect.stringContaining('screenshot-sanity-'), + diff: expect.stringContaining('screenshot-sanity-diff'), + message: expect.stringContaining(`Screenshot comparison failed`), + name: 'toHaveScreenshot', + pass: false, + }); + + expect.soft(stripAnsi(e.toString())).toContain(`Error: Screenshot comparison failed: + + 250000 pixels (ratio 1.00 of all image pixels) are different. + +Expected:`); +}); diff --git a/tests/page/expect-matcher-result.spec.ts-snapshots/screenshot-sanity-chromium.png b/tests/page/expect-matcher-result.spec.ts-snapshots/screenshot-sanity-chromium.png new file mode 100644 index 0000000000..122a4f0ae0 Binary files /dev/null and b/tests/page/expect-matcher-result.spec.ts-snapshots/screenshot-sanity-chromium.png differ diff --git a/tests/page/expect-matcher-result.spec.ts-snapshots/screenshot-sanity-firefox.png b/tests/page/expect-matcher-result.spec.ts-snapshots/screenshot-sanity-firefox.png new file mode 100644 index 0000000000..122a4f0ae0 Binary files /dev/null and b/tests/page/expect-matcher-result.spec.ts-snapshots/screenshot-sanity-firefox.png differ diff --git a/tests/page/expect-matcher-result.spec.ts-snapshots/screenshot-sanity-webkit.png b/tests/page/expect-matcher-result.spec.ts-snapshots/screenshot-sanity-webkit.png new file mode 100644 index 0000000000..122a4f0ae0 Binary files /dev/null and b/tests/page/expect-matcher-result.spec.ts-snapshots/screenshot-sanity-webkit.png differ