diff --git a/packages/html-reporter/src/testResultView.tsx b/packages/html-reporter/src/testResultView.tsx index bb18422dd0..3a562f3fcf 100644 --- a/packages/html-reporter/src/testResultView.tsx +++ b/packages/html-reporter/src/testResultView.tsx @@ -152,7 +152,8 @@ export const TestResultView: React.FC<{ function classifyErrors(testErrors: string[], diffs: ImageDiff[]) { return testErrors.map(error => { - if (error.includes('Screenshot comparison failed:')) { + const firstLine = error.split('\n')[0]; + if (firstLine.includes('toHaveScreenshot') || firstLine.includes('toMatchSnapshot')) { const matchingDiff = diffs.find(diff => { const attachmentName = diff.actual?.attachment.name; return attachmentName && error.includes(attachmentName); diff --git a/packages/playwright-core/src/client/page.ts b/packages/playwright-core/src/client/page.ts index 6654294edd..87317c9018 100644 --- a/packages/playwright-core/src/client/page.ts +++ b/packages/playwright-core/src/client/page.ts @@ -589,7 +589,7 @@ export class Page extends ChannelOwner implements api.Page return result.binary; } - async _expectScreenshot(options: ExpectScreenshotOptions): Promise<{ actual?: Buffer, previous?: Buffer, diff?: Buffer, errorMessage?: string, log?: string[]}> { + async _expectScreenshot(options: ExpectScreenshotOptions): Promise<{ actual?: Buffer, previous?: Buffer, diff?: Buffer, errorMessage?: string, log?: string[], timeout?: number}> { const mask = options?.mask ? options?.mask.map(locator => ({ frame: (locator as Locator)._frame._channel, selector: (locator as Locator)._selector, diff --git a/packages/playwright-core/src/protocol/validator.ts b/packages/playwright-core/src/protocol/validator.ts index 7bad26f498..b6dbc0bbf7 100644 --- a/packages/playwright-core/src/protocol/validator.ts +++ b/packages/playwright-core/src/protocol/validator.ts @@ -1193,6 +1193,7 @@ scheme.PageExpectScreenshotResult = tObject({ errorMessage: tOptional(tString), actual: tOptional(tBinary), previous: tOptional(tBinary), + timeout: tOptional(tNumber), log: tOptional(tArray(tString)), }); scheme.PageScreenshotParams = tObject({ diff --git a/packages/playwright-core/src/server/page.ts b/packages/playwright-core/src/server/page.ts index b78bd91ee1..d46df2df9b 100644 --- a/packages/playwright-core/src/server/page.ts +++ b/packages/playwright-core/src/server/page.ts @@ -674,11 +674,12 @@ export class Page extends SdkObject { throw e; let errorMessage = e.message; if (e instanceof TimeoutError && intermediateResult?.previous) - errorMessage = `Failed to take two consecutive stable screenshots. ${e.message}`; + errorMessage = `Failed to take two consecutive stable screenshots.`; return { log: e.message ? [...metadata.log, e.message] : metadata.log, ...intermediateResult, errorMessage, + ...((e instanceof TimeoutError) ? { timeout: callTimeout } : {}), }; }); } diff --git a/packages/playwright/src/matchers/toMatchSnapshot.ts b/packages/playwright/src/matchers/toMatchSnapshot.ts index 86504062d3..edcead15ca 100644 --- a/packages/playwright/src/matchers/toMatchSnapshot.ts +++ b/packages/playwright/src/matchers/toMatchSnapshot.ts @@ -18,7 +18,7 @@ import type { Locator, Page } from 'playwright-core'; import type { ExpectScreenshotOptions, Page as PageEx } from 'playwright-core/lib/client/page'; import { currentTestInfo } from '../common/globals'; import type { ImageComparatorOptions, Comparator } from 'playwright-core/lib/utils'; -import { getComparator, sanitizeForFilePath } from 'playwright-core/lib/utils'; +import { getComparator, isString, sanitizeForFilePath } from 'playwright-core/lib/utils'; import { addSuffixToFilePath, trimLongString, callLogText, @@ -31,7 +31,7 @@ import path from 'path'; import { mime } from 'playwright-core/lib/utilsBundle'; import type { TestInfoImpl } from '../worker/testInfo'; import type { ExpectMatcherState } from '../../types/test'; -import type { MatcherResult } from './matcherHint'; +import { matcherHint, type MatcherResult } from './matcherHint'; import type { FullProjectInternal } from '../common/config'; type NameOrSegments = string | string[]; @@ -250,16 +250,10 @@ class SnapshotHelper { expected: Buffer | string | undefined, previous: Buffer | string | undefined, diff: Buffer | string | undefined, - diffError: string | undefined, - log: string[] | undefined, - title = `${this.kind} comparison failed:`): ImageMatcherResult { - const output = [ - colors.red(title), - '', - ]; - if (diffError) - output.push(indent(diffError, ' ')); - + header: string, + diffError: string, + log: string[] | undefined): ImageMatcherResult { + const output = [`${header}${indent(diffError, ' ')}`]; if (expected !== undefined) { // Copy the expectation inside the `test-results/` folder for backwards compatibility, // so that one can upload `test-results/` directory and have all the data inside. @@ -338,7 +332,9 @@ export function toMatchSnapshot( return helper.createMatcherResult(helper.expectedPath + ' running with --update-snapshots, writing actual.', true); } - return helper.handleDifferent(received, expected, undefined, result.diff, result.errorMessage, undefined); + const receiver = isString(received) ? 'string' : 'Buffer'; + const header = matcherHint(this, undefined, 'toMatchSnapshot', receiver, undefined, undefined); + return helper.handleDifferent(received, expected, undefined, result.diff, header, result.errorMessage, undefined); } export function toHaveScreenshotStepTitle( @@ -410,13 +406,16 @@ export async function toHaveScreenshot( if (helper.updateSnapshots === 'none' && !hasSnapshot) return helper.createMatcherResult(`A snapshot doesn't exist at ${helper.expectedPath}.`, false); + const receiver = locator ? 'locator' : 'page'; if (!hasSnapshot) { // Regenerate a new screenshot by waiting until two screenshots are the same. - const { actual, previous, diff, errorMessage, log } = await page._expectScreenshot(expectScreenshotOptions); + const { actual, previous, diff, errorMessage, log, timeout } = await page._expectScreenshot(expectScreenshotOptions); // We tried re-generating new snapshot but failed. // This can be due to e.g. spinning animation, so we want to show it as a diff. - if (errorMessage) - return helper.handleDifferent(actual, undefined, previous, diff, undefined, log, errorMessage); + if (errorMessage) { + const header = matcherHint(this, locator, 'toHaveScreenshot', receiver, undefined, undefined, timeout); + return helper.handleDifferent(actual, undefined, previous, diff, header, errorMessage, log); + } // We successfully generated new screenshot. return helper.handleMissing(actual!); @@ -427,7 +426,7 @@ export async function toHaveScreenshot( // - regular matcher (i.e. not a `.not`) // - perhaps an 'all' flag to update non-matching screenshots expectScreenshotOptions.expected = await fs.promises.readFile(helper.expectedPath); - const { actual, previous, diff, errorMessage, log } = await page._expectScreenshot(expectScreenshotOptions); + const { actual, previous, diff, errorMessage, log, timeout } = await page._expectScreenshot(expectScreenshotOptions); if (!errorMessage) return helper.handleMatching(); @@ -440,7 +439,8 @@ export async function toHaveScreenshot( return helper.createMatcherResult(helper.expectedPath + ' running with --update-snapshots, writing actual.', true); } - return helper.handleDifferent(actual, expectScreenshotOptions.expected, previous, diff, errorMessage, log); + const header = matcherHint(this, undefined, 'toHaveScreenshot', receiver, undefined, undefined, timeout); + return helper.handleDifferent(actual, expectScreenshotOptions.expected, previous, diff, header, errorMessage, log); } function writeFileSync(aPath: string, content: Buffer | string) { diff --git a/packages/protocol/src/channels.ts b/packages/protocol/src/channels.ts index 7fcb815468..60afe65b25 100644 --- a/packages/protocol/src/channels.ts +++ b/packages/protocol/src/channels.ts @@ -2193,6 +2193,7 @@ export type PageExpectScreenshotResult = { errorMessage?: string, actual?: Binary, previous?: Binary, + timeout?: number, log?: string[], }; export type PageScreenshotParams = { diff --git a/packages/protocol/src/protocol.yml b/packages/protocol/src/protocol.yml index 98428cae2f..d16931dd05 100644 --- a/packages/protocol/src/protocol.yml +++ b/packages/protocol/src/protocol.yml @@ -1501,6 +1501,7 @@ Page: errorMessage: string? actual: binary? previous: binary? + timeout: number? log: type: array? items: string diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts index 11169fadff..51734be9ff 100644 --- a/tests/playwright-test/reporter-html.spec.ts +++ b/tests/playwright-test/reporter-html.spec.ts @@ -330,7 +330,9 @@ for (const useIntermediateMergeReport of [true, false] as const) { await expect(page.locator('text=Image mismatch')).toHaveCount(1); await expect(page.locator('text=Snapshot mismatch')).toHaveCount(0); await expect(page.locator('.chip-header', { hasText: 'Screenshots' })).toHaveCount(0); - await expect(page.getByTestId('test-result-image-mismatch-tabs').locator('div')).toHaveText([ + const errorChip = page.getByTestId('test-screenshot-error-view'); + await expect(errorChip).toContainText('Failed to take two consecutive stable screenshots.'); + await expect(errorChip.getByTestId('test-result-image-mismatch-tabs').locator('div')).toHaveText([ 'Diff', 'Actual', 'Previous',