diff --git a/docs/src/test-api/class-testinfoerror.md b/docs/src/test-api/class-testinfoerror.md index 66e78ecabd..de0a561176 100644 --- a/docs/src/test-api/class-testinfoerror.md +++ b/docs/src/test-api/class-testinfoerror.md @@ -16,6 +16,17 @@ Error message. Set when [Error] (or its subclass) has been thrown. Error stack. Set when [Error] (or its subclass) has been thrown. +## property: TestInfoError.matcherResult +* since: v1.49 +- type: ?<[Object]> + - `name` <[string]> + - `message` ?<[string]> Failure message + - `log` ?<[Array]<[string]>> Call log + - `expected` ?<[any]> + - `actual` ?<[any]> + +Expect matcher result. + ## property: TestInfoError.value * since: v1.10 - type: ?<[string]> diff --git a/docs/src/test-reporter-api/class-testerror.md b/docs/src/test-reporter-api/class-testerror.md index 7a872c63fc..7df42482fa 100644 --- a/docs/src/test-reporter-api/class-testerror.md +++ b/docs/src/test-reporter-api/class-testerror.md @@ -28,6 +28,17 @@ The value that was thrown. Set when anything except the [Error] (or its subclass Error location in the source code. +## property: TestError.matcherResult +* since: v1.49 +- type: ?<[Object]> + - `name` <[string]> + - `message` ?<[string]> Failure message + - `log` ?<[Array]<[string]>> Call log + - `expected` ?<[any]> + - `actual` ?<[any]> + +Expect matcher result. + ## property: TestError.snippet * since: v1.33 - type: ?<[string]> diff --git a/packages/html-reporter/src/testResultView.tsx b/packages/html-reporter/src/testResultView.tsx index 48a24a2391..bef5368811 100644 --- a/packages/html-reporter/src/testResultView.tsx +++ b/packages/html-reporter/src/testResultView.tsx @@ -14,7 +14,7 @@ limitations under the License. */ -import type { TestAttachment, TestCase, TestResult, TestStep } from './types'; +import type { ErrorDetails, TestAttachment, TestCase, TestResult, TestStep } from './types'; import * as React from 'react'; import { TreeItem } from './treeItem'; import { msToString } from './utils'; @@ -150,26 +150,28 @@ export const TestResultView: React.FC<{ ; }; -function classifyErrors(testErrors: string[], diffs: ImageDiff[]) { +function classifyErrors(testErrors: ErrorDetails[], diffs: ImageDiff[]) { return testErrors.map(error => { - if (error.includes('Screenshot comparison failed:')) { + if (error.matcherResult?.name === 'toHaveScreenshot' && error.matcherResult?.actual && error.matcherResult?.expected) { const matchingDiff = diffs.find(diff => { const attachmentName = diff.actual?.attachment.name; - return attachmentName && error.includes(attachmentName); + return attachmentName && error.matcherResult?.actual.endsWith(attachmentName); }); - + const errorSuffix = ['Call log:', + ...(error.matcherResult.log?.map(line => ' - ' + line) || []), + '', + error.snippet, + ].join('\n'); if (matchingDiff) { - const lines = error.split('\n'); - const index = lines.findIndex(line => /Expected:|Previous:|Received:/.test(line)); - const errorPrefix = index !== -1 ? lines.slice(0, index).join('\n') : lines[0]; - - const diffIndex = lines.findIndex(line => / +Diff:/.test(line)); - const errorSuffix = diffIndex !== -1 ? lines.slice(diffIndex + 2).join('\n') : lines.slice(1).join('\n'); - - return { type: 'screenshot', diff: matchingDiff, errorPrefix, errorSuffix }; + return { + type: 'screenshot', + diff: matchingDiff, + errorPrefix: error.matcherResult?.message, + errorSuffix + }; } } - return { type: 'regular', error }; + return { type: 'regular', error: error.message }; }); } diff --git a/packages/html-reporter/src/types.ts b/packages/html-reporter/src/types.ts index 733e88e8b9..dd3ce4f07d 100644 --- a/packages/html-reporter/src/types.ts +++ b/packages/html-reporter/src/types.ts @@ -83,6 +83,21 @@ export type TestCase = Omit & { results: TestResult[]; }; +type MatcherResult = { + name: string; + message?: string; + log?: string[]; + expected?: any; + actual?: any; +}; + +export type ErrorDetails = { + message: string; + location?: Location; + matcherResult?: MatcherResult; + snippet?: string; +}; + export type TestAttachment = { name: string; body?: string; @@ -95,7 +110,7 @@ export type TestResult = { startTime: string; duration: number; steps: TestStep[]; - errors: string[]; + errors: ErrorDetails[]; attachments: TestAttachment[]; status: 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted'; }; diff --git a/packages/playwright/src/matchers/matcherHint.ts b/packages/playwright/src/matchers/matcherHint.ts index 8a78932c68..dc29f2dab4 100644 --- a/packages/playwright/src/matchers/matcherHint.ts +++ b/packages/playwright/src/matchers/matcherHint.ts @@ -33,6 +33,7 @@ export function matcherHint(state: ExpectMatcherState, locator: Locator | undefi export type MatcherResult = { name: string; + shortMessage?: string; expected: E; message: () => string; pass: boolean; diff --git a/packages/playwright/src/matchers/toMatchSnapshot.ts b/packages/playwright/src/matchers/toMatchSnapshot.ts index b3fa3f556e..1e878f8092 100644 --- a/packages/playwright/src/matchers/toMatchSnapshot.ts +++ b/packages/playwright/src/matchers/toMatchSnapshot.ts @@ -185,8 +185,9 @@ class SnapshotHelper { this.kind = this.mimeType.startsWith('image/') ? 'Screenshot' : 'Snapshot'; } - createMatcherResult(message: string, pass: boolean, log?: string[]): ImageMatcherResult { + createMatcherResult(message: string, pass: boolean, log?: string[], shortMessage?: string): ImageMatcherResult { const unfiltered: ImageMatcherResult = { + shortMessage, name: this.matcherName, expected: this.expectedPath, actual: this.actualPath, @@ -202,7 +203,7 @@ class SnapshotHelper { const isWriteMissingMode = this.updateSnapshots === 'all' || this.updateSnapshots === 'missing'; const message = `A snapshot doesn't exist at ${this.expectedPath}${isWriteMissingMode ? ', matchers using ".not" won\'t write them automatically.' : '.'}`; // NOTE: 'isNot' matcher implies inversed value. - return this.createMatcherResult(message, true); + return this.createMatcherResult(message, true, undefined, message); } handleDifferentNegated(): ImageMatcherResult { @@ -217,7 +218,7 @@ class SnapshotHelper { indent('Expected result should be different from the actual one.', ' '), ].join('\n'); // NOTE: 'isNot' matcher implies inversed value. - return this.createMatcherResult(message, true); + return this.createMatcherResult(message, true, undefined, message); } handleMissing(actual: Buffer | string): ImageMatcherResult { @@ -231,14 +232,14 @@ class SnapshotHelper { if (this.updateSnapshots === 'all') { /* eslint-disable no-console */ console.log(message); - return this.createMatcherResult(message, true); + return this.createMatcherResult(message, true, undefined, message); } if (this.updateSnapshots === 'missing') { this.testInfo._hasNonRetriableError = true; this.testInfo._failWithError(new Error(message)); return this.createMatcherResult('', true); } - return this.createMatcherResult(message, false); + return this.createMatcherResult(message, false, undefined, message); } handleDifferent( @@ -256,6 +257,8 @@ class SnapshotHelper { if (diffError) output.push(indent(diffError, ' ')); + const shortMessage = output.join('\n'); + if (expected !== undefined) { // Copy the expectation inside the `test-results/` folder for backwards compatibility, // so that one can upload `test-results/` directory and have all the data inside. @@ -284,7 +287,7 @@ class SnapshotHelper { else output.push(''); - return this.createMatcherResult(output.join('\n'), false, log); + return this.createMatcherResult(output.join('\n'), false, log, shortMessage); } handleMatching(): ImageMatcherResult { diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.ts index 4249429a36..7e01d003ef 100644 --- a/packages/playwright/src/reporters/base.ts +++ b/packages/playwright/src/reporters/base.ts @@ -29,9 +29,19 @@ type Annotation = { location?: Location; }; +type MatcherResult = { + name: string; + message?: string; + log?: string[]; + expected?: any; + actual?: any; +}; + type ErrorDetails = { message: string; location?: Location; + matcherResult?: MatcherResult; + snippet?: string; }; type TestSummary = { @@ -383,6 +393,8 @@ export function formatResultFailure(test: TestCase, result: TestResult, initialI errorDetails.push({ message: indent(formattedError.message, initialIndent), location: formattedError.location, + matcherResult: error.matcherResult, + snippet: error.snippet, }); } return errorDetails; diff --git a/packages/playwright/src/reporters/html.ts b/packages/playwright/src/reporters/html.ts index 5aada7e495..9c6a92be20 100644 --- a/packages/playwright/src/reporters/html.ts +++ b/packages/playwright/src/reporters/html.ts @@ -485,7 +485,7 @@ class HtmlBuilder { startTime: result.startTime.toISOString(), retry: result.retry, steps: dedupeSteps(result.steps).map(s => this._createTestStep(s)), - errors: formatResultFailure(test, result, '', true).map(error => error.message), + errors: formatResultFailure(test, result, '', true), status: result.status, attachments: this._serializeAttachments([ ...result.attachments, diff --git a/packages/playwright/src/util.ts b/packages/playwright/src/util.ts index 460b3de07e..4abdec02d1 100644 --- a/packages/playwright/src/util.ts +++ b/packages/playwright/src/util.ts @@ -25,22 +25,37 @@ import type { TestInfoError } from './../types/test'; import type { Location } from './../types/testReporter'; import { calculateSha1, isRegExp, isString, sanitizeForFilePath, stringifyStackFrames } from 'playwright-core/lib/utils'; import type { RawStack } from 'playwright-core/lib/utils'; +import type { MatcherResult } from './matchers/matcherHint'; const PLAYWRIGHT_TEST_PATH = path.join(__dirname, '..'); const PLAYWRIGHT_CORE_PATH = path.dirname(require.resolve('playwright-core/package.json')); -export function filterStackTrace(e: Error): { message: string, stack: string } { +export function filterStackTrace(e: Error): { message: string, stack: string, matcherResult: TestInfoError['matcherResult'] } { const name = e.name ? e.name + ': ' : ''; if (process.env.PWDEBUGIMPL) - return { message: name + e.message, stack: e.stack || '' }; + return { message: name + e.message, stack: e.stack || '', matcherResult: filterMatcherResult(e) }; const stackLines = stringifyStackFrames(filteredStackTrace(e.stack?.split('\n') || [])); return { + matcherResult: filterMatcherResult(e), message: name + e.message, stack: `${name}${e.message}${stackLines.map(line => '\n' + line).join('')}` }; } +function filterMatcherResult(e: Error): TestInfoError['matcherResult'] | undefined { + const matcherResult = (e as any).matcherResult as MatcherResult; + if (!matcherResult) + return undefined; + return { + name: matcherResult.name, + message: matcherResult.shortMessage, + log: matcherResult.log, + expected: matcherResult.expected, + actual: matcherResult.actual, + }; +} + export function filterStackFile(file: string) { if (!process.env.PWDEBUGIMPL && file.startsWith(PLAYWRIGHT_TEST_PATH)) return false; diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index 3f1179e34f..112bdc72e2 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -8379,6 +8379,27 @@ export interface TestInfo { * Information about an error thrown during test execution. */ export interface TestInfoError { + /** + * Expect matcher result. + */ + matcherResult?: { + name: string; + + /** + * Failure message + */ + message?: string; + + /** + * Call log + */ + log?: Array; + + expected?: any; + + actual?: any; + }; + /** * Error message. Set when [Error] (or its subclass) has been thrown. */ diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.ts index a9d1f020ae..5316ce3ff0 100644 --- a/packages/playwright/types/testReporter.d.ts +++ b/packages/playwright/types/testReporter.d.ts @@ -559,6 +559,27 @@ export interface TestError { */ location?: Location; + /** + * Expect matcher result. + */ + matcherResult?: { + name: string; + + /** + * Failure message + */ + message?: string; + + /** + * Call log + */ + log?: Array; + + expected?: any; + + actual?: any; + }; + /** * Error message. Set when [Error] (or its subclass) has been thrown. */