From 78c5ff0817c8f11656eac1838d0db7b82ecf7b65 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Thu, 17 Oct 2024 17:11:35 -0700 Subject: [PATCH] Revert "revert reporters" This reverts commit a0f905234a4bee885818cf173efa3c3c317e1aa8. --- packages/html-reporter/src/testResultView.tsx | 32 +++++++++++-------- packages/html-reporter/src/types.ts | 13 +++++++- packages/playwright/src/reporters/base.ts | 25 +++++++++++++-- packages/playwright/src/reporters/html.ts | 2 +- packages/playwright/src/reporters/json.ts | 3 +- 5 files changed, 56 insertions(+), 19 deletions(-) diff --git a/packages/html-reporter/src/testResultView.tsx b/packages/html-reporter/src/testResultView.tsx index bb18422dd0..f6cc86d1c6 100644 --- a/packages/html-reporter/src/testResultView.tsx +++ b/packages/html-reporter/src/testResultView.tsx @@ -14,7 +14,7 @@ limitations under the License. */ -import type { TestAttachment, TestCase, TestResult, TestStep } from './types'; +import type { ErrorDetails, TestAttachment, TestCase, TestResult, TestStep } from './types'; import * as React from 'react'; import { TreeItem } from './treeItem'; import { msToString } from './utils'; @@ -150,26 +150,30 @@ export const TestResultView: React.FC<{ ; }; -function classifyErrors(testErrors: string[], diffs: ImageDiff[]) { +function classifyErrors(testErrors: ErrorDetails[], diffs: ImageDiff[]) { return testErrors.map(error => { - if (error.includes('Screenshot comparison failed:')) { + if (error.shortMessage?.includes('Screenshot comparison failed:') && error.actual && error.expected) { const matchingDiff = diffs.find(diff => { const attachmentName = diff.actual?.attachment.name; - return attachmentName && error.includes(attachmentName); + return attachmentName && error.actual.endsWith(attachmentName); }); - + const errorSuffix = ['Call log:', + ...(error.log?.map(line => ' - ' + line) || []), + '', + error.snippet, + '', + error.callStack, + ].join('\n'); if (matchingDiff) { - const lines = error.split('\n'); - const index = lines.findIndex(line => /Expected:|Previous:|Received:/.test(line)); - const errorPrefix = index !== -1 ? lines.slice(0, index).join('\n') : lines[0]; - - const diffIndex = lines.findIndex(line => / +Diff:/.test(line)); - const errorSuffix = diffIndex !== -1 ? lines.slice(diffIndex + 2).join('\n') : lines.slice(1).join('\n'); - - return { type: 'screenshot', diff: matchingDiff, errorPrefix, errorSuffix }; + return { + type: 'screenshot', + diff: matchingDiff, + errorPrefix: error.shortMessage, + errorSuffix + }; } } - return { type: 'regular', error }; + return { type: 'regular', error: error.message }; }); } diff --git a/packages/html-reporter/src/types.ts b/packages/html-reporter/src/types.ts index 733e88e8b9..ad06ba1b6e 100644 --- a/packages/html-reporter/src/types.ts +++ b/packages/html-reporter/src/types.ts @@ -83,6 +83,17 @@ export type TestCase = Omit & { results: TestResult[]; }; +export type ErrorDetails = { + message: string; + location?: Location; + shortMessage?: string; + log?: string[]; + expected?: any; + actual?: any; + snippet?: string; + callStack?: string; +}; + export type TestAttachment = { name: string; body?: string; @@ -95,7 +106,7 @@ export type TestResult = { startTime: string; duration: number; steps: TestStep[]; - errors: string[]; + errors: ErrorDetails[]; attachments: TestAttachment[]; status: 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted'; }; diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.ts index 4249429a36..f27b803389 100644 --- a/packages/playwright/src/reporters/base.ts +++ b/packages/playwright/src/reporters/base.ts @@ -32,6 +32,17 @@ type Annotation = { type ErrorDetails = { message: string; location?: Location; + callStack?: string; +}; + +type TestResultErrorDetails = ErrorDetails & { + shortMessage?: string; + log?: string[]; + expected?: any; + actual?: any; + + snippet?: string; + stack?: string; }; type TestSummary = { @@ -364,8 +375,8 @@ function quotePathIfNeeded(path: string): string { return path; } -export function formatResultFailure(test: TestCase, result: TestResult, initialIndent: string, highlightCode: boolean): ErrorDetails[] { - const errorDetails: ErrorDetails[] = []; +export function formatResultFailure(test: TestCase, result: TestResult, initialIndent: string, highlightCode: boolean): TestResultErrorDetails[] { + const errorDetails: TestResultErrorDetails[] = []; if (result.status === 'passed' && test.expectedStatus === 'failed') { errorDetails.push({ @@ -383,6 +394,12 @@ export function formatResultFailure(test: TestCase, result: TestResult, initialI errorDetails.push({ message: indent(formattedError.message, initialIndent), location: formattedError.location, + shortMessage: error.shortMessage, + log: error.log, + expected: error.expected, + actual: error.actual, + snippet: error.snippet, + callStack: formattedError.callStack, }); } return errorDetails; @@ -462,9 +479,12 @@ export function formatError(error: TestError, highlightCode: boolean): ErrorDeta tokens.push(snippet); } + let callStack; if (parsedStack && parsedStack.stackLines.length) { tokens.push(''); tokens.push(colors.dim(parsedStack.stackLines.join('\n'))); + // TODO: pass raw lines. + callStack = colors.dim(parsedStack.stackLines.join('\n')); } let location = error.location; @@ -474,6 +494,7 @@ export function formatError(error: TestError, highlightCode: boolean): ErrorDeta return { location, message: tokens.join('\n'), + callStack, }; } diff --git a/packages/playwright/src/reporters/html.ts b/packages/playwright/src/reporters/html.ts index 584c11bae8..c789c8aa66 100644 --- a/packages/playwright/src/reporters/html.ts +++ b/packages/playwright/src/reporters/html.ts @@ -485,7 +485,7 @@ class HtmlBuilder { startTime: result.startTime.toISOString(), retry: result.retry, steps: dedupeSteps(result.steps).map(s => this._createTestStep(s)), - errors: formatResultFailure(test, result, '', true).map(error => error.message), + errors: formatResultFailure(test, result, '', true), status: result.status, attachments: this._serializeAttachments([ ...result.attachments, diff --git a/packages/playwright/src/reporters/json.ts b/packages/playwright/src/reporters/json.ts index e2b7ddf872..df66bf5d4b 100644 --- a/packages/playwright/src/reporters/json.ts +++ b/packages/playwright/src/reporters/json.ts @@ -222,7 +222,8 @@ class JSONReporter implements ReporterV2 { } private _serializeError(error: TestError): JSONReportError { - return formatError(error, true); + const { message, location } = formatError(error, true); + return { message, location }; } private _serializeTestStep(step: TestStep): JSONReportTestStep {