chore(html): expose expect matcher result on TestError

docs/src/test-api/class-testinfoerror.md

@@ -16,6 +16,17 @@ Error message. Set when [Error] (or its subclass) has been thrown.
 
 Error stack. Set when [Error] (or its subclass) has been thrown.
 
+## property: TestInfoError.matcherResult
+* since: v1.49
+- type: ?<[Object]>
+  - `name` <[string]>
+  - `message` ?<[string]> Failure message
+  - `log` ?<[Array]<[string]>> Call log
+  - `expected` ?<[any]>
+  - `actual` ?<[any]>
+
+Expect matcher result.
+
 ## property: TestInfoError.value
 * since: v1.10
 - type: ?<[string]>

docs/src/test-reporter-api/class-testerror.md

@@ -28,6 +28,17 @@ The value that was thrown. Set when anything except the [Error] (or its subclass
 
 Error location in the source code.
 
+## property: TestError.matcherResult
+* since: v1.49
+- type: ?<[Object]>
+  - `name` <[string]>
+  - `message` ?<[string]> Failure message
+  - `log` ?<[Array]<[string]>> Call log
+  - `expected` ?<[any]>
+  - `actual` ?<[any]>
+
+Expect matcher result.
+
 ## property: TestError.snippet
 * since: v1.33
 - type: ?<[string]>

packages/html-reporter/src/testResultView.tsx

@@ -14,7 +14,7 @@
   limitations under the License.
 */
 
-import type { TestAttachment, TestCase, TestResult, TestStep } from './types';
+import type { ErrorDetails, TestAttachment, TestCase, TestResult, TestStep } from './types';
 import * as React from 'react';
 import { TreeItem } from './treeItem';
 import { msToString } from './utils';
@@ -150,26 +150,28 @@ export const TestResultView: React.FC<{
   </div>;
 };
 
-function classifyErrors(testErrors: string[], diffs: ImageDiff[]) {
+function classifyErrors(testErrors: ErrorDetails[], diffs: ImageDiff[]) {
   return testErrors.map(error => {
-    if (error.includes('Screenshot comparison failed:')) {
+    if (error.matcherResult?.name === 'toHaveScreenshot' && error.matcherResult?.actual && error.matcherResult?.expected) {
       const matchingDiff = diffs.find(diff => {
         const attachmentName = diff.actual?.attachment.name;
-        return attachmentName && error.includes(attachmentName);
+        return attachmentName && error.matcherResult?.actual.endsWith(attachmentName);
       });
-
+      const errorSuffix = ['Call log:',
+        ...(error.matcherResult.log?.map(line => '  - ' + line) || []),
+        '',
+        error.snippet,
+      ].join('\n');
       if (matchingDiff) {
-        const lines = error.split('\n');
-        const index = lines.findIndex(line => /Expected:|Previous:|Received:/.test(line));
-        const errorPrefix = index !== -1 ? lines.slice(0, index).join('\n') : lines[0];
-
-        const diffIndex = lines.findIndex(line => / +Diff:/.test(line));
-        const errorSuffix = diffIndex !== -1 ? lines.slice(diffIndex + 2).join('\n') : lines.slice(1).join('\n');
-
-        return { type: 'screenshot', diff: matchingDiff, errorPrefix, errorSuffix };
+        return {
+          type: 'screenshot',
+          diff: matchingDiff,
+          errorPrefix: error.matcherResult?.message,
+          errorSuffix
+        };
       }
     }
-    return { type: 'regular', error };
+    return { type: 'regular', error: error.message };
   });
 }
 

packages/html-reporter/src/types.ts

@@ -83,6 +83,21 @@ export type TestCase = Omit<TestCaseSummary, 'results'> & {
   results: TestResult[];
 };
 
+type MatcherResult = {
+  name: string;
+  message?: string;
+  log?: string[];
+  expected?: any;
+  actual?: any;
+};
+
+export type ErrorDetails = {
+  message: string;
+  location?: Location;
+  matcherResult?: MatcherResult;
+  snippet?: string;
+};
+
 export type TestAttachment = {
   name: string;
   body?: string;
@@ -95,7 +110,7 @@ export type TestResult = {
   startTime: string;
   duration: number;
   steps: TestStep[];
-  errors: string[];
+  errors: ErrorDetails[];
   attachments: TestAttachment[];
   status: 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted';
 };

packages/playwright/src/matchers/matcherHint.ts

@@ -33,6 +33,7 @@ export function matcherHint(state: ExpectMatcherState, locator: Locator | undefi
 
 export type MatcherResult<E, A> = {
   name: string;
+  shortMessage?: string;
   expected: E;
   message: () => string;
   pass: boolean;

packages/playwright/src/matchers/toMatchSnapshot.ts

@@ -185,8 +185,9 @@ class SnapshotHelper {
     this.kind = this.mimeType.startsWith('image/') ? 'Screenshot' : 'Snapshot';
   }
 
-  createMatcherResult(message: string, pass: boolean, log?: string[]): ImageMatcherResult {
+  createMatcherResult(message: string, pass: boolean, log?: string[], shortMessage?: string): ImageMatcherResult {
     const unfiltered: ImageMatcherResult = {
+      shortMessage,
       name: this.matcherName,
       expected: this.expectedPath,
       actual: this.actualPath,
@@ -202,7 +203,7 @@ class SnapshotHelper {
     const isWriteMissingMode = this.updateSnapshots === 'all' || this.updateSnapshots === 'missing';
     const message = `A snapshot doesn't exist at ${this.expectedPath}${isWriteMissingMode ? ', matchers using ".not" won\'t write them automatically.' : '.'}`;
     // NOTE: 'isNot' matcher implies inversed value.
-    return this.createMatcherResult(message, true);
+    return this.createMatcherResult(message, true, undefined, message);
   }
 
   handleDifferentNegated(): ImageMatcherResult {
@@ -217,7 +218,7 @@ class SnapshotHelper {
       indent('Expected result should be different from the actual one.', '  '),
     ].join('\n');
     // NOTE: 'isNot' matcher implies inversed value.
-    return this.createMatcherResult(message, true);
+    return this.createMatcherResult(message, true, undefined, message);
   }
 
   handleMissing(actual: Buffer | string): ImageMatcherResult {
@@ -231,14 +232,14 @@ class SnapshotHelper {
     if (this.updateSnapshots === 'all') {
       /* eslint-disable no-console */
       console.log(message);
-      return this.createMatcherResult(message, true);
+      return this.createMatcherResult(message, true, undefined, message);
     }
     if (this.updateSnapshots === 'missing') {
       this.testInfo._hasNonRetriableError = true;
       this.testInfo._failWithError(new Error(message));
       return this.createMatcherResult('', true);
     }
-    return this.createMatcherResult(message, false);
+    return this.createMatcherResult(message, false, undefined, message);
   }
 
   handleDifferent(
@@ -256,6 +257,8 @@ class SnapshotHelper {
     if (diffError)
       output.push(indent(diffError, '  '));
 
+    const shortMessage = output.join('\n');
+
     if (expected !== undefined) {
       // Copy the expectation inside the `test-results/` folder for backwards compatibility,
       // so that one can upload `test-results/` directory and have all the data inside.
@@ -284,7 +287,7 @@ class SnapshotHelper {
     else
       output.push('');
 
-    return this.createMatcherResult(output.join('\n'), false, log);
+    return this.createMatcherResult(output.join('\n'), false, log, shortMessage);
   }
 
   handleMatching(): ImageMatcherResult {

packages/playwright/src/reporters/base.ts

@@ -29,9 +29,19 @@ type Annotation = {
   location?: Location;
 };
 
+type MatcherResult = {
+  name: string;
+  message?: string;
+  log?: string[];
+  expected?: any;
+  actual?: any;
+};
+
 type ErrorDetails = {
   message: string;
   location?: Location;
+  matcherResult?: MatcherResult;
+  snippet?: string;
 };
 
 type TestSummary = {
@@ -383,6 +393,8 @@ export function formatResultFailure(test: TestCase, result: TestResult, initialI
     errorDetails.push({
       message: indent(formattedError.message, initialIndent),
       location: formattedError.location,
+      matcherResult: error.matcherResult,
+      snippet: error.snippet,
     });
   }
   return errorDetails;

packages/playwright/src/reporters/html.ts

@@ -485,7 +485,7 @@ class HtmlBuilder {
       startTime: result.startTime.toISOString(),
       retry: result.retry,
       steps: dedupeSteps(result.steps).map(s => this._createTestStep(s)),
-      errors: formatResultFailure(test, result, '', true).map(error => error.message),
+      errors: formatResultFailure(test, result, '', true),
       status: result.status,
       attachments: this._serializeAttachments([
         ...result.attachments,

packages/playwright/src/util.ts

@@ -25,22 +25,37 @@ import type { TestInfoError } from './../types/test';
 import type { Location } from './../types/testReporter';
 import { calculateSha1, isRegExp, isString, sanitizeForFilePath, stringifyStackFrames } from 'playwright-core/lib/utils';
 import type { RawStack } from 'playwright-core/lib/utils';
+import type { MatcherResult } from './matchers/matcherHint';
 
 const PLAYWRIGHT_TEST_PATH = path.join(__dirname, '..');
 const PLAYWRIGHT_CORE_PATH = path.dirname(require.resolve('playwright-core/package.json'));
 
-export function filterStackTrace(e: Error): { message: string, stack: string } {
+export function filterStackTrace(e: Error): { message: string, stack: string, matcherResult: TestInfoError['matcherResult'] } {
   const name = e.name ? e.name + ': ' : '';
   if (process.env.PWDEBUGIMPL)
-    return { message: name + e.message, stack: e.stack || '' };
+    return { message: name + e.message, stack: e.stack || '', matcherResult: filterMatcherResult(e) };
 
   const stackLines = stringifyStackFrames(filteredStackTrace(e.stack?.split('\n') || []));
   return {
+    matcherResult: filterMatcherResult(e),
     message: name + e.message,
     stack: `${name}${e.message}${stackLines.map(line => '\n' + line).join('')}`
   };
 }
 
+function filterMatcherResult(e: Error): TestInfoError['matcherResult'] | undefined {
+  const matcherResult = (e as any).matcherResult as MatcherResult<unknown, unknown>;
+  if (!matcherResult)
+    return undefined;
+  return {
+    name: matcherResult.name,
+    message: matcherResult.shortMessage,
+    log: matcherResult.log,
+    expected: matcherResult.expected,
+    actual: matcherResult.actual,
+  };
+}
+
 export function filterStackFile(file: string) {
   if (!process.env.PWDEBUGIMPL && file.startsWith(PLAYWRIGHT_TEST_PATH))
     return false;

packages/playwright/types/test.d.ts

@@ -8379,6 +8379,27 @@ export interface TestInfo {
  * Information about an error thrown during test execution.
  */
 export interface TestInfoError {
+  /**
+   * Expect matcher result.
+   */
+  matcherResult?: {
+    name: string;
+
+    /**
+     * Failure message
+     */
+    message?: string;
+
+    /**
+     * Call log
+     */
+    log?: Array<string>;
+
+    expected?: any;
+
+    actual?: any;
+  };
+
   /**
    * Error message. Set when [Error] (or its subclass) has been thrown.
    */

packages/playwright/types/testReporter.d.ts

@@ -559,6 +559,27 @@ export interface TestError {
    */
   location?: Location;
 
+  /**
+   * Expect matcher result.
+   */
+  matcherResult?: {
+    name: string;
+
+    /**
+     * Failure message
+     */
+    message?: string;
+
+    /**
+     * Call log
+     */
+    log?: Array<string>;
+
+    expected?: any;
+
+    actual?: any;
+  };
+
   /**
    * Error message. Set when [Error] (or its subclass) has been thrown.
    */