chore: add snippet to the json report (#17567)

packages/playwright-test/src/reporters/json.ts

@@ -16,8 +16,8 @@
 
 import fs from 'fs';
 import path from 'path';
-import type { FullConfig, TestCase, Suite, TestResult, TestError, TestStep, FullResult, Location, Reporter, JSONReport, JSONReportSuite, JSONReportSpec, JSONReportTest, JSONReportTestResult, JSONReportTestStep } from '../../types/testReporter';
+import type { FullConfig, TestCase, Suite, TestResult, TestError, TestStep, FullResult, Location, Reporter, JSONReport, JSONReportSuite, JSONReportSpec, JSONReportTest, JSONReportTestResult, JSONReportTestStep, JSONReportError } from '../../types/testReporter';
-import { prepareErrorStack } from './base';
+import { formatError, prepareErrorStack } from './base';
 import { MultiMap } from 'playwright-core/lib/utils/multimap';
 import { assert } from 'playwright-core/lib/utils';
 
@@ -183,6 +183,7 @@ class JSONReporter implements Reporter {
       status: result.status,
       duration: result.duration,
       error: result.error,
+      errors: result.errors.map(e => this._serializeError(e)),
       stdout: result.stdout.map(s => stdioEntry(s)),
       stderr: result.stderr.map(s => stdioEntry(s)),
       retry: result.retry,
@@ -200,6 +201,10 @@ class JSONReporter implements Reporter {
     return jsonResult;
   }
 
+  private _serializeError(error: TestError): JSONReportError {
+    return formatError(this.config, error, true);
+  }
+
   private _serializeTestStep(step: TestStep): JSONReportTestStep {
     const steps = step.steps.filter(s => s.category === 'test.step');
     return {

packages/playwright-test/src/testInfo.ts

@@ -63,16 +63,13 @@ export class TestInfoImpl implements TestInfo {
   currentStep: TestStepInternal | undefined;
 
   get error(): TestError | undefined {
-    return this.errors.length > 0 ? this.errors[0] : undefined;
+    return this.errors[0];
   }
 
   set error(e: TestError | undefined) {
     if (e === undefined)
       throw new Error('Cannot assign testInfo.error undefined value!');
-    if (!this.errors.length)
+    this.errors[0] = e;
-      this.errors.push(e);
-    else
-      this.errors[0] = e;
   }
 
   get timeout(): number {

packages/playwright-test/types/testReporter.d.ts

@@ -486,11 +486,17 @@ export interface JSONReportTest {
   status: 'skipped' | 'expected' | 'unexpected' | 'flaky';
 }
 
+export interface JSONReportError {
+  message: string;
+  location?: Location;
+}
+
 export interface JSONReportTestResult {
   workerIndex: number;
   status: TestStatus | undefined;
   duration: number;
   error: TestError | undefined;
+  errors: JSONReportError[];
   stdout: JSONReportSTDIOEntry[];
   stderr: JSONReportSTDIOEntry[];
   retry: number;

tests/playwright-test/reporter-json.spec.ts

@@ -133,11 +133,17 @@ test('should show steps', async ({ runInlineTest }) => {
   expect(result.exitCode).toBe(1);
   expect(result.report.suites.length).toBe(1);
   expect(result.report.suites[0].specs.length).toBe(1);
-  expect(result.report.suites[0].specs[0].tests[0].results[0].steps![0].title).toBe('math works in a step');
+  const testResult = result.report.suites[0].specs[0].tests[0].results[0];
-  expect(result.report.suites[0].specs[0].tests[0].results[0].steps![0].steps![0].title).toBe('nested step');
+  const steps = testResult.steps!;
-  expect(result.report.suites[0].specs[0].tests[0].results[0].steps![0].steps![0].steps![0].title).toBe('deeply nested step');
+  expect(steps[0].title).toBe('math works in a step');
-  expect(result.report.suites[0].specs[0].tests[0].results[0].steps![0].steps![0].steps![0].steps).toBeUndefined();
+  expect(steps[0].steps![0].title).toBe('nested step');
-  expect(result.report.suites[0].specs[0].tests[0].results[0].steps![1].error).not.toBeUndefined();
+  expect(steps[0].steps![0].steps![0].title).toBe('deeply nested step');
+  expect(steps[0].steps![0].steps![0].steps).toBeUndefined();
+  expect(steps[1].error).not.toBeUndefined();
+  expect(testResult.errors).toHaveLength(1);
+  const snippet = stripAnsi(testResult.errors[0].message);
+  expect(snippet).toContain('failing step');
+  expect(snippet).toContain('expect(2 + 2).toBe(5)');
 });
 
 test('should display tags separately from title', async ({ runInlineTest }) => {

utils/generate_types/index.js

@@ -602,6 +602,7 @@ class TypesGenerator {
       ignoreMissing: new Set([
         'FullResult',
         'JSONReport',
+        'JSONReportError',
         'JSONReportSuite',
         'JSONReportSpec',
         'JSONReportTest',

utils/generate_types/overrides-testReporter.d.ts

@@ -97,11 +97,17 @@ export interface JSONReportTest {
   status: 'skipped' | 'expected' | 'unexpected' | 'flaky';
 }
 
+export interface JSONReportError {
+  message: string;
+  location?: Location;
+}
+
 export interface JSONReportTestResult {
   workerIndex: number;
   status: TestStatus | undefined;
   duration: number;
   error: TestError | undefined;
+  errors: JSONReportError[];
   stdout: JSONReportSTDIOEntry[];
   stderr: JSONReportSTDIOEntry[];
   retry: number;