chore: markdown report details (#24237)

packages/playwright-test/src/reporters/base.ts

@@ -275,7 +275,7 @@ export function formatFailure(config: FullConfig, test: TestCase, options: {inde
   lines.push(colors.red(header));
   for (const result of test.results) {
     const resultLines: string[] = [];
-    const errors = formatResultFailure(config, test, result, '    ', colors.enabled);
+    const errors = formatResultFailure(test, result, '    ', colors.enabled);
     if (!errors.length)
       continue;
     const retryLines = [];
@@ -342,7 +342,7 @@ export function formatFailure(config: FullConfig, test: TestCase, options: {inde
   };
 }
 
-export function formatResultFailure(config: FullConfig, test: TestCase, result: TestResult, initialIndent: string, highlightCode: boolean): ErrorDetails[] {
+export function formatResultFailure(test: TestCase, result: TestResult, initialIndent: string, highlightCode: boolean): ErrorDetails[] {
   const errorDetails: ErrorDetails[] = [];
 
   if (result.status === 'passed' && test.expectedStatus === 'failed') {
@@ -357,7 +357,7 @@ export function formatResultFailure(config: FullConfig, test: TestCase, result:
   }
 
   for (const error of result.errors) {
-    const formattedError = formatError(config, error, highlightCode);
+    const formattedError = formatError(error, highlightCode);
     errorDetails.push({
       message: indent(formattedError.message, initialIndent),
       location: formattedError.location,
@@ -418,7 +418,7 @@ function formatTestHeader(config: FullConfig, test: TestCase, options: { indent?
   return separator(fullHeader);
 }
 
-export function formatError(config: FullConfig, error: TestError, highlightCode: boolean): ErrorDetails {
+export function formatError(error: TestError, highlightCode: boolean): ErrorDetails {
   const message = error.message || error.value || '';
   const stack = error.stack;
   if (!stack && !error.location)

packages/playwright-test/src/reporters/dot.ts

@@ -66,7 +66,7 @@ class DotReporter extends BaseReporter {
 
   override onError(error: TestError): void {
     super.onError(error);
-    console.log('\n' + formatError(this.config, error, colors.enabled).message);
+    console.log('\n' + formatError(error, colors.enabled).message);
     this._counter = 0;
   }
 

packages/playwright-test/src/reporters/github.ts

@@ -69,7 +69,7 @@ export class GitHubReporter extends BaseReporter {
   }
 
   override onError(error: TestError) {
-    const errorMessage = formatError(this.config, error, false).message;
+    const errorMessage = formatError(error, false).message;
     this.githubLogger.error(errorMessage);
   }
 

packages/playwright-test/src/reporters/json.ts

@@ -208,7 +208,7 @@ class JSONReporter extends EmptyReporter {
   }
 
   private _serializeError(error: TestError): JSONReportError {
-    return formatError(this.config, error, true);
+    return formatError(error, true);
   }
 
   private _serializeTestStep(step: TestStep): JSONReportTestStep {

packages/playwright-test/src/reporters/line.ts

@@ -106,7 +106,7 @@ class LineReporter extends BaseReporter {
   override onError(error: TestError): void {
     super.onError(error);
 
-    const message = formatError(this.config, error, colors.enabled).message + '\n\n';
+    const message = formatError(error, colors.enabled).message + '\n\n';
     if (!process.env.PW_TEST_DEBUG_REPORTERS)
       process.stdout.write(`\u001B[1A\u001B[2K`);
     process.stdout.write(message);

packages/playwright-test/src/reporters/list.ts

@@ -243,7 +243,7 @@ class ListReporter extends BaseReporter {
   override onError(error: TestError): void {
     super.onError(error);
     this._maybeWriteNewLine();
-    const message = formatError(this.config, error, colors.enabled).message + '\n';
+    const message = formatError(error, colors.enabled).message + '\n';
     this._updateLineCountAndNewLineFlagForOutput(message);
     process.stdout.write(message);
   }

packages/playwright-test/src/reporters/markdown.ts

@@ -17,7 +17,7 @@
 import fs from 'fs';
 import path from 'path';
 import type { FullResult, TestCase } from '../../types/testReporter';
-import { BaseReporter, formatTestTitle } from './base';
+import { BaseReporter, formatError, formatTestTitle, stripAnsiEscapes } from './base';
 
 type MarkdownReporterOptions = {
   configDir: string,
@@ -41,31 +41,66 @@ class MarkdownReporter extends BaseReporter {
     await super.onEnd(result);
     const summary = this.generateSummary();
     const lines: string[] = [];
-    lines.push(`:x: <b>failed: ${summary.unexpected.length}</b>`);
-    this._printTestList(summary.unexpected, lines);
+    if (summary.unexpected.length) {
+      lines.push(`**${summary.unexpected.length} failed**`);
+      this._printTestList(':x:', summary.unexpected, lines);
+    }
     if (summary.flaky.length) {
-      lines.push(`:warning: <b>flaky: ${summary.flaky.length}</b>`);
-      this._printTestList(summary.flaky, lines);
+      lines.push(`**${summary.flaky.length} flaky**`);
+      this._printTestList(':warning:', summary.flaky, lines);
     }
     if (summary.interrupted.length) {
-      lines.push(`:warning: <b>interrupted: ${summary.interrupted.length}</b>`);
-      this._printTestList(summary.interrupted, lines);
+      lines.push(`**${summary.interrupted.length} interrupted**`);
+      this._printTestList(':warning:', summary.interrupted, lines);
     }
-    if (summary.skipped) {
-      lines.push(`:ballot_box_with_check: <b>skipped: ${summary.skipped}</b>`);
-      lines.push(``);
-    }
-    lines.push(`:white_check_mark: <b>passed: ${summary.expected}</b>`);
+    const skipped = summary.skipped ? `, ${summary.skipped} skipped` : '';
+    lines.push(`**${summary.expected} passed${skipped}**`);
+    lines.push(`:heavy_check_mark::heavy_check_mark::heavy_check_mark:`);
     lines.push(``);
 
+    if (summary.unexpected.length || summary.flaky.length) {
+      lines.push(`<details>`);
+      lines.push(``);
+      if (summary.unexpected.length)
+        this._printTestListDetails(':x:', summary.unexpected, lines);
+      if (summary.flaky.length)
+        this._printTestListDetails(':warning:', summary.flaky, lines);
+      lines.push(`</details>`);
+    }
+
     const reportFile = path.resolve(this._options.configDir, this._options.outputFile || 'report.md');
     await fs.promises.mkdir(path.dirname(reportFile), { recursive: true });
     await fs.promises.writeFile(reportFile, lines.join('\n'));
   }
 
-  private _printTestList(tests: TestCase[], lines: string[]) {
+  private _printTestList(prefix: string, tests: TestCase[], lines: string[]) {
     for (const test of tests)
-      lines.push(` - ${formatTestTitle(this.config, test)}`);
+      lines.push(`${prefix} ${formatTestTitle(this.config, test)}`);
+    lines.push(``);
+  }
+
+  private _printTestListDetails(prefix: string, tests: TestCase[], lines: string[]) {
+    for (const test of tests)
+      this._printTestDetails(prefix, test, lines);
+  }
+
+  private _printTestDetails(prefix: string, test: TestCase, lines: string[]) {
+    lines.push(`${prefix} <b> ${formatTestTitle(this.config, test)} </b>`);
+    let retry = 0;
+    for (const result of test.results) {
+      if (result.status === 'passed')
+        break;
+      if (retry)
+        lines.push(`<b>Retry ${retry}:</b>`);
+      retry++;
+      if (result.error?.snippet) {
+        lines.push(``);
+        lines.push('```');
+        lines.push(stripAnsiEscapes(formatError(result.error, false).message));
+        lines.push('```');
+        lines.push(``);
+      }
+    }
     lines.push(``);
   }
 }

packages/playwright-test/src/reporters/raw.ts

@@ -246,7 +246,7 @@ class RawReporter {
       startTime: result.startTime.toISOString(),
       duration: result.duration,
       status: result.status,
-      errors: formatResultFailure(this.config, test, result, '', true).map(error => error.message),
+      errors: formatResultFailure(test, result, '', true).map(error => error.message),
       attachments: this.generateAttachments(result.attachments, result),
       steps: dedupeSteps(result.steps.map(step => this._serializeStep(test, step)))
     };

packages/playwright-test/src/runner/reporters.ts

@@ -102,6 +102,6 @@ class ListModeReporter extends EmptyReporter {
 
   override onError(error: TestError) {
     // eslint-disable-next-line no-console
-    console.error('\n' + formatError(this.config, error, false).message);
+    console.error('\n' + formatError(error, false).message);
   }
 }

tests/playwright-test/reporter-markdown.spec.ts

@@ -62,17 +62,34 @@ test('simple report', async ({ runInlineTest }) => {
   const { exitCode } = await runInlineTest(files);
   expect(exitCode).toBe(1);
   const reportFile = await fs.promises.readFile(test.info().outputPath('report.md'));
-  expect(reportFile.toString()).toBe(`:x: <b>failed: 2</b>
- - a.test.js:6:11 › failing 1
- - b.test.js:6:11 › failing 2
+  expect(reportFile.toString()).toContain(`**2 failed**
+:x: a.test.js:6:11 › failing 1
+:x: b.test.js:6:11 › failing 2
 
-:warning: <b>flaky: 2</b>
- - a.test.js:9:11 › flaky 1
- - c.test.js:6:11 › flaky 2
+**2 flaky**
+:warning: a.test.js:9:11 › flaky 1
+:warning: c.test.js:6:11 › flaky 2
 
-:ballot_box_with_check: <b>skipped: 3</b>
+**3 passed, 3 skipped**
+:heavy_check_mark::heavy_check_mark::heavy_check_mark:
 
-:white_check_mark: <b>passed: 3</b>
+<details>
+
+:x: <b> a.test.js:6:11 › failing 1 </b>
+`);
+
+  expect(reportFile.toString()).toContain(`Error: expect(received).toBe(expected) // Object.is equality
+
+Expected: 2
+Received: 1
+
+   5 |       });
+   6 |       test('failing 1', async ({}) => {
+>  7 |         expect(1).toBe(2);
+     |                   ^
+   8 |       });
+   9 |       test('flaky 1', async ({}) => {
+  10 |         expect(test.info().retry).toBe(1);
 `);
 });
 
@@ -94,8 +111,7 @@ test('custom report file', async ({ runInlineTest }) => {
   const { exitCode } = await runInlineTest(files);
   expect(exitCode).toBe(0);
   const reportFile = await fs.promises.readFile(test.info().outputPath('my-report.md'));
-  expect(reportFile.toString()).toBe(`:x: <b>failed: 0</b>
-
-:white_check_mark: <b>passed: 1</b>
+  expect(reportFile.toString()).toBe(`**1 passed**
+:heavy_check_mark::heavy_check_mark::heavy_check_mark:
 `);
 });