chore: remove details from markdown reports (#26961)

- remove error details from the reports
- collapse flaky tests by default
- limit comment to 65365 character

GitHub API has comment length limit 65536 chars:
```
Unhandled error: HttpError: Validation Failed: {"resource":"IssueComment","code":"unprocessable","field":"data","message":"Body is too long (maximum is 65536 characters)"}
```

.github/workflows/create_test_report.yml

@@ -98,15 +98,21 @@ jobs:
           core.notice('Report url: ' + reportUrl);
           const mergeWorkflowUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
           const reportMd = await fs.promises.readFile('report.md', 'utf8');
+          function formatComment(lines) {
+            let body = lines.join('\n');
+            if (body.length > 65535)
+              body = body.substring(0, 65000) + `... ${body.length - 65000} more characters`;
+            return body;
+          }
           const { data: response } = await github.rest.issues.createComment({
             ...context.repo,
             issue_number: prNumber,
-            body: [
+            body: formatComment([
               `### [Test results](${reportUrl}) for "${{ github.event.workflow_run.name }}"`,
               reportMd,
               '',
               `Merge [workflow run](${mergeWorkflowUrl}).`
-            ].join('\n'),
+            ]),
           });
           core.info('Posted comment: ' + response.html_url);
 

packages/playwright/src/reporters/markdown.ts

@@ -16,9 +16,9 @@
 
 import fs from 'fs';
 import path from 'path';
-import type { FullResult, TestCase, TestError } from '../../types/testReporter';
+import type { FullResult, TestCase } from '../../types/testReporter';
-import { BaseReporter, formatError, formatTestTitle, stripAnsiEscapes } from './base';
 import { resolveReporterOutputPath } from '../util';
+import { BaseReporter, formatTestTitle } from './base';
 
 type MarkdownReporterOptions = {
   configDir: string,
@@ -49,73 +49,32 @@ class MarkdownReporter extends BaseReporter {
       this._printTestList(':x:', summary.unexpected, lines);
     }
     if (summary.flaky.length) {
-      lines.push(`**${summary.flaky.length} flaky**`);
+      lines.push(`<details>`);
-      this._printTestList(':warning:', summary.flaky, lines);
+      lines.push(`<summary><b>${summary.flaky.length} flaky</b></summary>`);
+      this._printTestList(':warning:', summary.flaky, lines, ' <br/>');
+      lines.push(`</details>`);
+      lines.push(``);
     }
     if (summary.interrupted.length) {
-      lines.push(`**${summary.interrupted.length} interrupted**`);
+      lines.push(`<details>`);
-      this._printTestList(':warning:', summary.interrupted, lines);
+      lines.push(`<summary><b>${summary.flaky.length} interrupted</b></summary>`);
+      this._printTestList(':warning:', summary.interrupted, lines, ' <br/>');
+      lines.push(`</details>`);
+      lines.push(``);
     }
     const skipped = summary.skipped ? `, ${summary.skipped} skipped` : '';
     lines.push(`**${summary.expected} passed${skipped}**`);
     lines.push(`:heavy_check_mark::heavy_check_mark::heavy_check_mark:`);
     lines.push(``);
 
-    if (summary.unexpected.length || summary.fatalErrors.length) {
-      lines.push(`<details>`);
-      lines.push(``);
-      if (summary.fatalErrors.length)
-        this._printFatalErrorDetails(summary.fatalErrors, lines);
-      if (summary.unexpected.length)
-        this._printTestListDetails(':x:', summary.unexpected, lines);
-      lines.push(`</details>`);
-    }
-
     const reportFile = resolveReporterOutputPath('report.md', this._options.configDir, this._options.outputFile);
     await fs.promises.mkdir(path.dirname(reportFile), { recursive: true });
     await fs.promises.writeFile(reportFile, lines.join('\n'));
   }
 
-  private _printTestList(prefix: string, tests: TestCase[], lines: string[]) {
+  private _printTestList(prefix: string, tests: TestCase[], lines: string[], suffix?: string) {
     for (const test of tests)
-      lines.push(`${prefix} ${formatTestTitle(this.config, test)}`);
+      lines.push(`${prefix} ${formatTestTitle(this.config, test)}${suffix || ''}`);
-    lines.push(``);
-  }
-
-  private _printTestListDetails(prefix: string, tests: TestCase[], lines: string[]) {
-    for (const test of tests)
-      this._printTestDetails(prefix, test, lines);
-  }
-
-  private _printTestDetails(prefix: string, test: TestCase, lines: string[]) {
-    lines.push(`${prefix} <b> ${formatTestTitle(this.config, test)} </b>`);
-    let retry = 0;
-    for (const result of test.results) {
-      if (result.status === 'passed')
-        break;
-      if (retry)
-        lines.push(`<b>Retry ${retry}:</b>`);
-      retry++;
-      if (result.error) {
-        lines.push(``);
-        lines.push('```');
-        lines.push(stripAnsiEscapes(formatError(result.error, false).message));
-        lines.push('```');
-        lines.push(``);
-      }
-    }
-    lines.push(``);
-  }
-
-  private _printFatalErrorDetails(errors: TestError[], lines: string[]) {
-    for (const error of errors) {
-      lines.push(`:x: <b>fatal error, not part of any test</b>`);
-      lines.push(``);
-      lines.push('```');
-      lines.push(stripAnsiEscapes(formatError(error, false).message));
-      lines.push('```');
-      lines.push(``);
-    }
     lines.push(``);
   }
 }

tests/playwright-test/reporter-markdown.spec.ts

@@ -66,30 +66,15 @@ test('simple report', async ({ runInlineTest }) => {
 :x: a.test.js:6:11 › failing 1
 :x: b.test.js:6:11 › failing 2
 
-**2 flaky**
+<details>
-:warning: a.test.js:9:11 › flaky 1
+<summary><b>2 flaky</b></summary>
-:warning: c.test.js:6:11 › flaky 2
+:warning: a.test.js:9:11 › flaky 1 <br/>
+:warning: c.test.js:6:11 › flaky 2 <br/>
+
+</details>
 
 **3 passed, 3 skipped**
 :heavy_check_mark::heavy_check_mark::heavy_check_mark:
-
-<details>
-
-:x: <b> a.test.js:6:11 › failing 1 </b>
-`);
-
-  expect(reportFile.toString()).toContain(`Error: expect(received).toBe(expected) // Object.is equality
-
-Expected: 2
-Received: 1
-
-   5 |       });
-   6 |       test('failing 1', async ({}) => {
->  7 |         expect(1).toBe(2);
-     |                   ^
-   8 |       });
-   9 |       test('flaky 1', async ({}) => {
-  10 |         expect(test.info().retry).toBe(1);
 `);
 });
 
@@ -141,12 +126,7 @@ test('report error without snippet', async ({ runInlineTest }) => {
 
 **0 passed**
 :heavy_check_mark::heavy_check_mark::heavy_check_mark:
-
-<details>
-
-:x: <b> a.test.js:3:11 › math 1 </b>
 `);
-  expect(reportFile.toString()).toContain(`Error: My error`);
 });
 
 test('report with worker error', async ({ runInlineTest }) => {
@@ -173,20 +153,5 @@ test('report with worker error', async ({ runInlineTest }) => {
   expect(reportFile.toString()).toContain(`**3 fatal errors, not part of any test**
 **0 passed**
 :heavy_check_mark::heavy_check_mark::heavy_check_mark:
-
-<details>
-
-:x: <b>fatal error, not part of any test</b>
 `);
-  expect(reportFile.toString()).toContain(`Error: My error 1
-
-   at a.test.js:3
-
-  1 |
-  2 |       import { test, expect } from '@playwright/test';
-> 3 |       throw new Error('My error 1');
-    |             ^
-  4 |     
-`);
-  expect(reportFile.toString()).toContain(`Error: No tests found`);
 });