From 15b9e5afdb82b2c372ce5dc0adc18540bb950d53 Mon Sep 17 00:00:00 2001
From: Yury Semikhatsky <yurys@chromium.org>
Date: Mon, 17 Jul 2023 13:50:00 -0700
Subject: [PATCH] fix: print fatal and no snippet errors in markdown report
 (#24263)

---
 .../playwright-test/src/reporters/markdown.ts | 24 ++++--
 .../playwright-test/reporter-markdown.spec.ts | 75 +++++++++++++++++++
 2 files changed, 94 insertions(+), 5 deletions(-)

diff --git a/packages/playwright-test/src/reporters/markdown.ts b/packages/playwright-test/src/reporters/markdown.ts
index c5a6e29b3c..f730ce80a5 100644
--- a/packages/playwright-test/src/reporters/markdown.ts
+++ b/packages/playwright-test/src/reporters/markdown.ts
@@ -16,7 +16,7 @@
 
 import fs from 'fs';
 import path from 'path';
-import type { FullResult, TestCase } from '../../types/testReporter';
+import type { FullResult, TestCase, TestError } from '../../types/testReporter';
 import { BaseReporter, formatError, formatTestTitle, stripAnsiEscapes } from './base';
 
 type MarkdownReporterOptions = {
@@ -41,6 +41,8 @@ class MarkdownReporter extends BaseReporter {
     await super.onEnd(result);
     const summary = this.generateSummary();
     const lines: string[] = [];
+    if (summary.fatalErrors.length)
+      lines.push(`**${summary.fatalErrors.length} fatal errors, not part of any test**`);
     if (summary.unexpected.length) {
       lines.push(`**${summary.unexpected.length} failed**`);
       this._printTestList(':x:', summary.unexpected, lines);
@@ -58,9 +60,11 @@ class MarkdownReporter extends BaseReporter {
     lines.push(`:heavy_check_mark::heavy_check_mark::heavy_check_mark:`);
     lines.push(``);
 
-    if (summary.unexpected.length || summary.flaky.length) {
+    if (summary.unexpected.length || summary.fatalErrors.length || summary.flaky.length) {
       lines.push(`<details>`);
       lines.push(``);
+      if (summary.fatalErrors.length)
+        this._printFatalErrorDetails(summary.fatalErrors, lines);
       if (summary.unexpected.length)
         this._printTestListDetails(':x:', summary.unexpected, lines);
       if (summary.flaky.length)
@@ -93,7 +97,7 @@ class MarkdownReporter extends BaseReporter {
       if (retry)
         lines.push(`<b>Retry ${retry}:</b>`);
       retry++;
-      if (result.error?.snippet) {
+      if (result.error) {
         lines.push(``);
         lines.push('```');
         lines.push(stripAnsiEscapes(formatError(result.error, false).message));
@@ -103,8 +107,18 @@ class MarkdownReporter extends BaseReporter {
     }
     lines.push(``);
   }
+
+  private _printFatalErrorDetails(errors: TestError[], lines: string[]) {
+    for (const error of errors) {
+      lines.push(`:x: <b>fatal error, not part of any test</b>`);
+      lines.push(``);
+      lines.push('```');
+      lines.push(stripAnsiEscapes(formatError(error, false).message));
+      lines.push('```');
+      lines.push(``);
+    }
+    lines.push(``);
+  }
 }
 
-
 export default MarkdownReporter;
-
diff --git a/tests/playwright-test/reporter-markdown.spec.ts b/tests/playwright-test/reporter-markdown.spec.ts
index a96eeb69bb..9b0b7f7ad0 100644
--- a/tests/playwright-test/reporter-markdown.spec.ts
+++ b/tests/playwright-test/reporter-markdown.spec.ts
@@ -114,4 +114,79 @@ test('custom report file', async ({ runInlineTest }) => {
   expect(reportFile.toString()).toBe(`**1 passed**
 :heavy_check_mark::heavy_check_mark::heavy_check_mark:
 `);
+});
+
+test('report error without snippet', async ({ runInlineTest }) => {
+  const files = {
+    'playwright.config.ts': `
+      module.exports = {
+        retries: 1,
+        reporter: 'markdown',
+      };
+    `,
+    'a.test.js': `
+      import { test, expect } from '@playwright/test';
+      test('math 1', async ({}) => {
+        const e = new Error('My error');
+        e.stack = null;
+        throw e;
+      });
+    `,
+  };
+
+  await runInlineTest(files);
+  const reportFile = await fs.promises.readFile(test.info().outputPath('report.md'));
+  expect(reportFile.toString()).toContain(`**1 failed**
+:x: a.test.js:3:11 › math 1
+
+**0 passed**
+:heavy_check_mark::heavy_check_mark::heavy_check_mark:
+
+<details>
+
+:x: <b> a.test.js:3:11 › math 1 </b>
+`);
+  expect(reportFile.toString()).toContain(`Error: My error`);
+});
+
+test('report with worker error', async ({ runInlineTest }) => {
+  const files = {
+    'playwright.config.ts': `
+      module.exports = {
+        retries: 1,
+        reporter: 'markdown',
+      };
+    `,
+    'a.test.js': `
+      import { test, expect } from '@playwright/test';
+      throw new Error('My error 1');
+    `,
+    'b.test.js': `
+      import { test, expect } from '@playwright/test';
+      throw new Error('My error 2');
+    `,
+  };
+
+  const { exitCode } = await runInlineTest(files);
+  expect(exitCode).toBe(1);
+  const reportFile = await fs.promises.readFile(test.info().outputPath('report.md'));
+  expect(reportFile.toString()).toContain(`**3 fatal errors, not part of any test**
+**0 passed**
+:heavy_check_mark::heavy_check_mark::heavy_check_mark:
+
+<details>
+
+:x: <b>fatal error, not part of any test</b>
+`);
+  expect(reportFile.toString()).toContain(`Error: My error 1
+
+   at a.test.js:3
+
+  1 |
+  2 |       import { test, expect } from '@playwright/test';
+> 3 |       throw new Error('My error 1');
+    |             ^
+  4 |     
+`);
+  expect(reportFile.toString()).toContain(`Error: No tests found`);
 });
\ No newline at end of file