From 4bb123d4b720b5c72566afa65d1a27686bc89726 Mon Sep 17 00:00:00 2001
From: Dmitry Gozman <dgozman@gmail.com>
Date: Fri, 12 Nov 2021 06:47:41 -0800
Subject: [PATCH] fix(test runner): route more errors to reporter (#10263)

For example, top-level errors in files or global setup issues.
---
 packages/playwright-test/src/runner.ts | 33 +++++++-----
 tests/playwright-test/reporter.spec.ts | 75 +++++++++++++++++++++++++-
 2 files changed, 95 insertions(+), 13 deletions(-)

diff --git a/packages/playwright-test/src/runner.ts b/packages/playwright-test/src/runner.ts
index 8ba1263a23..6fdfb52223 100644
--- a/packages/playwright-test/src/runner.ts
+++ b/packages/playwright-test/src/runner.ts
@@ -20,7 +20,7 @@ import * as fs from 'fs';
 import * as path from 'path';
 import { promisify } from 'util';
 import { Dispatcher, TestGroup } from './dispatcher';
-import { createFileMatcher, createTitleMatcher, FilePatternFilter, monotonicTime } from './util';
+import { createFileMatcher, createTitleMatcher, FilePatternFilter, monotonicTime, serializeError } from './util';
 import { TestCase, Suite } from './test';
 import { Loader } from './loader';
 import { FullResult, Reporter, TestError } from '../types/testReporter';
@@ -97,18 +97,27 @@ export class Runner {
 
   async run(list: boolean, filePatternFilters: FilePatternFilter[], projectNames?: string[]): Promise<FullResult> {
     this._reporter = await this._createReporter(list);
-    const config = this._loader.fullConfig();
-    const globalDeadline = config.globalTimeout ? config.globalTimeout + monotonicTime() : 0;
-    const { result, timedOut } = await raceAgainstDeadline(this._run(list, filePatternFilters, projectNames), globalDeadline);
-    if (timedOut) {
-      const actualResult: FullResult = { status: 'timedout' };
-      if (this._didBegin)
-        await this._reporter.onEnd?.(actualResult);
-      else
-        this._reporter.onError?.(createStacklessError(`Timed out waiting ${config.globalTimeout / 1000}s for the entire test run`));
-      return actualResult;
+    try {
+      const config = this._loader.fullConfig();
+      const globalDeadline = config.globalTimeout ? config.globalTimeout + monotonicTime() : 0;
+      const { result, timedOut } = await raceAgainstDeadline(this._run(list, filePatternFilters, projectNames), globalDeadline);
+      if (timedOut) {
+        const actualResult: FullResult = { status: 'timedout' };
+        if (this._didBegin)
+          await this._reporter.onEnd?.(actualResult);
+        else
+          this._reporter.onError?.(createStacklessError(`Timed out waiting ${config.globalTimeout / 1000}s for the entire test run`));
+        return actualResult;
+      }
+      return result!;
+    } catch (e) {
+      const result: FullResult = { status: 'failed' };
+      try {
+        this._reporter.onError?.(serializeError(e));
+      } catch (ignored) {
+      }
+      return result;
     }
-    return result!;
   }
 
   async _run(list: boolean, testFileReFilters: FilePatternFilter[], projectNames?: string[]): Promise<FullResult> {
diff --git a/tests/playwright-test/reporter.spec.ts b/tests/playwright-test/reporter.spec.ts
index 932aa324e6..a97aa66d15 100644
--- a/tests/playwright-test/reporter.spec.ts
+++ b/tests/playwright-test/reporter.spec.ts
@@ -26,7 +26,9 @@ class Reporter {
   onStdErr() {}
   onTestEnd(test, result) {}
   onTimeout() {}
-  onError() {}
+  onError(error) {
+    console.log('\\n%%got error: ' + error.message);
+  }
   onEnd() {
     console.log('\\n%%end');
   }
@@ -405,6 +407,77 @@ test('should show nice stacks for locators', async ({ runInlineTest }) => {
   ]);
 });
 
+test('should report forbid-only error to reporter', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'reporter.ts': smallReporterJS,
+    'playwright.config.ts': `
+      module.exports = {
+        reporter: './reporter',
+      };
+    `,
+    'a.test.ts': `
+      pwt.test.only('pass', () => {});
+    `
+  }, { 'reporter': '', 'forbid-only': true });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain(`%%got error: =====================================\n --forbid-only found a focused test.`);
+});
+
+test('should report no-tests error to reporter', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'reporter.ts': smallReporterJS,
+    'playwright.config.ts': `
+      module.exports = {
+        reporter: './reporter',
+      };
+    `
+  }, { 'reporter': '' });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain(`%%got error: =================\n no tests found.`);
+});
+
+test('should report require error to reporter', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'reporter.ts': smallReporterJS,
+    'playwright.config.ts': `
+      module.exports = {
+        reporter: './reporter',
+      };
+    `,
+    'a.spec.js': `
+      throw new Error('Oh my!');
+    `,
+  }, { 'reporter': '' });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain(`%%got error: Oh my!`);
+});
+
+test('should report global setup error to reporter', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'reporter.ts': smallReporterJS,
+    'playwright.config.ts': `
+      module.exports = {
+        reporter: './reporter',
+        globalSetup: './globalSetup',
+      };
+    `,
+    'globalSetup.ts': `
+      module.exports = () => {
+        throw new Error('Oh my!');
+      };
+    `,
+    'a.spec.js': `
+      pwt.test('test', () => {});
+    `,
+  }, { 'reporter': '' });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain(`%%got error: Oh my!`);
+});
+
 function stripEscapedAscii(str: string) {
   return str.replace(/\\u00[a-z0-9][a-z0-9]\[[^m]+m/g, '');
 }