chore: run global setup before onBegin (#20285)

docs/src/test-reporter-api/class-reporter.md

@@ -79,11 +79,13 @@ export default defineConfig({
 ```
 
 Here is a typical order of reporter calls:
+* [`method: Reporter.onConfigure`] is called once config has been resolved.
 * [`method: Reporter.onBegin`] is called once with a root suite that contains all other suites and tests. Learn more about [suites hierarchy][Suite].
 * [`method: Reporter.onTestBegin`] is called for each test run. It is given a [TestCase] that is executed, and a [TestResult] that is almost empty. Test result will be populated while the test runs (for example, with steps and stdio) and will get final `status` once the test finishes.
 * [`method: Reporter.onStepBegin`] and [`method: Reporter.onStepEnd`] are called for each executed step inside the test. When steps are executed, test run has not finished yet.
 * [`method: Reporter.onTestEnd`] is called when test run has finished. By this time, [TestResult] is complete and you can use [`property: TestResult.status`], [`property: TestResult.error`] and more.
 * [`method: Reporter.onEnd`] is called once after all tests that should run had finished.
+* [`method: Reporter.onExit`] is called before test runner exits.
 
 Additionally, [`method: Reporter.onStdOut`] and [`method: Reporter.onStdErr`] are called when standard output is produced in the worker process, possibly during a test execution,
 and [`method: Reporter.onError`] is called when something went wrong outside of the test execution.
@@ -107,7 +109,16 @@ Resolved configuration.
 
 The root suite that contains all projects, files and test cases.
 
+## optional method: Reporter.onConfigure
+* since: v1.30
 
+Called once config is resolved.
+
+### param: Reporter.onConfigure.config
+* since: v1.30
+- `config` <[TestConfig]>
+
+Resolved configuration.
 
 ## optional async method: Reporter.onEnd
 * since: v1.10
@@ -125,8 +136,10 @@ Result of the full test run.
 * `'timedout'` - The [`property: TestConfig.globalTimeout`] has been reached.
 * `'interrupted'` - Interrupted by the user.
 
+## optional method: Reporter.onExit
+* since: v1.30
 
-
+Called before test runner exits.
 
 ## optional method: Reporter.onError
 * since: v1.10

packages/playwright-test/src/plugins/dockerPlugin.ts

@@ -15,14 +15,14 @@
  */
 
 import type { TestRunnerPlugin } from '.';
-import type { FullConfig, Reporter, Suite } from '../../types/testReporter';
+import type { FullConfig, Reporter } from '../../types/testReporter';
 import { colors } from 'playwright-core/lib/utilsBundle';
 import { checkDockerEngineIsRunningOrDie, containerInfo } from 'playwright-core/lib/containers/docker';
 
 export const dockerPlugin: TestRunnerPlugin = {
   name: 'playwright:docker',
 
-  async setup(config: FullConfig, configDir: string, rootSuite: Suite, reporter: Reporter) {
+  async setup(config: FullConfig, configDir: string, reporter: Reporter) {
     if (!process.env.PLAYWRIGHT_DOCKER)
       return;
 

packages/playwright-test/src/plugins/index.ts

@@ -20,7 +20,8 @@ import type { FullConfig } from '../types';
 
 export interface TestRunnerPlugin {
   name: string;
-  setup?(config: FullConfig, configDir: string, rootSuite: Suite, reporter: Reporter): Promise<void>;
+  setup?(config: FullConfig, configDir: string, reporter: Reporter): Promise<void>;
+  begin?(suite: Suite): Promise<void>;
   teardown?(): Promise<void>;
 }
 

packages/playwright-test/src/plugins/vitePlugin.ts

@@ -46,11 +46,16 @@ export function createPlugin(
   registerSourceFile: string,
   frameworkPluginFactory: () => Promise<Plugin>): TestRunnerPlugin {
   let configDir: string;
+  let config: FullConfig;
   return {
     name: 'playwright-vite-plugin',
 
-    setup: async (config: FullConfig, configDirectory: string, suite: Suite) => {
+    setup: async (configObject: FullConfig, configDirectory: string) => {
+      config = configObject;
       configDir = configDirectory;
+    },
+
+    begin: async (suite: Suite) => {
       const use = config.projects[0].use as CtConfig;
       const port = use.ctPort || 3100;
       const viteConfig: InlineConfig = use.ctViteConfig || {};

packages/playwright-test/src/plugins/webServerPlugin.ts

@@ -21,7 +21,7 @@ import net from 'net';
 import { debug } from 'playwright-core/lib/utilsBundle';
 import { raceAgainstTimeout, launchProcess } from 'playwright-core/lib/utils';
 
-import type { FullConfig, Reporter, Suite } from '../../types/testReporter';
+import type { FullConfig, Reporter } from '../../types/testReporter';
 import type { TestRunnerPlugin } from '.';
 import type { FullConfigInternal } from '../types';
 import { envWithoutExperimentalLoaderOptions } from '../cli';
@@ -57,7 +57,7 @@ export class WebServerPlugin implements TestRunnerPlugin {
     this._checkPortOnly = checkPortOnly;
   }
 
-  public async setup(config: FullConfig, configDir: string, rootSuite: Suite, reporter: Reporter) {
+  public async setup(config: FullConfig, configDir: string, reporter: Reporter) {
     this._reporter = reporter;
     this._isAvailable = getIsAvailableFunction(this._options.url, this._checkPortOnly, !!this._options.ignoreHTTPSErrors, this._reporter.onStdErr?.bind(this._reporter));
     this._options.cwd = this._options.cwd ? path.resolve(configDir, this._options.cwd) : configDir;

packages/playwright-test/src/reporters/base.ts

@@ -17,8 +17,8 @@
 import { colors, ms as milliseconds, parseStackTraceLine } from 'playwright-core/lib/utilsBundle';
 import fs from 'fs';
 import path from 'path';
-import type { FullConfig, TestCase, Suite, TestResult, TestError, FullResult, TestStep, Location } from '../../types/testReporter';
-import type { FullConfigInternal, ReporterInternal } from '../types';
+import type { FullConfig, TestCase, Suite, TestResult, TestError, FullResult, TestStep, Location, Reporter } from '../../types/testReporter';
+import type { FullConfigInternal } from '../types';
 import { codeFrameColumns } from '../babelBundle';
 import { monotonicTime } from 'playwright-core/lib/utils';
 
@@ -46,7 +46,7 @@ type TestSummary = {
   fatalErrors: TestError[];
 };
 
-export class BaseReporter implements ReporterInternal  {
+export class BaseReporter implements Reporter {
   duration = 0;
   config!: FullConfigInternal;
   suite!: Suite;
@@ -63,9 +63,12 @@ export class BaseReporter implements ReporterInternal  {
     this._ttyWidthForTest = parseInt(process.env.PWTEST_TTY_WIDTH || '', 10);
   }
 
-  onBegin(config: FullConfig, suite: Suite) {
+  onConfigure(config: FullConfig) {
     this.monotonicStartTime = monotonicTime();
     this.config = config as FullConfigInternal;
+  }
+
+  onBegin(config: FullConfig, suite: Suite) {
     this.suite = suite;
     this.totalTestCount = suite.allTests().length;
   }

packages/playwright-test/src/reporters/html.ts

@@ -20,13 +20,13 @@ import { open } from '../utilsBundle';
 import path from 'path';
 import type { TransformCallback } from 'stream';
 import { Transform } from 'stream';
-import type { FullConfig, Suite } from '../../types/testReporter';
+import type { FullConfig, Reporter, Suite } from '../../types/testReporter';
 import { HttpServer, assert, calculateSha1, monotonicTime, copyFileAndMakeWritable, removeFolders } from 'playwright-core/lib/utils';
 import type { JsonAttachment, JsonReport, JsonSuite, JsonTestCase, JsonTestResult, JsonTestStep } from './raw';
 import RawReporter from './raw';
 import { stripAnsiEscapes } from './base';
 import { getPackageJsonPath, sanitizeForFilePath } from '../util';
-import type { FullConfigInternal, Metadata, ReporterInternal } from '../types';
+import type { FullConfigInternal, Metadata } from '../types';
 import type { ZipFile } from 'playwright-core/lib/zipBundle';
 import { yazl } from 'playwright-core/lib/zipBundle';
 import { mime } from 'playwright-core/lib/utilsBundle';
@@ -47,7 +47,7 @@ type HtmlReporterOptions = {
   port?: number,
 };
 
-class HtmlReporter implements ReporterInternal {
+class HtmlReporter implements Reporter {
   private config!: FullConfigInternal;
   private suite!: Suite;
   private _montonicStartTime: number = 0;
@@ -64,9 +64,12 @@ class HtmlReporter implements ReporterInternal {
     return false;
   }
 
-  onBegin(config: FullConfig, suite: Suite) {
+  onConfigure(config: FullConfig) {
     this._montonicStartTime = monotonicTime();
     this.config = config as FullConfigInternal;
+  }
+
+  onBegin(config: FullConfig, suite: Suite) {
     const { outputFolder, open } = this._resolveOptions();
     this._outputFolder = outputFolder;
     this._open = open;
@@ -112,7 +115,7 @@ class HtmlReporter implements ReporterInternal {
     this._buildResult = await builder.build({ ...this.config.metadata, duration }, reports);
   }
 
-  async _onExit() {
+  async onExit() {
     if (process.env.CI)
       return;
 

packages/playwright-test/src/reporters/json.ts

@@ -39,8 +39,11 @@ class JSONReporter implements Reporter {
     return !this._outputFile;
   }
 
-  onBegin(config: FullConfig, suite: Suite) {
+  onConfigure(config: FullConfig) {
     this.config = config;
+  }
+
+  onBegin(config: FullConfig, suite: Suite) {
     this.suite = suite;
   }
 

packages/playwright-test/src/reporters/junit.ts

@@ -48,8 +48,11 @@ class JUnitReporter implements Reporter {
     return !this.outputFile;
   }
 
-  onBegin(config: FullConfig, suite: Suite) {
+  onConfigure(config: FullConfig) {
     this.config = config;
+  }
+
+  onBegin(config: FullConfig, suite: Suite) {
     this.suite = suite;
     this.timestamp = Date.now();
     this.startTime = monotonicTime();

packages/playwright-test/src/reporters/multiplexer.ts

@@ -14,13 +14,12 @@
  * limitations under the License.
  */
 
-import type { FullConfig, Suite, TestCase, TestError, TestResult, FullResult, TestStep } from '../../types/testReporter';
-import type { ReporterInternal } from '../types';
+import type { FullConfig, Suite, TestCase, TestError, TestResult, FullResult, TestStep, Reporter } from '../../types/testReporter';
 
-export class Multiplexer implements ReporterInternal {
-  private _reporters: ReporterInternal[];
+export class Multiplexer implements Reporter {
+  private _reporters: Reporter[];
 
-  constructor(reporters: ReporterInternal[]) {
+  constructor(reporters: Reporter[]) {
     this._reporters = reporters;
   }
 
@@ -28,6 +27,11 @@ export class Multiplexer implements ReporterInternal {
     return this._reporters.some(r => r.printsToStdio ? r.printsToStdio() : true);
   }
 
+  onConfigure(config: FullConfig) {
+    for (const reporter of this._reporters)
+      reporter.onConfigure?.(config);
+  }
+
   onBegin(config: FullConfig, suite: Suite) {
     for (const reporter of this._reporters)
       reporter.onBegin?.(config, suite);
@@ -58,9 +62,9 @@ export class Multiplexer implements ReporterInternal {
       await Promise.resolve().then(() => reporter.onEnd?.(result)).catch(e => console.error('Error in reporter', e));
   }
 
-  async _onExit() {
+  async onExit() {
     for (const reporter of this._reporters)
-      await Promise.resolve().then(() => reporter._onExit?.()).catch(e => console.error('Error in reporter', e));
+      await Promise.resolve().then(() => reporter.onExit?.()).catch(e => console.error('Error in reporter', e));
   }
 
   onError(error: TestError) {

packages/playwright-test/src/runner.ts

@@ -17,7 +17,7 @@
 
 import * as fs from 'fs';
 import * as path from 'path';
-import { raceAgainstTimeout } from 'playwright-core/lib/utils';
+import { monotonicTime, raceAgainstTimeout } from 'playwright-core/lib/utils';
 import { colors, minimatch, rimraf } from 'playwright-core/lib/utilsBundle';
 import { promisify } from 'util';
 import type { FullResult, Reporter, TestError } from '../types/testReporter';
@@ -41,7 +41,7 @@ import { Multiplexer } from './reporters/multiplexer';
 import { SigIntWatcher } from './sigIntWatcher';
 import type { TestCase } from './test';
 import { Suite } from './test';
-import type { Config, FullConfigInternal, FullProjectInternal, ReporterInternal } from './types';
+import type { Config, FullConfigInternal, FullProjectInternal } from './types';
 import { createFileMatcher, createFileMatcherFromFilters, createTitleMatcher, serializeError } from './util';
 import type { Matcher, TestFileFilter } from './util';
 import { setFatalErrorSink } from './globals';
@@ -83,7 +83,7 @@ export type ConfigCLIOverrides = {
 
 export class Runner {
   private _configLoader: ConfigLoader;
-  private _reporter!: ReporterInternal;
+  private _reporter!: Reporter;
   private _plugins: TestRunnerPlugin[] = [];
   private _fatalErrors: TestError[] = [];
 
@@ -176,30 +176,6 @@ export class Runner {
     return new Multiplexer(reporters);
   }
 
-  async runAllTests(options: RunOptions): Promise<FullResult> {
-    this._reporter = await this._createReporter(!!options.listOnly);
-    const config = this._configLoader.fullConfig();
-    const result = await raceAgainstTimeout(() => this._run(options), config.globalTimeout);
-    let fullResult: FullResult;
-    if (result.timedOut) {
-      this._reporter.onError?.(createStacklessError(
-          `Timed out waiting ${config.globalTimeout / 1000}s for the entire test run`));
-      fullResult = { status: 'timedout' };
-    } else {
-      fullResult = result.result;
-    }
-    await this._reporter.onEnd?.(fullResult);
-
-    // Calling process.exit() might truncate large stdout/stderr output.
-    // See https://github.com/nodejs/node/issues/6456.
-    // See https://github.com/nodejs/node/issues/12921
-    await new Promise<void>(resolve => process.stdout.write('', () => resolve()));
-    await new Promise<void>(resolve => process.stderr.write('', () => resolve()));
-
-    await this._reporter._onExit?.();
-    return fullResult;
-  }
-
   async listTestFiles(projectNames: string[] | undefined): Promise<any> {
     const projects = this._collectProjects(projectNames);
     const filesByProject = await this._collectFiles(projects, []);
@@ -401,7 +377,77 @@ export class Runner {
     }
   }
 
-  private async _run(options: RunOptions): Promise<FullResult> {
+  async runAllTests(options: RunOptions): Promise<FullResult> {
+    this._reporter = await this._createReporter(!!options.listOnly);
+    const config = this._configLoader.fullConfig();
+    const deadline = config.globalTimeout ? monotonicTime() + config.globalTimeout : 1 << 30;
+
+    // Run configure.
+    this._reporter.onConfigure?.(config);
+
+    // Run global setup.
+    let globalTearDown: (() => Promise<void>) | undefined;
+    {
+      const remainingTime = deadline - monotonicTime();
+      const raceResult = await raceAgainstTimeout(async () => {
+        const result: FullResult = { status: 'passed' };
+        globalTearDown = await this._performGlobalSetup(config, result);
+        return result;
+      }, remainingTime);
+
+      let result: FullResult;
+      if (raceResult.timedOut) {
+        this._reporter.onError?.(createStacklessError(
+            `Timed out waiting ${config.globalTimeout / 1000}s for the global setup to run`));
+        result = { status: 'timedout' } as FullResult;
+      } else {
+        result = raceResult.result;
+      }
+      if (result.status !== 'passed')
+        return result;
+    }
+
+    // Run the tests.
+    let fullResult: FullResult;
+    {
+      const remainingTime = deadline - monotonicTime();
+      const raceResult = await raceAgainstTimeout(async () => {
+        try {
+          return await this._innerRun(options);
+        } catch (e) {
+          this._reporter.onError?.(serializeError(e));
+          return { status: 'failed' } as FullResult;
+        } finally {
+          await globalTearDown?.();
+        }
+      }, remainingTime);
+
+      // If timed out, bail.
+      let result: FullResult;
+      if (raceResult.timedOut) {
+        this._reporter.onError?.(createStacklessError(
+            `Timed out waiting ${config.globalTimeout / 1000}s for the entire test run`));
+        result = { status: 'timedout' };
+      } else {
+        result = raceResult.result;
+      }
+
+      // Report end.
+      await this._reporter.onEnd?.(result);
+      fullResult = result;
+    }
+
+    // Calling process.exit() might truncate large stdout/stderr output.
+    // See https://github.com/nodejs/node/issues/6456.
+    // See https://github.com/nodejs/node/issues/12921
+    await new Promise<void>(resolve => process.stdout.write('', () => resolve()));
+    await new Promise<void>(resolve => process.stderr.write('', () => resolve()));
+
+    await this._reporter.onExit?.();
+    return fullResult;
+  }
+
+  private async _innerRun(options: RunOptions): Promise<FullResult> {
     const config = this._configLoader.fullConfig();
     // Each entry is an array of test groups that can be run concurrently. All
     // test groups from the previos entries must finish before entry starts.
@@ -415,6 +461,10 @@ export class Runner {
 
     config._maxConcurrentTestGroups = testGroups.length;
 
+    const result: FullResult = { status: 'passed' };
+    for (const plugin of this._plugins)
+      await plugin.begin?.(rootSuite);
+
     // Report begin
     this._reporter.onBegin?.(config, rootSuite);
 
@@ -433,12 +483,6 @@ export class Runner {
     if (!await this._removeOutputDirs(options))
       return { status: 'failed' };
 
-    // Run Global setup.
-    const result: FullResult = { status: 'passed' };
-    const globalTearDown = await this._performGlobalSetup(config, rootSuite, result);
-    if (result.status !== 'passed')
-      return result;
-
     if (config._ignoreSnapshots) {
       this._reporter.onStdOut?.(colors.dim([
         'NOTE: running with "ignoreSnapshots" option. All of the following asserts are silently ignored:',
@@ -449,19 +493,12 @@ export class Runner {
     }
 
     // Run tests.
-    try {
-      const dispatchResult = await this._dispatchToWorkers(testGroups);
-      if (dispatchResult === 'signal') {
-        result.status = 'interrupted';
-      } else {
-        const failed = dispatchResult === 'workererror' || rootSuite.allTests().some(test => !test.ok());
-        result.status = failed ? 'failed' : 'passed';
-      }
-    } catch (e) {
-      this._reporter.onError?.(serializeError(e));
-      return { status: 'failed' };
-    } finally {
-      await globalTearDown?.();
+    const dispatchResult = await this._dispatchToWorkers(testGroups);
+    if (dispatchResult === 'signal') {
+      result.status = 'interrupted';
+    } else {
+      const failed = dispatchResult === 'workererror' || rootSuite.allTests().some(test => !test.ok());
+      result.status = failed ? 'failed' : 'passed';
     }
     return result;
   }
@@ -510,7 +547,7 @@ export class Runner {
     return true;
   }
 
-  private async _performGlobalSetup(config: FullConfigInternal, rootSuite: Suite, result: FullResult): Promise<(() => Promise<void>) | undefined> {
+  private async _performGlobalSetup(config: FullConfigInternal, result: FullResult): Promise<(() => Promise<void>) | undefined> {
     let globalSetupResult: any = undefined;
 
     const pluginsThatWereSetUp: TestRunnerPlugin[] = [];
@@ -545,7 +582,7 @@ export class Runner {
       // config's global setup.
       for (const plugin of this._plugins) {
         await Promise.race([
-          plugin.setup?.(config, config._configDir, rootSuite, this._reporter),
+          plugin.setup?.(config, config._configDir, this._reporter),
           sigintWatcher.promise(),
         ]);
         if (sigintWatcher.hadSignal())

packages/playwright-test/src/types.ts

@@ -15,7 +15,7 @@
  */
 
 import type { Fixtures, TestInfoError, Project } from '../types/test';
-import type { Location, Reporter } from '../types/testReporter';
+import type { Location } from '../types/testReporter';
 import type { FullConfig as FullConfigPublic, FullProject as FullProjectPublic } from './types';
 export * from '../types/test';
 export type { Location } from '../types/testReporter';
@@ -70,8 +70,4 @@ export interface FullProjectInternal extends FullProjectPublic {
   snapshotPathTemplate: string;
 }
 
-export interface ReporterInternal extends Reporter {
-  _onExit?(): void | Promise<void>;
-}
-
 export type ContextReuseMode = 'none' | 'force' | 'when-possible';

packages/playwright-test/types/testReporter.d.ts

@@ -349,6 +349,8 @@ export interface FullResult {
  * ```
  *
  * Here is a typical order of reporter calls:
+ * - [reporter.onConfigure(config)](https://playwright.dev/docs/api/class-reporter#reporter-on-configure) is called
+ *   once config has been resolved.
  * - [reporter.onBegin(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-on-begin) is called
  *   once with a root suite that contains all other suites and tests. Learn more about [suites hierarchy][Suite].
  * - [reporter.onTestBegin(test, result)](https://playwright.dev/docs/api/class-reporter#reporter-on-test-begin) is
@@ -365,6 +367,8 @@ export interface FullResult {
  *   [testResult.error](https://playwright.dev/docs/api/class-testresult#test-result-error) and more.
  * - [reporter.onEnd(result)](https://playwright.dev/docs/api/class-reporter#reporter-on-end) is called once after
  *   all tests that should run had finished.
+ * - [reporter.onExit()](https://playwright.dev/docs/api/class-reporter#reporter-on-exit) is called before test
+ *   runner exits.
  *
  * Additionally,
  * [reporter.onStdOut(chunk, test, result)](https://playwright.dev/docs/api/class-reporter#reporter-on-std-out) and
@@ -379,6 +383,11 @@ export interface FullResult {
  * to enhance user experience.
  */
 export interface Reporter {
+  /**
+   * Called once config is resolved.
+   * @param config Resolved configuration.
+   */
+  onConfigure?(config: FullConfig): void;
   /**
    * Called once before running tests. All tests have been already discovered and put into a hierarchy of [Suite]s.
    * @param config Resolved configuration.
@@ -397,6 +406,11 @@ export interface Reporter {
    * - `'interrupted'` - Interrupted by the user.
    */
   onEnd?(result: FullResult): void | Promise<void>;
+  /**
+   * Called before test runner exits.
+   */
+  onExit?(): void;
+
   /**
    * Called on some global error, for example unhandled exception in the worker process.
    * @param error The error.

tests/playwright-test/global-setup.spec.ts

@@ -137,11 +137,7 @@ test('globalTeardown does not run when globalSetup times out', async ({ runInlin
       });
     `,
   });
-  // We did not run tests, so we should only have 1 skipped test.
-  expect(result.skipped).toBe(1);
-  expect(result.passed).toBe(0);
-  expect(result.failed).toBe(0);
-  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain('Timed out waiting 1s for the global setup to run');
   expect(result.output).not.toContain('teardown=');
 });
 

utils/generate_types/overrides-testReporter.d.ts

@@ -44,6 +44,7 @@ export interface FullResult {
 }
 
 export interface Reporter {
+  onConfigure?(config: FullConfig): void;
   onBegin?(config: FullConfig, suite: Suite): void;
   onEnd?(result: FullResult): void | Promise<void>;
 }