diff --git a/docs/src/test-reporter-api/class-reporter.md b/docs/src/test-reporter-api/class-reporter.md index c657a5fe03..5f7760f72b 100644 --- a/docs/src/test-reporter-api/class-reporter.md +++ b/docs/src/test-reporter-api/class-reporter.md @@ -79,11 +79,13 @@ export default defineConfig({ ``` Here is a typical order of reporter calls: +* [`method: Reporter.onConfigure`] is called once config has been resolved. * [`method: Reporter.onBegin`] is called once with a root suite that contains all other suites and tests. Learn more about [suites hierarchy][Suite]. * [`method: Reporter.onTestBegin`] is called for each test run. It is given a [TestCase] that is executed, and a [TestResult] that is almost empty. Test result will be populated while the test runs (for example, with steps and stdio) and will get final `status` once the test finishes. * [`method: Reporter.onStepBegin`] and [`method: Reporter.onStepEnd`] are called for each executed step inside the test. When steps are executed, test run has not finished yet. * [`method: Reporter.onTestEnd`] is called when test run has finished. By this time, [TestResult] is complete and you can use [`property: TestResult.status`], [`property: TestResult.error`] and more. * [`method: Reporter.onEnd`] is called once after all tests that should run had finished. +* [`method: Reporter.onExit`] is called before test runner exits. Additionally, [`method: Reporter.onStdOut`] and [`method: Reporter.onStdErr`] are called when standard output is produced in the worker process, possibly during a test execution, and [`method: Reporter.onError`] is called when something went wrong outside of the test execution. @@ -107,7 +109,16 @@ Resolved configuration. The root suite that contains all projects, files and test cases. +## optional method: Reporter.onConfigure +* since: v1.30 +Called once config is resolved. + +### param: Reporter.onConfigure.config +* since: v1.30 +- `config` <[TestConfig]> + +Resolved configuration. ## optional async method: Reporter.onEnd * since: v1.10 @@ -125,8 +136,10 @@ Result of the full test run. * `'timedout'` - The [`property: TestConfig.globalTimeout`] has been reached. * `'interrupted'` - Interrupted by the user. +## optional method: Reporter.onExit +* since: v1.30 - +Called before test runner exits. ## optional method: Reporter.onError * since: v1.10 diff --git a/packages/playwright-test/src/plugins/dockerPlugin.ts b/packages/playwright-test/src/plugins/dockerPlugin.ts index a20b39e85e..399dfe8147 100644 --- a/packages/playwright-test/src/plugins/dockerPlugin.ts +++ b/packages/playwright-test/src/plugins/dockerPlugin.ts @@ -15,14 +15,14 @@ */ import type { TestRunnerPlugin } from '.'; -import type { FullConfig, Reporter, Suite } from '../../types/testReporter'; +import type { FullConfig, Reporter } from '../../types/testReporter'; import { colors } from 'playwright-core/lib/utilsBundle'; import { checkDockerEngineIsRunningOrDie, containerInfo } from 'playwright-core/lib/containers/docker'; export const dockerPlugin: TestRunnerPlugin = { name: 'playwright:docker', - async setup(config: FullConfig, configDir: string, rootSuite: Suite, reporter: Reporter) { + async setup(config: FullConfig, configDir: string, reporter: Reporter) { if (!process.env.PLAYWRIGHT_DOCKER) return; diff --git a/packages/playwright-test/src/plugins/index.ts b/packages/playwright-test/src/plugins/index.ts index 7255604749..729dca87d8 100644 --- a/packages/playwright-test/src/plugins/index.ts +++ b/packages/playwright-test/src/plugins/index.ts @@ -20,7 +20,8 @@ import type { FullConfig } from '../types'; export interface TestRunnerPlugin { name: string; - setup?(config: FullConfig, configDir: string, rootSuite: Suite, reporter: Reporter): Promise; + setup?(config: FullConfig, configDir: string, reporter: Reporter): Promise; + begin?(suite: Suite): Promise; teardown?(): Promise; } diff --git a/packages/playwright-test/src/plugins/vitePlugin.ts b/packages/playwright-test/src/plugins/vitePlugin.ts index 8ab189c24e..ddf1b4aab6 100644 --- a/packages/playwright-test/src/plugins/vitePlugin.ts +++ b/packages/playwright-test/src/plugins/vitePlugin.ts @@ -46,11 +46,16 @@ export function createPlugin( registerSourceFile: string, frameworkPluginFactory: () => Promise): TestRunnerPlugin { let configDir: string; + let config: FullConfig; return { name: 'playwright-vite-plugin', - setup: async (config: FullConfig, configDirectory: string, suite: Suite) => { + setup: async (configObject: FullConfig, configDirectory: string) => { + config = configObject; configDir = configDirectory; + }, + + begin: async (suite: Suite) => { const use = config.projects[0].use as CtConfig; const port = use.ctPort || 3100; const viteConfig: InlineConfig = use.ctViteConfig || {}; diff --git a/packages/playwright-test/src/plugins/webServerPlugin.ts b/packages/playwright-test/src/plugins/webServerPlugin.ts index c2ee57c669..b87095ffbf 100644 --- a/packages/playwright-test/src/plugins/webServerPlugin.ts +++ b/packages/playwright-test/src/plugins/webServerPlugin.ts @@ -21,7 +21,7 @@ import net from 'net'; import { debug } from 'playwright-core/lib/utilsBundle'; import { raceAgainstTimeout, launchProcess } from 'playwright-core/lib/utils'; -import type { FullConfig, Reporter, Suite } from '../../types/testReporter'; +import type { FullConfig, Reporter } from '../../types/testReporter'; import type { TestRunnerPlugin } from '.'; import type { FullConfigInternal } from '../types'; import { envWithoutExperimentalLoaderOptions } from '../cli'; @@ -57,7 +57,7 @@ export class WebServerPlugin implements TestRunnerPlugin { this._checkPortOnly = checkPortOnly; } - public async setup(config: FullConfig, configDir: string, rootSuite: Suite, reporter: Reporter) { + public async setup(config: FullConfig, configDir: string, reporter: Reporter) { this._reporter = reporter; this._isAvailable = getIsAvailableFunction(this._options.url, this._checkPortOnly, !!this._options.ignoreHTTPSErrors, this._reporter.onStdErr?.bind(this._reporter)); this._options.cwd = this._options.cwd ? path.resolve(configDir, this._options.cwd) : configDir; diff --git a/packages/playwright-test/src/reporters/base.ts b/packages/playwright-test/src/reporters/base.ts index 7acf9ce35f..031d61f695 100644 --- a/packages/playwright-test/src/reporters/base.ts +++ b/packages/playwright-test/src/reporters/base.ts @@ -17,8 +17,8 @@ import { colors, ms as milliseconds, parseStackTraceLine } from 'playwright-core/lib/utilsBundle'; import fs from 'fs'; import path from 'path'; -import type { FullConfig, TestCase, Suite, TestResult, TestError, FullResult, TestStep, Location } from '../../types/testReporter'; -import type { FullConfigInternal, ReporterInternal } from '../types'; +import type { FullConfig, TestCase, Suite, TestResult, TestError, FullResult, TestStep, Location, Reporter } from '../../types/testReporter'; +import type { FullConfigInternal } from '../types'; import { codeFrameColumns } from '../babelBundle'; import { monotonicTime } from 'playwright-core/lib/utils'; @@ -46,7 +46,7 @@ type TestSummary = { fatalErrors: TestError[]; }; -export class BaseReporter implements ReporterInternal { +export class BaseReporter implements Reporter { duration = 0; config!: FullConfigInternal; suite!: Suite; @@ -63,9 +63,12 @@ export class BaseReporter implements ReporterInternal { this._ttyWidthForTest = parseInt(process.env.PWTEST_TTY_WIDTH || '', 10); } - onBegin(config: FullConfig, suite: Suite) { + onConfigure(config: FullConfig) { this.monotonicStartTime = monotonicTime(); this.config = config as FullConfigInternal; + } + + onBegin(config: FullConfig, suite: Suite) { this.suite = suite; this.totalTestCount = suite.allTests().length; } diff --git a/packages/playwright-test/src/reporters/html.ts b/packages/playwright-test/src/reporters/html.ts index c800bc22ad..b1674efa9b 100644 --- a/packages/playwright-test/src/reporters/html.ts +++ b/packages/playwright-test/src/reporters/html.ts @@ -20,13 +20,13 @@ import { open } from '../utilsBundle'; import path from 'path'; import type { TransformCallback } from 'stream'; import { Transform } from 'stream'; -import type { FullConfig, Suite } from '../../types/testReporter'; +import type { FullConfig, Reporter, Suite } from '../../types/testReporter'; import { HttpServer, assert, calculateSha1, monotonicTime, copyFileAndMakeWritable, removeFolders } from 'playwright-core/lib/utils'; import type { JsonAttachment, JsonReport, JsonSuite, JsonTestCase, JsonTestResult, JsonTestStep } from './raw'; import RawReporter from './raw'; import { stripAnsiEscapes } from './base'; import { getPackageJsonPath, sanitizeForFilePath } from '../util'; -import type { FullConfigInternal, Metadata, ReporterInternal } from '../types'; +import type { FullConfigInternal, Metadata } from '../types'; import type { ZipFile } from 'playwright-core/lib/zipBundle'; import { yazl } from 'playwright-core/lib/zipBundle'; import { mime } from 'playwright-core/lib/utilsBundle'; @@ -47,7 +47,7 @@ type HtmlReporterOptions = { port?: number, }; -class HtmlReporter implements ReporterInternal { +class HtmlReporter implements Reporter { private config!: FullConfigInternal; private suite!: Suite; private _montonicStartTime: number = 0; @@ -64,9 +64,12 @@ class HtmlReporter implements ReporterInternal { return false; } - onBegin(config: FullConfig, suite: Suite) { + onConfigure(config: FullConfig) { this._montonicStartTime = monotonicTime(); this.config = config as FullConfigInternal; + } + + onBegin(config: FullConfig, suite: Suite) { const { outputFolder, open } = this._resolveOptions(); this._outputFolder = outputFolder; this._open = open; @@ -112,7 +115,7 @@ class HtmlReporter implements ReporterInternal { this._buildResult = await builder.build({ ...this.config.metadata, duration }, reports); } - async _onExit() { + async onExit() { if (process.env.CI) return; diff --git a/packages/playwright-test/src/reporters/json.ts b/packages/playwright-test/src/reporters/json.ts index 6f7cb71322..021480f928 100644 --- a/packages/playwright-test/src/reporters/json.ts +++ b/packages/playwright-test/src/reporters/json.ts @@ -39,8 +39,11 @@ class JSONReporter implements Reporter { return !this._outputFile; } - onBegin(config: FullConfig, suite: Suite) { + onConfigure(config: FullConfig) { this.config = config; + } + + onBegin(config: FullConfig, suite: Suite) { this.suite = suite; } diff --git a/packages/playwright-test/src/reporters/junit.ts b/packages/playwright-test/src/reporters/junit.ts index 4dc1076276..81e530504b 100644 --- a/packages/playwright-test/src/reporters/junit.ts +++ b/packages/playwright-test/src/reporters/junit.ts @@ -48,8 +48,11 @@ class JUnitReporter implements Reporter { return !this.outputFile; } - onBegin(config: FullConfig, suite: Suite) { + onConfigure(config: FullConfig) { this.config = config; + } + + onBegin(config: FullConfig, suite: Suite) { this.suite = suite; this.timestamp = Date.now(); this.startTime = monotonicTime(); diff --git a/packages/playwright-test/src/reporters/multiplexer.ts b/packages/playwright-test/src/reporters/multiplexer.ts index df55babd79..444abfc4fe 100644 --- a/packages/playwright-test/src/reporters/multiplexer.ts +++ b/packages/playwright-test/src/reporters/multiplexer.ts @@ -14,13 +14,12 @@ * limitations under the License. */ -import type { FullConfig, Suite, TestCase, TestError, TestResult, FullResult, TestStep } from '../../types/testReporter'; -import type { ReporterInternal } from '../types'; +import type { FullConfig, Suite, TestCase, TestError, TestResult, FullResult, TestStep, Reporter } from '../../types/testReporter'; -export class Multiplexer implements ReporterInternal { - private _reporters: ReporterInternal[]; +export class Multiplexer implements Reporter { + private _reporters: Reporter[]; - constructor(reporters: ReporterInternal[]) { + constructor(reporters: Reporter[]) { this._reporters = reporters; } @@ -28,6 +27,11 @@ export class Multiplexer implements ReporterInternal { return this._reporters.some(r => r.printsToStdio ? r.printsToStdio() : true); } + onConfigure(config: FullConfig) { + for (const reporter of this._reporters) + reporter.onConfigure?.(config); + } + onBegin(config: FullConfig, suite: Suite) { for (const reporter of this._reporters) reporter.onBegin?.(config, suite); @@ -58,9 +62,9 @@ export class Multiplexer implements ReporterInternal { await Promise.resolve().then(() => reporter.onEnd?.(result)).catch(e => console.error('Error in reporter', e)); } - async _onExit() { + async onExit() { for (const reporter of this._reporters) - await Promise.resolve().then(() => reporter._onExit?.()).catch(e => console.error('Error in reporter', e)); + await Promise.resolve().then(() => reporter.onExit?.()).catch(e => console.error('Error in reporter', e)); } onError(error: TestError) { diff --git a/packages/playwright-test/src/runner.ts b/packages/playwright-test/src/runner.ts index d1b37755a1..83b13f1f48 100644 --- a/packages/playwright-test/src/runner.ts +++ b/packages/playwright-test/src/runner.ts @@ -17,7 +17,7 @@ import * as fs from 'fs'; import * as path from 'path'; -import { raceAgainstTimeout } from 'playwright-core/lib/utils'; +import { monotonicTime, raceAgainstTimeout } from 'playwright-core/lib/utils'; import { colors, minimatch, rimraf } from 'playwright-core/lib/utilsBundle'; import { promisify } from 'util'; import type { FullResult, Reporter, TestError } from '../types/testReporter'; @@ -41,7 +41,7 @@ import { Multiplexer } from './reporters/multiplexer'; import { SigIntWatcher } from './sigIntWatcher'; import type { TestCase } from './test'; import { Suite } from './test'; -import type { Config, FullConfigInternal, FullProjectInternal, ReporterInternal } from './types'; +import type { Config, FullConfigInternal, FullProjectInternal } from './types'; import { createFileMatcher, createFileMatcherFromFilters, createTitleMatcher, serializeError } from './util'; import type { Matcher, TestFileFilter } from './util'; import { setFatalErrorSink } from './globals'; @@ -83,7 +83,7 @@ export type ConfigCLIOverrides = { export class Runner { private _configLoader: ConfigLoader; - private _reporter!: ReporterInternal; + private _reporter!: Reporter; private _plugins: TestRunnerPlugin[] = []; private _fatalErrors: TestError[] = []; @@ -176,30 +176,6 @@ export class Runner { return new Multiplexer(reporters); } - async runAllTests(options: RunOptions): Promise { - this._reporter = await this._createReporter(!!options.listOnly); - const config = this._configLoader.fullConfig(); - const result = await raceAgainstTimeout(() => this._run(options), config.globalTimeout); - let fullResult: FullResult; - if (result.timedOut) { - this._reporter.onError?.(createStacklessError( - `Timed out waiting ${config.globalTimeout / 1000}s for the entire test run`)); - fullResult = { status: 'timedout' }; - } else { - fullResult = result.result; - } - await this._reporter.onEnd?.(fullResult); - - // Calling process.exit() might truncate large stdout/stderr output. - // See https://github.com/nodejs/node/issues/6456. - // See https://github.com/nodejs/node/issues/12921 - await new Promise(resolve => process.stdout.write('', () => resolve())); - await new Promise(resolve => process.stderr.write('', () => resolve())); - - await this._reporter._onExit?.(); - return fullResult; - } - async listTestFiles(projectNames: string[] | undefined): Promise { const projects = this._collectProjects(projectNames); const filesByProject = await this._collectFiles(projects, []); @@ -401,7 +377,77 @@ export class Runner { } } - private async _run(options: RunOptions): Promise { + async runAllTests(options: RunOptions): Promise { + this._reporter = await this._createReporter(!!options.listOnly); + const config = this._configLoader.fullConfig(); + const deadline = config.globalTimeout ? monotonicTime() + config.globalTimeout : 1 << 30; + + // Run configure. + this._reporter.onConfigure?.(config); + + // Run global setup. + let globalTearDown: (() => Promise) | undefined; + { + const remainingTime = deadline - monotonicTime(); + const raceResult = await raceAgainstTimeout(async () => { + const result: FullResult = { status: 'passed' }; + globalTearDown = await this._performGlobalSetup(config, result); + return result; + }, remainingTime); + + let result: FullResult; + if (raceResult.timedOut) { + this._reporter.onError?.(createStacklessError( + `Timed out waiting ${config.globalTimeout / 1000}s for the global setup to run`)); + result = { status: 'timedout' } as FullResult; + } else { + result = raceResult.result; + } + if (result.status !== 'passed') + return result; + } + + // Run the tests. + let fullResult: FullResult; + { + const remainingTime = deadline - monotonicTime(); + const raceResult = await raceAgainstTimeout(async () => { + try { + return await this._innerRun(options); + } catch (e) { + this._reporter.onError?.(serializeError(e)); + return { status: 'failed' } as FullResult; + } finally { + await globalTearDown?.(); + } + }, remainingTime); + + // If timed out, bail. + let result: FullResult; + if (raceResult.timedOut) { + this._reporter.onError?.(createStacklessError( + `Timed out waiting ${config.globalTimeout / 1000}s for the entire test run`)); + result = { status: 'timedout' }; + } else { + result = raceResult.result; + } + + // Report end. + await this._reporter.onEnd?.(result); + fullResult = result; + } + + // Calling process.exit() might truncate large stdout/stderr output. + // See https://github.com/nodejs/node/issues/6456. + // See https://github.com/nodejs/node/issues/12921 + await new Promise(resolve => process.stdout.write('', () => resolve())); + await new Promise(resolve => process.stderr.write('', () => resolve())); + + await this._reporter.onExit?.(); + return fullResult; + } + + private async _innerRun(options: RunOptions): Promise { const config = this._configLoader.fullConfig(); // Each entry is an array of test groups that can be run concurrently. All // test groups from the previos entries must finish before entry starts. @@ -415,6 +461,10 @@ export class Runner { config._maxConcurrentTestGroups = testGroups.length; + const result: FullResult = { status: 'passed' }; + for (const plugin of this._plugins) + await plugin.begin?.(rootSuite); + // Report begin this._reporter.onBegin?.(config, rootSuite); @@ -433,12 +483,6 @@ export class Runner { if (!await this._removeOutputDirs(options)) return { status: 'failed' }; - // Run Global setup. - const result: FullResult = { status: 'passed' }; - const globalTearDown = await this._performGlobalSetup(config, rootSuite, result); - if (result.status !== 'passed') - return result; - if (config._ignoreSnapshots) { this._reporter.onStdOut?.(colors.dim([ 'NOTE: running with "ignoreSnapshots" option. All of the following asserts are silently ignored:', @@ -449,19 +493,12 @@ export class Runner { } // Run tests. - try { - const dispatchResult = await this._dispatchToWorkers(testGroups); - if (dispatchResult === 'signal') { - result.status = 'interrupted'; - } else { - const failed = dispatchResult === 'workererror' || rootSuite.allTests().some(test => !test.ok()); - result.status = failed ? 'failed' : 'passed'; - } - } catch (e) { - this._reporter.onError?.(serializeError(e)); - return { status: 'failed' }; - } finally { - await globalTearDown?.(); + const dispatchResult = await this._dispatchToWorkers(testGroups); + if (dispatchResult === 'signal') { + result.status = 'interrupted'; + } else { + const failed = dispatchResult === 'workererror' || rootSuite.allTests().some(test => !test.ok()); + result.status = failed ? 'failed' : 'passed'; } return result; } @@ -510,7 +547,7 @@ export class Runner { return true; } - private async _performGlobalSetup(config: FullConfigInternal, rootSuite: Suite, result: FullResult): Promise<(() => Promise) | undefined> { + private async _performGlobalSetup(config: FullConfigInternal, result: FullResult): Promise<(() => Promise) | undefined> { let globalSetupResult: any = undefined; const pluginsThatWereSetUp: TestRunnerPlugin[] = []; @@ -545,7 +582,7 @@ export class Runner { // config's global setup. for (const plugin of this._plugins) { await Promise.race([ - plugin.setup?.(config, config._configDir, rootSuite, this._reporter), + plugin.setup?.(config, config._configDir, this._reporter), sigintWatcher.promise(), ]); if (sigintWatcher.hadSignal()) diff --git a/packages/playwright-test/src/types.ts b/packages/playwright-test/src/types.ts index c3eb795722..1922460591 100644 --- a/packages/playwright-test/src/types.ts +++ b/packages/playwright-test/src/types.ts @@ -15,7 +15,7 @@ */ import type { Fixtures, TestInfoError, Project } from '../types/test'; -import type { Location, Reporter } from '../types/testReporter'; +import type { Location } from '../types/testReporter'; import type { FullConfig as FullConfigPublic, FullProject as FullProjectPublic } from './types'; export * from '../types/test'; export type { Location } from '../types/testReporter'; @@ -70,8 +70,4 @@ export interface FullProjectInternal extends FullProjectPublic { snapshotPathTemplate: string; } -export interface ReporterInternal extends Reporter { - _onExit?(): void | Promise; -} - export type ContextReuseMode = 'none' | 'force' | 'when-possible'; diff --git a/packages/playwright-test/types/testReporter.d.ts b/packages/playwright-test/types/testReporter.d.ts index 5697b94f3e..b2f7c0f6af 100644 --- a/packages/playwright-test/types/testReporter.d.ts +++ b/packages/playwright-test/types/testReporter.d.ts @@ -349,6 +349,8 @@ export interface FullResult { * ``` * * Here is a typical order of reporter calls: + * - [reporter.onConfigure(config)](https://playwright.dev/docs/api/class-reporter#reporter-on-configure) is called + * once config has been resolved. * - [reporter.onBegin(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-on-begin) is called * once with a root suite that contains all other suites and tests. Learn more about [suites hierarchy][Suite]. * - [reporter.onTestBegin(test, result)](https://playwright.dev/docs/api/class-reporter#reporter-on-test-begin) is @@ -365,6 +367,8 @@ export interface FullResult { * [testResult.error](https://playwright.dev/docs/api/class-testresult#test-result-error) and more. * - [reporter.onEnd(result)](https://playwright.dev/docs/api/class-reporter#reporter-on-end) is called once after * all tests that should run had finished. + * - [reporter.onExit()](https://playwright.dev/docs/api/class-reporter#reporter-on-exit) is called before test + * runner exits. * * Additionally, * [reporter.onStdOut(chunk, test, result)](https://playwright.dev/docs/api/class-reporter#reporter-on-std-out) and @@ -379,6 +383,11 @@ export interface FullResult { * to enhance user experience. */ export interface Reporter { + /** + * Called once config is resolved. + * @param config Resolved configuration. + */ + onConfigure?(config: FullConfig): void; /** * Called once before running tests. All tests have been already discovered and put into a hierarchy of [Suite]s. * @param config Resolved configuration. @@ -397,6 +406,11 @@ export interface Reporter { * - `'interrupted'` - Interrupted by the user. */ onEnd?(result: FullResult): void | Promise; + /** + * Called before test runner exits. + */ + onExit?(): void; + /** * Called on some global error, for example unhandled exception in the worker process. * @param error The error. diff --git a/tests/playwright-test/global-setup.spec.ts b/tests/playwright-test/global-setup.spec.ts index 6fbb51cbb1..8c01eeddf2 100644 --- a/tests/playwright-test/global-setup.spec.ts +++ b/tests/playwright-test/global-setup.spec.ts @@ -137,11 +137,7 @@ test('globalTeardown does not run when globalSetup times out', async ({ runInlin }); `, }); - // We did not run tests, so we should only have 1 skipped test. - expect(result.skipped).toBe(1); - expect(result.passed).toBe(0); - expect(result.failed).toBe(0); - expect(result.exitCode).toBe(1); + expect(result.output).toContain('Timed out waiting 1s for the global setup to run'); expect(result.output).not.toContain('teardown='); }); diff --git a/utils/generate_types/overrides-testReporter.d.ts b/utils/generate_types/overrides-testReporter.d.ts index 484c63a7f2..77834dbf71 100644 --- a/utils/generate_types/overrides-testReporter.d.ts +++ b/utils/generate_types/overrides-testReporter.d.ts @@ -44,6 +44,7 @@ export interface FullResult { } export interface Reporter { + onConfigure?(config: FullConfig): void; onBegin?(config: FullConfig, suite: Suite): void; onEnd?(result: FullResult): void | Promise; }