diff --git a/docs/src/test-reporter-api/class-reporter.md b/docs/src/test-reporter-api/class-reporter.md index 5f7760f72b..db05d7a7d6 100644 --- a/docs/src/test-reporter-api/class-reporter.md +++ b/docs/src/test-reporter-api/class-reporter.md @@ -79,13 +79,11 @@ export default defineConfig({ ``` Here is a typical order of reporter calls: -* [`method: Reporter.onConfigure`] is called once config has been resolved. * [`method: Reporter.onBegin`] is called once with a root suite that contains all other suites and tests. Learn more about [suites hierarchy][Suite]. * [`method: Reporter.onTestBegin`] is called for each test run. It is given a [TestCase] that is executed, and a [TestResult] that is almost empty. Test result will be populated while the test runs (for example, with steps and stdio) and will get final `status` once the test finishes. * [`method: Reporter.onStepBegin`] and [`method: Reporter.onStepEnd`] are called for each executed step inside the test. When steps are executed, test run has not finished yet. * [`method: Reporter.onTestEnd`] is called when test run has finished. By this time, [TestResult] is complete and you can use [`property: TestResult.status`], [`property: TestResult.error`] and more. * [`method: Reporter.onEnd`] is called once after all tests that should run had finished. -* [`method: Reporter.onExit`] is called before test runner exits. Additionally, [`method: Reporter.onStdOut`] and [`method: Reporter.onStdErr`] are called when standard output is produced in the worker process, possibly during a test execution, and [`method: Reporter.onError`] is called when something went wrong outside of the test execution. @@ -109,17 +107,6 @@ Resolved configuration. The root suite that contains all projects, files and test cases. -## optional method: Reporter.onConfigure -* since: v1.30 - -Called once config is resolved. - -### param: Reporter.onConfigure.config -* since: v1.30 -- `config` <[TestConfig]> - -Resolved configuration. - ## optional async method: Reporter.onEnd * since: v1.10 @@ -136,11 +123,6 @@ Result of the full test run. * `'timedout'` - The [`property: TestConfig.globalTimeout`] has been reached. * `'interrupted'` - Interrupted by the user. -## optional method: Reporter.onExit -* since: v1.30 - -Called before test runner exits. - ## optional method: Reporter.onError * since: v1.10 diff --git a/packages/playwright-test/src/cli.ts b/packages/playwright-test/src/cli.ts index 17a9d7bc4c..a1872a22dc 100644 --- a/packages/playwright-test/src/cli.ts +++ b/packages/playwright-test/src/cli.ts @@ -170,7 +170,7 @@ async function runTests(args: string[], opts: { [key: string]: any }) { const grepInvertMatcher = opts.grepInvert ? createTitleMatcher(forceRegExp(opts.grepInvert)) : () => false; const testTitleMatcher = (title: string) => !grepInvertMatcher(title) && grepMatcher(title); - const result = await runner.runAllTests({ + const status = await runner.runAllTests({ listOnly: !!opts.list, testFileFilters, testTitleMatcher, @@ -179,9 +179,9 @@ async function runTests(args: string[], opts: { [key: string]: any }) { }); await stopProfiling(undefined); - if (result.status === 'interrupted') + if (status === 'interrupted') process.exit(130); - process.exit(result.status === 'passed' ? 0 : 1); + process.exit(status === 'passed' ? 0 : 1); } diff --git a/packages/playwright-test/src/globals.ts b/packages/playwright-test/src/globals.ts index a14adbac1d..9468be9673 100644 --- a/packages/playwright-test/src/globals.ts +++ b/packages/playwright-test/src/globals.ts @@ -35,14 +35,14 @@ export function currentlyLoadingFileSuite() { return currentFileSuite; } -let _fatalErrors: TestError[] | undefined; -export function setFatalErrorSink(fatalErrors: TestError[]) { - _fatalErrors = fatalErrors; +let _fatalErrorSink: ((fatalError: TestError) => void) | undefined; +export function setFatalErrorSink(fatalErrorSink: (fatalError: TestError) => void) { + _fatalErrorSink = fatalErrorSink; } export function addFatalError(message: string, location: Location) { - if (_fatalErrors) - _fatalErrors.push({ message: `Error: ${message}`, location }); + if (_fatalErrorSink) + _fatalErrorSink({ message: `Error: ${message}`, location }); else throw new Error(`${formatLocation(location)}: ${message}`); } diff --git a/packages/playwright-test/src/loaderHost.ts b/packages/playwright-test/src/loaderHost.ts index 2b4220764d..0300697b1a 100644 --- a/packages/playwright-test/src/loaderHost.ts +++ b/packages/playwright-test/src/loaderHost.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import type { TestError } from '../reporter'; +import type { Reporter, TestError } from '../reporter'; import type { SerializedConfig } from './ipc'; import { ProcessHost } from './processHost'; import { Suite } from './test'; @@ -28,9 +28,9 @@ export class LoaderHost extends ProcessHost { await this.startRunner(config, true, {}); } - async loadTestFiles(files: string[], loadErrors: TestError[]): Promise { + async loadTestFiles(files: string[], reporter: Reporter): Promise { const result = await this.sendMessage({ method: 'loadTestFiles', params: { files } }) as any; - loadErrors.push(...result.loadErrors); + result.loadErrors.forEach((e: TestError) => reporter.onError?.(e)); return Suite._deepParse(result.rootSuite); } } diff --git a/packages/playwright-test/src/loaderMain.ts b/packages/playwright-test/src/loaderMain.ts index bc6dd88293..4991bfaaa8 100644 --- a/packages/playwright-test/src/loaderMain.ts +++ b/packages/playwright-test/src/loaderMain.ts @@ -38,7 +38,7 @@ export class LoaderMain extends ProcessRunner { async loadTestFiles(params: { files: string[] }) { const loadErrors: TestError[] = []; - setFatalErrorSink(loadErrors); + setFatalErrorSink(error => loadErrors.push(error)); const configLoader = await this._configLoader(); const rootSuite = await loadTestFilesInProcess(configLoader.fullConfig(), params.files, loadErrors); return { rootSuite: rootSuite._deepSerialize(), loadErrors }; diff --git a/packages/playwright-test/src/reporters/DEPS.list b/packages/playwright-test/src/reporters/DEPS.list index b9a2866feb..54a397388f 100644 --- a/packages/playwright-test/src/reporters/DEPS.list +++ b/packages/playwright-test/src/reporters/DEPS.list @@ -1,4 +1,5 @@ [*] -../util.ts ../babelBundle.ts +../test.ts +../util.ts ../utilsBundle.ts diff --git a/packages/playwright-test/src/reporters/base.ts b/packages/playwright-test/src/reporters/base.ts index 031d61f695..db58b0e7e9 100644 --- a/packages/playwright-test/src/reporters/base.ts +++ b/packages/playwright-test/src/reporters/base.ts @@ -63,12 +63,9 @@ export class BaseReporter implements Reporter { this._ttyWidthForTest = parseInt(process.env.PWTEST_TTY_WIDTH || '', 10); } - onConfigure(config: FullConfig) { + onBegin(config: FullConfig, suite: Suite) { this.monotonicStartTime = monotonicTime(); this.config = config as FullConfigInternal; - } - - onBegin(config: FullConfig, suite: Suite) { this.suite = suite; this.totalTestCount = suite.allTests().length; } diff --git a/packages/playwright-test/src/reporters/html.ts b/packages/playwright-test/src/reporters/html.ts index b1674efa9b..4672e78a26 100644 --- a/packages/playwright-test/src/reporters/html.ts +++ b/packages/playwright-test/src/reporters/html.ts @@ -64,12 +64,9 @@ class HtmlReporter implements Reporter { return false; } - onConfigure(config: FullConfig) { + onBegin(config: FullConfig, suite: Suite) { this._montonicStartTime = monotonicTime(); this.config = config as FullConfigInternal; - } - - onBegin(config: FullConfig, suite: Suite) { const { outputFolder, open } = this._resolveOptions(); this._outputFolder = outputFolder; this._open = open; @@ -116,10 +113,10 @@ class HtmlReporter implements Reporter { } async onExit() { - if (process.env.CI) + if (process.env.CI || !this._buildResult) return; - const { ok, singleTestId } = this._buildResult!; + const { ok, singleTestId } = this._buildResult; const shouldOpen = this._open === 'always' || (!ok && this._open === 'on-failure'); if (shouldOpen) { await showHTMLReport(this._outputFolder, this._options.host, this._options.port, singleTestId); diff --git a/packages/playwright-test/src/reporters/json.ts b/packages/playwright-test/src/reporters/json.ts index 021480f928..6f7cb71322 100644 --- a/packages/playwright-test/src/reporters/json.ts +++ b/packages/playwright-test/src/reporters/json.ts @@ -39,11 +39,8 @@ class JSONReporter implements Reporter { return !this._outputFile; } - onConfigure(config: FullConfig) { - this.config = config; - } - onBegin(config: FullConfig, suite: Suite) { + this.config = config; this.suite = suite; } diff --git a/packages/playwright-test/src/reporters/junit.ts b/packages/playwright-test/src/reporters/junit.ts index 81e530504b..4dc1076276 100644 --- a/packages/playwright-test/src/reporters/junit.ts +++ b/packages/playwright-test/src/reporters/junit.ts @@ -48,11 +48,8 @@ class JUnitReporter implements Reporter { return !this.outputFile; } - onConfigure(config: FullConfig) { - this.config = config; - } - onBegin(config: FullConfig, suite: Suite) { + this.config = config; this.suite = suite; this.timestamp = Date.now(); this.startTime = monotonicTime(); diff --git a/packages/playwright-test/src/reporters/multiplexer.ts b/packages/playwright-test/src/reporters/multiplexer.ts index 444abfc4fe..4615c4fb18 100644 --- a/packages/playwright-test/src/reporters/multiplexer.ts +++ b/packages/playwright-test/src/reporters/multiplexer.ts @@ -14,10 +14,22 @@ * limitations under the License. */ -import type { FullConfig, Suite, TestCase, TestError, TestResult, FullResult, TestStep, Reporter } from '../../types/testReporter'; +import type { FullConfig, TestCase, TestError, TestResult, FullResult, TestStep, Reporter } from '../../types/testReporter'; +import { Suite } from '../test'; + +type StdIOChunk = { + type: 'stdout' | 'stderr'; + chunk: string | Buffer; + test?: TestCase; + result?: TestResult; +}; export class Multiplexer implements Reporter { private _reporters: Reporter[]; + private _deferredErrors: TestError[] | null = []; + private _deferredStdIO: StdIOChunk[] | null = []; + hasErrors = false; + private _config!: FullConfig; constructor(reporters: Reporter[]) { this._reporters = reporters; @@ -28,13 +40,26 @@ export class Multiplexer implements Reporter { } onConfigure(config: FullConfig) { - for (const reporter of this._reporters) - reporter.onConfigure?.(config); + this._config = config; } onBegin(config: FullConfig, suite: Suite) { for (const reporter of this._reporters) reporter.onBegin?.(config, suite); + + const errors = this._deferredErrors!; + this._deferredErrors = null; + for (const error of errors) + this.onError(error); + + const stdios = this._deferredStdIO!; + this._deferredStdIO = null; + for (const stdio of stdios) { + if (stdio.type === 'stdout') + this.onStdOut(stdio.chunk, stdio.test, stdio.result); + else + this.onStdErr(stdio.chunk, stdio.test, stdio.result); + } } onTestBegin(test: TestCase, result: TestResult) { @@ -43,11 +68,20 @@ export class Multiplexer implements Reporter { } onStdOut(chunk: string | Buffer, test?: TestCase, result?: TestResult) { + if (this._deferredStdIO) { + this._deferredStdIO.push({ chunk, test, result, type: 'stdout' }); + return; + } for (const reporter of this._reporters) wrap(() => reporter.onStdOut?.(chunk, test, result)); } onStdErr(chunk: string | Buffer, test?: TestCase, result?: TestResult) { + if (this._deferredStdIO) { + this._deferredStdIO.push({ chunk, test, result, type: 'stderr' }); + return; + } + for (const reporter of this._reporters) wrap(() => reporter.onStdErr?.(chunk, test, result)); } @@ -57,17 +91,28 @@ export class Multiplexer implements Reporter { wrap(() => reporter.onTestEnd?.(test, result)); } - async onEnd(result: FullResult) { + async onEnd() { } + + async onExit(result: FullResult) { + if (this._deferredErrors) { + // onBegin was not reported, emit it. + this.onBegin(this._config, new Suite('', 'root')); + } + for (const reporter of this._reporters) await Promise.resolve().then(() => reporter.onEnd?.(result)).catch(e => console.error('Error in reporter', e)); - } - async onExit() { for (const reporter of this._reporters) - await Promise.resolve().then(() => reporter.onExit?.()).catch(e => console.error('Error in reporter', e)); + await Promise.resolve().then(() => (reporter as any).onExit?.()).catch(e => console.error('Error in reporter', e)); } onError(error: TestError) { + this.hasErrors = true; + + if (this._deferredErrors) { + this._deferredErrors.push(error); + return; + } for (const reporter of this._reporters) wrap(() => reporter.onError?.(error)); } diff --git a/packages/playwright-test/src/runner.ts b/packages/playwright-test/src/runner.ts index 83b13f1f48..69a1cfa308 100644 --- a/packages/playwright-test/src/runner.ts +++ b/packages/playwright-test/src/runner.ts @@ -17,7 +17,7 @@ import * as fs from 'fs'; import * as path from 'path'; -import { monotonicTime, raceAgainstTimeout } from 'playwright-core/lib/utils'; +import { monotonicTime } from 'playwright-core/lib/utils'; import { colors, minimatch, rimraf } from 'playwright-core/lib/utilsBundle'; import { promisify } from 'util'; import type { FullResult, Reporter, TestError } from '../types/testReporter'; @@ -38,7 +38,6 @@ import JUnitReporter from './reporters/junit'; import LineReporter from './reporters/line'; import ListReporter from './reporters/list'; import { Multiplexer } from './reporters/multiplexer'; -import { SigIntWatcher } from './sigIntWatcher'; import type { TestCase } from './test'; import { Suite } from './test'; import type { Config, FullConfigInternal, FullProjectInternal } from './types'; @@ -48,6 +47,7 @@ import { setFatalErrorSink } from './globals'; import { buildFileSuiteForProject, filterOnly, filterSuite, filterSuiteWithOnlySemantics, filterTestsRemoveEmptySuites } from './suiteUtils'; import { LoaderHost } from './loaderHost'; import { loadTestFilesInProcess } from './testLoader'; +import { TaskRunner } from './taskRunner'; const removeFolderAsync = promisify(rimraf); const readDirAsync = promisify(fs.readdir); @@ -55,7 +55,7 @@ const readFileAsync = promisify(fs.readFile); export const kDefaultConfigFiles = ['playwright.config.ts', 'playwright.config.js', 'playwright.config.mjs']; type RunOptions = { - listOnly?: boolean; + listOnly: boolean; testFileFilters: TestFileFilter[]; testTitleMatcher: Matcher; projectFilter?: string[]; @@ -83,14 +83,12 @@ export type ConfigCLIOverrides = { export class Runner { private _configLoader: ConfigLoader; - private _reporter!: Reporter; + private _reporter!: Multiplexer; private _plugins: TestRunnerPlugin[] = []; - private _fatalErrors: TestError[] = []; constructor(configCLIOverrides?: ConfigCLIOverrides) { this._configLoader = new ConfigLoader(configCLIOverrides); setRunnerToAddPluginsTo(this); - setFatalErrorSink(this._fatalErrors); } addPlugin(plugin: TestRunnerPlugin) { @@ -223,7 +221,6 @@ export class Runner { const filesByProject = new Map(); const fileToProjectName = new Map(); const commandLineFileMatcher = commandLineFileFilters.length ? createFileMatcherFromFilters(commandLineFileFilters) : () => true; - for (const project of projects) { const allFiles = await collectFiles(project.testDir, project._respectGitIgnore); const testMatch = createFileMatcher(project.testMatch); @@ -243,18 +240,10 @@ export class Runner { return filesByProject; } - private async _collectTestGroups(options: RunOptions): Promise<{ rootSuite: Suite, testGroups: TestGroup[] }> { + private async _loadAllTests(options: RunOptions): Promise<{ rootSuite: Suite, testGroups: TestGroup[] }> { const config = this._configLoader.fullConfig(); const projects = this._collectProjects(options.projectFilter); const filesByProject = await this._collectFiles(projects, options.testFileFilters); - const rootSuite = await this._createFilteredRootSuite(options, filesByProject); - - const testGroups = createTestGroups(rootSuite.suites, config.workers); - return { rootSuite, testGroups }; - } - - private async _createFilteredRootSuite(options: RunOptions, filesByProject: Map): Promise { - const config = this._configLoader.fullConfig(); const allTestFiles = new Set(); for (const files of filesByProject.values()) files.forEach(file => allTestFiles.add(file)); @@ -263,7 +252,7 @@ export class Runner { const preprocessRoot = await this._loadTests(allTestFiles); // Complain about duplicate titles. - this._fatalErrors.push(...createDuplicateTitlesErrors(config, preprocessRoot)); + createDuplicateTitlesErrors(config, preprocessRoot).forEach(e => this._reporter.onError(e)); // Filter tests to respect line/column filter. filterByFocusedLine(preprocessRoot, options.testFileFilters); @@ -272,13 +261,24 @@ export class Runner { if (config.forbidOnly) { const onlyTestsAndSuites = preprocessRoot._getOnlyItems(); if (onlyTestsAndSuites.length > 0) - this._fatalErrors.push(...createForbidOnlyErrors(config, onlyTestsAndSuites)); + createForbidOnlyErrors(config, onlyTestsAndSuites).forEach(e => this._reporter.onError(e)); } // Filter only. if (!options.listOnly) filterOnly(preprocessRoot); + const rootSuite = await this._createRootSuite(preprocessRoot, options, filesByProject); + + // Do not create test groups when listing. + if (options.listOnly) + return { rootSuite, testGroups: [] }; + + const testGroups = createTestGroups(rootSuite.suites, config.workers); + return { rootSuite, testGroups }; + } + + private async _createRootSuite(preprocessRoot: Suite, options: RunOptions, filesByProject: Map): Promise { // Generate projects. const fileSuites = new Map(); for (const fileSuite of preprocessRoot.suites) @@ -321,12 +321,17 @@ export class Runner { const loaderHost = new LoaderHost(); await loaderHost.start(this._configLoader.serializedConfig()); try { - return await loaderHost.loadTestFiles([...testFiles], this._fatalErrors); + return await loaderHost.loadTestFiles([...testFiles], this._reporter); } finally { await loaderHost.stop(); } } - return loadTestFilesInProcess(this._configLoader.fullConfig(), [...testFiles], this._fatalErrors); + const loadErrors: TestError[] = []; + try { + return await loadTestFilesInProcess(this._configLoader.fullConfig(), [...testFiles], loadErrors); + } finally { + loadErrors.forEach(e => this._reporter.onError(e)); + } } private _filterForCurrentShard(rootSuite: Suite, testGroups: TestGroup[]) { @@ -377,147 +382,131 @@ export class Runner { } } - async runAllTests(options: RunOptions): Promise { - this._reporter = await this._createReporter(!!options.listOnly); + async runAllTests(options: RunOptions): Promise { const config = this._configLoader.fullConfig(); - const deadline = config.globalTimeout ? monotonicTime() + config.globalTimeout : 1 << 30; + // Legacy webServer support. + this._plugins.push(...webServerPluginsForConfig(config)); + // Docker support. + this._plugins.push(dockerPlugin); - // Run configure. - this._reporter.onConfigure?.(config); + this._reporter = await this._createReporter(options.listOnly); + setFatalErrorSink(error => this._reporter.onError(error)); + const taskRunner = new TaskRunner(this._reporter, config.globalTimeout); - // Run global setup. - let globalTearDown: (() => Promise) | undefined; - { - const remainingTime = deadline - monotonicTime(); - const raceResult = await raceAgainstTimeout(async () => { - const result: FullResult = { status: 'passed' }; - globalTearDown = await this._performGlobalSetup(config, result); - return result; - }, remainingTime); - - let result: FullResult; - if (raceResult.timedOut) { - this._reporter.onError?.(createStacklessError( - `Timed out waiting ${config.globalTimeout / 1000}s for the global setup to run`)); - result = { status: 'timedout' } as FullResult; - } else { - result = raceResult.result; - } - if (result.status !== 'passed') - return result; + // Setup the plugins. + for (const plugin of this._plugins) { + taskRunner.addTask('plugin setup', async () => { + await plugin.setup?.(config, config._configDir, this._reporter); + return () => plugin.teardown?.(); + }); } - // Run the tests. - let fullResult: FullResult; - { - const remainingTime = deadline - monotonicTime(); - const raceResult = await raceAgainstTimeout(async () => { - try { - return await this._innerRun(options); - } catch (e) { - this._reporter.onError?.(serializeError(e)); - return { status: 'failed' } as FullResult; - } finally { - await globalTearDown?.(); + // Run global setup & teardown. + if (config.globalSetup || config.globalTeardown) { + taskRunner.addTask('global setup', async () => { + const setupHook = config.globalSetup ? await this._configLoader.loadGlobalHook(config.globalSetup) : undefined; + const teardownHook = config.globalTeardown ? await this._configLoader.loadGlobalHook(config.globalTeardown) : undefined; + const globalSetupResult = setupHook ? await setupHook(this._configLoader.fullConfig()) : undefined; + return async () => { + if (typeof globalSetupResult === 'function') + await globalSetupResult(); + await teardownHook?.(config); + }; + }); + } + + let status: FullResult['status'] = 'passed'; + + // Load tests. + let loadedTests!: { rootSuite: Suite, testGroups: TestGroup[] }; + taskRunner.addTask('load tests', async () => { + loadedTests = await this._loadAllTests(options); + if (this._reporter.hasErrors) { + status = 'failed'; + taskRunner.stop(); + return; + } + + // Fail when no tests. + if (!loadedTests.rootSuite.allTests().length && !options.passWithNoTests) { + this._reporter.onError(createNoTestsError()); + status = 'failed'; + taskRunner.stop(); + return; + } + + if (!options.listOnly) { + this._filterForCurrentShard(loadedTests.rootSuite, loadedTests.testGroups); + config._maxConcurrentTestGroups = loadedTests.testGroups.length; + } + }); + + if (!options.listOnly) { + taskRunner.addTask('prepare to run', async () => { + // Remove output directores. + if (!await this._removeOutputDirs(options)) { + status = 'failed'; + taskRunner.stop(); + return; } - }, remainingTime); + }); - // If timed out, bail. - let result: FullResult; - if (raceResult.timedOut) { - this._reporter.onError?.(createStacklessError( - `Timed out waiting ${config.globalTimeout / 1000}s for the entire test run`)); - result = { status: 'timedout' }; - } else { - result = raceResult.result; - } - - // Report end. - await this._reporter.onEnd?.(result); - fullResult = result; + taskRunner.addTask('plugin begin', async () => { + for (const plugin of this._plugins) + await plugin.begin?.(loadedTests.rootSuite); + }); } + taskRunner.addTask('report begin', async () => { + this._reporter.onBegin?.(config, loadedTests.rootSuite); + return async () => { + await this._reporter.onEnd(); + }; + }); + + if (!options.listOnly) { + let dispatcher: Dispatcher; + + taskRunner.addTask('setup workers', async () => { + const { rootSuite, testGroups } = loadedTests; + + if (config._ignoreSnapshots) { + this._reporter.onStdOut(colors.dim([ + 'NOTE: running with "ignoreSnapshots" option. All of the following asserts are silently ignored:', + '- expect().toMatchSnapshot()', + '- expect().toHaveScreenshot()', + '', + ].join('\n'))); + } + + dispatcher = new Dispatcher(this._configLoader, testGroups, this._reporter); + + return async () => { + // Stop will stop workers and mark some tests as interrupted. + await dispatcher.stop(); + if (dispatcher.hasWorkerErrors() || rootSuite.allTests().some(test => !test.ok())) + status = 'failed'; + }; + }); + + taskRunner.addTask('test suite', async () => { + await dispatcher.run(); + }); + } + + const deadline = config.globalTimeout ? monotonicTime() + config.globalTimeout : 0; + + this._reporter.onConfigure(config); + const taskStatus = await taskRunner.run(deadline); + if (status === 'passed' && taskStatus !== 'passed') + status = taskStatus; + await this._reporter.onExit({ status }); // Calling process.exit() might truncate large stdout/stderr output. // See https://github.com/nodejs/node/issues/6456. // See https://github.com/nodejs/node/issues/12921 await new Promise(resolve => process.stdout.write('', () => resolve())); await new Promise(resolve => process.stderr.write('', () => resolve())); - - await this._reporter.onExit?.(); - return fullResult; - } - - private async _innerRun(options: RunOptions): Promise { - const config = this._configLoader.fullConfig(); - // Each entry is an array of test groups that can be run concurrently. All - // test groups from the previos entries must finish before entry starts. - const { rootSuite, testGroups } = await this._collectTestGroups(options); - - // Fail when no tests. - if (!rootSuite.allTests().length && !options.passWithNoTests) - this._fatalErrors.push(createNoTestsError()); - - this._filterForCurrentShard(rootSuite, testGroups); - - config._maxConcurrentTestGroups = testGroups.length; - - const result: FullResult = { status: 'passed' }; - for (const plugin of this._plugins) - await plugin.begin?.(rootSuite); - - // Report begin - this._reporter.onBegin?.(config, rootSuite); - - // Bail out on errors prior to running global setup. - if (this._fatalErrors.length) { - for (const error of this._fatalErrors) - this._reporter.onError?.(error); - return { status: 'failed' }; - } - - // Bail out if list mode only, don't do any work. - if (options.listOnly) - return { status: 'passed' }; - - // Remove output directores. - if (!await this._removeOutputDirs(options)) - return { status: 'failed' }; - - if (config._ignoreSnapshots) { - this._reporter.onStdOut?.(colors.dim([ - 'NOTE: running with "ignoreSnapshots" option. All of the following asserts are silently ignored:', - '- expect().toMatchSnapshot()', - '- expect().toHaveScreenshot()', - '', - ].join('\n'))); - } - - // Run tests. - const dispatchResult = await this._dispatchToWorkers(testGroups); - if (dispatchResult === 'signal') { - result.status = 'interrupted'; - } else { - const failed = dispatchResult === 'workererror' || rootSuite.allTests().some(test => !test.ok()); - result.status = failed ? 'failed' : 'passed'; - } - return result; - } - - private async _dispatchToWorkers(stageGroups: TestGroup[]): Promise<'success'|'signal'|'workererror'> { - const dispatcher = new Dispatcher(this._configLoader, [...stageGroups], this._reporter); - const sigintWatcher = new SigIntWatcher(); - await Promise.race([dispatcher.run(), sigintWatcher.promise()]); - if (!sigintWatcher.hadSignal()) { - // We know for sure there was no Ctrl+C, so we remove custom SIGINT handler - // as soon as we can. - sigintWatcher.disarm(); - } - await dispatcher.stop(); - if (sigintWatcher.hadSignal()) - return 'signal'; - if (dispatcher.hasWorkerErrors()) - return 'workererror'; - return 'success'; + return status; } private async _removeOutputDirs(options: RunOptions): Promise { @@ -541,89 +530,11 @@ export class Runner { } }))); } catch (e) { - this._reporter.onError?.(serializeError(e)); + this._reporter.onError(serializeError(e)); return false; } return true; } - - private async _performGlobalSetup(config: FullConfigInternal, result: FullResult): Promise<(() => Promise) | undefined> { - let globalSetupResult: any = undefined; - - const pluginsThatWereSetUp: TestRunnerPlugin[] = []; - const sigintWatcher = new SigIntWatcher(); - - const tearDown = async () => { - await this._runAndReportError(async () => { - if (globalSetupResult && typeof globalSetupResult === 'function') - await globalSetupResult(this._configLoader.fullConfig()); - }, result); - - await this._runAndReportError(async () => { - if (globalSetupResult && config.globalTeardown) - await (await this._configLoader.loadGlobalHook(config.globalTeardown))(this._configLoader.fullConfig()); - }, result); - - for (const plugin of pluginsThatWereSetUp.reverse()) { - await this._runAndReportError(async () => { - await plugin.teardown?.(); - }, result); - } - }; - - // Legacy webServer support. - this._plugins.push(...webServerPluginsForConfig(config)); - - // Docker support. - this._plugins.push(dockerPlugin); - - await this._runAndReportError(async () => { - // First run the plugins, if plugin is a web server we want it to run before the - // config's global setup. - for (const plugin of this._plugins) { - await Promise.race([ - plugin.setup?.(config, config._configDir, this._reporter), - sigintWatcher.promise(), - ]); - if (sigintWatcher.hadSignal()) - break; - pluginsThatWereSetUp.push(plugin); - } - - // Then do global setup. - if (!sigintWatcher.hadSignal()) { - if (config.globalSetup) { - const hook = await this._configLoader.loadGlobalHook(config.globalSetup); - await Promise.race([ - Promise.resolve().then(() => hook(this._configLoader.fullConfig())).then((r: any) => globalSetupResult = r || ''), - sigintWatcher.promise(), - ]); - } else { - // Make sure we run the teardown. - globalSetupResult = ''; - } - } - }, result); - - sigintWatcher.disarm(); - - if (result.status !== 'passed' || sigintWatcher.hadSignal()) { - await tearDown(); - result.status = sigintWatcher.hadSignal() ? 'interrupted' : 'failed'; - return; - } - - return tearDown; - } - - private async _runAndReportError(callback: () => Promise, result: FullResult) { - try { - await callback(); - } catch (e) { - result.status = 'failed'; - this._reporter.onError?.(serializeError(e)); - } - } } function createFileMatcherFromFilter(filter: TestFileFilter) { diff --git a/packages/playwright-test/src/taskRunner.ts b/packages/playwright-test/src/taskRunner.ts new file mode 100644 index 0000000000..e1f04095cb --- /dev/null +++ b/packages/playwright-test/src/taskRunner.ts @@ -0,0 +1,132 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { debug } from 'playwright-core/lib/utilsBundle'; +import { ManualPromise, monotonicTime } from 'playwright-core/lib/utils'; +import type { FullResult, Reporter } from '../reporter'; +import { SigIntWatcher } from './sigIntWatcher'; +import { serializeError } from './util'; + +type TaskTeardown = () => Promise | undefined; +type Task = () => Promise | undefined; + +export class TaskRunner { + private _tasks: { name: string, task: Task }[] = []; + private _reporter: Reporter; + private _hasErrors = false; + private _interrupted = false; + private _isTearDown = false; + private _globalTimeoutForError: number; + + constructor(reporter: Reporter, globalTimeoutForError: number) { + this._reporter = reporter; + this._globalTimeoutForError = globalTimeoutForError; + } + + addTask(name: string, task: Task) { + this._tasks.push({ name, task }); + } + + stop() { + this._interrupted = true; + } + + async run(deadline: number): Promise { + const sigintWatcher = new SigIntWatcher(); + const timeoutWatcher = new TimeoutWatcher(deadline); + const teardownRunner = new TaskRunner(this._reporter, this._globalTimeoutForError); + teardownRunner._isTearDown = true; + try { + let currentTaskName: string | undefined; + + const taskLoop = async () => { + for (const { name, task } of this._tasks) { + currentTaskName = name; + if (this._interrupted) + break; + debug('pw:test:task')(`"${name}" started`); + try { + const teardown = await task(); + if (teardown) + teardownRunner._tasks.unshift({ name: `teardown for ${name}`, task: teardown }); + } catch (e) { + debug('pw:test:task')(`error in "${name}": `, e); + this._reporter.onError?.(serializeError(e)); + if (!this._isTearDown) + this._interrupted = true; + this._hasErrors = true; + } + debug('pw:test:task')(`"${name}" finished`); + } + }; + + await Promise.race([ + taskLoop(), + sigintWatcher.promise(), + timeoutWatcher.promise, + ]); + + // Prevent subsequent tasks from running. + this._interrupted = true; + + let status: FullResult['status'] = 'passed'; + if (sigintWatcher.hadSignal()) { + status = 'interrupted'; + } else if (timeoutWatcher.timedOut()) { + this._reporter.onError?.({ message: `Timed out waiting ${this._globalTimeoutForError / 1000}s for the ${currentTaskName} to run` }); + status = 'timedout'; + } else if (this._hasErrors) { + status = 'failed'; + } + + return status; + } finally { + sigintWatcher.disarm(); + timeoutWatcher.disarm(); + if (!this._isTearDown) + await teardownRunner.run(deadline); + } + } +} + +export class TimeoutWatcher { + private _timedOut = false; + readonly promise = new ManualPromise(); + private _timer: NodeJS.Timeout | undefined; + + constructor(deadline: number) { + if (!deadline) + return; + + if (deadline - monotonicTime() <= 0) { + this._timedOut = true; + this.promise.resolve(); + return; + } + this._timer = setTimeout(() => { + this._timedOut = true; + this.promise.resolve(); + }, deadline - monotonicTime()); + } + + timedOut(): boolean { + return this._timedOut; + } + + disarm() { + clearTimeout(this._timer); + } +} diff --git a/packages/playwright-test/types/testReporter.d.ts b/packages/playwright-test/types/testReporter.d.ts index b2f7c0f6af..5697b94f3e 100644 --- a/packages/playwright-test/types/testReporter.d.ts +++ b/packages/playwright-test/types/testReporter.d.ts @@ -349,8 +349,6 @@ export interface FullResult { * ``` * * Here is a typical order of reporter calls: - * - [reporter.onConfigure(config)](https://playwright.dev/docs/api/class-reporter#reporter-on-configure) is called - * once config has been resolved. * - [reporter.onBegin(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-on-begin) is called * once with a root suite that contains all other suites and tests. Learn more about [suites hierarchy][Suite]. * - [reporter.onTestBegin(test, result)](https://playwright.dev/docs/api/class-reporter#reporter-on-test-begin) is @@ -367,8 +365,6 @@ export interface FullResult { * [testResult.error](https://playwright.dev/docs/api/class-testresult#test-result-error) and more. * - [reporter.onEnd(result)](https://playwright.dev/docs/api/class-reporter#reporter-on-end) is called once after * all tests that should run had finished. - * - [reporter.onExit()](https://playwright.dev/docs/api/class-reporter#reporter-on-exit) is called before test - * runner exits. * * Additionally, * [reporter.onStdOut(chunk, test, result)](https://playwright.dev/docs/api/class-reporter#reporter-on-std-out) and @@ -383,11 +379,6 @@ export interface FullResult { * to enhance user experience. */ export interface Reporter { - /** - * Called once config is resolved. - * @param config Resolved configuration. - */ - onConfigure?(config: FullConfig): void; /** * Called once before running tests. All tests have been already discovered and put into a hierarchy of [Suite]s. * @param config Resolved configuration. @@ -406,11 +397,6 @@ export interface Reporter { * - `'interrupted'` - Interrupted by the user. */ onEnd?(result: FullResult): void | Promise; - /** - * Called before test runner exits. - */ - onExit?(): void; - /** * Called on some global error, for example unhandled exception in the worker process. * @param error The error. diff --git a/tests/playwright-test/exit-code.spec.ts b/tests/playwright-test/exit-code.spec.ts index 9126ba9af6..d007d67540 100644 --- a/tests/playwright-test/exit-code.spec.ts +++ b/tests/playwright-test/exit-code.spec.ts @@ -130,7 +130,7 @@ test('should respect global timeout', async ({ runInlineTest }) => { ` }, { 'timeout': 100000, 'global-timeout': 3000 }); expect(exitCode).toBe(1); - expect(output).toContain('Timed out waiting 3s for the entire test run'); + expect(output).toContain('Timed out waiting 3s for the test suite to run'); expect(monotonicTime() - now).toBeGreaterThan(2900); }); diff --git a/tests/playwright-test/global-setup.spec.ts b/tests/playwright-test/global-setup.spec.ts index 8c01eeddf2..8ff7737251 100644 --- a/tests/playwright-test/global-setup.spec.ts +++ b/tests/playwright-test/global-setup.spec.ts @@ -321,3 +321,65 @@ test('globalSetup auth should compile', async ({ runTSC }) => { const result = await runTSC(authFiles); expect(result.exitCode).toBe(0); }); + +test('teardown order', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + for (let i = 1; i < 4; ++i) { + pwt._addRunnerPlugin(() => ({ + setup: () => console.log('\\n%%setup ' + i), + teardown: () => console.log('\\n%%teardown ' + i), + })); + } + export default {}; + `, + 'a.test.js': ` + pwt.test('test', () => {}); + `, + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + expect(stripAnsi(result.output).split('\n').filter(line => line.startsWith('%%'))).toEqual([ + '%%setup 1', + '%%setup 2', + '%%setup 3', + '%%teardown 3', + '%%teardown 2', + '%%teardown 1', + ]); +}); + +test('teardown after error', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + for (let i = 1; i < 4; ++i) { + pwt._addRunnerPlugin(() => ({ + setup: () => console.log('\\n%%setup ' + i), + teardown: () => { + console.log('\\n%%teardown ' + i); + throw new Error('failed teardown ' + i) + }, + })); + } + export default {}; + `, + 'a.test.js': ` + pwt.test('test', () => {}); + `, + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + const output = stripAnsi(result.output); + expect(output).toContain('Error: failed teardown 1'); + expect(output).toContain('Error: failed teardown 2'); + expect(output).toContain('Error: failed teardown 3'); + expect(output).toContain('throw new Error(\'failed teardown'); + expect(stripAnsi(result.output).split('\n').filter(line => line.startsWith('%%'))).toEqual([ + '%%setup 1', + '%%setup 2', + '%%setup 3', + '%%teardown 3', + '%%teardown 2', + '%%teardown 1', + ]); +}); diff --git a/utils/generate_types/overrides-testReporter.d.ts b/utils/generate_types/overrides-testReporter.d.ts index 77834dbf71..484c63a7f2 100644 --- a/utils/generate_types/overrides-testReporter.d.ts +++ b/utils/generate_types/overrides-testReporter.d.ts @@ -44,7 +44,6 @@ export interface FullResult { } export interface Reporter { - onConfigure?(config: FullConfig): void; onBegin?(config: FullConfig, suite: Suite): void; onEnd?(result: FullResult): void | Promise; }