diff --git a/packages/playwright-core/src/server/dispatchers/localUtilsDispatcher.ts b/packages/playwright-core/src/server/dispatchers/localUtilsDispatcher.ts index 25ba643015..42c73f2d95 100644 --- a/packages/playwright-core/src/server/dispatchers/localUtilsDispatcher.ts +++ b/packages/playwright-core/src/server/dispatchers/localUtilsDispatcher.ts @@ -102,7 +102,9 @@ export class LocalUtilsDispatcher extends Dispatcher<{ guid: string }, channels. // New file, just compress the entries. await fs.promises.mkdir(path.dirname(params.zipFile), { recursive: true }); zipFile.end(undefined, () => { - zipFile.outputStream.pipe(fs.createWriteStream(params.zipFile)).on('close', () => promise.resolve()); + zipFile.outputStream.pipe(fs.createWriteStream(params.zipFile)) + .on('close', () => promise.resolve()) + .on('error', error => promise.reject(error)); }); await promise; await this._deleteStackSession(params.stacksId); diff --git a/packages/playwright-core/src/server/trace/recorder/tracing.ts b/packages/playwright-core/src/server/trace/recorder/tracing.ts index 1656842016..25c5d915c2 100644 --- a/packages/playwright-core/src/server/trace/recorder/tracing.ts +++ b/packages/playwright-core/src/server/trace/recorder/tracing.ts @@ -329,7 +329,7 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps const artifact = new Artifact(this._context, zipFileName); artifact.reportFinished(); result.resolve(artifact); - }); + }).on('error', error => result.reject(error)); return result; } diff --git a/packages/playwright-core/src/utils/traceUtils.ts b/packages/playwright-core/src/utils/traceUtils.ts index 791744f01f..ca2a27ed59 100644 --- a/packages/playwright-core/src/utils/traceUtils.ts +++ b/packages/playwright-core/src/utils/traceUtils.ts @@ -91,7 +91,7 @@ export async function mergeTraceFiles(fileName: string, temporaryTraceFiles: str Promise.all(temporaryTraceFiles.map(tempFile => fs.promises.unlink(tempFile))).then(() => { mergePromise.resolve(); }); - }); + }).on('error', error => mergePromise.reject(error)); }); await mergePromise; } diff --git a/packages/playwright-test/src/isomorphic/teleReceiver.ts b/packages/playwright-test/src/isomorphic/teleReceiver.ts index fc67985fb4..44d02b0423 100644 --- a/packages/playwright-test/src/isomorphic/teleReceiver.ts +++ b/packages/playwright-test/src/isomorphic/teleReceiver.ts @@ -14,11 +14,12 @@ * limitations under the License. */ -import type { FullConfig, FullResult, Location, Reporter, TestError, TestResult, TestStatus, TestStep } from '../../types/testReporter'; +import type { FullConfig, FullResult, Location, TestError, TestResult, TestStatus, TestStep } from '../../types/testReporter'; import type { Annotation } from '../common/config'; import type { FullProject, Metadata } from '../../types/test'; import type * as reporterTypes from '../../types/testReporter'; import type { SuitePrivate } from '../../types/reporterPrivate'; +import type { ReporterV2 } from '../reporters/reporterV2'; export type JsonLocation = Location; export type JsonError = string; @@ -121,14 +122,16 @@ export type JsonEvent = { export class TeleReporterReceiver { private _rootSuite: TeleSuite; private _pathSeparator: string; - private _reporter: Reporter; + private _reporter: ReporterV2; private _tests = new Map(); private _rootDir!: string; + private _listOnly = false; private _clearPreviousResultsWhenTestBegins: boolean = false; private _reuseTestCases: boolean; private _reportConfig: MergeReporterConfig | undefined; + private _config!: FullConfig; - constructor(pathSeparator: string, reporter: Reporter, reuseTestCases: boolean, reportConfig?: MergeReporterConfig) { + constructor(pathSeparator: string, reporter: ReporterV2, reuseTestCases: boolean, reportConfig?: MergeReporterConfig) { this._rootSuite = new TeleSuite('', 'root'); this._pathSeparator = pathSeparator; this._reporter = reporter; @@ -136,10 +139,14 @@ export class TeleReporterReceiver { this._reportConfig = reportConfig; } - dispatch(message: JsonEvent): Promise | undefined { + dispatch(message: JsonEvent): Promise | void { const { method, params } = message; + if (method === 'onConfigure') { + this._onConfigure(params.config); + return; + } if (method === 'onBegin') { - this._onBegin(params.config, params.projects); + this._onBegin(params.projects); return; } if (method === 'onTestBegin') { @@ -176,8 +183,14 @@ export class TeleReporterReceiver { this._clearPreviousResultsWhenTestBegins = true; } - private _onBegin(config: JsonConfig, projects: JsonProject[]) { + private _onConfigure(config: JsonConfig) { this._rootDir = this._reportConfig?.rootDir || config.rootDir; + this._listOnly = config.listOnly; + this._config = this._parseConfig(config); + this._reporter.onConfigure(this._config); + } + + private _onBegin(projects: JsonProject[]) { for (const project of projects) { let projectSuite = this._rootSuite.suites.find(suite => suite.project()!.id === project.id); if (!projectSuite) { @@ -191,7 +204,7 @@ export class TeleReporterReceiver { // Remove deleted tests when listing. Empty suites will be auto-filtered // in the UI layer. - if (config.listOnly) { + if (this._listOnly) { const testIds = new Set(); const collectIds = (suite: JsonSuite) => { suite.tests.map(t => t.testId).forEach(testId => testIds.add(testId)); @@ -206,7 +219,7 @@ export class TeleReporterReceiver { filterTests(projectSuite); } } - this._reporter.onBegin?.(this._parseConfig(config), this._rootSuite); + this._reporter.onBegin?.(this._rootSuite); } private _onTestBegin(testId: string, payload: JsonTestResultStart) { @@ -289,11 +302,11 @@ export class TeleReporterReceiver { } } - private _onEnd(result: FullResult): Promise | undefined { - return this._reporter.onEnd?.(result) || undefined; + private _onEnd(result: FullResult): Promise | void { + return this._reporter.onEnd?.(result); } - private _onExit(): Promise | undefined { + private _onExit(): Promise | void { return this._reporter.onExit?.(); } diff --git a/packages/playwright-test/src/plugins/index.ts b/packages/playwright-test/src/plugins/index.ts index 19640084fb..58a1c2477e 100644 --- a/packages/playwright-test/src/plugins/index.ts +++ b/packages/playwright-test/src/plugins/index.ts @@ -15,11 +15,11 @@ */ import type { FullConfig, Suite } from '../../types/testReporter'; -import type { InternalReporter } from '../reporters/internalReporter'; +import type { ReporterV2 } from '../reporters/reporterV2'; export interface TestRunnerPlugin { name: string; - setup?(config: FullConfig, configDir: string, reporter: InternalReporter): Promise; + setup?(config: FullConfig, configDir: string, reporter: ReporterV2): Promise; begin?(suite: Suite): Promise; end?(): Promise; teardown?(): Promise; diff --git a/packages/playwright-test/src/plugins/webServerPlugin.ts b/packages/playwright-test/src/plugins/webServerPlugin.ts index f48fa379ed..d381c6c5bd 100644 --- a/packages/playwright-test/src/plugins/webServerPlugin.ts +++ b/packages/playwright-test/src/plugins/webServerPlugin.ts @@ -23,7 +23,7 @@ import type { FullConfig } from '../../types/testReporter'; import type { TestRunnerPlugin } from '.'; import type { FullConfigInternal } from '../common/config'; import { envWithoutExperimentalLoaderOptions } from '../util'; -import type { InternalReporter } from '../reporters/internalReporter'; +import type { ReporterV2 } from '../reporters/reporterV2'; export type WebServerPluginOptions = { @@ -50,7 +50,7 @@ export class WebServerPlugin implements TestRunnerPlugin { private _processExitedPromise!: Promise; private _options: WebServerPluginOptions; private _checkPortOnly: boolean; - private _reporter?: InternalReporter; + private _reporter?: ReporterV2; name = 'playwright:webserver'; constructor(options: WebServerPluginOptions, checkPortOnly: boolean) { @@ -58,7 +58,7 @@ export class WebServerPlugin implements TestRunnerPlugin { this._checkPortOnly = checkPortOnly; } - public async setup(config: FullConfig, configDir: string, reporter: InternalReporter) { + public async setup(config: FullConfig, configDir: string, reporter: ReporterV2) { this._reporter = reporter; this._isAvailable = getIsAvailableFunction(this._options.url, this._checkPortOnly, !!this._options.ignoreHTTPSErrors, this._reporter.onStdErr?.bind(this._reporter)); this._options.cwd = this._options.cwd ? path.resolve(configDir, this._options.cwd) : configDir; @@ -152,7 +152,7 @@ async function isPortUsed(port: number): Promise { return await innerIsPortUsed('127.0.0.1') || await innerIsPortUsed('::1'); } -async function isURLAvailable(url: URL, ignoreHTTPSErrors: boolean, onStdErr: InternalReporter['onStdErr']) { +async function isURLAvailable(url: URL, ignoreHTTPSErrors: boolean, onStdErr: ReporterV2['onStdErr']) { let statusCode = await httpStatusCode(url, ignoreHTTPSErrors, onStdErr); if (statusCode === 404 && url.pathname === '/') { const indexUrl = new URL(url); @@ -162,7 +162,7 @@ async function isURLAvailable(url: URL, ignoreHTTPSErrors: boolean, onStdErr: In return statusCode >= 200 && statusCode < 404; } -async function httpStatusCode(url: URL, ignoreHTTPSErrors: boolean, onStdErr: InternalReporter['onStdErr']): Promise { +async function httpStatusCode(url: URL, ignoreHTTPSErrors: boolean, onStdErr: ReporterV2['onStdErr']): Promise { return new Promise(resolve => { debugWebServer(`HTTP GET: ${url}`); httpRequest({ @@ -195,7 +195,7 @@ async function waitFor(waitFn: () => Promise, cancellationToken: { canc } } -function getIsAvailableFunction(url: string, checkPortOnly: boolean, ignoreHTTPSErrors: boolean, onStdErr: InternalReporter['onStdErr']) { +function getIsAvailableFunction(url: string, checkPortOnly: boolean, ignoreHTTPSErrors: boolean, onStdErr: ReporterV2['onStdErr']) { const urlObject = new URL(url); if (!checkPortOnly) return () => isURLAvailable(urlObject, ignoreHTTPSErrors, onStdErr); diff --git a/packages/playwright-test/src/reporters/base.ts b/packages/playwright-test/src/reporters/base.ts index 0eb11ad397..7481965c1e 100644 --- a/packages/playwright-test/src/reporters/base.ts +++ b/packages/playwright-test/src/reporters/base.ts @@ -118,7 +118,7 @@ export class BaseReporter implements Reporter { } protected generateStartingMessage() { - const jobs = this.config.workers; + const jobs = this.config.metadata.actualWorkers ?? this.config.workers; const shardDetails = this.config.shard ? `, shard ${this.config.shard.current} of ${this.config.shard.total}` : ''; if (!this.totalTestCount) return ''; diff --git a/packages/playwright-test/src/reporters/blob.ts b/packages/playwright-test/src/reporters/blob.ts index cb8e5b8eb2..6eb33b0536 100644 --- a/packages/playwright-test/src/reporters/blob.ts +++ b/packages/playwright-test/src/reporters/blob.ts @@ -57,15 +57,16 @@ export class BlobReporter extends TeleReporterEmitter { }); } - printsToStdio() { - return false; + override onConfigure(config: FullConfig) { + this._outputDir = path.resolve(this._options.configDir, this._options.outputDir || 'blob-report'); + this._reportName = this._computeReportName(config); + super.onConfigure(config); } - override onBegin(config: FullConfig<{}, {}>, suite: Suite): void { - this._outputDir = path.resolve(this._options.configDir, this._options.outputDir || 'blob-report'); + override onBegin(suite: Suite): void { + // Note: config.outputDir is cleared betwee onConfigure and onBegin, so we call mkdir here. fs.mkdirSync(path.join(this._outputDir, 'resources'), { recursive: true }); - this._reportName = this._computeReportName(config); - super.onBegin(config, suite); + super.onBegin(suite); } override async onEnd(result: FullResult): Promise { @@ -79,14 +80,16 @@ export class BlobReporter extends TeleReporterEmitter { const zipFileName = path.join(this._outputDir, this._reportName + '.zip'); zipFile.outputStream.pipe(fs.createWriteStream(zipFileName)).on('close', () => { zipFinishPromise.resolve(undefined); - }); + }).on('error', error => zipFinishPromise.reject(error)); zipFile.addReadStream(content, this._reportName + '.jsonl'); zipFile.end(); await Promise.all([ ...this._copyFilePromises, // Requires Node v14.18.0+ - zipFinishPromise.catch(e => console.error(`Failed to write report ${zipFileName}: ${e}`)) + zipFinishPromise.catch(e => { + throw new Error(`Failed to write report ${zipFileName}: ` + e.message); + }), ]); } diff --git a/packages/playwright-test/src/reporters/internalReporter.ts b/packages/playwright-test/src/reporters/internalReporter.ts index 485e01d05b..e7fbff9827 100644 --- a/packages/playwright-test/src/reporters/internalReporter.ts +++ b/packages/playwright-test/src/reporters/internalReporter.ts @@ -17,44 +17,29 @@ import fs from 'fs'; import { colors } from 'playwright-core/lib/utilsBundle'; import { codeFrameColumns } from '../transform/babelBundle'; -import type { FullConfig, TestCase, TestError, TestResult, FullResult, TestStep, Reporter } from '../../types/testReporter'; +import type { FullConfig, TestCase, TestError, TestResult, FullResult, TestStep } from '../../types/testReporter'; import { Suite } from '../common/test'; -import type { FullConfigInternal } from '../common/config'; import { Multiplexer } from './multiplexer'; import { prepareErrorStack, relativeFilePath } from './base'; +import type { ReporterV2 } from './reporterV2'; -type StdIOChunk = { - chunk: string | Buffer; - test?: TestCase; - result?: TestResult; -}; - -export class InternalReporter { +export class InternalReporter implements ReporterV2 { private _multiplexer: Multiplexer; - private _deferred: { error?: TestError, stdout?: StdIOChunk, stderr?: StdIOChunk }[] | null = []; - private _config!: FullConfigInternal; + private _didBegin = false; + private _config!: FullConfig; - constructor(reporters: Reporter[]) { + constructor(reporters: ReporterV2[]) { this._multiplexer = new Multiplexer(reporters); } - onConfigure(config: FullConfigInternal) { + onConfigure(config: FullConfig) { this._config = config; + this._multiplexer.onConfigure(config); } - onBegin(config: FullConfig, suite: Suite) { - this._multiplexer.onBegin(config, suite); - - const deferred = this._deferred!; - this._deferred = null; - for (const item of deferred) { - if (item.error) - this.onError(item.error); - if (item.stdout) - this.onStdOut(item.stdout.chunk, item.stdout.test, item.stdout.result); - if (item.stderr) - this.onStdErr(item.stderr.chunk, item.stderr.test, item.stderr.result); - } + onBegin(suite: Suite) { + this._didBegin = true; + this._multiplexer.onBegin(suite); } onTestBegin(test: TestCase, result: TestResult) { @@ -62,19 +47,10 @@ export class InternalReporter { } onStdOut(chunk: string | Buffer, test?: TestCase, result?: TestResult) { - if (this._deferred) { - this._deferred.push({ stdout: { chunk, test, result } }); - return; - } this._multiplexer.onStdOut(chunk, test, result); } onStdErr(chunk: string | Buffer, test?: TestCase, result?: TestResult) { - if (this._deferred) { - this._deferred.push({ stderr: { chunk, test, result } }); - return; - } - this._multiplexer.onStdErr(chunk, test, result); } @@ -84,9 +60,9 @@ export class InternalReporter { } async onEnd(result: FullResult) { - if (this._deferred) { + if (!this._didBegin) { // onBegin was not reported, emit it. - this.onBegin(this._config.config, new Suite('', 'root')); + this.onBegin(new Suite('', 'root')); } await this._multiplexer.onEnd(result); } @@ -96,11 +72,7 @@ export class InternalReporter { } onError(error: TestError) { - if (this._deferred) { - this._deferred.push({ error }); - return; - } - addLocationAndSnippetToError(this._config.config, error); + addLocationAndSnippetToError(this._config, error); this._multiplexer.onError(error); } @@ -113,14 +85,18 @@ export class InternalReporter { this._multiplexer.onStepEnd(test, result, step); } + printsToStdio() { + return this._multiplexer.printsToStdio(); + } + private _addSnippetToTestErrors(test: TestCase, result: TestResult) { for (const error of result.errors) - addLocationAndSnippetToError(this._config.config, error, test.location.file); + addLocationAndSnippetToError(this._config, error, test.location.file); } private _addSnippetToStepError(test: TestCase, step: TestStep) { if (step.error) - addLocationAndSnippetToError(this._config.config, step.error, test.location.file); + addLocationAndSnippetToError(this._config, step.error, test.location.file); } } diff --git a/packages/playwright-test/src/reporters/merge.ts b/packages/playwright-test/src/reporters/merge.ts index bb9bf7ece3..8f1fac6d94 100644 --- a/packages/playwright-test/src/reporters/merge.ts +++ b/packages/playwright-test/src/reporters/merge.ts @@ -73,13 +73,16 @@ async function extractReportFromZip(file: string): Promise { async function mergeEvents(dir: string, shardReportFiles: string[]) { const events: JsonEvent[] = []; + const configureEvents: JsonEvent[] = []; const beginEvents: JsonEvent[] = []; const endEvents: JsonEvent[] = []; for (const reportFile of shardReportFiles) { const reportJsonl = await extractReportFromZip(path.join(dir, reportFile)); const parsedEvents = parseEvents(reportJsonl); for (const event of parsedEvents) { - if (event.method === 'onBegin') + if (event.method === 'onConfigure') + configureEvents.push(event); + else if (event.method === 'onBegin') beginEvents.push(event); else if (event.method === 'onEnd') endEvents.push(event); @@ -89,13 +92,12 @@ async function mergeEvents(dir: string, shardReportFiles: string[]) { events.push(event); } } - return [mergeBeginEvents(beginEvents), ...events, mergeEndEvents(endEvents), { method: 'onExit', params: undefined }]; + return [mergeConfigureEvents(configureEvents), mergeBeginEvents(beginEvents), ...events, mergeEndEvents(endEvents), { method: 'onExit', params: undefined }]; } -function mergeBeginEvents(beginEvents: JsonEvent[]): JsonEvent { - if (!beginEvents.length) - throw new Error('No begin events found'); - const projects: JsonProject[] = []; +function mergeConfigureEvents(configureEvents: JsonEvent[]): JsonEvent { + if (!configureEvents.length) + throw new Error('No configure events found'); let config: JsonConfig = { configFile: undefined, globalTimeout: 0, @@ -108,8 +110,21 @@ function mergeBeginEvents(beginEvents: JsonEvent[]): JsonEvent { workers: 0, listOnly: false }; - for (const event of beginEvents) { + for (const event of configureEvents) config = mergeConfigs(config, event.params.config); + return { + method: 'onConfigure', + params: { + config, + } + }; +} + +function mergeBeginEvents(beginEvents: JsonEvent[]): JsonEvent { + if (!beginEvents.length) + throw new Error('No begin events found'); + const projects: JsonProject[] = []; + for (const event of beginEvents) { const shardProjects: JsonProject[] = event.params.projects; for (const shardProject of shardProjects) { const mergedProject = projects.find(p => p.id === shardProject.id); @@ -122,7 +137,6 @@ function mergeBeginEvents(beginEvents: JsonEvent[]): JsonEvent { return { method: 'onBegin', params: { - config, projects, } }; @@ -136,6 +150,7 @@ function mergeConfigs(to: JsonConfig, from: JsonConfig): JsonConfig { ...to.metadata, ...from.metadata, totalTime: to.metadata.totalTime + from.metadata.totalTime, + actualWorkers: (to.metadata.actualWorkers || 0) + (from.metadata.actualWorkers || 0), }, workers: to.workers + from.workers, }; diff --git a/packages/playwright-test/src/reporters/multiplexer.ts b/packages/playwright-test/src/reporters/multiplexer.ts index 2869af396c..48c89f4cee 100644 --- a/packages/playwright-test/src/reporters/multiplexer.ts +++ b/packages/playwright-test/src/reporters/multiplexer.ts @@ -14,64 +14,78 @@ * limitations under the License. */ -import type { FullConfig, TestCase, TestError, TestResult, FullResult, TestStep, Reporter } from '../../types/testReporter'; +import type { FullConfig, TestCase, TestError, TestResult, FullResult, TestStep } from '../../types/testReporter'; import type { Suite } from '../common/test'; +import type { ReporterV2 } from './reporterV2'; -export class Multiplexer implements Reporter { - private _reporters: Reporter[]; +export class Multiplexer implements ReporterV2 { + private _reporters: ReporterV2[]; - constructor(reporters: Reporter[]) { + constructor(reporters: ReporterV2[]) { this._reporters = reporters; } - onBegin(config: FullConfig, suite: Suite) { + onConfigure(config: FullConfig) { for (const reporter of this._reporters) - wrap(() => reporter.onBegin?.(config, suite)); + wrap(() => reporter.onConfigure(config)); + } + + onBegin(suite: Suite) { + for (const reporter of this._reporters) + wrap(() => reporter.onBegin(suite)); } onTestBegin(test: TestCase, result: TestResult) { for (const reporter of this._reporters) - wrap(() => reporter.onTestBegin?.(test, result)); + wrap(() => reporter.onTestBegin(test, result)); } onStdOut(chunk: string | Buffer, test?: TestCase, result?: TestResult) { for (const reporter of this._reporters) - wrap(() => reporter.onStdOut?.(chunk, test, result)); + wrap(() => reporter.onStdOut(chunk, test, result)); } onStdErr(chunk: string | Buffer, test?: TestCase, result?: TestResult) { for (const reporter of this._reporters) - wrap(() => reporter.onStdErr?.(chunk, test, result)); + wrap(() => reporter.onStdErr(chunk, test, result)); } onTestEnd(test: TestCase, result: TestResult) { for (const reporter of this._reporters) - wrap(() => reporter.onTestEnd?.(test, result)); + wrap(() => reporter.onTestEnd(test, result)); } async onEnd(result: FullResult) { for (const reporter of this._reporters) - await wrapAsync(() => reporter.onEnd?.(result)); + await wrapAsync(() => reporter.onEnd(result)); } async onExit() { for (const reporter of this._reporters) - await wrapAsync(() => reporter.onExit?.()); + await wrapAsync(() => reporter.onExit()); } onError(error: TestError) { for (const reporter of this._reporters) - wrap(() => reporter.onError?.(error)); + wrap(() => reporter.onError(error)); } onStepBegin(test: TestCase, result: TestResult, step: TestStep) { for (const reporter of this._reporters) - wrap(() => reporter.onStepBegin?.(test, result, step)); + wrap(() => reporter.onStepBegin(test, result, step)); } onStepEnd(test: TestCase, result: TestResult, step: TestStep) { for (const reporter of this._reporters) - wrap(() => reporter.onStepEnd?.(test, result, step)); + wrap(() => reporter.onStepEnd(test, result, step)); + } + + printsToStdio(): boolean { + return this._reporters.some(r => { + let prints = true; + wrap(() => prints = r.printsToStdio()); + return prints; + }); } } diff --git a/packages/playwright-test/src/reporters/reporterV2.ts b/packages/playwright-test/src/reporters/reporterV2.ts new file mode 100644 index 0000000000..6a59c729d5 --- /dev/null +++ b/packages/playwright-test/src/reporters/reporterV2.ts @@ -0,0 +1,119 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { FullConfig, TestCase, TestError, TestResult, FullResult, TestStep, Reporter, Suite } from '../../types/testReporter'; + +export interface ReporterV2 { + onConfigure(config: FullConfig): void; + onBegin(suite: Suite): void; + onTestBegin(test: TestCase, result: TestResult): void; + onStdOut(chunk: string | Buffer, test?: TestCase, result?: TestResult): void; + onStdErr(chunk: string | Buffer, test?: TestCase, result?: TestResult): void; + onTestEnd(test: TestCase, result: TestResult): void; + onEnd(result: FullResult): void | Promise; + onExit(): void | Promise; + onError(error: TestError): void; + onStepBegin(test: TestCase, result: TestResult, step: TestStep): void; + onStepEnd(test: TestCase, result: TestResult, step: TestStep): void; + printsToStdio(): boolean; +} + +type StdIOChunk = { + chunk: string | Buffer; + test?: TestCase; + result?: TestResult; +}; + +export class ReporterV2Wrapper implements ReporterV2 { + private _reporter: Reporter; + private _deferred: { error?: TestError, stdout?: StdIOChunk, stderr?: StdIOChunk }[] | null = []; + private _config!: FullConfig; + + constructor(reporter: Reporter) { + this._reporter = reporter; + } + + onConfigure(config: FullConfig) { + this._config = config; + } + + onBegin(suite: Suite) { + this._reporter.onBegin?.(this._config, suite); + + const deferred = this._deferred!; + this._deferred = null; + for (const item of deferred) { + if (item.error) + this.onError(item.error); + if (item.stdout) + this.onStdOut(item.stdout.chunk, item.stdout.test, item.stdout.result); + if (item.stderr) + this.onStdErr(item.stderr.chunk, item.stderr.test, item.stderr.result); + } + } + + onTestBegin(test: TestCase, result: TestResult) { + this._reporter.onTestBegin?.(test, result); + } + + onStdOut(chunk: string | Buffer, test?: TestCase, result?: TestResult) { + if (this._deferred) { + this._deferred.push({ stdout: { chunk, test, result } }); + return; + } + this._reporter.onStdOut?.(chunk, test, result); + } + + onStdErr(chunk: string | Buffer, test?: TestCase, result?: TestResult) { + if (this._deferred) { + this._deferred.push({ stderr: { chunk, test, result } }); + return; + } + this._reporter.onStdErr?.(chunk, test, result); + } + + onTestEnd(test: TestCase, result: TestResult) { + this._reporter.onTestEnd?.(test, result); + } + + async onEnd(result: FullResult) { + await this._reporter.onEnd?.(result); + } + + async onExit() { + await this._reporter.onExit?.(); + } + + onError(error: TestError) { + if (this._deferred) { + this._deferred.push({ error }); + return; + } + this._reporter.onError?.(error); + } + + onStepBegin(test: TestCase, result: TestResult, step: TestStep) { + this._reporter.onStepBegin?.(test, result, step); + } + + onStepEnd(test: TestCase, result: TestResult, step: TestStep) { + this._reporter.onStepEnd?.(test, result, step); + } + + printsToStdio() { + return this._reporter.printsToStdio ? this._reporter.printsToStdio() : true; + } +} diff --git a/packages/playwright-test/src/reporters/teleEmitter.ts b/packages/playwright-test/src/reporters/teleEmitter.ts index 338c340cd5..159800c317 100644 --- a/packages/playwright-test/src/reporters/teleEmitter.ts +++ b/packages/playwright-test/src/reporters/teleEmitter.ts @@ -17,13 +17,14 @@ import path from 'path'; import { createGuid } from 'playwright-core/lib/utils'; import type { SuitePrivate } from '../../types/reporterPrivate'; -import type { FullConfig, FullResult, Location, Reporter, TestError, TestResult, TestStep } from '../../types/testReporter'; +import type { FullConfig, FullResult, Location, TestError, TestResult, TestStep } from '../../types/testReporter'; import { FullConfigInternal, FullProjectInternal } from '../common/config'; import type { Suite, TestCase } from '../common/test'; import type { JsonAttachment, JsonConfig, JsonEvent, JsonProject, JsonStdIOType, JsonSuite, JsonTestCase, JsonTestEnd, JsonTestResultEnd, JsonTestResultStart, JsonTestStepEnd, JsonTestStepStart } from '../isomorphic/teleReceiver'; import { serializeRegexPatterns } from '../isomorphic/teleReceiver'; +import type { ReporterV2 } from './reporterV2'; -export class TeleReporterEmitter implements Reporter { +export class TeleReporterEmitter implements ReporterV2 { private _messageSink: (message: JsonEvent) => void; private _rootDir!: string; private _skipBuffers: boolean; @@ -33,10 +34,14 @@ export class TeleReporterEmitter implements Reporter { this._skipBuffers = skipBuffers; } - onBegin(config: FullConfig, suite: Suite) { + onConfigure(config: FullConfig) { this._rootDir = config.rootDir; + this._messageSink({ method: 'onConfigure', params: { config: this._serializeConfig(config) } }); + } + + onBegin(suite: Suite) { const projects = suite.suites.map(projectSuite => this._serializeProject(projectSuite)); - this._messageSink({ method: 'onBegin', params: { config: this._serializeConfig(config), projects } }); + this._messageSink({ method: 'onBegin', params: { projects } }); } onTestBegin(test: TestCase, result: TestResult): void { @@ -96,11 +101,11 @@ export class TeleReporterEmitter implements Reporter { }); } - onStdOut(chunk: string | Buffer, test: void | TestCase, result: void | TestResult): void { + onStdOut(chunk: string | Buffer, test?: TestCase, result?: TestResult): void { this._onStdIO('stdout', chunk, test, result); } - onStdErr(chunk: string | Buffer, test: void | TestCase, result: void | TestResult): void { + onStdErr(chunk: string | Buffer, test?: TestCase, result?: TestResult): void { this._onStdIO('stderr', chunk, test, result); } @@ -120,6 +125,10 @@ export class TeleReporterEmitter implements Reporter { async onExit() { } + printsToStdio() { + return false; + } + private _serializeConfig(config: FullConfig): JsonConfig { return { configFile: this._relativePath(config.configFile), diff --git a/packages/playwright-test/src/runner/dispatcher.ts b/packages/playwright-test/src/runner/dispatcher.ts index f32e9ab376..00544878e5 100644 --- a/packages/playwright-test/src/runner/dispatcher.ts +++ b/packages/playwright-test/src/runner/dispatcher.ts @@ -24,7 +24,7 @@ import { ManualPromise } from 'playwright-core/lib/utils'; import { WorkerHost } from './workerHost'; import type { TestGroup } from './testGroups'; import type { FullConfigInternal } from '../common/config'; -import type { InternalReporter } from '../reporters/internalReporter'; +import type { ReporterV2 } from '../reporters/reporterV2'; type TestResultData = { result: TestResult; @@ -46,14 +46,14 @@ export class Dispatcher { private _testById = new Map(); private _config: FullConfigInternal; - private _reporter: InternalReporter; + private _reporter: ReporterV2; private _hasWorkerErrors = false; private _failureCount = 0; private _extraEnvByProjectId: EnvByProjectId = new Map(); private _producedEnvByProjectId: EnvByProjectId = new Map(); - constructor(config: FullConfigInternal, reporter: InternalReporter) { + constructor(config: FullConfigInternal, reporter: ReporterV2) { this._config = config; this._reporter = reporter; } diff --git a/packages/playwright-test/src/runner/reporters.ts b/packages/playwright-test/src/runner/reporters.ts index de3d3af937..b19769415c 100644 --- a/packages/playwright-test/src/runner/reporters.ts +++ b/packages/playwright-test/src/runner/reporters.ts @@ -31,9 +31,10 @@ import type { BuiltInReporter, FullConfigInternal } from '../common/config'; import { loadReporter } from './loadUtils'; import { BlobReporter } from '../reporters/blob'; import type { ReporterDescription } from '../../types/test'; +import { type ReporterV2, ReporterV2Wrapper } from '../reporters/reporterV2'; -export async function createReporters(config: FullConfigInternal, mode: 'list' | 'run' | 'ui' | 'merge', descriptions?: ReporterDescription[]): Promise { - const defaultReporters: {[key in BuiltInReporter]: new(arg: any) => Reporter} = { +export async function createReporters(config: FullConfigInternal, mode: 'list' | 'run' | 'ui' | 'merge', descriptions?: ReporterDescription[]): Promise { + const defaultReporters: { [key in Exclude]: new(arg: any) => Reporter } = { dot: mode === 'list' ? ListModeReporter : DotReporter, line: mode === 'list' ? ListModeReporter : LineReporter, list: mode === 'list' ? ListModeReporter : ListReporter, @@ -42,42 +43,40 @@ export async function createReporters(config: FullConfigInternal, mode: 'list' | junit: JUnitReporter, null: EmptyReporter, html: mode === 'ui' ? LineReporter : HtmlReporter, - blob: BlobReporter, markdown: MarkdownReporter, }; - const reporters: Reporter[] = []; + const reporters: ReporterV2[] = []; descriptions ??= config.config.reporter; for (const r of descriptions) { const [name, arg] = r; const options = { ...arg, configDir: config.configDir }; - if (name in defaultReporters) { - reporters.push(new defaultReporters[name as keyof typeof defaultReporters](options)); + if (name === 'blob') { + reporters.push(new BlobReporter(options)); + } else if (name in defaultReporters) { + reporters.push(new ReporterV2Wrapper(new defaultReporters[name as keyof typeof defaultReporters](options))); } else { const reporterConstructor = await loadReporter(config, name); - reporters.push(new reporterConstructor(options)); + reporters.push(new ReporterV2Wrapper(new reporterConstructor(options))); } } if (process.env.PW_TEST_REPORTER) { const reporterConstructor = await loadReporter(config, process.env.PW_TEST_REPORTER); - reporters.push(new reporterConstructor()); + reporters.push(new ReporterV2Wrapper(new reporterConstructor())); } - const someReporterPrintsToStdio = reporters.some(r => { - const prints = r.printsToStdio ? r.printsToStdio() : true; - return prints; - }); + const someReporterPrintsToStdio = reporters.some(r => r.printsToStdio()); if (reporters.length && !someReporterPrintsToStdio) { // Add a line/dot/list-mode reporter for convenience. // Important to put it first, jsut in case some other reporter stalls onEnd. if (mode === 'list') - reporters.unshift(new ListModeReporter()); + reporters.unshift(new ReporterV2Wrapper(new ListModeReporter())); else - reporters.unshift(!process.env.CI ? new LineReporter({ omitFailures: true }) : new DotReporter()); + reporters.unshift(new ReporterV2Wrapper(!process.env.CI ? new LineReporter({ omitFailures: true }) : new DotReporter())); } return reporters; } -export class ListModeReporter implements Reporter { +class ListModeReporter implements Reporter { private config!: FullConfig; onBegin(config: FullConfig, suite: Suite): void { diff --git a/packages/playwright-test/src/runner/runner.ts b/packages/playwright-test/src/runner/runner.ts index 23f6d72b9f..082b2b9529 100644 --- a/packages/playwright-test/src/runner/runner.ts +++ b/packages/playwright-test/src/runner/runner.ts @@ -74,7 +74,7 @@ export class Runner { : createTaskRunner(config, reporter); const testRun = new TestRun(config, reporter); - reporter.onConfigure(config); + reporter.onConfigure(config.config); if (!listOnly && config.ignoreSnapshots) { reporter.onStdOut(colors.dim([ diff --git a/packages/playwright-test/src/runner/taskRunner.ts b/packages/playwright-test/src/runner/taskRunner.ts index 1cce1089db..98c24f7bf2 100644 --- a/packages/playwright-test/src/runner/taskRunner.ts +++ b/packages/playwright-test/src/runner/taskRunner.ts @@ -19,20 +19,20 @@ import { ManualPromise, monotonicTime } from 'playwright-core/lib/utils'; import type { FullResult, TestError } from '../../reporter'; import { SigIntWatcher } from './sigIntWatcher'; import { serializeError } from '../util'; -import type { InternalReporter } from '../reporters/internalReporter'; +import type { ReporterV2 } from '../reporters/reporterV2'; type TaskTeardown = () => Promise | undefined; export type Task = (context: Context, errors: TestError[], softErrors: TestError[]) => Promise | undefined; export class TaskRunner { private _tasks: { name: string, task: Task }[] = []; - private _reporter: InternalReporter; + private _reporter: ReporterV2; private _hasErrors = false; private _interrupted = false; private _isTearDown = false; private _globalTimeoutForError: number; - constructor(reporter: InternalReporter, globalTimeoutForError: number) { + constructor(reporter: ReporterV2, globalTimeoutForError: number) { this._reporter = reporter; this._globalTimeoutForError = globalTimeoutForError; } diff --git a/packages/playwright-test/src/runner/tasks.ts b/packages/playwright-test/src/runner/tasks.ts index d2ba6286aa..49a2963991 100644 --- a/packages/playwright-test/src/runner/tasks.ts +++ b/packages/playwright-test/src/runner/tasks.ts @@ -20,7 +20,7 @@ import { promisify } from 'util'; import { debug, rimraf } from 'playwright-core/lib/utilsBundle'; import { Dispatcher, type EnvByProjectId } from './dispatcher'; import type { TestRunnerPluginRegistration } from '../plugins'; -import type { InternalReporter } from '../reporters/internalReporter'; +import type { ReporterV2 } from '../reporters/reporterV2'; import { createTestGroups, type TestGroup } from '../runner/testGroups'; import type { Task } from './taskRunner'; import { TaskRunner } from './taskRunner'; @@ -46,7 +46,7 @@ export type Phase = { }; export class TestRun { - readonly reporter: InternalReporter; + readonly reporter: ReporterV2; readonly config: FullConfigInternal; rootSuite: Suite | undefined = undefined; readonly phases: Phase[] = []; @@ -55,13 +55,13 @@ export class TestRun { projectType: Map = new Map(); projectSuites: Map = new Map(); - constructor(config: FullConfigInternal, reporter: InternalReporter) { + constructor(config: FullConfigInternal, reporter: ReporterV2) { this.config = config; this.reporter = reporter; } } -export function createTaskRunner(config: FullConfigInternal, reporter: InternalReporter): TaskRunner { +export function createTaskRunner(config: FullConfigInternal, reporter: ReporterV2): TaskRunner { const taskRunner = new TaskRunner(reporter, config.config.globalTimeout); addGlobalSetupTasks(taskRunner, config); taskRunner.addTask('load tests', createLoadTask('in-process', { filterOnly: true, failOnLoadErrors: true })); @@ -69,13 +69,13 @@ export function createTaskRunner(config: FullConfigInternal, reporter: InternalR return taskRunner; } -export function createTaskRunnerForWatchSetup(config: FullConfigInternal, reporter: InternalReporter): TaskRunner { +export function createTaskRunnerForWatchSetup(config: FullConfigInternal, reporter: ReporterV2): TaskRunner { const taskRunner = new TaskRunner(reporter, 0); addGlobalSetupTasks(taskRunner, config); return taskRunner; } -export function createTaskRunnerForWatch(config: FullConfigInternal, reporter: InternalReporter, additionalFileMatcher?: Matcher): TaskRunner { +export function createTaskRunnerForWatch(config: FullConfigInternal, reporter: ReporterV2, additionalFileMatcher?: Matcher): TaskRunner { const taskRunner = new TaskRunner(reporter, 0); taskRunner.addTask('load tests', createLoadTask('out-of-process', { filterOnly: true, failOnLoadErrors: false, additionalFileMatcher })); addRunTasks(taskRunner, config); @@ -100,7 +100,7 @@ function addRunTasks(taskRunner: TaskRunner, config: FullConfigInternal return taskRunner; } -export function createTaskRunnerForList(config: FullConfigInternal, reporter: InternalReporter, mode: 'in-process' | 'out-of-process', options: { failOnLoadErrors: boolean }): TaskRunner { +export function createTaskRunnerForList(config: FullConfigInternal, reporter: ReporterV2, mode: 'in-process' | 'out-of-process', options: { failOnLoadErrors: boolean }): TaskRunner { const taskRunner = new TaskRunner(reporter, config.config.globalTimeout); taskRunner.addTask('load tests', createLoadTask(mode, { ...options, filterOnly: false })); taskRunner.addTask('report begin', createReportBeginTask()); @@ -110,7 +110,7 @@ export function createTaskRunnerForList(config: FullConfigInternal, reporter: In function createReportBeginTask(): Task { return async ({ config, reporter, rootSuite }) => { const montonicStartTime = monotonicTime(); - reporter.onBegin(config.config, rootSuite!); + reporter.onBegin(rootSuite!); return async () => { config.config.metadata.totalTime = monotonicTime() - montonicStartTime; }; @@ -228,7 +228,7 @@ function createPhasesTask(): Task { } } - testRun.config.config.workers = Math.min(testRun.config.config.workers, maxConcurrentTestGroups); + testRun.config.config.metadata.actualWorkers = Math.min(testRun.config.config.workers, maxConcurrentTestGroups); }; } diff --git a/packages/playwright-test/src/runner/uiMode.ts b/packages/playwright-test/src/runner/uiMode.ts index 834cc2b0ec..d5b8cc181a 100644 --- a/packages/playwright-test/src/runner/uiMode.ts +++ b/packages/playwright-test/src/runner/uiMode.ts @@ -28,6 +28,7 @@ import type { FSWatcher } from 'chokidar'; import { open } from 'playwright-core/lib/utilsBundle'; import ListReporter from '../reporters/list'; import type { OpenTraceViewerOptions, Transport } from 'playwright-core/lib/server/trace/viewer/traceViewer'; +import { ReporterV2Wrapper } from '../reporters/reporterV2'; class UIMode { private _config: FullConfigInternal; @@ -67,9 +68,9 @@ class UIMode { } async runGlobalSetup(): Promise { - const reporter = new InternalReporter([new ListReporter()]); + const reporter = new InternalReporter([new ReporterV2Wrapper(new ListReporter())]); const taskRunner = createTaskRunnerForWatchSetup(this._config, reporter); - reporter.onConfigure(this._config); + reporter.onConfigure(this._config.config); const testRun = new TestRun(this._config, reporter); const { status, cleanup: globalCleanup } = await taskRunner.runDeferCleanup(testRun, 0); await reporter.onEnd({ status }); @@ -167,7 +168,7 @@ class UIMode { const taskRunner = createTaskRunnerForList(this._config, reporter, 'out-of-process', { failOnLoadErrors: false }); const testRun = new TestRun(this._config, reporter); clearCompilationCache(); - reporter.onConfigure(this._config); + reporter.onConfigure(this._config.config); const status = await taskRunner.run(testRun, 0); await reporter.onEnd({ status }); await reporter.onExit(); @@ -191,7 +192,7 @@ class UIMode { const taskRunner = createTaskRunnerForWatch(this._config, reporter); const testRun = new TestRun(this._config, reporter); clearCompilationCache(); - reporter.onConfigure(this._config); + reporter.onConfigure(this._config.config); const stop = new ManualPromise(); const run = taskRunner.run(testRun, 0, stop).then(async status => { await reporter.onEnd({ status }); diff --git a/packages/playwright-test/src/runner/watchMode.ts b/packages/playwright-test/src/runner/watchMode.ts index 11d633019e..4bd8ffa52c 100644 --- a/packages/playwright-test/src/runner/watchMode.ts +++ b/packages/playwright-test/src/runner/watchMode.ts @@ -31,6 +31,7 @@ import { enquirer } from '../utilsBundle'; import { separator } from '../reporters/base'; import { PlaywrightServer } from 'playwright-core/lib/remote/playwrightServer'; import ListReporter from '../reporters/list'; +import { ReporterV2Wrapper } from '../reporters/reporterV2'; class FSWatcher { private _dirtyTestFiles = new Map>(); @@ -112,10 +113,10 @@ export async function runWatchModeLoop(config: FullConfigInternal): Promise => { return sendMessage('list', {}); let rootSuite: Suite; - let loadErrors: TestError[]; + const loadErrors: TestError[] = []; const progress: Progress = { passed: 0, failed: 0, @@ -606,12 +606,13 @@ const refreshRootSuite = (eraseResults: boolean): Promise => { }; let config: FullConfig; receiver = new TeleReporterReceiver(pathSeparator, { - onBegin: (c: FullConfig, suite: Suite) => { - if (!rootSuite) { - rootSuite = suite; - loadErrors = []; - } + onConfigure: (c: FullConfig) => { config = c; + }, + + onBegin: (suite: Suite) => { + if (!rootSuite) + rootSuite = suite; progress.passed = 0; progress.failed = 0; progress.skipped = 0; @@ -639,8 +640,18 @@ const refreshRootSuite = (eraseResults: boolean): Promise => { onError: (error: TestError) => { xtermDataSource.write((error.stack || error.value || '') + '\n'); loadErrors.push(error); - throttleUpdateRootSuite(config, rootSuite, loadErrors, progress); + throttleUpdateRootSuite(config, rootSuite ?? new TeleSuite('', 'root'), loadErrors, progress); }, + + printsToStdio: () => { + return false; + }, + + onStdOut: () => {}, + onStdErr: () => {}, + onExit: () => {}, + onStepBegin: () => {}, + onStepEnd: () => {}, }, true); receiver._setClearPreviousResultsWhenTestBegins(); return sendMessage('list', {}); diff --git a/tests/playwright-test/reporter-blob.spec.ts b/tests/playwright-test/reporter-blob.spec.ts index 53ad62c7d7..c33b6d47cd 100644 --- a/tests/playwright-test/reporter-blob.spec.ts +++ b/tests/playwright-test/reporter-blob.spec.ts @@ -836,8 +836,8 @@ test('preserve config fields', async ({ runInlineTest, mergeReports }) => { ` }; - await runInlineTest(files, { shard: `1/3` }); - await runInlineTest(files, { shard: `3/3` }); + await runInlineTest(files, { shard: `1/3`, workers: 1 }); + await runInlineTest(files, { shard: `3/3`, workers: 1 }); const mergeConfig = { reportSlowTests: {