From 6aefa02e91370ce18e51030046e661ad6c25f38b Mon Sep 17 00:00:00 2001 From: Dmitry Gozman Date: Tue, 29 Jun 2021 10:55:46 -0700 Subject: [PATCH] feat(test runner): improve reporters api (#7370) - onEnd may return a Promise - onEnd now takes a result for the full run - onTimeout is replaced with onEnd(result) --- src/test/dispatcher.ts | 2 +- src/test/reporter.ts | 6 ++++-- src/test/reporters/base.ts | 15 ++++++--------- src/test/reporters/dot.ts | 11 +++-------- src/test/reporters/empty.ts | 5 ++--- src/test/reporters/json.ts | 8 ++------ src/test/reporters/junit.ts | 4 ++-- src/test/reporters/line.ts | 6 +++--- src/test/reporters/list.ts | 6 +++--- src/test/reporters/multiplexer.ts | 11 +++-------- src/test/runner.ts | 12 ++++++++---- tests/playwright-test/config.spec.ts | 3 ++- 12 files changed, 39 insertions(+), 50 deletions(-) diff --git a/src/test/dispatcher.ts b/src/test/dispatcher.ts index 2199a486fa..1b67dc67ba 100644 --- a/src/test/dispatcher.ts +++ b/src/test/dispatcher.ts @@ -171,7 +171,7 @@ export class Dispatcher { if (params.fatalError) { for (const { testId } of remaining) { const { test, result } = this._testById.get(testId)!; - this._reporter.onTestBegin?.(test); + this._reporter.onTestBegin(test); result.error = params.fatalError; this._reportTestEnd(test, result, 'failed'); failedTestIds.add(testId); diff --git a/src/test/reporter.ts b/src/test/reporter.ts index 6b15b9d46d..b58e9a39c3 100644 --- a/src/test/reporter.ts +++ b/src/test/reporter.ts @@ -60,13 +60,15 @@ export interface TestResult { stdout: (string | Buffer)[]; stderr: (string | Buffer)[]; } +export type FullResult = { + status: 'passed' | 'failed' | 'timedout' | 'interrupted'; +}; export interface Reporter { onBegin(config: FullConfig, suite: Suite): void; onTestBegin(test: Test): void; onStdOut(chunk: string | Buffer, test?: Test): void; onStdErr(chunk: string | Buffer, test?: Test): void; onTestEnd(test: Test, result: TestResult): void; - onTimeout(timeout: number): void; onError(error: TestError): void; - onEnd(): void; + onEnd(result: FullResult): void | Promise; } diff --git a/src/test/reporters/base.ts b/src/test/reporters/base.ts index 27776b75e7..fcf07ba133 100644 --- a/src/test/reporters/base.ts +++ b/src/test/reporters/base.ts @@ -21,7 +21,7 @@ import fs from 'fs'; import milliseconds from 'ms'; import path from 'path'; import StackUtils from 'stack-utils'; -import { FullConfig, TestStatus, Test, Spec, Suite, TestResult, TestError, Reporter } from '../reporter'; +import { FullConfig, TestStatus, Test, Spec, Suite, TestResult, TestError, Reporter, FullResult } from '../reporter'; const stackUtils = new StackUtils(); @@ -29,7 +29,7 @@ export class BaseReporter implements Reporter { duration = 0; config!: FullConfig; suite!: Suite; - timeout: number = 0; + result!: FullResult; fileDurations = new Map(); monotonicStartTime: number = 0; @@ -66,12 +66,9 @@ export class BaseReporter implements Reporter { console.log(formatError(error)); } - onTimeout(timeout: number) { - this.timeout = timeout; - } - - onEnd() { + async onEnd(result: FullResult) { this.duration = monotonicTime() - this.monotonicStartTime; + this.result = result; } private _printSlowTests() { @@ -123,8 +120,8 @@ export class BaseReporter implements Reporter { console.log(colors.yellow(` ${skipped} skipped`)); if (expected) console.log(colors.green(` ${expected} passed`) + colors.dim(` (${milliseconds(this.duration)})`)); - if (this.timeout) - console.log(colors.red(` Timed out waiting ${this.timeout / 1000}s for the entire test run`)); + if (this.result.status === 'timedout') + console.log(colors.red(` Timed out waiting ${this.config.globalTimeout / 1000}s for the entire test run`)); } private _printTestHeaders(tests: Test[]) { diff --git a/src/test/reporters/dot.ts b/src/test/reporters/dot.ts index 93739460ec..b17e1f2b9c 100644 --- a/src/test/reporters/dot.ts +++ b/src/test/reporters/dot.ts @@ -16,7 +16,7 @@ import colors from 'colors/safe'; import { BaseReporter } from './base'; -import { Test, TestResult } from '../reporter'; +import { FullResult, Test, TestResult } from '../reporter'; class DotReporter extends BaseReporter { private _counter = 0; @@ -42,13 +42,8 @@ class DotReporter extends BaseReporter { } } - onTimeout(timeout: number) { - super.onTimeout(timeout); - this.onEnd(); - } - - onEnd() { - super.onEnd(); + async onEnd(result: FullResult) { + await super.onEnd(result); process.stdout.write('\n'); this.epilogue(true); } diff --git a/src/test/reporters/empty.ts b/src/test/reporters/empty.ts index 4adc84034e..234531c1ea 100644 --- a/src/test/reporters/empty.ts +++ b/src/test/reporters/empty.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { FullConfig, TestResult, Test, Suite, TestError, Reporter } from '../reporter'; +import { FullConfig, TestResult, Test, Suite, TestError, Reporter, FullResult } from '../reporter'; class EmptyReporter implements Reporter { onBegin(config: FullConfig, suite: Suite) {} @@ -22,9 +22,8 @@ class EmptyReporter implements Reporter { onStdOut(chunk: string | Buffer, test?: Test) {} onStdErr(chunk: string | Buffer, test?: Test) {} onTestEnd(test: Test, result: TestResult) {} - onTimeout(timeout: number) {} onError(error: TestError) {} - onEnd() {} + async onEnd(result: FullResult) {} } export default EmptyReporter; diff --git a/src/test/reporters/json.ts b/src/test/reporters/json.ts index 2ecd590b5c..56f8092218 100644 --- a/src/test/reporters/json.ts +++ b/src/test/reporters/json.ts @@ -17,7 +17,7 @@ import fs from 'fs'; import path from 'path'; import EmptyReporter from './empty'; -import { FullConfig, Test, Suite, Spec, TestResult, TestError } from '../reporter'; +import { FullConfig, Test, Suite, Spec, TestResult, TestError, FullResult } from '../reporter'; interface SerializedSuite { title: string; @@ -50,15 +50,11 @@ class JSONReporter extends EmptyReporter { this.suite = suite; } - onTimeout() { - this.onEnd(); - } - onError(error: TestError): void { this._errors.push(error); } - onEnd() { + async onEnd(result: FullResult) { outputReport(this._serializeReport(), this._outputFile); } diff --git a/src/test/reporters/junit.ts b/src/test/reporters/junit.ts index 7c793bdc46..6b7bc56d30 100644 --- a/src/test/reporters/junit.ts +++ b/src/test/reporters/junit.ts @@ -17,7 +17,7 @@ import fs from 'fs'; import path from 'path'; import EmptyReporter from './empty'; -import { FullConfig, Suite, Test } from '../reporter'; +import { FullConfig, FullResult, Suite, Test } from '../reporter'; import { monotonicTime } from '../util'; import { formatFailure, formatTestTitle, stripAscii } from './base'; @@ -45,7 +45,7 @@ class JUnitReporter extends EmptyReporter { this.startTime = monotonicTime(); } - onEnd() { + async onEnd(result: FullResult) { const duration = monotonicTime() - this.startTime; const children: XMLEntry[] = []; for (const suite of this.suite.suites) diff --git a/src/test/reporters/line.ts b/src/test/reporters/line.ts index a53a1607bc..5da7f60ab8 100644 --- a/src/test/reporters/line.ts +++ b/src/test/reporters/line.ts @@ -16,7 +16,7 @@ import colors from 'colors/safe'; import { BaseReporter, formatFailure, formatTestTitle } from './base'; -import { FullConfig, Test, Suite, TestResult } from '../reporter'; +import { FullConfig, Test, Suite, TestResult, FullResult } from '../reporter'; class LineReporter extends BaseReporter { private _total = 0; @@ -64,9 +64,9 @@ class LineReporter extends BaseReporter { } } - onEnd() { + async onEnd(result: FullResult) { process.stdout.write(`\u001B[1A\u001B[2K`); - super.onEnd(); + await super.onEnd(result); this.epilogue(false); } } diff --git a/src/test/reporters/list.ts b/src/test/reporters/list.ts index d185fef36d..9b36f16e9a 100644 --- a/src/test/reporters/list.ts +++ b/src/test/reporters/list.ts @@ -19,7 +19,7 @@ import colors from 'colors/safe'; // @ts-ignore import milliseconds from 'ms'; import { BaseReporter, formatTestTitle } from './base'; -import { FullConfig, Suite, Test, TestResult } from '../reporter'; +import { FullConfig, FullResult, Suite, Test, TestResult } from '../reporter'; // Allow it in the Visual Studio Code Terminal and the new Windows Terminal const DOES_NOT_SUPPORT_UTF8_IN_TERMINAL = process.platform === 'win32' && process.env.TERM_PROGRAM !== 'vscode' && !process.env.WT_SESSION; @@ -107,8 +107,8 @@ class ListReporter extends BaseReporter { } } - onEnd() { - super.onEnd(); + async onEnd(result: FullResult) { + await super.onEnd(result); process.stdout.write('\n'); this.epilogue(true); } diff --git a/src/test/reporters/multiplexer.ts b/src/test/reporters/multiplexer.ts index 64a9162058..4c96dd8ab7 100644 --- a/src/test/reporters/multiplexer.ts +++ b/src/test/reporters/multiplexer.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { FullConfig, Suite, Test, TestError, TestResult, Reporter } from '../reporter'; +import { FullConfig, Suite, Test, TestError, TestResult, Reporter, FullResult } from '../reporter'; export class Multiplexer implements Reporter { private _reporters: Reporter[]; @@ -48,14 +48,9 @@ export class Multiplexer implements Reporter { reporter.onTestEnd(test, result); } - onTimeout(timeout: number) { + async onEnd(result: FullResult) { for (const reporter of this._reporters) - reporter.onTimeout(timeout); - } - - onEnd() { - for (const reporter of this._reporters) - reporter.onEnd(); + await reporter.onEnd(result); } onError(error: TestError) { diff --git a/src/test/runner.ts b/src/test/runner.ts index 50954e2d0b..934ae4f166 100644 --- a/src/test/runner.ts +++ b/src/test/runner.ts @@ -99,7 +99,7 @@ export class Runner { if (timedOut) { if (!this._didBegin) this._reporter.onBegin(config, new Suite('')); - this._reporter.onTimeout(config.globalTimeout); + await this._reporter.onEnd({ status: 'timedout' }); await this._flushOutput(); return 'failed'; } @@ -251,11 +251,15 @@ export class Runner { await dispatcher.stop(); hasWorkerErrors = dispatcher.hasWorkerErrors(); } - this._reporter.onEnd(); - if (sigint) + if (sigint) { + await this._reporter.onEnd({ status: 'interrupted' }); return { status: 'sigint' }; - return { status: hasWorkerErrors || rootSuite.findSpec(spec => !spec.ok()) ? 'failed' : 'passed' }; + } + + const failed = hasWorkerErrors || rootSuite.findSpec(spec => !spec.ok()); + await this._reporter.onEnd({ status: failed ? 'failed' : 'passed' }); + return { status: failed ? 'failed' : 'passed' }; } finally { if (globalSetupResult && typeof globalSetupResult === 'function') await globalSetupResult(this._loader.fullConfig()); diff --git a/tests/playwright-test/config.spec.ts b/tests/playwright-test/config.spec.ts index 84acfd3a8c..170de478f9 100644 --- a/tests/playwright-test/config.spec.ts +++ b/tests/playwright-test/config.spec.ts @@ -350,7 +350,8 @@ test('should work with custom reporter', async ({ runInlineTest }) => { onError() { console.log('\\n%%reporter-error%%'); } - onEnd() { + async onEnd() { + await new Promise(f => setTimeout(f, 500)); console.log('\\n%%reporter-end%%' + this.options.end); } }