diff --git a/test-runner/src/cli.ts b/test-runner/src/cli.ts index aa749d49e5..7611540d62 100644 --- a/test-runner/src/cli.ts +++ b/test-runner/src/cli.ts @@ -17,7 +17,7 @@ import program from 'commander'; import * as fs from 'fs'; import * as path from 'path'; -import { collectTests, runTests, RunnerConfig } from '.'; +import { run, RunnerConfig } from '.'; import PytestReporter from './reporters/pytest'; import DotReporter from './reporters/dot'; import ListReporter from './reporters/list'; @@ -48,6 +48,7 @@ program const testDir = path.resolve(process.cwd(), command.args[0]); const config: RunnerConfig = { debug: command.debug, + forbidOnly: command.forbidOnly, quiet: command.quiet, grep: command.grep, jobs: command.jobs, @@ -58,25 +59,6 @@ program trialRun: command.trialRun, updateSnapshots: command.updateSnapshots }; - const files = collectFiles(testDir, '', command.args.slice(1)); - const suite = collectTests(config, files); - if (command.forbidOnly) { - const hasOnly = suite.eachTest(t => t.only) || suite.eachSuite(s => s.only); - if (hasOnly) { - console.error('====================================='); - console.error(' --forbid-only found a focused test.'); - console.error('====================================='); - process.exit(1); - } - } - - const total = suite.total(); - if (!total) { - console.error('================='); - console.error(' no tests found.'); - console.error('================='); - process.exit(1); - } const reporterList = command.reporter.split(','); const reporterObjects: Reporter[] = reporterList.map(c => { @@ -90,9 +72,24 @@ program process.exit(1); } }); - await runTests(config, suite, new Multiplexer(reporterObjects)); - const hasFailures = suite.eachTest(t => t.error); - process.exit(hasFailures ? 1 : 0); + + const files = collectFiles(testDir, '', command.args.slice(1)); + const result = await run(config, files, new Multiplexer(reporterObjects)); + if (result === 'forbid-only') { + console.error('====================================='); + console.error(' --forbid-only found a focused test.'); + console.error('====================================='); + process.exit(1); + } + + if (result === 'no-tests') { + console.error('================='); + console.error(' no tests found.'); + console.error('================='); + process.exit(1); + } + + process.exit(result === 'failed' ? 1 : 0); }); program.parse(process.argv); diff --git a/test-runner/src/index.ts b/test-runner/src/index.ts index c6c2c6191c..6e07b73f48 100644 --- a/test-runner/src/index.ts +++ b/test-runner/src/index.ts @@ -23,7 +23,7 @@ import { registerFixture as registerFixtureT, registerWorkerFixture as registerW import { Reporter } from './reporter'; import { Runner } from './runner'; import { RunnerConfig } from './runnerConfig'; -import { Suite, Test } from './test'; +import { Test } from './test'; import { Matrix, TestCollector } from './testCollector'; import { installTransform } from './transform'; export { parameters, registerParameter } from './fixtures'; @@ -58,7 +58,9 @@ export function registerWorkerFixture(name, fn); }; -export function collectTests(config: RunnerConfig, files: string[]): Suite { +type RunResult = 'passed' | 'failed' | 'forbid-only' | 'no-tests'; + +export async function run(config: RunnerConfig, files: string[], reporter: Reporter): Promise { const revertBabelRequire = installTransform(); let hasSetup = false; try { @@ -74,10 +76,17 @@ export function collectTests(config: RunnerConfig, files: string[]): Suite { revertBabelRequire(); const testCollector = new TestCollector(files, matrix, config); - return testCollector.suite; -} + const suite = testCollector.suite; + if (config.forbidOnly) { + const hasOnly = suite.findTest(t => t.only) || suite.eachSuite(s => s.only); + if (hasOnly) + return 'forbid-only'; + } + + const total = suite.total(); + if (!total) + return 'no-tests'; -export async function runTests(config: RunnerConfig, suite: Suite, reporter: Reporter) { // Trial run does not need many workers, use one. const jobs = (config.trialRun || config.debug) ? 1 : config.jobs; const runner = new Runner(suite, { ...config, jobs }, reporter); @@ -91,4 +100,5 @@ export async function runTests(config: RunnerConfig, suite: Suite, reporter: Rep for (const f of afterFunctions) await f(); } + return suite.findTest(t => t.error) ? 'failed' : 'passed'; } diff --git a/test-runner/src/reporter.ts b/test-runner/src/reporter.ts index 81693ae4d1..39872d8492 100644 --- a/test-runner/src/reporter.ts +++ b/test-runner/src/reporter.ts @@ -20,10 +20,10 @@ import { Suite, Test } from './test'; export interface Reporter { onBegin(config: RunnerConfig, suite: Suite): void; onTest(test: Test): void; - onPending(test: Test): void; - onStdOut(test: Test, chunk: string | Buffer); - onStdErr(test: Test, chunk: string | Buffer); - onPass(test: Test): void; - onFail(test: Test): void; + onSkippedTest(test: Test): void; + onTestStdOut(test: Test, chunk: string | Buffer); + onTestStdErr(test: Test, chunk: string | Buffer); + onTestPassed(test: Test): void; + onTestFailed(test: Test): void; onEnd(): void; } diff --git a/test-runner/src/reporters/base.ts b/test-runner/src/reporters/base.ts index 784182d66a..2e8c67871f 100644 --- a/test-runner/src/reporters/base.ts +++ b/test-runner/src/reporters/base.ts @@ -29,7 +29,7 @@ import { Suite, Test } from '../test'; const stackUtils = new StackUtils() export class BaseReporter implements Reporter { - pending: Test[] = []; + skipped: Test[] = []; passes: Test[] = []; failures: Test[] = []; timeouts: Test[] = []; @@ -54,25 +54,25 @@ export class BaseReporter implements Reporter { onTest(test: Test) { } - onPending(test: Test) { - this.pending.push(test); + onSkippedTest(test: Test) { + this.skipped.push(test); } - onStdOut(test: Test, chunk: string | Buffer) { + onTestStdOut(test: Test, chunk: string | Buffer) { if (!this.config.quiet) process.stdout.write(chunk); } - onStdErr(test: Test, chunk: string | Buffer) { + onTestStdErr(test: Test, chunk: string | Buffer) { if (!this.config.quiet) process.stderr.write(chunk); } - onPass(test: Test) { + onTestPassed(test: Test) { this.passes.push(test); } - onFail(test: Test) { + onTestFailed(test: Test) { if (test.duration >= test.timeout) this.timeouts.push(test); else @@ -88,8 +88,8 @@ export class BaseReporter implements Reporter { console.log(colors.green(` ${this.passes.length} passed`) + colors.dim(` (${milliseconds(this.duration)})`)); - if (this.pending.length) - console.log(colors.yellow(` ${this.pending.length} skipped`)); + if (this.skipped.length) + console.log(colors.yellow(` ${this.skipped.length} skipped`)); if (this.failures.length) { console.log(colors.red(` ${this.failures.length} failed`)); diff --git a/test-runner/src/reporters/dot.ts b/test-runner/src/reporters/dot.ts index 82657dd7d0..bbfb01c2e3 100644 --- a/test-runner/src/reporters/dot.ts +++ b/test-runner/src/reporters/dot.ts @@ -19,18 +19,18 @@ import { BaseReporter } from './base'; import { Test } from '../test'; class DotReporter extends BaseReporter { - onPending(test: Test) { - super.onPending(test); + onSkippedTest(test: Test) { + super.onSkippedTest(test); process.stdout.write(colors.yellow('∘')) } - onPass(test: Test) { - super.onPass(test); + onTestPassed(test: Test) { + super.onTestPassed(test); process.stdout.write(colors.green('·')); } - onFail(test: Test) { - super.onFail(test); + onTestFailed(test: Test) { + super.onTestFailed(test); if (test.duration >= test.timeout) process.stdout.write(colors.red('T')); else diff --git a/test-runner/src/reporters/json.ts b/test-runner/src/reporters/json.ts index ea541e0d10..909e20a951 100644 --- a/test-runner/src/reporters/json.ts +++ b/test-runner/src/reporters/json.ts @@ -33,7 +33,7 @@ class JSONReporter extends BaseReporter { } private _serializeSuite(suite: Suite): any { - if (!suite.eachTest(test => true)) + if (!suite.findTest(test => true)) return null; const suites = suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s); return { @@ -50,7 +50,7 @@ class JSONReporter extends BaseReporter { title: test.title, file: test.file, only: test.only, - pending: test.pending, + skipped: test.skipped, slow: test.slow, duration: test.duration, timeout: test.timeout, diff --git a/test-runner/src/reporters/list.ts b/test-runner/src/reporters/list.ts index ecbea96472..75b990c471 100644 --- a/test-runner/src/reporters/list.ts +++ b/test-runner/src/reporters/list.ts @@ -32,21 +32,21 @@ class ListReporter extends BaseReporter { process.stdout.write(' ' + colors.gray(test.fullTitle() + ': ')); } - onPending(test: Test) { - super.onPending(test); + onSkippedTest(test: Test) { + super.onSkippedTest(test); process.stdout.write(colors.green(' - ') + colors.cyan(test.fullTitle())); process.stdout.write('\n'); } - onPass(test: Test) { - super.onPass(test); + onTestPassed(test: Test) { + super.onTestPassed(test); process.stdout.write('\u001b[2K\u001b[0G'); process.stdout.write(colors.green(' ✓ ') + colors.gray(test.fullTitle())); process.stdout.write('\n'); } - onFail(test: Test) { - super.onFail(test); + onTestFailed(test: Test) { + super.onTestFailed(test); process.stdout.write('\u001b[2K\u001b[0G'); process.stdout.write(colors.red(` ${++this._failure}) ` + test.fullTitle())); process.stdout.write('\n'); diff --git a/test-runner/src/reporters/multiplexer.ts b/test-runner/src/reporters/multiplexer.ts index 89fa5009d5..614c6f647d 100644 --- a/test-runner/src/reporters/multiplexer.ts +++ b/test-runner/src/reporters/multiplexer.ts @@ -35,29 +35,29 @@ export class Multiplexer implements Reporter { reporter.onTest(test); } - onPending(test: Test) { + onSkippedTest(test: Test) { for (const reporter of this._reporters) - reporter.onPending(test); + reporter.onSkippedTest(test); } - onStdOut(test: Test, chunk: string | Buffer) { + onTestStdOut(test: Test, chunk: string | Buffer) { for (const reporter of this._reporters) - reporter.onStdOut(test, chunk); + reporter.onTestStdOut(test, chunk); } - onStdErr(test: Test, chunk: string | Buffer) { + onTestStdErr(test: Test, chunk: string | Buffer) { for (const reporter of this._reporters) - reporter.onStdErr(test, chunk); + reporter.onTestStdErr(test, chunk); } - onPass(test: Test) { + onTestPassed(test: Test) { for (const reporter of this._reporters) - reporter.onPass(test); + reporter.onTestPassed(test); } - onFail(test: Test) { + onTestFailed(test: Test) { for (const reporter of this._reporters) - reporter.onFail(test); + reporter.onTestFailed(test); } onEnd() { diff --git a/test-runner/src/reporters/pytest.ts b/test-runner/src/reporters/pytest.ts index 3815267c33..ffec65c2f4 100644 --- a/test-runner/src/reporters/pytest.ts +++ b/test-runner/src/reporters/pytest.ts @@ -84,30 +84,30 @@ class PytestReporter extends BaseReporter { row.startTime = Date.now(); } - onPending(test: Test) { - super.onPending(test); + onSkippedTest(test: Test) { + super.onSkippedTest(test); this._append(test, colors.yellow('∘')); this._progress.push('S'); this._throttler.schedule(); } - onStdOut(test: Test, chunk: string | Buffer) { + onTestStdOut(test: Test, chunk: string | Buffer) { this._repaint(chunk); } - onStdErr(test: Test, chunk: string | Buffer) { + onTestStdErr(test: Test, chunk: string | Buffer) { this._repaint(chunk); } - onPass(test: Test) { - super.onPass(test); + onTestPassed(test: Test) { + super.onTestPassed(test); this._append(test, colors.green('✓')); this._progress.push('P'); this._throttler.schedule(); } - onFail(test: Test) { - super.onFail(test); + onTestFailed(test: Test) { + super.onTestFailed(test); const title = test.duration >= test.timeout ? colors.red('T') : colors.red('F'); const row = this._append(test, title); row.failed = true; @@ -148,8 +148,8 @@ class PytestReporter extends BaseReporter { const status = []; if (this.passes.length) status.push(colors.green(`${this.passes.length} passed`)); - if (this.pending.length) - status.push(colors.yellow(`${this.pending.length} skipped`)); + if (this.skipped.length) + status.push(colors.yellow(`${this.skipped.length} skipped`)); if (this.failures.length) status.push(colors.red(`${this.failures.length} failed`)); if (this.timeouts.length) diff --git a/test-runner/src/runner.ts b/test-runner/src/runner.ts index f0cd9f008f..4f2d5c10d9 100644 --- a/test-runner/src/runner.ts +++ b/test-runner/src/runner.ts @@ -28,7 +28,7 @@ export class Runner { private _workers = new Set(); private _freeWorkers: Worker[] = []; private _workerClaimers: (() => void)[] = []; - stats: { duration: number; failures: number; passes: number; pending: number; tests: number; }; + stats: { duration: number; failures: number; passes: number; skipped: number; tests: number; }; private _testById = new Map(); private _queue: TestRunnerEntry[] = []; @@ -44,13 +44,13 @@ export class Runner { duration: 0, failures: 0, passes: 0, - pending: 0, + skipped: 0, tests: 0, }; this._suite = suite; for (const suite of this._suite.suites) { - suite.eachTest(test => { + suite.findTest(test => { this._testById.set(`${test._ordinal}@${suite.file}::[${suite._configurationString}]`, test); }); } @@ -67,7 +67,9 @@ export class Runner { const result: TestRunnerEntry[] = []; for (const suite of this._suite.suites) { const ordinals: number[] = []; - suite.eachTest(test => ordinals.push(test._ordinal) && false); + suite.findTest(test => ordinals.push(test._ordinal) && false); + if (!ordinals.length) + continue; result.push({ ordinals, file: suite.file, @@ -109,15 +111,26 @@ export class Runner { let doneCallback; const result = new Promise(f => doneCallback = f); worker.once('done', params => { - // When worker encounters error, we will restart it. - if (params.error || params.fatalError) { - this._restartWorker(worker); - // If there are remaining tests, we will queue them. - if (params.remaining.length && !params.fatalError) - this._queue.unshift({ ...entry, ordinals: params.remaining }); - } else { + if (!params.failedTestId && !params.fatalError) { this._workerAvailable(worker); + doneCallback(); + return; } + + // When worker encounters error, we will restart it. + this._restartWorker(worker); + + // In case of fatal error, we are done with the entry. + if (params.fatalError) { + doneCallback(); + return; + } + + const remaining = params.remaining; + if (params.remaining.length) + this._queue.unshift({ ...entry, ordinals: remaining }); + + // This job is over, we just scheduled another one. doneCallback(); }); return result; @@ -149,30 +162,30 @@ export class Runner { ++this.stats.tests; this._reporter.onTest(this._updateTest(params.test)); }); - worker.on('pending', params => { + worker.on('skipped', params => { ++this.stats.tests; - ++this.stats.pending; - this._reporter.onPending(this._updateTest(params.test)); + ++this.stats.skipped; + this._reporter.onSkippedTest(this._updateTest(params.test)); }); worker.on('pass', params => { ++this.stats.passes; - this._reporter.onPass(this._updateTest(params.test)); + this._reporter.onTestPassed(this._updateTest(params.test)); }); worker.on('fail', params => { ++this.stats.failures; - this._reporter.onFail(this._updateTest(params.test)); + this._reporter.onTestFailed(this._updateTest(params.test)); }); worker.on('stdout', params => { const chunk = chunkFromParams(params); const test = this._testById.get(params.testId); test.stdout.push(chunk); - this._reporter.onStdOut(test, chunk); + this._reporter.onTestStdOut(test, chunk); }); worker.on('stderr', params => { const chunk = chunkFromParams(params); const test = this._testById.get(params.testId); test.stderr.push(chunk); - this._reporter.onStdErr(test, chunk); + this._reporter.onTestStdErr(test, chunk); }); worker.on('exit', () => { this._workers.delete(worker); @@ -279,7 +292,7 @@ class InProcessWorker extends Worker { delete require.cache[entry.file]; const { TestRunner } = require('./testRunner'); const testRunner = new TestRunner(entry, this.runner._config, 0); - for (const event of ['test', 'pending', 'pass', 'fail', 'done', 'stdout', 'stderr']) + for (const event of ['test', 'skipped', 'pass', 'fail', 'done', 'stdout', 'stderr']) testRunner.on(event, this.emit.bind(this, event)); testRunner.run(); } diff --git a/test-runner/src/runnerConfig.ts b/test-runner/src/runnerConfig.ts index 8f6cf5511d..91aa18d39e 100644 --- a/test-runner/src/runnerConfig.ts +++ b/test-runner/src/runnerConfig.ts @@ -15,6 +15,7 @@ */ export type RunnerConfig = { + forbidOnly?: boolean; jobs: number; outputDir: string; snapshotDir: string; diff --git a/test-runner/src/spec.ts b/test-runner/src/spec.ts index 2e27fca9b0..35358aa067 100644 --- a/test-runner/src/spec.ts +++ b/test-runner/src/spec.ts @@ -64,9 +64,9 @@ export function spec(suite: Suite, file: string, timeout: number): () => void { if (only) test.only = true; if (!only && specs.skip && specs.skip[0]) - test.pending = true; + test.skipped = true; if (!only && specs.fail && specs.fail[0]) - test.pending = true; + test.skipped = true; suite._addTest(test); return test; }); @@ -79,9 +79,9 @@ export function spec(suite: Suite, file: string, timeout: number): () => void { if (only) child.only = true; if (!only && specs.skip && specs.skip[0]) - child.pending = true; + child.skipped = true; if (!only && specs.fail && specs.fail[0]) - child.pending = true; + child.skipped = true; suites.unshift(child); fn(); suites.shift(); diff --git a/test-runner/src/test.ts b/test-runner/src/test.ts index e30f538b7a..3172aae989 100644 --- a/test-runner/src/test.ts +++ b/test-runner/src/test.ts @@ -21,7 +21,7 @@ export class Test { title: string; file: string; only = false; - pending = false; + skipped = false; slow = false; duration = 0; timeout = 0; @@ -53,7 +53,7 @@ export class Test { test.suite = this.suite; test.only = this.only; test.file = this.file; - test.pending = this.pending; + test.skipped = this.skipped; test.timeout = this.timeout; test._overriddenFn = this._overriddenFn; return test; @@ -66,7 +66,7 @@ export class Suite { suites: Suite[] = []; tests: Test[] = []; only = false; - pending = false; + skipped = false; file: string; configuration: Configuration; _configurationString: string; @@ -87,14 +87,14 @@ export class Suite { total(): number { let count = 0; - this.eachTest(fn => { + this.findTest(fn => { ++count; }); return count; } - _isPending(): boolean { - return this.pending || (this.parent && this.parent._isPending()); + _isSkipped(): boolean { + return this.skipped || (this.parent && this.parent._isSkipped()); } _addTest(test: Test) { @@ -117,9 +117,9 @@ export class Suite { return false; } - eachTest(fn: (test: Test) => boolean | void): boolean { + findTest(fn: (test: Test) => boolean | void): boolean { for (const suite of this.suites) { - if (suite.eachTest(fn)) + if (suite.findTest(fn)) return true; } for (const test of this.tests) { @@ -133,13 +133,13 @@ export class Suite { const suite = new Suite(this.title); suite.only = this.only; suite.file = this.file; - suite.pending = this.pending; + suite.skipped = this.skipped; return suite; } _renumber() { let ordinal = 0; - this.eachTest((test: Test) => { + this.findTest((test: Test) => { // All tests are identified with their ordinals. test._ordinal = ordinal++; }); @@ -151,8 +151,8 @@ export class Suite { _hasTestsToRun(): boolean { let found = false; - this.eachTest(test => { - if (!test.pending) { + this.findTest(test => { + if (!test.skipped) { found = true; return true; } diff --git a/test-runner/src/testCollector.ts b/test-runner/src/testCollector.ts index f559e7d2fd..2dcae6404a 100644 --- a/test-runner/src/testCollector.ts +++ b/test-runner/src/testCollector.ts @@ -60,7 +60,7 @@ export class TestCollector { const workerGeneratorConfigurations = new Map(); - suite.eachTest((test: Test) => { + suite.findTest((test: Test) => { // Get all the fixtures that the test needs. const fixtures = fixturesForCallback(test.fn); diff --git a/test-runner/src/testRunner.ts b/test-runner/src/testRunner.ts index 693a033674..be7c82d5cc 100644 --- a/test-runner/src/testRunner.ts +++ b/test-runner/src/testRunner.ts @@ -48,8 +48,7 @@ export type SerializedTest = { }; export class TestRunner extends EventEmitter { - private _currentOrdinal = -1; - private _failedWithError: any | undefined; + private _failedTestId: string | undefined; private _fatalError: any | undefined; private _file: any; private _ordinals: Set; @@ -136,15 +135,14 @@ export class TestRunner extends EventEmitter { } private async _runTest(test: Test) { - if (this._failedWithError) + if (this._failedTestId) return false; this._test = test; - const ordinal = ++this._currentOrdinal; - if (this._ordinals.size && !this._ordinals.has(ordinal)) + if (this._ordinals.size && !this._ordinals.has(test._ordinal)) return; - this._remaining.delete(ordinal); - if (test.pending || test.suite._isPending()) { - this.emit('pending', { test: this._serializeTest() }); + this._remaining.delete(test._ordinal); + if (test.skipped || test.suite._isSkipped()) { + this.emit('skipped', { test: this._serializeTest() }); return; } @@ -154,11 +152,11 @@ export class TestRunner extends EventEmitter { test._startTime = Date.now(); if (!this._trialRun) await this._testWrapper(test)(); - this.emit('pass', { test: this._serializeTest(true) }); await this._runHooks(test.suite, 'afterEach', 'after'); + this.emit('pass', { test: this._serializeTest(true) }); } catch (error) { test.error = serializeError(error); - this._failedWithError = test.error; + this._failedTestId = this._testId(); this.emit('fail', { test: this._serializeTest(true) }); } this._test = null; @@ -180,7 +178,7 @@ export class TestRunner extends EventEmitter { private _reportDone() { this.emit('done', { - error: this._failedWithError, + failedTestId: this._failedTestId, fatalError: this._fatalError, remaining: [...this._remaining], }); diff --git a/test-runner/src/worker.ts b/test-runner/src/worker.ts index 536c19eb31..b6938a9609 100644 --- a/test-runner/src/worker.ts +++ b/test-runner/src/worker.ts @@ -69,7 +69,7 @@ process.on('message', async message => { } if (message.method === 'run') { testRunner = new TestRunner(message.params.entry, message.params.config, workerId); - for (const event of ['test', 'pending', 'pass', 'fail', 'done', 'stdout', 'stderr']) + for (const event of ['test', 'skipped', 'pass', 'fail', 'done', 'stdout', 'stderr']) testRunner.on(event, sendMessageToParent.bind(null, event)); await testRunner.run(); testRunner = null;