feat(test): shrink api to run only, rename pending to skipped (#3636)

test-runner/src/cli.ts

@@ -17,7 +17,7 @@
 import program from 'commander';
 import * as fs from 'fs';
 import * as path from 'path';
-import { collectTests, runTests, RunnerConfig } from '.';
+import { run, RunnerConfig } from '.';
 import PytestReporter from './reporters/pytest';
 import DotReporter from './reporters/dot';
 import ListReporter from './reporters/list';
@@ -48,6 +48,7 @@ program
     const testDir = path.resolve(process.cwd(), command.args[0]);
     const config: RunnerConfig = {
       debug: command.debug,
+      forbidOnly: command.forbidOnly,
       quiet: command.quiet,
       grep: command.grep,
       jobs: command.jobs,
@@ -58,25 +59,6 @@ program
       trialRun: command.trialRun,
       updateSnapshots: command.updateSnapshots
     };
-    const files = collectFiles(testDir, '', command.args.slice(1));
-    const suite = collectTests(config, files);
-    if (command.forbidOnly) {
-      const hasOnly = suite.eachTest(t => t.only) || suite.eachSuite(s => s.only);
-      if (hasOnly) {
-        console.error('=====================================');
-        console.error(' --forbid-only found a focused test.');
-        console.error('=====================================');
-        process.exit(1);
-      }
-    }
-
-    const total = suite.total();
-    if (!total) {
-      console.error('=================');
-      console.error(' no tests found.');
-      console.error('=================');
-      process.exit(1);
-    }
 
     const reporterList = command.reporter.split(',');
     const reporterObjects: Reporter[] = reporterList.map(c => {
@@ -90,9 +72,24 @@ program
         process.exit(1);
       }
     });
-    await runTests(config, suite, new Multiplexer(reporterObjects));
-    const hasFailures = suite.eachTest(t => t.error);
-    process.exit(hasFailures ? 1 : 0);
+
+    const files = collectFiles(testDir, '', command.args.slice(1));
+    const result = await run(config, files, new Multiplexer(reporterObjects));
+    if (result === 'forbid-only') {
+      console.error('=====================================');
+      console.error(' --forbid-only found a focused test.');
+      console.error('=====================================');
+      process.exit(1);
+    }
+
+    if (result === 'no-tests') {
+      console.error('=================');
+      console.error(' no tests found.');
+      console.error('=================');
+      process.exit(1);
+    }
+
+    process.exit(result === 'failed' ? 1 : 0);
   });
 
 program.parse(process.argv);

test-runner/src/index.ts

@@ -23,7 +23,7 @@ import { registerFixture as registerFixtureT, registerWorkerFixture as registerW
 import { Reporter } from './reporter';
 import { Runner } from './runner';
 import { RunnerConfig } from './runnerConfig';
-import { Suite, Test } from './test';
+import { Test } from './test';
 import { Matrix, TestCollector } from './testCollector';
 import { installTransform } from './transform';
 export { parameters, registerParameter } from './fixtures';
@@ -58,7 +58,9 @@ export function registerWorkerFixture<T extends keyof (WorkerState & FixturePara
   registerWorkerFixtureT<RunnerConfig>(name, fn);
 };
 
-export function collectTests(config: RunnerConfig, files: string[]): Suite {
+type RunResult = 'passed' | 'failed' | 'forbid-only' | 'no-tests';
+
+export async function run(config: RunnerConfig, files: string[], reporter: Reporter): Promise<RunResult> {
   const revertBabelRequire = installTransform();
   let hasSetup = false;
   try {
@@ -74,10 +76,17 @@ export function collectTests(config: RunnerConfig, files: string[]): Suite {
   revertBabelRequire();
 
   const testCollector = new TestCollector(files, matrix, config);
-  return testCollector.suite;
-}
+  const suite = testCollector.suite;
+  if (config.forbidOnly) {
+    const hasOnly = suite.findTest(t => t.only) || suite.eachSuite(s => s.only);
+    if (hasOnly)
+      return 'forbid-only';
+  }
+
+  const total = suite.total();
+  if (!total)
+    return 'no-tests';
 
-export async function runTests(config: RunnerConfig, suite: Suite, reporter: Reporter) {
   // Trial run does not need many workers, use one.
   const jobs = (config.trialRun || config.debug) ? 1 : config.jobs;
   const runner = new Runner(suite, { ...config, jobs }, reporter);
@@ -91,4 +100,5 @@ export async function runTests(config: RunnerConfig, suite: Suite, reporter: Rep
     for (const f of afterFunctions)
       await f();
   }
+  return suite.findTest(t => t.error) ? 'failed' : 'passed';
 }

test-runner/src/reporter.ts

@@ -20,10 +20,10 @@ import { Suite, Test } from './test';
 export interface Reporter {
   onBegin(config: RunnerConfig, suite: Suite): void;
   onTest(test: Test): void;
-  onPending(test: Test): void;
-  onStdOut(test: Test, chunk: string | Buffer);
-  onStdErr(test: Test, chunk: string | Buffer);
-  onPass(test: Test): void;
-  onFail(test: Test): void;
+  onSkippedTest(test: Test): void;
+  onTestStdOut(test: Test, chunk: string | Buffer);
+  onTestStdErr(test: Test, chunk: string | Buffer);
+  onTestPassed(test: Test): void;
+  onTestFailed(test: Test): void;
   onEnd(): void;
 }

test-runner/src/reporters/base.ts

@@ -29,7 +29,7 @@ import { Suite, Test } from '../test';
 const stackUtils = new StackUtils()
 
 export class BaseReporter implements Reporter  {
-  pending: Test[] = [];
+  skipped: Test[] = [];
   passes: Test[] = [];
   failures: Test[] = [];
   timeouts: Test[] = [];
@@ -54,25 +54,25 @@ export class BaseReporter implements Reporter  {
   onTest(test: Test) {
   }
 
-  onPending(test: Test) {
-    this.pending.push(test);
+  onSkippedTest(test: Test) {
+    this.skipped.push(test);
   }
 
-  onStdOut(test: Test, chunk: string | Buffer) {
+  onTestStdOut(test: Test, chunk: string | Buffer) {
     if (!this.config.quiet)
       process.stdout.write(chunk);
   }
 
-  onStdErr(test: Test, chunk: string | Buffer) {
+  onTestStdErr(test: Test, chunk: string | Buffer) {
     if (!this.config.quiet)
       process.stderr.write(chunk);
   }
 
-  onPass(test: Test) {
+  onTestPassed(test: Test) {
     this.passes.push(test);
   }
 
-  onFail(test: Test) {
+  onTestFailed(test: Test) {
     if (test.duration >= test.timeout)
       this.timeouts.push(test);
     else
@@ -88,8 +88,8 @@ export class BaseReporter implements Reporter  {
 
     console.log(colors.green(`  ${this.passes.length} passed`) + colors.dim(` (${milliseconds(this.duration)})`));  
 
-    if (this.pending.length)
-      console.log(colors.yellow(`  ${this.pending.length} skipped`));
+    if (this.skipped.length)
+      console.log(colors.yellow(`  ${this.skipped.length} skipped`));
 
     if (this.failures.length) {
       console.log(colors.red(`  ${this.failures.length} failed`));

test-runner/src/reporters/dot.ts

@@ -19,18 +19,18 @@ import { BaseReporter } from './base';
 import { Test } from '../test';
 
 class DotReporter extends BaseReporter {
-  onPending(test: Test) {
-    super.onPending(test);
+  onSkippedTest(test: Test) {
+    super.onSkippedTest(test);
     process.stdout.write(colors.yellow('∘'))
   }
   
-  onPass(test: Test) {
-    super.onPass(test);
+  onTestPassed(test: Test) {
+    super.onTestPassed(test);
     process.stdout.write(colors.green('·'));
   }
   
-  onFail(test: Test) {
-    super.onFail(test);
+  onTestFailed(test: Test) {
+    super.onTestFailed(test);
     if (test.duration >= test.timeout)
       process.stdout.write(colors.red('T'));
     else

test-runner/src/reporters/json.ts

@@ -33,7 +33,7 @@ class JSONReporter extends BaseReporter {
   }
 
   private _serializeSuite(suite: Suite): any {
-    if (!suite.eachTest(test => true))
+    if (!suite.findTest(test => true))
       return null;
     const suites = suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s);
     return {
@@ -50,7 +50,7 @@ class JSONReporter extends BaseReporter {
       title: test.title,
       file: test.file,
       only: test.only,
-      pending: test.pending,
+      skipped: test.skipped,
       slow: test.slow,
       duration: test.duration,
       timeout: test.timeout,

test-runner/src/reporters/list.ts

@@ -32,21 +32,21 @@ class ListReporter extends BaseReporter {
     process.stdout.write('    ' + colors.gray(test.fullTitle() + ': '));
   }
 
-  onPending(test: Test) {
-    super.onPending(test);
+  onSkippedTest(test: Test) {
+    super.onSkippedTest(test);
     process.stdout.write(colors.green('  - ') + colors.cyan(test.fullTitle()));
     process.stdout.write('\n');
   }
 
-  onPass(test: Test) {
-    super.onPass(test);
+  onTestPassed(test: Test) {
+    super.onTestPassed(test);
     process.stdout.write('\u001b[2K\u001b[0G');
     process.stdout.write(colors.green('  ✓ ') + colors.gray(test.fullTitle()));
     process.stdout.write('\n');
   }
 
-  onFail(test: Test) {
-    super.onFail(test);
+  onTestFailed(test: Test) {
+    super.onTestFailed(test);
     process.stdout.write('\u001b[2K\u001b[0G');
     process.stdout.write(colors.red(`  ${++this._failure}) ` + test.fullTitle()));
     process.stdout.write('\n');

test-runner/src/reporters/multiplexer.ts

@@ -35,29 +35,29 @@ export class Multiplexer implements Reporter {
       reporter.onTest(test);
   }
 
-  onPending(test: Test) {
+  onSkippedTest(test: Test) {
     for (const reporter of this._reporters)
-      reporter.onPending(test);
+      reporter.onSkippedTest(test);
   }
 
-  onStdOut(test: Test, chunk: string | Buffer) {
+  onTestStdOut(test: Test, chunk: string | Buffer) {
     for (const reporter of this._reporters)
-      reporter.onStdOut(test, chunk);    
+      reporter.onTestStdOut(test, chunk);
   }
 
-  onStdErr(test: Test, chunk: string | Buffer) {
+  onTestStdErr(test: Test, chunk: string | Buffer) {
     for (const reporter of this._reporters)
-      reporter.onStdErr(test, chunk);    
+      reporter.onTestStdErr(test, chunk);    
   }
 
-  onPass(test: Test) {
+  onTestPassed(test: Test) {
     for (const reporter of this._reporters)
-      reporter.onPass(test);
+      reporter.onTestPassed(test);
   }
 
-  onFail(test: Test) {
+  onTestFailed(test: Test) {
     for (const reporter of this._reporters)
-      reporter.onFail(test);
+      reporter.onTestFailed(test);
   }
 
   onEnd() {

test-runner/src/reporters/pytest.ts

@@ -84,30 +84,30 @@ class PytestReporter extends BaseReporter {
       row.startTime = Date.now();
   }
 
-  onPending(test: Test) {
-    super.onPending(test);
+  onSkippedTest(test: Test) {
+    super.onSkippedTest(test);
     this._append(test, colors.yellow('∘'));
     this._progress.push('S');
     this._throttler.schedule();
   }
 
-  onStdOut(test: Test, chunk: string | Buffer) {
+  onTestStdOut(test: Test, chunk: string | Buffer) {
     this._repaint(chunk);
   }
 
-  onStdErr(test: Test, chunk: string | Buffer) {
+  onTestStdErr(test: Test, chunk: string | Buffer) {
     this._repaint(chunk);
   }
 
-  onPass(test: Test) {
-    super.onPass(test);
+  onTestPassed(test: Test) {
+    super.onTestPassed(test);
     this._append(test, colors.green('✓'));
     this._progress.push('P');
     this._throttler.schedule();
   }
 
-  onFail(test: Test) {
-    super.onFail(test);
+  onTestFailed(test: Test) {
+    super.onTestFailed(test);
     const title = test.duration >= test.timeout ? colors.red('T') : colors.red('F');
     const row = this._append(test, title);
     row.failed = true;
@@ -148,8 +148,8 @@ class PytestReporter extends BaseReporter {
     const status = [];
     if (this.passes.length)
       status.push(colors.green(`${this.passes.length} passed`));
-    if (this.pending.length)
-      status.push(colors.yellow(`${this.pending.length} skipped`));
+    if (this.skipped.length)
+      status.push(colors.yellow(`${this.skipped.length} skipped`));
     if (this.failures.length)
       status.push(colors.red(`${this.failures.length} failed`));
     if (this.timeouts.length)

test-runner/src/runner.ts

@@ -28,7 +28,7 @@ export class Runner {
   private _workers = new Set<Worker>();
   private _freeWorkers: Worker[] = [];
   private _workerClaimers: (() => void)[] = [];
-  stats: { duration: number; failures: number; passes: number; pending: number; tests: number; };
+  stats: { duration: number; failures: number; passes: number; skipped: number; tests: number; };
 
   private _testById = new Map<string, Test>();
   private _queue: TestRunnerEntry[] = [];
@@ -44,13 +44,13 @@ export class Runner {
       duration: 0,
       failures: 0,
       passes: 0,
-      pending: 0,
+      skipped: 0,
       tests: 0,
     };
 
     this._suite = suite;
     for (const suite of this._suite.suites) {
-      suite.eachTest(test => {
+      suite.findTest(test => {
         this._testById.set(`${test._ordinal}@${suite.file}::[${suite._configurationString}]`, test);
       });
     }
@@ -67,7 +67,9 @@ export class Runner {
     const result: TestRunnerEntry[] = [];
     for (const suite of this._suite.suites) {
       const ordinals: number[] = [];
-      suite.eachTest(test => ordinals.push(test._ordinal) && false);
+      suite.findTest(test => ordinals.push(test._ordinal) && false);
+      if (!ordinals.length)
+        continue;
       result.push({
         ordinals,
         file: suite.file,
@@ -109,15 +111,26 @@ export class Runner {
     let doneCallback;
     const result = new Promise(f => doneCallback = f);
     worker.once('done', params => {
-      // When worker encounters error, we will restart it.
-      if (params.error || params.fatalError) {
-        this._restartWorker(worker);
-        // If there are remaining tests, we will queue them.
-        if (params.remaining.length && !params.fatalError)
-          this._queue.unshift({ ...entry, ordinals: params.remaining });
-      } else {
+      if (!params.failedTestId && !params.fatalError) {
         this._workerAvailable(worker);
+        doneCallback();
+        return;
       }
+
+      // When worker encounters error, we will restart it.
+      this._restartWorker(worker);
+
+      // In case of fatal error, we are done with the entry.
+      if (params.fatalError) {
+        doneCallback();
+        return;
+      }
+
+      const remaining = params.remaining;
+      if (params.remaining.length)
+        this._queue.unshift({ ...entry, ordinals: remaining });
+
+      // This job is over, we just scheduled another one.
       doneCallback();
     });
     return result;
@@ -149,30 +162,30 @@ export class Runner {
       ++this.stats.tests;
       this._reporter.onTest(this._updateTest(params.test));
     });
-    worker.on('pending', params => {
+    worker.on('skipped', params => {
       ++this.stats.tests;
-      ++this.stats.pending;
-      this._reporter.onPending(this._updateTest(params.test));
+      ++this.stats.skipped;
+      this._reporter.onSkippedTest(this._updateTest(params.test));
     });
     worker.on('pass', params => {
       ++this.stats.passes;
-      this._reporter.onPass(this._updateTest(params.test));
+      this._reporter.onTestPassed(this._updateTest(params.test));
     });
     worker.on('fail', params => {
       ++this.stats.failures;
-      this._reporter.onFail(this._updateTest(params.test));
+      this._reporter.onTestFailed(this._updateTest(params.test));
     });
     worker.on('stdout', params => {
       const chunk = chunkFromParams(params);
       const test = this._testById.get(params.testId);
       test.stdout.push(chunk);
-      this._reporter.onStdOut(test, chunk);
+      this._reporter.onTestStdOut(test, chunk);
     });
     worker.on('stderr', params => {
       const chunk = chunkFromParams(params);
       const test = this._testById.get(params.testId);
       test.stderr.push(chunk);
-      this._reporter.onStdErr(test, chunk);
+      this._reporter.onTestStdErr(test, chunk);
     });
     worker.on('exit', () => {
       this._workers.delete(worker);
@@ -279,7 +292,7 @@ class InProcessWorker extends Worker {
     delete require.cache[entry.file];
     const { TestRunner } = require('./testRunner');
     const testRunner = new TestRunner(entry, this.runner._config, 0);
-    for (const event of ['test', 'pending', 'pass', 'fail', 'done', 'stdout', 'stderr'])
+    for (const event of ['test', 'skipped', 'pass', 'fail', 'done', 'stdout', 'stderr'])
       testRunner.on(event, this.emit.bind(this, event));
     testRunner.run();
   }

test-runner/src/runnerConfig.ts

@@ -15,6 +15,7 @@
  */
 
 export type RunnerConfig = {
+  forbidOnly?: boolean;
   jobs: number;
   outputDir: string;
   snapshotDir: string;

test-runner/src/spec.ts

@@ -64,9 +64,9 @@ export function spec(suite: Suite, file: string, timeout: number): () => void {
     if (only)
       test.only = true;
     if (!only && specs.skip && specs.skip[0])
-      test.pending = true;
+      test.skipped = true;
     if (!only && specs.fail && specs.fail[0])
-      test.pending = true;
+      test.skipped = true;
     suite._addTest(test);
     return test;
   });
@@ -79,9 +79,9 @@ export function spec(suite: Suite, file: string, timeout: number): () => void {
     if (only)
       child.only = true;
     if (!only && specs.skip && specs.skip[0])
-      child.pending = true;
+      child.skipped = true;
     if (!only && specs.fail && specs.fail[0])
-      child.pending = true;
+      child.skipped = true;
     suites.unshift(child);
     fn();
     suites.shift();

test-runner/src/test.ts

@@ -21,7 +21,7 @@ export class Test {
   title: string;
   file: string;
   only = false;
-  pending = false;
+  skipped = false;
   slow = false;
   duration = 0;
   timeout = 0;
@@ -53,7 +53,7 @@ export class Test {
     test.suite = this.suite;
     test.only = this.only;
     test.file = this.file;
-    test.pending = this.pending;
+    test.skipped = this.skipped;
     test.timeout = this.timeout;
     test._overriddenFn = this._overriddenFn;
     return test;
@@ -66,7 +66,7 @@ export class Suite {
   suites: Suite[] = [];
   tests: Test[] = [];
   only = false;
-  pending = false;
+  skipped = false;
   file: string;
   configuration: Configuration;
   _configurationString: string;
@@ -87,14 +87,14 @@ export class Suite {
 
   total(): number {
     let count = 0;
-    this.eachTest(fn => {
+    this.findTest(fn => {
       ++count;
     });
     return count;
   }
 
-  _isPending(): boolean {
-    return this.pending || (this.parent && this.parent._isPending());
+  _isSkipped(): boolean {
+    return this.skipped || (this.parent && this.parent._isSkipped());
   }
 
   _addTest(test: Test) {
@@ -117,9 +117,9 @@ export class Suite {
     return false;
   }
 
-  eachTest(fn: (test: Test) => boolean | void): boolean {
+  findTest(fn: (test: Test) => boolean | void): boolean {
     for (const suite of this.suites) {
-      if (suite.eachTest(fn))
+      if (suite.findTest(fn))
         return true;
     }
     for (const test of this.tests) {
@@ -133,13 +133,13 @@ export class Suite {
     const suite = new Suite(this.title);
     suite.only = this.only;
     suite.file = this.file;
-    suite.pending = this.pending;
+    suite.skipped = this.skipped;
     return suite;
   }
 
   _renumber() {
     let ordinal = 0;
-    this.eachTest((test: Test) => {
+    this.findTest((test: Test) => {
       // All tests are identified with their ordinals.
       test._ordinal = ordinal++;
     });
@@ -151,8 +151,8 @@ export class Suite {
 
   _hasTestsToRun(): boolean {
     let found = false;
-    this.eachTest(test => {
-      if (!test.pending) {
+    this.findTest(test => {
+      if (!test.skipped) {
         found = true;
         return true;
       }

test-runner/src/testCollector.ts

@@ -60,7 +60,7 @@ export class TestCollector {
 
     const workerGeneratorConfigurations = new Map();
 
-    suite.eachTest((test: Test) => {
+    suite.findTest((test: Test) => {
       // Get all the fixtures that the test needs.
       const fixtures = fixturesForCallback(test.fn);
 

test-runner/src/testRunner.ts

@@ -48,8 +48,7 @@ export type SerializedTest = {
 };
 
 export class TestRunner extends EventEmitter {
-  private _currentOrdinal = -1;
-  private _failedWithError: any | undefined;
+  private _failedTestId: string | undefined;
   private _fatalError: any | undefined;
   private _file: any;
   private _ordinals: Set<number>;
@@ -136,15 +135,14 @@ export class TestRunner extends EventEmitter {
   }
 
   private async _runTest(test: Test) {
-    if (this._failedWithError)
+    if (this._failedTestId)
       return false;
     this._test = test;
-    const ordinal = ++this._currentOrdinal;
-    if (this._ordinals.size && !this._ordinals.has(ordinal))
+    if (this._ordinals.size && !this._ordinals.has(test._ordinal))
       return;
-    this._remaining.delete(ordinal);
-    if (test.pending || test.suite._isPending()) {
-      this.emit('pending', { test: this._serializeTest() });
+    this._remaining.delete(test._ordinal);
+    if (test.skipped || test.suite._isSkipped()) {
+      this.emit('skipped', { test: this._serializeTest() });
       return;
     }
 
@@ -154,11 +152,11 @@ export class TestRunner extends EventEmitter {
       test._startTime = Date.now();
       if (!this._trialRun)
         await this._testWrapper(test)();
-      this.emit('pass', { test: this._serializeTest(true) });
       await this._runHooks(test.suite, 'afterEach', 'after');
+      this.emit('pass', { test: this._serializeTest(true) });
     } catch (error) {
       test.error = serializeError(error);
-      this._failedWithError = test.error;
+      this._failedTestId = this._testId();
       this.emit('fail', { test: this._serializeTest(true) });
     }
     this._test = null;
@@ -180,7 +178,7 @@ export class TestRunner extends EventEmitter {
 
   private _reportDone() {
     this.emit('done', {
-      error: this._failedWithError,
+      failedTestId: this._failedTestId,
       fatalError: this._fatalError,
       remaining: [...this._remaining],
     });

test-runner/src/worker.ts

@@ -69,7 +69,7 @@ process.on('message', async message => {
   }
   if (message.method === 'run') {
     testRunner = new TestRunner(message.params.entry, message.params.config, workerId);
-    for (const event of ['test', 'pending', 'pass', 'fail', 'done', 'stdout', 'stderr'])
+    for (const event of ['test', 'skipped', 'pass', 'fail', 'done', 'stdout', 'stderr'])
       testRunner.on(event, sendMessageToParent.bind(null, event));
     await testRunner.run();
     testRunner = null;