chore(testrunner): extract runtime api and use it from cli (#3585)

test-runner/src/cli.ts

@@ -14,21 +14,11 @@
  * limitations under the License.
  */
 
+import program from 'commander';
 import * as fs from 'fs';
 import * as path from 'path';
-import program from 'commander';
+import { collectTests, runTests, RunnerConfig } from '.';
 import { reporters } from './reporters';
-import { installTransform } from './transform';
-import { Runner } from './runner';
-import { TestCollector } from './testCollector';
-
-let beforeFunction;
-let afterFunction;
-let matrix = {};
-
-global['before'] = (fn => beforeFunction = fn);
-global['after'] = (fn => afterFunction = fn);
-global['matrix'] = (m => matrix = m);
 
 program
   .version('Version ' + /** @type {any} */ (require)('../package.json').version)
@@ -43,39 +33,32 @@ program
   .option('--timeout <timeout>', 'Specify test timeout threshold (in milliseconds), default: 10000', '10000')
   .option('-u, --update-snapshots', 'Use this flag to re-record every snapshot that fails during this test run')
   .action(async (command) => {
-    // Collect files]
     const testDir = path.resolve(process.cwd(), command.args[0]);
-    const files = collectFiles(testDir, '', command.args.slice(1));
+    const config: RunnerConfig = {
-
+      debug: command.debug,
-    const revertBabelRequire = installTransform();
+      quiet: command.quiet,
-    let hasSetup = false;
-    try {
-      hasSetup = fs.statSync(path.join(testDir, 'setup.js')).isFile();
-    } catch (e) {
-    }
-    try {
-      hasSetup = hasSetup || fs.statSync(path.join(testDir, 'setup.ts')).isFile();
-    } catch (e) {
-    }
-
-    if (hasSetup)
-      require(path.join(testDir, 'setup'));
-    revertBabelRequire();
-
-    const testCollector = new TestCollector(files, matrix, {
-      forbidOnly: command.forbidOnly || undefined,
       grep: command.grep,
+      jobs: command.jobs,
+      outputDir: command.output,
+      snapshotDir: path.join(testDir, '__snapshots__'),
+      testDir,
       timeout: command.timeout,
-    });
+      trialRun: command.trialRun,
-    const rootSuite = testCollector.suite;
+      updateSnapshots: command.updateSnapshots
-    if (command.forbidOnly && testCollector.hasOnly()) {
+    };
-      console.error('=====================================');
+    const files = collectFiles(testDir, '', command.args.slice(1));
-      console.error(' --forbid-only found a focused test.');
+    const suite = collectTests(config, files);
-      console.error('=====================================');
+    if (command.forbidOnly) {
-      process.exit(1);
+      const hasOnly = suite.eachTest(t => t.only) || suite.eachSuite(s => s.only);
+      if (hasOnly) {
+        console.error('=====================================');
+        console.error(' --forbid-only found a focused test.');
+        console.error('=====================================');
+        process.exit(1);
+      }
     }
 
-    const total = rootSuite.total();
+    const total = suite.total();
     if (!total) {
       console.error('=================');
       console.error(' no tests found.');
@@ -83,38 +66,15 @@ program
       process.exit(1);
     }
 
-    // Trial run does not need many workers, use one.
-    const jobs = (command.trialRun || command.debug) ? 1 : command.jobs;
-    const runner = new Runner(rootSuite, total, {
-      debug: command.debug,
-      quiet: command.quiet,
-      grep: command.grep,
-      jobs,
-      outputDir: command.output,
-      snapshotDir: path.join(testDir, '__snapshots__'),
-      testDir,
-      timeout: command.timeout,
-      trialRun: command.trialRun,
-      updateSnapshots: command.updateSnapshots
-    });
     const reporterFactory = reporters[command.reporter || 'dot'];
-    new reporterFactory(runner);
+    await runTests(config, suite, reporterFactory);
-
+    const hasFailures = suite.eachTest(t => t.error);
-    try {
+    process.exit(hasFailures ? 1 : 0);
-      if (beforeFunction)
-        await beforeFunction();
-      await runner.run();
-      await runner.stop();
-    } finally {
-      if (afterFunction)
-        await afterFunction();
-    }
-    process.exit(runner.stats.failures ? 1 : 0);
   });
 
 program.parse(process.argv);
 
-function collectFiles(testDir, dir, filters) {
+function collectFiles(testDir: string, dir: string, filters: string[]): string[] {
   const fullDir = path.join(testDir, dir);
   if (fs.statSync(fullDir).isFile())
     return [fullDir];

test-runner/src/fixturesUI.ts

@@ -51,6 +51,7 @@ function specBuilder(modifiers, specCallback) {
 
 export function fixturesUI(suite: Suite, file: string, timeout: number): () => void {
   const suites = [suite];
+  suite.file = file;
 
   const it = specBuilder(['skip', 'fail', 'slow', 'only'], (specs, title, fn) => {
     const suite = suites[0];
@@ -66,14 +67,13 @@ export function fixturesUI(suite: Suite, file: string, timeout: number): () => v
       test.pending = true;
     if (!only && specs.fail && specs.fail[0])
       test.pending = true;
-    test.pending = test.pending || suite.isPending();
+    suite._addTest(test);
-    suite.addTest(test);
     return test;
   });
 
   const describe = specBuilder(['skip', 'fail', 'only'], (specs, title, fn) => {
     const child = new Suite(title, suites[0]);
-    suites[0].addSuite(child);
+    suites[0]._addSuite(child);
     child.file = file;
     const only = specs.only && specs.only[0];
     if (only)

test-runner/src/index.ts

@@ -15,12 +15,28 @@
  * limitations under the License.
  */
 
+import * as fs from 'fs';
+import * as path from 'path';
 import './builtin.fixtures';
 import './expect';
 import { registerFixture as registerFixtureT, registerWorkerFixture as registerWorkerFixtureT } from './fixtures';
+import { reporters } from './reporters';
+import { Runner } from './runner';
 import { RunnerConfig } from './runnerConfig';
-import { Test } from './test';
+import { Suite, Test } from './test';
+import { Matrix, TestCollector } from './testCollector';
+import { installTransform } from './transform';
 export { parameters, registerParameter } from './fixtures';
+export { RunnerConfig } from './runnerConfig';
+export { Suite, Test } from './test';
+
+let beforeFunctions: Function[] = [];
+let afterFunctions: Function[] = [];
+let matrix: Matrix = {};
+
+global['before'] = (fn: Function) => beforeFunctions.push(fn);
+global['after'] = (fn: Function) => afterFunctions.push(fn);
+global['matrix'] = (m: Matrix) => matrix = m;
 
 export function registerFixture<T extends keyof TestState>(name: T, fn: (params: FixtureParameters & WorkerState & TestState, runTest: (arg: TestState[T]) => Promise<void>, config: RunnerConfig, test: Test) => Promise<void>) {
   registerFixtureT<RunnerConfig, T>(name, fn);
@@ -29,3 +45,39 @@ export function registerFixture<T extends keyof TestState>(name: T, fn: (params:
 export function registerWorkerFixture<T extends keyof (WorkerState & FixtureParameters)>(name: T, fn: (params: FixtureParameters & WorkerState, runTest: (arg: (WorkerState & FixtureParameters)[T]) => Promise<void>, config: RunnerConfig) => Promise<void>) {
   registerWorkerFixtureT<RunnerConfig, T>(name, fn);
 };
+
+export function collectTests(config: RunnerConfig, files: string[]): Suite {
+  const revertBabelRequire = installTransform();
+  let hasSetup = false;
+  try {
+    hasSetup = fs.statSync(path.join(config.testDir, 'setup.js')).isFile();
+  } catch (e) {
+  }
+  try {
+    hasSetup = hasSetup || fs.statSync(path.join(config.testDir, 'setup.ts')).isFile();
+  } catch (e) {
+  }
+  if (hasSetup)
+    require(path.join(config.testDir, 'setup'));
+  revertBabelRequire();
+
+  const testCollector = new TestCollector(files, matrix, config);
+  return testCollector.suite;
+}
+
+export async function runTests(config: RunnerConfig, suite: Suite, reporterFactory: any) {
+  // Trial run does not need many workers, use one.
+  const jobs = (config.trialRun || config.debug) ? 1 : config.jobs;
+  const runner = new Runner(suite, { ...config, jobs });
+  new reporterFactory(runner);
+
+  try {
+    for (const f of beforeFunctions)
+      await f();
+    await runner.run();
+    await runner.stop();
+  } finally {
+    for (const f of afterFunctions)
+      await f();
+  }
+}

test-runner/src/reporters.ts

@@ -179,20 +179,22 @@ export class JSONReporter extends BaseReporter {
     runner.once('end', () => {
       const result = {
         config: this.config,
-        tests: this.suite.tests.map(test => this._serializeTest(test)),
+        suites: this.suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s)
-        suites: this.suite.suites.map(suite => this._serializeSuite(suite))
       };
       console.log(JSON.stringify(result, undefined, 2));
     });
   }
 
   private _serializeSuite(suite: Suite): any {
+    if (!suite.eachTest(test => true))
+      return null;
+    const suites = suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s);
     return {
       title: suite.title,
       file: suite.file,
       configuration: suite.configuration,
       tests: suite.tests.map(test => this._serializeTest(test)),
-      suites: suite.suites.map(suite => this._serializeSuite(suite))
+      suites: suites.length ? suites : undefined
     };
   }
 

test-runner/src/runner.ts

@@ -36,7 +36,7 @@ export class Runner extends EventEmitter {
   readonly _config: RunnerConfig;
   private _suite: Suite;
 
-  constructor(suite: Suite, total: number, config: RunnerConfig) {
+  constructor(suite: Suite, config: RunnerConfig) {
     super();
 
     this._config = config;
@@ -68,6 +68,7 @@ export class Runner extends EventEmitter {
     });
 
     if (process.stdout.isTTY) {
+      const total = suite.total();
       console.log();
       const jobs = Math.min(config.jobs, this._testsByConfiguredFile.size);
       console.log(`Running ${total} test${ total > 1 ? 's' : '' } using ${jobs} worker${ jobs > 1 ? 's' : ''}`);

test-runner/src/test.ts

@@ -39,7 +39,15 @@ export class Test {
     this.fn = fn;
   }
 
-  clone(): Test {
+  titlePath(): string[] {
+    return [...this.suite.titlePath(), this.title];
+  }
+
+  fullTitle(): string {
+    return this.titlePath().join(' ');
+  }
+
+  _clone(): Test {
     const test = new Test(this.title, this.fn);
     test.suite = this.suite;
     test.only = this.only;
@@ -49,14 +57,6 @@ export class Test {
     test._overriddenFn = this._overriddenFn;
     return test;
   }
-
-  titlePath(): string[] {
-    return [...this.suite.titlePath(), this.title];
-  }
-
-  fullTitle(): string {
-    return this.titlePath().join(' ');
-  }
 }
 
 export class Suite {
@@ -91,22 +91,30 @@ export class Suite {
     return count;
   }
 
-  isPending(): boolean {
+  _isPending(): boolean {
-    return this.pending || (this.parent && this.parent.isPending());
+    return this.pending || (this.parent && this.parent._isPending());
   }
 
-  addTest(test: Test) {
+  _addTest(test: Test) {
     test.suite = this;
     this.tests.push(test);
     this._entries.push(test);
   }
 
-  addSuite(suite: Suite) {
+  _addSuite(suite: Suite) {
     suite.parent = this;
     this.suites.push(suite);
     this._entries.push(suite);
   }
 
+  eachSuite(fn: (suite: Suite) => boolean | void): boolean {
+    for (const suite of this.suites) {
+      if (suite.eachSuite(fn))
+        return true;
+    }
+    return false;
+  }
+
   eachTest(fn: (test: Test) => boolean | void): boolean {
     for (const suite of this.suites) {
       if (suite.eachTest(fn))
@@ -119,7 +127,7 @@ export class Suite {
     return false;
   }
 
-  clone(): Suite {
+  _clone(): Suite {
     const suite = new Suite(this.title);
     suite.only = this.only;
     suite.file = this.file;

test-runner/src/testCollector.ts

@@ -18,21 +18,26 @@ import path from 'path';
 import { fixturesForCallback } from './fixtures';
 import { Configuration, Test, Suite } from './test';
 import { fixturesUI } from './fixturesUI';
+import { RunnerConfig } from './runnerConfig';
+
+export type Matrix = {
+  [key: string]: string[]
+};
 
 export class TestCollector {
   suite: Suite;
 
-  private _matrix: { [key: string]: string; };
+  private _matrix: Matrix;
-  private _options: any;
+  private _config: RunnerConfig;
   private _grep: RegExp;
   private _hasOnly: boolean;
 
-  constructor(files: string[], matrix: { [key: string] : string }, options) {
+  constructor(files: string[], matrix: Matrix, config: RunnerConfig) {
     this._matrix = matrix;
-    this._options = options;
+    this._config = config;
     this.suite = new Suite('');
-    if (options.grep) {
+    if (config.grep) {
-      const match = options.grep.match(/^\/(.*)\/(g|i|)$|.*/);
+      const match = config.grep.match(/^\/(.*)\/(g|i|)$|.*/);
       this._grep = new RegExp(match[1] || match[0], match[2]);
     }
 
@@ -46,9 +51,9 @@ export class TestCollector {
     return this._hasOnly;
   }
 
-  _addFile(file) {
+  _addFile(file: string) {
     const suite = new Suite('');
-    const revertBabelRequire = fixturesUI(suite, file, this._options.timeout);
+    const revertBabelRequire = fixturesUI(suite, file, this._config.timeout);
     require(file);
     revertBabelRequire();
     suite._renumber();
@@ -95,30 +100,30 @@ export class TestCollector {
     // Only include the tests that requested these generations.
     for (const [hash, {configurationObject, configurationString, tests}] of workerGeneratorConfigurations.entries()) {
       const clone = this._cloneSuite(suite, configurationObject, configurationString, tests);
-      this.suite.addSuite(clone);
+      this.suite._addSuite(clone);
       clone.title = path.basename(file) + (hash.length ? `::[${hash}]` : '');
     }
   }
 
   _cloneSuite(suite: Suite, configurationObject: Configuration, configurationString: string, tests: Set<Test>) {
-    const copy = suite.clone();
+    const copy = suite._clone();
     copy.only = suite.only;
     copy.configuration = configurationObject;
     for (const entry of suite._entries) {
       if (entry instanceof Suite) {
-        copy.addSuite(this._cloneSuite(entry, configurationObject, configurationString, tests));
+        copy._addSuite(this._cloneSuite(entry, configurationObject, configurationString, tests));
       } else {
         const test = entry;
         if (!tests.has(test))
           continue;
         if (this._grep && !this._grep.test(test.fullTitle()))
           continue;
-        const testCopy = test.clone();
+        const testCopy = test._clone();
         testCopy.only = test.only;
         testCopy._ordinal = test._ordinal;
         testCopy._configurationObject = configurationObject;
         testCopy._configurationString = configurationString;
-        copy.addTest(testCopy);
+        copy._addTest(testCopy);
       }
     }
     return copy;

test-runner/src/testRunner.ts

@@ -107,7 +107,7 @@ export class TestRunner extends EventEmitter {
     if (this._ordinals.size && !this._ordinals.has(ordinal))
       return;
     this._remaining.delete(ordinal);
-    if (test.pending) {
+    if (test.pending || test.suite._isPending()) {
       this.emit('pending', { test: this._serializeTest(test) });
       return;
     }