chore(testrunner): introduce test result, reuse it in ipc (#3644)

.eslintignore

@@ -1,8 +1,5 @@
 test/assets/modernizr.js
-third_party/*
-utils/browser/playwright-web.js
 utils/doclint/check_public_api/test/
-utils/testrunner/examples/
 lib/
 *.js
 src/generated/*
@@ -14,5 +11,7 @@ src/server/webkit/protocol.ts
 /electron-types.d.ts
 utils/generate_types/overrides.d.ts
 utils/generate_types/test/test.ts
-test/
+/test/
-test-runner/
+node_modules/
+browser_patches/*/checkout/
+packages/**/*.d.ts

.eslintrc.js

@@ -2,7 +2,6 @@ module.exports = {
     parser: '@typescript-eslint/parser',
     plugins: ['@typescript-eslint', 'notice'],
     parserOptions: {
-      project: ['./tsconfig.json', './test/tsconfig.json'],
       ecmaVersion: 9,
       sourceType: 'module',
     },
@@ -113,8 +112,5 @@ module.exports = {
             "mustMatch": "Copyright",
             "templateFile": "./utils/copyright.js",
         }],
-
-        // type-aware rules
-        "@typescript-eslint/no-unnecessary-type-assertion": 2,
     }
 };

package.json

@@ -13,7 +13,7 @@
     "ftest": "cross-env BROWSER=firefox node test-runner/cli test/",
     "wtest": "cross-env BROWSER=webkit node test-runner/cli test/",
     "test": "node test-runner/cli test/",
-    "eslint": "[ \"$CI\" = true ] && eslint --quiet -f codeframe --ext js,ts ./src || eslint --ext js,ts ./src",
+    "eslint": "[ \"$CI\" = true ] && eslint --quiet -f codeframe --ext js,ts . || eslint --ext js,ts .",
     "tsc": "tsc -p .",
     "tsc-installer": "tsc -p ./src/install/tsconfig.json",
     "doc": "node utils/doclint/cli.js",

src/utils/browserPaths.ts

@@ -32,7 +32,9 @@ export type BrowserDescriptor = {
 export const hostPlatform = ((): BrowserPlatform => {
   const platform = os.platform();
   if (platform === 'darwin') {
-    const macVersion = execSync('sw_vers -productVersion').toString('utf8').trim().split('.').slice(0, 2).join('.');
+    const macVersion = execSync('sw_vers -productVersion', {
+      stdio: ['ignore', 'pipe', 'ignore']
+    }).toString('utf8').trim().split('.').slice(0, 2).join('.');
     return `mac${macVersion}` as BrowserPlatform;
   }
   if (platform === 'linux') {

test-runner/src/builtin.fixtures.ts

@@ -30,7 +30,7 @@ declare global {
     repeat(n: number): DescribeFunction;
   };
 
-  type ItFunction<STATE> = ((name: string, inner: (state: STATE) => Promise<void>) => void) & {
+  type ItFunction<STATE> = ((name: string, inner: (state: STATE) => Promise<void> | void) => void) & {
     fail(condition: boolean): ItFunction<STATE>;
     skip(condition: boolean): ItFunction<STATE>;
     slow(): ItFunction<STATE>;

test-runner/src/cli.ts

@@ -33,65 +33,65 @@ export const reporters = {
 };
 
 program
-  .version('Version ' + /** @type {any} */ (require)('../package.json').version)
+    .version('Version ' + /** @type {any} */ (require)('../package.json').version)
-  .option('--forbid-only', 'Fail if exclusive test(s) encountered', false)
+    .option('--forbid-only', 'Fail if exclusive test(s) encountered', false)
-  .option('-g, --grep <grep>', 'Only run tests matching this string or regexp', '.*')
+    .option('-g, --grep <grep>', 'Only run tests matching this string or regexp', '.*')
-  .option('-j, --jobs <jobs>', 'Number of concurrent jobs for --parallel; use 1 to run in serial, default: (number of CPU cores / 2)', Math.ceil(require('os').cpus().length / 2) as any)
+    .option('-j, --jobs <jobs>', 'Number of concurrent jobs for --parallel; use 1 to run in serial, default: (number of CPU cores / 2)', Math.ceil(require('os').cpus().length / 2) as any)
-  .option('--reporter <reporter>', 'Specify reporter to use, comma-separated, can be "dot", "list", "json"', 'dot')
+    .option('--reporter <reporter>', 'Specify reporter to use, comma-separated, can be "dot", "list", "json"', 'dot')
-  .option('--trial-run', 'Only collect the matching tests and report them as passing')
+    .option('--trial-run', 'Only collect the matching tests and report them as passing')
-  .option('--quiet', 'Suppress stdio', false)
+    .option('--quiet', 'Suppress stdio', false)
-  .option('--debug', 'Run tests in-process for debugging', false)
+    .option('--debug', 'Run tests in-process for debugging', false)
-  .option('--output <outputDir>', 'Folder for output artifacts, default: test-results', path.join(process.cwd(), 'test-results'))
+    .option('--output <outputDir>', 'Folder for output artifacts, default: test-results', path.join(process.cwd(), 'test-results'))
-  .option('--timeout <timeout>', 'Specify test timeout threshold (in milliseconds), default: 10000', '10000')
+    .option('--timeout <timeout>', 'Specify test timeout threshold (in milliseconds), default: 10000', '10000')
-  .option('-u, --update-snapshots', 'Use this flag to re-record every snapshot that fails during this test run')
+    .option('-u, --update-snapshots', 'Use this flag to re-record every snapshot that fails during this test run')
-  .action(async (command) => {
+    .action(async command => {
-    const testDir = path.resolve(process.cwd(), command.args[0]);
+      const testDir = path.resolve(process.cwd(), command.args[0]);
-    const config: RunnerConfig = {
+      const config: RunnerConfig = {
-      debug: command.debug,
+        debug: command.debug,
-      forbidOnly: command.forbidOnly,
+        forbidOnly: command.forbidOnly,
-      quiet: command.quiet,
+        quiet: command.quiet,
-      grep: command.grep,
+        grep: command.grep,
-      jobs: command.jobs,
+        jobs: command.jobs,
-      outputDir: command.output,
+        outputDir: command.output,
-      snapshotDir: path.join(testDir, '__snapshots__'),
+        snapshotDir: path.join(testDir, '__snapshots__'),
-      testDir,
+        testDir,
-      timeout: command.timeout,
+        timeout: command.timeout,
-      trialRun: command.trialRun,
+        trialRun: command.trialRun,
-      updateSnapshots: command.updateSnapshots
+        updateSnapshots: command.updateSnapshots
-    };
+      };
 
-    const reporterList = command.reporter.split(',');
+      const reporterList = command.reporter.split(',');
-    const reporterObjects: Reporter[] = reporterList.map(c => {
+      const reporterObjects: Reporter[] = reporterList.map(c => {
-      if (reporters[c])
+        if (reporters[c])
-        return new reporters[c]();
+          return new reporters[c]();
-      try {
+        try {
-        const p = path.resolve(process.cwd(), c);
+          const p = path.resolve(process.cwd(), c);
-        return new (require(p).default);
+          return new (require(p).default)();
-      } catch (e) {
+        } catch (e) {
-        console.error('Invalid reporter ' + c, e);
+          console.error('Invalid reporter ' + c, e);
+          process.exit(1);
+        }
+      });
+
+      const files = collectFiles(testDir, '', command.args.slice(1));
+      const result = await run(config, files, new Multiplexer(reporterObjects));
+      if (result === 'forbid-only') {
+        console.error('=====================================');
+        console.error(' --forbid-only found a focused test.');
+        console.error('=====================================');
         process.exit(1);
       }
+
+      if (result === 'no-tests') {
+        console.error('=================');
+        console.error(' no tests found.');
+        console.error('=================');
+        process.exit(1);
+      }
+
+      process.exit(result === 'failed' ? 1 : 0);
     });
 
-    const files = collectFiles(testDir, '', command.args.slice(1));
-    const result = await run(config, files, new Multiplexer(reporterObjects));
-    if (result === 'forbid-only') {
-      console.error('=====================================');
-      console.error(' --forbid-only found a focused test.');
-      console.error('=====================================');
-      process.exit(1);
-    }
-
-    if (result === 'no-tests') {
-      console.error('=================');
-      console.error(' no tests found.');
-      console.error('=================');
-      process.exit(1);
-    }
-
-    process.exit(result === 'failed' ? 1 : 0);
-  });
-
 program.parse(process.argv);
 
 function collectFiles(testDir: string, dir: string, filters: string[]): string[] {

test-runner/src/expect.ts

@@ -35,7 +35,7 @@ export function initializeImageMatcher(config: RunnerConfig) {
   function toMatchImage(received: Buffer, name: string, options?: { threshold?: number }) {
     const { pass, message } = compare(received, name, config, testFile, options);
     return { pass, message: () => message };
-  };
+  }
   expect.extend({ toMatchImage });
 }
 

test-runner/src/fixtures.ts

@@ -15,14 +15,21 @@
  */
 
 import debug from 'debug';
-import { Test, serializeError } from './test';
+import { RunnerConfig } from './runnerConfig';
+import { serializeError, Test, TestResult } from './test';
 
 type Scope = 'test' | 'worker';
 
 type FixtureRegistration = {
   name: string;
   scope: Scope;
-  fn: Function; 
+  fn: Function;
+};
+
+export type TestInfo = {
+  config: RunnerConfig;
+  test: Test;
+  result: TestResult;
 };
 
 const registrations = new Map<string, FixtureRegistration>();
@@ -36,8 +43,8 @@ export function setParameters(params: any) {
     registerWorkerFixture(name, async ({}, test) => await test(parameters[name]));
 }
 
-class Fixture<Config> {
+class Fixture {
-  pool: FixturePool<Config>;
+  pool: FixturePool;
   name: string;
   scope: Scope;
   fn: Function;
@@ -50,7 +57,7 @@ class Fixture<Config> {
   _setup = false;
   _teardown = false;
 
-  constructor(pool: FixturePool<Config>, name: string, scope: Scope, fn: any) {
+  constructor(pool: FixturePool, name: string, scope: Scope, fn: any) {
     this.pool = pool;
     this.name = name;
     this.scope = scope;
@@ -61,11 +68,11 @@ class Fixture<Config> {
     this.value = this.hasGeneratorValue ? parameters[name] : null;
   }
 
-  async setup(config: Config, test?: Test) {
+  async setup(config: RunnerConfig, info?: TestInfo) {
     if (this.hasGeneratorValue)
       return;
     for (const name of this.deps) {
-      await this.pool.setupFixture(name, config, test);
+      await this.pool.setupFixture(name, config, info);
       this.pool.instances.get(name).usages.add(this.name);
     }
 
@@ -77,11 +84,12 @@ class Fixture<Config> {
     const setupFence = new Promise((f, r) => { setupFenceFulfill = f; setupFenceReject = r; });
     const teardownFence = new Promise(f => this._teardownFenceCallback = f);
     debug('pw:test:hook')(`setup "${this.name}"`);
+    const param = info || config;
     this._tearDownComplete = this.fn(params, async (value: any) => {
       this.value = value;
       setupFenceFulfill();
       return await teardownFence;
-    }, config, test).catch((e: any) => setupFenceReject(e));
+    }, param).catch((e: any) => setupFenceReject(e));
     await setupFence;
     this._setup = true;
   }
@@ -107,13 +115,13 @@ class Fixture<Config> {
   }
 }
 
-export class FixturePool<Config> {
+export class FixturePool {
-  instances: Map<string, Fixture<Config>>;
+  instances: Map<string, Fixture>;
   constructor() {
     this.instances = new Map();
   }
 
-  async setupFixture(name: string, config: Config, test?: Test) {
+  async setupFixture(name: string, config: RunnerConfig, info?: TestInfo) {
     let fixture = this.instances.get(name);
     if (fixture)
       return fixture;
@@ -123,45 +131,48 @@ export class FixturePool<Config> {
     const { scope, fn } = registrations.get(name);
     fixture = new Fixture(this, name, scope, fn);
     this.instances.set(name, fixture);
-    await fixture.setup(config, test);
+    await fixture.setup(config, info);
     return fixture;
   }
 
   async teardownScope(scope: string) {
-    for (const [name, fixture] of this.instances) {
+    for (const [, fixture] of this.instances) {
       if (fixture.scope === scope)
         await fixture.teardown();
     }
   }
 
-  async resolveParametersAndRun(fn: Function, config: Config, test?: Test) {
+  async resolveParametersAndRun(fn: Function, config: RunnerConfig, info?: TestInfo) {
     const names = fixtureParameterNames(fn);
     for (const name of names)
-      await this.setupFixture(name, config, test);
+      await this.setupFixture(name, config, info);
     const params = {};
     for (const n of names)
       params[n] = this.instances.get(n).value;
     return fn(params);
   }
 
-  wrapTestCallback(callback: any, timeout: number, config: Config, test: Test) {
+  async runTestWithFixtures(fn: Function, timeout: number, info: TestInfo) {
-    if (!callback)
+    let timer: NodeJS.Timer;
-      return callback;
+    const timerPromise = new Promise(f => timer = setTimeout(f, timeout));
-    return async() => {
+    try {
-      let timer: NodeJS.Timer;
+      await Promise.race([
-      let timerPromise = new Promise(f => timer = setTimeout(f, timeout));
+        this.resolveParametersAndRun(fn, info.config, info).then(() => {
-      try {
+          info.result.status = 'passed';
-        await Promise.race([
+          clearTimeout(timer);
-          this.resolveParametersAndRun(callback, config, test).then(() => clearTimeout(timer)),
+        }),
-          timerPromise.then(() => Promise.reject(new Error(`Timeout of ${timeout}ms exceeded`)))
+        timerPromise.then(() => {
-        ]);
+          info.result.status = 'timedOut';
-      } catch (e) {
+          Promise.reject(new Error(`Timeout of ${timeout}ms exceeded`));
-        test.error = serializeError(e);
+        })
-        throw e;
+      ]);
-      } finally {
+    } catch (e) {
-        await this.teardownScope('test');
+      info.result.status = 'failed';
-      }
+      info.result.error = serializeError(e);
-    };
+      throw e;
+    } finally {
+      await this.teardownScope('test');
+    }
   }
 }
 
@@ -171,10 +182,10 @@ export function fixturesForCallback(callback: Function): string[] {
     for (const name of fixtureParameterNames(callback)) {
       if (name in names)
         continue;
-        names.add(name);
+      names.add(name);
-      if (!registrations.has(name)) {
+      if (!registrations.has(name))
         throw new Error('Using undefined fixture ' + name);
-      }
+
       const { fn } = registrations.get(name);
       visit(fn);
     }
@@ -190,7 +201,7 @@ function fixtureParameterNames(fn: Function): string[] {
   const match = text.match(/async(?:\s+function)?\s*\(\s*{\s*([^}]*)\s*}/);
   if (!match || !match[1].trim())
     return [];
-  let signature = match[1];
+  const signature = match[1];
   return signature.split(',').map((t: string) => t.trim());
 }
 
@@ -205,15 +216,15 @@ function innerRegisterFixture(name: string, scope: Scope, fn: Function, caller:
   if (!registrationsByFile.has(file))
     registrationsByFile.set(file, []);
   registrationsByFile.get(file).push(registration);
-};
+}
 
-export function registerFixture<Config>(name: string, fn: (params: any, runTest: (arg: any) => Promise<void>, config: Config, test: Test) => Promise<void>) {
+export function registerFixture(name: string, fn: (params: any, runTest: (arg: any) => Promise<void>, info: TestInfo) => Promise<void>) {
   innerRegisterFixture(name, 'test', fn, registerFixture);
-};
+}
 
-export function registerWorkerFixture<Config>(name: string, fn: (params: any, runTest: (arg: any) => Promise<void>, config: Config) => Promise<void>) {
+export function registerWorkerFixture(name: string, fn: (params: any, runTest: (arg: any) => Promise<void>, config: RunnerConfig) => Promise<void>) {
   innerRegisterFixture(name, 'worker', fn, registerWorkerFixture);
-};
+}
 
 export function registerParameter(name: string, fn: () => any) {
   registerWorkerFixture(name, async ({}: any, test: Function) => await test(parameters[name]));
@@ -236,7 +247,7 @@ export function lookupRegistrations(file: string, scope: Scope) {
   const deps = new Set<string>();
   collectRequires(file, deps);
   const allDeps = [...deps].reverse();
-  let result = new Map();
+  const result = new Map();
   for (const dep of allDeps) {
     const registrationList = registrationsByFile.get(dep);
     if (!registrationList)
@@ -244,7 +255,7 @@ export function lookupRegistrations(file: string, scope: Scope) {
     for (const r of registrationList) {
       if (scope && r.scope !== scope)
         continue;
-        result.set(r.name, r);
+      result.set(r.name, r);
     }
   }
   return result;

test-runner/src/golden.ts

@@ -70,7 +70,7 @@ function compareText(actual: Buffer, expectedBuffer: Buffer): { diff?: object; e
   };
 }
 
-export function compare(actual: Buffer, name: string, config: RunnerConfig, testFile: string, options?: { threshold?: number } ): { pass: boolean; message?: string; } {
+export function compare(actual: Buffer, name: string, config: RunnerConfig, testFile: string, options?: { threshold?: number }): { pass: boolean; message?: string; } {
   let expectedPath: string;
   const relativeTestFile = path.relative(config.testDir, testFile);
   const testAssetsDir = relativeTestFile.replace(/\.spec\.[jt]s/, '');
@@ -125,8 +125,8 @@ export function compare(actual: Buffer, name: string, config: RunnerConfig, test
   }
   fs.writeFileSync(actualPath, actual);
   if (result.diff)
-    fs.writeFileSync(diffPath, result.diff);  
+    fs.writeFileSync(diffPath, result.diff);
-  
+
   const output = [
     c.red(`Image comparison failed:`),
   ];

test-runner/src/index.ts

@@ -19,11 +19,10 @@ import * as fs from 'fs';
 import * as path from 'path';
 import './builtin.fixtures';
 import './expect';
-import { registerFixture as registerFixtureT, registerWorkerFixture as registerWorkerFixtureT } from './fixtures';
+import { registerFixture as registerFixtureT, registerWorkerFixture as registerWorkerFixtureT, TestInfo } from './fixtures';
 import { Reporter } from './reporter';
 import { Runner } from './runner';
 import { RunnerConfig } from './runnerConfig';
-import { Test } from './test';
 import { Matrix, TestCollector } from './testCollector';
 import { installTransform } from './transform';
 export { parameters, registerParameter } from './fixtures';
@@ -42,21 +41,21 @@ declare global {
   }
 }
 
-let beforeFunctions: Function[] = [];
+const beforeFunctions: Function[] = [];
-let afterFunctions: Function[] = [];
+const afterFunctions: Function[] = [];
 let matrix: Matrix = {};
 
 global['before'] = (fn: Function) => beforeFunctions.push(fn);
 global['after'] = (fn: Function) => afterFunctions.push(fn);
 global['matrix'] = (m: Matrix) => matrix = m;
 
-export function registerFixture<T extends keyof TestState>(name: T, fn: (params: FixtureParameters & WorkerState & TestState, runTest: (arg: TestState[T]) => Promise<void>, config: RunnerConfig, test: Test) => Promise<void>) {
+export function registerFixture<T extends keyof TestState>(name: T, fn: (params: FixtureParameters & WorkerState & TestState, runTest: (arg: TestState[T]) => Promise<void>, info: TestInfo) => Promise<void>) {
-  registerFixtureT<RunnerConfig>(name, fn);
+  registerFixtureT(name, fn);
-};
+}
 
-export function registerWorkerFixture<T extends keyof (WorkerState & FixtureParameters)>(name: T, fn: (params: FixtureParameters & WorkerState, runTest: (arg: (WorkerState & FixtureParameters)[T]) => Promise<void>, config: RunnerConfig) => Promise<void>) {
+export function registerWorkerFixture<T extends keyof(WorkerState & FixtureParameters)>(name: T, fn: (params: FixtureParameters & WorkerState, runTest: (arg: (WorkerState & FixtureParameters)[T]) => Promise<void>, config: RunnerConfig) => Promise<void>) {
-  registerWorkerFixtureT<RunnerConfig>(name, fn);
+  registerWorkerFixtureT(name, fn);
-};
+}
 
 type RunResult = 'passed' | 'failed' | 'forbid-only' | 'no-tests';
 
@@ -100,5 +99,7 @@ export async function run(config: RunnerConfig, files: string[], reporter: Repor
     for (const f of afterFunctions)
       await f();
   }
-  return suite.findTest(t => t.error) ? 'failed' : 'passed';
+  return suite.findTest(test => {
+    return !!test.results.find(result => result.status === 'failed' || result.status === 'timedOut');
+  }) ? 'failed' : 'passed';
 }

test-runner/src/reporter.ts

@@ -15,15 +15,13 @@
  */
 
 import { RunnerConfig } from './runnerConfig';
-import { Suite, Test } from './test';
+import { Suite, Test, TestResult } from './test';
 
 export interface Reporter {
   onBegin(config: RunnerConfig, suite: Suite): void;
-  onTest(test: Test): void;
+  onTestBegin(test: Test): void;
-  onSkippedTest(test: Test): void;
   onTestStdOut(test: Test, chunk: string | Buffer);
   onTestStdErr(test: Test, chunk: string | Buffer);
-  onTestPassed(test: Test): void;
+  onTestEnd(test: Test, result: TestResult);
-  onTestFailed(test: Test): void;
   onEnd(): void;
 }

test-runner/src/reporters/base.ts

@@ -24,15 +24,15 @@ import StackUtils from 'stack-utils';
 import terminalLink from 'terminal-link';
 import { Reporter } from '../reporter';
 import { RunnerConfig } from '../runnerConfig';
-import { Suite, Test } from '../test';
+import { Suite, Test, TestResult } from '../test';
 
-const stackUtils = new StackUtils()
+const stackUtils = new StackUtils();
 
 export class BaseReporter implements Reporter  {
   skipped: Test[] = [];
-  passes: Test[] = [];
+  passed: { test: Test, result: TestResult }[] = [];
-  failures: Test[] = [];
+  failed: { test: Test, result: TestResult }[] = [];
-  timeouts: Test[] = [];
+  timedOut: { test: Test, result: TestResult }[] = [];
   duration = 0;
   startTime: number;
   config: RunnerConfig;
@@ -51,11 +51,7 @@ export class BaseReporter implements Reporter  {
     this.suite = suite;
   }
 
-  onTest(test: Test) {
+  onTestBegin(test: Test) {
-  }
-
-  onSkippedTest(test: Test) {
-    this.skipped.push(test);
   }
 
   onTestStdOut(test: Test, chunk: string | Buffer) {
@@ -68,15 +64,13 @@ export class BaseReporter implements Reporter  {
       process.stderr.write(chunk);
   }
 
-  onTestPassed(test: Test) {
+  onTestEnd(test: Test, result: TestResult) {
-    this.passes.push(test);
+    switch (result.status) {
-  }
+      case 'skipped': this.skipped.push(test); break;
-
+      case 'passed': this.passed.push({ test, result }); break;
-  onTestFailed(test: Test) {
+      case 'failed': this.failed.push({ test, result }); break;
-    if (test.duration >= test.timeout)
+      case 'timedOut': this.timedOut.push({ test, result }); break;
-      this.timeouts.push(test);
+    }
-    else
-      this.failures.push(test);
   }
 
   onEnd() {
@@ -86,56 +80,61 @@ export class BaseReporter implements Reporter  {
   epilogue() {
     console.log('');
 
-    console.log(colors.green(`  ${this.passes.length} passed`) + colors.dim(` (${milliseconds(this.duration)})`));  
+    console.log(colors.green(`  ${this.passed.length} passed`) + colors.dim(` (${milliseconds(this.duration)})`));
 
     if (this.skipped.length)
       console.log(colors.yellow(`  ${this.skipped.length} skipped`));
 
-    if (this.failures.length) {
+    if (this.failed.length) {
-      console.log(colors.red(`  ${this.failures.length} failed`));
+      console.log(colors.red(`  ${this.failed.length} failed`));
       console.log('');
-      this._printFailures(this.failures);
+      this._printFailures(this.failed);
     }
 
-    if (this.timeouts.length) {
+    if (this.timedOut.length) {
-      console.log(colors.red(`  ${this.timeouts.length} timed out`));
+      console.log(colors.red(`  ${this.timedOut.length} timed out`));
       console.log('');
-      this._printFailures(this.timeouts);
+      this._printFailures(this.timedOut);
     }
   }
 
-  private _printFailures(failures: Test[]) {
+  private _printFailures(failures: { test: Test, result: TestResult}[]) {
-    failures.forEach((failure, index) => {
+    failures.forEach(({test, result}, index) => {
-      console.log(this.formatFailure(failure, index + 1));
+      console.log(this.formatFailure(test, result, index + 1));
     });
   }
 
-  formatFailure(failure: Test, index?: number): string {
+  formatFailure(test: Test, failure: TestResult, index?: number): string {
     const tokens: string[] = [];
-    const relativePath = path.relative(process.cwd(), failure.file);
+    const relativePath = path.relative(process.cwd(), test.file);
-    const header = `  ${index ? index + ')' : ''} ${terminalLink(relativePath, `file://${os.hostname()}${failure.file}`)} › ${failure.title}`;
+    const header = `  ${index ? index + ')' : ''} ${terminalLink(relativePath, `file://${os.hostname()}${test.file}`)} › ${test.title}`;
     tokens.push(colors.bold(colors.red(header)));
-    const stack = failure.error.stack;
+    if (failure.status === 'timedOut') {
-    if (stack) {
       tokens.push('');
-      const messageLocation = failure.error.stack.indexOf(failure.error.message);
+      tokens.push(indent(colors.red(`Timeout of ${test.timeout}ms exceeded.`), '    '));
-      const preamble = failure.error.stack.substring(0, messageLocation + failure.error.message.length);
+    } else {
-      tokens.push(indent(preamble, '    '));
+      const stack = failure.error.stack;
-      const position = positionInFile(stack, failure.file);
+      if (stack) {
-      if (position) {
-        const source = fs.readFileSync(failure.file, 'utf8');
         tokens.push('');
-        tokens.push(indent(codeFrameColumns(source, {
+        const messageLocation = failure.error.stack.indexOf(failure.error.message);
+        const preamble = failure.error.stack.substring(0, messageLocation + failure.error.message.length);
+        tokens.push(indent(preamble, '    '));
+        const position = positionInFile(stack, test.file);
+        if (position) {
+          const source = fs.readFileSync(test.file, 'utf8');
+          tokens.push('');
+          tokens.push(indent(codeFrameColumns(source, {
             start: position,
           },
           { highlightCode: true}
-        ), '    '));
+          ), '    '));
+        }
+        tokens.push('');
+        tokens.push(indent(colors.dim(stack.substring(preamble.length + 1)), '    '));
+      } else {
+        tokens.push('');
+        tokens.push(indent(String(failure.error), '    '));
       }
-      tokens.push('');
-      tokens.push(indent(colors.dim(stack.substring(preamble.length + 1)), '    '));
-    } else {
-      tokens.push('');
-      tokens.push(indent(String(failure.error), '    '));
     }
     tokens.push('');
     return tokens.join('\n');

test-runner/src/reporters/dot.ts

@@ -16,27 +16,19 @@
 
 import colors from 'colors/safe';
 import { BaseReporter } from './base';
-import { Test } from '../test';
+import { Test, TestResult } from '../test';
 
 class DotReporter extends BaseReporter {
-  onSkippedTest(test: Test) {
+  onTestEnd(test: Test, result: TestResult) {
-    super.onSkippedTest(test);
+    super.onTestEnd(test, result);
-    process.stdout.write(colors.yellow('∘'))
+    switch (result.status) {
+      case 'skipped': process.stdout.write(colors.yellow('∘')); break;
+      case 'passed': process.stdout.write(colors.green('·')); break;
+      case 'failed': process.stdout.write(colors.red('F')); break;
+      case 'timedOut': process.stdout.write(colors.red('T')); break;
+    }
   }
-  
+
-  onTestPassed(test: Test) {
-    super.onTestPassed(test);
-    process.stdout.write(colors.green('·'));
-  }
-  
-  onTestFailed(test: Test) {
-    super.onTestFailed(test);
-    if (test.duration >= test.timeout)
-      process.stdout.write(colors.red('T'));
-    else
-      process.stdout.write(colors.red('F'));
-  } 
- 
   onEnd() {
     super.onEnd();
     process.stdout.write('\n');

test-runner/src/reporters/json.ts

@@ -15,7 +15,7 @@
  */
 
 import { BaseReporter } from './base';
-import { Suite, Test } from '../test';
+import { Suite, Test, TestResult } from '../test';
 import * as fs from 'fs';
 
 class JSONReporter extends BaseReporter {
@@ -50,14 +50,19 @@ class JSONReporter extends BaseReporter {
       title: test.title,
       file: test.file,
       only: test.only,
-      skipped: test.skipped,
       slow: test.slow,
-      duration: test.duration,
       timeout: test.timeout,
-      error: test.error,
+      results: test.results.map(r => this._serializeTestResult(r))
-      stdout: test.stdout.map(s => stdioEntry(s)),
+    };
-      stderr: test.stderr.map(s => stdioEntry(s)),
+  }
-      data: test.data
+
+  private _serializeTestResult(result: TestResult): any {
+    return {
+      duration: result.duration,
+      error: result.error,
+      stdout: result.stdout.map(s => stdioEntry(s)),
+      stderr: result.stderr.map(s => stdioEntry(s)),
+      data: result.data
     };
   }
 }
@@ -65,7 +70,7 @@ class JSONReporter extends BaseReporter {
 function stdioEntry(s: string | Buffer): any {
   if (typeof s === 'string')
     return { text: s };
-  return { buffer: s.toString('base64') }
+  return { buffer: s.toString('base64') };
 }
 
 export default JSONReporter;

test-runner/src/reporters/list.ts

@@ -17,7 +17,7 @@
 import colors from 'colors/safe';
 import { BaseReporter } from './base';
 import { RunnerConfig } from '../runnerConfig';
-import { Suite, Test } from '../test';
+import { Suite, Test, TestResult } from '../test';
 
 class ListReporter extends BaseReporter {
   _failure = 0;
@@ -27,29 +27,22 @@ class ListReporter extends BaseReporter {
     console.log();
   }
 
-  onTest(test: Test) {
+  onTestBegin(test: Test) {
-    super.onTest(test);
+    super.onTestBegin(test);
     process.stdout.write('    ' + colors.gray(test.fullTitle() + ': '));
   }
 
-  onSkippedTest(test: Test) {
+  onTestEnd(test: Test, result: TestResult) {
-    super.onSkippedTest(test);
+    super.onTestEnd(test, result);
-    process.stdout.write(colors.green('  - ') + colors.cyan(test.fullTitle()));
+    let text = '';
-    process.stdout.write('\n');
+    switch (result.status) {
-  }
+      case 'skipped': text = colors.green('  - ') + colors.cyan(test.fullTitle()); break;
-
+      case 'passed': text = '\u001b[2K\u001b[0G' + colors.green('  ✓ ') + colors.gray(test.fullTitle()); break;
-  onTestPassed(test: Test) {
+      case 'failed':
-    super.onTestPassed(test);
+        // fall through
-    process.stdout.write('\u001b[2K\u001b[0G');
+      case 'timedOut': text = '\u001b[2K\u001b[0G' + colors.red(`  ${++this._failure}) ` + test.fullTitle()); break;
-    process.stdout.write(colors.green('  ✓ ') + colors.gray(test.fullTitle()));
+    }
-    process.stdout.write('\n');
+    process.stdout.write(text + '\n');
-  }
-
-  onTestFailed(test: Test) {
-    super.onTestFailed(test);
-    process.stdout.write('\u001b[2K\u001b[0G');
-    process.stdout.write(colors.red(`  ${++this._failure}) ` + test.fullTitle()));
-    process.stdout.write('\n');
   }
 
   onEnd() {

test-runner/src/reporters/multiplexer.ts

@@ -15,7 +15,7 @@
  */
 
 import { RunnerConfig } from '../runnerConfig';
-import { Suite, Test } from '../test';
+import { Suite, Test, TestResult } from '../test';
 import { Reporter } from '../reporter';
 
 export class Multiplexer implements Reporter {
@@ -30,14 +30,9 @@ export class Multiplexer implements Reporter {
       reporter.onBegin(config, suite);
   }
 
-  onTest(test: Test) {
+  onTestBegin(test: Test) {
     for (const reporter of this._reporters)
-      reporter.onTest(test);
+      reporter.onTestBegin(test);
-  }
-
-  onSkippedTest(test: Test) {
-    for (const reporter of this._reporters)
-      reporter.onSkippedTest(test);
   }
 
   onTestStdOut(test: Test, chunk: string | Buffer) {
@@ -47,17 +42,12 @@ export class Multiplexer implements Reporter {
 
   onTestStdErr(test: Test, chunk: string | Buffer) {
     for (const reporter of this._reporters)
-      reporter.onTestStdErr(test, chunk);    
+      reporter.onTestStdErr(test, chunk);
   }
 
-  onTestPassed(test: Test) {
+  onTestEnd(test: Test, result: TestResult) {
     for (const reporter of this._reporters)
-      reporter.onTestPassed(test);
+      reporter.onTestEnd(test, result);
-  }
-
-  onTestFailed(test: Test) {
-    for (const reporter of this._reporters)
-      reporter.onTestFailed(test);
   }
 
   onEnd() {

test-runner/src/reporters/pytest.ts

@@ -17,12 +17,12 @@
 import colors from 'colors/safe';
 import milliseconds from 'ms';
 import * as path from 'path';
-import { Test, Suite, Configuration } from '../test';
+import { Test, Suite, Configuration, TestResult } from '../test';
 import { BaseReporter } from './base';
 import { RunnerConfig } from '../runnerConfig';
 
 const cursorPrevLine = '\u001B[F';
-const eraseLine = '\u001B[2K'
+const eraseLine = '\u001B[2K';
 
 type Row = {
   id: string;
@@ -43,7 +43,6 @@ class PytestReporter extends BaseReporter {
   private _suiteIds = new Map<Suite, string>();
   private _lastOrdinal = 0;
   private _visibleRows: number;
-  private _failed = false;
   private _total: number;
   private _progress: string[] = [];
   private _throttler = new Throttler(250, () => this._repaint());
@@ -77,20 +76,13 @@ class PytestReporter extends BaseReporter {
     }
   }
 
-  onTest(test: Test) {
+  onTestBegin(test: Test) {
-    super.onTest(test);
+    super.onTestBegin(test);
     const row = this._rows.get(this._id(test));
     if (!row.startTime)
       row.startTime = Date.now();
   }
 
-  onSkippedTest(test: Test) {
-    super.onSkippedTest(test);
-    this._append(test, colors.yellow('∘'));
-    this._progress.push('S');
-    this._throttler.schedule();
-  }
-
   onTestStdOut(test: Test, chunk: string | Buffer) {
     this._repaint(chunk);
   }
@@ -99,21 +91,32 @@ class PytestReporter extends BaseReporter {
     this._repaint(chunk);
   }
 
-  onTestPassed(test: Test) {
+  onTestEnd(test: Test, result: TestResult) {
-    super.onTestPassed(test);
+    super.onTestEnd(test, result);
-    this._append(test, colors.green('✓'));
+    switch (result.status) {
-    this._progress.push('P');
+      case 'skipped': {
-    this._throttler.schedule();
+        this._append(test, colors.yellow('∘'));
-  }
+        this._progress.push('S');
-
+        this._throttler.schedule();
-  onTestFailed(test: Test) {
+        break;
-    super.onTestFailed(test);
+      }
-    const title = test.duration >= test.timeout ? colors.red('T') : colors.red('F');
+      case 'passed': {
-    const row = this._append(test, title);
+        this._append(test, colors.green('✓'));
-    row.failed = true;
+        this._progress.push('P');
-    this._failed = true;
+        this._throttler.schedule();
-    this._progress.push('F');
+        break;
-    this._repaint(this.formatFailure(test) + '\n');
+      }
+      case 'failed':
+        // fall through
+      case 'timedOut': {
+        const title = result.status === 'timedOut' ? colors.red('T') : colors.red('F');
+        const row = this._append(test, title);
+        row.failed = true;
+        this._progress.push('F');
+        this._repaint(this.formatFailure(test, result) + '\n');
+        break;
+      }
+    }
   }
 
   private _append(test: Test, s: string): Row {
@@ -146,14 +149,14 @@ class PytestReporter extends BaseReporter {
     }
 
     const status = [];
-    if (this.passes.length)
+    if (this.passed.length)
-      status.push(colors.green(`${this.passes.length} passed`));
+      status.push(colors.green(`${this.passed.length} passed`));
     if (this.skipped.length)
       status.push(colors.yellow(`${this.skipped.length} skipped`));
-    if (this.failures.length)
+    if (this.failed.length)
-      status.push(colors.red(`${this.failures.length} failed`));
+      status.push(colors.red(`${this.failed.length} failed`));
-    if (this.timeouts.length)
+    if (this.timedOut.length)
-      status.push(colors.red(`${this.timeouts.length} timed out`));
+      status.push(colors.red(`${this.timedOut.length} timed out`));
     status.push(colors.dim(`(${milliseconds(Date.now() - this.startTime)})`));
 
     for (let i = lines.length; i < this._visibleRows; ++i)
@@ -214,7 +217,7 @@ const yellowBar = colors.yellow('▇');
 function serializeConfiguration(configuration: Configuration): string {
   const tokens = [];
   for (const { name, value } of configuration)
-      tokens.push(`${name}=${value}`);
+    tokens.push(`${name}=${value}`);
   return tokens.join(', ');
 }
 

test-runner/src/runner.ts

@@ -19,8 +19,8 @@ import crypto from 'crypto';
 import path from 'path';
 import { EventEmitter } from 'events';
 import { lookupRegistrations, FixturePool } from './fixtures';
-import { Suite, Test } from './test';
+import { Suite, Test, TestResult } from './test';
-import { TestRunnerEntry, SerializedTest } from './testRunner';
+import { TestRunnerEntry } from './testRunner';
 import { RunnerConfig } from './runnerConfig';
 import { Reporter } from './reporter';
 
@@ -28,9 +28,8 @@ export class Runner {
   private _workers = new Set<Worker>();
   private _freeWorkers: Worker[] = [];
   private _workerClaimers: (() => void)[] = [];
-  stats: { duration: number; failures: number; passes: number; skipped: number; tests: number; };
 
-  private _testById = new Map<string, Test>();
+  private _testById = new Map<string, { test: Test, result: TestResult }>();
   private _queue: TestRunnerEntry[] = [];
   private _stopCallback: () => void;
   readonly _config: RunnerConfig;
@@ -40,18 +39,11 @@ export class Runner {
   constructor(suite: Suite, config: RunnerConfig, reporter: Reporter) {
     this._config = config;
     this._reporter = reporter;
-    this.stats = {
-      duration: 0,
-      failures: 0,
-      passes: 0,
-      skipped: 0,
-      tests: 0,
-    };
 
     this._suite = suite;
     for (const suite of this._suite.suites) {
       suite.findTest(test => {
-        this._testById.set(`${test._ordinal}@${suite.file}::[${suite._configurationString}]`, test);
+        this._testById.set(`${test._ordinal}@${suite.file}::[${suite._configurationString}]`, { test, result: test._appendResult() });
       });
     }
 
@@ -59,7 +51,7 @@ export class Runner {
       const total = suite.total();
       console.log();
       const jobs = Math.min(config.jobs, suite.suites.length);
-      console.log(`Running ${total} test${ total > 1 ? 's' : '' } using ${jobs} worker${ jobs > 1 ? 's' : ''}`);
+      console.log(`Running ${total} test${total > 1 ? 's' : ''} using ${jobs} worker${jobs > 1 ? 's' : ''}`);
     }
   }
 
@@ -158,33 +150,29 @@ export class Runner {
 
   _createWorker() {
     const worker = this._config.debug ? new InProcessWorker(this) : new OopWorker(this);
-    worker.on('test', params => {
+    worker.on('testBegin', params => {
-      ++this.stats.tests;
+      const { test } = this._testById.get(params.id);
-      this._reporter.onTest(this._updateTest(params.test));
+      this._reporter.onTestBegin(test);
     });
-    worker.on('skipped', params => {
+    worker.on('testEnd', params => {
-      ++this.stats.tests;
+      const workerResult: TestResult = params.result;
-      ++this.stats.skipped;
+      // We were accumulating these below.
-      this._reporter.onSkippedTest(this._updateTest(params.test));
+      delete workerResult.stdout;
+      delete workerResult.stderr;
+      const { test, result } = this._testById.get(params.id);
+      Object.assign(result, workerResult);
+      this._reporter.onTestEnd(test, result);
     });
-    worker.on('pass', params => {
+    worker.on('testStdOut', params => {
-      ++this.stats.passes;
-      this._reporter.onTestPassed(this._updateTest(params.test));
-    });
-    worker.on('fail', params => {
-      ++this.stats.failures;
-      this._reporter.onTestFailed(this._updateTest(params.test));
-    });
-    worker.on('stdout', params => {
       const chunk = chunkFromParams(params);
-      const test = this._testById.get(params.testId);
+      const { test, result } = this._testById.get(params.id);
-      test.stdout.push(chunk);
+      result.stdout.push(chunk);
       this._reporter.onTestStdOut(test, chunk);
     });
-    worker.on('stderr', params => {
+    worker.on('testStdErr', params => {
       const chunk = chunkFromParams(params);
-      const test = this._testById.get(params.testId);
+      const { test, result } = this._testById.get(params.id);
-      test.stderr.push(chunk);
+      result.stderr.push(chunk);
       this._reporter.onTestStdErr(test, chunk);
     });
     worker.on('exit', () => {
@@ -201,14 +189,6 @@ export class Runner {
     this._createWorker();
   }
 
-  _updateTest(serialized: SerializedTest): Test {
-    const test = this._testById.get(serialized.id);
-    test.duration = serialized.duration;
-    test.error = serialized.error;
-    test.data = serialized.data;
-    return test;
-  }
-
   async stop() {
     const result = new Promise(f => this._stopCallback = f);
     for (const worker of this._workers)
@@ -253,7 +233,7 @@ class OopWorker extends Worker {
       stdio: ['ignore', 'ignore', 'ignore', 'ipc']
     });
     this.process.on('exit', () => this.emit('exit'));
-    this.process.on('error', (e) => {});  // do not yell at a send to dead process.
+    this.process.on('error', e => {});  // do not yell at a send to dead process.
     this.process.on('message', message => {
       const { method, params } = message;
       this.emit(method, params);
@@ -276,11 +256,11 @@ class OopWorker extends Worker {
 }
 
 class InProcessWorker extends Worker {
-  fixturePool: FixturePool<RunnerConfig>;
+  fixturePool: FixturePool;
 
   constructor(runner: Runner) {
     super(runner);
-    this.fixturePool = require('./testRunner').fixturePool as FixturePool<RunnerConfig>;
+    this.fixturePool = require('./testRunner').fixturePool as FixturePool;
   }
 
   async init() {
@@ -292,7 +272,7 @@ class InProcessWorker extends Worker {
     delete require.cache[entry.file];
     const { TestRunner } = require('./testRunner');
     const testRunner = new TestRunner(entry, this.runner._config, 0);
-    for (const event of ['test', 'skipped', 'pass', 'fail', 'done', 'stdout', 'stderr'])
+    for (const event of ['testBegin', 'testStdOut', 'testStdErr', 'testEnd', 'done'])
       testRunner.on(event, this.emit.bind(this, event));
     testRunner.run();
   }

test-runner/src/spec.ts

@@ -64,9 +64,9 @@ export function spec(suite: Suite, file: string, timeout: number): () => void {
     if (only)
       test.only = true;
     if (!only && specs.skip && specs.skip[0])
-      test.skipped = true;
+      test._skipped = true;
     if (!only && specs.fail && specs.fail[0])
-      test.skipped = true;
+      test._skipped = true;
     suite._addTest(test);
     return test;
   });

test-runner/src/test.ts

@@ -21,15 +21,11 @@ export class Test {
   title: string;
   file: string;
   only = false;
-  skipped = false;
+  _skipped = false;
   slow = false;
-  duration = 0;
   timeout = 0;
   fn: Function;
-  error: any;
+  results: TestResult[] = [];
-  stdout: (string | Buffer)[] = [];
-  stderr: (string | Buffer)[] = [];
-  data: any = {};
 
   _ordinal: number;
   _overriddenFn: Function;
@@ -48,18 +44,38 @@ export class Test {
     return this.titlePath().join(' ');
   }
 
+  _appendResult(): TestResult {
+    const result: TestResult = {
+      duration: 0,
+      status: 'none',
+      stdout: [],
+      stderr: [],
+      data: {}
+    };
+    this.results.push(result);
+    return result;
+  }
+
   _clone(): Test {
     const test = new Test(this.title, this.fn);
     test.suite = this.suite;
     test.only = this.only;
     test.file = this.file;
-    test.skipped = this.skipped;
     test.timeout = this.timeout;
     test._overriddenFn = this._overriddenFn;
     return test;
   }
 }
 
+export type TestResult = {
+  duration: number;
+  status: 'none' | 'passed' | 'failed' | 'timedOut' | 'skipped';
+  error?: any;
+  stdout: (string | Buffer)[];
+  stderr: (string | Buffer)[];
+  data: any;
+}
+
 export class Suite {
   title: string;
   parent?: Suite;
@@ -152,7 +168,7 @@ export class Suite {
   _hasTestsToRun(): boolean {
     let found = false;
     this.findTest(test => {
-      if (!test.skipped) {
+      if (!test._skipped) {
         found = true;
         return true;
       }
@@ -164,7 +180,7 @@ export class Suite {
 export function serializeConfiguration(configuration: Configuration): string {
   const tokens = [];
   for (const { name, value } of configuration)
-      tokens.push(`${name}=${value}`);
+    tokens.push(`${name}=${value}`);
   return tokens.join(', ');
 }
 
@@ -173,7 +189,7 @@ export function serializeError(error: Error | any): any {
     return {
       message: error.message,
       stack: error.stack
-    }
+    };
   }
   return trimCycles(error);
 }
@@ -181,13 +197,13 @@ export function serializeError(error: Error | any): any {
 function trimCycles(obj: any): any {
   const cache = new Set();
   return JSON.parse(
-    JSON.stringify(obj, function(key, value) {
+      JSON.stringify(obj, function(key, value) {
-      if (typeof value === 'object' && value !== null) {
+        if (typeof value === 'object' && value !== null) {
-        if (cache.has(value))
+          if (cache.has(value))
-          return '' + value;
+            return '' + value;
-        cache.add(value);
+          cache.add(value);
-      }
+        }
-      return value;
+        return value;
-    })
+      })
   );
 }

test-runner/src/testCollector.ts

@@ -71,7 +71,7 @@ export class TestCollector {
         const values = this._matrix[name];
         if (!values)
           continue;
-        let state = generatorConfigurations.length ? generatorConfigurations.slice() : [[]];
+        const state = generatorConfigurations.length ? generatorConfigurations.slice() : [[]];
         generatorConfigurations.length = 0;
         for (const gen of state) {
           for (const value of values)

test-runner/src/testRunner.ts

@@ -14,15 +14,15 @@
  * limitations under the License.
  */
 
-import { FixturePool, rerunRegistrations, setParameters } from './fixtures';
+import { FixturePool, rerunRegistrations, setParameters, TestInfo } from './fixtures';
 import { EventEmitter } from 'events';
 import { setCurrentTestFile } from './expect';
-import { Test, Suite, Configuration, serializeError } from './test';
+import { Test, Suite, Configuration, serializeError, TestResult } from './test';
 import { spec } from './spec';
 import { RunnerConfig } from './runnerConfig';
 import * as util from 'util';
 
-export const fixturePool = new FixturePool<RunnerConfig>();
+export const fixturePool = new FixturePool();
 
 export type TestRunnerEntry = {
   file: string;
@@ -40,13 +40,6 @@ function chunkToParams(chunk: Buffer | string):  { text?: string, buffer?: strin
   return { text: chunk };
 }
 
-export type SerializedTest = {
-  id: string,
-  error: any,
-  duration: number,
-  data: any[]
-};
-
 export class TestRunner extends EventEmitter {
   private _failedTestId: string | undefined;
   private _fatalError: any | undefined;
@@ -58,7 +51,10 @@ export class TestRunner extends EventEmitter {
   private _parsedGeneratorConfiguration: any = {};
   private _config: RunnerConfig;
   private _timeout: number;
-  private _test: Test | null = null;
+  private _testId: string | null;
+  private _stdOutBuffer: (string | Buffer)[] = [];
+  private _stdErrBuffer: (string | Buffer)[] = [];
+  private _testResult: TestResult | null = null;
 
   constructor(entry: TestRunnerEntry, config: RunnerConfig, workerId: number) {
     super();
@@ -81,21 +77,32 @@ export class TestRunner extends EventEmitter {
 
   fatalError(error: Error | any) {
     this._fatalError = serializeError(error);
-    if (this._test) {
+    if (this._testResult) {
-      this._test.error = this._fatalError;
+      this._testResult.error = this._fatalError;
-      this.emit('fail', {
+      this.emit('testEnd', {
-        test: this._serializeTest(),
+        id: this._testId,
+        result: this._testResult
       });
     }
     this._reportDone();
   }
 
   stdout(chunk: string | Buffer) {
-    this.emit('stdout', { testId: this._testId(), ...chunkToParams(chunk) })
+    this._stdOutBuffer.push(chunk);
+    if (!this._testId)
+      return;
+    for (const c of this._stdOutBuffer)
+      this.emit('testStdOut', { id: this._testId, ...chunkToParams(c) });
+    this._stdOutBuffer = [];
   }
 
   stderr(chunk: string | Buffer) {
-    this.emit('stderr', { testId: this._testId(), ...chunkToParams(chunk) })
+    this._stdErrBuffer.push(chunk);
+    if (!this._testId)
+      return;
+    for (const c of this._stdErrBuffer)
+      this.emit('testStdErr', { id: this._testId, ...chunkToParams(c) });
+    this._stdErrBuffer = [];
   }
 
   async run() {
@@ -120,11 +127,11 @@ export class TestRunner extends EventEmitter {
       this._reportDone();
     }
     for (const entry of suite._entries) {
-      if (entry instanceof Suite) {
+      if (entry instanceof Suite)
         await this._runSuite(entry);
-      } else {
+      else
         await this._runTest(entry);
-      }
+
     }
     try {
       await this._runHooks(suite, 'afterAll', 'after');
@@ -137,32 +144,52 @@ export class TestRunner extends EventEmitter {
   private async _runTest(test: Test) {
     if (this._failedTestId)
       return false;
-    this._test = test;
     if (this._ordinals.size && !this._ordinals.has(test._ordinal))
       return;
     this._remaining.delete(test._ordinal);
-    if (test.skipped || test.suite._isSkipped()) {
+
-      this.emit('skipped', { test: this._serializeTest() });
+    const id = `${test._ordinal}@${this._configuredFile}`;
+    this._testId = id;
+    this.emit('testBegin', { id });
+
+    const result: TestResult = {
+      duration: 0,
+      status: 'none',
+      stdout: [],
+      stderr: [],
+      data: {}
+    };
+    this._testResult = result;
+
+    if (test._skipped || test.suite._isSkipped()) {
+      result.status = 'skipped';
+      this.emit('testEnd', { id, result });
       return;
     }
 
-    this.emit('test', { test: this._serializeTest() });
+    const startTime = Date.now();
     try {
-      await this._runHooks(test.suite, 'beforeEach', 'before');
+      const testInfo = { config: this._config, test, result };
-      test._startTime = Date.now();
+      await this._runHooks(test.suite, 'beforeEach', 'before', testInfo);
-      if (!this._trialRun)
+      if (!this._trialRun) {
-        await this._testWrapper(test)();
+        const timeout = test.slow ? this._timeout * 3 : this._timeout;
-      await this._runHooks(test.suite, 'afterEach', 'after');
+        await fixturePool.runTestWithFixtures(test.fn, timeout, testInfo);
-      this.emit('pass', { test: this._serializeTest(true) });
+      }
+      await this._runHooks(test.suite, 'afterEach', 'after', testInfo);
+      result.duration = Date.now() - startTime;
+      this.emit('testEnd', { id, result });
     } catch (error) {
-      test.error = serializeError(error);
+      result.error = serializeError(error);
-      this._failedTestId = this._testId();
+      result.status = 'failed';
-      this.emit('fail', { test: this._serializeTest(true) });
+      result.duration = Date.now() - startTime;
+      this._failedTestId = this._testId;
+      this.emit('testEnd', { id, result });
     }
-    this._test = null;
+    this._testResult = null;
+    this._testId = null;
   }
 
-  private async _runHooks(suite: Suite, type: string, dir: 'before' | 'after') {
+  private async _runHooks(suite: Suite, type: string, dir: 'before' | 'after', testInfo?: TestInfo) {
     if (!suite._hasTestsToRun())
       return;
     const all = [];
@@ -173,7 +200,7 @@ export class TestRunner extends EventEmitter {
     if (dir === 'before')
       all.reverse();
     for (const hook of all)
-      await fixturePool.resolveParametersAndRun(hook, this._config);
+      await fixturePool.resolveParametersAndRun(hook, this._config, testInfo);
   }
 
   private _reportDone() {
@@ -183,22 +210,4 @@ export class TestRunner extends EventEmitter {
       remaining: [...this._remaining],
     });
   }
-
-  private _testWrapper(test: Test) {
-    const timeout = test.slow ? this._timeout * 3 : this._timeout;
-    return fixturePool.wrapTestCallback(test.fn, timeout, { ...this._config }, test);
-  }
-
-  private _testId() {
-    return `${this._test._ordinal}@${this._configuredFile}`;
-  }
-
-  private _serializeTest(full = false): SerializedTest {
-    return {
-      id: this._testId(),
-      error: this._test.error,
-      duration: Date.now() - this._test._startTime,
-      data: full ? this._test.data : undefined
-    };
-  }
 }

test-runner/src/transform.ts

@@ -56,7 +56,7 @@ export function installTransform(): () => void {
     sourceMaps.set(filename, sourceMapPath);
     if (fs.existsSync(codePath))
       return fs.readFileSync(codePath, 'utf8');
-    
+
     const result = babel.transformFileSync(filename, {
       presets: [
         ['@babel/preset-env', { targets: {node: 'current'} }],
@@ -67,7 +67,7 @@ export function installTransform(): () => void {
     if (result.code) {
       fs.mkdirSync(path.dirname(cachePath), {recursive: true});
       if (result.map)
-        fs.writeFileSync(sourceMapPath, JSON.stringify(result.map), 'utf8');  
+        fs.writeFileSync(sourceMapPath, JSON.stringify(result.map), 'utf8');
       fs.writeFileSync(codePath, result.code, 'utf8');
     }
     return result.code;

test-runner/src/worker.ts

@@ -69,7 +69,7 @@ process.on('message', async message => {
   }
   if (message.method === 'run') {
     testRunner = new TestRunner(message.params.entry, message.params.config, workerId);
-    for (const event of ['test', 'skipped', 'pass', 'fail', 'done', 'stdout', 'stderr'])
+    for (const event of ['testBegin', 'testStdOut', 'testStdErr', 'testEnd', 'done'])
       testRunner.on(event, sendMessageToParent.bind(null, event));
     await testRunner.run();
     testRunner = null;

test-runner/test/assets/test-data-visible-in-fixture.js

@@ -18,9 +18,9 @@ const { registerFixture } = require('../../');
 const fs = require('fs');
 const path = require('path');
 
-registerFixture('postProcess', async ({}, runTest, config, test) => {
+registerFixture('postProcess', async ({}, runTest, info) => {
   await runTest('');
-  test.data['myname'] = 'myvalue';
+  info.result.data['myname'] = 'myvalue';
 });
 
 it('ensure fixture handles test error', async ({ postProcess }) => {

test-runner/test/assets/test-error-visible-in-fixture.js

@@ -18,9 +18,10 @@ const { registerFixture } = require('../../');
 const fs = require('fs');
 const path = require('path');
 
-registerFixture('postProcess', async ({}, runTest, config, test) => {
+registerFixture('postProcess', async ({}, runTest, info) => {
 	await runTest('');
-	fs.writeFileSync(path.join(config.outputDir, 'test-error-visible-in-fixture.txt'), JSON.stringify(test.error, undefined, 2));
+	const { config, result } = info;
+	fs.writeFileSync(path.join(config.outputDir, 'test-error-visible-in-fixture.txt'), JSON.stringify(result.error, undefined, 2));
 });
 
 it('ensure fixture handles test error', async ({ postProcess }) => {

test-runner/test/assets/typescript.ts

@@ -1,5 +1,20 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 import '../../';
-import { abc } from "./global-foo";
+import './global-foo';
 
 it('should find global foo', () => {
   expect(global['foo']).toBe(true);

test-runner/test/exit-code.spec.ts

@@ -23,66 +23,66 @@ import '../lib';
 
 const removeFolderAsync = promisify(rimraf);
 
-it('should fail', async() => {
+it('should fail', async () => {
   const result = await runTest('one-failure.js');
   expect(result.exitCode).toBe(1);
   expect(result.passed).toBe(0);
   expect(result.failed).toBe(1);
 });
 
-it('should succeed', async() => {
+it('should succeed', async () => {
   const result = await runTest('one-success.js');
   expect(result.exitCode).toBe(0);
   expect(result.passed).toBe(1);
   expect(result.failed).toBe(0);
 });
 
-it('should access error in fixture', async() => {
+it('should access error in fixture', async () => {
   const result = await runTest('test-error-visible-in-fixture.js');
   expect(result.exitCode).toBe(1);
   const data = JSON.parse(fs.readFileSync(path.join(__dirname, 'test-results', 'test-error-visible-in-fixture.txt')).toString());
   expect(data.message).toContain('Object.is equality');
 });
 
-it('should access data in fixture', async() => {
+it('should access data in fixture', async () => {
   const result = await runTest('test-data-visible-in-fixture.js');
   expect(result.exitCode).toBe(1);
   const data = JSON.parse(fs.readFileSync(path.join(__dirname, 'test-results', 'results.json')).toString());
-  const test = data.suites[0].tests[0];
+  const testResult = data.suites[0].tests[0].results[0];
-  expect(test.data).toEqual({ 'myname': 'myvalue' });
+  expect(testResult.data).toEqual({ 'myname': 'myvalue' });
-  expect(test.stdout).toEqual([{ text: 'console.log\n' }]);
+  expect(testResult.stdout).toEqual([{ text: 'console.log\n' }]);
-  expect(test.stderr).toEqual([{ text: 'console.error\n' }]);
+  expect(testResult.stderr).toEqual([{ text: 'console.error\n' }]);
 });
 
-it('should handle worker fixture timeout', async() => {
+it('should handle worker fixture timeout', async () => {
   const result = await runTest('worker-fixture-timeout.js', 1000);
   expect(result.exitCode).toBe(1);
   expect(result.output).toContain('Timeout of 1000ms');
 });
 
-it('should handle worker fixture error', async() => {
+it('should handle worker fixture error', async () => {
   const result = await runTest('worker-fixture-error.js');
   expect(result.exitCode).toBe(1);
   expect(result.output).toContain('Worker failed');
 });
 
-it('should collect stdio', async() => {
+it('should collect stdio', async () => {
   const result = await runTest('stdio.js');
   expect(result.exitCode).toBe(0);
   const data = JSON.parse(fs.readFileSync(path.join(__dirname, 'test-results', 'results.json')).toString());
-  const test = data.suites[0].tests[0];
+  const testResult = data.suites[0].tests[0].results[0];
-  const { stdout, stderr } = test;
+  const { stdout, stderr } = testResult;
   expect(stdout).toEqual([{ text: 'stdout text' }, { buffer: Buffer.from('stdout buffer').toString('base64') }]);
   expect(stderr).toEqual([{ text: 'stderr text' }, { buffer: Buffer.from('stderr buffer').toString('base64') }]);
 });
 
-it('should work with typescript', async() => {
+it('should work with typescript', async () => {
   const result = await runTest('typescript.ts');
   expect(result.exitCode).toBe(0);
 });
 
 async function runTest(filePath: string, timeout = 10000) {
-  const outputDir = path.join(__dirname, 'test-results')
+  const outputDir = path.join(__dirname, 'test-results');
   await removeFolderAsync(outputDir).catch(e => {});
 
   const { output, status } = spawnSync('node', [
@@ -102,7 +102,7 @@ async function runTest(filePath: string, timeout = 10000) {
   return {
     exitCode: status,
     output: output.toString(),
-    passed: parseInt(passed),
+    passed: parseInt(passed, 10),
-    failed: parseInt(failed || '0')
+    failed: parseInt(failed || '0', 10)
-  }
+  };
 }

test/playwright.fixtures.ts

@@ -189,10 +189,11 @@ registerFixture('context', async ({browser}, test) => {
   await context.close();
 });
 
-registerFixture('page', async ({context}, runTest, config, test) => {
+registerFixture('page', async ({context}, runTest, info) => {
   const page = await context.newPage();
   await runTest(page);
-  if (test.error) {
+  const { test, config, result } = info;
+  if (result.status === 'failed' || result.status === 'timedOut') {
     const relativePath = path.relative(config.testDir, test.file).replace(/\.spec\.[jt]s/, '');
     const sanitizedTitle = test.title.replace(/[^\w\d]+/g, '_');
     const assetPath = path.join(config.outputDir, relativePath, sanitizedTitle) + '-failed.png';