feat(testrunner): introduce pytest-style reporter (#3594)

test-runner/src/cli.ts

@@ -21,18 +21,20 @@ import { collectTests, runTests, RunnerConfig } from '.';
 import { DotReporter } from './reporters/dot';
 import { ListReporter } from './reporters/list';
 import { JSONReporter } from './reporters/json';
+import { PytestReporter } from './reporters/pytest';
 
 export const reporters = {
   'dot': DotReporter,
   'list': ListReporter,
-  'json': JSONReporter
+  'json': JSONReporter,
+  'pytest': PytestReporter,
 };
 
 program
   .version('Version ' + /** @type {any} */ (require)('../package.json').version)
   .option('--forbid-only', 'Fail if exclusive test(s) encountered', false)
   .option('-g, --grep <grep>', 'Only run tests matching this string or regexp', '.*')
-  .option('-j, --jobs <jobs>', 'Number of concurrent jobs for --parallel; use 1 to run in serial, default: (number of CPU cores / 2)', Math.ceil(require('os').cpus().length / 2).toString())
+  .option('-j, --jobs <jobs>', 'Number of concurrent jobs for --parallel; use 1 to run in serial, default: (number of CPU cores / 2)', Math.ceil(require('os').cpus().length / 2) as any)
   .option('--reporter <reporter>', 'Specify reporter to use', '')
   .option('--trial-run', 'Only collect the matching tests and report them as passing')
   .option('--quiet', 'Suppress stdio', false)

test-runner/src/reporters/base.ts

@@ -32,6 +32,7 @@ export class BaseReporter implements Reporter  {
   pending: Test[] = [];
   passes: Test[] = [];
   failures: Test[] = [];
+  timeouts: Test[] = [];
   duration = 0;
   startTime: number;
   config: RunnerConfig;
@@ -62,7 +63,10 @@ export class BaseReporter implements Reporter  {
   }
 
   onFail(test: Test) {
-    this.failures.push(test);
+    if (test.duration >= test.timeout)
+      this.timeouts.push(test);
+    else
+      this.failures.push(test);
   }
 
   onEnd() {
@@ -72,43 +76,53 @@ export class BaseReporter implements Reporter  {
   epilogue() {
     console.log('');
 
-    console.log(colors.green(`  ${this.passes.length} passing`) + colors.dim(` (${milliseconds(this.duration)})`));  
+    console.log(colors.green(`  ${this.passes.length} passed`) + colors.dim(` (${milliseconds(this.duration)})`));  
 
     if (this.pending.length)
       console.log(colors.yellow(`  ${this.pending.length} skipped`));
 
-    if (this.failures.length) {  
-      console.log(colors.red(`  ${this.failures.length} failing`));
+    if (this.failures.length) {
+      console.log(colors.red(`  ${this.failures.length} failed`));
       console.log('');
-      this.failures.forEach((failure, index) => {
-        const relativePath = path.relative(process.cwd(), failure.file);
-        const header = `  ${index +1}. ${terminalLink(relativePath, `file://${os.hostname()}${failure.file}`)} › ${failure.title}`;
-        console.log(colors.bold(colors.red(header)));
-        const stack = failure.error.stack;
-        if (stack) {
+      this._printFailures(this.failures);
+    }
+
+    if (this.timeouts.length) {
+      console.log(colors.red(`  ${this.timeouts.length} timed out`));
+      console.log('');
+      this._printFailures(this.timeouts);
+    }
+  }
+
+  private _printFailures(failures: Test[]) {
+    failures.forEach((failure, index) => {
+      const relativePath = path.relative(process.cwd(), failure.file);
+      const header = `  ${index +1}. ${terminalLink(relativePath, `file://${os.hostname()}${failure.file}`)} › ${failure.title}`;
+      console.log(colors.bold(colors.red(header)));
+      const stack = failure.error.stack;
+      if (stack) {
+        console.log('');
+        const messageLocation = failure.error.stack.indexOf(failure.error.message);
+        const preamble = failure.error.stack.substring(0, messageLocation + failure.error.message.length);
+        console.log(indent(preamble, '    '));
+        const position = positionInFile(stack, failure.file);
+        if (position) {
+          const source = fs.readFileSync(failure.file, 'utf8');
           console.log('');
-          const messageLocation = failure.error.stack.indexOf(failure.error.message);
-          const preamble = failure.error.stack.substring(0, messageLocation + failure.error.message.length);
-          console.log(indent(preamble, '    '));
-          const position = positionInFile(stack, failure.file);
-          if (position) {
-            const source = fs.readFileSync(failure.file, 'utf8');
-            console.log('');
-            console.log(indent(codeFrameColumns(source, {
-                start: position,
-              },
-              { highlightCode: true}
-            ), '    '));
-          }
-          console.log('');
-          console.log(indent(colors.dim(stack.substring(preamble.length + 1)), '    '));
-        } else {
-          console.log('');
-          console.log(indent(String(failure.error), '    '));
+          console.log(indent(codeFrameColumns(source, {
+              start: position,
+            },
+            { highlightCode: true}
+          ), '    '));
         }
         console.log('');
-      });
-    }
+        console.log(indent(colors.dim(stack.substring(preamble.length + 1)), '    '));
+      } else {
+        console.log('');
+        console.log(indent(String(failure.error), '    '));
+      }
+      console.log('');
+    });
   }
 }
 

test-runner/src/reporters/dot.ts

@@ -26,7 +26,7 @@ export class DotReporter extends BaseReporter {
   
   onPass(test: Test) {
     super.onPass(test);
-    process.stdout.write(colors.green('\u00B7'));
+    process.stdout.write(colors.green('·'));
   }
   
   onFail(test: Test) {

test-runner/src/reporters/list.ts

@@ -36,21 +36,21 @@ export class ListReporter extends BaseReporter {
     super.onPending(test);
     process.stdout.write(colors.green('  - ') + colors.cyan(test.fullTitle()));
     process.stdout.write('\n');
-}
+  }
 
   onPass(test: Test) {
     super.onPass(test);
     process.stdout.write('\u001b[2K\u001b[0G');
     process.stdout.write(colors.green('  ✓ ') + colors.gray(test.fullTitle()));
     process.stdout.write('\n');
-}
+  }
 
   onFail(test: Test) {
     super.onFail(test);
     process.stdout.write('\u001b[2K\u001b[0G');
     process.stdout.write(colors.red(`  ${++this._failure}) ` + test.fullTitle()));
     process.stdout.write('\n');
-}
+  }
 
   onEnd() {
     super.onEnd();

test-runner/src/reporters/pytest.ts

@@ -0,0 +1,244 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import colors from 'colors/safe';
+import milliseconds from 'ms';
+import * as path from 'path';
+import { Test, Suite, Configuration } from '../test';
+import { BaseReporter } from './base';
+import { RunnerConfig } from '../runnerConfig';
+
+const cursorPrevLine = '\u001B[F';
+const eraseLine = '\u001B[2K'
+
+type Row = {
+  id: string;
+  relativeFile: string;
+  configuration: string;
+  ordinal: number;
+  track: string[];
+  total: number;
+  failed: boolean;
+  startTime: number;
+  finishTime: number;
+};
+
+const statusRows = 2;
+
+export class PytestReporter extends BaseReporter {
+  private _rows = new Map<string, Row>();
+  private _suiteIds = new Map<Suite, string>();
+  private _lastOrdinal = 0;
+  private _visibleRows: number;
+  private _failed = false;
+  private _total: number;
+  private _progress: string[] = [];
+  private _throttler = new Throttler(250, () => this._repaint());
+
+  onBegin(config: RunnerConfig, rootSuite: Suite) {
+    super.onBegin(config, rootSuite);
+    this._total = rootSuite.total();
+
+    const jobs = Math.min(config.jobs, rootSuite.suites.length);
+    this._visibleRows = jobs + Math.min(jobs, 3);  // 3 buffer rows for completed (green) workers.
+    for (let i = 0; i < this._visibleRows + statusRows; ++i)  // 4 rows for status
+      process.stdout.write('\n');
+
+    for (const s of rootSuite.suites) {
+      const relativeFile = path.relative(this.config.testDir, s.file);
+      const configurationString = serializeConfiguration(s.configuration);
+      const id = relativeFile + `::[${configurationString}]`;
+      this._suiteIds.set(s, id);
+      const row = {
+        id,
+        relativeFile,
+        configuration: configurationString,
+        ordinal: this._lastOrdinal++,
+        track: [],
+        total: s.total(),
+        failed: false,
+        startTime: 0,
+        finishTime: 0,
+      };
+      this._rows.set(id, row);
+    }
+  }
+
+  onTest(test: Test) {
+    super.onTest(test);
+    const row = this._rows.get(this._id(test));
+    if (!row.startTime)
+      row.startTime = Date.now();
+  }
+
+  onPending(test: Test) {
+    super.onPending(test);
+    this._append(test, colors.yellow('∘'));
+    this._progress.push('S');
+  }
+
+  onPass(test: Test) {
+    super.onPass(test);
+    this._append(test, colors.green('✓'));
+    this._progress.push('P');
+  }
+
+  onFail(test: Test) {
+    super.onFail(test);
+    const title = test.duration >= test.timeout ? colors.red('T') : colors.red('F');
+    const row = this._append(test, title);
+    row.failed = true;
+    this._failed = true;
+    this._progress.push('F');
+  }
+
+  onEnd() {
+    super.onEnd();
+    this._repaint();
+    if (this._failed)
+      this.epilogue();
+  }
+
+  private _append(test: Test, s: string): Row {
+    const testId = this._id(test);
+    const row = this._rows.get(testId);
+    row.track.push(s);
+    if (row.track.length === row.total)
+      row.finishTime = Date.now();
+    this._throttler.schedule();
+    return row;
+  }
+
+  private _repaint() {
+    const rowList = [...this._rows.values()];
+    const running = rowList.filter(r => r.startTime && !r.finishTime);
+    const finished = rowList.filter(r => r.finishTime).sort((a, b) => b.finishTime - a.finishTime);
+    const finishedToPrint = finished.slice(0, this._visibleRows - running.length);
+    const lines = [];
+    for (const row of finishedToPrint.concat(running)) {
+      const remaining = row.total - row.track.length;
+      const remainder = '·'.repeat(remaining);
+      let title = row.relativeFile;
+      if (row.finishTime) {
+        if (row.failed)
+          title = colors.red(row.relativeFile);
+        else
+          title = colors.green(row.relativeFile);
+      }
+      const configuration = ` [${colors.gray(row.configuration)}]`;
+      lines.push(' ' + title + configuration + ' ' + row.track.join('') + colors.gray(remainder));
+    }
+
+    const status = [];
+    if (this.passes.length)
+      status.push(colors.green(`${this.passes.length} passed`));
+    if (this.pending.length)
+      status.push(colors.yellow(`${this.pending.length} skipped`));
+    if (this.failures.length)
+      status.push(colors.red(`${this.failures.length} failed`));
+    if (this.timeouts.length)
+      status.push(colors.red(`${this.timeouts.length} timed out`));
+    status.push(colors.dim(`(${milliseconds(Date.now() - this.startTime)})`));
+
+    for (let i = lines.length; i < this._visibleRows; ++i)
+      lines.push('');
+    lines.push(this._paintProgress(this._progress.length, this._total));
+    lines.push(status.join('  '));
+    lines.push('');
+
+    process.stdout.write((cursorPrevLine + eraseLine).repeat(this._visibleRows + statusRows));
+    process.stdout.write(lines.join('\n'));
+  }
+
+  private _id(test: Test): string {
+    for (let suite = test.suite; suite; suite = suite.parent) {
+      if (this._suiteIds.has(suite))
+        return this._suiteIds.get(suite);
+    }
+    return '';
+  }
+
+  private _paintProgress(worked: number, total: number) {
+    const length = Math.min(total, 80);
+    const cellSize = Math.ceil(total / length);
+    const cellNum = (total / cellSize) | 0;
+    const bars: string[] = [];
+    for (let i = 0; i < cellNum; ++i) {
+      let bar = blankBar;
+      if (worked < cellSize * i) {
+        bars.push(bar);
+        continue;
+      }
+      bar = greenBar;
+      for (let j = i * cellSize; j < worked && j < (i + 1) * cellSize; ++j) {
+        if (worked < j)
+          continue;
+        if (this._progress[j] === 'F') {
+          bar = redBar;
+          break;
+        }
+        if (this._progress[j] === 'S') {
+          bar = yellowBar;
+          break;
+        }
+      }
+      bars.push(bar);
+    }
+    const barLength = length * worked / total | 0;
+    return '[' + bars.join('') + '] ' + worked + '/' + total;
+  }
+}
+
+const blankBar = '-';
+const redBar = colors.red('▇');
+const greenBar = colors.green('▇');
+const yellowBar = colors.yellow('▇');
+
+function serializeConfiguration(configuration: Configuration): string {
+  const tokens = [];
+  for (const { name, value } of configuration)
+      tokens.push(`${name}=${value}`);
+  return tokens.join(', ');
+}
+
+class Throttler {
+  private _timeout: number;
+  private _callback: () => void;
+  private _lastFire = 0;
+  private _timer: NodeJS.Timeout | null = null;
+
+  constructor(timeout: number, callback: () => void) {
+    this._timeout = timeout;
+    this._callback = callback;
+  }
+
+  schedule() {
+    const time = Date.now();
+    const timeRemaining = this._lastFire + this._timeout - time;
+    if (timeRemaining <= 0) {
+      this._fire();
+      return;
+    }
+    if (!this._timer)
+      this._timer = setTimeout(() => this._fire(), timeRemaining);
+  }
+
+  private _fire() {
+    this._timer = null;
+    this._lastFire = Date.now();
+    this._callback();
+  }
+}

test-runner/src/runner.ts

@@ -208,6 +208,9 @@ class Worker extends EventEmitter {
     this.runner = runner;
   }
 
+  run(entry: TestRunnerEntry) {
+  }
+
   stop() {
   }
 }
@@ -290,7 +293,7 @@ class InProcessWorker extends Worker {
     initializeImageMatcher(this.runner._config);
   }
 
-  async run(entry) {
+  async run(entry: TestRunnerEntry) {
     delete require.cache[entry.file];
     const { TestRunner } = require('./testRunner');
     const testRunner = new TestRunner(entry, this.runner._config, 0);

test-runner/src/test.ts

@@ -157,3 +157,10 @@ export class Suite {
     return found;
   }
 }
+
+export function serializeConfiguration(configuration: Configuration): string {
+  const tokens = [];
+  for (const { name, value } of configuration)
+      tokens.push(`${name}=${value}`);
+  return tokens.join(', ');
+}

test-runner/src/testCollector.ts

@@ -16,7 +16,7 @@
 
 import path from 'path';
 import { fixturesForCallback } from './fixtures';
-import { Test, Suite } from './test';
+import { Test, Suite, serializeConfiguration } from './test';
 import { spec } from './spec';
 import { RunnerConfig } from './runnerConfig';
 
@@ -85,10 +85,7 @@ export class TestCollector {
 
       for (const configuration of generatorConfigurations) {
         // Serialize configuration as readable string, we will use it as a hash.
-        const tokens = [];
-        for (const { name, value } of configuration)
-          tokens.push(`${name}=${value}`);
-        const configurationString = tokens.join(', ');
+        const configurationString = serializeConfiguration(configuration);
         // Allocate worker for this configuration, add test into it.
         if (!workerGeneratorConfigurations.has(configurationString))
           workerGeneratorConfigurations.set(configurationString, { configuration, configurationString, tests: new Set() });

test-runner/test/exit-code.spec.ts

@@ -20,25 +20,25 @@ import path from 'path';
 it('should fail', async() => {
   const result = runTest('one-failure.js');
   expect(result.exitCode).toBe(1);
-  expect(result.passing).toBe(0);
-  expect(result.failing).toBe(1);
+  expect(result.passed).toBe(0);
+  expect(result.failed).toBe(1);
 });
 
 it('should succeed', async() => {
   const result = runTest('one-success.js');
   expect(result.exitCode).toBe(0);
-  expect(result.passing).toBe(1);
-  expect(result.failing).toBe(0);
+  expect(result.passed).toBe(1);
+  expect(result.failed).toBe(0);
 });
 
 function runTest(filePath: string) {
   const {output, status} = spawnSync('node', [path.join(__dirname, '..', 'cli.js'), path.join(__dirname, 'assets', filePath)]);
-  const passing = (/  (\d+) passing/.exec(output.toString()) || [])[1];
-  const failing = (/  (\d+) failing/.exec(output.toString()) || [])[1];
+  const passed = (/  (\d+) passed/.exec(output.toString()) || [])[1];
+  const failed = (/  (\d+) failed/.exec(output.toString()) || [])[1];
   return {
     exitCode: status,
     output,
-    passing: parseInt(passing),
-    failing: parseInt(failing || '0')
+    passed: parseInt(passed),
+    failed: parseInt(failed || '0')
   }
 }