feat(testrunner): allow external reporters (#3603)

.github/workflows/auto_roll.yml

@@ -27,13 +27,14 @@ jobs:
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
     # Enable core dumps in the subshell.
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000"
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --reporter=dot,json"
       env:
         BROWSER: ${{ matrix.browser }}
         DEBUG: "pw:*,-pw:wrapped*,-pw:test*"
         DEBUG_FILE: "testrun.log"
         FFPATH: ${{ steps.build-browser.outputs.FFPATH }}
         WKPATH: ${{ steps.build-browser.outputs.WKPATH }}
+        PWRUNNER_JSON_REPORT: "test-results.json"
     - uses: actions/upload-artifact@v1
       if: failure()
       with:

.github/workflows/tests.yml

@@ -37,16 +37,22 @@ jobs:
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
     # Enable core dumps in the subshell.
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 && npm run coverage"
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --reporter=dot,json && npm run coverage"
       env:
         BROWSER: ${{ matrix.browser }}
         DEBUG: "pw:*,-pw:wrapped*,-pw:test*"
         DEBUG_FILE: "testrun.log"
+        PWRUNNER_JSON_REPORT: "test-results.json"
     - uses: actions/upload-artifact@v1
       if: failure()
       with:
         name: ${{ matrix.browser }}-${{ matrix.os }}-test-results
         path: test-results
+    - uses: actions/upload-artifact@v1
+      if: ${{ always() }}
+      with:
+        name: ${{ matrix.browser }}-${{ matrix.os }}-test-results.json
+        path: test-results.json
     - uses: actions/upload-artifact@v1
       if: ${{ always() }}
       with:
@@ -68,11 +74,12 @@ jobs:
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
     - run: npm run build
-    - run: node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000
+    - run: node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --reporter=dot,json
       env:
         BROWSER: ${{ matrix.browser }}
         DEBUG: "pw:*,-pw:wrapped*,-pw:test*"
         DEBUG_FILE: "testrun.log"
+        PWRUNNER_JSON_REPORT: "test-results.json"
     - uses: actions/upload-artifact@v1
       if: failure()
       with:
@@ -83,6 +90,11 @@ jobs:
       with:
         name: ${{ matrix.browser }}-mac-testrun.log
         path: testrun.log
+    - uses: actions/upload-artifact@v1
+      if: ${{ always() }}
+      with:
+        name: ${{ matrix.browser }}-mac-test-results.json
+        path: test-results.json
 
   test_win:
     name: "Windows"
@@ -102,12 +114,13 @@ jobs:
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
     - run: npm run build
-    - run: node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000
+    - run: node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --reporter=dot,json
       shell: bash
       env:
         BROWSER: ${{ matrix.browser }}
         DEBUG: "pw:*,-pw:wrapped*,-pw:test*"
         DEBUG_FILE: "testrun.log"
+        PWRUNNER_JSON_REPORT: "test-results.json"
     - uses: actions/upload-artifact@v1
       if: failure()
       with:
@@ -118,6 +131,11 @@ jobs:
       with:
         name: ${{ matrix.browser }}-win-testrun.log
         path: testrun.log
+    - uses: actions/upload-artifact@v1
+      if: ${{ always() }}
+      with:
+        name: ${{ matrix.browser }}-win-test-results.json
+        path: test-results.json
 
   test-package-installations:
     runs-on: ubuntu-latest
@@ -159,17 +177,23 @@ jobs:
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
     # Enable core dumps in the subshell.
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000"
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --reporter=dot,json"
       if: ${{ always() }}
       env:
         BROWSER: ${{ matrix.browser }}
         HEADLESS: "false"
         DEBUG_FILE: "testrun.log"
+        PWRUNNER_JSON_REPORT: "test-results.json"
     - uses: actions/upload-artifact@v1
       if: failure()
       with:
         name: headful-${{ matrix.browser }}-linux-test-results
         path: test-results
+    - uses: actions/upload-artifact@v1
+      if: ${{ always() }}
+      with:
+        name: headful-${{ matrix.browser }}-linux-test-results.json
+        path: test-results.json
     - uses: actions/upload-artifact@v1
       if: ${{ always() }}
       with:
@@ -197,17 +221,23 @@ jobs:
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
     # Enable core dumps in the subshell.
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000"
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --reporter=dot,json"
       env:
         BROWSER: ${{ matrix.browser }}
         DEBUG: "pw:*,-pw:wrapped*,-pw:test*"
         DEBUG_FILE: "testrun.log"
         PWWIRE: true
+        PWRUNNER_JSON_REPORT: "test-results.json"
     - uses: actions/upload-artifact@v1
       if: failure()
       with:
         name: wire-${{ matrix.browser }}-linux-test-results
         path: test-results
+    - uses: actions/upload-artifact@v1
+      if: ${{ always() }}
+      with:
+        name: wire-${{ matrix.browser }}-linux-test-results.json
+        path: test-results.json
     - uses: actions/upload-artifact@v1
       if: ${{ always() }}
       with:

test-runner/src/cli.ts

@@ -18,10 +18,12 @@ import program from 'commander';
 import * as fs from 'fs';
 import * as path from 'path';
 import { collectTests, runTests, RunnerConfig } from '.';
-import { DotReporter } from './reporters/dot';
+import PytestReporter from './reporters/pytest';
-import { ListReporter } from './reporters/list';
+import DotReporter from './reporters/dot';
-import { JSONReporter } from './reporters/json';
+import ListReporter from './reporters/list';
-import { PytestReporter } from './reporters/pytest';
+import JSONReporter from './reporters/json';
+import { Reporter } from './reporter';
+import { Multiplexer } from './reporters/multiplexer';
 
 export const reporters = {
   'dot': DotReporter,
@@ -35,7 +37,7 @@ program
   .option('--forbid-only', 'Fail if exclusive test(s) encountered', false)
   .option('-g, --grep <grep>', 'Only run tests matching this string or regexp', '.*')
   .option('-j, --jobs <jobs>', 'Number of concurrent jobs for --parallel; use 1 to run in serial, default: (number of CPU cores / 2)', Math.ceil(require('os').cpus().length / 2) as any)
-  .option('--reporter <reporter>', 'Specify reporter to use', '')
+  .option('--reporter <reporter>', 'Specify reporter to use, comma-separated, can be "dot", "list", "json"', 'dot')
   .option('--trial-run', 'Only collect the matching tests and report them as passing')
   .option('--quiet', 'Suppress stdio', false)
   .option('--debug', 'Run tests in-process for debugging', false)
@@ -76,8 +78,19 @@ program
       process.exit(1);
     }
 
-    const reporter = new (reporters[command.reporter || 'dot'])();
+    const reporterList = command.reporter.split(',');
-    await runTests(config, suite, reporter);
+    const reporterObjects: Reporter[] = reporterList.map(c => {
+      if (reporters[c])
+        return new reporters[c]();
+      try {
+        const p = path.resolve(process.cwd(), c);
+        return new (require(p).default);
+      } catch (e) {
+        console.error('Invalid reporter ' + c, e);
+        process.exit(1);
+      }
+    });
+    await runTests(config, suite, new Multiplexer(reporterObjects));
     const hasFailures = suite.eachTest(t => t.error);
     process.exit(hasFailures ? 1 : 0);
   });

test-runner/src/reporters/dot.ts

@@ -18,7 +18,7 @@ import colors from 'colors/safe';
 import { BaseReporter } from './base';
 import { Test } from '../test';
 
-export class DotReporter extends BaseReporter {  
+class DotReporter extends BaseReporter {
   onPending(test: Test) {
     super.onPending(test);
     process.stdout.write(colors.yellow('∘'))
@@ -43,3 +43,5 @@ export class DotReporter extends BaseReporter {
     this.epilogue();
   }
 }
+
+export default DotReporter;

test-runner/src/reporters/json.ts

@@ -16,15 +16,20 @@
 
 import { BaseReporter } from './base';
 import { Suite, Test } from '../test';
+import * as fs from 'fs';
 
-export class JSONReporter extends BaseReporter {
+class JSONReporter extends BaseReporter {
   onEnd() {
     super.onEnd();
     const result = {
       config: this.config,
       suites: this.suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s)
     };
-    console.log(JSON.stringify(result, undefined, 2));
+    const report = JSON.stringify(result, undefined, 2);
+    if (process.env.PWRUNNER_JSON_REPORT)
+      fs.writeFileSync(process.env.PWRUNNER_JSON_REPORT, report);
+    else
+      console.log(report);
   }
 
   private _serializeSuite(suite: Suite): any {
@@ -53,3 +58,5 @@ export class JSONReporter extends BaseReporter {
     };
   }
 }
+
+export default JSONReporter;

test-runner/src/reporters/list.ts

@@ -19,7 +19,7 @@ import { BaseReporter } from './base';
 import { RunnerConfig } from '../runnerConfig';
 import { Suite, Test } from '../test';
 
-export class ListReporter extends BaseReporter {
+class ListReporter extends BaseReporter {
   _failure = 0;
 
   onBegin(config: RunnerConfig, suite: Suite) {
@@ -58,3 +58,5 @@ export class ListReporter extends BaseReporter {
     this.epilogue();
   }
 }
+
+export default ListReporter;

test-runner/src/reporters/multiplexer.ts

@@ -0,0 +1,57 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { RunnerConfig } from '../runnerConfig';
+import { Suite, Test } from '../test';
+import { Reporter } from '../reporter';
+
+export class Multiplexer implements Reporter {
+  private _reporters: Reporter[];
+
+  constructor(reporters: Reporter[]) {
+    this._reporters = reporters;
+  }
+
+  onBegin(config: RunnerConfig, suite: Suite) {
+    for (const reporter of this._reporters)
+      reporter.onBegin(config, suite);
+  }
+
+  onTest(test: Test) {
+    for (const reporter of this._reporters)
+      reporter.onTest(test);
+  }
+
+  onPending(test: Test) {
+    for (const reporter of this._reporters)
+      reporter.onPending(test);
+  }
+
+  onPass(test: Test) {
+    for (const reporter of this._reporters)
+      reporter.onPass(test);
+  }
+
+  onFail(test: Test) {
+    for (const reporter of this._reporters)
+      reporter.onFail(test);
+  }
+
+  onEnd() {
+    for (const reporter of this._reporters)
+      reporter.onEnd();
+  }
+}

test-runner/src/reporters/pytest.ts

@@ -38,7 +38,7 @@ type Row = {
 
 const statusRows = 2;
 
-export class PytestReporter extends BaseReporter {
+class PytestReporter extends BaseReporter {
   private _rows = new Map<string, Row>();
   private _suiteIds = new Map<Suite, string>();
   private _lastOrdinal = 0;
@@ -197,7 +197,6 @@ export class PytestReporter extends BaseReporter {
       }
       bars.push(bar);
     }
-    const barLength = length * worked / total | 0;
     return '[' + bars.join('') + '] ' + worked + '/' + total;
   }
 }
@@ -242,3 +241,5 @@ class Throttler {
     this._callback();
   }
 }
+
+export default PytestReporter;