diff --git a/.github/workflows/auto_roll.yml b/.github/workflows/auto_roll.yml index 5ba8eadad9..e6781dd055 100644 --- a/.github/workflows/auto_roll.yml +++ b/.github/workflows/auto_roll.yml @@ -27,13 +27,14 @@ jobs: # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR # Wrap `npm run` in a subshell to redirect STDERR to file. # Enable core dumps in the subshell. - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000" + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --reporter=dot,json" env: BROWSER: ${{ matrix.browser }} DEBUG: "pw:*,-pw:wrapped*,-pw:test*" DEBUG_FILE: "testrun.log" FFPATH: ${{ steps.build-browser.outputs.FFPATH }} WKPATH: ${{ steps.build-browser.outputs.WKPATH }} + PWRUNNER_JSON_REPORT: "test-results.json" - uses: actions/upload-artifact@v1 if: failure() with: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8581ebcc08..8d8d5cdb4a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -37,16 +37,22 @@ jobs: # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR # Wrap `npm run` in a subshell to redirect STDERR to file. # Enable core dumps in the subshell. - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 && npm run coverage" + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --reporter=dot,json && npm run coverage" env: BROWSER: ${{ matrix.browser }} DEBUG: "pw:*,-pw:wrapped*,-pw:test*" DEBUG_FILE: "testrun.log" + PWRUNNER_JSON_REPORT: "test-results.json" - uses: actions/upload-artifact@v1 if: failure() with: name: ${{ matrix.browser }}-${{ matrix.os }}-test-results path: test-results + - uses: actions/upload-artifact@v1 + if: ${{ always() }} + with: + name: ${{ matrix.browser }}-${{ matrix.os }}-test-results.json + path: test-results.json - uses: actions/upload-artifact@v1 if: ${{ always() }} with: @@ -68,11 +74,12 @@ jobs: - uses: microsoft/playwright-github-action@v1 - run: npm ci - run: npm run build - - run: node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 + - run: node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --reporter=dot,json env: BROWSER: ${{ matrix.browser }} DEBUG: "pw:*,-pw:wrapped*,-pw:test*" DEBUG_FILE: "testrun.log" + PWRUNNER_JSON_REPORT: "test-results.json" - uses: actions/upload-artifact@v1 if: failure() with: @@ -83,6 +90,11 @@ jobs: with: name: ${{ matrix.browser }}-mac-testrun.log path: testrun.log + - uses: actions/upload-artifact@v1 + if: ${{ always() }} + with: + name: ${{ matrix.browser }}-mac-test-results.json + path: test-results.json test_win: name: "Windows" @@ -102,12 +114,13 @@ jobs: - uses: microsoft/playwright-github-action@v1 - run: npm ci - run: npm run build - - run: node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 + - run: node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --reporter=dot,json shell: bash env: BROWSER: ${{ matrix.browser }} DEBUG: "pw:*,-pw:wrapped*,-pw:test*" DEBUG_FILE: "testrun.log" + PWRUNNER_JSON_REPORT: "test-results.json" - uses: actions/upload-artifact@v1 if: failure() with: @@ -118,6 +131,11 @@ jobs: with: name: ${{ matrix.browser }}-win-testrun.log path: testrun.log + - uses: actions/upload-artifact@v1 + if: ${{ always() }} + with: + name: ${{ matrix.browser }}-win-test-results.json + path: test-results.json test-package-installations: runs-on: ubuntu-latest @@ -159,17 +177,23 @@ jobs: # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR # Wrap `npm run` in a subshell to redirect STDERR to file. # Enable core dumps in the subshell. - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000" + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --reporter=dot,json" if: ${{ always() }} env: BROWSER: ${{ matrix.browser }} HEADLESS: "false" DEBUG_FILE: "testrun.log" + PWRUNNER_JSON_REPORT: "test-results.json" - uses: actions/upload-artifact@v1 if: failure() with: name: headful-${{ matrix.browser }}-linux-test-results path: test-results + - uses: actions/upload-artifact@v1 + if: ${{ always() }} + with: + name: headful-${{ matrix.browser }}-linux-test-results.json + path: test-results.json - uses: actions/upload-artifact@v1 if: ${{ always() }} with: @@ -197,17 +221,23 @@ jobs: # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR # Wrap `npm run` in a subshell to redirect STDERR to file. # Enable core dumps in the subshell. - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000" + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --reporter=dot,json" env: BROWSER: ${{ matrix.browser }} DEBUG: "pw:*,-pw:wrapped*,-pw:test*" DEBUG_FILE: "testrun.log" PWWIRE: true + PWRUNNER_JSON_REPORT: "test-results.json" - uses: actions/upload-artifact@v1 if: failure() with: name: wire-${{ matrix.browser }}-linux-test-results path: test-results + - uses: actions/upload-artifact@v1 + if: ${{ always() }} + with: + name: wire-${{ matrix.browser }}-linux-test-results.json + path: test-results.json - uses: actions/upload-artifact@v1 if: ${{ always() }} with: diff --git a/test-runner/src/cli.ts b/test-runner/src/cli.ts index db2cad4484..aa749d49e5 100644 --- a/test-runner/src/cli.ts +++ b/test-runner/src/cli.ts @@ -18,10 +18,12 @@ import program from 'commander'; import * as fs from 'fs'; import * as path from 'path'; import { collectTests, runTests, RunnerConfig } from '.'; -import { DotReporter } from './reporters/dot'; -import { ListReporter } from './reporters/list'; -import { JSONReporter } from './reporters/json'; -import { PytestReporter } from './reporters/pytest'; +import PytestReporter from './reporters/pytest'; +import DotReporter from './reporters/dot'; +import ListReporter from './reporters/list'; +import JSONReporter from './reporters/json'; +import { Reporter } from './reporter'; +import { Multiplexer } from './reporters/multiplexer'; export const reporters = { 'dot': DotReporter, @@ -35,7 +37,7 @@ program .option('--forbid-only', 'Fail if exclusive test(s) encountered', false) .option('-g, --grep ', 'Only run tests matching this string or regexp', '.*') .option('-j, --jobs ', 'Number of concurrent jobs for --parallel; use 1 to run in serial, default: (number of CPU cores / 2)', Math.ceil(require('os').cpus().length / 2) as any) - .option('--reporter ', 'Specify reporter to use', '') + .option('--reporter ', 'Specify reporter to use, comma-separated, can be "dot", "list", "json"', 'dot') .option('--trial-run', 'Only collect the matching tests and report them as passing') .option('--quiet', 'Suppress stdio', false) .option('--debug', 'Run tests in-process for debugging', false) @@ -76,8 +78,19 @@ program process.exit(1); } - const reporter = new (reporters[command.reporter || 'dot'])(); - await runTests(config, suite, reporter); + const reporterList = command.reporter.split(','); + const reporterObjects: Reporter[] = reporterList.map(c => { + if (reporters[c]) + return new reporters[c](); + try { + const p = path.resolve(process.cwd(), c); + return new (require(p).default); + } catch (e) { + console.error('Invalid reporter ' + c, e); + process.exit(1); + } + }); + await runTests(config, suite, new Multiplexer(reporterObjects)); const hasFailures = suite.eachTest(t => t.error); process.exit(hasFailures ? 1 : 0); }); diff --git a/test-runner/src/reporters/dot.ts b/test-runner/src/reporters/dot.ts index e7acbd7fd7..82657dd7d0 100644 --- a/test-runner/src/reporters/dot.ts +++ b/test-runner/src/reporters/dot.ts @@ -18,7 +18,7 @@ import colors from 'colors/safe'; import { BaseReporter } from './base'; import { Test } from '../test'; -export class DotReporter extends BaseReporter { +class DotReporter extends BaseReporter { onPending(test: Test) { super.onPending(test); process.stdout.write(colors.yellow('∘')) @@ -43,3 +43,5 @@ export class DotReporter extends BaseReporter { this.epilogue(); } } + +export default DotReporter; diff --git a/test-runner/src/reporters/json.ts b/test-runner/src/reporters/json.ts index 403cdff01f..0b998b2f84 100644 --- a/test-runner/src/reporters/json.ts +++ b/test-runner/src/reporters/json.ts @@ -16,15 +16,20 @@ import { BaseReporter } from './base'; import { Suite, Test } from '../test'; +import * as fs from 'fs'; -export class JSONReporter extends BaseReporter { +class JSONReporter extends BaseReporter { onEnd() { super.onEnd(); const result = { config: this.config, suites: this.suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s) }; - console.log(JSON.stringify(result, undefined, 2)); + const report = JSON.stringify(result, undefined, 2); + if (process.env.PWRUNNER_JSON_REPORT) + fs.writeFileSync(process.env.PWRUNNER_JSON_REPORT, report); + else + console.log(report); } private _serializeSuite(suite: Suite): any { @@ -53,3 +58,5 @@ export class JSONReporter extends BaseReporter { }; } } + +export default JSONReporter; diff --git a/test-runner/src/reporters/list.ts b/test-runner/src/reporters/list.ts index ea940e46eb..ecbea96472 100644 --- a/test-runner/src/reporters/list.ts +++ b/test-runner/src/reporters/list.ts @@ -19,7 +19,7 @@ import { BaseReporter } from './base'; import { RunnerConfig } from '../runnerConfig'; import { Suite, Test } from '../test'; -export class ListReporter extends BaseReporter { +class ListReporter extends BaseReporter { _failure = 0; onBegin(config: RunnerConfig, suite: Suite) { @@ -58,3 +58,5 @@ export class ListReporter extends BaseReporter { this.epilogue(); } } + +export default ListReporter; diff --git a/test-runner/src/reporters/multiplexer.ts b/test-runner/src/reporters/multiplexer.ts new file mode 100644 index 0000000000..50f7d67b78 --- /dev/null +++ b/test-runner/src/reporters/multiplexer.ts @@ -0,0 +1,57 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { RunnerConfig } from '../runnerConfig'; +import { Suite, Test } from '../test'; +import { Reporter } from '../reporter'; + +export class Multiplexer implements Reporter { + private _reporters: Reporter[]; + + constructor(reporters: Reporter[]) { + this._reporters = reporters; + } + + onBegin(config: RunnerConfig, suite: Suite) { + for (const reporter of this._reporters) + reporter.onBegin(config, suite); + } + + onTest(test: Test) { + for (const reporter of this._reporters) + reporter.onTest(test); + } + + onPending(test: Test) { + for (const reporter of this._reporters) + reporter.onPending(test); + } + + onPass(test: Test) { + for (const reporter of this._reporters) + reporter.onPass(test); + } + + onFail(test: Test) { + for (const reporter of this._reporters) + reporter.onFail(test); + } + + onEnd() { + for (const reporter of this._reporters) + reporter.onEnd(); + } +} diff --git a/test-runner/src/reporters/pytest.ts b/test-runner/src/reporters/pytest.ts index 56bfa61af6..56f5a5c03a 100644 --- a/test-runner/src/reporters/pytest.ts +++ b/test-runner/src/reporters/pytest.ts @@ -38,7 +38,7 @@ type Row = { const statusRows = 2; -export class PytestReporter extends BaseReporter { +class PytestReporter extends BaseReporter { private _rows = new Map(); private _suiteIds = new Map(); private _lastOrdinal = 0; @@ -197,7 +197,6 @@ export class PytestReporter extends BaseReporter { } bars.push(bar); } - const barLength = length * worked / total | 0; return '[' + bars.join('') + '] ' + worked + '/' + total; } } @@ -242,3 +241,5 @@ class Throttler { this._callback(); } } + +export default PytestReporter;