diff --git a/docs/src/test-reporter-api/class-reporter.md b/docs/src/test-reporter-api/class-reporter.md index 4fa27a5d20..4bc25e4e57 100644 --- a/docs/src/test-reporter-api/class-reporter.md +++ b/docs/src/test-reporter-api/class-reporter.md @@ -237,3 +237,9 @@ Test that has been finished. - `result` <[TestResult]> Result of the test run. + + +## method: Reporter.printsToStdio +- returns: <[boolean]> + +Whether this reporter uses stdio for reporting. When it does not, Playwright Test could add some output to enhance user experience. diff --git a/docs/src/test-reporters-js.md b/docs/src/test-reporters-js.md index fb3b3104ab..7314a832b8 100644 --- a/docs/src/test-reporters-js.md +++ b/docs/src/test-reporters-js.md @@ -305,22 +305,22 @@ npx playwright show-report my-report ### JSON reporter -JSON reporter produces an object with all information about the test run. It is usually used together with some terminal reporter like `dot` or `line`. +JSON reporter produces an object with all information about the test run. Most likely you want to write the JSON to a file. When running with `--reporter=json`, use `PLAYWRIGHT_JSON_OUTPUT_NAME` environment variable: ```bash bash-flavor=bash -PLAYWRIGHT_JSON_OUTPUT_NAME=results.json npx playwright test --reporter=json,dot +PLAYWRIGHT_JSON_OUTPUT_NAME=results.json npx playwright test --reporter=json ``` ```bash bash-flavor=batch set PLAYWRIGHT_JSON_OUTPUT_NAME=results.json -npx playwright test --reporter=json,dot +npx playwright test --reporter=json ``` ```bash bash-flavor=powershell $env:PLAYWRIGHT_JSON_OUTPUT_NAME="results.json" -npx playwright test --reporter=json,dot +npx playwright test --reporter=json ``` In configuration file, pass options directly: @@ -348,22 +348,22 @@ export default config; ### JUnit reporter -JUnit reporter produces a JUnit-style xml report. It is usually used together with some terminal reporter like `dot` or `line`. +JUnit reporter produces a JUnit-style xml report. Most likely you want to write the report to an xml file. When running with `--reporter=junit`, use `PLAYWRIGHT_JUNIT_OUTPUT_NAME` environment variable: ```bash bash-flavor=bash -PLAYWRIGHT_JUNIT_OUTPUT_NAME=results.xml npx playwright test --reporter=junit,line +PLAYWRIGHT_JUNIT_OUTPUT_NAME=results.xml npx playwright test --reporter=junit ``` ```bash bash-flavor=batch set PLAYWRIGHT_JUNIT_OUTPUT_NAME=results.xml -npx playwright test --reporter=junit,line +npx playwright test --reporter=junit ``` ```bash bash-flavor=powershell $env:PLAYWRIGHT_JUNIT_OUTPUT_NAME="results.xml" -npx playwright test --reporter=junit,line +npx playwright test --reporter=junit ``` In configuration file, pass options directly: @@ -391,7 +391,7 @@ export default config; ### GitHub Actions annotations -You can use the built in `github` reporter to get automatic failure annotations when running in GitHub actions. Use it with some other reporter, for example `'dot'` and/or `'json'`. +You can use the built in `github` reporter to get automatic failure annotations when running in GitHub actions. Note that all other reporters work on GitHub Actions as well, but do not provide annotations. @@ -403,7 +403,7 @@ Note that all other reporters work on GitHub Actions as well, but do not provide const config = { // 'github' for GitHub Actions CI to generate annotations, plus a concise 'dot' // default 'list' when running locally - reporter: process.env.CI ? [ ['github'], ['dot'] ] : 'list', + reporter: process.env.CI ? 'github' : 'list', }; module.exports = config; @@ -416,7 +416,7 @@ import { PlaywrightTestConfig } from '@playwright/test'; const config: PlaywrightTestConfig = { // 'github' for GitHub Actions CI to generate annotations, plus a concise 'dot' // default 'list' when running locally - reporter: process.env.CI ? [ ['github'], ['dot'] ] : 'list', + reporter: process.env.CI ? 'github' : 'list', }; export default config; ``` diff --git a/packages/playwright-test/src/reporters/dot.ts b/packages/playwright-test/src/reporters/dot.ts index 0175b82607..66e007858f 100644 --- a/packages/playwright-test/src/reporters/dot.ts +++ b/packages/playwright-test/src/reporters/dot.ts @@ -21,6 +21,10 @@ import { FullResult, TestCase, TestResult } from '../../types/testReporter'; class DotReporter extends BaseReporter { private _counter = 0; + printsToStdio() { + return true; + } + override onStdOut(chunk: string | Buffer, test?: TestCase, result?: TestResult) { super.onStdOut(chunk, test, result); if (!this.config.quiet) diff --git a/packages/playwright-test/src/reporters/github.ts b/packages/playwright-test/src/reporters/github.ts index ab0d05eb6a..11453bee64 100644 --- a/packages/playwright-test/src/reporters/github.ts +++ b/packages/playwright-test/src/reporters/github.ts @@ -59,6 +59,10 @@ class GitHubLogger { export class GitHubReporter extends BaseReporter { githubLogger = new GitHubLogger(); + printsToStdio() { + return false; + } + override async onEnd(result: FullResult) { super.onEnd(result); this._printAnnotations(); diff --git a/packages/playwright-test/src/reporters/html.ts b/packages/playwright-test/src/reporters/html.ts index 95310fdf88..2768b4ccfa 100644 --- a/packages/playwright-test/src/reporters/html.ts +++ b/packages/playwright-test/src/reporters/html.ts @@ -19,7 +19,7 @@ import fs from 'fs'; import open from 'open'; import path from 'path'; import { Transform, TransformCallback } from 'stream'; -import { FullConfig, Suite } from '../../types/testReporter'; +import { FullConfig, Suite, Reporter } from '../../types/testReporter'; import { HttpServer } from 'playwright-core/lib/utils/httpServer'; import { calculateSha1, removeFolders } from 'playwright-core/lib/utils/utils'; import RawReporter, { JsonReport, JsonSuite, JsonTestCase, JsonTestResult, JsonTestStep, JsonAttachment } from './raw'; @@ -104,7 +104,7 @@ type TestEntry = { testCaseSummary: TestCaseSummary }; -class HtmlReporter { +class HtmlReporter implements Reporter { private config!: FullConfig; private suite!: Suite; private _outputFolder: string | undefined; @@ -116,6 +116,10 @@ class HtmlReporter { this._open = options.open || 'on-failure'; } + printsToStdio() { + return false; + } + onBegin(config: FullConfig, suite: Suite) { this.config = config; this.suite = suite; diff --git a/packages/playwright-test/src/reporters/json.ts b/packages/playwright-test/src/reporters/json.ts index 8ed09a705e..69c4bf9251 100644 --- a/packages/playwright-test/src/reporters/json.ts +++ b/packages/playwright-test/src/reporters/json.ts @@ -97,7 +97,11 @@ class JSONReporter implements Reporter { private _outputFile: string | undefined; constructor(options: { outputFile?: string } = {}) { - this._outputFile = options.outputFile; + this._outputFile = options.outputFile || process.env[`PLAYWRIGHT_JSON_OUTPUT_NAME`]; + } + + printsToStdio() { + return !this._outputFile; } onBegin(config: FullConfig, suite: Suite) { @@ -270,7 +274,6 @@ class JSONReporter implements Reporter { function outputReport(report: JSONReport, outputFile: string | undefined) { const reportString = JSON.stringify(report, undefined, 2); - outputFile = outputFile || process.env[`PLAYWRIGHT_JSON_OUTPUT_NAME`]; if (outputFile) { fs.mkdirSync(path.dirname(outputFile), { recursive: true }); fs.writeFileSync(outputFile, reportString); diff --git a/packages/playwright-test/src/reporters/junit.ts b/packages/playwright-test/src/reporters/junit.ts index d491d99139..79c068f53d 100644 --- a/packages/playwright-test/src/reporters/junit.ts +++ b/packages/playwright-test/src/reporters/junit.ts @@ -32,10 +32,14 @@ class JUnitReporter implements Reporter { private stripANSIControlSequences = false; constructor(options: { outputFile?: string, stripANSIControlSequences?: boolean } = {}) { - this.outputFile = options.outputFile; + this.outputFile = options.outputFile || process.env[`PLAYWRIGHT_JUNIT_OUTPUT_NAME`]; this.stripANSIControlSequences = options.stripANSIControlSequences || false; } + printsToStdio() { + return !this.outputFile; + } + onBegin(config: FullConfig, suite: Suite) { this.config = config; this.suite = suite; @@ -69,10 +73,9 @@ class JUnitReporter implements Reporter { serializeXML(root, tokens, this.stripANSIControlSequences); const reportString = tokens.join('\n'); - const outputFile = this.outputFile || process.env[`PLAYWRIGHT_JUNIT_OUTPUT_NAME`]; - if (outputFile) { - fs.mkdirSync(path.dirname(outputFile), { recursive: true }); - fs.writeFileSync(outputFile, reportString); + if (this.outputFile) { + fs.mkdirSync(path.dirname(this.outputFile), { recursive: true }); + fs.writeFileSync(this.outputFile, reportString); } else { console.log(reportString); } diff --git a/packages/playwright-test/src/reporters/line.ts b/packages/playwright-test/src/reporters/line.ts index 0d428f6264..4629b462dc 100644 --- a/packages/playwright-test/src/reporters/line.ts +++ b/packages/playwright-test/src/reporters/line.ts @@ -24,6 +24,10 @@ class LineReporter extends BaseReporter { private _failures = 0; private _lastTest: TestCase | undefined; + printsToStdio() { + return true; + } + override onBegin(config: FullConfig, suite: Suite) { super.onBegin(config, suite); this._total = suite.allTests().length; diff --git a/packages/playwright-test/src/reporters/list.ts b/packages/playwright-test/src/reporters/list.ts index 2672801655..16f352378f 100644 --- a/packages/playwright-test/src/reporters/list.ts +++ b/packages/playwright-test/src/reporters/list.ts @@ -38,6 +38,10 @@ class ListReporter extends BaseReporter { this._liveTerminal = process.stdout.isTTY || process.env.PWTEST_SKIP_TEST_OUTPUT || !!this._ttyWidthForTest; } + printsToStdio() { + return true; + } + override onBegin(config: FullConfig, suite: Suite) { super.onBegin(config, suite); console.log(); diff --git a/packages/playwright-test/src/reporters/multiplexer.ts b/packages/playwright-test/src/reporters/multiplexer.ts index 54bb789dfe..f841033a9c 100644 --- a/packages/playwright-test/src/reporters/multiplexer.ts +++ b/packages/playwright-test/src/reporters/multiplexer.ts @@ -23,6 +23,10 @@ export class Multiplexer implements Reporter { this._reporters = reporters; } + printsToStdio() { + return this._reporters.some(r => r.printsToStdio ? r.printsToStdio() : true); + } + onBegin(config: FullConfig, suite: Suite) { for (const reporter of this._reporters) reporter.onBegin?.(config, suite); diff --git a/packages/playwright-test/src/runner.ts b/packages/playwright-test/src/runner.ts index 0d6974d5a9..b2aab29990 100644 --- a/packages/playwright-test/src/runner.ts +++ b/packages/playwright-test/src/runner.ts @@ -80,13 +80,6 @@ export class Runner { html: HtmlReporter, }; const reporters: Reporter[] = []; - const reporterConfig = this._loader.fullConfig().reporter; - if (reporterConfig.length === 1 && reporterConfig[0][0] === 'html') { - // For html reporter, add a line/dot report for convenience. - // Important to put html last because it stalls onEnd. - reporterConfig.unshift([process.stdout.isTTY && !process.env.CI ? 'line' : 'dot', { omitFailures: true }]); - } - for (const r of this._loader.fullConfig().reporter) { const [name, arg] = r; if (name in defaultReporters) { @@ -96,6 +89,15 @@ export class Runner { reporters.push(new reporterConstructor(arg)); } } + const someReporterPrintsToStdio = reporters.some(r => { + const prints = r.printsToStdio ? r.printsToStdio() : true; + return prints; + }); + if (reporters.length && !someReporterPrintsToStdio) { + // Add a line/dot report for convenience. + // Important to put it first, jsut in case some other reporter stalls onEnd. + reporters.unshift(process.stdout.isTTY && !process.env.CI ? new LineReporter({ omitFailures: true }) : new DotReporter({ omitFailures: true })); + } return new Multiplexer(reporters); } diff --git a/packages/playwright-test/types/testReporter.d.ts b/packages/playwright-test/types/testReporter.d.ts index 86acc7603d..a2226bd436 100644 --- a/packages/playwright-test/types/testReporter.d.ts +++ b/packages/playwright-test/types/testReporter.d.ts @@ -356,6 +356,11 @@ export interface FullResult { * went wrong outside of the test execution. */ export interface Reporter { + /** + * Whether this reporter uses stdio for reporting. When it does not, Playwright Test could add some output to enhance user + * experience. + */ + printsToStdio?(): boolean; /** * Called once before running tests. All tests have been already discovered and put into a hierarchy of [Suite]s. * @param config Resolved configuration. diff --git a/tests/playwright-test/reporter-json.spec.ts b/tests/playwright-test/reporter-json.spec.ts index f9b0d5714b..d1626dfd56 100644 --- a/tests/playwright-test/reporter-json.spec.ts +++ b/tests/playwright-test/reporter-json.spec.ts @@ -15,7 +15,8 @@ */ import * as path from 'path'; -import { test, expect } from './playwright-test-fixtures'; +import * as fs from 'fs'; +import { test, expect, stripAscii } from './playwright-test-fixtures'; test('should support spec.ok', async ({ runInlineTest }) => { const result = await runInlineTest({ @@ -200,3 +201,20 @@ test('should have error position in results', async ({ expect(result.report.suites[0].specs[0].tests[0].results[0].errorLocation.line).toBe(7); expect(result.report.suites[0].specs[0].tests[0].results[0].errorLocation.column).toBe(23); }); + +test('should add dot in addition to file json', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { reporter: [['json', { outputFile: 'a.json' }]] }; + `, + 'a.test.js': ` + const { test } = pwt; + test('one', async ({}) => { + expect(1).toBe(1); + }); + `, + }, { reporter: '' }); + expect(result.exitCode).toBe(0); + expect(stripAscii(result.output)).toContain('ยท'); + expect(fs.existsSync(testInfo.outputPath('a.json'))).toBeTruthy(); +}); diff --git a/utils/generate_types/overrides-testReporter.d.ts b/utils/generate_types/overrides-testReporter.d.ts index 4a72b7b788..882718e4f8 100644 --- a/utils/generate_types/overrides-testReporter.d.ts +++ b/utils/generate_types/overrides-testReporter.d.ts @@ -89,6 +89,7 @@ export interface FullResult { } export interface Reporter { + printsToStdio?(): boolean; onBegin?(config: FullConfig, suite: Suite): void; onTestBegin?(test: TestCase, result: TestResult): void; onStdOut?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void;