feat(reporters): augment non-stdio reporters with dot/line (#10003)

docs/src/test-reporter-api/class-reporter.md

@@ -237,3 +237,9 @@ Test that has been finished.
 - `result` <[TestResult]>
 
 Result of the test run.
+
+
+## method: Reporter.printsToStdio
+- returns: <[boolean]>
+
+Whether this reporter uses stdio for reporting. When it does not, Playwright Test could add some output to enhance user experience.

docs/src/test-reporters-js.md

@@ -305,22 +305,22 @@ npx playwright show-report my-report
 
 ### JSON reporter
 
-JSON reporter produces an object with all information about the test run. It is usually used together with some terminal reporter like `dot` or `line`.
+JSON reporter produces an object with all information about the test run.
 
 Most likely you want to write the JSON to a file. When running with `--reporter=json`, use `PLAYWRIGHT_JSON_OUTPUT_NAME` environment variable:
 
 ```bash bash-flavor=bash
-PLAYWRIGHT_JSON_OUTPUT_NAME=results.json npx playwright test --reporter=json,dot
+PLAYWRIGHT_JSON_OUTPUT_NAME=results.json npx playwright test --reporter=json
 ```
 
 ```bash bash-flavor=batch
 set PLAYWRIGHT_JSON_OUTPUT_NAME=results.json
-npx playwright test --reporter=json,dot
+npx playwright test --reporter=json
 ```
 
 ```bash bash-flavor=powershell
 $env:PLAYWRIGHT_JSON_OUTPUT_NAME="results.json"
-npx playwright test --reporter=json,dot
+npx playwright test --reporter=json
 ```
 
 In configuration file, pass options directly:
@@ -348,22 +348,22 @@ export default config;
 
 ### JUnit reporter
 
-JUnit reporter produces a JUnit-style xml report. It is usually used together with some terminal reporter like `dot` or `line`.
+JUnit reporter produces a JUnit-style xml report.
 
 Most likely you want to write the report to an xml file. When running with `--reporter=junit`, use `PLAYWRIGHT_JUNIT_OUTPUT_NAME` environment variable:
 
 ```bash bash-flavor=bash
-PLAYWRIGHT_JUNIT_OUTPUT_NAME=results.xml npx playwright test --reporter=junit,line
+PLAYWRIGHT_JUNIT_OUTPUT_NAME=results.xml npx playwright test --reporter=junit
 ```
 
 ```bash bash-flavor=batch
 set PLAYWRIGHT_JUNIT_OUTPUT_NAME=results.xml
-npx playwright test --reporter=junit,line
+npx playwright test --reporter=junit
 ```
 
 ```bash bash-flavor=powershell
 $env:PLAYWRIGHT_JUNIT_OUTPUT_NAME="results.xml"
-npx playwright test --reporter=junit,line
+npx playwright test --reporter=junit
 ```
 
 In configuration file, pass options directly:
@@ -391,7 +391,7 @@ export default config;
 
 ### GitHub Actions annotations
 
-You can use the built in `github` reporter to get automatic failure annotations when running in GitHub actions. Use it with some other reporter, for example `'dot'` and/or `'json'`.
+You can use the built in `github` reporter to get automatic failure annotations when running in GitHub actions.
 
 Note that all other reporters work on GitHub Actions as well, but do not provide annotations.
 
@@ -403,7 +403,7 @@ Note that all other reporters work on GitHub Actions as well, but do not provide
 const config = {
   // 'github' for GitHub Actions CI to generate annotations, plus a concise 'dot'
   // default 'list' when running locally
-  reporter: process.env.CI ? [ ['github'], ['dot'] ] : 'list',
+  reporter: process.env.CI ? 'github' : 'list',
 };
 
 module.exports = config;
@@ -416,7 +416,7 @@ import { PlaywrightTestConfig } from '@playwright/test';
 const config: PlaywrightTestConfig = {
   // 'github' for GitHub Actions CI to generate annotations, plus a concise 'dot'
   // default 'list' when running locally
-  reporter: process.env.CI ? [ ['github'], ['dot'] ] : 'list',
+  reporter: process.env.CI ? 'github' : 'list',
 };
 export default config;
 ```

packages/playwright-test/src/reporters/dot.ts

@@ -21,6 +21,10 @@ import { FullResult, TestCase, TestResult } from '../../types/testReporter';
 class DotReporter extends BaseReporter {
   private _counter = 0;
 
+  printsToStdio() {
+    return true;
+  }
+
   override onStdOut(chunk: string | Buffer, test?: TestCase, result?: TestResult) {
     super.onStdOut(chunk, test, result);
     if (!this.config.quiet)

packages/playwright-test/src/reporters/github.ts

@@ -59,6 +59,10 @@ class GitHubLogger {
 export class GitHubReporter extends BaseReporter {
   githubLogger = new GitHubLogger();
 
+  printsToStdio() {
+    return false;
+  }
+
   override async onEnd(result: FullResult) {
     super.onEnd(result);
     this._printAnnotations();

packages/playwright-test/src/reporters/html.ts

@@ -19,7 +19,7 @@ import fs from 'fs';
 import open from 'open';
 import path from 'path';
 import { Transform, TransformCallback } from 'stream';
-import { FullConfig, Suite } from '../../types/testReporter';
+import { FullConfig, Suite, Reporter } from '../../types/testReporter';
 import { HttpServer } from 'playwright-core/lib/utils/httpServer';
 import { calculateSha1, removeFolders } from 'playwright-core/lib/utils/utils';
 import RawReporter, { JsonReport, JsonSuite, JsonTestCase, JsonTestResult, JsonTestStep, JsonAttachment } from './raw';
@@ -104,7 +104,7 @@ type TestEntry = {
   testCaseSummary: TestCaseSummary
 };
 
-class HtmlReporter {
+class HtmlReporter implements Reporter {
   private config!: FullConfig;
   private suite!: Suite;
   private _outputFolder: string | undefined;
@@ -116,6 +116,10 @@ class HtmlReporter {
     this._open = options.open || 'on-failure';
   }
 
+  printsToStdio() {
+    return false;
+  }
+
   onBegin(config: FullConfig, suite: Suite) {
     this.config = config;
     this.suite = suite;

packages/playwright-test/src/reporters/json.ts

@@ -97,7 +97,11 @@ class JSONReporter implements Reporter {
   private _outputFile: string | undefined;
 
   constructor(options: { outputFile?: string } = {}) {
-    this._outputFile = options.outputFile;
+    this._outputFile = options.outputFile || process.env[`PLAYWRIGHT_JSON_OUTPUT_NAME`];
+  }
+
+  printsToStdio() {
+    return !this._outputFile;
   }
 
   onBegin(config: FullConfig, suite: Suite) {
@@ -270,7 +274,6 @@ class JSONReporter implements Reporter {
 
 function outputReport(report: JSONReport, outputFile: string | undefined) {
   const reportString = JSON.stringify(report, undefined, 2);
-  outputFile = outputFile || process.env[`PLAYWRIGHT_JSON_OUTPUT_NAME`];
   if (outputFile) {
     fs.mkdirSync(path.dirname(outputFile), { recursive: true });
     fs.writeFileSync(outputFile, reportString);

packages/playwright-test/src/reporters/junit.ts

@@ -32,10 +32,14 @@ class JUnitReporter implements Reporter {
   private stripANSIControlSequences = false;
 
   constructor(options: { outputFile?: string, stripANSIControlSequences?: boolean } = {}) {
-    this.outputFile = options.outputFile;
+    this.outputFile = options.outputFile || process.env[`PLAYWRIGHT_JUNIT_OUTPUT_NAME`];
     this.stripANSIControlSequences = options.stripANSIControlSequences || false;
   }
 
+  printsToStdio() {
+    return !this.outputFile;
+  }
+
   onBegin(config: FullConfig, suite: Suite) {
     this.config = config;
     this.suite = suite;
@@ -69,10 +73,9 @@ class JUnitReporter implements Reporter {
 
     serializeXML(root, tokens, this.stripANSIControlSequences);
     const reportString = tokens.join('\n');
-    const outputFile = this.outputFile || process.env[`PLAYWRIGHT_JUNIT_OUTPUT_NAME`];
-    if (outputFile) {
-      fs.mkdirSync(path.dirname(outputFile), { recursive: true });
-      fs.writeFileSync(outputFile, reportString);
+    if (this.outputFile) {
+      fs.mkdirSync(path.dirname(this.outputFile), { recursive: true });
+      fs.writeFileSync(this.outputFile, reportString);
     } else {
       console.log(reportString);
     }

packages/playwright-test/src/reporters/line.ts

@@ -24,6 +24,10 @@ class LineReporter extends BaseReporter {
   private _failures = 0;
   private _lastTest: TestCase | undefined;
 
+  printsToStdio() {
+    return true;
+  }
+
   override onBegin(config: FullConfig, suite: Suite) {
     super.onBegin(config, suite);
     this._total = suite.allTests().length;

packages/playwright-test/src/reporters/list.ts

@@ -38,6 +38,10 @@ class ListReporter extends BaseReporter {
     this._liveTerminal = process.stdout.isTTY || process.env.PWTEST_SKIP_TEST_OUTPUT || !!this._ttyWidthForTest;
   }
 
+  printsToStdio() {
+    return true;
+  }
+
   override onBegin(config: FullConfig, suite: Suite) {
     super.onBegin(config, suite);
     console.log();

packages/playwright-test/src/reporters/multiplexer.ts

@@ -23,6 +23,10 @@ export class Multiplexer implements Reporter {
     this._reporters = reporters;
   }
 
+  printsToStdio() {
+    return this._reporters.some(r => r.printsToStdio ? r.printsToStdio() : true);
+  }
+
   onBegin(config: FullConfig, suite: Suite) {
     for (const reporter of this._reporters)
       reporter.onBegin?.(config, suite);

packages/playwright-test/src/runner.ts

@@ -80,13 +80,6 @@ export class Runner {
       html: HtmlReporter,
     };
     const reporters: Reporter[] = [];
-    const reporterConfig = this._loader.fullConfig().reporter;
-    if (reporterConfig.length === 1 && reporterConfig[0][0] === 'html') {
-      // For html reporter, add a line/dot report for convenience.
-      // Important to put html last because it stalls onEnd.
-      reporterConfig.unshift([process.stdout.isTTY && !process.env.CI ? 'line' : 'dot', { omitFailures: true }]);
-    }
-
     for (const r of this._loader.fullConfig().reporter) {
       const [name, arg] = r;
       if (name in defaultReporters) {
@@ -96,6 +89,15 @@ export class Runner {
         reporters.push(new reporterConstructor(arg));
       }
     }
+    const someReporterPrintsToStdio = reporters.some(r => {
+      const prints = r.printsToStdio ? r.printsToStdio() : true;
+      return prints;
+    });
+    if (reporters.length && !someReporterPrintsToStdio) {
+      // Add a line/dot report for convenience.
+      // Important to put it first, jsut in case some other reporter stalls onEnd.
+      reporters.unshift(process.stdout.isTTY && !process.env.CI ? new LineReporter({ omitFailures: true }) : new DotReporter({ omitFailures: true }));
+    }
     return new Multiplexer(reporters);
   }
 

packages/playwright-test/types/testReporter.d.ts

@@ -356,6 +356,11 @@ export interface FullResult {
  * went wrong outside of the test execution.
  */
 export interface Reporter {
+  /**
+   * Whether this reporter uses stdio for reporting. When it does not, Playwright Test could add some output to enhance user
+   * experience.
+   */
+  printsToStdio?(): boolean;
   /**
    * Called once before running tests. All tests have been already discovered and put into a hierarchy of [Suite]s.
    * @param config Resolved configuration.

tests/playwright-test/reporter-json.spec.ts

@@ -15,7 +15,8 @@
  */
 
 import * as path from 'path';
-import { test, expect } from './playwright-test-fixtures';
+import * as fs from 'fs';
+import { test, expect, stripAscii } from './playwright-test-fixtures';
 
 test('should support spec.ok', async ({ runInlineTest }) => {
   const result = await runInlineTest({
@@ -200,3 +201,20 @@ test('should have error position in results', async ({
   expect(result.report.suites[0].specs[0].tests[0].results[0].errorLocation.line).toBe(7);
   expect(result.report.suites[0].specs[0].tests[0].results[0].errorLocation.column).toBe(23);
 });
+
+test('should add dot in addition to file json', async ({ runInlineTest }, testInfo) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { reporter: [['json', { outputFile: 'a.json' }]] };
+    `,
+    'a.test.js': `
+      const { test } = pwt;
+      test('one', async ({}) => {
+        expect(1).toBe(1);
+      });
+    `,
+  }, { reporter: '' });
+  expect(result.exitCode).toBe(0);
+  expect(stripAscii(result.output)).toContain('·');
+  expect(fs.existsSync(testInfo.outputPath('a.json'))).toBeTruthy();
+});

utils/generate_types/overrides-testReporter.d.ts

@@ -89,6 +89,7 @@ export interface FullResult {
 }
 
 export interface Reporter {
+  printsToStdio?(): boolean;
   onBegin?(config: FullConfig, suite: Suite): void;
   onTestBegin?(test: TestCase, result: TestResult): void;
   onStdOut?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void;