fix(test runner): handle istty in line reporter (#15455)

When running without tty, line reporter outputs a line for each
percent of the tests, thus limiting the output to ~100 lines.

In addition, reporters now support PLAYWRIGHT_LIVE_TERMINAL
env variable to force tty mode.

packages/playwright-test/src/reporters/base.ts

@@ -20,6 +20,7 @@ import path from 'path';
 import type { FullConfig, TestCase, Suite, TestResult, TestError, Reporter, FullResult, TestStep, Location } from '../../types/testReporter';
 import type { FullConfigInternal } from '../types';
 import { codeFrameColumns } from '../babelBundle';
+import { getAsBooleanFromENV } from 'playwright-core/lib/utils';
 
 export type TestResultOutput = { chunk: string | Buffer, type: 'stdout' | 'stderr' };
 export const kOutputSymbol = Symbol('output');
@@ -54,10 +55,12 @@ export class BaseReporter implements Reporter  {
   private monotonicStartTime: number = 0;
   private _omitFailures: boolean;
   private readonly _ttyWidthForTest: number;
+  readonly liveTerminal: boolean;
 
   constructor(options: { omitFailures?: boolean } = {}) {
     this._omitFailures = options.omitFailures || false;
     this._ttyWidthForTest = parseInt(process.env.PWTEST_TTY_WIDTH || '', 10);
+    this.liveTerminal = process.stdout.isTTY || getAsBooleanFromENV('PLAYWRIGHT_LIVE_TERMINAL');
   }
 
   onBegin(config: FullConfig, suite: Suite) {

packages/playwright-test/src/reporters/line.ts

@@ -18,10 +18,14 @@ import { colors } from 'playwright-core/lib/utilsBundle';
 import { BaseReporter, formatFailure, formatTestTitle } from './base';
 import type { FullConfig, TestCase, Suite, TestResult, FullResult, TestStep } from '../../types/testReporter';
 
+const lineUp = process.env.PW_TEST_DEBUG_REPORTERS ? '<lineup>' : '\u001B[1A';
+const erase = process.env.PW_TEST_DEBUG_REPORTERS ? '<erase>' : '\u001B[2K';
+
 class LineReporter extends BaseReporter {
   private _current = 0;
   private _failures = 0;
   private _lastTest: TestCase | undefined;
+  private _lastPercent = -1;
 
   printsToStdio() {
     return true;
@@ -30,7 +34,8 @@ class LineReporter extends BaseReporter {
   override onBegin(config: FullConfig, suite: Suite) {
     super.onBegin(config, suite);
     console.log(this.generateStartingMessage());
-    console.log();
+    if (this.liveTerminal)
+      console.log();
   }
 
   override onStdOut(chunk: string | Buffer, test?: TestCase, result?: TestResult) {
@@ -46,8 +51,8 @@ class LineReporter extends BaseReporter {
   private _dumpToStdio(test: TestCase | undefined, chunk: string | Buffer, stream: NodeJS.WriteStream) {
     if (this.config.quiet)
       return;
-    if (!process.env.PW_TEST_DEBUG_REPORTERS)
+    if (this.liveTerminal)
-      stream.write(`\u001B[1A\u001B[2K`);
+      stream.write(lineUp + erase);
     if (test && this._lastTest !== test) {
       // Write new header for the output.
       const title = colors.gray(formatTestTitle(this.config, test));
@@ -80,8 +85,8 @@ class LineReporter extends BaseReporter {
   override onTestEnd(test: TestCase, result: TestResult) {
     super.onTestEnd(test, result);
     if (!this.willRetry(test) && (test.outcome() === 'flaky' || test.outcome() === 'unexpected')) {
-      if (!process.env.PW_TEST_DEBUG_REPORTERS)
+      if (this.liveTerminal)
-        process.stdout.write(`\u001B[1A\u001B[2K`);
+        process.stdout.write(lineUp + erase);
       console.log(formatFailure(this.config, test, {
         index: ++this._failures
       }).message);
@@ -90,20 +95,27 @@ class LineReporter extends BaseReporter {
   }
 
   private _updateLine(test: TestCase, result: TestResult, step?: TestStep) {
+    // Do not report 100% until done.
+    const percent = Math.min(99, Math.round(this._current / this.totalTestCount * 100));
     const retriesPrefix = this.totalTestCount < this._current ? ` (retries)` : ``;
     const prefix = `[${this._current}/${this.totalTestCount}]${retriesPrefix} `;
     const currentRetrySuffix = result.retry ? colors.yellow(` (retry #${result.retry})`) : '';
     const title = formatTestTitle(this.config, test, step) + currentRetrySuffix;
-    if (process.env.PW_TEST_DEBUG_REPORTERS)
+    if (this.liveTerminal) {
-      process.stdout.write(`${prefix + title}\n`);
+      process.stdout.write(lineUp + erase + prefix + this.fitToScreen(title, prefix) + '\n');
-    else
+    } else {
-      process.stdout.write(`\u001B[1A\u001B[2K${prefix + this.fitToScreen(title, prefix)}\n`);
+      if (percent !== this._lastPercent)
+        process.stdout.write(`[${percent}%] ${title}\n`);
+    }
+    this._lastPercent = percent;
   }
 
   override async onEnd(result: FullResult) {
-    if (!process.env.PW_TEST_DEBUG_REPORTERS)
-      process.stdout.write(`\u001B[1A\u001B[2K`);
     await super.onEnd(result);
+    if (this.liveTerminal)
+      process.stdout.write(lineUp + erase);
+    else
+      process.stdout.write(`[100%]\n`);
     this.epilogue(false);
   }
 }

packages/playwright-test/src/reporters/list.ts

@@ -28,11 +28,9 @@ class ListReporter extends BaseReporter {
   private _lastRow = 0;
   private _testRows = new Map<TestCase, number>();
   private _needNewLine = false;
-  private readonly _liveTerminal: string | boolean | undefined;
 
   constructor(options: { omitFailures?: boolean } = {}) {
     super(options);
-    this._liveTerminal = process.stdout.isTTY || !!process.env.PWTEST_TTY_WIDTH;
   }
 
   printsToStdio() {
@@ -46,7 +44,7 @@ class ListReporter extends BaseReporter {
   }
 
   onTestBegin(test: TestCase, result: TestResult) {
-    if (this._liveTerminal) {
+    if (this.liveTerminal) {
       if (this._needNewLine) {
         this._needNewLine = false;
         process.stdout.write('\n');
@@ -70,7 +68,7 @@ class ListReporter extends BaseReporter {
   }
 
   onStepBegin(test: TestCase, result: TestResult, step: TestStep) {
-    if (!this._liveTerminal)
+    if (!this.liveTerminal)
       return;
     if (step.category !== 'test.step')
       return;
@@ -78,7 +76,7 @@ class ListReporter extends BaseReporter {
   }
 
   onStepEnd(test: TestCase, result: TestResult, step: TestStep) {
-    if (!this._liveTerminal)
+    if (!this.liveTerminal)
       return;
     if (step.category !== 'test.step')
       return;
@@ -90,7 +88,7 @@ class ListReporter extends BaseReporter {
       return;
     const text = chunk.toString('utf-8');
     this._needNewLine = text[text.length - 1] !== '\n';
-    if (this._liveTerminal) {
+    if (this.liveTerminal) {
       const newLineCount = text.split('\n').length - 1;
       this._lastRow += newLineCount;
     }
@@ -119,7 +117,7 @@ class ListReporter extends BaseReporter {
       text += this._retrySuffix(result) + colors.dim(` (${milliseconds(result.duration)})`);
     }
 
-    if (this._liveTerminal) {
+    if (this.liveTerminal) {
       this._updateTestLine(test, text, prefix);
     } else {
       if (this._needNewLine) {

tests/playwright-test/hooks.spec.ts

@@ -631,7 +631,7 @@ test('should not hang and report results when worker process suddenly exits duri
       test('failing due to afterall', () => {});
       test.afterAll(() => { process.exit(0); });
     `
-  }, { reporter: 'line' });
+  }, { reporter: 'line' }, { PLAYWRIGHT_LIVE_TERMINAL: '1' });
   expect(result.exitCode).toBe(1);
   expect(result.passed).toBe(0);
   expect(result.failed).toBe(1);

tests/playwright-test/playwright-test-fixtures.ts

@@ -284,6 +284,10 @@ export function stripAnsi(str: string): string {
   return str.replace(asciiRegex, '');
 }
 
+export function trimLineEnds(text: string): string {
+  return text.split('\n').map(line => line.trimEnd()).join('\n');
+}
+
 export function countTimes(s: string, sub: string): number {
   let result = 0;
   for (let index = 0; index !== -1;) {

tests/playwright-test/reporter-json.spec.ts

@@ -230,7 +230,7 @@ test('should add line in addition to file json without CI', async ({ runInlineTe
         expect(1).toBe(1);
       });
     `,
-  }, { reporter: '' }, { PW_TEST_DEBUG_REPORTERS: '1' });
+  }, { reporter: '' }, { PLAYWRIGHT_LIVE_TERMINAL: '1' });
   expect(result.exitCode).toBe(0);
   expect(stripAnsi(result.output)).toContain('[1/1] a.test.js:6:7 › one');
   expect(fs.existsSync(testInfo.outputPath('a.json'))).toBeTruthy();

tests/playwright-test/reporter-line.spec.ts

@@ -14,73 +14,203 @@
  * limitations under the License.
  */
 
-import { test, expect, stripAnsi } from './playwright-test-fixtures';
+import { test, expect, stripAnsi, trimLineEnds } from './playwright-test-fixtures';
 
-test('render unexpected after retry', async ({ runInlineTest }) => {
+test('should work with tty', async ({ runInlineTest }, testInfo) => {
   const result = await runInlineTest({
     'a.test.js': `
       const { test } = pwt;
-      test('one', async ({}) => {
+      test.skip('skipped test', async ({}) => {
-        expect(1).toBe(0);
       });
-    `,
+      test('flaky test', async ({}, testInfo) => {
-  }, { retries: 3, reporter: 'line' });
-  const text = stripAnsi(result.output);
-  expect(text).toContain('[1/1] a.test.js:6:7 › one');
-  expect(text).toContain('[2/1] (retries) a.test.js:6:7 › one (retry #1)');
-  expect(text).toContain('[3/1] (retries) a.test.js:6:7 › one (retry #2)');
-  expect(text).toContain('[4/1] (retries) a.test.js:6:7 › one (retry #3)');
-  expect(text).toContain('1 failed');
-  expect(text).toContain('1) a.test');
-  expect(text).not.toContain('2) a.test');
-  expect(text).toContain('Retry #1 ----');
-  expect(text).toContain('Retry #2 ----');
-  expect(text).toContain('Retry #3 ----');
-  expect(result.exitCode).toBe(1);
-});
-
-test('render flaky', async ({ runInlineTest }) => {
-  const result = await runInlineTest({
-    'a.test.js': `
-      const { test } = pwt;
-      test('one', async ({}, testInfo) => {
-        expect(testInfo.retry).toBe(3);
-      });
-    `,
-  }, { retries: 3, reporter: 'line' });
-  const text = stripAnsi(result.output);
-  expect(text).toContain('1 flaky');
-  expect(result.exitCode).toBe(0);
-});
-
-test('should print flaky failures', async ({ runInlineTest }) => {
-  const result = await runInlineTest({
-    'a.spec.ts': `
-      const { test } = pwt;
-      test('foobar', async ({}, testInfo) => {
         expect(testInfo.retry).toBe(1);
       });
-    `
+      test('passing test', async ({}) => {
-  }, { retries: '1', reporter: 'line' });
+      });
-  expect(result.exitCode).toBe(0);
+      test('failing test', async ({}) => {
-  expect(result.flaky).toBe(1);
-  expect(stripAnsi(result.output)).toContain('expect(testInfo.retry).toBe(1)');
-});
-
-test('should work on CI', async ({ runInlineTest }) => {
-  const result = await runInlineTest({
-    'a.test.js': `
-      const { test } = pwt;
-      test('one', async ({}) => {
         expect(1).toBe(0);
       });
     `,
-  }, { reporter: 'line' }, { CI: '1' });
+  }, { retries: '1', reporter: 'line', workers: '1' }, {
-  const text = stripAnsi(result.output);
+    PLAYWRIGHT_LIVE_TERMINAL: '1',
-  expect(text).toContain('[1/1] a.test.js:6:7 › one');
+    FORCE_COLOR: '0',
-  expect(text).toContain('1 failed');
+    PW_TEST_DEBUG_REPORTERS: '1',
-  expect(text).toContain('1) a.test');
+  });
   expect(result.exitCode).toBe(1);
+  expect(trimLineEnds(result.output)).toContain(trimLineEnds(`Running 4 tests using 1 worker
+
+<lineup><erase>[1/4] a.test.js:6:12 › skipped test
+<lineup><erase>[2/4] a.test.js:8:7 › flaky test
+<lineup><erase>[3/4] a.test.js:8:7 › flaky test (retry #1)
+<lineup><erase>  1) a.test.js:8:7 › flaky test ====================================================================
+
+    Error: expect(received).toBe(expected) // Object.is equality
+
+    Expected: 1
+    Received: 0
+
+       7 |       });
+       8 |       test('flaky test', async ({}, testInfo) => {
+    >  9 |         expect(testInfo.retry).toBe(1);
+         |                                ^
+      10 |       });
+      11 |       test('passing test', async ({}) => {
+      12 |       });
+
+        at ${testInfo.outputPath('a.test.js')}:9:32
+
+
+<lineup><erase>[4/4] a.test.js:11:7 › passing test
+<lineup><erase>[5/4] (retries) a.test.js:13:7 › failing test
+<lineup><erase>[6/4] (retries) a.test.js:13:7 › failing test (retry #1)
+<lineup><erase>  2) a.test.js:13:7 › failing test =================================================================
+
+    Error: expect(received).toBe(expected) // Object.is equality
+
+    Expected: 0
+    Received: 1
+
+      12 |       });
+      13 |       test('failing test', async ({}) => {
+    > 14 |         expect(1).toBe(0);
+         |                   ^
+      15 |       });
+      16 |
+
+        at ${testInfo.outputPath('a.test.js')}:14:19
+
+    Retry #1 ---------------------------------------------------------------------------------------
+
+    Error: expect(received).toBe(expected) // Object.is equality
+
+    Expected: 0
+    Received: 1
+
+      12 |       });
+      13 |       test('failing test', async ({}) => {
+    > 14 |         expect(1).toBe(0);
+         |                   ^
+      15 |       });
+      16 |
+
+        at ${testInfo.outputPath('a.test.js')}:14:19
+
+
+<lineup><erase>
+  1 failed
+    a.test.js:13:7 › failing test ==================================================================
+  1 flaky
+    a.test.js:8:7 › flaky test =====================================================================
+  1 skipped
+  1 passed`));
+});
+
+test('should work with non-tty', async ({ runInlineTest }, testInfo) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const { test } = pwt;
+      test.skip('skipped test', async ({}) => {
+      });
+      test('flaky test', async ({}, testInfo) => {
+        expect(testInfo.retry).toBe(1);
+      });
+      test('passing test', async ({}) => {
+      });
+      test('failing test', async ({}) => {
+        expect(1).toBe(0);
+      });
+    `,
+  }, { retries: '1', reporter: 'line', workers: '1' }, {
+    FORCE_COLOR: '0',
+    PW_TEST_DEBUG_REPORTERS: '1',
+  });
+  expect(result.exitCode).toBe(1);
+  expect(trimLineEnds(result.output)).toContain(trimLineEnds(`Running 4 tests using 1 worker
+[25%] a.test.js:6:12 › skipped test
+[50%] a.test.js:8:7 › flaky test
+[75%] a.test.js:8:7 › flaky test (retry #1)
+  1) a.test.js:8:7 › flaky test ====================================================================
+
+    Error: expect(received).toBe(expected) // Object.is equality
+
+    Expected: 1
+    Received: 0
+
+       7 |       });
+       8 |       test('flaky test', async ({}, testInfo) => {
+    >  9 |         expect(testInfo.retry).toBe(1);
+         |                                ^
+      10 |       });
+      11 |       test('passing test', async ({}) => {
+      12 |       });
+
+        at ${testInfo.outputPath('a.test.js')}:9:32
+
+
+[99%] a.test.js:11:7 › passing test
+  2) a.test.js:13:7 › failing test =================================================================
+
+    Error: expect(received).toBe(expected) // Object.is equality
+
+    Expected: 0
+    Received: 1
+
+      12 |       });
+      13 |       test('failing test', async ({}) => {
+    > 14 |         expect(1).toBe(0);
+         |                   ^
+      15 |       });
+      16 |
+
+        at ${testInfo.outputPath('a.test.js')}:14:19
+
+    Retry #1 ---------------------------------------------------------------------------------------
+
+    Error: expect(received).toBe(expected) // Object.is equality
+
+    Expected: 0
+    Received: 1
+
+      12 |       });
+      13 |       test('failing test', async ({}) => {
+    > 14 |         expect(1).toBe(0);
+         |                   ^
+      15 |       });
+      16 |
+
+        at ${testInfo.outputPath('a.test.js')}:14:19
+
+
+[100%]
+
+  1 failed
+    a.test.js:13:7 › failing test ==================================================================
+  1 flaky
+    a.test.js:8:7 › flaky test =====================================================================
+  1 skipped
+  1 passed`));
+});
+
+test('should spare status updates in non-tty mode', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const { test } = pwt;
+      for (let i = 0; i < 300; i++) {
+        test('test' + i, () => {});
+      }
+    `,
+  }, { reporter: 'line', workers: '1' }, {
+    FORCE_COLOR: '0',
+    PW_TEST_DEBUG_REPORTERS: '1',
+  });
+  expect(result.exitCode).toBe(0);
+  const lines = [`Running 300 tests using 1 worker`, `[0%] a.test.js:7:9 › test0`];
+  for (let i = 1; i <= 99; i++)
+    lines.push(`[${i}%] a.test.js:7:9 › test${3 * i - 2}`);
+  lines.push('[100%]');
+  lines.push('');
+  lines.push('  300 passed');
+  expect(trimLineEnds(result.output)).toContain(lines.join('\n'));
 });
 
 test('should print output', async ({ runInlineTest }) => {

tests/playwright-test/reporter-list.spec.ts

@@ -66,7 +66,7 @@ test('render steps', async ({ runInlineTest }) => {
         });
       });
     `,
-  }, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PWTEST_TTY_WIDTH: '80' });
+  }, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PLAYWRIGHT_LIVE_TERMINAL: '1' });
   const text = stripAnsi(result.output);
   const lines = text.split('\n').filter(l => l.startsWith('0 :'));
   lines.pop(); // Remove last item that contains [v] and time in ms.
@@ -94,7 +94,7 @@ test('render retries', async ({ runInlineTest }) => {
         expect(testInfo.retry).toBe(1);
       });
     `,
-  }, { reporter: 'list', retries: '1' }, { PW_TEST_DEBUG_REPORTERS: '1', PWTEST_TTY_WIDTH: '80' });
+  }, { reporter: 'list', retries: '1' }, { PW_TEST_DEBUG_REPORTERS: '1', PLAYWRIGHT_LIVE_TERMINAL: '1' });
   const text = stripAnsi(result.output);
   const lines = text.split('\n').filter(l => l.startsWith('0 :') || l.startsWith('1 :')).map(l => l.replace(/[\dm]+s/, 'XXms'));
 
@@ -123,7 +123,7 @@ test('should truncate long test names', async ({ runInlineTest }) => {
       test.skip('skipped very long name', async () => {
       });
     `,
-  }, { reporter: 'list', retries: 0 }, { PWTEST_TTY_WIDTH: 50 });
+  }, { reporter: 'list', retries: 0 }, { PLAYWRIGHT_LIVE_TERMINAL: '1', PWTEST_TTY_WIDTH: 50 });
   expect(result.exitCode).toBe(1);
 
   const lines = stripAnsi(result.output).split('\n').slice(3, 11);