feat(reporters): show retry #x when running a test (#11003)

packages/playwright-test/src/reporters/base.ts

@@ -64,9 +64,11 @@ export class BaseReporter implements Reporter  {
   monotonicStartTime: number = 0;
   private printTestOutput = !process.env.PWTEST_SKIP_TEST_OUTPUT;
   protected _omitFailures: boolean;
+  private readonly _ttyWidthForTest: number;
 
   constructor(options: { omitFailures?: boolean } = {}) {
     this._omitFailures = options.omitFailures || false;
+    this._ttyWidthForTest = parseInt(process.env.PWTEST_TTY_WIDTH || '', 10);
   }
 
   onBegin(config: FullConfig, suite: Suite) {
@@ -108,6 +110,10 @@ export class BaseReporter implements Reporter  {
     this.result = result;
   }
 
+  protected ttyWidth() {
+    return this._ttyWidthForTest || (process.env.PWTEST_SKIP_TEST_OUTPUT ? 80 : process.stdout.columns || 0);
+  }
+
   protected generateStartingMessage() {
     const jobs = Math.min(this.config.workers, (this.config as any).__testGroupsCount);
     const shardDetails = this.config.shard ? `, shard ${this.config.shard.current} of ${this.config.shard.total}` : '';
@@ -429,3 +435,22 @@ const asciiRegex = new RegExp('[\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*
 export function stripAnsiEscapes(str: string): string {
   return str.replace(asciiRegex, '');
 }
+
+// Leaves enough space for the "suffix" to also fit.
+export function fitToScreen(line: string, width: number, suffix?: string): string {
+  const suffixLength = suffix ? stripAnsiEscapes(suffix).length : 0;
+  width -= suffixLength;
+  if (line.length <= width)
+    return line;
+  let m;
+  let ansiLen = 0;
+  asciiRegex.lastIndex = 0;
+  while ((m = asciiRegex.exec(line)) !== null) {
+    const visibleLen = m.index - ansiLen;
+    if (visibleLen >= width)
+      break;
+    ansiLen += m[0].length;
+  }
+  // Truncate and reset all colors.
+  return line.substr(0, width + ansiLen) + '\u001b[0m';
+}

packages/playwright-test/src/reporters/line.ts

@@ -15,7 +15,7 @@
  */
 
 import colors from 'colors/safe';
-import { BaseReporter, formatFailure, formatTestTitle } from './base';
+import { BaseReporter, fitToScreen, formatFailure, formatTestTitle } from './base';
 import { FullConfig, TestCase, Suite, TestResult, FullResult } from '../../types/testReporter';
 
 class LineReporter extends BaseReporter {
@@ -50,7 +50,8 @@ class LineReporter extends BaseReporter {
       stream.write(`\u001B[1A\u001B[2K`);
     if (test && this._lastTest !== test) {
       // Write new header for the output.
-      stream.write(colors.gray(formatTestTitle(this.config, test) + `\n`));
+      const title = colors.gray(formatTestTitle(this.config, test));
+      stream.write(fitToScreen(title, this.ttyWidth()) + `\n`);
       this._lastTest = test;
     }
 
@@ -60,15 +61,15 @@ class LineReporter extends BaseReporter {
 
   override onTestEnd(test: TestCase, result: TestResult) {
     super.onTestEnd(test, result);
-    const width = process.env.PWTEST_SKIP_TEST_OUTPUT ? 79 : process.stdout.columns! - 1;
     if (!test.title.startsWith('beforeAll') && !test.title.startsWith('afterAll'))
       ++this._current;
     const retriesSuffix = this.totalTestCount < this._current ? ` (retries)` : ``;
-    const title = `[${this._current}/${this.totalTestCount}]${retriesSuffix} ${formatTestTitle(this.config, test)}`.substring(0, width);
+    const title = `[${this._current}/${this.totalTestCount}]${retriesSuffix} ${formatTestTitle(this.config, test)}`;
+    const suffix = result.retry ? ` (retry #${result.retry})` : '';
     if (process.env.PWTEST_SKIP_TEST_OUTPUT)
-      process.stdout.write(`${title}\n`);
+      process.stdout.write(`${title + suffix}\n`);
     else
-      process.stdout.write(`\u001B[1A\u001B[2K${title}\n`);
+      process.stdout.write(`\u001B[1A\u001B[2K${fitToScreen(title, this.ttyWidth(), suffix) + colors.yellow(suffix)}\n`);
 
     if (!this.willRetry(test) && (test.outcome() === 'flaky' || test.outcome() === 'unexpected')) {
       if (!process.env.PWTEST_SKIP_TEST_OUTPUT)

packages/playwright-test/src/reporters/list.ts

@@ -17,7 +17,7 @@
 /* eslint-disable no-console */
 import colors from 'colors/safe';
 import milliseconds from 'ms';
-import { BaseReporter, formatTestTitle } from './base';
+import { BaseReporter, fitToScreen, formatTestTitle } from './base';
 import { FullConfig, FullResult, Suite, TestCase, TestResult, TestStep } from '../../types/testReporter';
 
 // Allow it in the Visual Studio Code Terminal and the new Windows Terminal
@@ -30,12 +30,10 @@ class ListReporter extends BaseReporter {
   private _testRows = new Map<TestCase, number>();
   private _needNewLine = false;
   private readonly _liveTerminal: string | boolean | undefined;
-  private readonly _ttyWidthForTest: number;
 
   constructor(options: { omitFailures?: boolean } = {}) {
     super(options);
-    this._ttyWidthForTest = parseInt(process.env.PWTEST_TTY_WIDTH || '', 10);
+    this._liveTerminal = process.stdout.isTTY || process.env.PWTEST_SKIP_TEST_OUTPUT || !!process.env.PWTEST_TTY_WIDTH;
-    this._liveTerminal = process.stdout.isTTY || process.env.PWTEST_SKIP_TEST_OUTPUT || !!this._ttyWidthForTest;
   }
 
   printsToStdio() {
@@ -48,7 +46,7 @@ class ListReporter extends BaseReporter {
     console.log();
   }
 
-  onTestBegin(test: TestCase) {
+  onTestBegin(test: TestCase, result: TestResult) {
     if (this._liveTerminal) {
       if (this._needNewLine) {
         this._needNewLine = false;
@@ -56,7 +54,8 @@ class ListReporter extends BaseReporter {
         this._lastRow++;
       }
       const line = '     ' + colors.gray(formatTestTitle(this.config, test));
-      process.stdout.write(this._fitToScreen(line, 0) + '\n');
+      const suffix = this._retrySuffix(result);
+      process.stdout.write(this._fitToScreen(line, suffix) + suffix + '\n');
     }
     this._testRows.set(test, this._lastRow++);
   }
@@ -76,7 +75,7 @@ class ListReporter extends BaseReporter {
       return;
     if (step.category !== 'test.step')
       return;
-    this._updateTestLine(test, '     ' + colors.gray(formatTestTitle(this.config, test, step)), '');
+    this._updateTestLine(test, '     ' + colors.gray(formatTestTitle(this.config, test, step)), this._retrySuffix(result));
   }
 
   onStepEnd(test: TestCase, result: TestResult, step: TestStep) {
@@ -84,7 +83,7 @@ class ListReporter extends BaseReporter {
       return;
     if (step.category !== 'test.step')
       return;
-    this._updateTestLine(test, '     ' + colors.gray(formatTestTitle(this.config, test, step.parent)), '');
+    this._updateTestLine(test, '     ' + colors.gray(formatTestTitle(this.config, test, step.parent)), this._retrySuffix(result));
   }
 
   private _dumpToStdio(test: TestCase | undefined, chunk: string | Buffer, stream: NodeJS.WriteStream) {
@@ -111,19 +110,20 @@ class ListReporter extends BaseReporter {
     } else {
       const statusMark = ('  ' + (result.status === 'passed' ? POSITIVE_STATUS_MARK : NEGATIVE_STATUS_MARK)).padEnd(5);
       if (result.status === test.expectedStatus)
-        text = '\u001b[2K\u001b[0G' + colors.green(statusMark) + colors.gray(title);
+        text = colors.green(statusMark) + colors.gray(title);
       else
-        text = '\u001b[2K\u001b[0G' + colors.red(statusMark + title);
+        text = colors.red(statusMark + title);
     }
+    const suffix = this._retrySuffix(result) + duration;
 
     if (this._liveTerminal) {
-      this._updateTestLine(test, text, duration);
+      this._updateTestLine(test, text, suffix);
     } else {
       if (this._needNewLine) {
         this._needNewLine = false;
         process.stdout.write('\n');
       }
-      process.stdout.write(text + duration);
+      process.stdout.write(text + suffix);
       process.stdout.write('\n');
     }
   }
@@ -140,32 +140,23 @@ class ListReporter extends BaseReporter {
     // Go up if needed
     if (testRow !== this._lastRow)
       process.stdout.write(`\u001B[${this._lastRow - testRow}A`);
-    // Erase line
+    // Erase line, go to the start
-    process.stdout.write('\u001B[2K');
+    process.stdout.write('\u001B[2K\u001B[0G');
-    process.stdout.write(this._fitToScreen(line, visibleLength(suffix)) + suffix);
+    process.stdout.write(this._fitToScreen(line, suffix) + suffix);
     // Go down if needed.
     if (testRow !== this._lastRow)
       process.stdout.write(`\u001B[${this._lastRow - testRow}E`);
   }
 
-  private _fitToScreen(line: string, suffixLength: number): string {
+  private _retrySuffix(result: TestResult) {
-    const ttyWidth = this._ttyWidth() - suffixLength;
+    return (result.retry ? colors.yellow(` (retry #${result.retry})`) : '');
-    if (!this._ttyWidth() || line.length <= ttyWidth)
-      return line;
-    let m;
-    let colorLen = 0;
-    while ((m = kColorsRe.exec(line)) !== null) {
-      const visibleLen = m.index - colorLen;
-      if (visibleLen >= ttyWidth)
-        break;
-      colorLen += m[0].length;
-    }
-    // Truncate and reset all colors.
-    return line.substr(0, ttyWidth + colorLen) + '\u001b[0m';
   }
 
-  private _ttyWidth(): number {
+  private _fitToScreen(line: string, suffix?: string): string {
-    return this._ttyWidthForTest || process.stdout.columns || 0;
+    const ttyWidth = this.ttyWidth();
+    if (!ttyWidth)
+      return line;
+    return fitToScreen(line, ttyWidth, suffix);
   }
 
   private _updateTestLineForTest(test: TestCase, line: string, suffix: string) {
@@ -180,10 +171,4 @@ class ListReporter extends BaseReporter {
   }
 }
 
-// Matches '\u001b[2K\u001b[0G' and all color codes.
-const kColorsRe = /\u001b\[2K\u001b\[0G|\x1B\[\d+m/g;
-function visibleLength(s: string): number {
-  return s.replace(kColorsRe, '').length;
-}
-
 export default ListReporter;

tests/playwright-test/reporter-line.spec.ts

@@ -27,15 +27,15 @@ test('render unexpected after retry', async ({ runInlineTest }) => {
   }, { retries: 3, reporter: 'line' });
   const text = stripAscii(result.output);
   expect(text).toContain('[1/1] a.test.js:6:7 › one');
-  expect(text).toContain('[2/1] (retries) a.test.js:6:7 › one');
+  expect(text).toContain('[2/1] (retries) a.test.js:6:7 › one (retry #1)');
-  expect(text).toContain('[3/1] (retries) a.test.js:6:7 › one');
+  expect(text).toContain('[3/1] (retries) a.test.js:6:7 › one (retry #2)');
-  expect(text).toContain('[4/1] (retries) a.test.js:6:7 › one');
+  expect(text).toContain('[4/1] (retries) a.test.js:6:7 › one (retry #3)');
   expect(text).toContain('1 failed');
   expect(text).toContain('1) a.test');
   expect(text).not.toContain('2) a.test');
-  expect(text).toContain('Retry #1');
+  expect(text).toContain('Retry #1 ----');
-  expect(text).toContain('Retry #2');
+  expect(text).toContain('Retry #2 ----');
-  expect(text).toContain('Retry #3');
+  expect(text).toContain('Retry #3 ----');
   expect(result.exitCode).toBe(1);
 });
 

tests/playwright-test/reporter-list.spec.ts

@@ -83,6 +83,25 @@ test('render steps', async ({ runInlineTest }) => {
   ]);
 });
 
+test('render retries', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.ts': `
+      const { test } = pwt;
+      test('flaky', async ({}, testInfo) => {
+        expect(testInfo.retry).toBe(1);
+      });
+    `,
+  }, { reporter: 'list', retries: '1' });
+  const text = stripAscii(result.output);
+  const lines = text.split('\n').filter(l => l.startsWith('0 :') || l.startsWith('1 :')).map(l => l.replace(/[\dm]+s/, 'XXms'));
+  const positiveStatusMarkPrefix = process.platform === 'win32' ? 'ok' : '✓ ';
+  const negativateStatusMarkPrefix = process.platform === 'win32' ? 'x ' : '✘ ';
+  expect(lines).toEqual([
+    `0 :   ${negativateStatusMarkPrefix} a.test.ts:6:7 › flaky (XXms)`,
+    `1 :   ${positiveStatusMarkPrefix} a.test.ts:6:7 › flaky (retry #1) (XXms)`,
+  ]);
+});
+
 test('should truncate long test names', async ({ runInlineTest }) => {
   const result = await runInlineTest({
     'playwright.config.ts': `
@@ -92,7 +111,7 @@ test('should truncate long test names', async ({ runInlineTest }) => {
     `,
     'a.test.ts': `
       const { test } = pwt;
-      test('fails long name', async ({}) => {
+      test('fails very long name', async ({}) => {
         expect(1).toBe(0);
       });
       test('passes', async ({}) => {
@@ -106,8 +125,8 @@ test('should truncate long test names', async ({ runInlineTest }) => {
   const text = stripAscii(result.output);
   const positiveStatusMarkPrefix = process.platform === 'win32' ? 'ok' : '✓ ';
   const negativateStatusMarkPrefix = process.platform === 'win32' ? 'x ' : '✘ ';
-  expect(text).toContain(`${negativateStatusMarkPrefix} [foo] › a.test.ts:6:7 › fails long`);
+  expect(text).toContain(`${negativateStatusMarkPrefix} [foo] › a.test.ts:6:7 › fails very`);
-  expect(text).not.toContain(`${negativateStatusMarkPrefix} [foo] › a.test.ts:6:7 › fails long name (`);
+  expect(text).not.toContain(`${negativateStatusMarkPrefix} [foo] › a.test.ts:6:7 › fails very long name (`);
   expect(text).toContain(`${positiveStatusMarkPrefix} [foo] › a.test.ts:9:7 › passes (`);
   expect(text).toContain(`${positiveStatusMarkPrefix} [foo] › a.test.ts:11:7 › passes 2 long`);
   expect(text).not.toContain(`${positiveStatusMarkPrefix} [foo] › a.test.ts:11:7 › passes 2 long name (`);