Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

feat(reporters): show retry #x when running a test (#11003)

Browse source
This commit is contained in:
Dmitry Gozman 2021-12-17 21:07:04 -08:00 committed by GitHub
parent 8f98074fc8
commit c9ba49936f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 82 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
packages/playwright-test/src/reporters/base.ts
View file

@ -64,9 +64,11 @@ export class BaseReporter implements Reporter {
monotonicStartTime: number = 0;
private printTestOutput = !process.env.PWTEST_SKIP_TEST_OUTPUT;
protected _omitFailures: boolean;
private readonly _ttyWidthForTest: number;
constructor(options: { omitFailures?: boolean } = {}) {
this._omitFailures = options.omitFailures || false;
this._ttyWidthForTest = parseInt(process.env.PWTEST_TTY_WIDTH || '', 10);
}
onBegin(config: FullConfig, suite: Suite) {
@ -108,6 +110,10 @@ export class BaseReporter implements Reporter {
this.result = result;
}
protected ttyWidth() {
return this._ttyWidthForTest || (process.env.PWTEST_SKIP_TEST_OUTPUT ? 80 : process.stdout.columns || 0);
}
protected generateStartingMessage() {
const jobs = Math.min(this.config.workers, (this.config as any).__testGroupsCount);
const shardDetails = this.config.shard ? `, shard ${this.config.shard.current} of ${this.config.shard.total}` : '';
@ -429,3 +435,22 @@ const asciiRegex = new RegExp('[\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*
export function stripAnsiEscapes(str: string): string {
return str.replace(asciiRegex, '');
}
// Leaves enough space for the "suffix" to also fit.
export function fitToScreen(line: string, width: number, suffix?: string): string {
const suffixLength = suffix ? stripAnsiEscapes(suffix).length : 0;
width -= suffixLength;
if (line.length <= width)
return line;
let m;
let ansiLen = 0;
asciiRegex.lastIndex = 0;
while ((m = asciiRegex.exec(line)) !== null) {
const visibleLen = m.index - ansiLen;
if (visibleLen >= width)
break;
ansiLen += m[0].length;
}
// Truncate and reset all colors.
return line.substr(0, width + ansiLen) + '\u001b[0m';
}

13
packages/playwright-test/src/reporters/line.ts
View file

@ -15,7 +15,7 @@
*/
import colors from 'colors/safe';
import { BaseReporter, formatFailure, formatTestTitle } from './base';
import { BaseReporter, fitToScreen, formatFailure, formatTestTitle } from './base';
import { FullConfig, TestCase, Suite, TestResult, FullResult } from '../../types/testReporter';
class LineReporter extends BaseReporter {
@ -50,7 +50,8 @@ class LineReporter extends BaseReporter {
stream.write(`\u001B[1A\u001B[2K`);
if (test && this._lastTest !== test) {
// Write new header for the output.
stream.write(colors.gray(formatTestTitle(this.config, test) + `\n`));
const title = colors.gray(formatTestTitle(this.config, test));
stream.write(fitToScreen(title, this.ttyWidth()) + `\n`);
this._lastTest = test;
}
@ -60,15 +61,15 @@ class LineReporter extends BaseReporter {
override onTestEnd(test: TestCase, result: TestResult) {
super.onTestEnd(test, result);
const width = process.env.PWTEST_SKIP_TEST_OUTPUT ? 79 : process.stdout.columns! - 1;
if (!test.title.startsWith('beforeAll') && !test.title.startsWith('afterAll'))
++this._current;
const retriesSuffix = this.totalTestCount < this._current ? ` (retries)` : ``;
const title = `[${this._current}/${this.totalTestCount}]${retriesSuffix} ${formatTestTitle(this.config, test)}`.substring(0, width);
const title = `[${this._current}/${this.totalTestCount}]${retriesSuffix} ${formatTestTitle(this.config, test)}`;
const suffix = result.retry ? ` (retry #${result.retry})` : '';
if (process.env.PWTEST_SKIP_TEST_OUTPUT)
process.stdout.write(`${title}\n`);
process.stdout.write(`${title + suffix}\n`);
else
process.stdout.write(`\u001B[1A\u001B[2K${title}\n`);
process.stdout.write(`\u001B[1A\u001B[2K${fitToScreen(title, this.ttyWidth(), suffix) + colors.yellow(suffix)}\n`);
if (!this.willRetry(test) && (test.outcome() === 'flaky' || test.outcome() === 'unexpected')) {
if (!process.env.PWTEST_SKIP_TEST_OUTPUT)

59
packages/playwright-test/src/reporters/list.ts
View file

@ -17,7 +17,7 @@
/* eslint-disable no-console */
import colors from 'colors/safe';
import milliseconds from 'ms';
import { BaseReporter, formatTestTitle } from './base';
import { BaseReporter, fitToScreen, formatTestTitle } from './base';
import { FullConfig, FullResult, Suite, TestCase, TestResult, TestStep } from '../../types/testReporter';
// Allow it in the Visual Studio Code Terminal and the new Windows Terminal
@ -30,12 +30,10 @@ class ListReporter extends BaseReporter {
private _testRows = new Map<TestCase, number>();
private _needNewLine = false;
private readonly _liveTerminal: string | boolean | undefined;
private readonly _ttyWidthForTest: number;
constructor(options: { omitFailures?: boolean } = {}) {
super(options);
this._ttyWidthForTest = parseInt(process.env.PWTEST_TTY_WIDTH || '', 10);
this._liveTerminal = process.stdout.isTTY || process.env.PWTEST_SKIP_TEST_OUTPUT || !!this._ttyWidthForTest;
this._liveTerminal = process.stdout.isTTY || process.env.PWTEST_SKIP_TEST_OUTPUT || !!process.env.PWTEST_TTY_WIDTH;
}
printsToStdio() {
@ -48,7 +46,7 @@ class ListReporter extends BaseReporter {
console.log();
}
onTestBegin(test: TestCase) {
onTestBegin(test: TestCase, result: TestResult) {
if (this._liveTerminal) {
if (this._needNewLine) {
this._needNewLine = false;
@ -56,7 +54,8 @@ class ListReporter extends BaseReporter {
this._lastRow++;
}
const line = ' ' + colors.gray(formatTestTitle(this.config, test));
process.stdout.write(this._fitToScreen(line, 0) + '\n');
const suffix = this._retrySuffix(result);
process.stdout.write(this._fitToScreen(line, suffix) + suffix + '\n');
}
this._testRows.set(test, this._lastRow++);
}
@ -76,7 +75,7 @@ class ListReporter extends BaseReporter {
return;
if (step.category !== 'test.step')
return;
this._updateTestLine(test, ' ' + colors.gray(formatTestTitle(this.config, test, step)), '');
this._updateTestLine(test, ' ' + colors.gray(formatTestTitle(this.config, test, step)), this._retrySuffix(result));
}
onStepEnd(test: TestCase, result: TestResult, step: TestStep) {
@ -84,7 +83,7 @@ class ListReporter extends BaseReporter {
return;
if (step.category !== 'test.step')
return;
this._updateTestLine(test, ' ' + colors.gray(formatTestTitle(this.config, test, step.parent)), '');
this._updateTestLine(test, ' ' + colors.gray(formatTestTitle(this.config, test, step.parent)), this._retrySuffix(result));
}
private _dumpToStdio(test: TestCase | undefined, chunk: string | Buffer, stream: NodeJS.WriteStream) {
@ -111,19 +110,20 @@ class ListReporter extends BaseReporter {
} else {
const statusMark = (' ' + (result.status === 'passed' ? POSITIVE_STATUS_MARK : NEGATIVE_STATUS_MARK)).padEnd(5);
if (result.status === test.expectedStatus)
text = '\u001b[2K\u001b[0G' + colors.green(statusMark) + colors.gray(title);
text = colors.green(statusMark) + colors.gray(title);
else
text = '\u001b[2K\u001b[0G' + colors.red(statusMark + title);
text = colors.red(statusMark + title);
}
const suffix = this._retrySuffix(result) + duration;
if (this._liveTerminal) {
this._updateTestLine(test, text, duration);
this._updateTestLine(test, text, suffix);
} else {
if (this._needNewLine) {
this._needNewLine = false;
process.stdout.write('\n');
}
process.stdout.write(text + duration);
process.stdout.write(text + suffix);
process.stdout.write('\n');
}
}
@ -140,32 +140,23 @@ class ListReporter extends BaseReporter {
// Go up if needed
if (testRow !== this._lastRow)
process.stdout.write(`\u001B[${this._lastRow - testRow}A`);
// Erase line
process.stdout.write('\u001B[2K');
process.stdout.write(this._fitToScreen(line, visibleLength(suffix)) + suffix);
// Erase line, go to the start
process.stdout.write('\u001B[2K\u001B[0G');
process.stdout.write(this._fitToScreen(line, suffix) + suffix);
// Go down if needed.
if (testRow !== this._lastRow)
process.stdout.write(`\u001B[${this._lastRow - testRow}E`);
}
private _fitToScreen(line: string, suffixLength: number): string {
const ttyWidth = this._ttyWidth() - suffixLength;
if (!this._ttyWidth() || line.length <= ttyWidth)
return line;
let m;
let colorLen = 0;
while ((m = kColorsRe.exec(line)) !== null) {
const visibleLen = m.index - colorLen;
if (visibleLen >= ttyWidth)
break;
colorLen += m[0].length;
}
// Truncate and reset all colors.
return line.substr(0, ttyWidth + colorLen) + '\u001b[0m';
private _retrySuffix(result: TestResult) {
return (result.retry ? colors.yellow(` (retry #${result.retry})`) : '');
}
private _ttyWidth(): number {
return this._ttyWidthForTest || process.stdout.columns || 0;
private _fitToScreen(line: string, suffix?: string): string {
const ttyWidth = this.ttyWidth();
if (!ttyWidth)
return line;
return fitToScreen(line, ttyWidth, suffix);
}
private _updateTestLineForTest(test: TestCase, line: string, suffix: string) {
@ -180,10 +171,4 @@ class ListReporter extends BaseReporter {
}
}
// Matches '\u001b[2K\u001b[0G' and all color codes.
const kColorsRe = /\u001b\[2K\u001b\[0G|\x1B\[\d+m/g;
function visibleLength(s: string): number {
return s.replace(kColorsRe, '').length;
}
export default ListReporter;

12
tests/playwright-test/reporter-line.spec.ts
View file

@ -27,15 +27,15 @@ test('render unexpected after retry', async ({ runInlineTest }) => {
}, { retries: 3, reporter: 'line' });
const text = stripAscii(result.output);
expect(text).toContain('[1/1] a.test.js:6:7 › one');
expect(text).toContain('[2/1] (retries) a.test.js:6:7 › one');
expect(text).toContain('[3/1] (retries) a.test.js:6:7 › one');
expect(text).toContain('[4/1] (retries) a.test.js:6:7 › one');
expect(text).toContain('[2/1] (retries) a.test.js:6:7 › one (retry #1)');
expect(text).toContain('[3/1] (retries) a.test.js:6:7 › one (retry #2)');
expect(text).toContain('[4/1] (retries) a.test.js:6:7 › one (retry #3)');
expect(text).toContain('1 failed');
expect(text).toContain('1) a.test');
expect(text).not.toContain('2) a.test');
expect(text).toContain('Retry #1');
expect(text).toContain('Retry #2');
expect(text).toContain('Retry #3');
expect(text).toContain('Retry #1 ----');
expect(text).toContain('Retry #2 ----');
expect(text).toContain('Retry #3 ----');
expect(result.exitCode).toBe(1);
});

25
tests/playwright-test/reporter-list.spec.ts
View file

@ -83,6 +83,25 @@ test('render steps', async ({ runInlineTest }) => {
]);
});
test('render retries', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
const { test } = pwt;
test('flaky', async ({}, testInfo) => {
expect(testInfo.retry).toBe(1);
});
`,
}, { reporter: 'list', retries: '1' });
const text = stripAscii(result.output);
const lines = text.split('\n').filter(l => l.startsWith('0 :') || l.startsWith('1 :')).map(l => l.replace(/[\dm]+s/, 'XXms'));
const positiveStatusMarkPrefix = process.platform === 'win32' ? 'ok' : '✓ ';
const negativateStatusMarkPrefix = process.platform === 'win32' ? 'x ' : '✘ ';
expect(lines).toEqual([
`0 : ${negativateStatusMarkPrefix} a.test.ts:6:7 › flaky (XXms)`,
`1 : ${positiveStatusMarkPrefix} a.test.ts:6:7 › flaky (retry #1) (XXms)`,
]);
});
test('should truncate long test names', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
@ -92,7 +111,7 @@ test('should truncate long test names', async ({ runInlineTest }) => {
`,
'a.test.ts': `
const { test } = pwt;
test('fails long name', async ({}) => {
test('fails very long name', async ({}) => {
expect(1).toBe(0);
});
test('passes', async ({}) => {
@ -106,8 +125,8 @@ test('should truncate long test names', async ({ runInlineTest }) => {
const text = stripAscii(result.output);
const positiveStatusMarkPrefix = process.platform === 'win32' ? 'ok' : '✓ ';
const negativateStatusMarkPrefix = process.platform === 'win32' ? 'x ' : '✘ ';
expect(text).toContain(`${negativateStatusMarkPrefix} [foo] › a.test.ts:6:7 › fails long`);
expect(text).not.toContain(`${negativateStatusMarkPrefix} [foo] › a.test.ts:6:7 › fails long name (`);
expect(text).toContain(`${negativateStatusMarkPrefix} [foo] › a.test.ts:6:7 › fails very`);
expect(text).not.toContain(`${negativateStatusMarkPrefix} [foo] › a.test.ts:6:7 › fails very long name (`);
expect(text).toContain(`${positiveStatusMarkPrefix} [foo] › a.test.ts:9:7 › passes (`);
expect(text).toContain(`${positiveStatusMarkPrefix} [foo] › a.test.ts:11:7 › passes 2 long`);
expect(text).not.toContain(`${positiveStatusMarkPrefix} [foo] › a.test.ts:11:7 › passes 2 long name (`);