Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

feat(line reporter): show testing progress (#15547)

Browse source 
Line reporter now shows stats in addition to the test name:
```
[chromium] › page/page-click-react.spec.ts:108:1 › should not retarget when element changes on hover
[21/93] Passed: 17 Flaky: 0 Failed: 0 Skipped: 4 (7s)
```
This commit is contained in:
Dmitry Gozman 2022-07-11 20:44:50 -07:00 committed by GitHub
parent 7727ebe758
commit e76d9b3b28
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 101 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
docs/src/test-reporters-js.md
View file

@ -151,7 +151,7 @@ Running 124 tests using 6 workers
### Line reporter ### Line reporter
Line reporter is more concise than the list reporter. It uses a single line to report last finished test, and prints failures when they occur. Line reporter is useful for large test suites where it shows the progress but does not spam the output by listing all the tests. Line reporter is more concise than the list reporter. It uses two lines to report currently running test and testing progress, and prints failures when they occur. Line reporter is useful for large test suites where it shows the progress but does not spam the output by listing all the tests.
```bash ```bash
npx playwright test --reporter=line npx playwright test --reporter=line
@ -190,7 +190,8 @@ Running 124 tests using 6 workers
Expected: 1 Expected: 1
Received: 0 Received: 0
[23/124] gitignore.spec.ts - should respect nested .gitignore example.spec.ts:8:3 › should navigate to playwright.dev
[23/124] Passed: 20 Flaky: 0 Failed: 0 Skipped: 2 (12s)
``` ```
### Dot reporter ### Dot reporter

37
packages/playwright-test/src/reporters/base.ts
View file

@ -49,13 +49,13 @@ export class BaseReporter implements Reporter {
duration = 0; duration = 0;
config!: FullConfigInternal; config!: FullConfigInternal;
suite!: Suite; suite!: Suite;
totalTestCount = 0;
result!: FullResult; result!: FullResult;
private fileDurations = new Map<string, number>(); private fileDurations = new Map<string, number>();
private monotonicStartTime: number = 0; private monotonicStartTime: number = 0;
private _omitFailures: boolean; private _omitFailures: boolean;
private readonly _ttyWidthForTest: number; private readonly _ttyWidthForTest: number;
readonly liveTerminal: boolean; readonly liveTerminal: boolean;
readonly stats = { started: 0, skipped: 0, completed: 0, total: 0, passed: 0, failed: 0, flaky: 0 };
constructor(options: { omitFailures?: boolean } = {}) { constructor(options: { omitFailures?: boolean } = {}) {
this._omitFailures = options.omitFailures || false; this._omitFailures = options.omitFailures || false;
@ -67,7 +67,7 @@ export class BaseReporter implements Reporter {
this.monotonicStartTime = monotonicTime(); this.monotonicStartTime = monotonicTime();
this.config = config as FullConfigInternal; this.config = config as FullConfigInternal;
this.suite = suite; this.suite = suite;
this.totalTestCount = suite.allTests().length; this.stats.total = suite.allTests().length;
} }
onStdOut(chunk: string | Buffer, test?: TestCase, result?: TestResult) { onStdOut(chunk: string | Buffer, test?: TestCase, result?: TestResult) {
@ -85,7 +85,21 @@ export class BaseReporter implements Reporter {
(result as any)[kOutputSymbol].push(output); (result as any)[kOutputSymbol].push(output);
} }
onTestBegin(test: TestCase, result: TestResult) {
this.stats.started++;
}
onTestEnd(test: TestCase, result: TestResult) { onTestEnd(test: TestCase, result: TestResult) {
this.stats.completed++;
if (!this.willRetry(test)) {
switch (test.outcome()) {
case 'skipped': this.stats.skipped++; break;
case 'expected': this.stats.passed++; break;
case 'unexpected': this.stats.failed++; break;
case 'flaky': this.stats.flaky++; break;
}
}
// Ignore any tests that are run in parallel. // Ignore any tests that are run in parallel.
for (let suite: Suite | undefined = test.parent; suite; suite = suite.parent) { for (let suite: Suite | undefined = test.parent; suite; suite = suite.parent) {
if ((suite as any)._parallelMode === 'parallel') if ((suite as any)._parallelMode === 'parallel')
@ -119,7 +133,24 @@ export class BaseReporter implements Reporter {
protected generateStartingMessage() { protected generateStartingMessage() {
const jobs = Math.min(this.config.workers, this.config._testGroupsCount); const jobs = Math.min(this.config.workers, this.config._testGroupsCount);
const shardDetails = this.config.shard ? `, shard ${this.config.shard.current} of ${this.config.shard.total}` : ''; const shardDetails = this.config.shard ? `, shard ${this.config.shard.current} of ${this.config.shard.total}` : '';
return `\nRunning ${this.totalTestCount} test${this.totalTestCount > 1 ? 's' : ''} using ${jobs} worker${jobs > 1 ? 's' : ''}${shardDetails}`; return `\nRunning ${this.stats.total} test${this.stats.total > 1 ? 's' : ''} using ${jobs} worker${jobs > 1 ? 's' : ''}${shardDetails}`;
}
protected generateStatsMessage(mode: 'started' | 'completed', done: boolean) {
// Do not report 100% until done.
const count = this.stats[mode];
const percent = Math.min(done ? 100 : 99, Math.round(count / this.stats.total * 100));
const maxExpected = done ? this.stats.total : this.stats.total - (mode === 'started' ? 0 : 1);
const retriesSuffix = count > maxExpected ? `+retries` : ``;
const message = [
`[${count}/${this.stats.total}${retriesSuffix}]`,
`${(this.stats.passed ? colors.green : colors.gray)('Passed: ' + this.stats.passed)}`,
`${(this.stats.flaky ? colors.red : colors.gray)('Flaky: ' + this.stats.flaky)}`,
`${(this.stats.failed ? colors.red : colors.gray)('Failed: ' + this.stats.failed)}`,
`${(this.stats.skipped ? colors.yellow : colors.gray)('Skipped: ' + this.stats.skipped)}`,
colors.gray(process.env.PW_TEST_DEBUG_REPORTERS ? `(XXms)` : `(${milliseconds((monotonicTime() - this.monotonicStartTime) | 0)})`),
].join(' ');
return { percent, message };
} }
protected getSlowTests(): [string, number][] { protected getSlowTests(): [string, number][] {

48
packages/playwright-test/src/reporters/line.ts
View file

@ -22,7 +22,6 @@ const lineUp = process.env.PW_TEST_DEBUG_REPORTERS ? '<lineup>' : '\u001B[1A';
const erase = process.env.PW_TEST_DEBUG_REPORTERS ? '<erase>' : '\u001B[2K'; const erase = process.env.PW_TEST_DEBUG_REPORTERS ? '<erase>' : '\u001B[2K';
class LineReporter extends BaseReporter { class LineReporter extends BaseReporter {
private _current = 0;
private _failures = 0; private _failures = 0;
private _lastTest: TestCase | undefined; private _lastTest: TestCase | undefined;
private _lastPercent = -1; private _lastPercent = -1;
@ -35,27 +34,31 @@ class LineReporter extends BaseReporter {
super.onBegin(config, suite); super.onBegin(config, suite);
console.log(this.generateStartingMessage()); console.log(this.generateStartingMessage());
if (this.liveTerminal) if (this.liveTerminal)
console.log(); console.log('\n');
} }
override onStdOut(chunk: string | Buffer, test?: TestCase, result?: TestResult) { override onStdOut(chunk: string | Buffer, test?: TestCase, result?: TestResult) {
super.onStdOut(chunk, test, result); super.onStdOut(chunk, test, result);
this._dumpToStdio(test, chunk, process.stdout); this._dumpToStdio(test, result, chunk, process.stdout);
} }
override onStdErr(chunk: string | Buffer, test?: TestCase, result?: TestResult) { override onStdErr(chunk: string | Buffer, test?: TestCase, result?: TestResult) {
super.onStdErr(chunk, test, result); super.onStdErr(chunk, test, result);
this._dumpToStdio(test, chunk, process.stderr); this._dumpToStdio(test, result, chunk, process.stderr);
} }
private _dumpToStdio(test: TestCase | undefined, chunk: string | Buffer, stream: NodeJS.WriteStream) { private _retrySuffix(result: TestResult | undefined) {
return result?.retry ? colors.yellow(` (retry #${result.retry})`) : '';
}
private _dumpToStdio(test: TestCase | undefined, result: TestResult | undefined, chunk: string | Buffer, stream: NodeJS.WriteStream) {
if (this.config.quiet) if (this.config.quiet)
return; return;
if (this.liveTerminal) if (this.liveTerminal)
stream.write(lineUp + erase); stream.write(lineUp + erase + lineUp + erase);
if (test && this._lastTest !== test) { if (test && this._lastTest !== test) {
// Write new header for the output. // Write new header for the output.
const title = colors.gray(formatTestTitle(this.config, test)); const title = colors.gray(formatTestTitle(this.config, test)) + this._retrySuffix(result);
stream.write(this.fitToScreen(title) + `\n`); stream.write(this.fitToScreen(title) + `\n`);
this._lastTest = test; this._lastTest = test;
} }
@ -64,11 +67,12 @@ class LineReporter extends BaseReporter {
if (chunk[chunk.length - 1] !== '\n') if (chunk[chunk.length - 1] !== '\n')
console.log(); console.log();
console.log(); if (this.liveTerminal)
console.log('\n');
} }
onTestBegin(test: TestCase, result: TestResult) { override onTestBegin(test: TestCase, result: TestResult) {
++this._current; super.onTestBegin(test, result);
this._updateLine(test, result, undefined); this._updateLine(test, result, undefined);
} }
@ -86,36 +90,34 @@ class LineReporter extends BaseReporter {
super.onTestEnd(test, result); super.onTestEnd(test, result);
if (!this.willRetry(test) && (test.outcome() === 'flaky' || test.outcome() === 'unexpected')) { if (!this.willRetry(test) && (test.outcome() === 'flaky' || test.outcome() === 'unexpected')) {
if (this.liveTerminal) if (this.liveTerminal)
process.stdout.write(lineUp + erase); process.stdout.write(lineUp + erase + lineUp + erase);
console.log(formatFailure(this.config, test, { console.log(formatFailure(this.config, test, {
index: ++this._failures index: ++this._failures
}).message); }).message);
console.log(); console.log();
if (this.liveTerminal)
process.stdout.write(this.fitToScreen(this.generateStatsMessage('started', false).message) + '\n');
} }
} }
private _updateLine(test: TestCase, result: TestResult, step?: TestStep) { private _updateLine(test: TestCase, result: TestResult, step?: TestStep) {
// Do not report 100% until done. const stats = this.generateStatsMessage('started', false);
const percent = Math.min(99, Math.round(this._current / this.totalTestCount * 100)); const title = formatTestTitle(this.config, test, step) + this._retrySuffix(result);
const retriesPrefix = this.totalTestCount < this._current ? ` (retries)` : ``;
const prefix = `[${this._current}/${this.totalTestCount}]${retriesPrefix} `;
const currentRetrySuffix = result.retry ? colors.yellow(` (retry #${result.retry})`) : '';
const title = formatTestTitle(this.config, test, step) + currentRetrySuffix;
if (this.liveTerminal) { if (this.liveTerminal) {
process.stdout.write(lineUp + erase + prefix + this.fitToScreen(title, prefix) + '\n'); process.stdout.write(lineUp + erase + lineUp + erase + this.fitToScreen(title) + '\n' + this.fitToScreen(stats.message) + '\n');
} else { } else {
if (percent !== this._lastPercent) if (stats.percent !== this._lastPercent)
process.stdout.write(`[${percent}%] ${title}\n`); process.stdout.write(this.fitToScreen(stats.message) + '\n');
} }
this._lastPercent = percent; this._lastPercent = stats.percent;
} }
override async onEnd(result: FullResult) { override async onEnd(result: FullResult) {
await super.onEnd(result); await super.onEnd(result);
if (this.liveTerminal) if (this.liveTerminal)
process.stdout.write(lineUp + erase); process.stdout.write(lineUp + erase + lineUp + erase);
else else
process.stdout.write(`[100%]\n`); process.stdout.write(this.fitToScreen(this.generateStatsMessage('started', true).message) + '\n');
this.epilogue(false); this.epilogue(false);
} }
} }

3
packages/playwright-test/src/reporters/list.ts
View file

@ -43,7 +43,8 @@ class ListReporter extends BaseReporter {
console.log(); console.log();
} }
onTestBegin(test: TestCase, result: TestResult) { override onTestBegin(test: TestCase, result: TestResult) {
super.onTestBegin(test, result);
if (this.liveTerminal) { if (this.liveTerminal) {
if (this._needNewLine) { if (this._needNewLine) {
this._needNewLine = false; this._needNewLine = false;

2
tests/playwright-test/hooks.spec.ts
View file

@ -636,7 +636,7 @@ test('should not hang and report results when worker process suddenly exits duri
expect(result.passed).toBe(0); expect(result.passed).toBe(0);
expect(result.failed).toBe(1); expect(result.failed).toBe(1);
expect(result.output).toContain('Worker process exited unexpectedly'); expect(result.output).toContain('Worker process exited unexpectedly');
expect(stripAnsi(result.output)).toContain('[1/1] a.spec.js:6:7 › failing due to afterall'); expect(stripAnsi(result.output)).toContain('a.spec.js:6:7 › failing due to afterall');
}); });
test('unhandled rejection during beforeAll should be reported and prevent more tests', async ({ runInlineTest }) => { test('unhandled rejection during beforeAll should be reported and prevent more tests', async ({ runInlineTest }) => {

2
tests/playwright-test/reporter-json.spec.ts
View file

@ -232,6 +232,6 @@ test('should add line in addition to file json without CI', async ({ runInlineTe
`, `,
}, { reporter: '' }, { PLAYWRIGHT_LIVE_TERMINAL: '1' }); }, { reporter: '' }, { PLAYWRIGHT_LIVE_TERMINAL: '1' });
expect(result.exitCode).toBe(0); expect(result.exitCode).toBe(0);
expect(stripAnsi(result.output)).toContain('[1/1] a.test.js:6:7 › one'); expect(stripAnsi(result.output)).toContain('a.test.js:6:7 › one');
expect(fs.existsSync(testInfo.outputPath('a.json'))).toBeTruthy(); expect(fs.existsSync(testInfo.outputPath('a.json'))).toBeTruthy();
}); });

58
tests/playwright-test/reporter-line.spec.ts
View file

@ -37,12 +37,13 @@ test('should work with tty', async ({ runInlineTest }, testInfo) => {
PW_TEST_DEBUG_REPORTERS: '1', PW_TEST_DEBUG_REPORTERS: '1',
}); });
expect(result.exitCode).toBe(1); expect(result.exitCode).toBe(1);
expect(trimLineEnds(result.output)).toContain(trimLineEnds(`Running 4 tests using 1 worker expect(trimLineEnds(result.output)).toContain(trimLineEnds(`<lineup><erase><lineup><erase>a.test.js:6:12 › skipped test
[1/4] Passed: 0 Flaky: 0 Failed: 0 Skipped: 0 (XXms)
<lineup><erase>[1/4] a.test.js:6:12 › skipped test <lineup><erase><lineup><erase>a.test.js:8:7 › flaky test
<lineup><erase>[2/4] a.test.js:8:7 › flaky test [2/4] Passed: 0 Flaky: 0 Failed: 0 Skipped: 1 (XXms)
<lineup><erase>[3/4] a.test.js:8:7 › flaky test (retry #1) <lineup><erase><lineup><erase>a.test.js:8:7 › flaky test (retry #1)
<lineup><erase> 1) a.test.js:8:7 › flaky test ==================================================================== [3/4] Passed: 0 Flaky: 0 Failed: 0 Skipped: 1 (XXms)
<lineup><erase><lineup><erase> 1) a.test.js:8:7 › flaky test ====================================================================
Error: expect(received).toBe(expected) // Object.is equality Error: expect(received).toBe(expected) // Object.is equality
@ -60,10 +61,14 @@ test('should work with tty', async ({ runInlineTest }, testInfo) => {
at ${testInfo.outputPath('a.test.js')}:9:32 at ${testInfo.outputPath('a.test.js')}:9:32
<lineup><erase>[4/4] a.test.js:11:7 › passing test [3/4] Passed: 0 Flaky: 1 Failed: 0 Skipped: 1 (XXms)
<lineup><erase>[5/4] (retries) a.test.js:13:7 › failing test <lineup><erase><lineup><erase>a.test.js:11:7 › passing test
<lineup><erase>[6/4] (retries) a.test.js:13:7 › failing test (retry #1) [4/4] Passed: 0 Flaky: 1 Failed: 0 Skipped: 1 (XXms)
<lineup><erase> 2) a.test.js:13:7 › failing test ================================================================= <lineup><erase><lineup><erase>a.test.js:13:7 › failing test
[5/4+retries] Passed: 1 Flaky: 1 Failed: 0 Skipped: 1 (XXms)
<lineup><erase><lineup><erase>a.test.js:13:7 › failing test (retry #1)
[6/4+retries] Passed: 1 Flaky: 1 Failed: 0 Skipped: 1 (XXms)
<lineup><erase><lineup><erase> 2) a.test.js:13:7 › failing test =================================================================
Error: expect(received).toBe(expected) // Object.is equality Error: expect(received).toBe(expected) // Object.is equality
@ -96,7 +101,8 @@ test('should work with tty', async ({ runInlineTest }, testInfo) => {
at ${testInfo.outputPath('a.test.js')}:14:19 at ${testInfo.outputPath('a.test.js')}:14:19
<lineup><erase> [6/4+retries] Passed: 1 Flaky: 1 Failed: 1 Skipped: 1 (XXms)
<lineup><erase><lineup><erase>
1 failed 1 failed
a.test.js:13:7 › failing test ================================================================== a.test.js:13:7 › failing test ==================================================================
1 flaky 1 flaky
@ -126,9 +132,9 @@ test('should work with non-tty', async ({ runInlineTest }, testInfo) => {
}); });
expect(result.exitCode).toBe(1); expect(result.exitCode).toBe(1);
expect(trimLineEnds(result.output)).toContain(trimLineEnds(`Running 4 tests using 1 worker expect(trimLineEnds(result.output)).toContain(trimLineEnds(`Running 4 tests using 1 worker
[25%] a.test.js:6:12 › skipped test [1/4] Passed: 0 Flaky: 0 Failed: 0 Skipped: 0 (XXms)
[50%] a.test.js:8:7 › flaky test [2/4] Passed: 0 Flaky: 0 Failed: 0 Skipped: 1 (XXms)
[75%] a.test.js:8:7 › flaky test (retry #1) [3/4] Passed: 0 Flaky: 0 Failed: 0 Skipped: 1 (XXms)
1) a.test.js:8:7 › flaky test ==================================================================== 1) a.test.js:8:7 › flaky test ====================================================================
Error: expect(received).toBe(expected) // Object.is equality Error: expect(received).toBe(expected) // Object.is equality
@ -147,7 +153,7 @@ test('should work with non-tty', async ({ runInlineTest }, testInfo) => {
at ${testInfo.outputPath('a.test.js')}:9:32 at ${testInfo.outputPath('a.test.js')}:9:32
[99%] a.test.js:11:7 › passing test [4/4] Passed: 0 Flaky: 1 Failed: 0 Skipped: 1 (XXms)
2) a.test.js:13:7 › failing test ================================================================= 2) a.test.js:13:7 › failing test =================================================================
Error: expect(received).toBe(expected) // Object.is equality Error: expect(received).toBe(expected) // Object.is equality
@ -181,7 +187,7 @@ test('should work with non-tty', async ({ runInlineTest }, testInfo) => {
at ${testInfo.outputPath('a.test.js')}:14:19 at ${testInfo.outputPath('a.test.js')}:14:19
[100%] [6/4+retries] Passed: 1 Flaky: 1 Failed: 1 Skipped: 1 (XXms)
1 failed 1 failed
a.test.js:13:7 › failing test ================================================================== a.test.js:13:7 › failing test ==================================================================
@ -204,10 +210,10 @@ test('should spare status updates in non-tty mode', async ({ runInlineTest }) =>
PW_TEST_DEBUG_REPORTERS: '1', PW_TEST_DEBUG_REPORTERS: '1',
}); });
expect(result.exitCode).toBe(0); expect(result.exitCode).toBe(0);
const lines = [`Running 300 tests using 1 worker`, `[0%] a.test.js:7:9 › test0`]; const lines = [`Running 300 tests using 1 worker`, `[1/300] Passed: 0 Flaky: 0 Failed: 0 Skipped: 0 (XXms)`];
for (let i = 1; i <= 99; i++) for (let i = 0; i < 99; i++)
lines.push(`[${i}%] a.test.js:7:9 › test${3 * i - 2}`); lines.push(`[${3 * i + 2}/300] Passed: ${3 * i + 1} Flaky: 0 Failed: 0 Skipped: 0 (XXms)`);
lines.push('[100%]'); lines.push('[300/300] Passed: 300 Flaky: 0 Failed: 0 Skipped: 0 (XXms)');
lines.push(''); lines.push('');
lines.push(' 300 passed'); lines.push(' 300 passed');
expect(trimLineEnds(result.output)).toContain(lines.join('\n')); expect(trimLineEnds(result.output)).toContain(lines.join('\n'));
@ -222,15 +228,21 @@ test('should print output', async ({ runInlineTest }) => {
process.stdout.write('two'); process.stdout.write('two');
console.log('full-line'); console.log('full-line');
}); });
test('one more', async ({}, testInfo) => {
console.log('yay');
});
` `
}, { reporter: 'line' }); }, { reporter: 'line' }, { PW_TEST_DEBUG_REPORTERS: '1' });
expect(result.exitCode).toBe(0); expect(result.exitCode).toBe(0);
expect(stripAnsi(result.output)).toContain([ expect(stripAnsi(result.output)).toContain([
'[1/2] Passed: 0 Flaky: 0 Failed: 0 Skipped: 0 (XXms)',
'a.spec.ts:6:7 › foobar', 'a.spec.ts:6:7 › foobar',
'one', 'one',
'',
'two', 'two',
'',
'full-line', 'full-line',
'[2/2] Passed: 1 Flaky: 0 Failed: 0 Skipped: 0 (XXms)',
'a.spec.ts:11:7 › one more',
'yay',
'[2/2] Passed: 2 Flaky: 0 Failed: 0 Skipped: 0 (XXms)',
].join('\n')); ].join('\n'));
}); });