feat(line reporter): show testing progress (#15547)
Line reporter now shows stats in addition to the test name: ``` [chromium] › page/page-click-react.spec.ts:108:1 › should not retarget when element changes on hover [21/93] Passed: 17 Flaky: 0 Failed: 0 Skipped: 4 (7s) ```
This commit is contained in:
Dmitry Gozman
GitHub
parent
7727ebe758
commit
e76d9b3b28
|
|
@ -151,7 +151,7 @@ Running 124 tests using 6 workers
|
|||
|
||||
### Line reporter
|
||||
|
||||
Line reporter is more concise than the list reporter. It uses a single line to report last finished test, and prints failures when they occur. Line reporter is useful for large test suites where it shows the progress but does not spam the output by listing all the tests.
|
||||
Line reporter is more concise than the list reporter. It uses two lines to report currently running test and testing progress, and prints failures when they occur. Line reporter is useful for large test suites where it shows the progress but does not spam the output by listing all the tests.
|
||||
|
||||
```bash
|
||||
npx playwright test --reporter=line
|
||||
|
|
@ -190,7 +190,8 @@ Running 124 tests using 6 workers
|
|||
Expected: 1
|
||||
Received: 0
|
||||
|
||||
[23/124] gitignore.spec.ts - should respect nested .gitignore
|
||||
example.spec.ts:8:3 › should navigate to playwright.dev
|
||||
[23/124] Passed: 20 Flaky: 0 Failed: 0 Skipped: 2 (12s)
|
||||
```
|
||||
|
||||
### Dot reporter
|
||||
|
|
|
|||
|
|
@ -49,13 +49,13 @@ export class BaseReporter implements Reporter {
|
|||
duration = 0;
|
||||
config!: FullConfigInternal;
|
||||
suite!: Suite;
|
||||
totalTestCount = 0;
|
||||
result!: FullResult;
|
||||
private fileDurations = new Map<string, number>();
|
||||
private monotonicStartTime: number = 0;
|
||||
private _omitFailures: boolean;
|
||||
private readonly _ttyWidthForTest: number;
|
||||
readonly liveTerminal: boolean;
|
||||
readonly stats = { started: 0, skipped: 0, completed: 0, total: 0, passed: 0, failed: 0, flaky: 0 };
|
||||
|
||||
constructor(options: { omitFailures?: boolean } = {}) {
|
||||
this._omitFailures = options.omitFailures || false;
|
||||
|
|
@ -67,7 +67,7 @@ export class BaseReporter implements Reporter {
|
|||
this.monotonicStartTime = monotonicTime();
|
||||
this.config = config as FullConfigInternal;
|
||||
this.suite = suite;
|
||||
this.totalTestCount = suite.allTests().length;
|
||||
this.stats.total = suite.allTests().length;
|
||||
}
|
||||
|
||||
onStdOut(chunk: string | Buffer, test?: TestCase, result?: TestResult) {
|
||||
|
|
@ -85,7 +85,21 @@ export class BaseReporter implements Reporter {
|
|||
(result as any)[kOutputSymbol].push(output);
|
||||
}
|
||||
|
||||
onTestBegin(test: TestCase, result: TestResult) {
|
||||
this.stats.started++;
|
||||
}
|
||||
|
||||
onTestEnd(test: TestCase, result: TestResult) {
|
||||
this.stats.completed++;
|
||||
if (!this.willRetry(test)) {
|
||||
switch (test.outcome()) {
|
||||
case 'skipped': this.stats.skipped++; break;
|
||||
case 'expected': this.stats.passed++; break;
|
||||
case 'unexpected': this.stats.failed++; break;
|
||||
case 'flaky': this.stats.flaky++; break;
|
||||
}
|
||||
}
|
||||
|
||||
// Ignore any tests that are run in parallel.
|
||||
for (let suite: Suite | undefined = test.parent; suite; suite = suite.parent) {
|
||||
if ((suite as any)._parallelMode === 'parallel')
|
||||
|
|
@ -119,7 +133,24 @@ export class BaseReporter implements Reporter {
|
|||
protected generateStartingMessage() {
|
||||
const jobs = Math.min(this.config.workers, this.config._testGroupsCount);
|
||||
const shardDetails = this.config.shard ? `, shard ${this.config.shard.current} of ${this.config.shard.total}` : '';
|
||||
return `\nRunning ${this.totalTestCount} test${this.totalTestCount > 1 ? 's' : ''} using ${jobs} worker${jobs > 1 ? 's' : ''}${shardDetails}`;
|
||||
return `\nRunning ${this.stats.total} test${this.stats.total > 1 ? 's' : ''} using ${jobs} worker${jobs > 1 ? 's' : ''}${shardDetails}`;
|
||||
}
|
||||
|
||||
protected generateStatsMessage(mode: 'started' | 'completed', done: boolean) {
|
||||
// Do not report 100% until done.
|
||||
const count = this.stats[mode];
|
||||
const percent = Math.min(done ? 100 : 99, Math.round(count / this.stats.total * 100));
|
||||
const maxExpected = done ? this.stats.total : this.stats.total - (mode === 'started' ? 0 : 1);
|
||||
const retriesSuffix = count > maxExpected ? `+retries` : ``;
|
||||
const message = [
|
||||
`[${count}/${this.stats.total}${retriesSuffix}]`,
|
||||
`${(this.stats.passed ? colors.green : colors.gray)('Passed: ' + this.stats.passed)}`,
|
||||
`${(this.stats.flaky ? colors.red : colors.gray)('Flaky: ' + this.stats.flaky)}`,
|
||||
`${(this.stats.failed ? colors.red : colors.gray)('Failed: ' + this.stats.failed)}`,
|
||||
`${(this.stats.skipped ? colors.yellow : colors.gray)('Skipped: ' + this.stats.skipped)}`,
|
||||
colors.gray(process.env.PW_TEST_DEBUG_REPORTERS ? `(XXms)` : `(${milliseconds((monotonicTime() - this.monotonicStartTime) | 0)})`),
|
||||
].join(' ');
|
||||
return { percent, message };
|
||||
}
|
||||
|
||||
protected getSlowTests(): [string, number][] {
|
||||
|
|
|
|||
|
|
@ -22,7 +22,6 @@ const lineUp = process.env.PW_TEST_DEBUG_REPORTERS ? '<lineup>' : '\u001B[1A';
|
|||
const erase = process.env.PW_TEST_DEBUG_REPORTERS ? '<erase>' : '\u001B[2K';
|
||||
|
||||
class LineReporter extends BaseReporter {
|
||||
private _current = 0;
|
||||
private _failures = 0;
|
||||
private _lastTest: TestCase | undefined;
|
||||
private _lastPercent = -1;
|
||||
|
|
@ -35,27 +34,31 @@ class LineReporter extends BaseReporter {
|
|||
super.onBegin(config, suite);
|
||||
console.log(this.generateStartingMessage());
|
||||
if (this.liveTerminal)
|
||||
console.log();
|
||||
console.log('\n');
|
||||
}
|
||||
|
||||
override onStdOut(chunk: string | Buffer, test?: TestCase, result?: TestResult) {
|
||||
super.onStdOut(chunk, test, result);
|
||||
this._dumpToStdio(test, chunk, process.stdout);
|
||||
this._dumpToStdio(test, result, chunk, process.stdout);
|
||||
}
|
||||
|
||||
override onStdErr(chunk: string | Buffer, test?: TestCase, result?: TestResult) {
|
||||
super.onStdErr(chunk, test, result);
|
||||
this._dumpToStdio(test, chunk, process.stderr);
|
||||
this._dumpToStdio(test, result, chunk, process.stderr);
|
||||
}
|
||||
|
||||
private _dumpToStdio(test: TestCase | undefined, chunk: string | Buffer, stream: NodeJS.WriteStream) {
|
||||
private _retrySuffix(result: TestResult | undefined) {
|
||||
return result?.retry ? colors.yellow(` (retry #${result.retry})`) : '';
|
||||
}
|
||||
|
||||
private _dumpToStdio(test: TestCase | undefined, result: TestResult | undefined, chunk: string | Buffer, stream: NodeJS.WriteStream) {
|
||||
if (this.config.quiet)
|
||||
return;
|
||||
if (this.liveTerminal)
|
||||
stream.write(lineUp + erase);
|
||||
stream.write(lineUp + erase + lineUp + erase);
|
||||
if (test && this._lastTest !== test) {
|
||||
// Write new header for the output.
|
||||
const title = colors.gray(formatTestTitle(this.config, test));
|
||||
const title = colors.gray(formatTestTitle(this.config, test)) + this._retrySuffix(result);
|
||||
stream.write(this.fitToScreen(title) + `\n`);
|
||||
this._lastTest = test;
|
||||
}
|
||||
|
|
@ -64,11 +67,12 @@ class LineReporter extends BaseReporter {
|
|||
if (chunk[chunk.length - 1] !== '\n')
|
||||
console.log();
|
||||
|
||||
console.log();
|
||||
if (this.liveTerminal)
|
||||
console.log('\n');
|
||||
}
|
||||
|
||||
onTestBegin(test: TestCase, result: TestResult) {
|
||||
++this._current;
|
||||
override onTestBegin(test: TestCase, result: TestResult) {
|
||||
super.onTestBegin(test, result);
|
||||
this._updateLine(test, result, undefined);
|
||||
}
|
||||
|
||||
|
|
@ -86,36 +90,34 @@ class LineReporter extends BaseReporter {
|
|||
super.onTestEnd(test, result);
|
||||
if (!this.willRetry(test) && (test.outcome() === 'flaky' || test.outcome() === 'unexpected')) {
|
||||
if (this.liveTerminal)
|
||||
process.stdout.write(lineUp + erase);
|
||||
process.stdout.write(lineUp + erase + lineUp + erase);
|
||||
console.log(formatFailure(this.config, test, {
|
||||
index: ++this._failures
|
||||
}).message);
|
||||
console.log();
|
||||
if (this.liveTerminal)
|
||||
process.stdout.write(this.fitToScreen(this.generateStatsMessage('started', false).message) + '\n');
|
||||
}
|
||||
}
|
||||
|
||||
private _updateLine(test: TestCase, result: TestResult, step?: TestStep) {
|
||||
// Do not report 100% until done.
|
||||
const percent = Math.min(99, Math.round(this._current / this.totalTestCount * 100));
|
||||
const retriesPrefix = this.totalTestCount < this._current ? ` (retries)` : ``;
|
||||
const prefix = `[${this._current}/${this.totalTestCount}]${retriesPrefix} `;
|
||||
const currentRetrySuffix = result.retry ? colors.yellow(` (retry #${result.retry})`) : '';
|
||||
const title = formatTestTitle(this.config, test, step) + currentRetrySuffix;
|
||||
const stats = this.generateStatsMessage('started', false);
|
||||
const title = formatTestTitle(this.config, test, step) + this._retrySuffix(result);
|
||||
if (this.liveTerminal) {
|
||||
process.stdout.write(lineUp + erase + prefix + this.fitToScreen(title, prefix) + '\n');
|
||||
process.stdout.write(lineUp + erase + lineUp + erase + this.fitToScreen(title) + '\n' + this.fitToScreen(stats.message) + '\n');
|
||||
} else {
|
||||
if (percent !== this._lastPercent)
|
||||
process.stdout.write(`[${percent}%] ${title}\n`);
|
||||
if (stats.percent !== this._lastPercent)
|
||||
process.stdout.write(this.fitToScreen(stats.message) + '\n');
|
||||
}
|
||||
this._lastPercent = percent;
|
||||
this._lastPercent = stats.percent;
|
||||
}
|
||||
|
||||
override async onEnd(result: FullResult) {
|
||||
await super.onEnd(result);
|
||||
if (this.liveTerminal)
|
||||
process.stdout.write(lineUp + erase);
|
||||
process.stdout.write(lineUp + erase + lineUp + erase);
|
||||
else
|
||||
process.stdout.write(`[100%]\n`);
|
||||
process.stdout.write(this.fitToScreen(this.generateStatsMessage('started', true).message) + '\n');
|
||||
this.epilogue(false);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -43,7 +43,8 @@ class ListReporter extends BaseReporter {
|
|||
console.log();
|
||||
}
|
||||
|
||||
onTestBegin(test: TestCase, result: TestResult) {
|
||||
override onTestBegin(test: TestCase, result: TestResult) {
|
||||
super.onTestBegin(test, result);
|
||||
if (this.liveTerminal) {
|
||||
if (this._needNewLine) {
|
||||
this._needNewLine = false;
|
||||
|
|
|
|||
|
|
@ -636,7 +636,7 @@ test('should not hang and report results when worker process suddenly exits duri
|
|||
expect(result.passed).toBe(0);
|
||||
expect(result.failed).toBe(1);
|
||||
expect(result.output).toContain('Worker process exited unexpectedly');
|
||||
expect(stripAnsi(result.output)).toContain('[1/1] a.spec.js:6:7 › failing due to afterall');
|
||||
expect(stripAnsi(result.output)).toContain('a.spec.js:6:7 › failing due to afterall');
|
||||
});
|
||||
|
||||
test('unhandled rejection during beforeAll should be reported and prevent more tests', async ({ runInlineTest }) => {
|
||||
|
|
|
|||
|
|
@ -232,6 +232,6 @@ test('should add line in addition to file json without CI', async ({ runInlineTe
|
|||
`,
|
||||
}, { reporter: '' }, { PLAYWRIGHT_LIVE_TERMINAL: '1' });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(stripAnsi(result.output)).toContain('[1/1] a.test.js:6:7 › one');
|
||||
expect(stripAnsi(result.output)).toContain('a.test.js:6:7 › one');
|
||||
expect(fs.existsSync(testInfo.outputPath('a.json'))).toBeTruthy();
|
||||
});
|
||||
|
|
|
|||
|
|
@ -37,12 +37,13 @@ test('should work with tty', async ({ runInlineTest }, testInfo) => {
|
|||
PW_TEST_DEBUG_REPORTERS: '1',
|
||||
});
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(trimLineEnds(result.output)).toContain(trimLineEnds(`Running 4 tests using 1 worker
|
||||
|
||||
<lineup><erase>[1/4] a.test.js:6:12 › skipped test
|
||||
<lineup><erase>[2/4] a.test.js:8:7 › flaky test
|
||||
<lineup><erase>[3/4] a.test.js:8:7 › flaky test (retry #1)
|
||||
<lineup><erase> 1) a.test.js:8:7 › flaky test ====================================================================
|
||||
expect(trimLineEnds(result.output)).toContain(trimLineEnds(`<lineup><erase><lineup><erase>a.test.js:6:12 › skipped test
|
||||
[1/4] Passed: 0 Flaky: 0 Failed: 0 Skipped: 0 (XXms)
|
||||
<lineup><erase><lineup><erase>a.test.js:8:7 › flaky test
|
||||
[2/4] Passed: 0 Flaky: 0 Failed: 0 Skipped: 1 (XXms)
|
||||
<lineup><erase><lineup><erase>a.test.js:8:7 › flaky test (retry #1)
|
||||
[3/4] Passed: 0 Flaky: 0 Failed: 0 Skipped: 1 (XXms)
|
||||
<lineup><erase><lineup><erase> 1) a.test.js:8:7 › flaky test ====================================================================
|
||||
|
||||
Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
|
|
@ -60,10 +61,14 @@ test('should work with tty', async ({ runInlineTest }, testInfo) => {
|
|||
at ${testInfo.outputPath('a.test.js')}:9:32
|
||||
|
||||
|
||||
<lineup><erase>[4/4] a.test.js:11:7 › passing test
|
||||
<lineup><erase>[5/4] (retries) a.test.js:13:7 › failing test
|
||||
<lineup><erase>[6/4] (retries) a.test.js:13:7 › failing test (retry #1)
|
||||
<lineup><erase> 2) a.test.js:13:7 › failing test =================================================================
|
||||
[3/4] Passed: 0 Flaky: 1 Failed: 0 Skipped: 1 (XXms)
|
||||
<lineup><erase><lineup><erase>a.test.js:11:7 › passing test
|
||||
[4/4] Passed: 0 Flaky: 1 Failed: 0 Skipped: 1 (XXms)
|
||||
<lineup><erase><lineup><erase>a.test.js:13:7 › failing test
|
||||
[5/4+retries] Passed: 1 Flaky: 1 Failed: 0 Skipped: 1 (XXms)
|
||||
<lineup><erase><lineup><erase>a.test.js:13:7 › failing test (retry #1)
|
||||
[6/4+retries] Passed: 1 Flaky: 1 Failed: 0 Skipped: 1 (XXms)
|
||||
<lineup><erase><lineup><erase> 2) a.test.js:13:7 › failing test =================================================================
|
||||
|
||||
Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
|
|
@ -96,7 +101,8 @@ test('should work with tty', async ({ runInlineTest }, testInfo) => {
|
|||
at ${testInfo.outputPath('a.test.js')}:14:19
|
||||
|
||||
|
||||
<lineup><erase>
|
||||
[6/4+retries] Passed: 1 Flaky: 1 Failed: 1 Skipped: 1 (XXms)
|
||||
<lineup><erase><lineup><erase>
|
||||
1 failed
|
||||
a.test.js:13:7 › failing test ==================================================================
|
||||
1 flaky
|
||||
|
|
@ -126,9 +132,9 @@ test('should work with non-tty', async ({ runInlineTest }, testInfo) => {
|
|||
});
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(trimLineEnds(result.output)).toContain(trimLineEnds(`Running 4 tests using 1 worker
|
||||
[25%] a.test.js:6:12 › skipped test
|
||||
[50%] a.test.js:8:7 › flaky test
|
||||
[75%] a.test.js:8:7 › flaky test (retry #1)
|
||||
[1/4] Passed: 0 Flaky: 0 Failed: 0 Skipped: 0 (XXms)
|
||||
[2/4] Passed: 0 Flaky: 0 Failed: 0 Skipped: 1 (XXms)
|
||||
[3/4] Passed: 0 Flaky: 0 Failed: 0 Skipped: 1 (XXms)
|
||||
1) a.test.js:8:7 › flaky test ====================================================================
|
||||
|
||||
Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
|
@ -147,7 +153,7 @@ test('should work with non-tty', async ({ runInlineTest }, testInfo) => {
|
|||
at ${testInfo.outputPath('a.test.js')}:9:32
|
||||
|
||||
|
||||
[99%] a.test.js:11:7 › passing test
|
||||
[4/4] Passed: 0 Flaky: 1 Failed: 0 Skipped: 1 (XXms)
|
||||
2) a.test.js:13:7 › failing test =================================================================
|
||||
|
||||
Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
|
@ -181,7 +187,7 @@ test('should work with non-tty', async ({ runInlineTest }, testInfo) => {
|
|||
at ${testInfo.outputPath('a.test.js')}:14:19
|
||||
|
||||
|
||||
[100%]
|
||||
[6/4+retries] Passed: 1 Flaky: 1 Failed: 1 Skipped: 1 (XXms)
|
||||
|
||||
1 failed
|
||||
a.test.js:13:7 › failing test ==================================================================
|
||||
|
|
@ -204,10 +210,10 @@ test('should spare status updates in non-tty mode', async ({ runInlineTest }) =>
|
|||
PW_TEST_DEBUG_REPORTERS: '1',
|
||||
});
|
||||
expect(result.exitCode).toBe(0);
|
||||
const lines = [`Running 300 tests using 1 worker`, `[0%] a.test.js:7:9 › test0`];
|
||||
for (let i = 1; i <= 99; i++)
|
||||
lines.push(`[${i}%] a.test.js:7:9 › test${3 * i - 2}`);
|
||||
lines.push('[100%]');
|
||||
const lines = [`Running 300 tests using 1 worker`, `[1/300] Passed: 0 Flaky: 0 Failed: 0 Skipped: 0 (XXms)`];
|
||||
for (let i = 0; i < 99; i++)
|
||||
lines.push(`[${3 * i + 2}/300] Passed: ${3 * i + 1} Flaky: 0 Failed: 0 Skipped: 0 (XXms)`);
|
||||
lines.push('[300/300] Passed: 300 Flaky: 0 Failed: 0 Skipped: 0 (XXms)');
|
||||
lines.push('');
|
||||
lines.push(' 300 passed');
|
||||
expect(trimLineEnds(result.output)).toContain(lines.join('\n'));
|
||||
|
|
@ -222,15 +228,21 @@ test('should print output', async ({ runInlineTest }) => {
|
|||
process.stdout.write('two');
|
||||
console.log('full-line');
|
||||
});
|
||||
test('one more', async ({}, testInfo) => {
|
||||
console.log('yay');
|
||||
});
|
||||
`
|
||||
}, { reporter: 'line' });
|
||||
}, { reporter: 'line' }, { PW_TEST_DEBUG_REPORTERS: '1' });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(stripAnsi(result.output)).toContain([
|
||||
'[1/2] Passed: 0 Flaky: 0 Failed: 0 Skipped: 0 (XXms)',
|
||||
'a.spec.ts:6:7 › foobar',
|
||||
'one',
|
||||
'',
|
||||
'two',
|
||||
'',
|
||||
'full-line',
|
||||
'[2/2] Passed: 1 Flaky: 0 Failed: 0 Skipped: 0 (XXms)',
|
||||
'a.spec.ts:11:7 › one more',
|
||||
'yay',
|
||||
'[2/2] Passed: 2 Flaky: 0 Failed: 0 Skipped: 0 (XXms)',
|
||||
].join('\n'));
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in a new issue