Revert "feat(line reporter): show stats, handle tty" (#12735)

This reverts commit be817d1a53, PR #12695.
Reason: found issues with stdout messing with stats.
This commit is contained in:
Dmitry Gozman 2022-03-14 10:37:43 -07:00 committed by GitHub
parent 9af88b6425
commit 237954212c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 91 additions and 298 deletions

View file

@ -19,7 +19,6 @@ import colors from 'colors/safe';
import fs from 'fs'; import fs from 'fs';
import milliseconds from 'ms'; import milliseconds from 'ms';
import path from 'path'; import path from 'path';
import { getAsBooleanFromENV } from 'playwright-core/lib/utils/utils';
import StackUtils from 'stack-utils'; import StackUtils from 'stack-utils';
import { FullConfig, TestCase, Suite, TestResult, TestError, Reporter, FullResult, TestStep, Location } from '../../types/testReporter'; import { FullConfig, TestCase, Suite, TestResult, TestError, Reporter, FullResult, TestStep, Location } from '../../types/testReporter';
@ -52,25 +51,23 @@ export class BaseReporter implements Reporter {
duration = 0; duration = 0;
config!: FullConfig; config!: FullConfig;
suite!: Suite; suite!: Suite;
totalTestCount = 0;
result!: FullResult; result!: FullResult;
private fileDurations = new Map<string, number>(); private fileDurations = new Map<string, number>();
private monotonicStartTime: number = 0; private monotonicStartTime: number = 0;
private _omitFailures: boolean; private _omitFailures: boolean;
private readonly _ttyWidthForTest: number; private readonly _ttyWidthForTest: number;
readonly liveTerminal: boolean;
readonly stats = { skipped: 0, complete: 0, total: 0, passed: 0, failed: 0, flaky: 0 };
constructor(options: { omitFailures?: boolean } = {}) { constructor(options: { omitFailures?: boolean } = {}) {
this._omitFailures = options.omitFailures || false; this._omitFailures = options.omitFailures || false;
this._ttyWidthForTest = parseInt(process.env.PWTEST_TTY_WIDTH || '', 10); this._ttyWidthForTest = parseInt(process.env.PWTEST_TTY_WIDTH || '', 10);
this.liveTerminal = process.stdout.isTTY || getAsBooleanFromENV('PLAYWRIGHT_LIVE_TERMINAL');
} }
onBegin(config: FullConfig, suite: Suite) { onBegin(config: FullConfig, suite: Suite) {
this.monotonicStartTime = monotonicTime(); this.monotonicStartTime = monotonicTime();
this.config = config; this.config = config;
this.suite = suite; this.suite = suite;
this.stats.total = suite.allTests().length; this.totalTestCount = suite.allTests().length;
} }
onStdOut(chunk: string | Buffer, test?: TestCase, result?: TestResult) { onStdOut(chunk: string | Buffer, test?: TestCase, result?: TestResult) {
@ -89,16 +86,6 @@ export class BaseReporter implements Reporter {
} }
onTestEnd(test: TestCase, result: TestResult) { onTestEnd(test: TestCase, result: TestResult) {
this.stats.complete++;
if (!this.willRetry(test)) {
switch (test.outcome()) {
case 'skipped': this.stats.skipped++; break;
case 'expected': this.stats.passed++; break;
case 'unexpected': this.stats.failed++; break;
case 'flaky': this.stats.flaky++; break;
}
}
// Ignore any tests that are run in parallel. // Ignore any tests that are run in parallel.
for (let suite: Suite | undefined = test.parent; suite; suite = suite.parent) { for (let suite: Suite | undefined = test.parent; suite; suite = suite.parent) {
if ((suite as any)._parallelMode === 'parallel') if ((suite as any)._parallelMode === 'parallel')
@ -132,23 +119,7 @@ export class BaseReporter implements Reporter {
protected generateStartingMessage() { protected generateStartingMessage() {
const jobs = Math.min(this.config.workers, (this.config as any).__testGroupsCount); const jobs = Math.min(this.config.workers, (this.config as any).__testGroupsCount);
const shardDetails = this.config.shard ? `, shard ${this.config.shard.current} of ${this.config.shard.total}` : ''; const shardDetails = this.config.shard ? `, shard ${this.config.shard.current} of ${this.config.shard.total}` : '';
return `\nRunning ${this.stats.total} test${this.stats.total > 1 ? 's' : ''} using ${jobs} worker${jobs > 1 ? 's' : ''}${shardDetails}`; return `\nRunning ${this.totalTestCount} test${this.totalTestCount > 1 ? 's' : ''} using ${jobs} worker${jobs > 1 ? 's' : ''}${shardDetails}`;
}
protected generateStatsMessage(done: boolean) {
// Do not report 100% until done.
const percent = Math.min(done ? 100 : 99, Math.round(this.stats.complete / this.stats.total * 100));
const maxExpected = done ? this.stats.total : this.stats.total - 1;
const retriesSuffix = this.stats.complete > maxExpected ? ` (retries)` : ``;
const message = [
`${percent}% [${this.stats.complete}/${this.stats.total}]${retriesSuffix}`,
`${(this.stats.passed ? colors.green : colors.gray)('Passed: ' + this.stats.passed)}`,
`${(this.stats.flaky ? colors.red : colors.gray)('Flaky: ' + this.stats.flaky)}`,
`${(this.stats.failed ? colors.red : colors.gray)('Failed: ' + this.stats.failed)}`,
`${(this.stats.skipped ? colors.yellow : colors.gray)('Skipped: ' + this.stats.skipped)}`,
colors.gray(process.env.PW_TEST_DEBUG_REPORTERS ? `(XXms)` : `(${milliseconds(monotonicTime() - this.monotonicStartTime)})`),
].join(' ');
return { percent, message };
} }
protected getSlowTests(): [string, number][] { protected getSlowTests(): [string, number][] {

View file

@ -18,13 +18,10 @@ import colors from 'colors/safe';
import { BaseReporter, formatFailure, formatTestTitle } from './base'; import { BaseReporter, formatFailure, formatTestTitle } from './base';
import { FullConfig, TestCase, Suite, TestResult, FullResult } from '../../types/testReporter'; import { FullConfig, TestCase, Suite, TestResult, FullResult } from '../../types/testReporter';
const lineUp = process.env.PW_TEST_DEBUG_REPORTERS ? '<lineup>' : '\u001B[1A';
const erase = process.env.PW_TEST_DEBUG_REPORTERS ? '<erase>' : '\u001B[2K';
class LineReporter extends BaseReporter { class LineReporter extends BaseReporter {
private _current = 0;
private _failures = 0; private _failures = 0;
private _lastTest: TestCase | undefined; private _lastTest: TestCase | undefined;
private _lastPercent = -1;
printsToStdio() { printsToStdio() {
return true; return true;
@ -33,55 +30,49 @@ class LineReporter extends BaseReporter {
override onBegin(config: FullConfig, suite: Suite) { override onBegin(config: FullConfig, suite: Suite) {
super.onBegin(config, suite); super.onBegin(config, suite);
console.log(this.generateStartingMessage()); console.log(this.generateStartingMessage());
if (this.liveTerminal) console.log();
console.log('\n');
} }
override onStdOut(chunk: string | Buffer, test?: TestCase, result?: TestResult) { override onStdOut(chunk: string | Buffer, test?: TestCase, result?: TestResult) {
super.onStdOut(chunk, test, result); super.onStdOut(chunk, test, result);
this._dumpToStdio(test, chunk, result, process.stdout); this._dumpToStdio(test, chunk, process.stdout);
} }
override onStdErr(chunk: string | Buffer, test?: TestCase, result?: TestResult) { override onStdErr(chunk: string | Buffer, test?: TestCase, result?: TestResult) {
super.onStdErr(chunk, test, result); super.onStdErr(chunk, test, result);
this._dumpToStdio(test, chunk, result, process.stderr); this._dumpToStdio(test, chunk, process.stderr);
} }
private _testTitleLine(test: TestCase, result: TestResult | undefined) { private _dumpToStdio(test: TestCase | undefined, chunk: string | Buffer, stream: NodeJS.WriteStream) {
const title = formatTestTitle(this.config, test);
const titleSuffix = result?.retry ? ` (retry #${result.retry})` : '';
return this.fitToScreen(title, titleSuffix) + colors.yellow(titleSuffix);
}
private _dumpToStdio(test: TestCase | undefined, chunk: string | Buffer, result: TestResult | undefined, stream: NodeJS.WriteStream) {
if (this.config.quiet) if (this.config.quiet)
return; return;
if (this.liveTerminal) if (!process.env.PW_TEST_DEBUG_REPORTERS)
stream.write(lineUp + erase + lineUp + erase); stream.write(`\u001B[1A\u001B[2K`);
if (test && this._lastTest !== test) { if (test && this._lastTest !== test) {
// Write new header for the output. // Write new header for the output.
stream.write(this._testTitleLine(test, result) + `\n`); const title = colors.gray(formatTestTitle(this.config, test));
stream.write(this.fitToScreen(title) + `\n`);
this._lastTest = test; this._lastTest = test;
} }
stream.write(chunk); stream.write(chunk);
console.log('\n'); console.log();
} }
override onTestEnd(test: TestCase, result: TestResult) { override onTestEnd(test: TestCase, result: TestResult) {
super.onTestEnd(test, result); super.onTestEnd(test, result);
const stats = this.generateStatsMessage(false); ++this._current;
if (this.liveTerminal) { const retriesSuffix = this.totalTestCount < this._current ? ` (retries)` : ``;
process.stdout.write(lineUp + erase + lineUp + erase + `${this._testTitleLine(test, result)}\n${this.fitToScreen(stats.message)}\n`); const title = `[${this._current}/${this.totalTestCount}]${retriesSuffix} ${formatTestTitle(this.config, test)}`;
} else { const suffix = result.retry ? ` (retry #${result.retry})` : '';
if (stats.percent !== this._lastPercent) if (process.env.PW_TEST_DEBUG_REPORTERS)
process.stdout.write(this.fitToScreen(stats.message) + '\n'); process.stdout.write(`${title + suffix}\n`);
} else
this._lastPercent = stats.percent; process.stdout.write(`\u001B[1A\u001B[2K${this.fitToScreen(title, suffix) + colors.yellow(suffix)}\n`);
if (!this.willRetry(test) && (test.outcome() === 'flaky' || test.outcome() === 'unexpected')) { if (!this.willRetry(test) && (test.outcome() === 'flaky' || test.outcome() === 'unexpected')) {
if (this.liveTerminal) if (!process.env.PW_TEST_DEBUG_REPORTERS)
process.stdout.write(lineUp + erase + lineUp + erase); process.stdout.write(`\u001B[1A\u001B[2K`);
console.log(formatFailure(this.config, test, { console.log(formatFailure(this.config, test, {
index: ++this._failures index: ++this._failures
}).message); }).message);
@ -90,9 +81,9 @@ class LineReporter extends BaseReporter {
} }
override async onEnd(result: FullResult) { override async onEnd(result: FullResult) {
if (!process.env.PW_TEST_DEBUG_REPORTERS)
process.stdout.write(`\u001B[1A\u001B[2K`);
await super.onEnd(result); await super.onEnd(result);
if (this.liveTerminal)
process.stdout.write(lineUp + erase + lineUp + erase);
this.epilogue(false); this.epilogue(false);
} }
} }

View file

@ -29,9 +29,11 @@ class ListReporter extends BaseReporter {
private _lastRow = 0; private _lastRow = 0;
private _testRows = new Map<TestCase, number>(); private _testRows = new Map<TestCase, number>();
private _needNewLine = false; private _needNewLine = false;
private readonly _liveTerminal: string | boolean | undefined;
constructor(options: { omitFailures?: boolean } = {}) { constructor(options: { omitFailures?: boolean } = {}) {
super(options); super(options);
this._liveTerminal = process.stdout.isTTY || !!process.env.PWTEST_TTY_WIDTH;
} }
printsToStdio() { printsToStdio() {
@ -45,7 +47,7 @@ class ListReporter extends BaseReporter {
} }
onTestBegin(test: TestCase, result: TestResult) { onTestBegin(test: TestCase, result: TestResult) {
if (this.liveTerminal) { if (this._liveTerminal) {
if (this._needNewLine) { if (this._needNewLine) {
this._needNewLine = false; this._needNewLine = false;
process.stdout.write('\n'); process.stdout.write('\n');
@ -69,7 +71,7 @@ class ListReporter extends BaseReporter {
} }
onStepBegin(test: TestCase, result: TestResult, step: TestStep) { onStepBegin(test: TestCase, result: TestResult, step: TestStep) {
if (!this.liveTerminal) if (!this._liveTerminal)
return; return;
if (step.category !== 'test.step') if (step.category !== 'test.step')
return; return;
@ -77,7 +79,7 @@ class ListReporter extends BaseReporter {
} }
onStepEnd(test: TestCase, result: TestResult, step: TestStep) { onStepEnd(test: TestCase, result: TestResult, step: TestStep) {
if (!this.liveTerminal) if (!this._liveTerminal)
return; return;
if (step.category !== 'test.step') if (step.category !== 'test.step')
return; return;
@ -89,7 +91,7 @@ class ListReporter extends BaseReporter {
return; return;
const text = chunk.toString('utf-8'); const text = chunk.toString('utf-8');
this._needNewLine = text[text.length - 1] !== '\n'; this._needNewLine = text[text.length - 1] !== '\n';
if (this.liveTerminal) { if (this._liveTerminal) {
const newLineCount = text.split('\n').length - 1; const newLineCount = text.split('\n').length - 1;
this._lastRow += newLineCount; this._lastRow += newLineCount;
} }
@ -114,7 +116,7 @@ class ListReporter extends BaseReporter {
} }
const suffix = this._retrySuffix(result) + duration; const suffix = this._retrySuffix(result) + duration;
if (this.liveTerminal) { if (this._liveTerminal) {
this._updateTestLine(test, text, suffix); this._updateTestLine(test, text, suffix);
} else { } else {
if (this._needNewLine) { if (this._needNewLine) {

View file

@ -631,12 +631,12 @@ test('should not hang and report results when worker process suddenly exits duri
test('failing due to afterall', () => {}); test('failing due to afterall', () => {});
test.afterAll(() => { process.exit(0); }); test.afterAll(() => { process.exit(0); });
` `
}, { reporter: 'line' }, { PLAYWRIGHT_LIVE_TERMINAL: '1' }); }, { reporter: 'line' });
expect(result.exitCode).toBe(1); expect(result.exitCode).toBe(1);
expect(result.passed).toBe(0); expect(result.passed).toBe(0);
expect(result.failed).toBe(1); expect(result.failed).toBe(1);
expect(result.output).toContain('Worker process exited unexpectedly'); expect(result.output).toContain('Worker process exited unexpectedly');
expect(stripAnsi(result.output)).toContain('a.spec.js:6:7 failing due to afterall'); expect(stripAnsi(result.output)).toContain('[1/1] a.spec.js:6:7 failing due to afterall');
}); });
test('unhandled rejection during beforeAll should be reported and prevent more tests', async ({ runInlineTest }) => { test('unhandled rejection during beforeAll should be reported and prevent more tests', async ({ runInlineTest }) => {

View file

@ -255,10 +255,6 @@ export function stripAnsi(str: string): string {
return str.replace(asciiRegex, ''); return str.replace(asciiRegex, '');
} }
export function trimLineEnds(text: string): string {
return text.split('\n').map(line => line.trimEnd()).join('\n');
}
export function countTimes(s: string, sub: string): number { export function countTimes(s: string, sub: string): number {
let result = 0; let result = 0;
for (let index = 0; index !== -1;) { for (let index = 0; index !== -1;) {

View file

@ -230,8 +230,8 @@ test('should add line in addition to file json without CI', async ({ runInlineTe
expect(1).toBe(1); expect(1).toBe(1);
}); });
`, `,
}, { reporter: '' }, { PLAYWRIGHT_LIVE_TERMINAL: '1' }); }, { reporter: '' }, { PW_TEST_DEBUG_REPORTERS: '1' });
expect(result.exitCode).toBe(0); expect(result.exitCode).toBe(0);
expect(stripAnsi(result.output)).toContain('a.test.js:6:7 one'); expect(stripAnsi(result.output)).toContain('[1/1] a.test.js:6:7 one');
expect(fs.existsSync(testInfo.outputPath('a.json'))).toBeTruthy(); expect(fs.existsSync(testInfo.outputPath('a.json'))).toBeTruthy();
}); });

View file

@ -14,238 +14,71 @@
* limitations under the License. * limitations under the License.
*/ */
import { test, expect, trimLineEnds, stripAnsi } from './playwright-test-fixtures'; import { test, expect, stripAnsi } from './playwright-test-fixtures';
test('should work with tty', async ({ runInlineTest }, testInfo) => { test('render unexpected after retry', async ({ runInlineTest }) => {
const result = await runInlineTest({ const result = await runInlineTest({
'a.test.js': ` 'a.test.js': `
const { test } = pwt; const { test } = pwt;
test('passing test', async ({}) => { test('one', async ({}) => {
});
test.skip('skipped test', async ({}) => {
});
test('flaky test', async ({}, testInfo) => {
expect(testInfo.retry).toBe(1);
});
test('failing test', async ({}) => {
expect(1).toBe(0); expect(1).toBe(0);
}); });
`, `,
}, { retries: '1', reporter: 'line', workers: '1' }, { }, { retries: 3, reporter: 'line' });
PLAYWRIGHT_LIVE_TERMINAL: '1',
FORCE_COLOR: '0',
PW_TEST_DEBUG_REPORTERS: '1',
});
expect(result.exitCode).toBe(1);
expect(trimLineEnds(result.output)).toContain(trimLineEnds(`Running 4 tests using 1 worker
<lineup><erase><lineup><erase>a.test.js:6:7 passing test
25% [1/4] Passed: 1 Flaky: 0 Failed: 0 Skipped: 0 (XXms)
<lineup><erase><lineup><erase>a.test.js:8:12 skipped test
50% [2/4] Passed: 1 Flaky: 0 Failed: 0 Skipped: 1 (XXms)
<lineup><erase><lineup><erase>a.test.js:10:7 flaky test
75% [3/4] Passed: 1 Flaky: 0 Failed: 0 Skipped: 1 (XXms)
<lineup><erase><lineup><erase>a.test.js:10:7 flaky test (retry #1)
99% [4/4] (retries) Passed: 1 Flaky: 1 Failed: 0 Skipped: 1 (XXms)
<lineup><erase><lineup><erase> 1) a.test.js:10:7 flaky test ===================================================================
Error: expect(received).toBe(expected) // Object.is equality
Expected: 1
Received: 0
9 | });
10 | test('flaky test', async ({}, testInfo) => {
> 11 | expect(testInfo.retry).toBe(1);
| ^
12 | });
13 | test('failing test', async ({}) => {
14 | expect(1).toBe(0);
at ${testInfo.outputPath('a.test.js')}:11:32
<lineup><erase><lineup><erase>a.test.js:13:7 failing test
99% [5/4] (retries) Passed: 1 Flaky: 1 Failed: 0 Skipped: 1 (XXms)
<lineup><erase><lineup><erase>a.test.js:13:7 failing test (retry #1)
99% [6/4] (retries) Passed: 1 Flaky: 1 Failed: 1 Skipped: 1 (XXms)
<lineup><erase><lineup><erase> 2) a.test.js:13:7 failing test =================================================================
Error: expect(received).toBe(expected) // Object.is equality
Expected: 0
Received: 1
12 | });
13 | test('failing test', async ({}) => {
> 14 | expect(1).toBe(0);
| ^
15 | });
16 |
at ${testInfo.outputPath('a.test.js')}:14:19
Retry #1 ---------------------------------------------------------------------------------------
Error: expect(received).toBe(expected) // Object.is equality
Expected: 0
Received: 1
12 | });
13 | test('failing test', async ({}) => {
> 14 | expect(1).toBe(0);
| ^
15 | });
16 |
at ${testInfo.outputPath('a.test.js')}:14:19
<lineup><erase><lineup><erase>
1 failed
a.test.js:13:7 failing test ==================================================================
1 flaky
a.test.js:10:7 flaky test ====================================================================
1 skipped
1 passed`));
});
test('should work with non-tty', async ({ runInlineTest }, testInfo) => {
const result = await runInlineTest({
'a.test.js': `
const { test } = pwt;
test('passing test', async ({}) => {
});
test.skip('skipped test', async ({}) => {
});
test('flaky test', async ({}, testInfo) => {
expect(testInfo.retry).toBe(1);
});
test('failing test', async ({}) => {
expect(1).toBe(0);
});
`,
}, { retries: '1', reporter: 'line', workers: '1' }, {
FORCE_COLOR: '0',
PW_TEST_DEBUG_REPORTERS: '1',
});
expect(result.exitCode).toBe(1);
expect(trimLineEnds(result.output)).toContain(trimLineEnds(`
Running 4 tests using 1 worker
25% [1/4] Passed: 1 Flaky: 0 Failed: 0 Skipped: 0 (XXms)
50% [2/4] Passed: 1 Flaky: 0 Failed: 0 Skipped: 1 (XXms)
75% [3/4] Passed: 1 Flaky: 0 Failed: 0 Skipped: 1 (XXms)
99% [4/4] (retries) Passed: 1 Flaky: 1 Failed: 0 Skipped: 1 (XXms)
1) a.test.js:10:7 flaky test ===================================================================
Error: expect(received).toBe(expected) // Object.is equality
Expected: 1
Received: 0
9 | });
10 | test('flaky test', async ({}, testInfo) => {
> 11 | expect(testInfo.retry).toBe(1);
| ^
12 | });
13 | test('failing test', async ({}) => {
14 | expect(1).toBe(0);
at ${testInfo.outputPath('a.test.js')}:11:32
2) a.test.js:13:7 failing test =================================================================
Error: expect(received).toBe(expected) // Object.is equality
Expected: 0
Received: 1
12 | });
13 | test('failing test', async ({}) => {
> 14 | expect(1).toBe(0);
| ^
15 | });
16 |
at ${testInfo.outputPath('a.test.js')}:14:19
Retry #1 ---------------------------------------------------------------------------------------
Error: expect(received).toBe(expected) // Object.is equality
Expected: 0
Received: 1
12 | });
13 | test('failing test', async ({}) => {
> 14 | expect(1).toBe(0);
| ^
15 | });
16 |
at ${testInfo.outputPath('a.test.js')}:14:19
1 failed
a.test.js:13:7 failing test ==================================================================
1 flaky
a.test.js:10:7 flaky test ====================================================================
1 skipped
1 passed`));
});
test('should respect tty width', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
const { test } = pwt;
test('passing test', async ({}) => {
});
test.skip('skipped test', async ({}) => {
});
test('flaky test', async ({}, testInfo) => {
expect(testInfo.retry).toBe(1);
});
test('failing test', async ({}) => {
expect(1).toBe(0);
});
`,
}, { retries: '1', reporter: 'line', workers: '1' }, {
PLAYWRIGHT_LIVE_TERMINAL: '1',
FORCE_COLOR: '0',
PWTEST_TTY_WIDTH: '30',
PW_TEST_DEBUG_REPORTERS: '1',
});
expect(result.exitCode).toBe(1);
const text = stripAnsi(result.output); const text = stripAnsi(result.output);
expect(text).toContain(`a.test.js:6:7 passing test`); expect(text).toContain('[1/1] a.test.js:6:7 one');
expect(text).toContain(`25% [1/4] Passed: 1 Flaky: 0 F`); expect(text).toContain('[2/1] (retries) a.test.js:6:7 one (retry #1)');
expect(text).not.toContain(`25% [1/4] Passed: 1 Flaky: 0 Fa`); expect(text).toContain('[3/1] (retries) a.test.js:6:7 one (retry #2)');
expect(text).toContain(`a.test.js:10:7 fl (retry #1)`); expect(text).toContain('[4/1] (retries) a.test.js:6:7 one (retry #3)');
expect(text).toContain(`99% [4/4] (retries) Passed: 1 `); expect(text).toContain('1 failed');
expect(text).not.toContain(`99% [4/4] (retries) Passed: 1 F`); expect(text).toContain('1) a.test');
expect(text).not.toContain('2) a.test');
expect(text).toContain('Retry #1 ----');
expect(text).toContain('Retry #2 ----');
expect(text).toContain('Retry #3 ----');
expect(result.exitCode).toBe(1);
}); });
test('should spare status updates in non-tty mode', async ({ runInlineTest }) => { test('render flaky', async ({ runInlineTest }) => {
const result = await runInlineTest({ const result = await runInlineTest({
'a.test.js': ` 'a.test.js': `
const { test } = pwt; const { test } = pwt;
for (let i = 0; i < 300; i++) { test('one', async ({}, testInfo) => {
test('test' + i, () => {}); expect(testInfo.retry).toBe(3);
} });
`, `,
}, { reporter: 'line', workers: '1' }, { }, { retries: 3, reporter: 'line' });
FORCE_COLOR: '0', const text = stripAnsi(result.output);
PW_TEST_DEBUG_REPORTERS: '1', expect(text).toContain('1 flaky');
});
expect(result.exitCode).toBe(0); expect(result.exitCode).toBe(0);
const lines = [`Running 300 tests using 1 worker`, `0% [1/300] Passed: 1 Flaky: 0 Failed: 0 Skipped: 0 (XXms)`]; });
for (let i = 1; i <= 99; i++)
lines.push(`${i}% [${3 * i - 1}/300] Passed: ${3 * i - 1} Flaky: 0 Failed: 0 Skipped: 0 (XXms)`); test('should print flaky failures', async ({ runInlineTest }) => {
lines.push(''); const result = await runInlineTest({
lines.push(' 300 passed'); 'a.spec.ts': `
expect(trimLineEnds(result.output)).toContain(lines.join('\n')); const { test } = pwt;
test('foobar', async ({}, testInfo) => {
expect(testInfo.retry).toBe(1);
});
`
}, { retries: '1', reporter: 'line' });
expect(result.exitCode).toBe(0);
expect(result.flaky).toBe(1);
expect(stripAnsi(result.output)).toContain('expect(testInfo.retry).toBe(1)');
});
test('should work on CI', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
const { test } = pwt;
test('one', async ({}) => {
expect(1).toBe(0);
});
`,
}, { reporter: 'line' }, { CI: '1' });
const text = stripAnsi(result.output);
expect(text).toContain('[1/1] a.test.js:6:7 one');
expect(text).toContain('1 failed');
expect(text).toContain('1) a.test');
expect(result.exitCode).toBe(1);
}); });

View file

@ -63,7 +63,7 @@ test('render steps', async ({ runInlineTest }) => {
}); });
}); });
`, `,
}, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PLAYWRIGHT_LIVE_TERMINAL: '1' }); }, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PWTEST_TTY_WIDTH: '80' });
const text = stripAnsi(result.output); const text = stripAnsi(result.output);
const lines = text.split('\n').filter(l => l.startsWith('0 :')); const lines = text.split('\n').filter(l => l.startsWith('0 :'));
lines.pop(); // Remove last item that contains [v] and time in ms. lines.pop(); // Remove last item that contains [v] and time in ms.
@ -91,7 +91,7 @@ test('render retries', async ({ runInlineTest }) => {
expect(testInfo.retry).toBe(1); expect(testInfo.retry).toBe(1);
}); });
`, `,
}, { reporter: 'list', retries: '1' }, { PW_TEST_DEBUG_REPORTERS: '1', PLAYWRIGHT_LIVE_TERMINAL: '1' }); }, { reporter: 'list', retries: '1' }, { PW_TEST_DEBUG_REPORTERS: '1', PWTEST_TTY_WIDTH: '80' });
const text = stripAnsi(result.output); const text = stripAnsi(result.output);
const lines = text.split('\n').filter(l => l.startsWith('0 :') || l.startsWith('1 :')).map(l => l.replace(/[\dm]+s/, 'XXms')); const lines = text.split('\n').filter(l => l.startsWith('0 :') || l.startsWith('1 :')).map(l => l.replace(/[\dm]+s/, 'XXms'));
const positiveStatusMarkPrefix = process.platform === 'win32' ? 'ok' : '✓ '; const positiveStatusMarkPrefix = process.platform === 'win32' ? 'ok' : '✓ ';
@ -121,7 +121,7 @@ test('should truncate long test names', async ({ runInlineTest }) => {
test.skip('skipped very long name', async () => { test.skip('skipped very long name', async () => {
}); });
`, `,
}, { reporter: 'list', retries: 0 }, { PLAYWRIGHT_LIVE_TERMINAL: '1', PWTEST_TTY_WIDTH: 50 }); }, { reporter: 'list', retries: 0 }, { PWTEST_TTY_WIDTH: 50 });
const text = stripAnsi(result.output); const text = stripAnsi(result.output);
const positiveStatusMarkPrefix = process.platform === 'win32' ? 'ok' : '✓ '; const positiveStatusMarkPrefix = process.platform === 'win32' ? 'ok' : '✓ ';
const negativateStatusMarkPrefix = process.platform === 'win32' ? 'x ' : '✘ '; const negativateStatusMarkPrefix = process.platform === 'win32' ? 'x ' : '✘ ';