Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

fix(test runner): show tests as interrupted when maxFailures stops them (#16178)

Browse source 
Previously, we marked these tests as skipped, and it was sometimes
confusing, because they did actually run and produce some output/artifacts.
This commit is contained in:
Dmitry Gozman 2022-08-03 15:25:25 -07:00 committed by GitHub
parent 744fa8b6e6
commit 03fe75251b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 12 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
packages/playwright-test/src/dispatcher.ts
View file

@ -196,8 +196,6 @@ export class Dispatcher {
const onTestBegin = (params: TestBeginPayload) => { const onTestBegin = (params: TestBeginPayload) => {
const data = this._testById.get(params.testId)!; const data = this._testById.get(params.testId)!;
if (this._hasReachedMaxFailures())
return;
const result = data.test._appendTestResult(); const result = data.test._appendTestResult();
data.resultByWorkerIndex.set(worker.workerIndex, { result, stepStack: new Set(), steps: new Map() }); data.resultByWorkerIndex.set(worker.workerIndex, { result, stepStack: new Set(), steps: new Map() });
result.workerIndex = worker.workerIndex; result.workerIndex = worker.workerIndex;
@ -208,8 +206,12 @@ export class Dispatcher {
const onTestEnd = (params: TestEndPayload) => { const onTestEnd = (params: TestEndPayload) => {
remainingByTestId.delete(params.testId); remainingByTestId.delete(params.testId);
if (this._hasReachedMaxFailures()) if (this._hasReachedMaxFailures()) {
return; // Do not show more than one error to avoid confusion, but report
// as interrupted to indicate that we did actually start the test.
params.status = 'interrupted';
params.errors = [];
}
const data = this._testById.get(params.testId)!; const data = this._testById.get(params.testId)!;
const test = data.test; const test = data.test;
const { result } = data.resultByWorkerIndex.get(worker.workerIndex)!; const { result } = data.resultByWorkerIndex.get(worker.workerIndex)!;

5
tests/playwright-test/max-failures.spec.ts
View file

@ -98,7 +98,7 @@ test('max-failures should stop workers', async ({ runInlineTest }) => {
test('passed short', async () => { test('passed short', async () => {
await new Promise(f => setTimeout(f, 1)); await new Promise(f => setTimeout(f, 1));
}); });
test('interrupted counts as skipped', async () => { test('interrupted reported as interrupted', async () => {
console.log('\\n%%interrupted'); console.log('\\n%%interrupted');
await new Promise(f => setTimeout(f, 5000)); await new Promise(f => setTimeout(f, 5000));
}); });
@ -110,7 +110,8 @@ test('max-failures should stop workers', async ({ runInlineTest }) => {
expect(result.exitCode).toBe(1); expect(result.exitCode).toBe(1);
expect(result.passed).toBe(2); expect(result.passed).toBe(2);
expect(result.failed).toBe(1); expect(result.failed).toBe(1);
expect(result.skipped).toBe(2); expect(result.interrupted).toBe(1);
expect(result.skipped).toBe(1);
expect(result.output).toContain('%%interrupted'); expect(result.output).toContain('%%interrupted');
expect(result.output).not.toContain('%%skipped'); expect(result.output).not.toContain('%%skipped');
}); });

6
tests/playwright-test/playwright.trace.spec.ts
View file

@ -241,7 +241,7 @@ test('should not override trace file in afterAll', async ({ runInlineTest, serve
expect(fs.existsSync(testInfo.outputPath('test-results', 'a-test-1', 'trace-1.zip'))).toBeTruthy(); expect(fs.existsSync(testInfo.outputPath('test-results', 'a-test-1', 'trace-1.zip'))).toBeTruthy();
}); });
test.fixme('should not retain traces for interrupted tests', async ({ runInlineTest }, testInfo) => { test('should retain traces for interrupted tests', async ({ runInlineTest }, testInfo) => {
const result = await runInlineTest({ const result = await runInlineTest({
'playwright.config.ts': ` 'playwright.config.ts': `
module.exports = { use: { trace: 'retain-on-failure' }, maxFailures: 1 }; module.exports = { use: { trace: 'retain-on-failure' }, maxFailures: 1 };
@ -261,9 +261,9 @@ test.fixme('should not retain traces for interrupted tests', async ({ runInlineT
expect(result.exitCode).toBe(1); expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1); expect(result.failed).toBe(1);
expect(result.skipped).toBe(1); expect(result.interrupted).toBe(1);
expect(fs.existsSync(testInfo.outputPath('test-results', 'a-test-1', 'trace.zip'))).toBeTruthy(); expect(fs.existsSync(testInfo.outputPath('test-results', 'a-test-1', 'trace.zip'))).toBeTruthy();
expect(fs.existsSync(testInfo.outputPath('test-results', 'b-test-2', 'trace.zip'))).toBeFalsy(); expect(fs.existsSync(testInfo.outputPath('test-results', 'b-test-2', 'trace.zip'))).toBeTruthy();
}); });
async function parseTrace(file: string): Promise<Map<string, Buffer>> { async function parseTrace(file: string): Promise<Map<string, Buffer>> {