fix(playwright): test unexpected result if failed and skipped
Checks if a test results are only failed and skipped. If so, it sets the result as unexpected instead of flaky, since they never passed. Fixes #28322
This commit is contained in:
parent
5d810f1e07
commit
49b293002c
|
|
@ -290,10 +290,14 @@ export class TestCase extends Base implements reporterTypes.TestCase {
|
||||||
return 'skipped';
|
return 'skipped';
|
||||||
|
|
||||||
const failures = results.filter(result => result.status !== 'skipped' && result.status !== 'interrupted' && result.status !== this.expectedStatus);
|
const failures = results.filter(result => result.status !== 'skipped' && result.status !== 'interrupted' && result.status !== this.expectedStatus);
|
||||||
|
const skipped = results.filter(result => result.status === 'skipped');
|
||||||
|
const passed = results.filter(result => result.status === 'passed');
|
||||||
if (!failures.length) // all passed
|
if (!failures.length) // all passed
|
||||||
return 'expected';
|
return 'expected';
|
||||||
if (failures.length === results.length) // all failed
|
if (failures.length === results.length) // all failed
|
||||||
return 'unexpected';
|
return 'unexpected';
|
||||||
|
if (failures.length && skipped.length && !passed.length) // some failed, none succedded and the rest where skipped
|
||||||
|
return 'unexpected';
|
||||||
return 'flaky'; // mixed bag
|
return 'flaky'; // mixed bag
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -140,6 +140,40 @@ test('test.describe.serial should work with retry', async ({ runInlineTest }) =>
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('test.describe.serial should work with retry when it fails in different places', async ({ runInlineTest }) => {
|
||||||
|
const result = await runInlineTest({
|
||||||
|
'a.test.ts': `
|
||||||
|
import { test, expect } from '@playwright/test';
|
||||||
|
test.describe.serial('serial suite', () => {
|
||||||
|
test('test1', async ({}, testInfo) => {
|
||||||
|
console.log('\\n%%test1');
|
||||||
|
expect(testInfo.retry).toEqual(0)
|
||||||
|
});
|
||||||
|
test('test2', async ({}, testInfo) => {
|
||||||
|
console.log('\\n%%test2');
|
||||||
|
expect(testInfo.retry).toEqual(1)
|
||||||
|
});
|
||||||
|
test('test3', async ({}) => {
|
||||||
|
console.log('\\n%%test3');
|
||||||
|
});
|
||||||
|
test('test4', async ({}) => {
|
||||||
|
console.log('\\n%%test4');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
`,
|
||||||
|
}, { retries: 1 });
|
||||||
|
expect(result.exitCode).toBe(1);
|
||||||
|
expect(result.passed).toBe(0);
|
||||||
|
expect(result.flaky).toBe(1);
|
||||||
|
expect(result.failed).toBe(1);
|
||||||
|
expect(result.skipped).toBe(2);
|
||||||
|
expect(result.outputLines).toEqual([
|
||||||
|
'test1',
|
||||||
|
'test2',
|
||||||
|
'test1',
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
test('test.describe.serial should work with retry and beforeAll failure', async ({ runInlineTest }) => {
|
test('test.describe.serial should work with retry and beforeAll failure', async ({ runInlineTest }) => {
|
||||||
const result = await runInlineTest({
|
const result = await runInlineTest({
|
||||||
'a.test.ts': `
|
'a.test.ts': `
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue