fix(playwright): test unexpected result if failed and skipped

Checks if a test results are only failed and skipped. If so, it sets the result as unexpected instead of flaky, since they never passed.

Fixes #28322
This commit is contained in:
BeeMargarida 2024-04-06 18:07:59 +01:00
parent 5d810f1e07
commit 49b293002c
2 changed files with 38 additions and 0 deletions

View file

@ -290,10 +290,14 @@ export class TestCase extends Base implements reporterTypes.TestCase {
return 'skipped';
const failures = results.filter(result => result.status !== 'skipped' && result.status !== 'interrupted' && result.status !== this.expectedStatus);
const skipped = results.filter(result => result.status === 'skipped');
const passed = results.filter(result => result.status === 'passed');
if (!failures.length) // all passed
return 'expected';
if (failures.length === results.length) // all failed
return 'unexpected';
if (failures.length && skipped.length && !passed.length) // some failed, none succedded and the rest where skipped
return 'unexpected';
return 'flaky'; // mixed bag
}

View file

@ -140,6 +140,40 @@ test('test.describe.serial should work with retry', async ({ runInlineTest }) =>
]);
});
test('test.describe.serial should work with retry when it fails in different places', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
import { test, expect } from '@playwright/test';
test.describe.serial('serial suite', () => {
test('test1', async ({}, testInfo) => {
console.log('\\n%%test1');
expect(testInfo.retry).toEqual(0)
});
test('test2', async ({}, testInfo) => {
console.log('\\n%%test2');
expect(testInfo.retry).toEqual(1)
});
test('test3', async ({}) => {
console.log('\\n%%test3');
});
test('test4', async ({}) => {
console.log('\\n%%test4');
});
});
`,
}, { retries: 1 });
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(0);
expect(result.flaky).toBe(1);
expect(result.failed).toBe(1);
expect(result.skipped).toBe(2);
expect(result.outputLines).toEqual([
'test1',
'test2',
'test1',
]);
});
test('test.describe.serial should work with retry and beforeAll failure', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `