Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

feat(test-runner): async test group filters

Browse source
This commit is contained in:
Mathias Leppich 2024-10-11 12:31:02 +02:00
parent c929ee4e67
commit d1e7b19c93
4 changed files with 127 additions and 105 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

56
packages/playwright/src/runner/loadUtils.ts
View file

@ -192,14 +192,45 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho
for (const projectSuite of rootSuite.suites)
testGroups.push(...createTestGroups(projectSuite, config.config.workers));
const testsInThisRun = await filterTestGroups(config, testGroups, filters);
// Update project suites, removing empty ones.
filterTestsRemoveEmptySuites(rootSuite, test => testsInThisRun.has(test));
}
// Now prepend dependency projects without filtration.
{
// Filtering 'only' and sharding might have reduced the number of top-level projects.
// Build the project closure to only include dependencies that are still needed.
const projectClosure = new Map(buildProjectsClosure(rootSuite.suites.map(suite => suite._fullProject!)));
// Clone file suites for dependency projects.
for (const [project, level] of projectClosure.entries()) {
if (level === 'dependency')
rootSuite._prependSuite(buildProjectSuite(project, projectSuites.get(project)!));
}
}
return rootSuite;
}
async function filterTestGroups(config: FullConfigInternal, testGroups: TestGroup[], filters: TestFilter[]): Promise<Set<TestCase>> {
let filteredTestGroups = testGroups.map(group => ({ tests: group.tests.map(test => test as reporterTypes.TestCase) }));
const allTests = new Set(filteredTestGroups.flatMap(group => group.tests));
for (const filter of filters) {
if ('filterTestGroups' in filter) {
filteredTestGroups = filter.filterTestGroups(filteredTestGroups, config.config);
const result = filter.filterTestGroups(filteredTestGroups, config.config);
filteredTestGroups = result instanceof Promise ? await result : result;
} else if ('filterTests' in filter) {
const result = filter.filterTests(filteredTestGroups.flatMap(group => group.tests), config.config);
const filteredTests = result instanceof Promise ? await result : result;
if (!Array.isArray(filteredTests))
throw new Error('Invalid filter result: tests should be an array');
const filteredTestsSet = new Set(filteredTests);
filteredTestGroups = filteredTestGroups.map(group => {
return { tests: filter.filterTests(group.tests, config.config) };
return {
tests: group.tests.filter(test => filteredTestsSet.has(test))
};
});
} else if (typeof filter === 'function') {
filteredTestGroups = filteredTestGroups.map(group => {
@ -225,26 +256,7 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho
}
}
}
const testsInThisRun = new Set(filteredTestGroups.flatMap(group => group.tests));
// Update project suites, removing empty ones.
filterTestsRemoveEmptySuites(rootSuite, test => testsInThisRun.has(test));
}
// Now prepend dependency projects without filtration.
{
// Filtering 'only' and sharding might have reduced the number of top-level projects.
// Build the project closure to only include dependencies that are still needed.
const projectClosure = new Map(buildProjectsClosure(rootSuite.suites.map(suite => suite._fullProject!)));
// Clone file suites for dependency projects.
for (const [project, level] of projectClosure.entries()) {
if (level === 'dependency')
rootSuite._prependSuite(buildProjectSuite(project, projectSuites.get(project)!));
}
}
return rootSuite;
return new Set(filteredTestGroups.flatMap(group => group.tests).map(test => test as TestCase));
}
function createShardFilter(shard: { total: number, current: number }): TestFilter {

4
packages/playwright/types/test.d.ts vendored
View file

@ -1886,8 +1886,8 @@ export type TestDetails = {
}
type TestFilterFunction = (test: TestCase) => boolean;
type TestsFilter = { filterTests(tests: TestCase[], config: FullConfig): TestCase[] };
type TestGroupsFilter = { filterTestGroups(testGroups: { tests: TestCase[] }[], config: FullConfig): { tests: TestCase[] }[] }
type TestsFilter = { filterTests(tests: TestCase[], config: FullConfig): Promise<TestCase[]> | TestCase[] };
type TestGroupsFilter = { filterTestGroups(testGroups: { tests: TestCase[] }[], config: FullConfig): Promise<{ tests: TestCase[] }[]> | { tests: TestCase[] }[] };
export type TestFilter = TestFilterFunction | TestsFilter | TestGroupsFilter;
interface SuiteFunction {

184
tests/playwright-test/test-filter.spec.ts
View file

@ -16,62 +16,7 @@
import { test, expect } from './playwright-test-fixtures';
test('config.filter function should work', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
filter: (test) => test.title === 'test1',
};
`,
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('test1', async () => { console.log('\\n%% test1'); });
test('test2', async () => { console.log('\\n%% test2'); });
`,
}, { workers: 2 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
result.outputLines.sort();
expect(result.outputLines).toEqual([
'test1',
]);
});
test('config.filter filterTests should work', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
filter: {
filterTests: (tests) => tests.filter((test, index) => index % 2 === 0),
},
};
`,
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('test1', async () => { console.log('\\n%% test1'); });
test('test2', async () => { console.log('\\n%% test2'); });
test('test3', async () => { console.log('\\n%% test3'); });
test('test4', async () => { console.log('\\n%% test4'); });
`,
}, { workers: 2 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(2);
result.outputLines.sort();
expect(result.outputLines).toEqual([
'test1',
'test3',
]);
});
test('config.filter filterTestGroups should work', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
filter: {
filterTestGroups: (testgroups) => testgroups.filter((testgroup, index) => index % 2 === 0),
},
};
`,
const testFiles = {
'a1.test.ts': `
import { test, expect } from '@playwright/test';
test('a1-test1', async () => { console.log('\\n%% a1-test1'); });
@ -90,29 +35,117 @@ test('config.filter filterTestGroups should work', async ({ runInlineTest }) =>
import { test, expect } from '@playwright/test';
test('a4-test1', async () => { console.log('\\n%% a4-test1'); });
`,
};
test('config.filter function should work', async ({ runInlineTest }) => {
const result = await runInlineTest({
...testFiles,
'playwright.config.ts': `
module.exports = {
filter: (test) => test.title === 'a2-test2',
};
`,
}, { workers: 2 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
result.outputLines.sort();
expect(result.outputLines).toEqual([
'a2-test2',
]);
});
test('config.filter filterTests should work', async ({ runInlineTest }) => {
const result = await runInlineTest({
...testFiles,
'playwright.config.ts': `
module.exports = {
filter: {
filterTests: (tests) => tests.filter((test, index) => index % 2 === 0),
},
};
`,
}, { workers: 2 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(3);
result.outputLines.sort();
expect(result.outputLines).toEqual([
'a1-test1',
'a2-test1',
'a3-test1',
]);
});
test('config.filter filterTestGroups should work', async ({ runInlineTest }) => {
const result = await runInlineTest({
...testFiles,
'playwright.config.ts': `
module.exports = {
filter: {
filterTestGroups: (testgroups) => testgroups.filter((testgroup, index) => index % 2 === 1),
},
};
`,
}, { workers: 2 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(3);
result.outputLines.sort(); // Due to parallel execution, the order of output lines is not deterministic.
expect(result.outputLines).toEqual([
'a2-test1',
'a2-test2',
'a4-test1',
]);
});
test('config.filter async filterTests should work', async ({ runInlineTest }) => {
const result = await runInlineTest({
...testFiles,
'playwright.config.ts': `
module.exports = {
filter: {
filterTests: (tests) => Promise.resolve(tests.filter((test, index) => index % 2 === 0)),
},
};
`,
}, { workers: 2 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(3);
result.outputLines.sort();
expect(result.outputLines).toEqual([
'a1-test1',
'a1-test2',
'a2-test1',
'a3-test1',
]);
});
test('config.filter async filterTestGroups should work', async ({ runInlineTest }) => {
const result = await runInlineTest({
...testFiles,
'playwright.config.ts': `
module.exports = {
filter: {
filterTestGroups: (testgroups) => Promise.resolve(testgroups.filter((testgroup, index) => index % 2 === 1)),
},
};
`,
}, { workers: 2 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(3);
result.outputLines.sort(); // Due to parallel execution, the order of output lines is not deterministic.
expect(result.outputLines).toEqual([
'a2-test1',
'a2-test2',
'a4-test1',
]);
});
test('config.filter invalid function should throw', async ({ runInlineTest }) => {
const result = await runInlineTest({
...testFiles,
'playwright.config.ts': `
module.exports = {
filter: (test) => undefined,
};
`,
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('test1', async () => { console.log('\\n%% test1'); });
test('test2', async () => { console.log('\\n%% test2'); });
`,
}, { workers: 2 });
expect(result.exitCode).toBe(1);
expect(result.output).toContain('Error: Invalid filter result: filter function should return a boolean');
@ -120,6 +153,7 @@ test('config.filter invalid function should throw', async ({ runInlineTest }) =>
test('config.filter invalid filterTests should throw', async ({ runInlineTest }) => {
const result = await runInlineTest({
...testFiles,
'playwright.config.ts': `
module.exports = {
filter: {
@ -127,13 +161,6 @@ test('config.filter invalid filterTests should throw', async ({ runInlineTest })
},
};
`,
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('test1', async () => { console.log('\\n%% test1'); });
test('test2', async () => { console.log('\\n%% test2'); });
test('test3', async () => { console.log('\\n%% test3'); });
test('test4', async () => { console.log('\\n%% test4'); });
`,
}, { workers: 2 });
expect(result.exitCode).toBe(1);
expect(result.output).toContain('Error: Invalid filter result: tests should be an array');
@ -141,6 +168,7 @@ test('config.filter invalid filterTests should throw', async ({ runInlineTest })
test('config.filter invalid filterTestGroups should throw', async ({ runInlineTest }) => {
const result = await runInlineTest({
...testFiles,
'playwright.config.ts': `
module.exports = {
filter: {
@ -148,24 +176,6 @@ test('config.filter invalid filterTestGroups should throw', async ({ runInlineTe
},
};
`,
'a1.test.ts': `
import { test, expect } from '@playwright/test';
test('a1-test1', async () => { console.log('\\n%% a1-test1'); });
test('a1-test2', async () => { console.log('\\n%% a1-test2'); });
`,
'a2.test.ts': `
import { test, expect } from '@playwright/test';
test('a2-test1', async () => { console.log('\\n%% a2-test1'); });
test('a2-test2', async () => { console.log('\\n%% a2-test2'); });
`,
'a3.test.ts': `
import { test, expect } from '@playwright/test';
test('a3-test1', async () => { console.log('\\n%% a3-test1'); });
`,
'a4.test.ts': `
import { test, expect } from '@playwright/test';
test('a4-test1', async () => { console.log('\\n%% a4-test1'); });
`,
}, { workers: 2 });
expect(result.exitCode).toBe(1);
expect(result.output).toContain('Error: Invalid filter result: test groups should be an array');

4
utils/generate_types/overrides-test.d.ts vendored
View file

@ -78,8 +78,8 @@ export type TestDetails = {
}
type TestFilterFunction = (test: TestCase) => boolean;
type TestsFilter = { filterTests(tests: TestCase[], config: FullConfig): TestCase[] };
type TestGroupsFilter = { filterTestGroups(testGroups: { tests: TestCase[] }[], config: FullConfig): { tests: TestCase[] }[] }
type TestsFilter = { filterTests(tests: TestCase[], config: FullConfig): Promise<TestCase[]> | TestCase[] };
type TestGroupsFilter = { filterTestGroups(testGroups: { tests: TestCase[] }[], config: FullConfig): Promise<{ tests: TestCase[] }[]> | { tests: TestCase[] }[] };
export type TestFilter = TestFilterFunction | TestsFilter | TestGroupsFilter;
interface SuiteFunction {