From d1e7b19c93eafe4365dd5d0cb3676717936109b2 Mon Sep 17 00:00:00 2001 From: Mathias Leppich Date: Fri, 11 Oct 2024 12:31:02 +0200 Subject: [PATCH] feat(test-runner): async test group filters --- packages/playwright/src/runner/loadUtils.ts | 80 ++++++----- packages/playwright/types/test.d.ts | 4 +- tests/playwright-test/test-filter.spec.ts | 144 +++++++++++--------- utils/generate_types/overrides-test.d.ts | 4 +- 4 files changed, 127 insertions(+), 105 deletions(-) diff --git a/packages/playwright/src/runner/loadUtils.ts b/packages/playwright/src/runner/loadUtils.ts index 1bc0fff3fe..faa1612fb6 100644 --- a/packages/playwright/src/runner/loadUtils.ts +++ b/packages/playwright/src/runner/loadUtils.ts @@ -192,40 +192,7 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho for (const projectSuite of rootSuite.suites) testGroups.push(...createTestGroups(projectSuite, config.config.workers)); - let filteredTestGroups = testGroups.map(group => ({ tests: group.tests.map(test => test as reporterTypes.TestCase) })); - const allTests = new Set(filteredTestGroups.flatMap(group => group.tests)); - for (const filter of filters) { - if ('filterTestGroups' in filter) { - filteredTestGroups = filter.filterTestGroups(filteredTestGroups, config.config); - } else if ('filterTests' in filter) { - filteredTestGroups = filteredTestGroups.map(group => { - return { tests: filter.filterTests(group.tests, config.config) }; - }); - } else if (typeof filter === 'function') { - filteredTestGroups = filteredTestGroups.map(group => { - return { - tests: group.tests.filter(test => { - const result = filter(test); - if (typeof result !== 'boolean') - throw new Error('Invalid filter result: filter function should return a boolean'); - return result; - }) - }; - }); - } - // check if filtered groups are still valid - if (!Array.isArray(filteredTestGroups)) - throw new Error('Invalid filter result: test groups should be an array'); - for (const group of filteredTestGroups) { - if (!Array.isArray(group.tests)) - throw new Error('Invalid filter result: tests should be an array'); - for (const test of group.tests) { - if (!allTests.has(test)) - throw new Error('Invalid filter result: test is not in the original list'); - } - } - } - const testsInThisRun = new Set(filteredTestGroups.flatMap(group => group.tests)); + const testsInThisRun = await filterTestGroups(config, testGroups, filters); // Update project suites, removing empty ones. filterTestsRemoveEmptySuites(rootSuite, test => testsInThisRun.has(test)); @@ -247,6 +214,51 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho return rootSuite; } +async function filterTestGroups(config: FullConfigInternal, testGroups: TestGroup[], filters: TestFilter[]): Promise> { + let filteredTestGroups = testGroups.map(group => ({ tests: group.tests.map(test => test as reporterTypes.TestCase) })); + const allTests = new Set(filteredTestGroups.flatMap(group => group.tests)); + for (const filter of filters) { + if ('filterTestGroups' in filter) { + const result = filter.filterTestGroups(filteredTestGroups, config.config); + filteredTestGroups = result instanceof Promise ? await result : result; + } else if ('filterTests' in filter) { + const result = filter.filterTests(filteredTestGroups.flatMap(group => group.tests), config.config); + const filteredTests = result instanceof Promise ? await result : result; + if (!Array.isArray(filteredTests)) + throw new Error('Invalid filter result: tests should be an array'); + const filteredTestsSet = new Set(filteredTests); + filteredTestGroups = filteredTestGroups.map(group => { + return { + tests: group.tests.filter(test => filteredTestsSet.has(test)) + }; + }); + } else if (typeof filter === 'function') { + filteredTestGroups = filteredTestGroups.map(group => { + return { + tests: group.tests.filter(test => { + const result = filter(test); + if (typeof result !== 'boolean') + throw new Error('Invalid filter result: filter function should return a boolean'); + return result; + }) + }; + }); + } + // check if filtered groups are still valid + if (!Array.isArray(filteredTestGroups)) + throw new Error('Invalid filter result: test groups should be an array'); + for (const group of filteredTestGroups) { + if (!Array.isArray(group.tests)) + throw new Error('Invalid filter result: tests should be an array'); + for (const test of group.tests) { + if (!allTests.has(test)) + throw new Error('Invalid filter result: test is not in the original list'); + } + } + } + return new Set(filteredTestGroups.flatMap(group => group.tests).map(test => test as TestCase)); +} + function createShardFilter(shard: { total: number, current: number }): TestFilter { return { filterTestGroups: (testGroups: TestGroup[]) => [...filterForShard(shard, testGroups).values()], diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index ce58bb3816..9c908caf74 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -1886,8 +1886,8 @@ export type TestDetails = { } type TestFilterFunction = (test: TestCase) => boolean; -type TestsFilter = { filterTests(tests: TestCase[], config: FullConfig): TestCase[] }; -type TestGroupsFilter = { filterTestGroups(testGroups: { tests: TestCase[] }[], config: FullConfig): { tests: TestCase[] }[] } +type TestsFilter = { filterTests(tests: TestCase[], config: FullConfig): Promise | TestCase[] }; +type TestGroupsFilter = { filterTestGroups(testGroups: { tests: TestCase[] }[], config: FullConfig): Promise<{ tests: TestCase[] }[]> | { tests: TestCase[] }[] }; export type TestFilter = TestFilterFunction | TestsFilter | TestGroupsFilter; interface SuiteFunction { diff --git a/tests/playwright-test/test-filter.spec.ts b/tests/playwright-test/test-filter.spec.ts index bc1da1db9f..899f21c902 100644 --- a/tests/playwright-test/test-filter.spec.ts +++ b/tests/playwright-test/test-filter.spec.ts @@ -16,29 +16,47 @@ import { test, expect } from './playwright-test-fixtures'; +const testFiles = { + 'a1.test.ts': ` + import { test, expect } from '@playwright/test'; + test('a1-test1', async () => { console.log('\\n%% a1-test1'); }); + test('a1-test2', async () => { console.log('\\n%% a1-test2'); }); + `, + 'a2.test.ts': ` + import { test, expect } from '@playwright/test'; + test('a2-test1', async () => { console.log('\\n%% a2-test1'); }); + test('a2-test2', async () => { console.log('\\n%% a2-test2'); }); + `, + 'a3.test.ts': ` + import { test, expect } from '@playwright/test'; + test('a3-test1', async () => { console.log('\\n%% a3-test1'); }); + `, + 'a4.test.ts': ` + import { test, expect } from '@playwright/test'; + test('a4-test1', async () => { console.log('\\n%% a4-test1'); }); + `, +}; + test('config.filter function should work', async ({ runInlineTest }) => { const result = await runInlineTest({ + ...testFiles, 'playwright.config.ts': ` module.exports = { - filter: (test) => test.title === 'test1', + filter: (test) => test.title === 'a2-test2', }; `, - 'a.test.ts': ` - import { test, expect } from '@playwright/test'; - test('test1', async () => { console.log('\\n%% test1'); }); - test('test2', async () => { console.log('\\n%% test2'); }); - `, }, { workers: 2 }); expect(result.exitCode).toBe(0); expect(result.passed).toBe(1); result.outputLines.sort(); expect(result.outputLines).toEqual([ - 'test1', + 'a2-test2', ]); }); test('config.filter filterTests should work', async ({ runInlineTest }) => { const result = await runInlineTest({ + ...testFiles, 'playwright.config.ts': ` module.exports = { filter: { @@ -46,73 +64,88 @@ test('config.filter filterTests should work', async ({ runInlineTest }) => { }, }; `, - 'a.test.ts': ` - import { test, expect } from '@playwright/test'; - test('test1', async () => { console.log('\\n%% test1'); }); - test('test2', async () => { console.log('\\n%% test2'); }); - test('test3', async () => { console.log('\\n%% test3'); }); - test('test4', async () => { console.log('\\n%% test4'); }); - `, }, { workers: 2 }); expect(result.exitCode).toBe(0); - expect(result.passed).toBe(2); + expect(result.passed).toBe(3); result.outputLines.sort(); expect(result.outputLines).toEqual([ - 'test1', - 'test3', + 'a1-test1', + 'a2-test1', + 'a3-test1', ]); }); test('config.filter filterTestGroups should work', async ({ runInlineTest }) => { const result = await runInlineTest({ + ...testFiles, 'playwright.config.ts': ` module.exports = { filter: { - filterTestGroups: (testgroups) => testgroups.filter((testgroup, index) => index % 2 === 0), + filterTestGroups: (testgroups) => testgroups.filter((testgroup, index) => index % 2 === 1), }, }; `, - 'a1.test.ts': ` - import { test, expect } from '@playwright/test'; - test('a1-test1', async () => { console.log('\\n%% a1-test1'); }); - test('a1-test2', async () => { console.log('\\n%% a1-test2'); }); - `, - 'a2.test.ts': ` - import { test, expect } from '@playwright/test'; - test('a2-test1', async () => { console.log('\\n%% a2-test1'); }); - test('a2-test2', async () => { console.log('\\n%% a2-test2'); }); - `, - 'a3.test.ts': ` - import { test, expect } from '@playwright/test'; - test('a3-test1', async () => { console.log('\\n%% a3-test1'); }); - `, - 'a4.test.ts': ` - import { test, expect } from '@playwright/test'; - test('a4-test1', async () => { console.log('\\n%% a4-test1'); }); - `, }, { workers: 2 }); expect(result.exitCode).toBe(0); expect(result.passed).toBe(3); result.outputLines.sort(); // Due to parallel execution, the order of output lines is not deterministic. + expect(result.outputLines).toEqual([ + 'a2-test1', + 'a2-test2', + 'a4-test1', + ]); +}); + +test('config.filter async filterTests should work', async ({ runInlineTest }) => { + const result = await runInlineTest({ + ...testFiles, + 'playwright.config.ts': ` + module.exports = { + filter: { + filterTests: (tests) => Promise.resolve(tests.filter((test, index) => index % 2 === 0)), + }, + }; + `, + }, { workers: 2 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(3); + result.outputLines.sort(); expect(result.outputLines).toEqual([ 'a1-test1', - 'a1-test2', + 'a2-test1', 'a3-test1', ]); }); +test('config.filter async filterTestGroups should work', async ({ runInlineTest }) => { + const result = await runInlineTest({ + ...testFiles, + 'playwright.config.ts': ` + module.exports = { + filter: { + filterTestGroups: (testgroups) => Promise.resolve(testgroups.filter((testgroup, index) => index % 2 === 1)), + }, + }; + `, + }, { workers: 2 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(3); + result.outputLines.sort(); // Due to parallel execution, the order of output lines is not deterministic. + expect(result.outputLines).toEqual([ + 'a2-test1', + 'a2-test2', + 'a4-test1', + ]); +}); + test('config.filter invalid function should throw', async ({ runInlineTest }) => { const result = await runInlineTest({ + ...testFiles, 'playwright.config.ts': ` module.exports = { filter: (test) => undefined, }; `, - 'a.test.ts': ` - import { test, expect } from '@playwright/test'; - test('test1', async () => { console.log('\\n%% test1'); }); - test('test2', async () => { console.log('\\n%% test2'); }); - `, }, { workers: 2 }); expect(result.exitCode).toBe(1); expect(result.output).toContain('Error: Invalid filter result: filter function should return a boolean'); @@ -120,6 +153,7 @@ test('config.filter invalid function should throw', async ({ runInlineTest }) => test('config.filter invalid filterTests should throw', async ({ runInlineTest }) => { const result = await runInlineTest({ + ...testFiles, 'playwright.config.ts': ` module.exports = { filter: { @@ -127,13 +161,6 @@ test('config.filter invalid filterTests should throw', async ({ runInlineTest }) }, }; `, - 'a.test.ts': ` - import { test, expect } from '@playwright/test'; - test('test1', async () => { console.log('\\n%% test1'); }); - test('test2', async () => { console.log('\\n%% test2'); }); - test('test3', async () => { console.log('\\n%% test3'); }); - test('test4', async () => { console.log('\\n%% test4'); }); - `, }, { workers: 2 }); expect(result.exitCode).toBe(1); expect(result.output).toContain('Error: Invalid filter result: tests should be an array'); @@ -141,6 +168,7 @@ test('config.filter invalid filterTests should throw', async ({ runInlineTest }) test('config.filter invalid filterTestGroups should throw', async ({ runInlineTest }) => { const result = await runInlineTest({ + ...testFiles, 'playwright.config.ts': ` module.exports = { filter: { @@ -148,24 +176,6 @@ test('config.filter invalid filterTestGroups should throw', async ({ runInlineTe }, }; `, - 'a1.test.ts': ` - import { test, expect } from '@playwright/test'; - test('a1-test1', async () => { console.log('\\n%% a1-test1'); }); - test('a1-test2', async () => { console.log('\\n%% a1-test2'); }); - `, - 'a2.test.ts': ` - import { test, expect } from '@playwright/test'; - test('a2-test1', async () => { console.log('\\n%% a2-test1'); }); - test('a2-test2', async () => { console.log('\\n%% a2-test2'); }); - `, - 'a3.test.ts': ` - import { test, expect } from '@playwright/test'; - test('a3-test1', async () => { console.log('\\n%% a3-test1'); }); - `, - 'a4.test.ts': ` - import { test, expect } from '@playwright/test'; - test('a4-test1', async () => { console.log('\\n%% a4-test1'); }); - `, }, { workers: 2 }); expect(result.exitCode).toBe(1); expect(result.output).toContain('Error: Invalid filter result: test groups should be an array'); diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 529419cefb..0cfe2a3aa3 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -78,8 +78,8 @@ export type TestDetails = { } type TestFilterFunction = (test: TestCase) => boolean; -type TestsFilter = { filterTests(tests: TestCase[], config: FullConfig): TestCase[] }; -type TestGroupsFilter = { filterTestGroups(testGroups: { tests: TestCase[] }[], config: FullConfig): { tests: TestCase[] }[] } +type TestsFilter = { filterTests(tests: TestCase[], config: FullConfig): Promise | TestCase[] }; +type TestGroupsFilter = { filterTestGroups(testGroups: { tests: TestCase[] }[], config: FullConfig): Promise<{ tests: TestCase[] }[]> | { tests: TestCase[] }[] }; export type TestFilter = TestFilterFunction | TestsFilter | TestGroupsFilter; interface SuiteFunction {