diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index ad5efdbb29..e6393b2758 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -164,7 +164,21 @@ export default defineConfig({ * since: v1.49 - type: ?<[TestFilter]|[Array]<[TestFilter]>> -Filter tests by passing a function. +Filter test cases by function. `TestFilter` can either be predicate function or an object with `filterTests` or `filterTestGroups` methods. + +**Usage** + +```js title="playwright.config.ts" +import { defineConfig } from '@playwright/test'; + +export default defineConfig({ + filter: (test) => test.title === 'some test', + // or + filter: { filterTests: (tests) => tests.filter((test, index) => index % 2 === 0) }, + // or + filter: { filterTestGroups: (testgroups) => testgroups.filter((testgroups, index) => index % 2 === 0) }, +}); +``` ## property: TestConfig.grep diff --git a/packages/playwright/src/runner/loadUtils.ts b/packages/playwright/src/runner/loadUtils.ts index 86a1da797b..c35e6edd97 100644 --- a/packages/playwright/src/runner/loadUtils.ts +++ b/packages/playwright/src/runner/loadUtils.ts @@ -15,6 +15,7 @@ */ import path from 'path'; +import type { TestFilter } from '../../types/test'; import type { FullConfig, Reporter, TestError } from '../../types/testReporter'; import type * as reporterTypes from '../../types/testReporter'; import { InProcessLoaderHost, OutOfProcessLoaderHost } from './loaderHost'; @@ -173,34 +174,58 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho } } - // Shard only the top-level projects. - if (config.config.shard || config.config.filter) { + const filters: TestFilter[] = []; + + if (config.config.filter) { + if (Array.isArray(config.config.filter)) + filters.push(...config.config.filter); + else + filters.push(config.config.filter); + } + + if (config.config.shard) + filters.push(createShardFilter(config.config.shard)); + + if (filters.length > 0) { // Create test groups for top-level projects. const testGroups: TestGroup[] = []; for (const projectSuite of rootSuite.suites) testGroups.push(...createTestGroups(projectSuite, config.config.workers)); - if (config.config.filter) { - const filters = Array.isArray(config.config.filter) ? config.config.filter : [config.config.filter]; - - const allTests = new Set(testGroups.flatMap(group => group.tests)); - - let filteredTests = [...allTests.values()]; - for (const filter of filters) - filteredTests = filter(filteredTests); - - const filteredTestSet = new Set(filteredTests); - for (const group of testGroups) - group.tests = group.tests.filter(test => filteredTestSet.has(test)); - } - - // Shard test groups. - const testGroupsInThisShard = config.config.shard ? filterForShard(config.config.shard, testGroups) : new Set(testGroups); - const testsInThisRun = new Set(); - for (const group of testGroupsInThisShard) { - for (const test of group.tests) - testsInThisRun.add(test); + let filteredTestGroups = testGroups.map(group => ({ tests: group.tests.map(test => test as reporterTypes.TestCase) })); + const allTests = new Set(filteredTestGroups.flatMap(group => group.tests)); + for (const filter of filters) { + if ('filterTestGroups' in filter) { + filteredTestGroups = filter.filterTestGroups(filteredTestGroups); + } else if ('filterTests' in filter) { + filteredTestGroups = filteredTestGroups.map(group => { + return { tests: filter.filterTests(group.tests) }; + }); + } else if (typeof filter === 'function') { + filteredTestGroups = filteredTestGroups.map(group => { + return { + tests: group.tests.filter(test => { + const result = filter(test); + if (typeof result !== 'boolean') + throw new Error('Invalid filter result: filter function should return a boolean'); + return result; + }) + }; + }); + } + // check if filtered groups are still valid + if (!Array.isArray(filteredTestGroups)) + throw new Error('Invalid filter result: test groups should be an array'); + for (const group of filteredTestGroups) { + if (!Array.isArray(group.tests)) + throw new Error('Invalid filter result: tests should be an array'); + for (const test of group.tests) { + if (!allTests.has(test)) + throw new Error('Invalid filter result: test is not in the original list'); + } + } } + const testsInThisRun = new Set(filteredTestGroups.flatMap(group => group.tests)); // Update project suites, removing empty ones. filterTestsRemoveEmptySuites(rootSuite, test => testsInThisRun.has(test)); @@ -222,6 +247,12 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho return rootSuite; } +function createShardFilter(shard: { total: number, current: number }): TestFilter { + return { + filterTestGroups: (testGroups: TestGroup[]) => [...filterForShard(shard, testGroups).values()], + }; +} + function createProjectSuite(project: FullProjectInternal, fileSuites: Suite[]): Suite { const projectSuite = new Suite(project.project.name, 'project'); for (const fileSuite of fileSuites) diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index 1485744d6b..91b9bdb37d 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -1037,7 +1037,24 @@ interface TestConfig { }; /** - * Filter tests by passing a function. + * Filter test cases by function. `TestFilter` can either be predicate function or an object with `filterTests` or + * `filterTestGroups` methods. + * + * **Usage** + * + * ```js + * // playwright.config.ts + * import { defineConfig } from '@playwright/test'; + * + * export default defineConfig({ + * filter: (test) => test.title === 'some test', + * // or + * filter: { filterTests: (tests) => tests.filter((test, index) => index % 2 === 0) }, + * // or + * filter: { filterTestGroups: (testgroups) => testgroups.filter((testgroups, index) => index % 2 === 0) }, + * }); + * ``` + * */ filter?: TestFilter|Array; @@ -1850,7 +1867,10 @@ export type TestDetails = { annotation?: TestDetailsAnnotation | TestDetailsAnnotation[]; } -export type TestFilter = (tests: TestCase[]) => TestCase[]; +type TestFilterFunction = (test: TestCase) => boolean; +type TestsFilter = { filterTests(tests: TestCase[]): TestCase[] } +type TestGroupsFilter = { filterTestGroups(testGroups: { tests: TestCase[] }[]): { tests: TestCase[] }[] } +export type TestFilter = TestFilterFunction | TestsFilter | TestGroupsFilter; interface SuiteFunction { /** diff --git a/tests/playwright-test/test-filter.spec.ts b/tests/playwright-test/test-filter.spec.ts index c2b3c04c1e..bc1da1db9f 100644 --- a/tests/playwright-test/test-filter.spec.ts +++ b/tests/playwright-test/test-filter.spec.ts @@ -16,11 +16,11 @@ import { test, expect } from './playwright-test-fixtures'; -test('config.filter should work', async ({ runInlineTest }) => { +test('config.filter function should work', async ({ runInlineTest }) => { const result = await runInlineTest({ 'playwright.config.ts': ` module.exports = { - filter: (tests) => tests.filter(test => test.title === 'test1'), + filter: (test) => test.title === 'test1', }; `, 'a.test.ts': ` @@ -28,10 +28,145 @@ test('config.filter should work', async ({ runInlineTest }) => { test('test1', async () => { console.log('\\n%% test1'); }); test('test2', async () => { console.log('\\n%% test2'); }); `, - }); + }, { workers: 2 }); expect(result.exitCode).toBe(0); expect(result.passed).toBe(1); + result.outputLines.sort(); expect(result.outputLines).toEqual([ 'test1', ]); }); + +test('config.filter filterTests should work', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + filter: { + filterTests: (tests) => tests.filter((test, index) => index % 2 === 0), + }, + }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('test1', async () => { console.log('\\n%% test1'); }); + test('test2', async () => { console.log('\\n%% test2'); }); + test('test3', async () => { console.log('\\n%% test3'); }); + test('test4', async () => { console.log('\\n%% test4'); }); + `, + }, { workers: 2 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(2); + result.outputLines.sort(); + expect(result.outputLines).toEqual([ + 'test1', + 'test3', + ]); +}); + +test('config.filter filterTestGroups should work', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + filter: { + filterTestGroups: (testgroups) => testgroups.filter((testgroup, index) => index % 2 === 0), + }, + }; + `, + 'a1.test.ts': ` + import { test, expect } from '@playwright/test'; + test('a1-test1', async () => { console.log('\\n%% a1-test1'); }); + test('a1-test2', async () => { console.log('\\n%% a1-test2'); }); + `, + 'a2.test.ts': ` + import { test, expect } from '@playwright/test'; + test('a2-test1', async () => { console.log('\\n%% a2-test1'); }); + test('a2-test2', async () => { console.log('\\n%% a2-test2'); }); + `, + 'a3.test.ts': ` + import { test, expect } from '@playwright/test'; + test('a3-test1', async () => { console.log('\\n%% a3-test1'); }); + `, + 'a4.test.ts': ` + import { test, expect } from '@playwright/test'; + test('a4-test1', async () => { console.log('\\n%% a4-test1'); }); + `, + }, { workers: 2 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(3); + result.outputLines.sort(); // Due to parallel execution, the order of output lines is not deterministic. + expect(result.outputLines).toEqual([ + 'a1-test1', + 'a1-test2', + 'a3-test1', + ]); +}); + +test('config.filter invalid function should throw', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + filter: (test) => undefined, + }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('test1', async () => { console.log('\\n%% test1'); }); + test('test2', async () => { console.log('\\n%% test2'); }); + `, + }, { workers: 2 }); + expect(result.exitCode).toBe(1); + expect(result.output).toContain('Error: Invalid filter result: filter function should return a boolean'); +}); + +test('config.filter invalid filterTests should throw', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + filter: { + filterTests: (tests) => undefined, + }, + }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('test1', async () => { console.log('\\n%% test1'); }); + test('test2', async () => { console.log('\\n%% test2'); }); + test('test3', async () => { console.log('\\n%% test3'); }); + test('test4', async () => { console.log('\\n%% test4'); }); + `, + }, { workers: 2 }); + expect(result.exitCode).toBe(1); + expect(result.output).toContain('Error: Invalid filter result: tests should be an array'); +}); + +test('config.filter invalid filterTestGroups should throw', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + filter: { + filterTestGroups: (testgroups) => undefined, + }, + }; + `, + 'a1.test.ts': ` + import { test, expect } from '@playwright/test'; + test('a1-test1', async () => { console.log('\\n%% a1-test1'); }); + test('a1-test2', async () => { console.log('\\n%% a1-test2'); }); + `, + 'a2.test.ts': ` + import { test, expect } from '@playwright/test'; + test('a2-test1', async () => { console.log('\\n%% a2-test1'); }); + test('a2-test2', async () => { console.log('\\n%% a2-test2'); }); + `, + 'a3.test.ts': ` + import { test, expect } from '@playwright/test'; + test('a3-test1', async () => { console.log('\\n%% a3-test1'); }); + `, + 'a4.test.ts': ` + import { test, expect } from '@playwright/test'; + test('a4-test1', async () => { console.log('\\n%% a4-test1'); }); + `, + }, { workers: 2 }); + expect(result.exitCode).toBe(1); + expect(result.output).toContain('Error: Invalid filter result: test groups should be an array'); +}); diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 16645e1abb..36f45d8eb7 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -77,7 +77,10 @@ export type TestDetails = { annotation?: TestDetailsAnnotation | TestDetailsAnnotation[]; } -export type TestFilter = (tests: TestCase[]) => TestCase[]; +type TestFilterFunction = (test: TestCase) => boolean; +type TestsFilter = { filterTests(tests: TestCase[]): TestCase[] } +type TestGroupsFilter = { filterTestGroups(testGroups: { tests: TestCase[] }[]): { tests: TestCase[] }[] } +export type TestFilter = TestFilterFunction | TestsFilter | TestGroupsFilter; interface SuiteFunction { (title: string, callback: () => void): void;