feat(test-runner): test groups filter

docs/src/test-api/class-testconfig.md

@@ -164,7 +164,21 @@ export default defineConfig({
 * since: v1.49
 - type: ?<[TestFilter]|[Array]<[TestFilter]>>
 
-Filter tests by passing a function.
+Filter test cases by function. `TestFilter` can either be predicate function or an object with `filterTests` or `filterTestGroups` methods.
+
+**Usage**
+
+```js title="playwright.config.ts"
+import { defineConfig } from '@playwright/test';
+
+export default defineConfig({
+  filter: (test) => test.title === 'some test',
+  // or
+  filter: { filterTests: (tests) => tests.filter((test, index) => index % 2 === 0) },
+  // or
+  filter: { filterTestGroups: (testgroups) => testgroups.filter((testgroups, index) => index % 2 === 0) },
+});
+```
 
 
 ## property: TestConfig.grep

packages/playwright/src/runner/loadUtils.ts

@@ -15,6 +15,7 @@
  */
 
 import path from 'path';
+import type { TestFilter } from '../../types/test';
 import type { FullConfig, Reporter, TestError } from '../../types/testReporter';
 import type * as reporterTypes from '../../types/testReporter';
 import { InProcessLoaderHost, OutOfProcessLoaderHost } from './loaderHost';
@@ -173,34 +174,58 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho
     }
   }
 
-  // Shard only the top-level projects.
-  if (config.config.shard || config.config.filter) {
+  const filters: TestFilter[] = [];
+
+  if (config.config.filter) {
+    if (Array.isArray(config.config.filter))
+      filters.push(...config.config.filter);
+    else
+      filters.push(config.config.filter);
+  }
+
+  if (config.config.shard)
+    filters.push(createShardFilter(config.config.shard));
+
+  if (filters.length > 0) {
     // Create test groups for top-level projects.
     const testGroups: TestGroup[] = [];
     for (const projectSuite of rootSuite.suites)
       testGroups.push(...createTestGroups(projectSuite, config.config.workers));
 
-    if (config.config.filter) {
-      const filters = Array.isArray(config.config.filter) ? config.config.filter : [config.config.filter];
-
-      const allTests = new Set<reporterTypes.TestCase>(testGroups.flatMap(group => group.tests));
-
-      let filteredTests = [...allTests.values()];
-      for (const filter of filters)
-        filteredTests = filter(filteredTests);
-
-      const filteredTestSet = new Set(filteredTests);
-      for (const group of testGroups)
-        group.tests = group.tests.filter(test => filteredTestSet.has(test));
+    let filteredTestGroups = testGroups.map(group => ({ tests: group.tests.map(test => test as reporterTypes.TestCase) }));
+    const allTests = new Set(filteredTestGroups.flatMap(group => group.tests));
+    for (const filter of filters) {
+      if ('filterTestGroups' in filter) {
+        filteredTestGroups = filter.filterTestGroups(filteredTestGroups);
+      } else if ('filterTests' in filter) {
+        filteredTestGroups = filteredTestGroups.map(group => {
+          return { tests: filter.filterTests(group.tests) };
+        });
+      } else if (typeof filter === 'function') {
+        filteredTestGroups = filteredTestGroups.map(group => {
+          return {
+            tests: group.tests.filter(test => {
+              const result = filter(test);
+              if (typeof result !== 'boolean')
+                throw new Error('Invalid filter result: filter function should return a boolean');
+              return result;
+            })
+          };
+        });
       }
-
-    // Shard test groups.
-    const testGroupsInThisShard = config.config.shard ? filterForShard(config.config.shard, testGroups) : new Set(testGroups);
-    const testsInThisRun = new Set<TestCase>();
-    for (const group of testGroupsInThisShard) {
-      for (const test of group.tests)
-        testsInThisRun.add(test);
+      // check if filtered groups are still valid
+      if (!Array.isArray(filteredTestGroups))
+        throw new Error('Invalid filter result: test groups should be an array');
+      for (const group of filteredTestGroups) {
+        if (!Array.isArray(group.tests))
+          throw new Error('Invalid filter result: tests should be an array');
+        for (const test of group.tests) {
+          if (!allTests.has(test))
+            throw new Error('Invalid filter result: test is not in the original list');
         }
+      }
+    }
+    const testsInThisRun = new Set(filteredTestGroups.flatMap(group => group.tests));
 
     // Update project suites, removing empty ones.
     filterTestsRemoveEmptySuites(rootSuite, test => testsInThisRun.has(test));
@@ -222,6 +247,12 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho
   return rootSuite;
 }
 
+function createShardFilter(shard: { total: number, current: number }): TestFilter {
+  return {
+    filterTestGroups: (testGroups: TestGroup[]) => [...filterForShard(shard, testGroups).values()],
+  };
+}
+
 function createProjectSuite(project: FullProjectInternal, fileSuites: Suite[]): Suite {
   const projectSuite = new Suite(project.project.name, 'project');
   for (const fileSuite of fileSuites)

packages/playwright/types/test.d.ts

@@ -1037,7 +1037,24 @@ interface TestConfig<TestArgs = {}, WorkerArgs = {}> {
   };
 
   /**
-   * Filter tests by passing a function.
+   * Filter test cases by function. `TestFilter` can either be predicate function or an object with `filterTests` or
+   * `filterTestGroups` methods.
+   *
+   * **Usage**
+   *
+   * ```js
+   * // playwright.config.ts
+   * import { defineConfig } from '@playwright/test';
+   *
+   * export default defineConfig({
+   *   filter: (test) => test.title === 'some test',
+   *   // or
+   *   filter: { filterTests: (tests) => tests.filter((test, index) => index % 2 === 0) },
+   *   // or
+   *   filter: { filterTestGroups: (testgroups) => testgroups.filter((testgroups, index) => index % 2 === 0) },
+   * });
+   * ```
+   *
    */
   filter?: TestFilter|Array<TestFilter>;
 
@@ -1850,7 +1867,10 @@ export type TestDetails = {
   annotation?: TestDetailsAnnotation | TestDetailsAnnotation[];
 }
 
-export type TestFilter = (tests: TestCase[]) => TestCase[];
+type TestFilterFunction = (test: TestCase) => boolean;
+type TestsFilter = { filterTests(tests: TestCase[]): TestCase[] }
+type TestGroupsFilter = { filterTestGroups(testGroups: { tests: TestCase[] }[]): { tests: TestCase[] }[] }
+export type TestFilter = TestFilterFunction | TestsFilter | TestGroupsFilter;
 
 interface SuiteFunction {
   /**

tests/playwright-test/test-filter.spec.ts

@@ -16,11 +16,11 @@
 
 import { test, expect } from './playwright-test-fixtures';
 
-test('config.filter should work', async ({ runInlineTest }) => {
+test('config.filter function should work', async ({ runInlineTest }) => {
   const result = await runInlineTest({
     'playwright.config.ts': `
       module.exports = {
-        filter: (tests) => tests.filter(test => test.title === 'test1'),
+        filter: (test) => test.title === 'test1',
       };
     `,
     'a.test.ts': `
@@ -28,10 +28,145 @@ test('config.filter should work', async ({ runInlineTest }) => {
       test('test1', async () => { console.log('\\n%% test1'); });
       test('test2', async () => { console.log('\\n%% test2'); });
     `,
-  });
+  }, { workers: 2 });
   expect(result.exitCode).toBe(0);
   expect(result.passed).toBe(1);
+  result.outputLines.sort();
   expect(result.outputLines).toEqual([
     'test1',
   ]);
 });
+
+test('config.filter filterTests should work', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = {
+        filter: {
+          filterTests: (tests) => tests.filter((test, index) => index % 2 === 0),
+        },
+      };
+    `,
+    'a.test.ts': `
+      import { test, expect } from '@playwright/test';
+      test('test1', async () => { console.log('\\n%% test1'); });
+      test('test2', async () => { console.log('\\n%% test2'); });
+      test('test3', async () => { console.log('\\n%% test3'); });
+      test('test4', async () => { console.log('\\n%% test4'); });
+    `,
+  }, { workers: 2 });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(2);
+  result.outputLines.sort();
+  expect(result.outputLines).toEqual([
+    'test1',
+    'test3',
+  ]);
+});
+
+test('config.filter filterTestGroups should work', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = {
+        filter: {
+          filterTestGroups: (testgroups) => testgroups.filter((testgroup, index) => index % 2 === 0),
+        },
+      };
+    `,
+    'a1.test.ts': `
+      import { test, expect } from '@playwright/test';
+      test('a1-test1', async () => { console.log('\\n%% a1-test1'); });
+      test('a1-test2', async () => { console.log('\\n%% a1-test2'); });
+    `,
+    'a2.test.ts': `
+      import { test, expect } from '@playwright/test';
+      test('a2-test1', async () => { console.log('\\n%% a2-test1'); });
+      test('a2-test2', async () => { console.log('\\n%% a2-test2'); });
+    `,
+    'a3.test.ts': `
+      import { test, expect } from '@playwright/test';
+      test('a3-test1', async () => { console.log('\\n%% a3-test1'); });
+    `,
+    'a4.test.ts': `
+      import { test, expect } from '@playwright/test';
+      test('a4-test1', async () => { console.log('\\n%% a4-test1'); });
+    `,
+  }, { workers: 2 });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(3);
+  result.outputLines.sort(); // Due to parallel execution, the order of output lines is not deterministic.
+  expect(result.outputLines).toEqual([
+    'a1-test1',
+    'a1-test2',
+    'a3-test1',
+  ]);
+});
+
+test('config.filter invalid function should throw', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = {
+        filter: (test) => undefined,
+      };
+    `,
+    'a.test.ts': `
+      import { test, expect } from '@playwright/test';
+      test('test1', async () => { console.log('\\n%% test1'); });
+      test('test2', async () => { console.log('\\n%% test2'); });
+    `,
+  }, { workers: 2 });
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain('Error: Invalid filter result: filter function should return a boolean');
+});
+
+test('config.filter invalid filterTests should throw', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = {
+        filter: {
+          filterTests: (tests) => undefined,
+        },
+      };
+    `,
+    'a.test.ts': `
+      import { test, expect } from '@playwright/test';
+      test('test1', async () => { console.log('\\n%% test1'); });
+      test('test2', async () => { console.log('\\n%% test2'); });
+      test('test3', async () => { console.log('\\n%% test3'); });
+      test('test4', async () => { console.log('\\n%% test4'); });
+    `,
+  }, { workers: 2 });
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain('Error: Invalid filter result: tests should be an array');
+});
+
+test('config.filter invalid filterTestGroups should throw', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = {
+        filter: {
+          filterTestGroups: (testgroups) => undefined,
+        },
+      };
+    `,
+    'a1.test.ts': `
+      import { test, expect } from '@playwright/test';
+      test('a1-test1', async () => { console.log('\\n%% a1-test1'); });
+      test('a1-test2', async () => { console.log('\\n%% a1-test2'); });
+    `,
+    'a2.test.ts': `
+      import { test, expect } from '@playwright/test';
+      test('a2-test1', async () => { console.log('\\n%% a2-test1'); });
+      test('a2-test2', async () => { console.log('\\n%% a2-test2'); });
+    `,
+    'a3.test.ts': `
+      import { test, expect } from '@playwright/test';
+      test('a3-test1', async () => { console.log('\\n%% a3-test1'); });
+    `,
+    'a4.test.ts': `
+      import { test, expect } from '@playwright/test';
+      test('a4-test1', async () => { console.log('\\n%% a4-test1'); });
+    `,
+  }, { workers: 2 });
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain('Error: Invalid filter result: test groups should be an array');
+});

utils/generate_types/overrides-test.d.ts

@@ -77,7 +77,10 @@ export type TestDetails = {
   annotation?: TestDetailsAnnotation | TestDetailsAnnotation[];
 }
 
-export type TestFilter = (tests: TestCase[]) => TestCase[];
+type TestFilterFunction = (test: TestCase) => boolean;
+type TestsFilter = { filterTests(tests: TestCase[]): TestCase[] }
+type TestGroupsFilter = { filterTestGroups(testGroups: { tests: TestCase[] }[]): { tests: TestCase[] }[] }
+export type TestFilter = TestFilterFunction | TestsFilter | TestGroupsFilter;
 
 interface SuiteFunction {
   (title: string, callback: () => void): void;