feat(test-runner): async test group filters

packages/playwright/src/runner/loadUtils.ts

@@ -192,40 +192,7 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho
     for (const projectSuite of rootSuite.suites)
       testGroups.push(...createTestGroups(projectSuite, config.config.workers));
 
-    let filteredTestGroups = testGroups.map(group => ({ tests: group.tests.map(test => test as reporterTypes.TestCase) }));
+    const testsInThisRun = await filterTestGroups(config, testGroups, filters);
-    const allTests = new Set(filteredTestGroups.flatMap(group => group.tests));
-    for (const filter of filters) {
-      if ('filterTestGroups' in filter) {
-        filteredTestGroups = filter.filterTestGroups(filteredTestGroups, config.config);
-      } else if ('filterTests' in filter) {
-        filteredTestGroups = filteredTestGroups.map(group => {
-          return { tests: filter.filterTests(group.tests, config.config) };
-        });
-      } else if (typeof filter === 'function') {
-        filteredTestGroups = filteredTestGroups.map(group => {
-          return {
-            tests: group.tests.filter(test => {
-              const result = filter(test);
-              if (typeof result !== 'boolean')
-                throw new Error('Invalid filter result: filter function should return a boolean');
-              return result;
-            })
-          };
-        });
-      }
-      // check if filtered groups are still valid
-      if (!Array.isArray(filteredTestGroups))
-        throw new Error('Invalid filter result: test groups should be an array');
-      for (const group of filteredTestGroups) {
-        if (!Array.isArray(group.tests))
-          throw new Error('Invalid filter result: tests should be an array');
-        for (const test of group.tests) {
-          if (!allTests.has(test))
-            throw new Error('Invalid filter result: test is not in the original list');
-        }
-      }
-    }
-    const testsInThisRun = new Set(filteredTestGroups.flatMap(group => group.tests));
 
     // Update project suites, removing empty ones.
     filterTestsRemoveEmptySuites(rootSuite, test => testsInThisRun.has(test));
@@ -247,6 +214,51 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho
   return rootSuite;
 }
 
+async function filterTestGroups(config: FullConfigInternal, testGroups: TestGroup[], filters: TestFilter[]): Promise<Set<TestCase>> {
+  let filteredTestGroups = testGroups.map(group => ({ tests: group.tests.map(test => test as reporterTypes.TestCase) }));
+  const allTests = new Set(filteredTestGroups.flatMap(group => group.tests));
+  for (const filter of filters) {
+    if ('filterTestGroups' in filter) {
+      const result = filter.filterTestGroups(filteredTestGroups, config.config);
+      filteredTestGroups = result instanceof Promise ? await result : result;
+    } else if ('filterTests' in filter) {
+      const result = filter.filterTests(filteredTestGroups.flatMap(group => group.tests), config.config);
+      const filteredTests = result instanceof Promise ? await result : result;
+      if (!Array.isArray(filteredTests))
+        throw new Error('Invalid filter result: tests should be an array');
+      const filteredTestsSet = new Set(filteredTests);
+      filteredTestGroups = filteredTestGroups.map(group => {
+        return {
+          tests: group.tests.filter(test => filteredTestsSet.has(test))
+        };
+      });
+    } else if (typeof filter === 'function') {
+      filteredTestGroups = filteredTestGroups.map(group => {
+        return {
+          tests: group.tests.filter(test => {
+            const result = filter(test);
+            if (typeof result !== 'boolean')
+              throw new Error('Invalid filter result: filter function should return a boolean');
+            return result;
+          })
+        };
+      });
+    }
+    // check if filtered groups are still valid
+    if (!Array.isArray(filteredTestGroups))
+      throw new Error('Invalid filter result: test groups should be an array');
+    for (const group of filteredTestGroups) {
+      if (!Array.isArray(group.tests))
+        throw new Error('Invalid filter result: tests should be an array');
+      for (const test of group.tests) {
+        if (!allTests.has(test))
+          throw new Error('Invalid filter result: test is not in the original list');
+      }
+    }
+  }
+  return new Set(filteredTestGroups.flatMap(group => group.tests).map(test => test as TestCase));
+}
+
 function createShardFilter(shard: { total: number, current: number }): TestFilter {
   return {
     filterTestGroups: (testGroups: TestGroup[]) => [...filterForShard(shard, testGroups).values()],

packages/playwright/types/test.d.ts

@@ -1886,8 +1886,8 @@ export type TestDetails = {
 }
 
 type TestFilterFunction = (test: TestCase) => boolean;
-type TestsFilter = { filterTests(tests: TestCase[], config: FullConfig): TestCase[] };
+type TestsFilter = { filterTests(tests: TestCase[], config: FullConfig): Promise<TestCase[]> | TestCase[] };
-type TestGroupsFilter = { filterTestGroups(testGroups: { tests: TestCase[] }[], config: FullConfig): { tests: TestCase[] }[] }
+type TestGroupsFilter = { filterTestGroups(testGroups: { tests: TestCase[] }[], config: FullConfig): Promise<{ tests: TestCase[] }[]> | { tests: TestCase[] }[] };
 export type TestFilter = TestFilterFunction | TestsFilter | TestGroupsFilter;
 
 interface SuiteFunction {

tests/playwright-test/test-filter.spec.ts

@@ -16,29 +16,47 @@
 
 import { test, expect } from './playwright-test-fixtures';
 
+const testFiles = {
+  'a1.test.ts': `
+    import { test, expect } from '@playwright/test';
+    test('a1-test1', async () => { console.log('\\n%% a1-test1'); });
+    test('a1-test2', async () => { console.log('\\n%% a1-test2'); });
+  `,
+  'a2.test.ts': `
+    import { test, expect } from '@playwright/test';
+    test('a2-test1', async () => { console.log('\\n%% a2-test1'); });
+    test('a2-test2', async () => { console.log('\\n%% a2-test2'); });
+  `,
+  'a3.test.ts': `
+    import { test, expect } from '@playwright/test';
+    test('a3-test1', async () => { console.log('\\n%% a3-test1'); });
+  `,
+  'a4.test.ts': `
+    import { test, expect } from '@playwright/test';
+    test('a4-test1', async () => { console.log('\\n%% a4-test1'); });
+  `,
+};
+
 test('config.filter function should work', async ({ runInlineTest }) => {
   const result = await runInlineTest({
+    ...testFiles,
     'playwright.config.ts': `
       module.exports = {
-        filter: (test) => test.title === 'test1',
+        filter: (test) => test.title === 'a2-test2',
       };
     `,
-    'a.test.ts': `
-      import { test, expect } from '@playwright/test';
-      test('test1', async () => { console.log('\\n%% test1'); });
-      test('test2', async () => { console.log('\\n%% test2'); });
-    `,
   }, { workers: 2 });
   expect(result.exitCode).toBe(0);
   expect(result.passed).toBe(1);
   result.outputLines.sort();
   expect(result.outputLines).toEqual([
-    'test1',
+    'a2-test2',
   ]);
 });
 
 test('config.filter filterTests should work', async ({ runInlineTest }) => {
   const result = await runInlineTest({
+    ...testFiles,
     'playwright.config.ts': `
       module.exports = {
         filter: {
@@ -46,73 +64,88 @@ test('config.filter filterTests should work', async ({ runInlineTest }) => {
         },
       };
     `,
-    'a.test.ts': `
-      import { test, expect } from '@playwright/test';
-      test('test1', async () => { console.log('\\n%% test1'); });
-      test('test2', async () => { console.log('\\n%% test2'); });
-      test('test3', async () => { console.log('\\n%% test3'); });
-      test('test4', async () => { console.log('\\n%% test4'); });
-    `,
   }, { workers: 2 });
   expect(result.exitCode).toBe(0);
-  expect(result.passed).toBe(2);
+  expect(result.passed).toBe(3);
   result.outputLines.sort();
   expect(result.outputLines).toEqual([
-    'test1',
+    'a1-test1',
-    'test3',
+    'a2-test1',
+    'a3-test1',
   ]);
 });
 
 test('config.filter filterTestGroups should work', async ({ runInlineTest }) => {
   const result = await runInlineTest({
+    ...testFiles,
     'playwright.config.ts': `
       module.exports = {
         filter: {
-          filterTestGroups: (testgroups) => testgroups.filter((testgroup, index) => index % 2 === 0),
+          filterTestGroups: (testgroups) => testgroups.filter((testgroup, index) => index % 2 === 1),
         },
       };
     `,
-    'a1.test.ts': `
-      import { test, expect } from '@playwright/test';
-      test('a1-test1', async () => { console.log('\\n%% a1-test1'); });
-      test('a1-test2', async () => { console.log('\\n%% a1-test2'); });
-    `,
-    'a2.test.ts': `
-      import { test, expect } from '@playwright/test';
-      test('a2-test1', async () => { console.log('\\n%% a2-test1'); });
-      test('a2-test2', async () => { console.log('\\n%% a2-test2'); });
-    `,
-    'a3.test.ts': `
-      import { test, expect } from '@playwright/test';
-      test('a3-test1', async () => { console.log('\\n%% a3-test1'); });
-    `,
-    'a4.test.ts': `
-      import { test, expect } from '@playwright/test';
-      test('a4-test1', async () => { console.log('\\n%% a4-test1'); });
-    `,
   }, { workers: 2 });
   expect(result.exitCode).toBe(0);
   expect(result.passed).toBe(3);
   result.outputLines.sort(); // Due to parallel execution, the order of output lines is not deterministic.
+  expect(result.outputLines).toEqual([
+    'a2-test1',
+    'a2-test2',
+    'a4-test1',
+  ]);
+});
+
+test('config.filter async filterTests should work', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    ...testFiles,
+    'playwright.config.ts': `
+      module.exports = {
+        filter: {
+          filterTests: (tests) => Promise.resolve(tests.filter((test, index) => index % 2 === 0)),
+        },
+      };
+    `,
+  }, { workers: 2 });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(3);
+  result.outputLines.sort();
   expect(result.outputLines).toEqual([
     'a1-test1',
-    'a1-test2',
+    'a2-test1',
     'a3-test1',
   ]);
 });
 
+test('config.filter async filterTestGroups should work', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    ...testFiles,
+    'playwright.config.ts': `
+      module.exports = {
+        filter: {
+          filterTestGroups: (testgroups) => Promise.resolve(testgroups.filter((testgroup, index) => index % 2 === 1)),
+        },
+      };
+    `,
+  }, { workers: 2 });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(3);
+  result.outputLines.sort(); // Due to parallel execution, the order of output lines is not deterministic.
+  expect(result.outputLines).toEqual([
+    'a2-test1',
+    'a2-test2',
+    'a4-test1',
+  ]);
+});
+
 test('config.filter invalid function should throw', async ({ runInlineTest }) => {
   const result = await runInlineTest({
+    ...testFiles,
     'playwright.config.ts': `
       module.exports = {
         filter: (test) => undefined,
       };
     `,
-    'a.test.ts': `
-      import { test, expect } from '@playwright/test';
-      test('test1', async () => { console.log('\\n%% test1'); });
-      test('test2', async () => { console.log('\\n%% test2'); });
-    `,
   }, { workers: 2 });
   expect(result.exitCode).toBe(1);
   expect(result.output).toContain('Error: Invalid filter result: filter function should return a boolean');
@@ -120,6 +153,7 @@ test('config.filter invalid function should throw', async ({ runInlineTest }) =>
 
 test('config.filter invalid filterTests should throw', async ({ runInlineTest }) => {
   const result = await runInlineTest({
+    ...testFiles,
     'playwright.config.ts': `
       module.exports = {
         filter: {
@@ -127,13 +161,6 @@ test('config.filter invalid filterTests should throw', async ({ runInlineTest })
         },
       };
     `,
-    'a.test.ts': `
-      import { test, expect } from '@playwright/test';
-      test('test1', async () => { console.log('\\n%% test1'); });
-      test('test2', async () => { console.log('\\n%% test2'); });
-      test('test3', async () => { console.log('\\n%% test3'); });
-      test('test4', async () => { console.log('\\n%% test4'); });
-    `,
   }, { workers: 2 });
   expect(result.exitCode).toBe(1);
   expect(result.output).toContain('Error: Invalid filter result: tests should be an array');
@@ -141,6 +168,7 @@ test('config.filter invalid filterTests should throw', async ({ runInlineTest })
 
 test('config.filter invalid filterTestGroups should throw', async ({ runInlineTest }) => {
   const result = await runInlineTest({
+    ...testFiles,
     'playwright.config.ts': `
       module.exports = {
         filter: {
@@ -148,24 +176,6 @@ test('config.filter invalid filterTestGroups should throw', async ({ runInlineTe
         },
       };
     `,
-    'a1.test.ts': `
-      import { test, expect } from '@playwright/test';
-      test('a1-test1', async () => { console.log('\\n%% a1-test1'); });
-      test('a1-test2', async () => { console.log('\\n%% a1-test2'); });
-    `,
-    'a2.test.ts': `
-      import { test, expect } from '@playwright/test';
-      test('a2-test1', async () => { console.log('\\n%% a2-test1'); });
-      test('a2-test2', async () => { console.log('\\n%% a2-test2'); });
-    `,
-    'a3.test.ts': `
-      import { test, expect } from '@playwright/test';
-      test('a3-test1', async () => { console.log('\\n%% a3-test1'); });
-    `,
-    'a4.test.ts': `
-      import { test, expect } from '@playwright/test';
-      test('a4-test1', async () => { console.log('\\n%% a4-test1'); });
-    `,
   }, { workers: 2 });
   expect(result.exitCode).toBe(1);
   expect(result.output).toContain('Error: Invalid filter result: test groups should be an array');

utils/generate_types/overrides-test.d.ts

@@ -78,8 +78,8 @@ export type TestDetails = {
 }
 
 type TestFilterFunction = (test: TestCase) => boolean;
-type TestsFilter = { filterTests(tests: TestCase[], config: FullConfig): TestCase[] };
+type TestsFilter = { filterTests(tests: TestCase[], config: FullConfig): Promise<TestCase[]> | TestCase[] };
-type TestGroupsFilter = { filterTestGroups(testGroups: { tests: TestCase[] }[], config: FullConfig): { tests: TestCase[] }[] }
+type TestGroupsFilter = { filterTestGroups(testGroups: { tests: TestCase[] }[], config: FullConfig): Promise<{ tests: TestCase[] }[]> | { tests: TestCase[] }[] };
 export type TestFilter = TestFilterFunction | TestsFilter | TestGroupsFilter;
 
 interface SuiteFunction {