feat(runner): allow filtering setup and tests (#18900)

Running `npx playwright test file:123` will have the following behavior
- if only test files match then only matching subset of tests will run
but all setup files will run as well
- if only setup files match the filter then only those setup tests will
run
- if both setup and test files match an error will be thrown

packages/playwright-test/src/reporters/base.ts

@@ -344,7 +344,7 @@ export function formatResultFailure(config: FullConfig, test: TestCase, result:
   return errorDetails;
 }
 
-function relativeFilePath(config: FullConfig, file: string): string {
+export function relativeFilePath(config: FullConfig, file: string): string {
   return path.relative(config.rootDir, file) || path.basename(file);
 }
 

packages/playwright-test/src/runner.ts

@@ -29,7 +29,7 @@ import type { TestRunnerPlugin } from './plugins';
 import { setRunnerToAddPluginsTo } from './plugins';
 import { dockerPlugin } from './plugins/dockerPlugin';
 import { webServerPluginsForConfig } from './plugins/webServerPlugin';
-import { formatError } from './reporters/base';
+import { formatError, relativeFilePath } from './reporters/base';
 import DotReporter from './reporters/dot';
 import EmptyReporter from './reporters/empty';
 import GitHubReporter from './reporters/github';
@@ -196,7 +196,7 @@ export class Runner {
 
   async listTestFiles(projectNames: string[] | undefined): Promise<any> {
     const projects = this._collectProjects(projectNames);
-    const { filesByProject } = await this._collectFiles(projects, () => true);
+    const { filesByProject } = await this._collectFiles(projects, []);
     const report: any = {
       projects: []
     };
@@ -235,12 +235,13 @@ export class Runner {
     return projects;
   }
 
-  private async _collectFiles(projects: FullProjectInternal[], testFileFilter: Matcher): Promise<{filesByProject: Map<FullProjectInternal, string[]>; setupFiles: Set<string>}> {
+  private async _collectFiles(projects: FullProjectInternal[], commandLineFileFilters: TestFileFilter[]): Promise<{filesByProject: Map<FullProjectInternal, string[]>; setupFiles: Set<string>, applyFilterToSetup: boolean}> {
     const extensions = ['.js', '.ts', '.mjs', '.tsx', '.jsx'];
     const testFileExtension = (file: string) => extensions.includes(path.extname(file));
     const filesByProject = new Map<FullProjectInternal, string[]>();
     const setupFiles = new Set<string>();
     const fileToProjectName = new Map<string, string>();
+    const commandLineFileMatcher = fileMatcherFrom(commandLineFileFilters);
     for (const project of projects) {
       const allFiles = await collectFiles(project.testDir, project._respectGitIgnore);
       const setupMatch = createFileMatcher(project._setup);
@@ -250,7 +251,7 @@ export class Runner {
         if (!testFileExtension(file))
           return false;
         const isSetup = setupMatch(file);
-        const isTest = !testIgnore(file) && testMatch(file) && testFileFilter(file);
+        const isTest = !testIgnore(file) && testMatch(file) && commandLineFileMatcher(file);
         if (!isTest && !isSetup)
           return false;
         if (isSetup && isTest)
@@ -270,13 +271,40 @@ export class Runner {
       });
       filesByProject.set(project, testFiles);
     }
-    return { filesByProject, setupFiles };
+
+    // If the filter didn't match any tests, apply it to the setup files.
+    const applyFilterToSetup = setupFiles.size === fileToProjectName.size;
+    if (applyFilterToSetup) {
+      // We now have only setup files in filesByProject, because all test files were filtered out.
+      for (const [project, setupFiles] of filesByProject) {
+        const filteredFiles = setupFiles.filter(commandLineFileMatcher);
+        if (filteredFiles.length)
+          filesByProject.set(project, filteredFiles);
+        else
+          filesByProject.delete(project);
+      }
+      for (const file of setupFiles) {
+        if (!commandLineFileMatcher(file))
+          setupFiles.delete(file);
+      }
+    } else if (commandLineFileFilters.length) {
+      const setupFile = [...setupFiles].find(commandLineFileMatcher);
+      // If the filter is not empty and it matches both setup and tests then it's an error: we allow
+      // to run either subset of tests with full setup or partial setup without any tests.
+      if (setupFile) {
+        const testFile = Array.from(fileToProjectName.keys()).find(f => !setupFiles.has(f));
+        const config = this._loader.fullConfig();
+        throw new Error(`Both setup and test files match command line filter.\n  Setup file: ${relativeFilePath(config, setupFile)}\n  Test file: ${relativeFilePath(config, testFile!)}`);
+      }
+    }
+
+    return { filesByProject, setupFiles, applyFilterToSetup };
   }
 
   private async _collectTestGroups(options: RunOptions, fatalErrors: TestError[]): Promise<{ rootSuite: Suite, projectSetupGroups: TestGroup[], testGroups: TestGroup[] }> {
     const config = this._loader.fullConfig();
     const projects = this._collectProjects(options.projectFilter);
-    const { filesByProject, setupFiles } = await this._collectFiles(projects, fileMatcherFrom(options.testFileFilters));
+    const { filesByProject, setupFiles, applyFilterToSetup } = await this._collectFiles(projects, options.testFileFilters);
 
     const allTestFiles = new Set<string>();
     for (const files of filesByProject.values())
@@ -298,7 +326,7 @@ export class Runner {
 
     // Filter tests to respect line/column filter.
     if (options.testFileFilters.length)
-      filterByFocusedLine(preprocessRoot, options.testFileFilters, setupFiles);
+      filterByFocusedLine(preprocessRoot, options.testFileFilters, applyFilterToSetup ? new Set() : setupFiles);
 
     // Complain about only.
     // TODO: check in project setup.

tests/playwright-test/playwright-test-fixtures.ts

@@ -316,7 +316,7 @@ export const test = base
 
           let timeline;
           try {
-            timeline = JSON.parse((await fs.promises.readFile(timelinePath, 'utf8')).toString('utf8'));
+            timeline = JSON.parse((await fs.promises.readFile(timelinePath, 'utf8')).toString());
           } catch (e) {
           }
           return {

tests/playwright-test/project-setup.spec.ts

@@ -425,3 +425,160 @@ test('test --list should enumerate setup tests as regular ones', async ({ runCom
   [p2] › b.test.ts:6:7 › test4
 Total: 5 tests in 5 files`);
 });
+
+test('should allow .only in setup files', async ({ runGroups }, testInfo) => {
+  const files = {
+    'playwright.config.ts': `
+      module.exports = {
+        projects: [
+          {
+            name: 'p1',
+            setup: /.*.setup.ts/,
+          },
+        ]
+      };`,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('test1', async () => { });
+      test('test2', async () => { });
+      test('test3', async () => { });
+      test('test4', async () => { });
+    `,
+    'a.setup.ts': `
+      const { test } = pwt;
+      test.only('setup1', async () => { });
+      test('setup2', async () => { });
+      test.only('setup3', async () => { });
+    `,
+  };
+
+  const { exitCode, passed, timeline, output } =  await runGroups(files);
+  expect(output).toContain('Running 2 tests using 1 worker');
+  expect(output).toContain('[p1] › a.setup.ts:5:12 › setup1');
+  expect(output).toContain('[p1] › a.setup.ts:7:12 › setup3');
+  expect(fileNames(timeline)).toEqual(['a.setup.ts']);
+  expect(exitCode).toBe(0);
+  expect(passed).toBe(2);
+});
+
+test('should allow filtering setup by file:line', async ({ runGroups }, testInfo) => {
+  const files = {
+    'playwright.config.ts': `
+      module.exports = {
+        projects: [
+          {
+            name: 'p1',
+            setup: /.*a.setup.ts/,
+          },
+          {
+            name: 'p2',
+            setup: /.*b.setup.ts/,
+          },
+        ]
+      };`,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('test1', async () => { });
+      test('test2', async () => { });
+      test('test3', async () => { });
+    `,
+    'a.setup.ts': `
+      const { test } = pwt;
+      test('setup1', async () => { });
+      test('setup2', async () => { });
+    `,
+    'b.setup.ts': `
+      const { test } = pwt;
+      test('setup1', async () => { });
+    `,
+    'b.test.ts': `
+      const { test } = pwt;
+      test('test1', async () => { });
+      test('test2', async () => { });
+      test('test3', async () => { });
+    `,
+  };
+
+  {
+    const { exitCode, passed, output } =  await runGroups(files, undefined, undefined, { additionalArgs: ['.*setup.ts$'] });
+    expect(output).toContain('Running 3 tests using 2 workers');
+    expect(output).toContain('[p1] › a.setup.ts:5:7 › setup1');
+    expect(output).toContain('[p1] › a.setup.ts:6:7 › setup2');
+    expect(output).toContain('[p2] › b.setup.ts:5:7 › setup1');
+    expect(exitCode).toBe(0);
+    expect(passed).toBe(3);
+  }
+  {
+    const { exitCode, passed, output } =  await runGroups(files, undefined, undefined, { additionalArgs: ['.*a.setup.ts:5'] });
+    expect(output).toContain('Running 1 test using 1 worker');
+    expect(output).toContain('[p1] › a.setup.ts:5:7 › setup1');
+    expect(exitCode).toBe(0);
+    expect(passed).toBe(1);
+  }
+});
+
+test('should prohibit filters matching both setup and test', async ({ runGroups }, testInfo) => {
+  const files = {
+    'playwright.config.ts': `
+      module.exports = {
+        projects: [
+          {
+            name: 'p1',
+            setup: /.*.setup.ts/,
+          },
+        ]
+      };`,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('test1', async () => { });
+      test('test2', async () => { });
+      test('test3', async () => { });
+    `,
+    'a.setup.ts': `
+      const { test } = pwt;
+      test('setup1', async () => { });
+      test('setup2', async () => { });
+    `,
+  };
+
+  const { exitCode, output } =  await runGroups(files, undefined, undefined, { additionalArgs: ['.*ts$'] });
+  expect(output).toContain('Error: Both setup and test files match command line filter.');
+  expect(exitCode).toBe(1);
+});
+
+test('should run all setup files if only tests match filter', async ({ runGroups }, testInfo) => {
+  const files = {
+    'playwright.config.ts': `
+      module.exports = {
+        projects: [
+          {
+            name: 'p1',
+            setup: /.*.setup.ts/,
+          },
+        ]
+      };`,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('test1', async () => { });
+      test('test2', async () => { });
+      test('test3', async () => { });
+    `,
+    'a.setup.ts': `
+      const { test } = pwt;
+      test('setup1', async () => { });
+      test('setup2', async () => { });
+    `,
+    'b.setup.ts': `
+      const { test } = pwt;
+      test('setup1', async () => { });
+    `,
+  };
+
+  const { exitCode, output } =  await runGroups(files, undefined, undefined, { additionalArgs: ['a.test.ts:7'] });
+  expect(exitCode).toBe(0);
+  expect(output).toContain('Running 4 tests using 2 workers');
+  expect(output).toContain('[p1] › a.setup.ts:5:7 › setup1');
+  expect(output).toContain('[p1] › a.setup.ts:6:7 › setup2');
+  expect(output).toContain('[p1] › b.setup.ts:5:7 › setup1');
+  expect(output).toContain('[p1] › a.test.ts:7:7 › test2');
+});