chore: extract --project-grep option for matching with regex (#29445)

Reference https://github.com/microsoft/playwright/issues/15128

docs/src/test-cli-js.md

@@ -92,7 +92,8 @@ Complete set of Playwright Test options is available in the [configuration file]
 | `--no-deps` | Ignore the dependencies between projects and behave as if they were not specified. |
 | `--output <dir>` | Directory for artifacts produced by tests, defaults to `test-results`. |
 | `--pass-with-no-tests` | Allows the test suite to pass when no files are found. |
-| `--project <name>` | Only run tests from the projects matching this regular expression. Defaults to running all projects defined in the configuration file.|
+| `--project <name>` | Only run tests from the specified [projects](./test-projects.md). Defaults to running all projects defined in the configuration file.|
+| `--project-grep <name>` | Only run tests from the projects matching this regular expression. Defaults to running all projects defined in the configuration file.|
 | `--quiet` | Whether to suppress stdout and stderr from the tests. |
 | `--repeat-each <N>` | Run each test `N` times, defaults to one. |
 | `--reporter <reporter>` | Choose a reporter: minimalist `dot`, concise `line` or detailed `list`. See [reporters](./test-reporters.md) for more information. |

packages/playwright/src/common/config.ts

@@ -48,6 +48,7 @@ export class FullConfigInternal {
   cliGrepInvert: string | undefined;
   cliTagFilter: string | undefined;
   cliProjectFilter?: string[];
+  cliProjectGrep?: string;
   cliListOnly = false;
   cliPassWithNoTests?: boolean;
   testIdMatcher?: Matcher;

packages/playwright/src/program.ts

@@ -64,7 +64,8 @@ function addListFilesCommand(program: Command) {
   const command = program.command('list-files [file-filter...]', { hidden: true });
   command.description('List files with Playwright Test tests');
   command.option('-c, --config <file>', `Configuration file, or a test directory with optional "playwright.config.{m,c}?{js,ts}"`);
-  command.option('--project <project-name...>', `Only run tests from the projects matching this regular expression (default: list all projects)`);
+  command.option('--project <project-name...>', `Only run tests from the specified list of projects (default: list all projects)`);
+  command.option('--project-grep <pattern>', `Only run tests from the projects matching this regular expression (default: list all projects)`);
   command.action(async (args, opts) => {
     try {
       await listTestFiles(opts);
@@ -166,12 +167,16 @@ async function runTests(args: string[], opts: { [key: string]: any }) {
   if (!config)
     return;
 
+  if (opts.project && opts.projectGrep)
+    throw new Error('Only one of --project and --project-grep can be specified.');
+
   config.cliArgs = args;
   config.cliGrep = opts.grep as string | undefined;
   config.cliGrepInvert = opts.grepInvert as string | undefined;
   config.cliTagFilter = opts.tag;
   config.cliListOnly = !!opts.list;
   config.cliProjectFilter = opts.project || undefined;
+  config.cliProjectGrep = opts.projectGrep || undefined;
   config.cliPassWithNoTests = !!opts.passWithNoTests;
 
   const runner = new Runner(config);
@@ -210,7 +215,9 @@ export async function withRunnerAndMutedWrite(configFile: string | undefined, ca
 }
 
 async function listTestFiles(opts: { [key: string]: any }) {
-  await withRunnerAndMutedWrite(opts.config, async runner => runner.listTestFiles(opts.project));
+  if (opts.project && opts.projectGrep)
+    throw new Error('Only one of --project and --project-grep can be specified.');
+  await withRunnerAndMutedWrite(opts.config, async runner => runner.listTestFiles(opts.project, opts.projectGrep));
 }
 
 async function mergeReports(reportDir: string | undefined, opts: { [key: string]: any }) {
@@ -323,7 +330,8 @@ const testOptions: [string, string][] = [
   ['--no-deps', 'Do not run project dependencies'],
   ['--output <dir>', `Folder for output artifacts (default: "test-results")`],
   ['--pass-with-no-tests', `Makes test run succeed even if no tests were found`],
-  ['--project <project-name...>', `Only run tests from the projects matching this regular expression (default: run all projects)`],
+  ['--project <project-name...>', `Only run tests from the specified list of projects(default: run all projects)`],
+  ['--project-grep <pattern>', `Only run tests from the projects matching this regular expression (default: run all projects)`],
   ['--quiet', `Suppress stdio`],
   ['--repeat-each <N>', `Run each test N times (default: 1)`],
   ['--reporter <reporter>', `Reporter to use, comma-separated, can be ${builtInReporters.map(name => `"${name}"`).join(', ')} (default: "${defaultReporter}")`],

packages/playwright/src/runner/loadUtils.ts

@@ -40,7 +40,7 @@ export async function collectProjectsAndTestFiles(testRun: TestRun, doNotRunTest
 
   // First collect all files for the projects in the command line, don't apply any file filters.
   const allFilesForProject = new Map<FullProjectInternal, string[]>();
-  const filteredProjects = filterProjects(config.projects, config.cliProjectFilter);
+  const filteredProjects = filterProjects(config.projects, config.cliProjectFilter, config.cliProjectGrep);
   for (const project of filteredProjects) {
     const files = await collectFilesForProject(project, fsCache);
     allFilesForProject.set(project, files);

packages/playwright/src/runner/projectUtils.ts

@@ -19,39 +19,48 @@ import path from 'path';
 import { minimatch } from 'playwright-core/lib/utilsBundle';
 import { promisify } from 'util';
 import type { FullProjectInternal } from '../common/config';
-import { createFileMatcher } from '../util';
+import { createFileMatcher, forceRegExp } from '../util';
 
 const readFileAsync = promisify(fs.readFile);
 const readDirAsync = promisify(fs.readdir);
 
-// The difference to forceRegExp is that we want to match the whole string.
+export function filterProjects(projects: FullProjectInternal[], projectNames?: string[], projectGrep?: string): FullProjectInternal[] {
-function forceBoundedRegExp(pattern: string): RegExp {
+  if (!projectNames && !projectGrep)
-  const match = pattern.match(/^\/(.*)\/([gi]*)$/);
-  if (match)
-    return new RegExp(match[1], match[2]);
-  return new RegExp(`^${pattern}$`, 'gi');
-}
-
-export function filterProjects(projects: FullProjectInternal[], projectNames?: string[]): FullProjectInternal[] {
-  if (!projectNames)
     return [...projects];
-  const unmatchedProjectFilters = new Set<string>(projectNames);
+
+  if (projectGrep) {
+    const regex = forceRegExp(projectGrep);
+    const result = projects.filter(project => {
+      regex.lastIndex = 0;
+      return regex.test(project.project.name);
+    });
+    if (!result.length)
+      throw new Error(`Projects matching "${projectGrep}" not found. Available projects: ${projects.map(p => `"${p.project.name}"`).join(', ')}`);
+    return result;
+  }
+
+  const projectNamesToFind = new Set<string>();
+  const unmatchedProjectNames = new Map<string, string>();
+  for (const name of projectNames!) {
+    const lowerCaseName = name.toLocaleLowerCase();
+    projectNamesToFind.add(lowerCaseName);
+    unmatchedProjectNames.set(lowerCaseName, name);
+  }
+
   const result = projects.filter(project => {
-    for (const projectName of projectNames) {
+    const lowerCaseName = project.project.name.toLocaleLowerCase();
-      if (forceBoundedRegExp(projectName).test(project.project.name)) {
+    if (projectNamesToFind.has(lowerCaseName)) {
-        unmatchedProjectFilters.delete(projectName);
+      unmatchedProjectNames.delete(lowerCaseName);
-        return true;
+      return true;
-      }
     }
     return false;
   });
-  if (unmatchedProjectFilters.size) {
+
-    const names = projects.map(p => p.project.name).filter(name => !!name);
+  if (unmatchedProjectNames.size) {
-    if (!names.length)
+    const unknownProjectNames = Array.from(unmatchedProjectNames.values()).map(n => `"${n}"`).join(', ');
-      throw new Error(`No named projects are specified in the configuration file`);
+    throw new Error(`Project(s) ${unknownProjectNames} not found. Available projects: ${projects.map(p => `"${p.project.name}"`).join(', ')}`);
-    const unknownProjectNames = Array.from(unmatchedProjectFilters.values()).map(n => `"${n}"`).join(', ');
-    throw new Error(`Project(s) ${unknownProjectNames} not found. Available named projects: ${names.map(name => `"${name}"`).join(', ')}`);
   }
+
   return result;
 }
 

packages/playwright/src/runner/runner.ts

@@ -48,8 +48,8 @@ export class Runner {
     this._config = config;
   }
 
-  async listTestFiles(projectNames: string[] | undefined): Promise<any> {
+  async listTestFiles(projectNames: string[] | undefined, projectGrep: string | undefined): Promise<any> {
-    const projects = filterProjects(this._config.projects, projectNames);
+    const projects = filterProjects(this._config.projects, projectNames, projectGrep);
     const report: ConfigListFilesReport = {
       projects: []
     };

packages/playwright/src/runner/tasks.ts

@@ -29,7 +29,7 @@ import type { FullConfigInternal, FullProjectInternal } from '../common/config';
 import { collectProjectsAndTestFiles, createRootSuite, loadFileSuites, loadGlobalHook } from './loadUtils';
 import type { Matcher } from '../util';
 import type { Suite } from '../common/test';
-import { buildDependentProjects, buildTeardownToSetupsMap } from './projectUtils';
+import { buildDependentProjects, buildTeardownToSetupsMap, filterProjects } from './projectUtils';
 import { FailureTracker } from './failureTracker';
 
 const readDirAsync = promisify(fs.readdir);
@@ -168,10 +168,8 @@ function createRemoveOutputDirsTask(): Task<TestRun> {
       if (process.env.PW_TEST_NO_REMOVE_OUTPUT_DIRS)
         return;
       const outputDirs = new Set<string>();
-      for (const p of config.projects) {
+      const projects = filterProjects(config.projects, config.cliProjectFilter, config.cliProjectGrep);
-        if (!config.cliProjectFilter || config.cliProjectFilter.includes(p.project.name))
+      projects.forEach(p => outputDirs.add(p.project.outputDir));
-          outputDirs.add(p.project.outputDir);
-      }
 
       await Promise.all(Array.from(outputDirs).map(outputDir => removeFolders([outputDir]).then(async ([error]) => {
         if (!error)

packages/playwright/src/runner/watchMode.ts

@@ -40,7 +40,7 @@ class FSWatcher {
 
   async update(config: FullConfigInternal) {
     const commandLineFileMatcher = config.cliArgs.length ? createFileMatcherFromArguments(config.cliArgs) : () => true;
-    const projects = filterProjects(config.projects, config.cliProjectFilter);
+    const projects = filterProjects(config.projects, config.cliProjectFilter, config.cliProjectGrep);
     const projectClosure = buildProjectsClosure(projects);
     const projectFilters = new Map<FullProjectInternal, Matcher>();
     for (const [project, type] of projectClosure) {
@@ -263,7 +263,7 @@ async function runChangedTests(config: FullConfigInternal, failedTestIdCollector
 
   // Collect all the affected projects, follow project dependencies.
   // Prepare to exclude all the projects that do not depend on this file, as if they did not exist.
-  const projects = filterProjects(config.projects, config.cliProjectFilter);
+  const projects = filterProjects(config.projects, config.cliProjectFilter, config.cliProjectGrep);
   const projectClosure = buildProjectsClosure(projects);
   const affectedProjects = affectedProjectsClosure([...projectClosure.keys()], [...filesByProject.keys()]);
   const affectsAnyDependency = [...affectedProjects].some(p => projectClosure.get(p) === 'dependency');
@@ -388,6 +388,8 @@ function printConfiguration(config: FullConfigInternal, title?: string) {
   const tokens: string[] = [];
   tokens.push(`${packageManagerCommand} playwright test`);
   tokens.push(...(config.cliProjectFilter || [])?.map(p => colors.blue(`--project ${p}`)));
+  if (config.cliProjectGrep)
+    tokens.push(colors.blue(`--project-grep ${config.cliProjectGrep}`));
   if (config.cliGrep)
     tokens.push(colors.red(`--grep ${config.cliGrep}`));
   if (config.cliArgs)

tests/playwright-test/config.spec.ts

@@ -245,34 +245,12 @@ test('should filter by project, case-insensitive', async ({ runInlineTest }) =>
   ]));
 });
 
-test('should filter by project and parse as RegExp', async ({ runInlineTest }) => {
+test('should filter by project-grep', async ({ runInlineTest }) => {
   const result = await runInlineTest({
     'playwright.config.js': `
       module.exports = {
         projects: [
-         { name: 'project-name' }
+         { name: 'project-name' },
-        ]
-      };
-    `,
-    'a.test.js': `
-      const { test } = require('@playwright/test');
-      test('one', async ({}) => {
-        console.log('%%' + test.info().project.name);
-      });    `
-  }, { project: '.*oj.*t-Na.?e' });
-  expect(result.exitCode).toBe(0);
-  expect(result.output).toContain('Running 1 test using 1 worker');
-  expect(new Set(result.outputLines)).toEqual(new Set([
-    'project-name',
-  ]));
-});
-
-test('should filter by project and only match if its full-match', async ({ runInlineTest }) => {
-  const result = await runInlineTest({
-    'playwright.config.js': `
-      module.exports = {
-        projects: [
-         { name: 'prefix-foobar-suffix' },
          { name: 'foobar' }
         ]
       };
@@ -282,10 +260,50 @@ test('should filter by project and only match if its full-match', async ({ runIn
       test('one', async ({}) => {
         console.log('%%' + test.info().project.name);
       });    `
-  }, { project: 'foobar' });
+  }, { '--project-grep': '.*oj.*t-Na.?e' });
   expect(result.exitCode).toBe(0);
   expect(result.output).toContain('Running 1 test using 1 worker');
-  expect(new Set(result.outputLines)).toEqual(new Set(['foobar']));
+  expect(new Set(result.outputLines)).toEqual(new Set([
+    'project-name',
+  ]));
+});
+
+test('should print nice error when the project grep does not match anything', async ({ runInlineTest }) => {
+  const { output, exitCode } = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { projects: [
+        { name: 'suite1' },
+        { name: 'suite2' },
+      ] };
+    `,
+    'a.test.ts': `
+      import { test, expect } from '@playwright/test';
+      test('pass', async ({}, testInfo) => {
+        console.log(testInfo.project.name);
+      });
+    `
+  }, { '--project-grep': ['aaa'] });
+  expect(exitCode).toBe(1);
+  expect(output).toContain('Error: Projects matching \"aaa\" not found. Available projects: \"suite1\", \"suite2\"');
+});
+
+test('should fail if both --project and --project-grep are passed', async ({ runInlineTest }) => {
+  const { output, exitCode } = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { projects: [
+        { name: 'suite1' },
+        { name: 'suite2' },
+      ] };
+    `,
+    'a.test.ts': `
+      import { test, expect } from '@playwright/test';
+      test('pass', async ({}, testInfo) => {
+        console.log(testInfo.project.name);
+      });
+    `
+  }, { '--project-grep': 'foo', '--project': 'bar' });
+  expect(exitCode).toBe(1);
+  expect(output).toContain('Only one of --project and --project-grep can be specified');
 });
 
 test('should filter by project and allow passing RegExp start/end flags', async ({ runInlineTest }) => {
@@ -304,7 +322,7 @@ test('should filter by project and allow passing RegExp start/end flags', async
       test('one', async ({}) => {
         console.log('%%' + test.info().project.name);
       });    `
-  }, { project: '/fooBar$/' });
+  }, { '--project-grep': '/fooBar$/' });
   expect(result.exitCode).toBe(0);
   expect(new Set(result.outputLines)).toEqual(new Set(['prefix-fooBar', 'fooBar']));
 });
@@ -323,7 +341,7 @@ test('should print nice error when project is unknown', async ({ runInlineTest }
     `
   }, { project: 'suite3' });
   expect(exitCode).toBe(1);
-  expect(output).toContain('Project(s) "suite3" not found. Available named projects: "suite1", "suite2"');
+  expect(output).toContain('Project(s) "suite3" not found. Available projects: "suite1", "suite2"');
 });
 
 test('should filter by project list, case-insensitive', async ({ runInlineTest }) => {
@@ -388,7 +406,7 @@ test('should print nice error when some of the projects are unknown', async ({ r
     `
   }, { project: ['suitE1', 'suIte3', 'SUite4'] });
   expect(exitCode).toBe(1);
-  expect(output).toContain('Project(s) "suIte3", "SUite4" not found. Available named projects: "suite1", "suite2"');
+  expect(output).toContain('Project(s) "suIte3", "SUite4" not found. Available projects: "suite1", "suite2"');
 });
 
 test('should work without config file', async ({ runInlineTest }) => {

tests/playwright-test/list-files.spec.ts

@@ -48,6 +48,30 @@ test('should list files', async ({ runCLICommand }) => {
   });
 });
 
+test('should support project-grep list files', async ({ runCLICommand }) => {
+  const result = await runCLICommand({
+    'playwright.config.ts': `
+      module.exports = { projects: [{ name: 'foo' }, { name: 'bar' }] };
+    `,
+    'a.test.js': ``
+  }, 'list-files', ['--project-grep', 'f.o']);
+  expect(result.exitCode).toBe(0);
+
+  const data = JSON.parse(result.stdout);
+  expect(data).toEqual({
+    projects: [
+      {
+        name: 'foo',
+        testDir: expect.stringContaining('list-files-should-support-project-grep-list-files-playwright-test'),
+        use: {},
+        files: [
+          expect.stringContaining('a.test.js')
+        ]
+      }
+    ]
+  });
+});
+
 test('should include testIdAttribute', async ({ runCLICommand }) => {
   const result = await runCLICommand({
     'playwright.config.ts': `

tests/playwright-test/playwright-test-fixtures.ts

@@ -105,7 +105,7 @@ function findPackageJSONDir(files: Files, dir: string) {
   return dir;
 }
 
-function startPlaywrightTest(childProcess: CommonFixtures['childProcess'], baseDir: string, params: any, env: NodeJS.ProcessEnv, options: RunOptions): TestChildProcess {
+function toParamList(params: any): string[] {
   const paramList: string[] = [];
   for (const key of Object.keys(params)) {
     for (const value of Array.isArray(params[key]) ? params[key] : [params[key]]) {
@@ -113,6 +113,11 @@ function startPlaywrightTest(childProcess: CommonFixtures['childProcess'], baseD
       paramList.push(params[key] === true ? `${k}` : `${k}=${value}`);
     }
   }
+  return paramList;
+}
+
+function startPlaywrightTest(childProcess: CommonFixtures['childProcess'], baseDir: string, params: any, env: NodeJS.ProcessEnv, options: RunOptions): TestChildProcess {
+  const paramList = toParamList(params);
   const args = ['test'];
   args.push(
       '--workers=2',