feat(runner): support multiple names in project filter (#8600)

src/test/cli.ts

@@ -57,7 +57,7 @@ export function addTestCommand(program: commander.CommanderStatic) {
   command.option('--reporter <reporter>', `Reporter to use, comma-separated, can be ${builtInReporters.map(name => `"${name}"`).join(', ')} (default: "${defaultReporter}")`);
   command.option('--retries <retries>', `Maximum retry count for flaky tests, zero for no retries (default: no retries)`);
   command.option('--shard <shard>', `Shard tests and execute only the selected shard, specify in the form "current/all", 1-based, for example "3/5"`);
-  command.option('--project <project-name>', `Only run tests from the specified project (default: run all projects)`);
+  command.option('--project <project-name...>', `Only run tests from the specified list of projects (default: run all projects)`);
   command.option('--timeout <timeout>', `Specify test timeout threshold in milliseconds, zero for unlimited (default: ${defaultTimeout})`);
   command.option('-u, --update-snapshots', `Update snapshots with actual results (default: only create missing snapshots)`);
   command.option('-x', `Stop after the first failure`);

src/test/runner.ts

@@ -93,11 +93,11 @@ export class Runner {
     this._loader.loadEmptyConfig(rootDir);
   }
 
-  async run(list: boolean, filePatternFilters: FilePatternFilter[], projectName?: string): Promise<RunResultStatus> {
+  async run(list: boolean, filePatternFilters: FilePatternFilter[], projectNames?: string[]): Promise<RunResultStatus> {
     this._reporter = await this._createReporter(list);
     const config = this._loader.fullConfig();
     const globalDeadline = config.globalTimeout ? config.globalTimeout + monotonicTime() : 0;
-    const { result, timedOut } = await raceAgainstDeadline(this._run(list, filePatternFilters, projectName), globalDeadline);
+    const { result, timedOut } = await raceAgainstDeadline(this._run(list, filePatternFilters, projectNames), globalDeadline);
     if (timedOut) {
       if (!this._didBegin)
         this._reporter.onBegin?.(config, new Suite(''));
@@ -137,18 +137,34 @@ export class Runner {
     await new Promise<void>(resolve => process.stderr.write('', () => resolve()));
   }
 
-  async _run(list: boolean, testFileReFilters: FilePatternFilter[], projectName?: string): Promise<RunResult> {
+  async _run(list: boolean, testFileReFilters: FilePatternFilter[], projectNames?: string[]): Promise<RunResult> {
     const testFileFilter = testFileReFilters.length ? createMatcher(testFileReFilters.map(e => e.re)) : () => true;
     const config = this._loader.fullConfig();
 
+    let projectsToFind: Set<string> | undefined;
+    let unknownProjects: Map<string, string> | undefined;
+    if (projectNames) {
+      projectsToFind = new Set();
+      unknownProjects = new Map();
+      projectNames.forEach(n => {
+        const name = n.toLocaleLowerCase();
+        projectsToFind!.add(name);
+        unknownProjects!.set(name, n);
+      });
+    }
     const projects = this._loader.projects().filter(project => {
-      return !projectName || project.config.name.toLocaleLowerCase() === projectName.toLocaleLowerCase();
+      if (!projectsToFind)
+        return true;
+      const name = project.config.name.toLocaleLowerCase();
+      unknownProjects!.delete(name);
+      return projectsToFind.has(name);
     });
-    if (projectName && !projects.length) {
+    if (unknownProjects && unknownProjects.size) {
       const names = this._loader.projects().map(p => p.config.name).filter(name => !!name);
       if (!names.length)
         throw new Error(`No named projects are specified in the configuration file`);
-      throw new Error(`Project "${projectName}" not found. Available named projects: ${names.map(name => `"${name}"`).join(', ')}`);
+      const unknownProjectNames = Array.from(unknownProjects.values()).map(n => `"${n}"`).join(', ');
+      throw new Error(`Project(s) ${unknownProjectNames} not found. Available named projects: ${names.map(name => `"${name}"`).join(', ')}`);
     }
 
     const files = new Map<ProjectImpl, string[]>();

tests/playwright-test/config.spec.ts

@@ -259,7 +259,77 @@ test('should print nice error when project is unknown', async ({ runInlineTest }
     `
   }, { project: 'suite3' });
   expect(exitCode).toBe(1);
-  expect(output).toContain('Project "suite3" not found. Available named projects: "suite1", "suite2"');
+  expect(output).toContain('Project(s) "suite3" not found. Available named projects: "suite1", "suite2"');
+});
+
+test('should filter by project list, case-insensitive', async ({ runInlineTest }) => {
+  const { passed, failed, output, skipped } = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { projects: [
+        { name: 'suite1' },
+        { name: 'suite2' },
+        { name: 'suite3' },
+        { name: 'suite4' },
+      ] };
+    `,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('pass', async ({}, testInfo) => {
+        console.log(testInfo.project.name);
+      });
+    `
+  }, { project: ['SUite2',  'Suite3'] });
+  expect(passed).toBe(2);
+  expect(failed).toBe(0);
+  expect(skipped).toBe(0);
+  expect(output).toContain('suite2');
+  expect(output).toContain('suite3');
+  expect(output).not.toContain('suite1');
+  expect(output).not.toContain('suite4');
+});
+
+test('should filter when duplicate project names exist', async ({ runInlineTest }) => {
+  const { passed, failed, output, skipped } = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { projects: [
+        { name: 'suite1' },
+        { name: 'suite2' },
+        { name: 'suite1' },
+        { name: 'suite4' },
+      ] };
+    `,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('pass', async ({}, testInfo) => {
+        console.log(testInfo.project.name);
+      });
+    `
+  }, { project: ['suite1',  'sUIte4'] });
+  expect(passed).toBe(3);
+  expect(failed).toBe(0);
+  expect(skipped).toBe(0);
+  expect(output).toContain('suite1');
+  expect(output).toContain('suite4');
+  expect(output).not.toContain('suite2');
+});
+
+test('should print nice error when some of the projects are unknown', async ({ runInlineTest }) => {
+  const { output, exitCode } = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { projects: [
+        { name: 'suite1' },
+        { name: 'suite2' },
+      ] };
+    `,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('pass', async ({}, testInfo) => {
+        console.log(testInfo.project.name);
+      });
+    `
+  }, { project: ['suitE1', 'suIte3', 'SUite4'] });
+  expect(exitCode).toBe(1);
+  expect(output).toContain('Project(s) "suIte3", "SUite4" not found. Available named projects: "suite1", "suite2"');
 });
 
 test('should work without config file', async ({ runInlineTest }) => {