From bafa426231e447163a3f2bbc1d7602dd8b97ff4a Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Thu, 2 Sep 2021 09:29:55 -0700 Subject: [PATCH] feat(runner): support multiple names in project filter (#8600) --- src/test/cli.ts | 2 +- src/test/runner.ts | 28 ++++++++--- tests/playwright-test/config.spec.ts | 72 +++++++++++++++++++++++++++- 3 files changed, 94 insertions(+), 8 deletions(-) diff --git a/src/test/cli.ts b/src/test/cli.ts index e151a7a8ff..86986b599d 100644 --- a/src/test/cli.ts +++ b/src/test/cli.ts @@ -57,7 +57,7 @@ export function addTestCommand(program: commander.CommanderStatic) { command.option('--reporter ', `Reporter to use, comma-separated, can be ${builtInReporters.map(name => `"${name}"`).join(', ')} (default: "${defaultReporter}")`); command.option('--retries ', `Maximum retry count for flaky tests, zero for no retries (default: no retries)`); command.option('--shard ', `Shard tests and execute only the selected shard, specify in the form "current/all", 1-based, for example "3/5"`); - command.option('--project ', `Only run tests from the specified project (default: run all projects)`); + command.option('--project ', `Only run tests from the specified list of projects (default: run all projects)`); command.option('--timeout ', `Specify test timeout threshold in milliseconds, zero for unlimited (default: ${defaultTimeout})`); command.option('-u, --update-snapshots', `Update snapshots with actual results (default: only create missing snapshots)`); command.option('-x', `Stop after the first failure`); diff --git a/src/test/runner.ts b/src/test/runner.ts index 5dc74e87f4..6379624820 100644 --- a/src/test/runner.ts +++ b/src/test/runner.ts @@ -93,11 +93,11 @@ export class Runner { this._loader.loadEmptyConfig(rootDir); } - async run(list: boolean, filePatternFilters: FilePatternFilter[], projectName?: string): Promise { + async run(list: boolean, filePatternFilters: FilePatternFilter[], projectNames?: string[]): Promise { this._reporter = await this._createReporter(list); const config = this._loader.fullConfig(); const globalDeadline = config.globalTimeout ? config.globalTimeout + monotonicTime() : 0; - const { result, timedOut } = await raceAgainstDeadline(this._run(list, filePatternFilters, projectName), globalDeadline); + const { result, timedOut } = await raceAgainstDeadline(this._run(list, filePatternFilters, projectNames), globalDeadline); if (timedOut) { if (!this._didBegin) this._reporter.onBegin?.(config, new Suite('')); @@ -137,18 +137,34 @@ export class Runner { await new Promise(resolve => process.stderr.write('', () => resolve())); } - async _run(list: boolean, testFileReFilters: FilePatternFilter[], projectName?: string): Promise { + async _run(list: boolean, testFileReFilters: FilePatternFilter[], projectNames?: string[]): Promise { const testFileFilter = testFileReFilters.length ? createMatcher(testFileReFilters.map(e => e.re)) : () => true; const config = this._loader.fullConfig(); + let projectsToFind: Set | undefined; + let unknownProjects: Map | undefined; + if (projectNames) { + projectsToFind = new Set(); + unknownProjects = new Map(); + projectNames.forEach(n => { + const name = n.toLocaleLowerCase(); + projectsToFind!.add(name); + unknownProjects!.set(name, n); + }); + } const projects = this._loader.projects().filter(project => { - return !projectName || project.config.name.toLocaleLowerCase() === projectName.toLocaleLowerCase(); + if (!projectsToFind) + return true; + const name = project.config.name.toLocaleLowerCase(); + unknownProjects!.delete(name); + return projectsToFind.has(name); }); - if (projectName && !projects.length) { + if (unknownProjects && unknownProjects.size) { const names = this._loader.projects().map(p => p.config.name).filter(name => !!name); if (!names.length) throw new Error(`No named projects are specified in the configuration file`); - throw new Error(`Project "${projectName}" not found. Available named projects: ${names.map(name => `"${name}"`).join(', ')}`); + const unknownProjectNames = Array.from(unknownProjects.values()).map(n => `"${n}"`).join(', '); + throw new Error(`Project(s) ${unknownProjectNames} not found. Available named projects: ${names.map(name => `"${name}"`).join(', ')}`); } const files = new Map(); diff --git a/tests/playwright-test/config.spec.ts b/tests/playwright-test/config.spec.ts index c742200608..d844351638 100644 --- a/tests/playwright-test/config.spec.ts +++ b/tests/playwright-test/config.spec.ts @@ -259,7 +259,77 @@ test('should print nice error when project is unknown', async ({ runInlineTest } ` }, { project: 'suite3' }); expect(exitCode).toBe(1); - expect(output).toContain('Project "suite3" not found. Available named projects: "suite1", "suite2"'); + expect(output).toContain('Project(s) "suite3" not found. Available named projects: "suite1", "suite2"'); +}); + +test('should filter by project list, case-insensitive', async ({ runInlineTest }) => { + const { passed, failed, output, skipped } = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { projects: [ + { name: 'suite1' }, + { name: 'suite2' }, + { name: 'suite3' }, + { name: 'suite4' }, + ] }; + `, + 'a.test.ts': ` + const { test } = pwt; + test('pass', async ({}, testInfo) => { + console.log(testInfo.project.name); + }); + ` + }, { project: ['SUite2', 'Suite3'] }); + expect(passed).toBe(2); + expect(failed).toBe(0); + expect(skipped).toBe(0); + expect(output).toContain('suite2'); + expect(output).toContain('suite3'); + expect(output).not.toContain('suite1'); + expect(output).not.toContain('suite4'); +}); + +test('should filter when duplicate project names exist', async ({ runInlineTest }) => { + const { passed, failed, output, skipped } = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { projects: [ + { name: 'suite1' }, + { name: 'suite2' }, + { name: 'suite1' }, + { name: 'suite4' }, + ] }; + `, + 'a.test.ts': ` + const { test } = pwt; + test('pass', async ({}, testInfo) => { + console.log(testInfo.project.name); + }); + ` + }, { project: ['suite1', 'sUIte4'] }); + expect(passed).toBe(3); + expect(failed).toBe(0); + expect(skipped).toBe(0); + expect(output).toContain('suite1'); + expect(output).toContain('suite4'); + expect(output).not.toContain('suite2'); +}); + +test('should print nice error when some of the projects are unknown', async ({ runInlineTest }) => { + const { output, exitCode } = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { projects: [ + { name: 'suite1' }, + { name: 'suite2' }, + ] }; + `, + 'a.test.ts': ` + const { test } = pwt; + test('pass', async ({}, testInfo) => { + console.log(testInfo.project.name); + }); + ` + }, { project: ['suitE1', 'suIte3', 'SUite4'] }); + expect(exitCode).toBe(1); + expect(output).toContain('Project(s) "suIte3", "SUite4" not found. Available named projects: "suite1", "suite2"'); }); test('should work without config file', async ({ runInlineTest }) => {