diff --git a/docs/src/test-cli-js.md b/docs/src/test-cli-js.md index 193e2cae6f..6eeec4abe2 100644 --- a/docs/src/test-cli-js.md +++ b/docs/src/test-cli-js.md @@ -92,7 +92,7 @@ Complete set of Playwright Test options is available in the [configuration file] | `--no-deps` | Ignore the dependencies between projects and behave as if they were not specified. | | `--output ` | Directory for artifacts produced by tests, defaults to `test-results`. | | `--pass-with-no-tests` | Allows the test suite to pass when no files are found. | -| `--project ` | Only run tests from one of the specified [projects](./test-projects.md). Defaults to running all projects defined in the configuration file.| +| `--project ` | Only run tests from the projects matching this regular expression. Defaults to running all projects defined in the configuration file.| | `--quiet` | Whether to suppress stdout and stderr from the tests. | | `--repeat-each ` | Run each test `N` times, defaults to one. | | `--reporter ` | Choose a reporter: minimalist `dot`, concise `line` or detailed `list`. See [reporters](./test-reporters.md) for more information. | diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts index ef6ad1c8d0..f3d3fc26c4 100644 --- a/packages/playwright/src/program.ts +++ b/packages/playwright/src/program.ts @@ -65,7 +65,7 @@ function addListFilesCommand(program: Command) { const command = program.command('list-files [file-filter...]', { hidden: true }); command.description('List files with Playwright Test tests'); command.option('-c, --config ', `Configuration file, or a test directory with optional "playwright.config.{m,c}?{js,ts}"`); - command.option('--project ', `Only run tests from the specified list of projects (default: list all projects)`); + command.option('--project ', `Only run tests from the projects matching this regular expression (default: list all projects)`); command.action(async (args, opts) => { try { await listTestFiles(opts); @@ -327,7 +327,7 @@ const testOptions: [string, string][] = [ ['--no-deps', 'Do not run project dependencies'], ['--output ', `Folder for output artifacts (default: "test-results")`], ['--pass-with-no-tests', `Makes test run succeed even if no tests were found`], - ['--project ', `Only run tests from the specified list of projects (default: run all projects)`], + ['--project ', `Only run tests from the projects matching this regular expression (default: run all projects)`], ['--quiet', `Suppress stdio`], ['--repeat-each ', `Run each test N times (default: 1)`], ['--reporter ', `Reporter to use, comma-separated, can be ${builtInReporters.map(name => `"${name}"`).join(', ')} (default: "${defaultReporter}")`], diff --git a/packages/playwright/src/runner/projectUtils.ts b/packages/playwright/src/runner/projectUtils.ts index 4469be3744..ee0ab7d3e7 100644 --- a/packages/playwright/src/runner/projectUtils.ts +++ b/packages/playwright/src/runner/projectUtils.ts @@ -24,26 +24,32 @@ import { createFileMatcher } from '../util'; const readFileAsync = promisify(fs.readFile); const readDirAsync = promisify(fs.readdir); +// The difference to forceRegExp is that we want to match the whole string. +function forceBoundedRegExp(pattern: string): RegExp { + const match = pattern.match(/^\/(.*)\/([gi]*)$/); + if (match) + return new RegExp(match[1], match[2]); + return new RegExp(`^${pattern}$`, 'gi'); +} + export function filterProjects(projects: FullProjectInternal[], projectNames?: string[]): FullProjectInternal[] { if (!projectNames) return [...projects]; - const projectsToFind = new Set(); - const unknownProjects = new Map(); - projectNames.forEach(n => { - const name = n.toLocaleLowerCase(); - projectsToFind.add(name); - unknownProjects.set(name, n); - }); + const unmatchedProjectFilters = new Set(projectNames); const result = projects.filter(project => { - const name = project.project.name.toLocaleLowerCase(); - unknownProjects.delete(name); - return projectsToFind.has(name); + for (const projectName of projectNames) { + if (forceBoundedRegExp(projectName).test(project.project.name)) { + unmatchedProjectFilters.delete(projectName); + return true; + } + } + return false; }); - if (unknownProjects.size) { + if (unmatchedProjectFilters.size) { const names = projects.map(p => p.project.name).filter(name => !!name); if (!names.length) throw new Error(`No named projects are specified in the configuration file`); - const unknownProjectNames = Array.from(unknownProjects.values()).map(n => `"${n}"`).join(', '); + const unknownProjectNames = Array.from(unmatchedProjectFilters.values()).map(n => `"${n}"`).join(', '); throw new Error(`Project(s) ${unknownProjectNames} not found. Available named projects: ${names.map(name => `"${name}"`).join(', ')}`); } return result; diff --git a/tests/playwright-test/config.spec.ts b/tests/playwright-test/config.spec.ts index 0d786de0ce..47a2dc04c0 100644 --- a/tests/playwright-test/config.spec.ts +++ b/tests/playwright-test/config.spec.ts @@ -223,7 +223,7 @@ test('should throw when test() is called in config file', async ({ runInlineTest }); test('should filter by project, case-insensitive', async ({ runInlineTest }) => { - const { passed, failed, output, skipped } = await runInlineTest({ + const { passed, failed, outputLines, skipped } = await runInlineTest({ 'playwright.config.ts': ` module.exports = { projects: [ { name: 'suite1' }, @@ -231,17 +231,82 @@ test('should filter by project, case-insensitive', async ({ runInlineTest }) => ] }; `, 'a.test.ts': ` - import { test, expect } from '@playwright/test'; + import { test } from '@playwright/test'; test('pass', async ({}, testInfo) => { - console.log(testInfo.project.name); + console.log('%%' + test.info().project.name); }); ` }, { project: 'SUite2' }); expect(passed).toBe(1); expect(failed).toBe(0); expect(skipped).toBe(0); - expect(output).toContain('suite2'); - expect(output).not.toContain('suite1'); + expect(new Set(outputLines)).toEqual(new Set([ + 'suite2', + ])); +}); + +test('should filter by project and parse as RegExp', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.js': ` + module.exports = { + projects: [ + { name: 'project-name' } + ] + }; + `, + 'a.test.js': ` + const { test } = require('@playwright/test'); + test('one', async ({}) => { + console.log('%%' + test.info().project.name); + }); ` + }, { project: '.*oj.*t-Na.?e' }); + expect(result.exitCode).toBe(0); + expect(result.output).toContain('Running 1 test using 1 worker'); + expect(new Set(result.outputLines)).toEqual(new Set([ + 'project-name', + ])); +}); + +test('should filter by project and only match if its full-match', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.js': ` + module.exports = { + projects: [ + { name: 'prefix-foobar-suffix' }, + { name: 'foobar' } + ] + }; + `, + 'a.test.js': ` + const { test } = require('@playwright/test'); + test('one', async ({}) => { + console.log('%%' + test.info().project.name); + }); ` + }, { project: 'foobar' }); + expect(result.exitCode).toBe(0); + expect(result.output).toContain('Running 1 test using 1 worker'); + expect(new Set(result.outputLines)).toEqual(new Set(['foobar'])); +}); + +test('should filter by project and allow passing RegExp start/end flags', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.js': ` + module.exports = { + projects: [ + { name: 'prefix-fooBar' }, + { name: 'fooBar' }, + { name: 'foobar' }, + ] + }; + `, + 'a.test.js': ` + const { test } = require('@playwright/test'); + test('one', async ({}) => { + console.log('%%' + test.info().project.name); + }); ` + }, { project: '/fooBar$/' }); + expect(result.exitCode).toBe(0); + expect(new Set(result.outputLines)).toEqual(new Set(['prefix-fooBar', 'fooBar'])); }); test('should print nice error when project is unknown', async ({ runInlineTest }) => { @@ -254,9 +319,7 @@ test('should print nice error when project is unknown', async ({ runInlineTest } `, 'a.test.ts': ` import { test, expect } from '@playwright/test'; - test('pass', async ({}, testInfo) => { - console.log(testInfo.project.name); - }); + test('pass', async ({}, testInfo) => {}); ` }, { project: 'suite3' }); expect(exitCode).toBe(1); @@ -264,7 +327,7 @@ test('should print nice error when project is unknown', async ({ runInlineTest } }); test('should filter by project list, case-insensitive', async ({ runInlineTest }) => { - const { passed, failed, output, skipped } = await runInlineTest({ + const { passed, failed, outputLines, skipped } = await runInlineTest({ 'playwright.config.ts': ` module.exports = { projects: [ { name: 'suite1' }, @@ -276,21 +339,18 @@ test('should filter by project list, case-insensitive', async ({ runInlineTest } 'a.test.ts': ` import { test, expect } from '@playwright/test'; test('pass', async ({}, testInfo) => { - console.log(testInfo.project.name); + console.log('%%' + test.info().project.name); }); ` }, { project: ['SUite2', 'Suite3'] }); expect(passed).toBe(2); expect(failed).toBe(0); expect(skipped).toBe(0); - expect(output).toContain('suite2'); - expect(output).toContain('suite3'); - expect(output).not.toContain('suite1'); - expect(output).not.toContain('suite4'); + expect(new Set(outputLines)).toEqual(new Set(['suite3', 'suite2'])); }); test('should filter when duplicate project names exist', async ({ runInlineTest }) => { - const { passed, failed, output, skipped } = await runInlineTest({ + const { passed, failed, outputLines, skipped } = await runInlineTest({ 'playwright.config.ts': ` module.exports = { projects: [ { name: 'suite1' }, @@ -302,16 +362,14 @@ test('should filter when duplicate project names exist', async ({ runInlineTest 'a.test.ts': ` import { test, expect } from '@playwright/test'; test('pass', async ({}, testInfo) => { - console.log(testInfo.project.name); + console.log('%%' + test.info().project.name); }); ` }, { project: ['suite1', 'sUIte4'] }); expect(passed).toBe(3); expect(failed).toBe(0); expect(skipped).toBe(0); - expect(output).toContain('suite1'); - expect(output).toContain('suite4'); - expect(output).not.toContain('suite2'); + expect(new Set(outputLines)).toEqual(new Set(['suite1', 'suite1', 'suite4'])); }); test('should print nice error when some of the projects are unknown', async ({ runInlineTest }) => {