diff --git a/docs/src/test-cli-js.md b/docs/src/test-cli-js.md index 61e204ab14..85e4d745c2 100644 --- a/docs/src/test-cli-js.md +++ b/docs/src/test-cli-js.md @@ -92,7 +92,8 @@ Complete set of Playwright Test options is available in the [configuration file] | `--no-deps` | Ignore the dependencies between projects and behave as if they were not specified. | | `--output ` | Directory for artifacts produced by tests, defaults to `test-results`. | | `--pass-with-no-tests` | Allows the test suite to pass when no files are found. | -| `--project ` | Only run tests from the projects matching this regular expression. Defaults to running all projects defined in the configuration file.| +| `--project ` | Only run tests from the specified [projects](./test-projects.md). Defaults to running all projects defined in the configuration file.| +| `--project-grep ` | Only run tests from the projects matching this regular expression. Defaults to running all projects defined in the configuration file.| | `--quiet` | Whether to suppress stdout and stderr from the tests. | | `--repeat-each ` | Run each test `N` times, defaults to one. | | `--reporter ` | Choose a reporter: minimalist `dot`, concise `line` or detailed `list`. See [reporters](./test-reporters.md) for more information. | diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts index 0c1a67c614..6cf31f604e 100644 --- a/packages/playwright/src/common/config.ts +++ b/packages/playwright/src/common/config.ts @@ -48,6 +48,7 @@ export class FullConfigInternal { cliGrepInvert: string | undefined; cliTagFilter: string | undefined; cliProjectFilter?: string[]; + cliProjectGrep?: string; cliListOnly = false; cliPassWithNoTests?: boolean; testIdMatcher?: Matcher; diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts index c02f0f8095..c46834f452 100644 --- a/packages/playwright/src/program.ts +++ b/packages/playwright/src/program.ts @@ -64,7 +64,8 @@ function addListFilesCommand(program: Command) { const command = program.command('list-files [file-filter...]', { hidden: true }); command.description('List files with Playwright Test tests'); command.option('-c, --config ', `Configuration file, or a test directory with optional "playwright.config.{m,c}?{js,ts}"`); - command.option('--project ', `Only run tests from the projects matching this regular expression (default: list all projects)`); + command.option('--project ', `Only run tests from the specified list of projects (default: list all projects)`); + command.option('--project-grep ', `Only run tests from the projects matching this regular expression (default: list all projects)`); command.action(async (args, opts) => { try { await listTestFiles(opts); @@ -166,12 +167,16 @@ async function runTests(args: string[], opts: { [key: string]: any }) { if (!config) return; + if (opts.project && opts.projectGrep) + throw new Error('Only one of --project and --project-grep can be specified.'); + config.cliArgs = args; config.cliGrep = opts.grep as string | undefined; config.cliGrepInvert = opts.grepInvert as string | undefined; config.cliTagFilter = opts.tag; config.cliListOnly = !!opts.list; config.cliProjectFilter = opts.project || undefined; + config.cliProjectGrep = opts.projectGrep || undefined; config.cliPassWithNoTests = !!opts.passWithNoTests; const runner = new Runner(config); @@ -210,7 +215,9 @@ export async function withRunnerAndMutedWrite(configFile: string | undefined, ca } async function listTestFiles(opts: { [key: string]: any }) { - await withRunnerAndMutedWrite(opts.config, async runner => runner.listTestFiles(opts.project)); + if (opts.project && opts.projectGrep) + throw new Error('Only one of --project and --project-grep can be specified.'); + await withRunnerAndMutedWrite(opts.config, async runner => runner.listTestFiles(opts.project, opts.projectGrep)); } async function mergeReports(reportDir: string | undefined, opts: { [key: string]: any }) { @@ -323,7 +330,8 @@ const testOptions: [string, string][] = [ ['--no-deps', 'Do not run project dependencies'], ['--output ', `Folder for output artifacts (default: "test-results")`], ['--pass-with-no-tests', `Makes test run succeed even if no tests were found`], - ['--project ', `Only run tests from the projects matching this regular expression (default: run all projects)`], + ['--project ', `Only run tests from the specified list of projects(default: run all projects)`], + ['--project-grep ', `Only run tests from the projects matching this regular expression (default: run all projects)`], ['--quiet', `Suppress stdio`], ['--repeat-each ', `Run each test N times (default: 1)`], ['--reporter ', `Reporter to use, comma-separated, can be ${builtInReporters.map(name => `"${name}"`).join(', ')} (default: "${defaultReporter}")`], diff --git a/packages/playwright/src/runner/loadUtils.ts b/packages/playwright/src/runner/loadUtils.ts index 5dc50c74a8..3bd5f1289d 100644 --- a/packages/playwright/src/runner/loadUtils.ts +++ b/packages/playwright/src/runner/loadUtils.ts @@ -40,7 +40,7 @@ export async function collectProjectsAndTestFiles(testRun: TestRun, doNotRunTest // First collect all files for the projects in the command line, don't apply any file filters. const allFilesForProject = new Map(); - const filteredProjects = filterProjects(config.projects, config.cliProjectFilter); + const filteredProjects = filterProjects(config.projects, config.cliProjectFilter, config.cliProjectGrep); for (const project of filteredProjects) { const files = await collectFilesForProject(project, fsCache); allFilesForProject.set(project, files); diff --git a/packages/playwright/src/runner/projectUtils.ts b/packages/playwright/src/runner/projectUtils.ts index ee0ab7d3e7..591e7f5966 100644 --- a/packages/playwright/src/runner/projectUtils.ts +++ b/packages/playwright/src/runner/projectUtils.ts @@ -19,39 +19,48 @@ import path from 'path'; import { minimatch } from 'playwright-core/lib/utilsBundle'; import { promisify } from 'util'; import type { FullProjectInternal } from '../common/config'; -import { createFileMatcher } from '../util'; +import { createFileMatcher, forceRegExp } from '../util'; const readFileAsync = promisify(fs.readFile); const readDirAsync = promisify(fs.readdir); -// The difference to forceRegExp is that we want to match the whole string. -function forceBoundedRegExp(pattern: string): RegExp { - const match = pattern.match(/^\/(.*)\/([gi]*)$/); - if (match) - return new RegExp(match[1], match[2]); - return new RegExp(`^${pattern}$`, 'gi'); -} - -export function filterProjects(projects: FullProjectInternal[], projectNames?: string[]): FullProjectInternal[] { - if (!projectNames) +export function filterProjects(projects: FullProjectInternal[], projectNames?: string[], projectGrep?: string): FullProjectInternal[] { + if (!projectNames && !projectGrep) return [...projects]; - const unmatchedProjectFilters = new Set(projectNames); + + if (projectGrep) { + const regex = forceRegExp(projectGrep); + const result = projects.filter(project => { + regex.lastIndex = 0; + return regex.test(project.project.name); + }); + if (!result.length) + throw new Error(`Projects matching "${projectGrep}" not found. Available projects: ${projects.map(p => `"${p.project.name}"`).join(', ')}`); + return result; + } + + const projectNamesToFind = new Set(); + const unmatchedProjectNames = new Map(); + for (const name of projectNames!) { + const lowerCaseName = name.toLocaleLowerCase(); + projectNamesToFind.add(lowerCaseName); + unmatchedProjectNames.set(lowerCaseName, name); + } + const result = projects.filter(project => { - for (const projectName of projectNames) { - if (forceBoundedRegExp(projectName).test(project.project.name)) { - unmatchedProjectFilters.delete(projectName); - return true; - } + const lowerCaseName = project.project.name.toLocaleLowerCase(); + if (projectNamesToFind.has(lowerCaseName)) { + unmatchedProjectNames.delete(lowerCaseName); + return true; } return false; }); - if (unmatchedProjectFilters.size) { - const names = projects.map(p => p.project.name).filter(name => !!name); - if (!names.length) - throw new Error(`No named projects are specified in the configuration file`); - const unknownProjectNames = Array.from(unmatchedProjectFilters.values()).map(n => `"${n}"`).join(', '); - throw new Error(`Project(s) ${unknownProjectNames} not found. Available named projects: ${names.map(name => `"${name}"`).join(', ')}`); + + if (unmatchedProjectNames.size) { + const unknownProjectNames = Array.from(unmatchedProjectNames.values()).map(n => `"${n}"`).join(', '); + throw new Error(`Project(s) ${unknownProjectNames} not found. Available projects: ${projects.map(p => `"${p.project.name}"`).join(', ')}`); } + return result; } diff --git a/packages/playwright/src/runner/runner.ts b/packages/playwright/src/runner/runner.ts index 2f265e5ebe..878c39714e 100644 --- a/packages/playwright/src/runner/runner.ts +++ b/packages/playwright/src/runner/runner.ts @@ -48,8 +48,8 @@ export class Runner { this._config = config; } - async listTestFiles(projectNames: string[] | undefined): Promise { - const projects = filterProjects(this._config.projects, projectNames); + async listTestFiles(projectNames: string[] | undefined, projectGrep: string | undefined): Promise { + const projects = filterProjects(this._config.projects, projectNames, projectGrep); const report: ConfigListFilesReport = { projects: [] }; diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.ts index 7f874aef4e..3862cad76f 100644 --- a/packages/playwright/src/runner/tasks.ts +++ b/packages/playwright/src/runner/tasks.ts @@ -29,7 +29,7 @@ import type { FullConfigInternal, FullProjectInternal } from '../common/config'; import { collectProjectsAndTestFiles, createRootSuite, loadFileSuites, loadGlobalHook } from './loadUtils'; import type { Matcher } from '../util'; import type { Suite } from '../common/test'; -import { buildDependentProjects, buildTeardownToSetupsMap } from './projectUtils'; +import { buildDependentProjects, buildTeardownToSetupsMap, filterProjects } from './projectUtils'; import { FailureTracker } from './failureTracker'; const readDirAsync = promisify(fs.readdir); @@ -168,10 +168,8 @@ function createRemoveOutputDirsTask(): Task { if (process.env.PW_TEST_NO_REMOVE_OUTPUT_DIRS) return; const outputDirs = new Set(); - for (const p of config.projects) { - if (!config.cliProjectFilter || config.cliProjectFilter.includes(p.project.name)) - outputDirs.add(p.project.outputDir); - } + const projects = filterProjects(config.projects, config.cliProjectFilter, config.cliProjectGrep); + projects.forEach(p => outputDirs.add(p.project.outputDir)); await Promise.all(Array.from(outputDirs).map(outputDir => removeFolders([outputDir]).then(async ([error]) => { if (!error) diff --git a/packages/playwright/src/runner/watchMode.ts b/packages/playwright/src/runner/watchMode.ts index 575653d3e2..8ce7972f87 100644 --- a/packages/playwright/src/runner/watchMode.ts +++ b/packages/playwright/src/runner/watchMode.ts @@ -40,7 +40,7 @@ class FSWatcher { async update(config: FullConfigInternal) { const commandLineFileMatcher = config.cliArgs.length ? createFileMatcherFromArguments(config.cliArgs) : () => true; - const projects = filterProjects(config.projects, config.cliProjectFilter); + const projects = filterProjects(config.projects, config.cliProjectFilter, config.cliProjectGrep); const projectClosure = buildProjectsClosure(projects); const projectFilters = new Map(); for (const [project, type] of projectClosure) { @@ -263,7 +263,7 @@ async function runChangedTests(config: FullConfigInternal, failedTestIdCollector // Collect all the affected projects, follow project dependencies. // Prepare to exclude all the projects that do not depend on this file, as if they did not exist. - const projects = filterProjects(config.projects, config.cliProjectFilter); + const projects = filterProjects(config.projects, config.cliProjectFilter, config.cliProjectGrep); const projectClosure = buildProjectsClosure(projects); const affectedProjects = affectedProjectsClosure([...projectClosure.keys()], [...filesByProject.keys()]); const affectsAnyDependency = [...affectedProjects].some(p => projectClosure.get(p) === 'dependency'); @@ -388,6 +388,8 @@ function printConfiguration(config: FullConfigInternal, title?: string) { const tokens: string[] = []; tokens.push(`${packageManagerCommand} playwright test`); tokens.push(...(config.cliProjectFilter || [])?.map(p => colors.blue(`--project ${p}`))); + if (config.cliProjectGrep) + tokens.push(colors.blue(`--project-grep ${config.cliProjectGrep}`)); if (config.cliGrep) tokens.push(colors.red(`--grep ${config.cliGrep}`)); if (config.cliArgs) diff --git a/tests/playwright-test/config.spec.ts b/tests/playwright-test/config.spec.ts index 47a2dc04c0..65cb58ef05 100644 --- a/tests/playwright-test/config.spec.ts +++ b/tests/playwright-test/config.spec.ts @@ -245,34 +245,12 @@ test('should filter by project, case-insensitive', async ({ runInlineTest }) => ])); }); -test('should filter by project and parse as RegExp', async ({ runInlineTest }) => { +test('should filter by project-grep', async ({ runInlineTest }) => { const result = await runInlineTest({ 'playwright.config.js': ` module.exports = { projects: [ - { name: 'project-name' } - ] - }; - `, - 'a.test.js': ` - const { test } = require('@playwright/test'); - test('one', async ({}) => { - console.log('%%' + test.info().project.name); - }); ` - }, { project: '.*oj.*t-Na.?e' }); - expect(result.exitCode).toBe(0); - expect(result.output).toContain('Running 1 test using 1 worker'); - expect(new Set(result.outputLines)).toEqual(new Set([ - 'project-name', - ])); -}); - -test('should filter by project and only match if its full-match', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'playwright.config.js': ` - module.exports = { - projects: [ - { name: 'prefix-foobar-suffix' }, + { name: 'project-name' }, { name: 'foobar' } ] }; @@ -282,10 +260,50 @@ test('should filter by project and only match if its full-match', async ({ runIn test('one', async ({}) => { console.log('%%' + test.info().project.name); }); ` - }, { project: 'foobar' }); + }, { '--project-grep': '.*oj.*t-Na.?e' }); expect(result.exitCode).toBe(0); expect(result.output).toContain('Running 1 test using 1 worker'); - expect(new Set(result.outputLines)).toEqual(new Set(['foobar'])); + expect(new Set(result.outputLines)).toEqual(new Set([ + 'project-name', + ])); +}); + +test('should print nice error when the project grep does not match anything', async ({ runInlineTest }) => { + const { output, exitCode } = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { projects: [ + { name: 'suite1' }, + { name: 'suite2' }, + ] }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('pass', async ({}, testInfo) => { + console.log(testInfo.project.name); + }); + ` + }, { '--project-grep': ['aaa'] }); + expect(exitCode).toBe(1); + expect(output).toContain('Error: Projects matching \"aaa\" not found. Available projects: \"suite1\", \"suite2\"'); +}); + +test('should fail if both --project and --project-grep are passed', async ({ runInlineTest }) => { + const { output, exitCode } = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { projects: [ + { name: 'suite1' }, + { name: 'suite2' }, + ] }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('pass', async ({}, testInfo) => { + console.log(testInfo.project.name); + }); + ` + }, { '--project-grep': 'foo', '--project': 'bar' }); + expect(exitCode).toBe(1); + expect(output).toContain('Only one of --project and --project-grep can be specified'); }); test('should filter by project and allow passing RegExp start/end flags', async ({ runInlineTest }) => { @@ -304,7 +322,7 @@ test('should filter by project and allow passing RegExp start/end flags', async test('one', async ({}) => { console.log('%%' + test.info().project.name); }); ` - }, { project: '/fooBar$/' }); + }, { '--project-grep': '/fooBar$/' }); expect(result.exitCode).toBe(0); expect(new Set(result.outputLines)).toEqual(new Set(['prefix-fooBar', 'fooBar'])); }); @@ -323,7 +341,7 @@ test('should print nice error when project is unknown', async ({ runInlineTest } ` }, { project: 'suite3' }); expect(exitCode).toBe(1); - expect(output).toContain('Project(s) "suite3" not found. Available named projects: "suite1", "suite2"'); + expect(output).toContain('Project(s) "suite3" not found. Available projects: "suite1", "suite2"'); }); test('should filter by project list, case-insensitive', async ({ runInlineTest }) => { @@ -388,7 +406,7 @@ test('should print nice error when some of the projects are unknown', async ({ r ` }, { project: ['suitE1', 'suIte3', 'SUite4'] }); expect(exitCode).toBe(1); - expect(output).toContain('Project(s) "suIte3", "SUite4" not found. Available named projects: "suite1", "suite2"'); + expect(output).toContain('Project(s) "suIte3", "SUite4" not found. Available projects: "suite1", "suite2"'); }); test('should work without config file', async ({ runInlineTest }) => { diff --git a/tests/playwright-test/list-files.spec.ts b/tests/playwright-test/list-files.spec.ts index 80df3a0e5b..0ef2eda304 100644 --- a/tests/playwright-test/list-files.spec.ts +++ b/tests/playwright-test/list-files.spec.ts @@ -48,6 +48,30 @@ test('should list files', async ({ runCLICommand }) => { }); }); +test('should support project-grep list files', async ({ runCLICommand }) => { + const result = await runCLICommand({ + 'playwright.config.ts': ` + module.exports = { projects: [{ name: 'foo' }, { name: 'bar' }] }; + `, + 'a.test.js': `` + }, 'list-files', ['--project-grep', 'f.o']); + expect(result.exitCode).toBe(0); + + const data = JSON.parse(result.stdout); + expect(data).toEqual({ + projects: [ + { + name: 'foo', + testDir: expect.stringContaining('list-files-should-support-project-grep-list-files-playwright-test'), + use: {}, + files: [ + expect.stringContaining('a.test.js') + ] + } + ] + }); +}); + test('should include testIdAttribute', async ({ runCLICommand }) => { const result = await runCLICommand({ 'playwright.config.ts': ` diff --git a/tests/playwright-test/playwright-test-fixtures.ts b/tests/playwright-test/playwright-test-fixtures.ts index c293dbb0c2..825068f51f 100644 --- a/tests/playwright-test/playwright-test-fixtures.ts +++ b/tests/playwright-test/playwright-test-fixtures.ts @@ -105,7 +105,7 @@ function findPackageJSONDir(files: Files, dir: string) { return dir; } -function startPlaywrightTest(childProcess: CommonFixtures['childProcess'], baseDir: string, params: any, env: NodeJS.ProcessEnv, options: RunOptions): TestChildProcess { +function toParamList(params: any): string[] { const paramList: string[] = []; for (const key of Object.keys(params)) { for (const value of Array.isArray(params[key]) ? params[key] : [params[key]]) { @@ -113,6 +113,11 @@ function startPlaywrightTest(childProcess: CommonFixtures['childProcess'], baseD paramList.push(params[key] === true ? `${k}` : `${k}=${value}`); } } + return paramList; +} + +function startPlaywrightTest(childProcess: CommonFixtures['childProcess'], baseDir: string, params: any, env: NodeJS.ProcessEnv, options: RunOptions): TestChildProcess { + const paramList = toParamList(params); const args = ['test']; args.push( '--workers=2',