fix: --grep and --grep-invert should intersect with config (#17716)

Fixes https://github.com/microsoft/playwright/issues/17405

packages/playwright-test/src/cli.ts

@@ -24,6 +24,7 @@ import { Runner, builtInReporters, kDefaultConfigFiles } from './runner';
 import type { ConfigCLIOverrides } from './runner';
 import { stopProfiling, startProfiling } from './profiler';
 import type { TestFileFilter } from './util';
+import { createTitleMatcher } from './util';
 import { showHTMLReport } from './reporters/html';
 import { baseFullConfig, defaultTimeout, fileIsModule } from './loader';
 import type { TraceMode } from './types';
@@ -163,9 +164,14 @@ async function runTests(args: string[], opts: { [key: string]: any }) {
     };
   });
 
+  const grepMatcher = opts.grep ? createTitleMatcher(forceRegExp(opts.grep)) : () => true;
+  const grepInvertMatcher = opts.grepInvert ? createTitleMatcher(forceRegExp(opts.grepInvert)) : () => false;
+  const testTitleMatcher = (title: string) => !grepInvertMatcher(title) && grepMatcher(title);
+
   const result = await runner.runAllTests({
     listOnly: !!opts.list,
     testFileFilters,
+    testTitleMatcher,
     projectFilter: opts.project || undefined,
     projectGroup: opts.group,
     passWithNoTests: opts.passWithNoTests,
@@ -208,8 +214,6 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrid
     forbidOnly: options.forbidOnly ? true : undefined,
     fullyParallel: options.fullyParallel ? true : undefined,
     globalTimeout: options.globalTimeout ? parseInt(options.globalTimeout, 10) : undefined,
-    grep: options.grep ? forceRegExp(options.grep) : undefined,
-    grepInvert: options.grepInvert ? forceRegExp(options.grepInvert) : undefined,
     maxFailures: options.x ? 1 : (options.maxFailures ? parseInt(options.maxFailures, 10) : undefined),
     outputDir: options.output ? path.resolve(process.cwd(), options.output) : undefined,
     quiet: options.quiet ? options.quiet : undefined,

packages/playwright-test/src/loader.ts

@@ -81,8 +81,6 @@ export class Loader {
     config.forbidOnly = takeFirst(this._configCLIOverrides.forbidOnly, config.forbidOnly);
     config.fullyParallel = takeFirst(this._configCLIOverrides.fullyParallel, config.fullyParallel);
     config.globalTimeout = takeFirst(this._configCLIOverrides.globalTimeout, config.globalTimeout);
-    config.grep = takeFirst(this._configCLIOverrides.grep, config.grep);
-    config.grepInvert = takeFirst(this._configCLIOverrides.grepInvert, config.grepInvert);
     config.maxFailures = takeFirst(this._configCLIOverrides.maxFailures, config.maxFailures);
     config.outputDir = takeFirst(this._configCLIOverrides.outputDir, config.outputDir);
     config.quiet = takeFirst(this._configCLIOverrides.quiet, config.quiet);
@@ -257,8 +255,6 @@ export class Loader {
 
   private _applyCLIOverridesToProject(projectConfig: Project) {
     projectConfig.fullyParallel = takeFirst(this._configCLIOverrides.fullyParallel, projectConfig.fullyParallel);
-    projectConfig.grep = takeFirst(this._configCLIOverrides.grep, projectConfig.grep);
-    projectConfig.grepInvert = takeFirst(this._configCLIOverrides.grepInvert, projectConfig.grepInvert);
     projectConfig.outputDir = takeFirst(this._configCLIOverrides.outputDir, projectConfig.outputDir);
     projectConfig.repeatEach = takeFirst(this._configCLIOverrides.repeatEach, projectConfig.repeatEach);
     projectConfig.retries = takeFirst(this._configCLIOverrides.retries, projectConfig.retries);

packages/playwright-test/src/runner.ts

@@ -63,7 +63,8 @@ type RunPhase = {
 
 type RunOptions = {
   listOnly?: boolean;
-  testFileFilters?: TestFileFilter[];
+  testFileFilters: TestFileFilter[];
+  testTitleMatcher: Matcher;
   projectFilter?: string[];
   projectGroup?: string;
   passWithNoTests?: boolean;
@@ -73,8 +74,6 @@ export type ConfigCLIOverrides = {
   forbidOnly?: boolean;
   fullyParallel?: boolean;
   globalTimeout?: number;
-  grep?: RegExp;
-  grepInvert?: RegExp;
   maxFailures?: number;
   outputDir?: string;
   quiet?: boolean;
@@ -183,7 +182,7 @@ export class Runner {
     return new Multiplexer(reporters);
   }
 
-  async runAllTests(options: RunOptions = {}): Promise<FullResult> {
+  async runAllTests(options: RunOptions): Promise<FullResult> {
     this._reporter = await this._createReporter(!!options.listOnly);
     const config = this._loader.fullConfig();
     const result = await raceAgainstTimeout(() => this._run(options), config.globalTimeout);
@@ -236,7 +235,7 @@ export class Runner {
       if (projectGroup)
         throw new Error('--group option can not be combined with --project');
     } else {
-      if (!projectGroup && config.groups?.default && !options.testFileFilters?.length)
+      if (!projectGroup && config.groups?.default && !options.testFileFilters.length)
         projectGroup = 'default';
       if (projectGroup) {
         if (config.shard)
@@ -289,7 +288,7 @@ export class Runner {
 
   private _runPhaseFromOptions(options: RunOptions): RunPhase {
     const testFileMatcher = fileMatcherFrom(options.testFileFilters);
-    const testTitleMatcher = () => true;
+    const testTitleMatcher = options.testTitleMatcher;
     const projects = options.projectFilter ?? this._loader.fullConfig().projects.map(p => p.name);
     return projects.map(projectName => ({
       projectName,
@@ -374,7 +373,7 @@ export class Runner {
 
       // 3. Filter tests to respect line/column filter.
       // TODO: figure out how this is supposed to work with groups.
-      if (options.testFileFilters?.length)
+      if (options.testFileFilters.length)
         filterByFocusedLine(preprocessRoot, options.testFileFilters);
 
       // 4. Complain about only.
@@ -397,6 +396,7 @@ export class Runner {
       for (const [project, files] of filesByProject) {
         const grepMatcher = createTitleMatcher(project.grep);
         const grepInvertMatcher = project.grepInvert ? createTitleMatcher(project.grepInvert) : null;
+        // TODO: also apply title matcher from options.
         const groupTitleMatcher = phase.find(p => p.projectName.toLocaleLowerCase() === project.name.toLocaleLowerCase())!.testTitleMatcher;
         const projectSuite = new Suite(project.name, 'project');
         projectSuite._projectConfig = project;

tests/playwright-test/test-grep.spec.ts

@@ -16,7 +16,7 @@
 
 import { test, expect } from './playwright-test-fixtures';
 
-test('config.glob should work', async ({ runInlineTest }) => {
+test('config.grep should work', async ({ runInlineTest }) => {
   const result = await runInlineTest({
     'playwright.config.ts': `
       module.exports = { grep: /test1/ };
@@ -32,7 +32,7 @@ test('config.glob should work', async ({ runInlineTest }) => {
   expect(result.output).toContain('%% test1');
 });
 
-test('config.globInvert should work', async ({ runInlineTest }) => {
+test('config.grepInvert should work', async ({ runInlineTest }) => {
   const result = await runInlineTest({
     'playwright.config.ts': `
       module.exports = { grepInvert: /test1/ };
@@ -48,7 +48,7 @@ test('config.globInvert should work', async ({ runInlineTest }) => {
   expect(result.output).toContain('%% test2');
 });
 
-test('project.glob should work', async ({ runInlineTest }) => {
+test('project.grep should work', async ({ runInlineTest }) => {
   const result = await runInlineTest({
     'playwright.config.ts': `
       module.exports = { projects: [ { grep: /test1/ } ] };
@@ -64,7 +64,7 @@ test('project.glob should work', async ({ runInlineTest }) => {
   expect(result.output).toContain('%% test1');
 });
 
-test('project.globInvert should work', async ({ runInlineTest }) => {
+test('project.grepInvert should work', async ({ runInlineTest }) => {
   const result = await runInlineTest({
     'playwright.config.ts': `
       module.exports = { projects: [ { grepInvert: /test1/ } ] };
@@ -79,3 +79,21 @@ test('project.globInvert should work', async ({ runInlineTest }) => {
   expect(result.passed).toBe(1);
   expect(result.output).toContain('%% test2');
 });
+
+test('config.grep should intercect with --grep and --grepInvert', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { grep: /test./, grepInvert: /test4/ };
+    `,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('test1', async () => { console.log('\\n%% test1'); });
+      test('test2', async () => { console.log('\\n%% test2'); });
+      test('test3', async () => { console.log('\\n%% test3'); });
+      test('test4', async () => { console.log('\\n%% test4'); });
+    `,
+  }, { 'grep': 'test[23]', 'grep-invert': '..st3' });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+  expect(result.output).toContain('%% test2');
+});