feat(test-runner): allow grepping in project (#12469)

docs/src/test-api/class-testproject.md

@@ -124,6 +124,20 @@ By default, **test files** are run in parallel. Tests in a single file are run i
 
 You can configure entire test project to concurrently run all tests in all files using this option.
 
+## property: TestProject.grep
+- type: <[RegExp]|[Array]<[RegExp]>>
+
+Filter to only run tests with a title matching one of the patterns. For example, passing `grep: /cart/` should only run tests with "cart" in the title. Also available globally and in the [command line](./test-cli.md) with the `-g` option.
+
+`grep` option is also useful for [tagging tests](./test-annotations.md#tag-tests).
+
+## property: TestProject.grepInvert
+- type: <[RegExp]|[Array]<[RegExp]>>
+
+Filter to only run tests with a title **not** matching one of the patterns. This is the opposite of [`property: TestProject.grep`]. Also available globally and in the [command line](./test-cli.md) with the `--grep-invert` option.
+
+`grepInvert` option is also useful for [tagging tests](./test-annotations.md#tag-tests).
+
 ## property: TestProject.metadata
 - type: <[Object]>
 

packages/playwright-test/src/loader.ts

@@ -205,6 +205,8 @@ export class Loader {
     const fullProject: FullProject = {
       fullyParallel: takeFirst(this._configOverrides.fullyParallel, projectConfig.fullyParallel, this._config.fullyParallel, undefined),
       expect: takeFirst(this._configOverrides.expect, projectConfig.expect, this._config.expect, undefined),
+      grep: takeFirst(this._configOverrides.grep, projectConfig.grep, this._config.grep, baseFullConfig.grep),
+      grepInvert: takeFirst(this._configOverrides.grepInvert, projectConfig.grepInvert, this._config.grepInvert, baseFullConfig.grepInvert),
       outputDir,
       repeatEach: takeFirst(this._configOverrides.repeatEach, projectConfig.repeatEach, this._config.repeatEach, 1),
       retries: takeFirst(this._configOverrides.retries, projectConfig.retries, this._config.retries, 0),

packages/playwright-test/src/runner.ts

@@ -280,10 +280,10 @@ export class Runner {
       fileSuites.set(fileSuite._requireFile, fileSuite);
 
     const outputDirs = new Set<string>();
-    const grepMatcher = createTitleMatcher(config.grep);
-    const grepInvertMatcher = config.grepInvert ? createTitleMatcher(config.grepInvert) : null;
     const rootSuite = new Suite('');
     for (const [project, files] of filesByProject) {
+      const grepMatcher = createTitleMatcher(project.config.grep);
+      const grepInvertMatcher = project.config.grepInvert ? createTitleMatcher(project.config.grepInvert) : null;
       const projectSuite = new Suite(project.config.name);
       projectSuite._projectConfig = project.config;
       if (project.config.fullyParallel)

packages/playwright-test/types/test.d.ts

@@ -133,6 +133,21 @@ interface TestProject {
    * You can configure entire test project to concurrently run all tests in all files using this option.
    */
   fullyParallel?: boolean;
+  /**
+   * Filter to only run tests with a title matching one of the patterns. For example, passing `grep: /cart/` should only run
+   * tests with "cart" in the title. Also available globally and in the [command line](https://playwright.dev/docs/test-cli) with the `-g` option.
+   *
+   * `grep` option is also useful for [tagging tests](https://playwright.dev/docs/test-annotations#tag-tests).
+   */
+  grep?: RegExp | RegExp[];
+  /**
+   * Filter to only run tests with a title **not** matching one of the patterns. This is the opposite of
+   * [testProject.grep](https://playwright.dev/docs/api/class-testproject#test-project-grep). Also available globally and in
+   * the [command line](https://playwright.dev/docs/test-cli) with the `--grep-invert` option.
+   *
+   * `grepInvert` option is also useful for [tagging tests](https://playwright.dev/docs/test-annotations#tag-tests).
+   */
+  grepInvert?: RegExp | RegExp[] | null;
   /**
    * Any JSON-serializable metadata that will be put directly to the test report.
    */

tests/playwright-test/test-grep.spec.ts

@@ -0,0 +1,81 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './playwright-test-fixtures';
+
+test('config.glob should work', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { grep: /test1/ };
+    `,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('test1', async () => { console.log('\\n%% test1'); });
+      test('test2', async () => { console.log('\\n%% test2'); });
+    `,
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+  expect(result.output).toContain('%% test1');
+});
+
+test('config.globInvert should work', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { grepInvert: /test1/ };
+    `,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('test1', async () => { console.log('\\n%% test1'); });
+      test('test2', async () => { console.log('\\n%% test2'); });
+    `,
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+  expect(result.output).toContain('%% test2');
+});
+
+test('project.glob should work', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { projects: [ { grep: /test1/ } ] };
+    `,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('test1', async () => { console.log('\\n%% test1'); });
+      test('test2', async () => { console.log('\\n%% test2'); });
+    `,
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+  expect(result.output).toContain('%% test1');
+});
+
+test('project.globInvert should work', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { projects: [ { grepInvert: /test1/ } ] };
+    `,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('test1', async () => { console.log('\\n%% test1'); });
+      test('test2', async () => { console.log('\\n%% test2'); });
+    `,
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+  expect(result.output).toContain('%% test2');
+});

utils/generate_types/overrides-test.d.ts

@@ -60,6 +60,8 @@ type ExpectSettings = {
 interface TestProject {
   expect?: ExpectSettings;
   fullyParallel?: boolean;
+  grep?: RegExp | RegExp[];
+  grepInvert?: RegExp | RegExp[] | null;
   metadata?: any;
   name?: string;
   snapshotDir?: string;