From 58242e35925eab1699e4eff6d2947f5fdaf4895d Mon Sep 17 00:00:00 2001 From: Mathias Leppich Date: Thu, 10 Oct 2024 20:02:08 +0200 Subject: [PATCH] feat(test-runner): test filter --- docs/src/test-api/class-fullconfig.md | 6 +++ docs/src/test-api/class-testconfig.md | 7 ++++ packages/playwright/src/common/config.ts | 3 +- .../playwright/src/isomorphic/teleReceiver.ts | 1 + packages/playwright/src/runner/loadUtils.ts | 25 ++++++++++--- packages/playwright/types/test.d.ts | 14 +++++++ tests/playwright-test/test-filter.spec.ts | 37 +++++++++++++++++++ utils/generate_types/overrides-test.d.ts | 4 ++ 8 files changed, 91 insertions(+), 6 deletions(-) create mode 100644 tests/playwright-test/test-filter.spec.ts diff --git a/docs/src/test-api/class-fullconfig.md b/docs/src/test-api/class-fullconfig.md index e6437ab314..5523142efe 100644 --- a/docs/src/test-api/class-fullconfig.md +++ b/docs/src/test-api/class-fullconfig.md @@ -40,6 +40,12 @@ See [`property: TestConfig.globalTeardown`]. See [`property: TestConfig.globalTimeout`]. +## property: FullConfig.filter +* since: v1.49 +- type: <[null]|[TestFilter]|[Array]<[TestFilter]>> + +See [`property: TestConfig.filter`]. + ## property: FullConfig.grep * since: v1.10 - type: <[RegExp]|[Array]<[RegExp]>> diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index d013f5e4ea..ad5efdbb29 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -160,6 +160,13 @@ export default defineConfig({ }); ``` +## property: TestConfig.filter +* since: v1.49 +- type: ?<[TestFilter]|[Array]<[TestFilter]>> + +Filter tests by passing a function. + + ## property: TestConfig.grep * since: v1.10 - type: ?<[RegExp]|[Array]<[RegExp]>> diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts index d7fb499645..174626d327 100644 --- a/packages/playwright/src/common/config.ts +++ b/packages/playwright/src/common/config.ts @@ -78,6 +78,7 @@ export class FullConfigInternal { globalSetup: takeFirst(resolveScript(userConfig.globalSetup, configDir), null), globalTeardown: takeFirst(resolveScript(userConfig.globalTeardown, configDir), null), globalTimeout: takeFirst(configCLIOverrides.globalTimeout, userConfig.globalTimeout, 0), + filter: takeFirst(userConfig.filter, undefined), grep: takeFirst(userConfig.grep, defaultGrep), grepInvert: takeFirst(userConfig.grepInvert, null), maxFailures: takeFirst(configCLIOverrides.debug ? 1 : undefined, configCLIOverrides.maxFailures, userConfig.maxFailures, 0), @@ -289,4 +290,4 @@ const configInternalSymbol = Symbol('configInternalSymbol'); export function getProjectId(project: FullProject): string { return (project as any).__projectId!; -} \ No newline at end of file +} diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts index 0c4408096d..238884fdbc 100644 --- a/packages/playwright/src/isomorphic/teleReceiver.ts +++ b/packages/playwright/src/isomorphic/teleReceiver.ts @@ -581,6 +581,7 @@ export const baseFullConfig: reporterTypes.FullConfig = { globalSetup: null, globalTeardown: null, globalTimeout: 0, + filter: null, grep: /.*/, grepInvert: null, maxFailures: 0, diff --git a/packages/playwright/src/runner/loadUtils.ts b/packages/playwright/src/runner/loadUtils.ts index 63a2307507..86a1da797b 100644 --- a/packages/playwright/src/runner/loadUtils.ts +++ b/packages/playwright/src/runner/loadUtils.ts @@ -16,6 +16,7 @@ import path from 'path'; import type { FullConfig, Reporter, TestError } from '../../types/testReporter'; +import type * as reporterTypes from '../../types/testReporter'; import { InProcessLoaderHost, OutOfProcessLoaderHost } from './loaderHost'; import { Suite } from '../common/test'; import type { TestCase } from '../common/test'; @@ -173,22 +174,36 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho } // Shard only the top-level projects. - if (config.config.shard) { + if (config.config.shard || config.config.filter) { // Create test groups for top-level projects. const testGroups: TestGroup[] = []; for (const projectSuite of rootSuite.suites) testGroups.push(...createTestGroups(projectSuite, config.config.workers)); + if (config.config.filter) { + const filters = Array.isArray(config.config.filter) ? config.config.filter : [config.config.filter]; + + const allTests = new Set(testGroups.flatMap(group => group.tests)); + + let filteredTests = [...allTests.values()]; + for (const filter of filters) + filteredTests = filter(filteredTests); + + const filteredTestSet = new Set(filteredTests); + for (const group of testGroups) + group.tests = group.tests.filter(test => filteredTestSet.has(test)); + } + // Shard test groups. - const testGroupsInThisShard = filterForShard(config.config.shard, testGroups); - const testsInThisShard = new Set(); + const testGroupsInThisShard = config.config.shard ? filterForShard(config.config.shard, testGroups) : new Set(testGroups); + const testsInThisRun = new Set(); for (const group of testGroupsInThisShard) { for (const test of group.tests) - testsInThisShard.add(test); + testsInThisRun.add(test); } // Update project suites, removing empty ones. - filterTestsRemoveEmptySuites(rootSuite, test => testsInThisShard.has(test)); + filterTestsRemoveEmptySuites(rootSuite, test => testsInThisRun.has(test)); } // Now prepend dependency projects without filtration. diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index 91fe2f2b5c..1485744d6b 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -18,6 +18,8 @@ import type { APIRequestContext, Browser, BrowserContext, BrowserContextOptions, Page, LaunchOptions, ViewportSize, Geolocation, HTTPCredentials, Locator, APIResponse, PageScreenshotOptions } from 'playwright-core'; export * from 'playwright-core'; +import type { TestCase } from './testReporter'; + export type ReporterDescription = Readonly< ['blob'] | ['blob', { outputDir?: string, fileName?: string }] | ['dot'] | @@ -1034,6 +1036,11 @@ interface TestConfig { }; }; + /** + * Filter tests by passing a function. + */ + filter?: TestFilter|Array; + /** * Whether to exit with an error if any tests or groups are marked as * [test.only(title[, details, body])](https://playwright.dev/docs/api/class-test#test-only) or @@ -1720,6 +1727,11 @@ export interface FullConfig { */ configFile?: string; + /** + * See [testConfig.filter](https://playwright.dev/docs/api/class-testconfig#test-config-filter). + */ + filter: null|TestFilter|Array; + /** * See [testConfig.forbidOnly](https://playwright.dev/docs/api/class-testconfig#test-config-forbid-only). */ @@ -1838,6 +1850,8 @@ export type TestDetails = { annotation?: TestDetailsAnnotation | TestDetailsAnnotation[]; } +export type TestFilter = (tests: TestCase[]) => TestCase[]; + interface SuiteFunction { /** * Declares a group of tests. diff --git a/tests/playwright-test/test-filter.spec.ts b/tests/playwright-test/test-filter.spec.ts new file mode 100644 index 0000000000..c2b3c04c1e --- /dev/null +++ b/tests/playwright-test/test-filter.spec.ts @@ -0,0 +1,37 @@ +/** + * Copyright Microsoft Corporation. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test, expect } from './playwright-test-fixtures'; + +test('config.filter should work', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + filter: (tests) => tests.filter(test => test.title === 'test1'), + }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('test1', async () => { console.log('\\n%% test1'); }); + test('test2', async () => { console.log('\\n%% test2'); }); + `, + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + expect(result.outputLines).toEqual([ + 'test1', + ]); +}); diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index be1fa7ee37..16645e1abb 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -17,6 +17,8 @@ import type { APIRequestContext, Browser, BrowserContext, BrowserContextOptions, Page, LaunchOptions, ViewportSize, Geolocation, HTTPCredentials, Locator, APIResponse, PageScreenshotOptions } from 'playwright-core'; export * from 'playwright-core'; +import type { TestCase } from './testReporter'; + export type ReporterDescription = Readonly< ['blob'] | ['blob', { outputDir?: string, fileName?: string }] | ['dot'] | @@ -75,6 +77,8 @@ export type TestDetails = { annotation?: TestDetailsAnnotation | TestDetailsAnnotation[]; } +export type TestFilter = (tests: TestCase[]) => TestCase[]; + interface SuiteFunction { (title: string, callback: () => void): void; (callback: () => void): void;