From 4b19d59ec53e6b2fcb57fd709225ea0540da22ae Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Tue, 1 Mar 2022 18:12:21 -0800 Subject: [PATCH] feat(test): introduce fully parallel mode (#12446) --- docs/src/test-api/class-testconfig.md | 8 +++ docs/src/test-api/class-testproject.md | 8 +++ docs/src/test-parallel-js.md | 3 +- packages/playwright-test/src/cli.ts | 2 + packages/playwright-test/src/loader.ts | 3 ++ packages/playwright-test/src/runner.ts | 2 + packages/playwright-test/types/test.d.ts | 24 +++++++++ tests/config/default.playwright.config.ts | 1 + tests/playwright-test/test-parallel.spec.ts | 60 +++++++++++++++++++++ utils/generate_types/overrides-test.d.ts | 3 ++ 10 files changed, 113 insertions(+), 1 deletion(-) diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index f58262d48f..f006b212e9 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -102,6 +102,14 @@ const config: PlaywrightTestConfig = { export default config; ``` +## property: TestConfig.fullyParallel +- type: <[boolean]> + +Playwright Test runs tests in parallel. In order to achieve that, it runs several worker processes that run at the same time. +By default, **test files** are run in parallel. Tests in a single file are run in order, in the same worker process. + +You can configure entire test run to concurrently execute all tests in all files using this option. + ## property: TestConfig.globalSetup - type: <[string]> diff --git a/docs/src/test-api/class-testproject.md b/docs/src/test-api/class-testproject.md index c9c4f4df95..5a745625a7 100644 --- a/docs/src/test-api/class-testproject.md +++ b/docs/src/test-api/class-testproject.md @@ -116,6 +116,14 @@ Configuration for the `expect` assertion library. Use [`property: TestConfig.expect`] to change this option for all projects. +## property: TestProject.fullyParallel +- type: <[boolean]> + +Playwright Test runs tests in parallel. In order to achieve that, it runs several worker processes that run at the same time. +By default, **test files** are run in parallel. Tests in a single file are run in order, in the same worker process. + +You can configure entire test project to concurrently run all tests in all files using this option. + ## property: TestProject.metadata - type: <[Object]> diff --git a/docs/src/test-parallel-js.md b/docs/src/test-parallel-js.md index 74a9e4d867..bb0c58ef77 100644 --- a/docs/src/test-parallel-js.md +++ b/docs/src/test-parallel-js.md @@ -6,7 +6,8 @@ title: "Parallelism and sharding" Playwright Test runs tests in parallel. In order to achieve that, it runs several worker processes that run at the same time. - By default, **test files** are run in parallel. Tests in a single file are run in order, in the same worker process. -- Group tests with [`test.describe.parallel`](#parallelize-tests-in-a-single-file) to run **tests in a single file** in parallel. +- Configure tests using [`test.describe.configure`](#parallelize-tests-in-a-single-file) to run **tests in a single file** in parallel. +- You can configure entire project to have all tests in all files to run in parallel using [`property: TestProject.fullyParallel`] or [`property: TestConfig.fullyParallel`] - To **disable** parallelism limit the number of [workers to one](#disable-parallelism). You can control the number of [parallel worker processes](#limit-workers) and [limit the number of failures](#limit-failures-and-fail-fast) in the whole test suite for efficiency. diff --git a/packages/playwright-test/src/cli.ts b/packages/playwright-test/src/cli.ts index 7c07929504..94317ba54e 100644 --- a/packages/playwright-test/src/cli.ts +++ b/packages/playwright-test/src/cli.ts @@ -40,6 +40,7 @@ export function addTestCommand(program: Command) { command.option('--debug', `Run tests with Playwright Inspector. Shortcut for "PWDEBUG=1" environment variable and "--timeout=0 --maxFailures=1 --headed --workers=1" options`); command.option('-c, --config ', `Configuration file, or a test directory with optional ${kDefaultConfigFiles.map(file => `"${file}"`).join('/')}`); command.option('--forbid-only', `Fail if test.only is called (default: false)`); + command.option('--fully-parallel', `Run all tests in parallel (default: false)`); command.option('-g, --grep ', `Only run tests matching this regular expression (default: ".*")`); command.option('-gv, --grep-invert ', `Only run tests that do not match this regular expression`); command.option('--global-timeout ', `Maximum time this test suite can run in milliseconds (default: unlimited)`); @@ -196,6 +197,7 @@ function overridesFromOptions(options: { [key: string]: any }): Config { const shardPair = options.shard ? options.shard.split('/').map((t: string) => parseInt(t, 10)) : undefined; return { forbidOnly: options.forbidOnly ? true : undefined, + fullyParallel: options.fullyParallel ? true : undefined, globalTimeout: options.globalTimeout ? parseInt(options.globalTimeout, 10) : undefined, grep: options.grep ? forceRegExp(options.grep) : undefined, grepInvert: options.grepInvert ? forceRegExp(options.grepInvert) : undefined, diff --git a/packages/playwright-test/src/loader.ts b/packages/playwright-test/src/loader.ts index f806581563..45a0a50bc9 100644 --- a/packages/playwright-test/src/loader.ts +++ b/packages/playwright-test/src/loader.ts @@ -93,6 +93,7 @@ export class Loader { this._fullConfig.rootDir = this._config.testDir || rootDir; this._fullConfig.forbidOnly = takeFirst(this._configOverrides.forbidOnly, this._config.forbidOnly, baseFullConfig.forbidOnly); + this._fullConfig.fullyParallel = takeFirst(this._configOverrides.fullyParallel, this._config.fullyParallel, baseFullConfig.fullyParallel); this._fullConfig.globalSetup = takeFirst(this._configOverrides.globalSetup, this._config.globalSetup, baseFullConfig.globalSetup); this._fullConfig.globalTeardown = takeFirst(this._configOverrides.globalTeardown, this._config.globalTeardown, baseFullConfig.globalTeardown); this._fullConfig.globalTimeout = takeFirst(this._configOverrides.globalTimeout, this._configOverrides.globalTimeout, this._config.globalTimeout, baseFullConfig.globalTimeout); @@ -202,6 +203,7 @@ export class Loader { if (!path.isAbsolute(snapshotDir)) snapshotDir = path.resolve(configDir, snapshotDir); const fullProject: FullProject = { + fullyParallel: takeFirst(this._configOverrides.fullyParallel, projectConfig.fullyParallel, this._config.fullyParallel, undefined), expect: takeFirst(this._configOverrides.expect, projectConfig.expect, this._config.expect, undefined), outputDir, repeatEach: takeFirst(this._configOverrides.repeatEach, projectConfig.repeatEach, this._config.repeatEach, 1), @@ -429,6 +431,7 @@ function validateProject(file: string, project: Project, title: string) { const baseFullConfig: FullConfig = { forbidOnly: false, + fullyParallel: false, globalSetup: null, globalTeardown: null, globalTimeout: 0, diff --git a/packages/playwright-test/src/runner.ts b/packages/playwright-test/src/runner.ts index 0fae2612a3..ad11d139c4 100644 --- a/packages/playwright-test/src/runner.ts +++ b/packages/playwright-test/src/runner.ts @@ -286,6 +286,8 @@ export class Runner { for (const [project, files] of filesByProject) { const projectSuite = new Suite(project.config.name); projectSuite._projectConfig = project.config; + if (project.config.fullyParallel) + projectSuite._parallelMode = 'parallel'; rootSuite._addSuite(projectSuite); for (const file of files) { const fileSuite = fileSuites.get(file); diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index ba9a766457..5e4bcbeb7e 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -125,6 +125,14 @@ interface TestProject { * all projects. */ expect?: ExpectSettings; + /** + * Playwright Test runs tests in parallel. In order to achieve that, it runs several worker processes that run at the same + * time. By default, **test files** are run in parallel. Tests in a single file are run in order, in the same worker + * process. + * + * You can configure entire test project to concurrently run all tests in all files using this option. + */ + fullyParallel?: boolean; /** * Any JSON-serializable metadata that will be put directly to the test report. */ @@ -437,6 +445,14 @@ interface TestConfig { * */ forbidOnly?: boolean; + /** + * Playwright Test runs tests in parallel. In order to achieve that, it runs several worker processes that run at the same + * time. By default, **test files** are run in parallel. Tests in a single file are run in order, in the same worker + * process. + * + * You can configure entire test run to concurrently execute all tests in all files using this option. + */ + fullyParallel?: boolean; /** * Path to the global setup file. This file will be required and run before all the tests. It must export a single function * that takes a [`TestConfig`] argument. @@ -907,6 +923,14 @@ export interface FullConfig { * */ forbidOnly: boolean; + /** + * Playwright Test runs tests in parallel. In order to achieve that, it runs several worker processes that run at the same + * time. By default, **test files** are run in parallel. Tests in a single file are run in order, in the same worker + * process. + * + * You can configure entire test run to concurrently execute all tests in all files using this option. + */ + fullyParallel: boolean; /** * Path to the global setup file. This file will be required and run before all the tests. It must export a single function * that takes a [`TestConfig`] argument. diff --git a/tests/config/default.playwright.config.ts b/tests/config/default.playwright.config.ts index c9bdafbea2..effee40413 100644 --- a/tests/config/default.playwright.config.ts +++ b/tests/config/default.playwright.config.ts @@ -49,6 +49,7 @@ const config: Config { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { fullyParallel: true }; + `, + 'a.test.ts': ` + const { test } = pwt; + test('test1', async ({}, testInfo) => { + console.log('\\n%% worker=' + testInfo.workerIndex); + await new Promise(f => setTimeout(f, 1000)); + }); + test('test2', async ({}, testInfo) => { + console.log('\\n%% worker=' + testInfo.workerIndex); + await new Promise(f => setTimeout(f, 1000)); + }); + test.describe('inner suite', () => { + test('test3', async ({}, testInfo) => { + console.log('\\n%% worker=' + testInfo.workerIndex); + await new Promise(f => setTimeout(f, 1000)); + }); + }); + `, + }, { workers: 3 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(3); + expect(result.output).toContain('%% worker=0'); + expect(result.output).toContain('%% worker=1'); + expect(result.output).toContain('%% worker=2'); +}); + +test('project.fullyParallel should work', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { projects: [ { fullyParallel: true } ] }; + `, + 'a.test.ts': ` + const { test } = pwt; + test('test1', async ({}, testInfo) => { + console.log('\\n%% worker=' + testInfo.workerIndex); + await new Promise(f => setTimeout(f, 1000)); + }); + test('test2', async ({}, testInfo) => { + console.log('\\n%% worker=' + testInfo.workerIndex); + await new Promise(f => setTimeout(f, 1000)); + }); + test.describe('inner suite', () => { + test('test3', async ({}, testInfo) => { + console.log('\\n%% worker=' + testInfo.workerIndex); + await new Promise(f => setTimeout(f, 1000)); + }); + }); + `, + }, { workers: 3 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(3); + expect(result.output).toContain('%% worker=0'); + expect(result.output).toContain('%% worker=1'); + expect(result.output).toContain('%% worker=2'); +}); diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index a5e33d76ca..359d922f85 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -59,6 +59,7 @@ type ExpectSettings = { interface TestProject { expect?: ExpectSettings; + fullyParallel?: boolean; metadata?: any; name?: string; snapshotDir?: string; @@ -117,6 +118,7 @@ type LiteralUnion = T | (U & { zz_IGNORE_ME?: never }); interface TestConfig { forbidOnly?: boolean; + fullyParallel?: boolean; globalSetup?: string; globalTeardown?: string; globalTimeout?: number; @@ -153,6 +155,7 @@ export interface Config extends TestConfig { export interface FullConfig { forbidOnly: boolean; + fullyParallel: boolean; globalSetup: string | null; globalTeardown: string | null; globalTimeout: number;