diff --git a/docs/src/test-api/class-test.md b/docs/src/test-api/class-test.md index 482fd140d8..5113ce3ebc 100644 --- a/docs/src/test-api/class-test.md +++ b/docs/src/test-api/class-test.md @@ -245,6 +245,78 @@ A callback that is run immediately when calling [`method: Test.describe.only`]. +## method: Test.describe.parallel + +Declares a group of tests that could be run in parallel. By default, tests in a single test file run one after another, but using [`method: Test.describe.parallel`] allows them to run in parallel. + +```js js-flavor=js +test.describe.parallel('group', () => { + test('runs in parallel 1', async ({ page }) => { + }); + test('runs in parallel 2', async ({ page }) => { + }); +}); +``` + +```js js-flavor=ts +test.describe.parallel('group', () => { + test('runs in parallel 1', async ({ page }) => { + }); + test('runs in parallel 2', async ({ page }) => { + }); +}); +``` + +Note that parallel tests are executed in separate processes and cannot share any state or global variables. Each of the parallel tests executes all relevant hooks. + +### param: Test.describe.parallel.title +- `title` <[string]> + +Group title. + +### param: Test.describe.parallel.callback +- `callback` <[function]> + +A callback that is run immediately when calling [`method: Test.describe.parallel`]. Any tests added in this callback will belong to the group. + + + +## method: Test.describe.parallel.only + +Declares a focused group of tests that could be run in parallel. By default, tests in a single test file run one after another, but using [`method: Test.describe.parallel`] allows them to run in parallel. If there are some focused tests or suites, all of them will be run but nothing else. + +```js js-flavor=js +test.describe.parallel.only('group', () => { + test('runs in parallel 1', async ({ page }) => { + }); + test('runs in parallel 2', async ({ page }) => { + }); +}); +``` + +```js js-flavor=ts +test.describe.parallel.only('group', () => { + test('runs in parallel 1', async ({ page }) => { + }); + test('runs in parallel 2', async ({ page }) => { + }); +}); +``` + +Note that parallel tests are executed in separate processes and cannot share any state or global variables. Each of the parallel tests executes all relevant hooks. + +### param: Test.describe.parallel.only.title +- `title` <[string]> + +Group title. + +### param: Test.describe.parallel.only.callback +- `callback` <[function]> + +A callback that is run immediately when calling [`method: Test.describe.parallel.only`]. Any tests added in this callback will belong to the group. + + + ## method: Test.describe.serial Declares a group of tests that should always be run serially. If one of the tests fails, all subsequent tests are skipped. All tests in a group are retried together. diff --git a/src/test/dispatcher.ts b/src/test/dispatcher.ts index 1bf557fe1e..77f48acb70 100644 --- a/src/test/dispatcher.ts +++ b/src/test/dispatcher.ts @@ -161,7 +161,7 @@ export class Dispatcher { let outermostSerialSuite: Suite | undefined; for (let parent = this._testById.get(params.failedTestId)!.test.parent; parent; parent = parent.parent) { - if (parent._serial) + if (parent._parallelMode === 'serial') outermostSerialSuite = parent; } diff --git a/src/test/runner.ts b/src/test/runner.ts index 6379624820..406bb4040f 100644 --- a/src/test/runner.ts +++ b/src/test/runner.ts @@ -472,47 +472,61 @@ function createTestGroups(rootSuite: Suite): TestGroup[] { // - They have a different repeatEachIndex - requires different workers. // - They have a different set of worker fixtures in the pool - requires different workers. // - They have a different requireFile - reuses the worker, but runs each requireFile separately. + // - They belong to a parallel suite. - // We try to preserve the order of tests when they require different workers - // by ordering different worker hashes sequentially. - const workerHashToOrdinal = new Map(); - const requireFileToOrdinal = new Map(); + // Using the map "workerHash -> requireFile -> group" makes us preserve the natural order + // of worker hashes and require files for the simple cases. + const groups = new Map>(); + + const createGroup = (test: TestCase): TestGroup => { + return { + workerHash: test._workerHash, + requireFile: test._requireFile, + repeatEachIndex: test._repeatEachIndex, + projectIndex: test._projectIndex, + tests: [], + }; + }; - const groupById = new Map(); for (const projectSuite of rootSuite.suites) { for (const test of projectSuite.allTests()) { - let workerHashOrdinal = workerHashToOrdinal.get(test._workerHash); - if (!workerHashOrdinal) { - workerHashOrdinal = workerHashToOrdinal.size + 1; - workerHashToOrdinal.set(test._workerHash, workerHashOrdinal); + let withWorkerHash = groups.get(test._workerHash); + if (!withWorkerHash) { + withWorkerHash = new Map(); + groups.set(test._workerHash, withWorkerHash); } - - let requireFileOrdinal = requireFileToOrdinal.get(test._requireFile); - if (!requireFileOrdinal) { - requireFileOrdinal = requireFileToOrdinal.size + 1; - requireFileToOrdinal.set(test._requireFile, requireFileOrdinal); - } - - const id = workerHashOrdinal * 10000 + requireFileOrdinal; - let group = groupById.get(id); - if (!group) { - group = { - workerHash: test._workerHash, - requireFile: test._requireFile, - repeatEachIndex: test._repeatEachIndex, - projectIndex: test._projectIndex, - tests: [], + let withRequireFile = withWorkerHash.get(test._requireFile); + if (!withRequireFile) { + withRequireFile = { + general: createGroup(test), + parallel: [], }; - groupById.set(id, group); + withWorkerHash.set(test._requireFile, withRequireFile); + } + + let insideParallel = false; + for (let parent = test.parent; parent; parent = parent.parent) + insideParallel = insideParallel || parent._parallelMode === 'parallel'; + + if (insideParallel) { + const group = createGroup(test); + group.tests.push(test); + withRequireFile.parallel.push(group); + } else { + withRequireFile.general.tests.push(test); } - group.tests.push(test); } } - // Sorting ids will preserve the natural order, because we - // replaced hashes with ordinals according to the natural ordering. - const ids = Array.from(groupById.keys()).sort(); - return ids.map(id => groupById.get(id)!); + const result: TestGroup[] = []; + for (const withWorkerHash of groups.values()) { + for (const withRequireFile of withWorkerHash.values()) { + if (withRequireFile.general.tests.length) + result.push(withRequireFile.general); + result.push(...withRequireFile.parallel); + } + } + return result; } class ListModeReporter implements Reporter { diff --git a/src/test/test.ts b/src/test/test.ts index 5fc97ae651..22d34754d3 100644 --- a/src/test/test.ts +++ b/src/test/test.ts @@ -56,7 +56,7 @@ export class Suite extends Base implements reporterTypes.Suite { _timeout: number | undefined; _annotations: Annotations = []; _modifiers: Modifier[] = []; - _serial = false; + _parallelMode: 'default' | 'serial' | 'parallel' = 'default'; _addTest(test: TestCase) { test.parent = this; @@ -110,7 +110,7 @@ export class Suite extends Base implements reporterTypes.Suite { suite._annotations = this._annotations.slice(); suite._modifiers = this._modifiers.slice(); suite._isDescribe = this._isDescribe; - suite._serial = this._serial; + suite._parallelMode = this._parallelMode; return suite; } } diff --git a/src/test/testType.ts b/src/test/testType.ts index 351338e5bb..563b2ee324 100644 --- a/src/test/testType.ts +++ b/src/test/testType.ts @@ -40,6 +40,8 @@ export class TestTypeImpl { test.only = wrapFunctionWithLocation(this._createTest.bind(this, 'only')); test.describe = wrapFunctionWithLocation(this._describe.bind(this, 'default')); test.describe.only = wrapFunctionWithLocation(this._describe.bind(this, 'only')); + test.describe.parallel = wrapFunctionWithLocation(this._describe.bind(this, 'parallel')); + test.describe.parallel.only = wrapFunctionWithLocation(this._describe.bind(this, 'parallel.only')); test.describe.serial = wrapFunctionWithLocation(this._describe.bind(this, 'serial')); test.describe.serial.only = wrapFunctionWithLocation(this._describe.bind(this, 'serial.only')); test.beforeEach = wrapFunctionWithLocation(this._hook.bind(this, 'beforeEach')); @@ -77,7 +79,7 @@ export class TestTypeImpl { test.expectedStatus = 'skipped'; } - private _describe(type: 'default' | 'only' | 'serial' | 'serial.only', location: Location, title: string, fn: Function) { + private _describe(type: 'default' | 'only' | 'serial' | 'serial.only' | 'parallel' | 'parallel.only', location: Location, title: string, fn: Function) { throwIfRunningInsideJest(); const suite = currentlyLoadingFileSuite(); if (!suite) @@ -98,10 +100,19 @@ export class TestTypeImpl { child.location = location; suite._addSuite(child); - if (type === 'only' || type === 'serial.only') + if (type === 'only' || type === 'serial.only' || type === 'parallel.only') child._only = true; if (type === 'serial' || type === 'serial.only') - child._serial = true; + child._parallelMode = 'serial'; + if (type === 'parallel' || type === 'parallel.only') + child._parallelMode = 'parallel'; + + for (let parent: Suite | undefined = suite; parent; parent = parent.parent) { + if (parent._parallelMode === 'serial' && child._parallelMode === 'parallel') + throw errorWithLocation(location, 'describe.parallel cannot be nested inside describe.serial'); + if (parent._parallelMode === 'parallel' && child._parallelMode === 'serial') + throw errorWithLocation(location, 'describe.serial cannot be nested inside describe.parallel'); + } setCurrentlyLoadingFileSuite(child); fn(); diff --git a/tests/playwright-test/test-parallel.spec.ts b/tests/playwright-test/test-parallel.spec.ts new file mode 100644 index 0000000000..2c408443a5 --- /dev/null +++ b/tests/playwright-test/test-parallel.spec.ts @@ -0,0 +1,60 @@ +/** + * Copyright Microsoft Corporation. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test, expect } from './playwright-test-fixtures'; + +test('test.describe.parallel should throw inside test.describe.serial', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': ` + const { test } = pwt; + test.describe.serial('serial suite', () => { + test.describe.parallel('parallel suite', () => { + }); + }); + `, + }); + expect(result.exitCode).toBe(1); + expect(result.output).toContain('a.test.ts:7:23: describe.parallel cannot be nested inside describe.serial'); +}); + +test('test.describe.parallel should work', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': ` + const { test } = pwt; + test.describe.parallel('parallel suite', () => { + test('test1', async ({}, testInfo) => { + console.log('\\n%% worker=' + testInfo.workerIndex); + await new Promise(f => setTimeout(f, 1000)); + }); + test('test2', async ({}, testInfo) => { + console.log('\\n%% worker=' + testInfo.workerIndex); + await new Promise(f => setTimeout(f, 1000)); + }); + test.describe('inner suite', () => { + test('test3', async ({}, testInfo) => { + console.log('\\n%% worker=' + testInfo.workerIndex); + await new Promise(f => setTimeout(f, 1000)); + }); + }); + }); + `, + }, { workers: 3 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(3); + expect(result.output).toContain('%% worker=0'); + expect(result.output).toContain('%% worker=1'); + expect(result.output).toContain('%% worker=2'); +}); diff --git a/tests/playwright-test/test-serial.spec.ts b/tests/playwright-test/test-serial.spec.ts index 9e4fd63ca3..054b5ddcb0 100644 --- a/tests/playwright-test/test-serial.spec.ts +++ b/tests/playwright-test/test-serial.spec.ts @@ -210,3 +210,17 @@ test('test.describe.serial should work with test.fail and retries', async ({ run '%%three', ]); }); + +test('test.describe.serial should throw inside test.describe.parallel', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': ` + const { test } = pwt; + test.describe.parallel('parallel suite', () => { + test.describe.serial('serial suite', () => { + }); + }); + `, + }); + expect(result.exitCode).toBe(1); + expect(result.output).toContain('a.test.ts:7:23: describe.serial cannot be nested inside describe.parallel'); +}); diff --git a/types/test.d.ts b/types/test.d.ts index 6baf3d0829..48b29520b4 100644 --- a/types/test.d.ts +++ b/types/test.d.ts @@ -1663,6 +1663,68 @@ export interface TestType { + * test('runs in parallel 1', async ({ page }) => { + * }); + * test('runs in parallel 2', async ({ page }) => { + * }); + * }); + * ``` + * + * ```js js-flavor=ts + * test.describe.parallel('group', () => { + * test('runs in parallel 1', async ({ page }) => { + * }); + * test('runs in parallel 2', async ({ page }) => { + * }); + * }); + * ``` + * + * Note that parallel tests are executed in separate processes and cannot share any state or global variables. Each of the + * parallel tests executes all relevant hooks. + * @param title Group title. + * @param callback A callback that is run immediately when calling [test.describe.parallel(title, callback)](https://playwright.dev/docs/api/class-test#test-describe-parallel). Any tests + * added in this callback will belong to the group. + */ + parallel: SuiteFunction & { + /** + * Declares a focused group of tests that could be run in parallel. By default, tests in a single test file run one after + * another, but using + * [test.describe.parallel(title, callback)](https://playwright.dev/docs/api/class-test#test-describe-parallel) allows them + * to run in parallel. If there are some focused tests or suites, all of them will be run but nothing else. + * + * ```js js-flavor=js + * test.describe.parallel.only('group', () => { + * test('runs in parallel 1', async ({ page }) => { + * }); + * test('runs in parallel 2', async ({ page }) => { + * }); + * }); + * ``` + * + * ```js js-flavor=ts + * test.describe.parallel.only('group', () => { + * test('runs in parallel 1', async ({ page }) => { + * }); + * test('runs in parallel 2', async ({ page }) => { + * }); + * }); + * ``` + * + * Note that parallel tests are executed in separate processes and cannot share any state or global variables. Each of the + * parallel tests executes all relevant hooks. + * @param title Group title. + * @param callback A callback that is run immediately when calling [test.describe.parallel.only(title, callback)](https://playwright.dev/docs/api/class-test#test-describe-parallel-only). + * Any tests added in this callback will belong to the group. + */ only: SuiteFunction; }; }; diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index f23ac97424..d165931f6c 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -226,6 +226,9 @@ export interface TestType Promise | void): void; skip(): void;