diff --git a/docs/src/test-api/class-test.md b/docs/src/test-api/class-test.md index 2736d5d47d..617437e810 100644 --- a/docs/src/test-api/class-test.md +++ b/docs/src/test-api/class-test.md @@ -248,6 +248,46 @@ Group title. A callback that is run immediately when calling [`method: Test.describe`]. Any tests added in this callback will belong to the group. +## method: Test.describe.configure + +Set execution mode of execution for the enclosing scope. Can be executed either on the top level or inside a describe. Configuration applies to the entire scope, regardless of whether it run before or after the test +declaration. + +Learn more about the execution modes [here](./test-parallel-js.md). + +```js js-flavor=js +// Run all the tests in the file concurrently using parallel workers. +test.describe.configure({ mode: 'parallel' }); +test('runs in parallel 1', async ({ page }) => {}); +test('runs in parallel 2', async ({ page }) => {}); +``` + +```js js-flavor=ts +// Run all the tests in the file concurrently using parallel workers. +test.describe.configure({ mode: 'parallel' }); +test('runs in parallel 1', async ({ page }) => {}); +test('runs in parallel 2', async ({ page }) => {}); +``` + +```js js-flavor=js +// Annotate tests as inter-dependent. +test.describe.configure({ mode: 'serial' }); +test('runs first', async ({ page }) => {}); +test('runs second', async ({ page }) => {}); +``` + +```js js-flavor=ts +// Annotate tests as inter-dependent. +test.describe.configure({ mode: 'serial' }); +test('runs first', async ({ page }) => {}); +test('runs second', async ({ page }) => {}); +``` + +### option: Test.describe.configure.mode +- `mode` <"parallel"|"serial"> + + + ## method: Test.describe.only Declares a focused group of tests. If there are some focused tests or suites, all of them will be run but nothing else. @@ -290,21 +330,19 @@ A callback that is run immediately when calling [`method: Test.describe.only`]. Declares a group of tests that could be run in parallel. By default, tests in a single test file run one after another, but using [`method: Test.describe.parallel`] allows them to run in parallel. +See [`method: Test.describe.configure`] for the preferred way of configuring the execution mode. + ```js js-flavor=js test.describe.parallel('group', () => { - test('runs in parallel 1', async ({ page }) => { - }); - test('runs in parallel 2', async ({ page }) => { - }); + test('runs in parallel 1', async ({ page }) => {}); + test('runs in parallel 2', async ({ page }) => {}); }); ``` ```js js-flavor=ts test.describe.parallel('group', () => { - test('runs in parallel 1', async ({ page }) => { - }); - test('runs in parallel 2', async ({ page }) => { - }); + test('runs in parallel 1', async ({ page }) => {}); + test('runs in parallel 2', async ({ page }) => {}); }); ``` @@ -342,25 +380,23 @@ A callback that is run immediately when calling [`method: Test.describe.parallel Declares a group of tests that should always be run serially. If one of the tests fails, all subsequent tests are skipped. All tests in a group are retried together. +See [`method: Test.describe.configure`] for the preferred way of configuring the execution mode. + :::note Using serial is not recommended. It is usually better to make your tests isolated, so they can be run independently. ::: ```js js-flavor=js test.describe.serial('group', () => { - test('runs first', async ({ page }) => { - }); - test('runs second', async ({ page }) => { - }); + test('runs first', async ({ page }) => {}); + test('runs second', async ({ page }) => {}); }); ``` ```js js-flavor=ts test.describe.serial('group', () => { - test('runs first', async ({ page }) => { - }); - test('runs second', async ({ page }) => { - }); + test('runs first', async ({ page }) => {}); + test('runs second', async ({ page }) => {}); }); ``` diff --git a/docs/src/test-auth-js.md b/docs/src/test-auth-js.md index ccd2965dd1..f2a4c80da6 100644 --- a/docs/src/test-auth-js.md +++ b/docs/src/test-auth-js.md @@ -293,30 +293,30 @@ order to achieve that: const { test } = require('@playwright/test'); -test.describe.serial('use the same page', () => { - /** @type {import('@playwright/test').Page} */ - let page; +test.describe.configure({ mode: 'serial' }); - test.beforeAll(async ({ browser }) => { - // Create page yourself and sign in. - page = await browser.newPage(); - await page.goto('https://github.com/login'); - await page.fill('input[name="user"]', 'user'); - await page.fill('input[name="password"]', 'password'); - await page.click('text=Sign in'); - }); +/** @type {import('@playwright/test').Page} */ +let page; - test.afterAll(async () => { - await page.close(); - }); +test.beforeAll(async ({ browser }) => { + // Create page yourself and sign in. + page = await browser.newPage(); + await page.goto('https://github.com/login'); + await page.fill('input[name="user"]', 'user'); + await page.fill('input[name="password"]', 'password'); + await page.click('text=Sign in'); +}); - test('first test', async () => { - // page is signed in. - }); +test.afterAll(async () => { + await page.close(); +}); - test('second test', async () => { - // page is signed in. - }); +test('first test', async () => { + // page is signed in. +}); + +test('second test', async () => { + // page is signed in. }); ``` @@ -325,29 +325,29 @@ test.describe.serial('use the same page', () => { import { test, Page } from '@playwright/test'; -test.describe.serial('use the same page', () => { - let page: Page; +test.describe.configure({ mode: 'serial' }); - test.beforeAll(async ({ browser }) => { - // Create page once and sign in. - page = await browser.newPage(); - await page.goto('https://github.com/login'); - await page.fill('input[name="user"]', 'user'); - await page.fill('input[name="password"]', 'password'); - await page.click('text=Sign in'); - }); +let page: Page; - test.afterAll(async () => { - await page.close(); - }); +test.beforeAll(async ({ browser }) => { + // Create page once and sign in. + page = await browser.newPage(); + await page.goto('https://github.com/login'); + await page.fill('input[name="user"]', 'user'); + await page.fill('input[name="password"]', 'password'); + await page.click('text=Sign in'); +}); - test('first test', async () => { - // page is signed in. - }); +test.afterAll(async () => { + await page.close(); +}); - test('second test', async () => { - // page is signed in. - }); +test('first test', async () => { + // page is signed in. +}); + +test('second test', async () => { + // page is signed in. }); ``` diff --git a/docs/src/test-parallel-js.md b/docs/src/test-parallel-js.md index ee43542980..74a9e4d867 100644 --- a/docs/src/test-parallel-js.md +++ b/docs/src/test-parallel-js.md @@ -72,18 +72,81 @@ Note that parallel tests are executed in separate worker processes and cannot sh ```js js-flavor=js const { test } = require('@playwright/test'); -test.describe.parallel('suite', () => { - test('runs in parallel 1', async ({ page }) => { /* ... */ }); - test('runs in parallel 2', async ({ page }) => { /* ... */ }); -}); +test.describe.configure({ mode: 'parallel' }); + +test('runs in parallel 1', async ({ page }) => { /* ... */ }); +test('runs in parallel 2', async ({ page }) => { /* ... */ }); ``` ```js js-flavor=ts import { test } from '@playwright/test'; -test.describe.parallel('suite', () => { - test('runs in parallel 1', async ({ page }) => { /* ... */ }); - test('runs in parallel 2', async ({ page }) => { /* ... */ }); +test.describe.configure({ mode: 'parallel' }); + +test('runs in parallel 1', async ({ page }) => { /* ... */ }); +test('runs in parallel 2', async ({ page }) => { /* ... */ }); +``` + +## Serial mode + +You can annotate inter-dependent tests as serial. If one of the serial tests +fails, all subsequent tests are skipped. All tests in a group are retried together. + +:::note +Using serial is not recommended. It is usually better to make your tests isolated, so they can be run independently. +::: + +```js js-flavor=js +// @ts-check + +const { test } = require('@playwright/test'); + +test.describe.configure({ mode: 'serial' }); + +/** @type {import('@playwright/test').Page} */ +let page; + +test.beforeAll(async ({ browser }) => { + page = await browser.newPage(); +}); + +test.afterAll(async () => { + await page.close(); +}); + +test('runs first', async () => { + await page.goto('https://playwright.dev/'); +}); + +test('runs second', async () => { + await page.click('text=Get Started'); +}); +``` + +```js js-flavor=ts +// example.spec.ts + +import { test, Page } from '@playwright/test'; + +// Annotate entire file as serial. +test.describe.configure({ mode: 'serial' }); + +let page: Page; + +test.beforeAll(async ({ browser }) => { + page = await browser.newPage(); +}); + +test.afterAll(async () => { + await page.close(); +}); + +test('runs first', async () => { + await page.goto('https://playwright.dev/'); +}); + +test('runs second', async () => { + await page.click('text=Get Started'); }); ``` diff --git a/docs/src/test-retries-js.md b/docs/src/test-retries-js.md index 7c150306e5..204caabf7c 100644 --- a/docs/src/test-retries-js.md +++ b/docs/src/test-retries-js.md @@ -143,23 +143,23 @@ Consider the following snippet that uses `test.describe.serial`: ```js js-flavor=js const { test } = require('@playwright/test'); -test.describe.serial('suite', () => { - test.beforeAll(async () => { /* ... */ }); - test('first good', async ({ page }) => { /* ... */ }); - test('second flaky', async ({ page }) => { /* ... */ }); - test('third good', async ({ page }) => { /* ... */ }); -}); +test.describe.configure({ mode: 'serial' }); + +test.beforeAll(async () => { /* ... */ }); +test('first good', async ({ page }) => { /* ... */ }); +test('second flaky', async ({ page }) => { /* ... */ }); +test('third good', async ({ page }) => { /* ... */ }); ``` ```js js-flavor=ts import { test } from '@playwright/test'; -test.describe.serial('suite', () => { - test.beforeAll(async () => { /* ... */ }); - test('first good', async ({ page }) => { /* ... */ }); - test('second flaky', async ({ page }) => { /* ... */ }); - test('third good', async ({ page }) => { /* ... */ }); -}); +test.describe.configure({ mode: 'serial' }); + +test.beforeAll(async () => { /* ... */ }); +test('first good', async ({ page }) => { /* ... */ }); +test('second flaky', async ({ page }) => { /* ... */ }); +test('third good', async ({ page }) => { /* ... */ }); ``` When running without [retries](#retries), all tests after the failure are skipped: @@ -195,25 +195,25 @@ Playwright Test creates an isolated [Page] object for each test. However, if you const { test } = require('@playwright/test'); -test.describe.serial('use the same page', () => { - /** @type {import('@playwright/test').Page} */ - let page; +test.describe.configure({ mode: 'serial' }); - test.beforeAll(async ({ browser }) => { - page = await browser.newPage(); - }); +/** @type {import('@playwright/test').Page} */ +let page; - test.afterAll(async () => { - await page.close(); - }); +test.beforeAll(async ({ browser }) => { + page = await browser.newPage(); +}); - test('runs first', async () => { - await page.goto('https://playwright.dev/'); - }); +test.afterAll(async () => { + await page.close(); +}); - test('runs second', async () => { - await page.click('text=Get Started'); - }); +test('runs first', async () => { + await page.goto('https://playwright.dev/'); +}); + +test('runs second', async () => { + await page.click('text=Get Started'); }); ``` @@ -222,23 +222,23 @@ test.describe.serial('use the same page', () => { import { test, Page } from '@playwright/test'; -test.describe.serial('use the same page', () => { - let page: Page; +test.describe.configure({ mode: 'serial' }); - test.beforeAll(async ({ browser }) => { - page = await browser.newPage(); - }); +let page: Page; - test.afterAll(async () => { - await page.close(); - }); +test.beforeAll(async ({ browser }) => { + page = await browser.newPage(); +}); - test('runs first', async () => { - await page.goto('https://playwright.dev/'); - }); +test.afterAll(async () => { + await page.close(); +}); - test('runs second', async () => { - await page.click('text=Get Started'); - }); +test('runs first', async () => { + await page.goto('https://playwright.dev/'); +}); + +test('runs second', async () => { + await page.click('text=Get Started'); }); ``` diff --git a/packages/playwright-test/src/testType.ts b/packages/playwright-test/src/testType.ts index b0af49a36c..3b3f795815 100644 --- a/packages/playwright-test/src/testType.ts +++ b/packages/playwright-test/src/testType.ts @@ -36,6 +36,7 @@ export class TestTypeImpl { test.only = wrapFunctionWithLocation(this._createTest.bind(this, 'only')); test.describe = wrapFunctionWithLocation(this._describe.bind(this, 'default')); test.describe.only = wrapFunctionWithLocation(this._describe.bind(this, 'only')); + test.describe.configure = wrapFunctionWithLocation(this._configure.bind(this)); test.describe.parallel = wrapFunctionWithLocation(this._describe.bind(this, 'parallel')); test.describe.parallel.only = wrapFunctionWithLocation(this._describe.bind(this, 'parallel.only')); test.describe.serial = wrapFunctionWithLocation(this._describe.bind(this, 'serial')); @@ -133,6 +134,26 @@ export class TestTypeImpl { } } + private _configure(location: Location, options: { mode?: 'parallel' | 'serial' }) { + throwIfRunningInsideJest(); + const suite = currentlyLoadingFileSuite(); + if (!suite) + throw errorWithLocation(location, `describe.configure() can only be called in a test file`); + + if (!options.mode) + return; + if (suite._parallelMode !== 'default') + throw errorWithLocation(location, 'Parallel mode is already assigned for the enclosing scope.'); + suite._parallelMode = options.mode; + + for (let parent: Suite | undefined = suite.parent; parent; parent = parent.parent) { + if (parent._parallelMode === 'serial' && suite._parallelMode === 'parallel') + throw errorWithLocation(location, 'describe.parallel cannot be nested inside describe.serial'); + if (parent._parallelMode === 'parallel' && suite._parallelMode === 'serial') + throw errorWithLocation(location, 'describe.serial cannot be nested inside describe.parallel'); + } + } + private _modifier(type: 'skip' | 'fail' | 'fixme' | 'slow', location: Location, ...modifierArgs: [arg?: any | Function, description?: string]) { const suite = currentlyLoadingFileSuite(); if (suite) { diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index 9919ad175e..5160660224 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -1640,15 +1640,16 @@ export interface TestType NOTE: Using serial is not recommended. It is usually better to make your tests isolated, so they can be run * independently. * * ```ts * test.describe.serial('group', () => { - * test('runs first', async ({ page }) => { - * }); - * test('runs second', async ({ page }) => { - * }); + * test('runs first', async ({ page }) => {}); + * test('runs second', async ({ page }) => {}); * }); * ``` * @@ -1685,12 +1686,13 @@ export interface TestType { - * test('runs in parallel 1', async ({ page }) => { - * }); - * test('runs in parallel 2', async ({ page }) => { - * }); + * test('runs in parallel 1', async ({ page }) => {}); + * test('runs in parallel 2', async ({ page }) => {}); * }); * ``` * @@ -1711,6 +1713,29 @@ export interface TestType {}); + * test('runs in parallel 2', async ({ page }) => {}); + * ``` + * + * ```ts + * // Annotate tests as inter-dependent. + * test.describe.configure({ mode: 'serial' }); + * test('runs first', async ({ page }) => {}); + * test('runs second', async ({ page }) => {}); + * ``` + * + * @param options + */ + configure: (options: { mode?: 'parallel' | 'serial' }) => void; }; /** * Declares a skipped test, similarly to diff --git a/tests/playwright-test/test-parallel.spec.ts b/tests/playwright-test/test-parallel.spec.ts index 2c408443a5..9f53e3dbaf 100644 --- a/tests/playwright-test/test-parallel.spec.ts +++ b/tests/playwright-test/test-parallel.spec.ts @@ -58,3 +58,61 @@ test('test.describe.parallel should work', async ({ runInlineTest }) => { expect(result.output).toContain('%% worker=1'); expect(result.output).toContain('%% worker=2'); }); + +test('test.describe.parallel should work in file', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': ` + const { test } = pwt; + test.describe.configure({ mode: 'parallel' }); + test('test1', async ({}, testInfo) => { + console.log('\\n%% worker=' + testInfo.workerIndex); + await new Promise(f => setTimeout(f, 1000)); + }); + test('test2', async ({}, testInfo) => { + console.log('\\n%% worker=' + testInfo.workerIndex); + await new Promise(f => setTimeout(f, 1000)); + }); + test.describe('inner suite', () => { + test('test3', async ({}, testInfo) => { + console.log('\\n%% worker=' + testInfo.workerIndex); + await new Promise(f => setTimeout(f, 1000)); + }); + }); + `, + }, { workers: 3 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(3); + expect(result.output).toContain('%% worker=0'); + expect(result.output).toContain('%% worker=1'); + expect(result.output).toContain('%% worker=2'); +}); + +test('test.describe.parallel should work in describe', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': ` + const { test } = pwt; + test.describe('parallel suite', () => { + test.describe.configure({ mode: 'parallel' }); + test('test1', async ({}, testInfo) => { + console.log('\\n%% worker=' + testInfo.workerIndex); + await new Promise(f => setTimeout(f, 1000)); + }); + test('test2', async ({}, testInfo) => { + console.log('\\n%% worker=' + testInfo.workerIndex); + await new Promise(f => setTimeout(f, 1000)); + }); + test.describe('inner suite', () => { + test('test3', async ({}, testInfo) => { + console.log('\\n%% worker=' + testInfo.workerIndex); + await new Promise(f => setTimeout(f, 1000)); + }); + }); + }); + `, + }, { workers: 3 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(3); + expect(result.output).toContain('%% worker=0'); + expect(result.output).toContain('%% worker=1'); + expect(result.output).toContain('%% worker=2'); +}); diff --git a/tests/playwright-test/test-serial.spec.ts b/tests/playwright-test/test-serial.spec.ts index 40e8761b1e..2ab6402e7d 100644 --- a/tests/playwright-test/test-serial.spec.ts +++ b/tests/playwright-test/test-serial.spec.ts @@ -55,6 +55,46 @@ test('test.describe.serial should work', async ({ runInlineTest }) => { ]); }); +test('test.describe.serial should work in describe', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': ` + const { test } = pwt; + test.describe('serial suite', () => { + test.describe.configure({ mode: 'serial' }); + test('test1', async ({}) => { + console.log('\\n%%test1'); + }); + test('test2', async ({}) => { + console.log('\\n%%test2'); + }); + + test.describe('inner suite', () => { + test('test3', async ({}) => { + console.log('\\n%%test3'); + expect(1).toBe(2); + }); + test('test4', async ({}) => { + console.log('\\n%%test4'); + }); + }); + + test('test5', async ({}) => { + console.log('\\n%%test5'); + }); + }); + `, + }); + expect(result.exitCode).toBe(1); + expect(result.passed).toBe(2); + expect(result.failed).toBe(1); + expect(result.skipped).toBe(2); + expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([ + '%%test1', + '%%test2', + '%%test3', + ]); +}); + test('test.describe.serial should work with retry', async ({ runInlineTest }) => { const result = await runInlineTest({ 'a.test.ts': ` diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 8707f0fd83..b553255e2e 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -244,6 +244,7 @@ export interface TestType void; }; skip(title: string, testFunction: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise | void): void; skip(): void; diff --git a/utils/generate_types/parseOverrides.js b/utils/generate_types/parseOverrides.js index 5dc69c15c3..765dffefd9 100644 --- a/utils/generate_types/parseOverrides.js +++ b/utils/generate_types/parseOverrides.js @@ -101,8 +101,8 @@ async function parseOverrides(filePath, commentForClass, commentForMethod, extra * @param {ts.Node} node */ function visitProperties(className, prefix, node) { - // This function supports structs like "a: { b: string; c: number }" - // and inserts comments for "a.b" and "a.c" + // This function supports structs like "a: { b: string; c: number, (): void }" + // and inserts comments for "a.b", "a.c", a. if (ts.isPropertySignature(node)) { const name = checker.getSymbolAtLocation(node.name).getName(); const pos = node.getStart(file, false); @@ -111,6 +111,12 @@ async function parseOverrides(filePath, commentForClass, commentForMethod, extra text: commentForMethod(className, `${prefix}.${name}`, 0), }); ts.forEachChild(node, child => visitProperties(className, `${prefix}.${name}`, child)); + } else if (ts.isCallSignatureDeclaration(node)) { + const pos = node.getStart(file, false); + replacers.push({ + pos, + text: commentForMethod(className, `${prefix}`, 0), + }); } else if (!ts.isMethodSignature(node)) { ts.forEachChild(node, child => visitProperties(className, prefix, child)); }