diff --git a/docs/src/test-api/class-test.md b/docs/src/test-api/class-test.md index c0f0f8bb3e..0afca6474a 100644 --- a/docs/src/test-api/class-test.md +++ b/docs/src/test-api/class-test.md @@ -307,8 +307,7 @@ A callback that is run immediately when calling [`method: Test.describe#2`]. Any ## method: Test.describe.configure * since: v1.10 -Set execution mode of execution for the enclosing scope. Can be executed either on the top level or inside a describe. Configuration applies to the entire scope, regardless of whether it run before or after the test -declaration. +Configures the enclosing scope. Can be executed either on the top level or inside a describe. Configuration applies to the entire scope, regardless of whether it run before or after the test declaration. Learn more about the execution modes [here](../test-parallel.md). @@ -344,10 +343,33 @@ test('runs first', async ({ page }) => {}); test('runs second', async ({ page }) => {}); ``` +Configuring retries and timeout: + +```js tab=js-js +// All tests in the file will be retried twice and have a timeout of 20 seconds. +test.describe.configure({ retries: 2, timeout: 20_000 }); +test('runs first', async ({ page }) => {}); +test('runs second', async ({ page }) => {}); +``` + +```js tab=js-ts +// All tests in the file will be retried twice and have a timeout of 20 seconds. +test.describe.configure({ retries: 2, timeout: 20_000 }); +test('runs first', async ({ page }) => {}); +test('runs second', async ({ page }) => {}); +``` + ### option: Test.describe.configure.mode * since: v1.10 - `mode` <[TestMode]<"parallel"|"serial">> +### option: Test.describe.configure.retries +* since: v1.28 +- `retries` <[int]> + +### option: Test.describe.configure.timeout +* since: v1.28 +- `timeout` <[int]> ## method: Test.describe.fixme @@ -1126,7 +1148,7 @@ const { test, expect } = require('@playwright/test'); test.describe('group', () => { // Applies to all tests in this group. - test.setTimeout(60000); + test.describe.configure({ timeout: 60000 }); test('test one', async () => { /* ... */ }); test('test two', async () => { /* ... */ }); @@ -1139,7 +1161,7 @@ import { test, expect } from '@playwright/test'; test.describe('group', () => { // Applies to all tests in this group. - test.setTimeout(60000); + test.describe.configure({ timeout: 60000 }); test('test one', async () => { /* ... */ }); test('test two', async () => { /* ... */ }); diff --git a/docs/src/test-api/class-testproject.md b/docs/src/test-api/class-testproject.md index 2375b45019..034c95add6 100644 --- a/docs/src/test-api/class-testproject.md +++ b/docs/src/test-api/class-testproject.md @@ -257,6 +257,8 @@ Use [`property: TestConfig.repeatEach`] to change this option for all projects. The maximum number of retry attempts given to failed tests. Learn more about [test retries](../test-retries.md#retries). +Use [`method: Test.describe.configure`] to change the number of retries for a specific file or a group of tests. + Use [`property: TestConfig.retries`] to change this option for all projects. ## property: TestProject.run @@ -392,7 +394,7 @@ Use [`property: TestConfig.testMatch`] to change this option for all projects. Timeout for each test in milliseconds. Defaults to 30 seconds. -This is a base timeout for all tests. In addition, each test can configure its own timeout with [`method: Test.setTimeout`]. +This is a base timeout for all tests. Each test can configure its own timeout with [`method: Test.setTimeout`]. Each file or a group of tests can configure the timeout with [`method: Test.describe.configure`]. Use [`property: TestConfig.timeout`] to change this option for all projects. diff --git a/docs/src/test-retries-js.md b/docs/src/test-retries-js.md index e368f56d80..d42e35ba26 100644 --- a/docs/src/test-retries-js.md +++ b/docs/src/test-retries-js.md @@ -134,6 +134,42 @@ test('my test', async ({ page }, testInfo) => { }); ``` +You can specify retries for a specific group of tests or a single file with [`method: Test.describe.configure`]. + +```js tab=js-js +const { test, expect } = require('@playwright/test'); + +test.describe(() => { + // All tests in this describe group will get 2 retry attempts. + test.describe.configure({ retries: 2 }); + + test('test 1', async ({ page }) => { + // ... + }); + + test('test 2', async ({ page }) => { + // ... + }); +}); +``` + +```js tab=js-ts +import { test, expect } from '@playwright/test'; + +test.describe(() => { + // All tests in this describe group will get 2 retry attempts. + test.describe.configure({ retries: 2 }); + + test('test 1', async ({ page }) => { + // ... + }); + + test('test 2', async ({ page }) => { + // ... + }); +}); +``` + ## Serial mode Use [`method: Test.describe.serial`] to group dependent tests to ensure they will always run together and in order. If one of the tests fails, all subsequent tests are skipped. All tests in the group are retried together. diff --git a/packages/playwright-test/src/loader.ts b/packages/playwright-test/src/loader.ts index 4ef30cde57..fa5b134956 100644 --- a/packages/playwright-test/src/loader.ts +++ b/packages/playwright-test/src/loader.ts @@ -398,6 +398,12 @@ class ProjectSuiteBuilder { const test = entry._clone(); to._addTest(test); test.retries = this._project.retries; + for (let parentSuite: Suite | undefined = to; parentSuite; parentSuite = parentSuite.parent) { + if (parentSuite._retries !== undefined) { + test.retries = parentSuite._retries; + break; + } + } const repeatEachIndexSuffix = repeatEachIndex ? ` (repeat:${repeatEachIndex})` : ''; // At the point of the query, suite is not yet attached to the project, so we only get file, describe and test titles. const testIdExpression = `[project=${this._project._id}]${test.titlePath().join('\x1e')}${repeatEachIndexSuffix}`; diff --git a/packages/playwright-test/src/test.ts b/packages/playwright-test/src/test.ts index 5e835a0531..0d70b92a46 100644 --- a/packages/playwright-test/src/test.ts +++ b/packages/playwright-test/src/test.ts @@ -46,6 +46,7 @@ export class Suite extends Base implements reporterTypes.Suite { _entries: (Suite | TestCase)[] = []; _hooks: { type: 'beforeEach' | 'afterEach' | 'beforeAll' | 'afterAll', fn: Function, location: Location }[] = []; _timeout: number | undefined; + _retries: number | undefined; _annotations: Annotation[] = []; _modifiers: Modifier[] = []; _parallelMode: 'default' | 'serial' | 'parallel' = 'default'; @@ -111,6 +112,7 @@ export class Suite extends Base implements reporterTypes.Suite { suite._use = this._use.slice(); suite._hooks = this._hooks.slice(); suite._timeout = this._timeout; + suite._retries = this._retries; suite._annotations = this._annotations.slice(); suite._modifiers = this._modifiers.slice(); suite._parallelMode = this._parallelMode; diff --git a/packages/playwright-test/src/testType.ts b/packages/playwright-test/src/testType.ts index fd35892382..2dc003f378 100644 --- a/packages/playwright-test/src/testType.ts +++ b/packages/playwright-test/src/testType.ts @@ -139,18 +139,24 @@ export class TestTypeImpl { suite._hooks.push({ type: name, fn, location }); } - private _configure(location: Location, options: { mode?: 'parallel' | 'serial' }) { + private _configure(location: Location, options: { mode?: 'parallel' | 'serial', retries?: number, timeout?: number }) { throwIfRunningInsideJest(); const suite = this._ensureCurrentSuite(location, `test.describe.configure()`); - if (!options.mode) - return; - if (suite._parallelMode !== 'default') - throw errorWithLocation(location, 'Parallel mode is already assigned for the enclosing scope.'); - suite._parallelMode = options.mode; - for (let parent: Suite | undefined = suite.parent; parent; parent = parent.parent) { - if (parent._parallelMode === 'serial' && suite._parallelMode === 'parallel') - throw errorWithLocation(location, 'describe.parallel cannot be nested inside describe.serial'); + if (options.timeout !== undefined) + suite._timeout = options.timeout; + + if (options.retries !== undefined) + suite._retries = options.retries; + + if (options.mode !== undefined) { + if (suite._parallelMode !== 'default') + throw errorWithLocation(location, 'Parallel mode is already assigned for the enclosing scope.'); + suite._parallelMode = options.mode; + for (let parent: Suite | undefined = suite.parent; parent; parent = parent.parent) { + if (parent._parallelMode === 'serial' && suite._parallelMode === 'parallel') + throw errorWithLocation(location, 'describe.parallel cannot be nested inside describe.serial'); + } } } diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index 0e3061debb..5d651ac92b 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -252,6 +252,9 @@ export interface FullProject { /** * The maximum number of retry attempts given to failed tests. Learn more about [test retries](https://playwright.dev/docs/test-retries#retries). * + * Use [test.describe.configure([options])](https://playwright.dev/docs/api/class-test#test-describe-configure) to change + * the number of retries for a specific file or a group of tests. + * * Use [testConfig.retries](https://playwright.dev/docs/api/class-testconfig#test-config-retries) to change this option for * all projects. */ @@ -344,8 +347,10 @@ export interface FullProject { /** * Timeout for each test in milliseconds. Defaults to 30 seconds. * - * This is a base timeout for all tests. In addition, each test can configure its own timeout with - * [test.setTimeout(timeout)](https://playwright.dev/docs/api/class-test#test-set-timeout). + * This is a base timeout for all tests. Each test can configure its own timeout with + * [test.setTimeout(timeout)](https://playwright.dev/docs/api/class-test#test-set-timeout). Each file or a group of tests + * can configure the timeout with + * [test.describe.configure([options])](https://playwright.dev/docs/api/class-test#test-describe-configure). * * Use [testConfig.timeout](https://playwright.dev/docs/api/class-testconfig#test-config-timeout) to change this option for * all projects. @@ -2017,8 +2022,8 @@ export interface TestType {}); * ``` * + * Configuring retries and timeout: + * + * ```js + * // All tests in the file will be retried twice and have a timeout of 20 seconds. + * test.describe.configure({ retries: 2, timeout: 20_000 }); + * test('runs first', async ({ page }) => {}); + * test('runs second', async ({ page }) => {}); + * ``` + * * @param options */ - configure: (options: { mode?: 'parallel' | 'serial' }) => void; + configure: (options: { mode?: 'parallel' | 'serial', retries?: number, timeout?: number }) => void; }; /** * Declares a skipped test, similarly to @@ -2372,7 +2386,7 @@ export interface TestType { * // Applies to all tests in this group. - * test.setTimeout(60000); + * test.describe.configure({ timeout: 60000 }); * * test('test one', async () => { /* ... *\/ }); * test('test two', async () => { /* ... *\/ }); @@ -4468,6 +4482,9 @@ interface TestProject { /** * The maximum number of retry attempts given to failed tests. Learn more about [test retries](https://playwright.dev/docs/test-retries#retries). * + * Use [test.describe.configure([options])](https://playwright.dev/docs/api/class-test#test-describe-configure) to change + * the number of retries for a specific file or a group of tests. + * * Use [testConfig.retries](https://playwright.dev/docs/api/class-testconfig#test-config-retries) to change this option for * all projects. */ @@ -4566,8 +4583,10 @@ interface TestProject { /** * Timeout for each test in milliseconds. Defaults to 30 seconds. * - * This is a base timeout for all tests. In addition, each test can configure its own timeout with - * [test.setTimeout(timeout)](https://playwright.dev/docs/api/class-test#test-set-timeout). + * This is a base timeout for all tests. Each test can configure its own timeout with + * [test.setTimeout(timeout)](https://playwright.dev/docs/api/class-test#test-set-timeout). Each file or a group of tests + * can configure the timeout with + * [test.describe.configure([options])](https://playwright.dev/docs/api/class-test#test-describe-configure). * * Use [testConfig.timeout](https://playwright.dev/docs/api/class-testconfig#test-config-timeout) to change this option for * all projects. diff --git a/tests/playwright-test/retry.spec.ts b/tests/playwright-test/retry.spec.ts index d37f04dd4f..601b855cf1 100644 --- a/tests/playwright-test/retry.spec.ts +++ b/tests/playwright-test/retry.spec.ts @@ -60,6 +60,61 @@ test('should retry based on config', async ({ runInlineTest }) => { expect(result.results.length).toBe(4); }); +test('should retry based on test.describe.configure', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.js': ` + module.exports = { retries: 2 }; + `, + 'a.test.js': ` + const { test } = pwt; + test.describe.configure({ retries: 1 }); + test('fail 1', ({}, testInfo) => { + console.log('%%fail1-' + testInfo.retry); + expect(1).toBe(2); + }); + `, + 'b.test.js': ` + const { test } = pwt; + test('fail 4', ({}, testInfo) => { + console.log('%%fail4-' + testInfo.retry); + expect(1).toBe(2); + }); + test.describe(() => { + test.describe.configure({ retries: 0 }); + test('fail 2', ({}, testInfo) => { + console.log('%%fail2-' + testInfo.retry); + expect(1).toBe(2); + }); + test.describe(() => { + test.describe.configure({ retries: 1 }); + test.describe(() => { + test('fail 3', ({}, testInfo) => { + console.log('%%fail3-' + testInfo.retry); + expect(1).toBe(2); + }); + }); + }); + }); + `, + }); + expect(result.exitCode).toBe(1); + expect(result.passed).toBe(0); + expect(result.failed).toBe(4); + expect(result.results.length).toBe(8); + expect(result.output).toContain('%%fail1-0'); + expect(result.output).toContain('%%fail1-1'); + expect(result.output).not.toContain('%%fail1-2'); + expect(result.output).toContain('%%fail4-0'); + expect(result.output).toContain('%%fail4-1'); + expect(result.output).toContain('%%fail4-2'); + expect(result.output).not.toContain('%%fail4-3'); + expect(result.output).toContain('%%fail2-0'); + expect(result.output).not.toContain('%%fail2-1'); + expect(result.output).toContain('%%fail3-0'); + expect(result.output).toContain('%%fail3-1'); + expect(result.output).not.toContain('%%fail3-2'); +}); + test('should retry timeout', async ({ runInlineTest }) => { const { exitCode, passed, failed, output } = await runInlineTest({ 'one-timeout.spec.js': ` diff --git a/tests/playwright-test/timeout.spec.ts b/tests/playwright-test/timeout.spec.ts index ebf9e34635..8faba8ab10 100644 --- a/tests/playwright-test/timeout.spec.ts +++ b/tests/playwright-test/timeout.spec.ts @@ -417,3 +417,27 @@ test('should run fixture teardowns after timeout with soft expect error', async contentType: 'text/plain', }); }); + +test('should respect test.describe.configure', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + const { test } = pwt; + test.describe.configure({ timeout: 1000 }); + test('test1', async ({}) => { + console.log('test1-' + test.info().timeout); + }); + test.describe(() => { + test.describe.configure({ timeout: 2000 }); + test.describe(() => { + test('test2', async ({}) => { + console.log('test2-' + test.info().timeout); + }); + }); + }); + ` + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(2); + expect(result.output).toContain('test1-1000'); + expect(result.output).toContain('test2-2000'); +}); diff --git a/tests/playwright-test/types-2.spec.ts b/tests/playwright-test/types-2.spec.ts index 597437426b..ea97e41b88 100644 --- a/tests/playwright-test/types-2.spec.ts +++ b/tests/playwright-test/types-2.spec.ts @@ -28,6 +28,7 @@ test('basics should work', async ({ runTSC }) => { test('my test', async({}, testInfo) => { expect(testInfo.title).toBe('my test'); testInfo.annotations[0].type; + test.setTimeout(123); }); test.skip('my test', async () => {}); test.fixme('my test', async () => {}); @@ -43,6 +44,8 @@ test('basics should work', async ({ runTSC }) => { test.describe.fixme('suite', () => {}); // @ts-expect-error test.foo(); + test.describe.configure({ mode: 'parallel' }); + test.describe.configure({ retries: 3, timeout: 123 }); ` }); expect(result.exitCode).toBe(0); diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index f784b81672..9e904b2bd9 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -131,7 +131,7 @@ export interface TestType void; + configure: (options: { mode?: 'parallel' | 'serial', retries?: number, timeout?: number }) => void; }; skip(title: string, testFunction: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise | void): void; skip(): void;