feat(test runner): test.describe.configure({ retries, timeout }) (#18388)

References #10825.

docs/src/test-api/class-test.md

@@ -307,8 +307,7 @@ A callback that is run immediately when calling [`method: Test.describe#2`]. Any
 ## method: Test.describe.configure
 * since: v1.10
 
-Set execution mode of execution for the enclosing scope. Can be executed either on the top level or inside a describe. Configuration applies to the entire scope, regardless of whether it run before or after the test
-declaration.
+Configures the enclosing scope. Can be executed either on the top level or inside a describe. Configuration applies to the entire scope, regardless of whether it run before or after the test declaration.
 
 Learn more about the execution modes [here](../test-parallel.md).
 
@@ -344,10 +343,33 @@ test('runs first', async ({ page }) => {});
 test('runs second', async ({ page }) => {});
 ```
 
+Configuring retries and timeout:
+
+```js tab=js-js
+// All tests in the file will be retried twice and have a timeout of 20 seconds.
+test.describe.configure({ retries: 2, timeout: 20_000 });
+test('runs first', async ({ page }) => {});
+test('runs second', async ({ page }) => {});
+```
+
+```js tab=js-ts
+// All tests in the file will be retried twice and have a timeout of 20 seconds.
+test.describe.configure({ retries: 2, timeout: 20_000 });
+test('runs first', async ({ page }) => {});
+test('runs second', async ({ page }) => {});
+```
+
 ### option: Test.describe.configure.mode
 * since: v1.10
 - `mode` <[TestMode]<"parallel"|"serial">>
 
+### option: Test.describe.configure.retries
+* since: v1.28
+- `retries` <[int]>
+
+### option: Test.describe.configure.timeout
+* since: v1.28
+- `timeout` <[int]>
 
 
 ## method: Test.describe.fixme
@@ -1126,7 +1148,7 @@ const { test, expect } = require('@playwright/test');
 
 test.describe('group', () => {
   // Applies to all tests in this group.
-  test.setTimeout(60000);
+  test.describe.configure({ timeout: 60000 });
 
   test('test one', async () => { /* ... */ });
   test('test two', async () => { /* ... */ });
@@ -1139,7 +1161,7 @@ import { test, expect } from '@playwright/test';
 
 test.describe('group', () => {
   // Applies to all tests in this group.
-  test.setTimeout(60000);
+  test.describe.configure({ timeout: 60000 });
 
   test('test one', async () => { /* ... */ });
   test('test two', async () => { /* ... */ });

docs/src/test-api/class-testproject.md

@@ -257,6 +257,8 @@ Use [`property: TestConfig.repeatEach`] to change this option for all projects.
 
 The maximum number of retry attempts given to failed tests. Learn more about [test retries](../test-retries.md#retries).
 
+Use [`method: Test.describe.configure`] to change the number of retries for a specific file or a group of tests.
+
 Use [`property: TestConfig.retries`] to change this option for all projects.
 
 ## property: TestProject.run
@@ -392,7 +394,7 @@ Use [`property: TestConfig.testMatch`] to change this option for all projects.
 
 Timeout for each test in milliseconds. Defaults to 30 seconds.
 
-This is a base timeout for all tests. In addition, each test can configure its own timeout with [`method: Test.setTimeout`].
+This is a base timeout for all tests. Each test can configure its own timeout with [`method: Test.setTimeout`]. Each file or a group of tests can configure the timeout with [`method: Test.describe.configure`].
 
 Use [`property: TestConfig.timeout`] to change this option for all projects.
 

docs/src/test-retries-js.md

@@ -134,6 +134,42 @@ test('my test', async ({ page }, testInfo) => {
 });
 ```
 
+You can specify retries for a specific group of tests or a single file with [`method: Test.describe.configure`].
+
+```js tab=js-js
+const { test, expect } = require('@playwright/test');
+
+test.describe(() => {
+  // All tests in this describe group will get 2 retry attempts.
+  test.describe.configure({ retries: 2 });
+
+  test('test 1', async ({ page }) => {
+    // ...
+  });
+
+  test('test 2', async ({ page }) => {
+    // ...
+  });
+});
+```
+
+```js tab=js-ts
+import { test, expect } from '@playwright/test';
+
+test.describe(() => {
+  // All tests in this describe group will get 2 retry attempts.
+  test.describe.configure({ retries: 2 });
+
+  test('test 1', async ({ page }) => {
+    // ...
+  });
+
+  test('test 2', async ({ page }) => {
+    // ...
+  });
+});
+```
+
 ## Serial mode
 
 Use [`method: Test.describe.serial`] to group dependent tests to ensure they will always run together and in order. If one of the tests fails, all subsequent tests are skipped. All tests in the group are retried together.

packages/playwright-test/src/loader.ts

@@ -398,6 +398,12 @@ class ProjectSuiteBuilder {
         const test = entry._clone();
         to._addTest(test);
         test.retries = this._project.retries;
+        for (let parentSuite: Suite | undefined = to; parentSuite; parentSuite = parentSuite.parent) {
+          if (parentSuite._retries !== undefined) {
+            test.retries = parentSuite._retries;
+            break;
+          }
+        }
         const repeatEachIndexSuffix = repeatEachIndex ? ` (repeat:${repeatEachIndex})` : '';
         // At the point of the query, suite is not yet attached to the project, so we only get file, describe and test titles.
         const testIdExpression = `[project=${this._project._id}]${test.titlePath().join('\x1e')}${repeatEachIndexSuffix}`;

packages/playwright-test/src/test.ts

@@ -46,6 +46,7 @@ export class Suite extends Base implements reporterTypes.Suite {
   _entries: (Suite | TestCase)[] = [];
   _hooks: { type: 'beforeEach' | 'afterEach' | 'beforeAll' | 'afterAll', fn: Function, location: Location }[] = [];
   _timeout: number | undefined;
+  _retries: number | undefined;
   _annotations: Annotation[] = [];
   _modifiers: Modifier[] = [];
   _parallelMode: 'default' | 'serial' | 'parallel' = 'default';
@@ -111,6 +112,7 @@ export class Suite extends Base implements reporterTypes.Suite {
     suite._use = this._use.slice();
     suite._hooks = this._hooks.slice();
     suite._timeout = this._timeout;
+    suite._retries = this._retries;
     suite._annotations = this._annotations.slice();
     suite._modifiers = this._modifiers.slice();
     suite._parallelMode = this._parallelMode;

packages/playwright-test/src/testType.ts

@@ -139,20 +139,26 @@ export class TestTypeImpl {
     suite._hooks.push({ type: name, fn, location });
   }
 
-  private _configure(location: Location, options: { mode?: 'parallel' | 'serial' }) {
+  private _configure(location: Location, options: { mode?: 'parallel' | 'serial', retries?: number, timeout?: number }) {
     throwIfRunningInsideJest();
     const suite = this._ensureCurrentSuite(location, `test.describe.configure()`);
-    if (!options.mode)
-      return;
+
+    if (options.timeout !== undefined)
+      suite._timeout = options.timeout;
+
+    if (options.retries !== undefined)
+      suite._retries = options.retries;
+
+    if (options.mode !== undefined) {
       if (suite._parallelMode !== 'default')
         throw errorWithLocation(location, 'Parallel mode is already assigned for the enclosing scope.');
       suite._parallelMode = options.mode;
-
       for (let parent: Suite | undefined = suite.parent; parent; parent = parent.parent) {
         if (parent._parallelMode === 'serial' && suite._parallelMode === 'parallel')
           throw errorWithLocation(location, 'describe.parallel cannot be nested inside describe.serial');
       }
     }
+  }
 
   private _modifier(type: 'skip' | 'fail' | 'fixme' | 'slow', location: Location, ...modifierArgs: [arg?: any | Function, description?: string]) {
     const suite = currentlyLoadingFileSuite();

packages/playwright-test/types/test.d.ts

@@ -252,6 +252,9 @@ export interface FullProject<TestArgs = {}, WorkerArgs = {}> {
   /**
    * The maximum number of retry attempts given to failed tests. Learn more about [test retries](https://playwright.dev/docs/test-retries#retries).
    *
+   * Use [test.describe.configure([options])](https://playwright.dev/docs/api/class-test#test-describe-configure) to change
+   * the number of retries for a specific file or a group of tests.
+   *
    * Use [testConfig.retries](https://playwright.dev/docs/api/class-testconfig#test-config-retries) to change this option for
    * all projects.
    */
@@ -344,8 +347,10 @@ export interface FullProject<TestArgs = {}, WorkerArgs = {}> {
   /**
    * Timeout for each test in milliseconds. Defaults to 30 seconds.
    *
-   * This is a base timeout for all tests. In addition, each test can configure its own timeout with
-   * [test.setTimeout(timeout)](https://playwright.dev/docs/api/class-test#test-set-timeout).
+   * This is a base timeout for all tests. Each test can configure its own timeout with
+   * [test.setTimeout(timeout)](https://playwright.dev/docs/api/class-test#test-set-timeout). Each file or a group of tests
+   * can configure the timeout with
+   * [test.describe.configure([options])](https://playwright.dev/docs/api/class-test#test-describe-configure).
    *
    * Use [testConfig.timeout](https://playwright.dev/docs/api/class-testconfig#test-config-timeout) to change this option for
    * all projects.
@@ -2017,8 +2022,8 @@ export interface TestType<TestArgs extends KeyValue, WorkerArgs extends KeyValue
   only: SuiteFunction;
     };
     /**
-   * Set execution mode of execution for the enclosing scope. Can be executed either on the top level or inside a describe.
-   * Configuration applies to the entire scope, regardless of whether it run before or after the test declaration.
+   * Configures the enclosing scope. Can be executed either on the top level or inside a describe. Configuration applies to
+   * the entire scope, regardless of whether it run before or after the test declaration.
    *
    * Learn more about the execution modes [here](https://playwright.dev/docs/test-parallel).
    *
@@ -2040,9 +2045,18 @@ export interface TestType<TestArgs extends KeyValue, WorkerArgs extends KeyValue
    * test('runs second', async ({ page }) => {});
    * ```
    *
+   * Configuring retries and timeout:
+   *
+   * ```js
+   * // All tests in the file will be retried twice and have a timeout of 20 seconds.
+   * test.describe.configure({ retries: 2, timeout: 20_000 });
+   * test('runs first', async ({ page }) => {});
+   * test('runs second', async ({ page }) => {});
+   * ```
+   *
    * @param options
    */
-  configure: (options: { mode?: 'parallel' | 'serial' }) => void;
+  configure: (options: { mode?: 'parallel' | 'serial', retries?: number, timeout?: number }) => void;
   };
   /**
    * Declares a skipped test, similarly to
@@ -2372,7 +2386,7 @@ export interface TestType<TestArgs extends KeyValue, WorkerArgs extends KeyValue
    *
    * test.describe('group', () => {
    *   // Applies to all tests in this group.
-   *   test.setTimeout(60000);
+   *   test.describe.configure({ timeout: 60000 });
    *
    *   test('test one', async () => { /* ... *\/ });
    *   test('test two', async () => { /* ... *\/ });
@@ -4468,6 +4482,9 @@ interface TestProject {
   /**
    * The maximum number of retry attempts given to failed tests. Learn more about [test retries](https://playwright.dev/docs/test-retries#retries).
    *
+   * Use [test.describe.configure([options])](https://playwright.dev/docs/api/class-test#test-describe-configure) to change
+   * the number of retries for a specific file or a group of tests.
+   *
    * Use [testConfig.retries](https://playwright.dev/docs/api/class-testconfig#test-config-retries) to change this option for
    * all projects.
    */
@@ -4566,8 +4583,10 @@ interface TestProject {
   /**
    * Timeout for each test in milliseconds. Defaults to 30 seconds.
    *
-   * This is a base timeout for all tests. In addition, each test can configure its own timeout with
-   * [test.setTimeout(timeout)](https://playwright.dev/docs/api/class-test#test-set-timeout).
+   * This is a base timeout for all tests. Each test can configure its own timeout with
+   * [test.setTimeout(timeout)](https://playwright.dev/docs/api/class-test#test-set-timeout). Each file or a group of tests
+   * can configure the timeout with
+   * [test.describe.configure([options])](https://playwright.dev/docs/api/class-test#test-describe-configure).
    *
    * Use [testConfig.timeout](https://playwright.dev/docs/api/class-testconfig#test-config-timeout) to change this option for
    * all projects.

tests/playwright-test/retry.spec.ts

@@ -60,6 +60,61 @@ test('should retry based on config', async ({ runInlineTest }) => {
   expect(result.results.length).toBe(4);
 });
 
+test('should retry based on test.describe.configure', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.js': `
+      module.exports = { retries: 2 };
+    `,
+    'a.test.js': `
+      const { test } = pwt;
+      test.describe.configure({ retries: 1 });
+      test('fail 1', ({}, testInfo) => {
+        console.log('%%fail1-' + testInfo.retry);
+        expect(1).toBe(2);
+      });
+    `,
+    'b.test.js': `
+      const { test } = pwt;
+      test('fail 4', ({}, testInfo) => {
+        console.log('%%fail4-' + testInfo.retry);
+        expect(1).toBe(2);
+      });
+      test.describe(() => {
+        test.describe.configure({ retries: 0 });
+        test('fail 2', ({}, testInfo) => {
+          console.log('%%fail2-' + testInfo.retry);
+          expect(1).toBe(2);
+        });
+        test.describe(() => {
+          test.describe.configure({ retries: 1 });
+          test.describe(() => {
+            test('fail 3', ({}, testInfo) => {
+              console.log('%%fail3-' + testInfo.retry);
+              expect(1).toBe(2);
+            });
+          });
+        });
+      });
+    `,
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.passed).toBe(0);
+  expect(result.failed).toBe(4);
+  expect(result.results.length).toBe(8);
+  expect(result.output).toContain('%%fail1-0');
+  expect(result.output).toContain('%%fail1-1');
+  expect(result.output).not.toContain('%%fail1-2');
+  expect(result.output).toContain('%%fail4-0');
+  expect(result.output).toContain('%%fail4-1');
+  expect(result.output).toContain('%%fail4-2');
+  expect(result.output).not.toContain('%%fail4-3');
+  expect(result.output).toContain('%%fail2-0');
+  expect(result.output).not.toContain('%%fail2-1');
+  expect(result.output).toContain('%%fail3-0');
+  expect(result.output).toContain('%%fail3-1');
+  expect(result.output).not.toContain('%%fail3-2');
+});
+
 test('should retry timeout', async ({ runInlineTest }) => {
   const { exitCode, passed, failed, output } = await runInlineTest({
     'one-timeout.spec.js': `

tests/playwright-test/timeout.spec.ts

@@ -417,3 +417,27 @@ test('should run fixture teardowns after timeout with soft expect error', async
     contentType: 'text/plain',
   });
 });
+
+test('should respect test.describe.configure', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const { test } = pwt;
+      test.describe.configure({ timeout: 1000 });
+      test('test1', async ({}) => {
+        console.log('test1-' + test.info().timeout);
+      });
+      test.describe(() => {
+        test.describe.configure({ timeout: 2000 });
+        test.describe(() => {
+          test('test2', async ({}) => {
+            console.log('test2-' + test.info().timeout);
+          });
+        });
+      });
+    `
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(2);
+  expect(result.output).toContain('test1-1000');
+  expect(result.output).toContain('test2-2000');
+});

tests/playwright-test/types-2.spec.ts

@@ -28,6 +28,7 @@ test('basics should work', async ({ runTSC }) => {
         test('my test', async({}, testInfo) => {
           expect(testInfo.title).toBe('my test');
           testInfo.annotations[0].type;
+          test.setTimeout(123);
         });
         test.skip('my test', async () => {});
         test.fixme('my test', async () => {});
@@ -43,6 +44,8 @@ test('basics should work', async ({ runTSC }) => {
       test.describe.fixme('suite', () => {});
       // @ts-expect-error
       test.foo();
+      test.describe.configure({ mode: 'parallel' });
+      test.describe.configure({ retries: 3, timeout: 123 });
     `
   });
   expect(result.exitCode).toBe(0);

utils/generate_types/overrides-test.d.ts

@@ -131,7 +131,7 @@ export interface TestType<TestArgs extends KeyValue, WorkerArgs extends KeyValue
     parallel: SuiteFunction & {
       only: SuiteFunction;
     };
-    configure: (options: { mode?: 'parallel' | 'serial' }) => void;
+    configure: (options: { mode?: 'parallel' | 'serial', retries?: number, timeout?: number }) => void;
   };
   skip(title: string, testFunction: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise<void> | void): void;
   skip(): void;