From fde2f6a77f9edc0bfc7e09d516ed9f8b2df97768 Mon Sep 17 00:00:00 2001 From: Dmitry Gozman Date: Fri, 19 Nov 2021 17:06:46 -0800 Subject: [PATCH] docs: separate doc for test timeouts (#10448) --- docs/src/test-api/class-test.md | 2 +- docs/src/test-api/class-testconfig.md | 6 +- docs/src/test-api/class-testinfo.md | 6 +- docs/src/test-api/class-testoptions.md | 4 + docs/src/test-assertions-js.md | 2 +- docs/src/test-cli-js.md | 6 +- docs/src/test-configuration-js.md | 4 +- docs/src/test-timeouts-js.md | 281 +++++++++++++++++++++++ packages/playwright-test/types/test.d.ts | 26 ++- 9 files changed, 317 insertions(+), 20 deletions(-) create mode 100644 docs/src/test-timeouts-js.md diff --git a/docs/src/test-api/class-test.md b/docs/src/test-api/class-test.md index 761e93254f..532b2ea7da 100644 --- a/docs/src/test-api/class-test.md +++ b/docs/src/test-api/class-test.md @@ -832,7 +832,7 @@ Test function that takes one or two arguments: an object with fixtures and optio ## method: Test.setTimeout -Changes the timeout for the test. +Changes the timeout for the test. Learn more about [various timeouts](./test-timeouts.md). ```js js-flavor=js const { test, expect } = require('@playwright/test'); diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index 4a5bfc7425..341037c10f 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -39,7 +39,7 @@ export default config; - `toMatchSnapshot` <[Object]> - `threshold` <[float]> Image matching threshold between zero (strict) and one (lax). -Configuration for the `expect` assertion library. +Configuration for the `expect` assertion library. Learn more about [various timeouts](./test-timeouts.md). ```js js-flavor=js // playwright.config.js @@ -161,7 +161,7 @@ export default config; ## property: TestConfig.globalTimeout - type: <[int]> -Maximum time in milliseconds the whole test suite can run. Zero timeout (default) disables this behavior. Useful on CI to prevent broken setup from running too long and wasting resources. +Maximum time in milliseconds the whole test suite can run. Zero timeout (default) disables this behavior. Useful on CI to prevent broken setup from running too long and wasting resources. Learn more about [various timeouts](./test-timeouts.md). ```js js-flavor=js // playwright.config.js @@ -491,7 +491,7 @@ export default config; Timeout for each test in milliseconds. Defaults to 30 seconds. -This is a base timeout for all tests. In addition, each test can configure its own timeout with [`method: Test.setTimeout`]. +This is a base timeout for all tests. In addition, each test can configure its own timeout with [`method: Test.setTimeout`]. Learn more about [various timeouts](./test-timeouts.md). ```js js-flavor=js // playwright.config.js diff --git a/docs/src/test-api/class-testinfo.md b/docs/src/test-api/class-testinfo.md index f55e21ca09..2d0a3a2e13 100644 --- a/docs/src/test-api/class-testinfo.md +++ b/docs/src/test-api/class-testinfo.md @@ -231,7 +231,7 @@ Specifies the retry number when the test is retried after a failure. The first t ## method: TestInfo.setTimeout -Changes the timeout for the currently running test. Zero means no timeout. +Changes the timeout for the currently running test. Zero means no timeout. Learn more about [various timeouts](./test-timeouts.md). Timeout is usually specified in the [configuration file](./test-configuration.md), but it could be useful to change the timeout in certain scenarios: @@ -342,7 +342,9 @@ Output written to `process.stdout` or `console.log` during the test execution. ## property: TestInfo.timeout - type: <[int]> -Timeout in milliseconds for the currently running test. Zero means no timeout. Timeout is usually specified in the [configuration file](./test-configuration.md) +Timeout in milliseconds for the currently running test. Zero means no timeout. Learn more about [various timeouts](./test-timeouts.md). + +Timeout is usually specified in the [configuration file](./test-configuration.md) ```js js-flavor=js const { test, expect } = require('@playwright/test'); diff --git a/docs/src/test-api/class-testoptions.md b/docs/src/test-api/class-testoptions.md index 5a9fd78d37..cee9bc653f 100644 --- a/docs/src/test-api/class-testoptions.md +++ b/docs/src/test-api/class-testoptions.md @@ -102,6 +102,8 @@ Default timeout for each Playwright action in milliseconds, defaults to 0 (no ti This is a default timeout for all Playwright actions, same as configured via [`method: Page.setDefaultTimeout`]. +Learn more about [various timeouts](./test-timeouts.md). + ## property: TestOptions.bypassCSP = %%-context-option-bypasscsp-%% ## property: TestOptions.channel = %%-browser-option-channel-%% @@ -145,6 +147,8 @@ Timeout for each navigation action in milliseconds. Defaults to 0 (no timeout). This is a default navigation timeout, same as configured via [`method: Page.setDefaultNavigationTimeout`]. +Learn more about [various timeouts](./test-timeouts.md). + ## property: TestOptions.offline = %%-context-option-offline-%% ## property: TestOptions.permissions = %%-context-option-permissions-%% diff --git a/docs/src/test-assertions-js.md b/docs/src/test-assertions-js.md index fb2296a183..7acc72c116 100644 --- a/docs/src/test-assertions-js.md +++ b/docs/src/test-assertions-js.md @@ -34,7 +34,7 @@ text. It will be re-fetching the node and checking it over and over, until the c reached. You can either pass this timeout or configure it once via the [`property: TestConfig.expect`] value in test config. -By default, the timeout for assertions is set to 5 seconds. +By default, the timeout for assertions is set to 5 seconds. Learn more about [various timeouts](./test-timeouts.md). ## expect(locator).toBeChecked([options]) - `options` diff --git a/docs/src/test-cli-js.md b/docs/src/test-cli-js.md index 1fb96cd9c0..422e5bb16e 100644 --- a/docs/src/test-cli-js.md +++ b/docs/src/test-cli-js.md @@ -77,7 +77,7 @@ Complete set of Playwright Test options is available in the [configuration file] - `--browser`: Run test in a specific browser. Available options are `"chromium"`, `"firefox"`, `"webkit"` or `"all"` to run tests in all three browsers at the same time. -- `--debug`: Run tests with Playwright Inspector. Shortcut for `PWDEBUG=1` environment variable and `--timeout=0 --maxFailures=1 --headed --workers=1` options +- `--debug`: Run tests with Playwright Inspector. Shortcut for `PWDEBUG=1` environment variable and `--timeout=0 --maxFailures=1 --headed --workers=1` options. - `-c ` or `--config `: Configuration file. If not passed, defaults to `playwright.config.ts` or `playwright.config.js` in the current directory. @@ -89,7 +89,7 @@ Complete set of Playwright Test options is available in the [configuration file] - `--grep-invert `: Only run tests **not** matching this regular expression. The opposite of `--grep`. -- `--global-timeout `: Total timeout for the whole test run in milliseconds. By default, there is no global timeout. +- `--global-timeout `: Total timeout for the whole test run in milliseconds. By default, there is no global timeout. Learn more about [various timeouts](./test-timeouts.md). - `--list`: List all the tests, but do not run them. @@ -109,7 +109,7 @@ Complete set of Playwright Test options is available in the [configuration file] - `--shard `: [Shard](./test-parallel.md#shard-tests-between-multiple-machines) tests and execute only selected shard, specified in the form `current/all`, 1-based, for example `3/5`. -- `--timeout `: Maximum timeout in milliseconds for each test, defaults to 30 seconds. +- `--timeout `: Maximum timeout in milliseconds for each test, defaults to 30 seconds. Learn more about [various timeouts](./test-timeouts.md). - `--update-snapshots` or `-u`: Whether to update [snapshots](./test-snapshots.md) with actual results instead of comparing them. Use this when snapshot expectations have changed. diff --git a/docs/src/test-configuration-js.md b/docs/src/test-configuration-js.md index 451f47563b..36b66b3b96 100644 --- a/docs/src/test-configuration-js.md +++ b/docs/src/test-configuration-js.md @@ -120,7 +120,7 @@ test.describe('locale block', () => { These are commonly used options for various scenarios. You usually set them globally in [configuration file](#global-configuration). -- `actionTimeout` - Timeout for each Playwright action in milliseconds. Defaults to `0` (no timeout). +- `actionTimeout` - Timeout for each Playwright action in milliseconds. Defaults to `0` (no timeout). Learn more about [various timeouts](./test-timeouts.md). - `baseURL` - Base URL used for all pages in the context. Allows navigating by using just the path, for example `page.goto('/settings')`. - `browserName` - Name of the browser that will run the tests, one of `chromium`, `firefox`, or `webkit`. - `bypassCSP` - Toggles bypassing Content-Security-Policy. Useful when CSP includes the production origin. @@ -556,7 +556,7 @@ In addition to configuring [Browser] or [BrowserContext], videos or screenshots, - `testDir`: Directory with the test files. - `testIgnore`: Glob patterns or regular expressions that should be ignored when looking for the test files. For example, `'**/test-assets'`. - `testMatch`: Glob patterns or regular expressions that match test files. For example, `'**/todo-tests/*.spec.ts'`. By default, Playwright Test runs `.*(test|spec)\.(js|ts|mjs)` files. -- `timeout`: Time in milliseconds given to each test. +- `timeout`: Time in milliseconds given to each test. Learn more about [various timeouts](./test-timeouts.md). - `webServer: { command: string, port: number, timeout?: number, reuseExistingServer?: boolean, cwd?: string, env?: object }` - Launch a process and wait that it's ready before the tests will start. See [launch web server](./test-advanced.md#launching-a-development-web-server-during-the-tests) configuration for examples. - `workers`: The maximum number of concurrent worker processes to use for parallelizing tests. diff --git a/docs/src/test-timeouts-js.md b/docs/src/test-timeouts-js.md new file mode 100644 index 0000000000..9f0613b847 --- /dev/null +++ b/docs/src/test-timeouts-js.md @@ -0,0 +1,281 @@ +--- +id: test-timeouts +title: "Timeouts" +--- + + + +## Overview + +Playwright Test has multiple configurable timeouts for various tasks. + +|Timeout |Default |Description | +|:----------|:----------------|:--------------------------------| +|Test timeout|30000 ms|Timeout for each test, includes test, hooks and fixtures:
Set default
{`config = { timeout: 60000 }`}
Override
```test.setTimeout(120000)``` | +|Expect timeout|5000 ms|Timeout for each assertion:
Set default
{`config = { expect: { timeout: 10000 } }`}
Override
```expect(locator).toBeVisible({ timeout: 10000 })```| +|Action timeout| no timeout |Timeout for each action:
Set default
{`config = { use: { actionTimeout: 10000 } }`}
Override
```locator.click({ timeout: 10000 })```| +|Navigation timeout| no timeout |Timeout for each navigation action:
Set default
{`config = { use: { navigationTimeout: 30000 } }`}
Override
```page.goto('/', { timeout: 30000 })```| +|Global timeout|no timeout |Global timeout for the whole test run:
Set in config
{`config = { globalTimeout: 60*60*1000 }`}
| + +## Test timeout + +Playwright Test enforces a timeout for each test, 30 seconds by default. Time spent by the test function, fixtures, `beforeEach` and `afterEach` hooks is included in the test timeout. + +Timed out test produces the following error: + +``` +example.spec.ts:3:1 › basic test =========================== + +Timeout of 30000ms exceeded. +``` + +The same test timeout also applies to `beforeAll` and `afterAll` hooks. + +### Set test timeout in the config + +```js js-flavor=js +// playwright.config.js +// @ts-check + +/** @type {import('@playwright/test').PlaywrightTestConfig} */ +const config = { + timeout: 5 * 60 * 1000, +}; + +module.exports = config; +``` + +```js js-flavor=ts +// playwright.config.ts +import { PlaywrightTestConfig } from '@playwright/test'; + +const config: PlaywrightTestConfig = { + timeout: 5 * 60 * 1000, +}; +export default config; +``` + +API reference: [`property: TestConfig.timeout`]. + +### Set timeout for a single test + +```js js-flavor=js +const { test, expect } = require('@playwright/test'); + +test('slow test', async ({ page }) => { + test.slow(); // Easy way to triple the default timeout + // ... +}); + +test('very slow test', async ({ page }) => { + test.setTimeout(120000); + // ... +}); +``` + +```js js-flavor=ts +import { test, expect } from '@playwright/test'; + +test('slow test', async ({ page }) => { + test.slow(); // Easy way to triple the default timeout + // ... +}); + +test('very slow test', async ({ page }) => { + test.setTimeout(120000); + // ... +}); +``` + +API reference: [`method: Test.setTimeout`] and [`method: Test.slow`]. + +### Change timeout from a hook or fixture + +```js js-flavor=js +const { test, expect } = require('@playwright/test'); + +test.beforeEach(async ({ page }, testInfo) => { + // Extend timeout for all tests running this hook by 30 seconds. + testInfo.setTimeout(testInfo.timeout + 30000); +}); +``` + +```js js-flavor=ts +import { test, expect } from '@playwright/test'; + +test.beforeEach(async ({ page }, testInfo) => { + // Extend timeout for all tests running this hook by 30 seconds. + testInfo.setTimeout(testInfo.timeout + 30000); +}); +``` + +API reference: [`method: TestInfo.setTimeout`]. + +## Expect timeout + +Web-first assertions like `expect(locator).toHaveText()` have a separate timeout, 5 seconds by default. Assertion timeout is unrelated to the test timeout. It produces the following error: + +``` +example.spec.ts:3:1 › basic test =========================== + +Error: expect(received).toHaveText(expected) + +Expected string: "my text" +Received string: "" +Call log: + - expect.toHaveText with timeout 5000ms + - waiting for selector "button" +``` + +### Set expect timeout in the config + +```js js-flavor=js +// playwright.config.js +// @ts-check + +/** @type {import('@playwright/test').PlaywrightTestConfig} */ +const config = { + expect: { + timeout: 10 * 1000, + }, +}; + +module.exports = config; +``` + +```js js-flavor=ts +// playwright.config.ts +import { PlaywrightTestConfig } from '@playwright/test'; + +const config: PlaywrightTestConfig = { + expect: { + timeout: 10 * 1000, + }, +}; +export default config; +``` + +API reference: [`property: TestConfig.expect`]. + +### Set timeout for a single assertion + +```js js-flavor=js +const { test, expect } = require('@playwright/test'); + +test('basic test', async ({ page }) => { + await expect(page.locator('button')).toHaveText('Sign in', { timeout: 10000 }); +}); +``` + +```js js-flavor=ts +import { test, expect } from '@playwright/test'; + +test('basic test', async ({ page }) => { + await expect(page.locator('button')).toHaveText('Sign in', { timeout: 10000 }); +}); +``` + +## Action and navigation timeouts + +Test usually performs some actions by calling Playwright APIs, for example `locator.click()`. These actions do not have a timeout by default, but you can set one. Action that timed out produces the following error: + +``` +example.spec.ts:3:1 › basic test =========================== + +locator.click: Timeout 1000ms exceeded. +=========================== logs =========================== +waiting for selector "button" +============================================================ +``` + +Playwright also allows to set a separate timeout for navigation actions like `page.goto()` because loading a page is usually slower. + +### Set action and navigation timeouts in the config + +```js js-flavor=js +// playwright.config.js +// @ts-check + +/** @type {import('@playwright/test').PlaywrightTestConfig} */ +const config = { + use: { + actionTimeout: 10 * 1000, + navigationTimeout: 30 * 1000, + }, +}; + +module.exports = config; +``` + +```js js-flavor=ts +// playwright.config.ts +import { PlaywrightTestConfig } from '@playwright/test'; + +const config: PlaywrightTestConfig = { + use: { + actionTimeout: 10 * 1000, + navigationTimeout: 30 * 1000, + }, +}; +export default config; +``` + +API reference: [`property: TestOptions.actionTimeout`] and [`property: TestOptions.navigationTimeout`]. + +### Set timeout for a single action + +```js js-flavor=js +const { test, expect } = require('@playwright/test'); + +test('basic test', async ({ page }) => { + await page.goto('https://playwright.dev', { timeout: 30000 }); + await page.locator('text=Get Started').click({ timeout: 10000 }); +}); +``` + +```js js-flavor=ts +import { test, expect } from '@playwright/test'; + +test('basic test', async ({ page }) => { + await page.goto('https://playwright.dev', { timeout: 30000 }); + await page.locator('text=Get Started').click({ timeout: 10000 }); +}); +``` + +## Global timeout + +Playwright Test supports a timeout for the whole test run. This prevents excess resource usage when everything went wrong. There is no default global timeout, but you can set a reasonable one in the config, for example one hour. Global timeout produces the following error: + +``` +Running 1000 tests using 10 workers + + 514 skipped + 486 passed + Timed out waiting 3600s for the entire test run +``` + +You can set global timeout in the config. + +```js js-flavor=js +// playwright.config.js +// @ts-check + +/** @type {import('@playwright/test').PlaywrightTestConfig} */ +const config = { + globalTimeout: 60 * 60 * 1000, +}; + +module.exports = config; +``` + +```js js-flavor=ts +// playwright.config.ts +import { PlaywrightTestConfig } from '@playwright/test'; + +const config: PlaywrightTestConfig = { + globalTimeout: 60 * 60 * 1000, +}; +export default config; +``` + +API reference: [`property: TestConfig.globalTimeout`]. diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index b04ede3078..60655dbda1 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -458,7 +458,8 @@ interface TestConfig { globalTeardown?: string; /** * Maximum time in milliseconds the whole test suite can run. Zero timeout (default) disables this behavior. Useful on CI - * to prevent broken setup from running too long and wasting resources. + * to prevent broken setup from running too long and wasting resources. Learn more about + * [various timeouts](https://playwright.dev/docs/test-timeouts). * * ```ts * // playwright.config.ts @@ -590,7 +591,7 @@ interface TestConfig { workers?: number; /** - * Configuration for the `expect` assertion library. + * Configuration for the `expect` assertion library. Learn more about [various timeouts](https://playwright.dev/docs/test-timeouts). * * ```ts * // playwright.config.ts @@ -737,7 +738,8 @@ interface TestConfig { * Timeout for each test in milliseconds. Defaults to 30 seconds. * * This is a base timeout for all tests. In addition, each test can configure its own timeout with - * [test.setTimeout(timeout)](https://playwright.dev/docs/api/class-test#test-set-timeout). + * [test.setTimeout(timeout)](https://playwright.dev/docs/api/class-test#test-set-timeout). Learn more about + * [various timeouts](https://playwright.dev/docs/test-timeouts). * * ```ts * // playwright.config.ts @@ -879,7 +881,8 @@ export interface FullConfig { globalTeardown: string | null; /** * Maximum time in milliseconds the whole test suite can run. Zero timeout (default) disables this behavior. Useful on CI - * to prevent broken setup from running too long and wasting resources. + * to prevent broken setup from running too long and wasting resources. Learn more about + * [various timeouts](https://playwright.dev/docs/test-timeouts). * * ```ts * // playwright.config.ts @@ -1234,7 +1237,8 @@ export interface TestInfo { slow(condition: boolean, description: string): void; /** - * Changes the timeout for the currently running test. Zero means no timeout. + * Changes the timeout for the currently running test. Zero means no timeout. Learn more about + * [various timeouts](https://playwright.dev/docs/test-timeouts). * * Timeout is usually specified in the [configuration file](https://playwright.dev/docs/test-configuration), but it could be useful to change the * timeout in certain scenarios: @@ -1272,8 +1276,10 @@ export interface TestInfo { */ expectedStatus: TestStatus; /** - * Timeout in milliseconds for the currently running test. Zero means no timeout. Timeout is usually specified in the - * [configuration file](https://playwright.dev/docs/test-configuration) + * Timeout in milliseconds for the currently running test. Zero means no timeout. Learn more about + * [various timeouts](https://playwright.dev/docs/test-timeouts). + * + * Timeout is usually specified in the [configuration file](https://playwright.dev/docs/test-configuration) * * ```ts * import { test, expect } from '@playwright/test'; @@ -2198,7 +2204,7 @@ export interface TestType boolean, description: string): void; /** - * Changes the timeout for the test. + * Changes the timeout for the test. Learn more about [various timeouts](https://playwright.dev/docs/test-timeouts). * * ```ts * import { test, expect } from '@playwright/test'; @@ -2729,6 +2735,8 @@ export interface PlaywrightTestOptions { * * This is a default timeout for all Playwright actions, same as configured via * [page.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-page#page-set-default-timeout). + * + * Learn more about [various timeouts](https://playwright.dev/docs/test-timeouts). */ actionTimeout: number | undefined; /** @@ -2736,6 +2744,8 @@ export interface PlaywrightTestOptions { * * This is a default navigation timeout, same as configured via * [page.setDefaultNavigationTimeout(timeout)](https://playwright.dev/docs/api/class-page#page-set-default-navigation-timeout). + * + * Learn more about [various timeouts](https://playwright.dev/docs/test-timeouts). */ navigationTimeout: number | undefined; }