diff --git a/docs/src/test-annotations-js.md b/docs/src/test-annotations-js.md index eaa0ea9749..8314d73491 100644 --- a/docs/src/test-annotations-js.md +++ b/docs/src/test-annotations-js.md @@ -9,9 +9,9 @@ title: "Annotations" Playwright Test supports test annotations to deal with failures, flakiness, skip, focus and tag tests: - [`method: Test.skip#1`] marks the test as irrelevant. Playwright Test does not run such a test. Use this annotation when the test is not applicable in some configuration. -- [`method: Test.fail`] marks the test as failing. Playwright Test will run this test and ensure it does indeed fail. If the test does not fail, Playwright Test will complain. +- [`method: Test.fail#1`] marks the test as failing. Playwright Test will run this test and ensure it does indeed fail. If the test does not fail, Playwright Test will complain. - [`method: Test.fixme#1`] marks the test as failing. Playwright Test will not run this test, as opposite to the `fail` annotation. Use `fixme` when running the test is slow or crashy. -- [`method: Test.slow`] marks the test as slow and triples the test timeout. +- [`method: Test.slow#1`] marks the test as slow and triples the test timeout. Annotations can be used on a single test or a group of tests. Annotations can be conditional, in which case they apply when the condition is truthy. Annotations may depend on test fixtures. There could be multiple annotations on the same test, possibly in different configurations. diff --git a/docs/src/test-api/class-test.md b/docs/src/test-api/class-test.md index 17b5a60891..c4370694d7 100644 --- a/docs/src/test-api/class-test.md +++ b/docs/src/test-api/class-test.md @@ -630,11 +630,9 @@ An object containing fixtures and/or options. Learn more about [fixtures format] -## method: Test.fail +## method: Test.fail#1 -Marks a test or a group of tests as "should fail". Playwright Test runs these tests and ensures that they are actually failing. This is useful for documentation purposes to acknowledge that some functionality is broken until it is fixed. - -Unconditional fail: +Unconditonally marks a test as "should fail". Playwright Test runs this test and ensures that it is actually failing. This is useful for documentation purposes to acknowledge that some functionality is broken until it is fixed. ```js js-flavor=js const { test, expect } = require('@playwright/test'); @@ -654,7 +652,9 @@ test('not yet ready', async ({ page }) => { }); ``` -Conditional fail a test with an optional description: +## method: Test.fail#2 + +Conditionally mark a test as "should fail" with an optional description. ```js js-flavor=js const { test, expect } = require('@playwright/test'); @@ -674,45 +674,56 @@ test('fail in WebKit', async ({ page, browserName }) => { }); ``` -Conditional fail for all tests in a file or [`method: Test.describe`] group: +### param: Test.fail#2.condition +- `condition` <[boolean]> -```js js-flavor=js -const { test, expect } = require('@playwright/test'); +Test is marked as "should fail" when the condition is `true`. -test.fail(({ browserName }) => browserName === 'webkit'); - -test('fail in WebKit 1', async ({ page }) => { - // ... -}); -test('fail in WebKit 2', async ({ page }) => { - // ... -}); -``` - -```js js-flavor=ts -import { test, expect } from '@playwright/test'; - -test.fail(({ browserName }) => browserName === 'webkit'); - -test('fail in WebKit 1', async ({ page }) => { - // ... -}); -test('fail in WebKit 2', async ({ page }) => { - // ... -}); -``` - -### param: Test.fail.condition -- `condition` <[void]|[boolean]|[function]\([Fixtures]\):[boolean]> - -Optional condition - either a boolean value, or a function that takes a fixtures object and returns a boolean. Test or tests are marked as "should fail" when the condition is `true`. - -### param: Test.fail.description -- `description` <[void]|[string]> +### param: Test.fail#2.description +- `description` <[string]> Optional description that will be reflected in a test report. +## method: Test.fail#3 + +Conditionally mark all tests in a file or [`method: Test.describe`] group as "should fail". + +```js js-flavor=js +const { test, expect } = require('@playwright/test'); + +test.fail(({ browserName }) => browserName === 'webkit'); + +test('fail in WebKit 1', async ({ page }) => { + // ... +}); +test('fail in WebKit 2', async ({ page }) => { + // ... +}); +``` + +```js js-flavor=ts +import { test, expect } from '@playwright/test'; + +test.fail(({ browserName }) => browserName === 'webkit'); + +test('fail in WebKit 1', async ({ page }) => { + // ... +}); +test('fail in WebKit 2', async ({ page }) => { + // ... +}); +``` + +### param: Test.fail#3.condition +- `callback` <[function]\([Fixtures]\):[boolean]> + +A function that returns whether to mark as "should fail", based on test fixtures. Test or tests are marked as "should fail" when the return value is `true`. + +### param: Test.fail#3.description +- `description` <[string]> + +Optional description that will be reflected in a test report. ## method: Test.fixme#1 @@ -824,12 +835,12 @@ test('broken in WebKit', async ({ page, browserName }) => { ### param: Test.fixme#3.condition - `condition` <[boolean]> -Test or tests are marked as "fixme" when the condition is `true`. +Test is marked as "fixme" when the condition is `true`. ### param: Test.fixme#3.description -- `description` <[void]|[string]> +- `description` <[string]> -An optional description that will be reflected in a test report. +Optional description that will be reflected in a test report. @@ -871,9 +882,9 @@ test('broken in WebKit 2', async ({ page }) => { A function that returns whether to mark as "fixme", based on test fixtures. Test or tests are marked as "fixme" when the return value is `true`. ### param: Test.fixme#4.description -- `description` <[void]|[string]> +- `description` <[string]> -An optional description that will be reflected in a test report. +Optional description that will be reflected in a test report. ## method: Test.info @@ -1089,12 +1100,12 @@ test.beforeEach(async ({ page }) => { ### param: Test.skip#3.condition - `condition` <[boolean]> -A skip condition. Test or tests are skipped when the condition is `true`. +A skip condition. Test is skipped when the condition is `true`. ### param: Test.skip#3.description - `description` <[void]|[string]> -An optional description that will be reflected in a test report. +Optional description that will be reflected in a test report. @@ -1136,17 +1147,15 @@ test('skip in WebKit 2', async ({ page }) => { A function that returns whether to skip, based on test fixtures. Test or tests are skipped when the return value is `true`. ### param: Test.skip#4.description -- `description` <[void]|[string]> +- `description` <[string]> -An optional description that will be reflected in a test report. +Optional description that will be reflected in a test report. -## method: Test.slow +## method: Test.slow#1 -Marks a test or a group of tests as "slow". Slow tests will be given triple the default timeout. - -Unconditional slow: +Unconditionally marks a test as "slow". Slow test will be given triple the default timeout. ```js js-flavor=js const { test, expect } = require('@playwright/test'); @@ -1166,7 +1175,9 @@ test('slow test', async ({ page }) => { }); ``` -Conditional slow a test with an optional description: +## method: Test.slow#2 + +Conditionally mark a test as "slow" with an optional description. Slow test will be given triple the default timeout. ```js js-flavor=js const { test, expect } = require('@playwright/test'); @@ -1186,7 +1197,20 @@ test('slow in WebKit', async ({ page, browserName }) => { }); ``` -Conditional slow for all tests in a file or [`method: Test.describe`] group: +### param: Test.slow#2.condition +- `condition` <[boolean]> + +Test is marked as "slow" when the condition is `true`. + +### param: Test.slow#2.description +- `description` <[string]> + +Optional description that will be reflected in a test report. + + +## method: Test.slow#3 + +Conditionally mark all tests in a file or [`method: Test.describe`] group as "slow". Slow tests will be given triple the default timeout. ```js js-flavor=js const { test, expect } = require('@playwright/test'); @@ -1214,13 +1238,13 @@ test('fail in WebKit 2', async ({ page }) => { }); ``` -### param: Test.slow.condition -- `condition` <[void]|[boolean]|[function]\([Fixtures]\):[boolean]> +### param: Test.slow#3.condition +- `callback` <[function]\([Fixtures]\):[boolean]> -Optional condition - either a boolean value, or a function that takes a fixtures object and returns a boolean. Test or tests are marked as "slow" when the condition is `true`. +A function that returns whether to mark as "slow", based on test fixtures. Test or tests are marked as "slow" when the return value is `true`. -### param: Test.slow.description -- `description` <[void]|[string]> +### param: Test.slow#3.description +- `description` <[string]> Optional description that will be reflected in a test report. diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index 06ef8c1c73..a4ecfac189 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -245,6 +245,11 @@ export default config; Any JSON-serializable metadata that will be put directly to the test report. +## property: TestConfig.name +- type: <[string]> + +Config name is visible in the report and during test execution, unless overridden by [`property: TestProject.name`]. + ## property: TestConfig.outputDir - type: <[string]> diff --git a/docs/src/test-api/class-testerror.md b/docs/src/test-api/class-testerror.md index b27b2cb8c9..bed490d78d 100644 --- a/docs/src/test-api/class-testerror.md +++ b/docs/src/test-api/class-testerror.md @@ -4,16 +4,16 @@ Information about an error thrown during test execution. ## property: TestError.message -- type: <[void]|[string]> +- type: <[string]> -Error message. Set when [Error] (or its subclass) has been thrown. +Optional error message. Set when [Error] (or its subclass) has been thrown. ## property: TestError.stack -- type: <[void]|[string]> +- type: <[string]> -Error stack. Set when [Error] (or its subclass) has been thrown. +Optional error stack. Set when [Error] (or its subclass) has been thrown. ## property: TestError.value -- type: <[void]|[string]> +- type: <[string]> -The value that was thrown. Set when anything except the [Error] (or its subclass) has been thrown. +Optional value that was thrown. Set when anything except the [Error] (or its subclass) has been thrown. diff --git a/docs/src/test-api/class-testinfo.md b/docs/src/test-api/class-testinfo.md index 4c1e62b5d4..b39e423938 100644 --- a/docs/src/test-api/class-testinfo.md +++ b/docs/src/test-api/class-testinfo.md @@ -25,7 +25,7 @@ test('basic test', async ({ page }, testInfo) => { ## property: TestInfo.annotations - type: <[Array]<[Object]>> - `type` <[string]> Annotation type, for example `'skip'` or `'fail'`. - - `description` <[void]|[string]> Optional description. + - `description` <[string]> Optional description. The list of annotations applicable to the current test. Includes annotations from the test, annotations from all [`method: Test.describe`] groups the test belongs to and file-level annotations for the test file. @@ -35,8 +35,8 @@ Learn more about [test annotations](../test-annotations.md). - type: <[Array]<[Object]>> - `name` <[string]> Attachment name. - `contentType` <[string]> Content type of this attachment to properly present in the report, for example `'application/json'` or `'image/png'`. - - `path` <[void]|[string]> Optional path on the filesystem to the attached file. - - `body` <[void]|[Buffer]> Optional attachment body used instead of a file. + - `path` <[string]> Optional path on the filesystem to the attached file. + - `body` <[Buffer]> Optional attachment body used instead of a file. The list of files or buffers attached to the current test. Some reporters show test attachments. @@ -103,7 +103,7 @@ after awaiting the attach call. - `body` <[string]|[Buffer]> Attachment body. Mutually exclusive with [`option: path`]. ### option: TestInfo.attach.contentType -- `contentType` <[void]|[string]> Optional content type of this attachment to properly present in the report, for example `'application/json'` or `'image/png'`. If omitted, content type is inferred based on the [`option: path`], or defaults to `text/plain` for [string] attachments and `application/octet-stream` for [Buffer] attachments. +- `contentType` <[string]> Content type of this attachment to properly present in the report, for example `'application/json'` or `'image/png'`. If omitted, content type is inferred based on the [`option: path`], or defaults to `text/plain` for [string] attachments and `application/octet-stream` for [Buffer] attachments. ### option: TestInfo.attach.path - `path` <[string]> Path on the filesystem to the attached file. Mutually exclusive with [`option: body`]. @@ -128,9 +128,9 @@ The number of milliseconds the test took to finish. Always zero before the test ## property: TestInfo.error -- type: <[void]|[TestError]> +- type: <[TestError]> -First error thrown during test execution, if any. This is equal to the first +Optional first error thrown during test execution, if any. This is equal to the first element in [`property: TestInfo.errors`]. ## property: TestInfo.errors @@ -144,7 +144,7 @@ Errors thrown during test execution, if any. Expected status for the currently running test. This is usually `'passed'`, except for a few cases: * `'skipped'` for skipped tests, e.g. with [`method: Test.skip#2`]; -* `'failed'` for tests marked as failed with [`method: Test.fail`]. +* `'failed'` for tests marked as failed with [`method: Test.fail#1`]. Expected status is usually compared with the actual [`property: TestInfo.status`]: @@ -166,41 +166,52 @@ test.afterEach(async ({}, testInfo) => { }); ``` -## method: TestInfo.fail +## method: TestInfo.fail#1 -Marks the currently running test as "should fail". Playwright Test ensures that this test is actually failing. This is similar to [`method: Test.fail`]. +Marks the currently running test as "should fail". Playwright Test runs theis tests and ensures that it is actually failing. This is useful for documentation purposes to acknowledge that some functionality is broken until it is fixed. This is similar to [`method: Test.fail#1`]. -### param: TestInfo.fail.condition -- `condition` <[void]|[boolean]> +## method: TestInfo.fail#2 -Optional condition - the test is marked as "should fail" when the condition is `true`. +Conditionally mark the currently running test as "should fail" with an optional description. This is similar to [`method: Test.fail#2`]. -### param: TestInfo.fail.description -- `description` <[void]|[string]> +### param: TestInfo.fail#2.condition +- `condition` <[boolean]> + +Test is marked as "should fail" when the condition is `true`. + +### param: TestInfo.fail#2.description +- `description` <[string]> Optional description that will be reflected in a test report. + ## property: TestInfo.file - type: <[string]> Absolute path to a file where the currently running test is declared. -## method: TestInfo.fixme -Marks the currently running test as "fixme". The test will be skipped, but the intention is to fix it. This is similar to [`method: Test.fixme#2`]. +## method: TestInfo.fixme#1 -### param: TestInfo.fixme.condition -- `condition` <[void]|[boolean]> +Mark a test as "fixme", with the intention to fix it. Test is immediately aborted. This is similar to [`method: Test.fixme#2`]. -Optional condition - the test is marked as "fixme" when the condition is `true`. +## method: TestInfo.fixme#2 -### param: TestInfo.fixme.description -- `description` <[void]|[string]> +Conditionally mark the currently running test as "fixme" with an optional description. This is similar to [`method: Test.fixme#3`]. + +### param: TestInfo.fixme#2.condition +- `condition` <[boolean]> + +Test is marked as "fixme" when the condition is `true`. + +### param: TestInfo.fixme#2.description +- `description` <[string]> Optional description that will be reflected in a test report. + ## property: TestInfo.fn -- type: <[function]\([TestArgs], [TestInfo]\)> +- type: <[function]> Test function as passed to `test(title, testFunction)`. @@ -248,7 +259,7 @@ test('example test', async ({}, testInfo) => { > However, this path must stay within the [`property: TestInfo.outputDir`] directory for each test (i.e. `test-results/a-test-title`), otherwise it will throw. ### param: TestInfo.outputPath.pathSegments -- `pathSegments` <[string...]> +- `...pathSegments` <[Array]<[string]>> Path segments to append at the end of the resulting path. @@ -338,34 +349,44 @@ test.beforeEach(async ({ page }, testInfo) => { Timeout in milliseconds. -## method: TestInfo.skip +## method: TestInfo.skip#1 -Skips the currently running test. This is similar to [`method: Test.skip#2`]. +Unconditionally skip the currently running test. Test is immediately aborted. This is similar to [`method: Test.skip#2`]. -### param: TestInfo.skip.condition -- `condition` <[void]|[boolean]> +## method: TestInfo.skip#2 -Optional condition - the test is skipped when the condition is `true`. +Conditionally skips the currently running test with an optional description. This is similar to [`method: Test.skip#3`]. -### param: TestInfo.skip.description -- `description` <[void]|[string]> +### param: TestInfo.skip#2.condition +- `condition` <[boolean]> + +A skip condition. Test is skipped when the condition is `true`. + +### param: TestInfo.skip#2.description +- `description` <[string]> Optional description that will be reflected in a test report. -## method: TestInfo.slow -Marks the currently running test as "slow", giving it triple the default timeout. This is similar to [`method: Test.slow`]. +## method: TestInfo.slow#1 -### param: TestInfo.slow.condition -- `condition` <[void]|[boolean]> +Marks the currently running test as "slow", giving it triple the default timeout. This is similar to [`method: Test.slow#1`]. -Optional condition - the test is marked as "slow" when the condition is `true`. +## method: TestInfo.slow#2 -### param: TestInfo.slow.description -- `description` <[void]|[string]> +Conditionally mark the currently running test as "slow" with an optional description, giving it triple the default timeout. This is similar to [`method: Test.slow#2`]. + +### param: TestInfo.slow#2.condition +- `condition` <[boolean]> + +Test is marked as "slow" when the condition is `true`. + +### param: TestInfo.slow#2.description +- `description` <[string]> Optional description that will be reflected in a test report. + ## method: TestInfo.snapshotPath - returns: <[string]> @@ -375,7 +396,7 @@ Returns a path to a snapshot file with the given `pathSegments`. Learn more abou > However, this path must stay within the snapshots directory for each test file (i.e. `a.spec.js-snapshots`), otherwise it will throw. ### param: TestInfo.snapshotPath.pathSegments -- `pathSegments` <[string...]> +- `...pathSegments` <[Array]<[string]>> The name of the snapshot or the path segments to define the snapshot file path. Snapshots with the same name in the same test file are expected to be the same. diff --git a/docs/src/test-reporter-api/class-suite.md b/docs/src/test-reporter-api/class-suite.md index b0b3d74c9e..99630fac00 100644 --- a/docs/src/test-reporter-api/class-suite.md +++ b/docs/src/test-reporter-api/class-suite.md @@ -25,14 +25,14 @@ Reporter is given a root suite in the [`method: Reporter.onBegin`] method. Returns the list of all test cases in this suite and its descendants, as opposite to [`property: Suite.tests`]. ## property: Suite.location -- type: <[void]|[Location]> +- type: <[Location]> -Location in the source where the suite is defined. Missing for root and project suites. +Optional location in the source where the suite is defined. Missing for root and project suites. ## property: Suite.parent -- type: <[void]|[Suite]> +- type: <[Suite]> -Parent suite or [void] for the root suite. +Optional parent suite, missing for the root suite. ## method: Suite.project - returns: <[void]|[TestProject]> diff --git a/docs/src/test-reporter-api/class-testcase.md b/docs/src/test-reporter-api/class-testcase.md index 74529d900a..17111f0b30 100644 --- a/docs/src/test-reporter-api/class-testcase.md +++ b/docs/src/test-reporter-api/class-testcase.md @@ -6,7 +6,7 @@ ## property: TestCase.annotations - type: <[Array]<[Object]>> - `type` <[string]> Annotation type, for example `'skip'` or `'fail'`. - - `description` <[void]|[string]> Optional description. + - `description` <[string]> Optional description. The list of annotations applicable to the current test. Includes annotations from the test, annotations from all [`method: Test.describe`] groups the test belongs to and file-level annotations for the test file. @@ -19,13 +19,13 @@ Learn more about [test annotations](../test-annotations.md). Expected test status. * Tests marked as [`method: Test.skip#1`] or [`method: Test.fixme#1`] are expected to be `'skipped'`. -* Tests marked as [`method: Test.fail`] are expected to be `'failed'`. +* Tests marked as [`method: Test.fail#1`] are expected to be `'failed'`. * Other tests are expected to be `'passed'`. See also [`property: TestResult.status`] for the actual status. ## property: TestCase.location -- type: <[void]|[Location]> +- type: <[Location]> Location in the source where the test is defined. @@ -66,7 +66,7 @@ Learn more about [test retries](../test-retries.md#retries). ## property: TestCase.timeout - type: <[float]> -The timeout given to the test. Affected by [`property: TestConfig.timeout`], [`property: TestProject.timeout`], [`method: Test.setTimeout`], [`method: Test.slow`] and [`method: TestInfo.setTimeout`]. +The timeout given to the test. Affected by [`property: TestConfig.timeout`], [`property: TestProject.timeout`], [`method: Test.setTimeout`], [`method: Test.slow#1`] and [`method: TestInfo.setTimeout`]. ## property: TestCase.title - type: <[string]> diff --git a/docs/src/test-reporter-api/class-testresult.md b/docs/src/test-reporter-api/class-testresult.md index 511f2b166c..27b2636386 100644 --- a/docs/src/test-reporter-api/class-testresult.md +++ b/docs/src/test-reporter-api/class-testresult.md @@ -7,8 +7,8 @@ A result of a single [TestCase] run. - type: <[Array]<[Object]>> - `name` <[string]> Attachment name. - `contentType` <[string]> Content type of this attachment to properly present in the report, for example `'application/json'` or `'image/png'`. - - `path` <[void]|[string]> Optional path on the filesystem to the attached file. - - `body` <[void]|[Buffer]> Optional attachment body used instead of a file. + - `path` <[string]> Optional path on the filesystem to the attached file. + - `body` <[Buffer]> Optional attachment body used instead of a file. The list of files or buffers attached during the test execution through [`property: TestInfo.attachments`]. @@ -18,9 +18,9 @@ The list of files or buffers attached during the test execution through [`proper Running time in milliseconds. ## property: TestResult.error -- type: <[void]|[TestError]> +- type: <[TestError]> -First error thrown during test execution, if any. This is equal to the first +Optional first error thrown during test execution, if any. This is equal to the first element in [`property: TestResult.errors`]. ## property: TestResult.errors diff --git a/docs/src/test-reporter-api/class-teststep.md b/docs/src/test-reporter-api/class-teststep.md index 43970bac41..3075415d7d 100644 --- a/docs/src/test-reporter-api/class-teststep.md +++ b/docs/src/test-reporter-api/class-teststep.md @@ -18,19 +18,19 @@ Step category to differentiate steps with different origin and verbosity. Built- Running time in milliseconds. ## property: TestStep.location -- type: <[void]|[Location]> +- type: <[Location]> -Location in the source where the step is defined. +Optional location in the source where the step is defined. ## property: TestStep.error -- type: <[void]|[TestError]> +- type: <[TestError]> -An error thrown during the step execution, if any. +Optional error thrown during the step execution, if any. ## property: TestStep.parent -- type: <[void]|[TestStep]> +- type: <[TestStep]> -Parent step, if any. +Optional parent step, if any. ## property: TestStep.startTime - type: <[Date]> diff --git a/docs/src/test-timeouts-js.md b/docs/src/test-timeouts-js.md index 1a4729dc83..5972bc124d 100644 --- a/docs/src/test-timeouts-js.md +++ b/docs/src/test-timeouts-js.md @@ -88,7 +88,7 @@ test('very slow test', async ({ page }) => { }); ``` -API reference: [`method: Test.setTimeout`] and [`method: Test.slow`]. +API reference: [`method: Test.setTimeout`] and [`method: Test.slow#1`]. ### Change timeout from a hook diff --git a/packages/playwright-test/src/dispatcher.ts b/packages/playwright-test/src/dispatcher.ts index 65c155ed28..81adc33577 100644 --- a/packages/playwright-test/src/dispatcher.ts +++ b/packages/playwright-test/src/dispatcher.ts @@ -239,7 +239,6 @@ export class Dispatcher { duration: -1, steps: [], location: params.location, - data: {}, }; steps.set(params.stepId, step); (parentStep || result).steps.push(step); diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index 0f1cc59f7e..ce1b9dc010 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -162,7 +162,7 @@ interface TestProject { * * The directory for each test can be accessed by * [testInfo.snapshotDir](https://playwright.dev/docs/api/class-testinfo#test-info-snapshot-dir) and - * [testInfo.snapshotPath(pathSegments)](https://playwright.dev/docs/api/class-testinfo#test-info-snapshot-path). + * [testInfo.snapshotPath(...pathSegments)](https://playwright.dev/docs/api/class-testinfo#test-info-snapshot-path). * * This path will serve as the base directory for each test file snapshot directory. Setting `snapshotDir` to * `'snapshots'`, the [testInfo.snapshotDir](https://playwright.dev/docs/api/class-testinfo#test-info-snapshot-dir) would @@ -176,10 +176,10 @@ interface TestProject { * [testProject.outputDir](https://playwright.dev/docs/api/class-testproject#test-project-output-dir) is created, * guaranteeing that test running in parallel do not conflict. This directory can be accessed by * [testInfo.outputDir](https://playwright.dev/docs/api/class-testinfo#test-info-output-dir) and - * [testInfo.outputPath(pathSegments)](https://playwright.dev/docs/api/class-testinfo#test-info-output-path). + * [testInfo.outputPath(...pathSegments)](https://playwright.dev/docs/api/class-testinfo#test-info-output-path). * * Here is an example that uses - * [testInfo.outputPath(pathSegments)](https://playwright.dev/docs/api/class-testinfo#test-info-output-path) to create a + * [testInfo.outputPath(...pathSegments)](https://playwright.dev/docs/api/class-testinfo#test-info-output-path) to create a * temporary file. * * ```ts @@ -713,6 +713,10 @@ interface TestConfig { * Any JSON-serializable metadata that will be put directly to the test report. */ metadata?: any; + /** + * Config name is visible in the report and during test execution, unless overridden by + * [testProject.name](https://playwright.dev/docs/api/class-testproject#test-project-name). + */ name?: string; /** * The base directory, relative to the config file, for snapshot files created with `toMatchSnapshot`. Defaults to @@ -720,7 +724,7 @@ interface TestConfig { * * The directory for each test can be accessed by * [testInfo.snapshotDir](https://playwright.dev/docs/api/class-testinfo#test-info-snapshot-dir) and - * [testInfo.snapshotPath(pathSegments)](https://playwright.dev/docs/api/class-testinfo#test-info-snapshot-path). + * [testInfo.snapshotPath(...pathSegments)](https://playwright.dev/docs/api/class-testinfo#test-info-snapshot-path). * * This path will serve as the base directory for each test file snapshot directory. Setting `snapshotDir` to * `'snapshots'`, the [testInfo.snapshotDir](https://playwright.dev/docs/api/class-testinfo#test-info-snapshot-dir) would @@ -744,10 +748,10 @@ interface TestConfig { * [testConfig.outputDir](https://playwright.dev/docs/api/class-testconfig#test-config-output-dir) is created, guaranteeing * that test running in parallel do not conflict. This directory can be accessed by * [testInfo.outputDir](https://playwright.dev/docs/api/class-testinfo#test-info-output-dir) and - * [testInfo.outputPath(pathSegments)](https://playwright.dev/docs/api/class-testinfo#test-info-output-path). + * [testInfo.outputPath(...pathSegments)](https://playwright.dev/docs/api/class-testinfo#test-info-output-path). * * Here is an example that uses - * [testInfo.outputPath(pathSegments)](https://playwright.dev/docs/api/class-testinfo#test-info-output-path) to create a + * [testInfo.outputPath(...pathSegments)](https://playwright.dev/docs/api/class-testinfo#test-info-output-path) to create a * temporary file. * * ```ts @@ -1172,24 +1176,6 @@ export interface FullConfig { export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped'; -/** - * Information about an error thrown during test execution. - */ -export interface TestError { - /** - * Error message. Set when [Error] (or its subclass) has been thrown. - */ - message?: string; - /** - * Error stack. Set when [Error] (or its subclass) has been thrown. - */ - stack?: string; - /** - * The value that was thrown. Set when anything except the [Error] (or its subclass) has been thrown. - */ - value?: string; -} - /** * `WorkerInfo` contains information about the worker that is running tests. It is available to * [test.beforeAll(hookFunction)](https://playwright.dev/docs/api/class-test#test-before-all) and @@ -1210,6 +1196,10 @@ export interface WorkerInfo { * Processed configuration from the [configuration file](https://playwright.dev/docs/test-configuration). */ config: FullConfig; + /** + * Processed project configuration from the [configuration file](https://playwright.dev/docs/test-configuration). + */ + project: FullProject; /** * The index of the worker between `0` and `workers - 1`. It is guaranteed that workers running at the same time have a * different `parallelIndex`. When a worker is restarted, for example after a failure, the new worker process has the same @@ -1219,10 +1209,7 @@ export interface WorkerInfo { * with Playwright Test. */ parallelIndex: number; - /** - * Processed project configuration from the [configuration file](https://playwright.dev/docs/test-configuration). - */ - project: FullProject; + /** * The unique index of the worker process that is running the test. When a worker is restarted, for example after a * failure, the new worker process gets a new unique `workerIndex`. @@ -1230,8 +1217,7 @@ export interface WorkerInfo { * Also available as `process.env.TEST_WORKER_INDEX`. Learn more about [parallelism and sharding](https://playwright.dev/docs/test-parallel) with * Playwright Test. */ - workerIndex: number; -} + workerIndex: number;} /** * `TestInfo` contains information about currently running test. It is available to any test function, @@ -1255,165 +1241,14 @@ export interface TestInfo { * Processed configuration from the [configuration file](https://playwright.dev/docs/test-configuration). */ config: FullConfig; - /** - * The index of the worker between `0` and `workers - 1`. It is guaranteed that workers running at the same time have a - * different `parallelIndex`. When a worker is restarted, for example after a failure, the new worker process has the same - * `parallelIndex`. - * - * Also available as `process.env.TEST_PARALLEL_INDEX`. Learn more about [parallelism and sharding](https://playwright.dev/docs/test-parallel) - * with Playwright Test. - */ - parallelIndex: number; /** * Processed project configuration from the [configuration file](https://playwright.dev/docs/test-configuration). */ project: FullProject; - /** - * The unique index of the worker process that is running the test. When a worker is restarted, for example after a - * failure, the new worker process gets a new unique `workerIndex`. - * - * Also available as `process.env.TEST_WORKER_INDEX`. Learn more about [parallelism and sharding](https://playwright.dev/docs/test-parallel) with - * Playwright Test. - */ - workerIndex: number; - - /** - * The title of the currently running test as passed to `test(title, testFunction)`. - */ - title: string; - /** - * The full title path starting with the project. - */ - titlePath: string[]; - /** - * Absolute path to a file where the currently running test is declared. - */ - file: string; - /** - * Line number where the currently running test is declared. - */ - line: number; - /** - * Column number where the currently running test is declared. - */ - column: number; - /** - * Test function as passed to `test(title, testFunction)`. - */ - fn: Function; - - /** - * Skips the currently running test. This is similar to - * [test.skip()](https://playwright.dev/docs/api/class-test#test-skip-2). - * @param condition Optional condition - the test is skipped when the condition is `true`. - * @param description Optional description that will be reflected in a test report. - */ - skip(): void; - /** - * Skips the currently running test. This is similar to - * [test.skip()](https://playwright.dev/docs/api/class-test#test-skip-2). - * @param condition Optional condition - the test is skipped when the condition is `true`. - * @param description Optional description that will be reflected in a test report. - */ - skip(condition: boolean): void; - /** - * Skips the currently running test. This is similar to - * [test.skip()](https://playwright.dev/docs/api/class-test#test-skip-2). - * @param condition Optional condition - the test is skipped when the condition is `true`. - * @param description Optional description that will be reflected in a test report. - */ - skip(condition: boolean, description: string): void; - - /** - * Marks the currently running test as "fixme". The test will be skipped, but the intention is to fix it. This is similar - * to [test.fixme()](https://playwright.dev/docs/api/class-test#test-fixme-2). - * @param condition Optional condition - the test is marked as "fixme" when the condition is `true`. - * @param description Optional description that will be reflected in a test report. - */ - fixme(): void; - /** - * Marks the currently running test as "fixme". The test will be skipped, but the intention is to fix it. This is similar - * to [test.fixme()](https://playwright.dev/docs/api/class-test#test-fixme-2). - * @param condition Optional condition - the test is marked as "fixme" when the condition is `true`. - * @param description Optional description that will be reflected in a test report. - */ - fixme(condition: boolean): void; - /** - * Marks the currently running test as "fixme". The test will be skipped, but the intention is to fix it. This is similar - * to [test.fixme()](https://playwright.dev/docs/api/class-test#test-fixme-2). - * @param condition Optional condition - the test is marked as "fixme" when the condition is `true`. - * @param description Optional description that will be reflected in a test report. - */ - fixme(condition: boolean, description: string): void; - - /** - * Marks the currently running test as "should fail". Playwright Test ensures that this test is actually failing. This is - * similar to [test.fail([condition, description])](https://playwright.dev/docs/api/class-test#test-fail). - * @param condition Optional condition - the test is marked as "should fail" when the condition is `true`. - * @param description Optional description that will be reflected in a test report. - */ - fail(): void; - /** - * Marks the currently running test as "should fail". Playwright Test ensures that this test is actually failing. This is - * similar to [test.fail([condition, description])](https://playwright.dev/docs/api/class-test#test-fail). - * @param condition Optional condition - the test is marked as "should fail" when the condition is `true`. - * @param description Optional description that will be reflected in a test report. - */ - fail(condition: boolean): void; - /** - * Marks the currently running test as "should fail". Playwright Test ensures that this test is actually failing. This is - * similar to [test.fail([condition, description])](https://playwright.dev/docs/api/class-test#test-fail). - * @param condition Optional condition - the test is marked as "should fail" when the condition is `true`. - * @param description Optional description that will be reflected in a test report. - */ - fail(condition: boolean, description: string): void; - - /** - * Marks the currently running test as "slow", giving it triple the default timeout. This is similar to - * [test.slow([condition, description])](https://playwright.dev/docs/api/class-test#test-slow). - * @param condition Optional condition - the test is marked as "slow" when the condition is `true`. - * @param description Optional description that will be reflected in a test report. - */ - slow(): void; - /** - * Marks the currently running test as "slow", giving it triple the default timeout. This is similar to - * [test.slow([condition, description])](https://playwright.dev/docs/api/class-test#test-slow). - * @param condition Optional condition - the test is marked as "slow" when the condition is `true`. - * @param description Optional description that will be reflected in a test report. - */ - slow(condition: boolean): void; - /** - * Marks the currently running test as "slow", giving it triple the default timeout. This is similar to - * [test.slow([condition, description])](https://playwright.dev/docs/api/class-test#test-slow). - * @param condition Optional condition - the test is marked as "slow" when the condition is `true`. - * @param description Optional description that will be reflected in a test report. - */ - slow(condition: boolean, description: string): void; - - /** - * Changes the timeout for the currently running test. Zero means no timeout. Learn more about - * [various timeouts](https://playwright.dev/docs/test-timeouts). - * - * Timeout is usually specified in the [configuration file](https://playwright.dev/docs/test-configuration), but it could be useful to change the - * timeout in certain scenarios: - * - * ```ts - * import { test, expect } from '@playwright/test'; - * - * test.beforeEach(async ({ page }, testInfo) => { - * // Extend timeout for all tests running this hook by 30 seconds. - * testInfo.setTimeout(testInfo.timeout + 30000); - * }); - * ``` - * - * @param timeout Timeout in milliseconds. - */ - setTimeout(timeout: number): void; /** * Expected status for the currently running test. This is usually `'passed'`, except for a few cases: * - `'skipped'` for skipped tests, e.g. with [test.skip()](https://playwright.dev/docs/api/class-test#test-skip-2); - * - `'failed'` for tests marked as failed with - * [test.fail([condition, description])](https://playwright.dev/docs/api/class-test#test-fail). + * - `'failed'` for tests marked as failed with [test.fail()](https://playwright.dev/docs/api/class-test#test-fail-1). * * Expected status is usually compared with the actual * [testInfo.status](https://playwright.dev/docs/api/class-testinfo#test-info-status): @@ -1430,22 +1265,23 @@ export interface TestInfo { */ expectedStatus: TestStatus; /** - * Timeout in milliseconds for the currently running test. Zero means no timeout. Learn more about - * [various timeouts](https://playwright.dev/docs/test-timeouts). + * Actual status for the currently running test. Available after the test has finished in + * [test.afterEach(hookFunction)](https://playwright.dev/docs/api/class-test#test-after-each) hook and fixtures. * - * Timeout is usually specified in the [configuration file](https://playwright.dev/docs/test-configuration) + * Status is usually compared with the + * [testInfo.expectedStatus](https://playwright.dev/docs/api/class-testinfo#test-info-expected-status): * * ```ts * import { test, expect } from '@playwright/test'; * - * test.beforeEach(async ({ page }, testInfo) => { - * // Extend timeout for all tests running this hook by 30 seconds. - * testInfo.setTimeout(testInfo.timeout + 30000); + * test.afterEach(async ({}, testInfo) => { + * if (testInfo.status !== testInfo.expectedStatus) + * console.log(`${testInfo.title} did not run as expected!`); * }); * ``` * */ - timeout: number; + status?: TestStatus; /** * The list of annotations applicable to the current test. Includes annotations from the test, annotations from all * [test.describe(title, callback)](https://playwright.dev/docs/api/class-test#test-describe) groups the test belongs to @@ -1453,7 +1289,18 @@ export interface TestInfo { * * Learn more about [test annotations](https://playwright.dev/docs/test-annotations). */ - annotations: { type: string, description?: string }[]; + annotations: Array<{ + /** + * Annotation type, for example `'skip'` or `'fail'`. + */ + type: string; + + /** + * Optional description. + */ + description?: string; + }>; + /** * The list of files or buffers attached to the current test. Some reporters show test attachments. * @@ -1461,7 +1308,28 @@ export interface TestInfo { * [testInfo.attach(name[, options])](https://playwright.dev/docs/api/class-testinfo#test-info-attach) instead of directly * pushing onto this array. */ - attachments: { name: string, path?: string, body?: Buffer, contentType: string }[]; + attachments: Array<{ + /** + * Attachment name. + */ + name: string; + + /** + * Content type of this attachment to properly present in the report, for example `'application/json'` or `'image/png'`. + */ + contentType: string; + + /** + * Optional path on the filesystem to the attached file. + */ + path?: string; + + /** + * Optional attachment body used instead of a file. + */ + body?: Buffer; + }>; + /** * Attach a value or a file from disk to the current test. Some reporters show test attachments. Either `path` or `body` * must be specified, but not both. @@ -1496,12 +1364,132 @@ export interface TestInfo { * @param name * @param options */ - attach(name: string, options?: { contentType?: string, path?: string, body?: string | Buffer }): Promise; + attach(name: string, options?: { + body?: string|Buffer; + + contentType?: string; + + path?: string; + }): void; + + /** + * Column number where the currently running test is declared. + */ + column: number; + + /** + * The number of milliseconds the test took to finish. Always zero before the test finishes, either successfully or not. + * Can be used in [test.afterEach(hookFunction)](https://playwright.dev/docs/api/class-test#test-after-each) hook. + */ + duration: number; + + /** + * Optional first error thrown during test execution, if any. This is equal to the first element in + * [testInfo.errors](https://playwright.dev/docs/api/class-testinfo#test-info-errors). + */ + error?: TestError; + + /** + * Errors thrown during test execution, if any. + */ + errors: Array; + + /** + * Marks the currently running test as "should fail". Playwright Test runs theis tests and ensures that it is actually + * failing. This is useful for documentation purposes to acknowledge that some functionality is broken until it is fixed. + * This is similar to [test.fail()](https://playwright.dev/docs/api/class-test#test-fail-1). + */ + fail(): void; + + /** + * Conditionally mark the currently running test as "should fail" with an optional description. This is similar to + * [test.fail(condition[, description])](https://playwright.dev/docs/api/class-test#test-fail-2). + * @param condition Test is marked as "should fail" when the condition is `true`. + * @param description Optional description that will be reflected in a test report. + */ + fail(condition: boolean, description?: string): void; + + /** + * Absolute path to a file where the currently running test is declared. + */ + file: string; + + /** + * Mark a test as "fixme", with the intention to fix it. Test is immediately aborted. This is similar to + * [test.fixme()](https://playwright.dev/docs/api/class-test#test-fixme-2). + */ + fixme(): void; + + /** + * Conditionally mark the currently running test as "fixme" with an optional description. This is similar to + * [test.fixme(condition[, description])](https://playwright.dev/docs/api/class-test#test-fixme-3). + * @param condition Test is marked as "fixme" when the condition is `true`. + * @param description Optional description that will be reflected in a test report. + */ + fixme(condition: boolean, description?: string): void; + + /** + * Test function as passed to `test(title, testFunction)`. + */ + fn: Function; + + /** + * Line number where the currently running test is declared. + */ + line: number; + + /** + * Absolute path to the snapshot output directory for this specific test. Each test suite gets its own directory so they + * cannot conflict. + */ + snapshotDir: string; + + /** + * Absolute path to the output directory for this specific test run. Each test run gets its own directory so they cannot + * conflict. + */ + outputDir: string; + + /** + * Returns a path inside the [testInfo.outputDir](https://playwright.dev/docs/api/class-testinfo#test-info-output-dir) + * where the test can safely put a temporary file. Guarantees that tests running in parallel will not interfere with each + * other. + * + * ```ts + * import { test, expect } from '@playwright/test'; + * import fs from 'fs'; + * + * test('example test', async ({}, testInfo) => { + * const file = testInfo.outputPath('dir', 'temporary-file.txt'); + * await fs.promises.writeFile(file, 'Put some data to the dir/temporary-file.txt', 'utf8'); + * }); + * ``` + * + * > Note that `pathSegments` accepts path segments to the test output directory such as `testInfo.outputPath('relative', + * 'path', 'to', 'output')`. + * > However, this path must stay within the + * [testInfo.outputDir](https://playwright.dev/docs/api/class-testinfo#test-info-output-dir) directory for each test (i.e. + * `test-results/a-test-title`), otherwise it will throw. + * @param pathSegments Path segments to append at the end of the resulting path. + */ + outputPath(...pathSegments: Array): string; + + /** + * The index of the worker between `0` and `workers - 1`. It is guaranteed that workers running at the same time have a + * different `parallelIndex`. When a worker is restarted, for example after a failure, the new worker process has the same + * `parallelIndex`. + * + * Also available as `process.env.TEST_PARALLEL_INDEX`. Learn more about [parallelism and sharding](https://playwright.dev/docs/test-parallel) + * with Playwright Test. + */ + parallelIndex: number; + /** * Specifies a unique repeat index when running in "repeat each" mode. This mode is enabled by passing `--repeat-each` to * the [command line](https://playwright.dev/docs/test-cli). */ repeatEachIndex: number; + /** * Specifies the retry number when the test is retried after a failure. The first test run has * [testInfo.retry](https://playwright.dev/docs/api/class-testinfo#test-info-retry) equal to zero, the first retry has it @@ -1526,63 +1514,56 @@ export interface TestInfo { * */ retry: number; + /** - * The number of milliseconds the test took to finish. Always zero before the test finishes, either successfully or not. - * Can be used in [test.afterEach(hookFunction)](https://playwright.dev/docs/api/class-test#test-after-each) hook. - */ - duration: number; - /** - * Actual status for the currently running test. Available after the test has finished in - * [test.afterEach(hookFunction)](https://playwright.dev/docs/api/class-test#test-after-each) hook and fixtures. + * Changes the timeout for the currently running test. Zero means no timeout. Learn more about + * [various timeouts](https://playwright.dev/docs/test-timeouts). * - * Status is usually compared with the - * [testInfo.expectedStatus](https://playwright.dev/docs/api/class-testinfo#test-info-expected-status): + * Timeout is usually specified in the [configuration file](https://playwright.dev/docs/test-configuration), but it could be useful to change the + * timeout in certain scenarios: * * ```ts * import { test, expect } from '@playwright/test'; * - * test.afterEach(async ({}, testInfo) => { - * if (testInfo.status !== testInfo.expectedStatus) - * console.log(`${testInfo.title} did not run as expected!`); + * test.beforeEach(async ({ page }, testInfo) => { + * // Extend timeout for all tests running this hook by 30 seconds. + * testInfo.setTimeout(testInfo.timeout + 30000); * }); * ``` * + * @param timeout Timeout in milliseconds. */ - status?: TestStatus; + setTimeout(timeout: number): void; + /** - * First error thrown during test execution, if any. This is equal to the first element in - * [testInfo.errors](https://playwright.dev/docs/api/class-testinfo#test-info-errors). + * Unconditionally skip the currently running test. Test is immediately aborted. This is similar to + * [test.skip()](https://playwright.dev/docs/api/class-test#test-skip-2). */ - error?: TestError; + skip(): void; + /** - * Errors thrown during test execution, if any. + * Conditionally skips the currently running test with an optional description. This is similar to + * [test.skip(condition[, description])](https://playwright.dev/docs/api/class-test#test-skip-3). + * @param condition A skip condition. Test is skipped when the condition is `true`. + * @param description Optional description that will be reflected in a test report. */ - errors: TestError[]; + skip(condition: boolean, description?: string): void; + /** - * Output written to `process.stdout` or `console.log` during the test execution. + * Marks the currently running test as "slow", giving it triple the default timeout. This is similar to + * [test.slow()](https://playwright.dev/docs/api/class-test#test-slow-1). */ - stdout: (string | Buffer)[]; + slow(): void; + /** - * Output written to `process.stderr` or `console.error` during the test execution. + * Conditionally mark the currently running test as "slow" with an optional description, giving it triple the default + * timeout. This is similar to + * [test.slow(condition[, description])](https://playwright.dev/docs/api/class-test#test-slow-2). + * @param condition Test is marked as "slow" when the condition is `true`. + * @param description Optional description that will be reflected in a test report. */ - stderr: (string | Buffer)[]; - /** - * Suffix used to differentiate snapshots between multiple test configurations. For example, if snapshots depend on the - * platform, you can set `testInfo.snapshotSuffix` equal to `process.platform`. In this case - * `expect(value).toMatchSnapshot(snapshotName)` will use different snapshots depending on the platform. Learn more about - * [snapshots](https://playwright.dev/docs/test-snapshots). - */ - snapshotSuffix: string; - /** - * Absolute path to the snapshot output directory for this specific test. Each test suite gets its own directory so they - * cannot conflict. - */ - snapshotDir: string; - /** - * Absolute path to the output directory for this specific test run. Each test run gets its own directory so they cannot - * conflict. - */ - outputDir: string; + slow(condition: boolean, description?: string): void; + /** * Returns a path to a snapshot file with the given `pathSegments`. Learn more about [snapshots](https://playwright.dev/docs/test-snapshots). * @@ -1592,31 +1573,62 @@ export interface TestInfo { * it will throw. * @param pathSegments The name of the snapshot or the path segments to define the snapshot file path. Snapshots with the same name in the same test file are expected to be the same. */ - snapshotPath: (...pathSegments: string[]) => string; + snapshotPath(...pathSegments: Array): string; + /** - * Returns a path inside the [testInfo.outputDir](https://playwright.dev/docs/api/class-testinfo#test-info-output-dir) - * where the test can safely put a temporary file. Guarantees that tests running in parallel will not interfere with each - * other. + * Suffix used to differentiate snapshots between multiple test configurations. For example, if snapshots depend on the + * platform, you can set `testInfo.snapshotSuffix` equal to `process.platform`. In this case + * `expect(value).toMatchSnapshot(snapshotName)` will use different snapshots depending on the platform. Learn more about + * [snapshots](https://playwright.dev/docs/test-snapshots). + */ + snapshotSuffix: string; + + /** + * Output written to `process.stderr` or `console.error` during the test execution. + */ + stderr: Array; + + /** + * Output written to `process.stdout` or `console.log` during the test execution. + */ + stdout: Array; + + /** + * Timeout in milliseconds for the currently running test. Zero means no timeout. Learn more about + * [various timeouts](https://playwright.dev/docs/test-timeouts). + * + * Timeout is usually specified in the [configuration file](https://playwright.dev/docs/test-configuration) * * ```ts * import { test, expect } from '@playwright/test'; - * import fs from 'fs'; * - * test('example test', async ({}, testInfo) => { - * const file = testInfo.outputPath('dir', 'temporary-file.txt'); - * await fs.promises.writeFile(file, 'Put some data to the dir/temporary-file.txt', 'utf8'); + * test.beforeEach(async ({ page }, testInfo) => { + * // Extend timeout for all tests running this hook by 30 seconds. + * testInfo.setTimeout(testInfo.timeout + 30000); * }); * ``` * - * > Note that `pathSegments` accepts path segments to the test output directory such as `testInfo.outputPath('relative', - * 'path', 'to', 'output')`. - * > However, this path must stay within the - * [testInfo.outputDir](https://playwright.dev/docs/api/class-testinfo#test-info-output-dir) directory for each test (i.e. - * `test-results/a-test-title`), otherwise it will throw. - * @param pathSegments Path segments to append at the end of the resulting path. */ - outputPath: (...pathSegments: string[]) => string; -} + timeout: number; + + /** + * The title of the currently running test as passed to `test(title, testFunction)`. + */ + title: string; + + /** + * The full title path starting with the project. + */ + titlePath: Array; + + /** + * The unique index of the worker process that is running the test. When a worker is restarted, for example after a + * failure, the new worker process gets a new unique `workerIndex`. + * + * Also available as `process.env.TEST_WORKER_INDEX`. Learn more about [parallelism and sharding](https://playwright.dev/docs/test-parallel) with + * Playwright Test. + */ + workerIndex: number;} interface SuiteFunction { (title: string, callback: () => void): void; @@ -1886,8 +1898,8 @@ export interface TestType boolean, description?: string): void; /** @@ -1970,8 +1982,8 @@ export interface TestType boolean, description?: string): void; /** - * Marks a test or a group of tests as "should fail". Playwright Test runs these tests and ensures that they are actually - * failing. This is useful for documentation purposes to acknowledge that some functionality is broken until it is fixed. - * - * Unconditional fail: + * Unconditonally marks a test as "should fail". Playwright Test runs this test and ensures that it is actually failing. + * This is useful for documentation purposes to acknowledge that some functionality is broken until it is fixed. * * ```ts * import { test, expect } from '@playwright/test'; @@ -2010,53 +2020,10 @@ export interface TestType { - * test.fail(browserName === 'webkit', 'This feature is not implemented for Mac yet'); - * // ... - * }); - * ``` - * - * Conditional fail for all tests in a file or - * [test.describe(title, callback)](https://playwright.dev/docs/api/class-test#test-describe) group: - * - * ```ts - * import { test, expect } from '@playwright/test'; - * - * test.fail(({ browserName }) => browserName === 'webkit'); - * - * test('fail in WebKit 1', async ({ page }) => { - * // ... - * }); - * test('fail in WebKit 2', async ({ page }) => { - * // ... - * }); - * ``` - * - * @param condition Optional condition - either a boolean value, or a function that takes a fixtures object and returns a boolean. Test or tests are marked as "should fail" when the condition is `true`. - * @param description Optional description that will be reflected in a test report. */ fail(): void; /** - * Marks a test or a group of tests as "should fail". Playwright Test runs these tests and ensures that they are actually - * failing. This is useful for documentation purposes to acknowledge that some functionality is broken until it is fixed. - * - * Unconditional fail: - * - * ```ts - * import { test, expect } from '@playwright/test'; - * - * test('not yet ready', async ({ page }) => { - * test.fail(); - * // ... - * }); - * ``` - * - * Conditional fail a test with an optional description: + * Conditionally mark a test as "should fail" with an optional description. * * ```ts * import { test, expect } from '@playwright/test'; @@ -2067,8 +2034,13 @@ export interface TestType boolean, description?: string): void; /** - * Marks a test or a group of tests as "should fail". Playwright Test runs these tests and ensures that they are actually - * failing. This is useful for documentation purposes to acknowledge that some functionality is broken until it is fixed. - * - * Unconditional fail: - * - * ```ts - * import { test, expect } from '@playwright/test'; - * - * test('not yet ready', async ({ page }) => { - * test.fail(); - * // ... - * }); - * ``` - * - * Conditional fail a test with an optional description: - * - * ```ts - * import { test, expect } from '@playwright/test'; - * - * test('fail in WebKit', async ({ page, browserName }) => { - * test.fail(browserName === 'webkit', 'This feature is not implemented for Mac yet'); - * // ... - * }); - * ``` - * - * Conditional fail for all tests in a file or - * [test.describe(title, callback)](https://playwright.dev/docs/api/class-test#test-describe) group: - * - * ```ts - * import { test, expect } from '@playwright/test'; - * - * test.fail(({ browserName }) => browserName === 'webkit'); - * - * test('fail in WebKit 1', async ({ page }) => { - * // ... - * }); - * test('fail in WebKit 2', async ({ page }) => { - * // ... - * }); - * ``` - * - * @param condition Optional condition - either a boolean value, or a function that takes a fixtures object and returns a boolean. Test or tests are marked as "should fail" when the condition is `true`. - * @param description Optional description that will be reflected in a test report. - */ - fail(condition: boolean, description: string): void; - /** - * Marks a test or a group of tests as "should fail". Playwright Test runs these tests and ensures that they are actually - * failing. This is useful for documentation purposes to acknowledge that some functionality is broken until it is fixed. - * - * Unconditional fail: - * - * ```ts - * import { test, expect } from '@playwright/test'; - * - * test('not yet ready', async ({ page }) => { - * test.fail(); - * // ... - * }); - * ``` - * - * Conditional fail a test with an optional description: - * - * ```ts - * import { test, expect } from '@playwright/test'; - * - * test('fail in WebKit', async ({ page, browserName }) => { - * test.fail(browserName === 'webkit', 'This feature is not implemented for Mac yet'); - * // ... - * }); - * ``` - * - * Conditional fail for all tests in a file or - * [test.describe(title, callback)](https://playwright.dev/docs/api/class-test#test-describe) group: - * - * ```ts - * import { test, expect } from '@playwright/test'; - * - * test.fail(({ browserName }) => browserName === 'webkit'); - * - * test('fail in WebKit 1', async ({ page }) => { - * // ... - * }); - * test('fail in WebKit 2', async ({ page }) => { - * // ... - * }); - * ``` - * - * @param condition Optional condition - either a boolean value, or a function that takes a fixtures object and returns a boolean. Test or tests are marked as "should fail" when the condition is `true`. - * @param description Optional description that will be reflected in a test report. - */ - fail(callback: (args: TestArgs & WorkerArgs) => boolean): void; - /** - * Marks a test or a group of tests as "should fail". Playwright Test runs these tests and ensures that they are actually - * failing. This is useful for documentation purposes to acknowledge that some functionality is broken until it is fixed. - * - * Unconditional fail: - * - * ```ts - * import { test, expect } from '@playwright/test'; - * - * test('not yet ready', async ({ page }) => { - * test.fail(); - * // ... - * }); - * ``` - * - * Conditional fail a test with an optional description: - * - * ```ts - * import { test, expect } from '@playwright/test'; - * - * test('fail in WebKit', async ({ page, browserName }) => { - * test.fail(browserName === 'webkit', 'This feature is not implemented for Mac yet'); - * // ... - * }); - * ``` - * - * Conditional fail for all tests in a file or - * [test.describe(title, callback)](https://playwright.dev/docs/api/class-test#test-describe) group: - * - * ```ts - * import { test, expect } from '@playwright/test'; - * - * test.fail(({ browserName }) => browserName === 'webkit'); - * - * test('fail in WebKit 1', async ({ page }) => { - * // ... - * }); - * test('fail in WebKit 2', async ({ page }) => { - * // ... - * }); - * ``` - * - * @param condition Optional condition - either a boolean value, or a function that takes a fixtures object and returns a boolean. Test or tests are marked as "should fail" when the condition is `true`. - * @param description Optional description that will be reflected in a test report. - */ - fail(callback: (args: TestArgs & WorkerArgs) => boolean, description: string): void; - /** - * Marks a test or a group of tests as "slow". Slow tests will be given triple the default timeout. - * - * Unconditional slow: + * Unconditionally marks a test as "slow". Slow test will be given triple the default timeout. * * ```ts * import { test, expect } from '@playwright/test'; @@ -2239,52 +2071,10 @@ export interface TestType { - * test.slow(browserName === 'webkit', 'This feature is slow on Mac'); - * // ... - * }); - * ``` - * - * Conditional slow for all tests in a file or - * [test.describe(title, callback)](https://playwright.dev/docs/api/class-test#test-describe) group: - * - * ```ts - * import { test, expect } from '@playwright/test'; - * - * test.slow(({ browserName }) => browserName === 'webkit'); - * - * test('slow in WebKit 1', async ({ page }) => { - * // ... - * }); - * test('fail in WebKit 2', async ({ page }) => { - * // ... - * }); - * ``` - * - * @param condition Optional condition - either a boolean value, or a function that takes a fixtures object and returns a boolean. Test or tests are marked as "slow" when the condition is `true`. - * @param description Optional description that will be reflected in a test report. */ slow(): void; /** - * Marks a test or a group of tests as "slow". Slow tests will be given triple the default timeout. - * - * Unconditional slow: - * - * ```ts - * import { test, expect } from '@playwright/test'; - * - * test('slow test', async ({ page }) => { - * test.slow(); - * // ... - * }); - * ``` - * - * Conditional slow a test with an optional description: + * Conditionally mark a test as "slow" with an optional description. Slow test will be given triple the default timeout. * * ```ts * import { test, expect } from '@playwright/test'; @@ -2295,53 +2085,14 @@ export interface TestType browserName === 'webkit'); - * - * test('slow in WebKit 1', async ({ page }) => { - * // ... - * }); - * test('fail in WebKit 2', async ({ page }) => { - * // ... - * }); - * ``` - * - * @param condition Optional condition - either a boolean value, or a function that takes a fixtures object and returns a boolean. Test or tests are marked as "slow" when the condition is `true`. + * @param condition Test is marked as "slow" when the condition is `true`. * @param description Optional description that will be reflected in a test report. */ - slow(condition: boolean): void; + slow(condition: boolean, description?: string): void; /** - * Marks a test or a group of tests as "slow". Slow tests will be given triple the default timeout. - * - * Unconditional slow: - * - * ```ts - * import { test, expect } from '@playwright/test'; - * - * test('slow test', async ({ page }) => { - * test.slow(); - * // ... - * }); - * ``` - * - * Conditional slow a test with an optional description: - * - * ```ts - * import { test, expect } from '@playwright/test'; - * - * test('slow in WebKit', async ({ page, browserName }) => { - * test.slow(browserName === 'webkit', 'This feature is slow on Mac'); - * // ... - * }); - * ``` - * - * Conditional slow for all tests in a file or - * [test.describe(title, callback)](https://playwright.dev/docs/api/class-test#test-describe) group: + * Conditionally mark all tests in a file or + * [test.describe(title, callback)](https://playwright.dev/docs/api/class-test#test-describe) group as "slow". Slow tests + * will be given triple the default timeout. * * ```ts * import { test, expect } from '@playwright/test'; @@ -2356,100 +2107,10 @@ export interface TestType { - * test.slow(); - * // ... - * }); - * ``` - * - * Conditional slow a test with an optional description: - * - * ```ts - * import { test, expect } from '@playwright/test'; - * - * test('slow in WebKit', async ({ page, browserName }) => { - * test.slow(browserName === 'webkit', 'This feature is slow on Mac'); - * // ... - * }); - * ``` - * - * Conditional slow for all tests in a file or - * [test.describe(title, callback)](https://playwright.dev/docs/api/class-test#test-describe) group: - * - * ```ts - * import { test, expect } from '@playwright/test'; - * - * test.slow(({ browserName }) => browserName === 'webkit'); - * - * test('slow in WebKit 1', async ({ page }) => { - * // ... - * }); - * test('fail in WebKit 2', async ({ page }) => { - * // ... - * }); - * ``` - * - * @param condition Optional condition - either a boolean value, or a function that takes a fixtures object and returns a boolean. Test or tests are marked as "slow" when the condition is `true`. - * @param description Optional description that will be reflected in a test report. - */ - slow(callback: (args: TestArgs & WorkerArgs) => boolean): void; - /** - * Marks a test or a group of tests as "slow". Slow tests will be given triple the default timeout. - * - * Unconditional slow: - * - * ```ts - * import { test, expect } from '@playwright/test'; - * - * test('slow test', async ({ page }) => { - * test.slow(); - * // ... - * }); - * ``` - * - * Conditional slow a test with an optional description: - * - * ```ts - * import { test, expect } from '@playwright/test'; - * - * test('slow in WebKit', async ({ page, browserName }) => { - * test.slow(browserName === 'webkit', 'This feature is slow on Mac'); - * // ... - * }); - * ``` - * - * Conditional slow for all tests in a file or - * [test.describe(title, callback)](https://playwright.dev/docs/api/class-test#test-describe) group: - * - * ```ts - * import { test, expect } from '@playwright/test'; - * - * test.slow(({ browserName }) => browserName === 'webkit'); - * - * test('slow in WebKit 1', async ({ page }) => { - * // ... - * }); - * test('fail in WebKit 2', async ({ page }) => { - * // ... - * }); - * ``` - * - * @param condition Optional condition - either a boolean value, or a function that takes a fixtures object and returns a boolean. Test or tests are marked as "slow" when the condition is `true`. - * @param description Optional description that will be reflected in a test report. - */ - slow(callback: (args: TestArgs & WorkerArgs) => boolean, description: string): void; + slow(callback: (args: TestArgs & WorkerArgs) => boolean, description?: string): void; /** * Changes the timeout for the test. Zero means no timeout. Learn more about [various timeouts](https://playwright.dev/docs/test-timeouts). * @@ -3166,5 +2827,24 @@ export const expect: Expect; export {}; +/** + * Information about an error thrown during test execution. + */ +export interface TestError { + /** + * Optional error message. Set when [Error] (or its subclass) has been thrown. + */ + message?: string; + + /** + * Optional error stack. Set when [Error] (or its subclass) has been thrown. + */ + stack?: string; + + /** + * Optional value that was thrown. Set when anything except the [Error] (or its subclass) has been thrown. + */ + value?: string; +} diff --git a/packages/playwright-test/types/testReporter.d.ts b/packages/playwright-test/types/testReporter.d.ts index 1916f4b4e3..74a0ed6ea6 100644 --- a/packages/playwright-test/types/testReporter.d.ts +++ b/packages/playwright-test/types/testReporter.d.ts @@ -18,24 +18,6 @@ import type { FullConfig, FullProject, TestStatus, TestError } from './test'; export type { FullConfig, TestStatus, TestError } from './test'; -/** - * Represents a location in the source code where [TestCase] or [Suite] is defined. - */ -export interface Location { - /** - * Path to the source file. - */ - file: string; - /** - * Line number in the source file. - */ - line: number; - /** - * Column number in the source file. - */ - column: number; -} - /** * `Suite` is a group of tests. All tests in Playwright Test form the following hierarchy: * - Root suite has a child suite for each [TestProject]. @@ -58,9 +40,37 @@ export interface Location { */ export interface Suite { /** - * Parent suite or [void] for the root suite. + * Configuration of the project this suite belongs to, or [void] for the root suite. + */ + project(): FullProject | undefined; + /** + * Returns the list of all test cases in this suite and its descendants, as opposite to + * [suite.tests](https://playwright.dev/docs/api/class-suite#suite-tests). + */ + allTests(): Array; + + /** + * Optional location in the source where the suite is defined. Missing for root and project suites. + */ + location?: Location; + + /** + * Optional parent suite, missing for the root suite. */ parent?: Suite; + + /** + * Child suites. See [Suite] for the hierarchy of suites. + */ + suites: Array; + + /** + * Test cases in the suite. Note that only test cases defined directly in this suite are in the list. Any test cases + * defined in nested [test.describe(title, callback)](https://playwright.dev/docs/api/class-test#test-describe) groups are + * listed in the child [suite.suites](https://playwright.dev/docs/api/class-suite#suite-suites). + */ + tests: Array; + /** * Suite title. * - Empty for root suite. @@ -70,34 +80,11 @@ export interface Suite { * group suite. */ title: string; - /** - * Location in the source where the suite is defined. Missing for root and project suites. - */ - location?: Location; - /** - * Child suites. See [Suite] for the hierarchy of suites. - */ - suites: Suite[]; - /** - * Test cases in the suite. Note that only test cases defined directly in this suite are in the list. Any test cases - * defined in nested [test.describe(title, callback)](https://playwright.dev/docs/api/class-test#test-describe) groups are - * listed in the child [suite.suites](https://playwright.dev/docs/api/class-suite#suite-suites). - */ - tests: TestCase[]; + /** * Returns a list of titles from the root down to this suite. */ - titlePath(): string[]; - /** - * Returns the list of all test cases in this suite and its descendants, as opposite to - * [suite.tests](https://playwright.dev/docs/api/class-suite#suite-tests). - */ - allTests(): TestCase[]; - /** - * Configuration of the project this suite belongs to, or [void] for the root suite. - */ - project(): FullProject | undefined; -} + titlePath(): Array;} /** * `TestCase` corresponds to every [test.(call)(title, testFunction)](https://playwright.dev/docs/api/class-test#test-call) @@ -106,44 +93,17 @@ export interface Suite { * or repeated multiple times, it will have multiple `TestCase` objects in corresponding projects' suites. */ export interface TestCase { - /** - * Suite this test case belongs to. - */ - parent: Suite; - /** - * Test title as passed to the [test.(call)(title, testFunction)](https://playwright.dev/docs/api/class-test#test-call) - * call. - */ - title: string; - /** - * Location in the source where the test is defined. - */ - location: Location; - /** - * Returns a list of titles from the root down to this test. - */ - titlePath(): string[]; /** * Expected test status. * - Tests marked as [test.skip(title, testFunction)](https://playwright.dev/docs/api/class-test#test-skip-1) or * [test.fixme(title, testFunction)](https://playwright.dev/docs/api/class-test#test-fixme-1) are expected to be * `'skipped'`. - * - Tests marked as [test.fail([condition, description])](https://playwright.dev/docs/api/class-test#test-fail) are - * expected to be `'failed'`. + * - Tests marked as [test.fail()](https://playwright.dev/docs/api/class-test#test-fail-1) are expected to be `'failed'`. * - Other tests are expected to be `'passed'`. * * See also [testResult.status](https://playwright.dev/docs/api/class-testresult#test-result-status) for the actual status. */ expectedStatus: TestStatus; - /** - * The timeout given to the test. Affected by - * [testConfig.timeout](https://playwright.dev/docs/api/class-testconfig#test-config-timeout), - * [testProject.timeout](https://playwright.dev/docs/api/class-testproject#test-project-timeout), - * [test.setTimeout(timeout)](https://playwright.dev/docs/api/class-test#test-set-timeout), - * [test.slow([condition, description])](https://playwright.dev/docs/api/class-test#test-slow) and - * [testInfo.setTimeout(timeout)](https://playwright.dev/docs/api/class-testinfo#test-info-set-timeout). - */ - timeout: number; /** * The list of annotations applicable to the current test. Includes annotations from the test, annotations from all * [test.describe(title, callback)](https://playwright.dev/docs/api/class-test#test-describe) groups the test belongs to @@ -154,138 +114,164 @@ export interface TestCase { * * Learn more about [test annotations](https://playwright.dev/docs/test-annotations). */ - annotations: { type: string, description?: string }[]; + annotations: Array<{ + /** + * Annotation type, for example `'skip'` or `'fail'`. + */ + type: string; + + /** + * Optional description. + */ + description?: string; + }>; + /** - * The maximum number of retries given to this test in the configuration. - * - * Learn more about [test retries](https://playwright.dev/docs/test-retries#retries). + * Location in the source where the test is defined. */ - retries: number; + location: Location; + /** - * Contains the repeat index when running in "repeat each" mode. This mode is enabled by passing `--repeat-each` to the - * [command line](https://playwright.dev/docs/test-cli). + * Whether the test is considered running fine. Non-ok tests fail the test run with non-zero exit code. */ - repeatEachIndex: number; - /** - * Results for each run of this test. - */ - results: TestResult[]; + ok(): boolean; + /** * Testing outcome for this test. Note that outcome is not the same as * [testResult.status](https://playwright.dev/docs/api/class-testresult#test-result-status): * - Test that is expected to fail and actually fails is `'expected'`. * - Test that passes on a second retry is `'flaky'`. */ - outcome(): 'skipped' | 'expected' | 'unexpected' | 'flaky'; + outcome(): "skipped"|"expected"|"unexpected"|"flaky"; + /** - * Whether the test is considered running fine. Non-ok tests fail the test run with non-zero exit code. + * Suite this test case belongs to. */ - ok(): boolean; -} + parent: Suite; + + /** + * Contains the repeat index when running in "repeat each" mode. This mode is enabled by passing `--repeat-each` to the + * [command line](https://playwright.dev/docs/test-cli). + */ + repeatEachIndex: number; + + /** + * Results for each run of this test. + */ + results: Array; + + /** + * The maximum number of retries given to this test in the configuration. + * + * Learn more about [test retries](https://playwright.dev/docs/test-retries#retries). + */ + retries: number; + + /** + * The timeout given to the test. Affected by + * [testConfig.timeout](https://playwright.dev/docs/api/class-testconfig#test-config-timeout), + * [testProject.timeout](https://playwright.dev/docs/api/class-testproject#test-project-timeout), + * [test.setTimeout(timeout)](https://playwright.dev/docs/api/class-test#test-set-timeout), + * [test.slow()](https://playwright.dev/docs/api/class-test#test-slow-1) and + * [testInfo.setTimeout(timeout)](https://playwright.dev/docs/api/class-testinfo#test-info-set-timeout). + */ + timeout: number; + + /** + * Test title as passed to the [test.(call)(title, testFunction)](https://playwright.dev/docs/api/class-test#test-call) + * call. + */ + title: string; + + /** + * Returns a list of titles from the root down to this test. + */ + titlePath(): Array;} /** * A result of a single [TestCase] run. */ export interface TestResult { - /** - * When test is retries multiple times, each retry attempt is given a sequential number. - * - * Learn more about [test retries](https://playwright.dev/docs/test-retries#retries). - */ - retry: number; - /** - * Index of the worker where the test was run. - * - * Learn more about [parallelism and sharding](https://playwright.dev/docs/test-parallel) with Playwright Test. - */ - workerIndex: number; - /** - * Start time of this particular test run. - */ - startTime: Date; - /** - * Running time in milliseconds. - */ - duration: number; /** * The status of this test result. See also * [testCase.expectedStatus](https://playwright.dev/docs/api/class-testcase#test-case-expected-status). */ status: TestStatus; - /** - * First error thrown during test execution, if any. This is equal to the first element in - * [testResult.errors](https://playwright.dev/docs/api/class-testresult#test-result-errors). - */ - error?: TestError; - /** - * Errors thrown during the test execution. - */ - errors: TestError[]; /** * The list of files or buffers attached during the test execution through * [testInfo.attachments](https://playwright.dev/docs/api/class-testinfo#test-info-attachments). */ - attachments: { name: string, path?: string, body?: Buffer, contentType: string }[]; - /** - * Anything written to the standard output during the test run. - */ - stdout: (string | Buffer)[]; - /** - * Anything written to the standard error during the test run. - */ - stderr: (string | Buffer)[]; - /** - * List of steps inside this test run. - */ - steps: TestStep[]; -} + attachments: Array<{ + /** + * Attachment name. + */ + name: string; + + /** + * Content type of this attachment to properly present in the report, for example `'application/json'` or `'image/png'`. + */ + contentType: string; + + /** + * Optional path on the filesystem to the attached file. + */ + path?: string; + + /** + * Optional attachment body used instead of a file. + */ + body?: Buffer; + }>; -/** - * Represents a step in the [TestRun]. - */ -export interface TestStep { - /** - * User-friendly test step title. - */ - title: string; - /** - * Returns a list of step titles from the root step down to this step. - */ - titlePath(): string[]; - /** - * Location in the source where the step is defined. - */ - location?: Location; - /** - * Parent step, if any. - */ - parent?: TestStep; - /** - * Step category to differentiate steps with different origin and verbosity. Built-in categories are: - * - `hook` for fixtures and hooks initialization and teardown - * - `expect` for expect calls - * - `pw:api` for Playwright API calls. - * - `test.step` for test.step API calls. - */ - category: string, - /** - * Start time of this particular test step. - */ - startTime: Date; /** * Running time in milliseconds. */ duration: number; + /** - * An error thrown during the step execution, if any. + * Optional first error thrown during test execution, if any. This is equal to the first element in + * [testResult.errors](https://playwright.dev/docs/api/class-testresult#test-result-errors). */ error?: TestError; + /** - * List of steps inside this step. + * Errors thrown during the test execution. */ - steps: TestStep[]; - data: { [key: string]: any }; -} + errors: Array; + + /** + * When test is retries multiple times, each retry attempt is given a sequential number. + * + * Learn more about [test retries](https://playwright.dev/docs/test-retries#retries). + */ + retry: number; + + /** + * Start time of this particular test run. + */ + startTime: Date; + + /** + * Anything written to the standard error during the test run. + */ + stderr: Array; + + /** + * Anything written to the standard output during the test run. + */ + stdout: Array; + + /** + * List of steps inside this test run. + */ + steps: Array; + + /** + * Index of the worker where the test was run. + * + * Learn more about [parallelism and sharding](https://playwright.dev/docs/test-parallel) with Playwright Test. + */ + workerIndex: number;} /** * Result of the full test run. @@ -442,5 +428,78 @@ export interface Reporter { export {}; +/** + * Represents a location in the source code where [TestCase] or [Suite] is defined. + */ +export interface Location { + /** + * Path to the source file. + */ + file: string; + + /** + * Line number in the source file. + */ + line: number; + + /** + * Column number in the source file. + */ + column: number; +} + +/** + * Represents a step in the [TestRun]. + */ +export interface TestStep { + /** + * Step category to differentiate steps with different origin and verbosity. Built-in categories are: + * - `hook` for fixtures and hooks initialization and teardown + * - `expect` for expect calls + * - `pw:api` for Playwright API calls. + * - `test.step` for test.step API calls. + */ + category: string; + + /** + * Running time in milliseconds. + */ + duration: number; + + /** + * Optional location in the source where the step is defined. + */ + location?: Location; + + /** + * Optional error thrown during the step execution, if any. + */ + error?: TestError; + + /** + * Optional parent step, if any. + */ + parent?: TestStep; + + /** + * Start time of this particular test step. + */ + startTime: Date; + + /** + * List of steps inside this step. + */ + steps: Array; + + /** + * User-friendly test step title. + */ + title: string; + + /** + * Returns a list of step titles from the root step down to this step. + */ + titlePath(): Array; +} diff --git a/utils/doclint/api_parser.js b/utils/doclint/api_parser.js index daf2529ad8..ff5d3bb26c 100644 --- a/utils/doclint/api_parser.js +++ b/utils/doclint/api_parser.js @@ -95,13 +95,14 @@ class ApiParser { if (!returnType) returnType = new Documentation.Type('void'); + const comments = extractComments(spec); let member; if (match[1] === 'event') - member = Documentation.Member.createEvent(extractLangs(spec), name, returnType, extractComments(spec)); + member = Documentation.Member.createEvent(extractLangs(spec), name, returnType, comments); if (match[1] === 'property') - member = Documentation.Member.createProperty(extractLangs(spec), name, returnType, extractComments(spec)); + member = Documentation.Member.createProperty(extractLangs(spec), name, returnType, comments, guessRequired(md.render(comments))); if (match[1] === 'method' || match[1] === 'async method') { - member = Documentation.Member.createMethod(extractLangs(spec), name, [], returnType, extractComments(spec)); + member = Documentation.Member.createMethod(extractLangs(spec), name, [], returnType, comments); if (match[1] === 'async method') member.async = true; } diff --git a/utils/generate_types/index.js b/utils/generate_types/index.js index dafdb71523..9f47ce4dc7 100644 --- a/utils/generate_types/index.js +++ b/utils/generate_types/index.js @@ -32,22 +32,29 @@ Error.stackTraceLimit = 50; class TypesGenerator { /** - * @param {Documentation} documentation + * @param {{ + * documentation: Documentation, + * classNames: Set, + * overridesToDocsClassMapping: Map, + * ignoreMissing: Set, + * }} options */ - constructor(documentation) { + constructor(options) { /** @type {Array<{name: string, properties: Member[]}>} */ this.objectDefinitions = []; /** @type {Set} */ this.handledMethods = new Set(); - this.documentation = documentation; + this.documentation = options.documentation; + this.classNames = options.classNames; + this.overridesToDocsClassMapping = options.overridesToDocsClassMapping; + this.ignoreMissing = options.ignoreMissing; } /** * @param {string} overridesFile - * @param {Map=} docsOnlyClassMapping * @returns {Promise} */ - async generateTypes(overridesFile, docsOnlyClassMapping) { + async generateTypes(overridesFile) { this.documentation.filterForLanguage('js'); this.documentation.copyDocsFromSuperclasses([]); @@ -82,22 +89,22 @@ class TypesGenerator { const handledClasses = new Set(); let overrides = await parseOverrides(overridesFile, className => { - const docClass = this.docClassForName(className, docsOnlyClassMapping); + const docClass = this.docClassForName(className); if (!docClass) return ''; handledClasses.add(className); return this.writeComment(docClass.comment) + '\n'; }, (className, methodName, overloadIndex) => { - const docClass = this.docClassForName(className, docsOnlyClassMapping); + const docClass = this.docClassForName(className); let method; if (docClass) { const methods = docClass.membersArray.filter(m => m.alias === methodName && m.kind !== 'event').sort((a, b) => a.overloadIndex - b.overloadIndex); // Use the last overload when not enough overloads are defined in docs. method = methods.find(m => m.overloadIndex === overloadIndex) || methods[methods.length - 1]; } - if (docsOnlyClassMapping && !method) + if (!method && this.canIgnoreMissingName(`${className}.${methodName}`)) return ''; - this.handledMethods.add(`${className}.${methodName}`); + this.handledMethods.add(`${className}.${methodName}#${overloadIndex}`); if (!method) { if (new Set(['on', 'addListener', 'off', 'removeListener', 'once']).has(methodName)) return ''; @@ -105,12 +112,15 @@ class TypesGenerator { } return this.memberJSDOC(method, ' ').trimLeft(); }, (className) => { - const docClass = this.docClassForName(className, docsOnlyClassMapping); - return (!docsOnlyClassMapping && docClass) ? this.classBody(docClass) : ''; + const docClass = this.docClassForName(className); + if (!docClass || !this.classNames.has(docClass.name)) + return ''; + return this.classBody(docClass); }); - const IGNORED_CLASSES = ['PlaywrightAssertions', 'LocatorAssertions', 'PageAssertions', 'APIResponseAssertions', 'ScreenshotAssertions']; - const classes = this.documentation.classesArray.filter(cls => !IGNORED_CLASSES.includes(cls.name)).filter(cls => !handledClasses.has(cls.name)); + const classes = this.documentation.classesArray + .filter(cls => this.classNames.has(cls.name)) + .filter(cls => !handledClasses.has(cls.name)); { const playwright = this.documentation.classesArray.find(c => c.name === 'Playwright'); playwright.membersArray = playwright.membersArray.filter(member => !['errors', 'devices'].includes(member.name)); @@ -121,7 +131,7 @@ class TypesGenerator { `// This file is generated by ${__filename.substring(path.join(__dirname, '..', '..').length).split(path.sep).join(path.posix.sep)}`, overrides, '', - docsOnlyClassMapping ? '' : classes.map(classDesc => { + classes.map(classDesc => { return (classDesc.name === 'Playwright') ? this.classBody(classDesc, true) : this.classToString(classDesc); }).join('\n'), this.objectDefinitionsToString(overrides), @@ -131,12 +141,20 @@ class TypesGenerator { /** * @param {string} name - * @param {Map | undefined} docsOnlyClassMapping */ - docClassForName(name, docsOnlyClassMapping) { - name = (docsOnlyClassMapping ? docsOnlyClassMapping.get(name) : undefined) || name; - const docClass = this.documentation.classes.get(name); - if (!docClass && !docsOnlyClassMapping) + canIgnoreMissingName(name) { + const parts = name.split('.'); + // Either the class is ignored, or a specific method. + return this.ignoreMissing.has(name) || this.ignoreMissing.has(parts[0]); + } + + /** + * @param {string} name + */ + docClassForName(name) { + const mappedName = (this.overridesToDocsClassMapping ? this.overridesToDocsClassMapping.get(name) : undefined) || name; + const docClass = this.documentation.classes.get(mappedName); + if (!docClass && !this.canIgnoreMissingName(name)) throw new Error(`Unknown override class ${name}`); return docClass; } @@ -265,26 +283,26 @@ class TypesGenerator { if (member.async) type = `Promise<${type}>`; // do this late, because we still want object definitions for overridden types - if (!this.hasOwnMethod(classDesc, member.alias)) + if (!this.hasOwnMethod(classDesc, member)) return ''; if (exportMembersAsGlobals) { const memberType = member.kind === 'method' ? `${args} => ${type}` : type; return `${jsdoc}${exportMembersAsGlobals ? 'export const ' : ''}${member.alias}: ${memberType};` } - return `${jsdoc}${member.alias}${args}: ${type};` + return `${jsdoc}${member.alias}${member.required ? '' : '?'}${args}: ${type};` }).filter(x => x).join('\n\n')); return parts.join('\n'); } /** * @param {Documentation.Class} classDesc - * @param {string} methodName + * @param {Documentation.Member} member */ - hasOwnMethod(classDesc, methodName) { - if (this.handledMethods.has(`${classDesc.name}.${methodName}`)) + hasOwnMethod(classDesc, member) { + if (this.handledMethods.has(`${classDesc.name}.${member.alias}#${member.overloadIndex}`)) return false; while (classDesc = this.parentClass(classDesc)) { - if (classDesc.members.has(methodName)) + if (classDesc.members.has(member.alias)) return false; } return true; @@ -479,9 +497,15 @@ class TypesGenerator { fs.mkdirSync(testTypesDir) writeFile(path.join(coreTypesDir, 'protocol.d.ts'), fs.readFileSync(path.join(PROJECT_DIR, 'packages', 'playwright-core', 'src', 'server', 'chromium', 'protocol.d.ts'), 'utf8')); + const assertionClasses = new Set(['PlaywrightAssertions', 'LocatorAssertions', 'PageAssertions', 'APIResponseAssertions', 'ScreenshotAssertions']); const apiDocumentation = parseApi(path.join(PROJECT_DIR, 'docs', 'src', 'api')); apiDocumentation.index(); - const apiTypesGenerator = new TypesGenerator(apiDocumentation); + const apiTypesGenerator = new TypesGenerator({ + documentation: apiDocumentation, + classNames: new Set(apiDocumentation.classesArray.map(cls => cls.name).filter(name => !assertionClasses.has(name))), + overridesToDocsClassMapping: new Map(), + ignoreMissing: new Set(), + }); let apiTypes = await apiTypesGenerator.generateTypes(path.join(__dirname, 'overrides.d.ts')); const namedDevices = Object.keys(devices).map(name => ` ${JSON.stringify(name)}: DeviceDescriptor;`).join('\n'); apiTypes += [ @@ -504,25 +528,41 @@ class TypesGenerator { const testOnlyDocumentation = parseApi(path.join(PROJECT_DIR, 'docs', 'src', 'test-api'), path.join(PROJECT_DIR, 'docs', 'src', 'api', 'params.md')); const testDocumentation = apiDocumentation.mergeWith(testOnlyDocumentation); - const testTypesGenerator = new TypesGenerator(testDocumentation); - const testClassMapping = new Map([ - ['TestType', 'Test'], - ['Config', 'TestConfig'], - ['FullConfig', 'TestConfig'], - ['Project', 'TestProject'], - ['PlaywrightWorkerOptions', 'TestOptions'], - ['PlaywrightTestOptions', 'TestOptions'], - ['PlaywrightWorkerArgs', 'Fixtures'], - ['PlaywrightTestArgs', 'Fixtures'], - ]); - let testTypes = await testTypesGenerator.generateTypes(path.join(__dirname, 'overrides-test.d.ts'), testClassMapping); + const testTypesGenerator = new TypesGenerator({ + documentation: testDocumentation, + classNames: new Set(['TestError', 'TestInfo', 'WorkerInfo']), + overridesToDocsClassMapping: new Map([ + ['TestType', 'Test'], + ['Config', 'TestConfig'], + ['FullConfig', 'TestConfig'], + ['Project', 'TestProject'], + ['PlaywrightWorkerOptions', 'TestOptions'], + ['PlaywrightTestOptions', 'TestOptions'], + ['PlaywrightWorkerArgs', 'Fixtures'], + ['PlaywrightTestArgs', 'Fixtures'], + ]), + ignoreMissing: new Set([ + 'FullConfig.version', + 'FullConfig.rootDir', + 'SuiteFunction', + 'TestFunction', + 'PlaywrightWorkerOptions.defaultBrowserType', + 'PlaywrightWorkerArgs.playwright', + ]), + }); + let testTypes = await testTypesGenerator.generateTypes(path.join(__dirname, 'overrides-test.d.ts')); testTypes = testTypes.replace(/( +)\n/g, '\n'); // remove trailing whitespace writeFile(path.join(testTypesDir, 'test.d.ts'), testTypes); const testReporterOnlyDocumentation = parseApi(path.join(PROJECT_DIR, 'docs', 'src', 'test-reporter-api')); const testReporterDocumentation = testDocumentation.mergeWith(testReporterOnlyDocumentation); - const testReporterTypesGenerator = new TypesGenerator(testReporterDocumentation); - let testReporterTypes = await testReporterTypesGenerator.generateTypes(path.join(__dirname, 'overrides-testReporter.d.ts'), new Map()); + const testReporterTypesGenerator = new TypesGenerator({ + documentation: testReporterDocumentation, + classNames: new Set(testReporterOnlyDocumentation.classesArray.map(cls => cls.name)), + overridesToDocsClassMapping: new Map(), + ignoreMissing: new Set(['FullResult']), + }); + let testReporterTypes = await testReporterTypesGenerator.generateTypes(path.join(__dirname, 'overrides-testReporter.d.ts')); testReporterTypes = testReporterTypes.replace(/( +)\n/g, '\n'); // remove trailing whitespace writeFile(path.join(testTypesDir, 'testReporter.d.ts'), testReporterTypes); diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 6dab8b0849..db356f64ad 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -187,67 +187,16 @@ export interface FullConfig { export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped'; -export interface TestError { - message?: string; - stack?: string; - value?: string; -} - export interface WorkerInfo { config: FullConfig; - parallelIndex: number; project: FullProject; - workerIndex: number; } export interface TestInfo { config: FullConfig; - parallelIndex: number; project: FullProject; - workerIndex: number; - - title: string; - titlePath: string[]; - file: string; - line: number; - column: number; - fn: Function; - - skip(): void; - skip(condition: boolean): void; - skip(condition: boolean, description: string): void; - - fixme(): void; - fixme(condition: boolean): void; - fixme(condition: boolean, description: string): void; - - fail(): void; - fail(condition: boolean): void; - fail(condition: boolean, description: string): void; - - slow(): void; - slow(condition: boolean): void; - slow(condition: boolean, description: string): void; - - setTimeout(timeout: number): void; expectedStatus: TestStatus; - timeout: number; - annotations: { type: string, description?: string }[]; - attachments: { name: string, path?: string, body?: Buffer, contentType: string }[]; - attach(name: string, options?: { contentType?: string, path?: string, body?: string | Buffer }): Promise; - repeatEachIndex: number; - retry: number; - duration: number; status?: TestStatus; - error?: TestError; - errors: TestError[]; - stdout: (string | Buffer)[]; - stderr: (string | Buffer)[]; - snapshotSuffix: string; - snapshotDir: string; - outputDir: string; - snapshotPath: (...pathSegments: string[]) => string; - outputPath: (...pathSegments: string[]) => string; } interface SuiteFunction { @@ -280,15 +229,11 @@ export interface TestType boolean, description?: string): void; fail(): void; - fail(condition: boolean): void; - fail(condition: boolean, description: string): void; - fail(callback: (args: TestArgs & WorkerArgs) => boolean): void; - fail(callback: (args: TestArgs & WorkerArgs) => boolean, description: string): void; + fail(condition: boolean, description?: string): void; + fail(callback: (args: TestArgs & WorkerArgs) => boolean, description?: string): void; slow(): void; - slow(condition: boolean): void; - slow(condition: boolean, description: string): void; - slow(callback: (args: TestArgs & WorkerArgs) => boolean): void; - slow(callback: (args: TestArgs & WorkerArgs) => boolean, description: string): void; + slow(condition: boolean, description?: string): void; + slow(callback: (args: TestArgs & WorkerArgs) => boolean, description?: string): void; setTimeout(timeout: number): void; beforeEach(inner: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise | any): void; afterEach(inner: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise | any): void; diff --git a/utils/generate_types/overrides-testReporter.d.ts b/utils/generate_types/overrides-testReporter.d.ts index 592eb6e421..c47fc677b8 100644 --- a/utils/generate_types/overrides-testReporter.d.ts +++ b/utils/generate_types/overrides-testReporter.d.ts @@ -17,63 +17,16 @@ import type { FullConfig, FullProject, TestStatus, TestError } from './test'; export type { FullConfig, TestStatus, TestError } from './test'; -export interface Location { - file: string; - line: number; - column: number; -} - export interface Suite { - parent?: Suite; - title: string; - location?: Location; - suites: Suite[]; - tests: TestCase[]; - titlePath(): string[]; - allTests(): TestCase[]; project(): FullProject | undefined; } export interface TestCase { - parent: Suite; - title: string; - location: Location; - titlePath(): string[]; expectedStatus: TestStatus; - timeout: number; - annotations: { type: string, description?: string }[]; - retries: number; - repeatEachIndex: number; - results: TestResult[]; - outcome(): 'skipped' | 'expected' | 'unexpected' | 'flaky'; - ok(): boolean; } export interface TestResult { - retry: number; - workerIndex: number; - startTime: Date; - duration: number; status: TestStatus; - error?: TestError; - errors: TestError[]; - attachments: { name: string, path?: string, body?: Buffer, contentType: string }[]; - stdout: (string | Buffer)[]; - stderr: (string | Buffer)[]; - steps: TestStep[]; -} - -export interface TestStep { - title: string; - titlePath(): string[]; - location?: Location; - parent?: TestStep; - category: string, - startTime: Date; - duration: number; - error?: TestError; - steps: TestStep[]; - data: { [key: string]: any }; } /** diff --git a/utils/generate_types/parseOverrides.js b/utils/generate_types/parseOverrides.js index 765dffefd9..ad80ea388f 100644 --- a/utils/generate_types/parseOverrides.js +++ b/utils/generate_types/parseOverrides.js @@ -117,7 +117,7 @@ async function parseOverrides(filePath, commentForClass, commentForMethod, extra pos, text: commentForMethod(className, `${prefix}`, 0), }); - } else if (!ts.isMethodSignature(node)) { + } else if (ts.isIntersectionTypeNode(node) || ts.isTypeLiteralNode(node)) { ts.forEachChild(node, child => visitProperties(className, prefix, child)); } }