diff --git a/docs/src/test-api/class-fixtures.md b/docs/src/test-api/class-fixtures.md index 4928afacea..1b9cda5791 100644 --- a/docs/src/test-api/class-fixtures.md +++ b/docs/src/test-api/class-fixtures.md @@ -201,9 +201,9 @@ test('basic test', async ({ page }) => { - type: <[Screenshot]<"off"|"on"|"only-on-failure">> Whether to automatically capture a screenshot after each test. Defaults to `'off'`. -1. `'off'`: Do not capture screenshots. -1. `'on'`: Capture screenshot after each test. -1. `'only-on-failure'`: Capture screenshot after each test failure. +* `'off'`: Do not capture screenshots. +* `'on'`: Capture screenshot after each test. +* `'only-on-failure'`: Capture screenshot after each test failure. Learn more about [automatic screenshots](./test-configuration.md#automatic-screenshots). @@ -215,10 +215,10 @@ Learn more about [automatic screenshots](./test-configuration.md#automatic-scree - type: <[Screenshot]<"off"|"on"|"retain-on-failure"|"on-first-retry">> Whether to record a trace for each test. Defaults to `'off'`. -1. `'off'`: Do not record a trace. -1. `'on'`: Record a trace for each test. -1. `'retain-on-failure'`: Record a trace for each test, but remove it from successful test runs. -1. `'on-first-retry'`: Record a trace only when retrying a test for the first time. +* `'off'`: Do not record a trace. +* `'on'`: Record a trace for each test. +* `'retain-on-failure'`: Record a trace for each test, but remove it from successful test runs. +* `'on-first-retry'`: Record a trace only when retrying a test for the first time. Learn more about [recording trace](./test-configuration.md#record-test-trace). @@ -232,10 +232,10 @@ Learn more about [recording trace](./test-configuration.md#record-test-trace). - `height` <[int]> Whether to record video for each test. Defaults to `'off'`. -1. `'off'`: Do not record video. -1. `'on'`: Record video for each test. -1. `'retain-on-failure'`: Record video for each test, but remove all videos from successful test runs. -1. `'on-first-retry'`: Record video only when retrying a test for the first time. +* `'off'`: Do not record video. +* `'on'`: Record video for each test. +* `'retain-on-failure'`: Record video for each test, but remove all videos from successful test runs. +* `'on-first-retry'`: Record video only when retrying a test for the first time. Learn more about [recording video](./test-configuration.md#record-video). diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index c20a2ab1a3..09e738aa6b 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -22,7 +22,7 @@ module.exports = config; ```js js-flavor=ts // playwright.config.ts -import { PlaywrightTestConfig, devices } from '@playwright/test'; +import { PlaywrightTestConfig } from '@playwright/test'; const config: PlaywrightTestConfig = { timeout: 30000, @@ -150,9 +150,9 @@ test('example test', async ({}, testInfo) => { - type: <[PreserveOutput]<"always"|"never"|"failures-only">> Whether to preserve test output in the [`property: TestConfig.outputDir`]. Defaults to `'always'`. -1. `'always'` - preserve output for all tests; -1. `'never'` - do not preserve output for any tests; -1. `'failures-only'` - only preserve output for failed tests. +* `'always'` - preserve output for all tests; +* `'never'` - do not preserve output for any tests; +* `'failures-only'` - only preserve output for failed tests. ## property: TestConfig.projects @@ -177,9 +177,9 @@ The number of times to repeat each test, useful for debugging flaky tests. - `1` <[Object]> An object with reporter options if any The list of reporters to use. Each reporter can be: -1. A builtin reporter name like `'list'` or `'json'`. -2. A module name like `'my-awesome-reporter'`. -3. A relative path to the reporter like `'./reporters/my-awesome-reporter.js'`. +* A builtin reporter name like `'list'` or `'json'`. +* A module name like `'my-awesome-reporter'`. +* A relative path to the reporter like `'./reporters/my-awesome-reporter.js'`. You can pass options to the reporter in a tuple like `['json', { outputFile: './report.json' }]`. @@ -260,9 +260,9 @@ This is a base timeout for all tests. In addition, each test can configure its o - type: <[UpdateSnapshots]<"all"|"none"|"missing">> Whether to update expected snapshots with the actual results produced by the test run. Defaults to `'missing'`. -1. `'all'` - All tests that are executed will update snapshots. -1. `'none'` - No snapshots are updated. -1. `'missing'` - Missing snapshots are created, for example when authoring a new test and running it for the first time. This is the default. +* `'all'` - All tests that are executed will update snapshots. +* `'none'` - No snapshots are updated. +* `'missing'` - Missing snapshots are created, for example when authoring a new test and running it for the first time. This is the default. Learn more about [snapshots](./test-snapshots.md). diff --git a/docs/src/test-api/class-testerror.md b/docs/src/test-api/class-testerror.md new file mode 100644 index 0000000000..b27b2cb8c9 --- /dev/null +++ b/docs/src/test-api/class-testerror.md @@ -0,0 +1,19 @@ +# class: TestError +* langs: js + +Information about an error thrown during test execution. + +## property: TestError.message +- type: <[void]|[string]> + +Error message. Set when [Error] (or its subclass) has been thrown. + +## property: TestError.stack +- type: <[void]|[string]> + +Error stack. Set when [Error] (or its subclass) has been thrown. + +## property: TestError.value +- type: <[void]|[string]> + +The value that was thrown. Set when anything except the [Error] (or its subclass) has been thrown. diff --git a/docs/src/test-api/class-testinfo.md b/docs/src/test-api/class-testinfo.md index 4adb2a854a..3545908af1 100644 --- a/docs/src/test-api/class-testinfo.md +++ b/docs/src/test-api/class-testinfo.md @@ -97,8 +97,8 @@ An error thrown during test execution, if any. - type: <[TestStatus]<"passed"|"failed"|"timedOut"|"skipped">> Expected status for the currently running test. This is usually `'passed'`, except for a few cases: -1. `'skipped'` for skipped tests, e.g. with [`method: Test.skip`]; -1. `'failed'` for tests marked as failed with [`method: Test.fail`]. +* `'skipped'` for skipped tests, e.g. with [`method: Test.skip`]; +* `'failed'` for tests marked as failed with [`method: Test.fail`]. Expected status is usually compared with the actual [`property: TestInfo.status`]: diff --git a/docs/src/test-reporter-api/class-location.md b/docs/src/test-reporter-api/class-location.md new file mode 100644 index 0000000000..af6c7d492a --- /dev/null +++ b/docs/src/test-reporter-api/class-location.md @@ -0,0 +1,19 @@ +# class: Location +* langs: js + +Represents a location in the source code where [TestCase] or [Suite] is defined. + +## property: Location.file +- type: <[string]> + +Path to the source file. + +## property: Location.line +- type: <[int]> + +Line number in the source file. + +## property: Location.column +- type: <[string]> + +Column number in the source file. diff --git a/docs/src/test-reporter-api/class-reporter.md b/docs/src/test-reporter-api/class-reporter.md new file mode 100644 index 0000000000..d78d836f31 --- /dev/null +++ b/docs/src/test-reporter-api/class-reporter.md @@ -0,0 +1,182 @@ +# class: Reporter +* langs: js + +Test runner notifies the reporter about various events during test execution. All methods of the reporter are optional. + +You can create a custom reporter my implementing a class with some of the reporter methods. Make sure to export this class as default. + +```js js-flavor=js +// my-awesome-reporter.js +// @ts-check + +/** @implements {import('@playwright/test/reporter').Reporter} */ +class MyReporter { + onBegin(config, suite) { + console.log(`Starting the run with ${suite.allTests().length} tests`); + } + + onTestBegin(test) { + console.log(`Starting test ${test.title}`); + } + + onTestEnd(test, result) { + console.log(`Finished test ${test.title}: ${result.status}`); + } + + onEnd(result) { + console.log(`Finished the run: ${result.status}`); + } +} + +module.exports = MyReporter; +``` + +```js js-flavor=ts +// playwright.config.ts +import { Reporter } from '@playwright/test/reporter'; + +class MyReporter implements Reporter { + onBegin(config, suite) { + console.log(`Starting the run with ${suite.allTests().length} tests`); + } + + onTestBegin(test) { + console.log(`Starting test ${test.title}`); + } + + onTestEnd(test, result) { + console.log(`Finished test ${test.title}: ${result.status}`); + } + + onEnd(result) { + console.log(`Finished the run: ${result.status}`); + } +} +export default MyReporter; +``` + +Now use this reporter with [`property: TestConfig.reporter`]. + +```js js-flavor=js +// playwright.config.js +// @ts-check + +/** @type {import('@playwright/test').PlaywrightTestConfig} */ +const config = { + reporter: './my-awesome-reporter.js', +}; + +module.exports = config; +``` + +```js js-flavor=ts +// playwright.config.ts +import { PlaywrightTestConfig } from '@playwright/test'; + +const config: PlaywrightTestConfig = { + reporter: './my-awesome-reporter.ts', +}; +export default config; +``` + +Learn more about [reporters](./test-reporters.md). + +## method: Reporter.onBegin + +Called once before running tests. All tests have been already discovered and put into a hierarchy of [Suite]s. + +### param: Reporter.onBegin.config +- `config` <[TestConfig]> + +Resolved configuration. + +### param: Reporter.onBegin.suite +- `suite` <[Suite]> + +The root suite that contains all projects, files and test cases. + + + +## async method: Reporter.onEnd + +Called after all tests has been run, or testing has been interrupted. Note that this method may return a [Promise] and Playwright Test will await it. + +### param: Reporter.onEnd.result +- `result` <[Object]> + - `status` <[FullStatus]<"passed"|"failed"|"timedout"|"interrupted">> + +Result of the full test run. +* `'passed'` - Everything went as expected. +* `'failed'` - Any test has failed. +* `'timedout'` - The [`property: TestConfig.globalTimeout`] has been reached. +* `'interrupted'` - Interrupted by the user. + + + + +## method: Reporter.onError + +Called on some global error, for example unhandled exception in the worker process. + +### param: Reporter.onError.error +- `error` <[TestError]> + +The error. + + +## method: Reporter.onStdErr + +Called when something has been written to the standard error in the worker process. + +### param: Reporter.onStdErr.chunk +- `chunk` <[string]|[Buffer]> + +Output chunk. + +### param: Reporter.onStdErr.test +- `test` <[void]|[TestCase]> + +Test that was running. Note that output may happen when to test is running, in which case this will be [void]. + + + +## method: Reporter.onStdOut + +Called when something has been written to the standard output in the worker process. + +### param: Reporter.onStdOut.chunk +- `chunk` <[string]|[Buffer]> + +Output chunk. + +### param: Reporter.onStdOut.test +- `test` <[void]|[TestCase]> + +Test that was running. Note that output may happen when to test is running, in which case this will be [void]. + + + +## method: Reporter.onTestBegin + +Called after a test has been started in the worker process. + +### param: Reporter.onTestBegin.test +- `test` <[TestCase]> + +Test that has been started. + + + +## method: Reporter.onTestEnd + +Called after a test has been finished in the worker process. + +### param: Reporter.onTestEnd.test +- `test` <[TestCase]> + +Test that has been finished. + +### param: Reporter.onTestEnd.result +- `result` <[TestResult]> + +Result of the test run. diff --git a/docs/src/test-reporter-api/class-suite.md b/docs/src/test-reporter-api/class-suite.md new file mode 100644 index 0000000000..dd43dbc82d --- /dev/null +++ b/docs/src/test-reporter-api/class-suite.md @@ -0,0 +1,55 @@ +# class: Suite +* langs: js + +`Suite` is a group of tests. All tests in Playwright Test form the following hierarchy: + +* Root suite has a child suite for each [TestProject]. + * Project suite #1. Has a child suite for each test file in the project. + * File suite #1 + * [TestCase] #1 + * [TestCase] #2 + * Suite corresponding to a [`method: Test.describe`] group + * [TestCase] #1 in a group + * [TestCase] #2 in a group + * < more test cases ... > + * File suite #2 + * < more file suites ... > + * Project suite #2 + * < more project suites ... > + +Reporter is given a root suite in the [`method: Reporter.onBegin`] method. + +## method: Suite.allTests +- returns: <[Array]<[TestCase]>> + +Returns the list of all test cases in this suite and its descendants, as opposite to [`property: Suite.tests`]. + +## property: Suite.location +- type: <[void]|[Location]> + +Location in the source where the suite is defined. Missing for root and project suites. + +## property: Suite.suites +- type: <[Array]<[Suite]>> + +Child suites. See [Suite] for the hierarchy of suites. + +## property: Suite.tests +- type: <[Array]<[TestCase]>> + +Test cases in the suite. Note that only test cases defined directly in this suite are in the list. Any test cases defined in nested [`method: Test.describe`] groups are listed +in the child [`property: Suite.suites`]. + +## property: Suite.title +- type: <[string]> + +Suite title. +* Empty for root suite. +* Project name for project suite. +* File path for file suite. +* Title passed to [`method: Test.describe`] for a group suite. + +## method: Suite.titlePath +- returns: <[Array]<[string]>> + +Returns a list of titles from the root down to this suite. diff --git a/docs/src/test-reporter-api/class-testcase.md b/docs/src/test-reporter-api/class-testcase.md new file mode 100644 index 0000000000..c2f1c3c45c --- /dev/null +++ b/docs/src/test-reporter-api/class-testcase.md @@ -0,0 +1,70 @@ +# class: TestCase +* langs: js + +`TestCase` corresponds to every [`method: Test.(call)`] call in a test file. When a single [`method: Test.(call)`] is running in multiple projects or repeated multiple times, it will have multiple `TestCase` objects in corresponding projects' suites. + +## property: TestCase.annotations +- type: <[Array]<[Object]>> + - `type` <[string]> Annotation type, for example `'skip'` or `'fail'`. + - `description` <[void]|[string]> Optional description. + +The list of annotations applicable to the current test. Includes annotations from the test, annotations from all [`method: Test.describe`] groups the test belongs to and file-level annotations for the test file. + +Annotations are available during test execution through [`property: TestInfo.annotations`]. + +Learn more about [test annotations](./test-annotations.md). + +## property: TestCase.expectedStatus +- type: <[TestStatus]<"passed"|"failed"|"timedOut"|"skipped">> + +Expected test status. +* Tests marked as [`method: Test.skip`] or [`method: Test.fixme`] are expected to be `'skipped'`. +* Tests marked as [`method: Test.fail`] are expected to be `'failed'`. +* Other tests are expected to be `'passed'`. + +See also [`property: TestResult.status`] for the actual status. + +## property: TestCase.location +- type: <[void]|[Location]> + +Location in the source where the test is defined. + +## method: TestCase.ok +- returns: <[boolean]> + +Whether the test is considered running fine. Non-ok tests fail the test run with non-zero exit code. + +## method: TestCase.outcome +- returns: <[TestOutcome]<"skipped"|"expected"|"unexpected"|"flaky">> + +Testing outcome for this test. Note that outcome is not the same as [`property: TestResult.status`]: +* Test that is expected to fail and actually fails is `'expected'`. +* Test that passes on a second retry is `'flaky'`. + +## property: TestCase.results +- type: <[Array]<[TestResult]>> + +Results for each run of this test. + +## property: TestCase.retries +- type: <[int]> + +The maximum number of retries given to this test in the configuration. + +Learn more about [test retries](./test-retries.md). + +## property: TestCase.timeout +- type: <[float]> + +The timeout given to the test. Affected by [`property: TestConfig.timeout`], [`property: TestProject.timeout`], [`method: Test.setTimeout`], [`method: Test.slow`] and [`method: TestInfo.setTimeout`]. + +## property: TestCase.title +- type: <[string]> + +Test title as passed to the [`method: Test.(call)`] call. + +## method: TestCase.titlePath +- returns: <[Array]<[string]>> + +Returns a list of titles from the root down to this test. + diff --git a/docs/src/test-reporter-api/class-testresult.md b/docs/src/test-reporter-api/class-testresult.md new file mode 100644 index 0000000000..966727f458 --- /dev/null +++ b/docs/src/test-reporter-api/class-testresult.md @@ -0,0 +1,57 @@ +# class: TestResult +* langs: js + +A result of a single [TestCase] run. + +## property: TestResult.attachments +- type: <[Array]<[Object]>> + - `name` <[string]> Attachment name. + - `contentType` <[string]> Content type of this attachment to properly present in the report, for example `'application/json'` or `'image/png'`. + - `path` <[void]|[string]> Optional path on the filesystem to the attached file. + - `body` <[void]|[Buffer]> Optional attachment body used instead of a file. + +The list of files or buffers attached during the test execution through [`property: TestInfo.attachments`]. + +## property: TestResult.duration +- type: <[float]> + +Running time in milliseconds. + +## property: TestResult.error +- type: <[void]|[TestError]> + +An error thrown during the test execution, if any. + +## property: TestResult.retry +- type: <[int]> + +When test is retries multiple times, each retry attempt is given a sequential number. + +Learn more about [test retries](./test-retries.md). + +## property: TestResult.startTime +- type: <[Date]> + +Start time of this particular test run. + +## property: TestResult.status +- type: <[void]|[TestStatus]<"passed"|"failed"|"timedOut"|"skipped">> + +The status of this test result. See also [`property: TestCase.expectedStatus`]. + +## property: TestResult.stderr +- type: <[Array]<[string]|[Buffer]>> + +Anything written to the standard error during the test run. + +## property: TestResult.stdout +- type: <[Array]<[string]|[Buffer]>> + +Anything written to the standard output during the test run. + +## property: TestResult.workerIndex +- type: <[int]> + +Index of the worker where the test was run. + +Learn more about [parallelism and sharding](./test-parallel.md) with Playwright Test. diff --git a/utils/doclint/cli.js b/utils/doclint/cli.js index 32c064e82b..e8be4c9e37 100755 --- a/utils/doclint/cli.js +++ b/utils/doclint/cli.js @@ -41,7 +41,9 @@ async function run() { apiDocumentation.filterForLanguage('js'); const testDocumentation = parseApi(path.join(PROJECT_DIR, 'docs', 'src', 'test-api'), path.join(PROJECT_DIR, 'docs', 'src', 'api', 'params.md')); testDocumentation.filterForLanguage('js'); - const documentation = apiDocumentation.mergeWith(testDocumentation); + const testRerpoterDocumentation = parseApi(path.join(PROJECT_DIR, 'docs', 'src', 'test-reporter-api')); + testRerpoterDocumentation.filterForLanguage('js'); + const documentation = apiDocumentation.mergeWith(testDocumentation).mergeWith(testRerpoterDocumentation); documentation.mergeWith(testDocumentation); // This validates member links.