diff --git a/docs/src/test-reporter-api/class-suite.md b/docs/src/test-reporter-api/class-suite.md index 80642def9e..f9661dc3fd 100644 --- a/docs/src/test-reporter-api/class-suite.md +++ b/docs/src/test-reporter-api/class-suite.md @@ -81,7 +81,7 @@ Returns a list of titles from the root down to this suite. ## property: Suite.type * since: v1.44 -- returns: <[SuiteType]<'root' | 'project' | 'file' | 'describe'>> +- returns: <[SuiteType]<"root"|"project"|"file"|"describe">> Returns the type of the suite. The Suites form the following hierarchy: `root` -> `project` -> `file` -> `describe` -> ...`describe` -> `test`. diff --git a/docs/src/test-reporter-api/class-testcase.md b/docs/src/test-reporter-api/class-testcase.md index 0abfa5c8d4..1b1dfb7845 100644 --- a/docs/src/test-reporter-api/class-testcase.md +++ b/docs/src/test-reporter-api/class-testcase.md @@ -110,6 +110,6 @@ Returns a list of titles from the root down to this test. ## property: TestCase.type * since: v1.44 -- returns: <[TestCaseType]<'test'>> +- returns: <[TestCaseType]<"test">> Returns type of the test. diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.ts index 5ed0d34eca..424e60889d 100644 --- a/packages/playwright/types/testReporter.d.ts +++ b/packages/playwright/types/testReporter.d.ts @@ -227,324 +227,6 @@ export interface FullProject { timeout: number; } -/** - * `Suite` is a group of tests. All tests in Playwright Test form the following hierarchy: - * - Root suite has a child suite for each {@link FullProject}. - * - Project suite #1. Has a child suite for each test file in the project. - * - File suite #1 - * - {@link TestCase} #1 - * - {@link TestCase} #2 - * - Suite corresponding to a - * [test.describe([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe) - * group - * - {@link TestCase} #1 in a group - * - {@link TestCase} #2 in a group - * - < more test cases ... > - * - File suite #2 - * - < more file suites ... > - * - Project suite #2 - * - < more project suites ... > - * - * Reporter is given a root suite in the - * [reporter.onBegin(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-on-begin) method. - */ -export interface Suite { - /** - * Returns the type of the suite. The Suites form the following hierarchy: `root` -> `project` -> `file` -> `describe` - * -> ...`describe` -> `test`. - */ - type: 'root' | 'project' | 'file' | 'describe'; - /** - * Returns the list of all test cases in this suite and its descendants, as opposite to - * [suite.tests](https://playwright.dev/docs/api/class-suite#suite-tests). - */ - allTests(): Array; - - /** - * Test cases and suites defined directly in this suite. The elements are returned in their declaration order. You can - * discriminate between different entry types using - * [testCase.type](https://playwright.dev/docs/api/class-testcase#test-case-type) and - * [suite.type](https://playwright.dev/docs/api/class-suite#suite-type). - */ - entries(): Array; - - /** - * Configuration of the project this suite belongs to, or [void] for the root suite. - */ - project(): FullProject|undefined; - - /** - * Returns a list of titles from the root down to this suite. - */ - titlePath(): Array; - - /** - * Location in the source where the suite is defined. Missing for root and project suites. - */ - location?: Location; - - /** - * Parent suite, missing for the root suite. - */ - parent?: Suite; - - /** - * Child suites. See {@link Suite} for the hierarchy of suites. - */ - suites: Array; - - /** - * Test cases in the suite. Note that only test cases defined directly in this suite are in the list. Any test cases - * defined in nested - * [test.describe([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe) groups are - * listed in the child [suite.suites](https://playwright.dev/docs/api/class-suite#suite-suites). - */ - tests: Array; - - /** - * Suite title. - * - Empty for root suite. - * - Project name for project suite. - * - File path for file suite. - * - Title passed to - * [test.describe([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe) for a - * group suite. - */ - title: string; -} - -/** - * `TestCase` corresponds to every - * [test.(call)(title[, details, body])](https://playwright.dev/docs/api/class-test#test-call) call in a test file. - * When a single [test.(call)(title[, details, body])](https://playwright.dev/docs/api/class-test#test-call) is - * running in multiple projects or repeated multiple times, it will have multiple `TestCase` objects in corresponding - * projects' suites. - */ -export interface TestCase { - /** - * Returns type of the test. - */ - type: 'test'; - /** - * Expected test status. - * - Tests marked as - * [test.skip([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-skip) - * or - * [test.fixme([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fixme) - * are expected to be `'skipped'`. - * - Tests marked as - * [test.fail([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fail) - * are expected to be `'failed'`. - * - Other tests are expected to be `'passed'`. - * - * See also [testResult.status](https://playwright.dev/docs/api/class-testresult#test-result-status) for the actual - * status. - */ - expectedStatus: TestStatus; - /** - * Whether the test is considered running fine. Non-ok tests fail the test run with non-zero exit code. - */ - ok(): boolean; - - /** - * Testing outcome for this test. Note that outcome is not the same as - * [testResult.status](https://playwright.dev/docs/api/class-testresult#test-result-status): - * - Test that is expected to fail and actually fails is `'expected'`. - * - Test that passes on a second retry is `'flaky'`. - */ - outcome(): "skipped"|"expected"|"unexpected"|"flaky"; - - /** - * Returns a list of titles from the root down to this test. - */ - titlePath(): Array; - - /** - * The list of annotations applicable to the current test. Includes: - * - annotations defined on the test or suite via - * [test.(call)(title[, details, body])](https://playwright.dev/docs/api/class-test#test-call) and - * [test.describe([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe); - * - annotations implicitly added by methods - * [test.skip([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-skip), - * [test.fixme([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fixme) - * and - * [test.fail([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fail); - * - annotations appended to - * [testInfo.annotations](https://playwright.dev/docs/api/class-testinfo#test-info-annotations) during the test - * execution. - * - * Annotations are available during test execution through - * [testInfo.annotations](https://playwright.dev/docs/api/class-testinfo#test-info-annotations). - * - * Learn more about [test annotations](https://playwright.dev/docs/test-annotations). - */ - annotations: Array<{ - /** - * Annotation type, for example `'skip'` or `'fail'`. - */ - type: string; - - /** - * Optional description. - */ - description?: string; - }>; - - /** - * Unique test ID that is computed based on the test file name, test title and project name. Test ID can be used as a - * history ID. - */ - id: string; - - /** - * Location in the source where the test is defined. - */ - location: Location; - - /** - * Suite this test case belongs to. - */ - parent: Suite; - - /** - * Contains the repeat index when running in "repeat each" mode. This mode is enabled by passing `--repeat-each` to - * the [command line](https://playwright.dev/docs/test-cli). - */ - repeatEachIndex: number; - - /** - * Results for each run of this test. - */ - results: Array; - - /** - * The maximum number of retries given to this test in the configuration. - * - * Learn more about [test retries](https://playwright.dev/docs/test-retries#retries). - */ - retries: number; - - /** - * The list of tags defined on the test or suite via - * [test.(call)(title[, details, body])](https://playwright.dev/docs/api/class-test#test-call) or - * [test.describe([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe), as well as - * `@`-tokens extracted from test and suite titles. - * - * Learn more about [test tags](https://playwright.dev/docs/test-annotations#tag-tests). - */ - tags: Array; - - /** - * The timeout given to the test. Affected by - * [testConfig.timeout](https://playwright.dev/docs/api/class-testconfig#test-config-timeout), - * [testProject.timeout](https://playwright.dev/docs/api/class-testproject#test-project-timeout), - * [test.setTimeout(timeout)](https://playwright.dev/docs/api/class-test#test-set-timeout), - * [test.slow([condition, callback, description])](https://playwright.dev/docs/api/class-test#test-slow) and - * [testInfo.setTimeout(timeout)](https://playwright.dev/docs/api/class-testinfo#test-info-set-timeout). - */ - timeout: number; - - /** - * Test title as passed to the - * [test.(call)(title[, details, body])](https://playwright.dev/docs/api/class-test#test-call) call. - */ - title: string; -} - -/** - * A result of a single {@link TestCase} run. - */ -export interface TestResult { - /** - * The status of this test result. See also - * [testCase.expectedStatus](https://playwright.dev/docs/api/class-testcase#test-case-expected-status). - */ - status: TestStatus; - /** - * The list of files or buffers attached during the test execution through - * [testInfo.attachments](https://playwright.dev/docs/api/class-testinfo#test-info-attachments). - */ - attachments: Array<{ - /** - * Attachment name. - */ - name: string; - - /** - * Content type of this attachment to properly present in the report, for example `'application/json'` or - * `'image/png'`. - */ - contentType: string; - - /** - * Optional path on the filesystem to the attached file. - */ - path?: string; - - /** - * Optional attachment body used instead of a file. - */ - body?: Buffer; - }>; - - /** - * Running time in milliseconds. - */ - duration: number; - - /** - * First error thrown during test execution, if any. This is equal to the first element in - * [testResult.errors](https://playwright.dev/docs/api/class-testresult#test-result-errors). - */ - error?: TestError; - - /** - * Errors thrown during the test execution. - */ - errors: Array; - - /** - * The index of the worker between `0` and `workers - 1`. It is guaranteed that workers running at the same time have - * a different `parallelIndex`. - */ - parallelIndex: number; - - /** - * When test is retries multiple times, each retry attempt is given a sequential number. - * - * Learn more about [test retries](https://playwright.dev/docs/test-retries#retries). - */ - retry: number; - - /** - * Start time of this particular test run. - */ - startTime: Date; - - /** - * Anything written to the standard error during the test run. - */ - stderr: Array; - - /** - * Anything written to the standard output during the test run. - */ - stdout: Array; - - /** - * List of steps inside this test run. - */ - steps: Array; - - /** - * Index of the worker where the test was run. If the test was not run a single time, for example when the user - * interrupted testing, the only result will have a `workerIndex` equal to `-1`. - * - * Learn more about [parallelism and sharding](https://playwright.dev/docs/test-parallel) with Playwright Test. - */ - workerIndex: number; -} - /** * Result of the full test run. */ @@ -866,6 +548,233 @@ export interface Location { line: number; } +/** + * `Suite` is a group of tests. All tests in Playwright Test form the following hierarchy: + * - Root suite has a child suite for each {@link FullProject}. + * - Project suite #1. Has a child suite for each test file in the project. + * - File suite #1 + * - {@link TestCase} #1 + * - {@link TestCase} #2 + * - Suite corresponding to a + * [test.describe([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe) + * group + * - {@link TestCase} #1 in a group + * - {@link TestCase} #2 in a group + * - < more test cases ... > + * - File suite #2 + * - < more file suites ... > + * - Project suite #2 + * - < more project suites ... > + * + * Reporter is given a root suite in the + * [reporter.onBegin(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-on-begin) method. + */ +export interface Suite { + /** + * Returns the list of all test cases in this suite and its descendants, as opposite to + * [suite.tests](https://playwright.dev/docs/api/class-suite#suite-tests). + */ + allTests(): Array; + + /** + * Test cases and suites defined directly in this suite. The elements are returned in their declaration order. You can + * discriminate between different entry types using + * [testCase.type](https://playwright.dev/docs/api/class-testcase#test-case-type) and + * [suite.type](https://playwright.dev/docs/api/class-suite#suite-type). + */ + entries(): Array; + + /** + * Configuration of the project this suite belongs to, or [void] for the root suite. + */ + project(): FullProject|undefined; + + /** + * Returns a list of titles from the root down to this suite. + */ + titlePath(): Array; + + /** + * Location in the source where the suite is defined. Missing for root and project suites. + */ + location?: Location; + + /** + * Parent suite, missing for the root suite. + */ + parent?: Suite; + + /** + * Child suites. See {@link Suite} for the hierarchy of suites. + */ + suites: Array; + + /** + * Test cases in the suite. Note that only test cases defined directly in this suite are in the list. Any test cases + * defined in nested + * [test.describe([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe) groups are + * listed in the child [suite.suites](https://playwright.dev/docs/api/class-suite#suite-suites). + */ + tests: Array; + + /** + * Suite title. + * - Empty for root suite. + * - Project name for project suite. + * - File path for file suite. + * - Title passed to + * [test.describe([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe) for a + * group suite. + */ + title: string; + + /** + * Returns the type of the suite. The Suites form the following hierarchy: `root` -> `project` -> `file` -> `describe` + * -> ...`describe` -> `test`. + */ + type: "root"|"project"|"file"|"describe"; +} + +/** + * `TestCase` corresponds to every + * [test.(call)(title[, details, body])](https://playwright.dev/docs/api/class-test#test-call) call in a test file. + * When a single [test.(call)(title[, details, body])](https://playwright.dev/docs/api/class-test#test-call) is + * running in multiple projects or repeated multiple times, it will have multiple `TestCase` objects in corresponding + * projects' suites. + */ +export interface TestCase { + /** + * Whether the test is considered running fine. Non-ok tests fail the test run with non-zero exit code. + */ + ok(): boolean; + + /** + * Testing outcome for this test. Note that outcome is not the same as + * [testResult.status](https://playwright.dev/docs/api/class-testresult#test-result-status): + * - Test that is expected to fail and actually fails is `'expected'`. + * - Test that passes on a second retry is `'flaky'`. + */ + outcome(): "skipped"|"expected"|"unexpected"|"flaky"; + + /** + * Returns a list of titles from the root down to this test. + */ + titlePath(): Array; + + /** + * The list of annotations applicable to the current test. Includes: + * - annotations defined on the test or suite via + * [test.(call)(title[, details, body])](https://playwright.dev/docs/api/class-test#test-call) and + * [test.describe([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe); + * - annotations implicitly added by methods + * [test.skip([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-skip), + * [test.fixme([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fixme) + * and + * [test.fail([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fail); + * - annotations appended to + * [testInfo.annotations](https://playwright.dev/docs/api/class-testinfo#test-info-annotations) during the test + * execution. + * + * Annotations are available during test execution through + * [testInfo.annotations](https://playwright.dev/docs/api/class-testinfo#test-info-annotations). + * + * Learn more about [test annotations](https://playwright.dev/docs/test-annotations). + */ + annotations: Array<{ + /** + * Annotation type, for example `'skip'` or `'fail'`. + */ + type: string; + + /** + * Optional description. + */ + description?: string; + }>; + + /** + * Expected test status. + * - Tests marked as + * [test.skip([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-skip) + * or + * [test.fixme([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fixme) + * are expected to be `'skipped'`. + * - Tests marked as + * [test.fail([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fail) + * are expected to be `'failed'`. + * - Other tests are expected to be `'passed'`. + * + * See also [testResult.status](https://playwright.dev/docs/api/class-testresult#test-result-status) for the actual + * status. + */ + expectedStatus: "passed"|"failed"|"timedOut"|"skipped"|"interrupted"; + + /** + * Unique test ID that is computed based on the test file name, test title and project name. Test ID can be used as a + * history ID. + */ + id: string; + + /** + * Location in the source where the test is defined. + */ + location: Location; + + /** + * Suite this test case belongs to. + */ + parent: Suite; + + /** + * Contains the repeat index when running in "repeat each" mode. This mode is enabled by passing `--repeat-each` to + * the [command line](https://playwright.dev/docs/test-cli). + */ + repeatEachIndex: number; + + /** + * Results for each run of this test. + */ + results: Array; + + /** + * The maximum number of retries given to this test in the configuration. + * + * Learn more about [test retries](https://playwright.dev/docs/test-retries#retries). + */ + retries: number; + + /** + * The list of tags defined on the test or suite via + * [test.(call)(title[, details, body])](https://playwright.dev/docs/api/class-test#test-call) or + * [test.describe([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe), as well as + * `@`-tokens extracted from test and suite titles. + * + * Learn more about [test tags](https://playwright.dev/docs/test-annotations#tag-tests). + */ + tags: Array; + + /** + * The timeout given to the test. Affected by + * [testConfig.timeout](https://playwright.dev/docs/api/class-testconfig#test-config-timeout), + * [testProject.timeout](https://playwright.dev/docs/api/class-testproject#test-project-timeout), + * [test.setTimeout(timeout)](https://playwright.dev/docs/api/class-test#test-set-timeout), + * [test.slow([condition, callback, description])](https://playwright.dev/docs/api/class-test#test-slow) and + * [testInfo.setTimeout(timeout)](https://playwright.dev/docs/api/class-testinfo#test-info-set-timeout). + */ + timeout: number; + + /** + * Test title as passed to the + * [test.(call)(title[, details, body])](https://playwright.dev/docs/api/class-test#test-call) call. + */ + title: string; + + /** + * Returns type of the test. + */ + type: "test"; +} + /** * Information about an error thrown during test execution. */ @@ -896,6 +805,101 @@ export interface TestError { value?: string; } +/** + * A result of a single {@link TestCase} run. + */ +export interface TestResult { + /** + * The list of files or buffers attached during the test execution through + * [testInfo.attachments](https://playwright.dev/docs/api/class-testinfo#test-info-attachments). + */ + attachments: Array<{ + /** + * Attachment name. + */ + name: string; + + /** + * Content type of this attachment to properly present in the report, for example `'application/json'` or + * `'image/png'`. + */ + contentType: string; + + /** + * Optional path on the filesystem to the attached file. + */ + path?: string; + + /** + * Optional attachment body used instead of a file. + */ + body?: Buffer; + }>; + + /** + * Running time in milliseconds. + */ + duration: number; + + /** + * First error thrown during test execution, if any. This is equal to the first element in + * [testResult.errors](https://playwright.dev/docs/api/class-testresult#test-result-errors). + */ + error?: TestError; + + /** + * Errors thrown during the test execution. + */ + errors: Array; + + /** + * The index of the worker between `0` and `workers - 1`. It is guaranteed that workers running at the same time have + * a different `parallelIndex`. + */ + parallelIndex: number; + + /** + * When test is retries multiple times, each retry attempt is given a sequential number. + * + * Learn more about [test retries](https://playwright.dev/docs/test-retries#retries). + */ + retry: number; + + /** + * Start time of this particular test run. + */ + startTime: Date; + + /** + * The status of this test result. See also + * [testCase.expectedStatus](https://playwright.dev/docs/api/class-testcase#test-case-expected-status). + */ + status: "passed"|"failed"|"timedOut"|"skipped"|"interrupted"; + + /** + * Anything written to the standard error during the test run. + */ + stderr: Array; + + /** + * Anything written to the standard output during the test run. + */ + stdout: Array; + + /** + * List of steps inside this test run. + */ + steps: Array; + + /** + * Index of the worker where the test was run. If the test was not run a single time, for example when the user + * interrupted testing, the only result will have a `workerIndex` equal to `-1`. + * + * Learn more about [parallelism and sharding](https://playwright.dev/docs/test-parallel) with Playwright Test. + */ + workerIndex: number; +} + /** * Represents a step in the [TestRun]. */ diff --git a/utils/generate_types/overrides-testReporter.d.ts b/utils/generate_types/overrides-testReporter.d.ts index 129faf30fc..e5bf37cdc8 100644 --- a/utils/generate_types/overrides-testReporter.d.ts +++ b/utils/generate_types/overrides-testReporter.d.ts @@ -29,19 +29,6 @@ export interface FullProject { use: UseOptions; } -export interface Suite { - type: 'root' | 'project' | 'file' | 'describe'; -} - -export interface TestCase { - type: 'test'; - expectedStatus: TestStatus; -} - -export interface TestResult { - status: TestStatus; -} - /** * Result of the full test run. */