From 167db03f051dc7591b2566e65ef6cbbd190b6a85 Mon Sep 17 00:00:00 2001 From: Dmitry Gozman Date: Fri, 16 Jul 2021 21:15:03 -0700 Subject: [PATCH] feat(test-runner): export reporter api as `@playwright/test/reporter` (#7692) --- packages/common/.npmignore | 3 + .../installation-tests/installation-tests.sh | 6 + .../playwright-test-types.ts | 57 ++++ packages/playwright-test/reporter.d.ts | 17 ++ packages/playwright-test/reporter.js | 17 ++ packages/playwright-test/reporter.mjs | 17 ++ src/test/dispatcher.ts | 2 +- src/test/ipc.ts | 2 +- src/test/loader.ts | 2 +- src/test/reporter.ts | 66 ----- src/test/reporters/base.ts | 2 +- src/test/reporters/dot.ts | 2 +- src/test/reporters/empty.ts | 2 +- src/test/reporters/json.ts | 2 +- src/test/reporters/junit.ts | 2 +- src/test/reporters/line.ts | 2 +- src/test/reporters/list.ts | 2 +- src/test/reporters/multiplexer.ts | 2 +- src/test/runner.ts | 2 +- src/test/test.ts | 2 +- src/test/types.ts | 4 +- types/testReporter.d.ts | 259 ++++++++++++++++++ 22 files changed, 391 insertions(+), 81 deletions(-) create mode 100644 packages/installation-tests/playwright-test-types.ts create mode 100644 packages/playwright-test/reporter.d.ts create mode 100644 packages/playwright-test/reporter.js create mode 100644 packages/playwright-test/reporter.mjs delete mode 100644 src/test/reporter.ts create mode 100644 types/testReporter.d.ts diff --git a/packages/common/.npmignore b/packages/common/.npmignore index ef19e0048e..d1993fb2a6 100644 --- a/packages/common/.npmignore +++ b/packages/common/.npmignore @@ -22,10 +22,13 @@ lib/**/injected/ # Include generated types and entrypoint. !types/* !index.d.ts +!reporter.d.ts # Include main entrypoint. !index.js +!reporter.js # Include main entrypoint for ES Modules. !index.mjs +!reporter.mjs # Include installer. !install.js # Include essentials. diff --git a/packages/installation-tests/installation-tests.sh b/packages/installation-tests/installation-tests.sh index f689cc9ec4..86bab8468a 100755 --- a/packages/installation-tests/installation-tests.sh +++ b/packages/installation-tests/installation-tests.sh @@ -53,6 +53,7 @@ function copy_test_scripts { cp "${SCRIPTS_PATH}/driver-client.js" . cp "${SCRIPTS_PATH}/sample.spec.js" . cp "${SCRIPTS_PATH}/read-json-report.js" . + cp "${SCRIPTS_PATH}/playwright-test-types.ts" . } function run_tests { @@ -107,6 +108,7 @@ function test_screencast { function test_typescript_types { initialize_test "${FUNCNAME[0]}" + copy_test_scripts # install all packages. PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_CORE_TGZ} @@ -114,6 +116,7 @@ function test_typescript_types { PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_FIREFOX_TGZ} PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_WEBKIT_TGZ} PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_CHROMIUM_TGZ} + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_TEST_TGZ} # typecheck all packages. for PKG_NAME in "playwright" \ @@ -126,6 +129,9 @@ function test_typescript_types { echo "import { Page } from '${PKG_NAME}';" > "${PKG_NAME}.ts" && npx -p typescript@3.7.5 tsc "${PKG_NAME}.ts" done; + echo "Checking types of @playwright/test" + echo npx -p typescript@3.7.5 tsc "playwright-test-types.ts" + echo "${FUNCNAME[0]} success" } diff --git a/packages/installation-tests/playwright-test-types.ts b/packages/installation-tests/playwright-test-types.ts new file mode 100644 index 0000000000..66e26ed39d --- /dev/null +++ b/packages/installation-tests/playwright-test-types.ts @@ -0,0 +1,57 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test, expect } from '@playwright/test'; +import { Reporter, Test } from '@playwright/test/reporter'; + +test.use({ locale: 'en-US' }); + +test.describe('block', () => { + test.beforeAll(async ({ browser }) => { + }); + test.afterAll(async ({ browser }) => { + }); + test.beforeEach(async ({ page }) => { + }); + test.afterEach(async ({ page }) => { + }); + test('should work', async ({ page, browserName }, testInfo) => { + test.skip(browserName === 'chromium'); + await page.click(testInfo.title); + testInfo.annotations.push({ type: 'foo' }); + await page.fill(testInfo.outputPath('foo', 'bar'), testInfo.outputDir); + }); +}); + +const test2 = test.extend<{ foo: string, bar: number }>({ + foo: '123', + bar: async ({ foo }, use) => { + await use(parseInt(foo, 10)); + }, +}); + +test2('should work 2', async ({ foo, bar }) => { + bar += parseInt(foo, 10); + expect(bar).toBe(123 * 2); +}); + +export class MyReporter implements Reporter { + onTestBegin(test: Test) { + test.titlePath().slice(); + if (test.results[0].status === test.expectedStatus) + console.log(`Nice test ${test.title} at ${test.location.file}`); + } +} diff --git a/packages/playwright-test/reporter.d.ts b/packages/playwright-test/reporter.d.ts new file mode 100644 index 0000000000..29a927d48e --- /dev/null +++ b/packages/playwright-test/reporter.d.ts @@ -0,0 +1,17 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './types/testReporter'; diff --git a/packages/playwright-test/reporter.js b/packages/playwright-test/reporter.js new file mode 100644 index 0000000000..485e880a66 --- /dev/null +++ b/packages/playwright-test/reporter.js @@ -0,0 +1,17 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// We only export types in reporter.d.ts. diff --git a/packages/playwright-test/reporter.mjs b/packages/playwright-test/reporter.mjs new file mode 100644 index 0000000000..485e880a66 --- /dev/null +++ b/packages/playwright-test/reporter.mjs @@ -0,0 +1,17 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// We only export types in reporter.d.ts. diff --git a/src/test/dispatcher.ts b/src/test/dispatcher.ts index e7955fd855..038f3195ab 100644 --- a/src/test/dispatcher.ts +++ b/src/test/dispatcher.ts @@ -18,7 +18,7 @@ import child_process from 'child_process'; import path from 'path'; import { EventEmitter } from 'events'; import { RunPayload, TestBeginPayload, TestEndPayload, DonePayload, TestOutputPayload, WorkerInitParams } from './ipc'; -import type { TestResult, Reporter, TestStatus } from './reporter'; +import type { TestResult, Reporter, TestStatus } from '../../types/testReporter'; import { Suite, Test } from './test'; import { Loader } from './loader'; diff --git a/src/test/ipc.ts b/src/test/ipc.ts index e02c3f1ed9..14304ade2d 100644 --- a/src/test/ipc.ts +++ b/src/test/ipc.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import type { TestError } from './reporter'; +import type { TestError } from '../../types/testReporter'; import type { Config, TestStatus } from './types'; export type SerializedLoaderData = { diff --git a/src/test/loader.ts b/src/test/loader.ts index f1d34829cd..6f0ca9f6e6 100644 --- a/src/test/loader.ts +++ b/src/test/loader.ts @@ -23,7 +23,7 @@ import { SerializedLoaderData } from './ipc'; import * as path from 'path'; import * as url from 'url'; import { ProjectImpl } from './project'; -import { Reporter } from './reporter'; +import { Reporter } from '../../types/testReporter'; import { LaunchConfig } from '../../types/test'; export class Loader { diff --git a/src/test/reporter.ts b/src/test/reporter.ts deleted file mode 100644 index 19728cf14e..0000000000 --- a/src/test/reporter.ts +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Copyright Microsoft Corporation. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import type { FullConfig, TestStatus, TestError } from './types'; -export type { FullConfig, TestStatus, TestError } from './types'; - -export interface Location { - file: string; - line: number; - column: number; -} -export interface Suite { - title: string; - location: Location; - suites: Suite[]; - tests: Test[]; - titlePath(): string[]; - allTests(): Test[]; -} -export interface Test { - title: string; - location: Location; - results: TestResult[]; - expectedStatus: TestStatus; - timeout: number; - annotations: { type: string, description?: string }[]; - retries: number; - titlePath(): string[]; - status(): 'skipped' | 'expected' | 'unexpected' | 'flaky'; - ok(): boolean; -} -export interface TestResult { - retry: number; - workerIndex: number, - duration: number; - status?: TestStatus; - error?: TestError; - attachments: { name: string, path?: string, body?: Buffer, contentType: string }[]; - stdout: (string | Buffer)[]; - stderr: (string | Buffer)[]; -} -export interface FullResult { - status: 'passed' | 'failed' | 'timedout' | 'interrupted'; -} -export interface Reporter { - onBegin?(config: FullConfig, suite: Suite): void; - onTestBegin?(test: Test): void; - onStdOut?(chunk: string | Buffer, test?: Test): void; - onStdErr?(chunk: string | Buffer, test?: Test): void; - onTestEnd?(test: Test, result: TestResult): void; - onError?(error: TestError): void; - onEnd?(result: FullResult): void | Promise; -} diff --git a/src/test/reporters/base.ts b/src/test/reporters/base.ts index c3cae17f1d..27fc004d6e 100644 --- a/src/test/reporters/base.ts +++ b/src/test/reporters/base.ts @@ -21,7 +21,7 @@ import fs from 'fs'; import milliseconds from 'ms'; import path from 'path'; import StackUtils from 'stack-utils'; -import { FullConfig, TestStatus, Test, Suite, TestResult, TestError, Reporter, FullResult } from '../reporter'; +import { FullConfig, TestStatus, Test, Suite, TestResult, TestError, Reporter, FullResult } from '../../../types/testReporter'; const stackUtils = new StackUtils(); diff --git a/src/test/reporters/dot.ts b/src/test/reporters/dot.ts index b17e1f2b9c..d98b3baf85 100644 --- a/src/test/reporters/dot.ts +++ b/src/test/reporters/dot.ts @@ -16,7 +16,7 @@ import colors from 'colors/safe'; import { BaseReporter } from './base'; -import { FullResult, Test, TestResult } from '../reporter'; +import { FullResult, Test, TestResult } from '../../../types/testReporter'; class DotReporter extends BaseReporter { private _counter = 0; diff --git a/src/test/reporters/empty.ts b/src/test/reporters/empty.ts index 382d1de43d..7dc691080f 100644 --- a/src/test/reporters/empty.ts +++ b/src/test/reporters/empty.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Reporter } from '../reporter'; +import { Reporter } from '../../../types/testReporter'; class EmptyReporter implements Reporter { } diff --git a/src/test/reporters/json.ts b/src/test/reporters/json.ts index f1e7b0662c..1f67f9d8d7 100644 --- a/src/test/reporters/json.ts +++ b/src/test/reporters/json.ts @@ -16,7 +16,7 @@ import fs from 'fs'; import path from 'path'; -import { FullConfig, Test, Suite, TestResult, TestError, FullResult, TestStatus, Location, Reporter } from '../reporter'; +import { FullConfig, Test, Suite, TestResult, TestError, FullResult, TestStatus, Location, Reporter } from '../../../types/testReporter'; export interface JSONReport { config: Omit & { diff --git a/src/test/reporters/junit.ts b/src/test/reporters/junit.ts index b341a519c0..0e2eea7e95 100644 --- a/src/test/reporters/junit.ts +++ b/src/test/reporters/junit.ts @@ -16,7 +16,7 @@ import fs from 'fs'; import path from 'path'; -import { FullConfig, FullResult, Reporter, Suite, Test } from '../reporter'; +import { FullConfig, FullResult, Reporter, Suite, Test } from '../../../types/testReporter'; import { monotonicTime } from '../util'; import { formatFailure, formatTestTitle, stripAscii } from './base'; diff --git a/src/test/reporters/line.ts b/src/test/reporters/line.ts index d6eef00559..02a27f9249 100644 --- a/src/test/reporters/line.ts +++ b/src/test/reporters/line.ts @@ -16,7 +16,7 @@ import colors from 'colors/safe'; import { BaseReporter, formatFailure, formatTestTitle } from './base'; -import { FullConfig, Test, Suite, TestResult, FullResult } from '../reporter'; +import { FullConfig, Test, Suite, TestResult, FullResult } from '../../../types/testReporter'; class LineReporter extends BaseReporter { private _total = 0; diff --git a/src/test/reporters/list.ts b/src/test/reporters/list.ts index f7437c4ace..5fdb7b83ee 100644 --- a/src/test/reporters/list.ts +++ b/src/test/reporters/list.ts @@ -19,7 +19,7 @@ import colors from 'colors/safe'; // @ts-ignore import milliseconds from 'ms'; import { BaseReporter, formatTestTitle } from './base'; -import { FullConfig, FullResult, Suite, Test, TestResult } from '../reporter'; +import { FullConfig, FullResult, Suite, Test, TestResult } from '../../../types/testReporter'; // Allow it in the Visual Studio Code Terminal and the new Windows Terminal const DOES_NOT_SUPPORT_UTF8_IN_TERMINAL = process.platform === 'win32' && process.env.TERM_PROGRAM !== 'vscode' && !process.env.WT_SESSION; diff --git a/src/test/reporters/multiplexer.ts b/src/test/reporters/multiplexer.ts index 3355260e93..9cbbfe53a7 100644 --- a/src/test/reporters/multiplexer.ts +++ b/src/test/reporters/multiplexer.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { FullConfig, Suite, Test, TestError, TestResult, Reporter, FullResult } from '../reporter'; +import { FullConfig, Suite, Test, TestError, TestResult, Reporter, FullResult } from '../../../types/testReporter'; export class Multiplexer implements Reporter { private _reporters: Reporter[]; diff --git a/src/test/runner.ts b/src/test/runner.ts index fbccc47cba..cb895699ff 100644 --- a/src/test/runner.ts +++ b/src/test/runner.ts @@ -24,7 +24,7 @@ import { Dispatcher } from './dispatcher'; import { createMatcher, FilePatternFilter, monotonicTime, raceAgainstDeadline } from './util'; import { Test, Suite } from './test'; import { Loader } from './loader'; -import { Reporter } from './reporter'; +import { Reporter } from '../../types/testReporter'; import { Multiplexer } from './reporters/multiplexer'; import DotReporter from './reporters/dot'; import LineReporter from './reporters/line'; diff --git a/src/test/test.ts b/src/test/test.ts index 692348020c..e3248bde1a 100644 --- a/src/test/test.ts +++ b/src/test/test.ts @@ -15,7 +15,7 @@ */ import type { FixturePool } from './fixtures'; -import * as reporterTypes from './reporter'; +import * as reporterTypes from '../../types/testReporter'; import type { TestTypeImpl } from './testType'; import { Annotations, Location } from './types'; diff --git a/src/test/types.ts b/src/test/types.ts index d9bf3b2898..c751003c16 100644 --- a/src/test/types.ts +++ b/src/test/types.ts @@ -15,9 +15,9 @@ */ import type { Fixtures } from '../../types/test'; -import type { Location } from './reporter'; +import type { Location } from '../../types/testReporter'; export * from '../../types/test'; -export { Location } from './reporter'; +export { Location } from '../../types/testReporter'; export type FixturesWithLocation = { fixtures: Fixtures; diff --git a/types/testReporter.d.ts b/types/testReporter.d.ts new file mode 100644 index 0000000000..64f2e3b0b8 --- /dev/null +++ b/types/testReporter.d.ts @@ -0,0 +1,259 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { FullConfig, TestStatus, TestError } from './test'; +export type { FullConfig, TestStatus, TestError } from './test'; + +/** + * Test or Suite location where it was defined. + */ +export interface Location { + /** + * Path to the file. + */ + file: string; + + /** + * Line number in the file. + */ + line: number; + + /** + * Column number in the file. + */ + column: number; +} + +/** + * A group of tests. All tests are reported in the following hierarchy: + * - Root suite + * - Project suite #1 (for each project) + * - File suite #1 (for each file in the project) + * - Suites for any describe() calls + * - Test #1 defined in the file or describe() group + * - Test #2 + * ... < more tests > + * - File suite #2 + * ... < more file suites > + * - Second project suite + * ... < more project suites > + */ +export interface Suite { + /** + * Suite title: + * - Empty for root suite. + * - Project name for project suite. + * - File path for file suite. + * - Title passed to describe() for describe suites + */ + title: string; + + /** + * Location where the suite is defined. + */ + location: Location; + + /** + * Child suites. + */ + suites: Suite[]; + + /** + * Tests in the suite. Note that only tests defined directly in this suite + * are in the list. Any tests defined in nested describe() groups are listed + * in the child `suites`. + */ + tests: Test[]; + + /** + * A list of titles from the root down to this suite. + */ + titlePath(): string[]; + + /** + * Returns the list of all tests in this suite and its descendants. + */ + allTests(): Test[]; +} + +/** + * A test, corresponds to test() call in a test file. When a single test() is + * running in multiple projects (or repeated multiple times), it will have multiple + * `Test` objects in corresponding projects' suites. + */ +export interface Test { + /** + * Test title as passed to the test() call. + */ + title: string; + + /** + * Location where the test is defined. + */ + location: Location; + + /** + * A list of titles from the root down to this test. + */ + titlePath(): string[]; + + /** + * Expected status. + * - Tests marked as test.skip() or test.fixme() are expected to be 'skipped'. + * - Tests marked as test.fail() are expected to be 'failed'. + * - Other tests are expected to be 'passed'. + */ + expectedStatus: TestStatus; + + /** + * The timeout given to the test. Affected by timeout in the configuration file, + * and calls to test.setTimeout() or test.slow(). + */ + timeout: number; + + /** + * Annotations collected for this test. For example, calling + * `test.skip(true, 'just because')` will produce an annotation + * `{ type: 'skip', description: 'just because' }`. + */ + annotations: { type: string, description?: string }[]; + + /** + * The maxmium number of retries given to this test in the configuration. + */ + retries: number; + + /** + * Results for each run of this test. + */ + results: TestResult[]; + + /** + * Overall test status. + */ + status(): 'skipped' | 'expected' | 'unexpected' | 'flaky'; + + /** + * Whether the test is considered running fine. + * Non-ok tests fail the test run with non-zero exit code. + */ + ok(): boolean; +} + +/** + * A result of a single test run. + */ +export interface TestResult { + /** + * When test is retries multiple times, each retry attempt is given a sequential number. + */ + retry: number; + + /** + * Index of the worker where the test was run. + */ + workerIndex: number, + + /** + * Running time in milliseconds. + */ + duration: number; + + /** + * The status of this test result. + */ + status?: TestStatus; + + /** + * An error from this test result, if any. + */ + error?: TestError; + + /** + * Any attachments created during the test run. + */ + attachments: { name: string, path?: string, body?: Buffer, contentType: string }[]; + + /** + * Anything written to the standard output during the test run. + */ + stdout: (string | Buffer)[]; + + /** + * Anything written to the standard error during the test run. + */ + stderr: (string | Buffer)[]; +} + +/** + * Result of the full test run. + */ +export interface FullResult { + /** + * Status: + * - 'passed' - everything went as expected. + * - 'failed' - any test has failed. + * - 'timedout' - the global time has been reached. + * - 'interrupted' - interrupted by the user. + */ + status: 'passed' | 'failed' | 'timedout' | 'interrupted'; +} + +/** + * Test runner notifies reporter about various events during the test run. + */ +export interface Reporter { + /** + * Called once before running tests. + * All tests have been already discovered and put into a hierarchy, see `Suite` description. + */ + onBegin?(config: FullConfig, suite: Suite): void; + + /** + * Called after a test has been started in the worker process. + */ + onTestBegin?(test: Test): void; + + /** + * Called when something has been written to the standard output in the worker process. + * When `test` is given, output happened while the test was running. + */ + onStdOut?(chunk: string | Buffer, test?: Test): void; + + /** + * Called when something has been written to the standard error in the worker process. + * When `test` is given, output happened while the test was running. + */ + onStdErr?(chunk: string | Buffer, test?: Test): void; + + /** + * Called after a test has been finished in the worker process. + */ + onTestEnd?(test: Test, result: TestResult): void; + + /** + * Called on some global error, for example unhandled expection in the worker process. + */ + onError?(error: TestError): void; + + /** + * Called after all tests has been run, or when testing has been interrupted. + */ + onEnd?(result: FullResult): void | Promise; +} + +// This is required to not export everything by default. See https://github.com/Microsoft/TypeScript/issues/19545#issuecomment-340490459 +export {};