diff --git a/docs/src/test-api/class-test.md b/docs/src/test-api/class-test.md index e0a5c4eac3..6e999bb225 100644 --- a/docs/src/test-api/class-test.md +++ b/docs/src/test-api/class-test.md @@ -709,6 +709,41 @@ Optional condition - either a boolean value, or a function that takes a fixtures Optional description that will be reflected in a test report. +## method: Test.step + +Declares a test step. + +```js js-flavor=js +const { test, expect } = require('@playwright/test'); + +test('test', async ({ page }) => { + await test.step('Log in', async () => { + // ... + }); +}); +``` + +```js js-flavor=ts +import { test, expect } from '@playwright/test'; + +test('test', async ({ page }) => { + await test.step('Log in', async () => { + // ... + }); +}); +``` + +### param: Test.step.title +- `title` <[string]> + +Step name. + + +### param: Test.step.body +- `body` <[function]\(\):[Promise]<[any]>> + +Step body. + ## method: Test.use diff --git a/src/test/expect.ts b/src/test/expect.ts index 166ea75459..7215acff14 100644 --- a/src/test/expect.ts +++ b/src/test/expect.ts @@ -39,9 +39,10 @@ import { toHaveValue } from './matchers/matchers'; import { toMatchSnapshot } from './matchers/toMatchSnapshot'; -import type { Expect, TestStatus } from './types'; +import type { Expect, TestError } from './types'; import matchers from 'expect/build/matchers'; import { currentTestInfo } from './globals'; +import { serializeError } from './util'; export const expect: Expect = expectLibrary as any; expectLibrary.setState({ expand: false }); @@ -78,22 +79,22 @@ function wrap(matcherName: string, matcher: any) { const infix = this.isNot ? '.not' : ''; const completeStep = testInfo._addStep('expect', `expect${infix}.${matcherName}`); + const stack = new Error().stack; const reportStepEnd = (result: any) => { - status = result.pass !== this.isNot ? 'passed' : 'failed'; - let error: Error | undefined; - if (status === 'failed') - error = new Error(result.message()); + const success = result.pass !== this.isNot; + let error: TestError | undefined; + if (!success) + error = { message: result.message(), stack }; completeStep?.(error); return result; }; const reportStepError = (error: Error) => { - completeStep?.(error); + completeStep?.(serializeError(error)); throw error; }; - let status: TestStatus = 'passed'; try { const result = matcher.call(this, ...args); if (result instanceof Promise) diff --git a/src/test/testType.ts b/src/test/testType.ts index 677929d73b..89c451ed37 100644 --- a/src/test/testType.ts +++ b/src/test/testType.ts @@ -19,7 +19,7 @@ import { currentlyLoadingFileSuite, currentTestInfo, setCurrentlyLoadingFileSuit import { TestCase, Suite } from './test'; import { wrapFunctionWithLocation } from './transform'; import { Fixtures, FixturesWithLocation, Location, TestType } from './types'; -import { errorWithLocation } from './util'; +import { errorWithLocation, serializeError } from './util'; const countByFile = new Map(); @@ -49,6 +49,7 @@ export class TestTypeImpl { test.fail = wrapFunctionWithLocation(this._modifier.bind(this, 'fail')); test.slow = wrapFunctionWithLocation(this._modifier.bind(this, 'slow')); test.setTimeout = wrapFunctionWithLocation(this._setTimeout.bind(this)); + test.step = wrapFunctionWithLocation(this._step.bind(this)); test.use = wrapFunctionWithLocation(this._use.bind(this)); test.extend = wrapFunctionWithLocation(this._extend.bind(this)); test.declare = wrapFunctionWithLocation(this._declare.bind(this)); @@ -146,7 +147,7 @@ export class TestTypeImpl { const testInfo = currentTestInfo(); if (!testInfo) - throw errorWithLocation(location, `test.setTimeout() can only be called from a test file`); + throw errorWithLocation(location, `test.setTimeout() can only be called from a test`); testInfo.setTimeout(timeout); } @@ -157,6 +158,20 @@ export class TestTypeImpl { suite._fixtureOverrides = { ...suite._fixtureOverrides, ...fixtures }; } + private async _step(location: Location, title: string, body: () => Promise): Promise { + const testInfo = currentTestInfo(); + if (!testInfo) + throw errorWithLocation(location, `test.step() can only be called from a test`); + const complete = testInfo._addStep('test.step', title); + try { + await body(); + complete(); + } catch (e) { + complete(serializeError(e)); + throw e; + } + } + private _extend(location: Location, fixtures: Fixtures) { const fixturesWithLocation = { fixtures, location }; return new TestTypeImpl([...this.fixtures, fixturesWithLocation]).test; diff --git a/src/test/types.ts b/src/test/types.ts index 0a123538f4..aade551240 100644 --- a/src/test/types.ts +++ b/src/test/types.ts @@ -25,7 +25,7 @@ export type FixturesWithLocation = { }; export type Annotations = { type: string, description?: string }[]; -export type CompleteStepCallback = (error?: TestError) => void; +export type CompleteStepCallback = (error?: Error | TestError) => void; export interface TestInfoImpl extends TestInfo { _testFinished: Promise; diff --git a/src/test/workerRunner.ts b/src/test/workerRunner.ts index 575025a07a..708b0b0431 100644 --- a/src/test/workerRunner.ts +++ b/src/test/workerRunner.ts @@ -278,7 +278,9 @@ export class WorkerRunner extends EventEmitter { wallTime: Date.now() }; this.emit('stepBegin', payload); - return (error?: TestError) => { + return (error?: Error | TestError) => { + if (error instanceof Error) + error = serializeError(error); const payload: StepEndPayload = { testId, stepId, diff --git a/tests/playwright-test/reporter.spec.ts b/tests/playwright-test/reporter.spec.ts index 0883b8fef0..de39471025 100644 --- a/tests/playwright-test/reporter.spec.ts +++ b/tests/playwright-test/reporter.spec.ts @@ -168,6 +168,8 @@ test('should report expect steps', async ({ runInlineTest }) => { } onStepEnd(test, result, step) { const copy = { ...step, startTime: undefined, duration: undefined }; + if (copy.error?.stack) + copy.error.stack = ''; console.log('%%%% end', JSON.stringify(copy)); } } @@ -197,13 +199,13 @@ test('should report expect steps', async ({ runInlineTest }) => { }, { reporter: '', workers: 1 }); expect(result.exitCode).toBe(1); - expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([ + expect(result.output.split('\n').filter(line => line.startsWith('%%')).map(stripEscapedAscii)).toEqual([ `%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, `%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, `%% begin {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\"}`, `%% end {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\"}`, `%% begin {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\"}`, - `%% end {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\",\"error\":{}}`, + `%% end {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\",\"error\":{\"message\":\"expect(received).toBeTruthy()\\n\\nReceived: false\",\"stack\":\"\"}}`, `%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, `%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`, `%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, @@ -236,6 +238,8 @@ test('should report api steps', async ({ runInlineTest }) => { } onStepEnd(test, result, step) { const copy = { ...step, startTime: undefined, duration: undefined }; + if (copy.error?.stack) + copy.error.stack = ''; console.log('%%%% end', JSON.stringify(copy)); } } @@ -259,7 +263,7 @@ test('should report api steps', async ({ runInlineTest }) => { }, { reporter: '', workers: 1 }); expect(result.exitCode).toBe(0); - expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([ + expect(result.output.split('\n').filter(line => line.startsWith('%%')).map(stripEscapedAscii)).toEqual([ `%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, `%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, `%% begin {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`, @@ -274,3 +278,109 @@ test('should report api steps', async ({ runInlineTest }) => { `%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`, ]); }); + + +test('should report api step failure', async ({ runInlineTest }) => { + const expectReporterJS = ` + class Reporter { + onStepBegin(test, result, step) { + const copy = { ...step, startTime: undefined, duration: undefined }; + console.log('%%%% begin', JSON.stringify(copy)); + } + onStepEnd(test, result, step) { + const copy = { ...step, startTime: undefined, duration: undefined }; + if (copy.error?.stack) + copy.error.stack = ''; + console.log('%%%% end', JSON.stringify(copy)); + } + } + module.exports = Reporter; + `; + + const result = await runInlineTest({ + 'reporter.ts': expectReporterJS, + 'playwright.config.ts': ` + module.exports = { + reporter: './reporter', + }; + `, + 'a.test.ts': ` + const { test } = pwt; + test('fail', async ({ page }) => { + await page.setContent(''); + await page.click('input', { timeout: 1 }); + }); + ` + }, { reporter: '', workers: 1 }); + + expect(result.exitCode).toBe(1); + expect(result.output.split('\n').filter(line => line.startsWith('%%')).map(stripEscapedAscii)).toEqual([ + `%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, + `%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, + `%% begin {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`, + `%% end {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`, + `%% begin {\"title\":\"page.setContent\",\"category\":\"pw:api\"}`, + `%% end {\"title\":\"page.setContent\",\"category\":\"pw:api\"}`, + `%% begin {\"title\":\"page.click\",\"category\":\"pw:api\"}`, + `%% end {\"title\":\"page.click\",\"category\":\"pw:api\",\"error\":{\"message\":\"page.click: Timeout 1ms exceeded.\\n=========================== logs ===========================\\nwaiting for selector \\\"input\\\"\\n============================================================\",\"stack\":\"\"}}`, + `%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, + `%% begin {\"title\":\"browserContext.close\",\"category\":\"pw:api\"}`, + `%% end {\"title\":\"browserContext.close\",\"category\":\"pw:api\"}`, + `%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`, + ]); +}); + +test('should report test.step', async ({ runInlineTest }) => { + const expectReporterJS = ` + class Reporter { + onStepBegin(test, result, step) { + const copy = { ...step, startTime: undefined, duration: undefined }; + console.log('%%%% begin', JSON.stringify(copy)); + } + onStepEnd(test, result, step) { + const copy = { ...step, startTime: undefined, duration: undefined }; + if (copy.error?.stack) + copy.error.stack = ''; + console.log('%%%% end', JSON.stringify(copy)); + } + } + module.exports = Reporter; + `; + + const result = await runInlineTest({ + 'reporter.ts': expectReporterJS, + 'playwright.config.ts': ` + module.exports = { + reporter: './reporter', + }; + `, + 'a.test.ts': ` + const { test } = pwt; + test('pass', async ({ page }) => { + await test.step('First step', async () => { + expect(1).toBe(2); + }); + }); + ` + }, { reporter: '', workers: 1 }); + + expect(result.exitCode).toBe(1); + expect(result.output.split('\n').filter(line => line.startsWith('%%')).map(stripEscapedAscii)).toEqual([ + `%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, + `%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, + `%% begin {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`, + `%% end {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`, + `%% begin {\"title\":\"First step\",\"category\":\"test.step\"}`, + `%% begin {\"title\":\"expect.toBe\",\"category\":\"expect\"}`, + `%% end {\"title\":\"expect.toBe\",\"category\":\"expect\",\"error\":{\"message\":\"expect(received).toBe(expected) // Object.is equality\\n\\nExpected: 2\\nReceived: 1\",\"stack\":\"\"}}`, + `%% end {\"title\":\"First step\",\"category\":\"test.step\",\"error\":{\"message\":\"expect(received).toBe(expected) // Object.is equality\\n\\nExpected: 2\\nReceived: 1\",\"stack\":\"\"}}`, + `%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, + `%% begin {\"title\":\"browserContext.close\",\"category\":\"pw:api\"}`, + `%% end {\"title\":\"browserContext.close\",\"category\":\"pw:api\"}`, + `%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`, + ]); +}); + +function stripEscapedAscii(str: string) { + return str.replace(/\\u00[a-z0-9][a-z0-9]\[[^m]+m/g, ''); +} diff --git a/types/test.d.ts b/types/test.d.ts index c3c5b96da3..039ff5269f 100644 --- a/types/test.d.ts +++ b/types/test.d.ts @@ -2093,6 +2093,33 @@ export interface TestType): void; + /** + * Declares a test step. + * + * ```js js-flavor=js + * const { test, expect } = require('@playwright/test'); + * + * test('test', async ({ page }) => { + * await test.step('Log in', async () => { + * // ... + * }); + * }); + * ``` + * + * ```js js-flavor=ts + * import { test, expect } from '@playwright/test'; + * + * test('test', async ({ page }) => { + * await test.step('Log in', async () => { + * // ... + * }); + * }); + * ``` + * + * @param title Step name. + * @param body Step body. + */ + step(title: string, body: () => Promise): Promise; /** * `expect` function can be used to create test assertions. Read * [expect library documentation](https://jestjs.io/docs/expect) for more details. diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 2b7a9c843e..3ad94ac2b8 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -246,6 +246,7 @@ export interface TestType Promise | any): void; afterAll(inner: (args: WorkerArgs, workerInfo: WorkerInfo) => Promise | any): void; use(fixtures: Fixtures<{}, {}, TestArgs, WorkerArgs>): void; + step(title: string, body: () => Promise): Promise; expect: Expect; declare(): TestType; extend(fixtures: Fixtures): TestType;