feat(test-runner): introduce test.step (#7956)

docs/src/test-api/class-test.md

@@ -709,6 +709,41 @@ Optional condition - either a boolean value, or a function that takes a fixtures
 Optional description that will be reflected in a test report.
 
 
+## method: Test.step
+
+Declares a test step.
+
+```js js-flavor=js
+const { test, expect } = require('@playwright/test');
+
+test('test', async ({ page }) => {
+  await test.step('Log in', async () => {
+    // ...
+  });
+});
+```
+
+```js js-flavor=ts
+import { test, expect } from '@playwright/test';
+
+test('test', async ({ page }) => {
+  await test.step('Log in', async () => {
+    // ...
+  });
+});
+```
+
+### param: Test.step.title
+- `title` <[string]>
+
+Step name.
+
+
+### param: Test.step.body
+- `body` <[function]\(\):[Promise]<[any]>>
+
+Step body.
+
 
 
 ## method: Test.use

src/test/expect.ts

@@ -39,9 +39,10 @@ import {
   toHaveValue
 } from './matchers/matchers';
 import { toMatchSnapshot } from './matchers/toMatchSnapshot';
-import type { Expect, TestStatus } from './types';
+import type { Expect, TestError } from './types';
 import matchers from 'expect/build/matchers';
 import { currentTestInfo } from './globals';
+import { serializeError } from './util';
 
 export const expect: Expect = expectLibrary as any;
 expectLibrary.setState({ expand: false });
@@ -78,22 +79,22 @@ function wrap(matcherName: string, matcher: any) {
 
     const infix = this.isNot ? '.not' : '';
     const completeStep = testInfo._addStep('expect', `expect${infix}.${matcherName}`);
+    const stack = new Error().stack;
 
     const reportStepEnd = (result: any) => {
-      status = result.pass !== this.isNot ? 'passed' : 'failed';
+      const success = result.pass !== this.isNot;
-      let error: Error | undefined;
+      let error: TestError | undefined;
-      if (status === 'failed')
+      if (!success)
-        error = new Error(result.message());
+        error = { message: result.message(), stack };
       completeStep?.(error);
       return result;
     };
 
     const reportStepError = (error: Error) => {
-      completeStep?.(error);
+      completeStep?.(serializeError(error));
       throw error;
     };
 
-    let status: TestStatus = 'passed';
     try {
       const result = matcher.call(this, ...args);
       if (result instanceof Promise)

src/test/testType.ts

@@ -19,7 +19,7 @@ import { currentlyLoadingFileSuite, currentTestInfo, setCurrentlyLoadingFileSuit
 import { TestCase, Suite } from './test';
 import { wrapFunctionWithLocation } from './transform';
 import { Fixtures, FixturesWithLocation, Location, TestType } from './types';
-import { errorWithLocation } from './util';
+import { errorWithLocation, serializeError } from './util';
 
 const countByFile = new Map<string, number>();
 
@@ -49,6 +49,7 @@ export class TestTypeImpl {
     test.fail = wrapFunctionWithLocation(this._modifier.bind(this, 'fail'));
     test.slow = wrapFunctionWithLocation(this._modifier.bind(this, 'slow'));
     test.setTimeout = wrapFunctionWithLocation(this._setTimeout.bind(this));
+    test.step = wrapFunctionWithLocation(this._step.bind(this));
     test.use = wrapFunctionWithLocation(this._use.bind(this));
     test.extend = wrapFunctionWithLocation(this._extend.bind(this));
     test.declare = wrapFunctionWithLocation(this._declare.bind(this));
@@ -146,7 +147,7 @@ export class TestTypeImpl {
 
     const testInfo = currentTestInfo();
     if (!testInfo)
-      throw errorWithLocation(location, `test.setTimeout() can only be called from a test file`);
+      throw errorWithLocation(location, `test.setTimeout() can only be called from a test`);
     testInfo.setTimeout(timeout);
   }
 
@@ -157,6 +158,20 @@ export class TestTypeImpl {
     suite._fixtureOverrides = { ...suite._fixtureOverrides, ...fixtures };
   }
 
+  private async _step(location: Location, title: string, body: () => Promise<void>): Promise<void> {
+    const testInfo = currentTestInfo();
+    if (!testInfo)
+      throw errorWithLocation(location, `test.step() can only be called from a test`);
+    const complete = testInfo._addStep('test.step', title);
+    try {
+      await body();
+      complete();
+    } catch (e) {
+      complete(serializeError(e));
+      throw e;
+    }
+  }
+
   private _extend(location: Location, fixtures: Fixtures) {
     const fixturesWithLocation = { fixtures, location };
     return new TestTypeImpl([...this.fixtures, fixturesWithLocation]).test;

src/test/types.ts

@@ -25,7 +25,7 @@ export type FixturesWithLocation = {
 };
 export type Annotations = { type: string, description?: string }[];
 
-export type CompleteStepCallback = (error?: TestError) => void;
+export type CompleteStepCallback = (error?: Error | TestError) => void;
 
 export interface TestInfoImpl extends TestInfo {
   _testFinished: Promise<void>;

src/test/workerRunner.ts

@@ -278,7 +278,9 @@ export class WorkerRunner extends EventEmitter {
           wallTime: Date.now()
         };
         this.emit('stepBegin', payload);
-        return (error?: TestError) => {
+        return (error?: Error | TestError) => {
+          if (error instanceof Error)
+            error = serializeError(error);
           const payload: StepEndPayload = {
             testId,
             stepId,

tests/playwright-test/reporter.spec.ts

@@ -168,6 +168,8 @@ test('should report expect steps', async ({ runInlineTest }) => {
       }
       onStepEnd(test, result, step) {
         const copy = { ...step, startTime: undefined, duration: undefined };
+        if (copy.error?.stack)
+          copy.error.stack = '<stack>';
         console.log('%%%% end', JSON.stringify(copy));
       }
     }
@@ -197,13 +199,13 @@ test('should report expect steps', async ({ runInlineTest }) => {
   }, { reporter: '', workers: 1 });
 
   expect(result.exitCode).toBe(1);
-  expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([
+  expect(result.output.split('\n').filter(line => line.startsWith('%%')).map(stripEscapedAscii)).toEqual([
     `%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
     `%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
     `%% begin {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\"}`,
     `%% end {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\"}`,
     `%% begin {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\"}`,
-    `%% end {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\",\"error\":{}}`,
+    `%% end {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\",\"error\":{\"message\":\"expect(received).toBeTruthy()\\n\\nReceived: false\",\"stack\":\"<stack>\"}}`,
     `%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
     `%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
     `%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
@@ -236,6 +238,8 @@ test('should report api steps', async ({ runInlineTest }) => {
       }
       onStepEnd(test, result, step) {
         const copy = { ...step, startTime: undefined, duration: undefined };
+        if (copy.error?.stack)
+          copy.error.stack = '<stack>';
         console.log('%%%% end', JSON.stringify(copy));
       }
     }
@@ -259,7 +263,7 @@ test('should report api steps', async ({ runInlineTest }) => {
   }, { reporter: '', workers: 1 });
 
   expect(result.exitCode).toBe(0);
-  expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([
+  expect(result.output.split('\n').filter(line => line.startsWith('%%')).map(stripEscapedAscii)).toEqual([
     `%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
     `%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
     `%% begin {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`,
@@ -274,3 +278,109 @@ test('should report api steps', async ({ runInlineTest }) => {
     `%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
   ]);
 });
+
+
+test('should report api step failure', async ({ runInlineTest }) => {
+  const expectReporterJS = `
+    class Reporter {
+      onStepBegin(test, result, step) {
+        const copy = { ...step, startTime: undefined, duration: undefined };
+        console.log('%%%% begin', JSON.stringify(copy));
+      }
+      onStepEnd(test, result, step) {
+        const copy = { ...step, startTime: undefined, duration: undefined };
+        if (copy.error?.stack)
+          copy.error.stack = '<stack>';
+        console.log('%%%% end', JSON.stringify(copy));
+      }
+    }
+    module.exports = Reporter;
+  `;
+
+  const result = await runInlineTest({
+    'reporter.ts': expectReporterJS,
+    'playwright.config.ts': `
+      module.exports = {
+        reporter: './reporter',
+      };
+    `,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('fail', async ({ page }) => {
+        await page.setContent('<button></button>');
+        await page.click('input', { timeout: 1 });
+      });
+    `
+  }, { reporter: '', workers: 1 });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.output.split('\n').filter(line => line.startsWith('%%')).map(stripEscapedAscii)).toEqual([
+    `%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
+    `%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
+    `%% begin {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`,
+    `%% end {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`,
+    `%% begin {\"title\":\"page.setContent\",\"category\":\"pw:api\"}`,
+    `%% end {\"title\":\"page.setContent\",\"category\":\"pw:api\"}`,
+    `%% begin {\"title\":\"page.click\",\"category\":\"pw:api\"}`,
+    `%% end {\"title\":\"page.click\",\"category\":\"pw:api\",\"error\":{\"message\":\"page.click: Timeout 1ms exceeded.\\n=========================== logs ===========================\\nwaiting for selector \\\"input\\\"\\n============================================================\",\"stack\":\"<stack>\"}}`,
+    `%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
+    `%% begin {\"title\":\"browserContext.close\",\"category\":\"pw:api\"}`,
+    `%% end {\"title\":\"browserContext.close\",\"category\":\"pw:api\"}`,
+    `%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
+  ]);
+});
+
+test('should report test.step', async ({ runInlineTest }) => {
+  const expectReporterJS = `
+    class Reporter {
+      onStepBegin(test, result, step) {
+        const copy = { ...step, startTime: undefined, duration: undefined };
+        console.log('%%%% begin', JSON.stringify(copy));
+      }
+      onStepEnd(test, result, step) {
+        const copy = { ...step, startTime: undefined, duration: undefined };
+        if (copy.error?.stack)
+          copy.error.stack = '<stack>';
+        console.log('%%%% end', JSON.stringify(copy));
+      }
+    }
+    module.exports = Reporter;
+  `;
+
+  const result = await runInlineTest({
+    'reporter.ts': expectReporterJS,
+    'playwright.config.ts': `
+      module.exports = {
+        reporter: './reporter',
+      };
+    `,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('pass', async ({ page }) => {
+        await test.step('First step', async () => {
+          expect(1).toBe(2);
+        });
+      });
+    `
+  }, { reporter: '', workers: 1 });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.output.split('\n').filter(line => line.startsWith('%%')).map(stripEscapedAscii)).toEqual([
+    `%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
+    `%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
+    `%% begin {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`,
+    `%% end {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`,
+    `%% begin {\"title\":\"First step\",\"category\":\"test.step\"}`,
+    `%% begin {\"title\":\"expect.toBe\",\"category\":\"expect\"}`,
+    `%% end {\"title\":\"expect.toBe\",\"category\":\"expect\",\"error\":{\"message\":\"expect(received).toBe(expected) // Object.is equality\\n\\nExpected: 2\\nReceived: 1\",\"stack\":\"<stack>\"}}`,
+    `%% end {\"title\":\"First step\",\"category\":\"test.step\",\"error\":{\"message\":\"expect(received).toBe(expected) // Object.is equality\\n\\nExpected: 2\\nReceived: 1\",\"stack\":\"<stack>\"}}`,
+    `%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
+    `%% begin {\"title\":\"browserContext.close\",\"category\":\"pw:api\"}`,
+    `%% end {\"title\":\"browserContext.close\",\"category\":\"pw:api\"}`,
+    `%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
+  ]);
+});
+
+function stripEscapedAscii(str: string) {
+  return str.replace(/\\u00[a-z0-9][a-z0-9]\[[^m]+m/g, '');
+}

types/test.d.ts

@@ -2093,6 +2093,33 @@ export interface TestType<TestArgs extends KeyValue, WorkerArgs extends KeyValue
    * @param fixtures An object with fixture definitions.
    */
   use(fixtures: Fixtures<{}, {}, TestArgs, WorkerArgs>): void;
+  /**
+   * Declares a test step.
+   *
+   * ```js js-flavor=js
+   * const { test, expect } = require('@playwright/test');
+   *
+   * test('test', async ({ page }) => {
+   *   await test.step('Log in', async () => {
+   *     // ...
+   *   });
+   * });
+   * ```
+   *
+   * ```js js-flavor=ts
+   * import { test, expect } from '@playwright/test';
+   *
+   * test('test', async ({ page }) => {
+   *   await test.step('Log in', async () => {
+   *     // ...
+   *   });
+   * });
+   * ```
+   *
+   * @param title Step name.
+   * @param body Step body.
+   */
+  step(title: string, body: () => Promise<any>): Promise<any>;
   /**
    * `expect` function can be used to create test assertions. Read
    * [expect library documentation](https://jestjs.io/docs/expect) for more details.

utils/generate_types/overrides-test.d.ts

@@ -246,6 +246,7 @@ export interface TestType<TestArgs extends KeyValue, WorkerArgs extends KeyValue
   beforeAll(inner: (args: WorkerArgs, workerInfo: WorkerInfo) => Promise<any> | any): void;
   afterAll(inner: (args: WorkerArgs, workerInfo: WorkerInfo) => Promise<any> | any): void;
   use(fixtures: Fixtures<{}, {}, TestArgs, WorkerArgs>): void;
+  step(title: string, body: () => Promise<any>): Promise<any>;
   expect: Expect;
   declare<T extends KeyValue = {}, W extends KeyValue = {}>(): TestType<TestArgs & T, WorkerArgs & W>;
   extend<T, W extends KeyValue = {}>(fixtures: Fixtures<T, W, TestArgs, WorkerArgs>): TestType<TestArgs & T, WorkerArgs & W>;