feat(test-runner): introduce steps (#7952)

.gitignore

@@ -16,4 +16,4 @@ drivers/
 .gradle/
 nohup.out
 .trace
-.tmp
+.tmp

docs/src/test-reporter-api/class-reporter.md

@@ -138,6 +138,10 @@ Output chunk.
 
 Test that was running. Note that output may happen when to test is running, in which case this will be [void].
 
+### param: Reporter.onStdErr.result
+- `result` <[void]|[TestResult]>
+
+Result of the test run, this object gets populated while the test runs.
 
 
 ## method: Reporter.onStdOut
@@ -154,7 +158,48 @@ Output chunk.
 
 Test that was running. Note that output may happen when to test is running, in which case this will be [void].
 
+### param: Reporter.onStdOut.result
+- `result` <[void]|[TestResult]>
 
+Result of the test run, this object gets populated while the test runs.
+
+## method: Reporter.onStepBegin
+
+Called when a test step started in the worker process.
+
+### param: Reporter.onStepBegin.test
+- `test` <[TestCase]>
+
+Test that has been started.
+
+### param: Reporter.onStepBegin.result
+- `result` <[TestResult]>
+
+Result of the test run, this object gets populated while the test runs.
+
+### param: Reporter.onStepBegin.step
+- `result` <[TestStep]>
+
+Test step instance.
+
+## method: Reporter.onStepEnd
+
+Called when a test step finished in the worker process.
+
+### param: Reporter.onStepEnd.test
+- `test` <[TestCase]>
+
+Test that has been finished.
+
+### param: Reporter.onStepEnd.result
+- `result` <[TestResult]>
+
+Result of the test run.
+
+### param: Reporter.onStepEnd.step
+- `result` <[TestStep]>
+
+Test step instance.
 
 ## method: Reporter.onTestBegin
 
@@ -165,6 +210,10 @@ Called after a test has been started in the worker process.
 
 Test that has been started.
 
+### param: Reporter.onTestBegin.result
+- `result` <[TestResult]>
+
+Result of the test run, this object gets populated while the test runs.
 
 
 ## method: Reporter.onTestEnd

docs/src/test-reporter-api/class-testresult.md

@@ -49,6 +49,11 @@ Anything written to the standard error during the test run.
 
 Anything written to the standard output during the test run.
 
+## property: TestResult.steps
+- type: <[Array]<[TestStep]>>
+
+List of steps inside this test run.
+
 ## property: TestResult.workerIndex
 - type: <[int]>
 

docs/src/test-reporter-api/class-teststep.md

@@ -0,0 +1,32 @@
+# class: TestStep
+* langs: js
+
+Represents a step in the [TestRun].
+
+## property: TestStep.category
+- type: <[string]>
+
+Step category to differentiate steps with different origin and verbosity. Built-in categories are:
+* `hook` for fixtures and hooks initialization and teardown
+* `expect` for expect calls
+* `pw:api` for Playwright API calls.
+
+## property: TestStep.duration
+- type: <[float]>
+
+Running time in milliseconds.
+
+## property: TestStep.error
+- type: <[void]|[TestError]>
+
+An error thrown during the step execution, if any.
+
+## property: TestStep.startTime
+- type: <[Date]>
+
+Start time of this particular test step.
+
+## property: TestStep.title
+- type: <[string]>
+
+User-friendly test step title.

src/client/channelOwner.ts

@@ -23,8 +23,6 @@ import { isUnderTest } from '../utils/utils';
 import type { Connection } from './connection';
 import type { ClientSideInstrumentation, Logger } from './types';
 
-let lastCallSeq = 0;
-
 export abstract class ChannelOwner<T extends channels.Channel = channels.Channel, Initializer = {}> extends EventEmitter {
   protected _connection: Connection;
   private _parent: ChannelOwner | undefined;
@@ -97,19 +95,19 @@ export abstract class ChannelOwner<T extends channels.Channel = channels.Channel
     const stackTrace = captureStackTrace();
     const { apiName, frameTexts } = stackTrace;
     const channel = this._createChannel({}, stackTrace);
-    const seq = ++lastCallSeq;
+    let csiCallback: ((e?: Error) => void) | undefined;
     try {
       logApiCall(logger, `=> ${apiName} started`);
-      this._csi?.onApiCall({ phase: 'begin', seq, apiName, frames: stackTrace.frames });
+      csiCallback = this._csi?.onApiCall(apiName);
       const result = await func(channel as any, stackTrace);
-      this._csi?.onApiCall({ phase: 'end', seq });
+      csiCallback?.();
       logApiCall(logger, `<= ${apiName} succeeded`);
       return result;
     } catch (e) {
       const innerError = ((process.env.PWDEBUGIMPL || isUnderTest()) && e.stack) ? '\n<inner error>\n' + e.stack : '';
       e.message = apiName + ': ' + e.message;
       e.stack = e.message + '\n' + frameTexts.join('\n') + innerError;
-      this._csi?.onApiCall({ phase: 'end', seq, error: e.stack });
+      csiCallback?.(e);
       logApiCall(logger, `<= ${apiName} failed`);
       throw e;
     }

src/client/types.ts

@@ -24,7 +24,7 @@ export interface Logger {
 }
 
 export interface ClientSideInstrumentation {
-  onApiCall(data: { phase: 'begin' | 'end', seq: number, apiName?: string, frames?: channels.StackFrame[], error?: string }): void;
+  onApiCall(name: string): (error?: Error) => void;
 }
 
 import { Size } from '../common/types';

src/test/dispatcher.ts

@@ -17,8 +17,8 @@
 import child_process from 'child_process';
 import path from 'path';
 import { EventEmitter } from 'events';
-import { RunPayload, TestBeginPayload, TestEndPayload, DonePayload, TestOutputPayload, WorkerInitParams, ProgressPayload } from './ipc';
+import { RunPayload, TestBeginPayload, TestEndPayload, DonePayload, TestOutputPayload, WorkerInitParams, StepBeginPayload, StepEndPayload } from './ipc';
-import type { TestResult, Reporter } from '../../types/testReporter';
+import type { TestResult, Reporter, TestStep } from '../../types/testReporter';
 import { TestCase } from './test';
 import { Loader } from './loader';
 
@@ -35,7 +35,7 @@ export class Dispatcher {
   private _freeWorkers: Worker[] = [];
   private _workerClaimers: (() => void)[] = [];
 
-  private _testById = new Map<string, { test: TestCase, result: TestResult }>();
+  private _testById = new Map<string, { test: TestCase, result: TestResult, steps: Map<string, TestStep> }>();
   private _queue: TestGroup[] = [];
   private _stopCallback = () => {};
   readonly _loader: Loader;
@@ -51,7 +51,8 @@ export class Dispatcher {
     for (const group of testGroups) {
       for (const test of group.tests) {
         const result = test._appendTestResult();
-        this._testById.set(test._id, { test, result });
+        // When changing this line, change the one in retry too.
+        this._testById.set(test._id, { test, result, steps: new Map() });
       }
     }
   }
@@ -136,7 +137,7 @@ export class Dispatcher {
             break;
           // There might be a single test that has started but has not finished yet.
           if (test._id !== lastStartedTestId)
-            this._reporter.onTestBegin?.(test);
+            this._reporter.onTestBegin?.(test, result);
           result.error = params.fatalError;
           result.status = first ? 'failed' : 'skipped';
           this._reportTestEnd(test, result);
@@ -155,6 +156,7 @@ export class Dispatcher {
         const pair = this._testById.get(testId)!;
         if (!this._isStopped && pair.test.expectedStatus === 'passed' && pair.test.results.length < pair.test.retries + 1) {
           pair.result = pair.test._appendTestResult();
+          pair.steps = new Map();
           remaining.unshift(pair.test);
         }
       }
@@ -215,7 +217,7 @@ export class Dispatcher {
       const { test, result: testRun  } = this._testById.get(params.testId)!;
       testRun.workerIndex = params.workerIndex;
       testRun.startTime = new Date(params.startWallTime);
-      this._reporter.onTestBegin?.(test);
+      this._reporter.onTestBegin?.(test, testRun);
     });
     worker.on('testEnd', (params: TestEndPayload) => {
       if (this._hasReachedMaxFailures())
@@ -235,23 +237,40 @@ export class Dispatcher {
       test.timeout = params.timeout;
       this._reportTestEnd(test, result);
     });
-    worker.on('progress', (params: ProgressPayload) => {
+    worker.on('stepBegin', (params: StepBeginPayload) => {
-      const { test } = this._testById.get(params.testId)!;
+      const { test, result, steps } = this._testById.get(params.testId)!;
-      (this._reporter as any)._onTestProgress?.(test, params.name, params.data);
+      const step: TestStep = {
+        title: params.title,
+        category: params.category,
+        startTime: new Date(params.wallTime),
+        duration: 0,
+      };
+      steps.set(params.stepId, step);
+      result.steps.push(step);
+      this._reporter.onStepBegin?.(test, result, step);
+    });
+    worker.on('stepEnd', (params: StepEndPayload) => {
+      const { test, result, steps } = this._testById.get(params.testId)!;
+      const step = steps.get(params.stepId)!;
+      step.duration = params.wallTime - step.startTime.getTime();
+      if (params.error)
+        step.error = params.error;
+      steps.delete(params.stepId);
+      this._reporter.onStepEnd?.(test, result, step);
     });
     worker.on('stdOut', (params: TestOutputPayload) => {
       const chunk = chunkFromParams(params);
       const pair = params.testId ? this._testById.get(params.testId) : undefined;
       if (pair)
         pair.result.stdout.push(chunk);
-      this._reporter.onStdOut?.(chunk, pair ? pair.test : undefined);
+      this._reporter.onStdOut?.(chunk, pair?.test, pair?.result);
     });
     worker.on('stdErr', (params: TestOutputPayload) => {
       const chunk = chunkFromParams(params);
       const pair = params.testId ? this._testById.get(params.testId) : undefined;
       if (pair)
         pair.result.stderr.push(chunk);
-      this._reporter.onStdErr?.(chunk, pair ? pair.test : undefined);
+      this._reporter.onStdErr?.(chunk, pair?.test, pair?.result);
     });
     worker.on('teardownError', ({error}) => {
       this._hasWorkerErrors = true;

src/test/expect.ts

@@ -39,7 +39,7 @@ import {
   toHaveValue
 } from './matchers/matchers';
 import { toMatchSnapshot } from './matchers/toMatchSnapshot';
-import type { Expect } from './types';
+import type { Expect, TestStatus } from './types';
 import matchers from 'expect/build/matchers';
 import { currentTestInfo } from './globals';
 
@@ -70,41 +70,37 @@ const customMatchers = {
   toMatchSnapshot,
 };
 
-let lastExpectSeq = 0;
-
 function wrap(matcherName: string, matcher: any) {
   return function(this: any, ...args: any[]) {
     const testInfo = currentTestInfo();
     if (!testInfo)
       return matcher.call(this, ...args);
 
-    const seq = ++lastExpectSeq;
+    const infix = this.isNot ? '.not' : '';
-    testInfo._progress('expect', { phase: 'begin', seq, matcherName });
+    const completeStep = testInfo._addStep('expect', `expect${infix}.${matcherName}`);
-    const endPayload: any = { phase: 'end', seq };
+
-    let isAsync = false;
+    const reportStepEnd = (result: any) => {
+      status = result.pass !== this.isNot ? 'passed' : 'failed';
+      let error: Error | undefined;
+      if (status === 'failed')
+        error = new Error(result.message());
+      completeStep?.(error);
+      return result;
+    };
+
+    const reportStepError = (error: Error) => {
+      completeStep?.(error);
+      throw error;
+    };
+
+    let status: TestStatus = 'passed';
     try {
       const result = matcher.call(this, ...args);
-      endPayload.pass = result.pass;
+      if (result instanceof Promise)
-      if (this.isNot)
+        return result.then(reportStepEnd).catch(reportStepError);
-        endPayload.isNot = this.isNot;
+      return reportStepEnd(result);
-      if (result.pass === this.isNot && result.message)
-        endPayload.message = result.message();
-      if (result instanceof Promise) {
-        isAsync = true;
-        return result.catch(e => {
-          endPayload.error = e.stack;
-          throw e;
-        }).finally(() => {
-          testInfo._progress('expect', endPayload);
-        });
-      }
-      return result;
     } catch (e) {
-      endPayload.error = e.stack;
+      reportStepError(e);
-      throw e;
-    } finally {
-      if (!isAsync)
-        testInfo._progress('expect', endPayload);
     }
   };
 }

src/test/index.ts

@@ -185,7 +185,9 @@ export const test = _baseTest.extend<PlaywrightTestArgs & PlaywrightTestOptions,
       };
       const context = await browser.newContext(combinedOptions);
       (context as any)._csi = {
-        onApiCall: (data: any) => (testInfo as any)._progress('pw:api', data),
+        onApiCall: (name: string) => {
+          return (testInfo as any)._addStep('pw:api', name);
+        },
       };
       context.setDefaultTimeout(actionTimeout || 0);
       context.setDefaultNavigationTimeout(navigationTimeout || actionTimeout || 0);

src/test/ipc.ts

@@ -46,10 +46,19 @@ export type TestEndPayload = {
   attachments: { name: string, path?: string, body?: string, contentType: string }[];
 };
 
-export type ProgressPayload = {
+export type StepBeginPayload = {
   testId: string;
-  name: string;
+  stepId: string;
-  data: any;
+  title: string;
+  category: string;
+  wallTime: number;  // milliseconds since unix epoch
+};
+
+export type StepEndPayload = {
+  testId: string;
+  stepId: string;
+  wallTime: number;  // milliseconds since unix epoch
+  error?: TestError;
 };
 
 export type TestEntry = {

src/test/reporters/multiplexer.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { FullConfig, Suite, TestCase, TestError, TestResult, Reporter, FullResult } from '../../../types/testReporter';
+import { FullConfig, Suite, TestCase, TestError, TestResult, Reporter, FullResult, TestStep } from '../../../types/testReporter';
 
 export class Multiplexer implements Reporter {
   private _reporters: Reporter[];
@@ -28,19 +28,19 @@ export class Multiplexer implements Reporter {
       reporter.onBegin?.(config, suite);
   }
 
-  onTestBegin(test: TestCase) {
+  onTestBegin(test: TestCase, result: TestResult) {
     for (const reporter of this._reporters)
-      reporter.onTestBegin?.(test);
+      reporter.onTestBegin?.(test, result);
   }
 
-  onStdOut(chunk: string | Buffer, test?: TestCase) {
+  onStdOut(chunk: string | Buffer, test?: TestCase, result?: TestResult) {
     for (const reporter of this._reporters)
-      reporter.onStdOut?.(chunk, test);
+      reporter.onStdOut?.(chunk, test, result);
   }
 
-  onStdErr(chunk: string | Buffer, test?: TestCase) {
+  onStdErr(chunk: string | Buffer, test?: TestCase, result?: TestResult) {
     for (const reporter of this._reporters)
-      reporter.onStdErr?.(chunk, test);
+      reporter.onStdErr?.(chunk, test, result);
   }
 
   onTestEnd(test: TestCase, result: TestResult) {
@@ -58,8 +58,13 @@ export class Multiplexer implements Reporter {
       reporter.onError?.(error);
   }
 
-  _onTestProgress(test: TestCase, name: string, data: any) {
+  onStepBegin(test: TestCase, result: TestResult, step: TestStep) {
     for (const reporter of this._reporters)
-      (reporter as any)._onTestProgress?.(test, name, data);
+      (reporter as any).onStepBegin?.(test, result, step);
+  }
+
+  onStepEnd(test: TestCase, result: TestResult, step: TestStep) {
+    for (const reporter of this._reporters)
+      (reporter as any).onStepEnd?.(test, result, step);
   }
 }

src/test/test.ts

@@ -173,6 +173,7 @@ export class TestCase extends Base implements reporterTypes.TestCase {
       stderr: [],
       attachments: [],
       status: 'skipped',
+      steps: []
     };
     this.results.push(result);
     return result;

src/test/types.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import type { Fixtures, TestInfo } from '../../types/test';
+import type { Fixtures, TestError, TestInfo } from '../../types/test';
 import type { Location } from '../../types/testReporter';
 export * from '../../types/test';
 export { Location } from '../../types/testReporter';
@@ -25,7 +25,9 @@ export type FixturesWithLocation = {
 };
 export type Annotations = { type: string, description?: string }[];
 
+export type CompleteStepCallback = (error?: TestError) => void;
+
 export interface TestInfoImpl extends TestInfo {
   _testFinished: Promise<void>;
-  _progress: (name: string, params: any) => void;
+  _addStep: (category: string, title: string) => CompleteStepCallback;
 }

src/test/worker.ts

@@ -74,7 +74,7 @@ process.on('message', async message => {
     workerIndex = initParams.workerIndex;
     startProfiling();
     workerRunner = new WorkerRunner(initParams);
-    for (const event of ['testBegin', 'testEnd', 'done', 'progress'])
+    for (const event of ['testBegin', 'testEnd', 'stepBegin', 'stepEnd', 'done'])
       workerRunner.on(event, sendMessageToParent.bind(null, event));
     return;
   }

src/test/workerRunner.ts

@@ -20,11 +20,11 @@ import rimraf from 'rimraf';
 import util from 'util';
 import { EventEmitter } from 'events';
 import { monotonicTime, DeadlineRunner, raceAgainstDeadline, serializeError } from './util';
-import { TestBeginPayload, TestEndPayload, RunPayload, TestEntry, DonePayload, WorkerInitParams } from './ipc';
+import { TestBeginPayload, TestEndPayload, RunPayload, TestEntry, DonePayload, WorkerInitParams, StepBeginPayload, StepEndPayload } from './ipc';
 import { setCurrentTestInfo } from './globals';
 import { Loader } from './loader';
 import { Modifier, Suite, TestCase } from './test';
-import { Annotations, TestError, TestInfo, TestInfoImpl, WorkerInfo } from './types';
+import { Annotations, CompleteStepCallback, TestError, TestInfo, TestInfoImpl, WorkerInfo } from './types';
 import { ProjectImpl } from './project';
 import { FixturePool, FixtureRunner } from './fixtures';
 
@@ -221,6 +221,7 @@ export class WorkerRunner extends EventEmitter {
     })();
 
     let testFinishedCallback = () => {};
+    let lastStepId = 0;
     const testInfo: TestInfoImpl = {
       ...this._workerInfo,
       title: test.title,
@@ -267,7 +268,26 @@ export class WorkerRunner extends EventEmitter {
           deadlineRunner.setDeadline(deadline());
       },
       _testFinished: new Promise(f => testFinishedCallback = f),
-      _progress: (name, data) => this.emit('progress', { testId, name, data }),
+      _addStep: (category: string, title: string) => {
+        const stepId = `${category}@${++lastStepId}`;
+        const payload: StepBeginPayload = {
+          testId,
+          stepId,
+          category,
+          title,
+          wallTime: Date.now()
+        };
+        this.emit('stepBegin', payload);
+        return (error?: TestError) => {
+          const payload: StepEndPayload = {
+            testId,
+            stepId,
+            wallTime: Date.now(),
+            error
+          };
+          this.emit('stepEnd', payload);
+        };
+      },
     };
 
     // Inherit test.setTimeout() from parent suites.
@@ -361,7 +381,8 @@ export class WorkerRunner extends EventEmitter {
     setCurrentTestInfo(currentTest ? currentTest.testInfo : null);
   }
 
-  private async _runTestWithBeforeHooks(test: TestCase, testInfo: TestInfo) {
+  private async _runTestWithBeforeHooks(test: TestCase, testInfo: TestInfoImpl) {
+    let completeStep: CompleteStepCallback | undefined;
     try {
       const beforeEachModifiers: Modifier[] = [];
       for (let s = test.parent; s; s = s.parent) {
@@ -375,6 +396,7 @@ export class WorkerRunner extends EventEmitter {
         const result = await this._fixtureRunner.resolveParametersAndRunHookOrTest(modifier.fn, 'test', testInfo);
         testInfo[modifier.type](!!result, modifier.description!);
       }
+      completeStep = testInfo._addStep('hook', 'Before Hooks');
       await this._runHooks(test.parent!, 'beforeEach', testInfo);
     } catch (error) {
       if (error instanceof SkipError) {
@@ -386,6 +408,7 @@ export class WorkerRunner extends EventEmitter {
       }
       // Continue running afterEach hooks even after the failure.
     }
+    completeStep?.(testInfo.error);
 
     // Do not run the test when beforeEach hook fails.
     if (this._isStopped || testInfo.status === 'failed' || testInfo.status === 'skipped')
@@ -409,8 +432,11 @@ export class WorkerRunner extends EventEmitter {
     }
   }
 
-  private async _runAfterHooks(test: TestCase, testInfo: TestInfo) {
+  private async _runAfterHooks(test: TestCase, testInfo: TestInfoImpl) {
+    let completeStep: CompleteStepCallback | undefined;
+    let teardownError: TestError | undefined;
     try {
+      completeStep = testInfo._addStep('hook', 'After Hooks');
       await this._runHooks(test.parent!, 'afterEach', testInfo);
     } catch (error) {
       if (!(error instanceof SkipError)) {
@@ -428,9 +454,12 @@ export class WorkerRunner extends EventEmitter {
       if (testInfo.status === 'passed')
         testInfo.status = 'failed';
       // Do not overwrite test failure error.
-      if (!('error' in  testInfo))
+      if (!('error' in  testInfo)) {
         testInfo.error = serializeError(error);
+        teardownError = testInfo.error;
+      }
     }
+    completeStep?.(teardownError);
   }
 
   private async _runHooks(suite: Suite, type: 'beforeEach' | 'afterEach', testInfo: TestInfo) {

tests/config/browserTest.ts

@@ -135,7 +135,7 @@ export const playwrightFixtures: Fixtures<PlaywrightTestOptions & PlaywrightTest
     await run(contextOptions);
   },
 
-  contextFactory: async ({ browser, contextOptions }, run) => {
+  contextFactory: async ({ browser, contextOptions }, run, testInfo) => {
     const contexts: BrowserContext[] = [];
     await run(async options => {
       const context = await browser.newContext({ ...contextOptions, ...options });

tests/playwright-test/reporter.spec.ts

@@ -159,13 +159,16 @@ test('should load reporter from node_modules', async ({ runInlineTest }) => {
   ]);
 });
 
-test('should report expect progress', async ({ runInlineTest }) => {
+test('should report expect steps', async ({ runInlineTest }) => {
   const expectReporterJS = `
     class Reporter {
-      _onTestProgress(test, name, data) {
+      onStepBegin(test, result, step) {
-        if (data.frames)
+        const copy = { ...step, startTime: undefined, duration: undefined };
-          data.frames = [];
+        console.log('%%%% begin', JSON.stringify(copy));
-        console.log('%%%%', name, JSON.stringify(data));
+      }
+      onStepEnd(test, result, step) {
+        const copy = { ...step, startTime: undefined, duration: undefined };
+        console.log('%%%% end', JSON.stringify(copy));
       }
     }
     module.exports = Reporter;
@@ -195,32 +198,45 @@ test('should report expect progress', async ({ runInlineTest }) => {
 
   expect(result.exitCode).toBe(1);
   expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([
-    `%% expect {\"phase\":\"begin\",\"seq\":1,\"matcherName\":\"toBeTruthy\"}`,
+    `%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
-    `%% expect {\"phase\":\"end\",\"seq\":1,\"pass\":true}`,
+    `%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
-    `%% expect {\"phase\":\"begin\",\"seq\":2,\"matcherName\":\"toBeTruthy\"}`,
+    `%% begin {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\"}`,
-    `%% expect {\"phase\":\"end\",\"seq\":2,\"pass\":false,\"message\":\"\\u001b[2mexpect(\\u001b[22m\\u001b[31mreceived\\u001b[39m\\u001b[2m).\\u001b[22mtoBeTruthy\\u001b[2m()\\u001b[22m\\n\\nReceived: \\u001b[31mfalse\\u001b[39m\"}`,
+    `%% end {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\"}`,
-
+    `%% begin {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\"}`,
-    `%% expect {\"phase\":\"begin\",\"seq\":1,\"matcherName\":\"toBeTruthy\"}`,
+    `%% end {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\",\"error\":{}}`,
-    `%% expect {\"phase\":\"end\",\"seq\":1,\"pass\":false,\"isNot\":true}`,
+    `%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
-
+    `%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
-    `%% pw:api {\"phase\":\"begin\",\"seq\":3,\"apiName\":\"browserContext.newPage\",\"frames\":[]}`,
+    `%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
-    `%% pw:api {\"phase\":\"end\",\"seq\":3}`,
+    `%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
-    `%% expect {\"phase\":\"begin\",\"seq\":2,\"matcherName\":\"toHaveTitle\"}`,
+    `%% begin {\"title\":\"expect.not.toBeTruthy\",\"category\":\"expect\"}`,
-    `%% pw:api {\"phase\":\"begin\",\"seq\":4,\"apiName\":\"page.title\",\"frames\":[]}`,
+    `%% end {\"title\":\"expect.not.toBeTruthy\",\"category\":\"expect\"}`,
-    `%% pw:api {\"phase\":\"end\",\"seq\":4}`,
+    `%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
-    `%% expect {\"phase\":\"end\",\"seq\":2,\"isNot\":true}`,
+    `%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
-    `%% pw:api {\"phase\":\"begin\",\"seq\":5,\"apiName\":\"browserContext.close\",\"frames\":[]}`,
+    `%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
-    `%% pw:api {\"phase\":\"end\",\"seq\":5}`,
+    `%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
+    `%% begin {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`,
+    `%% end {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`,
+    `%% begin {\"title\":\"expect.not.toHaveTitle\",\"category\":\"expect\"}`,
+    `%% begin {\"title\":\"page.title\",\"category\":\"pw:api\"}`,
+    `%% end {\"title\":\"page.title\",\"category\":\"pw:api\"}`,
+    `%% end {\"title\":\"expect.not.toHaveTitle\",\"category\":\"expect\"}`,
+    `%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
+    `%% begin {\"title\":\"browserContext.close\",\"category\":\"pw:api\"}`,
+    `%% end {\"title\":\"browserContext.close\",\"category\":\"pw:api\"}`,
+    `%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
   ]);
 });
 
-test('should report log progress', async ({ runInlineTest }) => {
+test('should report api steps', async ({ runInlineTest }) => {
   const expectReporterJS = `
     class Reporter {
-      _onTestProgress(test, name, data) {
+      onStepBegin(test, result, step) {
-        if (data.frames)
+        const copy = { ...step, startTime: undefined, duration: undefined };
-          data.frames = [];
+        console.log('%%%% begin', JSON.stringify(copy));
-        console.log('%%%%', name, JSON.stringify(data));
+      }
+      onStepEnd(test, result, step) {
+        const copy = { ...step, startTime: undefined, duration: undefined };
+        console.log('%%%% end', JSON.stringify(copy));
       }
     }
     module.exports = Reporter;
@@ -244,13 +260,17 @@ test('should report log progress', async ({ runInlineTest }) => {
 
   expect(result.exitCode).toBe(0);
   expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([
-    `%% pw:api {\"phase\":\"begin\",\"seq\":3,\"apiName\":\"browserContext.newPage\",\"frames\":[]}`,
+    `%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
-    `%% pw:api {\"phase\":\"end\",\"seq\":3}`,
+    `%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
-    `%% pw:api {\"phase\":\"begin\",\"seq\":4,\"apiName\":\"page.setContent\",\"frames\":[]}`,
+    `%% begin {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`,
-    `%% pw:api {\"phase\":\"end\",\"seq\":4}`,
+    `%% end {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`,
-    `%% pw:api {\"phase\":\"begin\",\"seq\":5,\"apiName\":\"page.click\",\"frames\":[]}`,
+    `%% begin {\"title\":\"page.setContent\",\"category\":\"pw:api\"}`,
-    `%% pw:api {\"phase\":\"end\",\"seq\":5}`,
+    `%% end {\"title\":\"page.setContent\",\"category\":\"pw:api\"}`,
-    `%% pw:api {\"phase\":\"begin\",\"seq\":6,\"apiName\":\"browserContext.close\",\"frames\":[]}`,
+    `%% begin {\"title\":\"page.click\",\"category\":\"pw:api\"}`,
-    `%% pw:api {\"phase\":\"end\",\"seq\":6}`,
+    `%% end {\"title\":\"page.click\",\"category\":\"pw:api\"}`,
+    `%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
+    `%% begin {\"title\":\"browserContext.close\",\"category\":\"pw:api\"}`,
+    `%% end {\"title\":\"browserContext.close\",\"category\":\"pw:api\"}`,
+    `%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
   ]);
 });

types/testReporter.d.ts

@@ -213,6 +213,39 @@ export interface TestResult {
    * Anything written to the standard error during the test run.
    */
   stderr: (string | Buffer)[];
+  /**
+   * List of steps inside this test run.
+   */
+  steps: TestStep[];
+}
+
+/**
+ * Represents a step in the [TestRun].
+ */
+export interface TestStep {
+  /**
+   * User-friendly test step title.
+   */
+  title: string;
+  /**
+   * Step category to differentiate steps with different origin and verbosity. Built-in categories are:
+   * - `hook` for fixtures and hooks initialization and teardown
+   * - `expect` for expect calls
+   * - `pw:api` for Playwright API calls.
+   */
+  category: string,
+  /**
+   * Start time of this particular test step.
+   */
+  startTime: Date;
+  /**
+   * Running time in milliseconds.
+   */
+  duration: number;
+  /**
+   * An error thrown during the step execution, if any.
+   */
+  error?: TestError;
 }
 
 /**
@@ -321,26 +354,43 @@ export interface Reporter {
   /**
    * Called after a test has been started in the worker process.
    * @param test Test that has been started.
+   * @param result Result of the test run, this object gets populated while the test runs.
    */
-  onTestBegin?(test: TestCase): void;
+  onTestBegin?(test: TestCase, result: TestResult): void;
   /**
    * Called when something has been written to the standard output in the worker process.
    * @param chunk Output chunk.
    * @param test Test that was running. Note that output may happen when to test is running, in which case this will be [void].
+   * @param result Result of the test run, this object gets populated while the test runs.
    */
-  onStdOut?(chunk: string | Buffer, test?: TestCase): void;
+  onStdOut?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void;
   /**
    * Called when something has been written to the standard error in the worker process.
    * @param chunk Output chunk.
    * @param test Test that was running. Note that output may happen when to test is running, in which case this will be [void].
+   * @param result Result of the test run, this object gets populated while the test runs.
    */
-  onStdErr?(chunk: string | Buffer, test?: TestCase): void;
+  onStdErr?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void;
   /**
    * Called after a test has been finished in the worker process.
    * @param test Test that has been finished.
    * @param result Result of the test run.
    */
   onTestEnd?(test: TestCase, result: TestResult): void;
+  /**
+   * Called when a test step started in the worker process.
+   * @param test Test that has been started.
+   * @param result Result of the test run, this object gets populated while the test runs.
+   * @param result Test step instance.
+   */
+  onStepBegin?(test: TestCase, result: TestResult, step: TestStep): void;
+  /**
+   * Called when a test step finished in the worker process.
+   * @param test Test that has been finished.
+   * @param result Result of the test run.
+   * @param result Test step instance.
+   */
+  onStepEnd?(test: TestCase, result: TestResult, step: TestStep): void;
   /**
    * Called on some global error, for example unhandled exception in the worker process.
    * @param error The error.

utils/generate_types/overrides-testReporter.d.ts

@@ -55,6 +55,15 @@ export interface TestResult {
   attachments: { name: string, path?: string, body?: Buffer, contentType: string }[];
   stdout: (string | Buffer)[];
   stderr: (string | Buffer)[];
+  steps: TestStep[];
+}
+
+export interface TestStep {
+  title: string;
+  category: string,
+  startTime: Date;
+  duration: number;
+  error?: TestError;
 }
 
 /**
@@ -73,10 +82,12 @@ export interface FullResult {
 
 export interface Reporter {
   onBegin?(config: FullConfig, suite: Suite): void;
-  onTestBegin?(test: TestCase): void;
+  onTestBegin?(test: TestCase, result: TestResult): void;
-  onStdOut?(chunk: string | Buffer, test?: TestCase): void;
+  onStdOut?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void;
-  onStdErr?(chunk: string | Buffer, test?: TestCase): void;
+  onStdErr?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void;
   onTestEnd?(test: TestCase, result: TestResult): void;
+  onStepBegin?(test: TestCase, result: TestResult, step: TestStep): void;
+  onStepEnd?(test: TestCase, result: TestResult, step: TestStep): void;
   onError?(error: TestError): void;
   onEnd?(result: FullResult): void | Promise<void>;
 }