feat(test-runner): report nested steps (#8266)

docs/src/test-reporter-api/class-teststep.md

@@ -26,6 +26,11 @@ An error thrown during the step execution, if any.
 
 Start time of this particular test step.
 
+## property: TestStep.steps
+- type: <[Array]<[TestStep]>>
+
+List of steps inside this step.
+
 ## property: TestStep.title
 - type: <[string]>
 

src/test/dispatcher.ts

@@ -35,7 +35,7 @@ export class Dispatcher {
   private _freeWorkers: Worker[] = [];
   private _workerClaimers: (() => void)[] = [];
 
-  private _testById = new Map<string, { test: TestCase, result: TestResult, steps: Map<string, TestStep> }>();
+  private _testById = new Map<string, { test: TestCase, result: TestResult, steps: Map<string, TestStep>, stepStack: Set<TestStep> }>();
   private _queue: TestGroup[] = [];
   private _stopCallback = () => {};
   readonly _loader: Loader;
@@ -52,7 +52,7 @@ export class Dispatcher {
       for (const test of group.tests) {
         const result = test._appendTestResult();
         // When changing this line, change the one in retry too.
-        this._testById.set(test._id, { test, result, steps: new Map() });
+        this._testById.set(test._id, { test, result, steps: new Map(), stepStack: new Set() });
       }
     }
   }
@@ -197,6 +197,7 @@ export class Dispatcher {
         if (!this._isStopped && pair.test.expectedStatus === 'passed' && pair.test.results.length < pair.test.retries + 1) {
           pair.result = pair.test._appendTestResult();
           pair.steps = new Map();
+          pair.stepStack = new Set();
           remaining.push(pair.test);
         }
       }
@@ -280,19 +281,22 @@ export class Dispatcher {
       this._reportTestEnd(test, result);
     });
     worker.on('stepBegin', (params: StepBeginPayload) => {
-      const { test, result, steps } = this._testById.get(params.testId)!;
+      const { test, result, steps, stepStack } = this._testById.get(params.testId)!;
       const step: TestStep = {
         title: params.title,
         category: params.category,
         startTime: new Date(params.wallTime),
         duration: 0,
+        steps: [],
       };
       steps.set(params.stepId, step);
-      result.steps.push(step);
+      const parentStep = [...stepStack].pop() || result;
+      parentStep.steps.push(step);
+      stepStack.add(step);
       this._reporter.onStepBegin?.(test, result, step);
     });
     worker.on('stepEnd', (params: StepEndPayload) => {
-      const { test, result, steps } = this._testById.get(params.testId)!;
+      const { test, result, steps, stepStack } = this._testById.get(params.testId)!;
       const step = steps.get(params.stepId);
       if (!step) {
         this._reporter.onStdErr?.('Internal error: step end without step begin: ' + params.stepId, test, result);
@@ -301,6 +305,7 @@ export class Dispatcher {
       step.duration = params.wallTime - step.startTime.getTime();
       if (params.error)
         step.error = params.error;
+      stepStack.delete(step);
       steps.delete(params.stepId);
       this._reporter.onStepEnd?.(test, result, step);
     });

src/test/reporters/html.ts

@@ -199,7 +199,7 @@ class HtmlReporter {
       attachments: await this._createAttachments(testId, result),
       stdout: result.stdout,
       stderr: result.stderr,
-      steps: this._serializeSteps(result.steps)
+      steps: serializeSteps(result.steps)
     };
   }
 
@@ -249,43 +249,19 @@ class HtmlReporter {
       sha1
     };
   }
-
-  private _serializeSteps(steps: TestStep[]): JsonTestStep[] {
-    const stepStack: TestStep[] = [];
-    const result: JsonTestStep[] = [];
-    const stepMap = new Map<TestStep, JsonTestStep>();
-    for (const step of steps) {
-      let lastStep = stepStack[stepStack.length - 1];
-      while (lastStep && !containsStep(lastStep, step)) {
-        stepStack.pop();
-        lastStep = stepStack[stepStack.length - 1];
-      }
-      const collection = stepMap.get(lastStep!)?.steps || result;
-      const jsonStep = {
-        title: step.title,
-        category: step.category,
-        startTime: step.startTime.toISOString(),
-        duration: step.duration,
-        error: step.error,
-        steps: []
-      };
-      collection.push(jsonStep);
-      stepMap.set(step, jsonStep);
-      stepStack.push(step);
-    }
-    return result;
-  }
 }
 
-
+function serializeSteps(steps: TestStep[]): JsonTestStep[] {
-function containsStep(outer: TestStep, inner: TestStep): boolean {
+  return steps.map(step => {
-  if (outer.startTime.getTime() > inner.startTime.getTime())
+    return {
-    return false;
+      title: step.title,
-  if (outer.startTime.getTime() + outer.duration < inner.startTime.getTime() + inner.duration)
+      category: step.category,
-    return false;
+      startTime: step.startTime.toISOString(),
-  if (outer.startTime.getTime() + outer.duration <= inner.startTime.getTime())
+      duration: step.duration,
-    return false;
+      error: step.error,
-  return true;
+      steps: serializeSteps(step.steps),
+    };
+  });
 }
 
 function isTextAttachment(contentType: string) {

tests/playwright-test/reporter.spec.ts

@@ -165,11 +165,11 @@ test('should report expect steps', async ({ runInlineTest }) => {
         process.stdout.write(chunk);
       }
       onStepBegin(test, result, step) {
-        const copy = { ...step, startTime: undefined, duration: undefined };
+        const copy = { ...step, startTime: undefined, duration: undefined, steps: undefined };
         console.log('%%%% begin', JSON.stringify(copy));
       }
       onStepEnd(test, result, step) {
-        const copy = { ...step, startTime: undefined, duration: undefined };
+        const copy = { ...step, startTime: undefined, duration: undefined, steps: undefined };
         if (copy.error?.stack)
           copy.error.stack = '<stack>';
         console.log('%%%% end', JSON.stringify(copy));
@@ -244,11 +244,11 @@ test('should report api steps', async ({ runInlineTest }) => {
         console.log('%%%% test end ' + test.title);
       }
       onStepBegin(test, result, step) {
-        const copy = { ...step, startTime: undefined, duration: undefined };
+        const copy = { ...step, startTime: undefined, duration: undefined, steps: undefined };
         console.log('%%%% begin', JSON.stringify(copy));
       }
       onStepEnd(test, result, step) {
-        const copy = { ...step, startTime: undefined, duration: undefined };
+        const copy = { ...step, startTime: undefined, duration: undefined, steps: undefined };
         if (copy.error?.stack)
           copy.error.stack = '<stack>';
         console.log('%%%% end', JSON.stringify(copy));
@@ -335,11 +335,11 @@ test('should report api step failure', async ({ runInlineTest }) => {
         process.stdout.write(chunk);
       }
       onStepBegin(test, result, step) {
-        const copy = { ...step, startTime: undefined, duration: undefined };
+        const copy = { ...step, startTime: undefined, duration: undefined, steps: undefined };
         console.log('%%%% begin', JSON.stringify(copy));
       }
       onStepEnd(test, result, step) {
-        const copy = { ...step, startTime: undefined, duration: undefined };
+        const copy = { ...step, startTime: undefined, duration: undefined, steps: undefined };
         if (copy.error?.stack)
           copy.error.stack = '<stack>';
         console.log('%%%% end', JSON.stringify(copy));
@@ -388,11 +388,11 @@ test('should report test.step', async ({ runInlineTest }) => {
         process.stdout.write(chunk);
       }
       onStepBegin(test, result, step) {
-        const copy = { ...step, startTime: undefined, duration: undefined };
+        const copy = { ...step, startTime: undefined, duration: undefined, steps: undefined };
         console.log('%%%% begin', JSON.stringify(copy));
       }
       onStepEnd(test, result, step) {
-        const copy = { ...step, startTime: undefined, duration: undefined };
+        const copy = { ...step, startTime: undefined, duration: undefined, steps: undefined };
         if (copy.error?.stack)
           copy.error.stack = '<stack>';
         console.log('%%%% end', JSON.stringify(copy));
@@ -435,6 +435,139 @@ test('should report test.step', async ({ runInlineTest }) => {
   ]);
 });
 
+test('should report api step hierarchy', async ({ runInlineTest }) => {
+  const expectReporterJS = `
+    class Reporter {
+      onBegin(config: FullConfig, suite: Suite) {
+        this.suite = suite;
+      }
+
+      async onEnd() {
+        const processSuite = (suite: Suite) => {
+          for (const child of suite.suites)
+            processSuite(child);
+          for (const test of suite.tests) {
+            for (const result of test.results) {
+              for (const step of result.steps) {
+                console.log('%% ' + JSON.stringify(step));
+              }
+            }
+          }
+        };
+        processSuite(this.suite);
+      }
+    }
+    module.exports = Reporter;
+  `;
+
+  const result = await runInlineTest({
+    'reporter.ts': expectReporterJS,
+    'playwright.config.ts': `
+      module.exports = {
+        reporter: './reporter',
+      };
+    `,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('pass', async ({ page }) => {
+        await test.step('outer step 1', async () => {
+          await test.step('inner step 1.1', async () => {});
+          await test.step('inner step 1.2', async () => {});
+        });
+        await test.step('outer step 2', async () => {
+          await test.step('inner step 2.1', async () => {});
+          await test.step('inner step 2.2', async () => {});
+        });
+      });
+    `
+  }, { reporter: '', workers: 1 });
+
+  expect(result.exitCode).toBe(0);
+  const objects = result.output.split('\n').filter(line => line.startsWith('%% ')).map(line => line.substring(3).trim()).filter(Boolean).map(line => JSON.parse(line));
+  const distill = step => {
+    step.duration = 1;
+    step.startTime = 'time';
+    step.steps.forEach(distill);
+  };
+  objects.forEach(distill);
+  expect(objects).toEqual([
+    {
+      category: 'hook',
+      title: 'Before Hooks',
+      duration: 1,
+      startTime: 'time',
+      steps: [
+        {
+          category: 'pw:api',
+          title: 'browserContext.newPage',
+          duration: 1,
+          startTime: 'time',
+          steps: [],
+        },
+      ],
+    },
+    {
+      category: 'test.step',
+      title: 'outer step 1',
+      duration: 1,
+      startTime: 'time',
+      steps: [
+        {
+          category: 'test.step',
+          title: 'inner step 1.1',
+          duration: 1,
+          startTime: 'time',
+          steps: [],
+        },
+        {
+          category: 'test.step',
+          title: 'inner step 1.2',
+          duration: 1,
+          startTime: 'time',
+          steps: [],
+        },
+      ],
+    },
+    {
+      category: 'test.step',
+      title: 'outer step 2',
+      duration: 1,
+      startTime: 'time',
+      steps: [
+        {
+          category: 'test.step',
+          title: 'inner step 2.1',
+          duration: 1,
+          startTime: 'time',
+          steps: [],
+        },
+        {
+          category: 'test.step',
+          title: 'inner step 2.2',
+          duration: 1,
+          startTime: 'time',
+          steps: [],
+        },
+      ],
+    },
+    {
+      category: 'hook',
+      title: 'After Hooks',
+      duration: 1,
+      startTime: 'time',
+      steps: [
+        {
+          category: 'pw:api',
+          title: 'browserContext.close',
+          duration: 1,
+          startTime: 'time',
+          steps: [],
+        },
+      ],
+    },
+  ]);
+});
+
 function stripEscapedAscii(str: string) {
   return str.replace(/\\u00[a-z0-9][a-z0-9]\[[^m]+m/g, '');
 }

types/testReporter.d.ts

@@ -246,6 +246,10 @@ export interface TestStep {
    * An error thrown during the step execution, if any.
    */
   error?: TestError;
+  /**
+   * List of steps inside this step.
+   */
+  steps: TestStep[];
 }
 
 /**

utils/generate_types/overrides-testReporter.d.ts

@@ -64,6 +64,7 @@ export interface TestStep {
   startTime: Date;
   duration: number;
   error?: TestError;
+  steps: TestStep[];
 }
 
 /**