fix: preserve test declaration order in html and merged report

Fixes https://github.com/microsoft/playwright/issues/29984

docs/src/test-reporter-api/class-suite.md

@@ -26,6 +26,12 @@ Reporter is given a root suite in the [`method: Reporter.onBegin`] method.
 
 Returns the list of all test cases in this suite and its descendants, as opposite to [`property: Suite.tests`].
 
+## method: Suite.entries
+* since: v1.44
+- type: <[Array]<[TestCase]|[Suite]>>
+
+Test cases and suites defined directly in this suite. The elements returned in their declaration order.
+
 ## property: Suite.location
 * since: v1.10
 - type: ?<[Location]>
@@ -72,3 +78,9 @@ Suite title.
 - returns: <[Array]<[string]>>
 
 Returns a list of titles from the root down to this suite.
+
+## property: Suite.type
+* since: v1.44
+- returns: <[SuiteType]<'root' | 'project' | 'file' | 'describe'>>
+
+Returns type of the suite.

docs/src/test-reporter-api/class-testcase.md

@@ -108,3 +108,8 @@ Test title as passed to the [`method: Test.(call)`] call.
 
 Returns a list of titles from the root down to this test.
 
+## property: TestCase.type
+* since: v1.44
+- returns: <[TestCaseType]<'test'>>
+
+Returns type of the test.

packages/playwright/src/common/test.ts

@@ -22,7 +22,7 @@ import type { Annotation, FixturesWithLocation, FullProjectInternal } from './co
 import type { FullProject } from '../../types/test';
 import type { Location } from '../../types/testReporter';
 
-class Base {
+abstract class Base {
   title: string;
   _only = false;
   _requireFile: string = '';
@@ -39,7 +39,7 @@ export type Modifier = {
   description: string | undefined
 };
 
-export class Suite extends Base {
+export class Suite extends Base implements reporterTypes.Suite {
   location?: Location;
   parent?: Suite;
   _use: FixturesWithLocation[] = [];
@@ -64,6 +64,14 @@ export class Suite extends Base {
     this._testTypeImpl = testTypeImpl;
   }
 
+  get type(): 'root' | 'project' | 'file' | 'describe' {
+    return this._type;
+  }
+
+  entries(): (reporterTypes.Suite | reporterTypes.TestCase)[] {
+    return this._entries;
+  }
+
   get suites(): Suite[] {
     return this._entries.filter(entry => entry instanceof Suite) as Suite[];
   }
@@ -240,6 +248,7 @@ export class TestCase extends Base implements reporterTypes.TestCase {
   results: reporterTypes.TestResult[] = [];
   location: Location;
   parent!: Suite;
+  type: 'test' = 'test';
 
   expectedStatus: reporterTypes.TestStatus = 'passed';
   timeout = 0;

packages/playwright/src/isomorphic/teleReceiver.ts

@@ -57,8 +57,9 @@ export type JsonProject = {
 export type JsonSuite = {
   title: string;
   location?: JsonLocation;
-  suites: JsonSuite[];
-  tests: JsonTestCase[];
+  entries: (JsonSuite | JsonTestCase)[];
+  suites?: JsonSuite[]; // used before v1.44
+  tests?: JsonTestCase[]; // used before v1.44
 };
 
 export type JsonTestCase = {
@@ -130,7 +131,7 @@ type TeleReporterReceiverOptions = {
 };
 
 export class TeleReporterReceiver {
-  private _rootSuite: TeleSuite;
+  private _rootSuite: TeleRootSuite;
   private _options: TeleReporterReceiverOptions;
   private _reporter: Partial<ReporterV2>;
   private _tests = new Map<string, TeleTestCase>();
@@ -138,14 +139,13 @@ export class TeleReporterReceiver {
   private _config!: reporterTypes.FullConfig;
 
   constructor(reporter: Partial<ReporterV2>, options: TeleReporterReceiverOptions) {
-    this._rootSuite = new TeleSuite('', 'root');
+    this._rootSuite = new TeleRootSuite();
     this._options = options;
     this._reporter = reporter;
   }
 
   reset() {
-    this._rootSuite.suites = [];
-    this._rootSuite.tests = [];
+    this._rootSuite.reset();
     this._tests.clear();
   }
 
@@ -202,13 +202,13 @@ export class TeleReporterReceiver {
   private _onProject(project: JsonProject) {
     let projectSuite = this._options.mergeProjects ? this._rootSuite.suites.find(suite => suite.project()!.name === project.name) : undefined;
     if (!projectSuite) {
-      projectSuite = new TeleSuite(project.name, 'project');
-      this._rootSuite.suites.push(projectSuite);
-      projectSuite.parent = this._rootSuite;
+      projectSuite = new TeleProjectSuite(project.name);
+      this._rootSuite._addSuite(projectSuite);
     }
     // Always update project in watch mode.
-    projectSuite._project = this._parseProject(project);
-    this._mergeSuitesInto(project.suites, projectSuite);
+    (projectSuite as TeleProjectSuite)._project = this._parseProject(project);
+    for (const suite of project.suites)
+      this._mergeSuiteInto(suite, projectSuite);
   }
 
   private _onBegin() {
@@ -336,31 +336,37 @@ export class TeleReporterReceiver {
     });
   }
 
-  private _mergeSuitesInto(jsonSuites: JsonSuite[], parent: TeleSuite) {
-    for (const jsonSuite of jsonSuites) {
-      let targetSuite = parent.suites.find(s => s.title === jsonSuite.title);
-      if (!targetSuite) {
-        targetSuite = new TeleSuite(jsonSuite.title, parent._type === 'project' ? 'file' : 'describe');
-        targetSuite.parent = parent;
-        parent.suites.push(targetSuite);
-      }
-      targetSuite.location = this._absoluteLocation(jsonSuite.location);
-      this._mergeSuitesInto(jsonSuite.suites, targetSuite);
-      this._mergeTestsInto(jsonSuite.tests, targetSuite);
+  private _mergeSuiteInto(jsonSuite: JsonSuite, parent: TeleSuite): void {
+    let targetSuite = parent.suites.find(s => s.title === jsonSuite.title);
+    if (!targetSuite) {
+      targetSuite = parent.type === 'project' ? new TeleFileSuite(jsonSuite.title) : new TeleDescribeSuite(jsonSuite.title);
+      parent._addSuite(targetSuite);
+    }
+    targetSuite.location = this._absoluteLocation(jsonSuite.location);
+    if (jsonSuite.entries) {
+      jsonSuite.entries.forEach(e => {
+        if ('testId' in e)
+          this._mergeTestInto(e, targetSuite!);
+        else
+          this._mergeSuiteInto(e, targetSuite!);
+      });
+    } else {
+      // before 1.44
+      for (const jsonChildSuite of jsonSuite.suites!)
+        this._mergeSuiteInto(jsonChildSuite, targetSuite);
+      for (const jsonTest of jsonSuite.tests!)
+        this._mergeTestInto(jsonTest, targetSuite);
     }
   }
 
-  private _mergeTestsInto(jsonTests: JsonTestCase[], parent: TeleSuite) {
-    for (const jsonTest of jsonTests) {
-      let targetTest = this._options.mergeTestCases ? parent.tests.find(s => s.title === jsonTest.title && s.repeatEachIndex === jsonTest.repeatEachIndex) : undefined;
-      if (!targetTest) {
-        targetTest = new TeleTestCase(jsonTest.testId, jsonTest.title, this._absoluteLocation(jsonTest.location), jsonTest.repeatEachIndex);
-        targetTest.parent = parent;
-        parent.tests.push(targetTest);
-        this._tests.set(targetTest.id, targetTest);
-      }
-      this._updateTest(jsonTest, targetTest);
+  private _mergeTestInto(jsonTest: JsonTestCase, parent: TeleSuite) {
+    let targetTest = this._options.mergeTestCases ? parent.tests.find(s => s.title === jsonTest.title && s.repeatEachIndex === jsonTest.repeatEachIndex) : undefined;
+    if (!targetTest) {
+      targetTest = new TeleTestCase(jsonTest.testId, jsonTest.title, this._absoluteLocation(jsonTest.location), jsonTest.repeatEachIndex);
+      parent._addTest(targetTest);
+      this._tests.set(targetTest.id, targetTest);
     }
+    this._updateTest(jsonTest, targetTest);
   }
 
   private _updateTest(payload: JsonTestCase, test: TeleTestCase): TeleTestCase {
@@ -391,32 +397,44 @@ export class TeleReporterReceiver {
   }
 }
 
-export class TeleSuite implements reporterTypes.Suite {
+export abstract class TeleSuite implements reporterTypes.Suite {
   title: string;
-  location?: reporterTypes.Location;
-  parent?: TeleSuite;
+  abstract location?: reporterTypes.Location;
+  abstract parent?: reporterTypes.Suite;
   _requireFile: string = '';
-  suites: TeleSuite[] = [];
-  tests: TeleTestCase[] = [];
   _timeout: number | undefined;
   _retries: number | undefined;
-  _project: TeleFullProject | undefined;
   _parallelMode: 'none' | 'default' | 'serial' | 'parallel' = 'none';
   readonly _type: 'root' | 'project' | 'file' | 'describe';
+  _entries: (TeleSuite | TeleTestCase)[] = [];
 
   constructor(title: string, type: 'root' | 'project' | 'file' | 'describe') {
     this.title = title;
     this._type = type;
   }
 
-  allTests(): TeleTestCase[] {
-    const result: TeleTestCase[] = [];
-    const visit = (suite: TeleSuite) => {
+  abstract type: 'root' | 'project' | 'file' | 'describe';
+
+  get suites(): TeleSuite[] {
+    return this._entries.filter(e => e.type !== 'test') as TeleSuite[];
+  }
+
+  get tests(): TeleTestCase[] {
+    return this._entries.filter(e => e.type === 'test') as TeleTestCase[];
+  }
+
+  entries(): (reporterTypes.Suite | reporterTypes.TestCase)[] {
+    return this._entries;
+  }
+
+  allTests(): reporterTypes.TestCase[] {
+    const result: reporterTypes.TestCase[] = [];
+    const visit = (suite: reporterTypes.Suite) => {
       for (const entry of [...suite.suites, ...suite.tests]) {
-        if (entry instanceof TeleSuite)
-          visit(entry);
-        else
+        if (entry.type === 'test')
           result.push(entry);
+        else
+          visit(entry);
       }
     };
     visit(this);
@@ -431,8 +449,72 @@ export class TeleSuite implements reporterTypes.Suite {
     return titlePath;
   }
 
-  project(): TeleFullProject | undefined {
-    return this._project ?? this.parent?.project();
+  abstract project(): TeleFullProject | undefined;
+
+  _addTest(test: TeleTestCase) {
+    test.parent = this;
+    this._entries.push(test);
+    this.tests.push(test);
+  }
+
+  _addSuite(suite: TeleSuite) {
+    suite.parent = this;
+    this._entries.push(suite);
+    this.suites.push(suite);
+  }
+}
+
+export class TeleDescribeSuite extends TeleSuite implements reporterTypes.DescribeSuite {
+  type: 'describe' = 'describe';
+  override location!: reporterTypes.Location;
+  override parent!: TeleDescribeSuite | TeleFileSuite;
+  constructor(title: string) {
+    super(title, 'project');
+  }
+  override project(): TeleFullProject {
+    return this.parent.project()!;
+  }
+}
+
+export class TeleFileSuite extends TeleSuite implements reporterTypes.FileSuite {
+  type: 'file' = 'file';
+  override location!: reporterTypes.Location;
+  override parent!: TeleProjectSuite;
+  constructor(title: string) {
+    super(title, 'project');
+  }
+  override project(): TeleFullProject {
+    return this.parent.project();
+  }
+}
+
+export class TeleProjectSuite extends TeleSuite implements reporterTypes.ProjectSuite {
+  type: 'project' = 'project';
+  _project!: TeleFullProject;
+  override location = undefined;
+  override parent!: TeleRootSuite;
+  constructor(title: string) {
+    super(title, 'project');
+  }
+  override project(): TeleFullProject {
+    return this._project!;
+  }
+}
+
+export class TeleRootSuite extends TeleSuite implements reporterTypes.RootSuite {
+  type: 'root' = 'root';
+  override location = undefined;
+  override parent = undefined;
+  // override suites: TeleProjectSuite[] = [];
+  constructor() {
+    super('', 'root');
+  }
+  override project(): undefined {
+    return undefined;
+  }
+
+  reset() {
+    this._entries = [];
   }
 }
 
@@ -442,6 +524,7 @@ export class TeleTestCase implements reporterTypes.TestCase {
   results: TeleTestResult[] = [];
   location: reporterTypes.Location;
   parent!: TeleSuite;
+  type: 'test' = 'test';
 
   expectedStatus: reporterTypes.TestStatus = 'passed';
   timeout = 0;

packages/playwright/src/reporters/html.ts

@@ -246,7 +246,7 @@ class HtmlBuilder {
         }
         const { testFile, testFileSummary } = fileEntry;
         const testEntries: TestEntry[] = [];
-        this._processJsonSuite(fileSuite, fileId, projectSuite.project()!.name, [], testEntries);
+        this._processSuite(fileSuite, fileId, projectSuite.project()!.name, [], testEntries);
         for (const test of testEntries) {
           testFile.tests.push(test.testCase);
           testFileSummary.tests.push(test.testCaseSummary);
@@ -346,10 +346,16 @@ class HtmlBuilder {
     this._dataZipFile.addBuffer(Buffer.from(JSON.stringify(data)), fileName);
   }
 
-  private _processJsonSuite(suite: Suite, fileId: string, projectName: string, path: string[], outTests: TestEntry[]) {
+  private _processSuite(suite: Suite, fileId: string, projectName: string, path: string[], outTests: TestEntry[]) {
     const newPath = [...path, suite.title];
-    suite.suites.forEach(s => this._processJsonSuite(s, fileId, projectName, newPath, outTests));
-    suite.tests.forEach(t => outTests.push(this._createTestEntry(fileId, t, projectName, newPath)));
+    suite.entries().forEach(e => {
+      if (e.type === 'test')
+        outTests.push(this._createTestEntry(fileId, e, projectName, newPath));
+      else
+        this._processSuite(e, fileId, projectName, newPath, outTests);
+    });
+    // suite.suites.forEach(s => this._processSuite(s, fileId, projectName, newPath, outTests));
+    // suite.tests.forEach(t => outTests.push(this._createTestEntry(fileId, t, projectName, newPath)));
   }
 
   private _createTestEntry(fileId: string, test: TestCasePublic, projectName: string, path: string[]): TestEntry {

packages/playwright/src/reporters/merge.ts

@@ -18,7 +18,7 @@ import fs from 'fs';
 import path from 'path';
 import type { ReporterDescription } from '../../types/test';
 import type { FullConfigInternal } from '../common/config';
-import type { JsonConfig, JsonEvent, JsonFullResult, JsonLocation, JsonProject, JsonSuite, JsonTestResultEnd, JsonTestStepStart } from '../isomorphic/teleReceiver';
+import type { JsonConfig, JsonEvent, JsonFullResult, JsonLocation, JsonProject, JsonSuite, JsonTestCase, JsonTestResultEnd, JsonTestStepStart } from '../isomorphic/teleReceiver';
 import { TeleReporterReceiver } from '../isomorphic/teleReceiver';
 import { JsonStringInternalizer, StringInternPool } from '../isomorphic/stringInternPool';
 import { createReporters } from '../runner/reporters';
@@ -387,14 +387,31 @@ class IdsPatcher {
   }
 
   private _updateTestIds(suite: JsonSuite) {
-    suite.tests.forEach(test => {
-      test.testId = this._mapTestId(test.testId);
-      if (this._botName) {
-        test.tags = test.tags || [];
-        test.tags.unshift('@' + this._botName);
-      }
-    });
-    suite.suites.forEach(suite => this._updateTestIds(suite));
+    if (suite.entries) { // Starting 1.44
+      suite.entries.forEach(entry => {
+        if ('testId' in entry)
+          this._updateTestId(entry);
+        else
+          this._updateTestIds(entry);
+      });
+    } else { // before 1.44
+      suite.tests!.forEach(test => {
+        test.testId = this._mapTestId(test.testId);
+        if (this._botName) {
+          test.tags = test.tags || [];
+          test.tags.unshift('@' + this._botName);
+        }
+      });
+      suite.suites!.forEach(suite => this._updateTestIds(suite));
+    }
+  }
+
+  private _updateTestId(test: JsonTestCase) {
+    test.testId = this._mapTestId(test.testId);
+    if (this._botName) {
+      test.tags = test.tags || [];
+      test.tags.unshift('@' + this._botName);
+    }
   }
 
   private _mapTestId(testId: string): string {
@@ -460,10 +477,19 @@ class PathSeparatorPatcher {
     this._updateLocation(suite.location);
     if (isFileSuite)
       suite.title = this._updatePath(suite.title);
-    for (const child of suite.suites)
-      this._updateSuite(child);
-    for (const test of suite.tests)
-      this._updateLocation(test.location);
+    if (suite.entries) { // starting 1.44
+      for (const entry of suite.entries) {
+        if ('testId' in entry)
+          this._updateLocation(entry.location);
+        else
+          this._updateSuite(entry);
+      }
+    } else { // before 1.44
+      for (const child of suite.suites!)
+        this._updateSuite(child);
+      for (const test of suite.tests!)
+        this._updateLocation(test.location);
+    }
   }
 
   private _updateLocation(location?: JsonLocation) {

packages/playwright/src/reporters/teleEmitter.ts

@@ -189,8 +189,11 @@ export class TeleReporterEmitter implements ReporterV2 {
     const result = {
       title: suite.title,
       location: this._relativeLocation(suite.location),
-      suites: suite.suites.map(s => this._serializeSuite(s)),
-      tests: suite.tests.map(t => this._serializeTest(t)),
+      entries: suite.entries().map(e => {
+        if (e.type === 'test')
+          return this._serializeTest(e);
+        return this._serializeSuite(e);
+      })
     };
     return result;
   }

packages/playwright/types/testReporter.d.ts

@@ -40,6 +40,10 @@ export type { FullConfig, TestStatus, FullProject } from './test';
  * [reporter.onBegin(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-on-begin) method.
  */
 export interface Suite {
+  /**
+   * Returns type of the suite.
+   */
+  type: 'root' | 'project' | 'file' | 'describe';
   /**
    * Configuration of the project this suite belongs to, or [void] for the root suite.
    */
@@ -50,6 +54,11 @@ export interface Suite {
    */
   allTests(): Array<TestCase>;
 
+  /**
+   * Test cases and suites defined directly in this suite. The elements returned in their declaration order.
+   */
+  entries(): Array<TestCase|Suite>;
+
   /**
    * Returns a list of titles from the root down to this suite.
    */
@@ -98,6 +107,10 @@ export interface Suite {
  * projects' suites.
  */
 export interface TestCase {
+  /**
+   * Returns type of the test.
+   */
+  type: 'test';
   /**
    * Expected test status.
    * - Tests marked as
@@ -224,6 +237,33 @@ export interface TestCase {
   title: string;
 }
 
+export interface FileSuite extends Suite {
+  type: 'file';
+  location: Location;
+  parent: Suite;
+//  suites: Array<DescribeSuite>;
+}
+
+export interface DescribeSuite extends Suite {
+  type: 'describe';
+  location: Location;
+  parent: Suite;
+//  suites: Array<DescribeSuite>;
+}
+
+export interface ProjectSuite extends Suite {
+  type: 'project';
+  location: undefined;
+//  suites: Array<FileSuite>;
+}
+
+export interface RootSuite extends Suite {
+  type: 'root';
+  parent: undefined;
+  location: undefined;
+//  suites: Array<ProjectSuite>;
+}
+
 /**
  * A result of a single {@link TestCase} run.
  */

packages/trace-viewer/src/ui/teleSuiteUpdater.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { TeleReporterReceiver, TeleSuite } from '@testIsomorphic/teleReceiver';
+import { TeleReporterReceiver, TeleRootSuite } from '@testIsomorphic/teleReceiver';
 import { statusEx } from '@testIsomorphic/testTree';
 import type { ReporterV2 } from 'playwright/src/reporters/reporterV2';
 import type * as reporterTypes from 'playwright/types/testReporter';
@@ -137,7 +137,7 @@ export class TeleSuiteUpdater {
 
   asModel(): TestModel {
     return {
-      rootSuite: this.rootSuite || new TeleSuite('', 'root'),
+      rootSuite: this.rootSuite || new TeleRootSuite(),
       config: this.config!,
       loadErrors: this.loadErrors,
       progress: this.progress,

packages/trace-viewer/src/ui/uiModeView.tsx

@@ -17,7 +17,7 @@
 import '@web/third_party/vscode/codicon.css';
 import '@web/common.css';
 import React from 'react';
-import { TeleSuite } from '@testIsomorphic/teleReceiver';
+import { TeleRootSuite } from '@testIsomorphic/teleReceiver';
 import { TeleSuiteUpdater } from './teleSuiteUpdater';
 import type { Progress } from './uiModeModel';
 import type { TeleTestCase } from '@testIsomorphic/teleReceiver';
@@ -226,7 +226,7 @@ export const UIModeView: React.FC<{}> = ({
   // Test tree is built from the model and filters.
   const { testTree } = React.useMemo(() => {
     if (!testModel)
-      return { testTree: new TestTree('', new TeleSuite('', 'root'), [], projectFilters, pathSeparator) };
+      return { testTree: new TestTree('', new TeleRootSuite(), [], projectFilters, pathSeparator) };
     const testTree = new TestTree('', testModel.rootSuite, testModel.loadErrors, projectFilters, pathSeparator);
     testTree.filterTree(filterText, statusFilters, runningState?.testIds);
     testTree.sortAndPropagateStatus();

tests/playwright-test/reporter-html.spec.ts

@@ -2058,6 +2058,39 @@ for (const useIntermediateMergeReport of [false, true] as const) {
       ]);
     });
 
+    test('html report should preserve declaration order within file', async ({ runInlineTest, showReport, page }) => {
+      test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/29984' });
+      await runInlineTest({
+        'main.spec.ts': `
+          import { test, expect } from '@playwright/test';
+          test('test 0', async ({}) => {});
+          test.describe('describe 1', () => {
+            test('test 1', async ({}) => {});
+            test.describe('describe 2', () => {
+              test('test 2', async ({}) => {});
+              test('test 3', async ({}) => {});
+              test('test 4', async ({}) => {});
+            });
+            test('test 5', async ({}) => {});
+          });
+          test('test 6', async ({}) => {});
+        `,
+      }, { reporter: 'html' }, { PW_TEST_HTML_REPORT_OPEN: 'never' });
+
+      await showReport();
+
+      // Failing test first, then sorted by the run order.
+      await expect(page.locator('.test-file-title')).toHaveText([
+        /test 0/,
+        /describe 1 › test 1/,
+        /describe 1 › describe 2 › test 2/,
+        /describe 1 › describe 2 › test 3/,
+        /describe 1 › describe 2 › test 4/,
+        /describe 1 › test 5/,
+        /test 6/,
+      ]);
+    });
+
     test('tests should filter by file', async ({ runInlineTest, showReport, page }) => {
       const result = await runInlineTest({
         'file-a.test.js': `

utils/generate_types/index.js

@@ -600,6 +600,10 @@ class TypesGenerator {
         'JSONReportTest',
         'JSONReportTestResult',
         'JSONReportTestStep',
+        'FileSuite',
+        'DescribeSuite',
+        'ProjectSuite',
+        'RootSuite',
       ]),
       includeExperimental,
     });

utils/generate_types/overrides-testReporter.d.ts

@@ -18,13 +18,42 @@ import type { FullConfig, FullProject, TestStatus, Metadata } from './test';
 export type { FullConfig, TestStatus, FullProject } from './test';
 
 export interface Suite {
+  type: 'root' | 'project' | 'file' | 'describe';
   project(): FullProject | undefined;
 }
 
 export interface TestCase {
+  type: 'test';
   expectedStatus: TestStatus;
 }
 
+export interface FileSuite extends Suite {
+  type: 'file';
+  location: Location;
+  parent: Suite;
+//  suites: Array<DescribeSuite>;
+}
+
+export interface DescribeSuite extends Suite {
+  type: 'describe';
+  location: Location;
+  parent: Suite;
+//  suites: Array<DescribeSuite>;
+}
+
+export interface ProjectSuite extends Suite {
+  type: 'project';
+  location: undefined;
+//  suites: Array<FileSuite>;
+}
+
+export interface RootSuite extends Suite {
+  type: 'root';
+  parent: undefined;
+  location: undefined;
+//  suites: Array<ProjectSuite>;
+}
+
 export interface TestResult {
   status: TestStatus;
 }