diff --git a/docs/src/test-reporter-api/class-suite.md b/docs/src/test-reporter-api/class-suite.md index 0c0c75691f..fc6b3e0aa9 100644 --- a/docs/src/test-reporter-api/class-suite.md +++ b/docs/src/test-reporter-api/class-suite.md @@ -26,6 +26,12 @@ Reporter is given a root suite in the [`method: Reporter.onBegin`] method. Returns the list of all test cases in this suite and its descendants, as opposite to [`property: Suite.tests`]. +## method: Suite.entries +* since: v1.44 +- type: <[Array]<[TestCase]|[Suite]>> + +Test cases and suites defined directly in this suite. The elements are returned in their declaration order. You can discriminate between different entry types using [`property: TestCase.type`] and [`property: Suite.type`]. + ## property: Suite.location * since: v1.10 - type: ?<[Location]> @@ -72,3 +78,10 @@ Suite title. - returns: <[Array]<[string]>> Returns a list of titles from the root down to this suite. + +## property: Suite.type +* since: v1.44 +- returns: <[SuiteType]<'root' | 'project' | 'file' | 'describe'>> + +Returns the type of the suite. The Suites form the following hierarchy: +`root` -> `project` -> `file` -> `describe` -> ...`describe` -> `test`. diff --git a/docs/src/test-reporter-api/class-testcase.md b/docs/src/test-reporter-api/class-testcase.md index d1e7b63927..0abfa5c8d4 100644 --- a/docs/src/test-reporter-api/class-testcase.md +++ b/docs/src/test-reporter-api/class-testcase.md @@ -108,3 +108,8 @@ Test title as passed to the [`method: Test.(call)`] call. Returns a list of titles from the root down to this test. +## property: TestCase.type +* since: v1.44 +- returns: <[TestCaseType]<'test'>> + +Returns type of the test. diff --git a/packages/playwright/src/common/test.ts b/packages/playwright/src/common/test.ts index 0d05ac5dfd..50f547b6ee 100644 --- a/packages/playwright/src/common/test.ts +++ b/packages/playwright/src/common/test.ts @@ -64,6 +64,14 @@ export class Suite extends Base { this._testTypeImpl = testTypeImpl; } + get type(): 'root' | 'project' | 'file' | 'describe' { + return this._type; + } + + entries() { + return this._entries; + } + get suites(): Suite[] { return this._entries.filter(entry => entry instanceof Suite) as Suite[]; } @@ -240,6 +248,7 @@ export class TestCase extends Base implements reporterTypes.TestCase { results: reporterTypes.TestResult[] = []; location: Location; parent!: Suite; + type: 'test' = 'test'; expectedStatus: reporterTypes.TestStatus = 'passed'; timeout = 0; diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts index e675ce336b..879d37e010 100644 --- a/packages/playwright/src/isomorphic/teleReceiver.ts +++ b/packages/playwright/src/isomorphic/teleReceiver.ts @@ -57,8 +57,7 @@ export type JsonProject = { export type JsonSuite = { title: string; location?: JsonLocation; - suites: JsonSuite[]; - tests: JsonTestCase[]; + entries: (JsonSuite | JsonTestCase)[]; }; export type JsonTestCase = { @@ -144,8 +143,7 @@ export class TeleReporterReceiver { } reset() { - this._rootSuite.suites = []; - this._rootSuite.tests = []; + this._rootSuite._entries = []; this._tests.clear(); } @@ -203,12 +201,12 @@ export class TeleReporterReceiver { let projectSuite = this._options.mergeProjects ? this._rootSuite.suites.find(suite => suite.project()!.name === project.name) : undefined; if (!projectSuite) { projectSuite = new TeleSuite(project.name, 'project'); - this._rootSuite.suites.push(projectSuite); - projectSuite.parent = this._rootSuite; + this._rootSuite._addSuite(projectSuite); } // Always update project in watch mode. projectSuite._project = this._parseProject(project); - this._mergeSuitesInto(project.suites, projectSuite); + for (const suite of project.suites) + this._mergeSuiteInto(suite, projectSuite); } private _onBegin() { @@ -336,31 +334,29 @@ export class TeleReporterReceiver { }); } - private _mergeSuitesInto(jsonSuites: JsonSuite[], parent: TeleSuite) { - for (const jsonSuite of jsonSuites) { - let targetSuite = parent.suites.find(s => s.title === jsonSuite.title); - if (!targetSuite) { - targetSuite = new TeleSuite(jsonSuite.title, parent._type === 'project' ? 'file' : 'describe'); - targetSuite.parent = parent; - parent.suites.push(targetSuite); - } - targetSuite.location = this._absoluteLocation(jsonSuite.location); - this._mergeSuitesInto(jsonSuite.suites, targetSuite); - this._mergeTestsInto(jsonSuite.tests, targetSuite); + private _mergeSuiteInto(jsonSuite: JsonSuite, parent: TeleSuite): void { + let targetSuite = parent.suites.find(s => s.title === jsonSuite.title); + if (!targetSuite) { + targetSuite = new TeleSuite(jsonSuite.title, parent.type === 'project' ? 'file' : 'describe'); + parent._addSuite(targetSuite); } + targetSuite.location = this._absoluteLocation(jsonSuite.location); + jsonSuite.entries.forEach(e => { + if ('testId' in e) + this._mergeTestInto(e, targetSuite!); + else + this._mergeSuiteInto(e, targetSuite!); + }); } - private _mergeTestsInto(jsonTests: JsonTestCase[], parent: TeleSuite) { - for (const jsonTest of jsonTests) { - let targetTest = this._options.mergeTestCases ? parent.tests.find(s => s.title === jsonTest.title && s.repeatEachIndex === jsonTest.repeatEachIndex) : undefined; - if (!targetTest) { - targetTest = new TeleTestCase(jsonTest.testId, jsonTest.title, this._absoluteLocation(jsonTest.location), jsonTest.repeatEachIndex); - targetTest.parent = parent; - parent.tests.push(targetTest); - this._tests.set(targetTest.id, targetTest); - } - this._updateTest(jsonTest, targetTest); + private _mergeTestInto(jsonTest: JsonTestCase, parent: TeleSuite) { + let targetTest = this._options.mergeTestCases ? parent.tests.find(s => s.title === jsonTest.title && s.repeatEachIndex === jsonTest.repeatEachIndex) : undefined; + if (!targetTest) { + targetTest = new TeleTestCase(jsonTest.testId, jsonTest.title, this._absoluteLocation(jsonTest.location), jsonTest.repeatEachIndex); + parent._addTest(targetTest); + this._tests.set(targetTest.id, targetTest); } + this._updateTest(jsonTest, targetTest); } private _updateTest(payload: JsonTestCase, test: TeleTestCase): TeleTestCase { @@ -395,28 +391,43 @@ export class TeleSuite implements reporterTypes.Suite { title: string; location?: reporterTypes.Location; parent?: TeleSuite; + _entries: (TeleSuite | TeleTestCase)[] = []; _requireFile: string = ''; - suites: TeleSuite[] = []; - tests: TeleTestCase[] = []; _timeout: number | undefined; _retries: number | undefined; _project: TeleFullProject | undefined; _parallelMode: 'none' | 'default' | 'serial' | 'parallel' = 'none'; - readonly _type: 'root' | 'project' | 'file' | 'describe'; + private readonly _type: 'root' | 'project' | 'file' | 'describe'; constructor(title: string, type: 'root' | 'project' | 'file' | 'describe') { this.title = title; this._type = type; } - allTests(): TeleTestCase[] { - const result: TeleTestCase[] = []; - const visit = (suite: TeleSuite) => { - for (const entry of [...suite.suites, ...suite.tests]) { - if (entry instanceof TeleSuite) - visit(entry); - else + get type() { + return this._type; + } + + get suites(): TeleSuite[] { + return this._entries.filter(e => e.type !== 'test') as TeleSuite[]; + } + + get tests(): TeleTestCase[] { + return this._entries.filter(e => e.type === 'test') as TeleTestCase[]; + } + + entries() { + return this._entries; + } + + allTests(): reporterTypes.TestCase[] { + const result: reporterTypes.TestCase[] = []; + const visit = (suite: reporterTypes.Suite) => { + for (const entry of suite.entries()) { + if (entry.type === 'test') result.push(entry); + else + visit(entry); } }; visit(this); @@ -434,6 +445,16 @@ export class TeleSuite implements reporterTypes.Suite { project(): TeleFullProject | undefined { return this._project ?? this.parent?.project(); } + + _addTest(test: TeleTestCase) { + test.parent = this; + this._entries.push(test); + } + + _addSuite(suite: TeleSuite) { + suite.parent = this; + this._entries.push(suite); + } } export class TeleTestCase implements reporterTypes.TestCase { @@ -442,6 +463,7 @@ export class TeleTestCase implements reporterTypes.TestCase { results: TeleTestResult[] = []; location: reporterTypes.Location; parent!: TeleSuite; + type: 'test' = 'test'; expectedStatus: reporterTypes.TestStatus = 'passed'; timeout = 0; diff --git a/packages/playwright/src/reporters/blob.ts b/packages/playwright/src/reporters/blob.ts index 019fa8b7ca..1c95658f6c 100644 --- a/packages/playwright/src/reporters/blob.ts +++ b/packages/playwright/src/reporters/blob.ts @@ -32,7 +32,7 @@ type BlobReporterOptions = { fileName?: string; }; -export const currentBlobReportVersion = 1; +export const currentBlobReportVersion = 2; export type BlobReportMetadata = { version: number; diff --git a/packages/playwright/src/reporters/html.ts b/packages/playwright/src/reporters/html.ts index 8f38346c83..9b67ea69b4 100644 --- a/packages/playwright/src/reporters/html.ts +++ b/packages/playwright/src/reporters/html.ts @@ -246,7 +246,7 @@ class HtmlBuilder { } const { testFile, testFileSummary } = fileEntry; const testEntries: TestEntry[] = []; - this._processJsonSuite(fileSuite, fileId, projectSuite.project()!.name, [], testEntries); + this._processSuite(fileSuite, fileId, projectSuite.project()!.name, [], testEntries); for (const test of testEntries) { testFile.tests.push(test.testCase); testFileSummary.tests.push(test.testCaseSummary); @@ -346,10 +346,14 @@ class HtmlBuilder { this._dataZipFile.addBuffer(Buffer.from(JSON.stringify(data)), fileName); } - private _processJsonSuite(suite: Suite, fileId: string, projectName: string, path: string[], outTests: TestEntry[]) { + private _processSuite(suite: Suite, fileId: string, projectName: string, path: string[], outTests: TestEntry[]) { const newPath = [...path, suite.title]; - suite.suites.forEach(s => this._processJsonSuite(s, fileId, projectName, newPath, outTests)); - suite.tests.forEach(t => outTests.push(this._createTestEntry(fileId, t, projectName, newPath))); + suite.entries().forEach(e => { + if (e.type === 'test') + outTests.push(this._createTestEntry(fileId, e, projectName, newPath)); + else + this._processSuite(e, fileId, projectName, newPath, outTests); + }); } private _createTestEntry(fileId: string, test: TestCasePublic, projectName: string, path: string[]): TestEntry { diff --git a/packages/playwright/src/reporters/merge.ts b/packages/playwright/src/reporters/merge.ts index 1aba179549..90eb22f1a7 100644 --- a/packages/playwright/src/reporters/merge.ts +++ b/packages/playwright/src/reporters/merge.ts @@ -18,7 +18,7 @@ import fs from 'fs'; import path from 'path'; import type { ReporterDescription } from '../../types/test'; import type { FullConfigInternal } from '../common/config'; -import type { JsonConfig, JsonEvent, JsonFullResult, JsonLocation, JsonProject, JsonSuite, JsonTestResultEnd, JsonTestStepStart } from '../isomorphic/teleReceiver'; +import type { JsonConfig, JsonEvent, JsonFullResult, JsonLocation, JsonProject, JsonSuite, JsonTestCase, JsonTestResultEnd, JsonTestStepStart } from '../isomorphic/teleReceiver'; import { TeleReporterReceiver } from '../isomorphic/teleReceiver'; import { JsonStringInternalizer, StringInternPool } from '../isomorphic/stringInternPool'; import { createReporters } from '../runner/reporters'; @@ -27,6 +27,7 @@ import { ZipFile } from 'playwright-core/lib/utils'; import { currentBlobReportVersion, type BlobReportMetadata } from './blob'; import { relativeFilePath } from '../util'; import type { TestError } from '../../types/testReporter'; +import type * as blobV1 from './versions/blobV1'; type StatusCallback = (message: string) => void; @@ -136,15 +137,17 @@ async function extractAndParseReports(dir: string, shardFiles: string[], interna const content = await zipFile.read(entryName); if (entryName.endsWith('.jsonl')) { fileName = reportNames.makeUnique(fileName); - const parsedEvents = parseCommonEvents(content); + let parsedEvents = parseCommonEvents(content); // Passing reviver to JSON.parse doesn't work, as the original strings // keep beeing used. To work around that we traverse the parsed events // as a post-processing step. internalizer.traverse(parsedEvents); + const metadata = findMetadata(parsedEvents, file); + parsedEvents = modernizer.modernize(metadata.version, parsedEvents); shardEvents.push({ file, localPath: fileName, - metadata: findMetadata(parsedEvents, file), + metadata, parsedEvents }); } @@ -386,14 +389,20 @@ class IdsPatcher { } private _updateTestIds(suite: JsonSuite) { - suite.tests.forEach(test => { - test.testId = this._mapTestId(test.testId); - if (this._botName) { - test.tags = test.tags || []; - test.tags.unshift('@' + this._botName); - } + suite.entries.forEach(entry => { + if ('testId' in entry) + this._updateTestId(entry); + else + this._updateTestIds(entry); }); - suite.suites.forEach(suite => this._updateTestIds(suite)); + } + + private _updateTestId(test: JsonTestCase) { + test.testId = this._mapTestId(test.testId); + if (this._botName) { + test.tags = test.tags || []; + test.tags.unshift('@' + this._botName); + } } private _mapTestId(testId: string): string { @@ -459,10 +468,12 @@ class PathSeparatorPatcher { this._updateLocation(suite.location); if (isFileSuite) suite.title = this._updatePath(suite.title); - for (const child of suite.suites) - this._updateSuite(child); - for (const test of suite.tests) - this._updateLocation(test.location); + for (const entry of suite.entries) { + if ('testId' in entry) + this._updateLocation(entry.location); + else + this._updateSuite(entry); + } } private _updateLocation(location?: JsonLocation) { @@ -507,3 +518,37 @@ class JsonEventPatchers { } } } + +class BlobModernizer { + modernize(fromVersion: number, events: JsonEvent[]): JsonEvent[] { + const result = []; + for (const event of events) + result.push(...this._modernize(fromVersion, event)); + return result; + } + + private _modernize(fromVersion: number, event: JsonEvent): JsonEvent[] { + let events = [event]; + for (let version = fromVersion; version < currentBlobReportVersion; ++version) + events = (this as any)[`_modernize_${version}_to_${version + 1}`].call(this, events); + return events; + } + + _modernize_1_to_2(events: JsonEvent[]): JsonEvent[] { + return events.map(event => { + if (event.method === 'onProject') { + const modernizeSuite = (suite: blobV1.JsonSuite): JsonSuite => { + const newSuites = suite.suites.map(modernizeSuite); + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { suites, tests, ...remainder } = suite; + return { entries: [...newSuites, ...tests], ...remainder }; + }; + const project = event.params.project; + project.suites = project.suites.map(modernizeSuite); + } + return event; + }); + } +} + +const modernizer = new BlobModernizer(); \ No newline at end of file diff --git a/packages/playwright/src/reporters/teleEmitter.ts b/packages/playwright/src/reporters/teleEmitter.ts index 2baef221aa..c374c5abe9 100644 --- a/packages/playwright/src/reporters/teleEmitter.ts +++ b/packages/playwright/src/reporters/teleEmitter.ts @@ -189,8 +189,11 @@ export class TeleReporterEmitter implements ReporterV2 { const result = { title: suite.title, location: this._relativeLocation(suite.location), - suites: suite.suites.map(s => this._serializeSuite(s)), - tests: suite.tests.map(t => this._serializeTest(t)), + entries: suite.entries().map(e => { + if (e.type === 'test') + return this._serializeTest(e); + return this._serializeSuite(e); + }) }; return result; } diff --git a/packages/playwright/src/reporters/versions/blobV1.ts b/packages/playwright/src/reporters/versions/blobV1.ts new file mode 100644 index 0000000000..5ea9350285 --- /dev/null +++ b/packages/playwright/src/reporters/versions/blobV1.ts @@ -0,0 +1,127 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { Metadata } from '../../../types/test'; +import type * as reporterTypes from '../../../types/testReporter'; + +export type JsonLocation = reporterTypes.Location; +export type JsonError = string; +export type JsonStackFrame = { file: string, line: number, column: number }; + +export type JsonStdIOType = 'stdout' | 'stderr'; + +export type JsonConfig = Pick; + +export type JsonPattern = { + s?: string; + r?: { source: string, flags: string }; +}; + +export type JsonProject = { + grep: JsonPattern[]; + grepInvert: JsonPattern[]; + metadata: Metadata; + name: string; + dependencies: string[]; + // This is relative to root dir. + snapshotDir: string; + // This is relative to root dir. + outputDir: string; + repeatEach: number; + retries: number; + suites: JsonSuite[]; + teardown?: string; + // This is relative to root dir. + testDir: string; + testIgnore: JsonPattern[]; + testMatch: JsonPattern[]; + timeout: number; +}; + +export type JsonSuite = { + title: string; + location?: JsonLocation; + suites: JsonSuite[]; + tests: JsonTestCase[]; +}; + +export type JsonTestCase = { + testId: string; + title: string; + location: JsonLocation; + retries: number; + tags?: string[]; + repeatEachIndex: number; +}; + +export type JsonTestEnd = { + testId: string; + expectedStatus: reporterTypes.TestStatus; + timeout: number; + annotations: { type: string, description?: string }[]; +}; + +export type JsonTestResultStart = { + id: string; + retry: number; + workerIndex: number; + parallelIndex: number; + startTime: number; +}; + +export type JsonAttachment = Omit & { base64?: string }; + +export type JsonTestResultEnd = { + id: string; + duration: number; + status: reporterTypes.TestStatus; + errors: reporterTypes.TestError[]; + attachments: JsonAttachment[]; +}; + +export type JsonTestStepStart = { + id: string; + parentStepId?: string; + title: string; + category: string, + startTime: number; + location?: reporterTypes.Location; +}; + +export type JsonTestStepEnd = { + id: string; + duration: number; + error?: reporterTypes.TestError; +}; + +export type JsonFullResult = { + status: reporterTypes.FullResult['status']; + startTime: number; + duration: number; +}; + +export type JsonEvent = { + method: string; + params: any +}; + +export type BlobReportMetadata = { + version: number; + userAgent: string; + name?: string; + shard?: { total: number, current: number }; + pathSeparator?: string; +}; diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.ts index ed6f62e71d..987d51e004 100644 --- a/packages/playwright/types/testReporter.d.ts +++ b/packages/playwright/types/testReporter.d.ts @@ -40,6 +40,11 @@ export type { FullConfig, TestStatus, FullProject } from './test'; * [reporter.onBegin(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-on-begin) method. */ export interface Suite { + /** + * Returns the type of the suite. The Suites form the following hierarchy: `root` -> `project` -> `file` -> `describe` + * -> ...`describe` -> `test`. + */ + type: 'root' | 'project' | 'file' | 'describe'; /** * Configuration of the project this suite belongs to, or [void] for the root suite. */ @@ -50,6 +55,14 @@ export interface Suite { */ allTests(): Array; + /** + * Test cases and suites defined directly in this suite. The elements are returned in their declaration order. You can + * discriminate between different entry types using + * [testCase.type](https://playwright.dev/docs/api/class-testcase#test-case-type) and + * [suite.type](https://playwright.dev/docs/api/class-suite#suite-type). + */ + entries(): Array; + /** * Returns a list of titles from the root down to this suite. */ @@ -98,6 +111,10 @@ export interface Suite { * projects' suites. */ export interface TestCase { + /** + * Returns type of the test. + */ + type: 'test'; /** * Expected test status. * - Tests marked as diff --git a/tests/assets/blob-1.42.zip b/tests/assets/blob-1.42.zip new file mode 100644 index 0000000000..da16671ccc Binary files /dev/null and b/tests/assets/blob-1.42.zip differ diff --git a/tests/playwright-test/reporter-blob.spec.ts b/tests/playwright-test/reporter-blob.spec.ts index e57d8ff223..2ddfcfcc96 100644 --- a/tests/playwright-test/reporter-blob.spec.ts +++ b/tests/playwright-test/reporter-blob.spec.ts @@ -1281,7 +1281,7 @@ test('blob report should include version', async ({ runInlineTest }) => { const events = await extractReport(test.info().outputPath('blob-report', 'report.zip'), test.info().outputPath('tmp')); const metadataEvent = events.find(e => e.method === 'onBlobReportMetadata'); - expect(metadataEvent.params.version).toBe(1); + expect(metadataEvent.params.version).toBe(2); expect(metadataEvent.params.userAgent).toBe(getUserAgent()); }); @@ -1703,3 +1703,52 @@ test('TestSuite.project() should return owning project', async ({ runInlineTest, expect(exitCode).toBe(0); expect(output).toContain(`test project: my-project`); }); + +test('open blob-1.42', async ({ runInlineTest, mergeReports }) => { + test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/29984' }); + await runInlineTest({ + 'echo-reporter.js': ` + export default class EchoReporter { + lines = []; + onTestBegin(test) { + this.lines.push(test.titlePath().join(' > ')); + } + onEnd() { + console.log(this.lines.join('\\n')); + } + }; + `, + 'merge.config.ts': `module.exports = { + testDir: 'mergeRoot', + reporter: './echo-reporter.js' + };`, + }); + + const blobDir = test.info().outputPath('blob-report'); + await fs.promises.mkdir(blobDir, { recursive: true }); + await fs.promises.copyFile(path.join(__dirname, '../assets/blob-1.42.zip'), path.join(blobDir, 'blob-1.42.zip')); + + const { exitCode, output } = await mergeReports(blobDir, undefined, { additionalArgs: ['--config', 'merge.config.ts'] }); + expect(exitCode).toBe(0); + expect(output).toContain(` > chromium > example.spec.ts > test 0 + > chromium > example.spec.ts > describe 1 > describe 2 > test 3 + > chromium > example.spec.ts > describe 1 > describe 2 > test 4 + > chromium > example.spec.ts > describe 1 > describe 2 > test 2 + > chromium > example.spec.ts > describe 1 > test 1 + > chromium > example.spec.ts > describe 1 > test 5 + > chromium > example.spec.ts > test 6 + > firefox > example.spec.ts > describe 1 > describe 2 > test 2 + > firefox > example.spec.ts > describe 1 > describe 2 > test 3 + > firefox > example.spec.ts > test 0 + > firefox > example.spec.ts > describe 1 > describe 2 > test 4 + > firefox > example.spec.ts > describe 1 > test 1 + > firefox > example.spec.ts > test 6 + > firefox > example.spec.ts > describe 1 > test 5 + > webkit > example.spec.ts > describe 1 > describe 2 > test 4 + > webkit > example.spec.ts > test 0 + > webkit > example.spec.ts > describe 1 > test 1 + > webkit > example.spec.ts > describe 1 > describe 2 > test 2 + > webkit > example.spec.ts > describe 1 > describe 2 > test 3 + > webkit > example.spec.ts > test 6 + > webkit > example.spec.ts > describe 1 > test 5`); +}); diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts index ad39504c64..f8a8215960 100644 --- a/tests/playwright-test/reporter-html.spec.ts +++ b/tests/playwright-test/reporter-html.spec.ts @@ -2058,6 +2058,39 @@ for (const useIntermediateMergeReport of [false, true] as const) { ]); }); + test('html report should preserve declaration order within file', async ({ runInlineTest, showReport, page }) => { + test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/29984' }); + await runInlineTest({ + 'main.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('test 0', async ({}) => {}); + test.describe('describe 1', () => { + test('test 1', async ({}) => {}); + test.describe('describe 2', () => { + test('test 2', async ({}) => {}); + test('test 3', async ({}) => {}); + test('test 4', async ({}) => {}); + }); + test('test 5', async ({}) => {}); + }); + test('test 6', async ({}) => {}); + `, + }, { reporter: 'html' }, { PW_TEST_HTML_REPORT_OPEN: 'never' }); + + await showReport(); + + // Failing test first, then sorted by the run order. + await expect(page.locator('.test-file-title')).toHaveText([ + /test 0/, + /describe 1 › test 1/, + /describe 1 › describe 2 › test 2/, + /describe 1 › describe 2 › test 3/, + /describe 1 › describe 2 › test 4/, + /describe 1 › test 5/, + /test 6/, + ]); + }); + test('tests should filter by file', async ({ runInlineTest, showReport, page }) => { const result = await runInlineTest({ 'file-a.test.js': ` diff --git a/utils/generate_types/overrides-testReporter.d.ts b/utils/generate_types/overrides-testReporter.d.ts index 9aeb9c9d31..8faa956928 100644 --- a/utils/generate_types/overrides-testReporter.d.ts +++ b/utils/generate_types/overrides-testReporter.d.ts @@ -18,10 +18,12 @@ import type { FullConfig, FullProject, TestStatus, Metadata } from './test'; export type { FullConfig, TestStatus, FullProject } from './test'; export interface Suite { + type: 'root' | 'project' | 'file' | 'describe'; project(): FullProject | undefined; } export interface TestCase { + type: 'test'; expectedStatus: TestStatus; }