From 3001c9ac736d2f1bb8ca00b95a4f994467bc7fa7 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Fri, 29 Mar 2024 10:12:33 -0700 Subject: [PATCH] fix: preserve test declaration order in html and merged report (#30159) * Add `Suite.entries` that returns tests and suites in their declaration order * Exposed `Suite.type` and `TestCase.type` for discriminating between different entry types. * Blob report format is updated to store entries instead of separate lists for suites and tests. * Bumped blob format version to 2, added modernizer. Fixes https://github.com/microsoft/playwright/issues/29984 --- docs/src/test-reporter-api/class-suite.md | 13 ++ docs/src/test-reporter-api/class-testcase.md | 5 + packages/playwright/src/common/test.ts | 9 ++ .../playwright/src/isomorphic/teleReceiver.ts | 98 ++++++++------ packages/playwright/src/reporters/blob.ts | 2 +- packages/playwright/src/reporters/html.ts | 12 +- packages/playwright/src/reporters/merge.ts | 73 ++++++++-- .../playwright/src/reporters/teleEmitter.ts | 7 +- .../src/reporters/versions/blobV1.ts | 127 ++++++++++++++++++ packages/playwright/types/testReporter.d.ts | 17 +++ tests/assets/blob-1.42.zip | Bin 0 -> 3931 bytes tests/playwright-test/reporter-blob.spec.ts | 51 ++++++- tests/playwright-test/reporter-html.spec.ts | 33 +++++ .../overrides-testReporter.d.ts | 2 + 14 files changed, 389 insertions(+), 60 deletions(-) create mode 100644 packages/playwright/src/reporters/versions/blobV1.ts create mode 100644 tests/assets/blob-1.42.zip diff --git a/docs/src/test-reporter-api/class-suite.md b/docs/src/test-reporter-api/class-suite.md index 0c0c75691f..fc6b3e0aa9 100644 --- a/docs/src/test-reporter-api/class-suite.md +++ b/docs/src/test-reporter-api/class-suite.md @@ -26,6 +26,12 @@ Reporter is given a root suite in the [`method: Reporter.onBegin`] method. Returns the list of all test cases in this suite and its descendants, as opposite to [`property: Suite.tests`]. +## method: Suite.entries +* since: v1.44 +- type: <[Array]<[TestCase]|[Suite]>> + +Test cases and suites defined directly in this suite. The elements are returned in their declaration order. You can discriminate between different entry types using [`property: TestCase.type`] and [`property: Suite.type`]. + ## property: Suite.location * since: v1.10 - type: ?<[Location]> @@ -72,3 +78,10 @@ Suite title. - returns: <[Array]<[string]>> Returns a list of titles from the root down to this suite. + +## property: Suite.type +* since: v1.44 +- returns: <[SuiteType]<'root' | 'project' | 'file' | 'describe'>> + +Returns the type of the suite. The Suites form the following hierarchy: +`root` -> `project` -> `file` -> `describe` -> ...`describe` -> `test`. diff --git a/docs/src/test-reporter-api/class-testcase.md b/docs/src/test-reporter-api/class-testcase.md index d1e7b63927..0abfa5c8d4 100644 --- a/docs/src/test-reporter-api/class-testcase.md +++ b/docs/src/test-reporter-api/class-testcase.md @@ -108,3 +108,8 @@ Test title as passed to the [`method: Test.(call)`] call. Returns a list of titles from the root down to this test. +## property: TestCase.type +* since: v1.44 +- returns: <[TestCaseType]<'test'>> + +Returns type of the test. diff --git a/packages/playwright/src/common/test.ts b/packages/playwright/src/common/test.ts index 0d05ac5dfd..50f547b6ee 100644 --- a/packages/playwright/src/common/test.ts +++ b/packages/playwright/src/common/test.ts @@ -64,6 +64,14 @@ export class Suite extends Base { this._testTypeImpl = testTypeImpl; } + get type(): 'root' | 'project' | 'file' | 'describe' { + return this._type; + } + + entries() { + return this._entries; + } + get suites(): Suite[] { return this._entries.filter(entry => entry instanceof Suite) as Suite[]; } @@ -240,6 +248,7 @@ export class TestCase extends Base implements reporterTypes.TestCase { results: reporterTypes.TestResult[] = []; location: Location; parent!: Suite; + type: 'test' = 'test'; expectedStatus: reporterTypes.TestStatus = 'passed'; timeout = 0; diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts index e675ce336b..879d37e010 100644 --- a/packages/playwright/src/isomorphic/teleReceiver.ts +++ b/packages/playwright/src/isomorphic/teleReceiver.ts @@ -57,8 +57,7 @@ export type JsonProject = { export type JsonSuite = { title: string; location?: JsonLocation; - suites: JsonSuite[]; - tests: JsonTestCase[]; + entries: (JsonSuite | JsonTestCase)[]; }; export type JsonTestCase = { @@ -144,8 +143,7 @@ export class TeleReporterReceiver { } reset() { - this._rootSuite.suites = []; - this._rootSuite.tests = []; + this._rootSuite._entries = []; this._tests.clear(); } @@ -203,12 +201,12 @@ export class TeleReporterReceiver { let projectSuite = this._options.mergeProjects ? this._rootSuite.suites.find(suite => suite.project()!.name === project.name) : undefined; if (!projectSuite) { projectSuite = new TeleSuite(project.name, 'project'); - this._rootSuite.suites.push(projectSuite); - projectSuite.parent = this._rootSuite; + this._rootSuite._addSuite(projectSuite); } // Always update project in watch mode. projectSuite._project = this._parseProject(project); - this._mergeSuitesInto(project.suites, projectSuite); + for (const suite of project.suites) + this._mergeSuiteInto(suite, projectSuite); } private _onBegin() { @@ -336,31 +334,29 @@ export class TeleReporterReceiver { }); } - private _mergeSuitesInto(jsonSuites: JsonSuite[], parent: TeleSuite) { - for (const jsonSuite of jsonSuites) { - let targetSuite = parent.suites.find(s => s.title === jsonSuite.title); - if (!targetSuite) { - targetSuite = new TeleSuite(jsonSuite.title, parent._type === 'project' ? 'file' : 'describe'); - targetSuite.parent = parent; - parent.suites.push(targetSuite); - } - targetSuite.location = this._absoluteLocation(jsonSuite.location); - this._mergeSuitesInto(jsonSuite.suites, targetSuite); - this._mergeTestsInto(jsonSuite.tests, targetSuite); + private _mergeSuiteInto(jsonSuite: JsonSuite, parent: TeleSuite): void { + let targetSuite = parent.suites.find(s => s.title === jsonSuite.title); + if (!targetSuite) { + targetSuite = new TeleSuite(jsonSuite.title, parent.type === 'project' ? 'file' : 'describe'); + parent._addSuite(targetSuite); } + targetSuite.location = this._absoluteLocation(jsonSuite.location); + jsonSuite.entries.forEach(e => { + if ('testId' in e) + this._mergeTestInto(e, targetSuite!); + else + this._mergeSuiteInto(e, targetSuite!); + }); } - private _mergeTestsInto(jsonTests: JsonTestCase[], parent: TeleSuite) { - for (const jsonTest of jsonTests) { - let targetTest = this._options.mergeTestCases ? parent.tests.find(s => s.title === jsonTest.title && s.repeatEachIndex === jsonTest.repeatEachIndex) : undefined; - if (!targetTest) { - targetTest = new TeleTestCase(jsonTest.testId, jsonTest.title, this._absoluteLocation(jsonTest.location), jsonTest.repeatEachIndex); - targetTest.parent = parent; - parent.tests.push(targetTest); - this._tests.set(targetTest.id, targetTest); - } - this._updateTest(jsonTest, targetTest); + private _mergeTestInto(jsonTest: JsonTestCase, parent: TeleSuite) { + let targetTest = this._options.mergeTestCases ? parent.tests.find(s => s.title === jsonTest.title && s.repeatEachIndex === jsonTest.repeatEachIndex) : undefined; + if (!targetTest) { + targetTest = new TeleTestCase(jsonTest.testId, jsonTest.title, this._absoluteLocation(jsonTest.location), jsonTest.repeatEachIndex); + parent._addTest(targetTest); + this._tests.set(targetTest.id, targetTest); } + this._updateTest(jsonTest, targetTest); } private _updateTest(payload: JsonTestCase, test: TeleTestCase): TeleTestCase { @@ -395,28 +391,43 @@ export class TeleSuite implements reporterTypes.Suite { title: string; location?: reporterTypes.Location; parent?: TeleSuite; + _entries: (TeleSuite | TeleTestCase)[] = []; _requireFile: string = ''; - suites: TeleSuite[] = []; - tests: TeleTestCase[] = []; _timeout: number | undefined; _retries: number | undefined; _project: TeleFullProject | undefined; _parallelMode: 'none' | 'default' | 'serial' | 'parallel' = 'none'; - readonly _type: 'root' | 'project' | 'file' | 'describe'; + private readonly _type: 'root' | 'project' | 'file' | 'describe'; constructor(title: string, type: 'root' | 'project' | 'file' | 'describe') { this.title = title; this._type = type; } - allTests(): TeleTestCase[] { - const result: TeleTestCase[] = []; - const visit = (suite: TeleSuite) => { - for (const entry of [...suite.suites, ...suite.tests]) { - if (entry instanceof TeleSuite) - visit(entry); - else + get type() { + return this._type; + } + + get suites(): TeleSuite[] { + return this._entries.filter(e => e.type !== 'test') as TeleSuite[]; + } + + get tests(): TeleTestCase[] { + return this._entries.filter(e => e.type === 'test') as TeleTestCase[]; + } + + entries() { + return this._entries; + } + + allTests(): reporterTypes.TestCase[] { + const result: reporterTypes.TestCase[] = []; + const visit = (suite: reporterTypes.Suite) => { + for (const entry of suite.entries()) { + if (entry.type === 'test') result.push(entry); + else + visit(entry); } }; visit(this); @@ -434,6 +445,16 @@ export class TeleSuite implements reporterTypes.Suite { project(): TeleFullProject | undefined { return this._project ?? this.parent?.project(); } + + _addTest(test: TeleTestCase) { + test.parent = this; + this._entries.push(test); + } + + _addSuite(suite: TeleSuite) { + suite.parent = this; + this._entries.push(suite); + } } export class TeleTestCase implements reporterTypes.TestCase { @@ -442,6 +463,7 @@ export class TeleTestCase implements reporterTypes.TestCase { results: TeleTestResult[] = []; location: reporterTypes.Location; parent!: TeleSuite; + type: 'test' = 'test'; expectedStatus: reporterTypes.TestStatus = 'passed'; timeout = 0; diff --git a/packages/playwright/src/reporters/blob.ts b/packages/playwright/src/reporters/blob.ts index 019fa8b7ca..1c95658f6c 100644 --- a/packages/playwright/src/reporters/blob.ts +++ b/packages/playwright/src/reporters/blob.ts @@ -32,7 +32,7 @@ type BlobReporterOptions = { fileName?: string; }; -export const currentBlobReportVersion = 1; +export const currentBlobReportVersion = 2; export type BlobReportMetadata = { version: number; diff --git a/packages/playwright/src/reporters/html.ts b/packages/playwright/src/reporters/html.ts index 8f38346c83..9b67ea69b4 100644 --- a/packages/playwright/src/reporters/html.ts +++ b/packages/playwright/src/reporters/html.ts @@ -246,7 +246,7 @@ class HtmlBuilder { } const { testFile, testFileSummary } = fileEntry; const testEntries: TestEntry[] = []; - this._processJsonSuite(fileSuite, fileId, projectSuite.project()!.name, [], testEntries); + this._processSuite(fileSuite, fileId, projectSuite.project()!.name, [], testEntries); for (const test of testEntries) { testFile.tests.push(test.testCase); testFileSummary.tests.push(test.testCaseSummary); @@ -346,10 +346,14 @@ class HtmlBuilder { this._dataZipFile.addBuffer(Buffer.from(JSON.stringify(data)), fileName); } - private _processJsonSuite(suite: Suite, fileId: string, projectName: string, path: string[], outTests: TestEntry[]) { + private _processSuite(suite: Suite, fileId: string, projectName: string, path: string[], outTests: TestEntry[]) { const newPath = [...path, suite.title]; - suite.suites.forEach(s => this._processJsonSuite(s, fileId, projectName, newPath, outTests)); - suite.tests.forEach(t => outTests.push(this._createTestEntry(fileId, t, projectName, newPath))); + suite.entries().forEach(e => { + if (e.type === 'test') + outTests.push(this._createTestEntry(fileId, e, projectName, newPath)); + else + this._processSuite(e, fileId, projectName, newPath, outTests); + }); } private _createTestEntry(fileId: string, test: TestCasePublic, projectName: string, path: string[]): TestEntry { diff --git a/packages/playwright/src/reporters/merge.ts b/packages/playwright/src/reporters/merge.ts index 1aba179549..90eb22f1a7 100644 --- a/packages/playwright/src/reporters/merge.ts +++ b/packages/playwright/src/reporters/merge.ts @@ -18,7 +18,7 @@ import fs from 'fs'; import path from 'path'; import type { ReporterDescription } from '../../types/test'; import type { FullConfigInternal } from '../common/config'; -import type { JsonConfig, JsonEvent, JsonFullResult, JsonLocation, JsonProject, JsonSuite, JsonTestResultEnd, JsonTestStepStart } from '../isomorphic/teleReceiver'; +import type { JsonConfig, JsonEvent, JsonFullResult, JsonLocation, JsonProject, JsonSuite, JsonTestCase, JsonTestResultEnd, JsonTestStepStart } from '../isomorphic/teleReceiver'; import { TeleReporterReceiver } from '../isomorphic/teleReceiver'; import { JsonStringInternalizer, StringInternPool } from '../isomorphic/stringInternPool'; import { createReporters } from '../runner/reporters'; @@ -27,6 +27,7 @@ import { ZipFile } from 'playwright-core/lib/utils'; import { currentBlobReportVersion, type BlobReportMetadata } from './blob'; import { relativeFilePath } from '../util'; import type { TestError } from '../../types/testReporter'; +import type * as blobV1 from './versions/blobV1'; type StatusCallback = (message: string) => void; @@ -136,15 +137,17 @@ async function extractAndParseReports(dir: string, shardFiles: string[], interna const content = await zipFile.read(entryName); if (entryName.endsWith('.jsonl')) { fileName = reportNames.makeUnique(fileName); - const parsedEvents = parseCommonEvents(content); + let parsedEvents = parseCommonEvents(content); // Passing reviver to JSON.parse doesn't work, as the original strings // keep beeing used. To work around that we traverse the parsed events // as a post-processing step. internalizer.traverse(parsedEvents); + const metadata = findMetadata(parsedEvents, file); + parsedEvents = modernizer.modernize(metadata.version, parsedEvents); shardEvents.push({ file, localPath: fileName, - metadata: findMetadata(parsedEvents, file), + metadata, parsedEvents }); } @@ -386,14 +389,20 @@ class IdsPatcher { } private _updateTestIds(suite: JsonSuite) { - suite.tests.forEach(test => { - test.testId = this._mapTestId(test.testId); - if (this._botName) { - test.tags = test.tags || []; - test.tags.unshift('@' + this._botName); - } + suite.entries.forEach(entry => { + if ('testId' in entry) + this._updateTestId(entry); + else + this._updateTestIds(entry); }); - suite.suites.forEach(suite => this._updateTestIds(suite)); + } + + private _updateTestId(test: JsonTestCase) { + test.testId = this._mapTestId(test.testId); + if (this._botName) { + test.tags = test.tags || []; + test.tags.unshift('@' + this._botName); + } } private _mapTestId(testId: string): string { @@ -459,10 +468,12 @@ class PathSeparatorPatcher { this._updateLocation(suite.location); if (isFileSuite) suite.title = this._updatePath(suite.title); - for (const child of suite.suites) - this._updateSuite(child); - for (const test of suite.tests) - this._updateLocation(test.location); + for (const entry of suite.entries) { + if ('testId' in entry) + this._updateLocation(entry.location); + else + this._updateSuite(entry); + } } private _updateLocation(location?: JsonLocation) { @@ -507,3 +518,37 @@ class JsonEventPatchers { } } } + +class BlobModernizer { + modernize(fromVersion: number, events: JsonEvent[]): JsonEvent[] { + const result = []; + for (const event of events) + result.push(...this._modernize(fromVersion, event)); + return result; + } + + private _modernize(fromVersion: number, event: JsonEvent): JsonEvent[] { + let events = [event]; + for (let version = fromVersion; version < currentBlobReportVersion; ++version) + events = (this as any)[`_modernize_${version}_to_${version + 1}`].call(this, events); + return events; + } + + _modernize_1_to_2(events: JsonEvent[]): JsonEvent[] { + return events.map(event => { + if (event.method === 'onProject') { + const modernizeSuite = (suite: blobV1.JsonSuite): JsonSuite => { + const newSuites = suite.suites.map(modernizeSuite); + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { suites, tests, ...remainder } = suite; + return { entries: [...newSuites, ...tests], ...remainder }; + }; + const project = event.params.project; + project.suites = project.suites.map(modernizeSuite); + } + return event; + }); + } +} + +const modernizer = new BlobModernizer(); \ No newline at end of file diff --git a/packages/playwright/src/reporters/teleEmitter.ts b/packages/playwright/src/reporters/teleEmitter.ts index 2baef221aa..c374c5abe9 100644 --- a/packages/playwright/src/reporters/teleEmitter.ts +++ b/packages/playwright/src/reporters/teleEmitter.ts @@ -189,8 +189,11 @@ export class TeleReporterEmitter implements ReporterV2 { const result = { title: suite.title, location: this._relativeLocation(suite.location), - suites: suite.suites.map(s => this._serializeSuite(s)), - tests: suite.tests.map(t => this._serializeTest(t)), + entries: suite.entries().map(e => { + if (e.type === 'test') + return this._serializeTest(e); + return this._serializeSuite(e); + }) }; return result; } diff --git a/packages/playwright/src/reporters/versions/blobV1.ts b/packages/playwright/src/reporters/versions/blobV1.ts new file mode 100644 index 0000000000..5ea9350285 --- /dev/null +++ b/packages/playwright/src/reporters/versions/blobV1.ts @@ -0,0 +1,127 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { Metadata } from '../../../types/test'; +import type * as reporterTypes from '../../../types/testReporter'; + +export type JsonLocation = reporterTypes.Location; +export type JsonError = string; +export type JsonStackFrame = { file: string, line: number, column: number }; + +export type JsonStdIOType = 'stdout' | 'stderr'; + +export type JsonConfig = Pick; + +export type JsonPattern = { + s?: string; + r?: { source: string, flags: string }; +}; + +export type JsonProject = { + grep: JsonPattern[]; + grepInvert: JsonPattern[]; + metadata: Metadata; + name: string; + dependencies: string[]; + // This is relative to root dir. + snapshotDir: string; + // This is relative to root dir. + outputDir: string; + repeatEach: number; + retries: number; + suites: JsonSuite[]; + teardown?: string; + // This is relative to root dir. + testDir: string; + testIgnore: JsonPattern[]; + testMatch: JsonPattern[]; + timeout: number; +}; + +export type JsonSuite = { + title: string; + location?: JsonLocation; + suites: JsonSuite[]; + tests: JsonTestCase[]; +}; + +export type JsonTestCase = { + testId: string; + title: string; + location: JsonLocation; + retries: number; + tags?: string[]; + repeatEachIndex: number; +}; + +export type JsonTestEnd = { + testId: string; + expectedStatus: reporterTypes.TestStatus; + timeout: number; + annotations: { type: string, description?: string }[]; +}; + +export type JsonTestResultStart = { + id: string; + retry: number; + workerIndex: number; + parallelIndex: number; + startTime: number; +}; + +export type JsonAttachment = Omit & { base64?: string }; + +export type JsonTestResultEnd = { + id: string; + duration: number; + status: reporterTypes.TestStatus; + errors: reporterTypes.TestError[]; + attachments: JsonAttachment[]; +}; + +export type JsonTestStepStart = { + id: string; + parentStepId?: string; + title: string; + category: string, + startTime: number; + location?: reporterTypes.Location; +}; + +export type JsonTestStepEnd = { + id: string; + duration: number; + error?: reporterTypes.TestError; +}; + +export type JsonFullResult = { + status: reporterTypes.FullResult['status']; + startTime: number; + duration: number; +}; + +export type JsonEvent = { + method: string; + params: any +}; + +export type BlobReportMetadata = { + version: number; + userAgent: string; + name?: string; + shard?: { total: number, current: number }; + pathSeparator?: string; +}; diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.ts index ed6f62e71d..987d51e004 100644 --- a/packages/playwright/types/testReporter.d.ts +++ b/packages/playwright/types/testReporter.d.ts @@ -40,6 +40,11 @@ export type { FullConfig, TestStatus, FullProject } from './test'; * [reporter.onBegin(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-on-begin) method. */ export interface Suite { + /** + * Returns the type of the suite. The Suites form the following hierarchy: `root` -> `project` -> `file` -> `describe` + * -> ...`describe` -> `test`. + */ + type: 'root' | 'project' | 'file' | 'describe'; /** * Configuration of the project this suite belongs to, or [void] for the root suite. */ @@ -50,6 +55,14 @@ export interface Suite { */ allTests(): Array; + /** + * Test cases and suites defined directly in this suite. The elements are returned in their declaration order. You can + * discriminate between different entry types using + * [testCase.type](https://playwright.dev/docs/api/class-testcase#test-case-type) and + * [suite.type](https://playwright.dev/docs/api/class-suite#suite-type). + */ + entries(): Array; + /** * Returns a list of titles from the root down to this suite. */ @@ -98,6 +111,10 @@ export interface Suite { * projects' suites. */ export interface TestCase { + /** + * Returns type of the test. + */ + type: 'test'; /** * Expected test status. * - Tests marked as diff --git a/tests/assets/blob-1.42.zip b/tests/assets/blob-1.42.zip new file mode 100644 index 0000000000000000000000000000000000000000..da16671ccc9e0b774f9bcef0a41bd5df867c12c5 GIT binary patch literal 3931 zcmV-h52Wx=O9KQH00;;O0QP-+SO5S3000000000001N;C0CHt;Z*p`lYIARHZ0(&( zb0f!*fbaV&Ci)~PXijClvWk1~TDx{E4hlO~-q?dehpeou8CoR30GifJjQQU;8k`Y` zM}rNZpoVvK8-PF~zs$-OF7w6hY}(wwRr9!z`5)fNv~6yhzueKspXayl-nVbz+Ujd~^A*kSz5UlW_r!nx z{teh``}LdYEX!LB*KYaD_Wk$LJ-MCDo15ls^YrvDpI(ac@3U#SefKz*H3@k6?&@EE zy1kR;rn$a;`|yhl*Ux{tZWql}^X_hz={xoW0#FYE8j9@~e<&*ENRFU9)ZTo#XaOX=qFAk_Ym`2Es?b7|+dpOw&k^7(7c z^NY#5>1-~|%@035zy6Ne?rT4MY8K5+^UXJJzq$UmuNDu&|LFeu_4V(+;(y$K{lnkd z9~Zy>>ciK~Q}?|uS~p55b=52$Z`x%+OGy9Z2X)9!h9Gp2TFhfmKxvFp)yw^LcN^X%^N ze%joqr(GguS@8Tey%h7yTW{dkkkGCq^ao3*e+>!!O2Y6`!p}qJv#{Nu>XYJkPB3`X zoC{fqF{!A+3!d)FOO^wq!j)9M$YAN=A3I@?!Qmu|SB59Rizd z;-*~wK;;b)Y^l6=b9q?mZr(k1F?*czGW&niMY+4>N1Nv9%O@w(zstORjzd`6v$i%)V zYl4e_VzUmByy#2F!~iTYJM!9TRf>%^*-OjBL9@;fi?`aukWz%Wl}yIS#E8Y9wFaUW zCB{oqTh-O*nK*^)Qb0%Ki|1gKQt1*hF@;c&auHv=hg@Po{(qZ^f0p#qZM#auzr6k} z5C4{j|0(mZkI{OUv~_4rR6&Hc^2!)_I2Y~3AxZ=mT~ZdaHS-)J58Hytic1kn$;?F; zb#ZxEQ`8A^w$3A#%t6dnDj6dW3nkAKb0LqEQxa{rbT@h)*3OpXlyWACwFQjCatV1j zV3uIAh$3KZEwlQ+nTNlXcem4KIM$f13i{qyGZ`3X*3elZfpa=L=90xGvKm;EPdSD{@oeaSoaRdh&yVxZ zbw~|_QDV?*2;&FR;L&6%VxwXeO902V6cb{xW>Ae0UIUC4b*i``8u3542cF_X9MY&f29QUnb^jy_u(D>d_UNU)pWsCuKgL%|HJd zwJq`cdzrS+)qI^ex~IaK46JU}%vqd6D1lXo5|U;UDQfV#fCT0kD#Xzm<9L+ZSvv^T zhpLQIdfcrc_s0XED5E`J0(SuO!ng*N)ry!&t zxg;Npu_d?!Y?G(p?PUz(QMk`DjC)XO4&&EVG4e3pJHea`oLsJ%GYX5!Rsy6{Pz$C& zoW0m+Fgc%3&!usD5F9g(M<^m{=hFHR-EOBC7?VS-M4^G;va!0Tq>FGmJqYb?BaCgl z0o>;bLOW3^2B86}c!8!)jY3EDEO&2J+yF7kDMQPNNetd|FhZ#mYdnipxm4+t$8FC% zhNx%%;AZ2_YaIocJRpe*)+W(L8%G+oSYM@6zJus+j~gXV)DD;RAp$}OTwF+r3QuFK`5LeUnhpzn>VlYudH4P6HfAsUXVP*mM|yYi?Ldj}Y~xU-=jSn`Zv^L;*i z83Wf~QQay41TRutzzmTE6S7a7)yeg$R#hg=V{osGw~cT@y}BEvVgMeXiWh}Jjre<* zF-HkOwLP&u1Y&TGqc;{%E900@YXD+ay6f3ph9{1~eV$UN2c>2y^tvi4dg3wDte#UG z8ypTyOiFu1o4m3?ipH)TSFngBZ~CWMaq#{KyJL7BL)3HrV{?%%Bx^)V%7Fqi7Zf4w z6PV)gdGWZ1C=8wo*COgUSg>g$DS?VcU!p>59G6fmg^+N$7O-d2m>ayssuoetS(ptX z8%;=-j1$m`jAI*gCPK^XjoL?qgD3a3hbu8EBvNh6O}%oI}(p87(dvPogDarj)!oeY1^Kx(@ojpg0*A5ZAk- zMih_=L_uXs#)P6YD0E=t5Z)AyHzJ4x*fMB*juc6u2IvynZw>TD3j?qq{9 z(HjJ(w4v^pLx!l7cI3jIO=AaiVRsN6?s21pn%c9c`VcM@(@o|nu+j<>ad6HV6O6My ztzCX{WZJJozSqGf1D$JqV|)-7d`iU`av~U%a}tzfOX!{T6*7gdbEIdVCr$=>;reU? zZ48LYSYi$1o?4jAn#m)-839-kz8P%0+L0jhYBBuCiTF^UJ* z-q=we0(DA33Av1^xIo4ifMkFaSakm6GrksTw68|tKF=1b9+Zl%I6xIEH2dp&;wTAa zUtGPAtO(+rB8UpeGdmp!B0GnPU5HVg?uk`SDUZQ@9#8B+spyFVR8i3rkC|rmTsXJ6 zH04cHDP+=-J;wr4I0VoJv^L4?xlahTFBIyVXZamP+c8KzSFCLy`H;YKhLTL~z7Q4L zjM{gE#7g!cZJ#&l+mcibY4??T8%SES5menVRL6`#TOgl;;o!kIy=O}j48JMJ@5tvz z?S$KXGvww+8boLOLeP?e(R}KtEca0$G(ZBUz&;l6N^qm6QZDi0J6_ zxu5Szb5*T58EDCk>o~w-&{?P0wMC6YpIks-=Z&}K^xm+C8VgjTfWNl7co@Pc!?yMs zTYWne7BI!69BomoT`*c_$r6bQ+O>b0jO^z>_58%-*9vd4Ecu(Z}}K0o>7k?TCh>_Mryy7;;(MqVY~TdGe6mg(!eI5`5L7;7vNQ1HRp7z=ttF;Gsw zk5Z7(E31bh_56Pm8%UCr&qT3Ec4#bSW1Mxd zbhV43y+{Z28FwIEMy6M&z%g5}Ts*?^K2i;A8I=nvmD2-e9rC^7b24yju3x~xHYxzE zONtPZ*O0uI=%Kr2CIvb@(VIFK@b4YRlYwJ+ePLWw<`Qfcq@n{l(`{#E(z*b9sZite zppP9`2aNXDb_5)TFiL&;T5dS`t}BibMr-dcs&9v`+XjXrl1c%%7&K9w_)?5cn*F)P-}qV38sANu zEE?~Ef&i#PDoDVbi|CV2v|Os^4Y2rm_H*>0)Vwn1byd9Z$X|b|ILaVCk1=~tDjIWu zDn>QtD7C { const events = await extractReport(test.info().outputPath('blob-report', 'report.zip'), test.info().outputPath('tmp')); const metadataEvent = events.find(e => e.method === 'onBlobReportMetadata'); - expect(metadataEvent.params.version).toBe(1); + expect(metadataEvent.params.version).toBe(2); expect(metadataEvent.params.userAgent).toBe(getUserAgent()); }); @@ -1703,3 +1703,52 @@ test('TestSuite.project() should return owning project', async ({ runInlineTest, expect(exitCode).toBe(0); expect(output).toContain(`test project: my-project`); }); + +test('open blob-1.42', async ({ runInlineTest, mergeReports }) => { + test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/29984' }); + await runInlineTest({ + 'echo-reporter.js': ` + export default class EchoReporter { + lines = []; + onTestBegin(test) { + this.lines.push(test.titlePath().join(' > ')); + } + onEnd() { + console.log(this.lines.join('\\n')); + } + }; + `, + 'merge.config.ts': `module.exports = { + testDir: 'mergeRoot', + reporter: './echo-reporter.js' + };`, + }); + + const blobDir = test.info().outputPath('blob-report'); + await fs.promises.mkdir(blobDir, { recursive: true }); + await fs.promises.copyFile(path.join(__dirname, '../assets/blob-1.42.zip'), path.join(blobDir, 'blob-1.42.zip')); + + const { exitCode, output } = await mergeReports(blobDir, undefined, { additionalArgs: ['--config', 'merge.config.ts'] }); + expect(exitCode).toBe(0); + expect(output).toContain(` > chromium > example.spec.ts > test 0 + > chromium > example.spec.ts > describe 1 > describe 2 > test 3 + > chromium > example.spec.ts > describe 1 > describe 2 > test 4 + > chromium > example.spec.ts > describe 1 > describe 2 > test 2 + > chromium > example.spec.ts > describe 1 > test 1 + > chromium > example.spec.ts > describe 1 > test 5 + > chromium > example.spec.ts > test 6 + > firefox > example.spec.ts > describe 1 > describe 2 > test 2 + > firefox > example.spec.ts > describe 1 > describe 2 > test 3 + > firefox > example.spec.ts > test 0 + > firefox > example.spec.ts > describe 1 > describe 2 > test 4 + > firefox > example.spec.ts > describe 1 > test 1 + > firefox > example.spec.ts > test 6 + > firefox > example.spec.ts > describe 1 > test 5 + > webkit > example.spec.ts > describe 1 > describe 2 > test 4 + > webkit > example.spec.ts > test 0 + > webkit > example.spec.ts > describe 1 > test 1 + > webkit > example.spec.ts > describe 1 > describe 2 > test 2 + > webkit > example.spec.ts > describe 1 > describe 2 > test 3 + > webkit > example.spec.ts > test 6 + > webkit > example.spec.ts > describe 1 > test 5`); +}); diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts index ad39504c64..f8a8215960 100644 --- a/tests/playwright-test/reporter-html.spec.ts +++ b/tests/playwright-test/reporter-html.spec.ts @@ -2058,6 +2058,39 @@ for (const useIntermediateMergeReport of [false, true] as const) { ]); }); + test('html report should preserve declaration order within file', async ({ runInlineTest, showReport, page }) => { + test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/29984' }); + await runInlineTest({ + 'main.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('test 0', async ({}) => {}); + test.describe('describe 1', () => { + test('test 1', async ({}) => {}); + test.describe('describe 2', () => { + test('test 2', async ({}) => {}); + test('test 3', async ({}) => {}); + test('test 4', async ({}) => {}); + }); + test('test 5', async ({}) => {}); + }); + test('test 6', async ({}) => {}); + `, + }, { reporter: 'html' }, { PW_TEST_HTML_REPORT_OPEN: 'never' }); + + await showReport(); + + // Failing test first, then sorted by the run order. + await expect(page.locator('.test-file-title')).toHaveText([ + /test 0/, + /describe 1 › test 1/, + /describe 1 › describe 2 › test 2/, + /describe 1 › describe 2 › test 3/, + /describe 1 › describe 2 › test 4/, + /describe 1 › test 5/, + /test 6/, + ]); + }); + test('tests should filter by file', async ({ runInlineTest, showReport, page }) => { const result = await runInlineTest({ 'file-a.test.js': ` diff --git a/utils/generate_types/overrides-testReporter.d.ts b/utils/generate_types/overrides-testReporter.d.ts index 9aeb9c9d31..8faa956928 100644 --- a/utils/generate_types/overrides-testReporter.d.ts +++ b/utils/generate_types/overrides-testReporter.d.ts @@ -18,10 +18,12 @@ import type { FullConfig, FullProject, TestStatus, Metadata } from './test'; export type { FullConfig, TestStatus, FullProject } from './test'; export interface Suite { + type: 'root' | 'project' | 'file' | 'describe'; project(): FullProject | undefined; } export interface TestCase { + type: 'test'; expectedStatus: TestStatus; }