fix(merger): total time is sum of shard total times (#23534)

packages/playwright-test/src/reporters/html.ts

@@ -20,7 +20,7 @@ import path from 'path';
 import type { TransformCallback } from 'stream';
 import { Transform } from 'stream';
 import type { FullConfig, Reporter, Suite } from '../../types/testReporter';
-import { HttpServer, assert, calculateSha1, monotonicTime, copyFileAndMakeWritable, removeFolders } from 'playwright-core/lib/utils';
+import { HttpServer, assert, calculateSha1, copyFileAndMakeWritable, removeFolders } from 'playwright-core/lib/utils';
 import type { JsonAttachment, JsonReport, JsonSuite, JsonTestCase, JsonTestResult, JsonTestStep } from './raw';
 import RawReporter from './raw';
 import { stripAnsiEscapes } from './base';
@@ -49,7 +49,6 @@ type HtmlReporterOptions = {
 class HtmlReporter implements Reporter {
   private config!: FullConfig;
   private suite!: Suite;
-  private _montonicStartTime: number = 0;
   private _options: HtmlReporterOptions;
   private _outputFolder!: string;
   private _attachmentsBaseURL!: string;
@@ -65,7 +64,6 @@ class HtmlReporter implements Reporter {
   }
 
   onBegin(config: FullConfig, suite: Suite) {
-    this._montonicStartTime = monotonicTime();
     this.config = config;
     const { outputFolder, open, attachmentsBaseURL } = this._resolveOptions();
     this._outputFolder = outputFolder;
@@ -102,7 +100,6 @@ class HtmlReporter implements Reporter {
   }
 
   async onEnd() {
-    const duration = monotonicTime() - this._montonicStartTime;
     const projectSuites = this.suite.suites;
     const reports = projectSuites.map(suite => {
       const rawReporter = new RawReporter();
@@ -111,7 +108,7 @@ class HtmlReporter implements Reporter {
     });
     await removeFolders([this._outputFolder]);
     const builder = new HtmlBuilder(this._outputFolder, this._attachmentsBaseURL);
-    this._buildResult = await builder.build({ ...this.config.metadata, duration }, reports);
+    this._buildResult = await builder.build(this.config.metadata, reports);
   }
 
   async onExit() {
@@ -208,7 +205,7 @@ class HtmlBuilder {
     this._attachmentsBaseURL = attachmentsBaseURL;
   }
 
-  async build(metadata: Metadata & { duration: number }, rawReports: JsonReport[]): Promise<{ ok: boolean, singleTestId: string | undefined }> {
+  async build(metadata: Metadata, rawReports: JsonReport[]): Promise<{ ok: boolean, singleTestId: string | undefined }> {
 
     const data = new Map<string, { testFile: TestFile, testFileSummary: TestFileSummary }>();
     for (const projectJson of rawReports) {
@@ -265,7 +262,7 @@ class HtmlBuilder {
       metadata,
       files: [...data.values()].map(e => e.testFileSummary),
       projectNames: rawReports.map(r => r.project.name),
-      stats: { ...[...data.values()].reduce((a, e) => addStats(a, e.testFileSummary.stats), emptyStats()), duration: metadata.duration }
+      stats: { ...[...data.values()].reduce((a, e) => addStats(a, e.testFileSummary.stats), emptyStats()), duration: metadata.totalTime }
     };
     htmlReport.files.sort((f1, f2) => {
       const w1 = f1.stats.unexpected * 1000 + f1.stats.flaky;

packages/playwright-test/src/reporters/internalReporter.ts

@@ -22,6 +22,7 @@ import { Suite } from '../common/test';
 import type { FullConfigInternal } from '../common/config';
 import { Multiplexer } from './multiplexer';
 import { prepareErrorStack, relativeFilePath } from './base';
+import { monotonicTime } from 'playwright-core/lib/utils';
 
 type StdIOChunk = {
   chunk: string | Buffer;
@@ -33,6 +34,7 @@ export class InternalReporter {
   private _multiplexer: Multiplexer;
   private _deferred: { error?: TestError, stdout?: StdIOChunk, stderr?: StdIOChunk }[] | null = [];
   private _config!: FullConfigInternal;
+  private _montonicStartTime: number = 0;
 
   constructor(reporters: Reporter[]) {
     this._multiplexer = new Multiplexer(reporters);
@@ -43,6 +45,7 @@ export class InternalReporter {
   }
 
   onBegin(config: FullConfig, suite: Suite) {
+    this._montonicStartTime = monotonicTime();
     this._multiplexer.onBegin(config, suite);
 
     const deferred = this._deferred!;
@@ -83,7 +86,9 @@ export class InternalReporter {
     this._multiplexer.onTestEnd(test, result);
   }
 
-  async onEnd() { }
+  async onEnd() {
+    this._config.config.metadata.totalTime = monotonicTime() - this._montonicStartTime;
+  }
 
   async onExit(result: FullResult) {
     if (this._deferred) {

packages/playwright-test/src/reporters/merge.ts

@@ -85,7 +85,9 @@ function mergeBeginEvents(beginEvents: JsonEvent[]): JsonEvent {
     configFile: undefined,
     globalTimeout: 0,
     maxFailures: 0,
-    metadata: {},
+    metadata: {
+      totalTime: 0,
+    },
     rootDir: '',
     version: '',
     workers: 0,
@@ -118,6 +120,7 @@ function mergeConfigs(to: JsonConfig, from: JsonConfig): JsonConfig {
     metadata: {
       ...to.metadata,
       ...from.metadata,
+      totalTime: to.metadata.totalTime + from.metadata.totalTime,
     },
     workers: to.workers + from.workers,
   };

tests/playwright-test/reporter-blob.spec.ts

@@ -283,6 +283,51 @@ test('be able to merge incomplete shards', async ({ runInlineTest, mergeReports,
   await expect(page.locator('.subnav-item:has-text("Skipped") .counter')).toHaveText('2');
 });
 
+test('total time is from test run not from merge', async ({ runInlineTest, mergeReports, showReport, page }) => {
+  const reportDir = test.info().outputPath('blob-report');
+  const files = {
+    'playwright.config.ts': `
+      module.exports = {
+        retries: 1,
+        reporter: [['blob', { outputDir: '${reportDir.replace(/\\/g, '/')}' }]]
+      };
+    `,
+    'a.test.js': `
+      import { test, expect } from '@playwright/test';
+      test('slow 1', async ({}) => {
+        await new Promise(f => setTimeout(f, 2000));
+        expect(1 + 1).toBe(2);
+      });
+    `,
+    'b.test.js': `
+      import { test, expect } from '@playwright/test';
+      test('slow 1', async ({}) => {
+        await new Promise(f => setTimeout(f, 1000));
+        expect(1 + 1).toBe(2);
+      });
+    `,
+  };
+  await runInlineTest(files, { shard: `1/2` });
+  await runInlineTest(files, { shard: `2/2` });
+
+  const { exitCode, output } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] });
+  expect(exitCode).toBe(0);
+
+  expect(output).toContain('To open last HTML report run:');
+  // console.log(output);
+
+  await showReport();
+
+  await expect(page.locator('.subnav-item:has-text("All") .counter')).toHaveText('2');
+  await expect(page.locator('.subnav-item:has-text("Passed") .counter')).toHaveText('2');
+
+  const durationText = await page.getByTestId('overall-duration').textContent();
+  // "Total time: 2.1s"
+  const time = /Total time: (\d+)(\.\d+)?s/.exec(durationText);
+  expect(time).toBeTruthy();
+  expect(parseInt(time[1], 10)).toBeGreaterThan(2);
+});
+
 test('merge into list report by default', async ({ runInlineTest, mergeReports }) => {
   const reportDir = test.info().outputPath('blob-report');
   const files = {
@@ -830,7 +875,8 @@ test('preserve config fields', async ({ runInlineTest, mergeReports }) => {
   expect(json.rootDir).toBe(test.info().outputDir);
   expect(json.globalTimeout).toBe(config.globalTimeout);
   expect(json.maxFailures).toBe(config.maxFailures);
-  expect(json.metadata).toEqual(config.metadata);
+  expect(json.metadata).toEqual(expect.objectContaining(config.metadata));
+  expect(json.metadata.totalTime).toBeTruthy();
   expect(json.workers).toBe(2);
   expect(json.version).toBeTruthy();
   expect(json.version).not.toEqual(test.info().config.version);