feat: sharded html report (#20700)

This implementation is based on the [original
PR](https://github.com/microsoft/playwright/pull/19691) by @kevin940726.
It makes the reporter produce single file when there is no sharding and
multiple out-of-line report-x-of-y.zip reports which are automatically
merged together when put in one folder.

References https://github.com/microsoft/playwright/issues/10437

Co-authored-by: Kai Hao <kevin830726@gmail.com>

packages/html-reporter/src/index.tsx

@@ -23,6 +23,7 @@ import * as ReactDOM from 'react-dom';
 import './colors.css';
 import type { LoadedReport } from './loadedReport';
 import { ReportView } from './reportView';
+import { mergeReports } from './mergeReports';
 // @ts-ignore
 const zipjs = zipImport as typeof zip;
 
@@ -31,8 +32,12 @@ const ReportLoader: React.FC = () => {
   React.useEffect(() => {
     if (report)
       return;
+    const shardTotal = window.playwrightShardTotal;
     const zipReport = new ZipReport();
-    zipReport.load().then(() => setReport(zipReport));
+    const loadPromise = shardTotal ?
+      zipReport.loadFromShards(shardTotal) :
+      zipReport.loadFromBase64(window.playwrightReportBase64!);
+    loadPromise.then(() => setReport(zipReport));
   }, [report]);
   return <ReportView report={report}></ReportView>;
 };
@@ -45,11 +50,27 @@ class ZipReport implements LoadedReport {
   private _entries = new Map<string, zip.Entry>();
   private _json!: HTMLReport;
 
-  async load() {
-    const zipReader = new zipjs.ZipReader(new zipjs.Data64URIReader((window as any).playwrightReportBase64), { useWebWorkers: false }) as zip.ZipReader;
+  async loadFromBase64(reportBase64: string) {
+    const zipReader = new zipjs.ZipReader(new zipjs.Data64URIReader(reportBase64), { useWebWorkers: false }) as zip.ZipReader;
+    this._json = await this._readReportAndTestEntries(zipReader);
+  }
+
+  async loadFromShards(shardTotal: number) {
+    const readers = [];
+    const paddedLen = String(shardTotal).length;
+    for (let i = 0; i < shardTotal; i++) {
+      const paddedNumber = String(i + 1).padStart(paddedLen, '0');
+      const fileName = `report-${paddedNumber}-of-${shardTotal}.zip`;
+      const zipReader = new zipjs.ZipReader(new zipjs.HttpReader(fileName), { useWebWorkers: false }) as zip.ZipReader;
+      readers.push(this._readReportAndTestEntries(zipReader));
+    }
+    this._json = mergeReports(await Promise.all(readers));
+  }
+
+  private async _readReportAndTestEntries(zipReader: zip.ZipReader): Promise<HTMLReport> {
     for (const entry of await zipReader.getEntries())
       this._entries.set(entry.filename, entry);
-    this._json = await this.entry('report.json') as HTMLReport;
+    return await this.entry('report.json') as HTMLReport;
   }
 
   json(): HTMLReport {
@@ -63,3 +84,4 @@ class ZipReport implements LoadedReport {
     return JSON.parse(await writer.getData());
   }
 }
+

packages/html-reporter/src/mergeReports.ts

@@ -0,0 +1,49 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { HTMLReport, Stats } from './types';
+
+export function mergeReports(reports: HTMLReport[]): HTMLReport {
+  const [report, ...rest] = reports;
+
+  for (const currentReport of rest) {
+    currentReport.files.forEach(file => {
+      const existingGroup = report.files.find(({ fileId }) => fileId === file.fileId);
+
+      if (existingGroup) {
+        existingGroup.tests.push(...file.tests);
+        mergeStats(existingGroup.stats, file.stats);
+      } else {
+        report.files.push(file);
+      }
+    });
+
+    mergeStats(report.stats, currentReport.stats);
+    report.metadata.duration += currentReport.metadata.duration;
+  }
+
+  return report;
+}
+
+function mergeStats(toStats: Stats, fromStats: Stats) {
+  toStats.total += fromStats.total;
+  toStats.expected += fromStats.expected;
+  toStats.unexpected += fromStats.unexpected;
+  toStats.flaky += fromStats.flaky;
+  toStats.skipped += fromStats.skipped;
+  toStats.duration += fromStats.duration;
+  toStats.ok = toStats.ok && fromStats.ok;
+}

packages/html-reporter/src/reportView.tsx

@@ -31,6 +31,7 @@ import './theme.css';
 
 declare global {
   interface Window {
+    playwrightShardTotal?: number;
     playwrightReportBase64?: string;
   }
 }

packages/playwright-test/src/reporters/html.ts

@@ -101,6 +101,7 @@ class HtmlReporter implements Reporter {
 
   async onEnd() {
     const duration = monotonicTime() - this._montonicStartTime;
+    const shard = this.config.shard;
     const projectSuites = this.suite.suites;
     const reports = projectSuites.map(suite => {
       const rawReporter = new RawReporter();
@@ -109,7 +110,7 @@ class HtmlReporter implements Reporter {
     });
     await removeFolders([this._outputFolder]);
     const builder = new HtmlBuilder(this._outputFolder);
-    this._buildResult = await builder.build({ ...this.config.metadata, duration }, reports);
+    this._buildResult = await builder.build({ ...this.config.metadata, duration }, reports, shard);
   }
 
   async _onExit() {
@@ -204,7 +205,7 @@ class HtmlBuilder {
     this._dataZipFile = new yazl.ZipFile();
   }
 
-  async build(metadata: Metadata & { duration: number }, rawReports: JsonReport[]): Promise<{ ok: boolean, singleTestId: string | undefined }> {
+  async build(metadata: Metadata & { duration: number }, rawReports: JsonReport[], shard: FullConfigInternal['shard']): Promise<{ ok: boolean, singleTestId: string | undefined }> {
 
     const data = new Map<string, { testFile: TestFile, testFileSummary: TestFileSummary }>();
     for (const projectJson of rawReports) {
@@ -289,17 +290,11 @@ class HtmlBuilder {
       }
     }
 
-    // Inline report data.
     const indexFile = path.join(this._reportFolder, 'index.html');
-    fs.appendFileSync(indexFile, '<script>\nwindow.playwrightReportBase64 = "data:application/zip;base64,');
-    await new Promise(f => {
-      this._dataZipFile!.end(undefined, () => {
-        this._dataZipFile!.outputStream
-            .pipe(new Base64Encoder())
-            .pipe(fs.createWriteStream(indexFile, { flags: 'a' })).on('close', f);
-      });
-    });
-    fs.appendFileSync(indexFile, '";</script>');
+    if (shard)
+      await this._writeShardedReport(indexFile, shard);
+    else
+      await this._writeInlineReport(indexFile);
 
     let singleTestId: string | undefined;
     if (htmlReport.stats.total === 1) {
@@ -310,6 +305,32 @@ class HtmlBuilder {
     return { ok, singleTestId };
   }
 
+  private async _writeShardedReport(indexFile: string, shard: { total: number, current: number }) {
+    // For each shard write same index.html and store report data in a separate report-num-of-total.zip
+    // so that they can all be copied in one folder.
+    await fs.promises.appendFile(indexFile, `<script>\nwindow.playwrightShardTotal=${shard.total};</script>`);
+    const paddedNumber = String(shard.current).padStart(String(shard.total).length, '0');
+    const reportZip = path.join(this._reportFolder, `report-${paddedNumber}-of-${shard.total}.zip`);
+    await new Promise(f => {
+      this._dataZipFile!.end(undefined, () => {
+        this._dataZipFile!.outputStream.pipe(fs.createWriteStream(reportZip)).on('close', f);
+      });
+    });
+  }
+
+  private async _writeInlineReport(indexFile: string) {
+    // Inline report data.
+    await fs.promises.appendFile(indexFile, '<script>\nwindow.playwrightReportBase64 = "data:application/zip;base64,');
+    await new Promise(f => {
+      this._dataZipFile!.end(undefined, () => {
+        this._dataZipFile!.outputStream
+            .pipe(new Base64Encoder())
+            .pipe(fs.createWriteStream(indexFile, { flags: 'a' })).on('close', f);
+      });
+    });
+    await fs.promises.appendFile(indexFile, '";</script>');
+  }
+
   private _addDataFile(fileName: string, data: any) {
     this._dataZipFile.addBuffer(Buffer.from(JSON.stringify(data)), fileName);
   }

tests/playwright-test/reporter-html.spec.ts

@@ -22,11 +22,11 @@ import type { HttpServer } from '../../packages/playwright-core/lib/utils';
 import { startHtmlReportServer } from '../../packages/playwright-test/lib/reporters/html';
 import { spawnAsync } from 'playwright-core/lib/utils';
 
-const test = baseTest.extend<{ showReport: () => Promise<void> }>({
+const test = baseTest.extend<{ showReport: (reportFolder?: string) => Promise<void> }>({
   showReport: async ({ page }, use, testInfo) => {
     let server: HttpServer | undefined;
-    await use(async () => {
-      const reportFolder = testInfo.outputPath('playwright-report');
+    await use(async (reportFolder?: string) => {
+      reportFolder ??=  testInfo.outputPath('playwright-report');
       server = startHtmlReportServer(reportFolder);
       const location = await server.start();
       await page.goto(location);
@@ -976,3 +976,66 @@ test.describe('report location', () => {
 });
 
 
+test('should shard report', async ({ runInlineTest, showReport, page }, testInfo) => {
+  const totalShards = 3;
+
+  const testFiles = {};
+  for (let i = 0; i < totalShards; i++) {
+    testFiles[`a-${i}.spec.ts`] = `
+      const { test } = pwt;
+      test('passes', async ({}) => { expect(2).toBe(2); });
+      test('fails', async ({}) => { expect(1).toBe(2); });
+      test('skipped', async ({}) => { test.skip('Does not work') });
+      test('flaky', async ({}, testInfo) => { expect(testInfo.retry).toBe(1); });
+    `;
+  }
+
+  const allReports = testInfo.outputPath(`aggregated-report`);
+  await fs.promises.mkdir(allReports, { recursive: true });
+
+  for (let i = 1; i <= totalShards; i++) {
+    const result = await runInlineTest(testFiles,
+        { 'reporter': 'dot,html', 'retries': 1, 'shard': `${i}/${totalShards}` },
+        { PW_TEST_HTML_REPORT_OPEN: 'never' },
+        { usesCustomReporters: true });
+
+
+    expect(result.exitCode).toBe(1);
+    const files = await fs.promises.readdir(testInfo.outputPath(`playwright-report`));
+    expect(new Set(files)).toEqual(new Set([
+      'index.html',
+      `report-${i}-of-${totalShards}.zip`
+    ]));
+    await Promise.all(files.map(name => fs.promises.rename(testInfo.outputPath(`playwright-report/${name}`), `${allReports}/${name}`)));
+  }
+
+  // Show aggregated report
+  await showReport(allReports);
+
+  await expect(page.locator('.subnav-item:has-text("All") .counter')).toHaveText('' + (4 * totalShards));
+  await expect(page.locator('.subnav-item:has-text("Passed") .counter')).toHaveText('' + totalShards);
+  await expect(page.locator('.subnav-item:has-text("Failed") .counter')).toHaveText('' + totalShards);
+  await expect(page.locator('.subnav-item:has-text("Flaky") .counter')).toHaveText('' + totalShards);
+  await expect(page.locator('.subnav-item:has-text("Skipped") .counter')).toHaveText('' + totalShards);
+
+  await expect(page.locator('.test-file-test-outcome-unexpected >> text=fails')).toHaveCount(totalShards);
+  await expect(page.locator('.test-file-test-outcome-flaky >> text=flaky')).toHaveCount(totalShards);
+  await expect(page.locator('.test-file-test-outcome-expected >> text=passes')).toHaveCount(totalShards);
+  await expect(page.locator('.test-file-test-outcome-skipped >> text=skipped')).toHaveCount(totalShards);
+});
+
+test('should pad report numbers with zeros', async ({ runInlineTest, showReport, page }, testInfo) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const { test } = pwt;
+      test('passes', async ({}) => {});
+    `,
+  }, { reporter: 'dot,html', shard: '3/100' }, { PW_TEST_HTML_REPORT_OPEN: 'never' });
+
+  expect(result.exitCode).toBe(0);
+  const files = await fs.promises.readdir(testInfo.outputPath(`playwright-report`));
+  expect(new Set(files)).toEqual(new Set([
+    'index.html',
+    `report-003-of-100.zip`
+  ]));
+});