chore: revert sharded html report (#20923)

We've decided not to ship it in the current form.

#10437

docs/src/test-reporters-js.md

@@ -310,37 +310,8 @@ Or if there is a custom folder name:
 npx playwright show-report my-report
 ```
 
-#### Sharded report
+> The `html` reporter currently does not support merging reports generated across multiple [`--shards`](./test-parallel.md#shard-tests-between-multiple-machines) into a single report. See [this](https://github.com/microsoft/playwright/issues/10437) issue for available third party solutions.
 
-When running tests on [multiple shards](./test-parallel.md#shard-tests-between-multiple-machines), the `html` reporter can automatically show test results from all shards in one page when configured with `sharded: true`.
-
-```js tab=js-js
-// playwright.config.js
-// @ts-check
-
-const { defineConfig } = require('@playwright/test');
-
-module.exports = defineConfig({
-  reporter: [['html', { sharded: true }]],
-});
-```
-
-```js tab=js-ts
-// playwright.config.ts
-import { defineConfig } from '@playwright/test';
-
-export default defineConfig({
-  reporter: [['html', { sharded: true }]],
-});
-```
-
-You can use sharded html report combined with a file hosting that allows serving html files.
-
-In your CI recipe, after running tests in each shard, upload all files from `playwright-report` directory to the **same location**. After that you can open `index.html` from the uploaded location directly in the browser.
-
-:::note
-The `html` report for each shard consists of `index.html` and a data file named like `report-003-of-100.zip`. It's ok to overwrite  `index.html` with one another when copying sharded reports to a single directory.
-:::
 
 ### JSON reporter
 

packages/html-reporter/src/headerView.css

@@ -30,7 +30,3 @@
     border-right: none;
   }
 }
-
-.header-view-status-line {
-  padding-right: '10px'
-}

packages/html-reporter/src/headerView.tsx

@@ -29,8 +29,7 @@ export const HeaderView: React.FC<React.PropsWithChildren<{
   filterText: string,
   setFilterText: (filterText: string) => void,
   projectNames: string[],
-  reportLoaderError?: string,
+}>> = ({ stats, filterText, setFilterText, projectNames }) => {
-}>> = ({ stats, filterText, setFilterText, projectNames, reportLoaderError }) => {
   React.useEffect(() => {
     (async () => {
       window.addEventListener('popstate', () => {
@@ -58,10 +57,9 @@ export const HeaderView: React.FC<React.PropsWithChildren<{
         }}></input>
       </form>
     </div>
-    {reportLoaderError && <div className='header-view-status-line pt-2' data-testid='loader-error' style={{ color: 'var(--color-danger-emphasis)', textAlign: 'right' }}>{reportLoaderError}</div>}
+    <div className='pt-2'>
-    <div className='header-view-status-line pt-2'>
       {projectNames.length === 1 && !!projectNames[0] && <span data-testid="project-name" style={{ color: 'var(--color-fg-subtle)', float: 'left' }}>Project: {projectNames[0]}</span>}
-      <span data-testid="overall-duration" style={{ color: 'var(--color-fg-subtle)', float: 'right' }}>Total time: {msToString(stats.duration)}</span>
+      <span data-testid="overall-duration" style={{ color: 'var(--color-fg-subtle)', paddingRight: '10px', float: 'right' }}>Total time: {msToString(stats.duration)}</span>
     </div>
   </>);
 };

packages/html-reporter/src/index.tsx

@@ -23,7 +23,6 @@ import * as ReactDOM from 'react-dom';
 import './colors.css';
 import type { LoadedReport } from './loadedReport';
 import { ReportView } from './reportView';
-import { mergeReports } from './mergeReports';
 // @ts-ignore
 const zipjs = zipImport as typeof zip;
 
@@ -32,12 +31,8 @@ const ReportLoader: React.FC = () => {
   React.useEffect(() => {
     if (report)
       return;
-    const shardTotal = window.playwrightShardTotal;
     const zipReport = new ZipReport();
-    const loadPromise = shardTotal ?
+    zipReport.load().then(() => setReport(zipReport));
-      zipReport.loadFromShards(shardTotal) :
-      zipReport.loadFromBase64(window.playwrightReportBase64!);
-    loadPromise.then(() => setReport(zipReport));
   }, [report]);
   return <ReportView report={report}></ReportView>;
 };
@@ -49,37 +44,12 @@ window.onload = () => {
 class ZipReport implements LoadedReport {
   private _entries = new Map<string, zip.Entry>();
   private _json!: HTMLReport;
-  private _loaderError: string | undefined;
 
-  async loadFromBase64(reportBase64: string) {
+  async load() {
-    const zipReader = new zipjs.ZipReader(new zipjs.Data64URIReader(reportBase64), { useWebWorkers: false }) as zip.ZipReader;
+    const zipReader = new zipjs.ZipReader(new zipjs.Data64URIReader((window as any).playwrightReportBase64), { useWebWorkers: false }) as zip.ZipReader;
-    this._json = await this._readReportAndTestEntries(zipReader);
-  }
-
-  async loadFromShards(shardTotal: number) {
-    const readers = [];
-    const paddedLen = String(shardTotal).length;
-    for (let i = 0; i < shardTotal; i++) {
-      const paddedNumber = String(i + 1).padStart(paddedLen, '0');
-      const fileName = `report-${paddedNumber}-of-${shardTotal}.zip`;
-      const zipReader = new zipjs.ZipReader(new zipjs.HttpReader(fileName), { useWebWorkers: false }) as zip.ZipReader;
-      readers.push(this._readReportAndTestEntries(zipReader).catch(e => {
-        // eslint-disable-next-line no-console
-        console.warn(e);
-        return undefined;
-      }));
-    }
-    const reportsOrErrors = await Promise.all(readers);
-    const reports = reportsOrErrors.filter(Boolean) as HTMLReport[];
-    if (reports.length < readers.length)
-      this._loaderError = `Only ${reports.length} of ${shardTotal} report shards loaded`;
-    this._json = mergeReports(reports);
-  }
-
-  private async _readReportAndTestEntries(zipReader: zip.ZipReader): Promise<HTMLReport> {
     for (const entry of await zipReader.getEntries())
       this._entries.set(entry.filename, entry);
-    return await this.entry('report.json') as HTMLReport;
+    this._json = await this.entry('report.json') as HTMLReport;
   }
 
   json(): HTMLReport {
@@ -92,9 +62,4 @@ class ZipReport implements LoadedReport {
     await reportEntry!.getData!(writer);
     return JSON.parse(await writer.getData());
   }
-
-  loaderError(): string | undefined {
-    return this._loaderError;
 }
-}
-

packages/html-reporter/src/loadedReport.ts

@@ -19,5 +19,4 @@ import type { HTMLReport } from './types';
 export interface LoadedReport {
   json(): HTMLReport;
   entry(name: string): Promise<Object | undefined>;
-  loaderError(): string | undefined;
 }

packages/html-reporter/src/mergeReports.ts

@@ -1,49 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import type { HTMLReport, Stats } from './types';
-
-export function mergeReports(reports: HTMLReport[]): HTMLReport {
-  const [report, ...rest] = reports;
-
-  for (const currentReport of rest) {
-    currentReport.files.forEach(file => {
-      const existingGroup = report.files.find(({ fileId }) => fileId === file.fileId);
-
-      if (existingGroup) {
-        existingGroup.tests.push(...file.tests);
-        mergeStats(existingGroup.stats, file.stats);
-      } else {
-        report.files.push(file);
-      }
-    });
-
-    mergeStats(report.stats, currentReport.stats);
-    report.metadata.duration += currentReport.metadata.duration;
-  }
-
-  return report;
-}
-
-function mergeStats(toStats: Stats, fromStats: Stats) {
-  toStats.total += fromStats.total;
-  toStats.expected += fromStats.expected;
-  toStats.unexpected += fromStats.unexpected;
-  toStats.flaky += fromStats.flaky;
-  toStats.skipped += fromStats.skipped;
-  toStats.duration += fromStats.duration;
-  toStats.ok = toStats.ok && fromStats.ok;
-}

packages/html-reporter/src/reportView.tsx

@@ -31,7 +31,6 @@ import './theme.css';
 
 declare global {
   interface Window {
-    playwrightShardTotal?: number;
     playwrightReportBase64?: string;
   }
 }
@@ -51,7 +50,7 @@ export const ReportView: React.FC<{
 
   return <div className='htmlreport vbox px-4 pb-4'>
     <main>
-      {report?.json() && <HeaderView stats={report.json().stats} filterText={filterText} setFilterText={setFilterText} projectNames={report.json().projectNames} reportLoaderError={report.loaderError()}></HeaderView>}
+      {report?.json() && <HeaderView stats={report.json().stats} filterText={filterText} setFilterText={setFilterText} projectNames={report.json().projectNames}></HeaderView>}
       {report?.json().metadata && <MetadataView {...report?.json().metadata as Metainfo} />}
       <Route predicate={testFilesRoutePredicate}>
         <TestFilesView report={report?.json()} filter={filter} expandedFiles={expandedFiles} setExpandedFiles={setExpandedFiles}></TestFilesView>

packages/playwright-test/src/reporters/html.ts

@@ -43,7 +43,6 @@ type HtmlReportOpenOption = 'always' | 'never' | 'on-failure';
 type HtmlReporterOptions = {
   outputFolder?: string,
   open?: HtmlReportOpenOption,
-  sharded?: boolean,
   host?: string,
   port?: number,
 };
@@ -54,7 +53,6 @@ class HtmlReporter implements Reporter {
   private _montonicStartTime: number = 0;
   private _options: HtmlReporterOptions;
   private _outputFolder!: string;
-  private _sharded!: boolean;
   private _open: string | undefined;
   private _buildResult: { ok: boolean, singleTestId: string | undefined } | undefined;
 
@@ -69,9 +67,8 @@ class HtmlReporter implements Reporter {
   onBegin(config: FullConfig, suite: Suite) {
     this._montonicStartTime = monotonicTime();
     this.config = config as FullConfigInternal;
-    const { outputFolder, open, sharded } = this._resolveOptions();
+    const { outputFolder, open } = this._resolveOptions();
     this._outputFolder = outputFolder;
-    this._sharded = sharded;
     this._open = open;
     const reportedWarnings = new Set<string>();
     for (const project of config.projects) {
@@ -92,20 +89,18 @@ class HtmlReporter implements Reporter {
     this.suite = suite;
   }
 
-  _resolveOptions(): { outputFolder: string, open: HtmlReportOpenOption, sharded: boolean } {
+  _resolveOptions(): { outputFolder: string, open: HtmlReportOpenOption } {
     let { outputFolder } = this._options;
     if (outputFolder)
       outputFolder = path.resolve(this.config._internal.configDir, outputFolder);
     return {
       outputFolder: reportFolderFromEnv() ?? outputFolder ?? defaultReportFolder(this.config._internal.configDir),
       open: process.env.PW_TEST_HTML_REPORT_OPEN as any || this._options.open || 'on-failure',
-      sharded: !!this._options.sharded
     };
   }
 
   async onEnd() {
     const duration = monotonicTime() - this._montonicStartTime;
-    const shard = this._sharded ? this.config.shard : null;
     const projectSuites = this.suite.suites;
     const reports = projectSuites.map(suite => {
       const rawReporter = new RawReporter();
@@ -114,7 +109,7 @@ class HtmlReporter implements Reporter {
     });
     await removeFolders([this._outputFolder]);
     const builder = new HtmlBuilder(this._outputFolder);
-    this._buildResult = await builder.build({ ...this.config.metadata, duration }, reports, shard);
+    this._buildResult = await builder.build({ ...this.config.metadata, duration }, reports);
   }
 
   async _onExit() {
@@ -209,7 +204,7 @@ class HtmlBuilder {
     this._dataZipFile = new yazl.ZipFile();
   }
 
-  async build(metadata: Metadata & { duration: number }, rawReports: JsonReport[], shard: FullConfigInternal['shard']): Promise<{ ok: boolean, singleTestId: string | undefined }> {
+  async build(metadata: Metadata & { duration: number }, rawReports: JsonReport[]): Promise<{ ok: boolean, singleTestId: string | undefined }> {
 
     const data = new Map<string, { testFile: TestFile, testFileSummary: TestFileSummary }>();
     for (const projectJson of rawReports) {
@@ -294,11 +289,17 @@ class HtmlBuilder {
       }
     }
 
+    // Inline report data.
     const indexFile = path.join(this._reportFolder, 'index.html');
-    if (shard)
+    fs.appendFileSync(indexFile, '<script>\nwindow.playwrightReportBase64 = "data:application/zip;base64,');
-      await this._writeShardedReport(indexFile, shard);
+    await new Promise(f => {
-    else
+      this._dataZipFile!.end(undefined, () => {
-      await this._writeInlineReport(indexFile);
+        this._dataZipFile!.outputStream
+            .pipe(new Base64Encoder())
+            .pipe(fs.createWriteStream(indexFile, { flags: 'a' })).on('close', f);
+      });
+    });
+    fs.appendFileSync(indexFile, '";</script>');
 
     let singleTestId: string | undefined;
     if (htmlReport.stats.total === 1) {
@@ -309,32 +310,6 @@ class HtmlBuilder {
     return { ok, singleTestId };
   }
 
-  private async _writeShardedReport(indexFile: string, shard: { total: number, current: number }) {
-    // For each shard write same index.html and store report data in a separate report-num-of-total.zip
-    // so that they can all be copied in one folder.
-    await fs.promises.appendFile(indexFile, `<script>\nwindow.playwrightShardTotal=${shard.total};</script>`);
-    const paddedNumber = String(shard.current).padStart(String(shard.total).length, '0');
-    const reportZip = path.join(this._reportFolder, `report-${paddedNumber}-of-${shard.total}.zip`);
-    await new Promise(f => {
-      this._dataZipFile!.end(undefined, () => {
-        this._dataZipFile!.outputStream.pipe(fs.createWriteStream(reportZip)).on('close', f);
-      });
-    });
-  }
-
-  private async _writeInlineReport(indexFile: string) {
-    // Inline report data.
-    await fs.promises.appendFile(indexFile, '<script>\nwindow.playwrightReportBase64 = "data:application/zip;base64,');
-    await new Promise(f => {
-      this._dataZipFile!.end(undefined, () => {
-        this._dataZipFile!.outputStream
-            .pipe(new Base64Encoder())
-            .pipe(fs.createWriteStream(indexFile, { flags: 'a' })).on('close', f);
-      });
-    });
-    await fs.promises.appendFile(indexFile, '";</script>');
-  }
-
   private _addDataFile(fileName: string, data: any) {
     this._dataZipFile.addBuffer(Buffer.from(JSON.stringify(data)), fileName);
   }

packages/playwright-test/types/test.d.ts

@@ -25,7 +25,7 @@ export type ReporterDescription =
   ['github'] |
   ['junit'] | ['junit', { outputFile?: string, stripANSIControlSequences?: boolean }] |
   ['json'] | ['json', { outputFile?: string }] |
-  ['html'] | ['html', { outputFolder?: string, open?: 'always' | 'never' | 'on-failure', sharded?: boolean }] |
+  ['html'] | ['html', { outputFolder?: string, open?: 'always' | 'never' | 'on-failure' }] |
   ['null'] |
   [string] | [string, any];
 

tests/playwright-test/reporter-html.spec.ts

@@ -985,173 +985,3 @@ test.describe('report location', () => {
     expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report'))).toBe(true);
   });
 });
-
-
-test('should shard report', async ({ runInlineTest, showReport, page }, testInfo) => {
-  const totalShards = 3;
-
-  const testFiles = {
-    'playwright.config.ts': `
-      module.exports = { reporter: [['html', { sharded: true }]] };
-    `,
-  };
-  for (let i = 0; i < totalShards; i++) {
-    testFiles[`a-${i}.spec.ts`] = `
-      import { test, expect } from '@playwright/test';
-      test('passes', async ({}) => { expect(2).toBe(2); });
-      test('fails', async ({}) => { expect(1).toBe(2); });
-      test('skipped', async ({}) => { test.skip('Does not work') });
-      test('flaky', async ({}, testInfo) => { expect(testInfo.retry).toBe(1); });
-    `;
-  }
-
-  const allReports = testInfo.outputPath(`aggregated-report`);
-  await fs.promises.mkdir(allReports, { recursive: true });
-
-  for (let i = 1; i <= totalShards; i++) {
-    const result = await runInlineTest(testFiles,
-        { 'retries': 1, 'shard': `${i}/${totalShards}` },
-        { PW_TEST_HTML_REPORT_OPEN: 'never' },
-        { usesCustomReporters: true });
-
-
-    expect(result.exitCode).toBe(1);
-    const files = await fs.promises.readdir(testInfo.outputPath(`playwright-report`));
-    expect(new Set(files)).toEqual(new Set([
-      'index.html',
-      `report-${i}-of-${totalShards}.zip`
-    ]));
-    await Promise.all(files.map(name => fs.promises.rename(testInfo.outputPath(`playwright-report/${name}`), `${allReports}/${name}`)));
-  }
-
-  // Show aggregated report
-  await showReport(allReports);
-
-  await expect(page.locator('.subnav-item:has-text("All") .counter')).toHaveText('' + (4 * totalShards));
-  await expect(page.locator('.subnav-item:has-text("Passed") .counter')).toHaveText('' + totalShards);
-  await expect(page.locator('.subnav-item:has-text("Failed") .counter')).toHaveText('' + totalShards);
-  await expect(page.locator('.subnav-item:has-text("Flaky") .counter')).toHaveText('' + totalShards);
-  await expect(page.locator('.subnav-item:has-text("Skipped") .counter')).toHaveText('' + totalShards);
-
-  await expect(page.locator('.test-file-test-outcome-unexpected >> text=fails')).toHaveCount(totalShards);
-  await expect(page.locator('.test-file-test-outcome-flaky >> text=flaky')).toHaveCount(totalShards);
-  await expect(page.locator('.test-file-test-outcome-expected >> text=passes')).toHaveCount(totalShards);
-  await expect(page.locator('.test-file-test-outcome-skipped >> text=skipped')).toHaveCount(totalShards);
-});
-
-test('should pad report numbers with zeros', async ({ runInlineTest }, testInfo) => {
-  const testFiles = {
-    'playwright.config.ts': `
-      module.exports = { reporter: [['html', { sharded: true }]] };
-    `,
-  };
-  for (let i = 0; i < 100; i++) {
-    testFiles[`a-${i}.spec.ts`] = `
-      import { test, expect } from '@playwright/test';
-      test('passes', async ({}) => { });
-    `;
-  }
-  const result = await runInlineTest(testFiles, { shard: '3/100' }, { PW_TEST_HTML_REPORT_OPEN: 'never' }, { usesCustomReporters: true });
-  expect(result.exitCode).toBe(0);
-  const files = await fs.promises.readdir(testInfo.outputPath(`playwright-report`));
-  expect(new Set(files)).toEqual(new Set([
-    'index.html',
-    `report-003-of-100.zip`
-  ]));
-});
-
-test('should show report with missing shards', async ({ runInlineTest, showReport, page }, testInfo) => {
-  const totalShards = 15;
-
-  const testFiles = {
-    'playwright.config.ts': `
-      module.exports = { reporter: [['html', { sharded: true }]] };
-    `,
-  };
-  for (let i = 0; i < totalShards; i++) {
-    testFiles[`a-${String(i).padStart(2, '0')}.spec.ts`] = `
-      import { test, expect } from '@playwright/test';
-      test('passes', async ({}) => { expect(2).toBe(2); });
-      test('fails', async ({}) => { expect(1).toBe(2); });
-      test('skipped', async ({}) => { test.skip('Does not work') });
-      test('flaky', async ({}, testInfo) => { expect(testInfo.retry).toBe(1); });
-    `;
-  }
-
-  const allReports = testInfo.outputPath(`aggregated-report`);
-  await fs.promises.mkdir(allReports, { recursive: true });
-
-  // Run tests in 2 out of 15 shards.
-  for (const i of [10, 13]) {
-    const result = await runInlineTest(testFiles,
-        { 'retries': 1, 'shard': `${i}/${totalShards}` },
-        { PW_TEST_HTML_REPORT_OPEN: 'never' },
-        { usesCustomReporters: true });
-
-
-    expect(result.exitCode).toBe(1);
-    const files = await fs.promises.readdir(testInfo.outputPath(`playwright-report`));
-    expect(new Set(files)).toEqual(new Set([
-      'index.html',
-      `report-${i}-of-${totalShards}.zip`
-    ]));
-    await Promise.all(files.map(name => fs.promises.rename(testInfo.outputPath(`playwright-report/${name}`), `${allReports}/${name}`)));
-  }
-
-  // Show aggregated report
-  await showReport(allReports);
-
-  await expect(page.getByText('Only 2 of 15 report shards loaded')).toBeVisible();
-
-  await expect(page.locator('.subnav-item:has-text("All") .counter')).toHaveText('8');
-  await expect(page.locator('.subnav-item:has-text("Passed") .counter')).toHaveText('2');
-  await expect(page.locator('.subnav-item:has-text("Failed") .counter')).toHaveText('2');
-  await expect(page.locator('.subnav-item:has-text("Flaky") .counter')).toHaveText('2');
-  await expect(page.locator('.subnav-item:has-text("Skipped") .counter')).toHaveText('2');
-
-  await expect(page.locator('.test-file-test-outcome-unexpected >> text=fails')).toHaveCount(2);
-  await expect(page.locator('.test-file-test-outcome-flaky >> text=flaky')).toHaveCount(2);
-  await expect(page.locator('.test-file-test-outcome-expected >> text=passes')).toHaveCount(2);
-  await expect(page.locator('.test-file-test-outcome-skipped >> text=skipped')).toHaveCount(2);
-});
-
-
-test('should produce single file report when shard: false', async ({ runInlineTest, showReport, page }, testInfo) => {
-  const totalShards = 5;
-
-  const testFiles = {};
-  for (let i = 0; i < totalShards; i++) {
-    testFiles[`a-${String(i).padStart(2, '0')}.spec.ts`] = `
-      import { test, expect } from '@playwright/test';
-      test('passes', async ({}) => { expect(2).toBe(2); });
-      test('fails', async ({}) => { expect(1).toBe(2); });
-      test('skipped', async ({}) => { test.skip('Does not work') });
-      test('flaky', async ({}, testInfo) => { expect(testInfo.retry).toBe(1); });
-    `;
-  }
-
-  // Run single shard.
-  const currentShard = 3;
-  const result = await runInlineTest(testFiles,
-      { 'reporter': 'dot,html', 'retries': 1, 'shard': `${currentShard}/${totalShards}` },
-      { PW_TEST_HTML_REPORT_OPEN: 'never' },
-      { usesCustomReporters: true });
-
-
-  expect(result.exitCode).toBe(1);
-  const files = await fs.promises.readdir(testInfo.outputPath(`playwright-report`));
-  expect(files).toEqual(['index.html']);
-
-  await showReport();
-
-  await expect(page.locator('.subnav-item:has-text("All") .counter')).toHaveText('4');
-  await expect(page.locator('.subnav-item:has-text("Passed") .counter')).toHaveText('1');
-  await expect(page.locator('.subnav-item:has-text("Failed") .counter')).toHaveText('1');
-  await expect(page.locator('.subnav-item:has-text("Flaky") .counter')).toHaveText('1');
-  await expect(page.locator('.subnav-item:has-text("Skipped") .counter')).toHaveText('1');
-
-  await expect(page.locator('.test-file-test-outcome-unexpected >> text=fails')).toHaveCount(1);
-  await expect(page.locator('.test-file-test-outcome-flaky >> text=flaky')).toHaveCount(1);
-  await expect(page.locator('.test-file-test-outcome-expected >> text=passes')).toHaveCount(1);
-  await expect(page.locator('.test-file-test-outcome-skipped >> text=skipped')).toHaveCount(1);
-});

utils/generate_types/overrides-test.d.ts

@@ -24,7 +24,7 @@ export type ReporterDescription =
   ['github'] |
   ['junit'] | ['junit', { outputFile?: string, stripANSIControlSequences?: boolean }] |
   ['json'] | ['json', { outputFile?: string }] |
-  ['html'] | ['html', { outputFolder?: string, open?: 'always' | 'never' | 'on-failure', sharded?: boolean }] |
+  ['html'] | ['html', { outputFolder?: string, open?: 'always' | 'never' | 'on-failure' }] |
   ['null'] |
   [string] | [string, any];