diff --git a/docs/src/test-reporters-js.md b/docs/src/test-reporters-js.md index e684e2af00..237e9e8447 100644 --- a/docs/src/test-reporters-js.md +++ b/docs/src/test-reporters-js.md @@ -310,37 +310,8 @@ Or if there is a custom folder name: npx playwright show-report my-report ``` -#### Sharded report +> The `html` reporter currently does not support merging reports generated across multiple [`--shards`](./test-parallel.md#shard-tests-between-multiple-machines) into a single report. See [this](https://github.com/microsoft/playwright/issues/10437) issue for available third party solutions. -When running tests on [multiple shards](./test-parallel.md#shard-tests-between-multiple-machines), the `html` reporter can automatically show test results from all shards in one page when configured with `sharded: true`. - -```js tab=js-js -// playwright.config.js -// @ts-check - -const { defineConfig } = require('@playwright/test'); - -module.exports = defineConfig({ - reporter: [['html', { sharded: true }]], -}); -``` - -```js tab=js-ts -// playwright.config.ts -import { defineConfig } from '@playwright/test'; - -export default defineConfig({ - reporter: [['html', { sharded: true }]], -}); -``` - -You can use sharded html report combined with a file hosting that allows serving html files. - -In your CI recipe, after running tests in each shard, upload all files from `playwright-report` directory to the **same location**. After that you can open `index.html` from the uploaded location directly in the browser. - -:::note -The `html` report for each shard consists of `index.html` and a data file named like `report-003-of-100.zip`. It's ok to overwrite `index.html` with one another when copying sharded reports to a single directory. -::: ### JSON reporter diff --git a/packages/html-reporter/src/headerView.css b/packages/html-reporter/src/headerView.css index 0ab2c3d839..ef29c442ca 100644 --- a/packages/html-reporter/src/headerView.css +++ b/packages/html-reporter/src/headerView.css @@ -30,7 +30,3 @@ border-right: none; } } - -.header-view-status-line { - padding-right: '10px' -} diff --git a/packages/html-reporter/src/headerView.tsx b/packages/html-reporter/src/headerView.tsx index 5ffa12ce45..f2ae20ce01 100644 --- a/packages/html-reporter/src/headerView.tsx +++ b/packages/html-reporter/src/headerView.tsx @@ -29,8 +29,7 @@ export const HeaderView: React.FC void, projectNames: string[], - reportLoaderError?: string, -}>> = ({ stats, filterText, setFilterText, projectNames, reportLoaderError }) => { +}>> = ({ stats, filterText, setFilterText, projectNames }) => { React.useEffect(() => { (async () => { window.addEventListener('popstate', () => { @@ -58,10 +57,9 @@ export const HeaderView: React.FC - {reportLoaderError &&
{reportLoaderError}
} -
+
{projectNames.length === 1 && !!projectNames[0] && Project: {projectNames[0]}} - Total time: {msToString(stats.duration)} + Total time: {msToString(stats.duration)}
); }; diff --git a/packages/html-reporter/src/index.tsx b/packages/html-reporter/src/index.tsx index 65a9fbecda..6f2d3b414f 100644 --- a/packages/html-reporter/src/index.tsx +++ b/packages/html-reporter/src/index.tsx @@ -23,7 +23,6 @@ import * as ReactDOM from 'react-dom'; import './colors.css'; import type { LoadedReport } from './loadedReport'; import { ReportView } from './reportView'; -import { mergeReports } from './mergeReports'; // @ts-ignore const zipjs = zipImport as typeof zip; @@ -32,12 +31,8 @@ const ReportLoader: React.FC = () => { React.useEffect(() => { if (report) return; - const shardTotal = window.playwrightShardTotal; const zipReport = new ZipReport(); - const loadPromise = shardTotal ? - zipReport.loadFromShards(shardTotal) : - zipReport.loadFromBase64(window.playwrightReportBase64!); - loadPromise.then(() => setReport(zipReport)); + zipReport.load().then(() => setReport(zipReport)); }, [report]); return ; }; @@ -49,37 +44,12 @@ window.onload = () => { class ZipReport implements LoadedReport { private _entries = new Map(); private _json!: HTMLReport; - private _loaderError: string | undefined; - async loadFromBase64(reportBase64: string) { - const zipReader = new zipjs.ZipReader(new zipjs.Data64URIReader(reportBase64), { useWebWorkers: false }) as zip.ZipReader; - this._json = await this._readReportAndTestEntries(zipReader); - } - - async loadFromShards(shardTotal: number) { - const readers = []; - const paddedLen = String(shardTotal).length; - for (let i = 0; i < shardTotal; i++) { - const paddedNumber = String(i + 1).padStart(paddedLen, '0'); - const fileName = `report-${paddedNumber}-of-${shardTotal}.zip`; - const zipReader = new zipjs.ZipReader(new zipjs.HttpReader(fileName), { useWebWorkers: false }) as zip.ZipReader; - readers.push(this._readReportAndTestEntries(zipReader).catch(e => { - // eslint-disable-next-line no-console - console.warn(e); - return undefined; - })); - } - const reportsOrErrors = await Promise.all(readers); - const reports = reportsOrErrors.filter(Boolean) as HTMLReport[]; - if (reports.length < readers.length) - this._loaderError = `Only ${reports.length} of ${shardTotal} report shards loaded`; - this._json = mergeReports(reports); - } - - private async _readReportAndTestEntries(zipReader: zip.ZipReader): Promise { + async load() { + const zipReader = new zipjs.ZipReader(new zipjs.Data64URIReader((window as any).playwrightReportBase64), { useWebWorkers: false }) as zip.ZipReader; for (const entry of await zipReader.getEntries()) this._entries.set(entry.filename, entry); - return await this.entry('report.json') as HTMLReport; + this._json = await this.entry('report.json') as HTMLReport; } json(): HTMLReport { @@ -92,9 +62,4 @@ class ZipReport implements LoadedReport { await reportEntry!.getData!(writer); return JSON.parse(await writer.getData()); } - - loaderError(): string | undefined { - return this._loaderError; - } } - diff --git a/packages/html-reporter/src/loadedReport.ts b/packages/html-reporter/src/loadedReport.ts index 44d278ef31..c1ff2d7530 100644 --- a/packages/html-reporter/src/loadedReport.ts +++ b/packages/html-reporter/src/loadedReport.ts @@ -19,5 +19,4 @@ import type { HTMLReport } from './types'; export interface LoadedReport { json(): HTMLReport; entry(name: string): Promise; - loaderError(): string | undefined; } diff --git a/packages/html-reporter/src/mergeReports.ts b/packages/html-reporter/src/mergeReports.ts deleted file mode 100644 index 981fde912f..0000000000 --- a/packages/html-reporter/src/mergeReports.ts +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import type { HTMLReport, Stats } from './types'; - -export function mergeReports(reports: HTMLReport[]): HTMLReport { - const [report, ...rest] = reports; - - for (const currentReport of rest) { - currentReport.files.forEach(file => { - const existingGroup = report.files.find(({ fileId }) => fileId === file.fileId); - - if (existingGroup) { - existingGroup.tests.push(...file.tests); - mergeStats(existingGroup.stats, file.stats); - } else { - report.files.push(file); - } - }); - - mergeStats(report.stats, currentReport.stats); - report.metadata.duration += currentReport.metadata.duration; - } - - return report; -} - -function mergeStats(toStats: Stats, fromStats: Stats) { - toStats.total += fromStats.total; - toStats.expected += fromStats.expected; - toStats.unexpected += fromStats.unexpected; - toStats.flaky += fromStats.flaky; - toStats.skipped += fromStats.skipped; - toStats.duration += fromStats.duration; - toStats.ok = toStats.ok && fromStats.ok; -} diff --git a/packages/html-reporter/src/reportView.tsx b/packages/html-reporter/src/reportView.tsx index 00cb770302..08dd0e11e9 100644 --- a/packages/html-reporter/src/reportView.tsx +++ b/packages/html-reporter/src/reportView.tsx @@ -31,7 +31,6 @@ import './theme.css'; declare global { interface Window { - playwrightShardTotal?: number; playwrightReportBase64?: string; } } @@ -51,7 +50,7 @@ export const ReportView: React.FC<{ return
- {report?.json() && } + {report?.json() && } {report?.json().metadata && } diff --git a/packages/playwright-test/src/reporters/html.ts b/packages/playwright-test/src/reporters/html.ts index cd5626d849..258e13d958 100644 --- a/packages/playwright-test/src/reporters/html.ts +++ b/packages/playwright-test/src/reporters/html.ts @@ -43,7 +43,6 @@ type HtmlReportOpenOption = 'always' | 'never' | 'on-failure'; type HtmlReporterOptions = { outputFolder?: string, open?: HtmlReportOpenOption, - sharded?: boolean, host?: string, port?: number, }; @@ -54,7 +53,6 @@ class HtmlReporter implements Reporter { private _montonicStartTime: number = 0; private _options: HtmlReporterOptions; private _outputFolder!: string; - private _sharded!: boolean; private _open: string | undefined; private _buildResult: { ok: boolean, singleTestId: string | undefined } | undefined; @@ -69,9 +67,8 @@ class HtmlReporter implements Reporter { onBegin(config: FullConfig, suite: Suite) { this._montonicStartTime = monotonicTime(); this.config = config as FullConfigInternal; - const { outputFolder, open, sharded } = this._resolveOptions(); + const { outputFolder, open } = this._resolveOptions(); this._outputFolder = outputFolder; - this._sharded = sharded; this._open = open; const reportedWarnings = new Set(); for (const project of config.projects) { @@ -92,20 +89,18 @@ class HtmlReporter implements Reporter { this.suite = suite; } - _resolveOptions(): { outputFolder: string, open: HtmlReportOpenOption, sharded: boolean } { + _resolveOptions(): { outputFolder: string, open: HtmlReportOpenOption } { let { outputFolder } = this._options; if (outputFolder) outputFolder = path.resolve(this.config._internal.configDir, outputFolder); return { outputFolder: reportFolderFromEnv() ?? outputFolder ?? defaultReportFolder(this.config._internal.configDir), open: process.env.PW_TEST_HTML_REPORT_OPEN as any || this._options.open || 'on-failure', - sharded: !!this._options.sharded }; } async onEnd() { const duration = monotonicTime() - this._montonicStartTime; - const shard = this._sharded ? this.config.shard : null; const projectSuites = this.suite.suites; const reports = projectSuites.map(suite => { const rawReporter = new RawReporter(); @@ -114,7 +109,7 @@ class HtmlReporter implements Reporter { }); await removeFolders([this._outputFolder]); const builder = new HtmlBuilder(this._outputFolder); - this._buildResult = await builder.build({ ...this.config.metadata, duration }, reports, shard); + this._buildResult = await builder.build({ ...this.config.metadata, duration }, reports); } async _onExit() { @@ -209,7 +204,7 @@ class HtmlBuilder { this._dataZipFile = new yazl.ZipFile(); } - async build(metadata: Metadata & { duration: number }, rawReports: JsonReport[], shard: FullConfigInternal['shard']): Promise<{ ok: boolean, singleTestId: string | undefined }> { + async build(metadata: Metadata & { duration: number }, rawReports: JsonReport[]): Promise<{ ok: boolean, singleTestId: string | undefined }> { const data = new Map(); for (const projectJson of rawReports) { @@ -294,11 +289,17 @@ class HtmlBuilder { } } + // Inline report data. const indexFile = path.join(this._reportFolder, 'index.html'); - if (shard) - await this._writeShardedReport(indexFile, shard); - else - await this._writeInlineReport(indexFile); + fs.appendFileSync(indexFile, ''); let singleTestId: string | undefined; if (htmlReport.stats.total === 1) { @@ -309,32 +310,6 @@ class HtmlBuilder { return { ok, singleTestId }; } - private async _writeShardedReport(indexFile: string, shard: { total: number, current: number }) { - // For each shard write same index.html and store report data in a separate report-num-of-total.zip - // so that they can all be copied in one folder. - await fs.promises.appendFile(indexFile, ``); - const paddedNumber = String(shard.current).padStart(String(shard.total).length, '0'); - const reportZip = path.join(this._reportFolder, `report-${paddedNumber}-of-${shard.total}.zip`); - await new Promise(f => { - this._dataZipFile!.end(undefined, () => { - this._dataZipFile!.outputStream.pipe(fs.createWriteStream(reportZip)).on('close', f); - }); - }); - } - - private async _writeInlineReport(indexFile: string) { - // Inline report data. - await fs.promises.appendFile(indexFile, ''); - } - private _addDataFile(fileName: string, data: any) { this._dataZipFile.addBuffer(Buffer.from(JSON.stringify(data)), fileName); } diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index a274e3591d..dc6b59223f 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -25,7 +25,7 @@ export type ReporterDescription = ['github'] | ['junit'] | ['junit', { outputFile?: string, stripANSIControlSequences?: boolean }] | ['json'] | ['json', { outputFile?: string }] | - ['html'] | ['html', { outputFolder?: string, open?: 'always' | 'never' | 'on-failure', sharded?: boolean }] | + ['html'] | ['html', { outputFolder?: string, open?: 'always' | 'never' | 'on-failure' }] | ['null'] | [string] | [string, any]; diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts index 5ddbdc53a8..564c28c76e 100644 --- a/tests/playwright-test/reporter-html.spec.ts +++ b/tests/playwright-test/reporter-html.spec.ts @@ -985,173 +985,3 @@ test.describe('report location', () => { expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report'))).toBe(true); }); }); - - -test('should shard report', async ({ runInlineTest, showReport, page }, testInfo) => { - const totalShards = 3; - - const testFiles = { - 'playwright.config.ts': ` - module.exports = { reporter: [['html', { sharded: true }]] }; - `, - }; - for (let i = 0; i < totalShards; i++) { - testFiles[`a-${i}.spec.ts`] = ` - import { test, expect } from '@playwright/test'; - test('passes', async ({}) => { expect(2).toBe(2); }); - test('fails', async ({}) => { expect(1).toBe(2); }); - test('skipped', async ({}) => { test.skip('Does not work') }); - test('flaky', async ({}, testInfo) => { expect(testInfo.retry).toBe(1); }); - `; - } - - const allReports = testInfo.outputPath(`aggregated-report`); - await fs.promises.mkdir(allReports, { recursive: true }); - - for (let i = 1; i <= totalShards; i++) { - const result = await runInlineTest(testFiles, - { 'retries': 1, 'shard': `${i}/${totalShards}` }, - { PW_TEST_HTML_REPORT_OPEN: 'never' }, - { usesCustomReporters: true }); - - - expect(result.exitCode).toBe(1); - const files = await fs.promises.readdir(testInfo.outputPath(`playwright-report`)); - expect(new Set(files)).toEqual(new Set([ - 'index.html', - `report-${i}-of-${totalShards}.zip` - ])); - await Promise.all(files.map(name => fs.promises.rename(testInfo.outputPath(`playwright-report/${name}`), `${allReports}/${name}`))); - } - - // Show aggregated report - await showReport(allReports); - - await expect(page.locator('.subnav-item:has-text("All") .counter')).toHaveText('' + (4 * totalShards)); - await expect(page.locator('.subnav-item:has-text("Passed") .counter')).toHaveText('' + totalShards); - await expect(page.locator('.subnav-item:has-text("Failed") .counter')).toHaveText('' + totalShards); - await expect(page.locator('.subnav-item:has-text("Flaky") .counter')).toHaveText('' + totalShards); - await expect(page.locator('.subnav-item:has-text("Skipped") .counter')).toHaveText('' + totalShards); - - await expect(page.locator('.test-file-test-outcome-unexpected >> text=fails')).toHaveCount(totalShards); - await expect(page.locator('.test-file-test-outcome-flaky >> text=flaky')).toHaveCount(totalShards); - await expect(page.locator('.test-file-test-outcome-expected >> text=passes')).toHaveCount(totalShards); - await expect(page.locator('.test-file-test-outcome-skipped >> text=skipped')).toHaveCount(totalShards); -}); - -test('should pad report numbers with zeros', async ({ runInlineTest }, testInfo) => { - const testFiles = { - 'playwright.config.ts': ` - module.exports = { reporter: [['html', { sharded: true }]] }; - `, - }; - for (let i = 0; i < 100; i++) { - testFiles[`a-${i}.spec.ts`] = ` - import { test, expect } from '@playwright/test'; - test('passes', async ({}) => { }); - `; - } - const result = await runInlineTest(testFiles, { shard: '3/100' }, { PW_TEST_HTML_REPORT_OPEN: 'never' }, { usesCustomReporters: true }); - expect(result.exitCode).toBe(0); - const files = await fs.promises.readdir(testInfo.outputPath(`playwright-report`)); - expect(new Set(files)).toEqual(new Set([ - 'index.html', - `report-003-of-100.zip` - ])); -}); - -test('should show report with missing shards', async ({ runInlineTest, showReport, page }, testInfo) => { - const totalShards = 15; - - const testFiles = { - 'playwright.config.ts': ` - module.exports = { reporter: [['html', { sharded: true }]] }; - `, - }; - for (let i = 0; i < totalShards; i++) { - testFiles[`a-${String(i).padStart(2, '0')}.spec.ts`] = ` - import { test, expect } from '@playwright/test'; - test('passes', async ({}) => { expect(2).toBe(2); }); - test('fails', async ({}) => { expect(1).toBe(2); }); - test('skipped', async ({}) => { test.skip('Does not work') }); - test('flaky', async ({}, testInfo) => { expect(testInfo.retry).toBe(1); }); - `; - } - - const allReports = testInfo.outputPath(`aggregated-report`); - await fs.promises.mkdir(allReports, { recursive: true }); - - // Run tests in 2 out of 15 shards. - for (const i of [10, 13]) { - const result = await runInlineTest(testFiles, - { 'retries': 1, 'shard': `${i}/${totalShards}` }, - { PW_TEST_HTML_REPORT_OPEN: 'never' }, - { usesCustomReporters: true }); - - - expect(result.exitCode).toBe(1); - const files = await fs.promises.readdir(testInfo.outputPath(`playwright-report`)); - expect(new Set(files)).toEqual(new Set([ - 'index.html', - `report-${i}-of-${totalShards}.zip` - ])); - await Promise.all(files.map(name => fs.promises.rename(testInfo.outputPath(`playwright-report/${name}`), `${allReports}/${name}`))); - } - - // Show aggregated report - await showReport(allReports); - - await expect(page.getByText('Only 2 of 15 report shards loaded')).toBeVisible(); - - await expect(page.locator('.subnav-item:has-text("All") .counter')).toHaveText('8'); - await expect(page.locator('.subnav-item:has-text("Passed") .counter')).toHaveText('2'); - await expect(page.locator('.subnav-item:has-text("Failed") .counter')).toHaveText('2'); - await expect(page.locator('.subnav-item:has-text("Flaky") .counter')).toHaveText('2'); - await expect(page.locator('.subnav-item:has-text("Skipped") .counter')).toHaveText('2'); - - await expect(page.locator('.test-file-test-outcome-unexpected >> text=fails')).toHaveCount(2); - await expect(page.locator('.test-file-test-outcome-flaky >> text=flaky')).toHaveCount(2); - await expect(page.locator('.test-file-test-outcome-expected >> text=passes')).toHaveCount(2); - await expect(page.locator('.test-file-test-outcome-skipped >> text=skipped')).toHaveCount(2); -}); - - -test('should produce single file report when shard: false', async ({ runInlineTest, showReport, page }, testInfo) => { - const totalShards = 5; - - const testFiles = {}; - for (let i = 0; i < totalShards; i++) { - testFiles[`a-${String(i).padStart(2, '0')}.spec.ts`] = ` - import { test, expect } from '@playwright/test'; - test('passes', async ({}) => { expect(2).toBe(2); }); - test('fails', async ({}) => { expect(1).toBe(2); }); - test('skipped', async ({}) => { test.skip('Does not work') }); - test('flaky', async ({}, testInfo) => { expect(testInfo.retry).toBe(1); }); - `; - } - - // Run single shard. - const currentShard = 3; - const result = await runInlineTest(testFiles, - { 'reporter': 'dot,html', 'retries': 1, 'shard': `${currentShard}/${totalShards}` }, - { PW_TEST_HTML_REPORT_OPEN: 'never' }, - { usesCustomReporters: true }); - - - expect(result.exitCode).toBe(1); - const files = await fs.promises.readdir(testInfo.outputPath(`playwright-report`)); - expect(files).toEqual(['index.html']); - - await showReport(); - - await expect(page.locator('.subnav-item:has-text("All") .counter')).toHaveText('4'); - await expect(page.locator('.subnav-item:has-text("Passed") .counter')).toHaveText('1'); - await expect(page.locator('.subnav-item:has-text("Failed") .counter')).toHaveText('1'); - await expect(page.locator('.subnav-item:has-text("Flaky") .counter')).toHaveText('1'); - await expect(page.locator('.subnav-item:has-text("Skipped") .counter')).toHaveText('1'); - - await expect(page.locator('.test-file-test-outcome-unexpected >> text=fails')).toHaveCount(1); - await expect(page.locator('.test-file-test-outcome-flaky >> text=flaky')).toHaveCount(1); - await expect(page.locator('.test-file-test-outcome-expected >> text=passes')).toHaveCount(1); - await expect(page.locator('.test-file-test-outcome-skipped >> text=skipped')).toHaveCount(1); -}); diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 52a517b96c..9451e00393 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -24,7 +24,7 @@ export type ReporterDescription = ['github'] | ['junit'] | ['junit', { outputFile?: string, stripANSIControlSequences?: boolean }] | ['json'] | ['json', { outputFile?: string }] | - ['html'] | ['html', { outputFolder?: string, open?: 'always' | 'never' | 'on-failure', sharded?: boolean }] | + ['html'] | ['html', { outputFolder?: string, open?: 'always' | 'never' | 'on-failure' }] | ['null'] | [string] | [string, any];