chore: revert sharded html report (#20923)
We've decided not to ship it in the current form. #10437
This commit is contained in:
Yury Semikhatsky
GitHub
parent
5899348936
commit
90c4e6f9b2
|
|
@ -310,37 +310,8 @@ Or if there is a custom folder name:
|
|||
npx playwright show-report my-report
|
||||
```
|
||||
|
||||
#### Sharded report
|
||||
> The `html` reporter currently does not support merging reports generated across multiple [`--shards`](./test-parallel.md#shard-tests-between-multiple-machines) into a single report. See [this](https://github.com/microsoft/playwright/issues/10437) issue for available third party solutions.
|
||||
|
||||
When running tests on [multiple shards](./test-parallel.md#shard-tests-between-multiple-machines), the `html` reporter can automatically show test results from all shards in one page when configured with `sharded: true`.
|
||||
|
||||
```js tab=js-js
|
||||
// playwright.config.js
|
||||
// @ts-check
|
||||
|
||||
const { defineConfig } = require('@playwright/test');
|
||||
|
||||
module.exports = defineConfig({
|
||||
reporter: [['html', { sharded: true }]],
|
||||
});
|
||||
```
|
||||
|
||||
```js tab=js-ts
|
||||
// playwright.config.ts
|
||||
import { defineConfig } from '@playwright/test';
|
||||
|
||||
export default defineConfig({
|
||||
reporter: [['html', { sharded: true }]],
|
||||
});
|
||||
```
|
||||
|
||||
You can use sharded html report combined with a file hosting that allows serving html files.
|
||||
|
||||
In your CI recipe, after running tests in each shard, upload all files from `playwright-report` directory to the **same location**. After that you can open `index.html` from the uploaded location directly in the browser.
|
||||
|
||||
:::note
|
||||
The `html` report for each shard consists of `index.html` and a data file named like `report-003-of-100.zip`. It's ok to overwrite `index.html` with one another when copying sharded reports to a single directory.
|
||||
:::
|
||||
|
||||
### JSON reporter
|
||||
|
||||
|
|
|
|||
|
|
@ -30,7 +30,3 @@
|
|||
border-right: none;
|
||||
}
|
||||
}
|
||||
|
||||
.header-view-status-line {
|
||||
padding-right: '10px'
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,8 +29,7 @@ export const HeaderView: React.FC<React.PropsWithChildren<{
|
|||
filterText: string,
|
||||
setFilterText: (filterText: string) => void,
|
||||
projectNames: string[],
|
||||
reportLoaderError?: string,
|
||||
}>> = ({ stats, filterText, setFilterText, projectNames, reportLoaderError }) => {
|
||||
}>> = ({ stats, filterText, setFilterText, projectNames }) => {
|
||||
React.useEffect(() => {
|
||||
(async () => {
|
||||
window.addEventListener('popstate', () => {
|
||||
|
|
@ -58,10 +57,9 @@ export const HeaderView: React.FC<React.PropsWithChildren<{
|
|||
}}></input>
|
||||
</form>
|
||||
</div>
|
||||
{reportLoaderError && <div className='header-view-status-line pt-2' data-testid='loader-error' style={{ color: 'var(--color-danger-emphasis)', textAlign: 'right' }}>{reportLoaderError}</div>}
|
||||
<div className='header-view-status-line pt-2'>
|
||||
<div className='pt-2'>
|
||||
{projectNames.length === 1 && !!projectNames[0] && <span data-testid="project-name" style={{ color: 'var(--color-fg-subtle)', float: 'left' }}>Project: {projectNames[0]}</span>}
|
||||
<span data-testid="overall-duration" style={{ color: 'var(--color-fg-subtle)', float: 'right' }}>Total time: {msToString(stats.duration)}</span>
|
||||
<span data-testid="overall-duration" style={{ color: 'var(--color-fg-subtle)', paddingRight: '10px', float: 'right' }}>Total time: {msToString(stats.duration)}</span>
|
||||
</div>
|
||||
</>);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -23,7 +23,6 @@ import * as ReactDOM from 'react-dom';
|
|||
import './colors.css';
|
||||
import type { LoadedReport } from './loadedReport';
|
||||
import { ReportView } from './reportView';
|
||||
import { mergeReports } from './mergeReports';
|
||||
// @ts-ignore
|
||||
const zipjs = zipImport as typeof zip;
|
||||
|
||||
|
|
@ -32,12 +31,8 @@ const ReportLoader: React.FC = () => {
|
|||
React.useEffect(() => {
|
||||
if (report)
|
||||
return;
|
||||
const shardTotal = window.playwrightShardTotal;
|
||||
const zipReport = new ZipReport();
|
||||
const loadPromise = shardTotal ?
|
||||
zipReport.loadFromShards(shardTotal) :
|
||||
zipReport.loadFromBase64(window.playwrightReportBase64!);
|
||||
loadPromise.then(() => setReport(zipReport));
|
||||
zipReport.load().then(() => setReport(zipReport));
|
||||
}, [report]);
|
||||
return <ReportView report={report}></ReportView>;
|
||||
};
|
||||
|
|
@ -49,37 +44,12 @@ window.onload = () => {
|
|||
class ZipReport implements LoadedReport {
|
||||
private _entries = new Map<string, zip.Entry>();
|
||||
private _json!: HTMLReport;
|
||||
private _loaderError: string | undefined;
|
||||
|
||||
async loadFromBase64(reportBase64: string) {
|
||||
const zipReader = new zipjs.ZipReader(new zipjs.Data64URIReader(reportBase64), { useWebWorkers: false }) as zip.ZipReader;
|
||||
this._json = await this._readReportAndTestEntries(zipReader);
|
||||
}
|
||||
|
||||
async loadFromShards(shardTotal: number) {
|
||||
const readers = [];
|
||||
const paddedLen = String(shardTotal).length;
|
||||
for (let i = 0; i < shardTotal; i++) {
|
||||
const paddedNumber = String(i + 1).padStart(paddedLen, '0');
|
||||
const fileName = `report-${paddedNumber}-of-${shardTotal}.zip`;
|
||||
const zipReader = new zipjs.ZipReader(new zipjs.HttpReader(fileName), { useWebWorkers: false }) as zip.ZipReader;
|
||||
readers.push(this._readReportAndTestEntries(zipReader).catch(e => {
|
||||
// eslint-disable-next-line no-console
|
||||
console.warn(e);
|
||||
return undefined;
|
||||
}));
|
||||
}
|
||||
const reportsOrErrors = await Promise.all(readers);
|
||||
const reports = reportsOrErrors.filter(Boolean) as HTMLReport[];
|
||||
if (reports.length < readers.length)
|
||||
this._loaderError = `Only ${reports.length} of ${shardTotal} report shards loaded`;
|
||||
this._json = mergeReports(reports);
|
||||
}
|
||||
|
||||
private async _readReportAndTestEntries(zipReader: zip.ZipReader): Promise<HTMLReport> {
|
||||
async load() {
|
||||
const zipReader = new zipjs.ZipReader(new zipjs.Data64URIReader((window as any).playwrightReportBase64), { useWebWorkers: false }) as zip.ZipReader;
|
||||
for (const entry of await zipReader.getEntries())
|
||||
this._entries.set(entry.filename, entry);
|
||||
return await this.entry('report.json') as HTMLReport;
|
||||
this._json = await this.entry('report.json') as HTMLReport;
|
||||
}
|
||||
|
||||
json(): HTMLReport {
|
||||
|
|
@ -92,9 +62,4 @@ class ZipReport implements LoadedReport {
|
|||
await reportEntry!.getData!(writer);
|
||||
return JSON.parse(await writer.getData());
|
||||
}
|
||||
|
||||
loaderError(): string | undefined {
|
||||
return this._loaderError;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -19,5 +19,4 @@ import type { HTMLReport } from './types';
|
|||
export interface LoadedReport {
|
||||
json(): HTMLReport;
|
||||
entry(name: string): Promise<Object | undefined>;
|
||||
loaderError(): string | undefined;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,49 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import type { HTMLReport, Stats } from './types';
|
||||
|
||||
export function mergeReports(reports: HTMLReport[]): HTMLReport {
|
||||
const [report, ...rest] = reports;
|
||||
|
||||
for (const currentReport of rest) {
|
||||
currentReport.files.forEach(file => {
|
||||
const existingGroup = report.files.find(({ fileId }) => fileId === file.fileId);
|
||||
|
||||
if (existingGroup) {
|
||||
existingGroup.tests.push(...file.tests);
|
||||
mergeStats(existingGroup.stats, file.stats);
|
||||
} else {
|
||||
report.files.push(file);
|
||||
}
|
||||
});
|
||||
|
||||
mergeStats(report.stats, currentReport.stats);
|
||||
report.metadata.duration += currentReport.metadata.duration;
|
||||
}
|
||||
|
||||
return report;
|
||||
}
|
||||
|
||||
function mergeStats(toStats: Stats, fromStats: Stats) {
|
||||
toStats.total += fromStats.total;
|
||||
toStats.expected += fromStats.expected;
|
||||
toStats.unexpected += fromStats.unexpected;
|
||||
toStats.flaky += fromStats.flaky;
|
||||
toStats.skipped += fromStats.skipped;
|
||||
toStats.duration += fromStats.duration;
|
||||
toStats.ok = toStats.ok && fromStats.ok;
|
||||
}
|
||||
|
|
@ -31,7 +31,6 @@ import './theme.css';
|
|||
|
||||
declare global {
|
||||
interface Window {
|
||||
playwrightShardTotal?: number;
|
||||
playwrightReportBase64?: string;
|
||||
}
|
||||
}
|
||||
|
|
@ -51,7 +50,7 @@ export const ReportView: React.FC<{
|
|||
|
||||
return <div className='htmlreport vbox px-4 pb-4'>
|
||||
<main>
|
||||
{report?.json() && <HeaderView stats={report.json().stats} filterText={filterText} setFilterText={setFilterText} projectNames={report.json().projectNames} reportLoaderError={report.loaderError()}></HeaderView>}
|
||||
{report?.json() && <HeaderView stats={report.json().stats} filterText={filterText} setFilterText={setFilterText} projectNames={report.json().projectNames}></HeaderView>}
|
||||
{report?.json().metadata && <MetadataView {...report?.json().metadata as Metainfo} />}
|
||||
<Route predicate={testFilesRoutePredicate}>
|
||||
<TestFilesView report={report?.json()} filter={filter} expandedFiles={expandedFiles} setExpandedFiles={setExpandedFiles}></TestFilesView>
|
||||
|
|
|
|||
|
|
@ -43,7 +43,6 @@ type HtmlReportOpenOption = 'always' | 'never' | 'on-failure';
|
|||
type HtmlReporterOptions = {
|
||||
outputFolder?: string,
|
||||
open?: HtmlReportOpenOption,
|
||||
sharded?: boolean,
|
||||
host?: string,
|
||||
port?: number,
|
||||
};
|
||||
|
|
@ -54,7 +53,6 @@ class HtmlReporter implements Reporter {
|
|||
private _montonicStartTime: number = 0;
|
||||
private _options: HtmlReporterOptions;
|
||||
private _outputFolder!: string;
|
||||
private _sharded!: boolean;
|
||||
private _open: string | undefined;
|
||||
private _buildResult: { ok: boolean, singleTestId: string | undefined } | undefined;
|
||||
|
||||
|
|
@ -69,9 +67,8 @@ class HtmlReporter implements Reporter {
|
|||
onBegin(config: FullConfig, suite: Suite) {
|
||||
this._montonicStartTime = monotonicTime();
|
||||
this.config = config as FullConfigInternal;
|
||||
const { outputFolder, open, sharded } = this._resolveOptions();
|
||||
const { outputFolder, open } = this._resolveOptions();
|
||||
this._outputFolder = outputFolder;
|
||||
this._sharded = sharded;
|
||||
this._open = open;
|
||||
const reportedWarnings = new Set<string>();
|
||||
for (const project of config.projects) {
|
||||
|
|
@ -92,20 +89,18 @@ class HtmlReporter implements Reporter {
|
|||
this.suite = suite;
|
||||
}
|
||||
|
||||
_resolveOptions(): { outputFolder: string, open: HtmlReportOpenOption, sharded: boolean } {
|
||||
_resolveOptions(): { outputFolder: string, open: HtmlReportOpenOption } {
|
||||
let { outputFolder } = this._options;
|
||||
if (outputFolder)
|
||||
outputFolder = path.resolve(this.config._internal.configDir, outputFolder);
|
||||
return {
|
||||
outputFolder: reportFolderFromEnv() ?? outputFolder ?? defaultReportFolder(this.config._internal.configDir),
|
||||
open: process.env.PW_TEST_HTML_REPORT_OPEN as any || this._options.open || 'on-failure',
|
||||
sharded: !!this._options.sharded
|
||||
};
|
||||
}
|
||||
|
||||
async onEnd() {
|
||||
const duration = monotonicTime() - this._montonicStartTime;
|
||||
const shard = this._sharded ? this.config.shard : null;
|
||||
const projectSuites = this.suite.suites;
|
||||
const reports = projectSuites.map(suite => {
|
||||
const rawReporter = new RawReporter();
|
||||
|
|
@ -114,7 +109,7 @@ class HtmlReporter implements Reporter {
|
|||
});
|
||||
await removeFolders([this._outputFolder]);
|
||||
const builder = new HtmlBuilder(this._outputFolder);
|
||||
this._buildResult = await builder.build({ ...this.config.metadata, duration }, reports, shard);
|
||||
this._buildResult = await builder.build({ ...this.config.metadata, duration }, reports);
|
||||
}
|
||||
|
||||
async _onExit() {
|
||||
|
|
@ -209,7 +204,7 @@ class HtmlBuilder {
|
|||
this._dataZipFile = new yazl.ZipFile();
|
||||
}
|
||||
|
||||
async build(metadata: Metadata & { duration: number }, rawReports: JsonReport[], shard: FullConfigInternal['shard']): Promise<{ ok: boolean, singleTestId: string | undefined }> {
|
||||
async build(metadata: Metadata & { duration: number }, rawReports: JsonReport[]): Promise<{ ok: boolean, singleTestId: string | undefined }> {
|
||||
|
||||
const data = new Map<string, { testFile: TestFile, testFileSummary: TestFileSummary }>();
|
||||
for (const projectJson of rawReports) {
|
||||
|
|
@ -294,11 +289,17 @@ class HtmlBuilder {
|
|||
}
|
||||
}
|
||||
|
||||
// Inline report data.
|
||||
const indexFile = path.join(this._reportFolder, 'index.html');
|
||||
if (shard)
|
||||
await this._writeShardedReport(indexFile, shard);
|
||||
else
|
||||
await this._writeInlineReport(indexFile);
|
||||
fs.appendFileSync(indexFile, '<script>\nwindow.playwrightReportBase64 = "data:application/zip;base64,');
|
||||
await new Promise(f => {
|
||||
this._dataZipFile!.end(undefined, () => {
|
||||
this._dataZipFile!.outputStream
|
||||
.pipe(new Base64Encoder())
|
||||
.pipe(fs.createWriteStream(indexFile, { flags: 'a' })).on('close', f);
|
||||
});
|
||||
});
|
||||
fs.appendFileSync(indexFile, '";</script>');
|
||||
|
||||
let singleTestId: string | undefined;
|
||||
if (htmlReport.stats.total === 1) {
|
||||
|
|
@ -309,32 +310,6 @@ class HtmlBuilder {
|
|||
return { ok, singleTestId };
|
||||
}
|
||||
|
||||
private async _writeShardedReport(indexFile: string, shard: { total: number, current: number }) {
|
||||
// For each shard write same index.html and store report data in a separate report-num-of-total.zip
|
||||
// so that they can all be copied in one folder.
|
||||
await fs.promises.appendFile(indexFile, `<script>\nwindow.playwrightShardTotal=${shard.total};</script>`);
|
||||
const paddedNumber = String(shard.current).padStart(String(shard.total).length, '0');
|
||||
const reportZip = path.join(this._reportFolder, `report-${paddedNumber}-of-${shard.total}.zip`);
|
||||
await new Promise(f => {
|
||||
this._dataZipFile!.end(undefined, () => {
|
||||
this._dataZipFile!.outputStream.pipe(fs.createWriteStream(reportZip)).on('close', f);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
private async _writeInlineReport(indexFile: string) {
|
||||
// Inline report data.
|
||||
await fs.promises.appendFile(indexFile, '<script>\nwindow.playwrightReportBase64 = "data:application/zip;base64,');
|
||||
await new Promise(f => {
|
||||
this._dataZipFile!.end(undefined, () => {
|
||||
this._dataZipFile!.outputStream
|
||||
.pipe(new Base64Encoder())
|
||||
.pipe(fs.createWriteStream(indexFile, { flags: 'a' })).on('close', f);
|
||||
});
|
||||
});
|
||||
await fs.promises.appendFile(indexFile, '";</script>');
|
||||
}
|
||||
|
||||
private _addDataFile(fileName: string, data: any) {
|
||||
this._dataZipFile.addBuffer(Buffer.from(JSON.stringify(data)), fileName);
|
||||
}
|
||||
|
|
|
|||
2
packages/playwright-test/types/test.d.ts
vendored
2
packages/playwright-test/types/test.d.ts
vendored
|
|
@ -25,7 +25,7 @@ export type ReporterDescription =
|
|||
['github'] |
|
||||
['junit'] | ['junit', { outputFile?: string, stripANSIControlSequences?: boolean }] |
|
||||
['json'] | ['json', { outputFile?: string }] |
|
||||
['html'] | ['html', { outputFolder?: string, open?: 'always' | 'never' | 'on-failure', sharded?: boolean }] |
|
||||
['html'] | ['html', { outputFolder?: string, open?: 'always' | 'never' | 'on-failure' }] |
|
||||
['null'] |
|
||||
[string] | [string, any];
|
||||
|
||||
|
|
|
|||
|
|
@ -985,173 +985,3 @@ test.describe('report location', () => {
|
|||
expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report'))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
test('should shard report', async ({ runInlineTest, showReport, page }, testInfo) => {
|
||||
const totalShards = 3;
|
||||
|
||||
const testFiles = {
|
||||
'playwright.config.ts': `
|
||||
module.exports = { reporter: [['html', { sharded: true }]] };
|
||||
`,
|
||||
};
|
||||
for (let i = 0; i < totalShards; i++) {
|
||||
testFiles[`a-${i}.spec.ts`] = `
|
||||
import { test, expect } from '@playwright/test';
|
||||
test('passes', async ({}) => { expect(2).toBe(2); });
|
||||
test('fails', async ({}) => { expect(1).toBe(2); });
|
||||
test('skipped', async ({}) => { test.skip('Does not work') });
|
||||
test('flaky', async ({}, testInfo) => { expect(testInfo.retry).toBe(1); });
|
||||
`;
|
||||
}
|
||||
|
||||
const allReports = testInfo.outputPath(`aggregated-report`);
|
||||
await fs.promises.mkdir(allReports, { recursive: true });
|
||||
|
||||
for (let i = 1; i <= totalShards; i++) {
|
||||
const result = await runInlineTest(testFiles,
|
||||
{ 'retries': 1, 'shard': `${i}/${totalShards}` },
|
||||
{ PW_TEST_HTML_REPORT_OPEN: 'never' },
|
||||
{ usesCustomReporters: true });
|
||||
|
||||
|
||||
expect(result.exitCode).toBe(1);
|
||||
const files = await fs.promises.readdir(testInfo.outputPath(`playwright-report`));
|
||||
expect(new Set(files)).toEqual(new Set([
|
||||
'index.html',
|
||||
`report-${i}-of-${totalShards}.zip`
|
||||
]));
|
||||
await Promise.all(files.map(name => fs.promises.rename(testInfo.outputPath(`playwright-report/${name}`), `${allReports}/${name}`)));
|
||||
}
|
||||
|
||||
// Show aggregated report
|
||||
await showReport(allReports);
|
||||
|
||||
await expect(page.locator('.subnav-item:has-text("All") .counter')).toHaveText('' + (4 * totalShards));
|
||||
await expect(page.locator('.subnav-item:has-text("Passed") .counter')).toHaveText('' + totalShards);
|
||||
await expect(page.locator('.subnav-item:has-text("Failed") .counter')).toHaveText('' + totalShards);
|
||||
await expect(page.locator('.subnav-item:has-text("Flaky") .counter')).toHaveText('' + totalShards);
|
||||
await expect(page.locator('.subnav-item:has-text("Skipped") .counter')).toHaveText('' + totalShards);
|
||||
|
||||
await expect(page.locator('.test-file-test-outcome-unexpected >> text=fails')).toHaveCount(totalShards);
|
||||
await expect(page.locator('.test-file-test-outcome-flaky >> text=flaky')).toHaveCount(totalShards);
|
||||
await expect(page.locator('.test-file-test-outcome-expected >> text=passes')).toHaveCount(totalShards);
|
||||
await expect(page.locator('.test-file-test-outcome-skipped >> text=skipped')).toHaveCount(totalShards);
|
||||
});
|
||||
|
||||
test('should pad report numbers with zeros', async ({ runInlineTest }, testInfo) => {
|
||||
const testFiles = {
|
||||
'playwright.config.ts': `
|
||||
module.exports = { reporter: [['html', { sharded: true }]] };
|
||||
`,
|
||||
};
|
||||
for (let i = 0; i < 100; i++) {
|
||||
testFiles[`a-${i}.spec.ts`] = `
|
||||
import { test, expect } from '@playwright/test';
|
||||
test('passes', async ({}) => { });
|
||||
`;
|
||||
}
|
||||
const result = await runInlineTest(testFiles, { shard: '3/100' }, { PW_TEST_HTML_REPORT_OPEN: 'never' }, { usesCustomReporters: true });
|
||||
expect(result.exitCode).toBe(0);
|
||||
const files = await fs.promises.readdir(testInfo.outputPath(`playwright-report`));
|
||||
expect(new Set(files)).toEqual(new Set([
|
||||
'index.html',
|
||||
`report-003-of-100.zip`
|
||||
]));
|
||||
});
|
||||
|
||||
test('should show report with missing shards', async ({ runInlineTest, showReport, page }, testInfo) => {
|
||||
const totalShards = 15;
|
||||
|
||||
const testFiles = {
|
||||
'playwright.config.ts': `
|
||||
module.exports = { reporter: [['html', { sharded: true }]] };
|
||||
`,
|
||||
};
|
||||
for (let i = 0; i < totalShards; i++) {
|
||||
testFiles[`a-${String(i).padStart(2, '0')}.spec.ts`] = `
|
||||
import { test, expect } from '@playwright/test';
|
||||
test('passes', async ({}) => { expect(2).toBe(2); });
|
||||
test('fails', async ({}) => { expect(1).toBe(2); });
|
||||
test('skipped', async ({}) => { test.skip('Does not work') });
|
||||
test('flaky', async ({}, testInfo) => { expect(testInfo.retry).toBe(1); });
|
||||
`;
|
||||
}
|
||||
|
||||
const allReports = testInfo.outputPath(`aggregated-report`);
|
||||
await fs.promises.mkdir(allReports, { recursive: true });
|
||||
|
||||
// Run tests in 2 out of 15 shards.
|
||||
for (const i of [10, 13]) {
|
||||
const result = await runInlineTest(testFiles,
|
||||
{ 'retries': 1, 'shard': `${i}/${totalShards}` },
|
||||
{ PW_TEST_HTML_REPORT_OPEN: 'never' },
|
||||
{ usesCustomReporters: true });
|
||||
|
||||
|
||||
expect(result.exitCode).toBe(1);
|
||||
const files = await fs.promises.readdir(testInfo.outputPath(`playwright-report`));
|
||||
expect(new Set(files)).toEqual(new Set([
|
||||
'index.html',
|
||||
`report-${i}-of-${totalShards}.zip`
|
||||
]));
|
||||
await Promise.all(files.map(name => fs.promises.rename(testInfo.outputPath(`playwright-report/${name}`), `${allReports}/${name}`)));
|
||||
}
|
||||
|
||||
// Show aggregated report
|
||||
await showReport(allReports);
|
||||
|
||||
await expect(page.getByText('Only 2 of 15 report shards loaded')).toBeVisible();
|
||||
|
||||
await expect(page.locator('.subnav-item:has-text("All") .counter')).toHaveText('8');
|
||||
await expect(page.locator('.subnav-item:has-text("Passed") .counter')).toHaveText('2');
|
||||
await expect(page.locator('.subnav-item:has-text("Failed") .counter')).toHaveText('2');
|
||||
await expect(page.locator('.subnav-item:has-text("Flaky") .counter')).toHaveText('2');
|
||||
await expect(page.locator('.subnav-item:has-text("Skipped") .counter')).toHaveText('2');
|
||||
|
||||
await expect(page.locator('.test-file-test-outcome-unexpected >> text=fails')).toHaveCount(2);
|
||||
await expect(page.locator('.test-file-test-outcome-flaky >> text=flaky')).toHaveCount(2);
|
||||
await expect(page.locator('.test-file-test-outcome-expected >> text=passes')).toHaveCount(2);
|
||||
await expect(page.locator('.test-file-test-outcome-skipped >> text=skipped')).toHaveCount(2);
|
||||
});
|
||||
|
||||
|
||||
test('should produce single file report when shard: false', async ({ runInlineTest, showReport, page }, testInfo) => {
|
||||
const totalShards = 5;
|
||||
|
||||
const testFiles = {};
|
||||
for (let i = 0; i < totalShards; i++) {
|
||||
testFiles[`a-${String(i).padStart(2, '0')}.spec.ts`] = `
|
||||
import { test, expect } from '@playwright/test';
|
||||
test('passes', async ({}) => { expect(2).toBe(2); });
|
||||
test('fails', async ({}) => { expect(1).toBe(2); });
|
||||
test('skipped', async ({}) => { test.skip('Does not work') });
|
||||
test('flaky', async ({}, testInfo) => { expect(testInfo.retry).toBe(1); });
|
||||
`;
|
||||
}
|
||||
|
||||
// Run single shard.
|
||||
const currentShard = 3;
|
||||
const result = await runInlineTest(testFiles,
|
||||
{ 'reporter': 'dot,html', 'retries': 1, 'shard': `${currentShard}/${totalShards}` },
|
||||
{ PW_TEST_HTML_REPORT_OPEN: 'never' },
|
||||
{ usesCustomReporters: true });
|
||||
|
||||
|
||||
expect(result.exitCode).toBe(1);
|
||||
const files = await fs.promises.readdir(testInfo.outputPath(`playwright-report`));
|
||||
expect(files).toEqual(['index.html']);
|
||||
|
||||
await showReport();
|
||||
|
||||
await expect(page.locator('.subnav-item:has-text("All") .counter')).toHaveText('4');
|
||||
await expect(page.locator('.subnav-item:has-text("Passed") .counter')).toHaveText('1');
|
||||
await expect(page.locator('.subnav-item:has-text("Failed") .counter')).toHaveText('1');
|
||||
await expect(page.locator('.subnav-item:has-text("Flaky") .counter')).toHaveText('1');
|
||||
await expect(page.locator('.subnav-item:has-text("Skipped") .counter')).toHaveText('1');
|
||||
|
||||
await expect(page.locator('.test-file-test-outcome-unexpected >> text=fails')).toHaveCount(1);
|
||||
await expect(page.locator('.test-file-test-outcome-flaky >> text=flaky')).toHaveCount(1);
|
||||
await expect(page.locator('.test-file-test-outcome-expected >> text=passes')).toHaveCount(1);
|
||||
await expect(page.locator('.test-file-test-outcome-skipped >> text=skipped')).toHaveCount(1);
|
||||
});
|
||||
|
|
|
|||
2
utils/generate_types/overrides-test.d.ts
vendored
2
utils/generate_types/overrides-test.d.ts
vendored
|
|
@ -24,7 +24,7 @@ export type ReporterDescription =
|
|||
['github'] |
|
||||
['junit'] | ['junit', { outputFile?: string, stripANSIControlSequences?: boolean }] |
|
||||
['json'] | ['json', { outputFile?: string }] |
|
||||
['html'] | ['html', { outputFolder?: string, open?: 'always' | 'never' | 'on-failure', sharded?: boolean }] |
|
||||
['html'] | ['html', { outputFolder?: string, open?: 'always' | 'never' | 'on-failure' }] |
|
||||
['null'] |
|
||||
[string] | [string, any];
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue