Yury Semikhatsky
GitHub
parent
a42567d549
commit
d59e0e10ce
|
|
@ -23,6 +23,7 @@ import { Runner } from './runner/runner';
|
|||
import { stopProfiling, startProfiling } from 'playwright-core/lib/utils';
|
||||
import { experimentalLoaderOption, fileIsModule } from './util';
|
||||
import { showHTMLReport } from './reporters/html';
|
||||
import { createMergedReport } from './reporters/blob';
|
||||
import { ConfigLoader, kDefaultConfigFiles, resolveConfigFile } from './common/configLoader';
|
||||
import type { ConfigCLIOverrides } from './common/ipc';
|
||||
import type { FullResult } from '../reporter';
|
||||
|
|
@ -34,6 +35,7 @@ export function addTestCommands(program: Command) {
|
|||
addTestCommand(program);
|
||||
addShowReportCommand(program);
|
||||
addListFilesCommand(program);
|
||||
addMergeReportsCommand(program);
|
||||
}
|
||||
|
||||
function addTestCommand(program: Command) {
|
||||
|
|
@ -90,6 +92,28 @@ Examples:
|
|||
$ npx playwright show-report playwright-report`);
|
||||
}
|
||||
|
||||
function addMergeReportsCommand(program: Command) {
|
||||
const command = program.command('merge-reports [dir]');
|
||||
command.description('merge multiple blob reports (for sharded tests) into a single report');
|
||||
command.action(async (dir, options) => {
|
||||
try {
|
||||
await mergeReports(dir, options);
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
process.exit(1);
|
||||
}
|
||||
});
|
||||
command.option('-c, --config <file>', `Configuration file. Can be used to specify additional configuration for the output report.`);
|
||||
command.option('--reporter <reporter>', 'Output report type', 'list');
|
||||
command.addHelpText('afterAll', `
|
||||
Arguments [dir]:
|
||||
Directory containing blob reports.
|
||||
|
||||
Examples:
|
||||
$ npx playwright merge-reports playwright-report`);
|
||||
}
|
||||
|
||||
|
||||
async function runTests(args: string[], opts: { [key: string]: any }) {
|
||||
await startProfiling();
|
||||
|
||||
|
|
@ -174,6 +198,24 @@ async function listTestFiles(opts: { [key: string]: any }) {
|
|||
});
|
||||
}
|
||||
|
||||
async function mergeReports(reportDir: string | undefined, opts: { [key: string]: any }) {
|
||||
let configFile = opts.config;
|
||||
if (configFile) {
|
||||
configFile = path.resolve(process.cwd(), configFile);
|
||||
if (!fs.existsSync(configFile))
|
||||
throw new Error(`${configFile} does not exist`);
|
||||
if (!fs.statSync(configFile).isFile())
|
||||
throw new Error(`${configFile} is not a file`);
|
||||
}
|
||||
if (restartWithExperimentalTsEsm(configFile))
|
||||
return;
|
||||
|
||||
const configLoader = new ConfigLoader();
|
||||
const config = await (configFile ? configLoader.loadConfigFile(configFile) : configLoader.loadEmptyConfig(process.cwd()));
|
||||
const dir = path.resolve(process.cwd(), reportDir || 'playwright-report');
|
||||
await createMergedReport(config, dir, opts.reporter || 'list');
|
||||
}
|
||||
|
||||
function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrides {
|
||||
const shardPair = options.shard ? options.shard.split('/').map((t: string) => parseInt(t, 10)) : undefined;
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -38,7 +38,6 @@ export class FullConfigInternal {
|
|||
configDir = '';
|
||||
configCLIOverrides: ConfigCLIOverrides = {};
|
||||
storeDir = '';
|
||||
maxConcurrentTestGroups = 0;
|
||||
ignoreSnapshots = false;
|
||||
webServers: Exclude<FullConfig['webServer'], null>[] = [];
|
||||
plugins: TestRunnerPluginRegistration[] = [];
|
||||
|
|
@ -246,7 +245,7 @@ export function toReporters(reporters: BuiltInReporter | ReporterDescription[] |
|
|||
return reporters;
|
||||
}
|
||||
|
||||
export const builtInReporters = ['list', 'line', 'dot', 'json', 'junit', 'null', 'github', 'html'] as const;
|
||||
export const builtInReporters = ['list', 'line', 'dot', 'json', 'junit', 'null', 'github', 'html', 'blob'] as const;
|
||||
export type BuiltInReporter = typeof builtInReporters[number];
|
||||
|
||||
export type ContextReuseMode = 'none' | 'force' | 'when-possible';
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ export type JsonConfig = {
|
|||
rootDir: string;
|
||||
configFile: string | undefined;
|
||||
listOnly: boolean;
|
||||
workers: number;
|
||||
};
|
||||
|
||||
export type JsonPattern = {
|
||||
|
|
@ -283,6 +284,7 @@ export class TeleReporterReceiver {
|
|||
const fullConfig = baseFullConfig;
|
||||
fullConfig.rootDir = config.rootDir;
|
||||
fullConfig.configFile = config.configFile;
|
||||
fullConfig.workers = config.workers;
|
||||
return fullConfig;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,3 +3,6 @@
|
|||
../isomorphic/**
|
||||
../util.ts
|
||||
../utilsBundle.ts
|
||||
|
||||
[blob.ts]
|
||||
../runner/loadUtils.ts
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ import type { FullConfig, TestCase, Suite, TestResult, TestError, FullResult, Te
|
|||
import type { SuitePrivate } from '../../types/reporterPrivate';
|
||||
import { codeFrameColumns } from '../common/babelBundle';
|
||||
import { monotonicTime } from 'playwright-core/lib/utils';
|
||||
import { FullConfigInternal } from '../common/config';
|
||||
|
||||
export type TestResultOutput = { chunk: string | Buffer, type: 'stdout' | 'stderr' };
|
||||
export const kOutputSymbol = Symbol('output');
|
||||
|
|
@ -122,7 +121,7 @@ export class BaseReporter implements Reporter {
|
|||
}
|
||||
|
||||
protected generateStartingMessage() {
|
||||
const jobs = Math.min(this.config.workers, FullConfigInternal.from(this.config).maxConcurrentTestGroups);
|
||||
const jobs = this.config.workers;
|
||||
const shardDetails = this.config.shard ? `, shard ${this.config.shard.current} of ${this.config.shard.total}` : '';
|
||||
if (!this.totalTestCount)
|
||||
return '';
|
||||
|
|
|
|||
228
packages/playwright-test/src/reporters/blob.ts
Normal file
228
packages/playwright-test/src/reporters/blob.ts
Normal file
|
|
@ -0,0 +1,228 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import type { EventEmitter } from 'events';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { ManualPromise, ZipFile } from 'playwright-core/lib/utils';
|
||||
import { yazl } from 'playwright-core/lib/zipBundle';
|
||||
import { Readable } from 'stream';
|
||||
import type { FullConfig, FullResult, Reporter } from '../../types/testReporter';
|
||||
import type { BuiltInReporter, FullConfigInternal } from '../common/config';
|
||||
import type { Suite } from '../common/test';
|
||||
import { TeleReporterReceiver, type JsonEvent, type JsonProject, type JsonSuite } from '../isomorphic/teleReceiver';
|
||||
import DotReporter from '../reporters/dot';
|
||||
import EmptyReporter from '../reporters/empty';
|
||||
import GitHubReporter from '../reporters/github';
|
||||
import JSONReporter from '../reporters/json';
|
||||
import JUnitReporter from '../reporters/junit';
|
||||
import LineReporter from '../reporters/line';
|
||||
import ListReporter from '../reporters/list';
|
||||
import { loadReporter } from '../runner/loadUtils';
|
||||
import HtmlReporter, { defaultReportFolder } from './html';
|
||||
import { TeleReporterEmitter } from './teleEmitter';
|
||||
|
||||
|
||||
type BlobReporterOptions = {
|
||||
configDir: string;
|
||||
outputDir?: string;
|
||||
};
|
||||
|
||||
export class BlobReporter extends TeleReporterEmitter {
|
||||
private _messages: any[] = [];
|
||||
private _options: BlobReporterOptions;
|
||||
private _outputFile!: string;
|
||||
|
||||
constructor(options: BlobReporterOptions) {
|
||||
super(message => this._messages.push(message));
|
||||
this._options = options;
|
||||
}
|
||||
|
||||
override onBegin(config: FullConfig<{}, {}>, suite: Suite): void {
|
||||
super.onBegin(config, suite);
|
||||
this._computeOutputFileName(config);
|
||||
}
|
||||
|
||||
override async onEnd(result: FullResult): Promise<void> {
|
||||
await super.onEnd(result);
|
||||
fs.mkdirSync(path.dirname(this._outputFile), { recursive: true });
|
||||
const lines = this._messages.map(m => JSON.stringify(m) + '\n');
|
||||
await zipReport(this._outputFile, lines);
|
||||
}
|
||||
|
||||
private _computeOutputFileName(config: FullConfig) {
|
||||
const outputDir = this._resolveOutputDir();
|
||||
let shardSuffix = '';
|
||||
if (config.shard) {
|
||||
const paddedNumber = `${config.shard.current}`.padStart(`${config.shard.total}`.length, '0');
|
||||
shardSuffix = `-${paddedNumber}-of-${config.shard.total}`;
|
||||
}
|
||||
this._outputFile = path.join(outputDir, `report${shardSuffix}.zip`);
|
||||
}
|
||||
|
||||
private _resolveOutputDir(): string {
|
||||
const { outputDir } = this._options;
|
||||
if (outputDir)
|
||||
return path.resolve(this._options.configDir, outputDir);
|
||||
return defaultReportFolder(this._options.configDir);
|
||||
}
|
||||
}
|
||||
|
||||
export async function createMergedReport(config: FullConfigInternal, dir: string, reporterName?: string) {
|
||||
const shardFiles = await sortedShardFiles(dir);
|
||||
const events = await mergeEvents(dir, shardFiles);
|
||||
|
||||
const defaultReporters: {[key in BuiltInReporter]: new(arg: any) => Reporter} = {
|
||||
dot: DotReporter,
|
||||
line: LineReporter,
|
||||
list: ListReporter,
|
||||
github: GitHubReporter,
|
||||
json: JSONReporter,
|
||||
junit: JUnitReporter,
|
||||
null: EmptyReporter,
|
||||
html: HtmlReporter,
|
||||
blob: BlobReporter,
|
||||
};
|
||||
reporterName ??= 'list';
|
||||
|
||||
const arg = config.config.reporter.find(([reporter, arg]) => reporter === reporterName)?.[1];
|
||||
const options = {
|
||||
...arg,
|
||||
configDir: process.cwd(),
|
||||
outputFolder: dir
|
||||
};
|
||||
|
||||
let reporter: Reporter | undefined;
|
||||
if (reporterName in defaultReporters) {
|
||||
reporter = new defaultReporters[reporterName as keyof typeof defaultReporters](options);
|
||||
} else {
|
||||
const reporterConstructor = await loadReporter(config, reporterName);
|
||||
reporter = new reporterConstructor(options);
|
||||
}
|
||||
|
||||
const receiver = new TeleReporterReceiver(path.sep, reporter);
|
||||
for (const event of events)
|
||||
await receiver.dispatch(event);
|
||||
console.log(`Done.`);
|
||||
}
|
||||
|
||||
async function mergeEvents(dir: string, shardFiles: string[]) {
|
||||
const events: JsonEvent[] = [];
|
||||
const beginEvents: JsonEvent[] = [];
|
||||
const endEvents: JsonEvent[] = [];
|
||||
for (const file of shardFiles) {
|
||||
const zipFile = new ZipFile(path.join(dir, file));
|
||||
const entryNames = await zipFile.entries();
|
||||
const reportEntryName = entryNames.find(e => e.endsWith('.jsonl'));
|
||||
if (!reportEntryName)
|
||||
throw new Error(`Zip file ${file} does not contain a .jsonl file`);
|
||||
const reportJson = await zipFile.read(reportEntryName);
|
||||
const parsedEvents = reportJson.toString().split('\n').filter(line => line.length).map(line => JSON.parse(line)) as JsonEvent[];
|
||||
for (const event of parsedEvents) {
|
||||
// TODO: show remaining events?
|
||||
if (event.method === 'onError')
|
||||
throw new Error('Error in shard: ' + file);
|
||||
if (event.method === 'onBegin')
|
||||
beginEvents.push(event);
|
||||
else if (event.method === 'onEnd')
|
||||
endEvents.push(event);
|
||||
else
|
||||
events.push(event);
|
||||
}
|
||||
|
||||
}
|
||||
return [mergeBeginEvents(beginEvents), ...events, mergeEndEvents(endEvents)];
|
||||
}
|
||||
|
||||
function mergeBeginEvents(beginEvents: JsonEvent[]): JsonEvent {
|
||||
if (!beginEvents.length)
|
||||
throw new Error('No begin events found');
|
||||
const projects: JsonProject[] = [];
|
||||
let totalWorkers = 0;
|
||||
for (const event of beginEvents) {
|
||||
totalWorkers += event.params.config.workers;
|
||||
const shardProjects: JsonProject[] = event.params.projects;
|
||||
for (const shardProject of shardProjects) {
|
||||
const mergedProject = projects.find(p => p.id === shardProject.id);
|
||||
if (!mergedProject)
|
||||
projects.push(shardProject);
|
||||
else
|
||||
mergeJsonSuites(shardProject.suites, mergedProject);
|
||||
}
|
||||
}
|
||||
const config = {
|
||||
...beginEvents[0].params.config,
|
||||
workers: totalWorkers,
|
||||
shard: undefined
|
||||
};
|
||||
return {
|
||||
method: 'onBegin',
|
||||
params: {
|
||||
config,
|
||||
projects,
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function mergeJsonSuites(jsonSuites: JsonSuite[], parent: JsonSuite | JsonProject) {
|
||||
for (const jsonSuite of jsonSuites) {
|
||||
const existingSuite = parent.suites.find(s => s.title === jsonSuite.title);
|
||||
if (!existingSuite) {
|
||||
parent.suites.push(jsonSuite);
|
||||
} else {
|
||||
mergeJsonSuites(jsonSuite.suites, existingSuite);
|
||||
existingSuite.tests.push(...jsonSuite.tests);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function mergeEndEvents(endEvents: JsonEvent[]): JsonEvent {
|
||||
const result: FullResult = { status: 'passed' };
|
||||
for (const event of endEvents) {
|
||||
const shardResult: FullResult = event.params.result;
|
||||
if (shardResult.status === 'failed')
|
||||
result.status = 'failed';
|
||||
else if (shardResult.status === 'timedout' && result.status !== 'failed')
|
||||
result.status = 'timedout';
|
||||
else if (shardResult.status === 'interrupted' && result.status !== 'failed' && result.status !== 'timedout')
|
||||
result.status = 'interrupted';
|
||||
}
|
||||
return {
|
||||
method: 'onEnd',
|
||||
params: {
|
||||
result
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async function sortedShardFiles(dir: string) {
|
||||
const files = await fs.promises.readdir(dir);
|
||||
return files.filter(file => file.endsWith('.zip')).sort();
|
||||
}
|
||||
|
||||
async function zipReport(zipFileName: string, lines: string[]) {
|
||||
const zipFile = new yazl.ZipFile();
|
||||
const result = new ManualPromise<undefined>();
|
||||
(zipFile as any as EventEmitter).on('error', error => result.reject(error));
|
||||
// TODO: feed events on the fly.
|
||||
const content = Readable.from(lines);
|
||||
zipFile.addReadStream(content, 'report.jsonl');
|
||||
zipFile.end();
|
||||
zipFile.outputStream.pipe(fs.createWriteStream(zipFileName)).on('close', () => {
|
||||
result.resolve(undefined);
|
||||
});
|
||||
await result;
|
||||
}
|
||||
|
|
@ -138,7 +138,7 @@ function reportFolderFromEnv(): string | undefined {
|
|||
return undefined;
|
||||
}
|
||||
|
||||
function defaultReportFolder(searchForPackageJson: string): string {
|
||||
export function defaultReportFolder(searchForPackageJson: string): string {
|
||||
let basePath = getPackageJsonPath(searchForPackageJson);
|
||||
if (basePath)
|
||||
basePath = path.dirname(basePath);
|
||||
|
|
|
|||
|
|
@ -124,6 +124,7 @@ export class TeleReporterEmitter implements Reporter {
|
|||
rootDir: config.rootDir,
|
||||
configFile: this._relativePath(config.configFile),
|
||||
listOnly: FullConfigInternal.from(config).listOnly,
|
||||
workers: config.workers,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ import { Multiplexer } from '../reporters/multiplexer';
|
|||
import type { Suite } from '../common/test';
|
||||
import type { BuiltInReporter, FullConfigInternal } from '../common/config';
|
||||
import { loadReporter } from './loadUtils';
|
||||
import { BlobReporter } from '../reporters/blob';
|
||||
|
||||
export async function createReporter(config: FullConfigInternal, mode: 'list' | 'watch' | 'run' | 'ui', additionalReporters: Reporter[] = []): Promise<Multiplexer> {
|
||||
const defaultReporters: {[key in BuiltInReporter]: new(arg: any) => Reporter} = {
|
||||
|
|
@ -40,6 +41,7 @@ export async function createReporter(config: FullConfigInternal, mode: 'list' |
|
|||
junit: JUnitReporter,
|
||||
null: EmptyReporter,
|
||||
html: mode === 'ui' ? LineReporter : HtmlReporter,
|
||||
blob: BlobReporter,
|
||||
};
|
||||
const reporters: Reporter[] = [];
|
||||
if (mode === 'watch') {
|
||||
|
|
|
|||
|
|
@ -177,7 +177,7 @@ function createLoadTask(mode: 'out-of-process' | 'in-process', shouldFilterOnly:
|
|||
|
||||
function createPhasesTask(): Task<TestRun> {
|
||||
return async testRun => {
|
||||
testRun.config.maxConcurrentTestGroups = 0;
|
||||
let maxConcurrentTestGroups = 0;
|
||||
|
||||
const processed = new Set<FullProjectInternal>();
|
||||
const projectToSuite = new Map(testRun.rootSuite!.suites.map(suite => [suite._projectConfig!, suite]));
|
||||
|
|
@ -206,9 +206,11 @@ function createPhasesTask(): Task<TestRun> {
|
|||
testGroupsInPhase += testGroups.length;
|
||||
}
|
||||
debug('pw:test:task')(`created phase #${testRun.phases.length} with ${phase.projects.map(p => p.project.project.name).sort()} projects, ${testGroupsInPhase} testGroups`);
|
||||
testRun.config.maxConcurrentTestGroups = Math.max(testRun.config.maxConcurrentTestGroups, testGroupsInPhase);
|
||||
maxConcurrentTestGroups = Math.max(maxConcurrentTestGroups, testGroupsInPhase);
|
||||
}
|
||||
}
|
||||
|
||||
testRun.config.config.workers = Math.min(testRun.config.config.workers, maxConcurrentTestGroups);
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
255
tests/playwright-test/reporter-blob.spec.ts
Normal file
255
tests/playwright-test/reporter-blob.spec.ts
Normal file
|
|
@ -0,0 +1,255 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import * as fs from 'fs';
|
||||
import type { HttpServer } from '../../packages/playwright-core/src/utils';
|
||||
import { startHtmlReportServer } from '../../packages/playwright-test/lib/reporters/html';
|
||||
import { type CliRunResult, type RunOptions, stripAnsi } from './playwright-test-fixtures';
|
||||
import { cleanEnv, cliEntrypoint, expect, test as baseTest } from './playwright-test-fixtures';
|
||||
|
||||
const DOES_NOT_SUPPORT_UTF8_IN_TERMINAL = process.platform === 'win32' && process.env.TERM_PROGRAM !== 'vscode' && !process.env.WT_SESSION;
|
||||
const POSITIVE_STATUS_MARK = DOES_NOT_SUPPORT_UTF8_IN_TERMINAL ? 'ok' : '✓ ';
|
||||
const NEGATIVE_STATUS_MARK = DOES_NOT_SUPPORT_UTF8_IN_TERMINAL ? 'x ' : '✘ ';
|
||||
|
||||
const test = baseTest.extend<{
|
||||
showReport: (reportFolder?: string) => Promise<void>,
|
||||
mergeReports: (reportFolder: string, env?: NodeJS.ProcessEnv, options?: RunOptions) => Promise<CliRunResult>
|
||||
}>({
|
||||
showReport: async ({ page }, use, testInfo) => {
|
||||
let server: HttpServer | undefined;
|
||||
await use(async (reportFolder?: string) => {
|
||||
reportFolder ??= testInfo.outputPath('playwright-report');
|
||||
server = startHtmlReportServer(reportFolder) as HttpServer;
|
||||
const location = await server.start();
|
||||
await page.goto(location);
|
||||
});
|
||||
await server?.stop();
|
||||
},
|
||||
mergeReports: async ({ childProcess, page }, use, testInfo) => {
|
||||
await use(async (reportFolder: string, env: NodeJS.ProcessEnv = {}, options: RunOptions = {}) => {
|
||||
const command = ['node', cliEntrypoint, 'merge-reports', reportFolder];
|
||||
if (options.additionalArgs)
|
||||
command.push(...options.additionalArgs);
|
||||
|
||||
const testProcess = childProcess({
|
||||
command,
|
||||
env: cleanEnv(env),
|
||||
// cwd,
|
||||
});
|
||||
const { exitCode } = await testProcess.exited;
|
||||
return { exitCode, output: testProcess.output.toString() };
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
test('should merge into html', async ({ runInlineTest, mergeReports, showReport, page }) => {
|
||||
const reportDir = test.info().outputPath('blob-report');
|
||||
const files = {
|
||||
'playwright.config.ts': `
|
||||
module.exports = {
|
||||
retries: 1,
|
||||
reporter: [['blob', { outputDir: '${reportDir.replace(/\\/g, '/')}' }]]
|
||||
};
|
||||
`,
|
||||
'a.test.js': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
test('math 1', async ({}) => {
|
||||
expect(1 + 1).toBe(2);
|
||||
});
|
||||
test('failing 1', async ({}) => {
|
||||
expect(1).toBe(2);
|
||||
});
|
||||
test('flaky 1', async ({}) => {
|
||||
expect(test.info().retry).toBe(1);
|
||||
});
|
||||
test.skip('skipped 1', async ({}) => {});
|
||||
`,
|
||||
'b.test.js': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
test('math 2', async ({}) => {
|
||||
expect(1 + 1).toBe(2);
|
||||
});
|
||||
test('failing 2', async ({}) => {
|
||||
expect(1).toBe(2);
|
||||
});
|
||||
test.skip('skipped 2', async ({}) => {});
|
||||
`,
|
||||
'c.test.js': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
test('math 3', async ({}) => {
|
||||
expect(1 + 1).toBe(2);
|
||||
});
|
||||
test('flaky 2', async ({}) => {
|
||||
expect(test.info().retry).toBe(1);
|
||||
});
|
||||
test.skip('skipped 3', async ({}) => {});
|
||||
`
|
||||
};
|
||||
const totalShards = 3;
|
||||
for (let i = 0; i < totalShards; i++)
|
||||
await runInlineTest(files, { shard: `${i + 1}/${totalShards}` });
|
||||
const reportFiles = await fs.promises.readdir(reportDir);
|
||||
reportFiles.sort();
|
||||
expect(reportFiles).toEqual(['report-1-of-3.zip', 'report-2-of-3.zip', 'report-3-of-3.zip']);
|
||||
const { exitCode } = await mergeReports(reportDir, {}, { additionalArgs: ['--reporter', 'html'] });
|
||||
expect(exitCode).toBe(0);
|
||||
|
||||
await showReport(reportDir);
|
||||
|
||||
await expect(page.locator('.subnav-item:has-text("All") .counter')).toHaveText('10');
|
||||
await expect(page.locator('.subnav-item:has-text("Passed") .counter')).toHaveText('3');
|
||||
await expect(page.locator('.subnav-item:has-text("Failed") .counter')).toHaveText('2');
|
||||
await expect(page.locator('.subnav-item:has-text("Flaky") .counter')).toHaveText('2');
|
||||
await expect(page.locator('.subnav-item:has-text("Skipped") .counter')).toHaveText('3');
|
||||
|
||||
await expect(page.locator('.test-file-test .test-file-title')).toHaveText(
|
||||
['failing 1', 'flaky 1', 'math 1', 'skipped 1', 'failing 2', 'math 2', 'skipped 2', 'flaky 2', 'math 3', 'skipped 3']);
|
||||
});
|
||||
|
||||
test('be able to merge incomplete shards', async ({ runInlineTest, mergeReports, showReport, page }) => {
|
||||
const reportDir = test.info().outputPath('blob-report');
|
||||
const files = {
|
||||
'playwright.config.ts': `
|
||||
module.exports = {
|
||||
retries: 1,
|
||||
reporter: [['blob', { outputDir: '${reportDir.replace(/\\/g, '/')}' }]]
|
||||
};
|
||||
`,
|
||||
'a.test.js': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
test('math 1', async ({}) => {
|
||||
});
|
||||
test('failing 1', async ({}) => {
|
||||
expect(1).toBe(2);
|
||||
});
|
||||
test.skip('skipped 1', async ({}) => {});
|
||||
`,
|
||||
'b.test.js': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
test('math 2', async ({}) => { });
|
||||
test('failing 2', async ({}) => {
|
||||
expect(1).toBe(2);
|
||||
});
|
||||
test.skip('skipped 2', async ({}) => {});
|
||||
`,
|
||||
'c.test.js': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
test('math 3', async ({}) => {
|
||||
expect(1 + 1).toBe(2);
|
||||
});
|
||||
test('flaky 2', async ({}) => {
|
||||
expect(test.info().retry).toBe(1);
|
||||
});
|
||||
test.skip('skipped 3', async ({}) => {});
|
||||
`
|
||||
};
|
||||
await runInlineTest(files, { shard: `1/3` });
|
||||
await runInlineTest(files, { shard: `3/3` });
|
||||
|
||||
const reportFiles = await fs.promises.readdir(reportDir);
|
||||
reportFiles.sort();
|
||||
expect(reportFiles).toEqual(['report-1-of-3.zip', 'report-3-of-3.zip']);
|
||||
const { exitCode } = await mergeReports(reportDir, {}, { additionalArgs: ['--reporter', 'html'] });
|
||||
expect(exitCode).toBe(0);
|
||||
|
||||
await showReport(reportDir);
|
||||
|
||||
await expect(page.locator('.subnav-item:has-text("All") .counter')).toHaveText('6');
|
||||
await expect(page.locator('.subnav-item:has-text("Passed") .counter')).toHaveText('2');
|
||||
await expect(page.locator('.subnav-item:has-text("Failed") .counter')).toHaveText('1');
|
||||
await expect(page.locator('.subnav-item:has-text("Flaky") .counter')).toHaveText('1');
|
||||
await expect(page.locator('.subnav-item:has-text("Skipped") .counter')).toHaveText('2');
|
||||
});
|
||||
|
||||
test('merge into list report by default', async ({ runInlineTest, mergeReports }) => {
|
||||
const reportDir = test.info().outputPath('blob-report');
|
||||
const files = {
|
||||
'playwright.config.ts': `
|
||||
module.exports = {
|
||||
retries: 1,
|
||||
reporter: [['blob', { outputDir: '${reportDir.replace(/\\/g, '/')}' }]]
|
||||
};
|
||||
`,
|
||||
'a.test.js': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
test('math 1', async ({}) => {
|
||||
expect(1 + 1).toBe(2);
|
||||
});
|
||||
test('failing 1', async ({}) => {
|
||||
expect(1).toBe(2);
|
||||
});
|
||||
test('flaky 1', async ({}) => {
|
||||
expect(test.info().retry).toBe(1);
|
||||
});
|
||||
test.skip('skipped 1', async ({}) => {});
|
||||
`,
|
||||
'b.test.js': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
test('math 2', async ({}) => {
|
||||
expect(1 + 1).toBe(2);
|
||||
});
|
||||
test('failing 2', async ({}) => {
|
||||
expect(1).toBe(2);
|
||||
});
|
||||
test.skip('skipped 2', async ({}) => {});
|
||||
`,
|
||||
'c.test.js': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
test('math 3', async ({}) => {
|
||||
expect(1 + 1).toBe(2);
|
||||
});
|
||||
test('flaky 2', async ({}) => {
|
||||
expect(test.info().retry).toBe(1);
|
||||
});
|
||||
test.skip('skipped 3', async ({}) => {});
|
||||
`
|
||||
};
|
||||
|
||||
const totalShards = 3;
|
||||
for (let i = 0; i < totalShards; i++)
|
||||
await runInlineTest(files, { shard: `${i + 1}/${totalShards}` });
|
||||
const reportFiles = await fs.promises.readdir(reportDir);
|
||||
reportFiles.sort();
|
||||
expect(reportFiles).toEqual(['report-1-of-3.zip', 'report-2-of-3.zip', 'report-3-of-3.zip']);
|
||||
const { exitCode, output } = await mergeReports(reportDir, { PW_TEST_DEBUG_REPORTERS: '1', PW_TEST_DEBUG_REPORTERS_PRINT_STEPS: '1', PWTEST_TTY_WIDTH: '80' }, { additionalArgs: ['--reporter', 'list'] });
|
||||
expect(exitCode).toBe(0);
|
||||
|
||||
const text = stripAnsi(output);
|
||||
expect(text).toContain('Running 10 tests using 3 workers');
|
||||
const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/\d+ms/, 'Xms'));
|
||||
expect(lines).toEqual([
|
||||
`0 : 1 a.test.js:3:11 › math 1`,
|
||||
`0 : ${POSITIVE_STATUS_MARK} 1 a.test.js:3:11 › math 1 (Xms)`,
|
||||
`1 : 2 a.test.js:6:11 › failing 1`,
|
||||
`1 : ${NEGATIVE_STATUS_MARK} 2 a.test.js:6:11 › failing 1 (Xms)`,
|
||||
`2 : 3 a.test.js:6:11 › failing 1 (retry #1)`,
|
||||
`2 : ${NEGATIVE_STATUS_MARK} 3 a.test.js:6:11 › failing 1 (retry #1) (Xms)`,
|
||||
`3 : 4 a.test.js:9:11 › flaky 1`,
|
||||
`3 : ${NEGATIVE_STATUS_MARK} 4 a.test.js:9:11 › flaky 1 (Xms)`,
|
||||
`4 : 5 a.test.js:9:11 › flaky 1 (retry #1)`,
|
||||
`4 : ${POSITIVE_STATUS_MARK} 5 a.test.js:9:11 › flaky 1 (retry #1) (Xms)`,
|
||||
`5 : 6 a.test.js:12:12 › skipped 1`,
|
||||
`5 : - 6 a.test.js:12:12 › skipped 1`,
|
||||
`6 : 7 b.test.js:3:11 › math 2`,
|
||||
`6 : ${POSITIVE_STATUS_MARK} 7 b.test.js:3:11 › math 2 (Xms)`,
|
||||
`7 : 8 b.test.js:6:11 › failing 2`,
|
||||
`7 : ${NEGATIVE_STATUS_MARK} 8 b.test.js:6:11 › failing 2 (Xms)`,
|
||||
`8 : 9 b.test.js:6:11 › failing 2 (retry #1)`,
|
||||
`8 : ${NEGATIVE_STATUS_MARK} 9 b.test.js:6:11 › failing 2 (retry #1) (Xms)`,
|
||||
`9 : 10 b.test.js:9:12 › skipped 2`,
|
||||
`9 : - 10 b.test.js:9:12 › skipped 2`
|
||||
]);
|
||||
});
|
||||
|
|
@ -68,6 +68,7 @@ test('render steps', async ({ runInlineTest }) => {
|
|||
`,
|
||||
}, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PW_TEST_DEBUG_REPORTERS_PRINT_STEPS: '1', PWTEST_TTY_WIDTH: '80' });
|
||||
const text = result.output;
|
||||
console.log(result.output)
|
||||
const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/\d+ms/, 'Xms'));
|
||||
lines.pop(); // Remove last item that contains [v] and time in ms.
|
||||
expect(lines).toEqual([
|
||||
|
|
|
|||
Loading…
Reference in a new issue