From d59e0e10cee6edf89e638deebd8d033bc1f3ccb3 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Fri, 7 Apr 2023 13:47:52 -0700 Subject: [PATCH] feat: blob reporter (#22244) #10437 --- packages/playwright-test/src/cli.ts | 42 +++ packages/playwright-test/src/common/config.ts | 3 +- .../src/isomorphic/teleReceiver.ts | 2 + .../playwright-test/src/reporters/DEPS.list | 3 + .../playwright-test/src/reporters/base.ts | 3 +- .../playwright-test/src/reporters/blob.ts | 228 ++++++++++++++++ .../playwright-test/src/reporters/html.ts | 2 +- .../src/reporters/teleEmitter.ts | 1 + .../playwright-test/src/runner/reporters.ts | 2 + packages/playwright-test/src/runner/tasks.ts | 6 +- tests/playwright-test/reporter-blob.spec.ts | 255 ++++++++++++++++++ tests/playwright-test/reporter-list.spec.ts | 1 + 12 files changed, 541 insertions(+), 7 deletions(-) create mode 100644 packages/playwright-test/src/reporters/blob.ts create mode 100644 tests/playwright-test/reporter-blob.spec.ts diff --git a/packages/playwright-test/src/cli.ts b/packages/playwright-test/src/cli.ts index 51b1bc2610..514f79455a 100644 --- a/packages/playwright-test/src/cli.ts +++ b/packages/playwright-test/src/cli.ts @@ -23,6 +23,7 @@ import { Runner } from './runner/runner'; import { stopProfiling, startProfiling } from 'playwright-core/lib/utils'; import { experimentalLoaderOption, fileIsModule } from './util'; import { showHTMLReport } from './reporters/html'; +import { createMergedReport } from './reporters/blob'; import { ConfigLoader, kDefaultConfigFiles, resolveConfigFile } from './common/configLoader'; import type { ConfigCLIOverrides } from './common/ipc'; import type { FullResult } from '../reporter'; @@ -34,6 +35,7 @@ export function addTestCommands(program: Command) { addTestCommand(program); addShowReportCommand(program); addListFilesCommand(program); + addMergeReportsCommand(program); } function addTestCommand(program: Command) { @@ -90,6 +92,28 @@ Examples: $ npx playwright show-report playwright-report`); } +function addMergeReportsCommand(program: Command) { + const command = program.command('merge-reports [dir]'); + command.description('merge multiple blob reports (for sharded tests) into a single report'); + command.action(async (dir, options) => { + try { + await mergeReports(dir, options); + } catch (e) { + console.error(e); + process.exit(1); + } + }); + command.option('-c, --config ', `Configuration file. Can be used to specify additional configuration for the output report.`); + command.option('--reporter ', 'Output report type', 'list'); + command.addHelpText('afterAll', ` +Arguments [dir]: + Directory containing blob reports. + +Examples: + $ npx playwright merge-reports playwright-report`); +} + + async function runTests(args: string[], opts: { [key: string]: any }) { await startProfiling(); @@ -174,6 +198,24 @@ async function listTestFiles(opts: { [key: string]: any }) { }); } +async function mergeReports(reportDir: string | undefined, opts: { [key: string]: any }) { + let configFile = opts.config; + if (configFile) { + configFile = path.resolve(process.cwd(), configFile); + if (!fs.existsSync(configFile)) + throw new Error(`${configFile} does not exist`); + if (!fs.statSync(configFile).isFile()) + throw new Error(`${configFile} is not a file`); + } + if (restartWithExperimentalTsEsm(configFile)) + return; + + const configLoader = new ConfigLoader(); + const config = await (configFile ? configLoader.loadConfigFile(configFile) : configLoader.loadEmptyConfig(process.cwd())); + const dir = path.resolve(process.cwd(), reportDir || 'playwright-report'); + await createMergedReport(config, dir, opts.reporter || 'list'); +} + function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrides { const shardPair = options.shard ? options.shard.split('/').map((t: string) => parseInt(t, 10)) : undefined; return { diff --git a/packages/playwright-test/src/common/config.ts b/packages/playwright-test/src/common/config.ts index b6ad4ed3f7..8610982298 100644 --- a/packages/playwright-test/src/common/config.ts +++ b/packages/playwright-test/src/common/config.ts @@ -38,7 +38,6 @@ export class FullConfigInternal { configDir = ''; configCLIOverrides: ConfigCLIOverrides = {}; storeDir = ''; - maxConcurrentTestGroups = 0; ignoreSnapshots = false; webServers: Exclude[] = []; plugins: TestRunnerPluginRegistration[] = []; @@ -246,7 +245,7 @@ export function toReporters(reporters: BuiltInReporter | ReporterDescription[] | return reporters; } -export const builtInReporters = ['list', 'line', 'dot', 'json', 'junit', 'null', 'github', 'html'] as const; +export const builtInReporters = ['list', 'line', 'dot', 'json', 'junit', 'null', 'github', 'html', 'blob'] as const; export type BuiltInReporter = typeof builtInReporters[number]; export type ContextReuseMode = 'none' | 'force' | 'when-possible'; diff --git a/packages/playwright-test/src/isomorphic/teleReceiver.ts b/packages/playwright-test/src/isomorphic/teleReceiver.ts index 081ac7757a..91edb7a3cf 100644 --- a/packages/playwright-test/src/isomorphic/teleReceiver.ts +++ b/packages/playwright-test/src/isomorphic/teleReceiver.ts @@ -28,6 +28,7 @@ export type JsonConfig = { rootDir: string; configFile: string | undefined; listOnly: boolean; + workers: number; }; export type JsonPattern = { @@ -283,6 +284,7 @@ export class TeleReporterReceiver { const fullConfig = baseFullConfig; fullConfig.rootDir = config.rootDir; fullConfig.configFile = config.configFile; + fullConfig.workers = config.workers; return fullConfig; } diff --git a/packages/playwright-test/src/reporters/DEPS.list b/packages/playwright-test/src/reporters/DEPS.list index 472e911a17..f2b0f3c999 100644 --- a/packages/playwright-test/src/reporters/DEPS.list +++ b/packages/playwright-test/src/reporters/DEPS.list @@ -3,3 +3,6 @@ ../isomorphic/** ../util.ts ../utilsBundle.ts + +[blob.ts] +../runner/loadUtils.ts diff --git a/packages/playwright-test/src/reporters/base.ts b/packages/playwright-test/src/reporters/base.ts index 53b554469e..e8f6477b9b 100644 --- a/packages/playwright-test/src/reporters/base.ts +++ b/packages/playwright-test/src/reporters/base.ts @@ -21,7 +21,6 @@ import type { FullConfig, TestCase, Suite, TestResult, TestError, FullResult, Te import type { SuitePrivate } from '../../types/reporterPrivate'; import { codeFrameColumns } from '../common/babelBundle'; import { monotonicTime } from 'playwright-core/lib/utils'; -import { FullConfigInternal } from '../common/config'; export type TestResultOutput = { chunk: string | Buffer, type: 'stdout' | 'stderr' }; export const kOutputSymbol = Symbol('output'); @@ -122,7 +121,7 @@ export class BaseReporter implements Reporter { } protected generateStartingMessage() { - const jobs = Math.min(this.config.workers, FullConfigInternal.from(this.config).maxConcurrentTestGroups); + const jobs = this.config.workers; const shardDetails = this.config.shard ? `, shard ${this.config.shard.current} of ${this.config.shard.total}` : ''; if (!this.totalTestCount) return ''; diff --git a/packages/playwright-test/src/reporters/blob.ts b/packages/playwright-test/src/reporters/blob.ts new file mode 100644 index 0000000000..1d55984f3d --- /dev/null +++ b/packages/playwright-test/src/reporters/blob.ts @@ -0,0 +1,228 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { EventEmitter } from 'events'; +import fs from 'fs'; +import path from 'path'; +import { ManualPromise, ZipFile } from 'playwright-core/lib/utils'; +import { yazl } from 'playwright-core/lib/zipBundle'; +import { Readable } from 'stream'; +import type { FullConfig, FullResult, Reporter } from '../../types/testReporter'; +import type { BuiltInReporter, FullConfigInternal } from '../common/config'; +import type { Suite } from '../common/test'; +import { TeleReporterReceiver, type JsonEvent, type JsonProject, type JsonSuite } from '../isomorphic/teleReceiver'; +import DotReporter from '../reporters/dot'; +import EmptyReporter from '../reporters/empty'; +import GitHubReporter from '../reporters/github'; +import JSONReporter from '../reporters/json'; +import JUnitReporter from '../reporters/junit'; +import LineReporter from '../reporters/line'; +import ListReporter from '../reporters/list'; +import { loadReporter } from '../runner/loadUtils'; +import HtmlReporter, { defaultReportFolder } from './html'; +import { TeleReporterEmitter } from './teleEmitter'; + + +type BlobReporterOptions = { + configDir: string; + outputDir?: string; +}; + +export class BlobReporter extends TeleReporterEmitter { + private _messages: any[] = []; + private _options: BlobReporterOptions; + private _outputFile!: string; + + constructor(options: BlobReporterOptions) { + super(message => this._messages.push(message)); + this._options = options; + } + + override onBegin(config: FullConfig<{}, {}>, suite: Suite): void { + super.onBegin(config, suite); + this._computeOutputFileName(config); + } + + override async onEnd(result: FullResult): Promise { + await super.onEnd(result); + fs.mkdirSync(path.dirname(this._outputFile), { recursive: true }); + const lines = this._messages.map(m => JSON.stringify(m) + '\n'); + await zipReport(this._outputFile, lines); + } + + private _computeOutputFileName(config: FullConfig) { + const outputDir = this._resolveOutputDir(); + let shardSuffix = ''; + if (config.shard) { + const paddedNumber = `${config.shard.current}`.padStart(`${config.shard.total}`.length, '0'); + shardSuffix = `-${paddedNumber}-of-${config.shard.total}`; + } + this._outputFile = path.join(outputDir, `report${shardSuffix}.zip`); + } + + private _resolveOutputDir(): string { + const { outputDir } = this._options; + if (outputDir) + return path.resolve(this._options.configDir, outputDir); + return defaultReportFolder(this._options.configDir); + } +} + +export async function createMergedReport(config: FullConfigInternal, dir: string, reporterName?: string) { + const shardFiles = await sortedShardFiles(dir); + const events = await mergeEvents(dir, shardFiles); + + const defaultReporters: {[key in BuiltInReporter]: new(arg: any) => Reporter} = { + dot: DotReporter, + line: LineReporter, + list: ListReporter, + github: GitHubReporter, + json: JSONReporter, + junit: JUnitReporter, + null: EmptyReporter, + html: HtmlReporter, + blob: BlobReporter, + }; + reporterName ??= 'list'; + + const arg = config.config.reporter.find(([reporter, arg]) => reporter === reporterName)?.[1]; + const options = { + ...arg, + configDir: process.cwd(), + outputFolder: dir + }; + + let reporter: Reporter | undefined; + if (reporterName in defaultReporters) { + reporter = new defaultReporters[reporterName as keyof typeof defaultReporters](options); + } else { + const reporterConstructor = await loadReporter(config, reporterName); + reporter = new reporterConstructor(options); + } + + const receiver = new TeleReporterReceiver(path.sep, reporter); + for (const event of events) + await receiver.dispatch(event); + console.log(`Done.`); +} + +async function mergeEvents(dir: string, shardFiles: string[]) { + const events: JsonEvent[] = []; + const beginEvents: JsonEvent[] = []; + const endEvents: JsonEvent[] = []; + for (const file of shardFiles) { + const zipFile = new ZipFile(path.join(dir, file)); + const entryNames = await zipFile.entries(); + const reportEntryName = entryNames.find(e => e.endsWith('.jsonl')); + if (!reportEntryName) + throw new Error(`Zip file ${file} does not contain a .jsonl file`); + const reportJson = await zipFile.read(reportEntryName); + const parsedEvents = reportJson.toString().split('\n').filter(line => line.length).map(line => JSON.parse(line)) as JsonEvent[]; + for (const event of parsedEvents) { + // TODO: show remaining events? + if (event.method === 'onError') + throw new Error('Error in shard: ' + file); + if (event.method === 'onBegin') + beginEvents.push(event); + else if (event.method === 'onEnd') + endEvents.push(event); + else + events.push(event); + } + + } + return [mergeBeginEvents(beginEvents), ...events, mergeEndEvents(endEvents)]; +} + +function mergeBeginEvents(beginEvents: JsonEvent[]): JsonEvent { + if (!beginEvents.length) + throw new Error('No begin events found'); + const projects: JsonProject[] = []; + let totalWorkers = 0; + for (const event of beginEvents) { + totalWorkers += event.params.config.workers; + const shardProjects: JsonProject[] = event.params.projects; + for (const shardProject of shardProjects) { + const mergedProject = projects.find(p => p.id === shardProject.id); + if (!mergedProject) + projects.push(shardProject); + else + mergeJsonSuites(shardProject.suites, mergedProject); + } + } + const config = { + ...beginEvents[0].params.config, + workers: totalWorkers, + shard: undefined + }; + return { + method: 'onBegin', + params: { + config, + projects, + } + }; +} + +function mergeJsonSuites(jsonSuites: JsonSuite[], parent: JsonSuite | JsonProject) { + for (const jsonSuite of jsonSuites) { + const existingSuite = parent.suites.find(s => s.title === jsonSuite.title); + if (!existingSuite) { + parent.suites.push(jsonSuite); + } else { + mergeJsonSuites(jsonSuite.suites, existingSuite); + existingSuite.tests.push(...jsonSuite.tests); + } + } +} + +function mergeEndEvents(endEvents: JsonEvent[]): JsonEvent { + const result: FullResult = { status: 'passed' }; + for (const event of endEvents) { + const shardResult: FullResult = event.params.result; + if (shardResult.status === 'failed') + result.status = 'failed'; + else if (shardResult.status === 'timedout' && result.status !== 'failed') + result.status = 'timedout'; + else if (shardResult.status === 'interrupted' && result.status !== 'failed' && result.status !== 'timedout') + result.status = 'interrupted'; + } + return { + method: 'onEnd', + params: { + result + } + }; +} + +async function sortedShardFiles(dir: string) { + const files = await fs.promises.readdir(dir); + return files.filter(file => file.endsWith('.zip')).sort(); +} + +async function zipReport(zipFileName: string, lines: string[]) { + const zipFile = new yazl.ZipFile(); + const result = new ManualPromise(); + (zipFile as any as EventEmitter).on('error', error => result.reject(error)); + // TODO: feed events on the fly. + const content = Readable.from(lines); + zipFile.addReadStream(content, 'report.jsonl'); + zipFile.end(); + zipFile.outputStream.pipe(fs.createWriteStream(zipFileName)).on('close', () => { + result.resolve(undefined); + }); + await result; +} diff --git a/packages/playwright-test/src/reporters/html.ts b/packages/playwright-test/src/reporters/html.ts index b31a4c1962..a4210dc45b 100644 --- a/packages/playwright-test/src/reporters/html.ts +++ b/packages/playwright-test/src/reporters/html.ts @@ -138,7 +138,7 @@ function reportFolderFromEnv(): string | undefined { return undefined; } -function defaultReportFolder(searchForPackageJson: string): string { +export function defaultReportFolder(searchForPackageJson: string): string { let basePath = getPackageJsonPath(searchForPackageJson); if (basePath) basePath = path.dirname(basePath); diff --git a/packages/playwright-test/src/reporters/teleEmitter.ts b/packages/playwright-test/src/reporters/teleEmitter.ts index 19afc54ded..513b24a6ed 100644 --- a/packages/playwright-test/src/reporters/teleEmitter.ts +++ b/packages/playwright-test/src/reporters/teleEmitter.ts @@ -124,6 +124,7 @@ export class TeleReporterEmitter implements Reporter { rootDir: config.rootDir, configFile: this._relativePath(config.configFile), listOnly: FullConfigInternal.from(config).listOnly, + workers: config.workers, }; } diff --git a/packages/playwright-test/src/runner/reporters.ts b/packages/playwright-test/src/runner/reporters.ts index 108d1ef7b0..b33575090d 100644 --- a/packages/playwright-test/src/runner/reporters.ts +++ b/packages/playwright-test/src/runner/reporters.ts @@ -29,6 +29,7 @@ import { Multiplexer } from '../reporters/multiplexer'; import type { Suite } from '../common/test'; import type { BuiltInReporter, FullConfigInternal } from '../common/config'; import { loadReporter } from './loadUtils'; +import { BlobReporter } from '../reporters/blob'; export async function createReporter(config: FullConfigInternal, mode: 'list' | 'watch' | 'run' | 'ui', additionalReporters: Reporter[] = []): Promise { const defaultReporters: {[key in BuiltInReporter]: new(arg: any) => Reporter} = { @@ -40,6 +41,7 @@ export async function createReporter(config: FullConfigInternal, mode: 'list' | junit: JUnitReporter, null: EmptyReporter, html: mode === 'ui' ? LineReporter : HtmlReporter, + blob: BlobReporter, }; const reporters: Reporter[] = []; if (mode === 'watch') { diff --git a/packages/playwright-test/src/runner/tasks.ts b/packages/playwright-test/src/runner/tasks.ts index 40bbcafd6c..dd4d573421 100644 --- a/packages/playwright-test/src/runner/tasks.ts +++ b/packages/playwright-test/src/runner/tasks.ts @@ -177,7 +177,7 @@ function createLoadTask(mode: 'out-of-process' | 'in-process', shouldFilterOnly: function createPhasesTask(): Task { return async testRun => { - testRun.config.maxConcurrentTestGroups = 0; + let maxConcurrentTestGroups = 0; const processed = new Set(); const projectToSuite = new Map(testRun.rootSuite!.suites.map(suite => [suite._projectConfig!, suite])); @@ -206,9 +206,11 @@ function createPhasesTask(): Task { testGroupsInPhase += testGroups.length; } debug('pw:test:task')(`created phase #${testRun.phases.length} with ${phase.projects.map(p => p.project.project.name).sort()} projects, ${testGroupsInPhase} testGroups`); - testRun.config.maxConcurrentTestGroups = Math.max(testRun.config.maxConcurrentTestGroups, testGroupsInPhase); + maxConcurrentTestGroups = Math.max(maxConcurrentTestGroups, testGroupsInPhase); } } + + testRun.config.config.workers = Math.min(testRun.config.config.workers, maxConcurrentTestGroups); }; } diff --git a/tests/playwright-test/reporter-blob.spec.ts b/tests/playwright-test/reporter-blob.spec.ts new file mode 100644 index 0000000000..a0e2c35531 --- /dev/null +++ b/tests/playwright-test/reporter-blob.spec.ts @@ -0,0 +1,255 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as fs from 'fs'; +import type { HttpServer } from '../../packages/playwright-core/src/utils'; +import { startHtmlReportServer } from '../../packages/playwright-test/lib/reporters/html'; +import { type CliRunResult, type RunOptions, stripAnsi } from './playwright-test-fixtures'; +import { cleanEnv, cliEntrypoint, expect, test as baseTest } from './playwright-test-fixtures'; + +const DOES_NOT_SUPPORT_UTF8_IN_TERMINAL = process.platform === 'win32' && process.env.TERM_PROGRAM !== 'vscode' && !process.env.WT_SESSION; +const POSITIVE_STATUS_MARK = DOES_NOT_SUPPORT_UTF8_IN_TERMINAL ? 'ok' : '✓ '; +const NEGATIVE_STATUS_MARK = DOES_NOT_SUPPORT_UTF8_IN_TERMINAL ? 'x ' : '✘ '; + +const test = baseTest.extend<{ + showReport: (reportFolder?: string) => Promise, + mergeReports: (reportFolder: string, env?: NodeJS.ProcessEnv, options?: RunOptions) => Promise + }>({ + showReport: async ({ page }, use, testInfo) => { + let server: HttpServer | undefined; + await use(async (reportFolder?: string) => { + reportFolder ??= testInfo.outputPath('playwright-report'); + server = startHtmlReportServer(reportFolder) as HttpServer; + const location = await server.start(); + await page.goto(location); + }); + await server?.stop(); + }, + mergeReports: async ({ childProcess, page }, use, testInfo) => { + await use(async (reportFolder: string, env: NodeJS.ProcessEnv = {}, options: RunOptions = {}) => { + const command = ['node', cliEntrypoint, 'merge-reports', reportFolder]; + if (options.additionalArgs) + command.push(...options.additionalArgs); + + const testProcess = childProcess({ + command, + env: cleanEnv(env), + // cwd, + }); + const { exitCode } = await testProcess.exited; + return { exitCode, output: testProcess.output.toString() }; + }); + } + }); + +test('should merge into html', async ({ runInlineTest, mergeReports, showReport, page }) => { + const reportDir = test.info().outputPath('blob-report'); + const files = { + 'playwright.config.ts': ` + module.exports = { + retries: 1, + reporter: [['blob', { outputDir: '${reportDir.replace(/\\/g, '/')}' }]] + }; + `, + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('math 1', async ({}) => { + expect(1 + 1).toBe(2); + }); + test('failing 1', async ({}) => { + expect(1).toBe(2); + }); + test('flaky 1', async ({}) => { + expect(test.info().retry).toBe(1); + }); + test.skip('skipped 1', async ({}) => {}); + `, + 'b.test.js': ` + import { test, expect } from '@playwright/test'; + test('math 2', async ({}) => { + expect(1 + 1).toBe(2); + }); + test('failing 2', async ({}) => { + expect(1).toBe(2); + }); + test.skip('skipped 2', async ({}) => {}); + `, + 'c.test.js': ` + import { test, expect } from '@playwright/test'; + test('math 3', async ({}) => { + expect(1 + 1).toBe(2); + }); + test('flaky 2', async ({}) => { + expect(test.info().retry).toBe(1); + }); + test.skip('skipped 3', async ({}) => {}); + ` + }; + const totalShards = 3; + for (let i = 0; i < totalShards; i++) + await runInlineTest(files, { shard: `${i + 1}/${totalShards}` }); + const reportFiles = await fs.promises.readdir(reportDir); + reportFiles.sort(); + expect(reportFiles).toEqual(['report-1-of-3.zip', 'report-2-of-3.zip', 'report-3-of-3.zip']); + const { exitCode } = await mergeReports(reportDir, {}, { additionalArgs: ['--reporter', 'html'] }); + expect(exitCode).toBe(0); + + await showReport(reportDir); + + await expect(page.locator('.subnav-item:has-text("All") .counter')).toHaveText('10'); + await expect(page.locator('.subnav-item:has-text("Passed") .counter')).toHaveText('3'); + await expect(page.locator('.subnav-item:has-text("Failed") .counter')).toHaveText('2'); + await expect(page.locator('.subnav-item:has-text("Flaky") .counter')).toHaveText('2'); + await expect(page.locator('.subnav-item:has-text("Skipped") .counter')).toHaveText('3'); + + await expect(page.locator('.test-file-test .test-file-title')).toHaveText( + ['failing 1', 'flaky 1', 'math 1', 'skipped 1', 'failing 2', 'math 2', 'skipped 2', 'flaky 2', 'math 3', 'skipped 3']); +}); + +test('be able to merge incomplete shards', async ({ runInlineTest, mergeReports, showReport, page }) => { + const reportDir = test.info().outputPath('blob-report'); + const files = { + 'playwright.config.ts': ` + module.exports = { + retries: 1, + reporter: [['blob', { outputDir: '${reportDir.replace(/\\/g, '/')}' }]] + }; + `, + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('math 1', async ({}) => { + }); + test('failing 1', async ({}) => { + expect(1).toBe(2); + }); + test.skip('skipped 1', async ({}) => {}); + `, + 'b.test.js': ` + import { test, expect } from '@playwright/test'; + test('math 2', async ({}) => { }); + test('failing 2', async ({}) => { + expect(1).toBe(2); + }); + test.skip('skipped 2', async ({}) => {}); + `, + 'c.test.js': ` + import { test, expect } from '@playwright/test'; + test('math 3', async ({}) => { + expect(1 + 1).toBe(2); + }); + test('flaky 2', async ({}) => { + expect(test.info().retry).toBe(1); + }); + test.skip('skipped 3', async ({}) => {}); + ` + }; + await runInlineTest(files, { shard: `1/3` }); + await runInlineTest(files, { shard: `3/3` }); + + const reportFiles = await fs.promises.readdir(reportDir); + reportFiles.sort(); + expect(reportFiles).toEqual(['report-1-of-3.zip', 'report-3-of-3.zip']); + const { exitCode } = await mergeReports(reportDir, {}, { additionalArgs: ['--reporter', 'html'] }); + expect(exitCode).toBe(0); + + await showReport(reportDir); + + await expect(page.locator('.subnav-item:has-text("All") .counter')).toHaveText('6'); + await expect(page.locator('.subnav-item:has-text("Passed") .counter')).toHaveText('2'); + await expect(page.locator('.subnav-item:has-text("Failed") .counter')).toHaveText('1'); + await expect(page.locator('.subnav-item:has-text("Flaky") .counter')).toHaveText('1'); + await expect(page.locator('.subnav-item:has-text("Skipped") .counter')).toHaveText('2'); +}); + +test('merge into list report by default', async ({ runInlineTest, mergeReports }) => { + const reportDir = test.info().outputPath('blob-report'); + const files = { + 'playwright.config.ts': ` + module.exports = { + retries: 1, + reporter: [['blob', { outputDir: '${reportDir.replace(/\\/g, '/')}' }]] + }; + `, + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('math 1', async ({}) => { + expect(1 + 1).toBe(2); + }); + test('failing 1', async ({}) => { + expect(1).toBe(2); + }); + test('flaky 1', async ({}) => { + expect(test.info().retry).toBe(1); + }); + test.skip('skipped 1', async ({}) => {}); + `, + 'b.test.js': ` + import { test, expect } from '@playwright/test'; + test('math 2', async ({}) => { + expect(1 + 1).toBe(2); + }); + test('failing 2', async ({}) => { + expect(1).toBe(2); + }); + test.skip('skipped 2', async ({}) => {}); + `, + 'c.test.js': ` + import { test, expect } from '@playwright/test'; + test('math 3', async ({}) => { + expect(1 + 1).toBe(2); + }); + test('flaky 2', async ({}) => { + expect(test.info().retry).toBe(1); + }); + test.skip('skipped 3', async ({}) => {}); + ` + }; + + const totalShards = 3; + for (let i = 0; i < totalShards; i++) + await runInlineTest(files, { shard: `${i + 1}/${totalShards}` }); + const reportFiles = await fs.promises.readdir(reportDir); + reportFiles.sort(); + expect(reportFiles).toEqual(['report-1-of-3.zip', 'report-2-of-3.zip', 'report-3-of-3.zip']); + const { exitCode, output } = await mergeReports(reportDir, { PW_TEST_DEBUG_REPORTERS: '1', PW_TEST_DEBUG_REPORTERS_PRINT_STEPS: '1', PWTEST_TTY_WIDTH: '80' }, { additionalArgs: ['--reporter', 'list'] }); + expect(exitCode).toBe(0); + + const text = stripAnsi(output); + expect(text).toContain('Running 10 tests using 3 workers'); + const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/\d+ms/, 'Xms')); + expect(lines).toEqual([ + `0 : 1 a.test.js:3:11 › math 1`, + `0 : ${POSITIVE_STATUS_MARK} 1 a.test.js:3:11 › math 1 (Xms)`, + `1 : 2 a.test.js:6:11 › failing 1`, + `1 : ${NEGATIVE_STATUS_MARK} 2 a.test.js:6:11 › failing 1 (Xms)`, + `2 : 3 a.test.js:6:11 › failing 1 (retry #1)`, + `2 : ${NEGATIVE_STATUS_MARK} 3 a.test.js:6:11 › failing 1 (retry #1) (Xms)`, + `3 : 4 a.test.js:9:11 › flaky 1`, + `3 : ${NEGATIVE_STATUS_MARK} 4 a.test.js:9:11 › flaky 1 (Xms)`, + `4 : 5 a.test.js:9:11 › flaky 1 (retry #1)`, + `4 : ${POSITIVE_STATUS_MARK} 5 a.test.js:9:11 › flaky 1 (retry #1) (Xms)`, + `5 : 6 a.test.js:12:12 › skipped 1`, + `5 : - 6 a.test.js:12:12 › skipped 1`, + `6 : 7 b.test.js:3:11 › math 2`, + `6 : ${POSITIVE_STATUS_MARK} 7 b.test.js:3:11 › math 2 (Xms)`, + `7 : 8 b.test.js:6:11 › failing 2`, + `7 : ${NEGATIVE_STATUS_MARK} 8 b.test.js:6:11 › failing 2 (Xms)`, + `8 : 9 b.test.js:6:11 › failing 2 (retry #1)`, + `8 : ${NEGATIVE_STATUS_MARK} 9 b.test.js:6:11 › failing 2 (retry #1) (Xms)`, + `9 : 10 b.test.js:9:12 › skipped 2`, + `9 : - 10 b.test.js:9:12 › skipped 2` + ]); +}); diff --git a/tests/playwright-test/reporter-list.spec.ts b/tests/playwright-test/reporter-list.spec.ts index 068765f3f6..549488c9e7 100644 --- a/tests/playwright-test/reporter-list.spec.ts +++ b/tests/playwright-test/reporter-list.spec.ts @@ -68,6 +68,7 @@ test('render steps', async ({ runInlineTest }) => { `, }, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PW_TEST_DEBUG_REPORTERS_PRINT_STEPS: '1', PWTEST_TTY_WIDTH: '80' }); const text = result.output; + console.log(result.output) const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/\d+ms/, 'Xms')); lines.pop(); // Remove last item that contains [v] and time in ms. expect(lines).toEqual([