feat: blob reporter (#22244)

#10437

packages/playwright-test/src/cli.ts

@@ -23,6 +23,7 @@ import { Runner } from './runner/runner';
 import { stopProfiling, startProfiling } from 'playwright-core/lib/utils';
 import { experimentalLoaderOption, fileIsModule } from './util';
 import { showHTMLReport } from './reporters/html';
+import { createMergedReport } from './reporters/blob';
 import { ConfigLoader, kDefaultConfigFiles, resolveConfigFile } from './common/configLoader';
 import type { ConfigCLIOverrides } from './common/ipc';
 import type { FullResult } from '../reporter';
@@ -34,6 +35,7 @@ export function addTestCommands(program: Command) {
   addTestCommand(program);
   addShowReportCommand(program);
   addListFilesCommand(program);
+  addMergeReportsCommand(program);
 }
 
 function addTestCommand(program: Command) {
@@ -90,6 +92,28 @@ Examples:
   $ npx playwright show-report playwright-report`);
 }
 
+function addMergeReportsCommand(program: Command) {
+  const command = program.command('merge-reports [dir]');
+  command.description('merge multiple blob reports (for sharded tests) into a single report');
+  command.action(async (dir, options) => {
+    try {
+      await mergeReports(dir, options);
+    } catch (e) {
+      console.error(e);
+      process.exit(1);
+    }
+  });
+  command.option('-c, --config <file>', `Configuration file. Can be used to specify additional configuration for the output report.`);
+  command.option('--reporter <reporter>', 'Output report type', 'list');
+  command.addHelpText('afterAll', `
+Arguments [dir]:
+  Directory containing blob reports.
+
+Examples:
+  $ npx playwright merge-reports playwright-report`);
+}
+
+
 async function runTests(args: string[], opts: { [key: string]: any }) {
   await startProfiling();
 
@@ -174,6 +198,24 @@ async function listTestFiles(opts: { [key: string]: any }) {
   });
 }
 
+async function mergeReports(reportDir: string | undefined, opts: { [key: string]: any }) {
+  let configFile = opts.config;
+  if (configFile) {
+    configFile = path.resolve(process.cwd(), configFile);
+    if (!fs.existsSync(configFile))
+      throw new Error(`${configFile} does not exist`);
+    if (!fs.statSync(configFile).isFile())
+      throw new Error(`${configFile} is not a file`);
+  }
+  if (restartWithExperimentalTsEsm(configFile))
+    return;
+
+  const configLoader = new ConfigLoader();
+  const config = await (configFile ? configLoader.loadConfigFile(configFile) : configLoader.loadEmptyConfig(process.cwd()));
+  const dir = path.resolve(process.cwd(), reportDir || 'playwright-report');
+  await createMergedReport(config, dir, opts.reporter || 'list');
+}
+
 function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrides {
   const shardPair = options.shard ? options.shard.split('/').map((t: string) => parseInt(t, 10)) : undefined;
   return {

packages/playwright-test/src/common/config.ts

@@ -38,7 +38,6 @@ export class FullConfigInternal {
   configDir = '';
   configCLIOverrides: ConfigCLIOverrides = {};
   storeDir = '';
-  maxConcurrentTestGroups = 0;
   ignoreSnapshots = false;
   webServers: Exclude<FullConfig['webServer'], null>[] = [];
   plugins: TestRunnerPluginRegistration[] = [];
@@ -246,7 +245,7 @@ export function toReporters(reporters: BuiltInReporter | ReporterDescription[] |
   return reporters;
 }
 
-export const builtInReporters = ['list', 'line', 'dot', 'json', 'junit', 'null', 'github', 'html'] as const;
+export const builtInReporters = ['list', 'line', 'dot', 'json', 'junit', 'null', 'github', 'html', 'blob'] as const;
 export type BuiltInReporter = typeof builtInReporters[number];
 
 export type ContextReuseMode = 'none' | 'force' | 'when-possible';

packages/playwright-test/src/isomorphic/teleReceiver.ts

@@ -28,6 +28,7 @@ export type JsonConfig = {
   rootDir: string;
   configFile: string | undefined;
   listOnly: boolean;
+  workers: number;
 };
 
 export type JsonPattern = {
@@ -283,6 +284,7 @@ export class TeleReporterReceiver {
     const fullConfig = baseFullConfig;
     fullConfig.rootDir = config.rootDir;
     fullConfig.configFile = config.configFile;
+    fullConfig.workers = config.workers;
     return fullConfig;
   }
 

packages/playwright-test/src/reporters/DEPS.list

@@ -3,3 +3,6 @@
 ../isomorphic/**
 ../util.ts
 ../utilsBundle.ts
+
+[blob.ts]
+../runner/loadUtils.ts

packages/playwright-test/src/reporters/base.ts

@@ -21,7 +21,6 @@ import type { FullConfig, TestCase, Suite, TestResult, TestError, FullResult, Te
 import type { SuitePrivate } from '../../types/reporterPrivate';
 import { codeFrameColumns } from '../common/babelBundle';
 import { monotonicTime } from 'playwright-core/lib/utils';
-import { FullConfigInternal } from '../common/config';
 
 export type TestResultOutput = { chunk: string | Buffer, type: 'stdout' | 'stderr' };
 export const kOutputSymbol = Symbol('output');
@@ -122,7 +121,7 @@ export class BaseReporter implements Reporter {
   }
 
   protected generateStartingMessage() {
-    const jobs = Math.min(this.config.workers, FullConfigInternal.from(this.config).maxConcurrentTestGroups);
+    const jobs = this.config.workers;
     const shardDetails = this.config.shard ? `, shard ${this.config.shard.current} of ${this.config.shard.total}` : '';
     if (!this.totalTestCount)
       return '';

packages/playwright-test/src/reporters/blob.ts

@@ -0,0 +1,228 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { EventEmitter } from 'events';
+import fs from 'fs';
+import path from 'path';
+import { ManualPromise, ZipFile } from 'playwright-core/lib/utils';
+import { yazl } from 'playwright-core/lib/zipBundle';
+import { Readable } from 'stream';
+import type { FullConfig, FullResult, Reporter } from '../../types/testReporter';
+import type { BuiltInReporter, FullConfigInternal } from '../common/config';
+import type { Suite } from '../common/test';
+import { TeleReporterReceiver, type JsonEvent, type JsonProject, type JsonSuite } from '../isomorphic/teleReceiver';
+import DotReporter from '../reporters/dot';
+import EmptyReporter from '../reporters/empty';
+import GitHubReporter from '../reporters/github';
+import JSONReporter from '../reporters/json';
+import JUnitReporter from '../reporters/junit';
+import LineReporter from '../reporters/line';
+import ListReporter from '../reporters/list';
+import { loadReporter } from '../runner/loadUtils';
+import HtmlReporter, { defaultReportFolder } from './html';
+import { TeleReporterEmitter } from './teleEmitter';
+
+
+type BlobReporterOptions = {
+  configDir: string;
+  outputDir?: string;
+};
+
+export class BlobReporter extends TeleReporterEmitter {
+  private _messages: any[] = [];
+  private _options: BlobReporterOptions;
+  private _outputFile!: string;
+
+  constructor(options: BlobReporterOptions) {
+    super(message => this._messages.push(message));
+    this._options = options;
+  }
+
+  override onBegin(config: FullConfig<{}, {}>, suite: Suite): void {
+    super.onBegin(config, suite);
+    this._computeOutputFileName(config);
+  }
+
+  override async onEnd(result: FullResult): Promise<void> {
+    await super.onEnd(result);
+    fs.mkdirSync(path.dirname(this._outputFile), { recursive: true });
+    const lines = this._messages.map(m => JSON.stringify(m) + '\n');
+    await zipReport(this._outputFile, lines);
+  }
+
+  private _computeOutputFileName(config: FullConfig) {
+    const outputDir = this._resolveOutputDir();
+    let shardSuffix = '';
+    if (config.shard) {
+      const paddedNumber = `${config.shard.current}`.padStart(`${config.shard.total}`.length, '0');
+      shardSuffix = `-${paddedNumber}-of-${config.shard.total}`;
+    }
+    this._outputFile = path.join(outputDir, `report${shardSuffix}.zip`);
+  }
+
+  private _resolveOutputDir(): string {
+    const { outputDir } = this._options;
+    if (outputDir)
+      return path.resolve(this._options.configDir, outputDir);
+    return defaultReportFolder(this._options.configDir);
+  }
+}
+
+export async function createMergedReport(config: FullConfigInternal, dir: string, reporterName?: string) {
+  const shardFiles = await sortedShardFiles(dir);
+  const events = await mergeEvents(dir, shardFiles);
+
+  const defaultReporters: {[key in BuiltInReporter]: new(arg: any) => Reporter} = {
+    dot: DotReporter,
+    line: LineReporter,
+    list: ListReporter,
+    github: GitHubReporter,
+    json: JSONReporter,
+    junit: JUnitReporter,
+    null: EmptyReporter,
+    html: HtmlReporter,
+    blob: BlobReporter,
+  };
+  reporterName ??= 'list';
+
+  const arg = config.config.reporter.find(([reporter, arg]) => reporter === reporterName)?.[1];
+  const options = {
+    ...arg,
+    configDir: process.cwd(),
+    outputFolder: dir
+  };
+
+  let reporter: Reporter | undefined;
+  if (reporterName in defaultReporters) {
+    reporter = new defaultReporters[reporterName as keyof typeof defaultReporters](options);
+  } else {
+    const reporterConstructor = await loadReporter(config, reporterName);
+    reporter = new reporterConstructor(options);
+  }
+
+  const receiver = new TeleReporterReceiver(path.sep, reporter);
+  for (const event of events)
+    await receiver.dispatch(event);
+  console.log(`Done.`);
+}
+
+async function mergeEvents(dir: string, shardFiles: string[]) {
+  const events: JsonEvent[] = [];
+  const beginEvents: JsonEvent[] = [];
+  const endEvents: JsonEvent[] = [];
+  for (const file of shardFiles) {
+    const zipFile = new ZipFile(path.join(dir, file));
+    const entryNames = await zipFile.entries();
+    const reportEntryName = entryNames.find(e => e.endsWith('.jsonl'));
+    if (!reportEntryName)
+      throw new Error(`Zip file ${file} does not contain a .jsonl file`);
+    const reportJson = await zipFile.read(reportEntryName);
+    const parsedEvents = reportJson.toString().split('\n').filter(line => line.length).map(line => JSON.parse(line)) as JsonEvent[];
+    for (const event of parsedEvents) {
+      // TODO: show remaining events?
+      if (event.method === 'onError')
+        throw new Error('Error in shard: ' + file);
+      if (event.method === 'onBegin')
+        beginEvents.push(event);
+      else if (event.method === 'onEnd')
+        endEvents.push(event);
+      else
+        events.push(event);
+    }
+
+  }
+  return [mergeBeginEvents(beginEvents), ...events, mergeEndEvents(endEvents)];
+}
+
+function mergeBeginEvents(beginEvents: JsonEvent[]): JsonEvent {
+  if (!beginEvents.length)
+    throw new Error('No begin events found');
+  const projects: JsonProject[] = [];
+  let totalWorkers = 0;
+  for (const event of beginEvents) {
+    totalWorkers += event.params.config.workers;
+    const shardProjects: JsonProject[] = event.params.projects;
+    for (const shardProject of shardProjects) {
+      const mergedProject = projects.find(p => p.id === shardProject.id);
+      if (!mergedProject)
+        projects.push(shardProject);
+      else
+        mergeJsonSuites(shardProject.suites, mergedProject);
+    }
+  }
+  const config = {
+    ...beginEvents[0].params.config,
+    workers: totalWorkers,
+    shard: undefined
+  };
+  return {
+    method: 'onBegin',
+    params: {
+      config,
+      projects,
+    }
+  };
+}
+
+function mergeJsonSuites(jsonSuites: JsonSuite[], parent: JsonSuite | JsonProject) {
+  for (const jsonSuite of jsonSuites) {
+    const existingSuite = parent.suites.find(s => s.title === jsonSuite.title);
+    if (!existingSuite) {
+      parent.suites.push(jsonSuite);
+    } else {
+      mergeJsonSuites(jsonSuite.suites, existingSuite);
+      existingSuite.tests.push(...jsonSuite.tests);
+    }
+  }
+}
+
+function mergeEndEvents(endEvents: JsonEvent[]): JsonEvent {
+  const result: FullResult = { status: 'passed' };
+  for (const event of endEvents) {
+    const shardResult: FullResult = event.params.result;
+    if (shardResult.status === 'failed')
+      result.status = 'failed';
+    else if (shardResult.status === 'timedout' && result.status !== 'failed')
+      result.status = 'timedout';
+    else if (shardResult.status === 'interrupted' && result.status !== 'failed' && result.status !== 'timedout')
+      result.status = 'interrupted';
+  }
+  return {
+    method: 'onEnd',
+    params: {
+      result
+    }
+  };
+}
+
+async function sortedShardFiles(dir: string) {
+  const files = await fs.promises.readdir(dir);
+  return files.filter(file => file.endsWith('.zip')).sort();
+}
+
+async function zipReport(zipFileName: string, lines: string[]) {
+  const zipFile = new yazl.ZipFile();
+  const result = new ManualPromise<undefined>();
+  (zipFile as any as EventEmitter).on('error', error => result.reject(error));
+  // TODO: feed events on the fly.
+  const content = Readable.from(lines);
+  zipFile.addReadStream(content, 'report.jsonl');
+  zipFile.end();
+  zipFile.outputStream.pipe(fs.createWriteStream(zipFileName)).on('close', () => {
+    result.resolve(undefined);
+  });
+  await result;
+}

packages/playwright-test/src/reporters/html.ts

@@ -138,7 +138,7 @@ function reportFolderFromEnv(): string | undefined {
   return undefined;
 }
 
-function defaultReportFolder(searchForPackageJson: string): string {
+export function defaultReportFolder(searchForPackageJson: string): string {
   let basePath = getPackageJsonPath(searchForPackageJson);
   if (basePath)
     basePath = path.dirname(basePath);

packages/playwright-test/src/reporters/teleEmitter.ts

@@ -124,6 +124,7 @@ export class TeleReporterEmitter implements Reporter {
       rootDir: config.rootDir,
       configFile: this._relativePath(config.configFile),
       listOnly: FullConfigInternal.from(config).listOnly,
+      workers: config.workers,
     };
   }
 

packages/playwright-test/src/runner/reporters.ts

@@ -29,6 +29,7 @@ import { Multiplexer } from '../reporters/multiplexer';
 import type { Suite } from '../common/test';
 import type { BuiltInReporter, FullConfigInternal } from '../common/config';
 import { loadReporter } from './loadUtils';
+import { BlobReporter } from '../reporters/blob';
 
 export async function createReporter(config: FullConfigInternal, mode: 'list' | 'watch' | 'run' | 'ui', additionalReporters: Reporter[] = []): Promise<Multiplexer> {
   const defaultReporters: {[key in BuiltInReporter]: new(arg: any) => Reporter} = {
@@ -40,6 +41,7 @@ export async function createReporter(config: FullConfigInternal, mode: 'list' |
     junit: JUnitReporter,
     null: EmptyReporter,
     html: mode === 'ui' ? LineReporter : HtmlReporter,
+    blob: BlobReporter,
   };
   const reporters: Reporter[] = [];
   if (mode === 'watch') {

packages/playwright-test/src/runner/tasks.ts

@@ -177,7 +177,7 @@ function createLoadTask(mode: 'out-of-process' | 'in-process', shouldFilterOnly:
 
 function createPhasesTask(): Task<TestRun> {
   return async testRun => {
-    testRun.config.maxConcurrentTestGroups = 0;
+    let maxConcurrentTestGroups = 0;
 
     const processed = new Set<FullProjectInternal>();
     const projectToSuite = new Map(testRun.rootSuite!.suites.map(suite => [suite._projectConfig!, suite]));
@@ -206,9 +206,11 @@ function createPhasesTask(): Task<TestRun> {
           testGroupsInPhase += testGroups.length;
         }
         debug('pw:test:task')(`created phase #${testRun.phases.length} with ${phase.projects.map(p => p.project.project.name).sort()} projects, ${testGroupsInPhase} testGroups`);
-        testRun.config.maxConcurrentTestGroups = Math.max(testRun.config.maxConcurrentTestGroups, testGroupsInPhase);
+        maxConcurrentTestGroups = Math.max(maxConcurrentTestGroups, testGroupsInPhase);
       }
     }
+
+    testRun.config.config.workers = Math.min(testRun.config.config.workers, maxConcurrentTestGroups);
   };
 }
 

tests/playwright-test/reporter-blob.spec.ts

@@ -0,0 +1,255 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as fs from 'fs';
+import type { HttpServer } from '../../packages/playwright-core/src/utils';
+import { startHtmlReportServer } from '../../packages/playwright-test/lib/reporters/html';
+import { type CliRunResult, type RunOptions, stripAnsi } from './playwright-test-fixtures';
+import { cleanEnv, cliEntrypoint, expect, test as baseTest } from './playwright-test-fixtures';
+
+const DOES_NOT_SUPPORT_UTF8_IN_TERMINAL = process.platform === 'win32' && process.env.TERM_PROGRAM !== 'vscode' && !process.env.WT_SESSION;
+const POSITIVE_STATUS_MARK = DOES_NOT_SUPPORT_UTF8_IN_TERMINAL ? 'ok' : '✓ ';
+const NEGATIVE_STATUS_MARK = DOES_NOT_SUPPORT_UTF8_IN_TERMINAL ? 'x ' : '✘ ';
+
+const test = baseTest.extend<{
+  showReport: (reportFolder?: string) => Promise<void>,
+  mergeReports: (reportFolder: string, env?: NodeJS.ProcessEnv, options?: RunOptions) => Promise<CliRunResult>
+      }>({
+        showReport: async ({ page }, use, testInfo) => {
+          let server: HttpServer | undefined;
+          await use(async (reportFolder?: string) => {
+            reportFolder ??=  testInfo.outputPath('playwright-report');
+            server = startHtmlReportServer(reportFolder) as HttpServer;
+            const location = await server.start();
+            await page.goto(location);
+          });
+          await server?.stop();
+        },
+        mergeReports: async ({ childProcess, page }, use, testInfo) => {
+          await use(async (reportFolder: string, env: NodeJS.ProcessEnv = {}, options: RunOptions = {}) => {
+            const command = ['node', cliEntrypoint, 'merge-reports', reportFolder];
+            if (options.additionalArgs)
+              command.push(...options.additionalArgs);
+
+            const testProcess = childProcess({
+              command,
+              env: cleanEnv(env),
+              // cwd,
+            });
+            const { exitCode } = await testProcess.exited;
+            return { exitCode, output: testProcess.output.toString() };
+          });
+        }
+      });
+
+test('should merge into html', async ({ runInlineTest, mergeReports, showReport, page }) => {
+  const reportDir = test.info().outputPath('blob-report');
+  const files = {
+    'playwright.config.ts': `
+      module.exports = {
+        retries: 1,
+        reporter: [['blob', { outputDir: '${reportDir.replace(/\\/g, '/')}' }]]
+      };
+    `,
+    'a.test.js': `
+      import { test, expect } from '@playwright/test';
+      test('math 1', async ({}) => {
+        expect(1 + 1).toBe(2);
+      });
+      test('failing 1', async ({}) => {
+        expect(1).toBe(2);
+      });
+      test('flaky 1', async ({}) => {
+        expect(test.info().retry).toBe(1);
+      });
+      test.skip('skipped 1', async ({}) => {});
+    `,
+    'b.test.js': `
+      import { test, expect } from '@playwright/test';
+      test('math 2', async ({}) => {
+        expect(1 + 1).toBe(2);
+      });
+      test('failing 2', async ({}) => {
+        expect(1).toBe(2);
+      });
+      test.skip('skipped 2', async ({}) => {});
+    `,
+    'c.test.js': `
+      import { test, expect } from '@playwright/test';
+      test('math 3', async ({}) => {
+        expect(1 + 1).toBe(2);
+      });
+      test('flaky 2', async ({}) => {
+        expect(test.info().retry).toBe(1);
+      });
+      test.skip('skipped 3', async ({}) => {});
+    `
+  };
+  const totalShards = 3;
+  for (let i = 0; i < totalShards; i++)
+    await runInlineTest(files, { shard: `${i + 1}/${totalShards}` });
+  const reportFiles = await fs.promises.readdir(reportDir);
+  reportFiles.sort();
+  expect(reportFiles).toEqual(['report-1-of-3.zip', 'report-2-of-3.zip', 'report-3-of-3.zip']);
+  const { exitCode } = await mergeReports(reportDir, {}, { additionalArgs: ['--reporter', 'html'] });
+  expect(exitCode).toBe(0);
+
+  await showReport(reportDir);
+
+  await expect(page.locator('.subnav-item:has-text("All") .counter')).toHaveText('10');
+  await expect(page.locator('.subnav-item:has-text("Passed") .counter')).toHaveText('3');
+  await expect(page.locator('.subnav-item:has-text("Failed") .counter')).toHaveText('2');
+  await expect(page.locator('.subnav-item:has-text("Flaky") .counter')).toHaveText('2');
+  await expect(page.locator('.subnav-item:has-text("Skipped") .counter')).toHaveText('3');
+
+  await expect(page.locator('.test-file-test .test-file-title')).toHaveText(
+      ['failing 1', 'flaky 1', 'math 1', 'skipped 1', 'failing 2', 'math 2', 'skipped 2', 'flaky 2', 'math 3', 'skipped 3']);
+});
+
+test('be able to merge incomplete shards', async ({ runInlineTest, mergeReports, showReport, page }) => {
+  const reportDir = test.info().outputPath('blob-report');
+  const files = {
+    'playwright.config.ts': `
+      module.exports = {
+        retries: 1,
+        reporter: [['blob', { outputDir: '${reportDir.replace(/\\/g, '/')}' }]]
+      };
+    `,
+    'a.test.js': `
+      import { test, expect } from '@playwright/test';
+      test('math 1', async ({}) => {
+      });
+      test('failing 1', async ({}) => {
+        expect(1).toBe(2);
+      });
+      test.skip('skipped 1', async ({}) => {});
+    `,
+    'b.test.js': `
+      import { test, expect } from '@playwright/test';
+      test('math 2', async ({}) => { });
+      test('failing 2', async ({}) => {
+        expect(1).toBe(2);
+      });
+      test.skip('skipped 2', async ({}) => {});
+    `,
+    'c.test.js': `
+      import { test, expect } from '@playwright/test';
+      test('math 3', async ({}) => {
+        expect(1 + 1).toBe(2);
+      });
+      test('flaky 2', async ({}) => {
+        expect(test.info().retry).toBe(1);
+      });
+      test.skip('skipped 3', async ({}) => {});
+    `
+  };
+  await runInlineTest(files, { shard: `1/3` });
+  await runInlineTest(files, { shard: `3/3` });
+
+  const reportFiles = await fs.promises.readdir(reportDir);
+  reportFiles.sort();
+  expect(reportFiles).toEqual(['report-1-of-3.zip', 'report-3-of-3.zip']);
+  const { exitCode } = await mergeReports(reportDir, {}, { additionalArgs: ['--reporter', 'html'] });
+  expect(exitCode).toBe(0);
+
+  await showReport(reportDir);
+
+  await expect(page.locator('.subnav-item:has-text("All") .counter')).toHaveText('6');
+  await expect(page.locator('.subnav-item:has-text("Passed") .counter')).toHaveText('2');
+  await expect(page.locator('.subnav-item:has-text("Failed") .counter')).toHaveText('1');
+  await expect(page.locator('.subnav-item:has-text("Flaky") .counter')).toHaveText('1');
+  await expect(page.locator('.subnav-item:has-text("Skipped") .counter')).toHaveText('2');
+});
+
+test('merge into list report by default', async ({ runInlineTest, mergeReports }) => {
+  const reportDir = test.info().outputPath('blob-report');
+  const files = {
+    'playwright.config.ts': `
+      module.exports = {
+        retries: 1,
+        reporter: [['blob', { outputDir: '${reportDir.replace(/\\/g, '/')}' }]]
+      };
+    `,
+    'a.test.js': `
+      import { test, expect } from '@playwright/test';
+      test('math 1', async ({}) => {
+        expect(1 + 1).toBe(2);
+      });
+      test('failing 1', async ({}) => {
+        expect(1).toBe(2);
+      });
+      test('flaky 1', async ({}) => {
+        expect(test.info().retry).toBe(1);
+      });
+      test.skip('skipped 1', async ({}) => {});
+    `,
+    'b.test.js': `
+      import { test, expect } from '@playwright/test';
+      test('math 2', async ({}) => {
+        expect(1 + 1).toBe(2);
+      });
+      test('failing 2', async ({}) => {
+        expect(1).toBe(2);
+      });
+      test.skip('skipped 2', async ({}) => {});
+    `,
+    'c.test.js': `
+      import { test, expect } from '@playwright/test';
+      test('math 3', async ({}) => {
+        expect(1 + 1).toBe(2);
+      });
+      test('flaky 2', async ({}) => {
+        expect(test.info().retry).toBe(1);
+      });
+      test.skip('skipped 3', async ({}) => {});
+    `
+  };
+
+  const totalShards = 3;
+  for (let i = 0; i < totalShards; i++)
+    await runInlineTest(files, { shard: `${i + 1}/${totalShards}` });
+  const reportFiles = await fs.promises.readdir(reportDir);
+  reportFiles.sort();
+  expect(reportFiles).toEqual(['report-1-of-3.zip', 'report-2-of-3.zip', 'report-3-of-3.zip']);
+  const { exitCode, output } = await mergeReports(reportDir, { PW_TEST_DEBUG_REPORTERS: '1', PW_TEST_DEBUG_REPORTERS_PRINT_STEPS: '1', PWTEST_TTY_WIDTH: '80' }, { additionalArgs: ['--reporter', 'list'] });
+  expect(exitCode).toBe(0);
+
+  const text = stripAnsi(output);
+  expect(text).toContain('Running 10 tests using 3 workers');
+  const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/\d+ms/, 'Xms'));
+  expect(lines).toEqual([
+    `0 :      1 a.test.js:3:11 › math 1`,
+    `0 :   ${POSITIVE_STATUS_MARK} 1 a.test.js:3:11 › math 1 (Xms)`,
+    `1 :      2 a.test.js:6:11 › failing 1`,
+    `1 :   ${NEGATIVE_STATUS_MARK} 2 a.test.js:6:11 › failing 1 (Xms)`,
+    `2 :      3 a.test.js:6:11 › failing 1 (retry #1)`,
+    `2 :   ${NEGATIVE_STATUS_MARK} 3 a.test.js:6:11 › failing 1 (retry #1) (Xms)`,
+    `3 :      4 a.test.js:9:11 › flaky 1`,
+    `3 :   ${NEGATIVE_STATUS_MARK} 4 a.test.js:9:11 › flaky 1 (Xms)`,
+    `4 :      5 a.test.js:9:11 › flaky 1 (retry #1)`,
+    `4 :   ${POSITIVE_STATUS_MARK} 5 a.test.js:9:11 › flaky 1 (retry #1) (Xms)`,
+    `5 :      6 a.test.js:12:12 › skipped 1`,
+    `5 :   -  6 a.test.js:12:12 › skipped 1`,
+    `6 :      7 b.test.js:3:11 › math 2`,
+    `6 :   ${POSITIVE_STATUS_MARK} 7 b.test.js:3:11 › math 2 (Xms)`,
+    `7 :      8 b.test.js:6:11 › failing 2`,
+    `7 :   ${NEGATIVE_STATUS_MARK} 8 b.test.js:6:11 › failing 2 (Xms)`,
+    `8 :      9 b.test.js:6:11 › failing 2 (retry #1)`,
+    `8 :   ${NEGATIVE_STATUS_MARK} 9 b.test.js:6:11 › failing 2 (retry #1) (Xms)`,
+    `9 :      10 b.test.js:9:12 › skipped 2`,
+    `9 :   -  10 b.test.js:9:12 › skipped 2`
+  ]);
+});

tests/playwright-test/reporter-list.spec.ts

@@ -68,6 +68,7 @@ test('render steps', async ({ runInlineTest }) => {
     `,
   }, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PW_TEST_DEBUG_REPORTERS_PRINT_STEPS: '1', PWTEST_TTY_WIDTH: '80' });
   const text = result.output;
+  console.log(result.output)
   const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/\d+ms/, 'Xms'));
   lines.pop(); // Remove last item that contains [v] and time in ms.
   expect(lines).toEqual([