From b5a6ce226d551c3c4d3b4be484e767686b356e8e Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Wed, 24 Apr 2024 17:58:19 -0700 Subject: [PATCH] project name in report file --- packages/playwright/src/common/config.ts | 1 - packages/playwright/src/program.ts | 16 +---- packages/playwright/src/reporters/blob.ts | 4 +- packages/playwright/src/runner/reporters.ts | 20 +++++- tests/playwright-test/reporter-blob.spec.ts | 72 +++++++++++++++------ 5 files changed, 74 insertions(+), 39 deletions(-) diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts index 83a26562d7..31819bdecc 100644 --- a/packages/playwright/src/common/config.ts +++ b/packages/playwright/src/common/config.ts @@ -48,7 +48,6 @@ export class FullConfigInternal { readonly webServers: NonNullable[]; readonly plugins: TestRunnerPluginRegistration[]; readonly projects: FullProjectInternal[] = []; - commandHash: string = ''; cliArgs: string[] = []; cliGrep: string | undefined; cliGrepInvert: string | undefined; diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts index a3a2b333e0..018eb5660d 100644 --- a/packages/playwright/src/program.ts +++ b/packages/playwright/src/program.ts @@ -20,7 +20,7 @@ import type { Command } from 'playwright-core/lib/utilsBundle'; import fs from 'fs'; import path from 'path'; import { Runner } from './runner/runner'; -import { stopProfiling, startProfiling, gracefullyProcessExitDoNotHang, calculateSha1 } from 'playwright-core/lib/utils'; +import { stopProfiling, startProfiling, gracefullyProcessExitDoNotHang } from 'playwright-core/lib/utils'; import { serializeError } from './util'; import { showHTMLReport } from './reporters/html'; import { createMergedReport } from './reporters/merge'; @@ -183,7 +183,6 @@ async function runTests(args: string[], opts: { [key: string]: any }) { if (!config) return; - config.commandHash = computeCommandHash(args, opts); config.cliArgs = args; config.cliGrep = opts.grep as string | undefined; config.cliGrepInvert = opts.grepInvert as string | undefined; @@ -202,19 +201,6 @@ async function runTests(args: string[], opts: { [key: string]: any }) { gracefullyProcessExitDoNotHang(exitCode); } -function computeCommandHash(args: string[], opts: { [key: string]: any }) { - const command = { - args, - opts: {} as any - }; - // Omit shard from the hash, shard number is added explicitely at the end of the blob report file name. - const keys = Object.keys(opts).filter(k => k !== 'shard'); - if (!keys.length && !args.length) - return ''; - keys.sort().forEach(key => command.opts[key] = opts[key]); - return calculateSha1(JSON.stringify(command)).substring(0, 7); -} - async function runTestServer(opts: { [key: string]: any }) { const host = opts.host || 'localhost'; const port = opts.port ? +opts.port : 0; diff --git a/packages/playwright/src/reporters/blob.ts b/packages/playwright/src/reporters/blob.ts index 864b338b7e..9461a4e0f7 100644 --- a/packages/playwright/src/reporters/blob.ts +++ b/packages/playwright/src/reporters/blob.ts @@ -16,7 +16,7 @@ import fs from 'fs'; import path from 'path'; -import { ManualPromise, calculateSha1, createGuid, getUserAgent, removeFolders } from 'playwright-core/lib/utils'; +import { ManualPromise, calculateSha1, createGuid, getUserAgent, removeFolders, sanitizeForFilePath } from 'playwright-core/lib/utils'; import { mime } from 'playwright-core/lib/utilsBundle'; import { Readable } from 'stream'; import type { EventEmitter } from 'events'; @@ -114,7 +114,7 @@ export class BlobReporter extends TeleReporterEmitter { return this._options.fileName; let reportName = 'report'; if (this._options._commandHash) - reportName += '-' + this._options._commandHash; + reportName += '-' + sanitizeForFilePath(this._options._commandHash); if (config.shard) { const paddedNumber = `${config.shard.current}`.padStart(`${config.shard.total}`.length, '0'); reportName = `${reportName}-${paddedNumber}`; diff --git a/packages/playwright/src/runner/reporters.ts b/packages/playwright/src/runner/reporters.ts index 3565437007..066c2e779a 100644 --- a/packages/playwright/src/runner/reporters.ts +++ b/packages/playwright/src/runner/reporters.ts @@ -32,6 +32,7 @@ import { loadReporter } from './loadUtils'; import { BlobReporter } from '../reporters/blob'; import type { ReporterDescription } from '../../types/test'; import { type ReporterV2, wrapReporterAsV2 } from '../reporters/reporterV2'; +import { calculateSha1 } from 'playwright-core/lib/utils'; export async function createReporters(config: FullConfigInternal, mode: 'list' | 'test' | 'merge', isTestServer: boolean, descriptions?: ReporterDescription[]): Promise { const defaultReporters: { [key in BuiltInReporter]: new(arg: any) => ReporterV2 } = { @@ -90,10 +91,27 @@ function reporterOptions(config: FullConfigInternal, mode: 'list' | 'test' | 'me configDir: config.configDir, _mode: mode, _isTestServer: isTestServer, - _commandHash: config.commandHash, + _commandHash: computeCommandHash(config), }; } +function computeCommandHash(config: FullConfigInternal) { + let parts = []; + // Include project names for readability. + if (config.cliProjectFilter) + parts.push(...config.cliProjectFilter); + const command = {} as any; + if (config.cliArgs.length) + command.cliArgs = config.cliArgs; + if (config.cliGrep) + command.cliGrep = config.cliGrep; + if (config.cliGrepInvert) + command.cliGrepInvert = config.cliGrepInvert; + if (Object.keys(command).length) + parts.push(calculateSha1(JSON.stringify(command)).substring(0, 7)); + return parts.join('-'); +} + class ListModeReporter extends EmptyReporter { private config!: FullConfig; diff --git a/tests/playwright-test/reporter-blob.spec.ts b/tests/playwright-test/reporter-blob.spec.ts index a434bb6f69..13c6d44102 100644 --- a/tests/playwright-test/reporter-blob.spec.ts +++ b/tests/playwright-test/reporter-blob.spec.ts @@ -274,7 +274,7 @@ test('should merge blob into blob', async ({ runInlineTest, mergeReports, showRe { const reportFiles = await fs.promises.readdir(reportDir); reportFiles.sort(); - expect(reportFiles).toEqual(['report-873189c-1.zip', 'report-873189c-2.zip']); + expect(reportFiles).toEqual(['report-1.zip', 'report-2.zip']); const { exitCode } = await mergeReports(reportDir, undefined, { additionalArgs: ['--reporter', 'blob'] }); expect(exitCode).toBe(0); } @@ -440,7 +440,7 @@ test('merge into list report by default', async ({ runInlineTest, mergeReports } await runInlineTest(files, { shard: `${i + 1}/${totalShards}` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' }); const reportFiles = await fs.promises.readdir(reportDir); reportFiles.sort(); - expect(reportFiles).toEqual([/report-.*-1.zip/, /report-.*-2.zip/, /report-.*-3.zip/].map(r => expect.stringMatching(r))); + expect(reportFiles).toEqual(['report-1.zip', 'report-2.zip', 'report-3.zip']); const { exitCode, output } = await mergeReports(reportDir, { PW_TEST_DEBUG_REPORTERS: '1', PW_TEST_DEBUG_REPORTERS_PRINT_STEPS: '1', PWTEST_TTY_WIDTH: '80' }, { additionalArgs: ['--reporter', 'list'] }); expect(exitCode).toBe(0); @@ -519,7 +519,7 @@ test('should print progress', async ({ runInlineTest, mergeReports }) => { await runInlineTest(files, { shard: `2/2` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' }); const reportFiles = await fs.promises.readdir(reportDir); reportFiles.sort(); - expect(reportFiles).toEqual([/report-.*-1.zip/, /report-.*-2.zip/].map(r => expect.stringMatching(r))); + expect(reportFiles).toEqual(['report-1.zip', 'report-2.zip']); const { exitCode, output } = await mergeReports(reportDir, { PW_TEST_HTML_REPORT_OPEN: 'never' }, { additionalArgs: ['--reporter', 'html'] }); expect(exitCode).toBe(0); @@ -570,7 +570,7 @@ test('preserve attachments', async ({ runInlineTest, mergeReports, showReport, p const reportFiles = await fs.promises.readdir(reportDir); reportFiles.sort(); - expect(reportFiles).toEqual(['report-873189c-1.zip']); + expect(reportFiles).toEqual(['report-1.zip']); const { exitCode } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] }); expect(exitCode).toBe(0); @@ -633,7 +633,7 @@ test('generate html with attachment urls', async ({ runInlineTest, mergeReports, const reportFiles = await fs.promises.readdir(reportDir); reportFiles.sort(); - expect(reportFiles).toEqual(['report-873189c-1.zip']); + expect(reportFiles).toEqual(['report-1.zip']); const { exitCode } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] }); expect(exitCode).toBe(0); @@ -707,7 +707,7 @@ test('resource names should not clash between runs', async ({ runInlineTest, sho const reportFiles = await fs.promises.readdir(reportDir); reportFiles.sort(); - expect(reportFiles).toEqual([/report-.*-1.zip/, /report-.*-2.zip/].map(r => expect.stringMatching(r))); + expect(reportFiles).toEqual(['report-1.zip', 'report-2.zip']); const { exitCode } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] }); expect(exitCode).toBe(0); @@ -780,7 +780,7 @@ test('multiple output reports', async ({ runInlineTest, mergeReports, showReport const reportFiles = await fs.promises.readdir(reportDir); reportFiles.sort(); - expect(reportFiles).toEqual([expect.stringMatching(/report-.*-1.zip/)]); + expect(reportFiles).toEqual(['report-1.zip']); const { exitCode, output } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html,line'] }); expect(exitCode).toBe(0); @@ -841,7 +841,7 @@ test('multiple output reports based on config', async ({ runInlineTest, mergeRep const reportFiles = await fs.promises.readdir(reportDir); reportFiles.sort(); - expect(reportFiles).toEqual([/report-.*-1.zip/, /report-.*-2.zip/].map(r => expect.stringMatching(r))); + expect(reportFiles).toEqual(['report-1.zip', 'report-2.zip']); const { exitCode, output } = await mergeReports(reportDir, undefined, { additionalArgs: ['--config', test.info().outputPath('merged/playwright.config.ts')] }); expect(exitCode).toBe(0); @@ -988,7 +988,7 @@ test('preserve config fields', async ({ runInlineTest, mergeReports }) => { const reportFiles = await fs.promises.readdir(reportDir); reportFiles.sort(); - expect(reportFiles).toEqual([/report-.*-1.zip/, /report-.*-3.zip/].map(r => expect.stringMatching(r))); + expect(reportFiles).toEqual(['report-1.zip', 'report-3.zip']); const { exitCode } = await mergeReports(reportDir, {}, { additionalArgs: ['--reporter', test.info().outputPath('echo-reporter.js'), '-c', test.info().outputPath('merge.config.ts')] }); expect(exitCode).toBe(0); const json = JSON.parse(fs.readFileSync(test.info().outputPath('config.json')).toString()); @@ -1144,7 +1144,7 @@ test('preserve steps in html report', async ({ runInlineTest, mergeReports, show await runInlineTest(files); const reportFiles = await fs.promises.readdir(reportDir); reportFiles.sort(); - expect(reportFiles).toEqual(['report-873189c.zip']); + expect(reportFiles).toEqual(['report.zip']); // Run merger in a different directory to make sure relative paths will not be resolved // relative to the current directory. const mergeCwd = test.info().outputPath('foo'); @@ -1277,9 +1277,9 @@ test('blob report should include version', async ({ runInlineTest }) => { // CI/1 is a part of user agent string, make sure it matches in the nested test runner. await runInlineTest(files, undefined, { CI: process.env.CI }); const reportFiles = await fs.promises.readdir(reportDir); - expect(reportFiles).toEqual(['report-873189c.zip']); + expect(reportFiles).toEqual(['report.zip']); - const events = await extractReport(test.info().outputPath('blob-report', 'report-873189c.zip'), test.info().outputPath('tmp')); + const events = await extractReport(test.info().outputPath('blob-report', 'report.zip'), test.info().outputPath('tmp')); const metadataEvent = events.find(e => e.method === 'onBlobReportMetadata'); expect(metadataEvent.params.version).toBe(2); expect(metadataEvent.params.userAgent).toBe(getUserAgent()); @@ -1330,7 +1330,7 @@ test('merge-reports should throw if report version is from the future', async ({ await runInlineTest(files, { shard: `2/2` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' }); const reportFiles = await fs.promises.readdir(reportDir); - expect(reportFiles).toEqual([/report-.*-1.zip/, /report-.*-2.zip/].map(r => expect.stringMatching(r))); + expect(reportFiles).toEqual(['report-1.zip', 'report-2.zip']); // Extract report and modify version. const reportZipFile = test.info().outputPath('blob-report', reportFiles[1]); @@ -1346,7 +1346,7 @@ test('merge-reports should throw if report version is from the future', async ({ const { exitCode, output } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] }); expect(exitCode).toBe(1); - expect(output).toContain(`Error: Blob report report-873189c-2.zip was created with a newer version of Playwright.`); + expect(output).toContain(`Error: Blob report report-2.zip was created with a newer version of Playwright.`); }); @@ -1383,7 +1383,7 @@ test('should merge blob reports with same name', async ({ runInlineTest, mergeRe ` }; await runInlineTest(files); - const reportZip = test.info().outputPath('blob-report', 'report-873189c.zip'); + const reportZip = test.info().outputPath('blob-report', 'report.zip'); const allReportsDir = test.info().outputPath('all-blob-reports'); await fs.promises.cp(reportZip, path.join(allReportsDir, 'report-1.zip')); await fs.promises.cp(reportZip, path.join(allReportsDir, 'report-2.zip')); @@ -1485,8 +1485,8 @@ test('merge reports with different rootDirs', async ({ runInlineTest, mergeRepor await runInlineTest(files2, { workers: 1 }, undefined, { additionalArgs: ['--config', test.info().outputPath('dir2/playwright.config.ts')] }); const allReportsDir = test.info().outputPath('all-blob-reports'); - await fs.promises.cp(test.info().outputPath('dir1', 'blob-report', 'report-546b561.zip'), path.join(allReportsDir, 'report-1.zip')); - await fs.promises.cp(test.info().outputPath('dir2', 'blob-report', 'report-f033fe4.zip'), path.join(allReportsDir, 'report-2.zip')); + await fs.promises.cp(test.info().outputPath('dir1', 'blob-report', 'report.zip'), path.join(allReportsDir, 'report-1.zip')); + await fs.promises.cp(test.info().outputPath('dir2', 'blob-report', 'report.zip'), path.join(allReportsDir, 'report-2.zip')); { const { exitCode, output } = await mergeReports(allReportsDir); @@ -1601,7 +1601,7 @@ test('merge reports with different rootDirs and path separators', async ({ runIn await fs.promises.mkdir(allReportsDir, { recursive: true }); // Extract report and change path separators. - const reportZipFile = test.info().outputPath('dir1', 'blob-report', 'report-e4e2def.zip'); + const reportZipFile = test.info().outputPath('dir1', 'blob-report', 'report.zip'); const events = await extractReport(reportZipFile, test.info().outputPath('tmp')); events.forEach(patchPathSeparators); @@ -1610,7 +1610,7 @@ test('merge reports with different rootDirs and path separators', async ({ runIn await zipReport(events, report1); // Copy second report as is. - await fs.promises.cp(test.info().outputPath('dir2', 'blob-report', 'report-b754021.zip'), path.join(allReportsDir, 'report-2.zip')); + await fs.promises.cp(test.info().outputPath('dir2', 'blob-report', 'report.zip'), path.join(allReportsDir, 'report-2.zip')); { const { exitCode, output } = await mergeReports(allReportsDir, undefined, { additionalArgs: ['--config', 'merge.config.ts'] }); @@ -1655,7 +1655,7 @@ test('merge reports without --config preserves path separators', async ({ runInl await fs.promises.mkdir(allReportsDir, { recursive: true }); // Extract report and change path separators. - const reportZipFile = test.info().outputPath('dir1', 'blob-report', 'report-6fabe8b.zip'); + const reportZipFile = test.info().outputPath('dir1', 'blob-report', 'report.zip'); const events = await extractReport(reportZipFile, test.info().outputPath('tmp')); events.forEach(patchPathSeparators); @@ -1795,3 +1795,35 @@ test('preserve static annotations when tests did not run', async ({ runInlineTes await page.goBack(); } }); + +test('project filter in report name', async ({ runInlineTest }) => { + const files = { + 'playwright.config.ts': ` + module.exports = { + reporter: 'blob', + projects: [ + { name: 'foo' }, + { name: 'b%/\\ar' }, + { name: 'baz' }, + ] + }; + `, + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('math 1 @smoke', async ({}) => {}); + `, + }; + + const reportDir = test.info().outputPath('blob-report'); + + { + await runInlineTest(files, { shard: `2/2`, project: 'foo' }); + const reportFiles = await fs.promises.readdir(reportDir); + expect(reportFiles.sort()).toEqual(['report-foo-2.zip']); + } + { + await runInlineTest(files, { shard: `1/2`, project: 'foo,b*r', grep: 'smoke' }); + const reportFiles = await fs.promises.readdir(reportDir); + expect(reportFiles.sort()).toEqual(['report-foo-b-r-6d9d49e-1.zip']); + } +}); \ No newline at end of file