diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts index 31819bdecc..83a26562d7 100644 --- a/packages/playwright/src/common/config.ts +++ b/packages/playwright/src/common/config.ts @@ -48,6 +48,7 @@ export class FullConfigInternal { readonly webServers: NonNullable[]; readonly plugins: TestRunnerPluginRegistration[]; readonly projects: FullProjectInternal[] = []; + commandHash: string = ''; cliArgs: string[] = []; cliGrep: string | undefined; cliGrepInvert: string | undefined; diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts index 018eb5660d..a3a2b333e0 100644 --- a/packages/playwright/src/program.ts +++ b/packages/playwright/src/program.ts @@ -20,7 +20,7 @@ import type { Command } from 'playwright-core/lib/utilsBundle'; import fs from 'fs'; import path from 'path'; import { Runner } from './runner/runner'; -import { stopProfiling, startProfiling, gracefullyProcessExitDoNotHang } from 'playwright-core/lib/utils'; +import { stopProfiling, startProfiling, gracefullyProcessExitDoNotHang, calculateSha1 } from 'playwright-core/lib/utils'; import { serializeError } from './util'; import { showHTMLReport } from './reporters/html'; import { createMergedReport } from './reporters/merge'; @@ -183,6 +183,7 @@ async function runTests(args: string[], opts: { [key: string]: any }) { if (!config) return; + config.commandHash = computeCommandHash(args, opts); config.cliArgs = args; config.cliGrep = opts.grep as string | undefined; config.cliGrepInvert = opts.grepInvert as string | undefined; @@ -201,6 +202,19 @@ async function runTests(args: string[], opts: { [key: string]: any }) { gracefullyProcessExitDoNotHang(exitCode); } +function computeCommandHash(args: string[], opts: { [key: string]: any }) { + const command = { + args, + opts: {} as any + }; + // Omit shard from the hash, shard number is added explicitely at the end of the blob report file name. + const keys = Object.keys(opts).filter(k => k !== 'shard'); + if (!keys.length && !args.length) + return ''; + keys.sort().forEach(key => command.opts[key] = opts[key]); + return calculateSha1(JSON.stringify(command)).substring(0, 7); +} + async function runTestServer(opts: { [key: string]: any }) { const host = opts.host || 'localhost'; const port = opts.port ? +opts.port : 0; diff --git a/packages/playwright/src/reporters/blob.ts b/packages/playwright/src/reporters/blob.ts index 1c95658f6c..864b338b7e 100644 --- a/packages/playwright/src/reporters/blob.ts +++ b/packages/playwright/src/reporters/blob.ts @@ -30,6 +30,7 @@ type BlobReporterOptions = { configDir: string; outputDir?: string; fileName?: string; + _commandHash: string; }; export const currentBlobReportVersion = 2; @@ -112,6 +113,8 @@ export class BlobReporter extends TeleReporterEmitter { if (this._options.fileName) return this._options.fileName; let reportName = 'report'; + if (this._options._commandHash) + reportName += '-' + this._options._commandHash; if (config.shard) { const paddedNumber = `${config.shard.current}`.padStart(`${config.shard.total}`.length, '0'); reportName = `${reportName}-${paddedNumber}`; diff --git a/packages/playwright/src/runner/reporters.ts b/packages/playwright/src/runner/reporters.ts index 084aed2912..3565437007 100644 --- a/packages/playwright/src/runner/reporters.ts +++ b/packages/playwright/src/runner/reporters.ts @@ -90,6 +90,7 @@ function reporterOptions(config: FullConfigInternal, mode: 'list' | 'test' | 'me configDir: config.configDir, _mode: mode, _isTestServer: isTestServer, + _commandHash: config.commandHash, }; } diff --git a/tests/playwright-test/reporter-blob.spec.ts b/tests/playwright-test/reporter-blob.spec.ts index 5e1cfef223..a434bb6f69 100644 --- a/tests/playwright-test/reporter-blob.spec.ts +++ b/tests/playwright-test/reporter-blob.spec.ts @@ -274,7 +274,7 @@ test('should merge blob into blob', async ({ runInlineTest, mergeReports, showRe { const reportFiles = await fs.promises.readdir(reportDir); reportFiles.sort(); - expect(reportFiles).toEqual(['report-1.zip', 'report-2.zip']); + expect(reportFiles).toEqual(['report-873189c-1.zip', 'report-873189c-2.zip']); const { exitCode } = await mergeReports(reportDir, undefined, { additionalArgs: ['--reporter', 'blob'] }); expect(exitCode).toBe(0); } @@ -440,7 +440,7 @@ test('merge into list report by default', async ({ runInlineTest, mergeReports } await runInlineTest(files, { shard: `${i + 1}/${totalShards}` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' }); const reportFiles = await fs.promises.readdir(reportDir); reportFiles.sort(); - expect(reportFiles).toEqual(['report-1.zip', 'report-2.zip', 'report-3.zip']); + expect(reportFiles).toEqual([/report-.*-1.zip/, /report-.*-2.zip/, /report-.*-3.zip/].map(r => expect.stringMatching(r))); const { exitCode, output } = await mergeReports(reportDir, { PW_TEST_DEBUG_REPORTERS: '1', PW_TEST_DEBUG_REPORTERS_PRINT_STEPS: '1', PWTEST_TTY_WIDTH: '80' }, { additionalArgs: ['--reporter', 'list'] }); expect(exitCode).toBe(0); @@ -519,7 +519,7 @@ test('should print progress', async ({ runInlineTest, mergeReports }) => { await runInlineTest(files, { shard: `2/2` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' }); const reportFiles = await fs.promises.readdir(reportDir); reportFiles.sort(); - expect(reportFiles).toEqual(['report-1.zip', 'report-2.zip']); + expect(reportFiles).toEqual([/report-.*-1.zip/, /report-.*-2.zip/].map(r => expect.stringMatching(r))); const { exitCode, output } = await mergeReports(reportDir, { PW_TEST_HTML_REPORT_OPEN: 'never' }, { additionalArgs: ['--reporter', 'html'] }); expect(exitCode).toBe(0); @@ -570,7 +570,7 @@ test('preserve attachments', async ({ runInlineTest, mergeReports, showReport, p const reportFiles = await fs.promises.readdir(reportDir); reportFiles.sort(); - expect(reportFiles).toEqual(['report-1.zip']); + expect(reportFiles).toEqual(['report-873189c-1.zip']); const { exitCode } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] }); expect(exitCode).toBe(0); @@ -633,7 +633,7 @@ test('generate html with attachment urls', async ({ runInlineTest, mergeReports, const reportFiles = await fs.promises.readdir(reportDir); reportFiles.sort(); - expect(reportFiles).toEqual(['report-1.zip']); + expect(reportFiles).toEqual(['report-873189c-1.zip']); const { exitCode } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] }); expect(exitCode).toBe(0); @@ -707,7 +707,7 @@ test('resource names should not clash between runs', async ({ runInlineTest, sho const reportFiles = await fs.promises.readdir(reportDir); reportFiles.sort(); - expect(reportFiles).toEqual(['report-1.zip', 'report-2.zip']); + expect(reportFiles).toEqual([/report-.*-1.zip/, /report-.*-2.zip/].map(r => expect.stringMatching(r))); const { exitCode } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] }); expect(exitCode).toBe(0); @@ -780,7 +780,7 @@ test('multiple output reports', async ({ runInlineTest, mergeReports, showReport const reportFiles = await fs.promises.readdir(reportDir); reportFiles.sort(); - expect(reportFiles).toEqual(['report-1.zip']); + expect(reportFiles).toEqual([expect.stringMatching(/report-.*-1.zip/)]); const { exitCode, output } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html,line'] }); expect(exitCode).toBe(0); @@ -841,7 +841,7 @@ test('multiple output reports based on config', async ({ runInlineTest, mergeRep const reportFiles = await fs.promises.readdir(reportDir); reportFiles.sort(); - expect(reportFiles).toEqual(['report-1.zip', 'report-2.zip']); + expect(reportFiles).toEqual([/report-.*-1.zip/, /report-.*-2.zip/].map(r => expect.stringMatching(r))); const { exitCode, output } = await mergeReports(reportDir, undefined, { additionalArgs: ['--config', test.info().outputPath('merged/playwright.config.ts')] }); expect(exitCode).toBe(0); @@ -988,7 +988,7 @@ test('preserve config fields', async ({ runInlineTest, mergeReports }) => { const reportFiles = await fs.promises.readdir(reportDir); reportFiles.sort(); - expect(reportFiles).toEqual(['report-1.zip', 'report-3.zip']); + expect(reportFiles).toEqual([/report-.*-1.zip/, /report-.*-3.zip/].map(r => expect.stringMatching(r))); const { exitCode } = await mergeReports(reportDir, {}, { additionalArgs: ['--reporter', test.info().outputPath('echo-reporter.js'), '-c', test.info().outputPath('merge.config.ts')] }); expect(exitCode).toBe(0); const json = JSON.parse(fs.readFileSync(test.info().outputPath('config.json')).toString()); @@ -1144,7 +1144,7 @@ test('preserve steps in html report', async ({ runInlineTest, mergeReports, show await runInlineTest(files); const reportFiles = await fs.promises.readdir(reportDir); reportFiles.sort(); - expect(reportFiles).toEqual(['report.zip']); + expect(reportFiles).toEqual(['report-873189c.zip']); // Run merger in a different directory to make sure relative paths will not be resolved // relative to the current directory. const mergeCwd = test.info().outputPath('foo'); @@ -1277,9 +1277,9 @@ test('blob report should include version', async ({ runInlineTest }) => { // CI/1 is a part of user agent string, make sure it matches in the nested test runner. await runInlineTest(files, undefined, { CI: process.env.CI }); const reportFiles = await fs.promises.readdir(reportDir); - expect(reportFiles).toEqual(['report.zip']); + expect(reportFiles).toEqual(['report-873189c.zip']); - const events = await extractReport(test.info().outputPath('blob-report', 'report.zip'), test.info().outputPath('tmp')); + const events = await extractReport(test.info().outputPath('blob-report', 'report-873189c.zip'), test.info().outputPath('tmp')); const metadataEvent = events.find(e => e.method === 'onBlobReportMetadata'); expect(metadataEvent.params.version).toBe(2); expect(metadataEvent.params.userAgent).toBe(getUserAgent()); @@ -1330,7 +1330,7 @@ test('merge-reports should throw if report version is from the future', async ({ await runInlineTest(files, { shard: `2/2` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' }); const reportFiles = await fs.promises.readdir(reportDir); - expect(reportFiles).toEqual(['report-1.zip', 'report-2.zip']); + expect(reportFiles).toEqual([/report-.*-1.zip/, /report-.*-2.zip/].map(r => expect.stringMatching(r))); // Extract report and modify version. const reportZipFile = test.info().outputPath('blob-report', reportFiles[1]); @@ -1346,7 +1346,7 @@ test('merge-reports should throw if report version is from the future', async ({ const { exitCode, output } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] }); expect(exitCode).toBe(1); - expect(output).toContain(`Error: Blob report report-2.zip was created with a newer version of Playwright.`); + expect(output).toContain(`Error: Blob report report-873189c-2.zip was created with a newer version of Playwright.`); }); @@ -1383,7 +1383,7 @@ test('should merge blob reports with same name', async ({ runInlineTest, mergeRe ` }; await runInlineTest(files); - const reportZip = test.info().outputPath('blob-report', 'report.zip'); + const reportZip = test.info().outputPath('blob-report', 'report-873189c.zip'); const allReportsDir = test.info().outputPath('all-blob-reports'); await fs.promises.cp(reportZip, path.join(allReportsDir, 'report-1.zip')); await fs.promises.cp(reportZip, path.join(allReportsDir, 'report-2.zip')); @@ -1485,8 +1485,8 @@ test('merge reports with different rootDirs', async ({ runInlineTest, mergeRepor await runInlineTest(files2, { workers: 1 }, undefined, { additionalArgs: ['--config', test.info().outputPath('dir2/playwright.config.ts')] }); const allReportsDir = test.info().outputPath('all-blob-reports'); - await fs.promises.cp(test.info().outputPath('dir1', 'blob-report', 'report.zip'), path.join(allReportsDir, 'report-1.zip')); - await fs.promises.cp(test.info().outputPath('dir2', 'blob-report', 'report.zip'), path.join(allReportsDir, 'report-2.zip')); + await fs.promises.cp(test.info().outputPath('dir1', 'blob-report', 'report-546b561.zip'), path.join(allReportsDir, 'report-1.zip')); + await fs.promises.cp(test.info().outputPath('dir2', 'blob-report', 'report-f033fe4.zip'), path.join(allReportsDir, 'report-2.zip')); { const { exitCode, output } = await mergeReports(allReportsDir); @@ -1601,7 +1601,7 @@ test('merge reports with different rootDirs and path separators', async ({ runIn await fs.promises.mkdir(allReportsDir, { recursive: true }); // Extract report and change path separators. - const reportZipFile = test.info().outputPath('dir1', 'blob-report', 'report.zip'); + const reportZipFile = test.info().outputPath('dir1', 'blob-report', 'report-e4e2def.zip'); const events = await extractReport(reportZipFile, test.info().outputPath('tmp')); events.forEach(patchPathSeparators); @@ -1610,7 +1610,7 @@ test('merge reports with different rootDirs and path separators', async ({ runIn await zipReport(events, report1); // Copy second report as is. - await fs.promises.cp(test.info().outputPath('dir2', 'blob-report', 'report.zip'), path.join(allReportsDir, 'report-2.zip')); + await fs.promises.cp(test.info().outputPath('dir2', 'blob-report', 'report-b754021.zip'), path.join(allReportsDir, 'report-2.zip')); { const { exitCode, output } = await mergeReports(allReportsDir, undefined, { additionalArgs: ['--config', 'merge.config.ts'] }); @@ -1655,7 +1655,7 @@ test('merge reports without --config preserves path separators', async ({ runInl await fs.promises.mkdir(allReportsDir, { recursive: true }); // Extract report and change path separators. - const reportZipFile = test.info().outputPath('dir1', 'blob-report', 'report.zip'); + const reportZipFile = test.info().outputPath('dir1', 'blob-report', 'report-6fabe8b.zip'); const events = await extractReport(reportZipFile, test.info().outputPath('tmp')); events.forEach(patchPathSeparators);