diff --git a/packages/playwright-test/src/reporters/blob.ts b/packages/playwright-test/src/reporters/blob.ts index 1c665ca818..212305f0dd 100644 --- a/packages/playwright-test/src/reporters/blob.ts +++ b/packages/playwright-test/src/reporters/blob.ts @@ -31,7 +31,7 @@ type BlobReporterOptions = { outputDir?: string; }; -const currentVersion = 1; +export const currentBlobReportVersion = 1; export type BlobReportMetadata = { version: number; @@ -54,7 +54,7 @@ export class BlobReporter extends TeleReporterEmitter { override onConfigure(config: FullConfig) { const metadata: BlobReportMetadata = { - version: currentVersion, + version: currentBlobReportVersion, projectSuffix: process.env.PWTEST_BLOB_SUFFIX, shard: config.shard ?? undefined, }; diff --git a/packages/playwright-test/src/reporters/merge.ts b/packages/playwright-test/src/reporters/merge.ts index b6167099c0..d3e34ce986 100644 --- a/packages/playwright-test/src/reporters/merge.ts +++ b/packages/playwright-test/src/reporters/merge.ts @@ -24,7 +24,7 @@ import { TeleReporterReceiver } from '../isomorphic/teleReceiver'; import { createReporters } from '../runner/reporters'; import { Multiplexer } from './multiplexer'; import { ZipFile } from 'playwright-core/lib/utils'; -import type { BlobReportMetadata } from './blob'; +import { currentBlobReportVersion, type BlobReportMetadata } from './blob'; import { relativeFilePath } from '../util'; type StatusCallback = (message: string) => void; @@ -102,7 +102,10 @@ async function extractAndParseReports(dir: string, shardFiles: string[], printSt function findMetadata(events: JsonEvent[], file: string): BlobReportMetadata { if (events[0]?.method !== 'onBlobReportMetadata') throw new Error(`No metadata event found in ${file}`); - return events[0].params; + const metadata = (events[0].params as BlobReportMetadata); + if (metadata.version > currentBlobReportVersion) + throw new Error(`Blob report ${file} was created with a newer version of Playwright.`); + return metadata; } async function mergeEvents(dir: string, shardReportFiles: string[], printStatus: StatusCallback) { diff --git a/tests/playwright-test/reporter-blob.spec.ts b/tests/playwright-test/reporter-blob.spec.ts index a6fe8b6064..724d6ceaa4 100644 --- a/tests/playwright-test/reporter-blob.spec.ts +++ b/tests/playwright-test/reporter-blob.spec.ts @@ -22,6 +22,8 @@ import type { HttpServer } from '../../packages/playwright-core/src/utils'; import { startHtmlReportServer } from '../../packages/playwright-test/lib/reporters/html'; import { expect as baseExpect, test as baseTest, stripAnsi } from './playwright-test-fixtures'; import extractZip from '../../packages/playwright-core/bundles/zip/node_modules/extract-zip'; +import * as yazl from '../../packages/playwright-core/bundles/zip/node_modules/yazl'; +import { Readable } from 'stream'; const DOES_NOT_SUPPORT_UTF8_IN_TERMINAL = process.platform === 'win32' && process.env.TERM_PROGRAM !== 'vscode' && !process.env.WT_SESSION; const POSITIVE_STATUS_MARK = DOES_NOT_SUPPORT_UTF8_IN_TERMINAL ? 'ok' : '✓ '; @@ -1225,7 +1227,7 @@ test('blob-report should be next to package.json', async ({ runInlineTest }, tes expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'tests', 'blob-report'))).toBe(false); }); -test('blob-report should include version', async ({ runInlineTest }) => { +test('blob report should include version', async ({ runInlineTest }) => { const reportDir = test.info().outputPath('blob-report'); const files = { 'playwright.config.ts': ` @@ -1249,3 +1251,57 @@ test('blob-report should include version', async ({ runInlineTest }) => { const metadataEvent = events.find(e => e.method === 'onBlobReportMetadata'); expect(metadataEvent.params.version).toBe(1); }); + +test('merge-reports should throw if report version is from the future', async ({ runInlineTest, mergeReports }) => { + const reportDir = test.info().outputPath('blob-report'); + const files = { + 'playwright.config.ts': ` + module.exports = { + reporter: [['blob']] + }; + `, + 'tests/a.test.js': ` + import { test, expect } from '@playwright/test'; + test('test 1', async ({}) => {}); + `, + 'tests/b.test.js': ` + import { test, expect } from '@playwright/test'; + test('test 1', async ({}) => {}); + `, + }; + await runInlineTest(files, { shard: `1/2` }); + await runInlineTest(files, { shard: `2/2` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' }); + + const reportFiles = await fs.promises.readdir(reportDir); + expect(reportFiles).toEqual(['report-1.zip', 'report-2.zip']); + + // Extract report and modify version. + const reportZipFile = test.info().outputPath('blob-report', reportFiles[1]); + await extractZip(reportZipFile, { dir: test.info().outputPath('tmp') }); + const reportFile = test.info().outputPath('tmp', reportFiles[1].replace(/\.zip$/, '.jsonl')); + const data = await fs.promises.readFile(reportFile, 'utf8'); + const events = data.split('\n').filter(Boolean).map(line => JSON.parse(line)); + const metadataEvent = events.find(e => e.method === 'onBlobReportMetadata'); + expect(metadataEvent.params.version).toBeTruthy(); + ++metadataEvent.params.version; + const modifiedLines = events.map(e => JSON.stringify(e) + '\n'); + + // Zip it back. + await fs.promises.rm(reportZipFile, { force: true }); + const zipFile = new yazl.ZipFile(); + const zipFinishPromise = new Promise((resolve, reject) => { + (zipFile as any).on('error', error => reject(error)); + zipFile.outputStream.pipe(fs.createWriteStream(reportZipFile)).on('close', () => { + resolve(undefined); + }).on('error', error => reject(error)); + }); + const content = Readable.from(modifiedLines); + zipFile.addReadStream(content, path.basename(reportFile)); + zipFile.end(); + await zipFinishPromise; + + const { exitCode, output } = await mergeReports(reportDir, {}, { additionalArgs: ['--reporter', 'html'] }); + expect(exitCode).toBe(1); + expect(output).toContain(`Error: Blob report report-2.zip was created with a newer version of Playwright.`); + +});