diff --git a/.github/workflows/tests_primary.yml b/.github/workflows/tests_primary.yml index 86365a6345..b117bb2267 100644 --- a/.github/workflows/tests_primary.yml +++ b/.github/workflows/tests_primary.yml @@ -55,7 +55,7 @@ jobs: - run: npx playwright install --with-deps ${{ matrix.browser }} chromium - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=${{ matrix.browser }} env: - PWTEST_BLOB_REPORT_NAME: "${{ matrix.browser }}-${{ matrix.os }}-node${{ matrix.node-version }}" + PWTEST_BOT_NAME: "${{ matrix.browser }}-${{ matrix.os }}-node${{ matrix.node-version }}" - run: node tests/config/checkCoverage.js ${{ matrix.browser }} - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() @@ -87,7 +87,7 @@ jobs: - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=chromium env: PWTEST_CHANNEL: chromium-tip-of-tree - PWTEST_BLOB_REPORT_NAME: "${{ matrix.os }}-chromium-tip-of-tree" + PWTEST_BOT_NAME: "${{ matrix.os }}-chromium-tip-of-tree" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash @@ -131,11 +131,11 @@ jobs: - run: npx playwright install --with-deps - run: npm run ttest -- --shard ${{ matrix.shard }} env: - PWTEST_BLOB_REPORT_NAME: "${{ matrix.os }}-node${{ matrix.node-version }}" + PWTEST_BOT_NAME: "${{ matrix.os }}-node${{ matrix.node-version }}" if: matrix.os != 'ubuntu-latest' - run: xvfb-run npm run ttest -- --shard ${{ matrix.shard }} env: - PWTEST_BLOB_REPORT_NAME: "${{ matrix.os }}-node${{ matrix.node-version }}" + PWTEST_BOT_NAME: "${{ matrix.os }}-node${{ matrix.node-version }}" if: matrix.os == 'ubuntu-latest' - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() @@ -162,7 +162,7 @@ jobs: - run: npx playwright install --with-deps - run: npm run test-html-reporter env: - PWTEST_BLOB_REPORT_NAME: "web-components-html-reporter" + PWTEST_BOT_NAME: "web-components-html-reporter" - name: Upload blob report if: always() uses: ./.github/actions/upload-blob-report @@ -172,7 +172,7 @@ jobs: - run: npm run test-web if: always() env: - PWTEST_BLOB_REPORT_NAME: "web-components-web" + PWTEST_BOT_NAME: "web-components-web" - name: Upload blob report if: always() uses: ./.github/actions/upload-blob-report @@ -206,7 +206,7 @@ jobs: - name: Run extension tests run: npm run test -- --workers=1 env: - PWTEST_BLOB_REPORT_NAME: "vscode-extension" + PWTEST_BOT_NAME: "vscode-extension" working-directory: ./playwright-vscode - name: Upload blob report if: always() @@ -240,11 +240,11 @@ jobs: - run: npm run itest if: matrix.os != 'ubuntu-latest' env: - PWTEST_BLOB_REPORT_NAME: "package-installations-${{ matrix.os }}" + PWTEST_BOT_NAME: "package-installations-${{ matrix.os }}" - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run itest if: matrix.os == 'ubuntu-latest' env: - PWTEST_BLOB_REPORT_NAME: "package-installations-${{ matrix.os }}" + PWTEST_BOT_NAME: "package-installations-${{ matrix.os }}" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash diff --git a/.github/workflows/tests_secondary.yml b/.github/workflows/tests_secondary.yml index 61429074c2..2a49d46a3b 100644 --- a/.github/workflows/tests_secondary.yml +++ b/.github/workflows/tests_secondary.yml @@ -42,7 +42,7 @@ jobs: - run: npx playwright install --with-deps ${{ matrix.browser }} chromium - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=${{ matrix.browser }} env: - PWTEST_BLOB_REPORT_NAME: "${{ matrix.browser }}-${{ matrix.os }}" + PWTEST_BOT_NAME: "${{ matrix.browser }}-${{ matrix.os }}" - run: node tests/config/checkCoverage.js ${{ matrix.browser }} - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() @@ -74,7 +74,7 @@ jobs: - run: npx playwright install --with-deps ${{ matrix.browser }} chromium - run: npm run test -- --project=${{ matrix.browser }} env: - PWTEST_BLOB_REPORT_NAME: "${{ matrix.browser }}-${{ matrix.os }}" + PWTEST_BOT_NAME: "${{ matrix.browser }}-${{ matrix.os }}" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash @@ -106,12 +106,12 @@ jobs: if: matrix.browser == 'firefox' shell: bash env: - PWTEST_BLOB_REPORT_NAME: "${{ matrix.browser }}-windows-latest" + PWTEST_BOT_NAME: "${{ matrix.browser }}-windows-latest" - run: npm run test -- --project=${{ matrix.browser }} if: matrix.browser != 'firefox' shell: bash env: - PWTEST_BLOB_REPORT_NAME: "${{ matrix.browser }}-windows-latest" + PWTEST_BOT_NAME: "${{ matrix.browser }}-windows-latest" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash @@ -175,11 +175,11 @@ jobs: - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=${{ matrix.browser }} --headed if: always() && startsWith(matrix.os, 'ubuntu-') env: - PWTEST_BLOB_REPORT_NAME: "${{ matrix.browser }}-headed-${{ matrix.os }}" + PWTEST_BOT_NAME: "${{ matrix.browser }}-headed-${{ matrix.os }}" - run: npm run test -- --project=${{ matrix.browser }} --headed if: always() && !startsWith(matrix.os, 'ubuntu-') env: - PWTEST_BLOB_REPORT_NAME: "${{ matrix.browser }}-headed-${{ matrix.os }}" + PWTEST_BOT_NAME: "${{ matrix.browser }}-headed-${{ matrix.os }}" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash @@ -210,7 +210,7 @@ jobs: - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run ctest env: PWTEST_MODE: ${{ matrix.mode }} - PWTEST_BLOB_REPORT_NAME: "${{ matrix.mode }}" + PWTEST_BOT_NAME: "${{ matrix.mode }}" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash @@ -247,7 +247,7 @@ jobs: env: PWTEST_TRACE: 1 PWTEST_CHANNEL: ${{ matrix.channel }} - PWTEST_BLOB_REPORT_NAME: "tracing-${{ matrix.channel || matrix.browser }}" + PWTEST_BOT_NAME: "tracing-${{ matrix.channel || matrix.browser }}" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash @@ -273,7 +273,7 @@ jobs: - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run ctest env: PWTEST_CHANNEL: chrome - PWTEST_BLOB_REPORT_NAME: "chrome-stable-linux" + PWTEST_BOT_NAME: "chrome-stable-linux" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash @@ -300,7 +300,7 @@ jobs: shell: bash env: PWTEST_CHANNEL: chrome - PWTEST_BLOB_REPORT_NAME: "chrome-stable-windows" + PWTEST_BOT_NAME: "chrome-stable-windows" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash @@ -326,7 +326,7 @@ jobs: - run: npm run ctest env: PWTEST_CHANNEL: chrome - PWTEST_BLOB_REPORT_NAME: "chrome-stable-mac" + PWTEST_BOT_NAME: "chrome-stable-mac" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash @@ -357,12 +357,12 @@ jobs: if: matrix.os == 'ubuntu-20.04' env: PWTEST_CHANNEL: chromium-tip-of-tree - PWTEST_BLOB_REPORT_NAME: "tip-of-tree-${{ matrix.os }}" + PWTEST_BOT_NAME: "tip-of-tree-${{ matrix.os }}" - run: npm run ctest if: matrix.os != 'ubuntu-20.04' env: PWTEST_CHANNEL: chromium-tip-of-tree - PWTEST_BLOB_REPORT_NAME: "tip-of-tree-${{ matrix.os }}" + PWTEST_BOT_NAME: "tip-of-tree-${{ matrix.os }}" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash @@ -393,12 +393,12 @@ jobs: if: matrix.os == 'ubuntu-latest' env: PWTEST_CHANNEL: chromium-tip-of-tree - PWTEST_BLOB_REPORT_NAME: "tip-of-tree-headed-${{ matrix.os }}" + PWTEST_BOT_NAME: "tip-of-tree-headed-${{ matrix.os }}" - run: npm run ctest -- --headed if: matrix.os != 'ubuntu-latest' env: PWTEST_CHANNEL: chromium-tip-of-tree - PWTEST_BLOB_REPORT_NAME: "tip-of-tree-headed-${{ matrix.os }}" + PWTEST_BOT_NAME: "tip-of-tree-headed-${{ matrix.os }}" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash @@ -424,7 +424,7 @@ jobs: - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run ftest env: PWTEST_CHANNEL: firefox-beta - PWTEST_BLOB_REPORT_NAME: "firefox-beta-linux" + PWTEST_BOT_NAME: "firefox-beta-linux" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash @@ -451,7 +451,7 @@ jobs: shell: bash env: PWTEST_CHANNEL: firefox-beta - PWTEST_BLOB_REPORT_NAME: "firefox-beta-windows" + PWTEST_BOT_NAME: "firefox-beta-windows" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash @@ -477,7 +477,7 @@ jobs: - run: npm run ftest env: PWTEST_CHANNEL: firefox-beta - PWTEST_BLOB_REPORT_NAME: "firefox-beta-mac" + PWTEST_BOT_NAME: "firefox-beta-mac" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash @@ -503,7 +503,7 @@ jobs: - run: npm run ctest env: PWTEST_CHANNEL: msedge - PWTEST_BLOB_REPORT_NAME: "edge-stable-mac" + PWTEST_BOT_NAME: "edge-stable-mac" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash @@ -530,7 +530,7 @@ jobs: shell: bash env: PWTEST_CHANNEL: msedge - PWTEST_BLOB_REPORT_NAME: "edge-stable-windows" + PWTEST_BOT_NAME: "edge-stable-windows" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash @@ -556,7 +556,7 @@ jobs: - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run ctest env: PWTEST_CHANNEL: msedge - PWTEST_BLOB_REPORT_NAME: "edge-stable-linux" + PWTEST_BOT_NAME: "edge-stable-linux" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash @@ -582,7 +582,7 @@ jobs: - run: npm run ctest env: PWTEST_CHANNEL: msedge-beta - PWTEST_BLOB_REPORT_NAME: "edge-beta-mac" + PWTEST_BOT_NAME: "edge-beta-mac" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash @@ -609,7 +609,7 @@ jobs: shell: bash env: PWTEST_CHANNEL: msedge-beta - PWTEST_BLOB_REPORT_NAME: "edge-beta-windows" + PWTEST_BOT_NAME: "edge-beta-windows" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash @@ -635,7 +635,7 @@ jobs: - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run ctest env: PWTEST_CHANNEL: msedge-beta - PWTEST_BLOB_REPORT_NAME: "edge-beta-linux" + PWTEST_BOT_NAME: "edge-beta-linux" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash @@ -661,7 +661,7 @@ jobs: - run: npm run ctest env: PWTEST_CHANNEL: msedge-dev - PWTEST_BLOB_REPORT_NAME: "edge-dev-mac" + PWTEST_BOT_NAME: "edge-dev-mac" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash @@ -688,7 +688,7 @@ jobs: shell: bash env: PWTEST_CHANNEL: msedge-dev - PWTEST_BLOB_REPORT_NAME: "edge-dev-windows" + PWTEST_BOT_NAME: "edge-dev-windows" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash @@ -714,7 +714,7 @@ jobs: - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run ctest env: PWTEST_CHANNEL: msedge-dev - PWTEST_BLOB_REPORT_NAME: "edge-dev-linux" + PWTEST_BOT_NAME: "edge-dev-linux" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash @@ -740,7 +740,7 @@ jobs: - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run ctest env: PWTEST_CHANNEL: chrome-beta - PWTEST_BLOB_REPORT_NAME: "chrome-beta-linux" + PWTEST_BOT_NAME: "chrome-beta-linux" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash @@ -767,7 +767,7 @@ jobs: shell: bash env: PWTEST_CHANNEL: chrome-beta - PWTEST_BLOB_REPORT_NAME: "chrome-beta-windows" + PWTEST_BOT_NAME: "chrome-beta-windows" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash @@ -793,7 +793,7 @@ jobs: - run: npm run ctest env: PWTEST_CHANNEL: chrome-beta - PWTEST_BLOB_REPORT_NAME: "chrome-beta-mac" + PWTEST_BOT_NAME: "chrome-beta-mac" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash @@ -833,7 +833,7 @@ jobs: - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=chromium env: PLAYWRIGHT_CHROMIUM_USE_HEADLESS_NEW: 1 - PWTEST_BLOB_REPORT_NAME: "headless-new" + PWTEST_BOT_NAME: "headless-new" - run: node tests/config/checkCoverage.js chromium - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() diff --git a/.github/workflows/tests_service.yml b/.github/workflows/tests_service.yml index 6450c4c3a1..d9b13ca13c 100644 --- a/.github/workflows/tests_service.yml +++ b/.github/workflows/tests_service.yml @@ -27,7 +27,7 @@ jobs: env: PWTEST_MODE: service2 PWTEST_TRACE: 1 - PWTEST_BLOB_REPORT_NAME: "${{ matrix.browser }}-${{ matrix.service-os }}-service" + PWTEST_BOT_NAME: "${{ matrix.browser }}-${{ matrix.service-os }}-service" PLAYWRIGHT_SERVICE_ACCESS_KEY: ${{ secrets.PLAYWRIGHT_SERVICE_ACCESS_KEY }} PLAYWRIGHT_SERVICE_URL: ${{ secrets.PLAYWRIGHT_SERVICE_URL }} PLAYWRIGHT_SERVICE_OS: ${{ matrix.service-os }} diff --git a/docs/src/test-reporters-js.md b/docs/src/test-reporters-js.md index f7cc0e3aee..ed4424f3e1 100644 --- a/docs/src/test-reporters-js.md +++ b/docs/src/test-reporters-js.md @@ -215,13 +215,13 @@ Blob reports contain all the details about the test run and can be used later to npx playwright test --reporter=blob ``` -By default, the report is written into the `blob-report` directory in the package.json directory or current working directory (if no package.json is found). The output directory can be overridden in the configuration file: +By default, the report is written into the `blob-report` directory in the package.json directory or current working directory (if no package.json is found). The report file name is `report.zip` or `report-.zip` when [sharding](./test-sharding.md) is used. Both output directory and report file name can be overridden in the configuration file: ```js title="playwright.config.ts" import { defineConfig } from '@playwright/test'; export default defineConfig({ - reporter: [['blob', { outputDir: 'my-report' }]], + reporter: [['blob', { outputDir: 'my-report', fileName: `report-${os.platform()}.zip` }]], }); ``` diff --git a/packages/playwright/src/reporters/blob.ts b/packages/playwright/src/reporters/blob.ts index 4cf401e698..f15fe6f87d 100644 --- a/packages/playwright/src/reporters/blob.ts +++ b/packages/playwright/src/reporters/blob.ts @@ -16,7 +16,7 @@ import fs from 'fs'; import path from 'path'; -import { ManualPromise, calculateSha1, createGuid, getUserAgent, removeFolders, sanitizeForFilePath } from 'playwright-core/lib/utils'; +import { ManualPromise, calculateSha1, createGuid, getUserAgent, removeFolders } from 'playwright-core/lib/utils'; import { mime } from 'playwright-core/lib/utilsBundle'; import { Readable } from 'stream'; import type { EventEmitter } from 'events'; @@ -29,6 +29,7 @@ import { resolveReporterOutputPath } from '../util'; type BlobReporterOptions = { configDir: string; outputDir?: string; + fileName?: string; }; export const currentBlobReportVersion = 1; @@ -51,6 +52,8 @@ export class BlobReporter extends TeleReporterEmitter { constructor(options: BlobReporterOptions) { super(message => this._messages.push(message), false); this._options = options; + if (this._options.fileName && !this._options.fileName.endsWith('.zip')) + throw new Error(`Blob report file name must end with .zip extension: ${this._options.fileName}`); this._salt = createGuid(); } @@ -58,7 +61,7 @@ export class BlobReporter extends TeleReporterEmitter { const metadata: BlobReportMetadata = { version: currentBlobReportVersion, userAgent: getUserAgent(), - name: config.botName || process.env.PWTEST_BLOB_REPORT_NAME, + name: config.botName, shard: config.shard ?? undefined, pathSeparator: path.sep, }; @@ -82,11 +85,11 @@ export class BlobReporter extends TeleReporterEmitter { const zipFile = new yazl.ZipFile(); const zipFinishPromise = new ManualPromise(); const finishPromise = zipFinishPromise.catch(e => { - throw new Error(`Failed to write report ${this._reportName + '.zip'}: ` + e.message); + throw new Error(`Failed to write report ${this._reportName}: ` + e.message); }); (zipFile as any as EventEmitter).on('error', error => zipFinishPromise.reject(error)); - const zipFileName = path.join(outputDir, this._reportName + '.zip'); + const zipFileName = path.join(outputDir, this._reportName); zipFile.outputStream.pipe(fs.createWriteStream(zipFileName)).on('close', () => { zipFinishPromise.resolve(undefined); }).on('error', error => zipFinishPromise.reject(error)); @@ -99,19 +102,17 @@ export class BlobReporter extends TeleReporterEmitter { const lines = this._messages.map(m => JSON.stringify(m) + '\n'); const content = Readable.from(lines); - zipFile.addReadStream(content, this._reportName + '.jsonl'); + zipFile.addReadStream(content, 'report.jsonl'); zipFile.end(); await finishPromise; } private _computeReportName(config: FullConfig) { - let reportName = 'report'; - if (process.env.PWTEST_BLOB_REPORT_NAME) - reportName += `-${sanitizeForFilePath(process.env.PWTEST_BLOB_REPORT_NAME)}`; + let reportName = this._options.fileName ?? 'report.zip'; if (config.shard) { const paddedNumber = `${config.shard.current}`.padStart(`${config.shard.total}`.length, '0'); - reportName += `-${paddedNumber}`; + reportName = `${reportName.slice(0, -4)}-${paddedNumber}.zip`; } return reportName; } diff --git a/packages/playwright/src/reporters/merge.ts b/packages/playwright/src/reporters/merge.ts index 442e55f26d..f206c592e8 100644 --- a/packages/playwright/src/reporters/merge.ts +++ b/packages/playwright/src/reporters/merge.ts @@ -321,7 +321,7 @@ function mergeEndEvents(endEvents: JsonEvent[]): JsonEvent { async function sortedShardFiles(dir: string) { const files = await fs.promises.readdir(dir); - return files.filter(file => file.startsWith('report') && file.endsWith('.zip')).sort(); + return files.filter(file => file.endsWith('.zip')).sort(); } function printStatusToStdout(message: string) { diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index eae28c116b..72d542f2b2 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -19,7 +19,7 @@ import type { APIRequestContext, Browser, BrowserContext, BrowserContextOptions, export * from 'playwright-core'; export type ReporterDescription = - ['blob'] | ['blob', { outputDir?: string }] | + ['blob'] | ['blob', { outputDir?: string, fileName?: string }] | ['dot'] | ['line'] | ['list'] | ['list', { printSteps?: boolean }] | diff --git a/tests/installation/playwright.config.ts b/tests/installation/playwright.config.ts index fc3dda9023..d2de1754fb 100644 --- a/tests/installation/playwright.config.ts +++ b/tests/installation/playwright.config.ts @@ -24,7 +24,7 @@ const reporters = () => { const result: ReporterDescription[] = process.env.CI ? [ ['dot'], ['json', { outputFile: path.join(outputDir, 'report.json') }], - ['blob'], + ['blob', { fileName: `${process.env.PWTEST_BOT_NAME}.zip` }], ] : [ ['list'], ['html', { open: 'on-failure' }] @@ -34,6 +34,7 @@ const reporters = () => { const outputDir = path.join(__dirname, '..', '..', 'test-results'); export default defineConfig({ + botName: process.env.PWTEST_BOT_NAME, globalSetup: path.join(__dirname, 'globalSetup'), outputDir, testIgnore: '**\/fixture-scripts/**', diff --git a/tests/library/playwright.config.ts b/tests/library/playwright.config.ts index fcaac7f726..c9530d156b 100644 --- a/tests/library/playwright.config.ts +++ b/tests/library/playwright.config.ts @@ -46,7 +46,7 @@ const reporters = () => { const result: ReporterDescription[] = process.env.CI ? [ ['dot'], ['json', { outputFile: path.join(outputDir, 'report.json') }], - ['blob'], + ['blob', { fileName: `${process.env.PWTEST_BOT_NAME}.zip` }], ] : [ ['html', { open: 'on-failure' }] ]; @@ -80,6 +80,7 @@ if (mode === 'service2') { } const config: Config = { + botName: process.env.PWTEST_BOT_NAME, testDir, outputDir, expect: { diff --git a/tests/playwright-test/playwright-test-fixtures.ts b/tests/playwright-test/playwright-test-fixtures.ts index b5bc38c631..76ca67537b 100644 --- a/tests/playwright-test/playwright-test-fixtures.ts +++ b/tests/playwright-test/playwright-test-fixtures.ts @@ -217,7 +217,6 @@ export function cleanEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { PW_TEST_REPORTER_WS_ENDPOINT: undefined, PW_TEST_SOURCE_TRANSFORM: undefined, PW_TEST_SOURCE_TRANSFORM_SCOPE: undefined, - PWTEST_BLOB_REPORT_NAME: undefined, TEST_WORKER_INDEX: undefined, TEST_PARALLEL_INDEX: undefined, NODE_OPTIONS: undefined, diff --git a/tests/playwright-test/reporter-blob.spec.ts b/tests/playwright-test/reporter-blob.spec.ts index c58bcd669d..71db0f2194 100644 --- a/tests/playwright-test/reporter-blob.spec.ts +++ b/tests/playwright-test/reporter-blob.spec.ts @@ -1173,23 +1173,11 @@ test('preserve steps in html report', async ({ runInlineTest, mergeReports, show await expect(page.getByText('expect.toBe')).toBeVisible(); }); -test('same project different suffixes', async ({ runInlineTest, mergeReports }) => { - const files = { - 'echo-reporter.js': ` - import fs from 'fs'; - - class EchoReporter { - onBegin(config, suite) { - const projects = suite.suites.map(s => s.project()).sort((a, b) => a.metadata.reportName.localeCompare(b.metadata.reportName)); - console.log('projectNames: ' + projects.map(p => p.name)); - console.log('reportNames: ' + projects.map(p => p.metadata.reportName)); - } - } - module.exports = EchoReporter; - `, +test('support fileName option', async ({ runInlineTest, mergeReports }) => { + const files = (fileSuffix: string) => ({ 'playwright.config.ts': ` module.exports = { - reporter: 'blob', + reporter: [['blob', { fileName: 'report-${fileSuffix}.zip' }]], projects: [ { name: 'foo' }, ] @@ -1199,16 +1187,14 @@ test('same project different suffixes', async ({ runInlineTest, mergeReports }) import { test, expect } from '@playwright/test'; test('math 1 @smoke', async ({}) => {}); `, - }; + }); - await runInlineTest(files, undefined, { PWTEST_BLOB_REPORT_NAME: 'first' }); - await runInlineTest(files, undefined, { PWTEST_BLOB_REPORT_NAME: 'second', PWTEST_BLOB_DO_NOT_REMOVE: '1' }); + await runInlineTest(files('one')); + await runInlineTest(files('two'), undefined, { PWTEST_BLOB_DO_NOT_REMOVE: '1' }); const reportDir = test.info().outputPath('blob-report'); - const { exitCode, output } = await mergeReports(reportDir, {}, { additionalArgs: ['--reporter', test.info().outputPath('echo-reporter.js')] }); - expect(exitCode).toBe(0); - expect(output).toContain(`projectNames: foo,foo`); - expect(output).toContain(`reportNames: first,second`); + const reportFiles = await fs.promises.readdir(reportDir); + expect(reportFiles.sort()).toEqual(['report-one.zip', 'report-two.zip']); }); test('preserve botName on projects', async ({ runInlineTest, mergeReports }) => { @@ -1218,7 +1204,7 @@ test('preserve botName on projects', async ({ runInlineTest, mergeReports }) => class EchoReporter { onBegin(config, suite) { - const projects = suite.suites.map(s => s.project()).sort((a, b) => a.metadata.reportName.localeCompare(b.metadata.reportName)); + const projects = suite.suites.map(s => s.project()).sort((a, b) => a.botName.localeCompare(b.botName)); console.log('projectNames: ' + projects.map(p => p.name)); console.log('botNames: ' + projects.map(p => p.botName)); } @@ -1227,7 +1213,7 @@ test('preserve botName on projects', async ({ runInlineTest, mergeReports }) => `, 'playwright.config.ts': ` module.exports = { - reporter: 'blob', + reporter: [['blob', { fileName: '${botName}.zip' }]], botName: '${botName}', projects: [ { name: 'foo' }, @@ -1240,8 +1226,8 @@ test('preserve botName on projects', async ({ runInlineTest, mergeReports }) => `, }); - await runInlineTest(files('first'), undefined, { PWTEST_BLOB_REPORT_NAME: 'first' }); - await runInlineTest(files('second'), undefined, { PWTEST_BLOB_REPORT_NAME: 'second', PWTEST_BLOB_DO_NOT_REMOVE: '1' }); + await runInlineTest(files('first')); + await runInlineTest(files('second'), undefined, { PWTEST_BLOB_DO_NOT_REMOVE: '1' }); const reportDir = test.info().outputPath('blob-report'); const { exitCode, output } = await mergeReports(reportDir, {}, { additionalArgs: ['--reporter', test.info().outputPath('echo-reporter.js')] }); @@ -1306,7 +1292,7 @@ test('blob report should include version', async ({ runInlineTest }) => { async function extractReport(reportZipFile: string, unzippedReportDir: string): Promise { await extractZip(reportZipFile, { dir: unzippedReportDir }); - const reportFile = path.join(unzippedReportDir, path.basename(reportZipFile).replace(/\.zip$/, '.jsonl')); + const reportFile = path.join(unzippedReportDir, 'report.jsonl'); const data = await fs.promises.readFile(reportFile, 'utf8'); const events = data.split('\n').filter(Boolean).map(line => JSON.parse(line)); return events; diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 7b5e8e8e12..382d4dacbe 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -18,7 +18,7 @@ import type { APIRequestContext, Browser, BrowserContext, BrowserContextOptions, export * from 'playwright-core'; export type ReporterDescription = - ['blob'] | ['blob', { outputDir?: string }] | + ['blob'] | ['blob', { outputDir?: string, fileName?: string }] | ['dot'] | ['line'] | ['list'] | ['list', { printSteps?: boolean }] |