From 3ccec7eae57f0e3d8a3909d12c835e22dcdce109 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Fri, 12 May 2023 09:26:04 -0700 Subject: [PATCH] feat(merge): generate html report with links to remote resources (#22968) --- packages/playwright-test/src/cli.ts | 10 +- .../playwright-test/src/reporters/html.ts | 7 -- .../playwright-test/src/reporters/merge.ts | 5 +- tests/playwright-test/reporter-blob.spec.ts | 94 +++++++++++++++++++ tests/playwright-test/reporter-html.spec.ts | 4 +- 5 files changed, 107 insertions(+), 13 deletions(-) diff --git a/packages/playwright-test/src/cli.ts b/packages/playwright-test/src/cli.ts index b68cbf27c1..bde82be103 100644 --- a/packages/playwright-test/src/cli.ts +++ b/packages/playwright-test/src/cli.ts @@ -88,6 +88,8 @@ Examples: $ npx playwright show-report playwright-report`); } +const kAttachmentModes: string[] = ['local', 'missing']; + function addMergeReportsCommand(program: Command) { const command = program.command('merge-reports [dir]', { hidden: true }); command.description('merge multiple blob reports (for sharded tests) into a single report'); @@ -101,6 +103,7 @@ function addMergeReportsCommand(program: Command) { }); command.option('-c, --config ', `Configuration file. Can be used to specify additional configuration for the output report.`); command.option('--reporter ', `Reporter to use, comma-separated, can be ${builtInReporters.map(name => `"${name}"`).join(', ')} (default: "${defaultReporter}")`); + command.option('--attachments ', `Whether the attachments are available locally. Supported values are ${kAttachmentModes.map(name => `"${name}"`).join(', ')} (default: "local")`); command.addHelpText('afterAll', ` Arguments [dir]: Directory containing blob reports. @@ -197,7 +200,12 @@ async function mergeReports(reportDir: string | undefined, opts: { [key: string] reporterDescriptions = config.config.reporter; if (!reporterDescriptions) reporterDescriptions = [[defaultReporter]]; - await createMergedReport(config, dir, reporterDescriptions!); + if (opts.attachments) { + if (!kAttachmentModes.includes(opts.attachments)) + throw new Error(`Invalid --attachments value "${opts.attachments}", must be one of ${kAttachmentModes.map(name => `"${name}"`).join(', ')}.`); + } + const resolveAttachmentPaths = opts.attachments !== 'missing'; + await createMergedReport(config, dir, reporterDescriptions!, resolveAttachmentPaths); } function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrides { diff --git a/packages/playwright-test/src/reporters/html.ts b/packages/playwright-test/src/reporters/html.ts index b514937764..f58b13ba35 100644 --- a/packages/playwright-test/src/reporters/html.ts +++ b/packages/playwright-test/src/reporters/html.ts @@ -37,8 +37,6 @@ type TestEntry = { testCaseSummary: TestCaseSummary }; -const kMissingContentType = 'x-playwright/missing'; - type HtmlReportOpenOption = 'always' | 'never' | 'on-failure'; type HtmlReporterOptions = { configDir: string, @@ -397,11 +395,6 @@ class HtmlBuilder { fs.mkdirSync(path.join(this._reportFolder, 'data'), { recursive: true }); fs.writeFileSync(path.join(this._reportFolder, 'data', sha1), buffer); } catch (e) { - return { - name: `Missing attachment "${a.name}"`, - contentType: kMissingContentType, - body: `Attachment file ${fileName} is missing`, - }; } return { name: a.name, diff --git a/packages/playwright-test/src/reporters/merge.ts b/packages/playwright-test/src/reporters/merge.ts index 56c1d256bf..f6072ff36f 100644 --- a/packages/playwright-test/src/reporters/merge.ts +++ b/packages/playwright-test/src/reporters/merge.ts @@ -23,10 +23,11 @@ import { TeleReporterReceiver, type JsonEvent, type JsonProject, type JsonSuite, import { createReporters } from '../runner/reporters'; import { Multiplexer } from './multiplexer'; -export async function createMergedReport(config: FullConfigInternal, dir: string, reporterDescriptions: ReporterDescription[]) { +export async function createMergedReport(config: FullConfigInternal, dir: string, reporterDescriptions: ReporterDescription[], resolvePaths: boolean) { const shardFiles = await sortedShardFiles(dir); const events = await mergeEvents(dir, shardFiles); - patchAttachmentPaths(events, dir); + if (resolvePaths) + patchAttachmentPaths(events, dir); const reporters = await createReporters(config, 'merge', reporterDescriptions); const receiver = new TeleReporterReceiver(path.sep, new Multiplexer(reporters)); diff --git a/tests/playwright-test/reporter-blob.spec.ts b/tests/playwright-test/reporter-blob.spec.ts index 2a473e52a1..1a58c1eca2 100644 --- a/tests/playwright-test/reporter-blob.spec.ts +++ b/tests/playwright-test/reporter-blob.spec.ts @@ -15,6 +15,8 @@ */ import * as fs from 'fs'; +import path from 'path'; +import url from 'url'; import type { HttpServer } from '../../packages/playwright-core/src/utils'; import { startHtmlReportServer } from '../../packages/playwright-test/lib/reporters/html'; import { type CliRunResult, type RunOptions, stripAnsi } from './playwright-test-fixtures'; @@ -405,14 +407,106 @@ test('preserve attachments', async ({ runInlineTest, mergeReports, showReport, p expect(exitCode).toBe(0); await showReport(); + + // Check file attachment. await page.getByText('first').click(); await expect(page.getByText('file-attachment')).toBeVisible(); + + // Check file attachment content. + const popupPromise = page.waitForEvent('popup'); + await page.getByText('file-attachment').click(); + const popup = await popupPromise; + await expect(popup.locator('body')).toHaveText('hello!'); + await popup.close(); await page.goBack(); await page.getByText('failing 1').click(); await expect(page.getByText('\'text-attachment\', { body: \'hi!\'')).toBeVisible(); }); +test('generate html with attachment urls', async ({ runInlineTest, mergeReports, page, server }) => { + test.slow(); + const reportDir = test.info().outputPath('blob-report'); + const files = { + 'playwright.config.ts': ` + module.exports = { + retries: 1, + use: { + trace: 'on' + }, + reporter: [['blob', { outputDir: '${reportDir.replace(/\\/g, '/')}' }]] + }; + `, + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + import fs from 'fs'; + + test('first', async ({}) => { + const attachmentPath = test.info().outputPath('foo.txt'); + fs.writeFileSync(attachmentPath, 'hello!'); + await test.info().attach('file-attachment', { path: attachmentPath }); + + console.log('console info'); + console.error('console error'); + }); + test('failing 1', async ({}) => { + await test.info().attach('text-attachment', { body: 'hi!' }); + expect(1).toBe(2); + }); + test.skip('skipped 1', async ({}) => {}); + `, + 'b.test.js': ` + import { test, expect } from '@playwright/test'; + test('math 2', async ({}) => { }); + test('failing 2', async ({}) => { + expect(1).toBe(2); + }); + test.skip('skipped 2', async ({}) => {}); + ` + }; + await runInlineTest(files, { shard: `1/2` }); + + const reportFiles = await fs.promises.readdir(reportDir); + reportFiles.sort(); + expect(reportFiles).toEqual([expect.stringMatching(/report-1-of-2.*.jsonl/), 'resources']); + const { exitCode } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html', '--attachments', 'missing'] }); + expect(exitCode).toBe(0); + + const htmlReportDir = test.info().outputPath('playwright-report'); + for (const entry of await fs.promises.readdir(htmlReportDir)) + await (fs.promises as any).cp(path.join(htmlReportDir, entry), path.join(reportDir, entry), { recursive: true }); + + const oldSeveFile = server.serveFile; + server.serveFile = async (req, res) => { + const pathName = url.parse(req.url!).pathname!; + const filePath = path.join(reportDir, pathName.substring(1)); + return oldSeveFile.call(server, req, res, filePath); + }; + + // Check file attachment. + await page.goto(`${server.PREFIX}/index.html`); + await page.getByText('first').click(); + await expect(page.getByText('file-attachment')).toBeVisible(); + + // Check file attachment content. + const popupPromise = page.waitForEvent('popup'); + await page.getByText('file-attachment').click(); + const popup = await popupPromise; + await expect(popup.locator('body')).toHaveText('hello!'); + await popup.close(); + await page.goBack(); + + // Check inline attachment. + await page.getByText('failing 1').click(); + await expect(page.getByText('\'text-attachment\', { body: \'hi!\'')).toBeVisible(); + await page.goBack(); + + // Check that trace loads. + await page.locator('div').filter({ hasText: /^a\.test\.js:13$/ }).getByRole('link', { name: 'View trace' }).click(); + await expect(page).toHaveTitle('Playwright Trace Viewer'); + await expect(page.getByTestId('action-list').locator('div').filter({ hasText: /^expect\.toBe$/ })).toBeVisible(); +}); + test('multiple output reports', async ({ runInlineTest, mergeReports, showReport, page }) => { test.slow(); const reportDir = test.info().outputPath('blob-report'); diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts index 53f29a9cf2..102121d428 100644 --- a/tests/playwright-test/reporter-html.spec.ts +++ b/tests/playwright-test/reporter-html.spec.ts @@ -98,9 +98,7 @@ test('should not throw when attachment is missing', async ({ runInlineTest, page await showReport(); await page.click('text=passes'); - await page.locator('text=Missing attachment "screenshot"').click(); - const screenshotFile = testInfo.outputPath('test-results', 'a-passes', 'screenshot.png'); - await expect(page.locator('.attachment-body')).toHaveText(`Attachment file ${screenshotFile} is missing`); + await expect(page.getByRole('link', { name: 'screenshot' })).toBeVisible(); }); test('should include image diff', async ({ runInlineTest, page, showReport }) => {