diff --git a/src/test/reporters/html.ts b/src/test/reporters/html.ts index d6bf8d65ed..cd997a1fc9 100644 --- a/src/test/reporters/html.ts +++ b/src/test/reporters/html.ts @@ -22,7 +22,7 @@ import { FullConfig, Suite } from '../../../types/testReporter'; import { HttpServer } from '../../utils/httpServer'; import { calculateSha1, removeFolders } from '../../utils/utils'; import { toPosixPath } from '../reporters/json'; -import RawReporter, { JsonReport, JsonSuite, JsonTestCase, JsonTestResult, JsonTestStep } from './raw'; +import RawReporter, { JsonReport, JsonSuite, JsonTestCase, JsonTestResult, JsonTestStep, JsonAttachment } from './raw'; export type Stats = { total: number; @@ -64,6 +64,8 @@ export type TestTreeItem = { ok: boolean; }; +export type TestAttachment = JsonAttachment; + export type TestFile = { fileId: string; path: string; @@ -83,6 +85,7 @@ export type TestResult = { duration: number; steps: TestStep[]; error?: string; + attachments: TestAttachment[]; status: 'passed' | 'failed' | 'timedOut' | 'skipped'; }; @@ -115,7 +118,7 @@ class HtmlReporter { await removeFolders([reportFolder]); new HtmlBuilder(reports, reportFolder, this.config.rootDir); - if (!process.env.CI) { + if (!process.env.CI && !process.env.PWTEST_SKIP_TEST_OUTPUT) { const server = new HttpServer(); server.routePrefix('/', (request, response) => { let relativePath = request.url!; @@ -139,12 +142,13 @@ class HtmlBuilder { private _reportFolder: string; private _tests = new Map(); private _rootDir: string; + private _dataFolder: string; constructor(rawReports: JsonReport[], outputDir: string, rootDir: string) { this._rootDir = rootDir; this._reportFolder = path.resolve(process.cwd(), outputDir); - const dataFolder = path.join(this._reportFolder, 'data'); - fs.mkdirSync(dataFolder, { recursive: true }); + this._dataFolder = path.join(this._reportFolder, 'data'); + fs.mkdirSync(this._dataFolder, { recursive: true }); const appFolder = path.join(__dirname, '..', '..', 'web', 'htmlReport'); for (const file of fs.readdirSync(appFolder)) fs.copyFileSync(path.join(appFolder, file), path.join(this._reportFolder, file)); @@ -162,7 +166,7 @@ class HtmlBuilder { path: relativeFileName, tests: tests.map(t => this._createTestCase(t)) }; - fs.writeFileSync(path.join(dataFolder, fileId + '.json'), JSON.stringify(testFile, undefined, 2)); + fs.writeFileSync(path.join(this._dataFolder, fileId + '.json'), JSON.stringify(testFile, undefined, 2)); } projects.push({ name: projectJson.project.name, @@ -170,7 +174,7 @@ class HtmlBuilder { stats: suites.reduce((a, s) => addStats(a, s.stats), emptyStats()), }); } - fs.writeFileSync(path.join(dataFolder, 'projects.json'), JSON.stringify(projects, undefined, 2)); + fs.writeFileSync(path.join(this._dataFolder, 'projects.json'), JSON.stringify(projects, undefined, 2)); } private _createTestCase(test: JsonTestCase): TestCase { @@ -178,7 +182,7 @@ class HtmlBuilder { testId: test.testId, title: test.title, location: this._relativeLocation(test.location), - results: test.results.map(r => this._createTestResult(r)) + results: test.results.map(r => this._createTestResult(test, r)) }; } @@ -190,6 +194,8 @@ class HtmlBuilder { for (const test of tests) { if (test.outcome === 'expected') ++stats.expected; + if (test.outcome === 'skipped') + ++stats.skipped; if (test.outcome === 'unexpected') ++stats.unexpected; if (test.outcome === 'flaky') @@ -221,7 +227,7 @@ class HtmlBuilder { }; } - private _createTestResult(result: JsonTestResult): TestResult { + private _createTestResult(test: JsonTestCase, result: JsonTestResult): TestResult { return { duration: result.duration, startTime: result.startTime, @@ -229,6 +235,22 @@ class HtmlBuilder { steps: result.steps.map(s => this._createTestStep(s)), error: result.error, status: result.status, + attachments: result.attachments.map(a => { + if (a.path) { + const fileName = 'data/' + test.testId + path.extname(a.path); + try { + fs.copyFileSync(a.path, path.join(this._reportFolder, fileName)); + } catch (e) { + } + return { + name: a.name, + contentType: a.contentType, + path: fileName, + body: a.body, + }; + } + return a; + }) }; } diff --git a/src/test/reporters/raw.ts b/src/test/reporters/raw.ts index b6b13206b5..260c9d8f73 100644 --- a/src/test/reporters/raw.ts +++ b/src/test/reporters/raw.ts @@ -67,13 +67,6 @@ export type JsonTestCase = { outcome: 'skipped' | 'expected' | 'unexpected' | 'flaky'; }; -export type TestAttachment = { - name: string; - path?: string; - body?: Buffer; - contentType: string; -}; - export type JsonAttachment = { name: string; body?: string; diff --git a/src/web/components/treeItem.tsx b/src/web/components/treeItem.tsx index 15369ea529..354bfb70d0 100644 --- a/src/web/components/treeItem.tsx +++ b/src/web/components/treeItem.tsx @@ -26,7 +26,7 @@ export const TreeItem: React.FunctionComponent<{ }> = ({ title, loadChildren, onClick, expandByDefault, depth, selected }) => { const [expanded, setExpanded] = React.useState(expandByDefault || false); const className = selected ? 'tree-item-title selected' : 'tree-item-title'; - return
+ return
{ onClick?.(); setExpanded(!expanded); }} >
diff --git a/src/web/htmlReport/htmlReport.css b/src/web/htmlReport/htmlReport.css index adea104852..e1be6c8b98 100644 --- a/src/web/htmlReport/htmlReport.css +++ b/src/web/htmlReport/htmlReport.css @@ -14,6 +14,14 @@ limitations under the License. */ +body { + --box-shadow-thick: rgb(0 0 0 / 10%) 0px 1.8px 1.9px, + rgb(0 0 0 / 15%) 0px 6.1px 6.3px, + rgb(0 0 0 / 10%) 0px -2px 4px, + rgb(0 0 0 / 15%) 0px -6.1px 12px, + rgb(0 0 0 / 25%) 0px 27px 28px; +} + .suite-tree-column { line-height: 18px; flex: auto; @@ -53,7 +61,7 @@ overflow: auto; margin: 20px; flex: none; - box-shadow: var(--box-shadow); + box-shadow: var(--box-shadow-thick); } .status-icon { @@ -81,28 +89,16 @@ flex: auto; display: flex; flex-direction: column; + max-width: 600px; + overflow: auto; } .test-overview-title { - padding: 10px 0; + padding: 30px 10px 10px; font-size: 18px; flex: none; } -.image-preview img { - max-width: 500px; - max-height: 500px; -} - -.image-preview { - position: relative; - display: flex; - align-items: center; - justify-content: center; - width: 550px; - height: 550px; -} - .test-result .tabbed-pane .tab-content { display: flex; align-items: center; @@ -130,6 +126,10 @@ color: white !important; } +.test-result > div { + flex: none; +} + .suite-tree-column .tab-strip, .test-case-column .tab-strip { border: none; @@ -185,10 +185,11 @@ .stats { background-color: gray; border-radius: 2px; - min-width: 10px; + min-width: 14px; color: white; margin: 0 2px; padding: 0 2px; + text-align: center; } .stats.expected { @@ -202,3 +203,10 @@ .stats.flaky { background-color: var(--yellow); } + +video, img { + flex: none; + box-shadow: var(--box-shadow-thick); + width: 80%; + margin: 20px auto; +} diff --git a/src/web/htmlReport/htmlReport.tsx b/src/web/htmlReport/htmlReport.tsx index b4f5ccc953..be3db0a212 100644 --- a/src/web/htmlReport/htmlReport.tsx +++ b/src/web/htmlReport/htmlReport.tsx @@ -21,7 +21,7 @@ import { SplitView } from '../components/splitView'; import { TreeItem } from '../components/treeItem'; import { TabbedPane } from '../traceViewer/ui/tabbedPane'; import { msToString } from '../uiUtils'; -import type { ProjectTreeItem, SuiteTreeItem, TestCase, TestResult, TestStep, TestTreeItem, Location, TestFile, Stats } from '../../test/reporters/html'; +import type { ProjectTreeItem, SuiteTreeItem, TestCase, TestResult, TestStep, TestTreeItem, Location, TestFile, Stats, TestAttachment } from '../../test/reporters/html'; type Filter = 'Failing' | 'All'; @@ -149,29 +149,45 @@ const TestCaseView: React.FC<{ } const [selectedResultIndex, setSelectedResultIndex] = React.useState(0); - return -
-
-
- { test &&
{test?.title}
} - { test &&
{renderLocation(test.location, true)}
} - { test && ({ - id: String(index), - title:
{statusIcon(result.status)} {retryLabel(index)}
, - render: () => - })) || []} selectedTab={String(selectedResultIndex)} setSelectedTab={id => setSelectedResultIndex(+id)} />} -
-
; + return
+ { test &&
{test?.title}
} + { test &&
{renderLocation(test.location, true)}
} + { test && ({ + id: String(index), + title:
{statusIcon(result.status)} {retryLabel(index)}
, + render: () => + })) || []} selectedTab={String(selectedResultIndex)} setSelectedTab={id => setSelectedResultIndex(+id)} />} +
; }; const TestResultView: React.FC<{ test: TestCase, result: TestResult, -}> = ({ test, result }) => { +}> = ({ result }) => { + + const { screenshots, videos, attachmentsMap } = React.useMemo(() => { + const attachmentsMap = new Map(); + const attachments = result?.attachments || []; + const screenshots = attachments.filter(a => a.name === 'screenshot'); + const videos = attachments.filter(a => a.name === 'video'); + for (const a of attachments) + attachmentsMap.set(a.name, a); + return { attachmentsMap, screenshots, videos }; + }, [ result ]); + return
{result.error && } {result.steps.map((step, i) => )} + {attachmentsMap.has('expected') && attachmentsMap.has('actual') && } + {!!screenshots &&
Screenshots
} + {screenshots.map((a, i) => )} + {!!videos.length &&
Videos
} + {videos.map((a, i) => )} + {!!result.attachments &&
Attachments
} + {result.attachments.map((a, i) => )}
; }; @@ -203,6 +219,48 @@ const StatsView: React.FC<{
; }; +export const AttachmentLink: React.FunctionComponent<{ + attachment: TestAttachment, +}> = ({ attachment }) => { + return + + {attachment.path && {attachment.name}} + {attachment.body && {attachment.name}} +
} loadChildren={attachment.body ? () => { + return [
${attachment.body}
]; + } : undefined} depth={0}>; +}; + +export const ImageDiff: React.FunctionComponent<{ + actual: TestAttachment, + expected: TestAttachment, + diff?: TestAttachment, +}> = ({ actual, expected, diff }) => { + const [selectedTab, setSelectedTab] = React.useState('actual'); + const tabs = []; + tabs.push({ + id: 'actual', + title: 'Actual', + render: () =>
+ }); + tabs.push({ + id: 'expected', + title: 'Expected', + render: () =>
+ }); + if (diff) { + tabs.push({ + id: 'diff', + title: 'Diff', + render: () =>
, + }); + } + return
+
Image mismatch
+ +
; +}; + function statusIcon(status: 'failed' | 'timedOut' | 'skipped' | 'passed' | 'expected' | 'unexpected' | 'flaky'): JSX.Element { switch (status) { case 'failed': diff --git a/tests/playwright-test/html-reporter.spec.ts b/tests/playwright-test/html-reporter.spec.ts index cf382bb8c1..1b289847fd 100644 --- a/tests/playwright-test/html-reporter.spec.ts +++ b/tests/playwright-test/html-reporter.spec.ts @@ -14,11 +14,117 @@ * limitations under the License. */ +import fs from 'fs'; +import path from 'path'; import { test, expect } from './playwright-test-fixtures'; -import * as path from 'path'; const kHTMLReporterPath = path.join(__dirname, '..', '..', 'lib', 'test', 'reporters', 'html.js'); +test('should generate report', async ({ runInlineTest }, testInfo) => { + await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { name: 'project-name' }; + `, + 'a.test.js': ` + const { test } = pwt; + test('passes', async ({}) => {}); + test('fails', async ({}) => { + expect(1).toBe(2); + }); + test('skip', async ({}) => { + test.skip('Does not work') + }); + test('flaky', async ({}, testInfo) => { + expect(testInfo.retry).toBe(1); + }); + `, + }, { reporter: 'dot,' + kHTMLReporterPath, retries: 1 }); + const report = testInfo.outputPath('playwright-report', 'data', 'projects.json'); + const reportObject = JSON.parse(fs.readFileSync(report, 'utf-8')); + delete reportObject[0].suites[0].duration; + delete reportObject[0].suites[0].location.line; + delete reportObject[0].suites[0].location.column; + + const fileNames = new Set(); + for (const test of reportObject[0].suites[0].tests) { + fileNames.add(testInfo.outputPath('playwright-report', 'data', test.fileId + '.json')); + delete test.testId; + delete test.fileId; + delete test.location.line; + delete test.location.column; + delete test.duration; + } + expect(reportObject[0]).toEqual({ + name: 'project-name', + suites: [ + { + title: 'a.test.js', + location: { + file: 'a.test.js' + }, + stats: { + total: 4, + expected: 1, + unexpected: 1, + flaky: 1, + skipped: 1, + ok: false + }, + suites: [], + tests: [ + { + location: { + file: 'a.test.js' + }, + title: 'passes', + outcome: 'expected', + ok: true + }, + { + location: { + file: 'a.test.js' + }, + title: 'fails', + outcome: 'unexpected', + ok: false + }, + { + location: { + file: 'a.test.js' + }, + title: 'skip', + outcome: 'skipped', + ok: true + }, + { + location: { + file: 'a.test.js' + }, + title: 'flaky', + outcome: 'flaky', + ok: true + } + ] + } + ], + stats: { + total: 4, + expected: 1, + unexpected: 1, + flaky: 1, + skipped: 1, + ok: false + } + }); + + expect(fileNames.size).toBe(1); + const fileName = fileNames.values().next().value; + const testCase = JSON.parse(fs.readFileSync(fileName, 'utf-8')); + expect(testCase.tests).toHaveLength(4); + expect(testCase.tests.map(t => t.title)).toEqual(['passes', 'fails', 'skip', 'flaky']); + expect(testCase).toBeTruthy(); +}); + test('should not throw when attachment is missing', async ({ runInlineTest }) => { const result = await runInlineTest({ 'playwright.config.ts': `