diff --git a/src/test/reporters/html.ts b/src/test/reporters/html.ts index 1be9ab1b71..26db3c03be 100644 --- a/src/test/reporters/html.ts +++ b/src/test/reporters/html.ts @@ -16,8 +16,9 @@ import fs from 'fs'; import path from 'path'; -import { Suite, TestError, TestStatus, Location, TestCase, TestResult, TestStep, FullConfig } from '../../../types/testReporter'; -import { BaseReporter, formatResultFailure } from './base'; +import { FullConfig, Location, Suite, TestCase, TestError, TestResult, TestStatus, TestStep } from '../../../types/testReporter'; +import { calculateFileSha1 } from '../../utils/utils'; +import { formatResultFailure } from './base'; import { serializePatterns, toPosixPath } from './json'; export type JsonStats = { expected: number, unexpected: number, flaky: number, skipped: number }; @@ -62,6 +63,22 @@ export type JsonTestCase = { outcome: 'skipped' | 'expected' | 'unexpected' | 'flaky'; }; +export type TestAttachment = { + name: string; + path?: string; + body?: Buffer; + contentType: string; + sha1?: string; +}; + +export type JsonAttachment = { + name: string; + path?: string; + body?: string; + contentType: string; + sha1?: string; +}; + export type JsonTestResult = { retry: number; workerIndex: number; @@ -70,7 +87,7 @@ export type JsonTestResult = { status: TestStatus; error?: TestError; failureSnippet?: string; - attachments: { name: string, path?: string, body?: Buffer, contentType: string }[]; + attachments: JsonAttachment[]; stdout: (string | Buffer)[]; stderr: (string | Buffer)[]; steps: JsonTestStep[]; @@ -85,15 +102,30 @@ export type JsonTestStep = { steps: JsonTestStep[]; }; -class HtmlReporter extends BaseReporter { - async onEnd() { - const targetFolder = process.env[`PLAYWRIGHT_HTML_REPORT`] || 'playwright-report'; - fs.mkdirSync(targetFolder, { recursive: true }); +class HtmlReporter { + private _targetFolder: string; + private config!: FullConfig; + private suite!: Suite; + + onBegin(config: FullConfig, suite: Suite) { + this.config = config; + this.suite = suite; + } + + constructor() { + this._targetFolder = process.env[`PLAYWRIGHT_HTML_REPORT`] || 'playwright-report'; + fs.mkdirSync(this._targetFolder, { recursive: true }); const appFolder = path.join(__dirname, '..', '..', 'web', 'htmlReport'); for (const file of fs.readdirSync(appFolder)) - fs.copyFileSync(path.join(appFolder, file), path.join(targetFolder, file)); + fs.copyFileSync(path.join(appFolder, file), path.join(this._targetFolder, file)); + } + + async onEnd() { const stats: JsonStats = { expected: 0, unexpected: 0, skipped: 0, flaky: 0 }; - const reportFile = path.join(targetFolder, 'report.json'); + this.suite.allTests().forEach(t => { + ++stats[t.outcome()]; + }); + const reportFile = path.join(this._targetFolder, 'report.json'); const output: JsonReport = { config: { ...this.config, @@ -113,7 +145,7 @@ class HtmlReporter extends BaseReporter { }) }, stats, - suites: this.suite.suites.map(s => this._serializeSuite(s)) + suites: await Promise.all(this.suite.suites.map(s => this._serializeSuite(s))) }; fs.writeFileSync(reportFile, JSON.stringify(output)); } @@ -128,16 +160,16 @@ class HtmlReporter extends BaseReporter { }; } - private _serializeSuite(suite: Suite): JsonSuite { + private async _serializeSuite(suite: Suite): Promise { return { title: suite.title, location: this._relativeLocation(suite.location), - suites: suite.suites.map(s => this._serializeSuite(s)), - tests: suite.tests.map(t => this._serializeTest(t)), + suites: await Promise.all(suite.suites.map(s => this._serializeSuite(s))), + tests: await Promise.all(suite.tests.map(t => this._serializeTest(t))), }; } - private _serializeTest(test: TestCase): JsonTestCase { + private async _serializeTest(test: TestCase): Promise { return { title: test.title, location: this._relativeLocation(test.location), @@ -147,11 +179,11 @@ class HtmlReporter extends BaseReporter { retries: test.retries, ok: test.ok(), outcome: test.outcome(), - results: test.results.map(r => this._serializeResult(test, r)), + results: await Promise.all(test.results.map(r => this._serializeResult(test, r))), }; } - private _serializeResult(test: TestCase, result: TestResult): JsonTestResult { + private async _serializeResult(test: TestCase, result: TestResult): Promise { return { retry: result.retry, workerIndex: result.workerIndex, @@ -160,13 +192,31 @@ class HtmlReporter extends BaseReporter { status: result.status, error: result.error, failureSnippet: formatResultFailure(test, result, '').join('') || undefined, - attachments: result.attachments, + attachments: await this._copyAttachments(result.attachments), stdout: result.stdout, stderr: result.stderr, steps: this._serializeSteps(result.steps) }; } + private async _copyAttachments(attachments: TestAttachment[]): Promise { + const result: JsonAttachment[] = []; + for (const attachment of attachments) { + if (attachment.path) { + const sha1 = await calculateFileSha1(attachment.path) + extension(attachment.contentType); + fs.copyFileSync(attachment.path, path.join(this._targetFolder, sha1)); + result.push({ + ...attachment, + body: undefined, + sha1 + }); + } else if (attachment.body) { + result.push({ ...attachment, body: attachment.body.toString('base64') }); + } + } + return result; + } + private _serializeSteps(steps: TestStep[]): JsonTestStep[] { const stepStack: TestStep[] = []; const result: JsonTestStep[] = []; @@ -205,4 +255,14 @@ function containsStep(outer: TestStep, inner: TestStep): boolean { return true; } +function extension(contentType: string) { + if (contentType === 'image/png') + return '.png'; + if (contentType === 'image/jpeg' || contentType === 'image/jpg') + return '.jpeg'; + if (contentType === 'video/webm') + return '.webm'; + return '.data'; +} + export default HtmlReporter; diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 1fbecfa48b..cb4aeb385d 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -16,6 +16,7 @@ import path from 'path'; import fs from 'fs'; +import stream from 'stream'; import removeFolder from 'rimraf'; import * as crypto from 'crypto'; import os from 'os'; @@ -266,6 +267,30 @@ export function monotonicTime(): number { return seconds * 1000 + (nanoseconds / 1000 | 0) / 1000; } +class HashStream extends stream.Writable { + private _hash = crypto.createHash('sha1'); + + _write(chunk: Buffer, encoding: string, done: () => void) { + this._hash.update(chunk); + done(); + } + + digest(): string { + return this._hash.digest('hex'); + } +} + +export async function calculateFileSha1(filename: string): Promise { + const hashStream = new HashStream(); + const stream = fs.createReadStream(filename); + stream.on('open', () => stream.pipe(hashStream)); + await new Promise((f, r) => { + hashStream.on('finish', f); + hashStream.on('error', r); + }); + return hashStream.digest(); +} + export function calculateSha1(buffer: Buffer | string): string { const hash = crypto.createHash('sha1'); hash.update(buffer); diff --git a/src/web/htmlReport/htmlReport.css b/src/web/htmlReport/htmlReport.css index 8f6eb75dc4..af2e794213 100644 --- a/src/web/htmlReport/htmlReport.css +++ b/src/web/htmlReport/htmlReport.css @@ -116,4 +116,37 @@ align-items: center; max-width: 450px; line-height: 24px; -} \ No newline at end of file +} + +.awesome { + font-size: 24px; + display: flex; + align-items: center; + justify-content: center; + height: 100%; +} + +.image-preview img { + max-width: 500px; + max-height: 500px; +} + +.test-result .tabbed-pane { + margin-top: 50px; + width: 550px; +} + +.image-preview { + position: relative; + display: flex; + align-items: center; + justify-content: center; + width: 550px; + height: 550px; +} + +.test-result .tabbed-pane .tab-content { + display: flex; + align-items: center; + justify-content: center; +} diff --git a/src/web/htmlReport/htmlReport.tsx b/src/web/htmlReport/htmlReport.tsx index df58150718..385659ef13 100644 --- a/src/web/htmlReport/htmlReport.tsx +++ b/src/web/htmlReport/htmlReport.tsx @@ -20,7 +20,7 @@ import { SplitView } from '../components/splitView'; import { TreeItem } from '../components/treeItem'; import { TabbedPane } from '../traceViewer/ui/tabbedPane'; import ansi2html from 'ansi-to-html'; -import { JsonLocation, JsonReport, JsonSuite, JsonTestCase, JsonTestResult, JsonTestStep } from '../../test/reporters/html'; +import type { JsonAttachment, JsonLocation, JsonReport, JsonSuite, JsonTestCase, JsonTestResult, JsonTestStep } from '../../test/reporters/html'; import { msToString } from '../uiUtils'; type Filter = 'Failing' | 'All'; @@ -38,11 +38,15 @@ export const Report: React.FC = () => { }, []); const [filter, setFilter] = React.useState('Failing'); - const failingTests = React.useMemo(() => { - const map = new Map(); - for (const project of report?.suites || []) - map.set(project, computeFailingTests(project)); - return map; + const { unexpectedTests, unexpectedTestCount } = React.useMemo(() => { + const unexpectedTests = new Map(); + let unexpectedTestCount = 0; + for (const project of report?.suites || []) { + const unexpected = computeUnexpectedTests(project); + unexpectedTestCount += unexpected.length; + unexpectedTests.set(project, unexpected); + } + return { unexpectedTests, unexpectedTestCount }; }, [report]); return
@@ -51,10 +55,11 @@ export const Report: React.FC = () => {
{filter === 'All' && report?.suites.map((s, i) => )} - {filter === 'Failing' && report?.suites.map((s, i) => { - const hasFailingTests = !!failingTests.get(s)?.length; - return hasFailingTests && ; + {filter === 'Failing' && !!unexpectedTestCount && report?.suites.map((s, i) => { + const hasUnexpectedOutcomes = !!unexpectedTests.get(s)?.length; + return hasUnexpectedOutcomes && ; })} + {filter === 'Failing' && !unexpectedTestCount &&
You are awesome!
}
; @@ -81,31 +86,31 @@ const ProjectTreeItem: React.FC<{ selectedTest?: JsonTestCase, setSelectedTest: (test: JsonTestCase) => void; }> = ({ suite, setSelectedTest, selectedTest }) => { - const location = renderLocation(suite?.location); + const location = renderLocation(suite?.location, true); return
{testSuiteErrorStatusIcon(suite) || statusIcon('passed')}
{suite?.title}
{!!suite?.location?.line && location &&
{location}
} } loadChildren={() => { - return suite?.suites.map((s, i) => ) || []; + return suite?.suites.map((s, i) => ) || []; }} depth={0} expandByDefault={true}>
; }; const ProjectFlatTreeItem: React.FC<{ suite?: JsonSuite; - failingTests: JsonTestCase[], + unexpectedTests: JsonTestCase[], selectedTest?: JsonTestCase, setSelectedTest: (test: JsonTestCase) => void; -}> = ({ suite, setSelectedTest, selectedTest, failingTests }) => { - const location = renderLocation(suite?.location); +}> = ({ suite, setSelectedTest, selectedTest, unexpectedTests }) => { + const location = renderLocation(suite?.location, true); return
{testSuiteErrorStatusIcon(suite) || statusIcon('passed')}
{suite?.title}
{!!suite?.location?.line && location &&
{location}
} } loadChildren={() => { - return failingTests.map((t, i) => ) || []; + return unexpectedTests.map((t, i) => ) || []; }} depth={0} expandByDefault={true}>
; }; @@ -114,14 +119,15 @@ const SuiteTreeItem: React.FC<{ selectedTest?: JsonTestCase, setSelectedTest: (test: JsonTestCase) => void; depth: number, -}> = ({ suite, setSelectedTest, selectedTest, depth }) => { - const location = renderLocation(suite?.location); + showFileName: boolean, +}> = ({ suite, setSelectedTest, selectedTest, showFileName, depth }) => { + const location = renderLocation(suite?.location, showFileName); return
{testSuiteErrorStatusIcon(suite) || statusIcon('passed')}
{suite?.title}
{!!suite?.location?.line && location &&
{location}
} } loadChildren={() => { - const suiteChildren = suite?.suites.map((s, i) => ) || []; + const suiteChildren = suite?.suites.map((s, i) => ) || []; const testChildren = suite?.tests.map((t, i) => ) || []; return [...suiteChildren, ...testChildren]; }} depth={depth}>
; @@ -163,14 +169,21 @@ const TestOverview: React.FC<{ test: JsonTestCase, result: JsonTestResult, }> = ({ test, result }) => { + const { attachments, screenshots } = React.useMemo(() => { + const attachments = new Map(); + const screenshots = result.attachments.filter(a => a.name === 'actual'); + for (const a of result.attachments) + attachments.set(a.name, a); + return { attachments, screenshots }; + }, [ result ]); return
{test?.title}
-
{renderLocation(test.location)}
{msToString(result.duration)}
- { result.failureSnippet &&
} - { result.steps.map((step, i) => ) } - {/*
{ JSON.stringify(result.steps, undefined, 2) }
*/} +
{renderLocation(test.location, true)}
{msToString(result.duration)}
+ {result.failureSnippet &&
} + {result.steps.map((step, i) => )} + {attachments.has('expected') && attachments.has('actual') && } + {!!screenshots.length &&
Screenshots
} + {screenshots.map(a =>
)}
; }; @@ -188,6 +201,36 @@ const StepTreeItem: React.FC<{ } : undefined} depth={depth}>; }; +export const ImageDiff: React.FunctionComponent<{ + actual: JsonAttachment, + expected: JsonAttachment, + diff?: JsonAttachment, +}> = ({ actual, expected, diff }) => { + const [selectedTab, setSelectedTab] = React.useState('actual'); + const tabs = []; + tabs.push({ + id: 'actual', + title: 'Actual', + render: () =>
+ }); + tabs.push({ + id: 'expected', + title: 'Expected', + render: () =>
+ }); + if (diff) { + tabs.push({ + id: 'diff', + title: 'Diff', + render: () =>
, + }); + } + return
+
Image mismatch
+ +
; +}; + function testSuiteErrorStatusIcon(suite?: JsonSuite): JSX.Element | undefined { if (!suite) return; @@ -231,13 +274,13 @@ function statusIcon(status: 'failed' | 'timedOut' | 'skipped' | 'passed' | 'expe } } -function computeFailingTests(suite: JsonSuite): JsonTestCase[] { +function computeUnexpectedTests(suite: JsonSuite): JsonTestCase[] { const failedTests: JsonTestCase[] = []; const visit = (suite: JsonSuite) => { for (const child of suite.suites) visit(child); for (const test of suite.tests) { - if (test.results.find(r => r.status === 'failed' || r.status === 'timedOut')) + if (test.outcome !== 'expected' && test.outcome !== 'skipped') failedTests.push(test); } }; @@ -245,10 +288,10 @@ function computeFailingTests(suite: JsonSuite): JsonTestCase[] { return failedTests; } -function renderLocation(location?: JsonLocation) { +function renderLocation(location: JsonLocation | undefined, showFileName: boolean) { if (!location) return ''; - return location.file + ':' + location.column; + return (showFileName ? location.file : '') + ':' + location.column; } function retryLabel(index: number) { diff --git a/tests/config/browserTest.ts b/tests/config/browserTest.ts index 763fece9bf..a446712b95 100644 --- a/tests/config/browserTest.ts +++ b/tests/config/browserTest.ts @@ -135,7 +135,7 @@ export const playwrightFixtures: Fixtures { + contextFactory: async ({ browser, contextOptions, video }, run, testInfo) => { const contexts: BrowserContext[] = []; await run(async options => { const context = await browser.newContext({ ...contextOptions, ...options }); @@ -147,7 +147,16 @@ export const playwrightFixtures: Fixtures context.close())); + await Promise.all(contexts.map(async context => { + const videos = context.pages().map(p => p.video()).filter(Boolean); + await context.close(); + for (const v of videos) { + const videoPath = await v.path(); + const savedPath = testInfo.outputPath(path.basename(videoPath)); + await v.saveAs(savedPath); + testInfo.attachments.push({ name: 'video', path: savedPath, contentType: 'video/webm' }); + } + })); }, context: async ({ contextFactory }, run) => {