From 6afa85927ee2fdb476155ab52d4b91ed873dfbd0 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Sun, 17 Oct 2021 19:58:06 -0800 Subject: [PATCH] chore(ui): redesign html report a bit (#9577) --- packages/playwright-core/src/web/common.css | 4 +- .../src/web/htmlReport/htmlReport.css | 256 +++++++++++-- .../src/web/htmlReport/htmlReport.tsx | 340 ++++++++++-------- .../src/web/traceViewer/ui/tabbedPane.css | 2 +- .../playwright-test/src/reporters/base.ts | 2 +- .../playwright-test/src/reporters/html.ts | 247 +++++++------ tests/playwright-test/reporter-html.spec.ts | 119 +++--- 7 files changed, 604 insertions(+), 366 deletions(-) diff --git a/packages/playwright-core/src/web/common.css b/packages/playwright-core/src/web/common.css index 0165f1fffb..e2720de4d8 100644 --- a/packages/playwright-core/src/web/common.css +++ b/packages/playwright-core/src/web/common.css @@ -23,9 +23,9 @@ --active-background: #333333; --color: #252423; --red: #F44336; - --green: #4CAF50; + --green: #367c39; --purple: #9C27B0; - --yellow: #FFC107; + --yellow: #ff9207; --white: #FFFFFF; --blue: #0b7ad5; --transparent-blue: #2196F355; diff --git a/packages/playwright-core/src/web/htmlReport/htmlReport.css b/packages/playwright-core/src/web/htmlReport/htmlReport.css index 1340a404de..7fa3b044ff 100644 --- a/packages/playwright-core/src/web/htmlReport/htmlReport.css +++ b/packages/playwright-core/src/web/htmlReport/htmlReport.css @@ -20,19 +20,31 @@ body { rgb(0 0 0 / 10%) 0px -2px 4px, rgb(0 0 0 / 15%) 0px -6.1px 12px, rgb(0 0 0 / 25%) 0px 27px 28px; + --color-border-default: #d0d7de; + --color-border-muted: #d8dee4; + --color-canvas-subtle: #f6f8fa; + --color-danger-fg: #cf222e; + --color-fg-default: #24292f; + --color-fg-muted: #57606a; + --color-page-header-bg: #f6f8fa; + --color-primer-border-active: #fd8c73; + --color-success-fg: #1a7f37; + --color-accent-fg: #0969da; + color: var(--color-fg-default); + overflow: auto; } .suite-tree-column { line-height: 18px; flex: auto; - overflow: auto; color: #616161; background-color: #f3f3f3; border-left: 1px solid #dfe1e5; } .test-case-column { - border-right: 1px solid #dfe1e5; + border-radius: 6px; + margin: 20px; } .tree-item-title { @@ -55,13 +67,12 @@ body { .error-message { white-space: pre; font-family: monospace; - background: #000; - color: white; - padding: 5px; overflow: auto; - margin: 20px; flex: none; - box-shadow: var(--box-shadow-thick); + padding: 0; + background-color: var(--color-canvas-subtle); + border-radius: 6px; + padding: 16px; } .status-icon { @@ -90,13 +101,7 @@ body { display: flex; flex-direction: column; padding: 0 16px; - overflow: auto; -} - -.test-overview-title { - padding: 30px 10px 10px; - font-size: 18px; - flex: none; + margin-bottom: 20px; } .test-result .tabbed-pane .tab-content { @@ -113,10 +118,6 @@ body { margin-left: 24px; } -.test-result .tree-item-title:not(.selected):hover { - background-color: #e8e8e8; -} - .test-result .tree-item-title.selected { background-color: #0060c0; color: white; @@ -130,11 +131,22 @@ body { flex: none; } -.suite-tree-column .tab-strip, -.test-case-column .tab-strip { - border: none; - box-shadow: none; - background-color: transparent; +.columns > .tab-strip { + font-size: 14px; + line-height: 30px; + color: var(--color-fg-default); + height: 48px; + background-color: var(--color-page-header-bg); + min-width: 70px; +} + +.tab-strip { + box-shadow: inset 0 -1px 0 var(--color-border-muted) !important; +} + +.columns > .tab-strip .tab-element.selected { + font-weight: 600; + border-bottom-color: var(--color-primer-border-active); } .suite-tree-column .tab-element, @@ -146,6 +158,10 @@ body { color: #aaa; } +.test-case-column .tab-strip { + background-color: inherit; +} + .suite-tree-column .tab-element.selected, .test-case-column .tab-element.selected { color: #555; @@ -156,8 +172,9 @@ body { display: flex; align-items: center; padding: 10px; - font-size: 18px; - cursor: pointer; + font-weight: 400; + font-size: 32px !important; + line-height: 1.25 !important; } .test-case-location { @@ -182,26 +199,25 @@ body { text-overflow: ellipsis; } +.stats-line { + padding-left: 5px; +} + .stats { - background-color: gray; - border-radius: 2px; - min-width: 14px; - color: white; margin: 0 2px; padding: 0 2px; - text-align: center; } .stats.expected { - background-color: var(--green); + color: var(--green); } .stats.unexpected { - background-color: var(--red); + color: var(--color-danger-fg); } .stats.flaky { - background-color: var(--yellow); + color: var(--yellow); } video, img { @@ -213,7 +229,179 @@ video, img { min-height: 300px; } -.columns { +.flow-container { max-width: 1280px; margin: 0 auto; + width: 100%; +} + +.file-summary-list .chip-body a:not(:nth-child(1)) .test-summary, +.failed-test:not(:nth-child(1)) { + border-top: 1px solid var(--color-border-default); +} + +.failed-file-subtitle { + padding-left: 5px; + font-weight: 600; + color: var(--color-danger-fg); +} + +.failed-test { + padding: 0 15px 0 10px; + line-height: 28px; +} + +.failed-test-title { + font-weight: 600; +} + +.failed-test-path { + padding: 5px 5px 0 0; + color: var(--color-fg-muted); +} + +.failed-test .error-message { + margin: 20px 0 0; +} + +.failed-test:hover { + background-color: var(--color-page-header-bg); +} + +a.no-decorations { + text-decoration: none; + color: initial; +} + +.chip-header { + display: flex; + align-items: center; + border: 1px solid var(--color-border-default); + border-top-left-radius: 6px; + border-top-right-radius: 6px; + background-color: var(--color-page-header-bg); + padding: 10px; + border-bottom: none; + margin-top: 20px; + font-weight: 600; +} + +.chip-header.expanded-false { + border: 1px solid var(--color-border-default); + border-radius: 6px; +} + +.chip-header.expanded-false, +.chip-header.expanded-true { + cursor: pointer; +} + +.chip-body { + border: 1px solid var(--color-border-default); + border-bottom-left-radius: 6px; + border-bottom-right-radius: 6px; + padding: 15px; +} + +.failed-tests { + padding-bottom: 20px; +} + +.file-summary-list .chip-body, +.failed-tests .chip-body { + padding: 0; +} + +.test-summary { + height: 38px; + line-height: 38px; + align-items: center; + padding: 0 10px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.test-summary:hover { + background-color: var(--color-page-header-bg); +} + +.test-summary-path { + padding: 0 0 0 5px; + color: var(--color-fg-muted); +} + +.octicon { + display: inline-block; + overflow: visible !important; + vertical-align: text-bottom; + fill: currentColor; +} + +.color-icon-success { + color: var(--color-success-fg) !important; +} + +.color-text-danger { + color: var(--color-danger-fg) !important; +} + +.color-text-warning { + color: var(--yellow) !important; +} + +.color-fg-muted { + color: var(--color-fg-muted) !important; +} + +.octicon { + margin-right: 7px; + flex: none; +} + +.label { + display: inline-block; + padding: 0 7px; + font-size: 12px; + font-weight: 500; + line-height: 18px; + border: 1px solid transparent; + border-radius: 2em; + background-color: #757575; + color: white; + margin-left: 10px; + flex: none; +} + +.label-color-0 { background-color: #2196f3; } +.label-color-1 { background-color: #4caf50; } +.label-color-2 { background-color: #ff9800; } +.label-color-3 { background-color: #ba68c8; } +.label-color-4 { background-color: #26a69a; } +.label-color-5 { background-color: #8d6e63; } +.label-color-6 { background-color: #607d8b; } +.label-color-7 { background-color: #fbc02d; } + +@media only screen and (max-width: 600px) { + .chip-header { + border-radius: 0 !important; + border-right: none !important; + border-left: none !important; + } + + .chip-body { + border-radius: 0 !important; + border-right: none !important; + border-left: none !important; + padding: 5px !important; + } + + .test-result { + padding: 0 !important; + } + + .test-case-column { + border-radius: 0 !important; + margin: 0 !important; + } } diff --git a/packages/playwright-core/src/web/htmlReport/htmlReport.tsx b/packages/playwright-core/src/web/htmlReport/htmlReport.tsx index 30db558b38..26853de0a5 100644 --- a/packages/playwright-core/src/web/htmlReport/htmlReport.tsx +++ b/packages/playwright-core/src/web/htmlReport/htmlReport.tsx @@ -17,138 +17,107 @@ import './htmlReport.css'; import * as React from 'react'; import ansi2html from 'ansi-to-html'; -import { SplitView } from '../components/splitView'; import { TreeItem } from '../components/treeItem'; import { TabbedPane } from '../traceViewer/ui/tabbedPane'; import { msToString } from '../uiUtils'; -import type { ProjectTreeItem, SuiteTreeItem, TestCase, TestResult, TestStep, TestTreeItem, Location, TestFile, Stats, TestAttachment } from '@playwright/test/src/reporters/html'; - -type Filter = 'failing' | 'all'; - -type TestId = { - fileId: string; - testId: string; -}; +import type { TestCase, TestResult, TestStep, TestFile, Stats, TestAttachment, HTMLReport, TestFileSummary } from '@playwright/test/src/reporters/html'; export const Report: React.FC = () => { - const [report, setReport] = React.useState([]); const [fetchError, setFetchError] = React.useState(); - const [testId, setTestId] = React.useState(); - const [filter, setFilter] = React.useState('failing'); + const [report, setReport] = React.useState(); + const [expandedFiles, setExpandedFiles] = React.useState>(new Set()); React.useEffect(() => { + if (report) + return; (async () => { try { - const result = await fetch('data/projects.json', { cache: 'no-cache' }); - const json = (await result.json()) as ProjectTreeItem[]; - const hasFailures = !!json.find(p => !p.stats.ok); - if (!hasFailures) - setFilter('all'); - setReport(json); + const report = await fetch('data/report.json', { cache: 'no-cache' }).then(r => r.json() as Promise); + if (report.files.length) + expandedFiles.add(report.files[0].fileId); + setReport(report); } catch (e) { setFetchError(e.message); } })(); - }, []); + }, [report, expandedFiles]); - return
- - -
-
-
setFilter('all')}>All
-
setFilter('failing')}>Failing
-
- {!fetchError && filter === 'all' && report?.map((project, i) => )} - {!fetchError && filter === 'failing' && report?.filter(p => !p.stats.ok).map((project, i) => )} -
- + return
+ {!fetchError &&
+ + expandedFiles.has(fileId)} setFileExpanded={(fileId, expanded) => { + const newExpanded = new Set(expandedFiles); + if (expanded) + newExpanded.add(fileId); + else + newExpanded.delete(fileId); + setExpandedFiles(newExpanded); + }}> + + + {!!report?.testIdToFileId && } + +
}
; }; -const ProjectTreeItemView: React.FC<{ - project: ProjectTreeItem; - testId?: TestId, - setTestId: (id: TestId) => void; - failingOnly: boolean; -}> = ({ project, testId, setTestId, failingOnly }) => { - const hasChildren = !(failingOnly && project.stats.ok); - return -
{project.name || 'Project'}
-
- -
- } loadChildren={hasChildren ? () => { - return project.suites.filter(s => !(failingOnly && s.stats.ok)).map((s, i) => ) || []; - } : undefined} depth={0} expandByDefault={true}>; +const AllTestFilesSummaryView: React.FC<{ + report?: HTMLReport; + isFileExpanded: (fileId: string) => boolean; + setFileExpanded: (fileId: string, expanded: boolean) => void; +}> = ({ report, isFileExpanded, setFileExpanded }) => { + return
+ {report && (report.files || []).map((file, i) => )} +
; }; -const SuiteTreeItemView: React.FC<{ - suite: SuiteTreeItem, - testId?: TestId, - setTestId: (id: TestId) => void; - failingOnly: boolean; - depth: number, -}> = ({ suite, testId, setTestId, failingOnly, depth }) => { - return -
{suite.title || ''}
-
- - - } loadChildren={() => { - const suiteChildren = suite.suites.filter(s => !(failingOnly && s.stats.ok)).map((s, i) => ) || []; - const suiteCount = suite.suites.length; - const testChildren = suite.tests.filter(t => !(failingOnly && t.ok)).map((t, i) => ) || []; - return [...suiteChildren, ...testChildren]; - }} depth={depth}>; -}; - -const TestTreeItemView: React.FC<{ - test: TestTreeItem, - testId?: TestId, - setTestId: (id: TestId) => void; - depth: number, -}> = ({ test, testId, setTestId, depth }) => { - return - {statusIcon(test.outcome)}
{test.title}
-
- {
{msToString(test.duration)}
} - - } selected={test.testId === testId?.testId} depth={depth} onClick={() => setTestId({ testId: test.testId, fileId: test.fileId })}>; +const TestFileSummaryView: React.FC<{ + report: HTMLReport; + file: TestFileSummary; + isFileExpanded: (fileId: string) => boolean; + setFileExpanded: (fileId: string, expanded: boolean) => void; +}> = ({ file, report, isFileExpanded, setFileExpanded }) => { + return setFileExpanded(file.fileId, expanded))} header={{file.fileName}}> + {file.tests.map((test, i) => +
+ {test.projectName} + {statusIcon(test.outcome)} + {test.title} + — {test.path.join(' › ')} +
+ )} + ; }; const TestCaseView: React.FC<{ - testId: TestId | undefined, -}> = ({ testId }) => { - const [file, setFile] = React.useState(); - + report: HTMLReport, +}> = ({ report }) => { + const [test, setTest] = React.useState(); React.useEffect(() => { (async () => { - if (!testId || file?.fileId === testId.fileId) + const testId = new URL(window.location.href).searchParams.get('testId'); + if (!testId || testId === test?.testId) return; - try { - const result = await fetch(`data/${testId.fileId}.json`, { cache: 'no-cache' }); - setFile((await result.json()) as TestFile); - } catch (e) { + const fileId = report.testIdToFileId[testId]; + if (!fileId) + return; + const result = await fetch(`/data/${fileId}.json`, { cache: 'no-cache' }); + const file = await result.json() as TestFile; + for (const t of file.tests) { + if (t.testId === testId) { + setTest(t); + break; + } } })(); - }); - - let test: TestCase | undefined; - if (file && testId) { - for (const t of file.tests) { - if (t.testId === testId.testId) { - test = t; - break; - } - } - } + }, [test, report]); const [selectedResultIndex, setSelectedResultIndex] = React.useState(0); return
- { test &&
{test?.title}
} - { test &&
{renderLocation(test.location, true)}
} - { test && {test?.title}
} + {test &&
{test.path.join(' › ')}
} + {test &&
{test.projectName}
} + {test && ({ id: String(index), title:
{statusIcon(result.status)} {retryLabel(index)}
, @@ -182,39 +151,49 @@ const TestResultView: React.FC<{ const actual = attachmentsMap.get('actual'); const diff = attachmentsMap.get('diff'); return
- {result.error && } - {result.steps.map((step, i) => )} + {result.error && + + } + {!!result.steps.length && + {result.steps.map((step, i) => )} + } {expected && actual &&
- - - - {diff && } + + + + + {diff && } +
} - {!!screenshots.length &&
Screenshots
} - {screenshots.map((a, i) => { - return
- + {!!screenshots.length && + {screenshots.map((a, i) => { + return
+ + +
; + })} + } + + {!!traces.length && + {traces.map((a, i) =>
+ +
)} + } + + {!!videos.length && + {videos.map((a, i) =>
+ -
; - })} +
)} + } - {!!traces.length &&
Traces
} - {traces.map((a, i) =>
- -
)} - - {!!videos.length &&
Videos
} - {videos.map((a, i) =>
- - -
)} - - {!!otherAttachments.length &&
Attachments
} - {otherAttachments.map((a, i) => )} + {!!otherAttachments.length && + {otherAttachments.map((a, i) => )} + }
; }; @@ -230,7 +209,7 @@ const StepTreeItem: React.FC<{ } loadChildren={step.steps.length + (step.error ? 1 : 0) ? () => { const children = step.steps.map((s, i) => ); if (step.error) - children.unshift(); + children.unshift(); return children; } : undefined} depth={depth}>; }; @@ -238,15 +217,16 @@ const StepTreeItem: React.FC<{ const StatsView: React.FC<{ stats: Stats }> = ({ stats }) => { - return
- {!!stats.expected &&
{stats.expected}
} - {!!stats.unexpected &&
{stats.unexpected}
} - {!!stats.flaky &&
{stats.flaky}
} - {!!stats.skipped &&
{stats.skipped}
} -
; + return + — + {!!stats.unexpected && {stats.unexpected} failed} + {!!stats.flaky && {stats.flaky} flaky} + {!!stats.expected && {stats.expected} passed} + {!!stats.skipped && {stats.skipped} skipped} + ; }; -export const AttachmentLink: React.FunctionComponent<{ +const AttachmentLink: React.FunctionComponent<{ attachment: TestAttachment, href?: string, }> = ({ attachment, href }) => { @@ -259,7 +239,7 @@ export const AttachmentLink: React.FunctionComponent<{ } : undefined} depth={0}>; }; -export const ImageDiff: React.FunctionComponent<{ +const ImageDiff: React.FunctionComponent<{ actual: TestAttachment, expected: TestAttachment, diff?: TestAttachment, @@ -284,7 +264,6 @@ export const ImageDiff: React.FunctionComponent<{ }); } return
-
Image mismatch
; }; @@ -293,25 +272,27 @@ function statusIcon(status: 'failed' | 'timedOut' | 'skipped' | 'passed' | 'expe switch (status) { case 'failed': case 'unexpected': - return ; + return ; case 'passed': case 'expected': - return ; + return ; case 'timedOut': return ; case 'flaky': - return ; + return ; case 'skipped': - return ; + return ; } } -function renderLocation(location: Location | undefined, showFileName: boolean) { - if (!location) - return ''; - return (showFileName ? location.file : '') + ':' + location.line; -} - function retryLabel(index: number) { if (!index) return 'Run'; @@ -320,10 +301,17 @@ function retryLabel(index: number) { const ErrorMessage: React.FC<{ error: string; -}> = ({ error }) => { + mode: 'dark' | 'light' +}> = ({ error, mode }) => { const html = React.useMemo(() => { - return new ansi2html({ colors: ansiColors }).toHtml(escapeHTML(error)); - }, [error]); + const config: any = { + fg: mode === 'dark' ? '#FFF' : '#252423', + bg: mode === 'dark' ? '#252423' : '#FFF', + }; + if (mode === 'dark') + config.colors = ansiColors; + return new ansi2html(config).toHtml(escapeHTML(error)); + }, [error, mode]); return
; }; @@ -349,3 +337,53 @@ const ansiColors = { function escapeHTML(text: string): string { return text.replace(/[&"<>]/g, c => ({ '&': '&', '"': '"', '<': '<', '>': '>' }[c]!)); } + +const Chip: React.FunctionComponent<{ + header: JSX.Element | string, + expanded?: boolean, + setExpanded?: (expanded: boolean) => void, + children?: any +}> = ({ header, expanded, setExpanded, children }) => { + return
+
setExpanded?.(!expanded)}> + {setExpanded &&
} + {header} +
+ { (!setExpanded || expanded) &&
{children}
} +
; +}; + +function navigate(href: string) { + window.history.pushState({}, '', href); + const navEvent = new PopStateEvent('popstate'); + window.dispatchEvent(navEvent); +} + +const Link: React.FunctionComponent<{ + href: string, + children: any +}> = ({ href, children }) => { + return { + event.preventDefault(); + navigate(href); + }} className='no-decorations' href={href}>{children}; +}; + +const Route: React.FunctionComponent<{ + params: string, + children: any +}> = ({ params, children }) => { + const initialParams = [...new URL(window.location.href).searchParams.keys()].join('&'); + const [currentParams, setCurrentParam] = React.useState(initialParams); + React.useEffect(() => { + const listener = () => { + const newParams = [...new URL(window.location.href).searchParams.keys()].join('&'); + setCurrentParam(newParams); + }; + window.addEventListener('popstate', listener); + return () => window.removeEventListener('popstate', listener); + }, []); + return currentParams === params ? children : null; +}; diff --git a/packages/playwright-core/src/web/traceViewer/ui/tabbedPane.css b/packages/playwright-core/src/web/traceViewer/ui/tabbedPane.css index 6611b5015b..cb207a063d 100644 --- a/packages/playwright-core/src/web/traceViewer/ui/tabbedPane.css +++ b/packages/playwright-core/src/web/traceViewer/ui/tabbedPane.css @@ -52,7 +52,7 @@ align-items: center; justify-content: center; user-select: none; - border-bottom: 3px solid transparent; + border-bottom: 2px solid transparent; outline: none; height: 100%; } diff --git a/packages/playwright-test/src/reporters/base.ts b/packages/playwright-test/src/reporters/base.ts index 240cde10d6..a83f6e9698 100644 --- a/packages/playwright-test/src/reporters/base.ts +++ b/packages/playwright-test/src/reporters/base.ts @@ -316,7 +316,7 @@ export function formatTestTitle(config: FullConfig, test: TestCase, step?: TestS const [, projectName, , ...titles] = test.titlePath(); const location = `${relativeTestPath(config, test)}:${test.location.line}:${test.location.column}`; const projectTitle = projectName ? `[${projectName}] › ` : ''; - return `${projectTitle}${location} › ${titles.join(' ')}${stepSuffix(step)}`; + return `${projectTitle}${location} › ${titles.join(' › ')}${stepSuffix(step)}`; } function formatTestHeader(config: FullConfig, test: TestCase, indent: string, index?: number): string { diff --git a/packages/playwright-test/src/reporters/html.ts b/packages/playwright-test/src/reporters/html.ts index 07ac742df5..32bd15e320 100644 --- a/packages/playwright-test/src/reporters/html.ts +++ b/packages/playwright-test/src/reporters/html.ts @@ -32,6 +32,7 @@ export type Stats = { flaky: number; skipped: number; ok: boolean; + duration: number; }; export type Location = { @@ -40,46 +41,43 @@ export type Location = { column: number; }; -export type ProjectTreeItem = { - name: string; - suites: SuiteTreeItem[]; - stats: Stats; +export type HTMLReport = { + files: TestFileSummary[]; + testIdToFileId: { [key: string]: string }; + projectNames: string[]; }; -export type SuiteTreeItem = { - title: string; - location?: Location; - duration: number; - suites: SuiteTreeItem[]; - tests: TestTreeItem[]; - stats: Stats; -}; - -export type TestTreeItem = { - testId: string, - fileId: string, - title: string; - location: Location; - duration: number; - outcome: 'skipped' | 'expected' | 'unexpected' | 'flaky'; - ok: boolean; -}; - -export type TestAttachment = JsonAttachment; - export type TestFile = { fileId: string; - path: string; + fileName: string; tests: TestCase[]; }; -export type TestCase = { +export type TestFileSummary = { + fileId: string; + fileName: string; + tests: TestCaseSummary[]; + stats: Stats; +}; + +export type TestCaseSummary = { testId: string, + fileId: string, title: string; + path: string[]; + projectName: string; location: Location; + outcome: 'skipped' | 'expected' | 'unexpected' | 'flaky'; + duration: number; + ok: boolean; +}; + +export type TestCase = TestCaseSummary & { results: TestResult[]; }; +export type TestAttachment = JsonAttachment; + export type TestResult = { retry: number; startTime: string; @@ -99,6 +97,11 @@ export type TestStep = { steps: TestStep[]; }; +type TestEntry = { + testCase: TestCase; + testCaseSummary: TestCaseSummary +}; + class HtmlReporter { private config!: FullConfig; private suite!: Suite; @@ -124,16 +127,18 @@ class HtmlReporter { const reportFolder = htmlReportFolder(this._outputFolder); await removeFolders([reportFolder]); const builder = new HtmlBuilder(reportFolder, this.config.rootDir); - const stats = builder.build(reports); + const ok = builder.build(reports); - if (!stats.ok && !process.env.CI && !process.env.PWTEST_SKIP_TEST_OUTPUT) { - await showHTMLReport(reportFolder); - } else { - console.log(''); - console.log('All tests passed. To open last HTML report run:'); - console.log(colors.cyan(` + if (!process.env.PWTEST_SKIP_TEST_OUTPUT) { + if (!ok && !process.env.CI && !process.env.PWTEST_SKIP_TEST_OUTPUT) { + await showHTMLReport(reportFolder); + } else { + console.log(''); + console.log('All tests passed. To open last HTML report run:'); + console.log(colors.cyan(` npx playwright show-report `)); + } } } } @@ -174,6 +179,7 @@ export async function showHTMLReport(reportFolder: string | undefined) { class HtmlBuilder { private _reportFolder: string; private _tests = new Map(); + private _testPath = new Map(); private _rootDir: string; private _dataFolder: string; @@ -183,7 +189,7 @@ class HtmlBuilder { this._dataFolder = path.join(this._reportFolder, 'data'); } - build(rawReports: JsonReport[]): Stats { + build(rawReports: JsonReport[]): boolean { fs.mkdirSync(this._dataFolder, { recursive: true }); // Copy app. @@ -195,84 +201,114 @@ class HtmlBuilder { const traceViewerFolder = path.join(require.resolve('playwright-core'), '..', 'lib', 'webpack', 'traceViewer'); const traceViewerTargetFolder = path.join(this._reportFolder, 'trace'); fs.mkdirSync(traceViewerTargetFolder, { recursive: true }); - // TODO (#9471): remove file filter when the babel build is fixed. - for (const file of fs.readdirSync(traceViewerFolder)) { - if (fs.statSync(path.join(traceViewerFolder, file)).isFile()) - fs.copyFileSync(path.join(traceViewerFolder, file), path.join(traceViewerTargetFolder, file)); - } + for (const file of fs.readdirSync(traceViewerFolder)) + fs.copyFileSync(path.join(traceViewerFolder, file), path.join(traceViewerTargetFolder, file)); - const projects: ProjectTreeItem[] = []; + const data = new Map(); for (const projectJson of rawReports) { - const suites: SuiteTreeItem[] = []; for (const file of projectJson.suites) { - const relativeFileName = this._relativeLocation(file.location).file; - const fileId = calculateSha1(projectJson.project.name + ':' + relativeFileName); - const tests: JsonTestCase[] = []; - suites.push(this._createSuiteTreeItem(file, fileId, tests)); - const testFile: TestFile = { - fileId, - path: relativeFileName, - tests: tests.map(t => this._createTestCase(t)) - }; - fs.writeFileSync(path.join(this._dataFolder, fileId + '.json'), JSON.stringify(testFile, undefined, 2)); + const fileName = this._relativeLocation(file.location).file; + const fileId = calculateSha1(fileName); + let fileEntry = data.get(fileId); + if (!fileEntry) { + fileEntry = { + testFile: { fileId, fileName, tests: [] }, + testFileSummary: { fileId, fileName, tests: [], stats: emptyStats() }, + }; + data.set(fileId, fileEntry); + } + const { testFile, testFileSummary } = fileEntry; + const testEntries: TestEntry[] = []; + this._processJsonSuite(file, fileId, projectJson.project.name, [], testEntries); + for (const test of testEntries) { + testFile.tests.push(test.testCase); + testFileSummary.tests.push(test.testCaseSummary); + } } - projects.push({ - name: projectJson.project.name, - suites, - stats: suites.reduce((a, s) => addStats(a, s.stats), emptyStats()), + } + + let ok = true; + const testIdToFileId: { [key: string]: string } = {}; + for (const [fileId, { testFile, testFileSummary }] of data) { + const stats = testFileSummary.stats; + for (const test of testFileSummary.tests) { + testIdToFileId[test.testId] = fileId; + if (test.outcome === 'expected') + ++stats.expected; + if (test.outcome === 'skipped') + ++stats.skipped; + if (test.outcome === 'unexpected') + ++stats.unexpected; + if (test.outcome === 'flaky') + ++stats.flaky; + ++stats.total; + stats.duration += test.duration; + } + stats.ok = stats.unexpected + stats.flaky === 0; + if (!stats.ok) + ok = false; + + testFileSummary.tests.sort((t1, t2) => { + const w1 = (t1.outcome === 'unexpected' ? 1000 : 0) + (t1.outcome === 'flaky' ? 1 : 0); + const w2 = (t2.outcome === 'unexpected' ? 1000 : 0) + (t2.outcome === 'flaky' ? 1 : 0); + if (w2 - w1) + return w2 - w1; + return t1.location.line - t2.location.line; }); + + fs.writeFileSync(path.join(this._dataFolder, fileId + '.json'), JSON.stringify(testFile, undefined, 2)); } - fs.writeFileSync(path.join(this._dataFolder, 'projects.json'), JSON.stringify(projects, undefined, 2)); - return projects.reduce((a, p) => addStats(a, p.stats), emptyStats()); - } - - private _createTestCase(test: JsonTestCase): TestCase { - return { - testId: test.testId, - title: test.title, - location: this._relativeLocation(test.location), - results: test.results.map(r => this._createTestResult(r)) + const htmlReport: HTMLReport = { + files: [...data.values()].map(e => e.testFileSummary), + testIdToFileId, + projectNames: rawReports.map(r => r.project.name) }; + htmlReport.files.sort((f1, f2) => { + const w1 = f1.stats.unexpected * 1000 + f1.stats.flaky; + const w2 = f2.stats.unexpected * 1000 + f2.stats.flaky; + return w2 - w1; + }); + fs.writeFileSync(path.join(this._dataFolder, 'report.json'), JSON.stringify(htmlReport, undefined, 2)); + return ok; } - private _createSuiteTreeItem(suite: JsonSuite, fileId: string, testCollector: JsonTestCase[]): SuiteTreeItem { - const suites = suite.suites.map(s => this._createSuiteTreeItem(s, fileId, testCollector)); - const tests = suite.tests.map(t => this._createTestTreeItem(t, fileId)); - testCollector.push(...suite.tests); - const stats = suites.reduce((a, s) => addStats(a, s.stats), emptyStats()); - for (const test of tests) { - if (test.outcome === 'expected') - ++stats.expected; - if (test.outcome === 'skipped') - ++stats.skipped; - if (test.outcome === 'unexpected') - ++stats.unexpected; - if (test.outcome === 'flaky') - ++stats.flaky; - ++stats.total; - } - stats.ok = stats.unexpected + stats.flaky === 0; - return { - title: suite.title, - location: this._relativeLocation(suite.location), - duration: suites.reduce((a, s) => a + s.duration, 0) + tests.reduce((a, t) => a + t.duration, 0), - stats, - suites, - tests - }; + private _processJsonSuite(suite: JsonSuite, fileId: string, projectName: string, path: string[], out: TestEntry[]) { + const newPath = [...path, suite.title]; + suite.suites.map(s => this._processJsonSuite(s, fileId, projectName, newPath, out)); + suite.tests.forEach(t => out.push(this._createTestEntry(t, fileId, projectName, newPath))); } - private _createTestTreeItem(test: JsonTestCase, fileId: string): TestTreeItem { + private _createTestEntry(test: JsonTestCase, fileId: string, projectName: string, path: string[]): TestEntry { const duration = test.results.reduce((a, r) => a + r.duration, 0); this._tests.set(test.testId, test); + const location = this._relativeLocation(test.location); + path = [location.file + ':' + location.line, ...path.slice(1)]; + this._testPath.set(test.testId, path); + return { - testId: test.testId, - fileId: fileId, - location: this._relativeLocation(test.location), - title: test.title, - duration, - outcome: test.outcome, - ok: test.ok + testCase: { + testId: test.testId, + fileId, + title: test.title, + projectName, + location, + duration, + outcome: test.outcome, + path, + results: test.results.map(r => this._createTestResult(r)), + ok: test.outcome === 'expected' || test.outcome === 'flaky', + }, + testCaseSummary: { + testId: test.testId, + fileId, + title: test.title, + projectName, + location, + duration, + outcome: test.outcome, + path, + ok: test.outcome === 'expected' || test.outcome === 'flaky', + }, }; } @@ -346,18 +382,9 @@ const emptyStats = (): Stats => { unexpected: 0, flaky: 0, skipped: 0, - ok: true + ok: true, + duration: 0, }; }; -const addStats = (stats: Stats, delta: Stats): Stats => { - stats.total += delta.total; - stats.skipped += delta.skipped; - stats.expected += delta.expected; - stats.unexpected += delta.unexpected; - stats.flaky += delta.flaky; - stats.ok = stats.ok && delta.ok; - return stats; -}; - export default HtmlReporter; diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts index 9ecfaa123c..66145b6f19 100644 --- a/tests/playwright-test/reporter-html.spec.ts +++ b/tests/playwright-test/reporter-html.spec.ts @@ -59,90 +59,78 @@ test('should generate report', async ({ runInlineTest }, testInfo) => { }); `, }, { reporter: 'dot,html', retries: 1 }); - const report = testInfo.outputPath('playwright-report', 'data', 'projects.json'); + const report = testInfo.outputPath('playwright-report', 'data', 'report.json'); const reportObject = JSON.parse(fs.readFileSync(report, 'utf-8')); - delete reportObject[0].suites[0].duration; - delete reportObject[0].suites[0].location.line; - delete reportObject[0].suites[0].location.column; + delete reportObject.testIdToFileId; + delete reportObject.files[0].fileId; + delete reportObject.files[0].stats.duration; const fileNames = new Set(); - for (const test of reportObject[0].suites[0].tests) { + for (const test of reportObject.files[0].tests) { fileNames.add(testInfo.outputPath('playwright-report', 'data', test.fileId + '.json')); delete test.testId; delete test.fileId; delete test.location.line; delete test.location.column; delete test.duration; + delete test.path; } - expect(reportObject[0]).toEqual({ - name: 'project-name', - suites: [ + expect(reportObject).toEqual({ + files: [ { - title: 'a.test.js', - location: { - file: 'a.test.js' - }, + fileName: 'a.test.js', + tests: [ + { + title: 'fails', + projectName: 'project-name', + location: { + file: 'a.test.js' + }, + outcome: 'unexpected', + ok: false + }, + { + title: 'flaky', + projectName: 'project-name', + location: { + file: 'a.test.js' + }, + outcome: 'flaky', + ok: true + }, + { + title: 'passes', + projectName: 'project-name', + location: { + file: 'a.test.js' + }, + outcome: 'expected', + ok: true + }, + { + title: 'skip', + projectName: 'project-name', + location: { + file: 'a.test.js' + }, + outcome: 'skipped', + ok: false + } + ], stats: { total: 4, expected: 1, unexpected: 1, flaky: 1, skipped: 1, - ok: false - }, - suites: [], - tests: [ - { - location: { - file: 'a.test.js' - }, - title: 'passes', - outcome: 'expected', - ok: true - }, - { - location: { - file: 'a.test.js' - }, - title: 'fails', - outcome: 'unexpected', - ok: false - }, - { - location: { - file: 'a.test.js' - }, - title: 'skip', - outcome: 'skipped', - ok: true - }, - { - location: { - file: 'a.test.js' - }, - title: 'flaky', - outcome: 'flaky', - ok: true - } - ] + ok: false, + } } ], - stats: { - total: 4, - expected: 1, - unexpected: 1, - flaky: 1, - skipped: 1, - ok: false - } + projectNames: [ + 'project-name' + ] }); - - expect(fileNames.size).toBe(1); - const fileName = fileNames.values().next().value; - const testCase = JSON.parse(fs.readFileSync(fileName, 'utf-8')); - expect(testCase.tests).toHaveLength(4); - expect(testCase.tests.map(t => t.title)).toEqual(['passes', 'fails', 'skip', 'flaky']); - expect(testCase).toBeTruthy(); }); test('should not throw when attachment is missing', async ({ runInlineTest }) => { @@ -185,7 +173,6 @@ test('should include image diff', async ({ runInlineTest, page, showReport }) => expect(result.failed).toBe(1); await showReport(); - await page.click('text=a.test.js'); await page.click('text=fails'); const imageDiff = page.locator('.test-image-mismatch'); const image = imageDiff.locator('img'); @@ -221,7 +208,6 @@ test('should include screenshot on failure', async ({ runInlineTest, page, showR expect(result.failed).toBe(1); await showReport(); - await page.click('text=a.test.js'); await page.click('text=fails'); await expect(page.locator('text=Screenshots')).toBeVisible(); await expect(page.locator('img')).toBeVisible(); @@ -245,7 +231,6 @@ test('should include stdio', async ({ runInlineTest, page, showReport }) => { expect(result.failed).toBe(1); await showReport(); - await page.click('text=a.test.js'); await page.click('text=fails'); await page.locator('text=stdout').click(); await expect(page.locator('.attachment-body')).toHaveText('First line\nSecond line');