feat(test-result): render image diff (#8061)

This commit is contained in:
Pavel Feldman 2021-08-07 15:47:03 -07:00 committed by GitHub
parent ca22055045
commit 40fb9d85e0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 218 additions and 48 deletions

View file

@ -16,8 +16,9 @@
import fs from 'fs';
import path from 'path';
import { Suite, TestError, TestStatus, Location, TestCase, TestResult, TestStep, FullConfig } from '../../../types/testReporter';
import { BaseReporter, formatResultFailure } from './base';
import { FullConfig, Location, Suite, TestCase, TestError, TestResult, TestStatus, TestStep } from '../../../types/testReporter';
import { calculateFileSha1 } from '../../utils/utils';
import { formatResultFailure } from './base';
import { serializePatterns, toPosixPath } from './json';
export type JsonStats = { expected: number, unexpected: number, flaky: number, skipped: number };
@ -62,6 +63,22 @@ export type JsonTestCase = {
outcome: 'skipped' | 'expected' | 'unexpected' | 'flaky';
};
export type TestAttachment = {
name: string;
path?: string;
body?: Buffer;
contentType: string;
sha1?: string;
};
export type JsonAttachment = {
name: string;
path?: string;
body?: string;
contentType: string;
sha1?: string;
};
export type JsonTestResult = {
retry: number;
workerIndex: number;
@ -70,7 +87,7 @@ export type JsonTestResult = {
status: TestStatus;
error?: TestError;
failureSnippet?: string;
attachments: { name: string, path?: string, body?: Buffer, contentType: string }[];
attachments: JsonAttachment[];
stdout: (string | Buffer)[];
stderr: (string | Buffer)[];
steps: JsonTestStep[];
@ -85,15 +102,30 @@ export type JsonTestStep = {
steps: JsonTestStep[];
};
class HtmlReporter extends BaseReporter {
async onEnd() {
const targetFolder = process.env[`PLAYWRIGHT_HTML_REPORT`] || 'playwright-report';
fs.mkdirSync(targetFolder, { recursive: true });
class HtmlReporter {
private _targetFolder: string;
private config!: FullConfig;
private suite!: Suite;
onBegin(config: FullConfig, suite: Suite) {
this.config = config;
this.suite = suite;
}
constructor() {
this._targetFolder = process.env[`PLAYWRIGHT_HTML_REPORT`] || 'playwright-report';
fs.mkdirSync(this._targetFolder, { recursive: true });
const appFolder = path.join(__dirname, '..', '..', 'web', 'htmlReport');
for (const file of fs.readdirSync(appFolder))
fs.copyFileSync(path.join(appFolder, file), path.join(targetFolder, file));
fs.copyFileSync(path.join(appFolder, file), path.join(this._targetFolder, file));
}
async onEnd() {
const stats: JsonStats = { expected: 0, unexpected: 0, skipped: 0, flaky: 0 };
const reportFile = path.join(targetFolder, 'report.json');
this.suite.allTests().forEach(t => {
++stats[t.outcome()];
});
const reportFile = path.join(this._targetFolder, 'report.json');
const output: JsonReport = {
config: {
...this.config,
@ -113,7 +145,7 @@ class HtmlReporter extends BaseReporter {
})
},
stats,
suites: this.suite.suites.map(s => this._serializeSuite(s))
suites: await Promise.all(this.suite.suites.map(s => this._serializeSuite(s)))
};
fs.writeFileSync(reportFile, JSON.stringify(output));
}
@ -128,16 +160,16 @@ class HtmlReporter extends BaseReporter {
};
}
private _serializeSuite(suite: Suite): JsonSuite {
private async _serializeSuite(suite: Suite): Promise<JsonSuite> {
return {
title: suite.title,
location: this._relativeLocation(suite.location),
suites: suite.suites.map(s => this._serializeSuite(s)),
tests: suite.tests.map(t => this._serializeTest(t)),
suites: await Promise.all(suite.suites.map(s => this._serializeSuite(s))),
tests: await Promise.all(suite.tests.map(t => this._serializeTest(t))),
};
}
private _serializeTest(test: TestCase): JsonTestCase {
private async _serializeTest(test: TestCase): Promise<JsonTestCase> {
return {
title: test.title,
location: this._relativeLocation(test.location),
@ -147,11 +179,11 @@ class HtmlReporter extends BaseReporter {
retries: test.retries,
ok: test.ok(),
outcome: test.outcome(),
results: test.results.map(r => this._serializeResult(test, r)),
results: await Promise.all(test.results.map(r => this._serializeResult(test, r))),
};
}
private _serializeResult(test: TestCase, result: TestResult): JsonTestResult {
private async _serializeResult(test: TestCase, result: TestResult): Promise<JsonTestResult> {
return {
retry: result.retry,
workerIndex: result.workerIndex,
@ -160,13 +192,31 @@ class HtmlReporter extends BaseReporter {
status: result.status,
error: result.error,
failureSnippet: formatResultFailure(test, result, '').join('') || undefined,
attachments: result.attachments,
attachments: await this._copyAttachments(result.attachments),
stdout: result.stdout,
stderr: result.stderr,
steps: this._serializeSteps(result.steps)
};
}
private async _copyAttachments(attachments: TestAttachment[]): Promise<JsonAttachment[]> {
const result: JsonAttachment[] = [];
for (const attachment of attachments) {
if (attachment.path) {
const sha1 = await calculateFileSha1(attachment.path) + extension(attachment.contentType);
fs.copyFileSync(attachment.path, path.join(this._targetFolder, sha1));
result.push({
...attachment,
body: undefined,
sha1
});
} else if (attachment.body) {
result.push({ ...attachment, body: attachment.body.toString('base64') });
}
}
return result;
}
private _serializeSteps(steps: TestStep[]): JsonTestStep[] {
const stepStack: TestStep[] = [];
const result: JsonTestStep[] = [];
@ -205,4 +255,14 @@ function containsStep(outer: TestStep, inner: TestStep): boolean {
return true;
}
function extension(contentType: string) {
if (contentType === 'image/png')
return '.png';
if (contentType === 'image/jpeg' || contentType === 'image/jpg')
return '.jpeg';
if (contentType === 'video/webm')
return '.webm';
return '.data';
}
export default HtmlReporter;

View file

@ -16,6 +16,7 @@
import path from 'path';
import fs from 'fs';
import stream from 'stream';
import removeFolder from 'rimraf';
import * as crypto from 'crypto';
import os from 'os';
@ -266,6 +267,30 @@ export function monotonicTime(): number {
return seconds * 1000 + (nanoseconds / 1000 | 0) / 1000;
}
class HashStream extends stream.Writable {
private _hash = crypto.createHash('sha1');
_write(chunk: Buffer, encoding: string, done: () => void) {
this._hash.update(chunk);
done();
}
digest(): string {
return this._hash.digest('hex');
}
}
export async function calculateFileSha1(filename: string): Promise<string> {
const hashStream = new HashStream();
const stream = fs.createReadStream(filename);
stream.on('open', () => stream.pipe(hashStream));
await new Promise((f, r) => {
hashStream.on('finish', f);
hashStream.on('error', r);
});
return hashStream.digest();
}
export function calculateSha1(buffer: Buffer | string): string {
const hash = crypto.createHash('sha1');
hash.update(buffer);

View file

@ -116,4 +116,37 @@
align-items: center;
max-width: 450px;
line-height: 24px;
}
}
.awesome {
font-size: 24px;
display: flex;
align-items: center;
justify-content: center;
height: 100%;
}
.image-preview img {
max-width: 500px;
max-height: 500px;
}
.test-result .tabbed-pane {
margin-top: 50px;
width: 550px;
}
.image-preview {
position: relative;
display: flex;
align-items: center;
justify-content: center;
width: 550px;
height: 550px;
}
.test-result .tabbed-pane .tab-content {
display: flex;
align-items: center;
justify-content: center;
}

View file

@ -20,7 +20,7 @@ import { SplitView } from '../components/splitView';
import { TreeItem } from '../components/treeItem';
import { TabbedPane } from '../traceViewer/ui/tabbedPane';
import ansi2html from 'ansi-to-html';
import { JsonLocation, JsonReport, JsonSuite, JsonTestCase, JsonTestResult, JsonTestStep } from '../../test/reporters/html';
import type { JsonAttachment, JsonLocation, JsonReport, JsonSuite, JsonTestCase, JsonTestResult, JsonTestStep } from '../../test/reporters/html';
import { msToString } from '../uiUtils';
type Filter = 'Failing' | 'All';
@ -38,11 +38,15 @@ export const Report: React.FC = () => {
}, []);
const [filter, setFilter] = React.useState<Filter>('Failing');
const failingTests = React.useMemo(() => {
const map = new Map<JsonSuite, JsonTestCase[]>();
for (const project of report?.suites || [])
map.set(project, computeFailingTests(project));
return map;
const { unexpectedTests, unexpectedTestCount } = React.useMemo(() => {
const unexpectedTests = new Map<JsonSuite, JsonTestCase[]>();
let unexpectedTestCount = 0;
for (const project of report?.suites || []) {
const unexpected = computeUnexpectedTests(project);
unexpectedTestCount += unexpected.length;
unexpectedTests.set(project, unexpected);
}
return { unexpectedTests, unexpectedTestCount };
}, [report]);
return <div className='hbox'>
@ -51,10 +55,11 @@ export const Report: React.FC = () => {
<TestCaseView test={selectedTest}></TestCaseView>
<div className='suite-tree'>
{filter === 'All' && report?.suites.map((s, i) => <ProjectTreeItem key={i} suite={s} setSelectedTest={setSelectedTest} selectedTest={selectedTest}></ProjectTreeItem>)}
{filter === 'Failing' && report?.suites.map((s, i) => {
const hasFailingTests = !!failingTests.get(s)?.length;
return hasFailingTests && <ProjectFlatTreeItem key={i} suite={s} setSelectedTest={setSelectedTest} selectedTest={selectedTest} failingTests={failingTests.get(s)!}></ProjectFlatTreeItem>;
{filter === 'Failing' && !!unexpectedTestCount && report?.suites.map((s, i) => {
const hasUnexpectedOutcomes = !!unexpectedTests.get(s)?.length;
return hasUnexpectedOutcomes && <ProjectFlatTreeItem key={i} suite={s} setSelectedTest={setSelectedTest} selectedTest={selectedTest} unexpectedTests={unexpectedTests.get(s)!}></ProjectFlatTreeItem>;
})}
{filter === 'Failing' && !unexpectedTestCount && <div className='awesome'>You are awesome!</div>}
</div>
</SplitView>
</div>;
@ -81,31 +86,31 @@ const ProjectTreeItem: React.FC<{
selectedTest?: JsonTestCase,
setSelectedTest: (test: JsonTestCase) => void;
}> = ({ suite, setSelectedTest, selectedTest }) => {
const location = renderLocation(suite?.location);
const location = renderLocation(suite?.location, true);
return <TreeItem title={<div className='hbox'>
<div style={{ flex: 'auto', alignItems: 'center', display: 'flex' }}>{testSuiteErrorStatusIcon(suite) || statusIcon('passed')}<div style={{ overflow: 'hidden', textOverflow: 'ellipsis' }}>{suite?.title}</div></div>
{!!suite?.location?.line && location && <div style={{ flex: 'none', padding: '0 4px', color: '#666' }}>{location}</div>}
</div>
} loadChildren={() => {
return suite?.suites.map((s, i) => <SuiteTreeItem key={i} suite={s} setSelectedTest={setSelectedTest} selectedTest={selectedTest} depth={1}></SuiteTreeItem>) || [];
return suite?.suites.map((s, i) => <SuiteTreeItem key={i} suite={s} setSelectedTest={setSelectedTest} selectedTest={selectedTest} depth={1} showFileName={true}></SuiteTreeItem>) || [];
}} depth={0} expandByDefault={true}></TreeItem>;
};
const ProjectFlatTreeItem: React.FC<{
suite?: JsonSuite;
failingTests: JsonTestCase[],
unexpectedTests: JsonTestCase[],
selectedTest?: JsonTestCase,
setSelectedTest: (test: JsonTestCase) => void;
}> = ({ suite, setSelectedTest, selectedTest, failingTests }) => {
const location = renderLocation(suite?.location);
}> = ({ suite, setSelectedTest, selectedTest, unexpectedTests }) => {
const location = renderLocation(suite?.location, true);
return <TreeItem title={<div className='hbox'>
<div style={{ flex: 'auto', alignItems: 'center', display: 'flex' }}>{testSuiteErrorStatusIcon(suite) || statusIcon('passed')}<div style={{ overflow: 'hidden', textOverflow: 'ellipsis' }}>{suite?.title}</div></div>
{!!suite?.location?.line && location && <div style={{ flex: 'none', padding: '0 4px', color: '#666' }}>{location}</div>}
</div>
} loadChildren={() => {
return failingTests.map((t, i) => <TestTreeItem key={i} test={t} setSelectedTest={setSelectedTest} selectedTest={selectedTest} showFileName={false} depth={1}></TestTreeItem>) || [];
return unexpectedTests.map((t, i) => <TestTreeItem key={i} test={t} setSelectedTest={setSelectedTest} selectedTest={selectedTest} showFileName={false} depth={1}></TestTreeItem>) || [];
}} depth={0} expandByDefault={true}></TreeItem>;
};
@ -114,14 +119,15 @@ const SuiteTreeItem: React.FC<{
selectedTest?: JsonTestCase,
setSelectedTest: (test: JsonTestCase) => void;
depth: number,
}> = ({ suite, setSelectedTest, selectedTest, depth }) => {
const location = renderLocation(suite?.location);
showFileName: boolean,
}> = ({ suite, setSelectedTest, selectedTest, showFileName, depth }) => {
const location = renderLocation(suite?.location, showFileName);
return <TreeItem title={<div className='hbox'>
<div style={{ flex: 'auto', alignItems: 'center', display: 'flex' }}>{testSuiteErrorStatusIcon(suite) || statusIcon('passed')}<div style={{ overflow: 'hidden', textOverflow: 'ellipsis' }}>{suite?.title}</div></div>
{!!suite?.location?.line && location && <div style={{ flex: 'none', padding: '0 4px', color: '#666' }}>{location}</div>}
</div>
} loadChildren={() => {
const suiteChildren = suite?.suites.map((s, i) => <SuiteTreeItem key={i} suite={s} setSelectedTest={setSelectedTest} selectedTest={selectedTest} depth={depth + 1}></SuiteTreeItem>) || [];
const suiteChildren = suite?.suites.map((s, i) => <SuiteTreeItem key={i} suite={s} setSelectedTest={setSelectedTest} selectedTest={selectedTest} depth={depth + 1} showFileName={false}></SuiteTreeItem>) || [];
const testChildren = suite?.tests.map((t, i) => <TestTreeItem key={i} test={t} setSelectedTest={setSelectedTest} selectedTest={selectedTest} showFileName={false} depth={depth + 1}></TestTreeItem>) || [];
return [...suiteChildren, ...testChildren];
}} depth={depth}></TreeItem>;
@ -163,14 +169,21 @@ const TestOverview: React.FC<{
test: JsonTestCase,
result: JsonTestResult,
}> = ({ test, result }) => {
const { attachments, screenshots } = React.useMemo(() => {
const attachments = new Map<string, JsonAttachment>();
const screenshots = result.attachments.filter(a => a.name === 'actual');
for (const a of result.attachments)
attachments.set(a.name, a);
return { attachments, screenshots };
}, [ result ]);
return <div className="test-result">
<div className='test-overview-title'>{test?.title}</div>
<div className='test-overview-property'>{renderLocation(test.location)}<div style={{ flex: 'auto' }}></div><div>{msToString(result.duration)}</div></div>
{ result.failureSnippet && <div className='error-message' dangerouslySetInnerHTML={{__html: new ansi2html({
colors: ansiColors
}).toHtml(result.failureSnippet.trim()) }}></div> }
{ result.steps.map((step, i) => <StepTreeItem key={i} step={step} depth={0}></StepTreeItem>) }
{/* <div style={{whiteSpace: 'pre'}}>{ JSON.stringify(result.steps, undefined, 2) }</div> */}
<div className='test-overview-property'>{renderLocation(test.location, true)}<div style={{ flex: 'auto' }}></div><div>{msToString(result.duration)}</div></div>
{result.failureSnippet && <div className='error-message' dangerouslySetInnerHTML={{ __html: new ansi2html({ colors: ansiColors }).toHtml(result.failureSnippet.trim()) }}></div>}
{result.steps.map((step, i) => <StepTreeItem key={i} step={step} depth={0}></StepTreeItem>)}
{attachments.has('expected') && attachments.has('actual') && <ImageDiff actual={attachments.get('actual')!} expected={attachments.get('expected')!} diff={attachments.get('diff')}></ImageDiff>}
{!!screenshots.length && <div className='test-overview-title'>Screenshots</div>}
{screenshots.map(a => <div className='image-preview'><img src={a.sha1} /></div>)}
</div>;
};
@ -188,6 +201,36 @@ const StepTreeItem: React.FC<{
} : undefined} depth={depth}></TreeItem>;
};
export const ImageDiff: React.FunctionComponent<{
actual: JsonAttachment,
expected: JsonAttachment,
diff?: JsonAttachment,
}> = ({ actual, expected, diff }) => {
const [selectedTab, setSelectedTab] = React.useState<string>('actual');
const tabs = [];
tabs.push({
id: 'actual',
title: 'Actual',
render: () => <div className='image-preview'><img src={actual.sha1}/></div>
});
tabs.push({
id: 'expected',
title: 'Expected',
render: () => <div className='image-preview'><img src={expected.sha1}/></div>
});
if (diff) {
tabs.push({
id: 'diff',
title: 'Diff',
render: () => <div className='image-preview'><img src={diff.sha1}/></div>,
});
}
return <div className='vbox test-image-mismatch'>
<div className='test-overview-title'>Image mismatch</div>
<TabbedPane tabs={tabs} selectedTab={selectedTab} setSelectedTab={setSelectedTab} />
</div>;
};
function testSuiteErrorStatusIcon(suite?: JsonSuite): JSX.Element | undefined {
if (!suite)
return;
@ -231,13 +274,13 @@ function statusIcon(status: 'failed' | 'timedOut' | 'skipped' | 'passed' | 'expe
}
}
function computeFailingTests(suite: JsonSuite): JsonTestCase[] {
function computeUnexpectedTests(suite: JsonSuite): JsonTestCase[] {
const failedTests: JsonTestCase[] = [];
const visit = (suite: JsonSuite) => {
for (const child of suite.suites)
visit(child);
for (const test of suite.tests) {
if (test.results.find(r => r.status === 'failed' || r.status === 'timedOut'))
if (test.outcome !== 'expected' && test.outcome !== 'skipped')
failedTests.push(test);
}
};
@ -245,10 +288,10 @@ function computeFailingTests(suite: JsonSuite): JsonTestCase[] {
return failedTests;
}
function renderLocation(location?: JsonLocation) {
function renderLocation(location: JsonLocation | undefined, showFileName: boolean) {
if (!location)
return '';
return location.file + ':' + location.column;
return (showFileName ? location.file : '') + ':' + location.column;
}
function retryLabel(index: number) {

View file

@ -135,7 +135,7 @@ export const playwrightFixtures: Fixtures<PlaywrightTestOptions & PlaywrightTest
await run(contextOptions);
},
contextFactory: async ({ browser, contextOptions }, run, testInfo) => {
contextFactory: async ({ browser, contextOptions, video }, run, testInfo) => {
const contexts: BrowserContext[] = [];
await run(async options => {
const context = await browser.newContext({ ...contextOptions, ...options });
@ -147,7 +147,16 @@ export const playwrightFixtures: Fixtures<PlaywrightTestOptions & PlaywrightTest
contexts.push(context);
return context;
});
await Promise.all(contexts.map(context => context.close()));
await Promise.all(contexts.map(async context => {
const videos = context.pages().map(p => p.video()).filter(Boolean);
await context.close();
for (const v of videos) {
const videoPath = await v.path();
const savedPath = testInfo.outputPath(path.basename(videoPath));
await v.saveAs(savedPath);
testInfo.attachments.push({ name: 'video', path: savedPath, contentType: 'video/webm' });
}
}));
},
context: async ({ contextFactory }, run) => {