feat(test-result): render image diff (#8061)

src/test/reporters/html.ts

@@ -16,8 +16,9 @@
 
 import fs from 'fs';
 import path from 'path';
-import { Suite, TestError, TestStatus, Location, TestCase, TestResult, TestStep, FullConfig } from '../../../types/testReporter';
+import { FullConfig, Location, Suite, TestCase, TestError, TestResult, TestStatus, TestStep } from '../../../types/testReporter';
-import { BaseReporter, formatResultFailure } from './base';
+import { calculateFileSha1 } from '../../utils/utils';
+import { formatResultFailure } from './base';
 import { serializePatterns, toPosixPath } from './json';
 
 export type JsonStats = { expected: number, unexpected: number, flaky: number, skipped: number };
@@ -62,6 +63,22 @@ export type JsonTestCase = {
   outcome: 'skipped' | 'expected' | 'unexpected' | 'flaky';
 };
 
+export type TestAttachment = {
+  name: string;
+  path?: string;
+  body?: Buffer;
+  contentType: string;
+  sha1?: string;
+};
+
+export type JsonAttachment = {
+  name: string;
+  path?: string;
+  body?: string;
+  contentType: string;
+  sha1?: string;
+};
+
 export type JsonTestResult = {
   retry: number;
   workerIndex: number;
@@ -70,7 +87,7 @@ export type JsonTestResult = {
   status: TestStatus;
   error?: TestError;
   failureSnippet?: string;
-  attachments: { name: string, path?: string, body?: Buffer, contentType: string }[];
+  attachments: JsonAttachment[];
   stdout: (string | Buffer)[];
   stderr: (string | Buffer)[];
   steps: JsonTestStep[];
@@ -85,15 +102,30 @@ export type JsonTestStep = {
   steps: JsonTestStep[];
 };
 
-class HtmlReporter extends BaseReporter {
+class HtmlReporter {
-  async onEnd() {
+  private _targetFolder: string;
-    const targetFolder = process.env[`PLAYWRIGHT_HTML_REPORT`] || 'playwright-report';
+  private config!: FullConfig;
-    fs.mkdirSync(targetFolder, { recursive: true });
+  private suite!: Suite;
+
+  onBegin(config: FullConfig, suite: Suite) {
+    this.config = config;
+    this.suite = suite;
+  }
+
+  constructor() {
+    this._targetFolder = process.env[`PLAYWRIGHT_HTML_REPORT`] || 'playwright-report';
+    fs.mkdirSync(this._targetFolder, { recursive: true });
     const appFolder = path.join(__dirname, '..', '..', 'web', 'htmlReport');
     for (const file of fs.readdirSync(appFolder))
-      fs.copyFileSync(path.join(appFolder, file), path.join(targetFolder, file));
+      fs.copyFileSync(path.join(appFolder, file), path.join(this._targetFolder, file));
+  }
+
+  async onEnd() {
     const stats: JsonStats = { expected: 0, unexpected: 0, skipped: 0, flaky: 0 };
-    const reportFile = path.join(targetFolder, 'report.json');
+    this.suite.allTests().forEach(t => {
+      ++stats[t.outcome()];
+    });
+    const reportFile = path.join(this._targetFolder, 'report.json');
     const output: JsonReport = {
       config: {
         ...this.config,
@@ -113,7 +145,7 @@ class HtmlReporter extends BaseReporter {
         })
       },
       stats,
-      suites: this.suite.suites.map(s => this._serializeSuite(s))
+      suites: await Promise.all(this.suite.suites.map(s => this._serializeSuite(s)))
     };
     fs.writeFileSync(reportFile, JSON.stringify(output));
   }
@@ -128,16 +160,16 @@ class HtmlReporter extends BaseReporter {
     };
   }
 
-  private _serializeSuite(suite: Suite): JsonSuite {
+  private async _serializeSuite(suite: Suite): Promise<JsonSuite> {
     return {
       title: suite.title,
       location: this._relativeLocation(suite.location),
-      suites: suite.suites.map(s => this._serializeSuite(s)),
+      suites: await Promise.all(suite.suites.map(s => this._serializeSuite(s))),
-      tests: suite.tests.map(t => this._serializeTest(t)),
+      tests: await Promise.all(suite.tests.map(t => this._serializeTest(t))),
     };
   }
 
-  private _serializeTest(test: TestCase): JsonTestCase {
+  private async _serializeTest(test: TestCase): Promise<JsonTestCase> {
     return {
       title: test.title,
       location: this._relativeLocation(test.location),
@@ -147,11 +179,11 @@ class HtmlReporter extends BaseReporter {
       retries: test.retries,
       ok: test.ok(),
       outcome: test.outcome(),
-      results: test.results.map(r => this._serializeResult(test, r)),
+      results: await Promise.all(test.results.map(r => this._serializeResult(test, r))),
     };
   }
 
-  private _serializeResult(test: TestCase, result: TestResult): JsonTestResult {
+  private async _serializeResult(test: TestCase, result: TestResult): Promise<JsonTestResult> {
     return {
       retry: result.retry,
       workerIndex: result.workerIndex,
@@ -160,13 +192,31 @@ class HtmlReporter extends BaseReporter {
       status: result.status,
       error: result.error,
       failureSnippet: formatResultFailure(test, result, '').join('') || undefined,
-      attachments: result.attachments,
+      attachments: await this._copyAttachments(result.attachments),
       stdout: result.stdout,
       stderr: result.stderr,
       steps: this._serializeSteps(result.steps)
     };
   }
 
+  private async _copyAttachments(attachments: TestAttachment[]): Promise<JsonAttachment[]> {
+    const result: JsonAttachment[] = [];
+    for (const attachment of attachments) {
+      if (attachment.path) {
+        const sha1 = await calculateFileSha1(attachment.path) + extension(attachment.contentType);
+        fs.copyFileSync(attachment.path, path.join(this._targetFolder, sha1));
+        result.push({
+          ...attachment,
+          body: undefined,
+          sha1
+        });
+      } else if (attachment.body) {
+        result.push({ ...attachment, body: attachment.body.toString('base64') });
+      }
+    }
+    return result;
+  }
+
   private _serializeSteps(steps: TestStep[]): JsonTestStep[] {
     const stepStack: TestStep[] = [];
     const result: JsonTestStep[] = [];
@@ -205,4 +255,14 @@ function containsStep(outer: TestStep, inner: TestStep): boolean {
   return true;
 }
 
+function extension(contentType: string) {
+  if (contentType === 'image/png')
+    return '.png';
+  if (contentType === 'image/jpeg' || contentType === 'image/jpg')
+    return '.jpeg';
+  if (contentType === 'video/webm')
+    return '.webm';
+  return '.data';
+}
+
 export default HtmlReporter;

src/utils/utils.ts

@@ -16,6 +16,7 @@
 
 import path from 'path';
 import fs from 'fs';
+import stream from 'stream';
 import removeFolder from 'rimraf';
 import * as crypto from 'crypto';
 import os from 'os';
@@ -266,6 +267,30 @@ export function monotonicTime(): number {
   return seconds * 1000 + (nanoseconds / 1000 | 0) / 1000;
 }
 
+class HashStream extends stream.Writable {
+  private _hash = crypto.createHash('sha1');
+
+  _write(chunk: Buffer, encoding: string, done: () => void) {
+    this._hash.update(chunk);
+    done();
+  }
+
+  digest(): string {
+    return this._hash.digest('hex');
+  }
+}
+
+export async function calculateFileSha1(filename: string): Promise<string> {
+  const hashStream = new HashStream();
+  const stream = fs.createReadStream(filename);
+  stream.on('open', () => stream.pipe(hashStream));
+  await new Promise((f, r) => {
+    hashStream.on('finish', f);
+    hashStream.on('error', r);
+  });
+  return hashStream.digest();
+}
+
 export function calculateSha1(buffer: Buffer | string): string {
   const hash = crypto.createHash('sha1');
   hash.update(buffer);

src/web/htmlReport/htmlReport.css

@@ -116,4 +116,37 @@
   align-items: center;
   max-width: 450px;
   line-height: 24px;
-}
+}
+
+.awesome {
+  font-size: 24px;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  height: 100%;
+}
+
+.image-preview img {
+  max-width: 500px;
+  max-height: 500px;
+}
+
+.test-result .tabbed-pane {
+  margin-top: 50px;
+  width: 550px;
+}
+
+.image-preview {
+  position: relative;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  width: 550px;
+  height: 550px;
+}
+
+.test-result .tabbed-pane .tab-content {
+  display: flex;
+  align-items: center;
+  justify-content: center;
+}

src/web/htmlReport/htmlReport.tsx

@@ -20,7 +20,7 @@ import { SplitView } from '../components/splitView';
 import { TreeItem } from '../components/treeItem';
 import { TabbedPane } from '../traceViewer/ui/tabbedPane';
 import ansi2html from 'ansi-to-html';
-import { JsonLocation, JsonReport, JsonSuite, JsonTestCase, JsonTestResult, JsonTestStep } from '../../test/reporters/html';
+import type { JsonAttachment, JsonLocation, JsonReport, JsonSuite, JsonTestCase, JsonTestResult, JsonTestStep } from '../../test/reporters/html';
 import { msToString } from '../uiUtils';
 
 type Filter = 'Failing' | 'All';
@@ -38,11 +38,15 @@ export const Report: React.FC = () => {
   }, []);
   const [filter, setFilter] = React.useState<Filter>('Failing');
 
-  const failingTests = React.useMemo(() => {
+  const { unexpectedTests, unexpectedTestCount } = React.useMemo(() => {
-    const map = new Map<JsonSuite, JsonTestCase[]>();
+    const unexpectedTests = new Map<JsonSuite, JsonTestCase[]>();
-    for (const project of report?.suites || [])
+    let unexpectedTestCount = 0;
-      map.set(project, computeFailingTests(project));
+    for (const project of report?.suites || []) {
-    return map;
+      const unexpected = computeUnexpectedTests(project);
+      unexpectedTestCount += unexpected.length;
+      unexpectedTests.set(project, unexpected);
+    }
+    return { unexpectedTests, unexpectedTestCount };
   }, [report]);
 
   return <div className='hbox'>
@@ -51,10 +55,11 @@ export const Report: React.FC = () => {
       <TestCaseView test={selectedTest}></TestCaseView>
       <div className='suite-tree'>
         {filter === 'All' && report?.suites.map((s, i) => <ProjectTreeItem key={i} suite={s} setSelectedTest={setSelectedTest} selectedTest={selectedTest}></ProjectTreeItem>)}
-        {filter === 'Failing' && report?.suites.map((s, i) => {
+        {filter === 'Failing' && !!unexpectedTestCount && report?.suites.map((s, i) => {
-          const hasFailingTests = !!failingTests.get(s)?.length;
+          const hasUnexpectedOutcomes = !!unexpectedTests.get(s)?.length;
-          return hasFailingTests && <ProjectFlatTreeItem key={i} suite={s} setSelectedTest={setSelectedTest} selectedTest={selectedTest} failingTests={failingTests.get(s)!}></ProjectFlatTreeItem>;
+          return hasUnexpectedOutcomes && <ProjectFlatTreeItem key={i} suite={s} setSelectedTest={setSelectedTest} selectedTest={selectedTest} unexpectedTests={unexpectedTests.get(s)!}></ProjectFlatTreeItem>;
         })}
+        {filter === 'Failing' && !unexpectedTestCount && <div className='awesome'>You are awesome!</div>}
       </div>
     </SplitView>
   </div>;
@@ -81,31 +86,31 @@ const ProjectTreeItem: React.FC<{
   selectedTest?: JsonTestCase,
   setSelectedTest: (test: JsonTestCase) => void;
 }> = ({ suite, setSelectedTest, selectedTest }) => {
-  const location = renderLocation(suite?.location);
+  const location = renderLocation(suite?.location, true);
 
   return <TreeItem title={<div className='hbox'>
     <div style={{ flex: 'auto', alignItems: 'center', display: 'flex' }}>{testSuiteErrorStatusIcon(suite) || statusIcon('passed')}<div style={{ overflow: 'hidden', textOverflow: 'ellipsis' }}>{suite?.title}</div></div>
     {!!suite?.location?.line && location && <div style={{ flex: 'none', padding: '0 4px', color: '#666' }}>{location}</div>}
   </div>
   } loadChildren={() => {
-    return suite?.suites.map((s, i) => <SuiteTreeItem key={i} suite={s} setSelectedTest={setSelectedTest} selectedTest={selectedTest} depth={1}></SuiteTreeItem>) || [];
+    return suite?.suites.map((s, i) => <SuiteTreeItem key={i} suite={s} setSelectedTest={setSelectedTest} selectedTest={selectedTest} depth={1} showFileName={true}></SuiteTreeItem>) || [];
   }} depth={0} expandByDefault={true}></TreeItem>;
 };
 
 const ProjectFlatTreeItem: React.FC<{
   suite?: JsonSuite;
-  failingTests: JsonTestCase[],
+  unexpectedTests: JsonTestCase[],
   selectedTest?: JsonTestCase,
   setSelectedTest: (test: JsonTestCase) => void;
-}> = ({ suite, setSelectedTest, selectedTest, failingTests }) => {
+}> = ({ suite, setSelectedTest, selectedTest, unexpectedTests }) => {
-  const location = renderLocation(suite?.location);
+  const location = renderLocation(suite?.location, true);
 
   return <TreeItem title={<div className='hbox'>
     <div style={{ flex: 'auto', alignItems: 'center', display: 'flex' }}>{testSuiteErrorStatusIcon(suite) || statusIcon('passed')}<div style={{ overflow: 'hidden', textOverflow: 'ellipsis' }}>{suite?.title}</div></div>
     {!!suite?.location?.line && location && <div style={{ flex: 'none', padding: '0 4px', color: '#666' }}>{location}</div>}
   </div>
   } loadChildren={() => {
-    return failingTests.map((t, i) => <TestTreeItem key={i} test={t} setSelectedTest={setSelectedTest} selectedTest={selectedTest} showFileName={false} depth={1}></TestTreeItem>) || [];
+    return unexpectedTests.map((t, i) => <TestTreeItem key={i} test={t} setSelectedTest={setSelectedTest} selectedTest={selectedTest} showFileName={false} depth={1}></TestTreeItem>) || [];
   }} depth={0} expandByDefault={true}></TreeItem>;
 };
 
@@ -114,14 +119,15 @@ const SuiteTreeItem: React.FC<{
   selectedTest?: JsonTestCase,
   setSelectedTest: (test: JsonTestCase) => void;
   depth: number,
-}> = ({ suite, setSelectedTest, selectedTest, depth }) => {
+  showFileName: boolean,
-  const location = renderLocation(suite?.location);
+}> = ({ suite, setSelectedTest, selectedTest, showFileName, depth }) => {
+  const location = renderLocation(suite?.location, showFileName);
   return <TreeItem title={<div className='hbox'>
     <div style={{ flex: 'auto', alignItems: 'center', display: 'flex' }}>{testSuiteErrorStatusIcon(suite) || statusIcon('passed')}<div style={{ overflow: 'hidden', textOverflow: 'ellipsis' }}>{suite?.title}</div></div>
     {!!suite?.location?.line && location && <div style={{ flex: 'none', padding: '0 4px', color: '#666' }}>{location}</div>}
   </div>
   } loadChildren={() => {
-    const suiteChildren = suite?.suites.map((s, i) => <SuiteTreeItem key={i} suite={s} setSelectedTest={setSelectedTest} selectedTest={selectedTest} depth={depth + 1}></SuiteTreeItem>) || [];
+    const suiteChildren = suite?.suites.map((s, i) => <SuiteTreeItem key={i} suite={s} setSelectedTest={setSelectedTest} selectedTest={selectedTest} depth={depth + 1} showFileName={false}></SuiteTreeItem>) || [];
     const testChildren = suite?.tests.map((t, i) => <TestTreeItem key={i} test={t} setSelectedTest={setSelectedTest} selectedTest={selectedTest} showFileName={false} depth={depth + 1}></TestTreeItem>) || [];
     return [...suiteChildren, ...testChildren];
   }} depth={depth}></TreeItem>;
@@ -163,14 +169,21 @@ const TestOverview: React.FC<{
   test: JsonTestCase,
   result: JsonTestResult,
 }> = ({ test, result }) => {
+  const { attachments, screenshots } = React.useMemo(() => {
+    const attachments = new Map<string, JsonAttachment>();
+    const screenshots = result.attachments.filter(a => a.name === 'actual');
+    for (const a of result.attachments)
+      attachments.set(a.name, a);
+    return { attachments, screenshots };
+  }, [ result ]);
   return <div className="test-result">
     <div className='test-overview-title'>{test?.title}</div>
-    <div className='test-overview-property'>{renderLocation(test.location)}<div style={{ flex: 'auto' }}></div><div>{msToString(result.duration)}</div></div>
+    <div className='test-overview-property'>{renderLocation(test.location, true)}<div style={{ flex: 'auto' }}></div><div>{msToString(result.duration)}</div></div>
-    { result.failureSnippet && <div className='error-message' dangerouslySetInnerHTML={{__html: new ansi2html({
+    {result.failureSnippet && <div className='error-message' dangerouslySetInnerHTML={{ __html: new ansi2html({ colors: ansiColors }).toHtml(result.failureSnippet.trim()) }}></div>}
-      colors: ansiColors
+    {result.steps.map((step, i) => <StepTreeItem key={i} step={step} depth={0}></StepTreeItem>)}
-    }).toHtml(result.failureSnippet.trim()) }}></div> }
+    {attachments.has('expected') && attachments.has('actual') && <ImageDiff actual={attachments.get('actual')!} expected={attachments.get('expected')!} diff={attachments.get('diff')}></ImageDiff>}
-    { result.steps.map((step, i) => <StepTreeItem key={i} step={step} depth={0}></StepTreeItem>) }
+    {!!screenshots.length && <div className='test-overview-title'>Screenshots</div>}
-    {/* <div style={{whiteSpace: 'pre'}}>{ JSON.stringify(result.steps, undefined, 2) }</div> */}
+    {screenshots.map(a => <div className='image-preview'><img src={a.sha1} /></div>)}
   </div>;
 };
 
@@ -188,6 +201,36 @@ const StepTreeItem: React.FC<{
   } : undefined} depth={depth}></TreeItem>;
 };
 
+export const ImageDiff: React.FunctionComponent<{
+  actual: JsonAttachment,
+  expected: JsonAttachment,
+  diff?: JsonAttachment,
+}> = ({ actual, expected, diff }) => {
+  const [selectedTab, setSelectedTab] = React.useState<string>('actual');
+  const tabs = [];
+  tabs.push({
+    id: 'actual',
+    title: 'Actual',
+    render: () => <div className='image-preview'><img src={actual.sha1}/></div>
+  });
+  tabs.push({
+    id: 'expected',
+    title: 'Expected',
+    render: () => <div className='image-preview'><img src={expected.sha1}/></div>
+  });
+  if (diff) {
+    tabs.push({
+      id: 'diff',
+      title: 'Diff',
+      render: () => <div className='image-preview'><img src={diff.sha1}/></div>,
+    });
+  }
+  return <div className='vbox test-image-mismatch'>
+    <div className='test-overview-title'>Image mismatch</div>
+    <TabbedPane tabs={tabs} selectedTab={selectedTab} setSelectedTab={setSelectedTab} />
+  </div>;
+};
+
 function testSuiteErrorStatusIcon(suite?: JsonSuite): JSX.Element | undefined {
   if (!suite)
     return;
@@ -231,13 +274,13 @@ function statusIcon(status: 'failed' | 'timedOut' | 'skipped' | 'passed' | 'expe
   }
 }
 
-function computeFailingTests(suite: JsonSuite): JsonTestCase[] {
+function computeUnexpectedTests(suite: JsonSuite): JsonTestCase[] {
   const failedTests: JsonTestCase[] = [];
   const visit = (suite: JsonSuite) => {
     for (const child of suite.suites)
       visit(child);
     for (const test of suite.tests) {
-      if (test.results.find(r => r.status === 'failed' || r.status === 'timedOut'))
+      if (test.outcome !== 'expected' && test.outcome !== 'skipped')
         failedTests.push(test);
     }
   };
@@ -245,10 +288,10 @@ function computeFailingTests(suite: JsonSuite): JsonTestCase[] {
   return failedTests;
 }
 
-function renderLocation(location?: JsonLocation) {
+function renderLocation(location: JsonLocation | undefined, showFileName: boolean) {
   if (!location)
     return '';
-  return location.file + ':' + location.column;
+  return (showFileName ? location.file : '') + ':' + location.column;
 }
 
 function retryLabel(index: number) {

tests/config/browserTest.ts

@@ -135,7 +135,7 @@ export const playwrightFixtures: Fixtures<PlaywrightTestOptions & PlaywrightTest
     await run(contextOptions);
   },
 
-  contextFactory: async ({ browser, contextOptions }, run, testInfo) => {
+  contextFactory: async ({ browser, contextOptions, video }, run, testInfo) => {
     const contexts: BrowserContext[] = [];
     await run(async options => {
       const context = await browser.newContext({ ...contextOptions, ...options });
@@ -147,7 +147,16 @@ export const playwrightFixtures: Fixtures<PlaywrightTestOptions & PlaywrightTest
       contexts.push(context);
       return context;
     });
-    await Promise.all(contexts.map(context => context.close()));
+    await Promise.all(contexts.map(async context => {
+      const videos = context.pages().map(p => p.video()).filter(Boolean);
+      await context.close();
+      for (const v of videos) {
+        const videoPath = await v.path();
+        const savedPath = testInfo.outputPath(path.basename(videoPath));
+        await v.saveAs(savedPath);
+        testInfo.attachments.push({ name: 'video', path: savedPath, contentType: 'video/webm' });
+      }
+    }));
   },
 
   context: async ({ contextFactory }, run) => {