feat(html): restore trace, video, screenshot (#8925)

2021-09-14 16:26:31 -07:00 · 2021-09-14 16:26:31 -07:00 · 5253a7eb54
parent e641bf2bed
commit 5253a7eb54
6 changed files with 237 additions and 50 deletions
--- a/src/test/reporters/html.ts
+++ b/src/test/reporters/html.ts
@ -22,7 +22,7 @@ import { FullConfig, Suite } from '../../../types/testReporter';
 import { HttpServer } from '../../utils/httpServer';
 import { calculateSha1, removeFolders } from '../../utils/utils';
 import { toPosixPath } from '../reporters/json';
-import RawReporter, { JsonReport, JsonSuite, JsonTestCase, JsonTestResult, JsonTestStep } from './raw';
+import RawReporter, { JsonReport, JsonSuite, JsonTestCase, JsonTestResult, JsonTestStep, JsonAttachment } from './raw';

 export type Stats = {
  total: number;
@ -64,6 +64,8 @@ export type TestTreeItem = {
  ok: boolean;
 };

+export type TestAttachment = JsonAttachment;
+
 export type TestFile = {
  fileId: string;
  path: string;
@ -83,6 +85,7 @@ export type TestResult = {
  duration: number;
  steps: TestStep[];
  error?: string;
+  attachments: TestAttachment[];
  status: 'passed' | 'failed' | 'timedOut' | 'skipped';
 };

@ -115,7 +118,7 @@ class HtmlReporter {
    await removeFolders([reportFolder]);
    new HtmlBuilder(reports, reportFolder, this.config.rootDir);

-    if (!process.env.CI) {
+    if (!process.env.CI && !process.env.PWTEST_SKIP_TEST_OUTPUT) {
      const server = new HttpServer();
      server.routePrefix('/', (request, response) => {
        let relativePath = request.url!;
@ -139,12 +142,13 @@ class HtmlBuilder {
  private _reportFolder: string;
  private _tests = new Map<string, JsonTestCase>();
  private _rootDir: string;
+  private _dataFolder: string;

  constructor(rawReports: JsonReport[], outputDir: string, rootDir: string) {
    this._rootDir = rootDir;
    this._reportFolder = path.resolve(process.cwd(), outputDir);
-    const dataFolder = path.join(this._reportFolder, 'data');
-    fs.mkdirSync(dataFolder, { recursive: true });
+    this._dataFolder = path.join(this._reportFolder, 'data');
+    fs.mkdirSync(this._dataFolder, { recursive: true });
    const appFolder = path.join(__dirname, '..', '..', 'web', 'htmlReport');
    for (const file of fs.readdirSync(appFolder))
      fs.copyFileSync(path.join(appFolder, file), path.join(this._reportFolder, file));
@ -162,7 +166,7 @@ class HtmlBuilder {
          path: relativeFileName,
          tests: tests.map(t => this._createTestCase(t))
        };
-        fs.writeFileSync(path.join(dataFolder, fileId + '.json'), JSON.stringify(testFile, undefined, 2));
+        fs.writeFileSync(path.join(this._dataFolder, fileId + '.json'), JSON.stringify(testFile, undefined, 2));
      }
      projects.push({
        name: projectJson.project.name,
@ -170,7 +174,7 @@ class HtmlBuilder {
        stats: suites.reduce((a, s) => addStats(a, s.stats), emptyStats()),
      });
    }
-    fs.writeFileSync(path.join(dataFolder, 'projects.json'), JSON.stringify(projects, undefined, 2));
+    fs.writeFileSync(path.join(this._dataFolder, 'projects.json'), JSON.stringify(projects, undefined, 2));
  }

  private _createTestCase(test: JsonTestCase): TestCase {
@ -178,7 +182,7 @@ class HtmlBuilder {
      testId: test.testId,
      title: test.title,
      location: this._relativeLocation(test.location),
-      results: test.results.map(r => this._createTestResult(r))
+      results: test.results.map(r => this._createTestResult(test, r))
    };
  }

@ -190,6 +194,8 @@ class HtmlBuilder {
    for (const test of tests) {
      if (test.outcome === 'expected')
        ++stats.expected;
+      if (test.outcome === 'skipped')
+        ++stats.skipped;
      if (test.outcome === 'unexpected')
        ++stats.unexpected;
      if (test.outcome === 'flaky')
@ -221,7 +227,7 @@ class HtmlBuilder {
    };
  }

-  private _createTestResult(result: JsonTestResult): TestResult {
+  private _createTestResult(test: JsonTestCase, result: JsonTestResult): TestResult {
    return {
      duration: result.duration,
      startTime: result.startTime,
@ -229,6 +235,22 @@ class HtmlBuilder {
      steps: result.steps.map(s => this._createTestStep(s)),
      error: result.error,
      status: result.status,
+      attachments: result.attachments.map(a => {
+        if (a.path) {
+          const fileName = 'data/' + test.testId + path.extname(a.path);
+          try {
+            fs.copyFileSync(a.path, path.join(this._reportFolder, fileName));
+          } catch (e) {
+          }
+          return {
+            name: a.name,
+            contentType: a.contentType,
+            path: fileName,
+            body: a.body,
+          };
+        }
+        return a;
+      })
    };
  }

--- a/src/test/reporters/raw.ts
+++ b/src/test/reporters/raw.ts
@ -67,13 +67,6 @@ export type JsonTestCase = {
  outcome: 'skipped' | 'expected' | 'unexpected' | 'flaky';
 };

-export type TestAttachment = {
-  name: string;
-  path?: string;
-  body?: Buffer;
-  contentType: string;
-};
-
 export type JsonAttachment = {
  name: string;
  body?: string;
--- a/src/web/components/treeItem.tsx
+++ b/src/web/components/treeItem.tsx
@ -26,7 +26,7 @@ export const TreeItem: React.FunctionComponent<{
 }> = ({ title, loadChildren, onClick, expandByDefault, depth, selected }) => {
  const [expanded, setExpanded] = React.useState(expandByDefault || false);
  const className = selected ? 'tree-item-title selected' : 'tree-item-title';
-  return <div style={{ display: 'flex', flexDirection: 'column', width: '100%' }}>
+  return <div style={{ display: 'flex', flexDirection: 'column', width: '100%', flex: 'none' }}>
    <div className={className} style={{ display: 'flex', flexDirection: 'row', alignItems: 'center', whiteSpace: 'nowrap', paddingLeft: depth * 16 + 4 }} onClick={() => { onClick?.(); setExpanded(!expanded); }} >
      <div className={'codicon codicon-' + (expanded ? 'chevron-down' : 'chevron-right')}
        style={{ cursor: 'pointer', color: 'var(--color)', visibility: loadChildren ? 'visible' : 'hidden' }} />
--- a/src/web/htmlReport/htmlReport.css
+++ b/src/web/htmlReport/htmlReport.css
@ -14,6 +14,14 @@
  limitations under the License.
 */

+body {
+  --box-shadow-thick: rgb(0 0 0 / 10%) 0px 1.8px 1.9px,
+  rgb(0 0 0 / 15%) 0px 6.1px 6.3px,
+  rgb(0 0 0 / 10%) 0px -2px 4px,
+  rgb(0 0 0 / 15%) 0px -6.1px 12px,
+  rgb(0 0 0 / 25%) 0px 27px 28px;
+}
+
 .suite-tree-column {
  line-height: 18px;
  flex: auto;
@ -53,7 +61,7 @@
  overflow: auto;
  margin: 20px;
  flex: none;
-  box-shadow: var(--box-shadow);
+  box-shadow: var(--box-shadow-thick);
 }

 .status-icon {
@ -81,28 +89,16 @@
  flex: auto;
  display: flex;
  flex-direction: column;
+  max-width: 600px;
+  overflow: auto;
 }

 .test-overview-title {
-  padding: 10px 0;
+  padding: 30px 10px 10px;
  font-size: 18px;
  flex: none;
 }

-.image-preview img {
-  max-width: 500px;
-  max-height: 500px;
-}
-
-.image-preview {
-  position: relative;
-  display: flex;
-  align-items: center;
-  justify-content: center;
-  width: 550px;
-  height: 550px;
-}
-
 .test-result .tabbed-pane .tab-content {
  display: flex;
  align-items: center;
@ -130,6 +126,10 @@
  color: white !important;
 }

+.test-result > div {
+  flex: none;
+}
+
 .suite-tree-column .tab-strip,
 .test-case-column .tab-strip {
  border: none;
@ -185,10 +185,11 @@
 .stats {
  background-color: gray;
  border-radius: 2px;
-  min-width: 10px;
+  min-width: 14px;
  color: white;
  margin: 0 2px;
  padding: 0 2px;
+  text-align: center;
 }

 .stats.expected {
@ -202,3 +203,10 @@
 .stats.flaky {
  background-color: var(--yellow);
 }
+
+video, img {
+  flex: none;
+  box-shadow: var(--box-shadow-thick);
+  width: 80%;
+  margin: 20px auto;
+}
--- a/src/web/htmlReport/htmlReport.tsx
+++ b/src/web/htmlReport/htmlReport.tsx
@ -21,7 +21,7 @@ import { SplitView } from '../components/splitView';
 import { TreeItem } from '../components/treeItem';
 import { TabbedPane } from '../traceViewer/ui/tabbedPane';
 import { msToString } from '../uiUtils';
-import type { ProjectTreeItem, SuiteTreeItem, TestCase, TestResult, TestStep, TestTreeItem, Location, TestFile, Stats } from '../../test/reporters/html';
+import type { ProjectTreeItem, SuiteTreeItem, TestCase, TestResult, TestStep, TestTreeItem, Location, TestFile, Stats, TestAttachment } from '../../test/reporters/html';

 type Filter = 'Failing' | 'All';

@ -149,29 +149,45 @@ const TestCaseView: React.FC<{
  }

  const [selectedResultIndex, setSelectedResultIndex] = React.useState(0);
-  return <SplitView sidebarSize={500} orientation='horizontal' sidebarIsFirst={true}>
-    <div className='test-details-column vbox'>
-    </div>
-    <div className='test-case-column vbox'>
-      { test && <div className='test-case-title'>{test?.title}</div> }
-      { test && <div className='test-case-location'>{renderLocation(test.location, true)}</div> }
-      { test && <TabbedPane tabs={
-        test.results.map((result, index) => ({
-          id: String(index),
-          title: <div style={{ display: 'flex', alignItems: 'center' }}>{statusIcon(result.status)} {retryLabel(index)}</div>,
-          render: () => <TestResultView test={test!} result={result}></TestResultView>
-        })) || []} selectedTab={String(selectedResultIndex)} setSelectedTab={id => setSelectedResultIndex(+id)} />}
-    </div>
-  </SplitView>;
+  return <div className='test-case-column vbox'>
+    { test && <div className='test-case-title'>{test?.title}</div> }
+    { test && <div className='test-case-location'>{renderLocation(test.location, true)}</div> }
+    { test && <TabbedPane tabs={
+      test.results.map((result, index) => ({
+        id: String(index),
+        title: <div style={{ display: 'flex', alignItems: 'center' }}>{statusIcon(result.status)} {retryLabel(index)}</div>,
+        render: () => <TestResultView test={test!} result={result}></TestResultView>
+      })) || []} selectedTab={String(selectedResultIndex)} setSelectedTab={id => setSelectedResultIndex(+id)} />}
+  </div>;
 };

 const TestResultView: React.FC<{
  test: TestCase,
  result: TestResult,
-}> = ({ test, result }) => {
+}> = ({ result }) => {
+
+  const { screenshots, videos, attachmentsMap } = React.useMemo(() => {
+    const attachmentsMap = new Map<string, TestAttachment>();
+    const attachments = result?.attachments || [];
+    const screenshots = attachments.filter(a => a.name === 'screenshot');
+    const videos = attachments.filter(a => a.name === 'video');
+    for (const a of attachments)
+      attachmentsMap.set(a.name, a);
+    return { attachmentsMap, screenshots, videos };
+  }, [ result ]);
+
  return <div className='test-result'>
    {result.error && <ErrorMessage key={-1} error={result.error}></ErrorMessage>}
    {result.steps.map((step, i) => <StepTreeItem key={i} step={step} depth={0}></StepTreeItem>)}
+    {attachmentsMap.has('expected') && attachmentsMap.has('actual') && <ImageDiff actual={attachmentsMap.get('actual')!} expected={attachmentsMap.get('expected')!} diff={attachmentsMap.get('diff')}></ImageDiff>}
+    {!!screenshots && <div className='test-overview-title'>Screenshots</div>}
+    {screenshots.map((a, i) => <img key={`screenshot-${i}`} src={a.path} />)}
+    {!!videos.length && <div className='test-overview-title'>Videos</div>}
+    {videos.map((a, i) => <video key={`video-${i}`} controls>
+      <source src={a.path} type={a.contentType}/>
+    </video>)}
+    {!!result.attachments && <div className='test-overview-title'>Attachments</div>}
+    {result.attachments.map((a, i) => <AttachmentLink key={`attachment-${i}`} attachment={a}></AttachmentLink>)}
  </div>;
 };

@ -203,6 +219,48 @@ const StatsView: React.FC<{
  </div>;
 };

+export const AttachmentLink: React.FunctionComponent<{
+  attachment: TestAttachment,
+}> = ({ attachment }) => {
+  return <TreeItem title={<div style={{ display: 'flex', alignItems: 'center', flex: 'auto' }}>
+    <span className={'codicon codicon-cloud-download'}></span>
+    {attachment.path && <a href={attachment.path} target='_blank'>{attachment.name}</a>}
+    {attachment.body && <span>{attachment.name}</span>}
+  </div>} loadChildren={attachment.body ? () => {
+    return [<div className='attachment-body'>${attachment.body}</div>];
+  } : undefined} depth={0}></TreeItem>;
+};
+
+export const ImageDiff: React.FunctionComponent<{
+ actual: TestAttachment,
+ expected: TestAttachment,
+ diff?: TestAttachment,
+}> = ({ actual, expected, diff }) => {
+  const [selectedTab, setSelectedTab] = React.useState<string>('actual');
+  const tabs = [];
+  tabs.push({
+    id: 'actual',
+    title: 'Actual',
+    render: () => <div className='image-preview'><img src={actual.path}/></div>
+  });
+  tabs.push({
+    id: 'expected',
+    title: 'Expected',
+    render: () => <div className='image-preview'><img src={expected.path}/></div>
+  });
+  if (diff) {
+    tabs.push({
+      id: 'diff',
+      title: 'Diff',
+      render: () => <div className='image-preview'><img src={diff.path}/></div>,
+    });
+  }
+  return <div className='vbox test-image-mismatch'>
+    <div className='test-overview-title'>Image mismatch</div>
+    <TabbedPane tabs={tabs} selectedTab={selectedTab} setSelectedTab={setSelectedTab} />
+  </div>;
+};
+
 function statusIcon(status: 'failed' | 'timedOut' | 'skipped' | 'passed' | 'expected' | 'unexpected' | 'flaky'): JSX.Element {
  switch (status) {
    case 'failed':
--- a/tests/playwright-test/html-reporter.spec.ts
+++ b/tests/playwright-test/html-reporter.spec.ts
@ -14,11 +14,117 @@
 * limitations under the License.
 */

+import fs from 'fs';
+import path from 'path';
 import { test, expect } from './playwright-test-fixtures';
-import * as path from 'path';

 const kHTMLReporterPath = path.join(__dirname, '..', '..', 'lib', 'test', 'reporters', 'html.js');

+test('should generate report', async ({ runInlineTest }, testInfo) => {
+  await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { name: 'project-name' };
+    `,
+    'a.test.js': `
+      const { test } = pwt;
+      test('passes', async ({}) => {});
+      test('fails', async ({}) => {
+        expect(1).toBe(2);
+      });
+      test('skip', async ({}) => {
+        test.skip('Does not work')
+      });
+      test('flaky', async ({}, testInfo) => {
+        expect(testInfo.retry).toBe(1);
+      });
+    `,
+  }, { reporter: 'dot,' + kHTMLReporterPath, retries: 1 });
+  const report = testInfo.outputPath('playwright-report', 'data', 'projects.json');
+  const reportObject = JSON.parse(fs.readFileSync(report, 'utf-8'));
+  delete reportObject[0].suites[0].duration;
+  delete reportObject[0].suites[0].location.line;
+  delete reportObject[0].suites[0].location.column;
+
+  const fileNames = new Set<string>();
+  for (const test of reportObject[0].suites[0].tests) {
+    fileNames.add(testInfo.outputPath('playwright-report', 'data', test.fileId + '.json'));
+    delete test.testId;
+    delete test.fileId;
+    delete test.location.line;
+    delete test.location.column;
+    delete test.duration;
+  }
+  expect(reportObject[0]).toEqual({
+    name: 'project-name',
+    suites: [
+      {
+        title: 'a.test.js',
+        location: {
+          file: 'a.test.js'
+        },
+        stats: {
+          total: 4,
+          expected: 1,
+          unexpected: 1,
+          flaky: 1,
+          skipped: 1,
+          ok: false
+        },
+        suites: [],
+        tests: [
+          {
+            location: {
+              file: 'a.test.js'
+            },
+            title: 'passes',
+            outcome: 'expected',
+            ok: true
+          },
+          {
+            location: {
+              file: 'a.test.js'
+            },
+            title: 'fails',
+            outcome: 'unexpected',
+            ok: false
+          },
+          {
+            location: {
+              file: 'a.test.js'
+            },
+            title: 'skip',
+            outcome: 'skipped',
+            ok: true
+          },
+          {
+            location: {
+              file: 'a.test.js'
+            },
+            title: 'flaky',
+            outcome: 'flaky',
+            ok: true
+          }
+        ]
+      }
+    ],
+    stats: {
+      total: 4,
+      expected: 1,
+      unexpected: 1,
+      flaky: 1,
+      skipped: 1,
+      ok: false
+    }
+  });
+
+  expect(fileNames.size).toBe(1);
+  const fileName = fileNames.values().next().value;
+  const testCase = JSON.parse(fs.readFileSync(fileName, 'utf-8'));
+  expect(testCase.tests).toHaveLength(4);
+  expect(testCase.tests.map(t => t.title)).toEqual(['passes', 'fails', 'skip', 'flaky']);
+  expect(testCase).toBeTruthy();
+});
+
 test('should not throw when attachment is missing', async ({ runInlineTest }) => {
  const result = await runInlineTest({
    'playwright.config.ts': `