test: add screenshots to html tests (#10925)

packages/html-reporter/playwright.config.ts

@@ -21,6 +21,11 @@ const config: PlaywrightTestConfig = {
   snapshotDir: 'snapshots',
   forbidOnly: !!process.env.CI,
   retries: process.env.CI ? 2 : 0,
+  reporter: process.env.CI ? [
+    ['html', { open: 'never' }],
+  ] : [
+    ['html', { open: 'on-failure' }]
+  ],
   use: {
     trace: 'on-first-retry',
   },

packages/html-reporter/src/chip.spec.tsx

@@ -15,36 +15,36 @@
  */
 
 import React from 'react';
-import { test, expect } from '../test/componentTest';
+import { expect, test } from '../test/componentTest';
-import { Chip, AutoChip } from './chip';
+import { AutoChip, Chip } from './chip';
 
 test.use({ webpack: require.resolve('../webpack.config.js') });
 test.use({ viewport: { width: 500, height: 500 } });
 
-test('chip expand collapse', async ({ render }) => {
+test('expand collapse', async ({ render, capture }) => {
   const component = await render(<AutoChip header='title'>
     Chip body
   </AutoChip>);
   await expect(component.locator('text=Chip body')).toBeVisible();
-  // expect(await component.screenshot()).toMatchSnapshot('expanded.png');
+  await capture(component, 'expanded');
   await component.locator('text=Title').click();
   await expect(component.locator('text=Chip body')).not.toBeVisible();
-  // expect(await component.screenshot()).toMatchSnapshot('collapsed.png');
+  await capture(component, 'collapsed');
   await component.locator('text=Title').click();
   await expect(component.locator('text=Chip body')).toBeVisible();
-  // expect(await component.screenshot()).toMatchSnapshot('expanded.png');
 });
 
-test('chip render long title', async ({ render }) => {
+test('render long title', async ({ render, capture }) => {
   const title = 'Extremely long title. '.repeat(10);
   const component = await render(<AutoChip header={title}>
     Chip body
   </AutoChip>);
   await expect(component).toContainText('Extremely long title.');
   await expect(component.locator('text=Extremely long title.')).toHaveAttribute('title', title);
+  await capture(component, 'long-title');
 });
 
-test('chip setExpanded is called', async ({ render }) => {
+test('setExpanded is called', async ({ render, capture }) => {
   const expandedValues: boolean[] = [];
   const component = await render(<Chip header='Title'
     setExpanded={(expanded: boolean) => expandedValues.push(expanded)}>

packages/html-reporter/src/headerView.spec.tsx

@@ -21,7 +21,7 @@ import { HeaderView } from './headerView';
 test.use({ webpack: require.resolve('../webpack.config.js') });
 test.use({ viewport: { width: 720, height: 200 } });
 
-test('should render counters', async ({ render }) => {
+test('should render counters', async ({ render, capture }) => {
   const component = await render(<HeaderView stats={{
     total: 100,
     expected: 42,
@@ -36,6 +36,7 @@ test('should render counters', async ({ render }) => {
   await expect(component.locator('a', { hasText: 'Failed' }).locator('.counter')).toHaveText('31');
   await expect(component.locator('a', { hasText: 'Flaky' }).locator('.counter')).toHaveText('17');
   await expect(component.locator('a', { hasText: 'Skipped' }).locator('.counter')).toHaveText('10');
+  await capture(component, 'counters');
 });
 
 test('should toggle filters', async ({ page, render: render }) => {

packages/html-reporter/src/testCaseView.spec.tsx

@@ -59,8 +59,9 @@ const testCase: TestCase = {
   results: [result]
 };
 
-test('should render counters', async ({ render }) => {
+test('should render test case', async ({ render, capture }) => {
   const component = await render(<TestCaseView projectNames={['chromium', 'webkit']} test={testCase}></TestCaseView>);
+  await capture(component, 'testcase');
   await expect(component.locator('text=Annotation text').first()).toBeVisible();
   await component.locator('text=Annotations').click();
   await expect(component.locator('text=Annotation text')).not.toBeVisible();
@@ -70,4 +71,5 @@ test('should render counters', async ({ render }) => {
   await expect(component.locator('text=Inner step')).toBeVisible();
   await expect(component.locator('text=test.spec.ts:42')).toBeVisible();
   await expect(component.locator('text=My test')).toBeVisible();
+  await capture(component, 'testcase-expanded');
 });

packages/html-reporter/src/testResultView.tsx

@@ -26,6 +26,8 @@ import { AttachmentLink } from './links';
 import { statusIcon } from './statusIcon';
 import './testResultView.css';
 
+const imageDiffNames = ['expected', 'actual', 'diff'];
+
 export const TestResultView: React.FC<{
   test: TestCase,
   result: TestResult,
@@ -34,16 +36,13 @@ export const TestResultView: React.FC<{
   const { screenshots, videos, traces, otherAttachments, attachmentsMap } = React.useMemo(() => {
     const attachmentsMap = new Map<string, TestAttachment>();
     const attachments = result?.attachments || [];
-    const otherAttachments: TestAttachment[] = [];
+    const otherAttachments = new Set<TestAttachment>(attachments);
-    const screenshots = attachments.filter(a => a.name === 'screenshot');
+    const screenshots = attachments.filter(a => a.contentType.startsWith('image/') && !imageDiffNames.includes(a.name));
     const videos = attachments.filter(a => a.name === 'video');
     const traces = attachments.filter(a => a.name === 'trace');
-    const knownNames = new Set(['screenshot', 'image', 'expected', 'actual', 'diff', 'video', 'trace']);
+    for (const a of attachments)
-    for (const a of attachments) {
       attachmentsMap.set(a.name, a);
-      if (!knownNames.has(a.name))
+    [...screenshots, ...videos, ...traces].forEach(a => otherAttachments.delete(a));
-        otherAttachments.push(a);
-    }
     return { attachmentsMap, screenshots, videos, otherAttachments, traces };
   }, [ result ]);
 
@@ -92,8 +91,8 @@ export const TestResultView: React.FC<{
       </div>)}
     </AutoChip>}
 
-    {!!otherAttachments.length && <AutoChip header='Attachments'>
+    {!!otherAttachments.size && <AutoChip header='Attachments'>
-      {otherAttachments.map((a, i) => <AttachmentLink key={`attachment-link-${i}`} attachment={a}></AttachmentLink>)}
+      {[...otherAttachments].map((a, i) => <AttachmentLink key={`attachment-link-${i}`} attachment={a}></AttachmentLink>)}
     </AutoChip>}
   </div>;
 };

packages/html-reporter/test/componentTest.ts

@@ -25,6 +25,7 @@ declare global {
 
 type TestFixtures = {
   render: (component: { type: string, props: Object }) => Promise<Locator>;
+  capture: (locator: Locator, name: string) => Promise<void>;
   webpack: string;
 };
 
@@ -68,6 +69,18 @@ export const test = baseTest.extend<TestFixtures>({
       return page.locator('#pw-root');
     });
   },
+
+  capture: async ({}, use, testInfo) => {
+    await use(async (locator: Locator, name: string) => {
+      const screenshotPath = path.join(__dirname, '..', 'screenshots', sanitizeForFilePath(path.basename(testInfo.file) + '-' + testInfo.title + '-' + name) + '.png');
+      testInfo.attachments.push({ name, path: screenshotPath, contentType: 'image/png' });
+      await locator.screenshot({ path: screenshotPath });
+    });
+  }
 });
 
+export function sanitizeForFilePath(s: string) {
+  return s.replace(/[\x00-\x2C\x2E-\x2F\x3A-\x40\x5B-\x60\x7B-\x7F]+/g, '-');
+}
+
 export { expect } from '@playwright/test';