test(html-reporter): add image diff tests (#13262)

packages/html-reporter/src/imageDiffView.css

@@ -0,0 +1,30 @@
+/*
+  Copyright (c) Microsoft Corporation.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+.image-diff-view .tabbed-pane .tab-content {
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  position: relative;
+}
+
+.image-diff-view img {
+  flex: none;
+  box-shadow: var(--box-shadow-thick);
+  margin: 24px auto;
+  min-width: 200px;
+  max-width: 80%;
+}

packages/html-reporter/src/imageDiffView.spec.tsx

@@ -0,0 +1,82 @@
+/**
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from 'react';
+import { test, expect } from '@playwright/experimental-ct-react/test';
+import { ImageDiff, ImageDiffView } from './imageDiffView';
+
+test.use({ viewport: { width: 1000, height: 800 } });
+
+const expectedPng = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAQAAAD9CzEMAAACMElEQVRYw+1XT0tCQRD/9Qci0Cw7mp1C6BMYnt5niMhPEEFCh07evNk54XnuGkhFehA/QxHkqYMEFWXpscMTipri7fqeu+vbfY+EoBkQ3Zn5zTo7MzsL/NNfoClkUUQNN3jCJ/ETfavRSpYkkSmFQzz8wMr4gaSp8OBJ2HCU4Iwd0kqGgd9GPxCccZ+0jWgWVW1wxlWy0qR51I3hv7lOllq7b4SC/+aGzr+QBadjEKgAykvzJGXwr/Lj4JfRk5hUSLKIa00HPUJRki0xeMWSWxVXmi5sddXKymqTyxdwquXAUVV3WREeLx3gTcNFWQY/jXtB8QIzgt4qTvAR4OCe0ATKCmrnmFMEM0Pp2BvrIisaFUdUjgKKZgYWSjjDLR5J+x13lATHuHSti6JBzQP+gq2QHXjfRaiJojbPgYqbmGFow0VpiyIW0/VIF9QKLzeBWA2MHmwCu8QJQV++Ps/joHQQH4HpuO0uobUeVztgIcr4Vnf4we9orWfUIWKHbEVyYKkPmaVpIVKICuo0ZYXWjHTITXWhsVYxkIDpUoKsla1i2Oz2QjvYG9fshu36GbFQ8DGyHNOuvRdOKZSDUtCFM7wyHeSM4XN8e7bOpd9F2gg+TRYal753bGkbuEjzMg0YW/yDV1czUDm+e43Byz86OnRwsYDMKXlmkYbeAOwffrtU/nGpXpwkXfPhVza+D9AiMAtrtOMYfVr0q8Wr1nh8n8ADZCJPqAk8AifyjP2n36cvkA6/Wln9MokAAAAASUVORK5CYII=';
+const actualPng = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAQAAAD9CzEMAAACcElEQVRYw+2XQU9TQRDHfxAiYoFE8IaHhhjjN3hw4+SJk0npGU+iCQYv1kQjIQInj6+foh4amhjvEi8EI3yAIk24lBgoBkmqqYfuLvt23763r8QmJszc3sz+/29nZ3Zm4Vr+BxkgoESFPY7o0OGIPSqUCBi4OvgUmzToOLTBJlO9g08QcuEEl3pByEQv8Ascp4JLPWYhG/gQ5QjAGVWWmSXPTUa5xxxLfDJ2V2bIF36ELW3hASuMxfqNsiSOvatbjPj9fU0tabPOjUTvHG/4pfxrPrvQg7PqteNA20c5zbkYiWubgmZbZJcTzvjKGneMZP6m1hST4CdpGvmhU0zzXX0/5VFk5V21iyaTboJQwa7STqH4Y1AE6ixCd9XKxHsHFFIpTo1AvVal56juDeFQZxi8KNaMjJJh2oiDH+RQmFfUtzSKXQPjifh+yGBcsnWNrUhZJVO0DIxxFeTAJigJU9X4nkRxYqF8FL4lm6AiTMuW5bMzaXcs36fCs2IT7AvTjHNvNsVjy3dO+O3bBLLE8jF9oemsblPuq3KzRB7PrZhl8/z2pBhTteAkyMUunI+0HzdFzk0gwzDtvKde8SU2o3TJu0MkD3k28bYtpFDMuA/ZnaZZKJ6707SkuhJXoKi6Cy1QDT7XM8U4LfdVEXfZSXnAMz6wTZ1zzqmzrVWGTvEi6bK7vK4bWqO/zUtF7FJJMcxB0nWtN5y3omje89Nr8OpSrKc1HL1lBjzUGosPxWWTDX2a/o8M4FFNbPrm2NKLFrMMXtk1dfCKjo5ZteY3AEeHX3/1HH7jxne/4HiP7314gPTlCdWHR2BfnrHX8u/lL/ENCdIFFeD3AAAAAElFTkSuQmCC';
+
+const actualAttachment = { name: 'screenshot-actual.png', path: actualPng, contentType: 'image/png', };
+const expectedAttachment = { name: 'screenshot-expected.png', path: expectedPng, contentType: 'image/png', };
+const diffAttachment = { name: 'screenshot-diff.png', path: expectedPng, contentType: 'image/png', };
+
+const imageDiff: ImageDiff = {
+  name: 'log in',
+  actual: { attachment: actualAttachment },
+  expected: { attachment: expectedAttachment, title: 'Expected' },
+  diff: { attachment: diffAttachment },
+};
+
+test('should render links', async ({ mount }) => {
+  const component = await mount(<ImageDiffView key='image-diff' imageDiff={imageDiff}></ImageDiffView>);
+  await expect(component.locator('a')).toHaveText([
+    'screenshot-actual.png',
+    'screenshot-expected.png',
+    'screenshot-diff.png',
+  ]);
+});
+
+test('should show actual by default', async ({ mount }) => {
+  const component = await mount(<ImageDiffView key='image-diff' imageDiff={imageDiff}></ImageDiffView>);
+  const sliderElement = component.locator('data-testid=test-result-image-mismatch-grip');
+  await expect.poll(() => sliderElement.evaluate(e => e.style.left), 'Actual slider is on the right').toBe('611px');
+
+  const images = component.locator('img');
+  const imageCount = await component.locator('img').count();
+  for (let i = 0; i < imageCount; ++i) {
+    const image = images.nth(i);
+    const box = await image.boundingBox();
+    expect(box).toEqual({ x: 400, y: 80, width: 200, height: 200 });
+  }
+});
+
+test('should switch to expected', async ({ mount }) => {
+  const component = await mount(<ImageDiffView key='image-diff' imageDiff={imageDiff}></ImageDiffView>);
+  await component.locator('text="Expected"').click();
+  const sliderElement = component.locator('data-testid=test-result-image-mismatch-grip');
+  await expect.poll(() => sliderElement.evaluate(e => e.style.left), 'Expected slider is on the left').toBe('371px');
+
+  const images = component.locator('img');
+  const imageCount = await component.locator('img').count();
+  for (let i = 0; i < imageCount; ++i) {
+    const image = images.nth(i);
+    const box = await image.boundingBox();
+    expect(box).toEqual({ x: 400, y: 80, width: 200, height: 200 });
+  }
+});
+
+test('should switch to diff', async ({ mount }) => {
+  const component = await mount(<ImageDiffView key='image-diff' imageDiff={imageDiff}></ImageDiffView>);
+  await component.locator('text="Diff"').click();
+
+  const image = component.locator('img');
+  const box = await image.boundingBox();
+  expect(box).toEqual({ x: 400, y: 80, width: 200, height: 200 });
+});

packages/html-reporter/src/imageDiffView.tsx

@@ -18,12 +18,14 @@ import type { TestAttachment } from '@playwright-test/reporters/html';
 import * as React from 'react';
 import { AttachmentLink } from './links';
 import { TabbedPane, TabbedPaneTab } from './tabbedPane';
+import './imageDiffView.css';
+import './tabbedPane.css';
 
 export type ImageDiff = {
   name: string,
-  left?: { attachment: TestAttachment, title: string },
+  expected?: { attachment: TestAttachment, title: string },
-  right?: { attachment: TestAttachment, title: string },
+  actual?: { attachment: TestAttachment },
-  diff?: { attachment: TestAttachment, title: string },
+  diff?: { attachment: TestAttachment },
 };
 
 export const ImageDiffView: React.FunctionComponent<{
@@ -51,28 +53,28 @@ export const ImageDiffView: React.FunctionComponent<{
       id: 'actual',
       title: 'Actual',
       render: () => <ImageDiffSlider sliderPosition={sliderPosition} setSliderPosition={setSliderPosition}>
-        <img src={diff.left!.attachment.path!} onLoad={() => onImageLoaded('right')} ref={imageElement} />
+        <img src={diff.expected!.attachment.path!} onLoad={() => onImageLoaded('right')} ref={imageElement} />
-        <img src={diff.right!.attachment.path!} style={{ boxShadow: 'none' }} />
+        <img src={diff.actual!.attachment.path!} />
       </ImageDiffSlider>,
     });
     tabs.push({
       id: 'expected',
-      title: diff.right!.title,
+      title: diff.expected!.title,
       render: () => <ImageDiffSlider sliderPosition={sliderPosition} setSliderPosition={setSliderPosition}>
-        <img src={diff.left!.attachment.path!} onLoad={() => onImageLoaded('left')} ref={imageElement} />
+        <img src={diff.expected!.attachment.path!} onLoad={() => onImageLoaded('left')} ref={imageElement} />
-        <img src={diff.right!.attachment.path!} style={{ boxShadow: 'none' }} />
+        <img src={diff.actual!.attachment.path!} style={{ boxShadow: 'none' }} />
       </ImageDiffSlider>,
     });
   } else {
     tabs.push({
       id: 'actual',
       title: 'Actual',
-      render: () => <img src={diff.left!.attachment.path!} onLoad={() => onImageLoaded()} />
+      render: () => <img src={diff.actual!.attachment.path!} onLoad={() => onImageLoaded()} />
     });
     tabs.push({
       id: 'expected',
-      title: diff.right!.title,
+      title: diff.expected!.title,
-      render: () => <img src={diff.right!.attachment.path!} onLoad={() => onImageLoaded()} />
+      render: () => <img src={diff.expected!.attachment.path!} onLoad={() => onImageLoaded()} />
     });
   }
   if (diff.diff) {
@@ -82,10 +84,10 @@ export const ImageDiffView: React.FunctionComponent<{
       render: () => <img src={diff.diff!.attachment.path} onLoad={() => onImageLoaded()} />
     });
   }
-  return <div className='vbox' data-testid='test-result-image-mismatch' ref={diffElement}>
+  return <div className='vbox image-diff-view' data-testid='test-result-image-mismatch' ref={diffElement}>
     <TabbedPane tabs={tabs} selectedTab={selectedTab} setSelectedTab={setSelectedTab} />
-    <AttachmentLink attachment={diff.left!.attachment}></AttachmentLink>
+    <AttachmentLink attachment={diff.actual!.attachment}></AttachmentLink>
-    <AttachmentLink attachment={diff.right!.attachment}></AttachmentLink>
+    <AttachmentLink attachment={diff.expected!.attachment}></AttachmentLink>
     {diff.diff && <AttachmentLink attachment={diff.diff.attachment}></AttachmentLink>}
   </div>;
 };
@@ -112,7 +114,13 @@ export const ImageDiffSlider: React.FC<{
   return <>
     {childrenArray[0]}
     <div style={{ ...absolute }}>
-      <div style={{ ...absolute, display: 'flex', zIndex: 50, clip: `rect(0, ${size}px, auto, 0)` }}>
+      <div style={{
+        ...absolute,
+        display: 'flex',
+        zIndex: 50,
+        clip: `rect(0, ${size}px, auto, 0)`,
+        backgroundColor: 'var(--color-canvas-default)',
+      }}>
         {childrenArray[1]}
       </div>
       <div

packages/html-reporter/src/testResultView.css

@@ -21,13 +21,6 @@
   margin-bottom: 24px;
 }
 
-.test-result .tabbed-pane .tab-content {
-  display: flex;
-  align-items: center;
-  justify-content: center;
-  position: relative;
-}
-
 .test-result > div {
   flex: none;
 }

packages/html-reporter/src/testResultView.tsx

@@ -40,20 +40,20 @@ function groupImageDiffs(screenshots: Set<TestAttachment>): ImageDiff[] {
       snapshotNameToImageDiff.set(snapshotName, imageDiff);
     }
     if (category === 'actual')
-      imageDiff.left = { attachment, title: 'Actual' };
+      imageDiff.actual = { attachment };
     if (category === 'expected')
-      imageDiff.right = { attachment, title: 'Expected' };
+      imageDiff.expected = { attachment, title: 'Expected' };
     if (category === 'previous')
-      imageDiff.right = { attachment, title: 'Previous' };
+      imageDiff.expected = { attachment, title: 'Previous' };
     if (category === 'diff')
-      imageDiff.diff = { attachment, title: 'Diff' };
+      imageDiff.diff = { attachment };
   }
   for (const [name, diff] of snapshotNameToImageDiff) {
-    if (!diff.left || !diff.right) {
+    if (!diff.actual || !diff.expected) {
       snapshotNameToImageDiff.delete(name);
     } else {
-      screenshots.delete(diff.left.attachment);
+      screenshots.delete(diff.actual.attachment);
-      screenshots.delete(diff.right.attachment);
+      screenshots.delete(diff.expected.attachment);
       screenshots.delete(diff.diff?.attachment!);
     }
   }

packages/html-reporter/src/tests.ts

@@ -16,6 +16,7 @@
 
 import { AutoChip, Chip } from './chip';
 import { HeaderView } from './headerView';
+import { ImageDiffView } from './imageDiffView';
 import { TestCaseView } from './testCaseView';
 import './theme.css';
 
@@ -25,5 +26,6 @@ register({
   AutoChip,
   Chip,
   HeaderView,
+  ImageDiffView,
   TestCaseView,
 });

tests/playwright-test/reporter-html.spec.ts

@@ -158,18 +158,14 @@ test('should include image diff', async ({ runInlineTest, page, showReport }) =>
   expect(set.size, 'Should be two images overlaid').toBe(2);
 
   const sliderElement = imageDiff.locator('data-testid=test-result-image-mismatch-grip');
-  await expect.poll(async () => {
+  await expect.poll(() => sliderElement.evaluate(e => e.style.left), 'Actual slider is on the right').toBe('590px');
-    return await sliderElement.evaluate(e => e.style.left);
-  }, 'Actual slider is on the right').toBe('590px');
 
   await imageDiff.locator('text="Expected"').click();
   set.add(await expectedImage.getAttribute('src'));
   set.add(await actualImage.getAttribute('src'));
   expect(set.size).toBe(2);
 
-  await expect.poll(async () => {
+  await expect.poll(() => sliderElement.evaluate(e => e.style.left), 'Expected slider is on the left').toBe('350px');
-    return await sliderElement.evaluate(e => e.style.left);
-  }, 'Actual slider is on the right').toBe('350px');
 
   await imageDiff.locator('text="Diff"').click();
   set.add(await imageDiff.locator('img').getAttribute('src'));