inline diiff

.gitignore

@@ -35,3 +35,4 @@ test-results
 .cache/
 .eslintcache
 playwright.env
+firefox

packages/html-reporter/src/testErrorView.tsx

@@ -17,20 +17,38 @@
 import ansi2html from 'ansi-to-html';
 import * as React from 'react';
 import './testErrorView.css';
+import type { ImageDiff } from '@web/shared/imageDiffView';
+import { ImageDiffView } from '@web/shared/imageDiffView';
 
 export const TestErrorView: React.FC<{
   error: string;
 }> = ({ error }) => {
-  const html = React.useMemo(() => {
+  const html = React.useMemo(() => ansiErrorToHtml(error), [error]);
+  return <div className='test-error-message' dangerouslySetInnerHTML={{ __html: html || '' }}></div>;
+};
+
+export const TestScreenshotErrorView: React.FC<{
+  errorPrefix?: string,
+  diff: ImageDiff,
+  errorSuffix?: string,
+}> = ({ errorPrefix, diff, errorSuffix }) => {
+  const prefixHtml = React.useMemo(() => ansiErrorToHtml(errorPrefix), [errorPrefix]);
+  const suffixHtml = React.useMemo(() => ansiErrorToHtml(errorSuffix), [errorSuffix]);
+  return <div className='test-error-message'>
+      <div dangerouslySetInnerHTML={{ __html: prefixHtml || '' }}></div>
+      <ImageDiffView key='image-diff' diff={diff} hideDetails={true} ></ImageDiffView>
+      <div dangerouslySetInnerHTML={{ __html: suffixHtml || '' }}></div>
+    </div>
+};
+
+function ansiErrorToHtml(text?: string): string {
   const config: any = {
     bg: 'var(--color-canvas-subtle)',
     fg: 'var(--color-fg-default)',
   };
   config.colors = ansiColors;
-    return new ansi2html(config).toHtml(escapeHTML(error));
+  return new ansi2html(config).toHtml(escapeHTML(text || ''));
-  }, [error]);
+}
-  return <div className='test-error-message' dangerouslySetInnerHTML={{ __html: html || '' }}></div>;
-};
 
 const ansiColors = {
   0: '#000',

packages/html-reporter/src/testResultView.tsx

@@ -24,7 +24,7 @@ import { AttachmentLink, generateTraceUrl } from './links';
 import { statusIcon } from './statusIcon';
 import type { ImageDiff } from '@web/shared/imageDiffView';
 import { ImageDiffView } from '@web/shared/imageDiffView';
-import { TestErrorView } from './testErrorView';
+import { TestErrorView, TestScreenshotErrorView } from './testErrorView';
 import './testResultView.css';
 
 function groupImageDiffs(screenshots: Set<TestAttachment>): ImageDiff[] {
@@ -67,7 +67,7 @@ export const TestResultView: React.FC<{
   anchor: 'video' | 'diff' | '',
 }> = ({ result, anchor }) => {
 
-  const { screenshots, videos, traces, otherAttachments, diffs, htmls } = React.useMemo(() => {
+  const { screenshots, videos, traces, otherAttachments, diffs, errors, htmls } = React.useMemo(() => {
     const attachments = result?.attachments || [];
     const screenshots = new Set(attachments.filter(a => a.contentType.startsWith('image/')));
     const videos = attachments.filter(a => a.name === 'video');
@@ -76,7 +76,8 @@ export const TestResultView: React.FC<{
     const otherAttachments = new Set<TestAttachment>(attachments);
     [...screenshots, ...videos, ...traces, ...htmls].forEach(a => otherAttachments.delete(a));
     const diffs = groupImageDiffs(screenshots);
-    return { screenshots: [...screenshots], videos, traces, otherAttachments, diffs, htmls };
+    const errors = classifyErrors(result.errors, diffs);
+    return { screenshots: [...screenshots], videos, traces, otherAttachments, diffs, errors, htmls };
   }, [result]);
 
   const videoRef = React.useRef<HTMLDivElement>(null);
@@ -94,8 +95,12 @@ export const TestResultView: React.FC<{
   }, [scrolled, anchor, setScrolled, videoRef]);
 
   return <div className='test-result'>
-    {!!result.errors.length && <AutoChip header='Errors'>
+    {!!errors.length && <AutoChip header='Errors'>
-      {result.errors.map((error, index) => <TestErrorView key={'test-result-error-message-' + index} error={error}></TestErrorView>)}
+      {errors.map((error, index) => {
+        if (error.type === 'screenshot')
+          return <TestScreenshotErrorView errorPrefix={error.errorPrefix} diff={error.diff!} errorSuffix={error.errorSuffix}></TestScreenshotErrorView>;
+        return <TestErrorView key={'test-result-error-message-' + index} error={error.error!}></TestErrorView>
+        })}
     </AutoChip>}
     {!!result.steps.length && <AutoChip header='Test Steps'>
       {result.steps.map((step, i) => <StepTreeItem key={`step-${i}`} step={step} depth={0}></StepTreeItem>)}
@@ -145,6 +150,38 @@ export const TestResultView: React.FC<{
   </div>;
 };
 
+function classifyErrors(testErrors: string[], diffs: ImageDiff[]) {
+  const errors = [];
+  for (const error of testErrors) {
+    let screenshotError;
+    if (error.includes('Screenshot comparison failed:')) {
+      for (const diff of diffs) {
+          if (!diff.actual?.attachment.name)
+            continue;
+        if (!error.includes(diff.actual!.attachment.name))
+          continue;
+        const index = error.search(/Expected:|Previous:|Received:/);
+        let errorPrefix;
+        if (index !== -1)
+          errorPrefix = error.slice(0, index);
+        else
+          errorPrefix = error.split('\n')[0];
+
+        const callLog = error.indexOf('Call log:');
+        let errorSuffix;
+        if (callLog !== -1)
+          errorSuffix = error.slice(callLog);
+        screenshotError = { type: 'screenshot', diff, errorPrefix, errorSuffix };
+      }
+    }
+    if (screenshotError)
+      errors.push(screenshotError);
+    else
+      errors.push({ type: 'regular', error });
+  }
+  return errors;
+}
+
 const StepTreeItem: React.FC<{
   step: TestStep;
   depth: number,

packages/playwright-core/src/server/page.ts

@@ -43,7 +43,7 @@ import type { TimeoutOptions } from '../common/types';
 import { isInvalidSelectorError } from '../utils/isomorphic/selectorParser';
 import { parseEvaluationResultValue, source } from './isomorphic/utilityScriptSerializers';
 import type { SerializedValue } from './isomorphic/utilityScriptSerializers';
-import { TargetClosedError } from './errors';
+import { TargetClosedError, TimeoutError } from './errors';
 import { asLocator } from '../utils';
 import { helper } from './helper';
 
@@ -672,10 +672,13 @@ export class Page extends SdkObject {
       // A: We want user to receive a friendly diff between actual and expected/previous.
       if (js.isJavaScriptErrorInEvaluate(e) || isInvalidSelectorError(e))
         throw e;
+      let errorMessage = e.message;
+      if (e instanceof TimeoutError && intermediateResult?.previous)
+        errorMessage = `Failed to take two consecutive stable screenshots. ${e.message}`;
       return {
         log: e.message ? [...metadata.log, e.message] : metadata.log,
         ...intermediateResult,
-        errorMessage: e.message,
+        errorMessage,
       };
     });
   }

packages/web/src/shared/imageDiffView.tsx

@@ -61,7 +61,9 @@ const checkerboardStyle: React.CSSProperties = {
 export const ImageDiffView: React.FC<{
   diff: ImageDiff,
   noTargetBlank?: boolean,
-}> = ({ diff, noTargetBlank }) => {
+  hideDetails?: boolean,
+}> = ({ diff, noTargetBlank, hideDetails }) => {
+  console.log('hideDetails', hideDetails);
   const [mode, setMode] = React.useState<'diff' | 'actual' | 'expected' | 'slider' | 'sxs'>(diff.diff ? 'diff' : 'actual');
   const [showSxsDiff, setShowSxsDiff] = React.useState<boolean>(false);
 
@@ -105,26 +107,26 @@ export const ImageDiffView: React.FC<{
         <div style={{ ...modeStyle, fontWeight: mode === 'slider' ? 600 : 'initial' }} onClick={() => setMode('slider')}>Slider</div>
       </div>
       <div style={{ display: 'flex', justifyContent: 'center', flex: 'auto', minHeight: fitHeight + 60 }}>
-        {diff.diff && mode === 'diff' && <ImageWithSize image={diffImage} alt='Diff' canvasWidth={fitWidth} canvasHeight={fitHeight} scale={scale}/>}
+        {diff.diff && mode === 'diff' && <ImageWithSize image={diffImage} alt='Diff' hideSize={hideDetails} canvasWidth={fitWidth} canvasHeight={fitHeight} scale={scale}/>}
-        {diff.diff && mode === 'actual' && <ImageWithSize image={actualImage}  alt='Actual' canvasWidth={fitWidth} canvasHeight={fitHeight} scale={scale}/>}
+        {diff.diff && mode === 'actual' && <ImageWithSize image={actualImage}  alt='Actual' hideSize={hideDetails} canvasWidth={fitWidth} canvasHeight={fitHeight} scale={scale}/>}
-        {diff.diff && mode === 'expected' && <ImageWithSize image={expectedImage}  alt={expectedImageTitle} canvasWidth={fitWidth} canvasHeight={fitHeight} scale={scale}/>}
+        {diff.diff && mode === 'expected' && <ImageWithSize image={expectedImage}  alt={expectedImageTitle} hideSize={hideDetails} canvasWidth={fitWidth} canvasHeight={fitHeight} scale={scale}/>}
-        {diff.diff && mode === 'slider' && <ImageDiffSlider expectedImage={expectedImage} actualImage={actualImage} canvasWidth={fitWidth} canvasHeight={fitHeight} scale={scale} expectedTitle={expectedImageTitle} />}
+        {diff.diff && mode === 'slider' && <ImageDiffSlider expectedImage={expectedImage} actualImage={actualImage} hideSize={hideDetails} canvasWidth={fitWidth} canvasHeight={fitHeight} scale={scale} expectedTitle={expectedImageTitle} />}
         {diff.diff && mode === 'sxs' && <div style={{ display: 'flex' }}>
-          <ImageWithSize image={expectedImage} title={expectedImageTitle} canvasWidth={sxsScale * imageWidth} canvasHeight={sxsScale * imageHeight} scale={sxsScale} />
+          <ImageWithSize image={expectedImage} title={expectedImageTitle} hideSize={hideDetails} canvasWidth={sxsScale * imageWidth} canvasHeight={sxsScale * imageHeight} scale={sxsScale} />
-          <ImageWithSize image={showSxsDiff ? diffImage : actualImage} title={showSxsDiff ? 'Diff' : 'Actual'} onClick={() => setShowSxsDiff(!showSxsDiff)} canvasWidth={sxsScale * imageWidth} canvasHeight={sxsScale * imageHeight} scale={sxsScale} />
+          <ImageWithSize image={showSxsDiff ? diffImage : actualImage} title={showSxsDiff ? 'Diff' : 'Actual'} onClick={() => setShowSxsDiff(!showSxsDiff)} hideSize={hideDetails} canvasWidth={sxsScale * imageWidth} canvasHeight={sxsScale * imageHeight} scale={sxsScale} />
         </div>}
-        {!diff.diff && mode === 'actual' && <ImageWithSize image={actualImage} title='Actual' canvasWidth={fitWidth} canvasHeight={fitHeight} scale={scale}/>}
+        {!diff.diff && mode === 'actual' && <ImageWithSize image={actualImage} title='Actual' hideSize={hideDetails} canvasWidth={fitWidth} canvasHeight={fitHeight} scale={scale}/>}
-        {!diff.diff && mode === 'expected' && <ImageWithSize image={expectedImage} title={expectedImageTitle} canvasWidth={fitWidth} canvasHeight={fitHeight} scale={scale}/>}
+        {!diff.diff && mode === 'expected' && <ImageWithSize image={expectedImage} title={expectedImageTitle} hideSize={hideDetails} canvasWidth={fitWidth} canvasHeight={fitHeight} scale={scale}/>}
         {!diff.diff && mode === 'sxs' && <div style={{ display: 'flex' }}>
           <ImageWithSize image={expectedImage} title={expectedImageTitle} canvasWidth={sxsScale * imageWidth} canvasHeight={sxsScale * imageHeight} scale={sxsScale} />
           <ImageWithSize image={actualImage} title='Actual' canvasWidth={sxsScale * imageWidth} canvasHeight={sxsScale * imageHeight} scale={sxsScale} />
         </div>}
       </div>
-      <div style={{ alignSelf: 'start', lineHeight: '18px', marginLeft: '15px' }}>
+      {!hideDetails && <div style={{ alignSelf: 'start', lineHeight: '18px', marginLeft: '15px' }}>
         <div>{diff.diff && <a target='_blank' href={diff.diff.attachment.path} rel='noreferrer'>{diff.diff.attachment.name}</a>}</div>
         <div><a target={noTargetBlank ? '' : '_blank'} href={diff.actual!.attachment.path} rel='noreferrer'>{diff.actual!.attachment.name}</a></div>
         <div><a target={noTargetBlank ? '' : '_blank'} href={diff.expected!.attachment.path} rel='noreferrer'>{diff.expected!.attachment.name}</a></div>
-      </div>
+      </div>}
     </>}
   </div>;
 };
@@ -136,7 +138,8 @@ export const ImageDiffSlider: React.FC<{
   canvasHeight: number,
   scale: number,
   expectedTitle: string,
-}> = ({ expectedImage, actualImage, canvasWidth, canvasHeight, scale, expectedTitle }) => {
+  hideSize?: boolean, 
+}> = ({ expectedImage, actualImage, canvasWidth, canvasHeight, scale, expectedTitle, hideSize }) => {
   const absoluteStyle: React.CSSProperties = {
     position: 'absolute',
     top: 0,
@@ -147,7 +150,7 @@ export const ImageDiffSlider: React.FC<{
   const sameSize = expectedImage.naturalWidth === actualImage.naturalWidth && expectedImage.naturalHeight === actualImage.naturalHeight;
 
   return <div style={{ flex: 'none', display: 'flex', alignItems: 'center', flexDirection: 'column', userSelect: 'none' }}>
-    <div style={{ margin: 5 }}>
+    {!hideSize && <div style={{ margin: 5 }}>
       {!sameSize && <span style={{ flex: 'none', margin: '0 5px' }}>Expected </span>}
       <span>{expectedImage.naturalWidth}</span>
       <span style={{ flex: 'none', margin: '0 5px' }}>x</span>
@@ -156,7 +159,7 @@ export const ImageDiffSlider: React.FC<{
       {!sameSize && <span>{actualImage.naturalWidth}</span>}
       {!sameSize && <span style={{ flex: 'none', margin: '0 5px' }}>x</span>}
       {!sameSize && <span>{actualImage.naturalHeight}</span>}
-    </div>
+    </div>}
     <div style={{ position: 'relative', width: canvasWidth, height: canvasHeight, margin: 15, ...checkerboardStyle }}>
       <ResizeView
         orientation={'horizontal'}
@@ -182,18 +185,19 @@ const ImageWithSize: React.FunctionComponent<{
   image: HTMLImageElement,
   title?: string,
   alt?: string,
+  hideSize?: boolean,
   canvasWidth: number,
   canvasHeight: number,
   scale: number,
   onClick?: () => void;
-}> = ({ image, title, alt, canvasWidth, canvasHeight, scale, onClick }) => {
+}> = ({ image, title, alt, hideSize, canvasWidth, canvasHeight, scale, onClick }) => {
   return <div style={{ flex: 'none', display: 'flex', alignItems: 'center', flexDirection: 'column' }}>
-    <div style={{ margin: 5 }}>
+    {!hideSize && <div style={{ margin: 5 }}>
       {title && <span style={{ flex: 'none', margin: '0 5px' }}>{title}</span>}
       <span>{image.naturalWidth}</span>
       <span style={{ flex: 'none', margin: '0 5px' }}>x</span>
       <span>{image.naturalHeight}</span>
-    </div>
+    </div>}
     <div style={{ display: 'flex', flex: 'none', width: canvasWidth, height: canvasHeight, margin: 15, ...checkerboardStyle }}>
       <img
         width={image.naturalWidth * scale}