diff --git a/.gitignore b/.gitignore index aadc481067..1f3b9a7a72 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,4 @@ test-results .cache/ .eslintcache playwright.env +firefox diff --git a/packages/html-reporter/src/testErrorView.tsx b/packages/html-reporter/src/testErrorView.tsx index 5208158b1c..80db4c9fb9 100644 --- a/packages/html-reporter/src/testErrorView.tsx +++ b/packages/html-reporter/src/testErrorView.tsx @@ -17,21 +17,39 @@ import ansi2html from 'ansi-to-html'; import * as React from 'react'; import './testErrorView.css'; +import type { ImageDiff } from '@web/shared/imageDiffView'; +import { ImageDiffView } from '@web/shared/imageDiffView'; export const TestErrorView: React.FC<{ error: string; }> = ({ error }) => { - const html = React.useMemo(() => { - const config: any = { - bg: 'var(--color-canvas-subtle)', - fg: 'var(--color-fg-default)', - }; - config.colors = ansiColors; - return new ansi2html(config).toHtml(escapeHTML(error)); - }, [error]); + const html = React.useMemo(() => ansiErrorToHtml(error), [error]); return
; }; +export const TestScreenshotErrorView: React.FC<{ + errorPrefix?: string, + diff: ImageDiff, + errorSuffix?: string, +}> = ({ errorPrefix, diff, errorSuffix }) => { + const prefixHtml = React.useMemo(() => ansiErrorToHtml(errorPrefix), [errorPrefix]); + const suffixHtml = React.useMemo(() => ansiErrorToHtml(errorSuffix), [errorSuffix]); + return
+
+ +
+
+}; + +function ansiErrorToHtml(text?: string): string { + const config: any = { + bg: 'var(--color-canvas-subtle)', + fg: 'var(--color-fg-default)', + }; + config.colors = ansiColors; + return new ansi2html(config).toHtml(escapeHTML(text || '')); +} + const ansiColors = { 0: '#000', 1: '#C00', diff --git a/packages/html-reporter/src/testResultView.tsx b/packages/html-reporter/src/testResultView.tsx index 8ee36d0cda..54d349eea8 100644 --- a/packages/html-reporter/src/testResultView.tsx +++ b/packages/html-reporter/src/testResultView.tsx @@ -24,7 +24,7 @@ import { AttachmentLink, generateTraceUrl } from './links'; import { statusIcon } from './statusIcon'; import type { ImageDiff } from '@web/shared/imageDiffView'; import { ImageDiffView } from '@web/shared/imageDiffView'; -import { TestErrorView } from './testErrorView'; +import { TestErrorView, TestScreenshotErrorView } from './testErrorView'; import './testResultView.css'; function groupImageDiffs(screenshots: Set): ImageDiff[] { @@ -67,7 +67,7 @@ export const TestResultView: React.FC<{ anchor: 'video' | 'diff' | '', }> = ({ result, anchor }) => { - const { screenshots, videos, traces, otherAttachments, diffs, htmls } = React.useMemo(() => { + const { screenshots, videos, traces, otherAttachments, diffs, errors, htmls } = React.useMemo(() => { const attachments = result?.attachments || []; const screenshots = new Set(attachments.filter(a => a.contentType.startsWith('image/'))); const videos = attachments.filter(a => a.name === 'video'); @@ -76,7 +76,8 @@ export const TestResultView: React.FC<{ const otherAttachments = new Set(attachments); [...screenshots, ...videos, ...traces, ...htmls].forEach(a => otherAttachments.delete(a)); const diffs = groupImageDiffs(screenshots); - return { screenshots: [...screenshots], videos, traces, otherAttachments, diffs, htmls }; + const errors = classifyErrors(result.errors, diffs); + return { screenshots: [...screenshots], videos, traces, otherAttachments, diffs, errors, htmls }; }, [result]); const videoRef = React.useRef(null); @@ -94,8 +95,12 @@ export const TestResultView: React.FC<{ }, [scrolled, anchor, setScrolled, videoRef]); return
- {!!result.errors.length && - {result.errors.map((error, index) => )} + {!!errors.length && + {errors.map((error, index) => { + if (error.type === 'screenshot') + return ; + return + })} } {!!result.steps.length && {result.steps.map((step, i) => )} @@ -145,6 +150,38 @@ export const TestResultView: React.FC<{
; }; +function classifyErrors(testErrors: string[], diffs: ImageDiff[]) { + const errors = []; + for (const error of testErrors) { + let screenshotError; + if (error.includes('Screenshot comparison failed:')) { + for (const diff of diffs) { + if (!diff.actual?.attachment.name) + continue; + if (!error.includes(diff.actual!.attachment.name)) + continue; + const index = error.search(/Expected:|Previous:|Received:/); + let errorPrefix; + if (index !== -1) + errorPrefix = error.slice(0, index); + else + errorPrefix = error.split('\n')[0]; + + const callLog = error.indexOf('Call log:'); + let errorSuffix; + if (callLog !== -1) + errorSuffix = error.slice(callLog); + screenshotError = { type: 'screenshot', diff, errorPrefix, errorSuffix }; + } + } + if (screenshotError) + errors.push(screenshotError); + else + errors.push({ type: 'regular', error }); + } + return errors; +} + const StepTreeItem: React.FC<{ step: TestStep; depth: number, diff --git a/packages/playwright-core/src/server/page.ts b/packages/playwright-core/src/server/page.ts index 8f5c0d989b..b78bd91ee1 100644 --- a/packages/playwright-core/src/server/page.ts +++ b/packages/playwright-core/src/server/page.ts @@ -43,7 +43,7 @@ import type { TimeoutOptions } from '../common/types'; import { isInvalidSelectorError } from '../utils/isomorphic/selectorParser'; import { parseEvaluationResultValue, source } from './isomorphic/utilityScriptSerializers'; import type { SerializedValue } from './isomorphic/utilityScriptSerializers'; -import { TargetClosedError } from './errors'; +import { TargetClosedError, TimeoutError } from './errors'; import { asLocator } from '../utils'; import { helper } from './helper'; @@ -672,10 +672,13 @@ export class Page extends SdkObject { // A: We want user to receive a friendly diff between actual and expected/previous. if (js.isJavaScriptErrorInEvaluate(e) || isInvalidSelectorError(e)) throw e; + let errorMessage = e.message; + if (e instanceof TimeoutError && intermediateResult?.previous) + errorMessage = `Failed to take two consecutive stable screenshots. ${e.message}`; return { log: e.message ? [...metadata.log, e.message] : metadata.log, ...intermediateResult, - errorMessage: e.message, + errorMessage, }; }); } diff --git a/packages/web/src/shared/imageDiffView.tsx b/packages/web/src/shared/imageDiffView.tsx index d7c48d16c0..0d26e12be5 100644 --- a/packages/web/src/shared/imageDiffView.tsx +++ b/packages/web/src/shared/imageDiffView.tsx @@ -61,7 +61,9 @@ const checkerboardStyle: React.CSSProperties = { export const ImageDiffView: React.FC<{ diff: ImageDiff, noTargetBlank?: boolean, -}> = ({ diff, noTargetBlank }) => { + hideDetails?: boolean, +}> = ({ diff, noTargetBlank, hideDetails }) => { + console.log('hideDetails', hideDetails); const [mode, setMode] = React.useState<'diff' | 'actual' | 'expected' | 'slider' | 'sxs'>(diff.diff ? 'diff' : 'actual'); const [showSxsDiff, setShowSxsDiff] = React.useState(false); @@ -105,26 +107,26 @@ export const ImageDiffView: React.FC<{
setMode('slider')}>Slider
- {diff.diff && mode === 'diff' && } - {diff.diff && mode === 'actual' && } - {diff.diff && mode === 'expected' && } - {diff.diff && mode === 'slider' && } + {diff.diff && mode === 'diff' && } + {diff.diff && mode === 'actual' && } + {diff.diff && mode === 'expected' && } + {diff.diff && mode === 'slider' && } {diff.diff && mode === 'sxs' &&
- - setShowSxsDiff(!showSxsDiff)} canvasWidth={sxsScale * imageWidth} canvasHeight={sxsScale * imageHeight} scale={sxsScale} /> + + setShowSxsDiff(!showSxsDiff)} hideSize={hideDetails} canvasWidth={sxsScale * imageWidth} canvasHeight={sxsScale * imageHeight} scale={sxsScale} />
} - {!diff.diff && mode === 'actual' && } - {!diff.diff && mode === 'expected' && } + {!diff.diff && mode === 'actual' && } + {!diff.diff && mode === 'expected' && } {!diff.diff && mode === 'sxs' &&
}
-
+ {!hideDetails && +
} } ; }; @@ -136,7 +138,8 @@ export const ImageDiffSlider: React.FC<{ canvasHeight: number, scale: number, expectedTitle: string, -}> = ({ expectedImage, actualImage, canvasWidth, canvasHeight, scale, expectedTitle }) => { + hideSize?: boolean, +}> = ({ expectedImage, actualImage, canvasWidth, canvasHeight, scale, expectedTitle, hideSize }) => { const absoluteStyle: React.CSSProperties = { position: 'absolute', top: 0, @@ -147,7 +150,7 @@ export const ImageDiffSlider: React.FC<{ const sameSize = expectedImage.naturalWidth === actualImage.naturalWidth && expectedImage.naturalHeight === actualImage.naturalHeight; return
-
+ {!hideSize &&
{!sameSize && Expected } {expectedImage.naturalWidth} x @@ -156,7 +159,7 @@ export const ImageDiffSlider: React.FC<{ {!sameSize && {actualImage.naturalWidth}} {!sameSize && x} {!sameSize && {actualImage.naturalHeight}} -
+
}
void; -}> = ({ image, title, alt, canvasWidth, canvasHeight, scale, onClick }) => { +}> = ({ image, title, alt, hideSize, canvasWidth, canvasHeight, scale, onClick }) => { return
-
+ {!hideSize &&
{title && {title}} {image.naturalWidth} x {image.naturalHeight} -
+
}