Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

feat(screenshots): when actual and expected have different sizes, pad and produce the diff image (#20208)

Browse source 
Also show sizes in the html report to easier spot the size mismatch
issue.

<img width="1030" alt="diff"
src="https://user-images.githubusercontent.com/9881434/213327632-b8fcd69c-8d08-460c-9de1-b5f4f8c56359.png">

Fixes #15802.
This commit is contained in:
Dmitry Gozman 2023-01-20 19:41:43 -08:00 committed by GitHub
parent 0cc0d168cd
commit b700c08dc5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 107 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
packages/html-reporter/src/imageDiffView.css
View file

@ -21,10 +21,27 @@
position: relative;
}
.image-diff-view img {
flex: none;
.image-diff-view .image-wrapper img {
flex: auto;
box-shadow: var(--box-shadow-thick);
margin: 24px auto;
min-width: 200px;
max-width: 80%;
}
.image-diff-view .image-wrapper {
flex: auto;
display: flex;
flex-direction: column;
align-items: center;
}
.image-diff-view .image-wrapper div {
flex: none;
align-self: stretch;
height: 2em;
font-weight: 500;
padding-top: 1em;
display: flex;
flex-direction: row;
}

6
packages/html-reporter/src/imageDiffView.spec.tsx
View file

@ -54,7 +54,7 @@ test('should show actual by default', async ({ mount }) => {
for (let i = 0; i < imageCount; ++i) {
const image = images.nth(i);
const box = await image.boundingBox();
expect(box).toEqual({ x: 400, y: 80, width: 200, height: 200 });
expect(box).toEqual({ x: 400, y: 108, width: 200, height: 200 });
}
});
@ -69,7 +69,7 @@ test('should switch to expected', async ({ mount }) => {
for (let i = 0; i < imageCount; ++i) {
const image = images.nth(i);
const box = await image.boundingBox();
expect(box).toEqual({ x: 400, y: 80, width: 200, height: 200 });
expect(box).toEqual({ x: 400, y: 108, width: 200, height: 200 });
}
});
@ -79,5 +79,5 @@ test('should switch to diff', async ({ mount }) => {
const image = component.locator('img');
const box = await image.boundingBox();
expect(box).toEqual({ x: 400, y: 80, width: 200, height: 200 });
expect(box).toEqual({ x: 400, y: 108, width: 200, height: 200 });
});

37
packages/html-reporter/src/imageDiffView.tsx
View file

@ -54,35 +54,35 @@ export const ImageDiffView: React.FunctionComponent<{
id: 'actual',
title: 'Actual',
render: () => <ImageDiffSlider sliderPosition={sliderPosition} setSliderPosition={setSliderPosition}>
<img src={diff.expected!.attachment.path!} onLoad={() => onImageLoaded('right')} ref={imageElement} />
<img src={diff.actual!.attachment.path!} />
<ImageWithSize src={diff.expected!.attachment.path!} onLoad={() => onImageLoaded('right')} imageRef={imageElement} style={{ boxShadow: 'none' }} />
<ImageWithSize src={diff.actual!.attachment.path!} />
</ImageDiffSlider>,
});
tabs.push({
id: 'expected',
title: diff.expected!.title,
render: () => <ImageDiffSlider sliderPosition={sliderPosition} setSliderPosition={setSliderPosition}>
<img src={diff.expected!.attachment.path!} onLoad={() => onImageLoaded('left')} ref={imageElement} />
<img src={diff.actual!.attachment.path!} style={{ boxShadow: 'none' }} />
<ImageWithSize src={diff.expected!.attachment.path!} onLoad={() => onImageLoaded('left')} imageRef={imageElement} />
<ImageWithSize src={diff.actual!.attachment.path!} style={{ boxShadow: 'none' }} />
</ImageDiffSlider>,
});
} else {
tabs.push({
id: 'actual',
title: 'Actual',
render: () => <img src={diff.actual!.attachment.path!} onLoad={() => onImageLoaded()} />
render: () => <ImageWithSize src={diff.actual!.attachment.path!} onLoad={() => onImageLoaded()} />
});
tabs.push({
id: 'expected',
title: diff.expected!.title,
render: () => <img src={diff.expected!.attachment.path!} onLoad={() => onImageLoaded()} />
render: () => <ImageWithSize src={diff.expected!.attachment.path!} onLoad={() => onImageLoaded()} />
});
}
if (diff.diff) {
tabs.push({
id: 'diff',
title: 'Diff',
render: () => <img src={diff.diff!.attachment.path} onLoad={() => onImageLoaded()} />
render: () => <ImageWithSize src={diff.diff!.attachment.path!} onLoad={() => onImageLoaded()} />
});
}
return <div className='vbox image-diff-view' data-testid='test-result-image-mismatch' ref={diffElement}>
@ -167,6 +167,29 @@ export const ImageDiffSlider: React.FC<React.PropsWithChildren<{
</>;
};
const ImageWithSize: React.FunctionComponent<{
src: string,
onLoad?: () => void,
imageRef?: React.RefObject<HTMLImageElement>,
style?: React.CSSProperties,
}> = ({ src, onLoad, imageRef, style }) => {
const newRef = React.useRef<HTMLImageElement>(null);
const ref = imageRef ?? newRef;
const [size, setSize] = React.useState<{ width: number, height: number } | null>(null);
return <div className='image-wrapper'>
<div>
<span style={{ flex: '1 1 0', textAlign: 'end' }}>{ size ? size.width : ''}</span>
<span style={{ flex: 'none', margin: '0 5px' }}>x</span>
<span style={{ flex: '1 1 0', textAlign: 'start' }}>{ size ? size.height : ''}</span>
</div>
<img src={src} onLoad={() => {
onLoad?.();
if (ref.current)
setSize({ width: ref.current.naturalWidth, height: ref.current.naturalHeight });
}} ref={ref} style={style} />
</div>;
};
const absolute: React.CSSProperties = {
position: 'absolute',
top: 0,

52
packages/playwright-core/src/utils/comparators.ts
View file

@ -47,27 +47,31 @@ function compareBuffersOrStrings(actualBuffer: Buffer | string, expectedBuffer:
return null;
}
type ImageData = { width: number, height: number, data: Buffer };
function compareImages(mimeType: string, actualBuffer: Buffer | string, expectedBuffer: Buffer, options: ImageComparatorOptions = {}): ComparatorResult {
if (!actualBuffer || !(actualBuffer instanceof Buffer))
return { errorMessage: 'Actual result should be a Buffer.' };
const actual = mimeType === 'image/png' ? PNG.sync.read(actualBuffer) : jpegjs.decode(actualBuffer, { maxMemoryUsageInMB: JPEG_JS_MAX_BUFFER_SIZE_IN_MB });
const expected = mimeType === 'image/png' ? PNG.sync.read(expectedBuffer) : jpegjs.decode(expectedBuffer, { maxMemoryUsageInMB: JPEG_JS_MAX_BUFFER_SIZE_IN_MB });
let actual: ImageData = mimeType === 'image/png' ? PNG.sync.read(actualBuffer) : jpegjs.decode(actualBuffer, { maxMemoryUsageInMB: JPEG_JS_MAX_BUFFER_SIZE_IN_MB });
let expected: ImageData = mimeType === 'image/png' ? PNG.sync.read(expectedBuffer) : jpegjs.decode(expectedBuffer, { maxMemoryUsageInMB: JPEG_JS_MAX_BUFFER_SIZE_IN_MB });
const size = { width: Math.max(expected.width, actual.width), height: Math.max(expected.height, actual.height) };
let sizesMismatchError = '';
if (expected.width !== actual.width || expected.height !== actual.height) {
return {
errorMessage: `Expected an image ${expected.width}px by ${expected.height}px, received ${actual.width}px by ${actual.height}px. `
};
sizesMismatchError = `Expected an image ${expected.width}px by ${expected.height}px, received ${actual.width}px by ${actual.height}px. `;
actual = resizeImage(actual, size);
expected = resizeImage(expected, size);
}
const diff = new PNG({ width: expected.width, height: expected.height });
const diff = new PNG({ width: size.width, height: size.height });
let count;
if (options._comparator === 'ssim-cie94') {
count = compare(expected.data, actual.data, diff.data, expected.width, expected.height, {
count = compare(expected.data, actual.data, diff.data, size.width, size.height, {
// All ΔE* formulae are originally designed to have the difference of 1.0 stand for a "just noticeable difference" (JND).
// See https://en.wikipedia.org/wiki/Color_difference#CIELAB_%CE%94E*
maxColorDeltaE94: 1.0,
});
} else if ((options._comparator ?? 'pixelmatch') === 'pixelmatch') {
count = pixelmatch(expected.data, actual.data, diff.data, expected.width, expected.height, {
count = pixelmatch(expected.data, actual.data, diff.data, size.width, size.height, {
threshold: options.threshold ?? 0.2,
});
} else {
@ -82,10 +86,10 @@ function compareImages(mimeType: string, actualBuffer: Buffer | string, expected
else
maxDiffPixels = maxDiffPixels1 ?? maxDiffPixels2 ?? 0;
const ratio = Math.ceil(count / (expected.width * expected.height) * 100) / 100;
return count > maxDiffPixels ? {
errorMessage: `${count} pixels (ratio ${ratio.toFixed(2)} of all image pixels) are different`,
diff: PNG.sync.write(diff),
} : null;
const pixelsMismatchError = count > maxDiffPixels ? `${count} pixels (ratio ${ratio.toFixed(2)} of all image pixels) are different.` : '';
if (pixelsMismatchError || sizesMismatchError)
return { errorMessage: sizesMismatchError + pixelsMismatchError, diff: PNG.sync.write(diff) };
return null;
}
function compareText(actual: Buffer | string, expectedBuffer: Buffer): ComparatorResult {
@ -122,3 +126,27 @@ function diff_prettyTerminal(diffs: [number, string][]) {
}
return html.join('');
}
function resizeImage(image: ImageData, size: { width: number, height: number }): ImageData {
if (image.width === size.width && image.height === size.height)
return image;
const buffer = new Uint8Array(size.width * size.height * 4);
for (let y = 0; y < size.height; y++) {
for (let x = 0; x < size.width; x++) {
const to = (y * size.width + x) * 4;
if (y < image.height && x < image.width) {
const from = (y * image.width + x) * 4;
buffer[to] = image.data[from];
buffer[to + 1] = image.data[from + 1];
buffer[to + 2] = image.data[from + 2];
buffer[to + 3] = image.data[from + 3];
} else {
buffer[to] = 0;
buffer[to + 1] = 0;
buffer[to + 2] = 0;
buffer[to + 3] = 0;
}
}
}
return { data: Buffer.from(buffer), width: size.width, height: size.height };
}

8
tests/playwright-test/golden.spec.ts
View file

@ -927,7 +927,7 @@ test('should attach expected/actual/diff', async ({ runInlineTest }, testInfo) =
]);
});
test('should attach expected/actual and no diff', async ({ runInlineTest }, testInfo) => {
test('should attach expected/actual/diff for different sizes', async ({ runInlineTest }, testInfo) => {
const result = await runInlineTest({
...files,
'a.spec.js-snapshots/snapshot.png':
@ -945,6 +945,7 @@ test('should attach expected/actual and no diff', async ({ runInlineTest }, test
const outputText = stripAnsi(result.output);
expect(outputText).toContain('Expected an image 2px by 2px, received 1px by 1px.');
expect(outputText).toContain('4 pixels (ratio 1.00 of all image pixels) are different.');
const attachments = outputText.split('\n').filter(l => l.startsWith('## ')).map(l => l.substring(3)).map(l => JSON.parse(l))[0];
for (const attachment of attachments)
attachment.path = attachment.path.replace(/\\/g, '/').replace(/.*test-results\//, '');
@ -959,6 +960,11 @@ test('should attach expected/actual and no diff', async ({ runInlineTest }, test
contentType: 'image/png',
path: 'a-is-a-test/snapshot-actual.png'
},
{
name: 'snapshot-diff.png',
contentType: 'image/png',
path: 'a-is-a-test/snapshot-diff.png'
},
]);
});

1
tests/playwright-test/reporter-html.spec.ts
View file

@ -137,6 +137,7 @@ test('should include image diff', async ({ runInlineTest, page, showReport }) =>
set.add(await expectedImage.getAttribute('src'));
set.add(await actualImage.getAttribute('src'));
expect(set.size, 'Should be two images overlaid').toBe(2);
await expect(imageDiff).toContainText('200x200');
const sliderElement = imageDiff.locator('data-testid=test-result-image-mismatch-grip');
await expect.poll(() => sliderElement.evaluate(e => e.style.left), 'Actual slider is on the right').toBe('590px');

8
tests/playwright-test/to-have-screenshot.spec.ts
View file

@ -943,7 +943,7 @@ test('should throw for invalid maxDiffPixelRatio values', async ({ runInlineTest
});
test('should attach expected/actual and no diff when sizes are different', async ({ runInlineTest }, testInfo) => {
test('should attach expected/actual/diff when sizes are different', async ({ runInlineTest }, testInfo) => {
const result = await runInlineTest({
...playwrightConfig({
snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}',
@ -962,6 +962,7 @@ test('should attach expected/actual and no diff when sizes are different', async
expect(result.exitCode).toBe(1);
const outputText = stripAnsi(result.output);
expect(outputText).toContain('Expected an image 2px by 2px, received 1280px by 720px.');
expect(outputText).toContain('4 pixels (ratio 0.01 of all image pixels) are different.');
const attachments = outputText.split('\n').filter(l => l.startsWith('## ')).map(l => l.substring(3)).map(l => JSON.parse(l))[0];
for (const attachment of attachments)
attachment.path = attachment.path.replace(/\\/g, '/').replace(/.*test-results\//, '');
@ -976,6 +977,11 @@ test('should attach expected/actual and no diff when sizes are different', async
contentType: 'image/png',
path: 'a-is-a-test/snapshot-actual.png'
},
{
name: 'snapshot-diff.png',
contentType: 'image/png',
path: 'a-is-a-test/snapshot-diff.png'
},
]);
});