diff --git a/.gitattributes b/.gitattributes index 653dde78b5..c2515b4ffe 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,5 +1,4 @@ # text files must be lf for golden file tests to work * text=auto eol=lf - # make project show as TS on GitHub *.js linguist-detectable=false diff --git a/docs/src/api/class-locatorassertions.md b/docs/src/api/class-locatorassertions.md index f7c5e7a2fa..c578608a74 100644 --- a/docs/src/api/class-locatorassertions.md +++ b/docs/src/api/class-locatorassertions.md @@ -1362,6 +1362,9 @@ Snapshot name. ### option: LocatorAssertions.toHaveScreenshot#1.scale = %%-screenshot-option-scale-default-css-%% * since: v1.23 +### option: LocatorAssertions.toHaveScreenshot#1.comparator = %%-assertions-comparator-%% +* since: v1.29 + ### option: LocatorAssertions.toHaveScreenshot#1.maxDiffPixels = %%-assertions-max-diff-pixels-%% * since: v1.23 @@ -1405,6 +1408,9 @@ Note that screenshot assertions only work with Playwright test runner. ### option: LocatorAssertions.toHaveScreenshot#2.scale = %%-screenshot-option-scale-default-css-%% * since: v1.23 +### option: LocatorAssertions.toHaveScreenshot#2.comparator = %%-assertions-comparator-%% +* since: v1.29 + ### option: LocatorAssertions.toHaveScreenshot#2.maxDiffPixels = %%-assertions-max-diff-pixels-%% * since: v1.23 diff --git a/docs/src/api/class-pageassertions.md b/docs/src/api/class-pageassertions.md index b1e68dc2f2..d52d562439 100644 --- a/docs/src/api/class-pageassertions.md +++ b/docs/src/api/class-pageassertions.md @@ -170,6 +170,9 @@ Snapshot name. ### option: PageAssertions.toHaveScreenshot#1.scale = %%-screenshot-option-scale-default-css-%% * since: v1.23 +### option: PageAssertions.toHaveScreenshot#1.comparator = %%-assertions-comparator-%% +* since: v1.29 + ### option: PageAssertions.toHaveScreenshot#1.maxDiffPixels = %%-assertions-max-diff-pixels-%% * since: v1.23 @@ -218,6 +221,9 @@ Note that screenshot assertions only work with Playwright test runner. ### option: PageAssertions.toHaveScreenshot#2.scale = %%-screenshot-option-scale-default-css-%% * since: v1.23 +### option: PageAssertions.toHaveScreenshot#2.comparator = %%-assertions-comparator-%% +* since: v1.29 + ### option: PageAssertions.toHaveScreenshot#2.maxDiffPixels = %%-assertions-max-diff-pixels-%% * since: v1.23 diff --git a/docs/src/api/class-snapshotassertions.md b/docs/src/api/class-snapshotassertions.md index a61a4752b1..faac9465a5 100644 --- a/docs/src/api/class-snapshotassertions.md +++ b/docs/src/api/class-snapshotassertions.md @@ -43,6 +43,9 @@ Note that matching snapshots only work with Playwright test runner. Snapshot name. +### option: SnapshotAssertions.toMatchSnapshot#1.comparator = %%-assertions-comparator-%% +* since: v1.29 + ### option: SnapshotAssertions.toMatchSnapshot#1.maxDiffPixels = %%-assertions-max-diff-pixels-%% * since: v1.22 @@ -79,6 +82,9 @@ Learn more about [visual comparisons](../test-snapshots.md). Note that matching snapshots only work with Playwright test runner. +### option: SnapshotAssertions.toMatchSnapshot#2.comparator = %%-assertions-comparator-%% +* since: v1.29 + ### option: SnapshotAssertions.toMatchSnapshot#2.maxDiffPixels = %%-assertions-max-diff-pixels-%% * since: v1.22 diff --git a/docs/src/api/params.md b/docs/src/api/params.md index e74d4847d1..a4bd5ab17d 100644 --- a/docs/src/api/params.md +++ b/docs/src/api/params.md @@ -800,6 +800,12 @@ Time to retry the assertion for. An acceptable amount of pixels that could be different. Default is configurable with `TestConfig.expect`. Unset by default. +## assertions-comparator +* langs: js +- `comparator` <[string]> Either `"pixelmatch"` or `"ssim-cie94"`. + +A comparator function to use when comparing images. + ## assertions-max-diff-pixel-ratio * langs: js - `maxDiffPixelRatio` <[float]> diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index bcf5c05e43..d799f1c0c9 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -39,14 +39,16 @@ export default config; - type: ?<[Object]> - `timeout` ?<[int]> Default timeout for async expect matchers in milliseconds, defaults to 5000ms. - `toHaveScreenshot` ?<[Object]> Configuration for the [`method: PageAssertions.toHaveScreenshot#1`] method. - - `threshold` ?<[float]> an acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between the same pixel in compared images, between zero (strict) and one (lax). Defaults to `0.2`. + - `comparator` ?<[string]> a comparator function to use, either `"pixelmatch"` or `"ssim-cie94"`. Defaults to `"pixelmatch"`. + - `threshold` ?<[float]> an acceptable perceived color difference between the same pixel in compared images, ranging from `0` (strict) and `1` (lax). `"pixelmatch"` comparator computes color difference in [YIQ color space](https://en.wikipedia.org/wiki/YIQ) and defaults `threshold` value to `0.2`. `"ssim-cie94"` comparator computes color difference by [CIE94](https://en.wikipedia.org/wiki/Color_difference#CIE94) and defaults `threshold` value to `0.01`. - `maxDiffPixels` ?<[int]> an acceptable amount of pixels that could be different, unset by default. - `maxDiffPixelRatio` ?<[float]> an acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default. - `animations` ?<[ScreenshotAnimations]<"allow"|"disabled">> See [`option: animations`] in [`method: Page.screenshot`]. Defaults to `"disabled"`. - `caret` ?<[ScreenshotCaret]<"hide"|"initial">> See [`option: caret`] in [`method: Page.screenshot`]. Defaults to `"hide"`. - `scale` ?<[ScreenshotScale]<"css"|"device">> See [`option: scale`] in [`method: Page.screenshot`]. Defaults to `"css"`. - `toMatchSnapshot` ?<[Object]> Configuration for the [`method: SnapshotAssertions.toMatchSnapshot#1`] method. - - `threshold` ?<[float]> an acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between the same pixel in compared images, between zero (strict) and one (lax). Defaults to `0.2`. + - `comparator` ?<[string]> a comparator function to use, either `"pixelmatch"` or `"ssim-cie94"`. Defaults to `"pixelmatch"`. + - `threshold` ?<[float]> an acceptable perceived color difference between the same pixel in compared images, ranging from `0` (strict) and `1` (lax). `"pixelmatch"` comparator computes color difference in [YIQ color space](https://en.wikipedia.org/wiki/YIQ) and defaults `threshold` value to `0.2`. `"ssim-cie94"` comparator computes color difference by [CIE94](https://en.wikipedia.org/wiki/Color_difference#CIE94) and defaults `threshold` value to `0.01`. - `maxDiffPixels` ?<[int]> an acceptable amount of pixels that could be different, unset by default. - `maxDiffPixelRatio` ?<[float]> an acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default. diff --git a/docs/src/test-api/class-testproject.md b/docs/src/test-api/class-testproject.md index 00dbc57416..ca1c99d511 100644 --- a/docs/src/test-api/class-testproject.md +++ b/docs/src/test-api/class-testproject.md @@ -110,14 +110,16 @@ export default config; - type: ?<[Object]> - `timeout` ?<[int]> Default timeout for async expect matchers in milliseconds, defaults to 5000ms. - `toHaveScreenshot` ?<[Object]> Configuration for the [`method: PageAssertions.toHaveScreenshot#1`] method. - - `threshold` ?<[float]> an acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between the same pixel in compared images, between zero (strict) and one (lax). Defaults to `0.2`. + - `comparator` ?<[string]> a comparator function to use, either `"pixelmatch"` or `"ssim-cie94"`. Defaults to `"pixelmatch"`. + - `threshold` ?<[float]> an acceptable perceived color difference between the same pixel in compared images, ranging from `0` (strict) and `1` (lax). `"pixelmatch"` comparator computes color difference in [YIQ color space](https://en.wikipedia.org/wiki/YIQ) and defaults `threshold` value to `0.2`. `"ssim-cie94"` comparator computes color difference by [CIE94](https://en.wikipedia.org/wiki/Color_difference#CIE94) and defaults `threshold` value to `0.01`. - `maxDiffPixels` ?<[int]> an acceptable amount of pixels that could be different, unset by default. - `maxDiffPixelRatio` ?<[float]> an acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default. - `animations` ?<[ScreenshotAnimations]<"allow"|"disabled">> See [`option: animations`] in [`method: Page.screenshot`]. Defaults to `"disabled"`. - `caret` ?<[ScreenshotCaret]<"hide"|"initial">> See [`option: caret`] in [`method: Page.screenshot`]. Defaults to `"hide"`. - `scale` ?<[ScreenshotScale]<"css"|"device">> See [`option: scale`] in [`method: Page.screenshot`]. Defaults to `"css"`. - `toMatchSnapshot` ?<[Object]> Configuration for the [`method: SnapshotAssertions.toMatchSnapshot#1`] method. - - `threshold` ?<[float]> an acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between the same pixel in compared images, between zero (strict) and one (lax). Defaults to `0.2`. + - `comparator` ?<[string]> a comparator function to use, either `"pixelmatch"` or `"ssim-cie94"`. Defaults to `"pixelmatch"`. + - `threshold` ?<[float]> an acceptable perceived color difference between the same pixel in compared images, ranging from `0` (strict) and `1` (lax). `"pixelmatch"` comparator computes color difference in [YIQ color space](https://en.wikipedia.org/wiki/YIQ) and defaults `threshold` value to `0.2`. `"ssim-cie94"` comparator computes color difference by [CIE94](https://en.wikipedia.org/wiki/Color_difference#CIE94) and defaults `threshold` value to `0.01`. - `maxDiffPixels` ?<[int]> an acceptable amount of pixels that could be different, unset by default. - `maxDiffPixelRatio` ?<[float]> an acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default. diff --git a/package-lock.json b/package-lock.json index bbbe20672c..79eb26b429 100644 --- a/package-lock.json +++ b/package-lock.json @@ -58,6 +58,7 @@ "react-dom": "^18.1.0", "rimraf": "^3.0.2", "socksv5": "0.0.6", + "ssim.js": "^3.5.0", "typescript": "^4.7.3", "vite": "^3.2.1", "ws": "^8.5.0", @@ -4876,6 +4877,12 @@ "license": "BSD-3-Clause", "optional": true }, + "node_modules/ssim.js": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/ssim.js/-/ssim.js-3.5.0.tgz", + "integrity": "sha512-Aj6Jl2z6oDmgYFFbQqK7fght19bXdOxY7Tj03nF+03M9gCBAjeIiO8/PlEGMfKDwYpw4q6iBqVq2YuREorGg/g==", + "dev": true + }, "node_modules/stack-trace": { "version": "0.0.10", "dev": true, @@ -9174,6 +9181,12 @@ "dev": true, "optional": true }, + "ssim.js": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/ssim.js/-/ssim.js-3.5.0.tgz", + "integrity": "sha512-Aj6Jl2z6oDmgYFFbQqK7fght19bXdOxY7Tj03nF+03M9gCBAjeIiO8/PlEGMfKDwYpw4q6iBqVq2YuREorGg/g==", + "dev": true + }, "stack-trace": { "version": "0.0.10", "dev": true diff --git a/package.json b/package.json index 23801a83f5..4493da9084 100644 --- a/package.json +++ b/package.json @@ -93,6 +93,7 @@ "react-dom": "^18.1.0", "rimraf": "^3.0.2", "socksv5": "0.0.6", + "ssim.js": "^3.5.0", "typescript": "^4.7.3", "vite": "^3.2.1", "ws": "^8.5.0", diff --git a/packages/playwright-core/package.json b/packages/playwright-core/package.json index a34ea7b2f4..a5a4870518 100644 --- a/packages/playwright-core/package.json +++ b/packages/playwright-core/package.json @@ -23,6 +23,10 @@ "./lib/grid/gridServer": "./lib/grid/gridServer.js", "./lib/outofprocess": "./lib/outofprocess.js", "./lib/utils": "./lib/utils/index.js", + "./lib/image_tools/stats": "./lib/image_tools/stats.js", + "./lib/image_tools/compare": "./lib/image_tools/compare.js", + "./lib/image_tools/imageChannel": "./lib/image_tools/imageChannel.js", + "./lib/image_tools/colorUtils": "./lib/image_tools/colorUtils.js", "./lib/common/userAgent": "./lib/common/userAgent.js", "./lib/containers/docker": "./lib/containers/docker.js", "./lib/utils/comparators": "./lib/utils/comparators.js", diff --git a/packages/playwright-core/src/image_tools/colorUtils.ts b/packages/playwright-core/src/image_tools/colorUtils.ts new file mode 100644 index 0000000000..1f2a6227c1 --- /dev/null +++ b/packages/playwright-core/src/image_tools/colorUtils.ts @@ -0,0 +1,99 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the 'License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export function blendWithWhite(c: number, a: number): number { + return 255 + (c - 255) * a; +} + +export function rgb2gray(r: number, g: number, b: number): number { + // NOTE: this is the exact integer formula from SSIM.js. + // See https://github.com/obartra/ssim/blob/ca8e3c6a6ff5f4f2e232239e0c3d91806f3c97d5/src/matlab/rgb2gray.ts#L56 + return (77 * r + 150 * g + 29 * b + 128) >> 8; +} + +// Percieved color difference defined by CIE94. +// See https://en.wikipedia.org/wiki/Color_difference#CIE94 +// +// The result of 1.0 is a "just-noticiable difference". +// +// Other results interpretation (taken from http://zschuessler.github.io/DeltaE/learn/): +// < 1.0 Not perceptible by human eyes. +// 1-2 Perceptible through close observation. +// 2-10 Perceptible at a glance. +// 11-49 Colors are more similar than opposite +// 100 Colors are exact opposite +export function colorDeltaE94(rgb1: number[], rgb2: number[]) { + const [l1, a1, b1] = xyz2lab(srgb2xyz(rgb1)); + const [l2, a2, b2] = xyz2lab(srgb2xyz(rgb2)); + const deltaL = l1 - l2; + const deltaA = a1 - a2; + const deltaB = b1 - b2; + const c1 = Math.sqrt(a1 ** 2 + b1 ** 2); + const c2 = Math.sqrt(a2 ** 2 + b2 ** 2); + const deltaC = c1 - c2; + let deltaH = deltaA ** 2 + deltaB ** 2 - deltaC ** 2; + deltaH = deltaH < 0 ? 0 : Math.sqrt(deltaH); + // The k1, k2, kL, kC, kH values for "graphic arts" applications. + // See https://en.wikipedia.org/wiki/Color_difference#CIE94 + const k1 = 0.045; + const k2 = 0.015; + const kL = 1; + const kC = 1; + const kH = 1; + + const sC = 1.0 + k1 * c1; + const sH = 1.0 + k2 * c1; + const sL = 1; + + return Math.sqrt((deltaL / sL / kL) ** 2 + (deltaC / sC / kC) ** 2 + (deltaH / sH / kH) ** 2); +} + +// sRGB -> 1-normalized XYZ (i.e. Y ∈ [0, 1]) with D65 illuminant +// See https://en.wikipedia.org/wiki/SRGB#From_sRGB_to_CIE_XYZ +export function srgb2xyz(rgb: number[]): number[] { + let r = rgb[0] / 255; + let g = rgb[1] / 255; + let b = rgb[2] / 255; + r = (r > 0.04045) ? Math.pow((r + 0.055) / 1.055, 2.4) : r / 12.92; + g = (g > 0.04045) ? Math.pow((g + 0.055) / 1.055, 2.4) : g / 12.92; + b = (b > 0.04045) ? Math.pow((b + 0.055) / 1.055, 2.4) : b / 12.92; + return [ + (r * 0.4124 + g * 0.3576 + b * 0.1805), + (r * 0.2126 + g * 0.7152 + b * 0.0722), + (r * 0.0193 + g * 0.1192 + b * 0.9505), + ]; +} + +const sigma_pow2 = 6 * 6 / 29 / 29; +const sigma_pow3 = 6 * 6 * 6 / 29 / 29 / 29; + +// 1-normalized CIE XYZ with D65 to L*a*b* +// See https://en.wikipedia.org/wiki/CIELAB_color_space#From_CIEXYZ_to_CIELAB +export function xyz2lab(xyz: number[]): number[] { + const x = xyz[0] / 0.950489; + const y = xyz[1]; + const z = xyz[2] / 1.088840; + + const fx = x > sigma_pow3 ? x ** (1 / 3) : x / 3 / sigma_pow2 + 4 / 29; + const fy = y > sigma_pow3 ? y ** (1 / 3) : y / 3 / sigma_pow2 + 4 / 29; + const fz = z > sigma_pow3 ? z ** (1 / 3) : z / 3 / sigma_pow2 + 4 / 29; + + const l = 116 * fy - 16; + const a = 500 * (fx - fy); + const b = 200 * (fy - fz); + + return [l, a, b]; +} diff --git a/packages/playwright-core/src/image_tools/compare.ts b/packages/playwright-core/src/image_tools/compare.ts new file mode 100644 index 0000000000..2748d5a91c --- /dev/null +++ b/packages/playwright-core/src/image_tools/compare.ts @@ -0,0 +1,107 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the 'License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { blendWithWhite, colorDeltaE94, rgb2gray } from './colorUtils'; +import { ImageChannel } from './imageChannel'; +import { ssim, FastStats } from './stats'; + +const SSIM_WINDOW_RADIUS = 5; +const VARIANCE_WINDOW_RADIUS = 1; + +function drawPixel(width: number, data: Buffer, x: number, y: number, r: number, g: number, b: number) { + const idx = (y * width + x) * 4; + data[idx + 0] = r; + data[idx + 1] = g; + data[idx + 2] = b; + data[idx + 3] = 255; +} + +type CompareOptions = { + maxColorDeltaE94: number; +}; + +export function compare(actual: Buffer, expected: Buffer, diff: Buffer, width: number, height: number, options: CompareOptions) { + const { + maxColorDeltaE94 + } = options; + const [r1, g1, b1] = ImageChannel.intoRGB(width, height, expected); + const [r2, g2, b2] = ImageChannel.intoRGB(width, height, actual); + + const drawRedPixel = (x: number, y: number) => drawPixel(width, diff, x, y, 255, 0, 0); + const drawYellowPixel = (x: number, y: number) => drawPixel(width, diff, x, y, 255, 255, 0); + const drawGrayPixel = (x: number, y: number) => { + const gray = rgb2gray(r1.get(x, y), g1.get(x, y), b1.get(x, y)); + const value = blendWithWhite(gray, 0.1); + drawPixel(width, diff, x, y, value, value, value); + }; + + let fastR, fastG, fastB; + + let diffCount = 0; + for (let y = 0; y < height; ++y){ + for (let x = 0; x < width; ++x) { + // Fast-path: equal pixels. + if (r1.get(x, y) === r2.get(x, y) && g1.get(x, y) === g2.get(x, y) && b1.get(x, y) === b2.get(x, y)) { + drawGrayPixel(x, y); + continue; + } + + // Compare pixel colors using the dE94 color difference formulae. + // The dE94 is normalized so that the value of 1.0 is the "just-noticeable-difference". + // Color difference below 1.0 is not noticeable to a human eye, so we can disregard it. + // See https://en.wikipedia.org/wiki/Color_difference + const delta = colorDeltaE94( + [r1.get(x, y), g1.get(x, y), b1.get(x, y)], + [r2.get(x, y), g2.get(x, y), b2.get(x, y)] + ); + + if (delta <= maxColorDeltaE94) { + drawGrayPixel(x, y); + continue; + } + + // if this pixel is a part of a flood fill of a 3x3 square then it cannot be + // anti-aliasing pixel so it must be a pixel difference. + if (!fastR || !fastG || !fastB) { + fastR = new FastStats(r1, r2); + fastG = new FastStats(g1, g2); + fastB = new FastStats(b1, b2); + } + const [varX1, varY1] = r1.boundXY(x - VARIANCE_WINDOW_RADIUS, y - VARIANCE_WINDOW_RADIUS); + const [varX2, varY2] = r1.boundXY(x + VARIANCE_WINDOW_RADIUS, y + VARIANCE_WINDOW_RADIUS); + const var1 = fastR.varianceC1(varX1, varY1, varX2, varY2) + fastG.varianceC1(varX1, varY1, varX2, varY2) + fastB.varianceC1(varX1, varY1, varX2, varY2); + const var2 = fastR.varianceC2(varX1, varY1, varX2, varY2) + fastG.varianceC2(varX1, varY1, varX2, varY2) + fastB.varianceC2(varX1, varY1, varX2, varY2); + if (var1 === 0 && var2 === 0) { + drawRedPixel(x, y); + ++diffCount; + continue; + } + + const [ssimX1, ssimY1] = r1.boundXY(x - SSIM_WINDOW_RADIUS, y - SSIM_WINDOW_RADIUS); + const [ssimX2, ssimY2] = r1.boundXY(x + SSIM_WINDOW_RADIUS, y + SSIM_WINDOW_RADIUS); + const ssimRGB = (ssim(fastR, ssimX1, ssimY1, ssimX2, ssimY2) + ssim(fastG, ssimX1, ssimY1, ssimX2, ssimY2) + ssim(fastB, ssimX1, ssimY1, ssimX2, ssimY2)) / 3.0; + const isAntialiassed = ssimRGB >= 0.99; + if (isAntialiassed) { + drawYellowPixel(x, y); + } else { + drawRedPixel(x, y); + ++diffCount; + } + } + } + + return diffCount; +} diff --git a/packages/playwright-core/src/image_tools/imageChannel.ts b/packages/playwright-core/src/image_tools/imageChannel.ts new file mode 100644 index 0000000000..a9c4687f8c --- /dev/null +++ b/packages/playwright-core/src/image_tools/imageChannel.ts @@ -0,0 +1,61 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the 'License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { blendWithWhite } from './colorUtils'; + +export class ImageChannel { + data: Uint8Array; + width: number; + height: number; + + static intoRGB(width: number, height: number, data: Buffer): ImageChannel[] { + const r = new Uint8Array(width * height); + const g = new Uint8Array(width * height); + const b = new Uint8Array(width * height); + for (let y = 0; y < height; ++y) { + for (let x = 0; x < width; ++x) { + const index = y * width + x; + const offset = index * 4; + const alpha = data[offset + 3] === 255 ? 1 : data[offset + 3] / 255; + r[index] = blendWithWhite(data[offset], alpha); + g[index] = blendWithWhite(data[offset + 1], alpha); + b[index] = blendWithWhite(data[offset + 2], alpha); + } + } + return [ + new ImageChannel(width, height, r), + new ImageChannel(width, height, g), + new ImageChannel(width, height, b), + ]; + } + + constructor(width: number, height: number, data: Uint8Array) { + this.data = data; + this.width = width; + this.height = height; + } + + get(x: number, y: number) { + return this.data[y * this.width + x]; + } + + boundXY(x: number, y: number) { + return [ + Math.min(Math.max(x, 0), this.width - 1), + Math.min(Math.max(y, 0), this.height - 1), + ]; + } +} diff --git a/packages/playwright-core/src/image_tools/stats.ts b/packages/playwright-core/src/image_tools/stats.ts new file mode 100644 index 0000000000..b35371b4a1 --- /dev/null +++ b/packages/playwright-core/src/image_tools/stats.ts @@ -0,0 +1,127 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the 'License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { ImageChannel } from './imageChannel'; + +export interface Stats { + c1: ImageChannel; + c2: ImageChannel; + + // Compute mean value. See https://en.wikipedia.org/wiki/Mean + meanC1(x1: number, y1: number, x2: number, y2: number): number; + meanC2(x1: number, y1: number, x2: number, y2: number): number; + // Compute **population** (not sample) variance. See https://en.wikipedia.org/wiki/Variance + varianceC1(x1: number, y1: number, x2: number, y2: number): number; + varianceC2(x1: number, y1: number, x2: number, y2: number): number; + // Compute covariance. See https://en.wikipedia.org/wiki/Covariance + covariance(x1: number, y1: number, x2: number, y2: number): number; +} + +// Image channel has a 8-bit depth. +const DYNAMIC_RANGE = 2 ** 8 - 1; + +export function ssim(stats: Stats, x1: number, y1: number, x2: number, y2: number): number { + const mean1 = stats.meanC1(x1, y1, x2, y2); + const mean2 = stats.meanC2(x1, y1, x2, y2); + const var1 = stats.varianceC1(x1, y1, x2, y2); + const var2 = stats.varianceC2(x1, y1, x2, y2); + const cov = stats.covariance(x1, y1, x2, y2); + const c1 = (0.01 * DYNAMIC_RANGE) ** 2; + const c2 = (0.03 * DYNAMIC_RANGE) ** 2; + return (2 * mean1 * mean2 + c1) * (2 * cov + c2) / (mean1 ** 2 + mean2 ** 2 + c1) / (var1 + var2 + c2); +} + +export class FastStats implements Stats { + c1: ImageChannel; + c2: ImageChannel; + + private _partialSumC1: number[]; + private _partialSumC2: number[]; + private _partialSumMult: number[]; + private _partialSumSq1: number[]; + private _partialSumSq2: number[]; + + constructor(c1: ImageChannel, c2: ImageChannel) { + this.c1 = c1; + this.c2 = c2; + const { width, height } = c1; + + this._partialSumC1 = new Array(width * height); + this._partialSumC2 = new Array(width * height); + this._partialSumSq1 = new Array(width * height); + this._partialSumSq2 = new Array(width * height); + this._partialSumMult = new Array(width * height); + + const recalc = (mx: number[], idx: number, initial: number, x: number, y: number) => { + mx[idx] = initial; + if (y > 0) + mx[idx] += mx[(y - 1) * width + x]; + if (x > 0) + mx[idx] += mx[y * width + x - 1]; + if (x > 0 && y > 0) + mx[idx] -= mx[(y - 1) * width + x - 1]; + }; + + for (let y = 0; y < height; ++y) { + for (let x = 0; x < width; ++x) { + const idx = y * width + x; + recalc(this._partialSumC1, idx, this.c1.data[idx], x, y); + recalc(this._partialSumC2, idx, this.c2.data[idx], x, y); + recalc(this._partialSumSq1, idx, this.c1.data[idx] * this.c1.data[idx], x, y); + recalc(this._partialSumSq2, idx, this.c2.data[idx] * this.c2.data[idx], x, y); + recalc(this._partialSumMult, idx, this.c1.data[idx] * this.c2.data[idx], x, y); + } + } + } + + _sum(partialSum: number[], x1: number, y1: number, x2: number, y2: number): number { + const width = this.c1.width; + let result = partialSum[y2 * width + x2]; + if (y1 > 0) + result -= partialSum[(y1 - 1) * width + x2]; + if (x1 > 0) + result -= partialSum[y2 * width + x1 - 1]; + if (x1 > 0 && y1 > 0) + result += partialSum[(y1 - 1) * width + x1 - 1]; + return result; + } + + meanC1(x1: number, y1: number, x2: number, y2: number): number { + const N = (y2 - y1 + 1) * (x2 - x1 + 1); + return this._sum(this._partialSumC1, x1, y1, x2, y2) / N; + } + + meanC2(x1: number, y1: number, x2: number, y2: number): number { + const N = (y2 - y1 + 1) * (x2 - x1 + 1); + return this._sum(this._partialSumC2, x1, y1, x2, y2) / N; + } + + varianceC1(x1: number, y1: number, x2: number, y2: number): number { + const N = (y2 - y1 + 1) * (x2 - x1 + 1); + return (this._sum(this._partialSumSq1, x1, y1, x2, y2) - (this._sum(this._partialSumC1, x1, y1, x2, y2) ** 2) / N) / N; + } + + varianceC2(x1: number, y1: number, x2: number, y2: number): number { + const N = (y2 - y1 + 1) * (x2 - x1 + 1); + return (this._sum(this._partialSumSq2, x1, y1, x2, y2) - (this._sum(this._partialSumC2, x1, y1, x2, y2) ** 2) / N) / N; + } + + covariance(x1: number, y1: number, x2: number, y2: number): number { + const N = (y2 - y1 + 1) * (x2 - x1 + 1); + return (this._sum(this._partialSumMult, x1, y1, x2, y2) - this._sum(this._partialSumC1, x1, y1, x2, y2) * this._sum(this._partialSumC2, x1, y1, x2, y2) / N) / N; + } +} + diff --git a/packages/playwright-core/src/protocol/validator.ts b/packages/playwright-core/src/protocol/validator.ts index 5a5133206d..4422903c9e 100644 --- a/packages/playwright-core/src/protocol/validator.ts +++ b/packages/playwright-core/src/protocol/validator.ts @@ -987,6 +987,7 @@ scheme.PageExpectScreenshotParams = tObject({ selector: tString, })), comparatorOptions: tOptional(tObject({ + comparator: tOptional(tString), maxDiffPixels: tOptional(tNumber), maxDiffPixelRatio: tOptional(tNumber), threshold: tOptional(tNumber), diff --git a/packages/playwright-core/src/utils/DEPS.list b/packages/playwright-core/src/utils/DEPS.list index e97f2d3792..514f09d407 100644 --- a/packages/playwright-core/src/utils/DEPS.list +++ b/packages/playwright-core/src/utils/DEPS.list @@ -2,5 +2,6 @@ ./ ../third_party/diff_match_patch ../third_party/pixelmatch +../image_tools/compare.ts ../utilsBundle.ts ../zipBundle.ts diff --git a/packages/playwright-core/src/utils/comparators.ts b/packages/playwright-core/src/utils/comparators.ts index 029764bbe9..a1279d6e37 100644 --- a/packages/playwright-core/src/utils/comparators.ts +++ b/packages/playwright-core/src/utils/comparators.ts @@ -17,26 +17,17 @@ import { colors, jpegjs } from '../utilsBundle'; import pixelmatch from '../third_party/pixelmatch'; +import { compare } from '../image_tools/compare'; import { diff_match_patch, DIFF_INSERT, DIFF_DELETE, DIFF_EQUAL } from '../third_party/diff_match_patch'; import { PNG } from '../utilsBundle'; -export type ImageComparatorOptions = { threshold?: number, maxDiffPixels?: number, maxDiffPixelRatio?: number }; +export type ImageComparatorOptions = { threshold?: number, maxDiffPixels?: number, maxDiffPixelRatio?: number, comparator?: string }; export type ComparatorResult = { diff?: Buffer; errorMessage: string; } | null; export type Comparator = (actualBuffer: Buffer | string, expectedBuffer: Buffer, options?: any) => ComparatorResult; -let customPNGComparator: Comparator | undefined; -if (process.env.PW_CUSTOM_PNG_COMPARATOR) { - try { - customPNGComparator = require(process.env.PW_CUSTOM_PNG_COMPARATOR); - if (typeof customPNGComparator !== 'function') - customPNGComparator = undefined; - } catch (e) { - } -} - export function getComparator(mimeType: string): Comparator { if (mimeType === 'image/png') - return customPNGComparator ?? compareImages.bind(null, 'image/png'); + return compareImages.bind(null, 'image/png'); if (mimeType === 'image/jpeg') return compareImages.bind(null, 'image/jpeg'); if (mimeType === 'text/plain') @@ -68,9 +59,18 @@ function compareImages(mimeType: string, actualBuffer: Buffer | string, expected }; } const diff = new PNG({ width: expected.width, height: expected.height }); - const count = pixelmatch(expected.data, actual.data, diff.data, expected.width, expected.height, { - threshold: options.threshold ?? 0.2, - }); + let count; + if (options.comparator === 'ssim-cie94') { + count = compare(expected.data, actual.data, diff.data, expected.width, expected.height, { + maxColorDeltaE94: (options.threshold ?? 0.01) * 100, + }); + } else if ((options.comparator ?? 'pixelmatch') === 'pixelmatch') { + count = pixelmatch(expected.data, actual.data, diff.data, expected.width, expected.height, { + threshold: options.threshold ?? 0.2, + }); + } else { + throw new Error(`Configuration specifies unknown comparator "${options.comparator}"`); + } const maxDiffPixels1 = options.maxDiffPixels; const maxDiffPixels2 = options.maxDiffPixelRatio !== undefined ? expected.width * expected.height * options.maxDiffPixelRatio : undefined; diff --git a/packages/playwright-test/src/matchers/toMatchSnapshot.ts b/packages/playwright-test/src/matchers/toMatchSnapshot.ts index ba4b21952a..f08ea37780 100644 --- a/packages/playwright-test/src/matchers/toMatchSnapshot.ts +++ b/packages/playwright-test/src/matchers/toMatchSnapshot.ts @@ -145,6 +145,7 @@ class SnapshotHelper { maxDiffPixels: options.maxDiffPixels, maxDiffPixelRatio: options.maxDiffPixelRatio, threshold: options.threshold, + comparator: options.comparator, }; this.kind = this.mimeType.startsWith('image/') ? 'Screenshot' : 'Snapshot'; } @@ -305,6 +306,7 @@ export async function toHaveScreenshot( const helper = new SnapshotHelper( testInfo, snapshotPathResolver, 'png', { + comparator: config?.comparator, maxDiffPixels: config?.maxDiffPixels, maxDiffPixelRatio: config?.maxDiffPixelRatio, threshold: config?.threshold, diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index 956d8afc61..fdd1d0a709 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -534,8 +534,16 @@ interface TestConfig { */ toHaveScreenshot?: { /** - * an acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between the - * same pixel in compared images, between zero (strict) and one (lax). Defaults to `0.2`. + * a comparator function to use, either `"pixelmatch"` or `"ssim-cie94"`. Defaults to `"pixelmatch"`. + */ + comparator?: string; + + /** + * an acceptable perceived color difference between the same pixel in compared images, ranging from `0` (strict) and + * `1` (lax). `"pixelmatch"` comparator computes color difference in + * [YIQ color space](https://en.wikipedia.org/wiki/YIQ) and defaults `threshold` value to `0.2`. `"ssim-cie94"` + * comparator computes color difference by [CIE94](https://en.wikipedia.org/wiki/Color_difference#CIE94) and defaults + * `threshold` value to `0.01`. */ threshold?: number; @@ -576,8 +584,16 @@ interface TestConfig { */ toMatchSnapshot?: { /** - * an acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between the - * same pixel in compared images, between zero (strict) and one (lax). Defaults to `0.2`. + * a comparator function to use, either `"pixelmatch"` or `"ssim-cie94"`. Defaults to `"pixelmatch"`. + */ + comparator?: string; + + /** + * an acceptable perceived color difference between the same pixel in compared images, ranging from `0` (strict) and + * `1` (lax). `"pixelmatch"` comparator computes color difference in + * [YIQ color space](https://en.wikipedia.org/wiki/YIQ) and defaults `threshold` value to `0.2`. `"ssim-cie94"` + * comparator computes color difference by [CIE94](https://en.wikipedia.org/wiki/Color_difference#CIE94) and defaults + * `threshold` value to `0.01`. */ threshold?: number; @@ -3845,6 +3861,11 @@ interface LocatorAssertions { */ caret?: "hide"|"initial"; + /** + * A comparator function to use when comparing images. + */ + comparator?: string; + /** * Specify locators that should be masked when the screenshot is taken. Masked elements will be overlaid with a pink * box `#FF00FF` that completely covers its bounding box. @@ -3922,6 +3943,11 @@ interface LocatorAssertions { */ caret?: "hide"|"initial"; + /** + * A comparator function to use when comparing images. + */ + comparator?: string; + /** * Specify locators that should be masked when the screenshot is taken. Masked elements will be overlaid with a pink * box `#FF00FF` that completely covers its bounding box. @@ -4170,6 +4196,11 @@ interface PageAssertions { height: number; }; + /** + * A comparator function to use when comparing images. + */ + comparator?: string; + /** * When true, takes a screenshot of the full scrollable page, instead of the currently visible viewport. Defaults to * `false`. @@ -4277,6 +4308,11 @@ interface PageAssertions { height: number; }; + /** + * A comparator function to use when comparing images. + */ + comparator?: string; + /** * When true, takes a screenshot of the full scrollable page, instead of the currently visible viewport. Defaults to * `false`. @@ -4407,6 +4443,11 @@ interface SnapshotAssertions { * @param options */ toMatchSnapshot(name: string|Array, options?: { + /** + * A comparator function to use when comparing images. + */ + comparator?: string; + /** * An acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1`. Default is * configurable with `TestConfig.expect`. Unset by default. @@ -4455,6 +4496,11 @@ interface SnapshotAssertions { * @param options */ toMatchSnapshot(options?: { + /** + * A comparator function to use when comparing images. + */ + comparator?: string; + /** * An acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1`. Default is * configurable with `TestConfig.expect`. Unset by default. @@ -4581,8 +4627,16 @@ interface TestProject { */ toHaveScreenshot?: { /** - * an acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between the - * same pixel in compared images, between zero (strict) and one (lax). Defaults to `0.2`. + * a comparator function to use, either `"pixelmatch"` or `"ssim-cie94"`. Defaults to `"pixelmatch"`. + */ + comparator?: string; + + /** + * an acceptable perceived color difference between the same pixel in compared images, ranging from `0` (strict) and + * `1` (lax). `"pixelmatch"` comparator computes color difference in + * [YIQ color space](https://en.wikipedia.org/wiki/YIQ) and defaults `threshold` value to `0.2`. `"ssim-cie94"` + * comparator computes color difference by [CIE94](https://en.wikipedia.org/wiki/Color_difference#CIE94) and defaults + * `threshold` value to `0.01`. */ threshold?: number; @@ -4623,8 +4677,16 @@ interface TestProject { */ toMatchSnapshot?: { /** - * an acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between the - * same pixel in compared images, between zero (strict) and one (lax). Defaults to `0.2`. + * a comparator function to use, either `"pixelmatch"` or `"ssim-cie94"`. Defaults to `"pixelmatch"`. + */ + comparator?: string; + + /** + * an acceptable perceived color difference between the same pixel in compared images, ranging from `0` (strict) and + * `1` (lax). `"pixelmatch"` comparator computes color difference in + * [YIQ color space](https://en.wikipedia.org/wiki/YIQ) and defaults `threshold` value to `0.2`. `"ssim-cie94"` + * comparator computes color difference by [CIE94](https://en.wikipedia.org/wiki/Color_difference#CIE94) and defaults + * `threshold` value to `0.01`. */ threshold?: number; diff --git a/packages/protocol/src/channels.ts b/packages/protocol/src/channels.ts index 6b451a99aa..985c8c405c 100644 --- a/packages/protocol/src/channels.ts +++ b/packages/protocol/src/channels.ts @@ -1825,6 +1825,7 @@ export type PageExpectScreenshotParams = { selector: string, }, comparatorOptions?: { + comparator?: string, maxDiffPixels?: number, maxDiffPixelRatio?: number, threshold?: number, @@ -1850,6 +1851,7 @@ export type PageExpectScreenshotOptions = { selector: string, }, comparatorOptions?: { + comparator?: string, maxDiffPixels?: number, maxDiffPixelRatio?: number, threshold?: number, diff --git a/packages/protocol/src/protocol.yml b/packages/protocol/src/protocol.yml index b630f0d18a..eb1ee8a668 100644 --- a/packages/protocol/src/protocol.yml +++ b/packages/protocol/src/protocol.yml @@ -1266,6 +1266,7 @@ Page: comparatorOptions: type: object? properties: + comparator: string? maxDiffPixels: number? maxDiffPixelRatio: number? threshold: number? diff --git a/tests/image_tools/fixtures.spec.ts b/tests/image_tools/fixtures.spec.ts new file mode 100644 index 0000000000..11dbf1e296 --- /dev/null +++ b/tests/image_tools/fixtures.spec.ts @@ -0,0 +1,91 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test, expect } from '../playwright-test/stable-test-runner'; +import { PNG } from 'playwright-core/lib/utilsBundle'; +import { compare } from 'playwright-core/lib/image_tools/compare'; +import fs from 'fs'; +import path from 'path'; + +function listFixtures(root: string, fixtures: Set = new Set()) { + for (const item of fs.readdirSync(root, { withFileTypes: true })) { + const p = path.join(root, item.name); + if (item.isDirectory()) + listFixtures(p, fixtures); + else if (item.isFile() && p.endsWith('-actual.png')) + fixtures.add(p.substring(0, p.length - '-actual.png'.length)); + } + return fixtures; +} + +const FIXTURES_DIR = path.join(__dirname, 'fixtures'); + +function declareFixtureTest(fixtureRoot: string, fixtureName: string, shouldMatch: boolean) { + test(path.relative(fixtureRoot, fixtureName), async ({}, testInfo) => { + const [actual, expected] = await Promise.all([ + fs.promises.readFile(fixtureName + '-actual.png'), + fs.promises.readFile(fixtureName + '-expected.png'), + ]); + testInfo.attach(fixtureName + '-actual.png', { + body: actual, + contentType: 'image/png', + }); + testInfo.attach(fixtureName + '-expected.png', { + body: expected, + contentType: 'image/png', + }); + const actualPNG = PNG.sync.read(actual); + const expectedPNG = PNG.sync.read(expected); + expect(actualPNG.width).toBe(expectedPNG.width); + expect(actualPNG.height).toBe(expectedPNG.height); + + const diffPNG = new PNG({ width: actualPNG.width, height: actualPNG.height }); + const diffCount = compare(actualPNG.data, expectedPNG.data, diffPNG.data, actualPNG.width, actualPNG.height, { + maxColorDeltaE94: 1.0, + }); + + testInfo.attach(fixtureName + '-diff.png', { + body: PNG.sync.write(diffPNG), + contentType: 'image/png', + }); + + if (shouldMatch) + expect(diffCount).toBe(0); + else + expect(diffCount).not.toBe(0); + }); +} + +test.describe('basic fixtures', () => { + test.describe.configure({ mode: 'parallel' }); + + for (const fixtureName of listFixtures(path.join(FIXTURES_DIR, 'should-match'))) + declareFixtureTest(FIXTURES_DIR, fixtureName, true /* shouldMatch */); + for (const fixtureName of listFixtures(path.join(FIXTURES_DIR, 'should-fail'))) + declareFixtureTest(FIXTURES_DIR, fixtureName, false /* shouldMatch */); +}); + +const customImageToolsFixtures = process.env.IMAGE_TOOLS_FIXTURES; +if (customImageToolsFixtures) { + test.describe('custom fixtures', () => { + test.describe.configure({ mode: 'parallel' }); + + for (const fixtureName of listFixtures(path.join(customImageToolsFixtures, 'should-match'))) + declareFixtureTest(customImageToolsFixtures, fixtureName, true /* shouldMatch */); + for (const fixtureName of listFixtures(path.join(customImageToolsFixtures, 'should-fail'))) + declareFixtureTest(customImageToolsFixtures, fixtureName, false /* shouldMatch */); + }); +} diff --git a/tests/image_tools/fixtures/should-fail/julia-ssim-trap/1-actual.png b/tests/image_tools/fixtures/should-fail/julia-ssim-trap/1-actual.png new file mode 100644 index 0000000000..11bce6daa3 Binary files /dev/null and b/tests/image_tools/fixtures/should-fail/julia-ssim-trap/1-actual.png differ diff --git a/tests/image_tools/fixtures/should-fail/julia-ssim-trap/1-expected.png b/tests/image_tools/fixtures/should-fail/julia-ssim-trap/1-expected.png new file mode 100644 index 0000000000..16646f5e63 Binary files /dev/null and b/tests/image_tools/fixtures/should-fail/julia-ssim-trap/1-expected.png differ diff --git a/tests/image_tools/fixtures/should-fail/julia-ssim-trap/README.md b/tests/image_tools/fixtures/should-fail/julia-ssim-trap/README.md new file mode 100644 index 0000000000..b34cebb958 --- /dev/null +++ b/tests/image_tools/fixtures/should-fail/julia-ssim-trap/README.md @@ -0,0 +1,11 @@ +# Julia SSIM trap + +[SSIM](https://en.wikipedia.org/wiki/Structural_similarity) is a metric used to compare image similarity. + +While original SSIM is computed against the luma channel (i.e. in a gray-scale), +the Julia language [computes a weighted combination of per-channel SSIM's](https://github.com/JuliaImages/ImageQualityIndexes.jl/blob/e014cee9bef7023a1047b6eb0cbe49fbf28f2fed/src/ssim.jl#L39-L41). + +This sample is a white image and a gray image that are reported equal by Julia SSIM. +It also traps all the suggestions for color-weighted SSIM given here: +https://dsp.stackexchange.com/questions/75187/how-to-apply-the-ssim-measure-on-rgb-images + diff --git a/tests/image_tools/fixtures/should-fail/original-ssim-trap/README.md b/tests/image_tools/fixtures/should-fail/original-ssim-trap/README.md new file mode 100644 index 0000000000..1bea83201b --- /dev/null +++ b/tests/image_tools/fixtures/should-fail/original-ssim-trap/README.md @@ -0,0 +1,8 @@ +# Original SSIM trap + +[SSIM](https://en.wikipedia.org/wiki/Structural_similarity) is a metric used to compare image similarity. + +The sample provides two different images. However, since the original SSIM implementation +[converts images into +gray-scale](https://github.com/obartra/ssim/blob/ca8e3c6a6ff5f4f2e232239e0c3d91806f3c97d5/src/index.ts#L104), +SSIM metric will yield a perfect match for these images. diff --git a/tests/image_tools/fixtures/should-fail/original-ssim-trap/sample-actual.png b/tests/image_tools/fixtures/should-fail/original-ssim-trap/sample-actual.png new file mode 100644 index 0000000000..467a057dec Binary files /dev/null and b/tests/image_tools/fixtures/should-fail/original-ssim-trap/sample-actual.png differ diff --git a/tests/image_tools/fixtures/should-fail/original-ssim-trap/sample-expected.png b/tests/image_tools/fixtures/should-fail/original-ssim-trap/sample-expected.png new file mode 100644 index 0000000000..cd762892c1 Binary files /dev/null and b/tests/image_tools/fixtures/should-fail/original-ssim-trap/sample-expected.png differ diff --git a/tests/image_tools/fixtures/should-fail/trivial/README.md b/tests/image_tools/fixtures/should-fail/trivial/README.md new file mode 100644 index 0000000000..01e9e6af59 --- /dev/null +++ b/tests/image_tools/fixtures/should-fail/trivial/README.md @@ -0,0 +1,3 @@ +# Trivial failing examples + +Some trivial failing examples diff --git a/tests/image_tools/fixtures/should-fail/trivial/equal-luma-actual.png b/tests/image_tools/fixtures/should-fail/trivial/equal-luma-actual.png new file mode 100644 index 0000000000..b405bbe3c0 Binary files /dev/null and b/tests/image_tools/fixtures/should-fail/trivial/equal-luma-actual.png differ diff --git a/tests/image_tools/fixtures/should-fail/trivial/equal-luma-expected.png b/tests/image_tools/fixtures/should-fail/trivial/equal-luma-expected.png new file mode 100644 index 0000000000..816f9db781 Binary files /dev/null and b/tests/image_tools/fixtures/should-fail/trivial/equal-luma-expected.png differ diff --git a/tests/image_tools/fixtures/should-fail/trivial/opposite-actual.png b/tests/image_tools/fixtures/should-fail/trivial/opposite-actual.png new file mode 100644 index 0000000000..cfe88317b8 Binary files /dev/null and b/tests/image_tools/fixtures/should-fail/trivial/opposite-actual.png differ diff --git a/tests/image_tools/fixtures/should-fail/trivial/opposite-expected.png b/tests/image_tools/fixtures/should-fail/trivial/opposite-expected.png new file mode 100644 index 0000000000..11bce6daa3 Binary files /dev/null and b/tests/image_tools/fixtures/should-fail/trivial/opposite-expected.png differ diff --git a/tests/image_tools/fixtures/should-fail/trivial/single-red-pixel-actual.png b/tests/image_tools/fixtures/should-fail/trivial/single-red-pixel-actual.png new file mode 100644 index 0000000000..f102657460 Binary files /dev/null and b/tests/image_tools/fixtures/should-fail/trivial/single-red-pixel-actual.png differ diff --git a/tests/image_tools/fixtures/should-fail/trivial/single-red-pixel-expected.png b/tests/image_tools/fixtures/should-fail/trivial/single-red-pixel-expected.png new file mode 100644 index 0000000000..edf139cf3a Binary files /dev/null and b/tests/image_tools/fixtures/should-fail/trivial/single-red-pixel-expected.png differ diff --git a/tests/image_tools/fixtures/should-match/crbug-919955/README.md b/tests/image_tools/fixtures/should-match/crbug-919955/README.md new file mode 100644 index 0000000000..518dfdbb36 --- /dev/null +++ b/tests/image_tools/fixtures/should-match/crbug-919955/README.md @@ -0,0 +1,5 @@ +# Chrome non-determenistic rendering + +Reported by: https://bugs.chromium.org/p/chromium/issues/detail?id=919955 + + diff --git a/tests/image_tools/fixtures/should-match/crbug-919955/example-1-actual.png b/tests/image_tools/fixtures/should-match/crbug-919955/example-1-actual.png new file mode 100644 index 0000000000..45591af30d Binary files /dev/null and b/tests/image_tools/fixtures/should-match/crbug-919955/example-1-actual.png differ diff --git a/tests/image_tools/fixtures/should-match/crbug-919955/example-1-expected.png b/tests/image_tools/fixtures/should-match/crbug-919955/example-1-expected.png new file mode 100644 index 0000000000..7d45cb0352 Binary files /dev/null and b/tests/image_tools/fixtures/should-match/crbug-919955/example-1-expected.png differ diff --git a/tests/image_tools/fixtures/should-match/crbug-919955/example-2-actual.png b/tests/image_tools/fixtures/should-match/crbug-919955/example-2-actual.png new file mode 100644 index 0000000000..ee2816b891 Binary files /dev/null and b/tests/image_tools/fixtures/should-match/crbug-919955/example-2-actual.png differ diff --git a/tests/image_tools/fixtures/should-match/crbug-919955/example-2-expected.png b/tests/image_tools/fixtures/should-match/crbug-919955/example-2-expected.png new file mode 100644 index 0000000000..5b760ccbfe Binary files /dev/null and b/tests/image_tools/fixtures/should-match/crbug-919955/example-2-expected.png differ diff --git a/tests/image_tools/fixtures/should-match/tiny-antialiasing-sample/README.md b/tests/image_tools/fixtures/should-match/tiny-antialiasing-sample/README.md new file mode 100644 index 0000000000..f65b71ed44 --- /dev/null +++ b/tests/image_tools/fixtures/should-match/tiny-antialiasing-sample/README.md @@ -0,0 +1,5 @@ +# Tiny anti-aliasing sample + +This is a 10x10 image sample with a 3 anti-aliased pixels in-between. This is actually +a cropped down snapshot of one of the [ubuntu-x86-vs-ubunu-arm samples](../ubuntu-x86-vs-ubuntu-arm/samples/stylings_stories-Stylings-stories-Texture-bar-should-use-custom-path-chrome/stylings-stories/texture/bar/should-use-custom-path-actual.png) handy for debugging +purposes. diff --git a/tests/image_tools/fixtures/should-match/tiny-antialiasing-sample/tiny-actual.png b/tests/image_tools/fixtures/should-match/tiny-antialiasing-sample/tiny-actual.png new file mode 100644 index 0000000000..1626202d59 Binary files /dev/null and b/tests/image_tools/fixtures/should-match/tiny-antialiasing-sample/tiny-actual.png differ diff --git a/tests/image_tools/fixtures/should-match/tiny-antialiasing-sample/tiny-expected.png b/tests/image_tools/fixtures/should-match/tiny-antialiasing-sample/tiny-expected.png new file mode 100644 index 0000000000..7f3d87a00b Binary files /dev/null and b/tests/image_tools/fixtures/should-match/tiny-antialiasing-sample/tiny-expected.png differ diff --git a/tests/image_tools/fixtures/should-match/trivial/README.md b/tests/image_tools/fixtures/should-match/trivial/README.md new file mode 100644 index 0000000000..919307f0f9 --- /dev/null +++ b/tests/image_tools/fixtures/should-match/trivial/README.md @@ -0,0 +1,3 @@ +# Equal small images + +Simple equal images. diff --git a/tests/image_tools/fixtures/should-match/trivial/black-actual.png b/tests/image_tools/fixtures/should-match/trivial/black-actual.png new file mode 100644 index 0000000000..cfe88317b8 Binary files /dev/null and b/tests/image_tools/fixtures/should-match/trivial/black-actual.png differ diff --git a/tests/image_tools/fixtures/should-match/trivial/black-expected.png b/tests/image_tools/fixtures/should-match/trivial/black-expected.png new file mode 100644 index 0000000000..cfe88317b8 Binary files /dev/null and b/tests/image_tools/fixtures/should-match/trivial/black-expected.png differ diff --git a/tests/image_tools/fixtures/should-match/trivial/white-actual.png b/tests/image_tools/fixtures/should-match/trivial/white-actual.png new file mode 100644 index 0000000000..11bce6daa3 Binary files /dev/null and b/tests/image_tools/fixtures/should-match/trivial/white-actual.png differ diff --git a/tests/image_tools/fixtures/should-match/trivial/white-expected.png b/tests/image_tools/fixtures/should-match/trivial/white-expected.png new file mode 100644 index 0000000000..11bce6daa3 Binary files /dev/null and b/tests/image_tools/fixtures/should-match/trivial/white-expected.png differ diff --git a/tests/image_tools/unit.spec.ts b/tests/image_tools/unit.spec.ts new file mode 100644 index 0000000000..f565a5d0a0 --- /dev/null +++ b/tests/image_tools/unit.spec.ts @@ -0,0 +1,119 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test } from '../playwright-test/stable-test-runner'; +import { ssim, FastStats } from 'playwright-core/lib/image_tools/stats'; +import { ImageChannel } from 'playwright-core/lib/image_tools/imageChannel'; +import { srgb2xyz, xyz2lab, colorDeltaE94 } from 'playwright-core/lib/image_tools/colorUtils'; +import referenceSSIM from 'ssim.js'; +import { randomPNG, assertEqual, grayChannel } from './utils'; + +test('srgb to lab conversion should work', async () => { + const srgb = [123, 81, 252]; + const [x, y, z] = srgb2xyz(srgb); + // Values obtained with http://colormine.org/convert/rgb-to-xyz + assertEqual(x, 0.28681495837305815); + assertEqual(y, 0.17124087944445404); + assertEqual(z, 0.938890585081072); + const [l, a, b] = xyz2lab([x, y, z]); + // Values obtained with http://colormine.org/convert/rgb-to-lab + assertEqual(l, 48.416007793699535); + assertEqual(a, 57.71275605467668); + assertEqual(b, -79.29993619401066); +}); + +test('colorDeltaE94 should work', async () => { + const rgb1 = [123, 81, 252]; + const rgb2 = [43, 201, 100]; + // Value obtained with http://colormine.org/delta-e-calculator/cie94 + assertEqual(colorDeltaE94(rgb1, rgb2), 71.2159); +}); + +test('fast stats and naive computation should match', async () => { + const N = 13, M = 17; + const png1 = randomPNG(N, M, 239); + const png2 = randomPNG(N, M, 261); + const [r1] = ImageChannel.intoRGB(png1.width, png1.height, png1.data); + const [r2] = ImageChannel.intoRGB(png2.width, png2.height, png2.data); + const fastStats = new FastStats(r1, r2); + + for (let x1 = 0; x1 < png1.width; ++x1) { + for (let y1 = 0; y1 < png1.height; ++y1) { + for (let x2 = x1; x2 < png1.width; ++x2) { + for (let y2 = y1; y2 < png1.height; ++y2) { + assertEqual(fastStats.meanC1(x1, y1, x2, y2), computeMean(r1, x1, y1, x2, y2)); + assertEqual(fastStats.varianceC1(x1, y1, x2, y2), computeVariance(r1, x1, y1, x2, y2)); + assertEqual(fastStats.covariance(x1, y1, x2, y2), computeCovariance(r1, r2, x1, y1, x2, y2)); + } + } + } + } +}); + +test('ssim + fastStats should match "weber" algorithm from ssim.js', async () => { + const N = 200; + const png1 = randomPNG(N, N, 239); + const png2 = randomPNG(N, N, 261); + const windowRadius = 5; + const refSSIM = referenceSSIM(png1 as any, png2 as any, { + downsample: false, + ssim: 'weber', + windowSize: windowRadius * 2 + 1, + }); + const gray1 = grayChannel(png1); + const gray2 = grayChannel(png2); + const fastStats = new FastStats(gray1, gray2); + for (let y = windowRadius; y < N - windowRadius; ++y) { + for (let x = windowRadius; x < N - windowRadius; ++x) { + const customSSIM = ssim(fastStats, x - windowRadius, y - windowRadius, x + windowRadius, y + windowRadius); + const reference = refSSIM.ssim_map.data[(y - windowRadius) * refSSIM.ssim_map.width + x - windowRadius]; + assertEqual(customSSIM, reference); + } + } +}); + +function computeMean(c: ImageChannel, x1: number, y1: number, x2: number, y2: number) { + let result = 0; + const N = (x2 - x1 + 1) * (y2 - y1 + 1); + for (let y = y1; y <= y2; ++y) { + for (let x = x1; x <= x2; ++x) + result += c.get(x, y); + } + return result / N; +} + +function computeVariance(c: ImageChannel, x1: number, y1: number, x2: number, y2: number) { + let result = 0; + const mean = computeMean(c, x1, y1, x2, y2); + const N = (x2 - x1 + 1) * (y2 - y1 + 1); + for (let y = y1; y <= y2; ++y) { + for (let x = x1; x <= x2; ++x) + result += (c.get(x, y) - mean) ** 2; + } + return result / N; +} + +function computeCovariance(c1: ImageChannel, c2: ImageChannel, x1: number, y1: number, x2: number, y2: number) { + const N = (x2 - x1 + 1) * (y2 - y1 + 1); + const mean1 = computeMean(c1, x1, y1, x2, y2); + const mean2 = computeMean(c2, x1, y1, x2, y2); + let result = 0; + for (let y = y1; y <= y2; ++y) { + for (let x = x1; x <= x2; ++x) + result += (c1.get(x, y) - mean1) * (c2.get(x, y) - mean2); + } + return result / N; +} diff --git a/tests/image_tools/utils.ts b/tests/image_tools/utils.ts new file mode 100644 index 0000000000..b4c61d5383 --- /dev/null +++ b/tests/image_tools/utils.ts @@ -0,0 +1,61 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PNG } from 'playwright-core/lib/utilsBundle'; +import { ImageChannel } from 'playwright-core/lib/image_tools/imageChannel'; + +// mulberry32 +export function createRandom(seed) { + return function() { + let t = seed += 0x6D2B79F5; + t = Math.imul(t ^ t >>> 15, t | 1); + t ^= t + Math.imul(t ^ t >>> 7, t | 61); + return ((t ^ t >>> 14) >>> 0) / 4294967296; + }; +} + +export function randomPNG(width, height, seed) { + const random = createRandom(seed); + const png = new PNG({ width, height }); + for (let i = 0; i < height; ++i) { + for (let j = 0; j < width; ++j) { + for (let k = 0; k < 4; ++k) + png.data[(i * width + j) * 4 + k] = (random() * 255) | 0; + } + } + return png; +} + +export function assertEqual(value1, value2) { + if (Math.abs(value1 - value2) >= 1e-3) + throw new Error(`ERROR: ${value1} is not equal to ${value2}`); +} + +// NOTE: this is exact formula from SSIM.js and it DOES NOT include alpha. +// We use it to better compare with original SSIM implementation. +export function grayChannel(image: PNG) { + const width = image.width; + const height = image.height; + const gray = new Uint8Array(image.width * image.height); + for (let y = 0; y < image.height; ++y) { + for (let x = 0; x < image.width; ++x) { + const index = y * image.width + x; + const offset = index * 4; + gray[index] = (77 * image.data[offset] + 150 * image.data[offset + 1] + 29 * image.data[offset + 2] + 128) >> 8; + } + } + return new ImageChannel(width, height, gray); +} diff --git a/tests/playwright-test/golden.spec.ts b/tests/playwright-test/golden.spec.ts index 8cb05152f2..102fa55530 100644 --- a/tests/playwright-test/golden.spec.ts +++ b/tests/playwright-test/golden.spec.ts @@ -620,6 +620,80 @@ test('should respect project threshold', async ({ runInlineTest }) => { expect(result.exitCode).toBe(0); }); +test('should respect comparator name', async ({ runInlineTest }) => { + const expected = fs.readFileSync(path.join(__dirname, '../image_tools/fixtures/should-match/tiny-antialiasing-sample/tiny-expected.png')); + const actual = fs.readFileSync(path.join(__dirname, '../image_tools/fixtures/should-match/tiny-antialiasing-sample/tiny-actual.png')); + const result = await runInlineTest({ + ...files, + 'a.spec.js-snapshots/snapshot.png': expected, + 'a.spec.js': ` + const { test } = require('./helper'); + test('should pass', ({}) => { + expect(Buffer.from('${actual.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png', { + threshold: 0, + comparator: 'ssim-cie94', + }); + }); + test('should fail', ({}) => { + expect(Buffer.from('${actual.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png', { + threshold: 0, + comparator: 'pixelmatch', + }); + }); + ` + }); + expect(result.exitCode).toBe(1); + expect(result.report.suites[0].specs[0].title).toBe('should pass'); + expect(result.report.suites[0].specs[0].ok).toBe(true); + expect(result.report.suites[0].specs[1].title).toBe('should fail'); + expect(result.report.suites[0].specs[1].ok).toBe(false); +}); + +test('should respect comparator in config', async ({ runInlineTest }) => { + const expected = fs.readFileSync(path.join(__dirname, '../image_tools/fixtures/should-match/tiny-antialiasing-sample/tiny-expected.png')); + const actual = fs.readFileSync(path.join(__dirname, '../image_tools/fixtures/should-match/tiny-antialiasing-sample/tiny-actual.png')); + const result = await runInlineTest({ + ...files, + 'playwright.config.ts': ` + module.exports = { + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + projects: [ + { + name: 'should-pass', + expect: { + toMatchSnapshot: { + comparator: 'ssim-cie94', + } + }, + }, + { + name: 'should-fail', + expect: { + toMatchSnapshot: { + comparator: 'pixelmatch', + } + }, + }, + ], + }; + `, + '__screenshots__/a.spec.js/snapshot.png': expected, + 'a.spec.js': ` + const { test } = require('./helper'); + test('test', ({}) => { + expect(Buffer.from('${actual.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png', { + threshold: 0, + }); + }); + ` + }); + expect(result.exitCode).toBe(1); + expect(result.report.suites[0].specs[0].tests[0].projectName).toBe('should-pass'); + expect(result.report.suites[0].specs[0].tests[0].status).toBe('expected'); + expect(result.report.suites[0].specs[0].tests[1].projectName).toBe('should-fail'); + expect(result.report.suites[0].specs[0].tests[1].status).toBe('unexpected'); +}); + test('should sanitize snapshot name when passed as string', async ({ runInlineTest }) => { const result = await runInlineTest({ ...files, diff --git a/tests/playwright-test/playwright.config.ts b/tests/playwright-test/playwright.config.ts index 518d56907e..64570a8f18 100644 --- a/tests/playwright-test/playwright.config.ts +++ b/tests/playwright-test/playwright.config.ts @@ -22,15 +22,20 @@ import * as path from 'path'; const outputDir = path.join(__dirname, '..', '..', 'test-results'); const config: Config = { - testDir: __dirname, - testIgnore: ['assets/**', 'stable-test-runner/**'], timeout: 30000, forbidOnly: !!process.env.CI, workers: process.env.CI ? 2 : undefined, preserveOutput: process.env.CI ? 'failures-only' : 'always', projects: [ { - name: 'playwright-test' + name: 'playwright-test', + testDir: __dirname, + testIgnore: ['assets/**', 'stable-test-runner/**'], + }, + { + name: 'image_tools', + testDir: path.join(__dirname, '../image_tools'), + testIgnore: [path.join(__dirname, '../fixtures/**')], }, ], reporter: process.env.CI ? [ diff --git a/tests/playwright-test/to-have-screenshot.spec.ts b/tests/playwright-test/to-have-screenshot.spec.ts index 0436b49b1c..dd121e8308 100644 --- a/tests/playwright-test/to-have-screenshot.spec.ts +++ b/tests/playwright-test/to-have-screenshot.spec.ts @@ -1022,6 +1022,78 @@ test('should update expectations with retries', async ({ runInlineTest }, testIn expect(comparePNGs(data, whiteImage)).toBe(null); }); +test('should respect comparator name', async ({ runInlineTest }) => { + const expected = fs.readFileSync(path.join(__dirname, '../image_tools/fixtures/should-match/tiny-antialiasing-sample/tiny-expected.png')); + const actualURL = pathToFileURL(path.join(__dirname, '../image_tools/fixtures/should-match/tiny-antialiasing-sample/tiny-actual.png')); + const result = await runInlineTest({ + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + }), + '__screenshots__/a.spec.js/snapshot.png': expected, + 'a.spec.js': ` + pwt.test('should pass', async ({ page }) => { + await page.goto('${actualURL}'); + await expect(page.locator('img')).toHaveScreenshot('snapshot.png', { + threshold: 0, + comparator: 'ssim-cie94', + }); + }); + pwt.test('should fail', async ({ page }) => { + await page.goto('${actualURL}'); + await expect(page.locator('img')).toHaveScreenshot('snapshot.png', { + threshold: 0, + comparator: 'pixelmatch', + }); + }); + ` + }); + expect(result.exitCode).toBe(1); + expect(result.report.suites[0].specs[0].title).toBe('should pass'); + expect(result.report.suites[0].specs[0].ok).toBe(true); + expect(result.report.suites[0].specs[1].title).toBe('should fail'); + expect(result.report.suites[0].specs[1].ok).toBe(false); +}); + +test('should respect comparator in config', async ({ runInlineTest }) => { + const expected = fs.readFileSync(path.join(__dirname, '../image_tools/fixtures/should-match/tiny-antialiasing-sample/tiny-expected.png')); + const actualURL = pathToFileURL(path.join(__dirname, '../image_tools/fixtures/should-match/tiny-antialiasing-sample/tiny-actual.png')); + const result = await runInlineTest({ + ...playwrightConfig({ + snapshotPathTemplate: '__screenshots__/{testFilePath}/{arg}{ext}', + projects: [ + { + name: 'should-pass', + expect: { + toHaveScreenshot: { + comparator: 'ssim-cie94', + } + }, + }, + { + name: 'should-fail', + expect: { + toHaveScreenshot: { + comparator: 'pixelmatch', + } + }, + }, + ], + }), + '__screenshots__/a.spec.js/snapshot.png': expected, + 'a.spec.js': ` + pwt.test('test', async ({ page }) => { + await page.goto('${actualURL}'); + await expect(page.locator('img')).toHaveScreenshot('snapshot.png', { threshold: 0, }); + }); + ` + }); + expect(result.exitCode).toBe(1); + expect(result.report.suites[0].specs[0].tests[0].projectName).toBe('should-pass'); + expect(result.report.suites[0].specs[0].tests[0].status).toBe('expected'); + expect(result.report.suites[0].specs[0].tests[1].projectName).toBe('should-fail'); + expect(result.report.suites[0].specs[0].tests[1].status).toBe('unexpected'); +}); + function playwrightConfig(obj: any) { return { 'playwright.config.js': `