From caec93ef42b4b93185cf63ecc95fb78cadb2fcf1 Mon Sep 17 00:00:00 2001 From: Andrey Lushnikov Date: Thu, 29 Dec 2022 23:48:28 -0800 Subject: [PATCH] test: enable ssim-cie94 image comparison algorithm for our tests (#19313) --- .../playwright-core/src/image_tools/compare.ts | 15 ++++++++------- tests/config/comparator.ts | 2 +- tests/library/chromium/oopif.spec.ts | 2 +- tests/library/headful.spec.ts | 4 ++-- tests/library/playwright.config.ts | 2 ++ tests/library/screenshot.spec.ts | 6 ++++-- tests/page/page-screenshot.spec.ts | 6 +++--- 7 files changed, 21 insertions(+), 16 deletions(-) diff --git a/packages/playwright-core/src/image_tools/compare.ts b/packages/playwright-core/src/image_tools/compare.ts index e1ffdc0766..9ce86d91ec 100644 --- a/packages/playwright-core/src/image_tools/compare.ts +++ b/packages/playwright-core/src/image_tools/compare.ts @@ -30,12 +30,12 @@ function drawPixel(width: number, data: Buffer, x: number, y: number, r: number, } type CompareOptions = { - maxColorDeltaE94: number; + maxColorDeltaE94?: number; }; -export function compare(actual: Buffer, expected: Buffer, diff: Buffer, width: number, height: number, options: CompareOptions) { +export function compare(actual: Buffer, expected: Buffer, diff: Buffer|null, width: number, height: number, options: CompareOptions = {}) { const { - maxColorDeltaE94 + maxColorDeltaE94 = 1.0, } = options; const paddingSize = Math.max(VARIANCE_WINDOW_RADIUS, SSIM_WINDOW_RADIUS); @@ -52,13 +52,14 @@ export function compare(actual: Buffer, expected: Buffer, diff: Buffer, width: n paddingColorOdd, }); - const drawRedPixel = (x: number, y: number) => drawPixel(width, diff, x - paddingSize, y - paddingSize, 255, 0, 0); - const drawYellowPixel = (x: number, y: number) => drawPixel(width, diff, x - paddingSize, y - paddingSize, 255, 255, 0); - const drawGrayPixel = (x: number, y: number) => { + const noop = (x: number, y: number) => {}; + const drawRedPixel = diff ? (x: number, y: number) => drawPixel(width, diff, x - paddingSize, y - paddingSize, 255, 0, 0) : noop; + const drawYellowPixel = diff ? (x: number, y: number) => drawPixel(width, diff, x - paddingSize, y - paddingSize, 255, 255, 0) : noop; + const drawGrayPixel = diff ? (x: number, y: number) => { const gray = rgb2gray(r1.get(x, y), g1.get(x, y), b1.get(x, y)); const value = blendWithWhite(gray, 0.1); drawPixel(width, diff, x - paddingSize, y - paddingSize, value, value, value); - }; + } : noop; let fastR, fastG, fastB; diff --git a/tests/config/comparator.ts b/tests/config/comparator.ts index 02dec5272c..1308033aa1 100644 --- a/tests/config/comparator.ts +++ b/tests/config/comparator.ts @@ -22,5 +22,5 @@ type ImageComparatorOptions = { threshold?: number, maxDiffPixels?: number, maxD export function comparePNGs(actual: Buffer, expected: Buffer, options: ImageComparatorOptions = {}): ComparatorResult { // Strict threshold by default in our tests. - return pngComparator(actual, expected, { threshold: 0, ...options }); + return pngComparator(actual, expected, { _comparator: 'ssim-cie94', threshold: 0, ...options }); } diff --git a/tests/library/chromium/oopif.spec.ts b/tests/library/chromium/oopif.spec.ts index cd61a75448..2903b80c3b 100644 --- a/tests/library/chromium/oopif.spec.ts +++ b/tests/library/chromium/oopif.spec.ts @@ -153,7 +153,7 @@ it('should take screenshot', async ({ page, browser, server }) => { await page.goto(server.PREFIX + '/dynamic-oopif.html'); expect(page.frames().length).toBe(2); expect(await countOOPIFs(browser)).toBe(1); - expect(await page.screenshot()).toMatchSnapshot('screenshot-oopif.png', { threshold: 0.3 }); + expect(await page.screenshot()).toMatchSnapshot('screenshot-oopif.png'); }); it('should load oopif iframes with subresources and route', async function({ page, browser, server }) { diff --git a/tests/library/headful.spec.ts b/tests/library/headful.spec.ts index b70e4481a0..ce17a4ed3c 100644 --- a/tests/library/headful.spec.ts +++ b/tests/library/headful.spec.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import pixelmatch from '../../packages/playwright-core/src/third_party/pixelmatch'; +import { compare } from 'playwright-core/lib/image_tools/compare'; import { PNG } from 'playwright-core/lib/utilsBundle'; import { expect, playwrightTest as it } from '../config/browserTest'; @@ -280,7 +280,7 @@ it('headless and headful should use same default fonts', async ({ page, headless const [image1, image2] = (await Promise.all([ page.screenshot(), headedPage.screenshot() ])).map(buffer => PNG.sync.read(buffer)); - const count = pixelmatch(image1.data, image2.data, null, image1.width, image2.height, { threshold: 0.01 }); + const count = compare(image1.data, image2.data, null, image1.width, image2.height); expect(count).toBe(0); } await headedBrowser.close(); diff --git a/tests/library/playwright.config.ts b/tests/library/playwright.config.ts index a05870d7e7..21e2f310e5 100644 --- a/tests/library/playwright.config.ts +++ b/tests/library/playwright.config.ts @@ -51,6 +51,8 @@ const config: Config { await context.close(); }); - browserTest('should work with device scale factor', async ({ browser, server }) => { + browserTest('should work with device scale factor', async ({ browser, server, isMac, browserName }) => { + browserTest.fixme(isMac && browserName === 'webkit'); const context = await browser.newContext({ viewport: { width: 320, height: 480 }, deviceScaleFactor: 2 }); const page = await context.newPage(); await page.goto(server.PREFIX + '/grid.html'); @@ -213,8 +214,9 @@ browserTest.describe('element screenshot', () => { await context.close(); }); - browserTest('element screenshot should work with device scale factor', async ({ browser, server, browserName }) => { + browserTest('element screenshot should work with device scale factor', async ({ browser, server, browserName, isMac }) => { browserTest.skip(browserName === 'firefox'); + browserTest.fixme(isMac && browserName === 'webkit'); const context = await browser.newContext({ viewport: { width: 320, height: 480 }, deviceScaleFactor: 2 }); const page = await context.newPage(); diff --git a/tests/page/page-screenshot.spec.ts b/tests/page/page-screenshot.spec.ts index 12258cdf39..de4c44c06e 100644 --- a/tests/page/page-screenshot.spec.ts +++ b/tests/page/page-screenshot.spec.ts @@ -272,7 +272,7 @@ it.describe('page screenshot', () => { await page.setViewportSize({ width: 500, height: 500 }); await page.goto(server.PREFIX + '/screenshots/canvas.html'); const screenshot = await page.screenshot(); - expect(screenshot).toMatchSnapshot('screenshot-canvas.png', { threshold: 0.4 }); + expect(screenshot).toMatchSnapshot('screenshot-canvas.png'); }); it('should capture canvas changes', async ({ page, isElectron, browserName, isMac, isWebView2 }) => { @@ -625,7 +625,7 @@ it.describe('page screenshot animations', () => { const buffer1 = await page.screenshot(); await rafraf(page); const buffer2 = await page.screenshot(); - expect(comparePNGs(buffer1, buffer2, { threshold: 0.2, maxDiffPixels: 50 })).not.toBe(null); + expect(comparePNGs(buffer1, buffer2, { maxDiffPixels: 50 })).not.toBe(null); }); it('should not capture infinite web animations', async ({ page, server }) => { @@ -645,7 +645,7 @@ it.describe('page screenshot animations', () => { const buffer1 = await page.screenshot(); await rafraf(page); const buffer2 = await page.screenshot(); - expect(comparePNGs(buffer1, buffer2, { threshold: 0.2, maxDiffPixels: 50 })).not.toBe(null); + expect(comparePNGs(buffer1, buffer2, { maxDiffPixels: 50 })).not.toBe(null); }); it('should fire transitionend for finite transitions', async ({ page, server }) => {