test: enable ssim-cie94 image comparison algorithm for our tests (#19313)

packages/playwright-core/src/image_tools/compare.ts

@@ -30,12 +30,12 @@ function drawPixel(width: number, data: Buffer, x: number, y: number, r: number,
 }
 
 type CompareOptions = {
-  maxColorDeltaE94: number;
+  maxColorDeltaE94?: number;
 };
 
-export function compare(actual: Buffer, expected: Buffer, diff: Buffer, width: number, height: number, options: CompareOptions) {
+export function compare(actual: Buffer, expected: Buffer, diff: Buffer|null, width: number, height: number, options: CompareOptions = {}) {
   const {
-    maxColorDeltaE94
+    maxColorDeltaE94 = 1.0,
   } = options;
 
   const paddingSize = Math.max(VARIANCE_WINDOW_RADIUS, SSIM_WINDOW_RADIUS);
@@ -52,13 +52,14 @@ export function compare(actual: Buffer, expected: Buffer, diff: Buffer, width: n
     paddingColorOdd,
   });
 
-  const drawRedPixel = (x: number, y: number) => drawPixel(width, diff, x - paddingSize, y - paddingSize, 255, 0, 0);
+  const noop = (x: number, y: number) => {};
-  const drawYellowPixel = (x: number, y: number) => drawPixel(width, diff, x - paddingSize, y - paddingSize, 255, 255, 0);
+  const drawRedPixel = diff ? (x: number, y: number) => drawPixel(width, diff, x - paddingSize, y - paddingSize, 255, 0, 0) : noop;
-  const drawGrayPixel = (x: number, y: number) => {
+  const drawYellowPixel = diff ? (x: number, y: number) => drawPixel(width, diff, x - paddingSize, y - paddingSize, 255, 255, 0) : noop;
+  const drawGrayPixel = diff ? (x: number, y: number) => {
     const gray = rgb2gray(r1.get(x, y), g1.get(x, y), b1.get(x, y));
     const value = blendWithWhite(gray, 0.1);
     drawPixel(width, diff, x - paddingSize, y - paddingSize, value, value, value);
-  };
+  } : noop;
 
   let fastR, fastG, fastB;
 

tests/config/comparator.ts

@@ -22,5 +22,5 @@ type ImageComparatorOptions = { threshold?: number, maxDiffPixels?: number, maxD
 
 export function comparePNGs(actual: Buffer, expected: Buffer, options: ImageComparatorOptions = {}): ComparatorResult {
   // Strict threshold by default in our tests.
-  return pngComparator(actual, expected, { threshold: 0, ...options });
+  return pngComparator(actual, expected, { _comparator: 'ssim-cie94', threshold: 0, ...options });
 }

tests/library/chromium/oopif.spec.ts

@@ -153,7 +153,7 @@ it('should take screenshot', async ({ page, browser, server }) => {
   await page.goto(server.PREFIX + '/dynamic-oopif.html');
   expect(page.frames().length).toBe(2);
   expect(await countOOPIFs(browser)).toBe(1);
-  expect(await page.screenshot()).toMatchSnapshot('screenshot-oopif.png', { threshold: 0.3 });
+  expect(await page.screenshot()).toMatchSnapshot('screenshot-oopif.png');
 });
 
 it('should load oopif iframes with subresources and route', async function({ page, browser, server }) {

tests/library/headful.spec.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import pixelmatch from '../../packages/playwright-core/src/third_party/pixelmatch';
+import { compare } from 'playwright-core/lib/image_tools/compare';
 import { PNG } from 'playwright-core/lib/utilsBundle';
 import { expect, playwrightTest as it } from '../config/browserTest';
 
@@ -280,7 +280,7 @@ it('headless and headful should use same default fonts', async ({ page, headless
     const [image1, image2] = (await Promise.all([
       page.screenshot(), headedPage.screenshot()
     ])).map(buffer => PNG.sync.read(buffer));
-    const count = pixelmatch(image1.data, image2.data, null, image1.width, image2.height, { threshold: 0.01 });
+    const count = compare(image1.data, image2.data, null, image1.width, image2.height);
     expect(count).toBe(0);
   }
   await headedBrowser.close();

tests/library/playwright.config.ts

@@ -51,6 +51,8 @@ const config: Config<CoverageWorkerOptions & PlaywrightWorkerOptions & Playwrigh
   outputDir,
   expect: {
     timeout: 10000,
+    toHaveScreenshot: { _comparator: 'ssim-cie94' } as any,
+    toMatchSnapshot: { _comparator: 'ssim-cie94' } as any,
   },
   maxFailures: 100,
   timeout: video ? 60000 : 30000,

tests/library/screenshot.spec.ts

@@ -72,7 +72,8 @@ browserTest.describe('page screenshot', () => {
     await context.close();
   });
 
-  browserTest('should work with device scale factor', async ({ browser, server }) => {
+  browserTest('should work with device scale factor', async ({ browser, server, isMac, browserName }) => {
+    browserTest.fixme(isMac && browserName === 'webkit');
     const context = await browser.newContext({ viewport: { width: 320, height: 480 }, deviceScaleFactor: 2 });
     const page = await context.newPage();
     await page.goto(server.PREFIX + '/grid.html');
@@ -213,8 +214,9 @@ browserTest.describe('element screenshot', () => {
     await context.close();
   });
 
-  browserTest('element screenshot should work with device scale factor', async ({ browser, server, browserName }) => {
+  browserTest('element screenshot should work with device scale factor', async ({ browser, server, browserName, isMac }) => {
     browserTest.skip(browserName === 'firefox');
+    browserTest.fixme(isMac && browserName === 'webkit');
 
     const context = await browser.newContext({ viewport: { width: 320, height: 480 }, deviceScaleFactor: 2 });
     const page = await context.newPage();

tests/page/page-screenshot.spec.ts

@@ -272,7 +272,7 @@ it.describe('page screenshot', () => {
     await page.setViewportSize({ width: 500, height: 500 });
     await page.goto(server.PREFIX + '/screenshots/canvas.html');
     const screenshot = await page.screenshot();
-    expect(screenshot).toMatchSnapshot('screenshot-canvas.png', { threshold: 0.4 });
+    expect(screenshot).toMatchSnapshot('screenshot-canvas.png');
   });
 
   it('should capture canvas changes', async ({ page, isElectron, browserName, isMac, isWebView2 }) => {
@@ -625,7 +625,7 @@ it.describe('page screenshot animations', () => {
     const buffer1 = await page.screenshot();
     await rafraf(page);
     const buffer2 = await page.screenshot();
-    expect(comparePNGs(buffer1, buffer2, { threshold: 0.2, maxDiffPixels: 50 })).not.toBe(null);
+    expect(comparePNGs(buffer1, buffer2, { maxDiffPixels: 50 })).not.toBe(null);
   });
 
   it('should not capture infinite web animations', async ({ page, server }) => {
@@ -645,7 +645,7 @@ it.describe('page screenshot animations', () => {
     const buffer1 = await page.screenshot();
     await rafraf(page);
     const buffer2 = await page.screenshot();
-    expect(comparePNGs(buffer1, buffer2, { threshold: 0.2, maxDiffPixels: 50 })).not.toBe(null);
+    expect(comparePNGs(buffer1, buffer2, { maxDiffPixels: 50 })).not.toBe(null);
   });
 
   it('should fire transitionend for finite transitions', async ({ page, server }) => {