fix(test-runner): handle negated toMatchSnapshot result (#7345)

src/test/expect.ts

@@ -21,7 +21,7 @@ import { compare } from './golden';
 
 export const expect: Expect = expectLibrary;
 
-function toMatchSnapshot(received: Buffer | string, nameOrOptions: string | { name: string, threshold?: number }, optOptions: { threshold?: number } = {}) {
+function toMatchSnapshot(this: ReturnType<Expect['getState']>, received: Buffer | string, nameOrOptions: string | { name: string, threshold?: number }, optOptions: { threshold?: number } = {}) {
   let options: { name: string, threshold?: number };
   const testInfo = currentTestInfo();
   if (!testInfo)
@@ -37,7 +37,16 @@ function toMatchSnapshot(received: Buffer | string, nameOrOptions: string | { na
   if (options.threshold === undefined && projectThreshold !== undefined)
     options.threshold = projectThreshold;
 
-  const { pass, message } = compare(received, options.name, testInfo.snapshotPath, testInfo.outputPath, testInfo.config.updateSnapshots, options);
+  const withNegateComparison = this.isNot;
+  const { pass, message } = compare(
+      received,
+      options.name,
+      testInfo.snapshotPath,
+      testInfo.outputPath,
+      testInfo.config.updateSnapshots,
+      withNegateComparison,
+      options
+  );
   return { pass, message: () => message };
 }
 

src/test/golden.ts

@@ -61,7 +61,7 @@ function compareImages(actualBuffer: Buffer | string, expectedBuffer: Buffer, mi
       errorMessage: `Sizes differ; expected image ${expected.width}px X ${expected.height}px, but got ${actual.width}px X ${actual.height}px. `
     };
   }
-  const diff = new PNG({width: expected.width, height: expected.height});
+  const diff = new PNG({ width: expected.width, height: expected.height });
   const count = pixelmatch(expected.data, actual.data, diff.data, expected.width, expected.height, { threshold: 0.2, ...options });
   return count > 0 ? { diff: PNG.sync.write(diff) } : null;
 }
@@ -80,21 +80,36 @@ function compareText(actual: Buffer | string, expectedBuffer: Buffer): { diff?:
   };
 }
 
-export function compare(actual: Buffer | string, name: string, snapshotPath: (name: string) => string, outputPath: (name: string) => string, updateSnapshots: UpdateSnapshots, options?: { threshold?: number }): { pass: boolean; message?: string; } {
+export function compare(
+  actual: Buffer | string,
+  name: string,
+  snapshotPath: (name: string) => string,
+  outputPath: (name: string) => string,
+  updateSnapshots: UpdateSnapshots,
+  withNegateComparison: boolean,
+  options?: { threshold?: number }
+): { pass: boolean; message?: string; } {
   const snapshotFile = snapshotPath(name);
+
   if (!fs.existsSync(snapshotFile)) {
-    const writingActual = updateSnapshots === 'all' || updateSnapshots === 'missing';
-    if (writingActual) {
+    const isWriteMissingMode = updateSnapshots === 'all' || updateSnapshots === 'missing';
+    const commonMissingSnapshotMessage = `${snapshotFile} is missing in snapshots`;
+    if (withNegateComparison) {
+      const message = `${commonMissingSnapshotMessage}${isWriteMissingMode ? ', matchers using ".not" won\'t write them automatically.' : '.'}`;
+      return { pass: true , message };
+    }
+    if (isWriteMissingMode) {
       fs.mkdirSync(path.dirname(snapshotFile), { recursive: true });
       fs.writeFileSync(snapshotFile, actual);
     }
-    const message = snapshotFile + ' is missing in snapshots' + (writingActual ? ', writing actual.' : '.');
+    const message = `${commonMissingSnapshotMessage}${isWriteMissingMode ? ', writing actual.' : '.'}`;
     if (updateSnapshots === 'all') {
       console.log(message);
       return { pass: true, message };
     }
     return { pass: false, message };
   }
+
   const expected = fs.readFileSync(snapshotFile);
   const extension = path.extname(snapshotFile).substring(1);
   const mimeType = extensionToMimeType[extension] || 'application/octet-string';
@@ -107,8 +122,27 @@ export function compare(actual: Buffer | string, name: string, snapshotPath: (na
   }
 
   const result = comparator(actual, expected, mimeType, options);
-  if (!result)
+  if (!result) {
+    if (withNegateComparison) {
+      const message = [
+        colors.red('Snapshot comparison failed:'),
+        '',
+        indent('Expected result should be different from the actual one.', '  '),
+      ].join('\n');
+      return {
+        pass: true,
+        message,
+      };
+    }
+
     return { pass: true };
+  }
+
+  if (withNegateComparison) {
+    return {
+      pass: false,
+    };
+  }
 
   if (updateSnapshots === 'all') {
     fs.mkdirSync(path.dirname(snapshotFile), { recursive: true });
@@ -119,6 +153,7 @@ export function compare(actual: Buffer | string, name: string, snapshotPath: (na
       message: snapshotFile + ' running with --update-snapshots, writing actual.'
     };
   }
+
   const outputFile = outputPath(name);
   const expectedPath = addSuffix(outputFile, '-expected');
   const actualPath = addSuffix(outputFile, '-actual');

tests/playwright-test/golden.spec.ts

@@ -17,7 +17,7 @@
 import colors from 'colors/safe';
 import * as fs from 'fs';
 import * as path from 'path';
-import { test, expect } from './playwright-test-fixtures';
+import { test, expect, stripAscii } from './playwright-test-fixtures';
 
 test('should support golden', async ({runInlineTest}) => {
   const result = await runInlineTest({
@@ -65,6 +65,79 @@ Line7`,
   expect(result.output).toContain('Line7');
 });
 
+test('should write detailed failure result to an output folder', async ({runInlineTest}, testInfo) => {
+  const result = await runInlineTest({
+    'a.spec.js-snapshots/snapshot.txt': `Hello world`,
+    'a.spec.js': `
+      const { test } = pwt;
+      test('is a test', ({}) => {
+        expect('Hello world updated').toMatchSnapshot('snapshot.txt');
+      });
+    `
+  });
+
+  expect(result.exitCode).toBe(1);
+  const outputText = stripAscii(result.output);
+  expect(outputText).toContain('Snapshot comparison failed:');
+  const expectedSnapshotArtifactPath = testInfo.outputPath('test-results', 'a-is-a-test', 'snapshot-expected.txt');
+  const actualSnapshotArtifactPath = testInfo.outputPath('test-results', 'a-is-a-test', 'snapshot-actual.txt');
+  expect(outputText).toContain(`Expected: ${expectedSnapshotArtifactPath}`);
+  expect(outputText).toContain(`Received: ${actualSnapshotArtifactPath}`);
+  expect(fs.existsSync(expectedSnapshotArtifactPath)).toBe(true);
+  expect(fs.existsSync(actualSnapshotArtifactPath)).toBe(true);
+});
+
+test("doesn\'t create comparison artifacts in an output folder for passed negated snapshot matcher", async ({runInlineTest}, testInfo) => {
+  const result = await runInlineTest({
+    'a.spec.js-snapshots/snapshot.txt': `Hello world`,
+    'a.spec.js': `
+      const { test } = pwt;
+      test('is a test', ({}) => {
+        expect('Hello world updated').not.toMatchSnapshot('snapshot.txt');
+      });
+    `
+  });
+
+  expect(result.exitCode).toBe(0);
+  const outputText = stripAscii(result.output);
+  const expectedSnapshotArtifactPath = testInfo.outputPath('test-results', 'a-is-a-test', 'snapshot-expected.txt');
+  const actualSnapshotArtifactPath = testInfo.outputPath('test-results', 'a-is-a-test', 'snapshot-actual.txt');
+  expect(outputText).not.toContain(`Expected: ${expectedSnapshotArtifactPath}`);
+  expect(outputText).not.toContain(`Received: ${actualSnapshotArtifactPath}`);
+  expect(fs.existsSync(expectedSnapshotArtifactPath)).toBe(false);
+  expect(fs.existsSync(actualSnapshotArtifactPath)).toBe(false);
+});
+
+test('should pass on different snapshots with negate matcher', async ({runInlineTest}) => {
+  const result = await runInlineTest({
+    'a.spec.js-snapshots/snapshot.txt': `Hello world`,
+    'a.spec.js': `
+      const { test } = pwt;
+      test('is a test', ({}) => {
+        expect('Hello world updated').not.toMatchSnapshot('snapshot.txt');
+      });
+    `
+  });
+
+  expect(result.exitCode).toBe(0);
+});
+
+test('should fail on same snapshots with negate matcher', async ({runInlineTest}) => {
+  const result = await runInlineTest({
+    'a.spec.js-snapshots/snapshot.txt': `Hello world`,
+    'a.spec.js': `
+      const { test } = pwt;
+      test('is a test', ({}) => {
+        expect('Hello world').not.toMatchSnapshot('snapshot.txt');
+      });
+    `
+  });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain('Snapshot comparison failed:');
+  expect(result.output).toContain('Expected result should be different from the actual one.');
+});
+
 test('should write missing expectations locally', async ({runInlineTest}, testInfo) => {
   const result = await runInlineTest({
     'a.spec.js': `
@@ -74,26 +147,101 @@ test('should write missing expectations locally', async ({runInlineTest}, testIn
       });
     `
   }, {}, { CI: '' });
+
   expect(result.exitCode).toBe(1);
-  expect(result.output).toContain('snapshot.txt is missing in snapshots, writing actual');
-  const data = fs.readFileSync(testInfo.outputPath('a.spec.js-snapshots/snapshot.txt'));
+  const snapshotOutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.txt');
+  expect(result.output).toContain(`${snapshotOutputPath} is missing in snapshots, writing actual`);
+  const data = fs.readFileSync(snapshotOutputPath);
   expect(data.toString()).toBe('Hello world');
 });
 
-test('should update expectations', async ({runInlineTest}, testInfo) => {
+test('shouldn\'t write missing expectations locally for negated matcher', async ({runInlineTest}, testInfo) => {
   const result = await runInlineTest({
-    'a.spec.js-snapshots/snapshot.txt': `Hello world`,
     'a.spec.js': `
       const { test } = pwt;
       test('is a test', ({}) => {
-        expect('Hello world updated').toMatchSnapshot('snapshot.txt');
+        expect('Hello world').not.toMatchSnapshot('snapshot.txt');
+      });
+    `
+  }, {}, { CI: '' });
+
+  expect(result.exitCode).toBe(1);
+  const snapshotOutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.txt');
+  expect(result.output).toContain(`${snapshotOutputPath} is missing in snapshots, matchers using ".not" won\'t write them automatically.`);
+  expect(fs.existsSync(snapshotOutputPath)).toBe(false);
+});
+
+test('should update snapshot with the update-snapshots flag', async ({runInlineTest}, testInfo) => {
+  const EXPECTED_SNAPSHOT = 'Hello world';
+  const ACTUAL_SNAPSHOT = 'Hello world updated';
+  const result = await runInlineTest({
+    'a.spec.js-snapshots/snapshot.txt': EXPECTED_SNAPSHOT,
+    'a.spec.js': `
+      const { test } = pwt;
+      test('is a test', ({}) => {
+        expect('${ACTUAL_SNAPSHOT}').toMatchSnapshot('snapshot.txt');
       });
     `
   }, { 'update-snapshots': true });
+
   expect(result.exitCode).toBe(0);
-  expect(result.output).toContain('snapshot.txt does not match, writing actual.');
-  const data = fs.readFileSync(testInfo.outputPath('a.spec.js-snapshots/snapshot.txt'));
-  expect(data.toString()).toBe('Hello world updated');
+  const snapshotOutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.txt');
+  expect(result.output).toContain(`${snapshotOutputPath} does not match, writing actual.`);
+  const data = fs.readFileSync(snapshotOutputPath);
+  expect(data.toString()).toBe(ACTUAL_SNAPSHOT);
+});
+
+test('shouldn\'t update snapshot with the update-snapshots flag for negated matcher', async ({runInlineTest}, testInfo) => {
+  const EXPECTED_SNAPSHOT = 'Hello world';
+  const ACTUAL_SNAPSHOT = 'Hello world updated';
+  const result = await runInlineTest({
+    'a.spec.js-snapshots/snapshot.txt': EXPECTED_SNAPSHOT,
+    'a.spec.js': `
+      const { test } = pwt;
+      test('is a test', ({}) => {
+        expect('${ACTUAL_SNAPSHOT}').not.toMatchSnapshot('snapshot.txt');
+      });
+    `
+  }, { 'update-snapshots': true });
+
+  expect(result.exitCode).toBe(0);
+  const snapshotOutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.txt');
+  const data = fs.readFileSync(snapshotOutputPath);
+  expect(data.toString()).toBe(EXPECTED_SNAPSHOT);
+});
+
+test('should silently write missing expectations locally with the update-snapshots flag', async ({runInlineTest}, testInfo) => {
+  const ACTUAL_SNAPSHOT = 'Hello world new';
+  const result = await runInlineTest({
+    'a.spec.js': `
+      const { test } = pwt;
+      test('is a test', ({}) => {
+        expect('${ACTUAL_SNAPSHOT}').toMatchSnapshot('snapshot.txt');
+      });
+    `
+  }, { 'update-snapshots': true });
+
+  expect(result.exitCode).toBe(0);
+  const snapshotOutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.txt');
+  expect(result.output).toContain(`${snapshotOutputPath} is missing in snapshots, writing actual`);
+  const data = fs.readFileSync(snapshotOutputPath);
+  expect(data.toString()).toBe(ACTUAL_SNAPSHOT);
+});
+
+test('should silently write missing expectations locally with the update-snapshots flag for negated matcher', async ({runInlineTest}, testInfo) => {
+  const result = await runInlineTest({
+    'a.spec.js': `
+      const { test } = pwt;
+      test('is a test', ({}) => {
+        expect('Hello world').not.toMatchSnapshot('snapshot.txt');
+      });
+    `
+  }, { 'update-snapshots': true });
+
+  expect(result.exitCode).toBe(1);
+  const snapshotOutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.txt');
+  expect(result.output).toContain(`${snapshotOutputPath} is missing in snapshots, matchers using ".not" won\'t write them automatically.`);
+  expect(fs.existsSync(snapshotOutputPath)).toBe(false);
 });
 
 test('should match multiple snapshots', async ({runInlineTest}) => {
@@ -206,7 +354,7 @@ test('should compare PNG images', async ({runInlineTest}) => {
   expect(result.exitCode).toBe(0);
 });
 
-test('should compare different PNG images', async ({runInlineTest}) => {
+test('should compare different PNG images', async ({runInlineTest}, testInfo) => {
   const result = await runInlineTest({
     'a.spec.js-snapshots/snapshot.png':
         Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+P+/HgAFhAJ/wlseKgAAAABJRU5ErkJggg==', 'base64'),
@@ -217,9 +365,19 @@ test('should compare different PNG images', async ({runInlineTest}) => {
       });
     `
   });
+
+  const outputText = stripAscii(result.output);
   expect(result.exitCode).toBe(1);
-  expect(result.output).toContain('Snapshot comparison failed');
-  expect(result.output).toContain('snapshot-diff.png');
+  expect(outputText).toContain('Snapshot comparison failed:');
+  const expectedSnapshotArtifactPath = testInfo.outputPath('test-results', 'a-is-a-test', 'snapshot-expected.png');
+  const actualSnapshotArtifactPath = testInfo.outputPath('test-results', 'a-is-a-test', 'snapshot-actual.png');
+  const diffSnapshotArtifactPath = testInfo.outputPath('test-results', 'a-is-a-test', 'snapshot-diff.png');
+  expect(outputText).toContain(`Expected: ${expectedSnapshotArtifactPath}`);
+  expect(outputText).toContain(`Received: ${actualSnapshotArtifactPath}`);
+  expect(outputText).toContain(`Diff: ${diffSnapshotArtifactPath}`);
+  expect(fs.existsSync(expectedSnapshotArtifactPath)).toBe(true);
+  expect(fs.existsSync(actualSnapshotArtifactPath)).toBe(true);
+  expect(fs.existsSync(diffSnapshotArtifactPath)).toBe(true);
 });
 
 test('should respect threshold', async ({runInlineTest}) => {