diff --git a/src/test/expect.ts b/src/test/expect.ts index fdde70568e..f73c9f4b9a 100644 --- a/src/test/expect.ts +++ b/src/test/expect.ts @@ -21,7 +21,7 @@ import { compare } from './golden'; export const expect: Expect = expectLibrary; -function toMatchSnapshot(received: Buffer | string, nameOrOptions: string | { name: string, threshold?: number }, optOptions: { threshold?: number } = {}) { +function toMatchSnapshot(this: ReturnType, received: Buffer | string, nameOrOptions: string | { name: string, threshold?: number }, optOptions: { threshold?: number } = {}) { let options: { name: string, threshold?: number }; const testInfo = currentTestInfo(); if (!testInfo) @@ -37,7 +37,16 @@ function toMatchSnapshot(received: Buffer | string, nameOrOptions: string | { na if (options.threshold === undefined && projectThreshold !== undefined) options.threshold = projectThreshold; - const { pass, message } = compare(received, options.name, testInfo.snapshotPath, testInfo.outputPath, testInfo.config.updateSnapshots, options); + const withNegateComparison = this.isNot; + const { pass, message } = compare( + received, + options.name, + testInfo.snapshotPath, + testInfo.outputPath, + testInfo.config.updateSnapshots, + withNegateComparison, + options + ); return { pass, message: () => message }; } diff --git a/src/test/golden.ts b/src/test/golden.ts index 43d3053790..460a29aa75 100644 --- a/src/test/golden.ts +++ b/src/test/golden.ts @@ -61,7 +61,7 @@ function compareImages(actualBuffer: Buffer | string, expectedBuffer: Buffer, mi errorMessage: `Sizes differ; expected image ${expected.width}px X ${expected.height}px, but got ${actual.width}px X ${actual.height}px. ` }; } - const diff = new PNG({width: expected.width, height: expected.height}); + const diff = new PNG({ width: expected.width, height: expected.height }); const count = pixelmatch(expected.data, actual.data, diff.data, expected.width, expected.height, { threshold: 0.2, ...options }); return count > 0 ? { diff: PNG.sync.write(diff) } : null; } @@ -80,21 +80,36 @@ function compareText(actual: Buffer | string, expectedBuffer: Buffer): { diff?: }; } -export function compare(actual: Buffer | string, name: string, snapshotPath: (name: string) => string, outputPath: (name: string) => string, updateSnapshots: UpdateSnapshots, options?: { threshold?: number }): { pass: boolean; message?: string; } { +export function compare( + actual: Buffer | string, + name: string, + snapshotPath: (name: string) => string, + outputPath: (name: string) => string, + updateSnapshots: UpdateSnapshots, + withNegateComparison: boolean, + options?: { threshold?: number } +): { pass: boolean; message?: string; } { const snapshotFile = snapshotPath(name); + if (!fs.existsSync(snapshotFile)) { - const writingActual = updateSnapshots === 'all' || updateSnapshots === 'missing'; - if (writingActual) { + const isWriteMissingMode = updateSnapshots === 'all' || updateSnapshots === 'missing'; + const commonMissingSnapshotMessage = `${snapshotFile} is missing in snapshots`; + if (withNegateComparison) { + const message = `${commonMissingSnapshotMessage}${isWriteMissingMode ? ', matchers using ".not" won\'t write them automatically.' : '.'}`; + return { pass: true , message }; + } + if (isWriteMissingMode) { fs.mkdirSync(path.dirname(snapshotFile), { recursive: true }); fs.writeFileSync(snapshotFile, actual); } - const message = snapshotFile + ' is missing in snapshots' + (writingActual ? ', writing actual.' : '.'); + const message = `${commonMissingSnapshotMessage}${isWriteMissingMode ? ', writing actual.' : '.'}`; if (updateSnapshots === 'all') { console.log(message); return { pass: true, message }; } return { pass: false, message }; } + const expected = fs.readFileSync(snapshotFile); const extension = path.extname(snapshotFile).substring(1); const mimeType = extensionToMimeType[extension] || 'application/octet-string'; @@ -102,13 +117,32 @@ export function compare(actual: Buffer | string, name: string, snapshotPath: (na if (!comparator) { return { pass: false, - message: 'Failed to find comparator with type ' + mimeType + ': ' + snapshotFile, + message: 'Failed to find comparator with type ' + mimeType + ': ' + snapshotFile, }; } const result = comparator(actual, expected, mimeType, options); - if (!result) + if (!result) { + if (withNegateComparison) { + const message = [ + colors.red('Snapshot comparison failed:'), + '', + indent('Expected result should be different from the actual one.', ' '), + ].join('\n'); + return { + pass: true, + message, + }; + } + return { pass: true }; + } + + if (withNegateComparison) { + return { + pass: false, + }; + } if (updateSnapshots === 'all') { fs.mkdirSync(path.dirname(snapshotFile), { recursive: true }); @@ -119,6 +153,7 @@ export function compare(actual: Buffer | string, name: string, snapshotPath: (na message: snapshotFile + ' running with --update-snapshots, writing actual.' }; } + const outputFile = outputPath(name); const expectedPath = addSuffix(outputFile, '-expected'); const actualPath = addSuffix(outputFile, '-actual'); diff --git a/tests/playwright-test/golden.spec.ts b/tests/playwright-test/golden.spec.ts index 5de00e74d2..78f3d53d75 100644 --- a/tests/playwright-test/golden.spec.ts +++ b/tests/playwright-test/golden.spec.ts @@ -17,7 +17,7 @@ import colors from 'colors/safe'; import * as fs from 'fs'; import * as path from 'path'; -import { test, expect } from './playwright-test-fixtures'; +import { test, expect, stripAscii } from './playwright-test-fixtures'; test('should support golden', async ({runInlineTest}) => { const result = await runInlineTest({ @@ -65,6 +65,79 @@ Line7`, expect(result.output).toContain('Line7'); }); +test('should write detailed failure result to an output folder', async ({runInlineTest}, testInfo) => { + const result = await runInlineTest({ + 'a.spec.js-snapshots/snapshot.txt': `Hello world`, + 'a.spec.js': ` + const { test } = pwt; + test('is a test', ({}) => { + expect('Hello world updated').toMatchSnapshot('snapshot.txt'); + }); + ` + }); + + expect(result.exitCode).toBe(1); + const outputText = stripAscii(result.output); + expect(outputText).toContain('Snapshot comparison failed:'); + const expectedSnapshotArtifactPath = testInfo.outputPath('test-results', 'a-is-a-test', 'snapshot-expected.txt'); + const actualSnapshotArtifactPath = testInfo.outputPath('test-results', 'a-is-a-test', 'snapshot-actual.txt'); + expect(outputText).toContain(`Expected: ${expectedSnapshotArtifactPath}`); + expect(outputText).toContain(`Received: ${actualSnapshotArtifactPath}`); + expect(fs.existsSync(expectedSnapshotArtifactPath)).toBe(true); + expect(fs.existsSync(actualSnapshotArtifactPath)).toBe(true); +}); + +test("doesn\'t create comparison artifacts in an output folder for passed negated snapshot matcher", async ({runInlineTest}, testInfo) => { + const result = await runInlineTest({ + 'a.spec.js-snapshots/snapshot.txt': `Hello world`, + 'a.spec.js': ` + const { test } = pwt; + test('is a test', ({}) => { + expect('Hello world updated').not.toMatchSnapshot('snapshot.txt'); + }); + ` + }); + + expect(result.exitCode).toBe(0); + const outputText = stripAscii(result.output); + const expectedSnapshotArtifactPath = testInfo.outputPath('test-results', 'a-is-a-test', 'snapshot-expected.txt'); + const actualSnapshotArtifactPath = testInfo.outputPath('test-results', 'a-is-a-test', 'snapshot-actual.txt'); + expect(outputText).not.toContain(`Expected: ${expectedSnapshotArtifactPath}`); + expect(outputText).not.toContain(`Received: ${actualSnapshotArtifactPath}`); + expect(fs.existsSync(expectedSnapshotArtifactPath)).toBe(false); + expect(fs.existsSync(actualSnapshotArtifactPath)).toBe(false); +}); + +test('should pass on different snapshots with negate matcher', async ({runInlineTest}) => { + const result = await runInlineTest({ + 'a.spec.js-snapshots/snapshot.txt': `Hello world`, + 'a.spec.js': ` + const { test } = pwt; + test('is a test', ({}) => { + expect('Hello world updated').not.toMatchSnapshot('snapshot.txt'); + }); + ` + }); + + expect(result.exitCode).toBe(0); +}); + +test('should fail on same snapshots with negate matcher', async ({runInlineTest}) => { + const result = await runInlineTest({ + 'a.spec.js-snapshots/snapshot.txt': `Hello world`, + 'a.spec.js': ` + const { test } = pwt; + test('is a test', ({}) => { + expect('Hello world').not.toMatchSnapshot('snapshot.txt'); + }); + ` + }); + + expect(result.exitCode).toBe(1); + expect(result.output).toContain('Snapshot comparison failed:'); + expect(result.output).toContain('Expected result should be different from the actual one.'); +}); + test('should write missing expectations locally', async ({runInlineTest}, testInfo) => { const result = await runInlineTest({ 'a.spec.js': ` @@ -74,26 +147,101 @@ test('should write missing expectations locally', async ({runInlineTest}, testIn }); ` }, {}, { CI: '' }); + expect(result.exitCode).toBe(1); - expect(result.output).toContain('snapshot.txt is missing in snapshots, writing actual'); - const data = fs.readFileSync(testInfo.outputPath('a.spec.js-snapshots/snapshot.txt')); + const snapshotOutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.txt'); + expect(result.output).toContain(`${snapshotOutputPath} is missing in snapshots, writing actual`); + const data = fs.readFileSync(snapshotOutputPath); expect(data.toString()).toBe('Hello world'); }); -test('should update expectations', async ({runInlineTest}, testInfo) => { +test('shouldn\'t write missing expectations locally for negated matcher', async ({runInlineTest}, testInfo) => { const result = await runInlineTest({ - 'a.spec.js-snapshots/snapshot.txt': `Hello world`, 'a.spec.js': ` const { test } = pwt; test('is a test', ({}) => { - expect('Hello world updated').toMatchSnapshot('snapshot.txt'); + expect('Hello world').not.toMatchSnapshot('snapshot.txt'); + }); + ` + }, {}, { CI: '' }); + + expect(result.exitCode).toBe(1); + const snapshotOutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.txt'); + expect(result.output).toContain(`${snapshotOutputPath} is missing in snapshots, matchers using ".not" won\'t write them automatically.`); + expect(fs.existsSync(snapshotOutputPath)).toBe(false); +}); + +test('should update snapshot with the update-snapshots flag', async ({runInlineTest}, testInfo) => { + const EXPECTED_SNAPSHOT = 'Hello world'; + const ACTUAL_SNAPSHOT = 'Hello world updated'; + const result = await runInlineTest({ + 'a.spec.js-snapshots/snapshot.txt': EXPECTED_SNAPSHOT, + 'a.spec.js': ` + const { test } = pwt; + test('is a test', ({}) => { + expect('${ACTUAL_SNAPSHOT}').toMatchSnapshot('snapshot.txt'); }); ` }, { 'update-snapshots': true }); + expect(result.exitCode).toBe(0); - expect(result.output).toContain('snapshot.txt does not match, writing actual.'); - const data = fs.readFileSync(testInfo.outputPath('a.spec.js-snapshots/snapshot.txt')); - expect(data.toString()).toBe('Hello world updated'); + const snapshotOutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.txt'); + expect(result.output).toContain(`${snapshotOutputPath} does not match, writing actual.`); + const data = fs.readFileSync(snapshotOutputPath); + expect(data.toString()).toBe(ACTUAL_SNAPSHOT); +}); + +test('shouldn\'t update snapshot with the update-snapshots flag for negated matcher', async ({runInlineTest}, testInfo) => { + const EXPECTED_SNAPSHOT = 'Hello world'; + const ACTUAL_SNAPSHOT = 'Hello world updated'; + const result = await runInlineTest({ + 'a.spec.js-snapshots/snapshot.txt': EXPECTED_SNAPSHOT, + 'a.spec.js': ` + const { test } = pwt; + test('is a test', ({}) => { + expect('${ACTUAL_SNAPSHOT}').not.toMatchSnapshot('snapshot.txt'); + }); + ` + }, { 'update-snapshots': true }); + + expect(result.exitCode).toBe(0); + const snapshotOutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.txt'); + const data = fs.readFileSync(snapshotOutputPath); + expect(data.toString()).toBe(EXPECTED_SNAPSHOT); +}); + +test('should silently write missing expectations locally with the update-snapshots flag', async ({runInlineTest}, testInfo) => { + const ACTUAL_SNAPSHOT = 'Hello world new'; + const result = await runInlineTest({ + 'a.spec.js': ` + const { test } = pwt; + test('is a test', ({}) => { + expect('${ACTUAL_SNAPSHOT}').toMatchSnapshot('snapshot.txt'); + }); + ` + }, { 'update-snapshots': true }); + + expect(result.exitCode).toBe(0); + const snapshotOutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.txt'); + expect(result.output).toContain(`${snapshotOutputPath} is missing in snapshots, writing actual`); + const data = fs.readFileSync(snapshotOutputPath); + expect(data.toString()).toBe(ACTUAL_SNAPSHOT); +}); + +test('should silently write missing expectations locally with the update-snapshots flag for negated matcher', async ({runInlineTest}, testInfo) => { + const result = await runInlineTest({ + 'a.spec.js': ` + const { test } = pwt; + test('is a test', ({}) => { + expect('Hello world').not.toMatchSnapshot('snapshot.txt'); + }); + ` + }, { 'update-snapshots': true }); + + expect(result.exitCode).toBe(1); + const snapshotOutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.txt'); + expect(result.output).toContain(`${snapshotOutputPath} is missing in snapshots, matchers using ".not" won\'t write them automatically.`); + expect(fs.existsSync(snapshotOutputPath)).toBe(false); }); test('should match multiple snapshots', async ({runInlineTest}) => { @@ -206,7 +354,7 @@ test('should compare PNG images', async ({runInlineTest}) => { expect(result.exitCode).toBe(0); }); -test('should compare different PNG images', async ({runInlineTest}) => { +test('should compare different PNG images', async ({runInlineTest}, testInfo) => { const result = await runInlineTest({ 'a.spec.js-snapshots/snapshot.png': Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+P+/HgAFhAJ/wlseKgAAAABJRU5ErkJggg==', 'base64'), @@ -217,9 +365,19 @@ test('should compare different PNG images', async ({runInlineTest}) => { }); ` }); + + const outputText = stripAscii(result.output); expect(result.exitCode).toBe(1); - expect(result.output).toContain('Snapshot comparison failed'); - expect(result.output).toContain('snapshot-diff.png'); + expect(outputText).toContain('Snapshot comparison failed:'); + const expectedSnapshotArtifactPath = testInfo.outputPath('test-results', 'a-is-a-test', 'snapshot-expected.png'); + const actualSnapshotArtifactPath = testInfo.outputPath('test-results', 'a-is-a-test', 'snapshot-actual.png'); + const diffSnapshotArtifactPath = testInfo.outputPath('test-results', 'a-is-a-test', 'snapshot-diff.png'); + expect(outputText).toContain(`Expected: ${expectedSnapshotArtifactPath}`); + expect(outputText).toContain(`Received: ${actualSnapshotArtifactPath}`); + expect(outputText).toContain(`Diff: ${diffSnapshotArtifactPath}`); + expect(fs.existsSync(expectedSnapshotArtifactPath)).toBe(true); + expect(fs.existsSync(actualSnapshotArtifactPath)).toBe(true); + expect(fs.existsSync(diffSnapshotArtifactPath)).toBe(true); }); test('should respect threshold', async ({runInlineTest}) => {