From a52eb0c9a08c7781a797553611695321445b37ca Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Thu, 5 Sep 2024 21:36:51 -0700 Subject: [PATCH] chore: expose matcherResult on TestError (#32455) --- .../class-testinfoerror-matcherresult.md | 35 +++++ docs/src/test-api/class-testinfoerror.md | 6 + .../class-testerror-matcherresult.md | 35 +++++ docs/src/test-reporter-api/class-testerror.md | 6 + packages/playwright/src/reporters/base.ts | 3 + packages/playwright/src/util.ts | 16 +- packages/playwright/types/test.d.ts | 36 +++++ packages/playwright/types/testReporter.d.ts | 36 +++++ tests/playwright-test/reporter-errors.spec.ts | 145 ++++++++++++++++++ utils/generate_types/overrides-test.d.ts | 1 + .../overrides-testReporter.d.ts | 1 + 11 files changed, 318 insertions(+), 2 deletions(-) create mode 100644 docs/src/test-api/class-testinfoerror-matcherresult.md create mode 100644 docs/src/test-reporter-api/class-testerror-matcherresult.md create mode 100644 tests/playwright-test/reporter-errors.spec.ts diff --git a/docs/src/test-api/class-testinfoerror-matcherresult.md b/docs/src/test-api/class-testinfoerror-matcherresult.md new file mode 100644 index 0000000000..f68e2db1ae --- /dev/null +++ b/docs/src/test-api/class-testinfoerror-matcherresult.md @@ -0,0 +1,35 @@ +# class: TestInfoErrorMatcherResult +* since: v1.48 +* langs: js + +Matcher-specific details for the error thrown during the `expect` call. + +## property: TestInfoErrorMatcherResult.actual +* since: v1.48 +- type: ?<[unknown]> + +Actual value. + +## property: TestInfoErrorMatcherResult.expected +* since: v1.48 +- type: ?<[unknown]> + +Expected value. + +## property: TestInfoErrorMatcherResult.name +* since: v1.48 +- type: ?<[string]> + +Matcher name. + +## property: TestInfoErrorMatcherResult.pass +* since: v1.48 +- type: <[string]> + +Whether the matcher passed. + +## property: TestInfoErrorMatcherResult.timeout +* since: v1.48 +- type: ?<[int]> + +Timeout that was used during matching. diff --git a/docs/src/test-api/class-testinfoerror.md b/docs/src/test-api/class-testinfoerror.md index 66e78ecabd..b084aaf54b 100644 --- a/docs/src/test-api/class-testinfoerror.md +++ b/docs/src/test-api/class-testinfoerror.md @@ -4,6 +4,12 @@ Information about an error thrown during test execution. +## property: TestInfoError.matcherResult +* since: v1.48 +- type: ?<[TestInfoErrorMatcherResult]> + +Matcher result details. + ## property: TestInfoError.message * since: v1.10 - type: ?<[string]> diff --git a/docs/src/test-reporter-api/class-testerror-matcherresult.md b/docs/src/test-reporter-api/class-testerror-matcherresult.md new file mode 100644 index 0000000000..b8d0f1dce7 --- /dev/null +++ b/docs/src/test-reporter-api/class-testerror-matcherresult.md @@ -0,0 +1,35 @@ +# class: TestErrorMatcherResult +* since: v1.48 +* langs: js + +Matcher-specific details for the error thrown during the `expect` call. + +## property: TestErrorMatcherResult.actual +* since: v1.48 +- type: ?<[unknown]> + +Actual value. + +## property: TestErrorMatcherResult.expected +* since: v1.48 +- type: ?<[unknown]> + +Expected value. + +## property: TestErrorMatcherResult.name +* since: v1.48 +- type: ?<[string]> + +Matcher name. + +## property: TestErrorMatcherResult.pass +* since: v1.48 +- type: <[string]> + +Whether the matcher passed. + +## property: TestErrorMatcherResult.timeout +* since: v1.48 +- type: ?<[int]> + +Timeout that was used during matching. diff --git a/docs/src/test-reporter-api/class-testerror.md b/docs/src/test-reporter-api/class-testerror.md index 7a872c63fc..cc59c3a4c6 100644 --- a/docs/src/test-reporter-api/class-testerror.md +++ b/docs/src/test-reporter-api/class-testerror.md @@ -4,6 +4,12 @@ Information about an error thrown during test execution. +## property: TestError.matcherResult +* since: v1.48 +- type: ?<[TestErrorMatcherResult]> + +Matcher result details. + ## property: TestError.message * since: v1.10 - type: ?<[string]> diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.ts index c9ce2f7bcd..22470b0954 100644 --- a/packages/playwright/src/reporters/base.ts +++ b/packages/playwright/src/reporters/base.ts @@ -32,6 +32,7 @@ type Annotation = { type ErrorDetails = { message: string; location?: Location; + matcherResult?: TestError['matcherResult']; }; type TestSummary = { @@ -399,6 +400,7 @@ export function formatResultFailure(test: TestCase, result: TestResult, initialI errorDetails.push({ message: indent(formattedError.message, initialIndent), location: formattedError.location, + matcherResult: formattedError.matcherResult }); } return errorDetails; @@ -489,6 +491,7 @@ export function formatError(error: TestError, highlightCode: boolean): ErrorDeta return { location, message: tokens.join('\n'), + matcherResult: error.matcherResult }; } diff --git a/packages/playwright/src/util.ts b/packages/playwright/src/util.ts index 06e18f206a..4c75d9d841 100644 --- a/packages/playwright/src/util.ts +++ b/packages/playwright/src/util.ts @@ -63,8 +63,20 @@ export function filteredStackTrace(rawStack: RawStack): StackFrame[] { } export function serializeError(error: Error | any): TestInfoError { - if (error instanceof Error) - return filterStackTrace(error); + if (error instanceof Error) { + const result: TestInfoError = filterStackTrace(error); + if ('matcherResult' in error && error.matcherResult) { + const matcherResult = (error.matcherResult as TestInfoError['matcherResult'])!; + result.matcherResult = { + pass: matcherResult.pass, + name: matcherResult.name, + expected: matcherResult.expected, + actual: matcherResult.actual, + timeout: matcherResult.timeout, + }; + } + return result; + } return { value: util.inspect(error) }; diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index 21bd59af5d..09b5c7efdb 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -6546,6 +6546,7 @@ export type MatcherReturnType = { expected?: unknown; actual?: any; log?: string[]; + timeout?: number; }; type MakeMatchers = { @@ -8221,10 +8222,45 @@ export interface TestInfo { workerIndex: number; } +/** + * Matcher-specific details for the error thrown during the `expect` call. + */ +export interface TestInfoErrorMatcherResult { + /** + * Actual value. + */ + actual?: unknown; + + /** + * Expected value. + */ + expected?: unknown; + + /** + * Matcher name. + */ + name?: string; + + /** + * Whether the matcher passed. + */ + pass: string; + + /** + * Timeout that was used during matching. + */ + timeout?: number; +} + /** * Information about an error thrown during test execution. */ export interface TestInfoError { + /** + * Matcher result details. + */ + matcherResult?: TestInfoErrorMatcherResult; + /** * Error message. Set when [Error] (or its subclass) has been thrown. */ diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.ts index 52073066f0..b600e3c801 100644 --- a/packages/playwright/types/testReporter.d.ts +++ b/packages/playwright/types/testReporter.d.ts @@ -284,6 +284,7 @@ export interface JSONReportTest { export interface JSONReportError { message: string; location?: Location; + matcherResult?: TestErrorMatcherResult; } export interface JSONReportTestResult { @@ -567,6 +568,36 @@ export interface TestCase { type: "test"; } +/** + * Matcher-specific details for the error thrown during the `expect` call. + */ +export interface TestErrorMatcherResult { + /** + * Actual value. + */ + actual?: unknown; + + /** + * Expected value. + */ + expected?: unknown; + + /** + * Matcher name. + */ + name?: string; + + /** + * Whether the matcher passed. + */ + pass: string; + + /** + * Timeout that was used during matching. + */ + timeout?: number; +} + /** * Information about an error thrown during test execution. */ @@ -576,6 +607,11 @@ export interface TestError { */ location?: Location; + /** + * Matcher result details. + */ + matcherResult?: TestErrorMatcherResult; + /** * Error message. Set when [Error] (or its subclass) has been thrown. */ diff --git a/tests/playwright-test/reporter-errors.spec.ts b/tests/playwright-test/reporter-errors.spec.ts new file mode 100644 index 0000000000..ad09237234 --- /dev/null +++ b/tests/playwright-test/reporter-errors.spec.ts @@ -0,0 +1,145 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test, expect } from './playwright-test-fixtures'; + +test('should report matcherResults for generic matchers', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.js': ` + import { test, expect as baseExpect } from '@playwright/test'; + const expect = baseExpect.soft; + test('fail', ({}) => { + expect(1).toBe(2); + expect(1).toBeCloseTo(2); + expect(undefined).toBeDefined(); + expect(1).toBeFalsy(); + expect(1).toBeGreaterThan(2); + expect(1).toBeGreaterThanOrEqual(2); + expect('a').toBeInstanceOf(Number); + expect(2).toBeLessThan(1); + expect(2).toBeLessThanOrEqual(1); + expect(1).toBeNaN(); + expect(1).toBeNull(); + expect(0).toBeTruthy(); + expect(1).toBeUndefined(); + expect([1]).toContain(2); + expect([1]).toContainEqual(2); + expect([1]).toEqual([2]); + expect([1]).toHaveLength(2); + expect({ a: 1 }).toHaveProperty('b'); + expect('a').toMatch(/b/); + expect({ a: 1 }).toMatchObject({ b: 2 }); + expect({ a: 1 }).toStrictEqual({ b: 2 }); + expect(() => {}).toThrow(); + expect(() => {}).toThrowError('a'); + }); + ` + }, { }); + expect(result.exitCode).toBe(1); + + const { errors } = result.report.suites[0].specs[0].tests[0].results[0]; + const matcherResults = errors.map(e => e.matcherResult); + expect(matcherResults).toEqual([ + { name: 'toBe', pass: false, expected: 2, actual: 1 }, + { pass: false }, + { pass: false }, + { pass: false }, + { pass: false }, + { pass: false }, + { pass: false }, + { pass: false }, + { pass: false }, + { pass: false }, + { pass: false }, + { pass: false }, + { pass: false }, + { pass: false }, + { pass: false }, + { name: 'toEqual', pass: false, expected: [2], actual: [1] }, + { pass: false }, + { pass: false }, + { pass: false }, + { pass: false }, + { name: 'toStrictEqual', pass: false, expected: { b: 2 }, actual: { a: 1 } }, + { pass: false }, + { pass: false }, + ]); +}); + +test('should report matcherResults for web matchers', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.js': ` + import { test, expect as baseExpect } from '@playwright/test'; + + const expect = baseExpect.configure({ soft: true, timeout: 1 }); + test('fail', async ({ page }) => { + await page.setContent('Hello
World
'); + await expect(page.locator('input')).toBeChecked(); + await expect(page.locator('input')).toBeDisabled(); + await expect(page.locator('textarea')).not.toBeEditable(); + await expect(page.locator('span')).toBeEmpty(); + await expect(page.locator('button')).not.toBeEnabled(); + await expect(page.locator('button')).toBeFocused(); + await expect(page.locator('span')).toBeHidden(); + await expect(page.locator('div')).not.toBeInViewport(); + await expect(page.locator('div')).not.toBeVisible(); + await expect(page.locator('span')).toContainText('World'); + await expect(page.locator('span')).toHaveAccessibleDescription('World'); + await expect(page.locator('span')).toHaveAccessibleName('World'); + await expect(page.locator('span')).toHaveAttribute('name', 'value'); + await expect(page.locator('span')).toHaveAttribute('name'); + await expect(page.locator('span')).toHaveClass('name'); + await expect(page.locator('span')).toHaveCount(2); + await expect(page.locator('span')).toHaveCSS('width', '10'); + await expect(page.locator('span')).toHaveId('id'); + await expect(page.locator('span')).toHaveJSProperty('name', 'value'); + await expect(page.locator('span')).toHaveRole('role'); + await expect(page.locator('span')).toHaveText('World'); + await expect(page.locator('textarea')).toHaveValue('value'); + await expect(page.locator('select')).toHaveValues(['value']); + }); + ` + }, { }); + expect(result.exitCode).toBe(1); + + const { errors } = result.report.suites[0].specs[0].tests[0].results[0]; + const matcherResults = errors.map(e => e.matcherResult); + expect(matcherResults).toEqual([ + { name: 'toBeChecked', pass: false, expected: 'checked', actual: 'unchecked', timeout: 1 }, + { name: 'toBeDisabled', pass: false, expected: 'disabled', actual: 'enabled', timeout: 1 }, + { name: 'toBeEditable', pass: true, expected: 'editable', actual: 'editable', timeout: 1 }, + { name: 'toBeEmpty', pass: false, expected: 'empty', actual: 'notEmpty', timeout: 1 }, + { name: 'toBeEnabled', pass: true, expected: 'enabled', actual: 'enabled', timeout: 1 }, + { name: 'toBeFocused', pass: false, expected: 'focused', actual: 'inactive', timeout: 1 }, + { name: 'toBeHidden', pass: false, expected: 'hidden', actual: 'visible', timeout: 1 }, + { name: 'toBeInViewport', pass: true, expected: 'in viewport', actual: 'in viewport', timeout: 1 }, + { name: 'toBeVisible', pass: true, expected: 'visible', actual: 'visible', timeout: 1 }, + { name: 'toContainText', pass: false, expected: 'World', actual: 'Hello', timeout: 1 }, + { name: 'toHaveAccessibleDescription', pass: false, expected: 'World', actual: '', timeout: 1 }, + { name: 'toHaveAccessibleName', pass: false, expected: 'World', actual: '', timeout: 1 }, + { name: 'toHaveAttribute', pass: false, expected: 'value', actual: null, timeout: 1 }, + { name: 'toHaveAttribute', pass: false, expected: 'have attribute', actual: 'not have attribute', timeout: 1 }, + { name: 'toHaveClass', pass: false, expected: 'name', actual: '', timeout: 1 }, + { name: 'toHaveCount', pass: false, expected: 2, actual: 1, timeout: 1 }, + { name: 'toHaveCSS', pass: false, expected: '10', actual: 'auto', timeout: 1 }, + { name: 'toHaveId', pass: false, expected: 'id', actual: '', timeout: 1 }, + { name: 'toHaveJSProperty', pass: false, expected: 'value', timeout: 1 }, + { name: 'toHaveRole', pass: false, expected: 'role', actual: '', timeout: 1 }, + { name: 'toHaveText', pass: false, expected: 'World', actual: 'Hello', timeout: 1 }, + { name: 'toHaveValue', pass: false, expected: 'value', actual: '', timeout: 1 }, + { name: 'toHaveValues', pass: false, expected: ['value'], actual: [], timeout: 1 }, + ]); +}); diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 8494671c76..90ef7fa75a 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -384,6 +384,7 @@ export type MatcherReturnType = { expected?: unknown; actual?: any; log?: string[]; + timeout?: number; }; type MakeMatchers = { diff --git a/utils/generate_types/overrides-testReporter.d.ts b/utils/generate_types/overrides-testReporter.d.ts index 51eab7e370..0b81724373 100644 --- a/utils/generate_types/overrides-testReporter.d.ts +++ b/utils/generate_types/overrides-testReporter.d.ts @@ -105,6 +105,7 @@ export interface JSONReportTest { export interface JSONReportError { message: string; location?: Location; + matcherResult?: TestErrorMatcherResult; } export interface JSONReportTestResult {