diff --git a/docs/src/test-api/class-testinfo.md b/docs/src/test-api/class-testinfo.md index bc73fc12ea..0b087c6b7a 100644 --- a/docs/src/test-api/class-testinfo.md +++ b/docs/src/test-api/class-testinfo.md @@ -128,12 +128,15 @@ The number of milliseconds the test took to finish. Always zero before the test ## property: TestInfo.error -- type: <[Object]> - - `message` <[void]|[string]> Error message. Set when `Error` (or its subclass) has been thrown. - - `stack` <[void]|[string]> Error stack. Set when `Error` (or its subclass) has been thrown. - - `value` <[void]|[string]> The thrown value. Set when anything except the `Error` (or its subclass) has been thrown. +- type: <[void]|[TestError]> -An error thrown during test execution, if any. +First error thrown during test execution, if any. This is equal to the first +element in [`property: TestInfo.errors`]. + +## property: TestInfo.errors +- type: <[Array]<[TestError]>> + +Errors thrown during test execution, if any. ## property: TestInfo.expectedStatus diff --git a/docs/src/test-assertions-js.md b/docs/src/test-assertions-js.md index cd541bc31e..f4e5c73021 100644 --- a/docs/src/test-assertions-js.md +++ b/docs/src/test-assertions-js.md @@ -19,10 +19,25 @@ expect(value).not.toEqual(0); await expect(locator).not.toContainText("some text"); ``` -You can also specify a custom error message as a second argument to the `expect` function, for example: +By default, failed assertion will terminate test execution. Playwright also +supports *soft assertions*: failed soft assertions **do not** terminate test execution, +but mark the test as failed. + +```js +// Make a few checks that will not stop the test when failed... +await expect.soft(page.locator('#status')).toHaveText('Success'); +await expect.soft(page.locator('#eta')).toHaveText('1 day'); + +// ... and continue the test to check more things. +await page.locator('#next-page').click(); +await expect.soft(page.locator('#title')).toHaveText('Make another order'); +``` + +You can specify a custom error message as a second argument to the `expect` function, for example: ```js expect(value, 'my custom error message').toBe(42); +expect.soft(value, 'my soft assertion').toBe(56); ``` diff --git a/docs/src/test-reporter-api/class-testresult.md b/docs/src/test-reporter-api/class-testresult.md index 7a96d057e0..6857eb12a0 100644 --- a/docs/src/test-reporter-api/class-testresult.md +++ b/docs/src/test-reporter-api/class-testresult.md @@ -20,7 +20,13 @@ Running time in milliseconds. ## property: TestResult.error - type: <[void]|[TestError]> -An error thrown during the test execution, if any. +First error thrown during test execution, if any. This is equal to the first +element in [`property: TestResult.errors`]. + +## property: TestResult.errors +- type: <[Array]<[TestError]>> + +Errors thrown during the test execution. ## property: TestResult.retry - type: <[int]> diff --git a/packages/html-reporter/src/testCaseView.spec.tsx b/packages/html-reporter/src/testCaseView.spec.tsx index 64eb73a855..1516c3649a 100644 --- a/packages/html-reporter/src/testCaseView.spec.tsx +++ b/packages/html-reporter/src/testCaseView.spec.tsx @@ -26,6 +26,7 @@ const result: TestResult = { retry: 0, startTime: new Date(0).toUTCString(), duration: 100, + errors: [], steps: [{ title: 'Outer step', startTime: new Date(100).toUTCString(), diff --git a/packages/html-reporter/src/testResultView.css b/packages/html-reporter/src/testResultView.css index 9f07a89cda..1d9c39e949 100644 --- a/packages/html-reporter/src/testResultView.css +++ b/packages/html-reporter/src/testResultView.css @@ -55,6 +55,7 @@ border-radius: 6px; padding: 16px; line-height: initial; + margin-bottom: 6px; } .test-result-counter { diff --git a/packages/html-reporter/src/testResultView.tsx b/packages/html-reporter/src/testResultView.tsx index 0a6c5a85a4..764cb25a14 100644 --- a/packages/html-reporter/src/testResultView.tsx +++ b/packages/html-reporter/src/testResultView.tsx @@ -51,8 +51,8 @@ export const TestResultView: React.FC<{ const diff = attachmentsMap.get('diff'); const hasImages = [actual?.contentType, expected?.contentType, diff?.contentType].some(v => v && /^image\//i.test(v)); return
- {result.error && - + {!!result.errors.length && + {result.errors.map((error, index) => )} } {!!result.steps.length && {result.steps.map((step, i) => )} diff --git a/packages/playwright-test/src/dispatcher.ts b/packages/playwright-test/src/dispatcher.ts index 1c3b8769bb..dcb1806a36 100644 --- a/packages/playwright-test/src/dispatcher.ts +++ b/packages/playwright-test/src/dispatcher.ts @@ -199,7 +199,8 @@ export class Dispatcher { const { result } = data.resultByWorkerIndex.get(worker.workerIndex)!; data.resultByWorkerIndex.delete(worker.workerIndex); result.duration = params.duration; - result.error = params.error; + result.errors = params.errors; + result.error = result.errors[0]; result.attachments = params.attachments.map(a => ({ name: a.name, path: a.path, @@ -292,7 +293,8 @@ export class Dispatcher { if (runningHookId) { const data = this._testById.get(runningHookId)!; const { result } = data.resultByWorkerIndex.get(worker.workerIndex)!; - result.error = params.fatalError; + result.errors = [params.fatalError]; + result.error = result.errors[0]; result.status = 'failed'; this._reporter.onTestEnd?.(data.test, result); } @@ -312,7 +314,8 @@ export class Dispatcher { if (test._type === 'test') this._reporter.onTestBegin?.(test, result); } - result.error = params.fatalError; + result.errors = [params.fatalError]; + result.error = result.errors[0]; result.status = first ? 'failed' : 'skipped'; this._reportTestEnd(test, result); failedTestIds.add(test._id); diff --git a/packages/playwright-test/src/expect.ts b/packages/playwright-test/src/expect.ts index 4524848567..96c7d9f564 100644 --- a/packages/playwright-test/src/expect.ts +++ b/packages/playwright-test/src/expect.ts @@ -92,15 +92,23 @@ export const printReceivedStringContainExpectedResult = ( // #endregion +function createExpect(actual: unknown, message: string|undefined, isSoft: boolean) { + if (message !== undefined && typeof message !== 'string') + throw new Error('expect(actual, optionalErrorMessage): optional error message must be a string.'); + return new Proxy(expectLibrary(actual), new ExpectMetaInfoProxyHandler(message || '', isSoft)); +} + export const expect: Expect = new Proxy(expectLibrary as any, { apply: function(target: any, thisArg: any, argumentsList: [actual: unknown, message: string|undefined]) { - const message = argumentsList[1]; - if (message !== undefined && typeof message !== 'string') - throw new Error('expect(actual, optionalErrorMessage): optional error message must be a string.'); - return new Proxy(expectLibrary.call(thisArg, argumentsList[0]), new ExpectMetaInfoProxyHandler(message || '')); + const [actual, message] = argumentsList; + return createExpect(actual, message, false /* isSoft */); } }); +expect.soft = (actual: unknown, message: string|undefined) => { + return createExpect(actual, message, true /* isSoft */); +}; + expectLibrary.setState({ expand: false }); const customMatchers = { toBeChecked, @@ -128,15 +136,18 @@ const customMatchers = { type ExpectMetaInfo = { message: string; + isSoft: boolean; }; let expectCallMetaInfo: undefined|ExpectMetaInfo = undefined; class ExpectMetaInfoProxyHandler { private _message: string; + private _isSoft: boolean; - constructor(message: string) { + constructor(message: string, isSoft: boolean) { this._message = message; + this._isSoft = isSoft; } get(target: any, prop: any, receiver: any): any { @@ -144,12 +155,26 @@ class ExpectMetaInfoProxyHandler { if (typeof value !== 'function') return new Proxy(value, this); return (...args: any[]) => { + const testInfo = currentTestInfo(); + if (!testInfo) + return value.call(target, ...args); + const handleError = (e: Error) => { + if (this._isSoft) + testInfo._failWithError(serializeError(e), false /* isHardError */); + else + throw e; + }; try { expectCallMetaInfo = { message: this._message, + isSoft: this._isSoft, }; - const result = value.call(target, ...args); + let result = value.call(target, ...args); + if ((result instanceof Promise)) + result = result.catch(handleError); return result; + } catch (e) { + handleError(e); } finally { expectCallMetaInfo = undefined; } @@ -172,10 +197,11 @@ function wrap(matcherName: string, matcher: any) { const stackLines = new Error().stack!.split('\n').slice(INTERNAL_STACK_LENGTH + 1); const frame = stackLines[0] ? stackUtils.parseLine(stackLines[0]) : undefined; const customMessage = expectCallMetaInfo?.message ?? ''; + const isSoft = expectCallMetaInfo?.isSoft ?? false; const step = testInfo._addStep({ location: frame && frame.file ? { file: path.resolve(process.cwd(), frame.file), line: frame.line || 0, column: frame.column || 0 } : undefined, category: 'expect', - title: customMessage || `expect${this.isNot ? '.not' : ''}.${matcherName}`, + title: customMessage || `expect${isSoft ? '.soft' : ''}${this.isNot ? '.not' : ''}.${matcherName}`, canHaveChildren: true, forceNoParent: false }); diff --git a/packages/playwright-test/src/index.ts b/packages/playwright-test/src/index.ts index 34d67fe69b..4e075578b5 100644 --- a/packages/playwright-test/src/index.ts +++ b/packages/playwright-test/src/index.ts @@ -380,7 +380,8 @@ export const test = _baseTest.extend({ const anyContext = leftoverContexts[0]; const pendingCalls = anyContext ? formatPendingCalls((anyContext as any)._connection.pendingProtocolCalls()) : ''; await Promise.all(leftoverContexts.filter(c => createdContexts.has(c)).map(c => c.close())); - testInfo.error = prependToTestError(testInfo.error, pendingCalls); + if (pendingCalls) + testInfo.error = prependToTestError(testInfo.error, pendingCalls); } }, { auto: true }], @@ -434,7 +435,8 @@ export const test = _baseTest.extend({ } })); - testInfo.error = prependToTestError(testInfo.error, prependToError); + if (prependToError) + testInfo.error = prependToTestError(testInfo.error, prependToError); }, context: async ({ _contextFactory }, use) => { diff --git a/packages/playwright-test/src/ipc.ts b/packages/playwright-test/src/ipc.ts index f31badf109..c20c63bb1d 100644 --- a/packages/playwright-test/src/ipc.ts +++ b/packages/playwright-test/src/ipc.ts @@ -39,7 +39,7 @@ export type TestEndPayload = { testId: string; duration: number; status: TestStatus; - error?: TestError; + errors: TestError[]; expectedStatus: TestStatus; annotations: { type: string, description?: string }[]; timeout: number; diff --git a/packages/playwright-test/src/matchers/golden.ts b/packages/playwright-test/src/matchers/golden.ts index 546d04bd68..cc95e3c325 100644 --- a/packages/playwright-test/src/matchers/golden.ts +++ b/packages/playwright-test/src/matchers/golden.ts @@ -23,7 +23,7 @@ import jpeg from 'jpeg-js'; import pixelmatch from 'pixelmatch'; import { diff_match_patch, DIFF_INSERT, DIFF_DELETE, DIFF_EQUAL } from '../third_party/diff_match_patch'; import { UpdateSnapshots } from '../types'; -import { addSuffixToFilePath } from '../util'; +import { addSuffixToFilePath, serializeError } from '../util'; import BlinkDiff from '../third_party/blink-diff'; import PNGImage from '../third_party/png-js'; import { TestInfoImpl } from '../testInfo'; @@ -129,8 +129,8 @@ export function compare( return { pass: true, message }; } if (updateSnapshots === 'missing') { - testInfo._appendErrorMessage(message); - return { pass: true, message }; + testInfo._failWithError(serializeError(new Error(message)), false /* isHardError */); + return { pass: true }; } return { pass: false, message }; } diff --git a/packages/playwright-test/src/reporters/base.ts b/packages/playwright-test/src/reporters/base.ts index 4371c5ca14..a779bb2dd4 100644 --- a/packages/playwright-test/src/reporters/base.ts +++ b/packages/playwright-test/src/reporters/base.ts @@ -33,11 +33,6 @@ type Annotation = { location?: Location; }; -type FailureDetails = { - tokens: string[]; - location?: Location; -}; - type ErrorDetails = { message: string; location?: Location; @@ -99,7 +94,7 @@ export class BaseReporter implements Reporter { } onError(error: TestError) { - console.log(formatError(this.config, error, colors.enabled).message); + console.log('\n' + formatError(this.config, error, colors.enabled).message); } async onEnd(result: FullResult) { @@ -232,14 +227,16 @@ export function formatFailure(config: FullConfig, test: TestCase, options: {inde lines.push(colors.red(header)); for (const result of test.results) { const resultLines: string[] = []; - const { tokens: resultTokens, location } = formatResultFailure(config, test, result, ' ', colors.enabled); - if (!resultTokens.length) + const errors = formatResultFailure(config, test, result, ' ', colors.enabled); + if (!errors.length) continue; + const retryLines = []; if (result.retry) { - resultLines.push(''); - resultLines.push(colors.gray(pad(` Retry #${result.retry}`, '-'))); + retryLines.push(''); + retryLines.push(colors.gray(pad(` Retry #${result.retry}`, '-'))); } - resultLines.push(...resultTokens); + resultLines.push(...retryLines); + resultLines.push(...errors.map(error => '\n' + error.message)); if (includeAttachments) { for (let i = 0; i < result.attachments.length; ++i) { const attachment = result.attachments[i]; @@ -277,11 +274,13 @@ export function formatFailure(config: FullConfig, test: TestCase, options: {inde resultLines.push(''); resultLines.push(colors.gray(pad('--- Test output', '-')) + '\n\n' + outputText + '\n' + pad('', '-')); } - annotations.push({ - location, - title, - message: [header, ...resultLines].join('\n'), - }); + for (const error of errors) { + annotations.push({ + location: error.location, + title, + message: [header, ...retryLines, error.message].join('\n'), + }); + } lines.push(...resultLines); } lines.push(''); @@ -291,25 +290,27 @@ export function formatFailure(config: FullConfig, test: TestCase, options: {inde }; } -export function formatResultFailure(config: FullConfig, test: TestCase, result: TestResult, initialIndent: string, highlightCode: boolean): FailureDetails { - const resultTokens: string[] = []; +export function formatResultFailure(config: FullConfig, test: TestCase, result: TestResult, initialIndent: string, highlightCode: boolean): ErrorDetails[] { + const errorDetails: ErrorDetails[] = []; + if (result.status === 'timedOut') { - resultTokens.push(''); - resultTokens.push(indent(colors.red(`Timeout of ${test.timeout}ms exceeded.`), initialIndent)); + errorDetails.push({ + message: indent(colors.red(`Timeout of ${test.timeout}ms exceeded.`), initialIndent), + }); + } else if (result.status === 'passed' && test.expectedStatus === 'failed') { + errorDetails.push({ + message: indent(colors.red(`Expected to fail, but passed.`), initialIndent), + }); } - if (result.status === 'passed' && test.expectedStatus === 'failed') { - resultTokens.push(''); - resultTokens.push(indent(colors.red(`Expected to fail, but passed.`), initialIndent)); + + for (const error of result.errors) { + const formattedError = formatError(config, error, highlightCode, test.location.file); + errorDetails.push({ + message: indent(formattedError.message, initialIndent), + location: formattedError.location, + }); } - let error: ErrorDetails | undefined = undefined; - if (result.error !== undefined) { - error = formatError(config, result.error, highlightCode, test.location.file); - resultTokens.push(indent(error.message, initialIndent)); - } - return { - tokens: resultTokens, - location: error?.location, - }; + return errorDetails; } function relativeFilePath(config: FullConfig, file: string): string { @@ -341,7 +342,7 @@ function formatTestHeader(config: FullConfig, test: TestCase, indent: string, in export function formatError(config: FullConfig, error: TestError, highlightCode: boolean, file?: string): ErrorDetails { const stack = error.stack; - const tokens = ['']; + const tokens = []; let location: Location | undefined; if (stack) { const parsed = prepareErrorStack(stack, file); diff --git a/packages/playwright-test/src/reporters/html.ts b/packages/playwright-test/src/reporters/html.ts index 29b656a47a..009e85f96a 100644 --- a/packages/playwright-test/src/reporters/html.ts +++ b/packages/playwright-test/src/reporters/html.ts @@ -93,7 +93,7 @@ export type TestResult = { startTime: string; duration: number; steps: TestStep[]; - error?: string; + errors: string[]; attachments: TestAttachment[]; status: 'passed' | 'failed' | 'timedOut' | 'skipped'; }; @@ -393,7 +393,7 @@ class HtmlBuilder { startTime: result.startTime, retry: result.retry, steps: result.steps.map(s => this._createTestStep(s)), - error: result.error, + errors: result.errors, status: result.status, attachments: result.attachments.map(a => { if (a.name === 'trace') diff --git a/packages/playwright-test/src/reporters/raw.ts b/packages/playwright-test/src/reporters/raw.ts index dad07522c3..9cb155c12d 100644 --- a/packages/playwright-test/src/reporters/raw.ts +++ b/packages/playwright-test/src/reporters/raw.ts @@ -83,7 +83,7 @@ export type JsonTestResult = { startTime: string; duration: number; status: TestStatus; - error?: JsonError; + errors: JsonError[]; attachments: JsonAttachment[]; steps: JsonTestStep[]; }; @@ -224,7 +224,7 @@ class RawReporter { startTime: result.startTime.toISOString(), duration: result.duration, status: result.status, - error: formatResultFailure(this.config, test, result, '', true).tokens.join('').trim(), + errors: formatResultFailure(this.config, test, result, '', true).map(error => error.message), attachments: this._createAttachments(result), steps: dedupeSteps(result.steps.map(step => this._serializeStep(test, step))) }; diff --git a/packages/playwright-test/src/test.ts b/packages/playwright-test/src/test.ts index 0ba40a7a2f..121b1d6f7b 100644 --- a/packages/playwright-test/src/test.ts +++ b/packages/playwright-test/src/test.ts @@ -191,7 +191,8 @@ export class TestCase extends Base implements reporterTypes.TestCase { stderr: [], attachments: [], status: 'skipped', - steps: [] + steps: [], + errors: [], }; this.results.push(result); return result; diff --git a/packages/playwright-test/src/testInfo.ts b/packages/playwright-test/src/testInfo.ts index ba647139a1..7913a66b8e 100644 --- a/packages/playwright-test/src/testInfo.ts +++ b/packages/playwright-test/src/testInfo.ts @@ -34,6 +34,7 @@ export class TestInfoImpl implements TestInfo { readonly _timeoutRunner: TimeoutRunner; readonly _startTime: number; readonly _startWallTime: number; + private _hasHardError: boolean = false; // ------------ TestInfo fields ------------ readonly repeatEachIndex: number; @@ -59,7 +60,20 @@ export class TestInfoImpl implements TestInfo { snapshotSuffix: string = ''; readonly outputDir: string; readonly snapshotDir: string; - error: TestError | undefined = undefined; + errors: TestError[] = []; + + get error(): TestError | undefined { + return this.errors.length > 0 ? this.errors[0] : undefined; + } + + set error(e: TestError | undefined) { + if (e === undefined) + throw new Error('Cannot assign testInfo.error undefined value!'); + if (!this.errors.length) + this.errors.push(e); + else + this.errors[0] = e; + } constructor( loader: Loader, @@ -168,7 +182,7 @@ export class TestInfoImpl implements TestInfo { this.status = 'skipped'; } else { const serialized = serializeError(error); - this._failWithError(serialized); + this._failWithError(serialized, true /* isHardError */); return serialized; } } @@ -178,25 +192,18 @@ export class TestInfoImpl implements TestInfo { return this._addStepImpl(data); } - _failWithError(error: TestError) { - // Do not overwrite any previous error and error status. + _failWithError(error: TestError, isHardError: boolean) { + // Do not overwrite any previous hard errors. // Some (but not all) scenarios include: // - expect() that fails after uncaught exception. // - fail after the timeout, e.g. due to fixture teardown. + if (isHardError && this._hasHardError) + return; + if (isHardError) + this._hasHardError = true; if (this.status === 'passed') this.status = 'failed'; - if (this.error === undefined) - this.error = error; - } - - _appendErrorMessage(message: string) { - // Do not overwrite any previous error status. - if (this.status === 'passed') - this.status = 'failed'; - if (this.error === undefined) - this.error = { value: 'Error: ' + message }; - else if (this.error.value) - this.error.value += '\nError: ' + message; + this.errors.push(error); } // ------------ TestInfo methods ------------ diff --git a/packages/playwright-test/src/util.ts b/packages/playwright-test/src/util.ts index 29e5556e45..59ca3e03f0 100644 --- a/packages/playwright-test/src/util.ts +++ b/packages/playwright-test/src/util.ts @@ -24,6 +24,7 @@ import { calculateSha1, isRegExp } from 'playwright-core/lib/utils/utils'; import { isInternalFileName } from 'playwright-core/lib/utils/stackTrace'; const PLAYWRIGHT_CORE_PATH = path.dirname(require.resolve('playwright-core')); +const EXPECT_PATH = path.dirname(require.resolve('expect')); const PLAYWRIGHT_TEST_PATH = path.join(__dirname, '..'); function filterStackTrace(e: Error) { @@ -46,7 +47,7 @@ function filterStackTrace(e: Error) { const functionName = callSite.getFunctionName() || undefined; if (!fileName) return true; - return !fileName.startsWith(PLAYWRIGHT_TEST_PATH) && !fileName.startsWith(PLAYWRIGHT_CORE_PATH) && !isInternalFileName(fileName, functionName); + return !fileName.startsWith(PLAYWRIGHT_TEST_PATH) && !fileName.startsWith(PLAYWRIGHT_CORE_PATH) && !fileName.startsWith(EXPECT_PATH) && !isInternalFileName(fileName, functionName); })); }; // eslint-disable-next-line @@ -202,9 +203,7 @@ export function getContainedPath(parentPath: string, subPath: string = ''): stri export const debugTest = debug('pw:test'); -export function prependToTestError(testError: TestError | undefined, message: string | undefined, location?: Location) { - if (!message) - return testError; +export function prependToTestError(testError: TestError | undefined, message: string, location?: Location): TestError { if (!testError) { if (!location) return { value: message }; diff --git a/packages/playwright-test/src/workerRunner.ts b/packages/playwright-test/src/workerRunner.ts index 9f46d915a5..52420c9f97 100644 --- a/packages/playwright-test/src/workerRunner.ts +++ b/packages/playwright-test/src/workerRunner.ts @@ -95,7 +95,7 @@ export class WorkerRunner extends EventEmitter { // and continuing to run tests in the same worker is problematic. Therefore, // we turn this into a fatal error and restart the worker anyway. if (this._currentTest && this._currentTest._test._type === 'test' && this._currentTest.expectedStatus !== 'failed') { - this._currentTest._failWithError(serializeError(error)); + this._currentTest._failWithError(serializeError(error), true /* isHardError */); } else { // No current test - fatal error. if (!this._fatalError) @@ -395,7 +395,7 @@ function buildTestEndPayload(testInfo: TestInfoImpl): TestEndPayload { testId: testInfo._test._id, duration: testInfo.duration, status: testInfo.status!, - error: testInfo.error, + errors: testInfo.errors, expectedStatus: testInfo.expectedStatus, annotations: testInfo.annotations, timeout: testInfo.timeout, diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index 9c3f948e6d..9919ad175e 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -1492,9 +1492,14 @@ export interface TestInfo { */ status?: TestStatus; /** - * An error thrown during test execution, if any. + * First error thrown during test execution, if any. This is equal to the first element in + * [testInfo.errors](https://playwright.dev/docs/api/class-testinfo#test-info-errors). */ error?: TestError; + /** + * Errors thrown during test execution, if any. + */ + errors: TestError[]; /** * Output written to `process.stdout` or `console.log` during the test execution. */ diff --git a/packages/playwright-test/types/testExpect.d.ts b/packages/playwright-test/types/testExpect.d.ts index 76b0417b9f..1a80948ff1 100644 --- a/packages/playwright-test/types/testExpect.d.ts +++ b/packages/playwright-test/types/testExpect.d.ts @@ -29,6 +29,7 @@ type MakeMatchers = PlaywrightTest.Matchers & export declare type Expect = { (actual: T, message?: string): MakeMatchers; + soft: (actual: T, message?: string) => MakeMatchers; // Sourced from node_modules/expect/build/types.d.ts assertions(arg0: number): void; diff --git a/packages/playwright-test/types/testReporter.d.ts b/packages/playwright-test/types/testReporter.d.ts index 18f92b26fc..1916f4b4e3 100644 --- a/packages/playwright-test/types/testReporter.d.ts +++ b/packages/playwright-test/types/testReporter.d.ts @@ -213,9 +213,14 @@ export interface TestResult { */ status: TestStatus; /** - * An error thrown during the test execution, if any. + * First error thrown during test execution, if any. This is equal to the first element in + * [testResult.errors](https://playwright.dev/docs/api/class-testresult#test-result-errors). */ error?: TestError; + /** + * Errors thrown during the test execution. + */ + errors: TestError[]; /** * The list of files or buffers attached during the test execution through * [testInfo.attachments](https://playwright.dev/docs/api/class-testinfo#test-info-attachments). diff --git a/tests/playwright-test/expect-soft.spec.ts b/tests/playwright-test/expect-soft.spec.ts new file mode 100644 index 0000000000..29af49f18a --- /dev/null +++ b/tests/playwright-test/expect-soft.spec.ts @@ -0,0 +1,80 @@ +/** + * Copyright Microsoft Corporation. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test, expect, stripAnsi } from './playwright-test-fixtures'; + +test('soft expects should compile', async ({ runTSC }) => { + const result = await runTSC({ + 'a.spec.ts': ` + const { test } = pwt; + test('should work', () => { + test.expect.soft(1+1).toBe(3); + test.expect.soft(1+1, 'custom error message').toBe(3); + }); + ` + }); + expect(result.exitCode).toBe(0); +}); + +test('soft expects should work', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + const { test } = pwt; + test('should work', () => { + test.expect.soft(1+1).toBe(3); + console.log('woof-woof'); + }); + ` + }); + expect(result.exitCode).toBe(1); + expect(stripAnsi(result.output)).toContain('woof-woof'); +}); + +test('should report a mixture of soft and non-soft errors', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + const { test } = pwt; + test('should work', ({}) => { + test.expect.soft(1+1, 'one plus one').toBe(3); + test.expect.soft(2*2, 'two times two').toBe(5); + test.expect(3/3, 'three div three').toBe(7); + test.expect.soft(6-4, 'six minus four').toBe(3); + }); + ` + }); + expect(result.exitCode).toBe(1); + expect(stripAnsi(result.output)).toContain('Error: one plus one'); + expect(stripAnsi(result.output)).toContain('Error: two times two'); + expect(stripAnsi(result.output)).toContain('Error: three div three'); + expect(stripAnsi(result.output)).not.toContain('Error: six minus four'); +}); + +test('testInfo should contain all soft expect errors', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + const { test } = pwt; + test('should work', ({}, testInfo) => { + test.expect.soft(1+1, 'one plus one').toBe(3); + test.expect.soft(2*2, 'two times two').toBe(5); + test.expect(testInfo.errors.length, 'must be exactly two errors').toBe(2); + }); + ` + }); + expect(result.exitCode).toBe(1); + expect(stripAnsi(result.output)).toContain('Error: one plus one'); + expect(stripAnsi(result.output)).toContain('Error: two times two'); + expect(stripAnsi(result.output)).not.toContain('Error: must be exactly two errors'); +}); diff --git a/tests/playwright-test/golden.spec.ts b/tests/playwright-test/golden.spec.ts index e1dfba4920..11fc014b2d 100644 --- a/tests/playwright-test/golden.spec.ts +++ b/tests/playwright-test/golden.spec.ts @@ -172,14 +172,18 @@ test('should write missing expectations locally twice and continue', async ({ ru expect(result.failed).toBe(1); const snapshot1OutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.txt'); - expect(result.output).toContain(`${snapshot1OutputPath} is missing in snapshots, writing actual`); + expect(result.output).toContain(`Error: ${snapshot1OutputPath} is missing in snapshots, writing actual`); expect(fs.readFileSync(snapshot1OutputPath, 'utf-8')).toBe('Hello world'); const snapshot2OutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot2.txt'); - expect(result.output).toContain(`${snapshot2OutputPath} is missing in snapshots, writing actual`); + expect(result.output).toContain(`Error: ${snapshot2OutputPath} is missing in snapshots, writing actual`); expect(fs.readFileSync(snapshot2OutputPath, 'utf-8')).toBe('Hello world2'); expect(result.output).toContain('Here we are!'); + + const stackLines = stripAnsi(result.output).split('\n').filter(line => line.includes(' at ')).filter(line => !line.includes(testInfo.outputPath())); + expect(result.output).toContain('a.spec.js:8'); + expect(stackLines.length).toBe(0); }); test('shouldn\'t write missing expectations locally for negated matcher', async ({ runInlineTest }, testInfo) => { diff --git a/tests/playwright-test/reporter-github.spec.ts b/tests/playwright-test/reporter-github.spec.ts index 44a2ef3e7e..d4c554893a 100644 --- a/tests/playwright-test/reporter-github.spec.ts +++ b/tests/playwright-test/reporter-github.spec.ts @@ -89,6 +89,6 @@ test('print GitHub annotations for global error', async ({ runInlineTest }) => { `, }, { reporter: 'github' }); const text = stripAnsi(result.output); - expect(text).toContain('::error ::%0AError: Oh my!%0A%0A'); + expect(text).toContain('::error ::Error: Oh my!%0A%0A'); expect(result.exitCode).toBe(1); }); diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 6eb666ea26..8707f0fd83 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -216,6 +216,7 @@ export interface TestInfo { duration: number; status?: TestStatus; error?: TestError; + errors: TestError[]; stdout: (string | Buffer)[]; stderr: (string | Buffer)[]; snapshotSuffix: string; diff --git a/utils/generate_types/overrides-testReporter.d.ts b/utils/generate_types/overrides-testReporter.d.ts index af511e8ab1..592eb6e421 100644 --- a/utils/generate_types/overrides-testReporter.d.ts +++ b/utils/generate_types/overrides-testReporter.d.ts @@ -56,6 +56,7 @@ export interface TestResult { duration: number; status: TestStatus; error?: TestError; + errors: TestError[]; attachments: { name: string, path?: string, body?: Buffer, contentType: string }[]; stdout: (string | Buffer)[]; stderr: (string | Buffer)[];