diff --git a/packages/playwright-test/src/reporters/base.ts b/packages/playwright-test/src/reporters/base.ts index cb11f1997c..bfb5df663b 100644 --- a/packages/playwright-test/src/reporters/base.ts +++ b/packages/playwright-test/src/reporters/base.ts @@ -275,7 +275,7 @@ export function formatFailure(config: FullConfig, test: TestCase, options: {inde lines.push(colors.red(header)); for (const result of test.results) { const resultLines: string[] = []; - const errors = formatResultFailure(config, test, result, ' ', colors.enabled); + const errors = formatResultFailure(test, result, ' ', colors.enabled); if (!errors.length) continue; const retryLines = []; @@ -342,7 +342,7 @@ export function formatFailure(config: FullConfig, test: TestCase, options: {inde }; } -export function formatResultFailure(config: FullConfig, test: TestCase, result: TestResult, initialIndent: string, highlightCode: boolean): ErrorDetails[] { +export function formatResultFailure(test: TestCase, result: TestResult, initialIndent: string, highlightCode: boolean): ErrorDetails[] { const errorDetails: ErrorDetails[] = []; if (result.status === 'passed' && test.expectedStatus === 'failed') { @@ -357,7 +357,7 @@ export function formatResultFailure(config: FullConfig, test: TestCase, result: } for (const error of result.errors) { - const formattedError = formatError(config, error, highlightCode); + const formattedError = formatError(error, highlightCode); errorDetails.push({ message: indent(formattedError.message, initialIndent), location: formattedError.location, @@ -418,7 +418,7 @@ function formatTestHeader(config: FullConfig, test: TestCase, options: { indent? return separator(fullHeader); } -export function formatError(config: FullConfig, error: TestError, highlightCode: boolean): ErrorDetails { +export function formatError(error: TestError, highlightCode: boolean): ErrorDetails { const message = error.message || error.value || ''; const stack = error.stack; if (!stack && !error.location) diff --git a/packages/playwright-test/src/reporters/dot.ts b/packages/playwright-test/src/reporters/dot.ts index e52f3c02b5..71599ff352 100644 --- a/packages/playwright-test/src/reporters/dot.ts +++ b/packages/playwright-test/src/reporters/dot.ts @@ -66,7 +66,7 @@ class DotReporter extends BaseReporter { override onError(error: TestError): void { super.onError(error); - console.log('\n' + formatError(this.config, error, colors.enabled).message); + console.log('\n' + formatError(error, colors.enabled).message); this._counter = 0; } diff --git a/packages/playwright-test/src/reporters/github.ts b/packages/playwright-test/src/reporters/github.ts index 9fe888edfe..85bd4f8525 100644 --- a/packages/playwright-test/src/reporters/github.ts +++ b/packages/playwright-test/src/reporters/github.ts @@ -69,7 +69,7 @@ export class GitHubReporter extends BaseReporter { } override onError(error: TestError) { - const errorMessage = formatError(this.config, error, false).message; + const errorMessage = formatError(error, false).message; this.githubLogger.error(errorMessage); } diff --git a/packages/playwright-test/src/reporters/json.ts b/packages/playwright-test/src/reporters/json.ts index 574ad28340..f85427bb4f 100644 --- a/packages/playwright-test/src/reporters/json.ts +++ b/packages/playwright-test/src/reporters/json.ts @@ -208,7 +208,7 @@ class JSONReporter extends EmptyReporter { } private _serializeError(error: TestError): JSONReportError { - return formatError(this.config, error, true); + return formatError(error, true); } private _serializeTestStep(step: TestStep): JSONReportTestStep { diff --git a/packages/playwright-test/src/reporters/line.ts b/packages/playwright-test/src/reporters/line.ts index 822009372b..66345aa879 100644 --- a/packages/playwright-test/src/reporters/line.ts +++ b/packages/playwright-test/src/reporters/line.ts @@ -106,7 +106,7 @@ class LineReporter extends BaseReporter { override onError(error: TestError): void { super.onError(error); - const message = formatError(this.config, error, colors.enabled).message + '\n\n'; + const message = formatError(error, colors.enabled).message + '\n\n'; if (!process.env.PW_TEST_DEBUG_REPORTERS) process.stdout.write(`\u001B[1A\u001B[2K`); process.stdout.write(message); diff --git a/packages/playwright-test/src/reporters/list.ts b/packages/playwright-test/src/reporters/list.ts index 480ddf35d3..40de8cb818 100644 --- a/packages/playwright-test/src/reporters/list.ts +++ b/packages/playwright-test/src/reporters/list.ts @@ -243,7 +243,7 @@ class ListReporter extends BaseReporter { override onError(error: TestError): void { super.onError(error); this._maybeWriteNewLine(); - const message = formatError(this.config, error, colors.enabled).message + '\n'; + const message = formatError(error, colors.enabled).message + '\n'; this._updateLineCountAndNewLineFlagForOutput(message); process.stdout.write(message); } diff --git a/packages/playwright-test/src/reporters/markdown.ts b/packages/playwright-test/src/reporters/markdown.ts index d0a4731ecb..c5a6e29b3c 100644 --- a/packages/playwright-test/src/reporters/markdown.ts +++ b/packages/playwright-test/src/reporters/markdown.ts @@ -17,7 +17,7 @@ import fs from 'fs'; import path from 'path'; import type { FullResult, TestCase } from '../../types/testReporter'; -import { BaseReporter, formatTestTitle } from './base'; +import { BaseReporter, formatError, formatTestTitle, stripAnsiEscapes } from './base'; type MarkdownReporterOptions = { configDir: string, @@ -41,31 +41,66 @@ class MarkdownReporter extends BaseReporter { await super.onEnd(result); const summary = this.generateSummary(); const lines: string[] = []; - lines.push(`:x: failed: ${summary.unexpected.length}`); - this._printTestList(summary.unexpected, lines); + if (summary.unexpected.length) { + lines.push(`**${summary.unexpected.length} failed**`); + this._printTestList(':x:', summary.unexpected, lines); + } if (summary.flaky.length) { - lines.push(`:warning: flaky: ${summary.flaky.length}`); - this._printTestList(summary.flaky, lines); + lines.push(`**${summary.flaky.length} flaky**`); + this._printTestList(':warning:', summary.flaky, lines); } if (summary.interrupted.length) { - lines.push(`:warning: interrupted: ${summary.interrupted.length}`); - this._printTestList(summary.interrupted, lines); + lines.push(`**${summary.interrupted.length} interrupted**`); + this._printTestList(':warning:', summary.interrupted, lines); } - if (summary.skipped) { - lines.push(`:ballot_box_with_check: skipped: ${summary.skipped}`); - lines.push(``); - } - lines.push(`:white_check_mark: passed: ${summary.expected}`); + const skipped = summary.skipped ? `, ${summary.skipped} skipped` : ''; + lines.push(`**${summary.expected} passed${skipped}**`); + lines.push(`:heavy_check_mark::heavy_check_mark::heavy_check_mark:`); lines.push(``); + if (summary.unexpected.length || summary.flaky.length) { + lines.push(`
`); + lines.push(``); + if (summary.unexpected.length) + this._printTestListDetails(':x:', summary.unexpected, lines); + if (summary.flaky.length) + this._printTestListDetails(':warning:', summary.flaky, lines); + lines.push(`
`); + } + const reportFile = path.resolve(this._options.configDir, this._options.outputFile || 'report.md'); await fs.promises.mkdir(path.dirname(reportFile), { recursive: true }); await fs.promises.writeFile(reportFile, lines.join('\n')); } - private _printTestList(tests: TestCase[], lines: string[]) { + private _printTestList(prefix: string, tests: TestCase[], lines: string[]) { for (const test of tests) - lines.push(` - ${formatTestTitle(this.config, test)}`); + lines.push(`${prefix} ${formatTestTitle(this.config, test)}`); + lines.push(``); + } + + private _printTestListDetails(prefix: string, tests: TestCase[], lines: string[]) { + for (const test of tests) + this._printTestDetails(prefix, test, lines); + } + + private _printTestDetails(prefix: string, test: TestCase, lines: string[]) { + lines.push(`${prefix} ${formatTestTitle(this.config, test)} `); + let retry = 0; + for (const result of test.results) { + if (result.status === 'passed') + break; + if (retry) + lines.push(`Retry ${retry}:`); + retry++; + if (result.error?.snippet) { + lines.push(``); + lines.push('```'); + lines.push(stripAnsiEscapes(formatError(result.error, false).message)); + lines.push('```'); + lines.push(``); + } + } lines.push(``); } } diff --git a/packages/playwright-test/src/reporters/raw.ts b/packages/playwright-test/src/reporters/raw.ts index 030389f67a..3b8bb8e282 100644 --- a/packages/playwright-test/src/reporters/raw.ts +++ b/packages/playwright-test/src/reporters/raw.ts @@ -246,7 +246,7 @@ class RawReporter { startTime: result.startTime.toISOString(), duration: result.duration, status: result.status, - errors: formatResultFailure(this.config, test, result, '', true).map(error => error.message), + errors: formatResultFailure(test, result, '', true).map(error => error.message), attachments: this.generateAttachments(result.attachments, result), steps: dedupeSteps(result.steps.map(step => this._serializeStep(test, step))) }; diff --git a/packages/playwright-test/src/runner/reporters.ts b/packages/playwright-test/src/runner/reporters.ts index c7851896d5..d77c8ad862 100644 --- a/packages/playwright-test/src/runner/reporters.ts +++ b/packages/playwright-test/src/runner/reporters.ts @@ -102,6 +102,6 @@ class ListModeReporter extends EmptyReporter { override onError(error: TestError) { // eslint-disable-next-line no-console - console.error('\n' + formatError(this.config, error, false).message); + console.error('\n' + formatError(error, false).message); } } diff --git a/tests/playwright-test/reporter-markdown.spec.ts b/tests/playwright-test/reporter-markdown.spec.ts index 55a9b992c1..a96eeb69bb 100644 --- a/tests/playwright-test/reporter-markdown.spec.ts +++ b/tests/playwright-test/reporter-markdown.spec.ts @@ -62,17 +62,34 @@ test('simple report', async ({ runInlineTest }) => { const { exitCode } = await runInlineTest(files); expect(exitCode).toBe(1); const reportFile = await fs.promises.readFile(test.info().outputPath('report.md')); - expect(reportFile.toString()).toBe(`:x: failed: 2 - - a.test.js:6:11 › failing 1 - - b.test.js:6:11 › failing 2 + expect(reportFile.toString()).toContain(`**2 failed** +:x: a.test.js:6:11 › failing 1 +:x: b.test.js:6:11 › failing 2 -:warning: flaky: 2 - - a.test.js:9:11 › flaky 1 - - c.test.js:6:11 › flaky 2 +**2 flaky** +:warning: a.test.js:9:11 › flaky 1 +:warning: c.test.js:6:11 › flaky 2 -:ballot_box_with_check: skipped: 3 +**3 passed, 3 skipped** +:heavy_check_mark::heavy_check_mark::heavy_check_mark: -:white_check_mark: passed: 3 +
+ +:x: a.test.js:6:11 › failing 1 +`); + + expect(reportFile.toString()).toContain(`Error: expect(received).toBe(expected) // Object.is equality + +Expected: 2 +Received: 1 + + 5 | }); + 6 | test('failing 1', async ({}) => { +> 7 | expect(1).toBe(2); + | ^ + 8 | }); + 9 | test('flaky 1', async ({}) => { + 10 | expect(test.info().retry).toBe(1); `); }); @@ -94,8 +111,7 @@ test('custom report file', async ({ runInlineTest }) => { const { exitCode } = await runInlineTest(files); expect(exitCode).toBe(0); const reportFile = await fs.promises.readFile(test.info().outputPath('my-report.md')); - expect(reportFile.toString()).toBe(`:x: failed: 0 - -:white_check_mark: passed: 1 + expect(reportFile.toString()).toBe(`**1 passed** +:heavy_check_mark::heavy_check_mark::heavy_check_mark: `); }); \ No newline at end of file