From 159e71982ef931b0ca6792023df5bfb20b3641ee Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Wed, 5 Apr 2023 13:03:42 -0700 Subject: [PATCH] chore: render failed steps in the basic reporters (#22200) Fixes #20532 --- .../playwright-test/src/reporters/base.ts | 35 ++++++++--- tests/playwright-test/reporter-line.spec.ts | 58 +++++++++++++++++++ tests/playwright-test/reporter-list.spec.ts | 20 ++++++- .../stable-test-runner/package-lock.json | 30 +++++----- .../stable-test-runner/package.json | 2 +- 5 files changed, 121 insertions(+), 24 deletions(-) diff --git a/packages/playwright-test/src/reporters/base.ts b/packages/playwright-test/src/reporters/base.ts index 8a840fb8e8..0164ab1394 100644 --- a/packages/playwright-test/src/reporters/base.ts +++ b/packages/playwright-test/src/reporters/base.ts @@ -144,17 +144,17 @@ export class BaseReporter implements Reporter { if (unexpected.length) { tokens.push(colors.red(` ${unexpected.length} failed`)); for (const test of unexpected) - tokens.push(colors.red(formatTestHeader(this.config, test, ' '))); + tokens.push(colors.red(formatTestHeader(this.config, test, { indent: ' ' }))); } if (interrupted.length) { tokens.push(colors.yellow(` ${interrupted.length} interrupted`)); for (const test of interrupted) - tokens.push(colors.yellow(formatTestHeader(this.config, test, ' '))); + tokens.push(colors.yellow(formatTestHeader(this.config, test, { indent: ' ' }))); } if (flaky.length) { tokens.push(colors.yellow(` ${flaky.length} flaky`)); for (const test of flaky) - tokens.push(colors.yellow(formatTestHeader(this.config, test, ' '))); + tokens.push(colors.yellow(formatTestHeader(this.config, test, { indent: ' ' }))); } if (skipped) tokens.push(colors.yellow(` ${skipped} skipped`)); @@ -251,7 +251,7 @@ export function formatFailure(config: FullConfig, test: TestCase, options: {inde const lines: string[] = []; const title = formatTestTitle(config, test); const annotations: Annotation[] = []; - const header = formatTestHeader(config, test, ' ', index); + const header = formatTestHeader(config, test, { indent: ' ', index, mode: 'error' }); lines.push(colors.red(header)); for (const result of test.results) { const resultLines: string[] = []; @@ -371,10 +371,31 @@ export function formatTestTitle(config: FullConfig, test: TestCase, step?: TestS return `${projectTitle}${location} › ${titles.join(' › ')}${stepSuffix(step)}`; } -function formatTestHeader(config: FullConfig, test: TestCase, indent: string, index?: number): string { +function formatTestHeader(config: FullConfig, test: TestCase, options: { indent?: string, index?: number, mode?: 'default' | 'error' } = {}): string { const title = formatTestTitle(config, test); - const header = `${indent}${index ? index + ') ' : ''}${title}`; - return separator(header); + const header = `${options.indent || ''}${options.index ? options.index + ') ' : ''}${title}`; + let fullHeader = header; + + // Render the path to the deepest failing test.step. + if (options.mode === 'error') { + const stepPaths = new Set(); + for (const result of test.results.filter(r => !!r.errors.length)) { + const stepPath: string[] = []; + const visit = (steps: TestStep[]) => { + const errors = steps.filter(s => s.error); + if (errors.length > 1) + return; + if (errors.length === 1 && errors[0].category === 'test.step') { + stepPath.push(errors[0].title); + visit(errors[0].steps); + } + }; + visit(result.steps); + stepPaths.add(['', ...stepPath].join(' › ')); + } + fullHeader = header + (stepPaths.size === 1 ? stepPaths.values().next().value : ''); + } + return separator(fullHeader); } export function formatError(config: FullConfig, error: TestError, highlightCode: boolean): ErrorDetails { diff --git a/tests/playwright-test/reporter-line.spec.ts b/tests/playwright-test/reporter-line.spec.ts index 5db4455e22..23ccaf878b 100644 --- a/tests/playwright-test/reporter-line.spec.ts +++ b/tests/playwright-test/reporter-line.spec.ts @@ -104,3 +104,61 @@ test('should print output', async ({ runInlineTest }) => { 'full-line', ].join('\n')); }); + +test('should render failed test steps', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', async ({}) => { + await test.step('outer 1.0', async () => { + await test.step('inner 1.1', async () => { + expect(1).toBe(2); + }); + }); + }); + `, + }, { reporter: 'line' }); + const text = result.output; + expect(text).toContain('1) a.test.ts:3:11 › passes › outer 1.0 › inner 1.1 ──'); + expect(result.exitCode).toBe(1); +}); + +test('should not render more than one failed test steps in header', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', async ({}) => { + await test.step('outer 1.0', async () => { + await test.step('inner 1.1', async () => { + expect.soft(1).toBe(2); + }); + await test.step('inner 1.2', async () => { + expect.soft(1).toBe(2); + }); + }); + }); + `, + }, { reporter: 'line' }); + const text = result.output; + expect(text).toContain('1) a.test.ts:3:11 › passes › outer 1.0 ──'); + expect(result.exitCode).toBe(1); +}); + +test('should not render more than one failed test steps in header (2)', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', async ({}) => { + await test.step('outer 1.0', async () => { + await test.step('inner 1.1', async () => { + expect.soft(1).toBe(2); + }); + }); + expect.soft(1).toBe(2); + }); + `, + }, { reporter: 'line' }); + const text = result.output; + expect(text).toContain('1) a.test.ts:3:11 › passes ──'); + expect(result.exitCode).toBe(1); +}); diff --git a/tests/playwright-test/reporter-list.spec.ts b/tests/playwright-test/reporter-list.spec.ts index a79cd4830d..068765f3f6 100644 --- a/tests/playwright-test/reporter-list.spec.ts +++ b/tests/playwright-test/reporter-list.spec.ts @@ -87,7 +87,7 @@ test('render steps', async ({ runInlineTest }) => { ]); }); -test('render steps inlint', async ({ runInlineTest }) => { +test('render steps inline', async ({ runInlineTest }) => { const result = await runInlineTest({ 'a.test.ts': ` import { test, expect } from '@playwright/test'; @@ -209,6 +209,24 @@ test('should truncate long test names', async ({ runInlineTest }) => { expect(lines[7]).toBe(` - 4 …› a.test.ts:10:12 › skipped very long name`); }); +test('render failed test steps', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', async ({}) => { + await test.step('outer 1.0', async () => { + await test.step('inner 1.1', async () => { + expect(1).toBe(2); + }); + }); + }); + `, + }, { reporter: 'list' }); + const text = result.output; + expect(text).toContain('1) a.test.ts:3:11 › passes › outer 1.0 › inner 1.1 ──'); + expect(result.exitCode).toBe(1); +}); + function simpleAnsiRenderer(text, ttyWidth) { let lineNumber = 0; let columnNumber = 0; diff --git a/tests/playwright-test/stable-test-runner/package-lock.json b/tests/playwright-test/stable-test-runner/package-lock.json index 7f3565a188..7499e0c905 100644 --- a/tests/playwright-test/stable-test-runner/package-lock.json +++ b/tests/playwright-test/stable-test-runner/package-lock.json @@ -5,16 +5,16 @@ "packages": { "": { "dependencies": { - "@playwright/test": "1.33.0-alpha-mar-20-2023" + "@playwright/test": "1.33.0-alpha-apr-4-2023" } }, "node_modules/@playwright/test": { - "version": "1.33.0-alpha-mar-20-2023", - "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.33.0-alpha-mar-20-2023.tgz", - "integrity": "sha512-547fmDpyQKHMsV17WSXQf1x2XkJ80raOpw+eHpmiROItsMpe7lqUC4OI1hgMxo5miuyPBWOABuuFnXKS11uaFw==", + "version": "1.33.0-alpha-apr-4-2023", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.33.0-alpha-apr-4-2023.tgz", + "integrity": "sha512-sURaHids6lQjVEi2qBm4eJzgg8khwbbwTO/nkmc/fH0xek7BQNo1zcJ+MvrYtarmCeKbMoOj1x/O8FBNgox61g==", "dependencies": { "@types/node": "*", - "playwright-core": "1.33.0-alpha-mar-20-2023" + "playwright-core": "1.33.0-alpha-apr-4-2023" }, "bin": { "playwright": "cli.js" @@ -45,9 +45,9 @@ } }, "node_modules/playwright-core": { - "version": "1.33.0-alpha-mar-20-2023", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.33.0-alpha-mar-20-2023.tgz", - "integrity": "sha512-cjyn0tVVtJa7Nd3/FZf3GHyyDZPEeVTLN2kZ/G3589yiV6oEZgKgxwtMuUK7NPctmctUFPN6eKaPihH6rpNKbA==", + "version": "1.33.0-alpha-apr-4-2023", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.33.0-alpha-apr-4-2023.tgz", + "integrity": "sha512-rPormzFZ2n1AkAqG9ZdeaLkv9mrIkBpidtU47r50RYhe/K5PJDg/kanc4YQzG1ZAPU/t+ijjf/OyGhI/p7piKA==", "bin": { "playwright": "cli.js" }, @@ -58,13 +58,13 @@ }, "dependencies": { "@playwright/test": { - "version": "1.33.0-alpha-mar-20-2023", - "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.33.0-alpha-mar-20-2023.tgz", - "integrity": "sha512-547fmDpyQKHMsV17WSXQf1x2XkJ80raOpw+eHpmiROItsMpe7lqUC4OI1hgMxo5miuyPBWOABuuFnXKS11uaFw==", + "version": "1.33.0-alpha-apr-4-2023", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.33.0-alpha-apr-4-2023.tgz", + "integrity": "sha512-sURaHids6lQjVEi2qBm4eJzgg8khwbbwTO/nkmc/fH0xek7BQNo1zcJ+MvrYtarmCeKbMoOj1x/O8FBNgox61g==", "requires": { "@types/node": "*", "fsevents": "2.3.2", - "playwright-core": "1.33.0-alpha-mar-20-2023" + "playwright-core": "1.33.0-alpha-apr-4-2023" } }, "@types/node": { @@ -79,9 +79,9 @@ "optional": true }, "playwright-core": { - "version": "1.33.0-alpha-mar-20-2023", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.33.0-alpha-mar-20-2023.tgz", - "integrity": "sha512-cjyn0tVVtJa7Nd3/FZf3GHyyDZPEeVTLN2kZ/G3589yiV6oEZgKgxwtMuUK7NPctmctUFPN6eKaPihH6rpNKbA==" + "version": "1.33.0-alpha-apr-4-2023", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.33.0-alpha-apr-4-2023.tgz", + "integrity": "sha512-rPormzFZ2n1AkAqG9ZdeaLkv9mrIkBpidtU47r50RYhe/K5PJDg/kanc4YQzG1ZAPU/t+ijjf/OyGhI/p7piKA==" } } } diff --git a/tests/playwright-test/stable-test-runner/package.json b/tests/playwright-test/stable-test-runner/package.json index c9709af6f6..98eba4e91b 100644 --- a/tests/playwright-test/stable-test-runner/package.json +++ b/tests/playwright-test/stable-test-runner/package.json @@ -1,6 +1,6 @@ { "private": true, "dependencies": { - "@playwright/test": "1.33.0-alpha-mar-20-2023" + "@playwright/test": "1.33.0-alpha-apr-4-2023" } }