From b9e5a934ee17bcde7aeaefd293209250cb21283a Mon Sep 17 00:00:00 2001 From: Dmitry Gozman Date: Mon, 22 Apr 2024 15:30:51 -0700 Subject: [PATCH] test: make expectations readable for more step reporting tests (#30468) --- tests/playwright-test/reporter.spec.ts | 310 ------------------------ tests/playwright-test/test-step.spec.ts | 216 ++++++++++++++++- 2 files changed, 209 insertions(+), 317 deletions(-) diff --git a/tests/playwright-test/reporter.spec.ts b/tests/playwright-test/reporter.spec.ts index 3c0f81e1f0..b3cf60c94f 100644 --- a/tests/playwright-test/reporter.spec.ts +++ b/tests/playwright-test/reporter.spec.ts @@ -40,41 +40,6 @@ class Reporter { module.exports = Reporter; `; -const stepsReporterJS = ` -class Reporter { - onStdOut(chunk) { - process.stdout.write(chunk); - } - distillStep(step) { - return { - ...step, - _startTime: undefined, - startTime: undefined, - duration: undefined, - parent: undefined, - data: undefined, - location: undefined, - steps: step.steps.length ? step.steps.map(s => this.distillStep(s)) : undefined, - }; - } - onStepBegin(test, result, step) { - console.log('%%%% begin', JSON.stringify(this.distillStep(step))); - } - onStepEnd(test, result, step) { - if (step.error?.stack) - step.error.stack = ''; - if (step.error?.location) - step.error.location = ''; - if (step.error?.snippet) - step.error.snippet = ''; - if (step.error?.message.includes('getaddrinfo')) - step.error.message = ''; - console.log('%%%% end', JSON.stringify(this.distillStep(step))); - } -} -module.exports = Reporter; -`; - for (const useIntermediateMergeReport of [false, true] as const) { test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => { test.use({ useIntermediateMergeReport }); @@ -244,232 +209,6 @@ for (const useIntermediateMergeReport of [false, true] as const) { ]); }); - test('should report expect steps', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'reporter.ts': stepsReporterJS, - 'playwright.config.ts': ` - module.exports = { - reporter: './reporter', - }; - `, - 'a.test.ts': ` - import { test, expect } from '@playwright/test'; - test('fail', async ({}) => { - expect(true).toBeTruthy(); - expect(false).toBeTruthy(); - }); - test('pass', async ({}) => { - expect(false).not.toBeTruthy(); - }); - test('async', async ({ page }) => { - await expect(page).not.toHaveTitle('False'); - }); - ` - }, { reporter: '', workers: 1 }); - - expect(result.exitCode).toBe(1); - expect(result.outputLines).toEqual([ - `begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, - `end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, - `begin {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\"}`, - `end {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\"}`, - `begin {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\"}`, - `end {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\",\"error\":{\"message\":\"Error: \\u001b[2mexpect(\\u001b[22m\\u001b[31mreceived\\u001b[39m\\u001b[2m).\\u001b[22mtoBeTruthy\\u001b[2m()\\u001b[22m\\n\\nReceived: \\u001b[31mfalse\\u001b[39m\",\"stack\":\"\",\"location\":\"\",\"snippet\":\"\"}}`, - `begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, - `end {\"title\":\"After Hooks\",\"category\":\"hook\"}`, - `begin {\"title\":\"Worker Cleanup\",\"category\":\"hook\"}`, - `end {\"title\":\"Worker Cleanup\",\"category\":\"hook\"}`, - `begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, - `end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, - `begin {\"title\":\"expect.not.toBeTruthy\",\"category\":\"expect\"}`, - `end {\"title\":\"expect.not.toBeTruthy\",\"category\":\"expect\"}`, - `begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, - `end {\"title\":\"After Hooks\",\"category\":\"hook\"}`, - `begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, - `begin {\"title\":\"fixture: browser\",\"category\":\"fixture\"}`, - `begin {\"title\":\"browserType.launch\",\"category\":\"pw:api\"}`, - `end {\"title\":\"browserType.launch\",\"category\":\"pw:api\"}`, - `end {\"title\":\"fixture: browser\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserType.launch\",\"category\":\"pw:api\"}]}`, - `begin {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, - `begin {\"title\":\"browser.newContext\",\"category\":\"pw:api\"}`, - `end {\"title\":\"browser.newContext\",\"category\":\"pw:api\"}`, - `end {\"title\":\"fixture: context\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browser.newContext\",\"category\":\"pw:api\"}]}`, - `begin {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, - `begin {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`, - `end {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`, - `end {\"title\":\"fixture: page\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}]}`, - `end {\"title\":\"Before Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"fixture: browser\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserType.launch\",\"category\":\"pw:api\"}]},{\"title\":\"fixture: context\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browser.newContext\",\"category\":\"pw:api\"}]},{\"title\":\"fixture: page\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}]}]}`, - `begin {\"title\":\"expect.not.toHaveTitle\",\"category\":\"expect\"}`, - `end {\"title\":\"expect.not.toHaveTitle\",\"category\":\"expect\"}`, - `begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, - `begin {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, - `end {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, - `begin {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, - `end {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, - `end {\"title\":\"After Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"fixture: page\",\"category\":\"fixture\"},{\"title\":\"fixture: context\",\"category\":\"fixture\"}]}`, - ]); - }); - - test('should report api steps', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'reporter.ts': stepsReporterJS, - 'playwright.config.ts': ` - module.exports = { - reporter: './reporter', - }; - `, - 'a.test.ts': ` - import { test, expect } from '@playwright/test'; - test('pass', async ({ page, request }) => { - await Promise.all([ - page.waitForNavigation(), - page.goto('data:text/html,'), - ]); - await page.click('button'); - await page.getByRole('button').click(); - await page.request.get('http://localhost2').catch(() => {}); - await request.get('http://localhost2').catch(() => {}); - }); - - test.describe('suite', () => { - let myPage; - test.beforeAll(async ({ browser }) => { - myPage = await browser.newPage(); - await myPage.setContent(''); - }); - - test('pass1', async () => { - await myPage.click('button'); - }); - test('pass2', async () => { - await myPage.click('button'); - }); - - test.afterAll(async () => { - await myPage.close(); - }); - }); - ` - }, { reporter: '', workers: 1 }); - - expect(result.exitCode).toBe(0); - expect(result.outputLines).toEqual([ - `begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, - `begin {\"title\":\"fixture: browser\",\"category\":\"fixture\"}`, - `begin {\"title\":\"browserType.launch\",\"category\":\"pw:api\"}`, - `end {\"title\":\"browserType.launch\",\"category\":\"pw:api\"}`, - `end {\"title\":\"fixture: browser\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserType.launch\",\"category\":\"pw:api\"}]}`, - `begin {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, - `begin {\"title\":\"browser.newContext\",\"category\":\"pw:api\"}`, - `end {\"title\":\"browser.newContext\",\"category\":\"pw:api\"}`, - `end {\"title\":\"fixture: context\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browser.newContext\",\"category\":\"pw:api\"}]}`, - `begin {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, - `begin {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`, - `end {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`, - `end {\"title\":\"fixture: page\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}]}`, - `begin {\"title\":\"fixture: request\",\"category\":\"fixture\"}`, - `begin {\"title\":\"apiRequest.newContext\",\"category\":\"pw:api\"}`, - `end {\"title\":\"apiRequest.newContext\",\"category\":\"pw:api\"}`, - `end {\"title\":\"fixture: request\",\"category\":\"fixture\",\"steps\":[{\"title\":\"apiRequest.newContext\",\"category\":\"pw:api\"}]}`, - `end {\"title\":\"Before Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"fixture: browser\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserType.launch\",\"category\":\"pw:api\"}]},{\"title\":\"fixture: context\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browser.newContext\",\"category\":\"pw:api\"}]},{\"title\":\"fixture: page\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}]},{\"title\":\"fixture: request\",\"category\":\"fixture\",\"steps\":[{\"title\":\"apiRequest.newContext\",\"category\":\"pw:api\"}]}]}`, - `begin {\"title\":\"page.waitForNavigation\",\"category\":\"pw:api\"}`, - `begin {\"title\":\"page.goto(data:text/html,)\",\"category\":\"pw:api\"}`, - `end {\"title\":\"page.waitForNavigation\",\"category\":\"pw:api\",\"steps\":[{\"title\":\"page.goto(data:text/html,)\",\"category\":\"pw:api\"}]}`, - `end {\"title\":\"page.goto(data:text/html,)\",\"category\":\"pw:api\"}`, - `begin {\"title\":\"page.click(button)\",\"category\":\"pw:api\"}`, - `end {\"title\":\"page.click(button)\",\"category\":\"pw:api\"}`, - `begin {\"title\":\"locator.getByRole('button').click\",\"category\":\"pw:api\"}`, - `end {\"title\":\"locator.getByRole('button').click\",\"category\":\"pw:api\"}`, - `begin {"title":"apiRequestContext.get(http://localhost2)","category":"pw:api"}`, - `end {"title":"apiRequestContext.get(http://localhost2)","category":"pw:api","error":{"message":"","stack":"","location":"","snippet":""}}`, - `begin {"title":"apiRequestContext.get(http://localhost2)","category":"pw:api"}`, - `end {"title":"apiRequestContext.get(http://localhost2)","category":"pw:api","error":{"message":"","stack":"","location":"","snippet":""}}`, - `begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, - `begin {\"title\":\"fixture: request\",\"category\":\"fixture\"}`, - `begin {\"title\":\"apiRequestContext.dispose\",\"category\":\"pw:api\"}`, - `end {\"title\":\"apiRequestContext.dispose\",\"category\":\"pw:api\"}`, - `end {\"title\":\"fixture: request\",\"category\":\"fixture\",\"steps\":[{\"title\":\"apiRequestContext.dispose\",\"category\":\"pw:api\"}]}`, - `begin {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, - `end {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, - `begin {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, - `end {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, - `end {\"title\":\"After Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"fixture: request\",\"category\":\"fixture\",\"steps\":[{\"title\":\"apiRequestContext.dispose\",\"category\":\"pw:api\"}]},{\"title\":\"fixture: page\",\"category\":\"fixture\"},{\"title\":\"fixture: context\",\"category\":\"fixture\"}]}`, - `begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, - `begin {\"title\":\"beforeAll hook\",\"category\":\"hook\"}`, - `begin {\"title\":\"browser.newPage\",\"category\":\"pw:api\"}`, - `end {\"title\":\"browser.newPage\",\"category\":\"pw:api\"}`, - `begin {\"title\":\"page.setContent\",\"category\":\"pw:api\"}`, - `end {\"title\":\"page.setContent\",\"category\":\"pw:api\"}`, - `end {\"title\":\"beforeAll hook\",\"category\":\"hook\",\"steps\":[{\"title\":\"browser.newPage\",\"category\":\"pw:api\"},{\"title\":\"page.setContent\",\"category\":\"pw:api\"}]}`, - `end {\"title\":\"Before Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"beforeAll hook\",\"category\":\"hook\",\"steps\":[{\"title\":\"browser.newPage\",\"category\":\"pw:api\"},{\"title\":\"page.setContent\",\"category\":\"pw:api\"}]}]}`, - `begin {\"title\":\"page.click(button)\",\"category\":\"pw:api\"}`, - `end {\"title\":\"page.click(button)\",\"category\":\"pw:api\"}`, - `begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, - `end {\"title\":\"After Hooks\",\"category\":\"hook\"}`, - `begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, - `end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, - `begin {\"title\":\"page.click(button)\",\"category\":\"pw:api\"}`, - `end {\"title\":\"page.click(button)\",\"category\":\"pw:api\"}`, - `begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, - `begin {\"title\":\"afterAll hook\",\"category\":\"hook\"}`, - `begin {\"title\":\"page.close\",\"category\":\"pw:api\"}`, - `end {\"title\":\"page.close\",\"category\":\"pw:api\"}`, - `end {\"title\":\"afterAll hook\",\"category\":\"hook\",\"steps\":[{\"title\":\"page.close\",\"category\":\"pw:api\"}]}`, - `end {\"title\":\"After Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"afterAll hook\",\"category\":\"hook\",\"steps\":[{\"title\":\"page.close\",\"category\":\"pw:api\"}]}]}`, - ]); - }); - - - test('should report api step failure', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'reporter.ts': stepsReporterJS, - 'playwright.config.ts': ` - module.exports = { - reporter: './reporter', - }; - `, - 'a.test.ts': ` - import { test, expect } from '@playwright/test'; - test('fail', async ({ page }) => { - await page.setContent(''); - await page.click('input', { timeout: 1 }); - }); - ` - }, { reporter: '', workers: 1 }); - - expect(result.exitCode).toBe(1); - expect(result.outputLines).toEqual([ - `begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, - `begin {\"title\":\"fixture: browser\",\"category\":\"fixture\"}`, - `begin {\"title\":\"browserType.launch\",\"category\":\"pw:api\"}`, - `end {\"title\":\"browserType.launch\",\"category\":\"pw:api\"}`, - `end {\"title\":\"fixture: browser\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserType.launch\",\"category\":\"pw:api\"}]}`, - `begin {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, - `begin {\"title\":\"browser.newContext\",\"category\":\"pw:api\"}`, - `end {\"title\":\"browser.newContext\",\"category\":\"pw:api\"}`, - `end {\"title\":\"fixture: context\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browser.newContext\",\"category\":\"pw:api\"}]}`, - `begin {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, - `begin {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`, - `end {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`, - `end {\"title\":\"fixture: page\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}]}`, - `end {\"title\":\"Before Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"fixture: browser\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserType.launch\",\"category\":\"pw:api\"}]},{\"title\":\"fixture: context\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browser.newContext\",\"category\":\"pw:api\"}]},{\"title\":\"fixture: page\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}]}]}`, - `begin {\"title\":\"page.setContent\",\"category\":\"pw:api\"}`, - `end {\"title\":\"page.setContent\",\"category\":\"pw:api\"}`, - `begin {\"title\":\"page.click(input)\",\"category\":\"pw:api\"}`, - `end {\"title\":\"page.click(input)\",\"category\":\"pw:api\",\"error\":{\"message\":\"TimeoutError: page.click: Timeout 1ms exceeded.\\nCall log:\\n \\u001b[2m- waiting for locator('input')\\u001b[22m\\n\",\"stack\":\"\",\"location\":\"\",\"snippet\":\"\"}}`, - `begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, - `begin {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, - `end {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, - `begin {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, - `end {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, - `end {\"title\":\"After Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"fixture: page\",\"category\":\"fixture\"},{\"title\":\"fixture: context\",\"category\":\"fixture\"}]}`, - `begin {\"title\":\"Worker Cleanup\",\"category\":\"hook\"}`, - `begin {\"title\":\"fixture: browser\",\"category\":\"fixture\"}`, - `end {\"title\":\"fixture: browser\",\"category\":\"fixture\"}`, - `end {\"title\":\"Worker Cleanup\",\"category\":\"hook\",\"steps\":[{\"title\":\"fixture: browser\",\"category\":\"fixture\"}]}`, - ]); - }); - test('should not have internal error when steps are finished after timeout', async ({ runInlineTest }) => { const result = await runInlineTest({ 'a.test.ts': ` @@ -493,55 +232,6 @@ for (const useIntermediateMergeReport of [false, true] as const) { expect(result.output).not.toContain('Internal error'); }); - test('should show nice stacks for locators', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'reporter.ts': stepsReporterJS, - 'playwright.config.ts': ` - module.exports = { - reporter: './reporter', - }; - `, - 'a.test.ts': ` - import { test, expect } from '@playwright/test'; - test('pass', async ({ page }) => { - await page.setContent(''); - const locator = page.locator('button'); - await locator.evaluate(e => e.innerText); - }); - ` - }, { reporter: '', workers: 1 }); - - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(0); - expect(result.output).not.toContain('Internal error'); - expect(result.outputLines).toEqual([ - `begin {"title":"Before Hooks","category":"hook"}`, - `begin {\"title\":\"fixture: browser\",\"category\":\"fixture\"}`, - `begin {\"title\":\"browserType.launch\",\"category\":\"pw:api\"}`, - `end {\"title\":\"browserType.launch\",\"category\":\"pw:api\"}`, - `end {\"title\":\"fixture: browser\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserType.launch\",\"category\":\"pw:api\"}]}`, - `begin {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, - `begin {\"title\":\"browser.newContext\",\"category\":\"pw:api\"}`, - `end {\"title\":\"browser.newContext\",\"category\":\"pw:api\"}`, - `end {\"title\":\"fixture: context\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browser.newContext\",\"category\":\"pw:api\"}]}`, - `begin {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, - `begin {"title":"browserContext.newPage","category":"pw:api"}`, - `end {"title":"browserContext.newPage","category":"pw:api"}`, - `end {\"title\":\"fixture: page\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}]}`, - `end {\"title\":\"Before Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"fixture: browser\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserType.launch\",\"category\":\"pw:api\"}]},{\"title\":\"fixture: context\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browser.newContext\",\"category\":\"pw:api\"}]},{\"title\":\"fixture: page\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}]}]}`, - `begin {"title":"page.setContent","category":"pw:api"}`, - `end {"title":"page.setContent","category":"pw:api"}`, - `begin {"title":"locator.evaluate(button)","category":"pw:api"}`, - `end {"title":"locator.evaluate(button)","category":"pw:api"}`, - `begin {"title":"After Hooks","category":"hook"}`, - `begin {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, - `end {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, - `begin {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, - `end {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, - `end {\"title\":\"After Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"fixture: page\",\"category\":\"fixture\"},{\"title\":\"fixture: context\",\"category\":\"fixture\"}]}`, - ]); - }); - test('should report forbid-only error to reporter', async ({ runInlineTest }) => { const result = await runInlineTest({ 'reporter.ts': smallReporterJS, diff --git a/tests/playwright-test/test-step.spec.ts b/tests/playwright-test/test-step.spec.ts index 4b49a622ca..c35070c945 100644 --- a/tests/playwright-test/test-step.spec.ts +++ b/tests/playwright-test/test-step.spec.ts @@ -23,11 +23,6 @@ function formatPrefix(str) { return str.padEnd(10, ' ') + '|'; } -function trimError(message) { - const lines = message.split('\\n'); - return lines[0]; -} - function formatLocation(location) { return ' @ ' + path.basename(location.file) + ':' + location.line; } @@ -45,9 +40,20 @@ function formatStack(indent, stack) { class Reporter { printErrorLocation: boolean; + skipErrorMessage: boolean; + constructor(options) { this.printErrorLocation = options.printErrorLocation; + this.skipErrorMessage = options.skipErrorMessage; } + + trimError(message) { + if (this.skipErrorMessage) + return ''; + const lines = message.split('\\n'); + return lines[0]; + } + onBegin(config: FullConfig, suite: Suite) { this.suite = suite; } @@ -68,7 +74,7 @@ class Reporter { console.log(formatPrefix(step.category) + indent + step.title + location); if (step.error) { const errorLocation = this.printErrorLocation ? formatLocation(step.error.location) : ''; - console.log(formatPrefix(step.category) + indent + '↪ error: ' + trimError(step.error.message) + errorLocation); + console.log(formatPrefix(step.category) + indent + '↪ error: ' + this.trimError(step.error.message) + errorLocation); if (this.printErrorLocation) console.log(formatStack(formatPrefix(step.category) + indent, step.error.stack)); } @@ -88,7 +94,7 @@ class Reporter { this.printStep(step, ''); for (const error of result.errors) { const errorLocation = this.printErrorLocation ? formatLocation(error.location) : ''; - console.log(formatPrefix('') + trimError(error.message) + errorLocation); + console.log(formatPrefix('') + this.trimError(error.message) + errorLocation); if (this.printErrorLocation) console.log(formatStack(formatPrefix(''), error.stack)); } @@ -1037,3 +1043,199 @@ fixture | fixture: page fixture | fixture: context `); }); + +test('should report expect steps', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': stepIndentReporter, + 'playwright.config.ts': ` + module.exports = { + reporter: './reporter', + }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('fail', async ({}) => { + expect(true).toBeTruthy(); + expect(false).toBeTruthy(); + }); + test('pass', async ({}) => { + expect(false).not.toBeTruthy(); + }); + test('async', async ({ page }) => { + await expect(page).not.toHaveTitle('False'); + }); + ` + }, { reporter: '', workers: 1 }); + + expect(result.exitCode).toBe(1); + expect(stripAnsi(result.output)).toBe(` +hook |Before Hooks +expect |expect.toBeTruthy @ a.test.ts:4 +expect |expect.toBeTruthy @ a.test.ts:5 +expect |↪ error: Error: expect(received).toBeTruthy() +hook |After Hooks +hook |Worker Cleanup + |Error: expect(received).toBeTruthy() +hook |Before Hooks +expect |expect.not.toBeTruthy @ a.test.ts:8 +hook |After Hooks +hook |Before Hooks +fixture | fixture: browser +pw:api | browserType.launch +fixture | fixture: context +pw:api | browser.newContext +fixture | fixture: page +pw:api | browserContext.newPage +expect |expect.not.toHaveTitle @ a.test.ts:11 +hook |After Hooks +fixture | fixture: page +fixture | fixture: context +`); +}); + +test('should report api steps', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': stepIndentReporter, + 'playwright.config.ts': `module.exports = { reporter: [['./reporter', { skipErrorMessage: true }]] };`, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('pass', async ({ page, request }) => { + await Promise.all([ + page.waitForNavigation(), + page.goto('data:text/html,'), + ]); + await page.click('button'); + await page.getByRole('button').click(); + await page.request.get('http://localhost2').catch(() => {}); + await request.get('http://localhost2').catch(() => {}); + }); + + test.describe('suite', () => { + let myPage; + test.beforeAll(async ({ browser }) => { + myPage = await browser.newPage(); + await myPage.setContent(''); + }); + + test('pass1', async () => { + await myPage.click('button'); + }); + test('pass2', async () => { + await myPage.click('button'); + }); + + test.afterAll(async () => { + await myPage.close(); + }); + }); + ` + }, { reporter: '', workers: 1 }); + + expect(result.exitCode).toBe(0); + expect(stripAnsi(result.output)).toBe(` +hook |Before Hooks +hook | beforeAll hook @ a.test.ts:16 +pw:api | browser.newPage @ a.test.ts:17 +pw:api | page.setContent @ a.test.ts:18 +pw:api |page.click(button) @ a.test.ts:22 +hook |After Hooks +hook |Before Hooks +pw:api |page.click(button) @ a.test.ts:25 +hook |After Hooks +hook | afterAll hook @ a.test.ts:28 +pw:api | page.close @ a.test.ts:29 +hook |Before Hooks +fixture | fixture: browser +pw:api | browserType.launch +fixture | fixture: context +pw:api | browser.newContext +fixture | fixture: page +pw:api | browserContext.newPage +fixture | fixture: request +pw:api | apiRequest.newContext +pw:api |page.waitForNavigation @ a.test.ts:5 +pw:api | page.goto(data:text/html,) @ a.test.ts:6 +pw:api |page.click(button) @ a.test.ts:8 +pw:api |locator.getByRole('button').click @ a.test.ts:9 +pw:api |apiRequestContext.get(http://localhost2) @ a.test.ts:10 +pw:api |↪ error: +pw:api |apiRequestContext.get(http://localhost2) @ a.test.ts:11 +pw:api |↪ error: +hook |After Hooks +fixture | fixture: request +pw:api | apiRequestContext.dispose +fixture | fixture: page +fixture | fixture: context +`); +}); + +test('should report api step failure', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': stepIndentReporter, + 'playwright.config.ts': ` + module.exports = { + reporter: './reporter', + }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('fail', async ({ page }) => { + await page.setContent(''); + await page.click('input', { timeout: 1 }); + }); + ` + }, { reporter: '', workers: 1 }); + + expect(result.exitCode).toBe(1); + expect(stripAnsi(result.output)).toBe(` +hook |Before Hooks +fixture | fixture: browser +pw:api | browserType.launch +fixture | fixture: context +pw:api | browser.newContext +fixture | fixture: page +pw:api | browserContext.newPage +pw:api |page.setContent @ a.test.ts:4 +pw:api |page.click(input) @ a.test.ts:5 +pw:api |↪ error: TimeoutError: page.click: Timeout 1ms exceeded. +hook |After Hooks +fixture | fixture: page +fixture | fixture: context +hook |Worker Cleanup +fixture | fixture: browser + |TimeoutError: page.click: Timeout 1ms exceeded. +`); +}); + +test('should show nice stacks for locators', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': stepIndentReporter, + 'playwright.config.ts': `module.exports = { reporter: [['./reporter', { printErrorLocation: true }]] };`, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('pass', async ({ page }) => { + await page.setContent(''); + const locator = page.locator('button'); + await locator.evaluate(e => e.innerText); + }); + ` + }, { reporter: '', workers: 1 }); + + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(0); + expect(result.output).not.toContain('Internal error'); + expect(stripAnsi(result.output)).toBe(` +hook |Before Hooks +fixture | fixture: browser +pw:api | browserType.launch +fixture | fixture: context +pw:api | browser.newContext +fixture | fixture: page +pw:api | browserContext.newPage +pw:api |page.setContent @ a.test.ts:4 +pw:api |locator.evaluate(button) @ a.test.ts:6 +hook |After Hooks +fixture | fixture: page +fixture | fixture: context +`); +});