diff --git a/packages/playwright-test/src/reporters/merge.ts b/packages/playwright-test/src/reporters/merge.ts index af4e36bb51..0dd8907af1 100644 --- a/packages/playwright-test/src/reporters/merge.ts +++ b/packages/playwright-test/src/reporters/merge.ts @@ -97,7 +97,7 @@ function mergeEvents(shardReports: string[]) { events.push(event); } } - return [mergeBeginEvents(beginEvents), ...events, mergeEndEvents(endEvents)]; + return [mergeBeginEvents(beginEvents), ...events, mergeEndEvents(endEvents), { method: 'onExit', params: undefined }]; } function mergeBeginEvents(beginEvents: JsonEvent[]): JsonEvent { diff --git a/packages/playwright-test/src/reporters/teleEmitter.ts b/packages/playwright-test/src/reporters/teleEmitter.ts index 2b2c50cda7..79b0c3e16a 100644 --- a/packages/playwright-test/src/reporters/teleEmitter.ts +++ b/packages/playwright-test/src/reporters/teleEmitter.ts @@ -122,6 +122,9 @@ export class TeleReporterEmitter implements Reporter { this._messageSink({ method: 'onEnd', params: { result } }); } + async onExit() { + } + private _serializeConfig(config: FullConfig): JsonConfig { return { rootDir: config.rootDir, diff --git a/tests/playwright-test/reporter-blob.spec.ts b/tests/playwright-test/reporter-blob.spec.ts index e0c5fb9040..171434900c 100644 --- a/tests/playwright-test/reporter-blob.spec.ts +++ b/tests/playwright-test/reporter-blob.spec.ts @@ -57,6 +57,105 @@ const test = baseTest.extend<{ test.use({ channel: 'chrome' }); +const echoReporterJs = ` +class EchoReporter { + onBegin(config, suite) { + console.log('onBegin'); + } + onTestBegin(test, result) { + console.log('onTestBegin'); + } + onStdOut(chunk, test, result) { + console.log('onStdOut'); + } + onStdErr(chunk, test, result) { + console.log('onStdErr'); + } + onTestEnd(test, result) { + console.log('onTestEnd'); + } + onEnd(result) { + console.log('onEnd'); + } + onExit() { + console.log('onExit'); + } + onError(error) { + console.log('onError'); + } + onStepBegin(test, result, step) { + } + onStepEnd(test, result, step) { + } +} +module.exports = EchoReporter; +`; + +test('should call methods in right order', async ({ runInlineTest, mergeReports, showReport, page }) => { + test.slow(); + const reportDir = test.info().outputPath('blob-report'); + const files = { + 'echo-reporter.js': echoReporterJs, + 'playwright.config.ts': ` + module.exports = { + retries: 1, + reporter: [['blob', { outputDir: '${reportDir.replace(/\\/g, '/')}' }]] + }; + `, + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('math 1', async ({}) => { + expect(1 + 1).toBe(2); + }); + test('failing 1', async ({}) => { + expect(1).toBe(2); + }); + test('flaky 1', async ({}) => { + expect(test.info().retry).toBe(1); + }); + test.skip('skipped 1', async ({}) => {}); + `, + 'b.test.js': ` + import { test, expect } from '@playwright/test'; + test('math 2', async ({}) => { + expect(1 + 1).toBe(2); + }); + test('failing 2', async ({}) => { + expect(1).toBe(2); + }); + test.skip('skipped 2', async ({}) => {}); + `, + 'c.test.js': ` + import { test, expect } from '@playwright/test'; + test('math 3', async ({}) => { + expect(1 + 1).toBe(2); + }); + test('flaky 2', async ({}) => { + expect(test.info().retry).toBe(1); + }); + test.skip('skipped 3', async ({}) => {}); + ` + }; + await runInlineTest(files, { shard: `1/3` }); + await runInlineTest(files, { shard: `3/3` }); + const reportFiles = await fs.promises.readdir(reportDir); + reportFiles.sort(); + expect(reportFiles).toEqual(['report-1-of-3.zip', 'report-3-of-3.zip']); + const { exitCode, output } = await mergeReports(reportDir, {}, { additionalArgs: ['--reporter', test.info().outputPath('echo-reporter.js')] }); + expect(exitCode).toBe(0); + const lines = output.split('\n').filter(l => l.trim().length); + expect(lines[0]).toBe('onBegin'); + expect(lines).toContain('onTestBegin'); + expect(lines).toContain('onEnd'); + expect(lines).toContain('onExit'); + expect(lines.indexOf('onBegin')).toBeLessThan(lines.indexOf('onTestBegin')); + expect(lines[lines.length - 2]).toBe('onEnd'); + expect(lines[lines.length - 1]).toBe('onExit'); + expect(lines.filter(l => l === 'onBegin').length).toBe(1); + expect(lines.filter(l => l === 'onEnd').length).toBe(1); + expect(lines.filter(l => l === 'onExit').length).toBe(1); +}); + test('should merge into html', async ({ runInlineTest, mergeReports, showReport, page }) => { test.slow(); const reportDir = test.info().outputPath('blob-report'); @@ -107,9 +206,11 @@ test('should merge into html', async ({ runInlineTest, mergeReports, showReport, const reportFiles = await fs.promises.readdir(reportDir); reportFiles.sort(); expect(reportFiles).toEqual(['report-1-of-3.zip', 'report-2-of-3.zip', 'report-3-of-3.zip']); - const { exitCode } = await mergeReports(reportDir, {}, { additionalArgs: ['--reporter', 'html'] }); + const { exitCode, output } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] }); expect(exitCode).toBe(0); + expect(output).toContain('To open last HTML report run:'); + await showReport(); await expect(page.locator('.subnav-item:has-text("All") .counter')).toHaveText('10'); @@ -166,7 +267,7 @@ test('be able to merge incomplete shards', async ({ runInlineTest, mergeReports, const reportFiles = await fs.promises.readdir(reportDir); reportFiles.sort(); expect(reportFiles).toEqual(['report-1-of-3.zip', 'report-3-of-3.zip']); - const { exitCode } = await mergeReports(reportDir, {}, { additionalArgs: ['--reporter', 'html'] }); + const { exitCode } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] }); expect(exitCode).toBe(0); await showReport(); @@ -300,7 +401,7 @@ test('preserve attachments', async ({ runInlineTest, mergeReports, showReport, p const reportFiles = await fs.promises.readdir(reportDir); reportFiles.sort(); expect(reportFiles).toEqual(['report-1-of-2.zip']); - const { exitCode } = await mergeReports(reportDir, {}, { additionalArgs: ['--reporter', 'html'] }); + const { exitCode } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] }); expect(exitCode).toBe(0); await showReport(); @@ -354,7 +455,7 @@ test('multiple output reports', async ({ runInlineTest, mergeReports, showReport const reportFiles = await fs.promises.readdir(reportDir); reportFiles.sort(); expect(reportFiles).toEqual(['report-1-of-2.zip']); - const { exitCode, output } = await mergeReports(reportDir, { 'PW_TEST_DEBUG_REPORTERS': '1' }, { additionalArgs: ['--reporter', 'html,line'] }); + const { exitCode, output } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never', 'PW_TEST_DEBUG_REPORTERS': '1' }, { additionalArgs: ['--reporter', 'html,line'] }); expect(exitCode).toBe(0); // Check that line reporter was called. @@ -374,7 +475,7 @@ test('multiple output reports based on config', async ({ runInlineTest, mergeRep const files = { 'merged/playwright.config.ts': ` module.exports = { - reporter: [['blob', { outputDir: 'merged-blob' }], ['html', { outputFolder: 'html' }], ['line']] + reporter: [['blob', { outputDir: 'merged-blob' }], ['html', { outputFolder: 'html', open: 'never' }], ['line']] }; `, 'playwright.config.ts': ` @@ -431,4 +532,4 @@ test('multiple output reports based on config', async ({ runInlineTest, mergeRep // Check report presence. expect((await fs.promises.stat(test.info().outputPath('merged/merged-blob/report.zip'))).isFile).toBeTruthy(); -}); \ No newline at end of file +});