From a867e738dbf3ff4b150c0a7d55389d97f8118620 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Wed, 2 Aug 2023 14:49:06 -0700 Subject: [PATCH] chore(merge): console status updates (#24559) --- packages/playwright-test/src/cli.ts | 5 +- .../playwright-test/src/reporters/merge.ts | 41 ++++++++++--- .../playwright-test/src/runner/reporters.ts | 2 +- tests/playwright-test/reporter-blob.spec.ts | 59 +++++++++++++++++++ tests/playwright-test/reporter-html.spec.ts | 6 +- tests/playwright-test/reporter-junit.spec.ts | 2 +- 6 files changed, 100 insertions(+), 15 deletions(-) diff --git a/packages/playwright-test/src/cli.ts b/packages/playwright-test/src/cli.ts index f1ee0bc64a..c88c336644 100644 --- a/packages/playwright-test/src/cli.ts +++ b/packages/playwright-test/src/cli.ts @@ -189,8 +189,11 @@ async function mergeReports(reportDir: string | undefined, opts: { [key: string] const configLoader = new ConfigLoader(); const config = await (configFile ? configLoader.loadConfigFile(configFile) : configLoader.loadEmptyConfig(process.cwd())); const dir = path.resolve(process.cwd(), reportDir || ''); - if (!(await fs.promises.stat(dir)).isDirectory()) + const dirStat = await fs.promises.stat(dir).catch(e => null); + if (!dirStat) throw new Error('Directory does not exist: ' + dir); + if (!dirStat.isDirectory()) + throw new Error(`"${dir}" is not a directory`); let reporterDescriptions: ReporterDescription[] | undefined = resolveReporterOption(opts.reporter); if (!reporterDescriptions && configFile) reporterDescriptions = config.config.reporter; diff --git a/packages/playwright-test/src/reporters/merge.ts b/packages/playwright-test/src/reporters/merge.ts index 991c36a132..791e68d6d9 100644 --- a/packages/playwright-test/src/reporters/merge.ts +++ b/packages/playwright-test/src/reporters/merge.ts @@ -25,19 +25,35 @@ import { createReporters } from '../runner/reporters'; import { Multiplexer } from './multiplexer'; import { ZipFile } from 'playwright-core/lib/utils'; import type { BlobReportMetadata } from './blob'; +import { relativeFilePath } from '../util'; + +type StatusCallback = (message: string) => void; export async function createMergedReport(config: FullConfigInternal, dir: string, reporterDescriptions: ReporterDescription[]) { + const reporters = await createReporters(config, 'merge', reporterDescriptions); + const multiplexer = new Multiplexer(reporters); + const receiver = new TeleReporterReceiver(path.sep, multiplexer, false, config.config); + + let printStatus: StatusCallback = () => {}; + if (!multiplexer.printsToStdio()) { + printStatus = printStatusToStdout; + printStatus(`merging reports from ${dir}`); + } + const shardFiles = await sortedShardFiles(dir); if (shardFiles.length === 0) throw new Error(`No report files found in ${dir}`); - const events = await mergeEvents(dir, shardFiles); + const events = await mergeEvents(dir, shardFiles, printStatus); patchAttachmentPaths(events, dir); - const reporters = await createReporters(config, 'merge', reporterDescriptions); - const receiver = new TeleReporterReceiver(path.sep, new Multiplexer(reporters), false, config.config); - - for (const event of events) + printStatus(`processing ${events.length} test events`); + for (const event of events) { + if (event.method === 'onEnd') + printStatus(`building final report`); await receiver.dispatch(event); + if (event.method === 'onEnd') + printStatus(`finished building report`); + } } function patchAttachmentPaths(events: JsonEvent[], resourceDir: string) { @@ -57,11 +73,13 @@ function parseEvents(reportJsonl: Buffer): JsonEvent[] { return reportJsonl.toString().split('\n').filter(line => line.length).map(line => JSON.parse(line)) as JsonEvent[]; } -async function extractAndParseReports(dir: string, shardFiles: string[]): Promise<{ metadata: BlobReportMetadata, parsedEvents: JsonEvent[] }[]> { +async function extractAndParseReports(dir: string, shardFiles: string[], printStatus: StatusCallback): Promise<{ metadata: BlobReportMetadata, parsedEvents: JsonEvent[] }[]> { const shardEvents = []; await fs.promises.mkdir(path.join(dir, 'resources'), { recursive: true }); for (const file of shardFiles) { - const zipFile = new ZipFile(path.join(dir, file)); + const absolutePath = path.join(dir, file); + printStatus(`extracting: ${relativeFilePath(absolutePath)}`); + const zipFile = new ZipFile(absolutePath); const entryNames = await zipFile.entries(); for (const entryName of entryNames) { const content = await zipFile.read(entryName); @@ -87,18 +105,19 @@ function findMetadata(events: JsonEvent[], file: string): BlobReportMetadata { return events[0].params; } -async function mergeEvents(dir: string, shardReportFiles: string[]) { +async function mergeEvents(dir: string, shardReportFiles: string[], printStatus: StatusCallback) { const events: JsonEvent[] = []; const configureEvents: JsonEvent[] = []; const beginEvents: JsonEvent[] = []; const endEvents: JsonEvent[] = []; - const shardEvents = await extractAndParseReports(dir, shardReportFiles); + const shardEvents = await extractAndParseReports(dir, shardReportFiles, printStatus); shardEvents.sort((a, b) => { const shardA = a.metadata.shard?.current ?? 0; const shardB = b.metadata.shard?.current ?? 0; return shardA - shardB; }); const allTestIds = new Set(); + printStatus(`merging events`); for (const { parsedEvents } of shardEvents) { for (const event of parsedEvents) { if (event.method === 'onConfigure') @@ -213,6 +232,10 @@ async function sortedShardFiles(dir: string) { return files.filter(file => file.startsWith('report-') && file.endsWith('.zip')).sort(); } +function printStatusToStdout(message: string) { + process.stdout.write(`${message}\n`); +} + class ProjectNamePatcher { private _testIds = new Set(); diff --git a/packages/playwright-test/src/runner/reporters.ts b/packages/playwright-test/src/runner/reporters.ts index d77c8ad862..f71b3d87ad 100644 --- a/packages/playwright-test/src/runner/reporters.ts +++ b/packages/playwright-test/src/runner/reporters.ts @@ -69,7 +69,7 @@ export async function createReporters(config: FullConfigInternal, mode: 'list' | // Important to put it first, jsut in case some other reporter stalls onEnd. if (mode === 'list') reporters.unshift(new ListModeReporter()); - else + else if (mode !== 'merge') reporters.unshift(!process.env.CI ? new LineReporter({ omitFailures: true }) : new DotReporter()); } return reporters; diff --git a/tests/playwright-test/reporter-blob.spec.ts b/tests/playwright-test/reporter-blob.spec.ts index 469a7695a1..035e8b706e 100644 --- a/tests/playwright-test/reporter-blob.spec.ts +++ b/tests/playwright-test/reporter-blob.spec.ts @@ -412,6 +412,65 @@ test('merge into list report by default', async ({ runInlineTest, mergeReports } ]); }); +test('should print progress', async ({ runInlineTest, mergeReports }) => { + const reportDir = test.info().outputPath('blob-report'); + const files = { + 'playwright.config.ts': ` + module.exports = { + retries: 1, + reporter: [['blob', { outputDir: '${reportDir.replace(/\\/g, '/')}' }]] + }; + `, + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('math 1', async ({}) => { + expect(1 + 1).toBe(2); + }); + test('failing 1', async ({}) => { + expect(1).toBe(2); + }); + test('flaky 1', async ({}) => { + expect(test.info().retry).toBe(1); + }); + test.skip('skipped 1', async ({}) => {}); + `, + 'b.test.js': ` + import { test, expect } from '@playwright/test'; + test('math 2', async ({}) => { + expect(1 + 1).toBe(2); + }); + test('failing 2', async ({}) => { + expect(1).toBe(2); + }); + test.skip('skipped 2', async ({}) => {}); + `, + 'c.test.js': ` + import { test, expect } from '@playwright/test'; + test('math 3', async ({}) => { + expect(1 + 1).toBe(2); + }); + test('flaky 2', async ({}) => { + expect(test.info().retry).toBe(1); + }); + test.skip('skipped 3', async ({}) => {}); + ` + }; + + await runInlineTest(files, { shard: `1/2` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' }); + await runInlineTest(files, { shard: `2/2` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' }); + const reportFiles = await fs.promises.readdir(reportDir); + reportFiles.sort(); + expect(reportFiles).toEqual([expect.stringMatching(/report-.*.zip/), expect.stringMatching(/report-.*.zip/)]); + const { exitCode, output } = await mergeReports(reportDir, { PW_TEST_HTML_REPORT_OPEN: 'never' }, { additionalArgs: ['--reporter', 'html'] }); + expect(exitCode).toBe(0); + + const lines = output.split('\n'); + expect(lines).toContainEqual(expect.stringMatching(/extracting: blob-report[\/\\]report-.*zip$/)); + expect(lines).toContainEqual(expect.stringMatching(/merging events$/)); + expect(lines).toContainEqual(expect.stringMatching(/building final report/)); + expect(lines).toContainEqual(expect.stringMatching(/finished building report/)); +}); + test('preserve attachments', async ({ runInlineTest, mergeReports, showReport, page }) => { const reportDir = test.info().outputPath('blob-report'); const files = { diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts index 33f9ddee00..74c6f5758b 100644 --- a/tests/playwright-test/reporter-html.spec.ts +++ b/tests/playwright-test/reporter-html.spec.ts @@ -344,7 +344,7 @@ for (const useIntermediateMergeReport of [false, true] as const) { video: 'on', trace: 'on', }, - reporter: [['html', { attachmentsBaseURL: 'https://some-url.com/' }]] + reporter: [['html', { attachmentsBaseURL: 'https://some-url.com/' }], ['line']] }; `, 'a.test.js': ` @@ -1019,7 +1019,7 @@ for (const useIntermediateMergeReport of [false, true] as const) { test('pass', ({}, testInfo) => { }); ` - }, { 'reporter': 'html' }, { PW_TEST_HTML_REPORT_OPEN: 'never' }, { + }, { 'reporter': 'html,line' }, { PW_TEST_HTML_REPORT_OPEN: 'never' }, { cwd: 'foo/bar/baz/tests', }); expect(result.exitCode).toBe(0); @@ -1043,7 +1043,7 @@ for (const useIntermediateMergeReport of [false, true] as const) { test('pass', ({}, testInfo) => { }); ` - }, { 'reporter': 'html' }, { 'PW_TEST_HTML_REPORT_OPEN': 'never', 'PLAYWRIGHT_HTML_REPORT': '../my-report' }, { + }, { 'reporter': 'html,line' }, { 'PW_TEST_HTML_REPORT_OPEN': 'never', 'PLAYWRIGHT_HTML_REPORT': '../my-report' }, { cwd: 'foo/bar/baz/tests', }); expect(result.exitCode).toBe(0); diff --git a/tests/playwright-test/reporter-junit.spec.ts b/tests/playwright-test/reporter-junit.spec.ts index ec8f2950a9..c3fe77fd05 100644 --- a/tests/playwright-test/reporter-junit.spec.ts +++ b/tests/playwright-test/reporter-junit.spec.ts @@ -438,7 +438,7 @@ for (const useIntermediateMergeReport of [false, true] as const) { test('pass', ({}, testInfo) => { }); ` - }, { 'reporter': 'junit' }, { 'PLAYWRIGHT_JUNIT_OUTPUT_NAME': '../my-report.xml' }, { + }, { 'reporter': 'junit,line' }, { 'PLAYWRIGHT_JUNIT_OUTPUT_NAME': '../my-report.xml' }, { cwd: 'foo/bar/baz/tests', }); expect(result.exitCode).toBe(0);