diff --git a/packages/playwright-test/src/reporters/html.ts b/packages/playwright-test/src/reporters/html.ts index eaa026dc75..24caa0b3c6 100644 --- a/packages/playwright-test/src/reporters/html.ts +++ b/packages/playwright-test/src/reporters/html.ts @@ -131,9 +131,12 @@ class HtmlReporter implements Reporter { private config!: FullConfigInternal; private suite!: Suite; private _options: HtmlReporterOptions; + private _outputFolder!: string; + private _open: string | undefined; constructor(options: HtmlReporterOptions = {}) { this._options = options; + console.log('HTML REPORTER OPTIONS', options); } printsToStdio() { @@ -142,6 +145,25 @@ class HtmlReporter implements Reporter { onBegin(config: FullConfig, suite: Suite) { this.config = config as FullConfigInternal; + const { outputFolder, open } = this._resolveOptions(); + this._outputFolder = outputFolder; + this._open = open; + const reportedWarnings = new Set(); + for (const project of config.projects) { + if (outputFolder.startsWith(project.outputDir) || project.outputDir.startsWith(outputFolder)) { + const key = outputFolder + '|' + project.outputDir; + if (reportedWarnings.has(key)) + continue; + reportedWarnings.add(key); + console.log(colors.red(`Configuration Error: HTML reporter output folder clashes with the tests output folder:`)); + console.log(` + html reporter folder: ${colors.bold(outputFolder)} + test results folder: ${colors.bold(project.outputDir)}`); + console.log(''); + console.log(`HTML reporter will clear its output directory prior to being generated, which will lead to the artifact loss. +`); + } + } this.suite = suite; } @@ -156,26 +178,25 @@ class HtmlReporter implements Reporter { } async onEnd() { - const { open, outputFolder } = this._resolveOptions(); const projectSuites = this.suite.suites; const reports = projectSuites.map(suite => { const rawReporter = new RawReporter(); const report = rawReporter.generateProjectReport(this.config, suite); return report; }); - await removeFolders([outputFolder]); - const builder = new HtmlBuilder(outputFolder); + await removeFolders([this._outputFolder]); + const builder = new HtmlBuilder(this._outputFolder); const { ok, singleTestId } = await builder.build(this.config.metadata, reports); if (process.env.CI) return; - const shouldOpen = open === 'always' || (!ok && open === 'on-failure'); + const shouldOpen = this._open === 'always' || (!ok && this._open === 'on-failure'); if (shouldOpen) { - await showHTMLReport(outputFolder, singleTestId); + await showHTMLReport(this._outputFolder, singleTestId); } else { - const relativeReportPath = outputFolder === standaloneDefaultFolder() ? '' : ' ' + path.relative(process.cwd(), outputFolder); + const relativeReportPath = this._outputFolder === standaloneDefaultFolder() ? '' : ' ' + path.relative(process.cwd(), this._outputFolder); console.log(''); console.log('To open last HTML report run:'); console.log(colors.cyan(` diff --git a/tests/config/commonFixtures.ts b/tests/config/commonFixtures.ts index 33ac045eb2..b629a762b8 100644 --- a/tests/config/commonFixtures.ts +++ b/tests/config/commonFixtures.ts @@ -32,7 +32,7 @@ export class TestChildProcess { process: ChildProcess; output = ''; onOutput?: () => void; - exited: Promise<{ exitCode: number | null, signal: string | null }>; + exited: Promise<{ exitCode: number, signal: string | null }>; exitCode: Promise; private _outputCallbacks = new Set<() => void>(); diff --git a/tests/playwright-test/playwright-test-fixtures.ts b/tests/playwright-test/playwright-test-fixtures.ts index 4abf4bdcec..20c1e385cb 100644 --- a/tests/playwright-test/playwright-test-fixtures.ts +++ b/tests/playwright-test/playwright-test-fixtures.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import type { JSONReport, JSONReportSuite } from '@playwright/test/reporter'; +import type { JSONReport, JSONReportSuite, JSONReportTestResult } from '@playwright/test/reporter'; import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; @@ -102,7 +102,7 @@ async function writeFiles(testInfo: TestInfo, files: Files) { const cliEntrypoint = path.join(__dirname, '../../packages/playwright-core/cli.js'); async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], baseDir: string, params: any, env: Env, options: RunOptions): Promise { - const paramList = []; + const paramList: string[] = []; for (const key of Object.keys(params)) { for (const value of Array.isArray(params[key]) ? params[key] : [params[key]]) { const k = key.startsWith('-') ? key : '--' + key; @@ -114,8 +114,9 @@ async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], b const args = ['node', cliEntrypoint, 'test']; if (!options.usesCustomOutputDir) args.push('--output=' + outputDir); + if (!options.usesCustomReporters) + args.push('--reporter=dot,json'); args.push( - '--reporter=dot,json', '--workers=2', ...paramList ); @@ -181,7 +182,7 @@ async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], b testProcess.output += '\n' + e.toString(); } - const results = []; + const results: JSONReportTestResult[] = []; function visitSuites(suites?: JSONReportSuite[]) { if (!suites) return; @@ -211,6 +212,7 @@ async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], b type RunOptions = { sendSIGINTAfter?: number; usesCustomOutputDir?: boolean; + usesCustomReporters?: boolean; additionalArgs?: string[]; cwd?: string, }; diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts index 3e6057353d..360ae7e5f7 100644 --- a/tests/playwright-test/reporter-html.spec.ts +++ b/tests/playwright-test/reporter-html.spec.ts @@ -16,7 +16,7 @@ import fs from 'fs'; import path from 'path'; -import { test as baseTest, expect, createImage } from './playwright-test-fixtures'; +import { test as baseTest, expect, createImage, stripAnsi } from './playwright-test-fixtures'; import type { HttpServer } from '../../packages/playwright-core/lib/utils/httpServer'; import { startHtmlReportServer } from '../../packages/playwright-test/lib/reporters/html'; import { spawnAsync } from 'playwright-core/lib/utils/spawnAsync'; @@ -860,3 +860,22 @@ test.describe('gitCommitInfo plugin', () => { await expect.soft(page.locator('data-test-id=metadata-chip')).not.toBeVisible(); }); }); + +test('should report clashing folders', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + reporter: [['html', { outputFolder: 'test-results/html-report' }]] + } + `, + 'a.test.js': ` + const { test } = pwt; + test('passes', async ({}) => { + }); + `, + }, {}, {}, { usesCustomReporters: true }); + expect(result.exitCode).toBe(0); + const output = stripAnsi(result.output); + expect(output).toContain('Configuration Error'); + expect(output).toContain('html-report'); +});