From 3112edb4cae8285f2fde2652a16c141211cebf1f Mon Sep 17 00:00:00 2001 From: Dmitry Gozman Date: Mon, 1 Aug 2022 09:01:23 -0700 Subject: [PATCH] feat(test runner): TestProject.projectSetup (#16063) `projectSetup` is a project-scoped alternative to `globalSetup`. It is only executed if at least one test from the project is scheduled to run. --- docs/src/test-advanced-js.md | 2 + docs/src/test-api/class-testproject.md | 57 +++++++++++++++++ packages/playwright-test/src/loader.ts | 8 ++- packages/playwright-test/src/runner.ts | 74 ++++++++++++++-------- packages/playwright-test/src/types.ts | 2 + packages/playwright-test/types/test.d.ts | 43 +++++++++++++ tests/playwright-test/config.spec.ts | 12 +++- tests/playwright-test/global-setup.spec.ts | 54 ++++++++++++---- tests/playwright-test/reporter.spec.ts | 4 +- 9 files changed, 214 insertions(+), 42 deletions(-) diff --git a/docs/src/test-advanced-js.md b/docs/src/test-advanced-js.md index 3d7f35cd38..ba6532123a 100644 --- a/docs/src/test-advanced-js.md +++ b/docs/src/test-advanced-js.md @@ -359,6 +359,8 @@ test('test', async ({ page }) => { }); ``` +You can also have project-specific setup with [`property: TestProject.projectSetup`]. It will only be executed if at least one test from a specific project should be run, while global setup is always executed at the start of the test session. + ### Capturing trace of failures during global setup In some instances, it may be useful to capture a trace of failures encountered during the global setup. In order to do this, you must [start tracing](./api/class-tracing.md#tracing-start) in your setup, and you must ensure that you [stop tracing](./api/class-tracing.md#tracing-stop) if an error occurs before that error is thrown. This can be achieved by wrapping your setup in a `try...catch` block. Here is an example that expands the global setup example to capture a trace. diff --git a/docs/src/test-api/class-testproject.md b/docs/src/test-api/class-testproject.md index 8642b7f435..9cfc039481 100644 --- a/docs/src/test-api/class-testproject.md +++ b/docs/src/test-api/class-testproject.md @@ -163,6 +163,63 @@ Metadata that will be put directly to the test report serialized as JSON. Project name is visible in the report and during test execution. +## property: TestProject.projectSetup +* since: v1.25 +- type: ?<[string]> + +Path to the project-specifc setup file. This file will be required and run before all the tests from this project. It must export a single function that takes a [`TestConfig`] argument. + +Project setup is similar to [`property: TestConfig.globalSetup`], but it is only executed if at least one test from this particular project should be run. Learn more about [global setup and teardown](../test-advanced.md#global-setup-and-teardown). + +```js tab=js-js +// playwright.config.js +// @ts-check + +/** @type {import('@playwright/test').PlaywrightTestConfig} */ +const config = { + projects: [ + { + name: 'Admin Portal', + projectSetup: './setup-admin', + }, + { + name: 'Customer Portal', + projectSetup: './setup-customer', + }, + ], +}; + +module.exports = config; +``` + +```js tab=js-ts +// playwright.config.ts +import { type PlaywrightTestConfig } from '@playwright/test'; + +const config: PlaywrightTestConfig = { + projects: [ + { + name: 'Admin Portal', + projectSetup: './setup-admin', + }, + { + name: 'Customer Portal', + projectSetup: './setup-customer', + }, + ], +}; +export default config; +``` + +## property: TestProject.projectTeardown +* since: v1.25 +- type: ?<[string]> + +Path to the project-specifc teardown file. This file will be required and run after all the tests from this project. It must export a single function. See also [`property: TestProject.projectSetup`]. + +Project teardown is similar to [`property: TestConfig.globalTeardown`], but it is only executed if at least one test from this particular project did run. Learn more about [global setup and teardown](../test-advanced.md#global-setup-and-teardown). + + ## property: TestProject.screenshotsDir * since: v1.10 * experimental diff --git a/packages/playwright-test/src/loader.ts b/packages/playwright-test/src/loader.ts index 827c336ed4..368f9daa43 100644 --- a/packages/playwright-test/src/loader.ts +++ b/packages/playwright-test/src/loader.ts @@ -212,7 +212,7 @@ export class Loader { return suite; } - async loadGlobalHook(file: string, name: string): Promise<(config: FullConfigInternal) => any> { + async loadGlobalHook(file: string): Promise<(config: FullConfigInternal) => any> { return this._requireOrImportDefaultFunction(path.resolve(this._fullConfig.rootDir, file), false); } @@ -257,6 +257,10 @@ export class Loader { projectConfig.testDir = path.resolve(this._configDir, projectConfig.testDir); if (projectConfig.outputDir !== undefined) projectConfig.outputDir = path.resolve(this._configDir, projectConfig.outputDir); + if (projectConfig.projectSetup) + projectConfig.projectSetup = resolveScript(projectConfig.projectSetup, this._configDir); + if (projectConfig.projectTeardown) + projectConfig.projectTeardown = resolveScript(projectConfig.projectTeardown, this._configDir); if ((projectConfig as any).screenshotsDir !== undefined) (projectConfig as any).screenshotsDir = path.resolve(this._configDir, (projectConfig as any).screenshotsDir); if (projectConfig.snapshotDir !== undefined) @@ -281,6 +285,8 @@ export class Loader { retries: takeFirst(projectConfig.retries, config.retries, 0), metadata: takeFirst(projectConfig.metadata, config.metadata, undefined), name, + _projectSetup: projectConfig.projectSetup, + _projectTeardown: projectConfig.projectTeardown, testDir, _respectGitIgnore: respectGitIgnore, snapshotDir, diff --git a/packages/playwright-test/src/runner.ts b/packages/playwright-test/src/runner.ts index cfa22a93ec..6e451226c6 100644 --- a/packages/playwright-test/src/runner.ts +++ b/packages/playwright-test/src/runner.ts @@ -216,7 +216,7 @@ export class Runner { const rootSuite = new Suite('', 'root'); this._reporter.onBegin?.(config, rootSuite); const result: FullResult = { status: 'passed' }; - const globalTearDown = await this._performGlobalSetup(config, rootSuite, result); + const globalTearDown = await this._performGlobalAndProjectSetup(config, rootSuite, config.projects, result); if (result.status !== 'passed') return; @@ -330,7 +330,6 @@ export class Runner { for (const fileSuite of preprocessRoot.suites) fileSuites.set(fileSuite._requireFile, fileSuite); - const outputDirs = new Set(); const rootSuite = new Suite('', 'root'); for (const [project, files] of filesByProject) { const grepMatcher = createTitleMatcher(project.grep); @@ -355,7 +354,6 @@ export class Runner { projectSuite._addSuite(builtSuite); } } - outputDirs.add(project.outputDir); } // 7. Fail when no tests. @@ -413,6 +411,7 @@ export class Runner { // 12. Remove output directores. try { + const outputDirs = new Set([...filesByProject.keys()].map(project => project.outputDir)); await Promise.all(Array.from(outputDirs).map(outputDir => removeFolderAsync(outputDir).catch(async error => { if ((error as any).code === 'EBUSY') { // We failed to remove folder, might be due to the whole folder being mounted inside a container: @@ -431,7 +430,7 @@ export class Runner { // 13. Run Global setup. const result: FullResult = { status: 'passed' }; - const globalTearDown = await this._performGlobalSetup(config, rootSuite, result); + const globalTearDown = await this._performGlobalAndProjectSetup(config, rootSuite, [...filesByProject.keys()], result); if (result.status !== 'passed') return result; @@ -465,22 +464,43 @@ export class Runner { return result; } - private async _performGlobalSetup(config: FullConfigInternal, rootSuite: Suite, result: FullResult): Promise<(() => Promise) | undefined> { - let globalSetupResult: any; + private async _performGlobalAndProjectSetup(config: FullConfigInternal, rootSuite: Suite, projects: FullProjectInternal[], result: FullResult): Promise<(() => Promise) | undefined> { + type SetupData = { + setupFile?: string | null; + teardownFile?: string | null; + setupResult?: any; + }; + + const setups: SetupData[] = []; + setups.push({ + setupFile: config.globalSetup, + teardownFile: config.globalTeardown, + setupResult: undefined, + }); + for (const project of projects) { + setups.push({ + setupFile: project._projectSetup, + teardownFile: project._projectTeardown, + setupResult: undefined, + }); + } + const pluginsThatWereSetUp: TestRunnerPlugin[] = []; const sigintWatcher = new SigIntWatcher(); const tearDown = async () => { - // Reverse to setup. - await this._runAndReportError(async () => { - if (globalSetupResult && typeof globalSetupResult === 'function') - await globalSetupResult(this._loader.fullConfig()); - }, result); + setups.reverse(); + for (const setup of setups) { + await this._runAndReportError(async () => { + if (setup.setupResult && typeof setup.setupResult === 'function') + await setup.setupResult(this._loader.fullConfig()); + }, result); - await this._runAndReportError(async () => { - if (globalSetupResult && config.globalTeardown) - await (await this._loader.loadGlobalHook(config.globalTeardown, 'globalTeardown'))(this._loader.fullConfig()); - }, result); + await this._runAndReportError(async () => { + if (setup.setupResult && setup.teardownFile) + await (await this._loader.loadGlobalHook(setup.teardownFile))(this._loader.fullConfig()); + }, result); + } for (const plugin of pluginsThatWereSetUp.reverse()) { await this._runAndReportError(async () => { @@ -505,17 +525,19 @@ export class Runner { pluginsThatWereSetUp.push(plugin); } - // The do global setup. - if (!sigintWatcher.hadSignal()) { - if (config.globalSetup) { - const hook = await this._loader.loadGlobalHook(config.globalSetup, 'globalSetup'); - await Promise.race([ - Promise.resolve().then(() => hook(this._loader.fullConfig())).then((r: any) => globalSetupResult = r || ''), - sigintWatcher.promise(), - ]); - } else { - // Make sure we run globalTeardown. - globalSetupResult = ''; + // Then do global setup and project setups. + for (const setup of setups) { + if (!sigintWatcher.hadSignal()) { + if (setup.setupFile) { + const hook = await this._loader.loadGlobalHook(setup.setupFile); + await Promise.race([ + Promise.resolve().then(() => hook(this._loader.fullConfig())).then((r: any) => setup.setupResult = r || ''), + sigintWatcher.promise(), + ]); + } else { + // Make sure we run the teardown. + setup.setupResult = ''; + } } } }, result); diff --git a/packages/playwright-test/src/types.ts b/packages/playwright-test/src/types.ts index 921a216fb7..f2a6d745f7 100644 --- a/packages/playwright-test/src/types.ts +++ b/packages/playwright-test/src/types.ts @@ -65,4 +65,6 @@ export interface FullProjectInternal extends FullProjectPublic { _expect: Project['expect']; _screenshotsDir: string; _respectGitIgnore: boolean; + _projectSetup?: string; + _projectTeardown?: string; } diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index b7ac91792a..300760fa57 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -4314,6 +4314,49 @@ interface TestProject { */ name?: string; + /** + * Path to the project-specifc setup file. This file will be required and run before all the tests from this project. It + * must export a single function that takes a [`TestConfig`] argument. + * + * Project setup is similar to + * [testConfig.globalSetup](https://playwright.dev/docs/api/class-testconfig#test-config-global-setup), but it is only + * executed if at least one test from this particular project should be run. Learn more about + * [global setup and teardown](https://playwright.dev/docs/test-advanced#global-setup-and-teardown). + * + * ```js + * // playwright.config.ts + * import { type PlaywrightTestConfig } from '@playwright/test'; + * + * const config: PlaywrightTestConfig = { + * projects: [ + * { + * name: 'Admin Portal', + * projectSetup: './setup-admin', + * }, + * { + * name: 'Customer Portal', + * projectSetup: './setup-customer', + * }, + * ], + * }; + * export default config; + * ``` + * + */ + projectSetup?: string; + + /** + * Path to the project-specifc teardown file. This file will be required and run after all the tests from this project. It + * must export a single function. See also + * [testProject.projectSetup](https://playwright.dev/docs/api/class-testproject#test-project-project-setup). + * + * Project teardown is similar to + * [testConfig.globalTeardown](https://playwright.dev/docs/api/class-testconfig#test-config-global-teardown), but it is + * only executed if at least one test from this particular project did run. Learn more about + * [global setup and teardown](https://playwright.dev/docs/test-advanced#global-setup-and-teardown). + */ + projectTeardown?: string; + /** * The base directory, relative to the config file, for snapshot files created with `toMatchSnapshot`. Defaults to * [testProject.testDir](https://playwright.dev/docs/api/class-testproject#test-project-test-dir). diff --git a/tests/playwright-test/config.spec.ts b/tests/playwright-test/config.spec.ts index 6ddc58f94b..bdf7263a94 100644 --- a/tests/playwright-test/config.spec.ts +++ b/tests/playwright-test/config.spec.ts @@ -410,8 +410,18 @@ test('should have correct types for the config', async ({ runTSC }) => { port: 8082, }, ], + globalSetup: './globalSetup', + // @ts-expect-error + globalTeardown: null, + projects: [ + { + name: 'project name', + projectSetup: './projectSetup', + projectTeardown: './projectTeardown', + } + ], }; - + export default config; ` }); diff --git a/tests/playwright-test/global-setup.spec.ts b/tests/playwright-test/global-setup.spec.ts index ad6a3e73e1..635c4c7e7b 100644 --- a/tests/playwright-test/global-setup.spec.ts +++ b/tests/playwright-test/global-setup.spec.ts @@ -17,35 +17,65 @@ import { test, expect, stripAnsi } from './playwright-test-fixtures'; test('globalSetup and globalTeardown should work', async ({ runInlineTest }) => { - const { results, output } = await runInlineTest({ - 'playwright.config.ts': ` + const result = await runInlineTest({ + 'dir/playwright.config.ts': ` import * as path from 'path'; module.exports = { + testDir: '..', globalSetup: './globalSetup', globalTeardown: path.join(__dirname, 'globalTeardown.ts'), + projects: [ + { name: 'p1', projectSetup: './projectSetup1', projectTeardown: './projectTeardown1' }, + { name: 'p2', projectSetup: './projectSetup2', projectTeardown: './projectTeardown2' }, + ] }; `, - 'globalSetup.ts': ` + 'dir/globalSetup.ts': ` module.exports = async () => { - await new Promise(f => setTimeout(f, 100)); - global.value = 42; - process.env.FOO = String(global.value); + console.log('\\n%%from-global-setup'); }; `, - 'globalTeardown.ts': ` + 'dir/globalTeardown.ts': ` module.exports = async () => { - console.log('teardown=' + global.value); + console.log('\\n%%from-global-teardown'); + }; + `, + 'dir/projectSetup1.ts': ` + module.exports = async () => { + console.log('\\n%%from-project-setup-1'); + }; + `, + 'dir/projectTeardown1.ts': ` + module.exports = async () => { + console.log('\\n%%from-project-teardown-1'); + }; + `, + 'dir/projectSetup2.ts': ` + module.exports = async () => { + console.log('\\n%%from-project-setup-2'); + }; + `, + 'dir/projectTeardown2.ts': ` + module.exports = async () => { + console.log('\\n%%from-project-teardown-2'); }; `, 'a.test.js': ` const { test } = pwt; test('should work', async ({}, testInfo) => { - expect(process.env.FOO).toBe('42'); + console.log('\\n%%from-test'); }); `, - }); - expect(results[0].status).toBe('passed'); - expect(output).toContain('teardown=42'); + }, { 'project': 'p2', 'config': 'dir' }); + expect(result.passed).toBe(1); + expect(result.failed).toBe(0); + expect(stripAnsi(result.output).split('\n').filter(line => line.startsWith('%%'))).toEqual([ + '%%from-global-setup', + '%%from-project-setup-2', + '%%from-test', + '%%from-project-teardown-2', + '%%from-global-teardown', + ]); }); test('standalone globalTeardown should work', async ({ runInlineTest }) => { diff --git a/tests/playwright-test/reporter.spec.ts b/tests/playwright-test/reporter.spec.ts index 427b0945ef..e26a2d1262 100644 --- a/tests/playwright-test/reporter.spec.ts +++ b/tests/playwright-test/reporter.spec.ts @@ -557,8 +557,8 @@ test('should report correct tests/suites when using grep', async ({ runInlineTes expect(result.output).toContain('%%test2'); expect(result.output).not.toContain('%%test3'); const fileSuite = result.report.suites[0]; - expect(fileSuite.suites.length).toBe(1); - expect(fileSuite.suites[0].specs.length).toBe(2); + expect(fileSuite.suites!.length).toBe(1); + expect(fileSuite.suites![0].specs.length).toBe(2); expect(fileSuite.specs.length).toBe(0); });