diff --git a/docs/src/test-api/class-testproject.md b/docs/src/test-api/class-testproject.md index 36d77b21e7..3748e32411 100644 --- a/docs/src/test-api/class-testproject.md +++ b/docs/src/test-api/class-testproject.md @@ -57,7 +57,7 @@ behaves as if they were not specified. Using dependencies allows global setup to produce traces and other artifacts, see the setup steps in the test report, etc. -For example: +**Usage** ```js // playwright.config.ts @@ -198,6 +198,52 @@ Use [`method: Test.describe.configure`] to change the number of retries for a sp Use [`property: TestConfig.retries`] to change this option for all projects. +## property: TestProject.teardown +* since: v1.34 +- type: ?<[string]> + +Name of a project that needs to run after this and any dependent projects have finished. Teardown is useful to cleanup any resources acquired by this project. + +Passing `--no-deps` argument ignores [`property: TestProject.teardown`] and behaves as if it was not specified. + +**Usage** + +A common pattern is a "setup" dependency that has a corresponding "teardown": + +```js +// playwright.config.ts +import { defineConfig } from '@playwright/test'; + +export default defineConfig({ + projects: [ + { + name: 'setup', + testMatch: /global.setup\.ts/, + teardown: 'teardown', + }, + { + name: 'teardown', + testMatch: /global.teardown\.ts/, + }, + { + name: 'chromium', + use: devices['Desktop Chrome'], + dependencies: ['setup'], + }, + { + name: 'firefox', + use: devices['Desktop Firefox'], + dependencies: ['setup'], + }, + { + name: 'webkit', + use: devices['Desktop Safari'], + dependencies: ['setup'], + }, + ], +}); +``` + ## property: TestProject.testDir * since: v1.10 - type: ?<[string]> diff --git a/packages/playwright-test/src/common/config.ts b/packages/playwright-test/src/common/config.ts index f91891e82e..198eff70ac 100644 --- a/packages/playwright-test/src/common/config.ts +++ b/packages/playwright-test/src/common/config.ts @@ -152,6 +152,7 @@ export class FullProjectInternal { readonly snapshotPathTemplate: string; id = ''; deps: FullProjectInternal[] = []; + teardown: FullProjectInternal | undefined; constructor(configDir: string, config: Config, fullConfig: FullConfigInternal, projectConfig: Project, configCLIOverrides: ConfigCLIOverrides, throwawayArtifactsPath: string) { this.fullConfig = fullConfig; @@ -174,6 +175,7 @@ export class FullProjectInternal { timeout: takeFirst(configCLIOverrides.timeout, projectConfig.timeout, config.timeout, defaultTimeout), use: mergeObjects(config.use, projectConfig.use, configCLIOverrides.use), dependencies: projectConfig.dependencies || [], + teardown: projectConfig.teardown, }; (this.project as any)[projectInternalSymbol] = this; this.fullyParallel = takeFirst(configCLIOverrides.fullyParallel, projectConfig.fullyParallel, config.fullyParallel, undefined); @@ -205,6 +207,7 @@ function resolveReporters(reporters: Config['reporter'], rootDir: string): Repor } function resolveProjectDependencies(projects: FullProjectInternal[]) { + const teardownToSetup = new Map(); for (const project of projects) { for (const dependencyName of project.project.dependencies) { const dependencies = projects.filter(p => p.project.name === dependencyName); @@ -214,6 +217,28 @@ function resolveProjectDependencies(projects: FullProjectInternal[]) { throw new Error(`Project dependencies should have unique names, reading ${dependencyName}`); project.deps.push(...dependencies); } + if (project.project.teardown) { + const teardowns = projects.filter(p => p.project.name === project.project.teardown); + if (!teardowns.length) + throw new Error(`Project '${project.project.name}' has unknown teardown project '${project.project.teardown}'`); + if (teardowns.length > 1) + throw new Error(`Project teardowns should have unique names, reading ${project.project.teardown}`); + const teardown = teardowns[0]; + project.teardown = teardown; + if (teardownToSetup.has(teardown)) + throw new Error(`Project ${teardown.project.name} can not be designated as teardown to multiple projects (${teardownToSetup.get(teardown)!.project.name} and ${project.project.name})`); + teardownToSetup.set(teardown, project); + } + } + for (const teardown of teardownToSetup.keys()) { + if (teardown.deps.length) + throw new Error(`Teardown project ${teardown.project.name} must not have dependencies`); + } + for (const project of projects) { + for (const dep of project.deps) { + if (teardownToSetup.has(dep)) + throw new Error(`Project ${project.project.name} must not depend on a teardown project ${dep.project.name}`); + } } } diff --git a/packages/playwright-test/src/common/configLoader.ts b/packages/playwright-test/src/common/configLoader.ts index 40898c43db..02006183c2 100644 --- a/packages/playwright-test/src/common/configLoader.ts +++ b/packages/playwright-test/src/common/configLoader.ts @@ -55,8 +55,10 @@ export class ConfigLoader { const fullConfig = await this._loadConfig(config, path.dirname(file), file); setCurrentConfig(fullConfig); if (ignoreProjectDependencies) { - for (const project of fullConfig.projects) + for (const project of fullConfig.projects) { project.deps = []; + project.teardown = undefined; + } } this._fullConfig = fullConfig; return fullConfig; diff --git a/packages/playwright-test/src/isomorphic/teleReceiver.ts b/packages/playwright-test/src/isomorphic/teleReceiver.ts index 7842855792..3382adfa16 100644 --- a/packages/playwright-test/src/isomorphic/teleReceiver.ts +++ b/packages/playwright-test/src/isomorphic/teleReceiver.ts @@ -48,6 +48,7 @@ export type JsonProject = { repeatEach: number; retries: number; suites: JsonSuite[]; + teardown?: string; testDir: string; testIgnore: JsonPattern[]; testMatch: JsonPattern[]; @@ -303,6 +304,7 @@ export class TeleReporterReceiver { grep: parseRegexPatterns(project.grep) as RegExp[], grepInvert: parseRegexPatterns(project.grepInvert) as RegExp[], dependencies: project.dependencies, + teardown: project.teardown, snapshotDir: this._absolutePath(project.snapshotDir), use: {}, }; diff --git a/packages/playwright-test/src/reporters/teleEmitter.ts b/packages/playwright-test/src/reporters/teleEmitter.ts index 5ba554bf0d..2b2c50cda7 100644 --- a/packages/playwright-test/src/reporters/teleEmitter.ts +++ b/packages/playwright-test/src/reporters/teleEmitter.ts @@ -151,6 +151,7 @@ export class TeleReporterEmitter implements Reporter { grepInvert: serializeRegexPatterns(project.grepInvert || []), dependencies: project.dependencies, snapshotDir: this._relativePath(project.snapshotDir), + teardown: project.teardown, }; return report; } diff --git a/packages/playwright-test/src/runner/projectUtils.ts b/packages/playwright-test/src/runner/projectUtils.ts index 664eadcbc5..d009323c39 100644 --- a/packages/playwright-test/src/runner/projectUtils.ts +++ b/packages/playwright-test/src/runner/projectUtils.ts @@ -49,6 +49,15 @@ export function filterProjects(projects: FullProjectInternal[], projectNames?: s return result; } +export function buildTeardownToSetupMap(projects: FullProjectInternal[]): Map { + const result = new Map(); + for (const project of projects) { + if (project.teardown) + result.set(project.teardown, project); + } + return result; +} + export function buildProjectsClosure(projects: FullProjectInternal[]): Map { const result = new Map(); const visit = (depth: number, project: FullProjectInternal) => { @@ -59,6 +68,8 @@ export function buildProjectsClosure(projects: FullProjectInternal[]): Map { + const reverseDeps = new Map(projects.map(p => ([p, []]))); + for (const project of projects) { + for (const dep of project.deps) + reverseDeps.get(dep)!.push(project); + } + const result = new Set(); + const visit = (depth: number, project: FullProjectInternal) => { + if (depth > 100) { + const error = new Error('Circular dependency detected between projects.'); + error.stack = ''; + throw error; + } + result.add(project); + for (const reverseDep of reverseDeps.get(project)!) + visit(depth + 1, reverseDep); + if (project.teardown) + visit(depth + 1, project.teardown); + }; + visit(0, forProject); + return result; +} + export async function collectFilesForProject(project: FullProjectInternal, fsCache = new Map()): Promise { const extensions = ['.js', '.ts', '.mjs', '.tsx', '.jsx']; const testFileExtension = (file: string) => extensions.includes(path.extname(file)); diff --git a/packages/playwright-test/src/runner/tasks.ts b/packages/playwright-test/src/runner/tasks.ts index f63b89c2c7..db3c5caa42 100644 --- a/packages/playwright-test/src/runner/tasks.ts +++ b/packages/playwright-test/src/runner/tasks.ts @@ -28,6 +28,7 @@ import type { FullConfigInternal, FullProjectInternal } from '../common/config'; import { collectProjectsAndTestFiles, createRootSuite, loadFileSuites, loadGlobalHook } from './loadUtils'; import type { Matcher } from '../util'; import type { Suite } from '../common/test'; +import { buildDependentProjects, buildTeardownToSetupMap } from './projectUtils'; const removeFolderAsync = promisify(rimraf); const readDirAsync = promisify(fs.readdir); @@ -182,13 +183,23 @@ function createPhasesTask(): Task { const processed = new Set(); const projectToSuite = new Map(testRun.rootSuite!.suites.map(suite => [suite._fullProject!, suite])); + const allProjects = [...projectToSuite.keys()]; + const teardownToSetup = buildTeardownToSetupMap(allProjects); + const teardownToSetupDependents = new Map(); + for (const [teardown, setup] of teardownToSetup) { + const closure = buildDependentProjects(setup, allProjects); + closure.delete(teardown); + teardownToSetupDependents.set(teardown, [...closure]); + } + for (let i = 0; i < projectToSuite.size; i++) { // Find all projects that have all their dependencies processed by previous phases. const phaseProjects: FullProjectInternal[] = []; for (const project of projectToSuite.keys()) { if (processed.has(project)) continue; - if (project.deps.find(p => !processed.has(p))) + const projectsThatShouldFinishFirst = [...project.deps, ...(teardownToSetupDependents.get(project) || [])]; + if (projectsThatShouldFinishFirst.find(p => !processed.has(p))) continue; phaseProjects.push(project); } @@ -229,6 +240,7 @@ function createRunTestsTask(): Task { const { phases } = testRun; const successfulProjects = new Set(); const extraEnvByProjectId: EnvByProjectId = new Map(); + const teardownToSetup = buildTeardownToSetupMap(phases.map(phase => phase.projects.map(p => p.project)).flat()); for (const { dispatcher, projects } of phases) { // Each phase contains dispatcher and a set of test groups. @@ -240,6 +252,9 @@ function createRunTestsTask(): Task { let extraEnv: Record = {}; for (const dep of project.deps) extraEnv = { ...extraEnv, ...extraEnvByProjectId.get(dep.id) }; + const setupForTeardown = teardownToSetup.get(project); + if (setupForTeardown) + extraEnv = { ...extraEnv, ...extraEnvByProjectId.get(setupForTeardown.id) }; extraEnvByProjectId.set(project.id, extraEnv); const hasFailedDeps = project.deps.some(p => !successfulProjects.has(p)); diff --git a/packages/playwright-test/src/runner/uiMode.ts b/packages/playwright-test/src/runner/uiMode.ts index 0b99a5c137..895c77cf45 100644 --- a/packages/playwright-test/src/runner/uiMode.ts +++ b/packages/playwright-test/src/runner/uiMode.ts @@ -44,8 +44,10 @@ class UIMode { process.env.PW_LIVE_TRACE_STACKS = '1'; config.cliListOnly = false; config.cliPassWithNoTests = true; - for (const project of config.projects) + for (const project of config.projects) { project.deps = []; + project.teardown = undefined; + } for (const p of config.projects) { p.project.retries = 0; diff --git a/packages/playwright-test/src/runner/watchMode.ts b/packages/playwright-test/src/runner/watchMode.ts index e446671371..c48122a227 100644 --- a/packages/playwright-test/src/runner/watchMode.ts +++ b/packages/playwright-test/src/runner/watchMode.ts @@ -308,6 +308,8 @@ function affectedProjectsClosure(projectClosure: FullProjectInternal[], affected if (result.has(dep)) result.add(p); } + if (p.teardown && result.has(p.teardown)) + result.add(p); } } return result; diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index d854521ab5..c4b39ced79 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -193,7 +193,7 @@ export interface FullProject { * Using dependencies allows global setup to produce traces and other artifacts, see the setup steps in the test * report, etc. * - * For example: + * **Usage** * * ```js * // playwright.config.ts @@ -284,6 +284,54 @@ export interface FullProject { * option for all projects. */ retries: number; + /** + * Name of a project that needs to run after this and any dependent projects have finished. Teardown is useful to + * cleanup any resources acquired by this project. + * + * Passing `--no-deps` argument ignores + * [testProject.teardown](https://playwright.dev/docs/api/class-testproject#test-project-teardown) and behaves as if + * it was not specified. + * + * **Usage** + * + * A common pattern is a "setup" dependency that has a corresponding "teardown": + * + * ```js + * // playwright.config.ts + * import { defineConfig } from '@playwright/test'; + * + * export default defineConfig({ + * projects: [ + * { + * name: 'setup', + * testMatch: /global.setup\.ts/, + * teardown: 'teardown', + * }, + * { + * name: 'teardown', + * testMatch: /global.teardown\.ts/, + * }, + * { + * name: 'chromium', + * use: devices['Desktop Chrome'], + * dependencies: ['setup'], + * }, + * { + * name: 'firefox', + * use: devices['Desktop Firefox'], + * dependencies: ['setup'], + * }, + * { + * name: 'webkit', + * use: devices['Desktop Safari'], + * dependencies: ['setup'], + * }, + * ], + * }); + * ``` + * + */ + teardown?: string; /** * Directory that will be recursively scanned for test files. Defaults to the directory of the configuration file. * @@ -5932,7 +5980,7 @@ interface TestProject { * Using dependencies allows global setup to produce traces and other artifacts, see the setup steps in the test * report, etc. * - * For example: + * **Usage** * * ```js * // playwright.config.ts @@ -6245,6 +6293,55 @@ interface TestProject { */ snapshotPathTemplate?: string; + /** + * Name of a project that needs to run after this and any dependent projects have finished. Teardown is useful to + * cleanup any resources acquired by this project. + * + * Passing `--no-deps` argument ignores + * [testProject.teardown](https://playwright.dev/docs/api/class-testproject#test-project-teardown) and behaves as if + * it was not specified. + * + * **Usage** + * + * A common pattern is a "setup" dependency that has a corresponding "teardown": + * + * ```js + * // playwright.config.ts + * import { defineConfig } from '@playwright/test'; + * + * export default defineConfig({ + * projects: [ + * { + * name: 'setup', + * testMatch: /global.setup\.ts/, + * teardown: 'teardown', + * }, + * { + * name: 'teardown', + * testMatch: /global.teardown\.ts/, + * }, + * { + * name: 'chromium', + * use: devices['Desktop Chrome'], + * dependencies: ['setup'], + * }, + * { + * name: 'firefox', + * use: devices['Desktop Firefox'], + * dependencies: ['setup'], + * }, + * { + * name: 'webkit', + * use: devices['Desktop Safari'], + * dependencies: ['setup'], + * }, + * ], + * }); + * ``` + * + */ + teardown?: string; + /** * Directory that will be recursively scanned for test files. Defaults to the directory of the configuration file. * diff --git a/tests/playwright-test/deps.spec.ts b/tests/playwright-test/deps.spec.ts index 3adabefab6..57a3ce5783 100644 --- a/tests/playwright-test/deps.spec.ts +++ b/tests/playwright-test/deps.spec.ts @@ -43,9 +43,10 @@ test('should inherit env changes from dependencies', async ({ runInlineTest }) = 'playwright.config.ts': ` module.exports = { projects: [ { name: 'A', testMatch: '**/a.spec.ts' }, - { name: 'B', testMatch: '**/b.spec.ts' }, + { name: 'B', testMatch: '**/b.spec.ts', teardown: 'E' }, { name: 'C', testMatch: '**/c.spec.ts', dependencies: ['A'] }, { name: 'D', testMatch: '**/d.spec.ts', dependencies: ['B'] }, + { name: 'E', testMatch: '**/e.spec.ts' }, ] }; `, 'a.spec.ts': ` @@ -75,11 +76,17 @@ test('should inherit env changes from dependencies', async ({ runInlineTest }) = console.log('\\n%%D-' + process.env.SET_IN_A + '-' + process.env.SET_IN_B + '-' + process.env.SET_OUTSIDE); }); `, + 'e.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('pass', async ({}, testInfo) => { + console.log('\\n%%E-' + process.env.SET_IN_A + '-' + process.env.SET_IN_B + '-' + process.env.SET_OUTSIDE); + }); + `, }, {}, { SET_OUTSIDE: 'outside' }); - expect(result.passed).toBe(4); + expect(result.passed).toBe(5); expect(result.failed).toBe(0); expect(result.skipped).toBe(0); - expect(result.outputLines.sort()).toEqual(['A', 'B', 'C-valuea-undefined-undefined', 'D-undefined-valueb-outside']); + expect(result.outputLines.sort()).toEqual(['A', 'B', 'C-valuea-undefined-undefined', 'D-undefined-valueb-outside', 'E-undefined-valueb-outside']); }); test('should not run projects with dependencies when --no-deps is passed', async ({ runInlineTest }) => { @@ -423,3 +430,132 @@ test('should run setup project with zero tests recursively', async ({ runInlineT expect(result.passed).toBe(2); expect(result.outputLines).toEqual(['A', 'C']); }); + +test('should run project with teardown', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + projects: [ + { name: 'A', teardown: 'B' }, + { name: 'B' }, + ], + };`, + 'test.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('test', async ({}, testInfo) => { + console.log('\\n%%' + testInfo.project.name); + }); + `, + }, { workers: 1 }, undefined, { additionalArgs: ['--project=A'] }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(2); + expect(result.outputLines).toEqual(['A', 'B']); +}); + +test('should run teardown after depedents', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + projects: [ + { name: 'A', teardown: 'E' }, + { name: 'B', dependencies: ['A'] }, + { name: 'C', dependencies: ['B'], teardown: 'D' }, + { name: 'D' }, + { name: 'E' }, + ], + };`, + 'test.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('test', async ({}, testInfo) => { + console.log('\\n%%' + testInfo.project.name); + }); + `, + }, { workers: 1 }, undefined, { additionalArgs: ['--project=C'] }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(5); + expect(result.outputLines).toEqual(['A', 'B', 'C', 'D', 'E']); +}); + +test('should run teardown after failure', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + projects: [ + { name: 'A', teardown: 'D' }, + { name: 'B', dependencies: ['A'] }, + { name: 'C', dependencies: ['B'] }, + { name: 'D' }, + ], + };`, + 'test.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('test', async ({}, testInfo) => { + console.log('\\n%%' + testInfo.project.name); + if (testInfo.project.name === 'A') + throw new Error('ouch'); + }); + `, + }, { workers: 1 }, undefined, { additionalArgs: ['--project=C'] }); + expect(result.exitCode).toBe(1); + expect(result.passed).toBe(1); + expect(result.failed).toBe(1); + expect(result.skipped).toBe(2); + expect(result.outputLines).toEqual(['A', 'D']); +}); + +test('should complain about teardown being a dependency', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + projects: [ + { name: 'A', teardown: 'B' }, + { name: 'B' }, + { name: 'C', dependencies: ['B'] }, + ], + };`, + 'test.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('test', () => {}); + `, + }); + expect(result.exitCode).toBe(1); + expect(result.output).toContain(`Project C must not depend on a teardown project B`); +}); + +test('should complain about teardown having a dependency', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + projects: [ + { name: 'A', teardown: 'B' }, + { name: 'B', dependencies: ['C'] }, + { name: 'C' }, + ], + };`, + 'test.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('test', () => {}); + `, + }); + expect(result.exitCode).toBe(1); + expect(result.output).toContain(`Teardown project B must not have dependencies`); +}); + +test('should complain about teardown used multiple times', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + projects: [ + { name: 'A', teardown: 'C' }, + { name: 'B', teardown: 'C' }, + { name: 'C' }, + ], + };`, + 'test.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('test', () => {}); + `, + }); + expect(result.exitCode).toBe(1); + expect(result.output).toContain(`Project C can not be designated as teardown to multiple projects (A and B)`); +}); diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 860b48aab5..1a34759636 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -36,7 +36,7 @@ export interface Project extends TestProject { // [internal] !!! DO NOT ADD TO THIS !!! // [internal] It is part of the public API and is computed from the user's config. -// [internal] If you need new fields internally, add them to FullConfigInternal instead. +// [internal] If you need new fields internally, add them to FullProjectInternal instead. export interface FullProject { grep: RegExp | RegExp[]; grepInvert: RegExp | RegExp[] | null; @@ -47,6 +47,7 @@ export interface FullProject { outputDir: string; repeatEach: number; retries: number; + teardown?: string; testDir: string; testIgnore: string | RegExp | (string | RegExp)[]; testMatch: string | RegExp | (string | RegExp)[];