feat(test runner): TestProject.projectSetup (#16063)

`projectSetup` is a project-scoped alternative to `globalSetup`.
It is only executed if at least one test from the project is scheduled to run.

docs/src/test-advanced-js.md

@@ -359,6 +359,8 @@ test('test', async ({ page }) => {
 });
 ```
 
+You can also have project-specific setup with [`property: TestProject.projectSetup`]. It will only be executed if at least one test from a specific project should be run, while global setup is always executed at the start of the test session.
+
 ### Capturing trace of failures during global setup
 
 In some instances, it may be useful to capture a trace of failures encountered during the global setup. In order to do this, you must [start tracing](./api/class-tracing.md#tracing-start) in your setup, and you must ensure that you [stop tracing](./api/class-tracing.md#tracing-stop) if an error occurs before that error is thrown. This can be achieved by wrapping your setup in a `try...catch` block.  Here is an example that expands the global setup example to capture a trace.

docs/src/test-api/class-testproject.md

@@ -163,6 +163,63 @@ Metadata that will be put directly to the test report serialized as JSON.
 Project name is visible in the report and during test execution.
 
 
+## property: TestProject.projectSetup
+* since: v1.25
+- type: ?<[string]>
+
+Path to the project-specifc setup file. This file will be required and run before all the tests from this project. It must export a single function that takes a [`TestConfig`] argument.
+
+Project setup is similar to [`property: TestConfig.globalSetup`], but it is only executed if at least one test from this particular project should be run. Learn more about [global setup and teardown](../test-advanced.md#global-setup-and-teardown).
+
+```js tab=js-js
+// playwright.config.js
+// @ts-check
+
+/** @type {import('@playwright/test').PlaywrightTestConfig} */
+const config = {
+  projects: [
+    {
+      name: 'Admin Portal',
+      projectSetup: './setup-admin',
+    },
+    {
+      name: 'Customer Portal',
+      projectSetup: './setup-customer',
+    },
+  ],
+};
+
+module.exports = config;
+```
+
+```js tab=js-ts
+// playwright.config.ts
+import { type PlaywrightTestConfig } from '@playwright/test';
+
+const config: PlaywrightTestConfig = {
+  projects: [
+    {
+      name: 'Admin Portal',
+      projectSetup: './setup-admin',
+    },
+    {
+      name: 'Customer Portal',
+      projectSetup: './setup-customer',
+    },
+  ],
+};
+export default config;
+```
+
+## property: TestProject.projectTeardown
+* since: v1.25
+- type: ?<[string]>
+
+Path to the project-specifc teardown file. This file will be required and run after all the tests from this project. It must export a single function. See also [`property: TestProject.projectSetup`].
+
+Project teardown is similar to [`property: TestConfig.globalTeardown`], but it is only executed if at least one test from this particular project did run. Learn more about [global setup and teardown](../test-advanced.md#global-setup-and-teardown).
+
+
 ## property: TestProject.screenshotsDir
 * since: v1.10
 * experimental

packages/playwright-test/src/loader.ts

@@ -212,7 +212,7 @@ export class Loader {
     return suite;
   }
 
-  async loadGlobalHook(file: string, name: string): Promise<(config: FullConfigInternal) => any> {
+  async loadGlobalHook(file: string): Promise<(config: FullConfigInternal) => any> {
     return this._requireOrImportDefaultFunction(path.resolve(this._fullConfig.rootDir, file), false);
   }
 
@@ -257,6 +257,10 @@ export class Loader {
       projectConfig.testDir = path.resolve(this._configDir, projectConfig.testDir);
     if (projectConfig.outputDir !== undefined)
       projectConfig.outputDir = path.resolve(this._configDir, projectConfig.outputDir);
+    if (projectConfig.projectSetup)
+      projectConfig.projectSetup = resolveScript(projectConfig.projectSetup, this._configDir);
+    if (projectConfig.projectTeardown)
+      projectConfig.projectTeardown = resolveScript(projectConfig.projectTeardown, this._configDir);
     if ((projectConfig as any).screenshotsDir !== undefined)
       (projectConfig as any).screenshotsDir = path.resolve(this._configDir, (projectConfig as any).screenshotsDir);
     if (projectConfig.snapshotDir !== undefined)
@@ -281,6 +285,8 @@ export class Loader {
       retries: takeFirst(projectConfig.retries, config.retries, 0),
       metadata: takeFirst(projectConfig.metadata, config.metadata, undefined),
       name,
+      _projectSetup: projectConfig.projectSetup,
+      _projectTeardown: projectConfig.projectTeardown,
       testDir,
       _respectGitIgnore: respectGitIgnore,
       snapshotDir,

packages/playwright-test/src/runner.ts

@@ -216,7 +216,7 @@ export class Runner {
     const rootSuite = new Suite('', 'root');
     this._reporter.onBegin?.(config, rootSuite);
     const result: FullResult = { status: 'passed' };
-    const globalTearDown = await this._performGlobalSetup(config, rootSuite, result);
+    const globalTearDown = await this._performGlobalAndProjectSetup(config, rootSuite, config.projects, result);
     if (result.status !== 'passed')
       return;
 
@@ -330,7 +330,6 @@ export class Runner {
     for (const fileSuite of preprocessRoot.suites)
       fileSuites.set(fileSuite._requireFile, fileSuite);
 
-    const outputDirs = new Set<string>();
     const rootSuite = new Suite('', 'root');
     for (const [project, files] of filesByProject) {
       const grepMatcher = createTitleMatcher(project.grep);
@@ -355,7 +354,6 @@ export class Runner {
             projectSuite._addSuite(builtSuite);
         }
       }
-      outputDirs.add(project.outputDir);
     }
 
     // 7. Fail when no tests.
@@ -413,6 +411,7 @@ export class Runner {
 
     // 12. Remove output directores.
     try {
+      const outputDirs = new Set([...filesByProject.keys()].map(project => project.outputDir));
       await Promise.all(Array.from(outputDirs).map(outputDir => removeFolderAsync(outputDir).catch(async error => {
         if ((error as any).code === 'EBUSY') {
           // We failed to remove folder, might be due to the whole folder being mounted inside a container:
@@ -431,7 +430,7 @@ export class Runner {
 
     // 13. Run Global setup.
     const result: FullResult = { status: 'passed' };
-    const globalTearDown = await this._performGlobalSetup(config, rootSuite, result);
+    const globalTearDown = await this._performGlobalAndProjectSetup(config, rootSuite, [...filesByProject.keys()], result);
     if (result.status !== 'passed')
       return result;
 
@@ -465,22 +464,43 @@ export class Runner {
     return result;
   }
 
-  private async _performGlobalSetup(config: FullConfigInternal, rootSuite: Suite, result: FullResult): Promise<(() => Promise<void>) | undefined> {
+  private async _performGlobalAndProjectSetup(config: FullConfigInternal, rootSuite: Suite, projects: FullProjectInternal[], result: FullResult): Promise<(() => Promise<void>) | undefined> {
-    let globalSetupResult: any;
+    type SetupData = {
+      setupFile?: string | null;
+      teardownFile?: string | null;
+      setupResult?: any;
+    };
+
+    const setups: SetupData[] = [];
+    setups.push({
+      setupFile: config.globalSetup,
+      teardownFile: config.globalTeardown,
+      setupResult: undefined,
+    });
+    for (const project of projects) {
+      setups.push({
+        setupFile: project._projectSetup,
+        teardownFile: project._projectTeardown,
+        setupResult: undefined,
+      });
+    }
+
     const pluginsThatWereSetUp: TestRunnerPlugin[] = [];
     const sigintWatcher = new SigIntWatcher();
 
     const tearDown = async () => {
-      // Reverse to setup.
+      setups.reverse();
-      await this._runAndReportError(async () => {
+      for (const setup of setups) {
-        if (globalSetupResult && typeof globalSetupResult === 'function')
+        await this._runAndReportError(async () => {
-          await globalSetupResult(this._loader.fullConfig());
+          if (setup.setupResult && typeof setup.setupResult === 'function')
-      }, result);
+            await setup.setupResult(this._loader.fullConfig());
+        }, result);
 
-      await this._runAndReportError(async () => {
+        await this._runAndReportError(async () => {
-        if (globalSetupResult && config.globalTeardown)
+          if (setup.setupResult && setup.teardownFile)
-          await (await this._loader.loadGlobalHook(config.globalTeardown, 'globalTeardown'))(this._loader.fullConfig());
+            await (await this._loader.loadGlobalHook(setup.teardownFile))(this._loader.fullConfig());
-      }, result);
+        }, result);
+      }
 
       for (const plugin of pluginsThatWereSetUp.reverse()) {
         await this._runAndReportError(async () => {
@@ -505,17 +525,19 @@ export class Runner {
         pluginsThatWereSetUp.push(plugin);
       }
 
-      // The do global setup.
+      // Then do global setup and project setups.
-      if (!sigintWatcher.hadSignal()) {
+      for (const setup of setups) {
-        if (config.globalSetup) {
+        if (!sigintWatcher.hadSignal()) {
-          const hook = await this._loader.loadGlobalHook(config.globalSetup, 'globalSetup');
+          if (setup.setupFile) {
-          await Promise.race([
+            const hook = await this._loader.loadGlobalHook(setup.setupFile);
-            Promise.resolve().then(() => hook(this._loader.fullConfig())).then((r: any) => globalSetupResult = r || '<noop>'),
+            await Promise.race([
-            sigintWatcher.promise(),
+              Promise.resolve().then(() => hook(this._loader.fullConfig())).then((r: any) => setup.setupResult = r || '<noop>'),
-          ]);
+              sigintWatcher.promise(),
-        } else {
+            ]);
-          // Make sure we run globalTeardown.
+          } else {
-          globalSetupResult = '<noop>';
+            // Make sure we run the teardown.
+            setup.setupResult = '<noop>';
+          }
         }
       }
     }, result);

packages/playwright-test/src/types.ts

@@ -65,4 +65,6 @@ export interface FullProjectInternal extends FullProjectPublic {
   _expect: Project['expect'];
   _screenshotsDir: string;
   _respectGitIgnore: boolean;
+  _projectSetup?: string;
+  _projectTeardown?: string;
 }

packages/playwright-test/types/test.d.ts

@@ -4314,6 +4314,49 @@ interface TestProject {
    */
   name?: string;
 
+  /**
+   * Path to the project-specifc setup file. This file will be required and run before all the tests from this project. It
+   * must export a single function that takes a [`TestConfig`] argument.
+   *
+   * Project setup is similar to
+   * [testConfig.globalSetup](https://playwright.dev/docs/api/class-testconfig#test-config-global-setup), but it is only
+   * executed if at least one test from this particular project should be run. Learn more about
+   * [global setup and teardown](https://playwright.dev/docs/test-advanced#global-setup-and-teardown).
+   *
+   * ```js
+   * // playwright.config.ts
+   * import { type PlaywrightTestConfig } from '@playwright/test';
+   *
+   * const config: PlaywrightTestConfig = {
+   *   projects: [
+   *     {
+   *       name: 'Admin Portal',
+   *       projectSetup: './setup-admin',
+   *     },
+   *     {
+   *       name: 'Customer Portal',
+   *       projectSetup: './setup-customer',
+   *     },
+   *   ],
+   * };
+   * export default config;
+   * ```
+   *
+   */
+  projectSetup?: string;
+
+  /**
+   * Path to the project-specifc teardown file. This file will be required and run after all the tests from this project. It
+   * must export a single function. See also
+   * [testProject.projectSetup](https://playwright.dev/docs/api/class-testproject#test-project-project-setup).
+   *
+   * Project teardown is similar to
+   * [testConfig.globalTeardown](https://playwright.dev/docs/api/class-testconfig#test-config-global-teardown), but it is
+   * only executed if at least one test from this particular project did run. Learn more about
+   * [global setup and teardown](https://playwright.dev/docs/test-advanced#global-setup-and-teardown).
+   */
+  projectTeardown?: string;
+
   /**
    * The base directory, relative to the config file, for snapshot files created with `toMatchSnapshot`. Defaults to
    * [testProject.testDir](https://playwright.dev/docs/api/class-testproject#test-project-test-dir).

tests/playwright-test/config.spec.ts

@@ -410,8 +410,18 @@ test('should have correct types for the config', async ({ runTSC }) => {
             port: 8082,
           },
         ],
+        globalSetup: './globalSetup',
+        // @ts-expect-error
+        globalTeardown: null,
+        projects: [
+          {
+            name: 'project name',
+            projectSetup: './projectSetup',
+            projectTeardown: './projectTeardown',
+          }
+        ],
       };
-      
+
       export default config;
   `
   });

tests/playwright-test/global-setup.spec.ts

@@ -17,35 +17,65 @@
 import { test, expect, stripAnsi } from './playwright-test-fixtures';
 
 test('globalSetup and globalTeardown should work', async ({ runInlineTest }) => {
-  const { results, output } = await runInlineTest({
+  const result = await runInlineTest({
-    'playwright.config.ts': `
+    'dir/playwright.config.ts': `
       import * as path from 'path';
       module.exports = {
+        testDir: '..',
         globalSetup: './globalSetup',
         globalTeardown: path.join(__dirname, 'globalTeardown.ts'),
+        projects: [
+          { name: 'p1', projectSetup: './projectSetup1', projectTeardown: './projectTeardown1' },
+          { name: 'p2', projectSetup: './projectSetup2', projectTeardown: './projectTeardown2' },
+        ]
       };
     `,
-    'globalSetup.ts': `
+    'dir/globalSetup.ts': `
       module.exports = async () => {
-        await new Promise(f => setTimeout(f, 100));
+        console.log('\\n%%from-global-setup');
-        global.value = 42;
-        process.env.FOO = String(global.value);
       };
     `,
-    'globalTeardown.ts': `
+    'dir/globalTeardown.ts': `
       module.exports = async () => {
-        console.log('teardown=' + global.value);
+        console.log('\\n%%from-global-teardown');
+      };
+    `,
+    'dir/projectSetup1.ts': `
+      module.exports = async () => {
+        console.log('\\n%%from-project-setup-1');
+      };
+    `,
+    'dir/projectTeardown1.ts': `
+      module.exports = async () => {
+        console.log('\\n%%from-project-teardown-1');
+      };
+    `,
+    'dir/projectSetup2.ts': `
+      module.exports = async () => {
+        console.log('\\n%%from-project-setup-2');
+      };
+    `,
+    'dir/projectTeardown2.ts': `
+      module.exports = async () => {
+        console.log('\\n%%from-project-teardown-2');
       };
     `,
     'a.test.js': `
       const { test } = pwt;
       test('should work', async ({}, testInfo) => {
-        expect(process.env.FOO).toBe('42');
+        console.log('\\n%%from-test');
       });
     `,
-  });
+  }, { 'project': 'p2', 'config': 'dir' });
-  expect(results[0].status).toBe('passed');
+  expect(result.passed).toBe(1);
-  expect(output).toContain('teardown=42');
+  expect(result.failed).toBe(0);
+  expect(stripAnsi(result.output).split('\n').filter(line => line.startsWith('%%'))).toEqual([
+    '%%from-global-setup',
+    '%%from-project-setup-2',
+    '%%from-test',
+    '%%from-project-teardown-2',
+    '%%from-global-teardown',
+  ]);
 });
 
 test('standalone globalTeardown should work', async ({ runInlineTest }) => {

tests/playwright-test/reporter.spec.ts

@@ -557,8 +557,8 @@ test('should report correct tests/suites when using grep', async ({ runInlineTes
   expect(result.output).toContain('%%test2');
   expect(result.output).not.toContain('%%test3');
   const fileSuite = result.report.suites[0];
-  expect(fileSuite.suites.length).toBe(1);
+  expect(fileSuite.suites!.length).toBe(1);
-  expect(fileSuite.suites[0].specs.length).toBe(2);
+  expect(fileSuite.suites![0].specs.length).toBe(2);
   expect(fileSuite.specs.length).toBe(0);
 });