feature(test-runner): multiple web servers (#15388)

Fixes #8206.

Since #8206 is a long-awaited (~ 1 year old), popular (~ 45 reactions, frequently requested in community channels, etc.), this PR aims to unblock folks.

Notably, we do not innovate on the `webServer` API, despite knowing we're not in love with it. We'll save the innovation for either Plugins or a new `LaunchConfigs` option. (We haven't yet arrived at a Plugin API we like, and instead of launching a new option guessing what the "better" launchConfig API would be, let's wait and see how folks use this new Array-variant of `webServer` which—despite its name—can be used for non-Web Server launches!

docs/src/test-advanced-js.md

@@ -207,6 +207,8 @@ test('test', async ({ page }) => {
 });
 ```
 
+Multiple web servers (or background processes) can be launched simultaneously by providing an array of `webServer` configurations. See [`property: TestConfig.webServer`] for additional examples and documentation.
+
 ## Global setup and teardown
 
 To set something up once before running all tests, use `globalSetup` option in the [configuration file](#configuration-object). Global setup file must export a single function that takes a config object. This function will be run once before all the tests.

docs/src/test-api/class-testconfig.md

@@ -646,7 +646,7 @@ export default config;
 
 ## property: TestConfig.webServer
 * since: v1.10
-- type: ?<[Object]>
+- type: ?<[Object]|[Array]<[Object]>>
   - `command` <[string]> Shell command to start. For example `npm run start`..
   - `port` ?<[int]> The port that your http server is expected to appear on. It does wait until it accepts connections. Exactly one of `port` or `url` is required.
   - `url` ?<[string]> The url on your http server that is expected to return a 2xx, 3xx, 400, 401, 402, or 403 status code when the server is ready to accept connections. Exactly one of `port` or `url` is required.
@@ -656,13 +656,13 @@ export default config;
   - `cwd` ?<[string]> Current working directory of the spawned process, defaults to the directory of the configuration file.
   - `env` ?<[Object]<[string], [string]>> Environment variables to set for the command, `process.env` by default.
 
-Launch a development web server during the tests.
+Launch a development web server (or multiple) during the tests.
 
-If the port is specified, the server will wait for it to be available on `127.0.0.1` or `::1`, before running the tests. If the url is specified, the server will wait for the URL to return a 2xx status code before running the tests.
+If the port is specified, Playwright Test will wait for it to be available on `127.0.0.1` or `::1`, before running the tests. If the url is specified, Playwright Test will wait for the URL to return a 2xx, 3xx, 400, 401, 402, or 403 status code before running the tests.
 
 For continuous integration, you may want to use the `reuseExistingServer: !process.env.CI` option which does not use an existing server on the CI. To see the stdout, you can set the `DEBUG=pw:webserver` environment variable.
 
-The `port` (but not the `url`) gets passed over to Playwright as a [`property: TestOptions.baseURL`]. For example port `8080` produces `baseURL` equal `http://localhost:8080`.
+The `port` (but not the `url`) gets passed over to Playwright as a [`property: TestOptions.baseURL`]. For example port `8080` produces `baseURL` equal `http://localhost:8080`. If `webServer` is specified as an array, you must explicitly configure the `baseURL` (even if it only has one entry).
 
 :::note
 It is also recommended to specify [`property: TestOptions.baseURL`] in the config, so that tests could use relative urls.
@@ -725,6 +725,59 @@ test('test', async ({ page }) => {
 });
 ```
 
+Multiple web servers (or background processes) can be launched:
+
+```js tab=js-ts
+// playwright.config.ts
+import type { PlaywrightTestConfig } from '@playwright/test';
+const config: PlaywrightTestConfig = {
+  webServer: [
+    {
+      command: 'npm run start',
+      port: 3000,
+      timeout: 120 * 1000,
+      reuseExistingServer: !process.env.CI,
+    },
+    {
+      command: 'npm run backend',
+      port: 3333,
+      timeout: 120 * 1000,
+      reuseExistingServer: !process.env.CI,
+    }
+  ],
+  use: {
+    baseURL: 'http://localhost:3000/',
+  },
+};
+export default config;
+```
+
+```js tab=js-js
+// playwright.config.js
+// @ts-check
+/** @type {import('@playwright/test').PlaywrightTestConfig} */
+const config = {
+  webServer: [
+    {
+      command: 'npm run start',
+      port: 3000,
+      timeout: 120 * 1000,
+      reuseExistingServer: !process.env.CI,
+    },
+    {
+      command: 'npm run backend',
+      port: 3333,
+      timeout: 120 * 1000,
+      reuseExistingServer: !process.env.CI,
+    }
+  ],
+  use: {
+    baseURL: 'http://localhost:3000/',
+  },
+};
+module.exports = config;
+```
+
 ## property: TestConfig.workers
 * since: v1.10
 - type: ?<[int]>

packages/playwright-test/src/loader.ts

@@ -141,7 +141,15 @@ export class Loader {
     this._fullConfig.shard = takeFirst(config.shard, baseFullConfig.shard);
     this._fullConfig.updateSnapshots = takeFirst(config.updateSnapshots, baseFullConfig.updateSnapshots);
     this._fullConfig.workers = takeFirst(config.workers, baseFullConfig.workers);
-    this._fullConfig.webServer = takeFirst(config.webServer, baseFullConfig.webServer);
+    const webServers = takeFirst(config.webServer, baseFullConfig.webServer);
+    if (Array.isArray(webServers)) { // multiple web server mode
+      // Due to previous choices, this value shows up to the user in globalSetup as part of FullConfig. Arrays are not supported by the old type.
+      this._fullConfig.webServer = null;
+      this._fullConfig._webServers = webServers;
+    } else if (webServers) { // legacy singleton mode
+      this._fullConfig.webServer = webServers;
+      this._fullConfig._webServers = [webServers];
+    }
     this._fullConfig.metadata = takeFirst(config.metadata, baseFullConfig.metadata);
     this._fullConfig.projects = (config.projects || [config]).map(p => this._resolveProject(config, this._fullConfig, p, throwawayArtifactsPath));
   }
@@ -610,6 +618,7 @@ export const baseFullConfig: FullConfigInternal = {
   version: require('../package.json').version,
   workers,
   webServer: null,
+  _webServers: [],
   _globalOutputDir: path.resolve(process.cwd()),
   _configDir: '',
   _testGroupsCount: 0,

packages/playwright-test/src/plugins/webServerPlugin.ts

@@ -24,6 +24,7 @@ import { launchProcess } from 'playwright-core/lib/utils/processLauncher';
 
 import type { FullConfig, Reporter } from '../../types/testReporter';
 import type { TestRunnerPlugin } from '.';
+import type { FullConfigInternal } from '../types';
 
 
 export type WebServerPluginOptions = {
@@ -202,18 +203,21 @@ export const webServer = (options: WebServerPluginOptions): TestRunnerPlugin =>
   return new WebServerPlugin(options, false, { onStdOut: d => console.log(d.toString()), onStdErr: d => console.error(d.toString()) });
 };
 
-export const webServerPluginForConfig = (config: FullConfig, reporter: Reporter): TestRunnerPlugin => {
+export const webServerPluginsForConfig = (config: FullConfigInternal, reporter: Reporter): TestRunnerPlugin[] => {
-  const webServer = config.webServer!;
+  const shouldSetBaseUrl = !!config.webServer;
-  if (webServer.port !== undefined && webServer.url !== undefined)
+  const webServerPlugins = [];
-    throw new Error(`Exactly one of 'port' or 'url' is required in config.webServer.`);
+  for (const webServerConfig of config._webServers) {
+    if (webServerConfig.port !== undefined && webServerConfig.url !== undefined)
+      throw new Error(`Exactly one of 'port' or 'url' is required in config.webServer.`);
 
-  const url = webServer.url || `http://localhost:${webServer.port}`;
+    const url = webServerConfig.url || `http://localhost:${webServerConfig.port}`;
 
-  // We only set base url when only the port is given. That's a legacy mode we have regrets about.
+    // We only set base url when only the port is given. That's a legacy mode we have regrets about.
-  if (!webServer.url)
+    if (shouldSetBaseUrl && !webServerConfig.url)
-    process.env.PLAYWRIGHT_TEST_BASE_URL = url;
+      process.env.PLAYWRIGHT_TEST_BASE_URL = url;
 
-  // TODO: replace with reporter once plugins are removed.
+    webServerPlugins.push(new WebServerPlugin({ ...webServerConfig,  url }, webServerConfig.port !== undefined, reporter));
-  // eslint-disable-next-line no-console
+  }
-  return new WebServerPlugin({ ...webServer, url }, webServer.port !== undefined, reporter);
+
+  return webServerPlugins;
 };

packages/playwright-test/src/runner.ts

@@ -43,7 +43,7 @@ import { raceAgainstTimeout } from 'playwright-core/lib/utils/timeoutRunner';
 import { SigIntWatcher } from './sigIntWatcher';
 import type { TestRunnerPlugin } from './plugins';
 import { setRunnerToAddPluginsTo } from './plugins';
-import { webServerPluginForConfig } from './plugins/webServerPlugin';
+import { webServerPluginsForConfig } from './plugins/webServerPlugin';
 import { MultiMap } from 'playwright-core/lib/utils/multimap';
 
 const removeFolderAsync = promisify(rimraf);
@@ -457,8 +457,7 @@ export class Runner {
     };
 
     // Legacy webServer support.
-    if (config.webServer)
+    this._plugins.push(...webServerPluginsForConfig(config, this._reporter));
-      this._plugins.push(webServerPluginForConfig(config, this._reporter));
 
     await this._runAndReportError(async () => {
       // First run the plugins, if plugin is a web server we want it to run before the

packages/playwright-test/src/types.ts

@@ -44,6 +44,11 @@ export interface FullConfigInternal extends FullConfigPublic {
   _globalOutputDir: string;
   _configDir: string;
   _testGroupsCount: number;
+  /**
+   * If populated, this should also be the first/only entry in _webServers. Legacy singleton `webServer` as well as those provided via an array in the user-facing playwright.config.{ts,js} will be in `_webServers`. The legacy field (`webServer`) field additionally stores the backwards-compatible singleton `webServer` since it had been showing up in globalSetup to the user.
+   */
+  webServer: FullConfigPublic['webServer'];
+  _webServers: Exclude<FullConfigPublic['webServer'], null>[];
 
   // Overrides the public field.
   projects: FullProjectInternal[];

packages/playwright-test/types/test.d.ts

@@ -412,17 +412,19 @@ interface TestConfig {
    */
   reporter?: LiteralUnion<'list'|'dot'|'line'|'github'|'json'|'junit'|'null'|'html', string> | ReporterDescription[];
   /**
-   * Launch a development web server during the tests.
+   * Launch a development web server (or multiple) during the tests.
    *
-   * If the port is specified, the server will wait for it to be available on `127.0.0.1` or `::1`, before running the tests.
+   * If the port is specified, Playwright Test will wait for it to be available on `127.0.0.1` or `::1`, before running the
-   * If the url is specified, the server will wait for the URL to return a 2xx status code before running the tests.
+   * tests. If the url is specified, Playwright Test will wait for the URL to return a 2xx, 3xx, 400, 401, 402, or 403 status
+   * code before running the tests.
    *
    * For continuous integration, you may want to use the `reuseExistingServer: !process.env.CI` option which does not use an
    * existing server on the CI. To see the stdout, you can set the `DEBUG=pw:webserver` environment variable.
    *
    * The `port` (but not the `url`) gets passed over to Playwright as a
    * [testOptions.baseURL](https://playwright.dev/docs/api/class-testoptions#test-options-base-url). For example port `8080`
-   * produces `baseURL` equal `http://localhost:8080`.
+   * produces `baseURL` equal `http://localhost:8080`. If `webServer` is specified as an array, you must explicitly configure
+   * the `baseURL` (even if it only has one entry).
    *
    * > NOTE: It is also recommended to specify
    * [testOptions.baseURL](https://playwright.dev/docs/api/class-testoptions#test-options-base-url) in the config, so that
@@ -457,6 +459,33 @@ interface TestConfig {
    * });
    * ```
    *
+   * Multiple web servers (or background processes) can be launched:
+   *
+   * ```js
+   * // playwright.config.ts
+   * import type { PlaywrightTestConfig } from '@playwright/test';
+   * const config: PlaywrightTestConfig = {
+   *   webServer: [
+   *     {
+   *       command: 'npm run start',
+   *       port: 3000,
+   *       timeout: 120 * 1000,
+   *       reuseExistingServer: !process.env.CI,
+   *     },
+   *     {
+   *       command: 'npm run backend',
+   *       port: 3333,
+   *       timeout: 120 * 1000,
+   *       reuseExistingServer: !process.env.CI,
+   *     }
+   *   ],
+   *   use: {
+   *     baseURL: 'http://localhost:3000/',
+   *   },
+   * };
+   * export default config;
+   * ```
+   *
    */
   webServer?: TestConfigWebServer;
   /**
@@ -1187,17 +1216,19 @@ export interface FullConfig<TestArgs = {}, WorkerArgs = {}> {
    */
   workers: number;
   /**
-   * Launch a development web server during the tests.
+   * Launch a development web server (or multiple) during the tests.
    *
-   * If the port is specified, the server will wait for it to be available on `127.0.0.1` or `::1`, before running the tests.
+   * If the port is specified, Playwright Test will wait for it to be available on `127.0.0.1` or `::1`, before running the
-   * If the url is specified, the server will wait for the URL to return a 2xx status code before running the tests.
+   * tests. If the url is specified, Playwright Test will wait for the URL to return a 2xx, 3xx, 400, 401, 402, or 403 status
+   * code before running the tests.
    *
    * For continuous integration, you may want to use the `reuseExistingServer: !process.env.CI` option which does not use an
    * existing server on the CI. To see the stdout, you can set the `DEBUG=pw:webserver` environment variable.
    *
    * The `port` (but not the `url`) gets passed over to Playwright as a
    * [testOptions.baseURL](https://playwright.dev/docs/api/class-testoptions#test-options-base-url). For example port `8080`
-   * produces `baseURL` equal `http://localhost:8080`.
+   * produces `baseURL` equal `http://localhost:8080`. If `webServer` is specified as an array, you must explicitly configure
+   * the `baseURL` (even if it only has one entry).
    *
    * > NOTE: It is also recommended to specify
    * [testOptions.baseURL](https://playwright.dev/docs/api/class-testoptions#test-options-base-url) in the config, so that
@@ -1232,6 +1263,33 @@ export interface FullConfig<TestArgs = {}, WorkerArgs = {}> {
    * });
    * ```
    *
+   * Multiple web servers (or background processes) can be launched:
+   *
+   * ```js
+   * // playwright.config.ts
+   * import type { PlaywrightTestConfig } from '@playwright/test';
+   * const config: PlaywrightTestConfig = {
+   *   webServer: [
+   *     {
+   *       command: 'npm run start',
+   *       port: 3000,
+   *       timeout: 120 * 1000,
+   *       reuseExistingServer: !process.env.CI,
+   *     },
+   *     {
+   *       command: 'npm run backend',
+   *       port: 3333,
+   *       timeout: 120 * 1000,
+   *       reuseExistingServer: !process.env.CI,
+   *     }
+   *   ],
+   *   use: {
+   *     baseURL: 'http://localhost:3000/',
+   *   },
+   * };
+   * export default config;
+   * ```
+   *
    */
   webServer: TestConfigWebServer | null;
 }

tests/playwright-test/web-server.spec.ts

@@ -43,7 +43,9 @@ test('should create a server', async ({ runInlineTest }, { workerIndex }) => {
       };
     `,
     'globalSetup.ts': `
-      module.exports = async () => {
+      const { expect } = pwt;
+      module.exports = async (config) => {
+        expect(config.webServer.port, "For backwards compatibility reasons, we ensure this shows up.").toBe(${port});
         const http = require("http");
         const response = await new Promise(resolve => {
           const request = http.request("http://localhost:${port}/hello", resolve);
@@ -431,16 +433,7 @@ test('should create multiple servers', async ({ runInlineTest }, { workerIndex }
   const port = workerIndex + 10500;
   const result = await runInlineTest({
     'test.spec.ts': `
-        import { webServer } from '@playwright/test/lib/plugins';
+        const { test } = pwt;
-        const { test, _addRunnerPlugin } = pwt;
-        _addRunnerPlugin(webServer({
-            command: 'node ${JSON.stringify(SIMPLE_SERVER_PATH)} ${port}',
-            url: 'http://localhost:${port}/port',
-        }));
-        _addRunnerPlugin(webServer({
-            command: 'node ${JSON.stringify(SIMPLE_SERVER_PATH)} ${port + 1}',
-            url: 'http://localhost:${port + 1}/port',
-        }));
 
         test('connect to the server', async ({page}) => {
           await page.goto('http://localhost:${port}/port');
@@ -452,12 +445,24 @@ test('should create multiple servers', async ({ runInlineTest }, { workerIndex }
       `,
     'playwright.config.ts': `
         module.exports = {
+          webServer: [
+            {
+              command: 'node ${JSON.stringify(SIMPLE_SERVER_PATH)} ${port}',
+              url: 'http://localhost:${port}/port',
+            },
+            {
+              command: 'node ${JSON.stringify(SIMPLE_SERVER_PATH)} ${port + 1}',
+              url: 'http://localhost:${port + 1}/port',
+            }
+          ],
           globalSetup: 'globalSetup.ts',
           globalTeardown: 'globalTeardown.ts',
         };
         `,
     'globalSetup.ts': `
-        module.exports = async () => {
+        const { expect } = pwt;
+        module.exports = async (config) => {
+          expect(config.webServer, "The public API defines this type as singleton or null, so if using array style we fallback to null to avoid having the type lie to the user.").toBe(null);
           const http = require("http");
           const response = await new Promise(resolve => {
             const request = http.request("http://localhost:${port}/hello", resolve);
@@ -568,4 +573,3 @@ test('should treat 3XX as available server', async ({ runInlineTest }, { workerI
   expect(result.output).toContain('[WebServer] listening');
   expect(result.output).toContain('[WebServer] error from server');
 });
-

utils/generate_types/index.js

@@ -426,7 +426,8 @@ class TypesGenerator {
       const name = namespace.map(n => n[0].toUpperCase() + n.substring(1)).join('');
       const shouldExport = exported[name];
       const properties = namespace[namespace.length - 1] === 'options' ? type.sortedProperties() : type.properties;
-      this.objectDefinitions.push({ name, properties });
+      if (!this.objectDefinitions.some(o => o.name === name))
+        this.objectDefinitions.push({ name, properties });
       if (shouldExport) {
         out = name;
       } else {