feat(test-runner): migrate to launch config/server (#7603)

docs/src/test-advanced.md

@@ -42,7 +42,7 @@ These options would be typically different between local development and CI oper
 - `reportSlowTests: { max: number, threshold: number } | null` - Whether to report slow tests. When `null`, slow tests are not reported. Otherwise, tests that took more than `threshold` milliseconds are reported as slow, but no more than `max` number of them. Passing zero as `max` reports all slow tests that exceed the threshold.
 - `shard: { total: number, current: number } | null` - [Shard](./test-parallel.md#shards) information.
 - `updateSnapshots: boolean` - Whether to update expected snapshots with the actual results produced by the test run.
-- `webServer: { command: string, port?: number, cwd?: string, timeout?: number, env?: object }` - Launch a web server before the tests will start. It will automaticially detect the port when it got printed to the stdout.
+- `launch: { command: string, waitForPort?: number, waitForPortTimeout?: number, strict?: boolean, cwd?: string, env?: object }[]` - Launch a process before the tests will start. When using `waitForPort` it will wait until the server is available, see [launch server](#launching-a-development-web-server-during-the-tests) configuration for examples. `strict` will verify that the `waitForPort` port is available instead of using it by default.
 - `workers: number` - The maximum number of concurrent worker processes to use for parallelizing tests.
 
 Note that each [test project](#projects) can provide its own test suite options, for example two projects can run different tests by providing different `testDir`s. However, test run options are shared between all projects.
@@ -203,20 +203,22 @@ export const test = base.extend<{ saveLogs: void }>({
 
 ## Launching a development web server during the tests
 
-To launch a web server during the tests, use the `webServer` option in the [configuration file](#configuration-object).
+To launch a server during the tests, use the `launch` option in the [configuration file](#configuration-object).
 
-Playwright Test does automatically detect if a localhost URL like `http://localhost:3000` gets printed to the stdout.
+You can specify a port via `waitForPort` or additional environment variables, see [here](#configuration-object). When a port is specified, the server will wait for it to be available before starting. For continuous integration, you may want to use the `strict` option which ensures that the port is available before starting the server.
-The port from the printed URL gets then used to check when its accepting requests and passed over to Playwright as a
+
-[`param: baseURL`] when creating the context [`method: Browser.newContext`]. You can also manually specify a `port` or additional environment variables, see [here](#configuration-object).
+The port gets then passed over to Playwright as a [`param: baseURL`] when creating the context [`method: Browser.newContext`].
 
 ```js js-flavor=ts
 // playwright.config.ts
 import { PlaywrightTestConfig } from '@playwright/test';
 
 const config: PlaywrightTestConfig = {
-  webServer: {
+  launch: {
     command: 'npm run start',
-    timeout: 120 * 1000,
+    waitForPort: 3000,
+    waitForPortTimeout: 120 * 1000,
+    strict: !!process.env.CI,
   },
 };
 
@@ -228,9 +230,11 @@ export default config;
 // @ts-check
 /** @type {import('@playwright/test').PlaywrightTestConfig} */
 const config = {
-  webServer: {
+  launch: {
     command: 'npm run start',
-    timeout: 120 * 1000,
+    waitForPort: 3000,
+    waitForPortTimeout: 120 * 1000,
+    strict: !!process.env.CI,
   },
 };
 

docs/src/test-configuration.md

@@ -478,7 +478,7 @@ In addition to configuring [Browser] or [BrowserContext], videos or screenshots,
 - `testIgnore`: Glob patterns or regular expressions that should be ignored when looking for the test files. For example, `'**/test-assets'`.
 - `testMatch`: Glob patterns or regular expressions that match test files. For example, `'**/todo-tests/*.spec.ts'`. By default, Playwright Test runs `.*(test|spec)\.(js|ts|mjs)` files.
 - `timeout`: Time in milliseconds given to each test.
-- `webServer: { command: string, port?: number, cwd?: string, timeout?: number, env?: object }` - Launch a web server before the tests will start. It will automatically detect the port when it got printed to the stdout.
+- `launch: { command: string, waitForPort?: number, waitForPortTimeout?: number, strict?: boolean, cwd?: string, env?: object }` - Launch a process before the tests will start. When using `waitForPort` it will wait until the server is available, see [launch server](./test-advanced.md#launching-a-development-web-server-during-the-tests) configuration for examples. `strict` will verify that the `waitForPort` port is available instead of using it by default.
 - `workers`: The maximum number of concurrent worker processes to use for parallelizing tests.
 
 You can specify these options in the configuration file. Note that testing options are **top-level**, do not put them into the `use` section.

src/test/launchServer.ts

@@ -19,7 +19,7 @@ import net from 'net';
 import os from 'os';
 import stream from 'stream';
 import { monotonicTime, raceAgainstDeadline } from './util';
-import { WebServerConfig } from '../../types/test';
+import { LaunchConfig } from '../../types/test';
 import { launchProcess } from '../utils/processLauncher';
 
 const DEFAULT_ENVIRONMENT_VARIABLES = {
@@ -28,57 +28,41 @@ const DEFAULT_ENVIRONMENT_VARIABLES = {
 
 const newProcessLogPrefixer = () => new stream.Transform({
   transform(this: stream.Transform, chunk: Buffer, encoding: string, callback: stream.TransformCallback) {
-    this.push(chunk.toString().split(os.EOL).map((line: string): string => line ? `[WebServer] ${line}` : line).join(os.EOL));
+    this.push(chunk.toString().split(os.EOL).map((line: string): string => line ? `[Launch] ${line}` : line).join(os.EOL));
     callback();
   },
 });
 
-export class WebServer {
+class LaunchServer {
   private _killProcess?: () => Promise<void>;
   private _processExitedPromise!: Promise<any>;
-  constructor(private readonly config: WebServerConfig) { }
+  constructor(private readonly config: LaunchConfig) { }
 
-  public static async create(config: WebServerConfig): Promise<WebServer> {
+  public static async create(config: LaunchConfig): Promise<LaunchServer> {
-    const webServer = new WebServer(config);
+    const launchServer = new LaunchServer(config);
-    if (config.port)
-      await webServer._verifyFreePort(config.port);
     try {
-      const port = await webServer._startWebServer();
+      await launchServer._startProcess();
-      await webServer._waitForAvailability(port);
+      await launchServer._waitForProcess();
-      const baseURL = `http://localhost:${port}`;
+      return launchServer;
-      process.env.PLAYWRIGHT_TEST_BASE_URL = baseURL;
-      console.log(`Using WebServer at '${baseURL}'.`);
-      return webServer;
     } catch (error) {
-      await webServer.kill();
+      await launchServer.kill();
       throw error;
     }
   }
 
-  private async _verifyFreePort(port: number) {
+  private async _startProcess(): Promise<void> {
-    const cancellationToken = { canceled: false };
-    const portIsUsed = await Promise.race([
-      new Promise(resolve => setTimeout(() => resolve(false), 100)),
-      waitForSocket(port, 100, cancellationToken),
-    ]);
-    cancellationToken.canceled = true;
-    if (portIsUsed)
-      throw new Error(`Port ${port} is used, make sure that nothing is running on the port`);
-  }
-
-  private async _startWebServer(): Promise<number> {
-    let collectPortResolve = (port: number) => { };
-    const collectPortPromise = new Promise<number>(resolve => collectPortResolve = resolve);
-    function collectPort(data: Buffer) {
-      const regExp = /http:\/\/localhost:(\d+)/.exec(data.toString());
-      if (regExp)
-        collectPortResolve(parseInt(regExp[1], 10));
-    }
-
     let processExitedReject = (error: Error) => { };
     this._processExitedPromise = new Promise((_, reject) => processExitedReject = reject);
 
-    console.log(`Starting WebServer with '${this.config.command}'...`);
+    if (this.config.waitForPort) {
+      const portIsUsed = !await canBindPort(this.config.waitForPort);
+      if (portIsUsed && this.config.strict)
+        throw new Error(`Port ${this.config.waitForPort} is used, make sure that nothing is running on the port or set strict:false in config.launch.`);
+      if (portIsUsed)
+        return;
+    }
+
+    console.log(`Launching '${this.config.command}'...`);
     const { launchedProcess, kill } = await launchProcess({
       command: this.config.command,
       env: {
@@ -91,26 +75,26 @@ export class WebServer {
       shell: true,
       attemptToGracefullyClose: async () => {},
       log: () => {},
-      onExit: code => processExitedReject(new Error(`WebServer was not able to start. Exit code: ${code}`)),
+      onExit: code => processExitedReject(new Error(`Process from config.launch was not able to start. Exit code: ${code}`)),
       tempDirectories: [],
     });
     this._killProcess = kill;
 
     launchedProcess.stderr.pipe(newProcessLogPrefixer()).pipe(process.stderr);
     launchedProcess.stdout.on('data', () => {});
+  }
 
-    if (this.config.port)
+  private async _waitForProcess() {
-      return this.config.port;
+    if (this.config.waitForPort) {
-    launchedProcess.stdout.on('data', collectPort);
+      await this._waitForAvailability(this.config.waitForPort);
-    const detectedPort = await Promise.race([
+      const baseURL = `http://localhost:${this.config.waitForPort}`;
-      this._processExitedPromise,
+      process.env.PLAYWRIGHT_TEST_BASE_URL = baseURL;
-      collectPortPromise,
+      console.log(`Using baseURL '${baseURL}' from config.launch.`);
-    ]);
+    }
-    return detectedPort;
   }
 
   private async _waitForAvailability(port: number) {
-    const launchTimeout = this.config.timeout || 60 * 1000;
+    const launchTimeout = this.config.waitForPortTimeout || 60 * 1000;
     const cancellationToken = { canceled: false };
     const { timedOut } = (await Promise.race([
       raceAgainstDeadline(waitForSocket(port, 100, cancellationToken), launchTimeout + monotonicTime()),
@@ -118,13 +102,25 @@ export class WebServer {
     ]));
     cancellationToken.canceled = true;
     if (timedOut)
-      throw new Error(`Timed out waiting ${launchTimeout}ms for WebServer"`);
+      throw new Error(`Timed out waiting ${launchTimeout}ms from config.launch.`);
   }
   public async kill() {
     await this._killProcess?.();
   }
 }
 
+async function canBindPort(port: number): Promise<boolean> {
+  return new Promise<boolean>(resolve => {
+    const server = net.createServer();
+    server.on('error', () => resolve(false));
+    server.listen(port, () => {
+      server.close(() => {
+        resolve(true);
+      });
+    });
+  });
+}
+
 async function waitForSocket(port: number, delay: number, cancellationToken: { canceled: boolean }) {
   while (!cancellationToken.canceled) {
     const connected = await new Promise(resolve => {
@@ -143,3 +139,25 @@ async function waitForSocket(port: number, delay: number, cancellationToken: { c
     await new Promise(x => setTimeout(x, delay));
   }
 }
+
+export class LaunchServers {
+  private readonly _servers: LaunchServer[] = [];
+
+  public static async create(configs: LaunchConfig[]): Promise<LaunchServers> {
+    const launchServers = new LaunchServers();
+    try {
+      for (const config of configs)
+        launchServers._servers.push(await LaunchServer.create(config));
+    } catch (error) {
+      for (const server of launchServers._servers)
+        await server.kill();
+      throw error;
+    }
+    return launchServers;
+  }
+
+  public async killAll() {
+    for (const server of this._servers)
+      await server.kill();
+  }
+}

src/test/loader.ts

@@ -24,6 +24,7 @@ import * as path from 'path';
 import * as url from 'url';
 import { ProjectImpl } from './project';
 import { Reporter } from './reporter';
+import { LaunchConfig } from '../../types/test';
 
 export class Loader {
   private _defaultConfig: Config;
@@ -98,7 +99,7 @@ export class Loader {
     this._fullConfig.shard = takeFirst(this._configOverrides.shard, this._config.shard, baseFullConfig.shard);
     this._fullConfig.updateSnapshots = takeFirst(this._configOverrides.updateSnapshots, this._config.updateSnapshots, baseFullConfig.updateSnapshots);
     this._fullConfig.workers = takeFirst(this._configOverrides.workers, this._config.workers, baseFullConfig.workers);
-    this._fullConfig.webServer = takeFirst(this._configOverrides.webServer, this._config.webServer, baseFullConfig.webServer);
+    this._fullConfig.launch = takeFirst(toLaunchServers(this._configOverrides.launch), toLaunchServers(this._config.launch), baseFullConfig.launch);
 
     for (const project of projects)
       this._addProject(project, this._fullConfig.rootDir);
@@ -227,6 +228,14 @@ function toReporters(reporters: 'dot' | 'line' | 'list' | 'junit' | 'json' | 'nu
   return reporters;
 }
 
+function toLaunchServers(launchConfigs?: LaunchConfig | LaunchConfig[]): LaunchConfig[]|undefined {
+  if (!launchConfigs)
+    return;
+  if (!Array.isArray(launchConfigs))
+    return [launchConfigs];
+  return launchConfigs;
+}
+
 function errorWithFile(file: string, message: string) {
   return new Error(`${file}: ${message}`);
 }
@@ -430,5 +439,5 @@ const baseFullConfig: FullConfig = {
   shard: null,
   updateSnapshots: 'missing',
   workers: 1,
-  webServer: null,
+  launch: [],
 };

src/test/runner.ts

@@ -35,7 +35,7 @@ import EmptyReporter from './reporters/empty';
 import { ProjectImpl } from './project';
 import { Minimatch } from 'minimatch';
 import { Config } from './types';
-import { WebServer } from './webServer';
+import { LaunchServers } from './launchServer';
 
 const removeFolderAsync = promisify(rimraf);
 const readDirAsync = promisify(fs.readdir);
@@ -167,7 +167,7 @@ export class Runner {
       testFiles.forEach(file => allTestFiles.add(file));
     }
 
-    const webServer: WebServer|null = config.webServer ? await WebServer.create(config.webServer) : null;
+    const launchServers = await LaunchServers.create(config.launch);
     let globalSetupResult: any;
     if (config.globalSetup)
       globalSetupResult = await (await this._loader.loadGlobalHook(config.globalSetup, 'globalSetup'))(this._loader.fullConfig());
@@ -267,7 +267,7 @@ export class Runner {
         await globalSetupResult(this._loader.fullConfig());
       if (config.globalTeardown)
         await (await this._loader.loadGlobalHook(config.globalTeardown, 'globalTeardown'))(this._loader.fullConfig());
-      await webServer?.kill();
+      await launchServers.killAll();
     }
   }
 }

tests/playwright-test/launch-server.spec.ts

@@ -32,9 +32,9 @@ test('should create a server', async ({ runInlineTest }, { workerIndex }) => {
     `,
     'playwright.config.ts': `
       module.exports = {
-        webServer: {
+        launch: {
           command: 'node ${JSON.stringify(path.join(__dirname, 'assets', 'simple-server.js'))} ${port}',
-          port: ${port},
+          waitForPort: ${port},
         },
         globalSetup: 'globalSetup.ts',
         globalTeardown: 'globalTeardown.ts',
@@ -61,7 +61,7 @@ test('should create a server', async ({ runInlineTest }, { workerIndex }) => {
   expect(result.passed).toBe(1);
   expect(result.report.suites[0].specs[0].tests[0].results[0].status).toContain('passed');
 
-  const expectedLogMessages = ['Starting WebServer', 'globalSetup', 'globalSetup teardown', 'globalTeardown-status-200'];
+  const expectedLogMessages = ['Launching ', 'globalSetup', 'globalSetup teardown', 'globalTeardown-status-200'];
   const actualLogMessages = expectedLogMessages.map(log => ({
     log,
     index: result.output.indexOf(log),
@@ -82,9 +82,9 @@ test('should create a server with environment variables', async ({ runInlineTest
     `,
     'playwright.config.ts': `
       module.exports = {
-        webServer: {
+        launch: {
           command: 'node ${JSON.stringify(path.join(__dirname, 'assets', 'simple-server.js'))} ${port}',
-          port: ${port},
+          waitForPort: ${port},
           env: {
             'FOO': 'BAR',
           }
@@ -110,40 +110,16 @@ test('should time out waiting for a server', async ({ runInlineTest }, { workerI
     `,
     'playwright.config.ts': `
       module.exports = {
-        webServer: {
+        launch: {
           command: 'node ${JSON.stringify(JSON.stringify(path.join(__dirname, 'assets', 'simple-server.js')))} ${port}',
-          port: ${port},
+          waitForPort: ${port},
-          timeout: 100,
+          waitForPortTimeout: 100,
         }
       };
     `,
   });
   expect(result.exitCode).toBe(1);
-  expect(result.output).toContain(`Timed out waiting 100ms for WebServer`);
+  expect(result.output).toContain(`Timed out waiting 100ms from config.launch.`);
-});
-
-test('should be able to detect the port from the process stdout', async ({ runInlineTest }, { workerIndex }) => {
-  const port = workerIndex + 10500;
-  const result = await runInlineTest({
-    'test.spec.ts': `
-      const { test } = pwt;
-      test('connect to the server', async ({baseURL, page}) => {
-        expect(baseURL).toBe('http://localhost:${port}');
-        await page.goto(baseURL + '/hello');
-        expect(await page.textContent('body')).toBe('hello');
-      });
-    `,
-    'playwright.config.ts': `
-      module.exports = {
-        webServer: {
-          command: 'node ${JSON.stringify(path.join(__dirname, 'assets', 'simple-server-with-stdout.js'))} ${port}',
-        }
-      };
-    `,
-  });
-  expect(result.exitCode).toBe(0);
-  expect(result.passed).toBe(1);
-  expect(result.report.suites[0].specs[0].tests[0].results[0].status).toContain('passed');
 });
 
 test('should be able to specify the baseURL without the server', async ({ runInlineTest }, { workerIndex }) => {
@@ -172,5 +148,97 @@ test('should be able to specify the baseURL without the server', async ({ runInl
   expect(result.exitCode).toBe(0);
   expect(result.passed).toBe(1);
   expect(result.report.suites[0].specs[0].tests[0].results[0].status).toContain('passed');
-  server.close();
+  await new Promise(resolve => server.close(resolve));
+});
+
+test('should be able to use an existing server when strict is false ', async ({ runInlineTest }, { workerIndex }) => {
+  const port = workerIndex + 10500;
+  const server = http.createServer((req: http.IncomingMessage, res: http.ServerResponse) => {
+    res.end('<html><body>hello</body></html>');
+  });
+  await new Promise(resolve => server.listen(port, resolve));
+  const result = await runInlineTest({
+    'test.spec.ts': `
+      const { test } = pwt;
+      test('connect to the server via the baseURL', async ({baseURL, page}) => {
+        await page.goto('/hello');
+        await page.waitForURL('/hello');
+        expect(page.url()).toBe('http://localhost:${port}/hello');
+        expect(await page.textContent('body')).toBe('hello');
+      });
+    `,
+    'playwright.config.ts': `
+      module.exports = {
+        launch: {
+          command: 'node ${JSON.stringify(path.join(__dirname, 'assets', 'simple-server.js'))} ${port}',
+          waitForPort: ${port},
+          strict: false,
+        }
+      };
+    `,
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+  expect(result.output).not.toContain('[Launch] ');
+  expect(result.report.suites[0].specs[0].tests[0].results[0].status).toContain('passed');
+  await new Promise(resolve => server.close(resolve));
+});
+
+test('should throw when a server is already running on the given port and strict is true ', async ({ runInlineTest }, { workerIndex }) => {
+  const port = workerIndex + 10500;
+  const server = http.createServer((req: http.IncomingMessage, res: http.ServerResponse) => {
+    res.end('<html><body>hello</body></html>');
+  });
+  await new Promise(resolve => server.listen(port, resolve));
+  const result = await runInlineTest({
+    'test.spec.ts': `
+      const { test } = pwt;
+      test('connect to the server via the baseURL', async ({baseURL, page}) => {
+        await page.goto('/hello');
+        await page.waitForURL('/hello');
+        expect(page.url()).toBe('http://localhost:${port}/hello');
+        expect(await page.textContent('body')).toBe('hello');
+      });
+    `,
+    'playwright.config.ts': `
+      module.exports = {
+        launch: {
+          command: 'node ${JSON.stringify(path.join(__dirname, 'assets', 'simple-server.js'))} ${port}',
+          waitForPort: ${port},
+          strict: true,
+        }
+      };
+    `,
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain(`Port ${port} is used, make sure that nothing is running on the port`);
+  await new Promise(resolve => server.close(resolve));
+});
+
+test('should create multiple servers', async ({ runInlineTest }, { workerIndex }) => {
+  const port1 = workerIndex + 10500;
+  const port2 = workerIndex + 10600;
+  const result = await runInlineTest({
+    'test.spec.ts': `
+      const { test } = pwt;
+      test('connect to the server via the baseURL', async ({baseURL, page}) => {
+        await page.goto('http://localhost:${port1}/hello');
+        await page.goto('http://localhost:${port2}/hello');
+      });
+    `,
+    'playwright.config.ts': `
+      module.exports = {
+        launch: [{
+          command: 'node ${JSON.stringify(path.join(__dirname, 'assets', 'simple-server.js'))} ${port1}',
+          waitForPort: ${port1},
+        },{
+          command: 'node ${JSON.stringify(path.join(__dirname, 'assets', 'simple-server.js'))} ${port2}',
+          waitForPort: ${port2},
+        }],
+      };
+    `,
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+  expect(result.report.suites[0].specs[0].tests[0].results[0].status).toContain('passed');
 });

types/test.d.ts

@@ -118,28 +118,32 @@ export interface Project<TestArgs = {}, WorkerArgs = {}> extends ProjectBase {
 
 export type FullProject<TestArgs = {}, WorkerArgs = {}> = Required<Project<TestArgs, WorkerArgs>>;
 
-export type WebServerConfig = {
+export type LaunchConfig = {
   /**
-   * Shell command to start the webserver. For example `npm run start`.
+   * Shell command to start. For example `npm run start`.
    */
   command: string,
   /**
-   * The port that your server is expected to appear on. If not specified, it does get automatically collected via the
+   * The port that your http server is expected to appear on. If specified it does wait until it accepts connections.
-   * command output when a localhost URL gets printed.
    */
-  port?: number,
+  waitForPort?: number,
   /**
-   * WebServer environment variables, process.env by default
+   * How long to wait for the process to start up and be available in milliseconds. Defaults to 60000.
+   */
+  waitForPortTimeout?: number,
+  /**
+   * If true it will verify that the given port via `waitForPort` is available and throw otherwise.
+   * This should commonly set to !!process.env.CI to allow the local dev server when running tests locally.
+   */
+  strict?: boolean
+  /**
+   * Environment variables, process.env by default
    */
   env?: Record<string, string>,
   /**
    * Current working directory of the spawned process. Default is process.cwd().
    */
   cwd?: string,
-  /**
-   * How long to wait for the server to start up in milliseconds. Defaults to 60000.
-   */
-  timeout?: number,
 };
 
 /**
@@ -233,7 +237,7 @@ interface ConfigBase {
   /**
    * Launch a web server before running tests.
    */
-   webServer?: WebServerConfig;
+  launch?: LaunchConfig | LaunchConfig[];
 
   /**
    * The maximum number of concurrent worker processes to use for parallelizing tests.
@@ -268,7 +272,7 @@ export interface FullConfig {
   shard: Shard;
   updateSnapshots: UpdateSnapshots;
   workers: number;
-  webServer: WebServerConfig | null;
+  launch: LaunchConfig[];
 }
 
 export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped';