feat(test-runner): re-enable web server (#7906)

Co-authored-by: Joel Einbinder <joel.einbinder@gmail.com>

docs/src/test-advanced-js.md

@@ -43,6 +43,7 @@ These options would be typically different between local development and CI oper
 - `shard: { total: number, current: number } | null` - [Shard](./test-parallel.md#shards) information.
 - `updateSnapshots: boolean` - Whether to update expected snapshots with the actual results produced by the test run.
 - `workers: number` - The maximum number of concurrent worker processes to use for parallelizing tests.
+- `webServer: { command: string, port: number, timeout?: number, reuseExistingServer?: boolean, cwd?: string, env?: object }` - Launch a process and wait that it's ready before the tests will start. See [launch web server](#launching-a-development-web-server-during-the-tests) configuration for examples.
 
 Note that each [test project](#projects) can provide its own test suite options, for example two projects can run different tests by providing different `testDir`s. However, test run options are shared between all projects.
 
@@ -200,6 +201,73 @@ export const test = base.extend<{ saveLogs: void }>({
 });
 ```
 
+## Launching a development web server during the tests
+
+To launch a server during the tests, use the `webServer` option in the [configuration file](#configuration-object).
+
+You can specify a port via `port` or additional environment variables, see [here](#configuration-object). The server will wait for it to be available before running the tests. For continuous integration, you may want to use the `reuseExistingServer: !process.env.CI` option which does not use an existing server on the CI.
+
+The port gets then passed over to Playwright as a [`param: baseURL`] when creating the context [`method: Browser.newContext`].
+
+```js js-flavor=ts
+// playwright.config.ts
+import { PlaywrightTestConfig } from '@playwright/test';
+const config: PlaywrightTestConfig = {
+  webServer: {
+    command: 'npm run start',
+    port: 3000,
+    timeout: 120 * 1000,
+    reuseExistingServer: !process.env.CI,
+  },
+};
+export default config;
+```
+
+```js js-flavor=js
+// playwright.config.js
+// @ts-check
+/** @type {import('@playwright/test').PlaywrightTestConfig} */
+const config = {
+  webServer: {
+    command: 'npm run start',
+    port: 3000,
+    timeout: 120 * 1000,
+    reuseExistingServer: !process.env.CI,
+  },
+};
+mode.exports = config;
+```
+
+Now you can use a relative path when navigating the page, or use `baseURL` fixture:
+
+```js js-flavor=ts
+// test.spec.ts
+import { test } = from '@playwright/test';
+test('test', async ({ page, baseURL }) => {
+  // baseURL is taken directly from your web server,
+  // e.g. http://localhost:3000
+  await page.goto(baseURL + '/bar');
+  // Alternatively, just use relative path, because baseURL is already
+  // set for the default context and page.
+  // For example, this will result in http://localhost:3000/foo
+  await page.goto('/foo');
+});
+```
+
+```js js-flavor=js
+// test.spec.js
+const { test } = require('@playwright/test');
+test('test', async ({ page, baseURL }) => {
+  // baseURL is taken directly from your web server,
+  // e.g. http://localhost:3000
+  await page.goto(baseURL + '/bar');
+  // Alternatively, just use relative path, because baseURL is already
+  // set for the default context and page.
+  // For example, this will result in http://localhost:3000/foo
+  await page.goto('/foo');
+});
+```
+
 ## Global setup and teardown
 
 To set something up once before running all tests, use `globalSetup` option in the [configuration file](#configuration-object).

docs/src/test-configuration-js.md

@@ -478,6 +478,7 @@ In addition to configuring [Browser] or [BrowserContext], videos or screenshots,
 - `testIgnore`: Glob patterns or regular expressions that should be ignored when looking for the test files. For example, `'**/test-assets'`.
 - `testMatch`: Glob patterns or regular expressions that match test files. For example, `'**/todo-tests/*.spec.ts'`. By default, Playwright Test runs `.*(test|spec)\.(js|ts|mjs)` files.
 - `timeout`: Time in milliseconds given to each test.
+- `webServer: { command: string, port: number, timeout?: number, reuseExistingServer?: boolean, cwd?: string, env?: object }` - Launch a process and wait that it's ready before the tests will start. See [launch web server](./test-advanced.md#launching-a-development-web-server-during-the-tests) configuration for examples.
 - `workers`: The maximum number of concurrent worker processes to use for parallelizing tests.
 
 You can specify these options in the configuration file. Note that testing options are **top-level**, do not put them into the `use` section.

src/test/loader.ts

@@ -25,7 +25,6 @@ import * as url from 'url';
 import * as fs from 'fs';
 import { ProjectImpl } from './project';
 import { Reporter } from '../../types/testReporter';
-import { LaunchConfig } from '../../types/test';
 import { BuiltInReporter, builtInReporters } from './runner';
 
 export class Loader {
@@ -101,7 +100,7 @@ export class Loader {
     this._fullConfig.shard = takeFirst(this._configOverrides.shard, this._config.shard, baseFullConfig.shard);
     this._fullConfig.updateSnapshots = takeFirst(this._configOverrides.updateSnapshots, this._config.updateSnapshots, baseFullConfig.updateSnapshots);
     this._fullConfig.workers = takeFirst(this._configOverrides.workers, this._config.workers, baseFullConfig.workers);
-    this._fullConfig._launch = takeFirst(toLaunchServers(this._configOverrides._launch), toLaunchServers(this._config._launch), baseFullConfig._launch);
+    this._fullConfig.webServer = takeFirst(this._configOverrides.webServer, this._config.webServer, baseFullConfig.webServer);
 
     for (const project of projects)
       this._addProject(project, this._fullConfig.rootDir);
@@ -230,14 +229,6 @@ function toReporters(reporters: BuiltInReporter | ReporterDescription[] | undefi
   return reporters;
 }
 
-function toLaunchServers(launchConfigs?: LaunchConfig | LaunchConfig[]): LaunchConfig[]|undefined {
-  if (!launchConfigs)
-    return;
-  if (!Array.isArray(launchConfigs))
-    return [launchConfigs];
-  return launchConfigs;
-}
-
 function validateConfig(file: string, config: Config) {
   if (typeof config !== 'object' || !config)
     throw errorWithFile(file, `Configuration file must export a single object`);
@@ -435,7 +426,7 @@ const baseFullConfig: FullConfig = {
   shard: null,
   updateSnapshots: 'missing',
   workers: 1,
-  _launch: [],
+  webServer: null,
 };
 
 function resolveReporters(reporters: Config['reporter'], rootDir: string): ReporterDescription[]|undefined {

src/test/runner.ts

@@ -35,7 +35,7 @@ import EmptyReporter from './reporters/empty';
 import { ProjectImpl } from './project';
 import { Minimatch } from 'minimatch';
 import { Config, FullConfig } from './types';
-import { LaunchServers } from './launchServer';
+import { WebServer } from './webServer';
 
 const removeFolderAsync = promisify(rimraf);
 const readDirAsync = promisify(fs.readdir);
@@ -167,7 +167,7 @@ export class Runner {
       testFiles.forEach(file => allTestFiles.add(file));
     }
 
-    const launchServers = await LaunchServers.create(config._launch);
+    const webServer = config.webServer && await WebServer.create(config.webServer);
     let globalSetupResult: any;
     if (config.globalSetup)
       globalSetupResult = await (await this._loader.loadGlobalHook(config.globalSetup, 'globalSetup'))(this._loader.fullConfig());
@@ -316,7 +316,7 @@ export class Runner {
         await globalSetupResult(this._loader.fullConfig());
       if (config.globalTeardown)
         await (await this._loader.loadGlobalHook(config.globalTeardown, 'globalTeardown'))(this._loader.fullConfig());
-      await launchServers.killAll();
+      await webServer?.kill();
     }
   }
 }

src/test/webServer.ts

@@ -13,13 +13,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-/* eslint-disable no-console */
 
 import net from 'net';
 import os from 'os';
 import stream from 'stream';
 import { monotonicTime, raceAgainstDeadline } from './util';
-import { LaunchConfig } from '../../types/test';
+import { WebServerConfig } from '../../types/test';
 import { launchProcess } from '../utils/processLauncher';
 
 const DEFAULT_ENVIRONMENT_VARIABLES = {
@@ -33,19 +32,19 @@ const newProcessLogPrefixer = () => new stream.Transform({
   },
 });
 
-class LaunchServer {
+export class WebServer {
   private _killProcess?: () => Promise<void>;
   private _processExitedPromise!: Promise<any>;
-  constructor(private readonly config: LaunchConfig) { }
+  constructor(private readonly config: WebServerConfig) { }
 
-  public static async create(config: LaunchConfig): Promise<LaunchServer> {
+  public static async create(config: WebServerConfig): Promise<WebServer> {
-    const launchServer = new LaunchServer(config);
+    const webServer = new WebServer(config);
     try {
-      await launchServer._startProcess();
+      await webServer._startProcess();
-      await launchServer._waitForProcess();
+      await webServer._waitForProcess();
-      return launchServer;
+      return webServer;
     } catch (error) {
-      await launchServer.kill();
+      await webServer.kill();
       throw error;
     }
   }
@@ -54,15 +53,13 @@ class LaunchServer {
     let processExitedReject = (error: Error) => { };
     this._processExitedPromise = new Promise((_, reject) => processExitedReject = reject);
 
-    if (this.config.waitForPort) {
+    const portIsUsed = !await canBindPort(this.config.port);
-      const portIsUsed = !await canBindPort(this.config.waitForPort);
+    if (portIsUsed) {
-      if (portIsUsed && this.config.strict)
+      if (this.config.reuseExistingServer)
-        throw new Error(`Port ${this.config.waitForPort} is used, make sure that nothing is running on the port or set strict:false in config.launch.`);
-      if (portIsUsed)
         return;
+      throw new Error(`Port ${this.config.port} is used, make sure that nothing is running on the port or set strict:false in config.launch.`);
     }
 
-    console.log(`Launching '${this.config.command}'...`);
     const { launchedProcess, kill } = await launchProcess({
       command: this.config.command,
       env: {
@@ -85,19 +82,16 @@ class LaunchServer {
   }
 
   private async _waitForProcess() {
-    if (this.config.waitForPort) {
+    await this._waitForAvailability();
-      await this._waitForAvailability(this.config.waitForPort);
+    const baseURL = `http://localhost:${this.config.port}`;
-      const baseURL = `http://localhost:${this.config.waitForPort}`;
+    process.env.PLAYWRIGHT_TEST_BASE_URL = baseURL;
-      process.env.PLAYWRIGHT_TEST_BASE_URL = baseURL;
-      console.log(`Using baseURL '${baseURL}' from config.launch.`);
-    }
   }
 
-  private async _waitForAvailability(port: number) {
+  private async _waitForAvailability() {
-    const launchTimeout = this.config.waitForPortTimeout || 60 * 1000;
+    const launchTimeout = this.config.timeout || 60 * 1000;
     const cancellationToken = { canceled: false };
     const { timedOut } = (await Promise.race([
-      raceAgainstDeadline(waitForSocket(port, 100, cancellationToken), launchTimeout + monotonicTime()),
+      raceAgainstDeadline(waitForSocket(this.config.port, 100, cancellationToken), launchTimeout + monotonicTime()),
       this._processExitedPromise,
     ]));
     cancellationToken.canceled = true;
@@ -139,25 +133,3 @@ async function waitForSocket(port: number, delay: number, cancellationToken: { c
     await new Promise(x => setTimeout(x, delay));
   }
 }
-
-export class LaunchServers {
-  private readonly _servers: LaunchServer[] = [];
-
-  public static async create(configs: LaunchConfig[]): Promise<LaunchServers> {
-    const launchServers = new LaunchServers();
-    try {
-      for (const config of configs)
-        launchServers._servers.push(await LaunchServer.create(config));
-    } catch (error) {
-      for (const server of launchServers._servers)
-        await server.kill();
-      throw error;
-    }
-    return launchServers;
-  }
-
-  public async killAll() {
-    for (const server of this._servers)
-      await server.kill();
-  }
-}

tests/playwright-test/web-server.spec.ts

@@ -32,9 +32,9 @@ test('should create a server', async ({ runInlineTest }, { workerIndex }) => {
     `,
     'playwright.config.ts': `
       module.exports = {
-        _launch: {
+        webServer: {
           command: 'node ${JSON.stringify(path.join(__dirname, 'assets', 'simple-server.js'))} ${port}',
-          waitForPort: ${port},
+          port: ${port},
         },
         globalSetup: 'globalSetup.ts',
         globalTeardown: 'globalTeardown.ts',
@@ -42,8 +42,19 @@ test('should create a server', async ({ runInlineTest }, { workerIndex }) => {
     `,
     'globalSetup.ts': `
       module.exports = async () => {
-        console.log('globalSetup')
+        const http = require("http");
-        return () => console.log('globalSetup teardown');
+        const response = await new Promise(resolve => {
+          const request = http.request("http://localhost:${port}/hello", resolve);
+          request.end();
+        })
+        console.log('globalSetup-status-'+response.statusCode)
+        return async () => {
+          const response = await new Promise(resolve => {
+            const request = http.request("http://localhost:${port}/hello", resolve);
+            request.end();
+          })
+          console.log('globalSetup-teardown-status-'+response.statusCode)
+        };
       };
     `,
     'globalTeardown.ts': `
@@ -61,7 +72,7 @@ test('should create a server', async ({ runInlineTest }, { workerIndex }) => {
   expect(result.passed).toBe(1);
   expect(result.report.suites[0].specs[0].tests[0].results[0].status).toContain('passed');
 
-  const expectedLogMessages = ['Launching ', 'globalSetup', 'globalSetup teardown', 'globalTeardown-status-200'];
+  const expectedLogMessages = ['globalSetup-status-200', 'globalSetup-teardown-status', 'globalTeardown-status-200'];
   const actualLogMessages = expectedLogMessages.map(log => ({
     log,
     index: result.output.indexOf(log),
@@ -82,9 +93,9 @@ test('should create a server with environment variables', async ({ runInlineTest
     `,
     'playwright.config.ts': `
       module.exports = {
-        _launch: {
+        webServer: {
           command: 'node ${JSON.stringify(path.join(__dirname, 'assets', 'simple-server.js'))} ${port}',
-          waitForPort: ${port},
+          port: ${port},
           env: {
             'FOO': 'BAR',
           }
@@ -110,10 +121,10 @@ test('should time out waiting for a server', async ({ runInlineTest }, { workerI
     `,
     'playwright.config.ts': `
       module.exports = {
-        _launch: {
+        webServer: {
           command: 'node ${JSON.stringify(JSON.stringify(path.join(__dirname, 'assets', 'simple-server.js')))} ${port}',
-          waitForPort: ${port},
+          port: ${port},
-          waitForPortTimeout: 100,
+          timeout: 100,
         }
       };
     `,
@@ -151,7 +162,7 @@ test('should be able to specify the baseURL without the server', async ({ runInl
   await new Promise(resolve => server.close(resolve));
 });
 
-test('should be able to use an existing server when strict is false ', async ({ runInlineTest }, { workerIndex }) => {
+test('should be able to use an existing server when reuseExistingServer:true ', async ({ runInlineTest }, { workerIndex }) => {
   const port = workerIndex + 10500;
   const server = http.createServer((req: http.IncomingMessage, res: http.ServerResponse) => {
     res.end('<html><body>hello</body></html>');
@@ -169,10 +180,10 @@ test('should be able to use an existing server when strict is false ', async ({
     `,
     'playwright.config.ts': `
       module.exports = {
-        _launch: {
+        webServer: {
           command: 'node ${JSON.stringify(path.join(__dirname, 'assets', 'simple-server.js'))} ${port}',
-          waitForPort: ${port},
+          port: ${port},
-          strict: false,
+          reuseExistingServer: true,
         }
       };
     `,
@@ -202,10 +213,10 @@ test('should throw when a server is already running on the given port and strict
     `,
     'playwright.config.ts': `
       module.exports = {
-        _launch: {
+        webServer: {
           command: 'node ${JSON.stringify(path.join(__dirname, 'assets', 'simple-server.js'))} ${port}',
-          waitForPort: ${port},
+          port: ${port},
-          strict: true,
+          reuseExistingServer: false,
         }
       };
     `,
@@ -214,31 +225,3 @@ test('should throw when a server is already running on the given port and strict
   expect(result.output).toContain(`Port ${port} is used, make sure that nothing is running on the port`);
   await new Promise(resolve => server.close(resolve));
 });
-
-test('should create multiple servers', async ({ runInlineTest }, { workerIndex }) => {
-  const port1 = workerIndex + 10500;
-  const port2 = workerIndex + 10600;
-  const result = await runInlineTest({
-    'test.spec.ts': `
-      const { test } = pwt;
-      test('connect to the server via the baseURL', async ({baseURL, page}) => {
-        await page.goto('http://localhost:${port1}/hello');
-        await page.goto('http://localhost:${port2}/hello');
-      });
-    `,
-    'playwright.config.ts': `
-      module.exports = {
-        _launch: [{
-          command: 'node ${JSON.stringify(path.join(__dirname, 'assets', 'simple-server.js'))} ${port1}',
-          waitForPort: ${port1},
-        },{
-          command: 'node ${JSON.stringify(path.join(__dirname, 'assets', 'simple-server.js'))} ${port2}',
-          waitForPort: ${port2},
-        }],
-      };
-    `,
-  });
-  expect(result.exitCode).toBe(0);
-  expect(result.passed).toBe(1);
-  expect(result.report.suites[0].specs[0].tests[0].results[0].status).toContain('passed');
-});

types/test.d.ts

@@ -479,24 +479,26 @@ export interface Project<TestArgs = {}, WorkerArgs = {}> extends TestProject {
 
 export type FullProject<TestArgs = {}, WorkerArgs = {}> = Required<Project<TestArgs, WorkerArgs>>;
 
-export type LaunchConfig = {
+export type WebServerConfig = {
   /**
    * Shell command to start. For example `npm run start`.
    */
   command: string,
   /**
-   * The port that your http server is expected to appear on. If specified it does wait until it accepts connections.
+   * The port that your http server is expected to appear on. It does wait until it accepts connections.
    */
-  waitForPort?: number,
+  port: number,
   /**
    * How long to wait for the process to start up and be available in milliseconds. Defaults to 60000.
    */
-  waitForPortTimeout?: number,
+  timeout?: number,
   /**
-   * If true it will verify that the given port via `waitForPort` is available and throw otherwise.
+   * If true, it will re-use an existing server on the port when available. If no server is running
-   * This should commonly set to !!process.env.CI to allow the local dev server when running tests locally.
+   * on that port, it will run the command to start a new server.
+   * If false, it will throw if an existing process is listening on the port.
+   * This should commonly set to !process.env.CI to allow the local dev server when running tests locally.
    */
-  strict?: boolean
+  reuseExistingServer?: boolean
   /**
    * Environment variables, process.env by default
    */
@@ -690,7 +692,7 @@ interface TestConfig {
    * Learn more about [snapshots](https://playwright.dev/docs/test-snapshots).
    */
   updateSnapshots?: UpdateSnapshots;
-  _launch?: LaunchConfig | LaunchConfig[];
+  webServer?: WebServerConfig;
   /**
    * The maximum number of concurrent worker processes to use for parallelizing tests.
    *
@@ -1051,7 +1053,7 @@ export interface FullConfig {
    * Playwright Test.
    */
   workers: number;
-  _launch: LaunchConfig[];
+  webServer: WebServerConfig | null;
 }
 
 export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped';

utils/generate_types/overrides-test.d.ts

@@ -62,24 +62,26 @@ export interface Project<TestArgs = {}, WorkerArgs = {}> extends TestProject {
 
 export type FullProject<TestArgs = {}, WorkerArgs = {}> = Required<Project<TestArgs, WorkerArgs>>;
 
-export type LaunchConfig = {
+export type WebServerConfig = {
   /**
    * Shell command to start. For example `npm run start`.
    */
   command: string,
   /**
-   * The port that your http server is expected to appear on. If specified it does wait until it accepts connections.
+   * The port that your http server is expected to appear on. It does wait until it accepts connections.
    */
-  waitForPort?: number,
+  port: number,
   /**
    * How long to wait for the process to start up and be available in milliseconds. Defaults to 60000.
    */
-  waitForPortTimeout?: number,
+  timeout?: number,
   /**
-   * If true it will verify that the given port via `waitForPort` is available and throw otherwise.
+   * If true, it will re-use an existing server on the port when available. If no server is running
-   * This should commonly set to !!process.env.CI to allow the local dev server when running tests locally.
+   * on that port, it will run the command to start a new server.
+   * If false, it will throw if an existing process is listening on the port.
+   * This should commonly set to !process.env.CI to allow the local dev server when running tests locally.
    */
-  strict?: boolean
+  reuseExistingServer?: boolean
   /**
    * Environment variables, process.env by default
    */
@@ -107,7 +109,7 @@ interface TestConfig {
   reportSlowTests?: ReportSlowTests;
   shard?: Shard;
   updateSnapshots?: UpdateSnapshots;
-  _launch?: LaunchConfig | LaunchConfig[];
+  webServer?: WebServerConfig;
   workers?: number;
 
   expect?: ExpectSettings;
@@ -145,7 +147,7 @@ export interface FullConfig {
   shard: Shard;
   updateSnapshots: UpdateSnapshots;
   workers: number;
-  _launch: LaunchConfig[];
+  webServer: WebServerConfig | null;
 }
 
 export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped';