feat(test-runner): wait for a url before starting tests (#10138)

The webServer configuration in @playwright/test now accepts a url as an
alternative to a port number to wait for a url to return a 2xx status code.

docs/src/test-api/class-testconfig.md

@@ -559,7 +559,8 @@ export default config;
 ## property: TestConfig.webServer
 - type: <[Object]>
   - `command` <[string]> Command which gets executed
-  - `port` <[int]> Port to wait on for the web server
+  - `port` <[int]> Port to wait on for the web server (exactly one of `port` or `url` is required)
+  - `url` <[string]> URL to wait on for the web server (exactly one of `port` or `url` is required)
   - `timeout` <[int]> Maximum duration to wait on until the web server is ready
   - `reuseExistingServer` <[boolean]> If true, reuse the existing server if it is already running, otherwise it will fail
   - `cwd` <[boolean]> Working directory to run the command in
@@ -567,11 +568,10 @@ export default config;
 
 Launch a development web server during the tests.
 
-The server will wait for it to be available on `127.0.0.1` or `::1` before running the tests. For continuous integration, you may want to use the `reuseExistingServer: !process.env.CI` option which does not use an existing server on the CI. To see the stdout, you can set the `DEBUG=pw:webserver` environment variable.
+If the port is specified, the server will wait for it to be available on `127.0.0.1` or `::1`, before running the tests. If the url is specified, the server will wait for the URL to return a 2xx status code before running the tests. For continuous integration, you may want to use the `reuseExistingServer: !process.env.CI` option which does not use an existing server on the CI. To see the stdout, you can set the `DEBUG=pw:webserver` environment variable.
 
-The port gets then passed over to Playwright as a `baseURL` when creating the context [`method: Browser.newContext`].
+The port or url gets then passed over to Playwright as a `baseURL` when creating the context [`method: Browser.newContext`].
-For example `8080` ends up in `baseURL` to be `http://localhost:8080`. If you want to use `https://` you need to manually specify
+For example port `8080` ends up in `baseURL` to be `http://localhost:8080`. If you want to instead use `https://` you need to manually specify the `baseURL` inside `use` or use a url instead of a port in the `webServer` configuration. The url ends up in `baseURL` without any change.
-the `baseURL` inside `use`.
 
 ```js js-flavor=ts
 // playwright.config.ts

packages/playwright-test/src/webServer.ts

@@ -14,6 +14,8 @@
  * limitations under the License.
  */
 
+import http from 'http';
+import https from 'https';
 import net from 'net';
 import os from 'os';
 import stream from 'stream';
@@ -36,9 +38,12 @@ const newProcessLogPrefixer = () => new stream.Transform({
 const debugWebServer = debug('pw:webserver');
 
 export class WebServer {
+  private _isAvailable: () => Promise<boolean>;
   private _killProcess?: () => Promise<void>;
   private _processExitedPromise!: Promise<any>;
-  constructor(private readonly config: WebServerConfig) { }
+  constructor(private readonly config: WebServerConfig) {
+    this._isAvailable = getIsAvailableFunction(config);
+  }
 
   public static async create(config: WebServerConfig): Promise<WebServer> {
     const webServer = new WebServer(config);
@@ -56,11 +61,11 @@ export class WebServer {
     let processExitedReject = (error: Error) => { };
     this._processExitedPromise = new Promise((_, reject) => processExitedReject = reject);
 
-    const portIsUsed = await isPortUsed(this.config.port);
+    const isAlreadyAvailable = await this._isAvailable();
-    if (portIsUsed) {
+    if (isAlreadyAvailable) {
       if (this.config.reuseExistingServer)
         return;
-      throw new Error(`Port ${this.config.port} is used, make sure that nothing is running on the port or set strict:false in config.webServer.`);
+      throw new Error(`${this.config.url ?? `http://localhost:${this.config.port}`} is already used, make sure that nothing is running on the port/url or set strict:false in config.webServer.`);
     }
 
     const { launchedProcess, kill } = await launchProcess({
@@ -86,7 +91,7 @@ export class WebServer {
 
   private async _waitForProcess() {
     await this._waitForAvailability();
-    const baseURL = `http://localhost:${this.config.port}`;
+    const baseURL = this.config.url ?? `http://localhost:${this.config.port}`;
     process.env.PLAYWRIGHT_TEST_BASE_URL = baseURL;
   }
 
@@ -94,7 +99,7 @@ export class WebServer {
     const launchTimeout = this.config.timeout || 60 * 1000;
     const cancellationToken = { canceled: false };
     const { timedOut } = (await Promise.race([
-      raceAgainstTimeout(() => waitForSocket(this.config.port, 100, cancellationToken), launchTimeout),
+      raceAgainstTimeout(() => waitFor(this._isAvailable, 100, cancellationToken), launchTimeout),
       this._processExitedPromise,
     ]));
     cancellationToken.canceled = true;
@@ -121,11 +126,34 @@ async function isPortUsed(port: number): Promise<boolean> {
   return await innerIsPortUsed('127.0.0.1') || await innerIsPortUsed('::1');
 }
 
-async function waitForSocket(port: number, delay: number, cancellationToken: { canceled: boolean }) {
+async function isURLAvailable(url: URL) {
+  return new Promise<boolean>(resolve => {
+    (url.protocol === 'https:' ? https : http).get(url, res => {
+      res.resume();
+      const statusCode = res.statusCode ?? 0;
+      resolve(statusCode >= 200 && statusCode < 300);
+    }).on('error', () => {
+      resolve(false);
+    });
+  });
+}
+
+async function waitFor(waitFn: () => Promise<boolean>, delay: number, cancellationToken: { canceled: boolean }) {
   while (!cancellationToken.canceled) {
-    const connected = await isPortUsed(port);
+    const connected = await waitFn();
     if (connected)
       return;
     await new Promise(x => setTimeout(x, delay));
   }
 }
+
+function getIsAvailableFunction({ url, port }: Pick<WebServerConfig, 'port' | 'url'>) {
+  if (url && typeof port === 'undefined') {
+    const urlObject = new URL(url);
+    return () => isURLAvailable(urlObject);
+  } else if (port && typeof url === 'undefined') {
+    return () => isPortUsed(port);
+  } else {
+    throw new Error(`Exactly one of 'port' or 'url' is required in config.webServer.`);
+  }
+}

packages/playwright-test/types/test.d.ts

@@ -355,16 +355,22 @@ export type WebServerConfig = {
   command: string,
   /**
    * The port that your http server is expected to appear on. It does wait until it accepts connections.
+   * Exactly one of `port` or `url` is required.
    */
-  port: number,
+  port?: number,
+  /**
+   * The url on your http server that is expected to return a 2xx status code when the server is ready to accept connections.
+   * Exactly one of `port` or `url` is required.
+   */
+  url?: string,
   /**
    * How long to wait for the process to start up and be available in milliseconds. Defaults to 60000.
    */
   timeout?: number,
   /**
-   * If true, it will re-use an existing server on the port when available. If no server is running
+   * If true, it will re-use an existing server on the port or url when available. If no server is running
-   * on that port, it will run the command to start a new server.
+   * on that port or url, it will run the command to start a new server.
-   * If false, it will throw if an existing process is listening on the port.
+   * If false, it will throw if an existing process is listening on the port or url.
    * This should commonly set to !process.env.CI to allow the local dev server when running tests locally.
    */
   reuseExistingServer?: boolean
@@ -570,14 +576,16 @@ interface TestConfig {
   /**
    * Launch a development web server during the tests.
    *
-   * The server will wait for it to be available on `127.0.0.1` or `::1` before running the tests. For continuous
+   * If the port is specified, the server will wait for it to be available on `127.0.0.1` or `::1`, before running the tests.
-   * integration, you may want to use the `reuseExistingServer: !process.env.CI` option which does not use an existing server
+   * If the url is specified, the server will wait for the URL to return a 2xx status code before running the tests. For
-   * on the CI. To see the stdout, you can set the `DEBUG=pw:webserver` environment variable.
+   * continuous integration, you may want to use the `reuseExistingServer: !process.env.CI` option which does not use an
+   * existing server on the CI. To see the stdout, you can set the `DEBUG=pw:webserver` environment variable.
    *
-   * The port gets then passed over to Playwright as a `baseURL` when creating the context
+   * The port or url gets then passed over to Playwright as a `baseURL` when creating the context
-   * [browser.newContext([options])](https://playwright.dev/docs/api/class-browser#browser-new-context). For example `8080`
+   * [browser.newContext([options])](https://playwright.dev/docs/api/class-browser#browser-new-context). For example port
-   * ends up in `baseURL` to be `http://localhost:8080`. If you want to use `https://` you need to manually specify the
+   * `8080` ends up in `baseURL` to be `http://localhost:8080`. If you want to instead use `https://` you need to manually
-   * `baseURL` inside `use`.
+   * specify the `baseURL` inside `use` or use a url instead of a port in the `webServer` configuration. The url ends up in
+   * `baseURL` without any change.
    *
    * ```ts
    * // playwright.config.ts
@@ -1059,14 +1067,16 @@ export interface FullConfig<TestArgs = {}, WorkerArgs = {}> {
   /**
    * Launch a development web server during the tests.
    *
-   * The server will wait for it to be available on `127.0.0.1` or `::1` before running the tests. For continuous
+   * If the port is specified, the server will wait for it to be available on `127.0.0.1` or `::1`, before running the tests.
-   * integration, you may want to use the `reuseExistingServer: !process.env.CI` option which does not use an existing server
+   * If the url is specified, the server will wait for the URL to return a 2xx status code before running the tests. For
-   * on the CI. To see the stdout, you can set the `DEBUG=pw:webserver` environment variable.
+   * continuous integration, you may want to use the `reuseExistingServer: !process.env.CI` option which does not use an
+   * existing server on the CI. To see the stdout, you can set the `DEBUG=pw:webserver` environment variable.
    *
-   * The port gets then passed over to Playwright as a `baseURL` when creating the context
+   * The port or url gets then passed over to Playwright as a `baseURL` when creating the context
-   * [browser.newContext([options])](https://playwright.dev/docs/api/class-browser#browser-new-context). For example `8080`
+   * [browser.newContext([options])](https://playwright.dev/docs/api/class-browser#browser-new-context). For example port
-   * ends up in `baseURL` to be `http://localhost:8080`. If you want to use `https://` you need to manually specify the
+   * `8080` ends up in `baseURL` to be `http://localhost:8080`. If you want to instead use `https://` you need to manually
-   * `baseURL` inside `use`.
+   * specify the `baseURL` inside `use` or use a url instead of a port in the `webServer` configuration. The url ends up in
+   * `baseURL` without any change.
    *
    * ```ts
    * // playwright.config.ts

tests/playwright-test/assets/simple-server-with-ready-route.js

@@ -0,0 +1,15 @@
+const { TestServer } = require('../../../utils/testserver/');
+TestServer.create(__dirname, process.argv[2] || 3000).then(server => {
+  console.log('listening on port', server.PORT);
+  let ready = false;
+  setTimeout(() => ready = true, 750);
+  server.setRoute('/ready', (message, response) => {
+    if (ready) {
+      response.statusCode = 200;
+      response.end('hello');
+    } else {
+      response.statusCode = 404;
+      response.end('not-ready');
+    }
+  });
+});

tests/playwright-test/web-server.spec.ts

@@ -108,6 +108,31 @@ test('should create a server with environment variables', async ({ runInlineTest
   expect(result.report.suites[0].specs[0].tests[0].results[0].status).toContain('passed');
 });
 
+test('should create a server with url', async ({ runInlineTest }, { workerIndex }) => {
+  const port = workerIndex + 10500;
+  const result = await runInlineTest({
+    'test.spec.ts': `
+      const { test } = pwt;
+      test('connect to the server', async ({baseURL, page}) => {
+        expect(baseURL).toBe('http://localhost:${port}/ready');
+        await page.goto(baseURL);
+        expect(await page.textContent('body')).toBe('hello');
+      });
+    `,
+    'playwright.config.ts': `
+      module.exports = {
+        webServer: {
+          command: 'node ${JSON.stringify(path.join(__dirname, 'assets', 'simple-server-with-ready-route.js'))} ${port}',
+          url: 'http://localhost:${port}/ready'
+        }
+      };
+    `,
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+  expect(result.report.suites[0].specs[0].tests[0].results[0].status).toContain('passed');
+});
+
 test('should time out waiting for a server', async ({ runInlineTest }, { workerIndex }) => {
   const port = workerIndex + 10500;
   const result = await runInlineTest({
@@ -133,6 +158,31 @@ test('should time out waiting for a server', async ({ runInlineTest }, { workerI
   expect(result.output).toContain(`Timed out waiting 100ms from config.webServer.`);
 });
 
+test('should time out waiting for a server with url', async ({ runInlineTest }, { workerIndex }) => {
+  const port = workerIndex + 10500;
+  const result = await runInlineTest({
+    'test.spec.ts': `
+      const { test } = pwt;
+      test('connect to the server', async ({baseURL, page}) => {
+        expect(baseURL).toBe('http://localhost:${port}/ready');
+        await page.goto(baseURL);
+        expect(await page.textContent('body')).toBe('hello');
+      });
+    `,
+    'playwright.config.ts': `
+      module.exports = {
+        webServer: {
+          command: 'node ${JSON.stringify(path.join(__dirname, 'assets', 'simple-server-with-ready-route.js'))} ${port}',
+          url: 'http://localhost:${port}/ready',
+          timeout: 300,
+        }
+      };
+    `,
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain(`Timed out waiting 300ms from config.webServer.`);
+});
+
 test('should be able to specify the baseURL without the server', async ({ runInlineTest }, { workerIndex }) => {
   const port = workerIndex + 10500;
   const server = http.createServer((req: http.IncomingMessage, res: http.ServerResponse) => {
@@ -256,7 +306,7 @@ test('should throw when a server is already running on the given port and strict
     `,
   });
   expect(result.exitCode).toBe(1);
-  expect(result.output).toContain(`Port ${port} is used, make sure that nothing is running on the port`);
+  expect(result.output).toContain(`http://localhost:${port} is already used, make sure that nothing is running on the port/url`);
   await new Promise(resolve => server.close(resolve));
 });
 
@@ -287,7 +337,7 @@ for (const host of ['localhost', '127.0.0.1', '0.0.0.0']) {
     `,
       });
       expect(result.exitCode).toBe(1);
-      expect(result.output).toContain(`Port ${port} is used, make sure that nothing is running on the port`);
+      expect(result.output).toContain(`http://localhost:${port} is already used, make sure that nothing is running on the port/url`);
     } finally {
       await new Promise(resolve => server.close(resolve));
     }

utils/generate_types/overrides-test.d.ts

@@ -73,16 +73,22 @@ export type WebServerConfig = {
   command: string,
   /**
    * The port that your http server is expected to appear on. It does wait until it accepts connections.
+   * Exactly one of `port` or `url` is required.
    */
-  port: number,
+  port?: number,
+  /**
+   * The url on your http server that is expected to return a 2xx status code when the server is ready to accept connections.
+   * Exactly one of `port` or `url` is required.
+   */
+  url?: string,
   /**
    * How long to wait for the process to start up and be available in milliseconds. Defaults to 60000.
    */
   timeout?: number,
   /**
-   * If true, it will re-use an existing server on the port when available. If no server is running
+   * If true, it will re-use an existing server on the port or url when available. If no server is running
-   * on that port, it will run the command to start a new server.
+   * on that port or url, it will run the command to start a new server.
-   * If false, it will throw if an existing process is listening on the port.
+   * If false, it will throw if an existing process is listening on the port or url.
    * This should commonly set to !process.env.CI to allow the local dev server when running tests locally.
    */
   reuseExistingServer?: boolean