diff --git a/docs/src/test-advanced.md b/docs/src/test-advanced.md index 36aa61688d..423925e4b6 100644 --- a/docs/src/test-advanced.md +++ b/docs/src/test-advanced.md @@ -42,7 +42,7 @@ These options would be typically different between local development and CI oper - `reportSlowTests: { max: number, threshold: number } | null` - Whether to report slow tests. When `null`, slow tests are not reported. Otherwise, tests that took more than `threshold` milliseconds are reported as slow, but no more than `max` number of them. Passing zero as `max` reports all slow tests that exceed the threshold. - `shard: { total: number, current: number } | null` - [Shard](./test-parallel.md#shards) information. - `updateSnapshots: boolean` - Whether to update expected snapshots with the actual results produced by the test run. -- `webServer: { command: string, port?: number, cwd?: string, timeout?: number, env?: object }` - Launch a web server before the tests will start. It will automaticially detect the port when it got printed to the stdout. +- `launch: { command: string, waitForPort?: number, waitForPortTimeout?: number, strict?: boolean, cwd?: string, env?: object }[]` - Launch a process before the tests will start. When using `waitForPort` it will wait until the server is available, see [launch server](#launching-a-development-web-server-during-the-tests) configuration for examples. `strict` will verify that the `waitForPort` port is available instead of using it by default. - `workers: number` - The maximum number of concurrent worker processes to use for parallelizing tests. Note that each [test project](#projects) can provide its own test suite options, for example two projects can run different tests by providing different `testDir`s. However, test run options are shared between all projects. @@ -203,20 +203,22 @@ export const test = base.extend<{ saveLogs: void }>({ ## Launching a development web server during the tests -To launch a web server during the tests, use the `webServer` option in the [configuration file](#configuration-object). +To launch a server during the tests, use the `launch` option in the [configuration file](#configuration-object). -Playwright Test does automatically detect if a localhost URL like `http://localhost:3000` gets printed to the stdout. -The port from the printed URL gets then used to check when its accepting requests and passed over to Playwright as a -[`param: baseURL`] when creating the context [`method: Browser.newContext`]. You can also manually specify a `port` or additional environment variables, see [here](#configuration-object). +You can specify a port via `waitForPort` or additional environment variables, see [here](#configuration-object). When a port is specified, the server will wait for it to be available before starting. For continuous integration, you may want to use the `strict` option which ensures that the port is available before starting the server. + +The port gets then passed over to Playwright as a [`param: baseURL`] when creating the context [`method: Browser.newContext`]. ```js js-flavor=ts // playwright.config.ts import { PlaywrightTestConfig } from '@playwright/test'; const config: PlaywrightTestConfig = { - webServer: { + launch: { command: 'npm run start', - timeout: 120 * 1000, + waitForPort: 3000, + waitForPortTimeout: 120 * 1000, + strict: !!process.env.CI, }, }; @@ -228,9 +230,11 @@ export default config; // @ts-check /** @type {import('@playwright/test').PlaywrightTestConfig} */ const config = { - webServer: { + launch: { command: 'npm run start', - timeout: 120 * 1000, + waitForPort: 3000, + waitForPortTimeout: 120 * 1000, + strict: !!process.env.CI, }, }; diff --git a/docs/src/test-configuration.md b/docs/src/test-configuration.md index 02f335ef2f..3cb984a7b9 100644 --- a/docs/src/test-configuration.md +++ b/docs/src/test-configuration.md @@ -478,7 +478,7 @@ In addition to configuring [Browser] or [BrowserContext], videos or screenshots, - `testIgnore`: Glob patterns or regular expressions that should be ignored when looking for the test files. For example, `'**/test-assets'`. - `testMatch`: Glob patterns or regular expressions that match test files. For example, `'**/todo-tests/*.spec.ts'`. By default, Playwright Test runs `.*(test|spec)\.(js|ts|mjs)` files. - `timeout`: Time in milliseconds given to each test. -- `webServer: { command: string, port?: number, cwd?: string, timeout?: number, env?: object }` - Launch a web server before the tests will start. It will automatically detect the port when it got printed to the stdout. +- `launch: { command: string, waitForPort?: number, waitForPortTimeout?: number, strict?: boolean, cwd?: string, env?: object }` - Launch a process before the tests will start. When using `waitForPort` it will wait until the server is available, see [launch server](./test-advanced.md#launching-a-development-web-server-during-the-tests) configuration for examples. `strict` will verify that the `waitForPort` port is available instead of using it by default. - `workers`: The maximum number of concurrent worker processes to use for parallelizing tests. You can specify these options in the configuration file. Note that testing options are **top-level**, do not put them into the `use` section. diff --git a/src/test/webServer.ts b/src/test/launchServer.ts similarity index 56% rename from src/test/webServer.ts rename to src/test/launchServer.ts index f1f0a889ac..4cd2d06839 100644 --- a/src/test/webServer.ts +++ b/src/test/launchServer.ts @@ -19,7 +19,7 @@ import net from 'net'; import os from 'os'; import stream from 'stream'; import { monotonicTime, raceAgainstDeadline } from './util'; -import { WebServerConfig } from '../../types/test'; +import { LaunchConfig } from '../../types/test'; import { launchProcess } from '../utils/processLauncher'; const DEFAULT_ENVIRONMENT_VARIABLES = { @@ -28,57 +28,41 @@ const DEFAULT_ENVIRONMENT_VARIABLES = { const newProcessLogPrefixer = () => new stream.Transform({ transform(this: stream.Transform, chunk: Buffer, encoding: string, callback: stream.TransformCallback) { - this.push(chunk.toString().split(os.EOL).map((line: string): string => line ? `[WebServer] ${line}` : line).join(os.EOL)); + this.push(chunk.toString().split(os.EOL).map((line: string): string => line ? `[Launch] ${line}` : line).join(os.EOL)); callback(); }, }); -export class WebServer { +class LaunchServer { private _killProcess?: () => Promise; private _processExitedPromise!: Promise; - constructor(private readonly config: WebServerConfig) { } + constructor(private readonly config: LaunchConfig) { } - public static async create(config: WebServerConfig): Promise { - const webServer = new WebServer(config); - if (config.port) - await webServer._verifyFreePort(config.port); + public static async create(config: LaunchConfig): Promise { + const launchServer = new LaunchServer(config); try { - const port = await webServer._startWebServer(); - await webServer._waitForAvailability(port); - const baseURL = `http://localhost:${port}`; - process.env.PLAYWRIGHT_TEST_BASE_URL = baseURL; - console.log(`Using WebServer at '${baseURL}'.`); - return webServer; + await launchServer._startProcess(); + await launchServer._waitForProcess(); + return launchServer; } catch (error) { - await webServer.kill(); + await launchServer.kill(); throw error; } } - private async _verifyFreePort(port: number) { - const cancellationToken = { canceled: false }; - const portIsUsed = await Promise.race([ - new Promise(resolve => setTimeout(() => resolve(false), 100)), - waitForSocket(port, 100, cancellationToken), - ]); - cancellationToken.canceled = true; - if (portIsUsed) - throw new Error(`Port ${port} is used, make sure that nothing is running on the port`); - } - - private async _startWebServer(): Promise { - let collectPortResolve = (port: number) => { }; - const collectPortPromise = new Promise(resolve => collectPortResolve = resolve); - function collectPort(data: Buffer) { - const regExp = /http:\/\/localhost:(\d+)/.exec(data.toString()); - if (regExp) - collectPortResolve(parseInt(regExp[1], 10)); - } - + private async _startProcess(): Promise { let processExitedReject = (error: Error) => { }; this._processExitedPromise = new Promise((_, reject) => processExitedReject = reject); - console.log(`Starting WebServer with '${this.config.command}'...`); + if (this.config.waitForPort) { + const portIsUsed = !await canBindPort(this.config.waitForPort); + if (portIsUsed && this.config.strict) + throw new Error(`Port ${this.config.waitForPort} is used, make sure that nothing is running on the port or set strict:false in config.launch.`); + if (portIsUsed) + return; + } + + console.log(`Launching '${this.config.command}'...`); const { launchedProcess, kill } = await launchProcess({ command: this.config.command, env: { @@ -91,26 +75,26 @@ export class WebServer { shell: true, attemptToGracefullyClose: async () => {}, log: () => {}, - onExit: code => processExitedReject(new Error(`WebServer was not able to start. Exit code: ${code}`)), + onExit: code => processExitedReject(new Error(`Process from config.launch was not able to start. Exit code: ${code}`)), tempDirectories: [], }); this._killProcess = kill; launchedProcess.stderr.pipe(newProcessLogPrefixer()).pipe(process.stderr); launchedProcess.stdout.on('data', () => {}); + } - if (this.config.port) - return this.config.port; - launchedProcess.stdout.on('data', collectPort); - const detectedPort = await Promise.race([ - this._processExitedPromise, - collectPortPromise, - ]); - return detectedPort; + private async _waitForProcess() { + if (this.config.waitForPort) { + await this._waitForAvailability(this.config.waitForPort); + const baseURL = `http://localhost:${this.config.waitForPort}`; + process.env.PLAYWRIGHT_TEST_BASE_URL = baseURL; + console.log(`Using baseURL '${baseURL}' from config.launch.`); + } } private async _waitForAvailability(port: number) { - const launchTimeout = this.config.timeout || 60 * 1000; + const launchTimeout = this.config.waitForPortTimeout || 60 * 1000; const cancellationToken = { canceled: false }; const { timedOut } = (await Promise.race([ raceAgainstDeadline(waitForSocket(port, 100, cancellationToken), launchTimeout + monotonicTime()), @@ -118,13 +102,25 @@ export class WebServer { ])); cancellationToken.canceled = true; if (timedOut) - throw new Error(`Timed out waiting ${launchTimeout}ms for WebServer"`); + throw new Error(`Timed out waiting ${launchTimeout}ms from config.launch.`); } public async kill() { await this._killProcess?.(); } } +async function canBindPort(port: number): Promise { + return new Promise(resolve => { + const server = net.createServer(); + server.on('error', () => resolve(false)); + server.listen(port, () => { + server.close(() => { + resolve(true); + }); + }); + }); +} + async function waitForSocket(port: number, delay: number, cancellationToken: { canceled: boolean }) { while (!cancellationToken.canceled) { const connected = await new Promise(resolve => { @@ -143,3 +139,25 @@ async function waitForSocket(port: number, delay: number, cancellationToken: { c await new Promise(x => setTimeout(x, delay)); } } + +export class LaunchServers { + private readonly _servers: LaunchServer[] = []; + + public static async create(configs: LaunchConfig[]): Promise { + const launchServers = new LaunchServers(); + try { + for (const config of configs) + launchServers._servers.push(await LaunchServer.create(config)); + } catch (error) { + for (const server of launchServers._servers) + await server.kill(); + throw error; + } + return launchServers; + } + + public async killAll() { + for (const server of this._servers) + await server.kill(); + } +} diff --git a/src/test/loader.ts b/src/test/loader.ts index bfc24291c6..a01c071bae 100644 --- a/src/test/loader.ts +++ b/src/test/loader.ts @@ -24,6 +24,7 @@ import * as path from 'path'; import * as url from 'url'; import { ProjectImpl } from './project'; import { Reporter } from './reporter'; +import { LaunchConfig } from '../../types/test'; export class Loader { private _defaultConfig: Config; @@ -98,7 +99,7 @@ export class Loader { this._fullConfig.shard = takeFirst(this._configOverrides.shard, this._config.shard, baseFullConfig.shard); this._fullConfig.updateSnapshots = takeFirst(this._configOverrides.updateSnapshots, this._config.updateSnapshots, baseFullConfig.updateSnapshots); this._fullConfig.workers = takeFirst(this._configOverrides.workers, this._config.workers, baseFullConfig.workers); - this._fullConfig.webServer = takeFirst(this._configOverrides.webServer, this._config.webServer, baseFullConfig.webServer); + this._fullConfig.launch = takeFirst(toLaunchServers(this._configOverrides.launch), toLaunchServers(this._config.launch), baseFullConfig.launch); for (const project of projects) this._addProject(project, this._fullConfig.rootDir); @@ -227,6 +228,14 @@ function toReporters(reporters: 'dot' | 'line' | 'list' | 'junit' | 'json' | 'nu return reporters; } +function toLaunchServers(launchConfigs?: LaunchConfig | LaunchConfig[]): LaunchConfig[]|undefined { + if (!launchConfigs) + return; + if (!Array.isArray(launchConfigs)) + return [launchConfigs]; + return launchConfigs; +} + function errorWithFile(file: string, message: string) { return new Error(`${file}: ${message}`); } @@ -430,5 +439,5 @@ const baseFullConfig: FullConfig = { shard: null, updateSnapshots: 'missing', workers: 1, - webServer: null, + launch: [], }; diff --git a/src/test/runner.ts b/src/test/runner.ts index 86646f8c31..867a246b47 100644 --- a/src/test/runner.ts +++ b/src/test/runner.ts @@ -35,7 +35,7 @@ import EmptyReporter from './reporters/empty'; import { ProjectImpl } from './project'; import { Minimatch } from 'minimatch'; import { Config } from './types'; -import { WebServer } from './webServer'; +import { LaunchServers } from './launchServer'; const removeFolderAsync = promisify(rimraf); const readDirAsync = promisify(fs.readdir); @@ -167,7 +167,7 @@ export class Runner { testFiles.forEach(file => allTestFiles.add(file)); } - const webServer: WebServer|null = config.webServer ? await WebServer.create(config.webServer) : null; + const launchServers = await LaunchServers.create(config.launch); let globalSetupResult: any; if (config.globalSetup) globalSetupResult = await (await this._loader.loadGlobalHook(config.globalSetup, 'globalSetup'))(this._loader.fullConfig()); @@ -267,7 +267,7 @@ export class Runner { await globalSetupResult(this._loader.fullConfig()); if (config.globalTeardown) await (await this._loader.loadGlobalHook(config.globalTeardown, 'globalTeardown'))(this._loader.fullConfig()); - await webServer?.kill(); + await launchServers.killAll(); } } } diff --git a/tests/playwright-test/web-server.spec.ts b/tests/playwright-test/launch-server.spec.ts similarity index 61% rename from tests/playwright-test/web-server.spec.ts rename to tests/playwright-test/launch-server.spec.ts index c00d697545..622aecd022 100644 --- a/tests/playwright-test/web-server.spec.ts +++ b/tests/playwright-test/launch-server.spec.ts @@ -32,9 +32,9 @@ test('should create a server', async ({ runInlineTest }, { workerIndex }) => { `, 'playwright.config.ts': ` module.exports = { - webServer: { + launch: { command: 'node ${JSON.stringify(path.join(__dirname, 'assets', 'simple-server.js'))} ${port}', - port: ${port}, + waitForPort: ${port}, }, globalSetup: 'globalSetup.ts', globalTeardown: 'globalTeardown.ts', @@ -61,7 +61,7 @@ test('should create a server', async ({ runInlineTest }, { workerIndex }) => { expect(result.passed).toBe(1); expect(result.report.suites[0].specs[0].tests[0].results[0].status).toContain('passed'); - const expectedLogMessages = ['Starting WebServer', 'globalSetup', 'globalSetup teardown', 'globalTeardown-status-200']; + const expectedLogMessages = ['Launching ', 'globalSetup', 'globalSetup teardown', 'globalTeardown-status-200']; const actualLogMessages = expectedLogMessages.map(log => ({ log, index: result.output.indexOf(log), @@ -82,9 +82,9 @@ test('should create a server with environment variables', async ({ runInlineTest `, 'playwright.config.ts': ` module.exports = { - webServer: { + launch: { command: 'node ${JSON.stringify(path.join(__dirname, 'assets', 'simple-server.js'))} ${port}', - port: ${port}, + waitForPort: ${port}, env: { 'FOO': 'BAR', } @@ -110,40 +110,16 @@ test('should time out waiting for a server', async ({ runInlineTest }, { workerI `, 'playwright.config.ts': ` module.exports = { - webServer: { + launch: { command: 'node ${JSON.stringify(JSON.stringify(path.join(__dirname, 'assets', 'simple-server.js')))} ${port}', - port: ${port}, - timeout: 100, + waitForPort: ${port}, + waitForPortTimeout: 100, } }; `, }); expect(result.exitCode).toBe(1); - expect(result.output).toContain(`Timed out waiting 100ms for WebServer`); -}); - -test('should be able to detect the port from the process stdout', async ({ runInlineTest }, { workerIndex }) => { - const port = workerIndex + 10500; - const result = await runInlineTest({ - 'test.spec.ts': ` - const { test } = pwt; - test('connect to the server', async ({baseURL, page}) => { - expect(baseURL).toBe('http://localhost:${port}'); - await page.goto(baseURL + '/hello'); - expect(await page.textContent('body')).toBe('hello'); - }); - `, - 'playwright.config.ts': ` - module.exports = { - webServer: { - command: 'node ${JSON.stringify(path.join(__dirname, 'assets', 'simple-server-with-stdout.js'))} ${port}', - } - }; - `, - }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(1); - expect(result.report.suites[0].specs[0].tests[0].results[0].status).toContain('passed'); + expect(result.output).toContain(`Timed out waiting 100ms from config.launch.`); }); test('should be able to specify the baseURL without the server', async ({ runInlineTest }, { workerIndex }) => { @@ -172,5 +148,97 @@ test('should be able to specify the baseURL without the server', async ({ runInl expect(result.exitCode).toBe(0); expect(result.passed).toBe(1); expect(result.report.suites[0].specs[0].tests[0].results[0].status).toContain('passed'); - server.close(); + await new Promise(resolve => server.close(resolve)); +}); + +test('should be able to use an existing server when strict is false ', async ({ runInlineTest }, { workerIndex }) => { + const port = workerIndex + 10500; + const server = http.createServer((req: http.IncomingMessage, res: http.ServerResponse) => { + res.end('hello'); + }); + await new Promise(resolve => server.listen(port, resolve)); + const result = await runInlineTest({ + 'test.spec.ts': ` + const { test } = pwt; + test('connect to the server via the baseURL', async ({baseURL, page}) => { + await page.goto('/hello'); + await page.waitForURL('/hello'); + expect(page.url()).toBe('http://localhost:${port}/hello'); + expect(await page.textContent('body')).toBe('hello'); + }); + `, + 'playwright.config.ts': ` + module.exports = { + launch: { + command: 'node ${JSON.stringify(path.join(__dirname, 'assets', 'simple-server.js'))} ${port}', + waitForPort: ${port}, + strict: false, + } + }; + `, + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + expect(result.output).not.toContain('[Launch] '); + expect(result.report.suites[0].specs[0].tests[0].results[0].status).toContain('passed'); + await new Promise(resolve => server.close(resolve)); +}); + +test('should throw when a server is already running on the given port and strict is true ', async ({ runInlineTest }, { workerIndex }) => { + const port = workerIndex + 10500; + const server = http.createServer((req: http.IncomingMessage, res: http.ServerResponse) => { + res.end('hello'); + }); + await new Promise(resolve => server.listen(port, resolve)); + const result = await runInlineTest({ + 'test.spec.ts': ` + const { test } = pwt; + test('connect to the server via the baseURL', async ({baseURL, page}) => { + await page.goto('/hello'); + await page.waitForURL('/hello'); + expect(page.url()).toBe('http://localhost:${port}/hello'); + expect(await page.textContent('body')).toBe('hello'); + }); + `, + 'playwright.config.ts': ` + module.exports = { + launch: { + command: 'node ${JSON.stringify(path.join(__dirname, 'assets', 'simple-server.js'))} ${port}', + waitForPort: ${port}, + strict: true, + } + }; + `, + }); + expect(result.exitCode).toBe(1); + expect(result.output).toContain(`Port ${port} is used, make sure that nothing is running on the port`); + await new Promise(resolve => server.close(resolve)); +}); + +test('should create multiple servers', async ({ runInlineTest }, { workerIndex }) => { + const port1 = workerIndex + 10500; + const port2 = workerIndex + 10600; + const result = await runInlineTest({ + 'test.spec.ts': ` + const { test } = pwt; + test('connect to the server via the baseURL', async ({baseURL, page}) => { + await page.goto('http://localhost:${port1}/hello'); + await page.goto('http://localhost:${port2}/hello'); + }); + `, + 'playwright.config.ts': ` + module.exports = { + launch: [{ + command: 'node ${JSON.stringify(path.join(__dirname, 'assets', 'simple-server.js'))} ${port1}', + waitForPort: ${port1}, + },{ + command: 'node ${JSON.stringify(path.join(__dirname, 'assets', 'simple-server.js'))} ${port2}', + waitForPort: ${port2}, + }], + }; + `, + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + expect(result.report.suites[0].specs[0].tests[0].results[0].status).toContain('passed'); }); diff --git a/types/test.d.ts b/types/test.d.ts index 819a555938..c60b345f46 100644 --- a/types/test.d.ts +++ b/types/test.d.ts @@ -118,28 +118,32 @@ export interface Project extends ProjectBase { export type FullProject = Required>; -export type WebServerConfig = { +export type LaunchConfig = { /** - * Shell command to start the webserver. For example `npm run start`. + * Shell command to start. For example `npm run start`. */ command: string, /** - * The port that your server is expected to appear on. If not specified, it does get automatically collected via the - * command output when a localhost URL gets printed. + * The port that your http server is expected to appear on. If specified it does wait until it accepts connections. */ - port?: number, + waitForPort?: number, /** - * WebServer environment variables, process.env by default + * How long to wait for the process to start up and be available in milliseconds. Defaults to 60000. + */ + waitForPortTimeout?: number, + /** + * If true it will verify that the given port via `waitForPort` is available and throw otherwise. + * This should commonly set to !!process.env.CI to allow the local dev server when running tests locally. + */ + strict?: boolean + /** + * Environment variables, process.env by default */ env?: Record, /** * Current working directory of the spawned process. Default is process.cwd(). */ cwd?: string, - /** - * How long to wait for the server to start up in milliseconds. Defaults to 60000. - */ - timeout?: number, }; /** @@ -233,7 +237,7 @@ interface ConfigBase { /** * Launch a web server before running tests. */ - webServer?: WebServerConfig; + launch?: LaunchConfig | LaunchConfig[]; /** * The maximum number of concurrent worker processes to use for parallelizing tests. @@ -268,7 +272,7 @@ export interface FullConfig { shard: Shard; updateSnapshots: UpdateSnapshots; workers: number; - webServer: WebServerConfig | null; + launch: LaunchConfig[]; } export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped';