From 385d489b35ad5bc3b28c78c05b9c04e87400bd22 Mon Sep 17 00:00:00 2001 From: Max Schmitt Date: Tue, 3 Aug 2021 23:24:14 +0200 Subject: [PATCH] feat(test-runner): re-enable web server (#7906) Co-authored-by: Joel Einbinder --- docs/src/test-advanced-js.md | 68 +++++++++++++++++ docs/src/test-configuration-js.md | 1 + src/test/loader.ts | 13 +--- src/test/runner.ts | 6 +- src/test/{launchServer.ts => webServer.ts} | 66 +++++------------ ...unch-server.spec.ts => web-server.spec.ts} | 73 +++++++------------ types/test.d.ts | 20 ++--- utils/generate_types/overrides-test.d.ts | 20 ++--- 8 files changed, 143 insertions(+), 124 deletions(-) rename src/test/{launchServer.ts => webServer.ts} (64%) rename tests/playwright-test/{launch-server.spec.ts => web-server.spec.ts} (81%) diff --git a/docs/src/test-advanced-js.md b/docs/src/test-advanced-js.md index 16b11c2d7b..835f698ed5 100644 --- a/docs/src/test-advanced-js.md +++ b/docs/src/test-advanced-js.md @@ -43,6 +43,7 @@ These options would be typically different between local development and CI oper - `shard: { total: number, current: number } | null` - [Shard](./test-parallel.md#shards) information. - `updateSnapshots: boolean` - Whether to update expected snapshots with the actual results produced by the test run. - `workers: number` - The maximum number of concurrent worker processes to use for parallelizing tests. +- `webServer: { command: string, port: number, timeout?: number, reuseExistingServer?: boolean, cwd?: string, env?: object }` - Launch a process and wait that it's ready before the tests will start. See [launch web server](#launching-a-development-web-server-during-the-tests) configuration for examples. Note that each [test project](#projects) can provide its own test suite options, for example two projects can run different tests by providing different `testDir`s. However, test run options are shared between all projects. @@ -200,6 +201,73 @@ export const test = base.extend<{ saveLogs: void }>({ }); ``` +## Launching a development web server during the tests + +To launch a server during the tests, use the `webServer` option in the [configuration file](#configuration-object). + +You can specify a port via `port` or additional environment variables, see [here](#configuration-object). The server will wait for it to be available before running the tests. For continuous integration, you may want to use the `reuseExistingServer: !process.env.CI` option which does not use an existing server on the CI. + +The port gets then passed over to Playwright as a [`param: baseURL`] when creating the context [`method: Browser.newContext`]. + +```js js-flavor=ts +// playwright.config.ts +import { PlaywrightTestConfig } from '@playwright/test'; +const config: PlaywrightTestConfig = { + webServer: { + command: 'npm run start', + port: 3000, + timeout: 120 * 1000, + reuseExistingServer: !process.env.CI, + }, +}; +export default config; +``` + +```js js-flavor=js +// playwright.config.js +// @ts-check +/** @type {import('@playwright/test').PlaywrightTestConfig} */ +const config = { + webServer: { + command: 'npm run start', + port: 3000, + timeout: 120 * 1000, + reuseExistingServer: !process.env.CI, + }, +}; +mode.exports = config; +``` + +Now you can use a relative path when navigating the page, or use `baseURL` fixture: + +```js js-flavor=ts +// test.spec.ts +import { test } = from '@playwright/test'; +test('test', async ({ page, baseURL }) => { + // baseURL is taken directly from your web server, + // e.g. http://localhost:3000 + await page.goto(baseURL + '/bar'); + // Alternatively, just use relative path, because baseURL is already + // set for the default context and page. + // For example, this will result in http://localhost:3000/foo + await page.goto('/foo'); +}); +``` + +```js js-flavor=js +// test.spec.js +const { test } = require('@playwright/test'); +test('test', async ({ page, baseURL }) => { + // baseURL is taken directly from your web server, + // e.g. http://localhost:3000 + await page.goto(baseURL + '/bar'); + // Alternatively, just use relative path, because baseURL is already + // set for the default context and page. + // For example, this will result in http://localhost:3000/foo + await page.goto('/foo'); +}); +``` + ## Global setup and teardown To set something up once before running all tests, use `globalSetup` option in the [configuration file](#configuration-object). diff --git a/docs/src/test-configuration-js.md b/docs/src/test-configuration-js.md index 64f7cfb0f6..c8d4cf941c 100644 --- a/docs/src/test-configuration-js.md +++ b/docs/src/test-configuration-js.md @@ -478,6 +478,7 @@ In addition to configuring [Browser] or [BrowserContext], videos or screenshots, - `testIgnore`: Glob patterns or regular expressions that should be ignored when looking for the test files. For example, `'**/test-assets'`. - `testMatch`: Glob patterns or regular expressions that match test files. For example, `'**/todo-tests/*.spec.ts'`. By default, Playwright Test runs `.*(test|spec)\.(js|ts|mjs)` files. - `timeout`: Time in milliseconds given to each test. +- `webServer: { command: string, port: number, timeout?: number, reuseExistingServer?: boolean, cwd?: string, env?: object }` - Launch a process and wait that it's ready before the tests will start. See [launch web server](./test-advanced.md#launching-a-development-web-server-during-the-tests) configuration for examples. - `workers`: The maximum number of concurrent worker processes to use for parallelizing tests. You can specify these options in the configuration file. Note that testing options are **top-level**, do not put them into the `use` section. diff --git a/src/test/loader.ts b/src/test/loader.ts index 19ddcd01dc..ad74e4d963 100644 --- a/src/test/loader.ts +++ b/src/test/loader.ts @@ -25,7 +25,6 @@ import * as url from 'url'; import * as fs from 'fs'; import { ProjectImpl } from './project'; import { Reporter } from '../../types/testReporter'; -import { LaunchConfig } from '../../types/test'; import { BuiltInReporter, builtInReporters } from './runner'; export class Loader { @@ -101,7 +100,7 @@ export class Loader { this._fullConfig.shard = takeFirst(this._configOverrides.shard, this._config.shard, baseFullConfig.shard); this._fullConfig.updateSnapshots = takeFirst(this._configOverrides.updateSnapshots, this._config.updateSnapshots, baseFullConfig.updateSnapshots); this._fullConfig.workers = takeFirst(this._configOverrides.workers, this._config.workers, baseFullConfig.workers); - this._fullConfig._launch = takeFirst(toLaunchServers(this._configOverrides._launch), toLaunchServers(this._config._launch), baseFullConfig._launch); + this._fullConfig.webServer = takeFirst(this._configOverrides.webServer, this._config.webServer, baseFullConfig.webServer); for (const project of projects) this._addProject(project, this._fullConfig.rootDir); @@ -230,14 +229,6 @@ function toReporters(reporters: BuiltInReporter | ReporterDescription[] | undefi return reporters; } -function toLaunchServers(launchConfigs?: LaunchConfig | LaunchConfig[]): LaunchConfig[]|undefined { - if (!launchConfigs) - return; - if (!Array.isArray(launchConfigs)) - return [launchConfigs]; - return launchConfigs; -} - function validateConfig(file: string, config: Config) { if (typeof config !== 'object' || !config) throw errorWithFile(file, `Configuration file must export a single object`); @@ -435,7 +426,7 @@ const baseFullConfig: FullConfig = { shard: null, updateSnapshots: 'missing', workers: 1, - _launch: [], + webServer: null, }; function resolveReporters(reporters: Config['reporter'], rootDir: string): ReporterDescription[]|undefined { diff --git a/src/test/runner.ts b/src/test/runner.ts index acf5817484..2992a8e9de 100644 --- a/src/test/runner.ts +++ b/src/test/runner.ts @@ -35,7 +35,7 @@ import EmptyReporter from './reporters/empty'; import { ProjectImpl } from './project'; import { Minimatch } from 'minimatch'; import { Config, FullConfig } from './types'; -import { LaunchServers } from './launchServer'; +import { WebServer } from './webServer'; const removeFolderAsync = promisify(rimraf); const readDirAsync = promisify(fs.readdir); @@ -167,7 +167,7 @@ export class Runner { testFiles.forEach(file => allTestFiles.add(file)); } - const launchServers = await LaunchServers.create(config._launch); + const webServer = config.webServer && await WebServer.create(config.webServer); let globalSetupResult: any; if (config.globalSetup) globalSetupResult = await (await this._loader.loadGlobalHook(config.globalSetup, 'globalSetup'))(this._loader.fullConfig()); @@ -316,7 +316,7 @@ export class Runner { await globalSetupResult(this._loader.fullConfig()); if (config.globalTeardown) await (await this._loader.loadGlobalHook(config.globalTeardown, 'globalTeardown'))(this._loader.fullConfig()); - await launchServers.killAll(); + await webServer?.kill(); } } } diff --git a/src/test/launchServer.ts b/src/test/webServer.ts similarity index 64% rename from src/test/launchServer.ts rename to src/test/webServer.ts index 4cd2d06839..adf691899d 100644 --- a/src/test/launchServer.ts +++ b/src/test/webServer.ts @@ -13,13 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/* eslint-disable no-console */ import net from 'net'; import os from 'os'; import stream from 'stream'; import { monotonicTime, raceAgainstDeadline } from './util'; -import { LaunchConfig } from '../../types/test'; +import { WebServerConfig } from '../../types/test'; import { launchProcess } from '../utils/processLauncher'; const DEFAULT_ENVIRONMENT_VARIABLES = { @@ -33,19 +32,19 @@ const newProcessLogPrefixer = () => new stream.Transform({ }, }); -class LaunchServer { +export class WebServer { private _killProcess?: () => Promise; private _processExitedPromise!: Promise; - constructor(private readonly config: LaunchConfig) { } + constructor(private readonly config: WebServerConfig) { } - public static async create(config: LaunchConfig): Promise { - const launchServer = new LaunchServer(config); + public static async create(config: WebServerConfig): Promise { + const webServer = new WebServer(config); try { - await launchServer._startProcess(); - await launchServer._waitForProcess(); - return launchServer; + await webServer._startProcess(); + await webServer._waitForProcess(); + return webServer; } catch (error) { - await launchServer.kill(); + await webServer.kill(); throw error; } } @@ -54,15 +53,13 @@ class LaunchServer { let processExitedReject = (error: Error) => { }; this._processExitedPromise = new Promise((_, reject) => processExitedReject = reject); - if (this.config.waitForPort) { - const portIsUsed = !await canBindPort(this.config.waitForPort); - if (portIsUsed && this.config.strict) - throw new Error(`Port ${this.config.waitForPort} is used, make sure that nothing is running on the port or set strict:false in config.launch.`); - if (portIsUsed) + const portIsUsed = !await canBindPort(this.config.port); + if (portIsUsed) { + if (this.config.reuseExistingServer) return; + throw new Error(`Port ${this.config.port} is used, make sure that nothing is running on the port or set strict:false in config.launch.`); } - console.log(`Launching '${this.config.command}'...`); const { launchedProcess, kill } = await launchProcess({ command: this.config.command, env: { @@ -85,19 +82,16 @@ class LaunchServer { } private async _waitForProcess() { - if (this.config.waitForPort) { - await this._waitForAvailability(this.config.waitForPort); - const baseURL = `http://localhost:${this.config.waitForPort}`; - process.env.PLAYWRIGHT_TEST_BASE_URL = baseURL; - console.log(`Using baseURL '${baseURL}' from config.launch.`); - } + await this._waitForAvailability(); + const baseURL = `http://localhost:${this.config.port}`; + process.env.PLAYWRIGHT_TEST_BASE_URL = baseURL; } - private async _waitForAvailability(port: number) { - const launchTimeout = this.config.waitForPortTimeout || 60 * 1000; + private async _waitForAvailability() { + const launchTimeout = this.config.timeout || 60 * 1000; const cancellationToken = { canceled: false }; const { timedOut } = (await Promise.race([ - raceAgainstDeadline(waitForSocket(port, 100, cancellationToken), launchTimeout + monotonicTime()), + raceAgainstDeadline(waitForSocket(this.config.port, 100, cancellationToken), launchTimeout + monotonicTime()), this._processExitedPromise, ])); cancellationToken.canceled = true; @@ -139,25 +133,3 @@ async function waitForSocket(port: number, delay: number, cancellationToken: { c await new Promise(x => setTimeout(x, delay)); } } - -export class LaunchServers { - private readonly _servers: LaunchServer[] = []; - - public static async create(configs: LaunchConfig[]): Promise { - const launchServers = new LaunchServers(); - try { - for (const config of configs) - launchServers._servers.push(await LaunchServer.create(config)); - } catch (error) { - for (const server of launchServers._servers) - await server.kill(); - throw error; - } - return launchServers; - } - - public async killAll() { - for (const server of this._servers) - await server.kill(); - } -} diff --git a/tests/playwright-test/launch-server.spec.ts b/tests/playwright-test/web-server.spec.ts similarity index 81% rename from tests/playwright-test/launch-server.spec.ts rename to tests/playwright-test/web-server.spec.ts index 2f01ea64b8..0391d692fa 100644 --- a/tests/playwright-test/launch-server.spec.ts +++ b/tests/playwright-test/web-server.spec.ts @@ -32,9 +32,9 @@ test('should create a server', async ({ runInlineTest }, { workerIndex }) => { `, 'playwright.config.ts': ` module.exports = { - _launch: { + webServer: { command: 'node ${JSON.stringify(path.join(__dirname, 'assets', 'simple-server.js'))} ${port}', - waitForPort: ${port}, + port: ${port}, }, globalSetup: 'globalSetup.ts', globalTeardown: 'globalTeardown.ts', @@ -42,8 +42,19 @@ test('should create a server', async ({ runInlineTest }, { workerIndex }) => { `, 'globalSetup.ts': ` module.exports = async () => { - console.log('globalSetup') - return () => console.log('globalSetup teardown'); + const http = require("http"); + const response = await new Promise(resolve => { + const request = http.request("http://localhost:${port}/hello", resolve); + request.end(); + }) + console.log('globalSetup-status-'+response.statusCode) + return async () => { + const response = await new Promise(resolve => { + const request = http.request("http://localhost:${port}/hello", resolve); + request.end(); + }) + console.log('globalSetup-teardown-status-'+response.statusCode) + }; }; `, 'globalTeardown.ts': ` @@ -61,7 +72,7 @@ test('should create a server', async ({ runInlineTest }, { workerIndex }) => { expect(result.passed).toBe(1); expect(result.report.suites[0].specs[0].tests[0].results[0].status).toContain('passed'); - const expectedLogMessages = ['Launching ', 'globalSetup', 'globalSetup teardown', 'globalTeardown-status-200']; + const expectedLogMessages = ['globalSetup-status-200', 'globalSetup-teardown-status', 'globalTeardown-status-200']; const actualLogMessages = expectedLogMessages.map(log => ({ log, index: result.output.indexOf(log), @@ -82,9 +93,9 @@ test('should create a server with environment variables', async ({ runInlineTest `, 'playwright.config.ts': ` module.exports = { - _launch: { + webServer: { command: 'node ${JSON.stringify(path.join(__dirname, 'assets', 'simple-server.js'))} ${port}', - waitForPort: ${port}, + port: ${port}, env: { 'FOO': 'BAR', } @@ -110,10 +121,10 @@ test('should time out waiting for a server', async ({ runInlineTest }, { workerI `, 'playwright.config.ts': ` module.exports = { - _launch: { + webServer: { command: 'node ${JSON.stringify(JSON.stringify(path.join(__dirname, 'assets', 'simple-server.js')))} ${port}', - waitForPort: ${port}, - waitForPortTimeout: 100, + port: ${port}, + timeout: 100, } }; `, @@ -151,7 +162,7 @@ test('should be able to specify the baseURL without the server', async ({ runInl await new Promise(resolve => server.close(resolve)); }); -test('should be able to use an existing server when strict is false ', async ({ runInlineTest }, { workerIndex }) => { +test('should be able to use an existing server when reuseExistingServer:true ', async ({ runInlineTest }, { workerIndex }) => { const port = workerIndex + 10500; const server = http.createServer((req: http.IncomingMessage, res: http.ServerResponse) => { res.end('hello'); @@ -169,10 +180,10 @@ test('should be able to use an existing server when strict is false ', async ({ `, 'playwright.config.ts': ` module.exports = { - _launch: { + webServer: { command: 'node ${JSON.stringify(path.join(__dirname, 'assets', 'simple-server.js'))} ${port}', - waitForPort: ${port}, - strict: false, + port: ${port}, + reuseExistingServer: true, } }; `, @@ -202,10 +213,10 @@ test('should throw when a server is already running on the given port and strict `, 'playwright.config.ts': ` module.exports = { - _launch: { + webServer: { command: 'node ${JSON.stringify(path.join(__dirname, 'assets', 'simple-server.js'))} ${port}', - waitForPort: ${port}, - strict: true, + port: ${port}, + reuseExistingServer: false, } }; `, @@ -214,31 +225,3 @@ test('should throw when a server is already running on the given port and strict expect(result.output).toContain(`Port ${port} is used, make sure that nothing is running on the port`); await new Promise(resolve => server.close(resolve)); }); - -test('should create multiple servers', async ({ runInlineTest }, { workerIndex }) => { - const port1 = workerIndex + 10500; - const port2 = workerIndex + 10600; - const result = await runInlineTest({ - 'test.spec.ts': ` - const { test } = pwt; - test('connect to the server via the baseURL', async ({baseURL, page}) => { - await page.goto('http://localhost:${port1}/hello'); - await page.goto('http://localhost:${port2}/hello'); - }); - `, - 'playwright.config.ts': ` - module.exports = { - _launch: [{ - command: 'node ${JSON.stringify(path.join(__dirname, 'assets', 'simple-server.js'))} ${port1}', - waitForPort: ${port1}, - },{ - command: 'node ${JSON.stringify(path.join(__dirname, 'assets', 'simple-server.js'))} ${port2}', - waitForPort: ${port2}, - }], - }; - `, - }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(1); - expect(result.report.suites[0].specs[0].tests[0].results[0].status).toContain('passed'); -}); diff --git a/types/test.d.ts b/types/test.d.ts index 039ff5269f..9c2c809fd8 100644 --- a/types/test.d.ts +++ b/types/test.d.ts @@ -479,24 +479,26 @@ export interface Project extends TestProject { export type FullProject = Required>; -export type LaunchConfig = { +export type WebServerConfig = { /** * Shell command to start. For example `npm run start`. */ command: string, /** - * The port that your http server is expected to appear on. If specified it does wait until it accepts connections. + * The port that your http server is expected to appear on. It does wait until it accepts connections. */ - waitForPort?: number, + port: number, /** * How long to wait for the process to start up and be available in milliseconds. Defaults to 60000. */ - waitForPortTimeout?: number, + timeout?: number, /** - * If true it will verify that the given port via `waitForPort` is available and throw otherwise. - * This should commonly set to !!process.env.CI to allow the local dev server when running tests locally. + * If true, it will re-use an existing server on the port when available. If no server is running + * on that port, it will run the command to start a new server. + * If false, it will throw if an existing process is listening on the port. + * This should commonly set to !process.env.CI to allow the local dev server when running tests locally. */ - strict?: boolean + reuseExistingServer?: boolean /** * Environment variables, process.env by default */ @@ -690,7 +692,7 @@ interface TestConfig { * Learn more about [snapshots](https://playwright.dev/docs/test-snapshots). */ updateSnapshots?: UpdateSnapshots; - _launch?: LaunchConfig | LaunchConfig[]; + webServer?: WebServerConfig; /** * The maximum number of concurrent worker processes to use for parallelizing tests. * @@ -1051,7 +1053,7 @@ export interface FullConfig { * Playwright Test. */ workers: number; - _launch: LaunchConfig[]; + webServer: WebServerConfig | null; } export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped'; diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 3ad94ac2b8..4a7a847f7f 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -62,24 +62,26 @@ export interface Project extends TestProject { export type FullProject = Required>; -export type LaunchConfig = { +export type WebServerConfig = { /** * Shell command to start. For example `npm run start`. */ command: string, /** - * The port that your http server is expected to appear on. If specified it does wait until it accepts connections. + * The port that your http server is expected to appear on. It does wait until it accepts connections. */ - waitForPort?: number, + port: number, /** * How long to wait for the process to start up and be available in milliseconds. Defaults to 60000. */ - waitForPortTimeout?: number, + timeout?: number, /** - * If true it will verify that the given port via `waitForPort` is available and throw otherwise. - * This should commonly set to !!process.env.CI to allow the local dev server when running tests locally. + * If true, it will re-use an existing server on the port when available. If no server is running + * on that port, it will run the command to start a new server. + * If false, it will throw if an existing process is listening on the port. + * This should commonly set to !process.env.CI to allow the local dev server when running tests locally. */ - strict?: boolean + reuseExistingServer?: boolean /** * Environment variables, process.env by default */ @@ -107,7 +109,7 @@ interface TestConfig { reportSlowTests?: ReportSlowTests; shard?: Shard; updateSnapshots?: UpdateSnapshots; - _launch?: LaunchConfig | LaunchConfig[]; + webServer?: WebServerConfig; workers?: number; expect?: ExpectSettings; @@ -145,7 +147,7 @@ export interface FullConfig { shard: Shard; updateSnapshots: UpdateSnapshots; workers: number; - _launch: LaunchConfig[]; + webServer: WebServerConfig | null; } export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped';