From 8028fb052aab7287d694e6a2966a1101a4f71890 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Mon, 3 Feb 2020 14:23:24 -0800 Subject: [PATCH] feat(route): migrate from request interception w/ events to page.route (#809) --- docs/api.md | 73 ++++++------- src/frames.ts | 2 +- src/helper.ts | 58 +++++++++++ src/network.ts | 4 + src/page.ts | 23 ++++- src/platform.ts | 19 ++-- test/chromium/chromium.spec.js | 5 +- test/chromium/oopif.spec.js | 3 +- test/interception.spec.js | 175 +++++++++++--------------------- test/webkit/provisional.spec.js | 22 ---- 10 files changed, 192 insertions(+), 192 deletions(-) diff --git a/docs/api.md b/docs/api.md index 751ba22f79..9e77934985 100644 --- a/docs/api.md +++ b/docs/api.md @@ -470,6 +470,7 @@ page.removeListener('request', logRequest); - [page.opener()](#pageopener) - [page.pdf([options])](#pagepdfoptions) - [page.reload([options])](#pagereloadoptions) +- [page.route(url, handler)](#pagerouteurl-handler) - [page.screenshot([options])](#pagescreenshotoptions) - [page.select(selector, value, options)](#pageselectselector-value-options) - [page.setCacheEnabled([enabled])](#pagesetcacheenabledenabled) @@ -478,7 +479,6 @@ page.removeListener('request', logRequest); - [page.setDefaultTimeout(timeout)](#pagesetdefaulttimeouttimeout) - [page.setExtraHTTPHeaders(headers)](#pagesetextrahttpheadersheaders) - [page.setOfflineMode(enabled)](#pagesetofflinemodeenabled) -- [page.setRequestInterception(enabled)](#pagesetrequestinterceptionenabled) - [page.setViewport(viewport)](#pagesetviewportviewport) - [page.title()](#pagetitle) - [page.tripleclick(selector[, options])](#pagetripleclickselector-options) @@ -585,7 +585,7 @@ const [popup] = await Promise.all([ - <[Request]> Emitted when a page issues a request. The [request] object is read-only. -In order to intercept and mutate requests, see `page.setRequestInterception(true)`. +In order to intercept and mutate requests, see `page.route()`. #### event: 'requestfailed' - <[Request]> @@ -1181,6 +1181,36 @@ The `format` options are: - `'networkidle2'` - consider navigation to be finished when there are no more than 2 network connections for at least `500` ms. - returns: <[Promise]<[Response]>> Promise which resolves to the main resource response. In case of multiple redirects, the navigation will resolve with the response of the last redirect. +#### page.route(url, handler) +- `url` <[string]|[RegExp]|[Function]> A glob pattern, regex pattern or predicate receiving [URL] to match while routing. +- `handler` <[Function]> handler function to router the request. +- returns: <[Promise]<[void]>>. + +Routing activates the request interception and enables `request.abort`, `request.continue` and +`request.respond` methods on the request. This provides the capability to modify network requests that are made by a page. + +Once request interception is enabled, every request matching the url pattern will stall unless it's continued, responded or aborted. +An example of a naïve request interceptor that aborts all image requests: + +```js +const page = await browser.newPage(); +await page.route('**/*.{png,jpg,jpeg}', request => request.abort()); +// await page.route(/\.(png|jpeg|jpg)$/, request => request.abort()); // <-- same thing +await page.goto('https://example.com'); +await browser.close(); +``` + +or the same snippet using a regex pattern instead: + +```js +const page = await browser.newPage(); +await page.route(/(\.png$)|(\.jpg$)/, request => request.abort()); +await page.goto('https://example.com'); +await browser.close(); +``` + +> **NOTE** Enabling request interception disables page caching. + #### page.screenshot([options]) - `options` <[Object]> Options object which might have the following properties: - `path` <[string]> The file path to save the image to. The screenshot type will be inferred from file extension. If `path` is a relative path, then it is resolved relative to [current working directory](https://nodejs.org/api/process.html#process_process_cwd). If no path is provided, the image won't be saved to the disk. @@ -1288,31 +1318,6 @@ The extra HTTP headers will be sent with every request the page initiates. - `enabled` <[boolean]> When `true`, enables offline mode for the page. - returns: <[Promise]> -#### page.setRequestInterception(enabled) -- `enabled` <[boolean]> Whether to enable request interception. -- returns: <[Promise]> - -Activating request interception enables `request.abort`, `request.continue` and -`request.respond` methods. This provides the capability to modify network requests that are made by a page. - -Once request interception is enabled, every request will stall unless it's continued, responded or aborted. -An example of a naïve request interceptor that aborts all image requests: - -```js -const page = await browser.newPage(); -await page.setRequestInterception(true); -page.on('request', interceptedRequest => { - if (interceptedRequest.url().endsWith('.png') || interceptedRequest.url().endsWith('.jpg')) - interceptedRequest.abort(); - else - interceptedRequest.continue(); -}); -await page.goto('https://example.com'); -await browser.close(); -``` - -> **NOTE** Enabling request interception disables page caching. - #### page.setViewport(viewport) - `viewport` <[Object]> - `width` <[number]> page width in pixels. **required** @@ -1501,7 +1506,7 @@ Shortcut for [page.mainFrame().waitForLoadState([options])](#framewaitforloadsta #### page.waitForNavigation([options]) - `options` <[Object]> Navigation parameters which might have the following properties: - `timeout` <[number]> Maximum navigation time in milliseconds, defaults to 30 seconds, pass `0` to disable timeout. The default value can be changed by using the [page.setDefaultNavigationTimeout(timeout)](#pagesetdefaultnavigationtimeouttimeout) or [page.setDefaultTimeout(timeout)](#pagesetdefaulttimeouttimeout) methods. - - `url` <[string]|[RegExp]|[Function]> URL string, URL regex pattern or predicate receiving [URL] to match while waiting for the navigation. + - `url` <[string]|[RegExp]|[Function]> A glob pattern, regex pattern or predicate receiving [URL] to match while waiting for the navigation. - `waitUntil` <"load"|"domcontentloaded"|"networkidle0"|"networkidle2"|[Array]<"load"|"domcontentloaded"|"networkidle0"|"networkidle2">> When to consider navigation succeeded, defaults to `load`. Given an array of event strings, navigation is considered to be successful after all events have been fired. Events can be either: - `'load'` - consider navigation to be finished when the `load` event is fired. - `'domcontentloaded'` - consider navigation to be finished when the `DOMContentLoaded` event is fired. @@ -2849,7 +2854,7 @@ If request gets a 'redirect' response, the request is successfully finished with - `failed` - A generic failure occurred. - returns: <[Promise]> -Aborts request. To use this, request interception should be enabled with `page.setRequestInterception`. +Aborts request. To use this, request interception should be enabled with `page.route`. Exception is immediately thrown if the request interception is not enabled. #### request.continue([overrides]) @@ -2859,12 +2864,11 @@ Exception is immediately thrown if the request interception is not enabled. - `headers` <[Object]> If set changes the request HTTP headers. Header values will be converted to a string. - returns: <[Promise]> -Continues request with optional request overrides. To use this, request interception should be enabled with `page.setRequestInterception`. +Continues request with optional request overrides. To use this, request interception should be enabled with `page.route`. Exception is immediately thrown if the request interception is not enabled. ```js -await page.setRequestInterception(true); -page.on('request', request => { +await page.route('**/*', request => { // Override headers const headers = Object.assign({}, request.headers(), { foo: 'bar', // set "foo" header @@ -2901,14 +2905,13 @@ page.on('requestfailed', request => { - returns: <[Promise]> Fulfills request with given response. To use this, request interception should -be enabled with `page.setRequestInterception`. Exception is thrown if +be enabled with `page.route`. Exception is thrown if request interception is not enabled. An example of fulfilling all requests with 404 responses: ```js -await page.setRequestInterception(true); -page.on('request', request => { +await page.route('**/*', request => { request.respond({ status: 404, contentType: 'text/plain', diff --git a/src/frames.ts b/src/frames.ts index 6bbe0f6b3b..3d841da09b 100644 --- a/src/frames.ts +++ b/src/frames.ts @@ -199,7 +199,7 @@ export class FrameManager { watcher._onNavigationRequest(frame, request); } if (!request._isFavicon) - this._page.emit(Events.Page.Request, request); + this._page._requestStarted(request); } requestReceivedResponse(response: network.Response) { diff --git a/src/helper.ts b/src/helper.ts index 062736bc11..c68af5e8a1 100644 --- a/src/helper.ts +++ b/src/helper.ts @@ -155,6 +155,62 @@ class Helper { clearTimeout(timeoutTimer); } } + + static globToRegex(glob: string): RegExp { + const tokens = ['^']; + let inGroup; + for (let i = 0; i < glob.length; ++i) { + const c = glob[i]; + if (escapeGlobChars.has(c)) { + tokens.push('\\' + c); + continue; + } + if (c === '*') { + const beforeDeep = glob[i - 1]; + let starCount = 1; + while (glob[i + 1] === '*') { + starCount++; + i++; + } + const afterDeep = glob[i + 1]; + const isDeep = starCount > 1 && + (beforeDeep === '/' || beforeDeep === undefined) && + (afterDeep === '/' || afterDeep === undefined); + if (isDeep) { + tokens.push('((?:[^/]*(?:\/|$))*)'); + i++; + } else { + tokens.push('([^/]*)'); + } + continue; + } + + switch (c) { + case '?': + tokens.push('.'); + break; + case '{': + inGroup = true; + tokens.push('('); + break; + case '}': + inGroup = false; + tokens.push(')'); + break; + case ',': + if (inGroup) { + tokens.push('|'); + break; + } + tokens.push('\\' + c); + break; + default: + tokens.push(c); + } + } + tokens.push('$'); + return new RegExp(tokens.join('')); + } } export function assert(value: any, message?: string) { @@ -162,4 +218,6 @@ export function assert(value: any, message?: string) { throw new Error(message); } +const escapeGlobChars = new Set(['/', '$', '^', '+', '.', '(', ')', '=', '!', '|']); + export const helper = Helper; diff --git a/src/network.ts b/src/network.ts index e776f20c97..35692a08fb 100644 --- a/src/network.ts +++ b/src/network.ts @@ -216,6 +216,10 @@ export class Request { assert(!this._interceptionHandled, 'Request is already handled!'); await this._delegate!.continue(overrides); } + + _isIntercepted(): boolean { + return !!this._delegate; + } } type GetResponseBodyCallback = () => Promise; diff --git a/src/page.ts b/src/page.ts index 46cf767194..9c5ab5e12d 100644 --- a/src/page.ts +++ b/src/page.ts @@ -111,6 +111,7 @@ export class Page extends platform.EventEmitter { private _workers = new Map(); readonly pdf: ((options?: types.PDFOptions) => Promise) | undefined; readonly coverage: Coverage | undefined; + readonly _requestHandlers: { url: types.URLMatch, handler: (request: network.Request) => void }[] = []; constructor(delegate: PageDelegate, browserContext: BrowserContext) { super(); @@ -416,11 +417,25 @@ export class Page extends platform.EventEmitter { await this._delegate.setCacheEnabled(enabled); } - async setRequestInterception(enabled: boolean) { - if (this._state.interceptNetwork === enabled) + async route(url: types.URLMatch, handler: (request: network.Request) => void) { + if (!this._state.interceptNetwork) { + this._state.interceptNetwork = true; + await this._delegate.setRequestInterception(true); + } + this._requestHandlers.push({ url, handler }); + } + + _requestStarted(request: network.Request) { + this.emit(Events.Page.Request, request); + if (!request._isIntercepted()) return; - this._state.interceptNetwork = enabled; - await this._delegate.setRequestInterception(enabled); + for (const { url, handler } of this._requestHandlers) { + if (platform.urlMatches(request.url(), url)) { + handler(request); + return; + } + } + request.continue(); } async setOfflineMode(enabled: boolean) { diff --git a/src/platform.ts b/src/platform.ts index bdef6a2665..8b14265a07 100644 --- a/src/platform.ts +++ b/src/platform.ts @@ -219,19 +219,20 @@ export function getMimeType(file: string): string | null { } export function urlMatches(urlString: string, match: types.URLMatch | undefined): boolean { - if (match === undefined) + if (match === undefined || match === '') return true; - if (typeof match === 'string') - return match === urlString; + if (helper.isString(match)) + match = helper.globToRegex(match); if (match instanceof RegExp) return match.test(urlString); - assert(typeof match === 'function', 'url parameter should be string, RegExp or function'); + if (typeof match === 'string' && match === urlString) + return true; + const url = new URL(urlString); + if (typeof match === 'string') + return url.pathname === match; - try { - return match(new URL(urlString)); - } catch (e) { - } - return false; + assert(typeof match === 'function', 'url parameter should be string, RegExp or function'); + return match(url); } export function pngToJpeg(buffer: Buffer): Buffer { diff --git a/test/chromium/chromium.spec.js b/test/chromium/chromium.spec.js index e6c5a3c2fc..13bb5f7202 100644 --- a/test/chromium/chromium.spec.js +++ b/test/chromium/chromium.spec.js @@ -224,7 +224,7 @@ module.exports.describe = function({testRunner, expect, playwright, FFOX, CHROMI }); describe('Chromium-Specific Page Tests', function() { - it('Page.setRequestInterception should work with intervention headers', async({server, page}) => { + it('Page.route should work with intervention headers', async({server, page}) => { server.setRoute('/intervention', (req, res) => res.end(`