Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

feat(proxy): enable per-context http proxy (#4280)

Browse source
This commit is contained in:
Pavel Feldman 2020-10-29 16:12:30 -07:00 committed by GitHub
parent ff7d6a2342
commit 914f6372ec
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 327 additions and 71 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
docs/api.md
View file

@ -223,6 +223,11 @@ Indicates that the browser is connected.
- `password` <[string]>
- `colorScheme` <"light"|"dark"|"no-preference"> Emulates `'prefers-colors-scheme'` media feature, supported values are `'light'`, `'dark'`, `'no-preference'`. See [page.emulateMedia(options)](#pageemulatemediaoptions) for more details. Defaults to '`light`'.
- `logger` <[Logger]> Logger sink for Playwright logging.
- `proxy` <[Object]> Network proxy settings to use with this context. Note that browser needs to be launched with the global proxy for this option to work. If all contexts override the proxy, global proxy will be never used and can be any string, for example `launch({ proxy: { server: 'per-proxy' } })`.
- `server` <[string]> Proxy to be used for all requests. HTTP and SOCKS proxies are supported, for example `http://myproxy.com:3128` or `socks5://myproxy.com:3128`. Short form `myproxy.com:3128` is considered an HTTP proxy.
- `bypass` <[string]> Optional coma-separated domains to bypass proxy, for example `".com, chromium.org, .domain.com"`.
- `username` <[string]> Optional username to use if HTTP proxy requires authentication.
- `password` <[string]> Optional password to use if HTTP proxy requires authentication.
- `videosPath` <[string]> Enables video recording for all pages to `videosPath` folder. If not specified, videos are not recorded. Make sure to await [`browserContext.close`](#browsercontextclose) for videos to be saved.
- `videoSize` <[Object]> Specifies dimensions of the automatically recorded video. Can only be used if `videosPath` is set. If not specified the size will be equal to `viewport`. If `viewport` is not configured explicitly the video size defaults to 1280x720. Actual picture of the page will be scaled down if necessary to fit specified size.
- `width` <[number]> Video frame width.
@ -272,6 +277,11 @@ Creates a new browser context. It won't share cookies/cache with other browser c
- `password` <[string]>
- `colorScheme` <"light"|"dark"|"no-preference"> Emulates `'prefers-colors-scheme'` media feature, supported values are `'light'`, `'dark'`, `'no-preference'`. See [page.emulateMedia(options)](#pageemulatemediaoptions) for more details. Defaults to '`light`'.
- `logger` <[Logger]> Logger sink for Playwright logging.
- `proxy` <[Object]> Network proxy settings to use with this context. Note that browser needs to be launched with the global proxy for this option to work. If all contexts override the proxy, global proxy will be never used and can be any string, for example `launch({ proxy: { server: 'per-proxy' } })`.
- `server` <[string]> Proxy to be used for all requests. HTTP and SOCKS proxies are supported, for example `http://myproxy.com:3128` or `socks5://myproxy.com:3128`. Short form `myproxy.com:3128` is considered an HTTP proxy.
- `bypass` <[string]> Optional coma-separated domains to bypass proxy, for example `".com, chromium.org, .domain.com"`.
- `username` <[string]> Optional username to use if HTTP proxy requires authentication.
- `password` <[string]> Optional password to use if HTTP proxy requires authentication.
- `videosPath` <[string]> Enables video recording for all pages to `videosPath` folder. If not specified, videos are not recorded. Make sure to await [`page.close`](#pagecloseoptions) for videos to be saved.
- `videoSize` <[Object]> Specifies dimensions of the automatically recorded video. Can only be used if `videosPath` is set. If not specified the size will be equal to `viewport`. If `viewport` is not configured explicitly the video size defaults to 1280x720. Actual picture of the page will be scaled down if necessary to fit specified size.
- `width` <[number]> Video frame width.

12
src/protocol/channels.ts
View file

@ -398,6 +398,12 @@ export type BrowserNewContextParams = {
omitContent?: boolean,
path: string,
},
proxy?: {
server: string,
bypass?: string,
username?: string,
password?: string,
},
};
export type BrowserNewContextOptions = {
noDefaultViewport?: boolean,
@ -442,6 +448,12 @@ export type BrowserNewContextOptions = {
omitContent?: boolean,
path: string,
},
proxy?: {
server: string,
bypass?: string,
username?: string,
password?: string,
},
};
export type BrowserNewContextResult = {
context: BrowserContextChannel,

7
src/protocol/protocol.yml
View file

@ -396,6 +396,13 @@ Browser:
properties:
omitContent: boolean?
path: string
proxy:
type: object?
properties:
server: string
bypass: string?
username: string?
password: string?
returns:
context: BrowserContext

6
src/protocol/validator.ts
View file

@ -233,6 +233,12 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme {
omitContent: tOptional(tBoolean),
path: tString,
})),
proxy: tOptional(tObject({
server: tString,
bypass: tOptional(tString),
username: tOptional(tString),
password: tOptional(tString),
})),
});
scheme.BrowserCrNewBrowserCDPSessionParams = tOptional(tObject({}));
scheme.BrowserCrStartTracingParams = tObject({

9
src/server/browserContext.ts
View file

@ -241,7 +241,7 @@ export abstract class BrowserContext extends EventEmitter {
}
protected _authenticateProxyViaHeader() {
const proxy = this._browser._options.proxy || { username: undefined, password: undefined };
const proxy = this._options.proxy || this._browser._options.proxy || { username: undefined, password: undefined };
const { username, password } = proxy;
if (username) {
this._options.httpCredentials = { username, password: password! };
@ -254,7 +254,7 @@ export abstract class BrowserContext extends EventEmitter {
}
protected _authenticateProxyViaCredentials() {
const proxy = this._browser._options.proxy;
const proxy = this._options.proxy || this._browser._options.proxy;
if (!proxy)
return;
const { username, password } = proxy;
@ -322,6 +322,11 @@ export function validateBrowserContextOptions(options: types.BrowserContextOptio
throw new Error(`"isMobile" option is not supported with null "viewport"`);
if (!options.viewport && !options.noDefaultViewport)
options.viewport = { width: 1280, height: 720 };
if (options.proxy) {
if (!browserOptions.proxy)
throw new Error(`Browser needs to be launched with the global proxy. If all contexts override the proxy, global proxy will be never used and can be any string, for example "launch({ proxy: { server: 'per-proxy' } })"`);
options.proxy = normalizeProxySettings(options.proxy);
}
verifyGeolocation(options.geolocation);
if (options.videoSize && !options.videosPath)
throw new Error(`"videoSize" option requires "videosPath" to be specified`);

6
src/server/chromium/crBrowser.ts
View file

@ -99,7 +99,11 @@ export class CRBrowser extends Browser {
async newContext(options: types.BrowserContextOptions = {}): Promise<BrowserContext> {
validateBrowserContextOptions(options, this._options);
const { browserContextId } = await this._session.send('Target.createBrowserContext', { disposeOnDetach: true });
const { browserContextId } = await this._session.send('Target.createBrowserContext', {
disposeOnDetach: true,
proxyServer: options.proxy ? options.proxy.server : undefined,
proxyBypassList: options.proxy ? options.proxy.bypass : undefined,
});
const context = new CRBrowserContext(this, browserContextId, options);
await context._initialize();
this._contexts.set(browserContextId, context);

73
src/server/firefox/ffBrowser.ts
View file

@ -18,7 +18,6 @@
import { assert } from '../../utils/utils';
import { Browser, BrowserOptions } from '../browser';
import { assertBrowserContextIsNotOwned, BrowserContext, validateBrowserContextOptions, verifyGeolocation } from '../browserContext';
import { helper, RegisteredListener } from '../helper';
import * as network from '../network';
import { Page, PageBinding } from '../page';
import { ConnectionTransport } from '../transport';
@ -31,7 +30,6 @@ export class FFBrowser extends Browser {
_connection: FFConnection;
readonly _ffPages: Map<string, FFPage>;
readonly _contexts: Map<string, FFBrowserContext>;
private _eventListeners: RegisteredListener[];
private _version = '';
static async connect(transport: ConnectionTransport, options: BrowserOptions): Promise<FFBrowser> {
@ -45,31 +43,8 @@ export class FFBrowser extends Browser {
browser._defaultContext = new FFBrowserContext(browser, undefined, options.persistent);
promises.push((browser._defaultContext as FFBrowserContext)._initialize());
}
if (options.proxy) {
const proxyServer = new URL(options.proxy.server);
let proxyPort = parseInt(proxyServer.port, 10);
let aType: 'http'|'https'|'socks'|'socks4' = 'http';
if (proxyServer.protocol === 'socks5:')
aType = 'socks';
else if (proxyServer.protocol === 'socks4:')
aType = 'socks4';
else if (proxyServer.protocol === 'https:')
aType = 'https';
if (proxyServer.port === '') {
if (proxyServer.protocol === 'http:')
proxyPort = 80;
else if (proxyServer.protocol === 'https:')
proxyPort = 443;
}
promises.push(browser._connection.send('Browser.setBrowserProxy', {
type: aType,
bypass: options.proxy.bypass ? options.proxy.bypass.split(',').map(domain => domain.trim()) : [],
host: proxyServer.hostname,
port: proxyPort,
username: options.proxy.username,
password: options.proxy.password,
}));
}
if (options.proxy)
promises.push(browser._connection.send('Browser.setBrowserProxy', toJugglerProxyOptions(options.proxy)));
await Promise.all(promises);
return browser;
}
@ -80,13 +55,11 @@ export class FFBrowser extends Browser {
this._ffPages = new Map();
this._contexts = new Map();
this._connection.on(ConnectionEvents.Disconnected, () => this._didClose());
this._eventListeners = [
helper.addEventListener(this._connection, 'Browser.attachedToTarget', this._onAttachedToTarget.bind(this)),
helper.addEventListener(this._connection, 'Browser.detachedFromTarget', this._onDetachedFromTarget.bind(this)),
helper.addEventListener(this._connection, 'Browser.downloadCreated', this._onDownloadCreated.bind(this)),
helper.addEventListener(this._connection, 'Browser.downloadFinished', this._onDownloadFinished.bind(this)),
helper.addEventListener(this._connection, 'Browser.screencastFinished', this._onScreencastFinished.bind(this)),
];
this._connection.on('Browser.attachedToTarget', this._onAttachedToTarget.bind(this));
this._connection.on('Browser.detachedFromTarget', this._onDetachedFromTarget.bind(this));
this._connection.on('Browser.downloadCreated', this._onDownloadCreated.bind(this));
this._connection.on('Browser.downloadFinished', this._onDownloadFinished.bind(this));
this._connection.on('Browser.screencastFinished', this._onScreencastFinished.bind(this));
}
async _initVersion() {
@ -239,6 +212,12 @@ export class FFBrowserContext extends BrowserContext {
});
}));
}
if (this._options.proxy) {
promises.push(this._browser._connection.send('Browser.setContextProxy', {
browserContextId: this._browserContextId,
...toJugglerProxyOptions(this._options.proxy)
}));
}
await Promise.all(promises);
}
@ -350,3 +329,29 @@ export class FFBrowserContext extends BrowserContext {
this._browser._contexts.delete(this._browserContextId);
}
}
function toJugglerProxyOptions(proxy: types.ProxySettings) {
const proxyServer = new URL(proxy.server);
let port = parseInt(proxyServer.port, 10);
let type: 'http' | 'https' | 'socks' | 'socks4' = 'http';
if (proxyServer.protocol === 'socks5:')
type = 'socks';
else if (proxyServer.protocol === 'socks4:')
type = 'socks4';
else if (proxyServer.protocol === 'https:')
type = 'https';
if (proxyServer.port === '') {
if (proxyServer.protocol === 'http:')
port = 80;
else if (proxyServer.protocol === 'https:')
port = 443;
}
return {
type,
bypass: proxy.bypass ? proxy.bypass.split(',').map(domain => domain.trim()) : [],
host: proxyServer.hostname,
port,
username: proxy.username,
password: proxy.password
};
}

1
src/server/types.ts
View file

@ -244,6 +244,7 @@ export type BrowserContextOptions = {
omitContent?: boolean,
path: string
},
proxy?: ProxySettings,
_tracePath?: string,
_traceResourcesPath?: string,
};

6
src/server/webkit/wkBrowser.ts
View file

@ -75,7 +75,11 @@ export class WKBrowser extends Browser {
async newContext(options: types.BrowserContextOptions = {}): Promise<BrowserContext> {
validateBrowserContextOptions(options, this._options);
const { browserContextId } = await this._browserSession.send('Playwright.createContext');
const createOptions = options.proxy ? {
proxyServer: options.proxy.server,
proxyBypassList: options.proxy.bypass
} : undefined;
const { browserContextId } = await this._browserSession.send('Playwright.createContext', createOptions);
options.userAgent = options.userAgent || DEFAULT_USER_AGENT;
const context = new WKBrowserContext(this, browserContextId, options);
await context._initialize();

208
test/browsercontext-proxy.spec.ts Normal file
View file

@ -0,0 +1,208 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { folio as baseFolio } from './fixtures';
const fixtures = baseFolio.extend();
fixtures.browserOptions.override(async ({ browserOptions }, run) => {
await run({ ...browserOptions, proxy: { server: 'per-proxy' } });
});
const { it, expect } = fixtures.build();
it('should throw for missing global proxy', async ({ browserType, browserOptions, server }) => {
delete browserOptions.proxy;
const browser = await browserType.launch(browserOptions);
const error = await browser.newContext({ proxy: { server: `localhost:${server.PORT}` } }).catch(e => e);
expect(error.toString()).toContain('Browser needs to be launched with the global proxy');
await browser.close();
});
it('should throw for bad server value', async ({ contextFactory, contextOptions }) => {
const error = await contextFactory({
...contextOptions,
// @ts-expect-error server must be a string
proxy: { server: 123 }
}).catch(e => e);
expect(error.message).toContain('proxy.server: expected string, got number');
});
it('should use proxy', async ({ contextFactory, contextOptions, server }) => {
server.setRoute('/target.html', async (req, res) => {
res.end('<html><title>Served by the proxy</title></html>');
});
const browser = await contextFactory({
...contextOptions,
proxy: { server: `localhost:${server.PORT}` }
});
const page = await browser.newPage();
await page.goto('http://non-existent.com/target.html');
expect(await page.title()).toBe('Served by the proxy');
await browser.close();
});
it('should use proxy twice', async ({ contextFactory, contextOptions, server }) => {
server.setRoute('/target.html', async (req, res) => {
res.end('<html><title>Served by the proxy</title></html>');
});
const browser = await contextFactory({
...contextOptions,
proxy: { server: `localhost:${server.PORT}` }
});
const page = await browser.newPage();
await page.goto('http://non-existent.com/target.html');
await page.goto('http://non-existent-2.com/target.html');
expect(await page.title()).toBe('Served by the proxy');
await browser.close();
});
it('should use proxy for second page', async ({contextFactory, contextOptions, server}) => {
server.setRoute('/target.html', async (req, res) => {
res.end('<html><title>Served by the proxy</title></html>');
});
const browser = await contextFactory({
...contextOptions,
proxy: { server: `localhost:${server.PORT}` }
});
const page = await browser.newPage();
await page.goto('http://non-existent.com/target.html');
expect(await page.title()).toBe('Served by the proxy');
const page2 = await browser.newPage();
await page2.goto('http://non-existent.com/target.html');
expect(await page2.title()).toBe('Served by the proxy');
await browser.close();
});
it('should work with IP:PORT notion', async ({contextFactory, contextOptions, server}) => {
server.setRoute('/target.html', async (req, res) => {
res.end('<html><title>Served by the proxy</title></html>');
});
const browser = await contextFactory({
...contextOptions,
proxy: { server: `127.0.0.1:${server.PORT}` }
});
const page = await browser.newPage();
await page.goto('http://non-existent.com/target.html');
expect(await page.title()).toBe('Served by the proxy');
await browser.close();
});
it('should authenticate', async ({contextFactory, contextOptions, server}) => {
server.setRoute('/target.html', async (req, res) => {
const auth = req.headers['proxy-authorization'];
if (!auth) {
res.writeHead(407, 'Proxy Authentication Required', {
'Proxy-Authenticate': 'Basic realm="Access to internal site"'
});
res.end();
} else {
res.end(`<html><title>${auth}</title></html>`);
}
});
const browser = await contextFactory({
...contextOptions,
proxy: { server: `localhost:${server.PORT}`, username: 'user', password: 'secret' }
});
const page = await browser.newPage();
await page.goto('http://non-existent.com/target.html');
expect(await page.title()).toBe('Basic ' + Buffer.from('user:secret').toString('base64'));
await browser.close();
});
it('should exclude patterns', (test, { browserName, headful }) => {
test.fixme(browserName === 'chromium' && headful, 'Chromium headful crashes with CHECK(!in_frame_tree_) in RenderFrameImpl::OnDeleteFrame.');
}, async ({contextFactory, contextOptions, server}) => {
server.setRoute('/target.html', async (req, res) => {
res.end('<html><title>Served by the proxy</title></html>');
});
// FYI: using long and weird domain names to avoid ATT DNS hijacking
// that resolves everything to some weird search results page.
//
// @see https://gist.github.com/CollinChaffin/24f6c9652efb3d6d5ef2f5502720ef00
const browser = await contextFactory({
...contextOptions,
proxy: { server: `localhost:${server.PORT}`, bypass: '1.non.existent.domain.for.the.test, 2.non.existent.domain.for.the.test, .another.test' }
});
const page = await browser.newPage();
await page.goto('http://0.non.existent.domain.for.the.test/target.html');
expect(await page.title()).toBe('Served by the proxy');
{
const error = await page.goto('http://1.non.existent.domain.for.the.test/target.html').catch(e => e);
expect(error.message).toBeTruthy();
}
{
const error = await page.goto('http://2.non.existent.domain.for.the.test/target.html').catch(e => e);
expect(error.message).toBeTruthy();
}
{
const error = await page.goto('http://foo.is.the.another.test/target.html').catch(e => e);
expect(error.message).toBeTruthy();
}
{
await page.goto('http://3.non.existent.domain.for.the.test/target.html');
expect(await page.title()).toBe('Served by the proxy');
}
await browser.close();
});
it('should use socks proxy', (test, { browserName, platform }) => {
test.flaky(platform === 'darwin' && browserName === 'webkit', 'Intermittent page.goto: The network connection was lost error on bots');
}, async ({ contextFactory, contextOptions, socksPort }) => {
const browser = await contextFactory({
...contextOptions,
proxy: { server: `socks5://localhost:${socksPort}` }
});
const page = await browser.newPage();
await page.goto('http://non-existent.com');
expect(await page.title()).toBe('Served by the SOCKS proxy');
await browser.close();
});
it('should use socks proxy in second page', (test, { browserName, platform }) => {
test.flaky(platform === 'darwin' && browserName === 'webkit', 'Intermittent page.goto: The network connection was lost error on bots');
}, async ({ contextFactory, contextOptions, socksPort }) => {
const browser = await contextFactory({
...contextOptions,
proxy: { server: `socks5://localhost:${socksPort}` }
});
const page = await browser.newPage();
await page.goto('http://non-existent.com');
expect(await page.title()).toBe('Served by the SOCKS proxy');
const page2 = await browser.newPage();
await page2.goto('http://non-existent.com');
expect(await page2.title()).toBe('Served by the SOCKS proxy');
await browser.close();
});
it('does launch without a port', async ({ contextFactory, contextOptions }) => {
const browser = await contextFactory({
...contextOptions,
proxy: { server: 'http://localhost' }
});
await browser.close();
});

26
test/http.fixtures.ts
View file

@ -16,11 +16,13 @@
import { folio as base } from 'folio';
import path from 'path';
import socks from 'socksv5';
import { TestServer } from '../utils/testserver';
type HttpWorkerFixtures = {
asset: (path: string) => string;
httpService: { server: TestServer, httpsServer: TestServer };
socksPort: number,
};
type HttpTestFixtures = {
@ -63,4 +65,28 @@ fixtures.httpsServer.init(async ({ httpService }, test) => {
await test(httpService.httpsServer);
});
fixtures.socksPort.init(async ({ testWorkerIndex }, run) => {
const server = socks.createServer((info, accept, deny) => {
let socket;
if ((socket = accept(true))) {
// Catch and ignore ECONNRESET errors.
socket.on('error', () => {});
const body = '<html><title>Served by the SOCKS proxy</title></html>';
socket.end([
'HTTP/1.1 200 OK',
'Connection: close',
'Content-Type: text/html',
'Content-Length: ' + Buffer.byteLength(body),
'',
body
].join('\r\n'));
}
});
const socksPort = 9107 + testWorkerIndex * 2;
server.listen(socksPort, 'localhost');
server.useAuth(socks.auth.None());
await run(socksPort);
server.close();
}, { scope: 'worker' });
export const folio = fixtures.build();

34
test/proxy.spec.ts
View file

@ -14,39 +14,7 @@
* limitations under the License.
*/
import { folio as baseFolio } from './fixtures';
import socks from 'socksv5';
const builder = baseFolio.extend<{}, {
socksPort: number,
}>();
builder.socksPort.init(async ({ testWorkerIndex }, run) => {
const server = socks.createServer((info, accept, deny) => {
let socket;
if ((socket = accept(true))) {
// Catch and ignore ECONNRESET errors.
socket.on('error', () => {});
const body = '<html><title>Served by the SOCKS proxy</title></html>';
socket.end([
'HTTP/1.1 200 OK',
'Connection: close',
'Content-Type: text/html',
'Content-Length: ' + Buffer.byteLength(body),
'',
body
].join('\r\n'));
}
});
const socksPort = 9107 + testWorkerIndex * 2;
server.listen(socksPort, 'localhost');
server.useAuth(socks.auth.None());
await run(socksPort);
server.close();
}, { scope: 'worker' });
const { it, expect } = builder.build();
import { it, expect } from './fixtures';
it('should throw for bad server value', async ({browserType, browserOptions}) => {
const error = await browserType.launch({