feat(proxy): enable per-context http proxy (#4280)

docs/api.md

@@ -223,6 +223,11 @@ Indicates that the browser is connected.
     - `password` <[string]>
   - `colorScheme` <"light"|"dark"|"no-preference"> Emulates `'prefers-colors-scheme'` media feature, supported values are `'light'`, `'dark'`, `'no-preference'`. See [page.emulateMedia(options)](#pageemulatemediaoptions) for more details. Defaults to '`light`'.
   - `logger` <[Logger]> Logger sink for Playwright logging.
+  - `proxy` <[Object]> Network proxy settings to use with this context. Note that browser needs to be launched with the global proxy for this option to work. If all contexts override the proxy, global proxy will be never used and can be any string, for example `launch({ proxy: { server: 'per-proxy' } })`.
+    - `server` <[string]> Proxy to be used for all requests. HTTP and SOCKS proxies are supported, for example `http://myproxy.com:3128` or `socks5://myproxy.com:3128`. Short form `myproxy.com:3128` is considered an HTTP proxy.
+    - `bypass` <[string]> Optional coma-separated domains to bypass proxy, for example `".com, chromium.org, .domain.com"`.
+    - `username` <[string]> Optional username to use if HTTP proxy requires authentication.
+    - `password` <[string]> Optional password to use if HTTP proxy requires authentication.
   - `videosPath` <[string]> Enables video recording for all pages to `videosPath` folder. If not specified, videos are not recorded. Make sure to await [`browserContext.close`](#browsercontextclose) for videos to be saved.
   - `videoSize` <[Object]> Specifies dimensions of the automatically recorded video. Can only be used if `videosPath` is set. If not specified the size will be equal to `viewport`. If `viewport` is not configured explicitly the video size defaults to 1280x720. Actual picture of the page will be scaled down if necessary to fit specified size.
     - `width` <[number]> Video frame width.
@@ -272,6 +277,11 @@ Creates a new browser context. It won't share cookies/cache with other browser c
     - `password` <[string]>
   - `colorScheme` <"light"|"dark"|"no-preference"> Emulates `'prefers-colors-scheme'` media feature, supported values are `'light'`, `'dark'`, `'no-preference'`. See [page.emulateMedia(options)](#pageemulatemediaoptions) for more details. Defaults to '`light`'.
   - `logger` <[Logger]> Logger sink for Playwright logging.
+  - `proxy` <[Object]> Network proxy settings to use with this context. Note that browser needs to be launched with the global proxy for this option to work. If all contexts override the proxy, global proxy will be never used and can be any string, for example `launch({ proxy: { server: 'per-proxy' } })`.
+    - `server` <[string]> Proxy to be used for all requests. HTTP and SOCKS proxies are supported, for example `http://myproxy.com:3128` or `socks5://myproxy.com:3128`. Short form `myproxy.com:3128` is considered an HTTP proxy.
+    - `bypass` <[string]> Optional coma-separated domains to bypass proxy, for example `".com, chromium.org, .domain.com"`.
+    - `username` <[string]> Optional username to use if HTTP proxy requires authentication.
+    - `password` <[string]> Optional password to use if HTTP proxy requires authentication.
   - `videosPath` <[string]> Enables video recording for all pages to `videosPath` folder. If not specified, videos are not recorded. Make sure to await [`page.close`](#pagecloseoptions) for videos to be saved.
   - `videoSize` <[Object]> Specifies dimensions of the automatically recorded video. Can only be used if `videosPath` is set. If not specified the size will be equal to `viewport`. If `viewport` is not configured explicitly the video size defaults to 1280x720. Actual picture of the page will be scaled down if necessary to fit specified size.
     - `width` <[number]> Video frame width.

src/protocol/channels.ts

@@ -398,6 +398,12 @@ export type BrowserNewContextParams = {
     omitContent?: boolean,
     path: string,
   },
+  proxy?: {
+    server: string,
+    bypass?: string,
+    username?: string,
+    password?: string,
+  },
 };
 export type BrowserNewContextOptions = {
   noDefaultViewport?: boolean,
@@ -442,6 +448,12 @@ export type BrowserNewContextOptions = {
     omitContent?: boolean,
     path: string,
   },
+  proxy?: {
+    server: string,
+    bypass?: string,
+    username?: string,
+    password?: string,
+  },
 };
 export type BrowserNewContextResult = {
   context: BrowserContextChannel,

src/protocol/protocol.yml

@@ -396,6 +396,13 @@ Browser:
           properties:
             omitContent: boolean?
             path: string
+        proxy:
+          type: object?
+          properties:
+            server: string
+            bypass: string?
+            username: string?
+            password: string?
       returns:
         context: BrowserContext
 

src/protocol/validator.ts

@@ -233,6 +233,12 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme {
       omitContent: tOptional(tBoolean),
       path: tString,
     })),
+    proxy: tOptional(tObject({
+      server: tString,
+      bypass: tOptional(tString),
+      username: tOptional(tString),
+      password: tOptional(tString),
+    })),
   });
   scheme.BrowserCrNewBrowserCDPSessionParams = tOptional(tObject({}));
   scheme.BrowserCrStartTracingParams = tObject({

src/server/browserContext.ts

@@ -241,7 +241,7 @@ export abstract class BrowserContext extends EventEmitter {
   }
 
   protected _authenticateProxyViaHeader() {
-    const proxy = this._browser._options.proxy || { username: undefined, password: undefined };
+    const proxy = this._options.proxy || this._browser._options.proxy || { username: undefined, password: undefined };
     const { username, password } = proxy;
     if (username) {
       this._options.httpCredentials = { username, password: password! };
@@ -254,7 +254,7 @@ export abstract class BrowserContext extends EventEmitter {
   }
 
   protected _authenticateProxyViaCredentials() {
-    const proxy = this._browser._options.proxy;
+    const proxy = this._options.proxy || this._browser._options.proxy;
     if (!proxy)
       return;
     const { username, password } = proxy;
@@ -322,6 +322,11 @@ export function validateBrowserContextOptions(options: types.BrowserContextOptio
     throw new Error(`"isMobile" option is not supported with null "viewport"`);
   if (!options.viewport && !options.noDefaultViewport)
     options.viewport = { width: 1280, height: 720 };
+  if (options.proxy) {
+    if (!browserOptions.proxy)
+      throw new Error(`Browser needs to be launched with the global proxy. If all contexts override the proxy, global proxy will be never used and can be any string, for example "launch({ proxy: { server: 'per-proxy' } })"`);
+    options.proxy = normalizeProxySettings(options.proxy);
+  }
   verifyGeolocation(options.geolocation);
   if (options.videoSize && !options.videosPath)
     throw new Error(`"videoSize" option requires "videosPath" to be specified`);

src/server/chromium/crBrowser.ts

@@ -99,7 +99,11 @@ export class CRBrowser extends Browser {
 
   async newContext(options: types.BrowserContextOptions = {}): Promise<BrowserContext> {
     validateBrowserContextOptions(options, this._options);
-    const { browserContextId } = await this._session.send('Target.createBrowserContext', { disposeOnDetach: true });
+    const { browserContextId } = await this._session.send('Target.createBrowserContext', {
+      disposeOnDetach: true,
+      proxyServer: options.proxy ? options.proxy.server : undefined,
+      proxyBypassList: options.proxy ? options.proxy.bypass : undefined,
+    });
     const context = new CRBrowserContext(this, browserContextId, options);
     await context._initialize();
     this._contexts.set(browserContextId, context);

src/server/firefox/ffBrowser.ts

@@ -18,7 +18,6 @@
 import { assert } from '../../utils/utils';
 import { Browser, BrowserOptions } from '../browser';
 import { assertBrowserContextIsNotOwned, BrowserContext, validateBrowserContextOptions, verifyGeolocation } from '../browserContext';
-import { helper, RegisteredListener } from '../helper';
 import * as network from '../network';
 import { Page, PageBinding } from '../page';
 import { ConnectionTransport } from '../transport';
@@ -31,7 +30,6 @@ export class FFBrowser extends Browser {
   _connection: FFConnection;
   readonly _ffPages: Map<string, FFPage>;
   readonly _contexts: Map<string, FFBrowserContext>;
-  private _eventListeners: RegisteredListener[];
   private _version = '';
 
   static async connect(transport: ConnectionTransport, options: BrowserOptions): Promise<FFBrowser> {
@@ -45,31 +43,8 @@ export class FFBrowser extends Browser {
       browser._defaultContext = new FFBrowserContext(browser, undefined, options.persistent);
       promises.push((browser._defaultContext as FFBrowserContext)._initialize());
     }
-    if (options.proxy) {
+    if (options.proxy)
-      const proxyServer = new URL(options.proxy.server);
+      promises.push(browser._connection.send('Browser.setBrowserProxy', toJugglerProxyOptions(options.proxy)));
-      let proxyPort = parseInt(proxyServer.port, 10);
-      let aType: 'http'|'https'|'socks'|'socks4' = 'http';
-      if (proxyServer.protocol === 'socks5:')
-        aType = 'socks';
-      else if (proxyServer.protocol === 'socks4:')
-        aType = 'socks4';
-      else if (proxyServer.protocol === 'https:')
-        aType = 'https';
-      if (proxyServer.port === '') {
-        if (proxyServer.protocol === 'http:')
-          proxyPort = 80;
-        else if (proxyServer.protocol === 'https:')
-          proxyPort = 443;
-      }
-      promises.push(browser._connection.send('Browser.setBrowserProxy', {
-        type: aType,
-        bypass: options.proxy.bypass ? options.proxy.bypass.split(',').map(domain => domain.trim()) : [],
-        host: proxyServer.hostname,
-        port: proxyPort,
-        username: options.proxy.username,
-        password: options.proxy.password,
-      }));
-    }
     await Promise.all(promises);
     return browser;
   }
@@ -80,13 +55,11 @@ export class FFBrowser extends Browser {
     this._ffPages = new Map();
     this._contexts = new Map();
     this._connection.on(ConnectionEvents.Disconnected, () => this._didClose());
-    this._eventListeners = [
+    this._connection.on('Browser.attachedToTarget', this._onAttachedToTarget.bind(this));
-      helper.addEventListener(this._connection, 'Browser.attachedToTarget', this._onAttachedToTarget.bind(this)),
+    this._connection.on('Browser.detachedFromTarget', this._onDetachedFromTarget.bind(this));
-      helper.addEventListener(this._connection, 'Browser.detachedFromTarget', this._onDetachedFromTarget.bind(this)),
+    this._connection.on('Browser.downloadCreated', this._onDownloadCreated.bind(this));
-      helper.addEventListener(this._connection, 'Browser.downloadCreated', this._onDownloadCreated.bind(this)),
+    this._connection.on('Browser.downloadFinished', this._onDownloadFinished.bind(this));
-      helper.addEventListener(this._connection, 'Browser.downloadFinished', this._onDownloadFinished.bind(this)),
+    this._connection.on('Browser.screencastFinished', this._onScreencastFinished.bind(this));
-      helper.addEventListener(this._connection, 'Browser.screencastFinished', this._onScreencastFinished.bind(this)),
-    ];
   }
 
   async _initVersion() {
@@ -239,6 +212,12 @@ export class FFBrowserContext extends BrowserContext {
         });
       }));
     }
+    if (this._options.proxy) {
+      promises.push(this._browser._connection.send('Browser.setContextProxy', {
+        browserContextId: this._browserContextId,
+        ...toJugglerProxyOptions(this._options.proxy)
+      }));
+    }
 
     await Promise.all(promises);
   }
@@ -350,3 +329,29 @@ export class FFBrowserContext extends BrowserContext {
     this._browser._contexts.delete(this._browserContextId);
   }
 }
+
+function toJugglerProxyOptions(proxy: types.ProxySettings) {
+  const proxyServer = new URL(proxy.server);
+  let port = parseInt(proxyServer.port, 10);
+  let type: 'http' | 'https' | 'socks' | 'socks4' = 'http';
+  if (proxyServer.protocol === 'socks5:')
+    type = 'socks';
+  else if (proxyServer.protocol === 'socks4:')
+    type = 'socks4';
+  else if (proxyServer.protocol === 'https:')
+    type = 'https';
+  if (proxyServer.port === '') {
+    if (proxyServer.protocol === 'http:')
+      port = 80;
+    else if (proxyServer.protocol === 'https:')
+      port = 443;
+  }
+  return {
+    type,
+    bypass: proxy.bypass ? proxy.bypass.split(',').map(domain => domain.trim()) : [],
+    host: proxyServer.hostname,
+    port,
+    username: proxy.username,
+    password: proxy.password
+  };
+}

src/server/types.ts

@@ -244,6 +244,7 @@ export type BrowserContextOptions = {
     omitContent?: boolean,
     path: string
   },
+  proxy?: ProxySettings,
   _tracePath?: string,
   _traceResourcesPath?: string,
 };

src/server/webkit/wkBrowser.ts

@@ -75,7 +75,11 @@ export class WKBrowser extends Browser {
 
   async newContext(options: types.BrowserContextOptions = {}): Promise<BrowserContext> {
     validateBrowserContextOptions(options, this._options);
-    const { browserContextId } = await this._browserSession.send('Playwright.createContext');
+    const createOptions = options.proxy ? {
+      proxyServer: options.proxy.server,
+      proxyBypassList: options.proxy.bypass
+    } : undefined;
+    const { browserContextId } = await this._browserSession.send('Playwright.createContext', createOptions);
     options.userAgent = options.userAgent || DEFAULT_USER_AGENT;
     const context = new WKBrowserContext(this, browserContextId, options);
     await context._initialize();

test/browsercontext-proxy.spec.ts

@@ -0,0 +1,208 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { folio as baseFolio } from './fixtures';
+
+const fixtures = baseFolio.extend();
+fixtures.browserOptions.override(async ({ browserOptions }, run) => {
+  await run({ ...browserOptions, proxy: { server: 'per-proxy' } });
+});
+
+const { it, expect } = fixtures.build();
+
+it('should throw for missing global proxy', async ({ browserType, browserOptions, server }) => {
+  delete browserOptions.proxy;
+  const browser = await browserType.launch(browserOptions);
+  const error = await browser.newContext({ proxy: { server: `localhost:${server.PORT}` } }).catch(e => e);
+  expect(error.toString()).toContain('Browser needs to be launched with the global proxy');
+  await browser.close();
+});
+
+it('should throw for bad server value', async ({ contextFactory, contextOptions }) => {
+  const error = await contextFactory({
+    ...contextOptions,
+    // @ts-expect-error server must be a string
+    proxy: { server: 123 }
+  }).catch(e => e);
+  expect(error.message).toContain('proxy.server: expected string, got number');
+});
+
+it('should use proxy', async ({ contextFactory, contextOptions, server }) => {
+  server.setRoute('/target.html', async (req, res) => {
+    res.end('<html><title>Served by the proxy</title></html>');
+  });
+  const browser = await contextFactory({
+    ...contextOptions,
+    proxy: { server: `localhost:${server.PORT}` }
+  });
+  const page = await browser.newPage();
+  await page.goto('http://non-existent.com/target.html');
+  expect(await page.title()).toBe('Served by the proxy');
+  await browser.close();
+});
+
+it('should use proxy twice', async ({ contextFactory, contextOptions, server }) => {
+  server.setRoute('/target.html', async (req, res) => {
+    res.end('<html><title>Served by the proxy</title></html>');
+  });
+  const browser = await contextFactory({
+    ...contextOptions,
+    proxy: { server: `localhost:${server.PORT}` }
+  });
+  const page = await browser.newPage();
+  await page.goto('http://non-existent.com/target.html');
+  await page.goto('http://non-existent-2.com/target.html');
+  expect(await page.title()).toBe('Served by the proxy');
+  await browser.close();
+});
+
+it('should use proxy for second page', async ({contextFactory, contextOptions, server}) => {
+  server.setRoute('/target.html', async (req, res) => {
+    res.end('<html><title>Served by the proxy</title></html>');
+  });
+  const browser = await contextFactory({
+    ...contextOptions,
+    proxy: { server: `localhost:${server.PORT}` }
+  });
+
+  const page = await browser.newPage();
+  await page.goto('http://non-existent.com/target.html');
+  expect(await page.title()).toBe('Served by the proxy');
+
+  const page2 = await browser.newPage();
+  await page2.goto('http://non-existent.com/target.html');
+  expect(await page2.title()).toBe('Served by the proxy');
+
+  await browser.close();
+});
+
+it('should work with IP:PORT notion', async ({contextFactory, contextOptions, server}) => {
+  server.setRoute('/target.html', async (req, res) => {
+    res.end('<html><title>Served by the proxy</title></html>');
+  });
+  const browser = await contextFactory({
+    ...contextOptions,
+    proxy: { server: `127.0.0.1:${server.PORT}` }
+  });
+  const page = await browser.newPage();
+  await page.goto('http://non-existent.com/target.html');
+  expect(await page.title()).toBe('Served by the proxy');
+  await browser.close();
+});
+
+it('should authenticate', async ({contextFactory, contextOptions, server}) => {
+  server.setRoute('/target.html', async (req, res) => {
+    const auth = req.headers['proxy-authorization'];
+    if (!auth) {
+      res.writeHead(407, 'Proxy Authentication Required', {
+        'Proxy-Authenticate': 'Basic realm="Access to internal site"'
+      });
+      res.end();
+    } else {
+      res.end(`<html><title>${auth}</title></html>`);
+    }
+  });
+  const browser = await contextFactory({
+    ...contextOptions,
+    proxy: { server: `localhost:${server.PORT}`, username: 'user', password: 'secret' }
+  });
+  const page = await browser.newPage();
+  await page.goto('http://non-existent.com/target.html');
+  expect(await page.title()).toBe('Basic ' + Buffer.from('user:secret').toString('base64'));
+  await browser.close();
+});
+
+it('should exclude patterns', (test, { browserName, headful }) => {
+  test.fixme(browserName === 'chromium' && headful, 'Chromium headful crashes with CHECK(!in_frame_tree_) in RenderFrameImpl::OnDeleteFrame.');
+}, async ({contextFactory, contextOptions, server}) => {
+  server.setRoute('/target.html', async (req, res) => {
+    res.end('<html><title>Served by the proxy</title></html>');
+  });
+  // FYI: using long and weird domain names to avoid ATT DNS hijacking
+  // that resolves everything to some weird search results page.
+  //
+  // @see https://gist.github.com/CollinChaffin/24f6c9652efb3d6d5ef2f5502720ef00
+  const browser = await contextFactory({
+    ...contextOptions,
+    proxy: { server: `localhost:${server.PORT}`, bypass: '1.non.existent.domain.for.the.test, 2.non.existent.domain.for.the.test, .another.test' }
+  });
+
+  const page = await browser.newPage();
+  await page.goto('http://0.non.existent.domain.for.the.test/target.html');
+  expect(await page.title()).toBe('Served by the proxy');
+
+  {
+    const error = await page.goto('http://1.non.existent.domain.for.the.test/target.html').catch(e => e);
+    expect(error.message).toBeTruthy();
+  }
+
+  {
+    const error = await page.goto('http://2.non.existent.domain.for.the.test/target.html').catch(e => e);
+    expect(error.message).toBeTruthy();
+  }
+
+  {
+    const error = await page.goto('http://foo.is.the.another.test/target.html').catch(e => e);
+    expect(error.message).toBeTruthy();
+  }
+
+  {
+    await page.goto('http://3.non.existent.domain.for.the.test/target.html');
+    expect(await page.title()).toBe('Served by the proxy');
+  }
+
+  await browser.close();
+});
+
+it('should use socks proxy', (test, { browserName, platform }) => {
+  test.flaky(platform === 'darwin' && browserName === 'webkit', 'Intermittent page.goto: The network connection was lost error on bots');
+}, async ({ contextFactory, contextOptions, socksPort }) => {
+  const browser = await contextFactory({
+    ...contextOptions,
+    proxy: { server: `socks5://localhost:${socksPort}` }
+  });
+  const page = await browser.newPage();
+  await page.goto('http://non-existent.com');
+  expect(await page.title()).toBe('Served by the SOCKS proxy');
+  await browser.close();
+});
+
+it('should use socks proxy in second page', (test, { browserName, platform }) => {
+  test.flaky(platform === 'darwin' && browserName === 'webkit', 'Intermittent page.goto: The network connection was lost error on bots');
+}, async ({ contextFactory, contextOptions, socksPort }) => {
+  const browser = await contextFactory({
+    ...contextOptions,
+    proxy: { server: `socks5://localhost:${socksPort}` }
+  });
+
+  const page = await browser.newPage();
+  await page.goto('http://non-existent.com');
+  expect(await page.title()).toBe('Served by the SOCKS proxy');
+
+  const page2 = await browser.newPage();
+  await page2.goto('http://non-existent.com');
+  expect(await page2.title()).toBe('Served by the SOCKS proxy');
+
+  await browser.close();
+});
+
+it('does launch without a port', async ({ contextFactory, contextOptions }) => {
+  const browser = await contextFactory({
+    ...contextOptions,
+    proxy: { server: 'http://localhost' }
+  });
+  await browser.close();
+});

test/http.fixtures.ts

@@ -16,11 +16,13 @@
 
 import { folio as base } from 'folio';
 import path from 'path';
+import socks from 'socksv5';
 import { TestServer } from '../utils/testserver';
 
 type HttpWorkerFixtures = {
   asset: (path: string) => string;
   httpService: { server: TestServer, httpsServer: TestServer };
+  socksPort: number,
 };
 
 type HttpTestFixtures = {
@@ -63,4 +65,28 @@ fixtures.httpsServer.init(async ({ httpService }, test) => {
   await test(httpService.httpsServer);
 });
 
+fixtures.socksPort.init(async ({ testWorkerIndex }, run) => {
+  const server = socks.createServer((info, accept, deny) => {
+    let socket;
+    if ((socket = accept(true))) {
+      // Catch and ignore ECONNRESET errors.
+      socket.on('error', () => {});
+      const body = '<html><title>Served by the SOCKS proxy</title></html>';
+      socket.end([
+        'HTTP/1.1 200 OK',
+        'Connection: close',
+        'Content-Type: text/html',
+        'Content-Length: ' + Buffer.byteLength(body),
+        '',
+        body
+      ].join('\r\n'));
+    }
+  });
+  const socksPort = 9107 + testWorkerIndex * 2;
+  server.listen(socksPort, 'localhost');
+  server.useAuth(socks.auth.None());
+  await run(socksPort);
+  server.close();
+}, { scope: 'worker' });
+
 export const folio = fixtures.build();

test/proxy.spec.ts

@@ -14,39 +14,7 @@
  * limitations under the License.
  */
 
-import { folio as baseFolio } from './fixtures';
+import { it, expect } from './fixtures';
-
-import socks from 'socksv5';
-
-const builder = baseFolio.extend<{}, {
-  socksPort: number,
-}>();
-
-builder.socksPort.init(async ({ testWorkerIndex }, run) => {
-  const server = socks.createServer((info, accept, deny) => {
-    let socket;
-    if ((socket = accept(true))) {
-      // Catch and ignore ECONNRESET errors.
-      socket.on('error', () => {});
-      const body = '<html><title>Served by the SOCKS proxy</title></html>';
-      socket.end([
-        'HTTP/1.1 200 OK',
-        'Connection: close',
-        'Content-Type: text/html',
-        'Content-Length: ' + Buffer.byteLength(body),
-        '',
-        body
-      ].join('\r\n'));
-    }
-  });
-  const socksPort = 9107 + testWorkerIndex * 2;
-  server.listen(socksPort, 'localhost');
-  server.useAuth(socks.auth.None());
-  await run(socksPort);
-  server.close();
-}, { scope: 'worker' });
-
-const { it, expect } = builder.build();
 
 it('should throw for bad server value', async ({browserType, browserOptions}) => {
   const error = await browserType.launch({