From 768a97cfdc4b7425d164a13139ed2d4df358f740 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Fri, 27 Aug 2021 08:26:19 -0700 Subject: [PATCH] feat(fetch): set user-agent and other default headers (#8491) --- src/server/browser.ts | 1 + src/server/chromium/crBrowser.ts | 6 ++++ src/server/fetch.ts | 30 ++++++++++------- src/server/firefox/ffBrowser.ts | 6 ++++ src/server/types.ts | 2 +- src/server/webkit/wkBrowser.ts | 4 +++ tests/browsercontext-fetch.spec.ts | 52 ++++++++++++++++++++++++++++++ 7 files changed, 89 insertions(+), 12 deletions(-) diff --git a/src/server/browser.ts b/src/server/browser.ts index bc338849f7..7c07e31700 100644 --- a/src/server/browser.ts +++ b/src/server/browser.ts @@ -78,6 +78,7 @@ export abstract class Browser extends SdkObject { abstract contexts(): BrowserContext[]; abstract isConnected(): boolean; abstract version(): string; + abstract userAgent(): string; _downloadCreated(page: Page, uuid: string, url: string, suggestedFilename?: string) { const download = new Download(page, this.options.downloadsPath || '', uuid, url, suggestedFilename); diff --git a/src/server/chromium/crBrowser.ts b/src/server/chromium/crBrowser.ts index 913f286da0..04ea30e89d 100644 --- a/src/server/chromium/crBrowser.ts +++ b/src/server/chromium/crBrowser.ts @@ -45,6 +45,7 @@ export class CRBrowser extends Browser { private _tracingRecording = false; private _tracingPath: string | null = ''; private _tracingClient: CRSession | undefined; + private _userAgent: string = ''; static async connect(transport: ConnectionTransport, options: BrowserOptions, devtools?: CRDevTools): Promise { const connection = new CRConnection(transport, options.protocolLogger, options.browserLogsCollector); @@ -57,6 +58,7 @@ export class CRBrowser extends Browser { const version = await session.send('Browser.getVersion'); browser._isMac = version.userAgent.includes('Macintosh'); browser._version = version.product.substring(version.product.indexOf('/') + 1); + browser._userAgent = version.userAgent; if (!options.persistent) { await session.send('Target.setAutoAttach', { autoAttach: true, waitForDebuggerOnStart: true, flatten: true }); return browser; @@ -107,6 +109,10 @@ export class CRBrowser extends Browser { return this._version; } + userAgent(): string { + return this._userAgent; + } + isClank(): boolean { return this.options.name === 'clank'; } diff --git a/src/server/fetch.ts b/src/server/fetch.ts index 631777348e..e592e57670 100644 --- a/src/server/fetch.ts +++ b/src/server/fetch.ts @@ -23,16 +23,24 @@ import * as types from './types'; export async function playwrightFetch(context: BrowserContext, params: types.FetchOptions): Promise<{fetchResponse?: types.FetchResponse, error?: string}> { try { - const cookies = await context.cookies(params.url); - const valueArray = cookies.map(c => `${c.name}=${c.value}`); - const clientCookie = params.headers?.['cookie']; - if (clientCookie) - valueArray.unshift(clientCookie); - const cookieHeader = valueArray.join('; '); - if (cookieHeader) { - if (!params.headers) - params.headers = {}; - params.headers['cookie'] = cookieHeader; + const headers: { [name: string]: string } = {}; + if (params.headers) { + for (const [name, value] of Object.entries(params.headers)) + headers[name.toLowerCase()] = value; + } + if (headers['user-agent'] === undefined) + headers['user-agent'] = context._options.userAgent || context._browser.userAgent(); + if (headers['accept'] === undefined) + headers['accept'] = '*/*'; + if (headers['accept-encoding'] === undefined) + headers['accept-encoding'] = 'gzip,deflate'; + + if (headers['cookie'] === undefined) { + const cookies = await context.cookies(params.url); + if (cookies.length) { + const valueArray = cookies.map(c => `${c.name}=${c.value}`); + headers['cookie'] = valueArray.join('; '); + } } if (!params.method) params.method = 'GET'; @@ -48,7 +56,7 @@ export async function playwrightFetch(context: BrowserContext, params: types.Fet // TODO(https://github.com/microsoft/playwright/issues/8381): set user agent const {fetchResponse, setCookie} = await sendRequest(new URL(params.url), { method: params.method, - headers: params.headers, + headers: headers, agent, maxRedirects: 20 }, params.postData); diff --git a/src/server/firefox/ffBrowser.ts b/src/server/firefox/ffBrowser.ts index da3ebc0743..a8931b38ff 100644 --- a/src/server/firefox/ffBrowser.ts +++ b/src/server/firefox/ffBrowser.ts @@ -32,6 +32,7 @@ export class FFBrowser extends Browser { readonly _ffPages: Map; readonly _contexts: Map; private _version = ''; + private _userAgent: string = ''; static async connect(transport: ConnectionTransport, options: BrowserOptions): Promise { const connection = new FFConnection(transport, options.protocolLogger, options.browserLogsCollector); @@ -68,6 +69,7 @@ export class FFBrowser extends Browser { async _initVersion() { const result = await this._connection.send('Browser.getInfo'); this._version = result.version.substring(result.version.indexOf('/') + 1); + this._userAgent = result.userAgent; } isConnected(): boolean { @@ -93,6 +95,10 @@ export class FFBrowser extends Browser { return this._version; } + userAgent(): string { + return this._userAgent; + } + _onDetachedFromTarget(payload: Protocol.Browser.detachedFromTargetPayload) { const ffPage = this._ffPages.get(payload.targetId)!; this._ffPages.delete(payload.targetId); diff --git a/src/server/types.ts b/src/server/types.ts index 985ca12806..da442574de 100644 --- a/src/server/types.ts +++ b/src/server/types.ts @@ -371,7 +371,7 @@ export type SetStorageState = { export type FetchOptions = { url: string, method?: string, - headers?: { [name: string]: string }, + headers?: { [name: string]: string }, postData?: Buffer, }; diff --git a/src/server/webkit/wkBrowser.ts b/src/server/webkit/wkBrowser.ts index 82917f678b..7fd3b10925 100644 --- a/src/server/webkit/wkBrowser.ts +++ b/src/server/webkit/wkBrowser.ts @@ -101,6 +101,10 @@ export class WKBrowser extends Browser { return BROWSER_VERSION; } + userAgent(): string { + return DEFAULT_USER_AGENT; + } + _onDownloadCreated(payload: Protocol.Playwright.downloadCreatedPayload) { const page = this._wkPages.get(payload.pageProxyId); if (!page) diff --git a/tests/browsercontext-fetch.spec.ts b/tests/browsercontext-fetch.spec.ts index 248ee94954..a2ee77ae54 100644 --- a/tests/browsercontext-fetch.spec.ts +++ b/tests/browsercontext-fetch.spec.ts @@ -68,6 +68,29 @@ it('should add session cookies to request', async ({context, server}) => { expect(req.headers.cookie).toEqual('username=John Doe'); }); +it('should not add context cookie if cookie header passed as a parameter', async ({context, server}) => { + await context.addCookies([{ + name: 'username', + value: 'John Doe', + domain: '.my.playwright.dev', + path: '/', + expires: -1, + httpOnly: false, + secure: false, + sameSite: 'Lax', + }]); + const [req] = await Promise.all([ + server.waitForRequest('/empty.html'), + // @ts-expect-error + context._fetch(`http://www.my.playwright.dev:${server.PORT}/empty.html`, { + headers: { + 'Cookie': 'foo=bar' + } + }), + ]); + expect(req.headers.cookie).toEqual('foo=bar'); +}); + it('should follow redirects', async ({context, server}) => { server.setRedirect('/redirect1', '/redirect2'); server.setRedirect('/redirect2', '/simple.json'); @@ -172,3 +195,32 @@ it('should support post data', async ({context, server}) => { expect(response.status()).toBe(200); expect(request.url).toBe('/simple.json'); }); + +it('should add default headers', async ({context, server, page}) => { + const [request] = await Promise.all([ + server.waitForRequest('/empty.html'), + // @ts-expect-error + context._fetch(server.EMPTY_PAGE) + ]); + expect(request.headers['accept']).toBe('*/*'); + const userAgent = await page.evaluate(() => navigator.userAgent); + expect(request.headers['user-agent']).toBe(userAgent); + expect(request.headers['accept-encoding']).toBe('gzip,deflate'); +}); + +it('should allow to override default headers', async ({context, server, page}) => { + const [request] = await Promise.all([ + server.waitForRequest('/empty.html'), + // @ts-expect-error + context._fetch(server.EMPTY_PAGE, { + headers: { + 'User-Agent': 'Playwright', + 'Accept': 'text/html', + 'Accept-Encoding': 'br' + } + }) + ]); + expect(request.headers['accept']).toBe('text/html'); + expect(request.headers['user-agent']).toBe('Playwright'); + expect(request.headers['accept-encoding']).toBe('br'); +});