feat(fetch): set user-agent and other default headers (#8491)

src/server/browser.ts

@@ -78,6 +78,7 @@ export abstract class Browser extends SdkObject {
   abstract contexts(): BrowserContext[];
   abstract isConnected(): boolean;
   abstract version(): string;
+  abstract userAgent(): string;
 
   _downloadCreated(page: Page, uuid: string, url: string, suggestedFilename?: string) {
     const download = new Download(page, this.options.downloadsPath || '', uuid, url, suggestedFilename);

src/server/chromium/crBrowser.ts

@@ -45,6 +45,7 @@ export class CRBrowser extends Browser {
   private _tracingRecording = false;
   private _tracingPath: string | null = '';
   private _tracingClient: CRSession | undefined;
+  private _userAgent: string = '';
 
   static async connect(transport: ConnectionTransport, options: BrowserOptions, devtools?: CRDevTools): Promise<CRBrowser> {
     const connection = new CRConnection(transport, options.protocolLogger, options.browserLogsCollector);
@@ -57,6 +58,7 @@ export class CRBrowser extends Browser {
     const version = await session.send('Browser.getVersion');
     browser._isMac = version.userAgent.includes('Macintosh');
     browser._version = version.product.substring(version.product.indexOf('/') + 1);
+    browser._userAgent = version.userAgent;
     if (!options.persistent) {
       await session.send('Target.setAutoAttach', { autoAttach: true, waitForDebuggerOnStart: true, flatten: true });
       return browser;
@@ -107,6 +109,10 @@ export class CRBrowser extends Browser {
     return this._version;
   }
 
+  userAgent(): string {
+    return this._userAgent;
+  }
+
   isClank(): boolean {
     return this.options.name === 'clank';
   }

src/server/fetch.ts

@@ -23,16 +23,24 @@ import * as types from './types';
 
 export async function playwrightFetch(context: BrowserContext, params: types.FetchOptions): Promise<{fetchResponse?: types.FetchResponse, error?: string}> {
   try {
-    const cookies = await context.cookies(params.url);
-    const valueArray = cookies.map(c => `${c.name}=${c.value}`);
-    const clientCookie = params.headers?.['cookie'];
-    if (clientCookie)
-      valueArray.unshift(clientCookie);
-    const cookieHeader = valueArray.join('; ');
-    if (cookieHeader) {
-      if (!params.headers)
-        params.headers = {};
-      params.headers['cookie'] = cookieHeader;
+    const headers: { [name: string]: string } = {};
+    if (params.headers) {
+      for (const [name, value] of Object.entries(params.headers))
+        headers[name.toLowerCase()] = value;
+    }
+    if (headers['user-agent'] === undefined)
+      headers['user-agent'] = context._options.userAgent || context._browser.userAgent();
+    if (headers['accept'] === undefined)
+      headers['accept'] = '*/*';
+    if (headers['accept-encoding'] === undefined)
+      headers['accept-encoding'] = 'gzip,deflate';
+
+    if (headers['cookie'] === undefined) {
+      const cookies = await context.cookies(params.url);
+      if (cookies.length) {
+        const valueArray = cookies.map(c => `${c.name}=${c.value}`);
+        headers['cookie'] = valueArray.join('; ');
+      }
     }
     if (!params.method)
       params.method = 'GET';
@@ -48,7 +56,7 @@ export async function playwrightFetch(context: BrowserContext, params: types.Fet
     // TODO(https://github.com/microsoft/playwright/issues/8381): set user agent
     const {fetchResponse, setCookie} = await sendRequest(new URL(params.url), {
       method: params.method,
-      headers: params.headers,
+      headers: headers,
       agent,
       maxRedirects: 20
     }, params.postData);

src/server/firefox/ffBrowser.ts

@@ -32,6 +32,7 @@ export class FFBrowser extends Browser {
   readonly _ffPages: Map<string, FFPage>;
   readonly _contexts: Map<string, FFBrowserContext>;
   private _version = '';
+  private _userAgent: string = '';
 
   static async connect(transport: ConnectionTransport, options: BrowserOptions): Promise<FFBrowser> {
     const connection = new FFConnection(transport, options.protocolLogger, options.browserLogsCollector);
@@ -68,6 +69,7 @@ export class FFBrowser extends Browser {
   async _initVersion() {
     const result = await this._connection.send('Browser.getInfo');
     this._version = result.version.substring(result.version.indexOf('/') + 1);
+    this._userAgent = result.userAgent;
   }
 
   isConnected(): boolean {
@@ -93,6 +95,10 @@ export class FFBrowser extends Browser {
     return this._version;
   }
 
+  userAgent(): string {
+    return this._userAgent;
+  }
+
   _onDetachedFromTarget(payload: Protocol.Browser.detachedFromTargetPayload) {
     const ffPage = this._ffPages.get(payload.targetId)!;
     this._ffPages.delete(payload.targetId);

src/server/types.ts

@@ -371,7 +371,7 @@ export type SetStorageState = {
 export type FetchOptions = {
   url: string,
   method?: string,
-  headers?:  { [name: string]: string },
+  headers?: { [name: string]: string },
   postData?: Buffer,
 };
 

src/server/webkit/wkBrowser.ts

@@ -101,6 +101,10 @@ export class WKBrowser extends Browser {
     return BROWSER_VERSION;
   }
 
+  userAgent(): string {
+    return DEFAULT_USER_AGENT;
+  }
+
   _onDownloadCreated(payload: Protocol.Playwright.downloadCreatedPayload) {
     const page = this._wkPages.get(payload.pageProxyId);
     if (!page)

tests/browsercontext-fetch.spec.ts

@@ -68,6 +68,29 @@ it('should add session cookies to request', async ({context, server}) => {
   expect(req.headers.cookie).toEqual('username=John Doe');
 });
 
+it('should not add context cookie if cookie header passed as a parameter', async ({context, server}) => {
+  await context.addCookies([{
+    name: 'username',
+    value: 'John Doe',
+    domain: '.my.playwright.dev',
+    path: '/',
+    expires: -1,
+    httpOnly: false,
+    secure: false,
+    sameSite: 'Lax',
+  }]);
+  const [req] = await Promise.all([
+    server.waitForRequest('/empty.html'),
+    // @ts-expect-error
+    context._fetch(`http://www.my.playwright.dev:${server.PORT}/empty.html`, {
+      headers: {
+        'Cookie': 'foo=bar'
+      }
+    }),
+  ]);
+  expect(req.headers.cookie).toEqual('foo=bar');
+});
+
 it('should follow redirects', async ({context, server}) => {
   server.setRedirect('/redirect1', '/redirect2');
   server.setRedirect('/redirect2', '/simple.json');
@@ -172,3 +195,32 @@ it('should support post data', async ({context, server}) => {
   expect(response.status()).toBe(200);
   expect(request.url).toBe('/simple.json');
 });
+
+it('should add default headers', async ({context, server, page}) => {
+  const [request] = await Promise.all([
+    server.waitForRequest('/empty.html'),
+    // @ts-expect-error
+    context._fetch(server.EMPTY_PAGE)
+  ]);
+  expect(request.headers['accept']).toBe('*/*');
+  const userAgent = await page.evaluate(() => navigator.userAgent);
+  expect(request.headers['user-agent']).toBe(userAgent);
+  expect(request.headers['accept-encoding']).toBe('gzip,deflate');
+});
+
+it('should allow to override default headers', async ({context, server, page}) => {
+  const [request] = await Promise.all([
+    server.waitForRequest('/empty.html'),
+    // @ts-expect-error
+    context._fetch(server.EMPTY_PAGE, {
+      headers: {
+        'User-Agent': 'Playwright',
+        'Accept': 'text/html',
+        'Accept-Encoding': 'br'
+      }
+    })
+  ]);
+  expect(request.headers['accept']).toBe('text/html');
+  expect(request.headers['user-agent']).toBe('Playwright');
+  expect(request.headers['accept-encoding']).toBe('br');
+});