fix(headers): report raw request headers on Chromium (#4207)

src/client/network.ts

@@ -51,7 +51,7 @@ export class Request extends ChannelOwner<channels.RequestChannel, channels.Requ
   private _redirectedFrom: Request | null = null;
   private _redirectedTo: Request | null = null;
   _failureText: string | null = null;
-  private _headers: Headers;
+  _headers: Headers;
   private _postData: Buffer | null;
   _timing: ResourceTiming;
 
@@ -258,6 +258,7 @@ export class Response extends ChannelOwner<channels.ResponseChannel, channels.Re
     super(parent, type, guid, initializer);
     this._headers = headersArrayToObject(initializer.headers, true /* lowerCase */);
     this._request = Request.from(this._initializer.request);
+    this._request._headers = headersArrayToObject(initializer.requestHeaders, true /* lowerCase */);
     Object.assign(this._request._timing, this._initializer.timing);
   }
 

src/dispatchers/networkDispatchers.ts

@@ -58,6 +58,7 @@ export class ResponseDispatcher extends Dispatcher<Response, channels.ResponseIn
       url: response.url(),
       status: response.status(),
       statusText: response.statusText(),
+      requestHeaders: response.request().headers(),
       headers: response.headers(),
       timing: response.timing()
     });

src/protocol/channels.ts

@@ -2152,6 +2152,10 @@ export type ResponseInitializer = {
   url: string,
   status: number,
   statusText: string,
+  requestHeaders: {
+    name: string,
+    value: string,
+  }[],
   headers: {
     name: string,
     value: string,

src/protocol/protocol.yml

@@ -1811,6 +1811,13 @@ Response:
     url: string
     status: number
     statusText: string
+    requestHeaders:
+      type: array
+      items:
+        type: object
+        properties:
+          name: string
+          value: string
     headers:
       type: array
       items:

src/server/chromium/crNetworkManager.ts

@@ -37,6 +37,7 @@ export class CRNetworkManager {
   private _protocolRequestInterceptionEnabled = false;
   private _requestIdToRequestPausedEvent = new Map<string, Protocol.Fetch.requestPausedPayload>();
   private _eventListeners: RegisteredListener[];
+  private _requestIdToExtraInfo = new Map<string, Protocol.Network.requestWillBeSentExtraInfoPayload>();
 
   constructor(client: CRSession, page: Page, parentManager: CRNetworkManager | null) {
     this._client = client;
@@ -50,6 +51,7 @@ export class CRNetworkManager {
       helper.addEventListener(session, 'Fetch.requestPaused', this._onRequestPaused.bind(this, workerFrame)),
       helper.addEventListener(session, 'Fetch.authRequired', this._onAuthRequired.bind(this)),
       helper.addEventListener(session, 'Network.requestWillBeSent', this._onRequestWillBeSent.bind(this, workerFrame)),
+      helper.addEventListener(session, 'Network.requestWillBeSentExtraInfo', this._onRequestWillBeSentExtraInfo.bind(this)),
       helper.addEventListener(session, 'Network.responseReceived', this._onResponseReceived.bind(this)),
       helper.addEventListener(session, 'Network.loadingFinished', this._onLoadingFinished.bind(this)),
       helper.addEventListener(session, 'Network.loadingFailed', this._onLoadingFailed.bind(this)),
@@ -116,9 +118,22 @@ export class CRNetworkManager {
       } else {
         this._requestIdToRequestWillBeSentEvent.set(event.requestId, event);
       }
-      return;
+    } else {
+      this._onRequest(workerFrame, event, null);
+    }
+    const extraInfo = this._requestIdToExtraInfo.get(event.requestId);
+    if (extraInfo)
+      this._onRequestWillBeSentExtraInfo(extraInfo);
+  }
+
+  _onRequestWillBeSentExtraInfo(event: Protocol.Network.requestWillBeSentExtraInfoPayload) {
+    const request = this._requestIdToRequest.get(event.requestId);
+    if (request) {
+      request.request._updateWithRawHeaders(headersObjectToArray(event.headers));
+      this._requestIdToExtraInfo.delete(event.requestId);
+    } else {
+      this._requestIdToExtraInfo.set(event.requestId, event);
     }
-    this._onRequest(workerFrame, event, null);
   }
 
   _onAuthRequired(event: Protocol.Fetch.authRequiredPayload) {

src/server/chromium/crPage.ts

@@ -128,19 +128,19 @@ export class CRPage implements PageDelegate {
   }
 
   async updateExtraHTTPHeaders(): Promise<void> {
-    await this._forAllFrameSessions(frame => frame._updateExtraHTTPHeaders());
+    await this._forAllFrameSessions(frame => frame._updateExtraHTTPHeaders(false));
   }
 
   async updateGeolocation(): Promise<void> {
-    await this._forAllFrameSessions(frame => frame._updateGeolocation());
+    await this._forAllFrameSessions(frame => frame._updateGeolocation(false));
   }
 
   async updateOffline(): Promise<void> {
-    await this._forAllFrameSessions(frame => frame._updateOffline());
+    await this._forAllFrameSessions(frame => frame._updateOffline(false));
   }
 
   async updateHttpCredentials(): Promise<void> {
-    await this._forAllFrameSessions(frame => frame._updateHttpCredentials());
+    await this._forAllFrameSessions(frame => frame._updateHttpCredentials(false));
   }
 
   async setViewportSize(viewportSize: types.Size): Promise<void> {
@@ -153,11 +153,11 @@ export class CRPage implements PageDelegate {
   }
 
   async updateEmulateMedia(): Promise<void> {
-    await this._forAllFrameSessions(frame => frame._updateEmulateMedia());
+    await this._forAllFrameSessions(frame => frame._updateEmulateMedia(false));
   }
 
   async updateRequestInterception(): Promise<void> {
-    await this._forAllFrameSessions(frame => frame._updateRequestInterception());
+    await this._forAllFrameSessions(frame => frame._updateRequestInterception(false));
   }
 
   async setFileChooserIntercepted(enabled: boolean) {
@@ -446,12 +446,12 @@ class FrameSession {
       promises.push(emulateLocale(this._client, options.locale));
     if (options.timezoneId)
       promises.push(emulateTimezone(this._client, options.timezoneId));
-    promises.push(this._updateGeolocation());
+    promises.push(this._updateGeolocation(true));
-    promises.push(this._updateExtraHTTPHeaders());
+    promises.push(this._updateExtraHTTPHeaders(true));
-    promises.push(this._updateRequestInterception());
+    promises.push(this._updateRequestInterception(true));
-    promises.push(this._updateOffline());
+    promises.push(this._updateOffline(true));
-    promises.push(this._updateHttpCredentials());
+    promises.push(this._updateHttpCredentials(true));
-    promises.push(this._updateEmulateMedia());
+    promises.push(this._updateEmulateMedia(true));
     for (const binding of this._crPage._browserContext._pageBindings.values())
       promises.push(this._initBinding(binding));
     for (const binding of this._crPage._page._pageBindings.values())
@@ -794,27 +794,31 @@ class FrameSession {
     }
   }
 
-  async _updateExtraHTTPHeaders(): Promise<void> {
+  async _updateExtraHTTPHeaders(initial: boolean): Promise<void> {
     const headers = network.mergeHeaders([
       this._crPage._browserContext._options.extraHTTPHeaders,
       this._page._state.extraHTTPHeaders
     ]);
-    await this._client.send('Network.setExtraHTTPHeaders', { headers: headersArrayToObject(headers, false /* lowerCase */) });
+    if (!initial || headers.length)
+      await this._client.send('Network.setExtraHTTPHeaders', { headers: headersArrayToObject(headers, false /* lowerCase */) });
   }
 
-  async _updateGeolocation(): Promise<void> {
+  async _updateGeolocation(initial: boolean): Promise<void> {
     const geolocation = this._crPage._browserContext._options.geolocation;
-    await this._client.send('Emulation.setGeolocationOverride', geolocation || {});
+    if (!initial || geolocation)
+      await this._client.send('Emulation.setGeolocationOverride', geolocation || {});
   }
 
-  async _updateOffline(): Promise<void> {
+  async _updateOffline(initial: boolean): Promise<void> {
     const offline = !!this._crPage._browserContext._options.offline;
-    await this._networkManager.setOffline(offline);
+    if (!initial || offline)
+      await this._networkManager.setOffline(offline);
   }
 
-  async _updateHttpCredentials(): Promise<void> {
+  async _updateHttpCredentials(initial: boolean): Promise<void> {
     const credentials = this._crPage._browserContext._options.httpCredentials || null;
-    await this._networkManager.authenticate(credentials);
+    if (!initial || credentials)
+      await this._networkManager.authenticate(credentials);
   }
 
   async _updateViewport(): Promise<void> {
@@ -855,13 +859,13 @@ class FrameSession {
     await Promise.all(promises);
   }
 
-  async _updateEmulateMedia(): Promise<void> {
+  async _updateEmulateMedia(initial: boolean): Promise<void> {
     const colorScheme = this._page._state.colorScheme || this._crPage._browserContext._options.colorScheme || 'light';
     const features = colorScheme ? [{ name: 'prefers-color-scheme', value: colorScheme }] : [];
     await this._client.send('Emulation.setEmulatedMedia', { media: this._page._state.mediaType || '', features });
   }
 
-  async _updateRequestInterception(): Promise<void> {
+  async _updateRequestInterception(initial: boolean): Promise<void> {
     await this._networkManager.setRequestInterception(this._page._needsRequestInterception());
   }
 

src/server/network.ts

@@ -169,6 +169,10 @@ export class Request {
       return null;
     return new Route(this, this._routeDelegate);
   }
+
+  _updateWithRawHeaders(headers: types.HeadersArray) {
+    this._headers = headers;
+  }
 }
 
 export class Route {

test/network-request.spec.ts

@@ -82,7 +82,19 @@ it('should return headers', async ({page, server, isChromium, isFirefox, isWebKi
 });
 
 it('should get the same headers as the server', (test, { browserName }) => {
-  test.fail(browserName === 'chromium' || browserName === 'webkit', 'Provisional headers differ from those in network stack');
+  test.fail(browserName === 'webkit', 'Provisional headers differ from those in network stack');
+}, async ({ page, server }) => {
+  let serverRequest;
+  server.setRoute('/empty.html', (request, response) => {
+    serverRequest = request;
+    response.end('done');
+  });
+  const response = await page.goto(server.PREFIX + '/empty.html');
+  expect(response.request().headers()).toEqual(serverRequest.headers);
+});
+
+it('should get the same headers as the server CORP', (test, { browserName }) => {
+  test.fail(browserName === 'webkit', 'Provisional headers differ from those in network stack');
 }, async ({page, server}) => {
   await page.goto(server.PREFIX + '/empty.html');
   let serverRequest;
@@ -91,14 +103,14 @@ it('should get the same headers as the server', (test, { browserName }) => {
     response.writeHead(200, { 'Access-Control-Allow-Origin': '*' });
     response.end('done');
   });
-  const requestPromise = page.waitForEvent('request');
+  const responsePromise = page.waitForEvent('response');
   const text = await page.evaluate(async url => {
     const data = await fetch(url);
     return data.text();
   }, server.CROSS_PROCESS_PREFIX + '/something');
-  const request = await requestPromise;
+  const response = await responsePromise;
   expect(text).toBe('done');
-  expect(request.headers()).toEqual(serverRequest.headers);
+  expect(response.request().headers()).toEqual(serverRequest.headers);
 });
 
 it('should return postData', async ({page, server}) => {