fix(har): rewrite sizes and make transferSize work in WK/Linux (#8504)

src/server/chromium/crNetworkManager.ts

@@ -361,8 +361,11 @@ export class CRNetworkManager {
     // Under certain conditions we never get the Network.responseReceived
     // event from protocol. @see https://crbug.com/883475
     const response = request.request._existingResponse();
-    if (response)
+    if (response) {
-      response._requestFinished(helper.secondsToRoundishMillis(event.timestamp - request._timestamp), undefined, event.encodedDataLength);
+      request.request._sizes.transferSize = event.encodedDataLength;
+      request.request._sizes.responseBodySize = event.encodedDataLength - response?.headersSize();
+      response._requestFinished(helper.secondsToRoundishMillis(event.timestamp - request._timestamp), undefined);
+    }
     this._requestIdToRequest.delete(request._requestId);
     if (request._interceptionId)
       this._attemptedAuthentications.delete(request._interceptionId);

src/server/firefox/ffNetworkManager.ts

@@ -116,13 +116,17 @@ export class FFNetworkManager {
     if (!request)
       return;
     const response = request.request._existingResponse()!;
+
+    request.request._sizes.transferSize = event.transferSize;
+    request.request._sizes.responseBodySize = event.transferSize - response.headersSize();
+
     // Keep redirected requests in the map for future reference as redirectedFrom.
     const isRedirected = response.status() >= 300 && response.status() <= 399;
     if (isRedirected) {
       response._requestFinished(this._relativeTiming(event.responseEndTime), 'Response body is unavailable for redirect responses');
     } else {
       this._requests.delete(request._id);
-      response._requestFinished(this._relativeTiming(event.responseEndTime), undefined, event.transferSize);
+      response._requestFinished(this._relativeTiming(event.responseEndTime), undefined);
     }
     this._page._frameManager.requestFinished(request.request);
   }

src/server/network.ts

@@ -78,6 +78,11 @@ export function stripFragmentFromUrl(url: string): string {
   return url.substring(0, url.indexOf('#'));
 }
 
+type RequestSizes = {
+  responseBodySize: number;
+  transferSize: number;
+};
+
 export class Request extends SdkObject {
   private _response: Response | null = null;
   private _redirectedFrom: Request | null;
@@ -95,6 +100,7 @@ export class Request extends SdkObject {
   private _waitForResponsePromise: Promise<Response | null>;
   private _waitForResponsePromiseCallback: (value: Response | null) => void = () => {};
   _responseEndTiming = -1;
+  _sizes: RequestSizes = { responseBodySize: 0, transferSize: 0 };
 
   constructor(frame: frames.Frame, redirectedFrom: Request | null, documentId: string | undefined,
     url: string, resourceType: string, method: string, postData: Buffer | null, headers: types.HeadersArray) {
@@ -193,6 +199,22 @@ export class Request extends SdkObject {
       this._headersMap.set('host', host);
     }
   }
+
+  bodySize(): number {
+    return this.postDataBuffer()?.length || 0;
+  }
+
+  headersSize(): number {
+    if (!this._response)
+      return 0;
+    let headersSize = 4; // 4 = 2 spaces + 2 line breaks (GET /path \r\n)
+    headersSize += this.method().length;
+    headersSize += (new URL(this.url())).pathname.length;
+    headersSize += 8; // httpVersion
+    for (const header of this._headers)
+      headersSize += header.name.length + header.value.length + 4; // 4 = ': ' + '\r\n'
+    return headersSize;
+  }
 }
 
 export class Route extends SdkObject {
@@ -302,8 +324,7 @@ export class Response extends SdkObject {
   private _serverAddrPromiseCallback: (arg?: RemoteAddr) => void = () => {};
   private _securityDetailsPromise: Promise<SecurityDetails|undefined>;
   private _securityDetailsPromiseCallback: (arg?: SecurityDetails) => void = () => {};
-  _httpVersion: string | undefined;
+  private _httpVersion: string | undefined;
-  _transferSize: number | undefined;
 
   constructor(request: Request, status: number, statusText: string, headers: types.HeadersArray, timing: ResourceTiming, getResponseBodyCallback: GetResponseBodyCallback, httpVersion?: string) {
     super(request.frame(), 'response');
@@ -337,9 +358,8 @@ export class Response extends SdkObject {
     this._securityDetailsPromiseCallback(securityDetails);
   }
 
-  _requestFinished(responseEndTiming: number, error?: string, transferSize?: number) {
+  _requestFinished(responseEndTiming: number, error?: string) {
     this._request._responseEndTiming = Math.max(responseEndTiming, this._timing.responseStart);
-    this._transferSize = transferSize;
     this._finishedPromiseCallback({ error });
   }
 
@@ -401,6 +421,33 @@ export class Response extends SdkObject {
   frame(): frames.Frame {
     return this._request.frame();
   }
+
+  transferSize(): number | undefined {
+    return this._request._sizes.transferSize;
+  }
+
+  bodySize(): number {
+    return this._request._sizes.responseBodySize;
+  }
+
+  httpVersion(): string {
+    if (!this._httpVersion)
+      return 'HTTP/1.1';
+    if (this._httpVersion === 'http/1.1')
+      return 'HTTP/1.1';
+    return this._httpVersion;
+  }
+
+  headersSize(): number {
+    let headersSize = 4; // 4 = 2 spaces + 2 line breaks (HTTP/1.1 200 Ok\r\n)
+    headersSize += 8; // httpVersion;
+    headersSize += 3; // statusCode;
+    headersSize += this.statusText().length;
+    for (const header of this.headers())
+      headersSize += header.name.length + header.value.length + 4; // 4 = ': ' + '\r\n'
+    headersSize += 2; // '\r\n'
+    return headersSize;
+  }
 }
 
 export class InterceptedResponse extends SdkObject {

src/server/supplements/har/harTracer.ts

@@ -14,13 +14,11 @@
  * limitations under the License.
  */
 
-import { URL } from 'url';
 import { BrowserContext } from '../../browserContext';
 import { helper } from '../../helper';
 import * as network from '../../network';
 import { Page } from '../../page';
 import * as har from './har';
-import * as types from '../../types';
 import { calculateSha1, monotonicTime } from '../../../utils/utils';
 import { eventsHelper, RegisteredListener } from '../../../utils/eventsHelper';
 import * as mime from 'mime';
@@ -158,7 +156,7 @@ export class HarTracer {
         queryString: [...url.searchParams].map(e => ({ name: e[0], value: e[1] })),
         postData: postDataForHar(request, this._options.content),
         headersSize: -1,
-        bodySize: calculateRequestBodySize(request) || 0,
+        bodySize: request.bodySize(),
       },
       response: {
         status: -1,
@@ -204,16 +202,15 @@ export class HarTracer {
     if (!response)
       return;
 
-    const httpVersion = normaliseHttpVersion(response._httpVersion);
+    const httpVersion = response.httpVersion();
-    const transferSize = response._transferSize || -1;
+    const transferSize = response.transferSize() || -1;
-    const headersSize = calculateResponseHeadersSize(httpVersion, response.status(), response.statusText(), response.headers());
+    const responseHeadersSize = response.headersSize();
-    const bodySize = transferSize !== -1 ? transferSize - headersSize : -1;
 
     harEntry.request.httpVersion = httpVersion;
-    harEntry.response.bodySize = bodySize;
+    harEntry.response.bodySize = response.bodySize();
-    harEntry.response.headersSize = headersSize;
+    harEntry.response.headersSize = responseHeadersSize;
     harEntry.response._transferSize = transferSize;
-    harEntry.request.headersSize = calculateRequestHeadersSize(request.method(), request.url(), httpVersion, request.headers());
+    harEntry.request.headersSize = request.headersSize();
 
     const promise = response.body().then(buffer => {
       const content = harEntry.response.content;
@@ -258,7 +255,7 @@ export class HarTracer {
     harEntry.response = {
       status: response.status(),
       statusText: response.statusText(),
-      httpVersion: normaliseHttpVersion(response._httpVersion),
+      httpVersion: response.httpVersion(),
       cookies: cookiesForHar(response.headerValue('set-cookie'), '\n'),
       headers: response.headers().map(header => ({ name: header.name, value: header.value })),
       content: {
@@ -397,33 +394,3 @@ function parseCookie(c: string): har.Cookie {
   }
   return cookie;
 }
-
-function calculateResponseHeadersSize(protocol: string, status: number, statusText: string , headers: types.HeadersArray) {
-  let rawHeaders = `${protocol} ${status} ${statusText}\r\n`;
-  for (const header of headers)
-    rawHeaders += `${header.name}: ${header.value}\r\n`;
-  rawHeaders += '\r\n';
-  return rawHeaders.length;
-}
-
-function calculateRequestHeadersSize(method: string, url: string, httpVersion: string, headers: types.HeadersArray) {
-  let rawHeaders = `${method} ${(new URL(url)).pathname} ${httpVersion}\r\n`;
-  for (const header of headers)
-    rawHeaders += `${header.name}: ${header.value}\r\n`;
-  return rawHeaders.length;
-}
-
-function normaliseHttpVersion(httpVersion?: string) {
-  if (!httpVersion)
-    return FALLBACK_HTTP_VERSION;
-  if (httpVersion === 'http/1.1')
-    return 'HTTP/1.1';
-  return httpVersion;
-}
-
-function calculateRequestBodySize(request: network.Request): number|undefined {
-  const postData = request.postDataBuffer();
-  if (!postData)
-    return;
-  return new TextEncoder().encode(postData.toString('utf8')).length;
-}

src/server/webkit/wkPage.ts

@@ -380,6 +380,7 @@ export class WKPage implements PageDelegate {
       eventsHelper.addEventListener(this._session, 'Network.responseReceived', e => this._onResponseReceived(e)),
       eventsHelper.addEventListener(this._session, 'Network.loadingFinished', e => this._onLoadingFinished(e)),
       eventsHelper.addEventListener(this._session, 'Network.loadingFailed', e => this._onLoadingFailed(e)),
+      eventsHelper.addEventListener(this._session, 'Network.dataReceived', e => this._onDataReceived(e)),
       eventsHelper.addEventListener(this._session, 'Network.webSocketCreated', e => this._page._frameManager.onWebSocketCreated(e.requestId, e.url)),
       eventsHelper.addEventListener(this._session, 'Network.webSocketWillSendHandshakeRequest', e => this._page._frameManager.onWebSocketRequest(e.requestId)),
       eventsHelper.addEventListener(this._session, 'Network.webSocketHandshakeResponseReceived', e => this._page._frameManager.onWebSocketResponse(e.requestId, e.response.status, e.response.statusText)),
@@ -1049,11 +1050,10 @@ export class WKPage implements PageDelegate {
         validFrom: responseReceivedPayload?.response.security?.certificate?.validFrom,
         validTo: responseReceivedPayload?.response.security?.certificate?.validUntil,
       });
-      const { responseBodyBytesReceived, responseHeaderBytesReceived } = event.metrics || {};
+      request.request._sizes.transferSize += response.headersSize();
-      const transferSize = responseBodyBytesReceived ? responseBodyBytesReceived + (responseHeaderBytesReceived || 0) : undefined;
       if (event.metrics?.protocol)
         response._setHttpVersion(event.metrics.protocol);
-      response._requestFinished(helper.secondsToRoundishMillis(event.timestamp - request._timestamp), undefined, transferSize);
+      response._requestFinished(helper.secondsToRoundishMillis(event.timestamp - request._timestamp), undefined);
     }
 
     this._requestIdToResponseReceivedPayloadEvent.delete(request._requestId);
@@ -1081,6 +1081,14 @@ export class WKPage implements PageDelegate {
     this._page._frameManager.requestFailed(request.request, event.errorText.includes('cancelled'));
   }
 
+  _onDataReceived(event: Protocol.Network.dataReceivedPayload) {
+    const request = this._requestIdToRequest.get(event.requestId);
+    if (!request)
+      return;
+    request.request._sizes.responseBodySize += event.encodedDataLength === -1 ? event.dataLength : event.encodedDataLength;
+    request.request._sizes.transferSize += event.encodedDataLength === -1 ? event.dataLength : event.encodedDataLength;
+  }
+
   async _grantPermissions(origin: string, permissions: string[]) {
     const webPermissionToProtocol = new Map<string, string>([
       ['geolocation', 'geolocation'],

tests/har.spec.ts

@@ -262,23 +262,34 @@ it('should include content', async ({ contextFactory, server }, testInfo) => {
   expect(log.entries[1].response.content.compression).toBe(0);
 });
 
-it('should include sizes', async ({ contextFactory, server, browserName, platform }, testInfo) => {
+it('should include sizes', async ({ contextFactory, server, asset }, testInfo) => {
-  it.fixme(browserName === 'webkit' && platform === 'linux', 'blocked by libsoup3');
-
   const { page, getLog } = await pageWithHar(contextFactory, testInfo);
   await page.goto(server.PREFIX + '/har.html');
   const log = await getLog();
-
+  expect(log.entries.length).toBe(2);
+  expect(log.entries[0].request.url.endsWith('har.html')).toBe(true);
   expect(log.entries[0].request.headersSize).toBeGreaterThanOrEqual(280);
-  expect(log.entries[0].response.bodySize).toBeGreaterThanOrEqual(96);
+  expect(log.entries[0].response.bodySize).toBe(fs.statSync(asset('har.html')).size);
-  expect(log.entries[0].response.headersSize).toBe(198);
+  expect(log.entries[0].response.headersSize).toBeGreaterThanOrEqual(198);
   expect(log.entries[0].response._transferSize).toBeGreaterThanOrEqual(294);
 
-  expect(log.entries[1].response.bodySize).toBeGreaterThanOrEqual(37);
+  expect(log.entries[1].request.url.endsWith('one-style.css')).toBe(true);
-  expect(log.entries[1].response.headersSize).toBe(197);
+  expect(log.entries[1].response.bodySize).toBe(fs.statSync(asset('one-style.css')).size);
+  expect(log.entries[1].response.headersSize).toBeGreaterThanOrEqual(197);
   expect(log.entries[1].response._transferSize).toBeGreaterThanOrEqual(234);
 });
 
+it('should work with gzip compression', async ({ contextFactory, server, browserName }, testInfo) => {
+  it.fixme(browserName === 'webkit');
+  const { page, getLog } = await pageWithHar(contextFactory, testInfo);
+  server.enableGzip('/simple.json');
+  const response = await page.goto(server.PREFIX + '/simple.json');
+  expect(response.headers()['content-encoding']).toBe('gzip');
+  const log = await getLog();
+  expect(log.entries.length).toBe(1);
+  expect(log.entries[0].response.content.compression).toBe(-20);
+});
+
 it('should calculate time', async ({ contextFactory, server }, testInfo) => {
   const { page, getLog } = await pageWithHar(contextFactory, testInfo);
   await page.goto(server.PREFIX + '/har.html');
@@ -286,15 +297,14 @@ it('should calculate time', async ({ contextFactory, server }, testInfo) => {
   expect(log.entries[0].time).toBeGreaterThan(0);
 });
 
-it('should report the correct _transferSize with PNG files', async ({ contextFactory, server, browserName, platform }, testInfo) => {
+it('should report the correct _transferSize with PNG files', async ({ contextFactory, server, asset }, testInfo) => {
-  it.fixme(browserName === 'webkit' && platform === 'linux', 'blocked by libsoup3');
   const { page, getLog } = await pageWithHar(contextFactory, testInfo);
   await page.goto(server.EMPTY_PAGE);
   await page.setContent(`
     <img src="${server.PREFIX}/pptr.png" />
   `);
   const log = await getLog();
-  expect(log.entries[1].response._transferSize).toBe(6323);
+  expect(log.entries[1].response._transferSize).toBeGreaterThan(fs.statSync(asset('pptr.png')).size);
 });
 
 it('should have -1 _transferSize when its a failed request', async ({ contextFactory, server }, testInfo) => {
@@ -311,14 +321,14 @@ it('should have -1 _transferSize when its a failed request', async ({ contextFac
   expect(log.entries[1].response._transferSize).toBe(-1);
 });
 
-it('should report the correct body size', async ({ contextFactory, server }, testInfo) => {
+it('should report the correct request body size', async ({ contextFactory, server }, testInfo) => {
   server.setRoute('/api', (req, res) => res.end());
   const { page, getLog } = await pageWithHar(contextFactory, testInfo);
   await page.goto(server.EMPTY_PAGE);
   await Promise.all([
-    page.waitForResponse(server.PREFIX + '/api'),
+    page.waitForResponse(server.PREFIX + '/api1'),
     page.evaluate(() => {
-      fetch('/api', {
+      fetch('/api1', {
         method: 'POST',
         body: 'abc123'
       });
@@ -328,6 +338,31 @@ it('should report the correct body size', async ({ contextFactory, server }, tes
   expect(log.entries[1].request.bodySize).toBe(6);
 });
 
+it('should report the correct request body size when the bodySize is 0', async ({ contextFactory, server }, testInfo) => {
+  server.setRoute('/api', (req, res) => res.end());
+  const { page, getLog } = await pageWithHar(contextFactory, testInfo);
+  await page.goto(server.EMPTY_PAGE);
+  await Promise.all([
+    page.waitForResponse(server.PREFIX + '/api2'),
+    page.evaluate(() => {
+      fetch('/api2', {
+        method: 'POST',
+        body: ''
+      });
+    })
+  ]);
+  const log = await getLog();
+  expect(log.entries[1].request.bodySize).toBe(0);
+});
+
+it('should report the correct response body size when the bodySize is 0', async ({ contextFactory, server }, testInfo) => {
+  server.setRoute('/empty.html', (req, res) => res.end(''));
+  const { page, getLog } = await pageWithHar(contextFactory, testInfo);
+  await page.goto(server.EMPTY_PAGE);
+  const log = await getLog();
+  expect(log.entries[0].response.bodySize).toBe(0);
+});
+
 it('should have popup requests', async ({ contextFactory, server }, testInfo) => {
   const { page, getLog } = await pageWithHar(contextFactory, testInfo);
   await page.goto(server.EMPTY_PAGE);