feat(har): add bodySize, transportSize, headersSize (#7470)

Co-authored-by: tnolet <tim@checklyhq.com>

.gitignore

@@ -7,6 +7,7 @@
 *.swp
 *.pyc
 .vscode
+.idea
 yarn.lock
 /src/generated/*
 lib/

src/server/chromium/crNetworkManager.ts

@@ -310,7 +310,7 @@ export class CRNetworkManager {
         responseStart: -1,
       };
     }
-    const response = new network.Response(request.request, responsePayload.status, responsePayload.statusText, headersObjectToArray(responsePayload.headers), timing, getResponseBody);
+    const response = new network.Response(request.request, responsePayload.status, responsePayload.statusText, headersObjectToArray(responsePayload.headers), timing, getResponseBody, responsePayload.protocol);
     if (responsePayload?.remoteIPAddress && typeof responsePayload?.remotePort === 'number') {
       response._serverAddrFinished({
         ipAddress: responsePayload.remoteIPAddress,
@@ -361,7 +361,7 @@ export class CRNetworkManager {
     // event from protocol. @see https://crbug.com/883475
     const response = request.request._existingResponse();
     if (response)
-      response._requestFinished(helper.secondsToRoundishMillis(event.timestamp - request._timestamp));
+      response._requestFinished(helper.secondsToRoundishMillis(event.timestamp - request._timestamp), undefined, event.encodedDataLength);
     this._requestIdToRequest.delete(request._requestId);
     if (request._interceptionId)
       this._attemptedAuthentications.delete(request._interceptionId);

src/server/firefox/ffNetworkManager.ts

@@ -118,7 +118,7 @@ export class FFNetworkManager {
       response._requestFinished(this._relativeTiming(event.responseEndTime), 'Response body is unavailable for redirect responses');
     } else {
       this._requests.delete(request._id);
-      response._requestFinished(this._relativeTiming(event.responseEndTime));
+      response._requestFinished(this._relativeTiming(event.responseEndTime), undefined, event.transferSize);
     }
     this._page._frameManager.requestFinished(request.request);
   }

src/server/network.ts

@@ -311,8 +311,10 @@ export class Response extends SdkObject {
   private _serverAddrPromiseCallback: (arg?: RemoteAddr) => void = () => {};
   private _securityDetailsPromise: Promise<SecurityDetails|undefined>;
   private _securityDetailsPromiseCallback: (arg?: SecurityDetails) => void = () => {};
+  _httpVersion: string | undefined;
+  _transferSize: number | undefined;
 
-  constructor(request: Request, status: number, statusText: string, headers: types.HeadersArray, timing: ResourceTiming, getResponseBodyCallback: GetResponseBodyCallback) {
+  constructor(request: Request, status: number, statusText: string, headers: types.HeadersArray, timing: ResourceTiming, getResponseBodyCallback: GetResponseBodyCallback, httpVersion?: string) {
     super(request.frame(), 'response');
     this._request = request;
     this._timing = timing;
@@ -333,6 +335,7 @@ export class Response extends SdkObject {
       this._finishedPromiseCallback = f;
     });
     this._request._setResponse(this);
+    this._httpVersion = httpVersion;
   }
 
   _serverAddrFinished(addr?: RemoteAddr) {
@@ -343,11 +346,16 @@ export class Response extends SdkObject {
     this._securityDetailsPromiseCallback(securityDetails);
   }
 
-  _requestFinished(responseEndTiming: number, error?: string) {
+  _requestFinished(responseEndTiming: number, error?: string, transferSize?: number) {
     this._request._responseEndTiming = Math.max(responseEndTiming, this._timing.responseStart);
+    this._transferSize = transferSize;
     this._finishedPromiseCallback({ error });
   }
 
+  _setHttpVersion(httpVersion: string) {
+    this._httpVersion = httpVersion;
+  }
+
   url(): string {
     return this._url;
   }

src/server/supplements/har/har.ts

@@ -81,6 +81,7 @@ export type Response = {
   redirectURL: string;
   headersSize: number;
   bodySize: number;
+  _transferSize: number;
 };
 
 export type Cookie = {

src/server/supplements/har/harTracer.ts

@@ -14,12 +14,16 @@
  * limitations under the License.
  */
 
+import { URL } from 'url';
 import fs from 'fs';
 import { BrowserContext } from '../../browserContext';
 import { helper } from '../../helper';
 import * as network from '../../network';
 import { Page } from '../../page';
 import * as har from './har';
+import * as types from '../../types';
+
+const FALLBACK_HTTP_VERSION = 'HTTP/1.1';
 
 type HarOptions = {
   path: string;
@@ -51,6 +55,7 @@ export class HarTracer {
     };
     context.on(BrowserContext.Events.Page, (page: Page) => this._ensurePageEntry(page));
     context.on(BrowserContext.Events.Request, (request: network.Request) => this._onRequest(request));
+    context.on(BrowserContext.Events.RequestFinished, (request: network.Request) => this._onRequestFinished(request).catch(() => {}));
     context.on(BrowserContext.Events.Response, (response: network.Response) => this._onResponse(response));
   }
 
@@ -128,27 +133,28 @@ export class HarTracer {
       request: {
         method: request.method(),
         url: request.url(),
-        httpVersion: 'HTTP/1.1',
+        httpVersion: FALLBACK_HTTP_VERSION,
         cookies: [],
         headers: [],
         queryString: [...url.searchParams].map(e => ({ name: e[0], value: e[1] })),
-        postData: undefined,
+        postData: postDataForHar(request),
         headersSize: -1,
-        bodySize: -1,
+        bodySize: calculateRequestBodySize(request) || 0,
       },
       response: {
         status: -1,
         statusText: '',
-        httpVersion: 'HTTP/1.1',
+        httpVersion: FALLBACK_HTTP_VERSION,
         cookies: [],
         headers: [],
         content: {
           size: -1,
-          mimeType: request.headerValue('content-type') || 'application/octet-stream',
+          mimeType: request.headerValue('content-type') || 'x-unknown',
         },
         headersSize: -1,
         bodySize: -1,
-        redirectURL: ''
+        redirectURL: '',
+        _transferSize: -1
       },
       cache: {
         beforeRequest: null,
@@ -168,29 +174,63 @@ export class HarTracer {
     this._entries.set(request, harEntry);
   }
 
+  private async _onRequestFinished(request: network.Request) {
+    const page = request.frame()._page;
+    const harEntry = this._entries.get(request)!;
+    const response = await request.response();
+
+    if (!response)
+      return;
+
+    const httpVersion = normaliseHttpVersion(response._httpVersion);
+    const transferSize = response._transferSize || -1;
+    const headersSize = calculateResponseHeadersSize(httpVersion, response.status(), response.statusText(), response.headers());
+    const bodySize = transferSize !== -1 ? transferSize - headersSize : -1;
+
+    harEntry.request.httpVersion = httpVersion;
+    harEntry.response.bodySize = bodySize;
+    harEntry.response.headersSize = headersSize;
+    harEntry.response._transferSize = transferSize;
+    harEntry.request.headersSize = calculateRequestHeadersSize(request.method(), request.url(), httpVersion, request.headers());
+
+    const promise = response.body().then(buffer => {
+      const content = harEntry.response.content;
+      content.size = buffer.length;
+      content.compression = harEntry.response.bodySize !== -1 ? buffer.length - harEntry.response.bodySize : 0;
+
+      if (!this._options.omitContent && buffer && buffer.length > 0) {
+        content.text = buffer.toString('base64');
+        content.encoding = 'base64';
+      }
+    }).catch(() => {});
+    this._addBarrier(page, promise);
+  }
+
   private _onResponse(response: network.Response) {
     const page = response.frame()._page;
     const pageEntry = this._ensurePageEntry(page);
     const harEntry = this._entries.get(response.request())!;
     // Rewrite provisional headers with actual
     const request = response.request();
+
     harEntry.request.headers = request.headers().map(header => ({ name: header.name, value: header.value }));
     harEntry.request.cookies = cookiesForHar(request.headerValue('cookie'), ';');
-    harEntry.request.postData = postDataForHar(request) || undefined;
+    harEntry.request.postData = postDataForHar(request);
 
     harEntry.response = {
       status: response.status(),
       statusText: response.statusText(),
-      httpVersion: 'HTTP/1.1',
+      httpVersion: normaliseHttpVersion(response._httpVersion),
       cookies: cookiesForHar(response.headerValue('set-cookie'), '\n'),
       headers: response.headers().map(header => ({ name: header.name, value: header.value })),
       content: {
         size: -1,
-        mimeType: response.headerValue('content-type') || 'application/octet-stream',
+        mimeType: response.headerValue('content-type') || 'x-unknown',
       },
       headersSize: -1,
       bodySize: -1,
-      redirectURL: ''
+      redirectURL: '',
+      _transferSize: -1
     };
     const timing = response.timing();
     if (pageEntry.startedDateTime.valueOf() > timing.startTime)
@@ -220,14 +260,6 @@ export class HarTracer {
       if (details)
         harEntry._securityDetails = details;
     }));
-
-    if (!this._options.omitContent && response.status() === 200) {
-      const promise = response.body().then(buffer => {
-        harEntry.response.content.text = buffer.toString('base64');
-        harEntry.response.content.encoding = 'base64';
-      }).catch(() => {});
-      this._addBarrier(page, promise);
-    }
   }
 
   async flush() {
@@ -246,10 +278,10 @@ export class HarTracer {
   }
 }
 
-function postDataForHar(request: network.Request): har.PostData | null {
+function postDataForHar(request: network.Request): har.PostData | undefined {
   const postData = request.postDataBuffer();
   if (!postData)
-    return null;
+    return;
 
   const contentType = request.headerValue('content-type') || 'application/octet-stream';
   const result: har.PostData = {
@@ -305,3 +337,33 @@ function parseCookie(c: string): har.Cookie {
   }
   return cookie;
 }
+
+function calculateResponseHeadersSize(protocol: string, status: number, statusText: string , headers: types.HeadersArray) {
+  let rawHeaders = `${protocol} ${status} ${statusText}\r\n`;
+  for (const header of headers)
+    rawHeaders += `${header.name}: ${header.value}\r\n`;
+  rawHeaders += '\r\n';
+  return rawHeaders.length;
+}
+
+function calculateRequestHeadersSize(method: string, url: string, httpVersion: string, headers: types.HeadersArray) {
+  let rawHeaders = `${method} ${(new URL(url)).pathname} ${httpVersion}\r\n`;
+  for (const header of headers)
+    rawHeaders += `${header.name}: ${header.value}\r\n`;
+  return rawHeaders.length;
+}
+
+function normaliseHttpVersion(httpVersion?: string) {
+  if (!httpVersion)
+    return FALLBACK_HTTP_VERSION;
+  if (httpVersion === 'http/1.1')
+    return 'HTTP/1.1';
+  return httpVersion;
+}
+
+function calculateRequestBodySize(request: network.Request): number|undefined {
+  const postData = request.postDataBuffer();
+  if (!postData)
+    return;
+  return new TextEncoder().encode(postData.toString('utf8')).length;
+}

src/server/webkit/wkPage.ts

@@ -1044,7 +1044,11 @@ export class WKPage implements PageDelegate {
         validFrom: responseReceivedPayload?.response.security?.certificate?.validFrom,
         validTo: responseReceivedPayload?.response.security?.certificate?.validUntil,
       });
-      response._requestFinished(helper.secondsToRoundishMillis(event.timestamp - request._timestamp));
+      const { responseBodyBytesReceived, responseHeaderBytesReceived } = event.metrics || {};
+      const transferSize = responseBodyBytesReceived ? responseBodyBytesReceived + (responseHeaderBytesReceived || 0) : undefined;
+      if (event.metrics?.protocol)
+        response._setHttpVersion(event.metrics.protocol);
+      response._requestFinished(helper.secondsToRoundishMillis(event.timestamp - request._timestamp), undefined, transferSize);
     }
 
     this._requestIdToResponseReceivedPayloadEvent.delete(request._requestId);

tests/har.spec.ts

@@ -16,8 +16,11 @@
  */
 
 import { browserTest as it, expect } from './config/browserTest';
+import * as path from 'path';
 import fs from 'fs';
+import http2 from 'http2';
 import type { BrowserContext, BrowserContextOptions } from '../index';
+import type { AddressInfo } from 'net';
 
 async function pageWithHar(contextFactory: (options?: BrowserContextOptions) => Promise<BrowserContext>, testInfo: any) {
   const harPath = testInfo.outputPath('test.har');
@@ -96,6 +99,7 @@ it('should include request', async ({ contextFactory, server }, testInfo) => {
   expect(entry.request.httpVersion).toBe('HTTP/1.1');
   expect(entry.request.headers.length).toBeGreaterThan(1);
   expect(entry.request.headers.find(h => h.name.toLowerCase() === 'user-agent')).toBeTruthy();
+  expect(entry.request.bodySize).toBe(0);
 });
 
 it('should include response', async ({ contextFactory, server }, testInfo) => {
@@ -242,15 +246,36 @@ it('should include content', async ({ contextFactory, server }, testInfo) => {
   await page.goto(server.PREFIX + '/har.html');
   const log = await getLog();
 
-  const content1 = log.entries[0].response.content;
+  expect(log.entries[0].response.httpVersion).toBe('HTTP/1.1');
-  expect(content1.encoding).toBe('base64');
+  expect(log.entries[0].response.content.encoding).toBe('base64');
-  expect(content1.mimeType).toBe('text/html; charset=utf-8');
+  expect(log.entries[0].response.content.mimeType).toBe('text/html; charset=utf-8');
-  expect(Buffer.from(content1.text, 'base64').toString()).toContain('HAR Page');
+  expect(Buffer.from(log.entries[0].response.content.text, 'base64').toString()).toContain('HAR Page');
+  expect(log.entries[0].response.content.size).toBeGreaterThanOrEqual(96);
+  expect(log.entries[0].response.content.compression).toBe(0);
 
-  const content2 = log.entries[1].response.content;
+  expect(log.entries[1].response.httpVersion).toBe('HTTP/1.1');
-  expect(content2.encoding).toBe('base64');
+  expect(log.entries[1].response.content.encoding).toBe('base64');
-  expect(content2.mimeType).toBe('text/css; charset=utf-8');
+  expect(log.entries[1].response.content.mimeType).toBe('text/css; charset=utf-8');
-  expect(Buffer.from(content2.text, 'base64').toString()).toContain('pink');
+  expect(Buffer.from(log.entries[1].response.content.text, 'base64').toString()).toContain('pink');
+  expect(log.entries[1].response.content.size).toBeGreaterThanOrEqual(37);
+  expect(log.entries[1].response.content.compression).toBe(0);
+});
+
+it('should include sizes', async ({ contextFactory, server, browserName, platform }, testInfo) => {
+  it.fixme(browserName === 'webkit' && platform === 'linux', 'blocked by libsoup3');
+
+  const { page, getLog } = await pageWithHar(contextFactory, testInfo);
+  await page.goto(server.PREFIX + '/har.html');
+  const log = await getLog();
+
+  expect(log.entries[0].request.headersSize).toBeGreaterThanOrEqual(280);
+  expect(log.entries[0].response.bodySize).toBeGreaterThanOrEqual(96);
+  expect(log.entries[0].response.headersSize).toBe(198);
+  expect(log.entries[0].response._transferSize).toBeGreaterThanOrEqual(294);
+
+  expect(log.entries[1].response.bodySize).toBeGreaterThanOrEqual(37);
+  expect(log.entries[1].response.headersSize).toBe(197);
+  expect(log.entries[1].response._transferSize).toBeGreaterThanOrEqual(234);
 });
 
 it('should calculate time', async ({ contextFactory, server }, testInfo) => {
@@ -260,6 +285,48 @@ it('should calculate time', async ({ contextFactory, server }, testInfo) => {
   expect(log.entries[0].time).toBeGreaterThan(0);
 });
 
+it('should report the correct _transferSize with PNG files', async ({ contextFactory, server, browserName, platform }, testInfo) => {
+  it.fixme(browserName === 'webkit' && platform === 'linux', 'blocked by libsoup3');
+  const { page, getLog } = await pageWithHar(contextFactory, testInfo);
+  await page.goto(server.EMPTY_PAGE);
+  await page.setContent(`
+    <img src="${server.PREFIX}/pptr.png" />
+  `);
+  const log = await getLog();
+  expect(log.entries[1].response._transferSize).toBe(6323);
+});
+
+it('should have -1 _transferSize when its a failed request', async ({ contextFactory, server }, testInfo) => {
+  const { page, getLog } = await pageWithHar(contextFactory, testInfo);
+  server.setRoute('/one-style.css', (req, res) => {
+    res.setHeader('Content-Type', 'text/css');
+    res.connection.destroy();
+  });
+  const failedRequests = [];
+  page.on('requestfailed', request => failedRequests.push(request));
+  await page.goto(server.PREFIX + '/har.html');
+  const log = await getLog();
+  expect(log.entries[1].request.url.endsWith('/one-style.css')).toBe(true);
+  expect(log.entries[1].response._transferSize).toBe(-1);
+});
+
+it('should report the correct body size', async ({ contextFactory, server }, testInfo) => {
+  server.setRoute('/api', (req, res) => res.end());
+  const { page, getLog } = await pageWithHar(contextFactory, testInfo);
+  await page.goto(server.EMPTY_PAGE);
+  await Promise.all([
+    page.waitForResponse(server.PREFIX + '/api'),
+    page.evaluate(() => {
+      fetch('/api', {
+        method: 'POST',
+        body: 'abc123'
+      });
+    })
+  ]);
+  const log = await getLog();
+  expect(log.entries[1].request.bodySize).toBe(6);
+});
+
 it('should have popup requests', async ({ contextFactory, server }, testInfo) => {
   const { page, getLog } = await pageWithHar(contextFactory, testInfo);
   await page.goto(server.EMPTY_PAGE);
@@ -385,3 +452,27 @@ it('should return security details directly from response', async ({ contextFact
   else
     expect(securityDetails).toEqual({issuer: 'puppeteer-tests', protocol: 'TLS 1.3', subjectName: 'puppeteer-tests', validFrom: 1550084863, validTo: 33086084863});
 });
+
+it('should contain http2 for http2 requests', async ({ contextFactory, browserName }, testInfo) => {
+  it.fixme(browserName === 'firefox' || browserName === 'webkit');
+
+  const server = http2.createSecureServer({
+    key: await fs.promises.readFile(path.join(__dirname, '..', 'utils', 'testserver', 'key.pem')),
+    cert: await fs.promises.readFile(path.join(__dirname, '..', 'utils', 'testserver', 'cert.pem')),
+  });
+  server.on('stream', stream => {
+    stream.respond({
+      'content-type': 'text/html; charset=utf-8',
+      ':status': 200
+    });
+    stream.end('<h1>Hello World</h1>');
+  });
+  server.listen(0);
+
+  const { page, getLog } = await pageWithHar(contextFactory, testInfo);
+  await page.goto(`https://localhost:${(server.address() as AddressInfo).port}`);
+  const log = await getLog();
+  expect(log.entries[0].request.httpVersion).toBe('h2');
+  expect(log.entries[0].response.httpVersion).toBe('h2');
+  server.close();
+});