diff --git a/.gitignore b/.gitignore index 40bc2f31c5..547c965244 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ *.swp *.pyc .vscode +.idea yarn.lock /src/generated/* lib/ diff --git a/src/server/chromium/crNetworkManager.ts b/src/server/chromium/crNetworkManager.ts index 8a777a5601..545516bd43 100644 --- a/src/server/chromium/crNetworkManager.ts +++ b/src/server/chromium/crNetworkManager.ts @@ -310,7 +310,7 @@ export class CRNetworkManager { responseStart: -1, }; } - const response = new network.Response(request.request, responsePayload.status, responsePayload.statusText, headersObjectToArray(responsePayload.headers), timing, getResponseBody); + const response = new network.Response(request.request, responsePayload.status, responsePayload.statusText, headersObjectToArray(responsePayload.headers), timing, getResponseBody, responsePayload.protocol); if (responsePayload?.remoteIPAddress && typeof responsePayload?.remotePort === 'number') { response._serverAddrFinished({ ipAddress: responsePayload.remoteIPAddress, @@ -361,7 +361,7 @@ export class CRNetworkManager { // event from protocol. @see https://crbug.com/883475 const response = request.request._existingResponse(); if (response) - response._requestFinished(helper.secondsToRoundishMillis(event.timestamp - request._timestamp)); + response._requestFinished(helper.secondsToRoundishMillis(event.timestamp - request._timestamp), undefined, event.encodedDataLength); this._requestIdToRequest.delete(request._requestId); if (request._interceptionId) this._attemptedAuthentications.delete(request._interceptionId); diff --git a/src/server/firefox/ffNetworkManager.ts b/src/server/firefox/ffNetworkManager.ts index 2e6d2d01d0..bb4fbe2506 100644 --- a/src/server/firefox/ffNetworkManager.ts +++ b/src/server/firefox/ffNetworkManager.ts @@ -118,7 +118,7 @@ export class FFNetworkManager { response._requestFinished(this._relativeTiming(event.responseEndTime), 'Response body is unavailable for redirect responses'); } else { this._requests.delete(request._id); - response._requestFinished(this._relativeTiming(event.responseEndTime)); + response._requestFinished(this._relativeTiming(event.responseEndTime), undefined, event.transferSize); } this._page._frameManager.requestFinished(request.request); } diff --git a/src/server/network.ts b/src/server/network.ts index 24411916c2..637997c35a 100644 --- a/src/server/network.ts +++ b/src/server/network.ts @@ -311,8 +311,10 @@ export class Response extends SdkObject { private _serverAddrPromiseCallback: (arg?: RemoteAddr) => void = () => {}; private _securityDetailsPromise: Promise; private _securityDetailsPromiseCallback: (arg?: SecurityDetails) => void = () => {}; + _httpVersion: string | undefined; + _transferSize: number | undefined; - constructor(request: Request, status: number, statusText: string, headers: types.HeadersArray, timing: ResourceTiming, getResponseBodyCallback: GetResponseBodyCallback) { + constructor(request: Request, status: number, statusText: string, headers: types.HeadersArray, timing: ResourceTiming, getResponseBodyCallback: GetResponseBodyCallback, httpVersion?: string) { super(request.frame(), 'response'); this._request = request; this._timing = timing; @@ -333,6 +335,7 @@ export class Response extends SdkObject { this._finishedPromiseCallback = f; }); this._request._setResponse(this); + this._httpVersion = httpVersion; } _serverAddrFinished(addr?: RemoteAddr) { @@ -343,11 +346,16 @@ export class Response extends SdkObject { this._securityDetailsPromiseCallback(securityDetails); } - _requestFinished(responseEndTiming: number, error?: string) { + _requestFinished(responseEndTiming: number, error?: string, transferSize?: number) { this._request._responseEndTiming = Math.max(responseEndTiming, this._timing.responseStart); + this._transferSize = transferSize; this._finishedPromiseCallback({ error }); } + _setHttpVersion(httpVersion: string) { + this._httpVersion = httpVersion; + } + url(): string { return this._url; } diff --git a/src/server/supplements/har/har.ts b/src/server/supplements/har/har.ts index e06b1c9cf9..e538bf220b 100644 --- a/src/server/supplements/har/har.ts +++ b/src/server/supplements/har/har.ts @@ -81,6 +81,7 @@ export type Response = { redirectURL: string; headersSize: number; bodySize: number; + _transferSize: number; }; export type Cookie = { diff --git a/src/server/supplements/har/harTracer.ts b/src/server/supplements/har/harTracer.ts index b34c3fdb38..1ce25d8e30 100644 --- a/src/server/supplements/har/harTracer.ts +++ b/src/server/supplements/har/harTracer.ts @@ -14,12 +14,16 @@ * limitations under the License. */ +import { URL } from 'url'; import fs from 'fs'; import { BrowserContext } from '../../browserContext'; import { helper } from '../../helper'; import * as network from '../../network'; import { Page } from '../../page'; import * as har from './har'; +import * as types from '../../types'; + +const FALLBACK_HTTP_VERSION = 'HTTP/1.1'; type HarOptions = { path: string; @@ -51,6 +55,7 @@ export class HarTracer { }; context.on(BrowserContext.Events.Page, (page: Page) => this._ensurePageEntry(page)); context.on(BrowserContext.Events.Request, (request: network.Request) => this._onRequest(request)); + context.on(BrowserContext.Events.RequestFinished, (request: network.Request) => this._onRequestFinished(request).catch(() => {})); context.on(BrowserContext.Events.Response, (response: network.Response) => this._onResponse(response)); } @@ -128,27 +133,28 @@ export class HarTracer { request: { method: request.method(), url: request.url(), - httpVersion: 'HTTP/1.1', + httpVersion: FALLBACK_HTTP_VERSION, cookies: [], headers: [], queryString: [...url.searchParams].map(e => ({ name: e[0], value: e[1] })), - postData: undefined, + postData: postDataForHar(request), headersSize: -1, - bodySize: -1, + bodySize: calculateRequestBodySize(request) || 0, }, response: { status: -1, statusText: '', - httpVersion: 'HTTP/1.1', + httpVersion: FALLBACK_HTTP_VERSION, cookies: [], headers: [], content: { size: -1, - mimeType: request.headerValue('content-type') || 'application/octet-stream', + mimeType: request.headerValue('content-type') || 'x-unknown', }, headersSize: -1, bodySize: -1, - redirectURL: '' + redirectURL: '', + _transferSize: -1 }, cache: { beforeRequest: null, @@ -168,29 +174,63 @@ export class HarTracer { this._entries.set(request, harEntry); } + private async _onRequestFinished(request: network.Request) { + const page = request.frame()._page; + const harEntry = this._entries.get(request)!; + const response = await request.response(); + + if (!response) + return; + + const httpVersion = normaliseHttpVersion(response._httpVersion); + const transferSize = response._transferSize || -1; + const headersSize = calculateResponseHeadersSize(httpVersion, response.status(), response.statusText(), response.headers()); + const bodySize = transferSize !== -1 ? transferSize - headersSize : -1; + + harEntry.request.httpVersion = httpVersion; + harEntry.response.bodySize = bodySize; + harEntry.response.headersSize = headersSize; + harEntry.response._transferSize = transferSize; + harEntry.request.headersSize = calculateRequestHeadersSize(request.method(), request.url(), httpVersion, request.headers()); + + const promise = response.body().then(buffer => { + const content = harEntry.response.content; + content.size = buffer.length; + content.compression = harEntry.response.bodySize !== -1 ? buffer.length - harEntry.response.bodySize : 0; + + if (!this._options.omitContent && buffer && buffer.length > 0) { + content.text = buffer.toString('base64'); + content.encoding = 'base64'; + } + }).catch(() => {}); + this._addBarrier(page, promise); + } + private _onResponse(response: network.Response) { const page = response.frame()._page; const pageEntry = this._ensurePageEntry(page); const harEntry = this._entries.get(response.request())!; // Rewrite provisional headers with actual const request = response.request(); + harEntry.request.headers = request.headers().map(header => ({ name: header.name, value: header.value })); harEntry.request.cookies = cookiesForHar(request.headerValue('cookie'), ';'); - harEntry.request.postData = postDataForHar(request) || undefined; + harEntry.request.postData = postDataForHar(request); harEntry.response = { status: response.status(), statusText: response.statusText(), - httpVersion: 'HTTP/1.1', + httpVersion: normaliseHttpVersion(response._httpVersion), cookies: cookiesForHar(response.headerValue('set-cookie'), '\n'), headers: response.headers().map(header => ({ name: header.name, value: header.value })), content: { size: -1, - mimeType: response.headerValue('content-type') || 'application/octet-stream', + mimeType: response.headerValue('content-type') || 'x-unknown', }, headersSize: -1, bodySize: -1, - redirectURL: '' + redirectURL: '', + _transferSize: -1 }; const timing = response.timing(); if (pageEntry.startedDateTime.valueOf() > timing.startTime) @@ -220,14 +260,6 @@ export class HarTracer { if (details) harEntry._securityDetails = details; })); - - if (!this._options.omitContent && response.status() === 200) { - const promise = response.body().then(buffer => { - harEntry.response.content.text = buffer.toString('base64'); - harEntry.response.content.encoding = 'base64'; - }).catch(() => {}); - this._addBarrier(page, promise); - } } async flush() { @@ -246,10 +278,10 @@ export class HarTracer { } } -function postDataForHar(request: network.Request): har.PostData | null { +function postDataForHar(request: network.Request): har.PostData | undefined { const postData = request.postDataBuffer(); if (!postData) - return null; + return; const contentType = request.headerValue('content-type') || 'application/octet-stream'; const result: har.PostData = { @@ -305,3 +337,33 @@ function parseCookie(c: string): har.Cookie { } return cookie; } + +function calculateResponseHeadersSize(protocol: string, status: number, statusText: string , headers: types.HeadersArray) { + let rawHeaders = `${protocol} ${status} ${statusText}\r\n`; + for (const header of headers) + rawHeaders += `${header.name}: ${header.value}\r\n`; + rawHeaders += '\r\n'; + return rawHeaders.length; +} + +function calculateRequestHeadersSize(method: string, url: string, httpVersion: string, headers: types.HeadersArray) { + let rawHeaders = `${method} ${(new URL(url)).pathname} ${httpVersion}\r\n`; + for (const header of headers) + rawHeaders += `${header.name}: ${header.value}\r\n`; + return rawHeaders.length; +} + +function normaliseHttpVersion(httpVersion?: string) { + if (!httpVersion) + return FALLBACK_HTTP_VERSION; + if (httpVersion === 'http/1.1') + return 'HTTP/1.1'; + return httpVersion; +} + +function calculateRequestBodySize(request: network.Request): number|undefined { + const postData = request.postDataBuffer(); + if (!postData) + return; + return new TextEncoder().encode(postData.toString('utf8')).length; +} diff --git a/src/server/webkit/wkPage.ts b/src/server/webkit/wkPage.ts index 9309e504a7..7373307fab 100644 --- a/src/server/webkit/wkPage.ts +++ b/src/server/webkit/wkPage.ts @@ -1044,7 +1044,11 @@ export class WKPage implements PageDelegate { validFrom: responseReceivedPayload?.response.security?.certificate?.validFrom, validTo: responseReceivedPayload?.response.security?.certificate?.validUntil, }); - response._requestFinished(helper.secondsToRoundishMillis(event.timestamp - request._timestamp)); + const { responseBodyBytesReceived, responseHeaderBytesReceived } = event.metrics || {}; + const transferSize = responseBodyBytesReceived ? responseBodyBytesReceived + (responseHeaderBytesReceived || 0) : undefined; + if (event.metrics?.protocol) + response._setHttpVersion(event.metrics.protocol); + response._requestFinished(helper.secondsToRoundishMillis(event.timestamp - request._timestamp), undefined, transferSize); } this._requestIdToResponseReceivedPayloadEvent.delete(request._requestId); diff --git a/tests/har.spec.ts b/tests/har.spec.ts index 760fb9bcc4..5d839ab823 100644 --- a/tests/har.spec.ts +++ b/tests/har.spec.ts @@ -16,8 +16,11 @@ */ import { browserTest as it, expect } from './config/browserTest'; +import * as path from 'path'; import fs from 'fs'; +import http2 from 'http2'; import type { BrowserContext, BrowserContextOptions } from '../index'; +import type { AddressInfo } from 'net'; async function pageWithHar(contextFactory: (options?: BrowserContextOptions) => Promise, testInfo: any) { const harPath = testInfo.outputPath('test.har'); @@ -96,6 +99,7 @@ it('should include request', async ({ contextFactory, server }, testInfo) => { expect(entry.request.httpVersion).toBe('HTTP/1.1'); expect(entry.request.headers.length).toBeGreaterThan(1); expect(entry.request.headers.find(h => h.name.toLowerCase() === 'user-agent')).toBeTruthy(); + expect(entry.request.bodySize).toBe(0); }); it('should include response', async ({ contextFactory, server }, testInfo) => { @@ -242,15 +246,36 @@ it('should include content', async ({ contextFactory, server }, testInfo) => { await page.goto(server.PREFIX + '/har.html'); const log = await getLog(); - const content1 = log.entries[0].response.content; - expect(content1.encoding).toBe('base64'); - expect(content1.mimeType).toBe('text/html; charset=utf-8'); - expect(Buffer.from(content1.text, 'base64').toString()).toContain('HAR Page'); + expect(log.entries[0].response.httpVersion).toBe('HTTP/1.1'); + expect(log.entries[0].response.content.encoding).toBe('base64'); + expect(log.entries[0].response.content.mimeType).toBe('text/html; charset=utf-8'); + expect(Buffer.from(log.entries[0].response.content.text, 'base64').toString()).toContain('HAR Page'); + expect(log.entries[0].response.content.size).toBeGreaterThanOrEqual(96); + expect(log.entries[0].response.content.compression).toBe(0); - const content2 = log.entries[1].response.content; - expect(content2.encoding).toBe('base64'); - expect(content2.mimeType).toBe('text/css; charset=utf-8'); - expect(Buffer.from(content2.text, 'base64').toString()).toContain('pink'); + expect(log.entries[1].response.httpVersion).toBe('HTTP/1.1'); + expect(log.entries[1].response.content.encoding).toBe('base64'); + expect(log.entries[1].response.content.mimeType).toBe('text/css; charset=utf-8'); + expect(Buffer.from(log.entries[1].response.content.text, 'base64').toString()).toContain('pink'); + expect(log.entries[1].response.content.size).toBeGreaterThanOrEqual(37); + expect(log.entries[1].response.content.compression).toBe(0); +}); + +it('should include sizes', async ({ contextFactory, server, browserName, platform }, testInfo) => { + it.fixme(browserName === 'webkit' && platform === 'linux', 'blocked by libsoup3'); + + const { page, getLog } = await pageWithHar(contextFactory, testInfo); + await page.goto(server.PREFIX + '/har.html'); + const log = await getLog(); + + expect(log.entries[0].request.headersSize).toBeGreaterThanOrEqual(280); + expect(log.entries[0].response.bodySize).toBeGreaterThanOrEqual(96); + expect(log.entries[0].response.headersSize).toBe(198); + expect(log.entries[0].response._transferSize).toBeGreaterThanOrEqual(294); + + expect(log.entries[1].response.bodySize).toBeGreaterThanOrEqual(37); + expect(log.entries[1].response.headersSize).toBe(197); + expect(log.entries[1].response._transferSize).toBeGreaterThanOrEqual(234); }); it('should calculate time', async ({ contextFactory, server }, testInfo) => { @@ -260,6 +285,48 @@ it('should calculate time', async ({ contextFactory, server }, testInfo) => { expect(log.entries[0].time).toBeGreaterThan(0); }); +it('should report the correct _transferSize with PNG files', async ({ contextFactory, server, browserName, platform }, testInfo) => { + it.fixme(browserName === 'webkit' && platform === 'linux', 'blocked by libsoup3'); + const { page, getLog } = await pageWithHar(contextFactory, testInfo); + await page.goto(server.EMPTY_PAGE); + await page.setContent(` + + `); + const log = await getLog(); + expect(log.entries[1].response._transferSize).toBe(6323); +}); + +it('should have -1 _transferSize when its a failed request', async ({ contextFactory, server }, testInfo) => { + const { page, getLog } = await pageWithHar(contextFactory, testInfo); + server.setRoute('/one-style.css', (req, res) => { + res.setHeader('Content-Type', 'text/css'); + res.connection.destroy(); + }); + const failedRequests = []; + page.on('requestfailed', request => failedRequests.push(request)); + await page.goto(server.PREFIX + '/har.html'); + const log = await getLog(); + expect(log.entries[1].request.url.endsWith('/one-style.css')).toBe(true); + expect(log.entries[1].response._transferSize).toBe(-1); +}); + +it('should report the correct body size', async ({ contextFactory, server }, testInfo) => { + server.setRoute('/api', (req, res) => res.end()); + const { page, getLog } = await pageWithHar(contextFactory, testInfo); + await page.goto(server.EMPTY_PAGE); + await Promise.all([ + page.waitForResponse(server.PREFIX + '/api'), + page.evaluate(() => { + fetch('/api', { + method: 'POST', + body: 'abc123' + }); + }) + ]); + const log = await getLog(); + expect(log.entries[1].request.bodySize).toBe(6); +}); + it('should have popup requests', async ({ contextFactory, server }, testInfo) => { const { page, getLog } = await pageWithHar(contextFactory, testInfo); await page.goto(server.EMPTY_PAGE); @@ -385,3 +452,27 @@ it('should return security details directly from response', async ({ contextFact else expect(securityDetails).toEqual({issuer: 'puppeteer-tests', protocol: 'TLS 1.3', subjectName: 'puppeteer-tests', validFrom: 1550084863, validTo: 33086084863}); }); + +it('should contain http2 for http2 requests', async ({ contextFactory, browserName }, testInfo) => { + it.fixme(browserName === 'firefox' || browserName === 'webkit'); + + const server = http2.createSecureServer({ + key: await fs.promises.readFile(path.join(__dirname, '..', 'utils', 'testserver', 'key.pem')), + cert: await fs.promises.readFile(path.join(__dirname, '..', 'utils', 'testserver', 'cert.pem')), + }); + server.on('stream', stream => { + stream.respond({ + 'content-type': 'text/html; charset=utf-8', + ':status': 200 + }); + stream.end('

Hello World

'); + }); + server.listen(0); + + const { page, getLog } = await pageWithHar(contextFactory, testInfo); + await page.goto(`https://localhost:${(server.address() as AddressInfo).port}`); + const log = await getLog(); + expect(log.entries[0].request.httpVersion).toBe('h2'); + expect(log.entries[0].response.httpVersion).toBe('h2'); + server.close(); +});