diff --git a/src/server/chromium/crNetworkManager.ts b/src/server/chromium/crNetworkManager.ts index 6cbfb8c5c5..696e11c516 100644 --- a/src/server/chromium/crNetworkManager.ts +++ b/src/server/chromium/crNetworkManager.ts @@ -361,8 +361,11 @@ export class CRNetworkManager { // Under certain conditions we never get the Network.responseReceived // event from protocol. @see https://crbug.com/883475 const response = request.request._existingResponse(); - if (response) - response._requestFinished(helper.secondsToRoundishMillis(event.timestamp - request._timestamp), undefined, event.encodedDataLength); + if (response) { + request.request._sizes.transferSize = event.encodedDataLength; + request.request._sizes.responseBodySize = event.encodedDataLength - response?.headersSize(); + response._requestFinished(helper.secondsToRoundishMillis(event.timestamp - request._timestamp), undefined); + } this._requestIdToRequest.delete(request._requestId); if (request._interceptionId) this._attemptedAuthentications.delete(request._interceptionId); diff --git a/src/server/firefox/ffNetworkManager.ts b/src/server/firefox/ffNetworkManager.ts index 2ef90ec310..c98edc4d5c 100644 --- a/src/server/firefox/ffNetworkManager.ts +++ b/src/server/firefox/ffNetworkManager.ts @@ -116,13 +116,17 @@ export class FFNetworkManager { if (!request) return; const response = request.request._existingResponse()!; + + request.request._sizes.transferSize = event.transferSize; + request.request._sizes.responseBodySize = event.transferSize - response.headersSize(); + // Keep redirected requests in the map for future reference as redirectedFrom. const isRedirected = response.status() >= 300 && response.status() <= 399; if (isRedirected) { response._requestFinished(this._relativeTiming(event.responseEndTime), 'Response body is unavailable for redirect responses'); } else { this._requests.delete(request._id); - response._requestFinished(this._relativeTiming(event.responseEndTime), undefined, event.transferSize); + response._requestFinished(this._relativeTiming(event.responseEndTime), undefined); } this._page._frameManager.requestFinished(request.request); } diff --git a/src/server/network.ts b/src/server/network.ts index 6c8a03e688..46068796b1 100644 --- a/src/server/network.ts +++ b/src/server/network.ts @@ -78,6 +78,11 @@ export function stripFragmentFromUrl(url: string): string { return url.substring(0, url.indexOf('#')); } +type RequestSizes = { + responseBodySize: number; + transferSize: number; +}; + export class Request extends SdkObject { private _response: Response | null = null; private _redirectedFrom: Request | null; @@ -95,6 +100,7 @@ export class Request extends SdkObject { private _waitForResponsePromise: Promise; private _waitForResponsePromiseCallback: (value: Response | null) => void = () => {}; _responseEndTiming = -1; + _sizes: RequestSizes = { responseBodySize: 0, transferSize: 0 }; constructor(frame: frames.Frame, redirectedFrom: Request | null, documentId: string | undefined, url: string, resourceType: string, method: string, postData: Buffer | null, headers: types.HeadersArray) { @@ -193,6 +199,22 @@ export class Request extends SdkObject { this._headersMap.set('host', host); } } + + bodySize(): number { + return this.postDataBuffer()?.length || 0; + } + + headersSize(): number { + if (!this._response) + return 0; + let headersSize = 4; // 4 = 2 spaces + 2 line breaks (GET /path \r\n) + headersSize += this.method().length; + headersSize += (new URL(this.url())).pathname.length; + headersSize += 8; // httpVersion + for (const header of this._headers) + headersSize += header.name.length + header.value.length + 4; // 4 = ': ' + '\r\n' + return headersSize; + } } export class Route extends SdkObject { @@ -302,8 +324,7 @@ export class Response extends SdkObject { private _serverAddrPromiseCallback: (arg?: RemoteAddr) => void = () => {}; private _securityDetailsPromise: Promise; private _securityDetailsPromiseCallback: (arg?: SecurityDetails) => void = () => {}; - _httpVersion: string | undefined; - _transferSize: number | undefined; + private _httpVersion: string | undefined; constructor(request: Request, status: number, statusText: string, headers: types.HeadersArray, timing: ResourceTiming, getResponseBodyCallback: GetResponseBodyCallback, httpVersion?: string) { super(request.frame(), 'response'); @@ -337,9 +358,8 @@ export class Response extends SdkObject { this._securityDetailsPromiseCallback(securityDetails); } - _requestFinished(responseEndTiming: number, error?: string, transferSize?: number) { + _requestFinished(responseEndTiming: number, error?: string) { this._request._responseEndTiming = Math.max(responseEndTiming, this._timing.responseStart); - this._transferSize = transferSize; this._finishedPromiseCallback({ error }); } @@ -401,6 +421,33 @@ export class Response extends SdkObject { frame(): frames.Frame { return this._request.frame(); } + + transferSize(): number | undefined { + return this._request._sizes.transferSize; + } + + bodySize(): number { + return this._request._sizes.responseBodySize; + } + + httpVersion(): string { + if (!this._httpVersion) + return 'HTTP/1.1'; + if (this._httpVersion === 'http/1.1') + return 'HTTP/1.1'; + return this._httpVersion; + } + + headersSize(): number { + let headersSize = 4; // 4 = 2 spaces + 2 line breaks (HTTP/1.1 200 Ok\r\n) + headersSize += 8; // httpVersion; + headersSize += 3; // statusCode; + headersSize += this.statusText().length; + for (const header of this.headers()) + headersSize += header.name.length + header.value.length + 4; // 4 = ': ' + '\r\n' + headersSize += 2; // '\r\n' + return headersSize; + } } export class InterceptedResponse extends SdkObject { diff --git a/src/server/supplements/har/harTracer.ts b/src/server/supplements/har/harTracer.ts index a5ac5ac419..b58ca2a3f7 100644 --- a/src/server/supplements/har/harTracer.ts +++ b/src/server/supplements/har/harTracer.ts @@ -14,13 +14,11 @@ * limitations under the License. */ -import { URL } from 'url'; import { BrowserContext } from '../../browserContext'; import { helper } from '../../helper'; import * as network from '../../network'; import { Page } from '../../page'; import * as har from './har'; -import * as types from '../../types'; import { calculateSha1, monotonicTime } from '../../../utils/utils'; import { eventsHelper, RegisteredListener } from '../../../utils/eventsHelper'; import * as mime from 'mime'; @@ -158,7 +156,7 @@ export class HarTracer { queryString: [...url.searchParams].map(e => ({ name: e[0], value: e[1] })), postData: postDataForHar(request, this._options.content), headersSize: -1, - bodySize: calculateRequestBodySize(request) || 0, + bodySize: request.bodySize(), }, response: { status: -1, @@ -204,16 +202,15 @@ export class HarTracer { if (!response) return; - const httpVersion = normaliseHttpVersion(response._httpVersion); - const transferSize = response._transferSize || -1; - const headersSize = calculateResponseHeadersSize(httpVersion, response.status(), response.statusText(), response.headers()); - const bodySize = transferSize !== -1 ? transferSize - headersSize : -1; + const httpVersion = response.httpVersion(); + const transferSize = response.transferSize() || -1; + const responseHeadersSize = response.headersSize(); harEntry.request.httpVersion = httpVersion; - harEntry.response.bodySize = bodySize; - harEntry.response.headersSize = headersSize; + harEntry.response.bodySize = response.bodySize(); + harEntry.response.headersSize = responseHeadersSize; harEntry.response._transferSize = transferSize; - harEntry.request.headersSize = calculateRequestHeadersSize(request.method(), request.url(), httpVersion, request.headers()); + harEntry.request.headersSize = request.headersSize(); const promise = response.body().then(buffer => { const content = harEntry.response.content; @@ -258,7 +255,7 @@ export class HarTracer { harEntry.response = { status: response.status(), statusText: response.statusText(), - httpVersion: normaliseHttpVersion(response._httpVersion), + httpVersion: response.httpVersion(), cookies: cookiesForHar(response.headerValue('set-cookie'), '\n'), headers: response.headers().map(header => ({ name: header.name, value: header.value })), content: { @@ -397,33 +394,3 @@ function parseCookie(c: string): har.Cookie { } return cookie; } - -function calculateResponseHeadersSize(protocol: string, status: number, statusText: string , headers: types.HeadersArray) { - let rawHeaders = `${protocol} ${status} ${statusText}\r\n`; - for (const header of headers) - rawHeaders += `${header.name}: ${header.value}\r\n`; - rawHeaders += '\r\n'; - return rawHeaders.length; -} - -function calculateRequestHeadersSize(method: string, url: string, httpVersion: string, headers: types.HeadersArray) { - let rawHeaders = `${method} ${(new URL(url)).pathname} ${httpVersion}\r\n`; - for (const header of headers) - rawHeaders += `${header.name}: ${header.value}\r\n`; - return rawHeaders.length; -} - -function normaliseHttpVersion(httpVersion?: string) { - if (!httpVersion) - return FALLBACK_HTTP_VERSION; - if (httpVersion === 'http/1.1') - return 'HTTP/1.1'; - return httpVersion; -} - -function calculateRequestBodySize(request: network.Request): number|undefined { - const postData = request.postDataBuffer(); - if (!postData) - return; - return new TextEncoder().encode(postData.toString('utf8')).length; -} diff --git a/src/server/webkit/wkPage.ts b/src/server/webkit/wkPage.ts index 44d96ff61f..3c49952023 100644 --- a/src/server/webkit/wkPage.ts +++ b/src/server/webkit/wkPage.ts @@ -380,6 +380,7 @@ export class WKPage implements PageDelegate { eventsHelper.addEventListener(this._session, 'Network.responseReceived', e => this._onResponseReceived(e)), eventsHelper.addEventListener(this._session, 'Network.loadingFinished', e => this._onLoadingFinished(e)), eventsHelper.addEventListener(this._session, 'Network.loadingFailed', e => this._onLoadingFailed(e)), + eventsHelper.addEventListener(this._session, 'Network.dataReceived', e => this._onDataReceived(e)), eventsHelper.addEventListener(this._session, 'Network.webSocketCreated', e => this._page._frameManager.onWebSocketCreated(e.requestId, e.url)), eventsHelper.addEventListener(this._session, 'Network.webSocketWillSendHandshakeRequest', e => this._page._frameManager.onWebSocketRequest(e.requestId)), eventsHelper.addEventListener(this._session, 'Network.webSocketHandshakeResponseReceived', e => this._page._frameManager.onWebSocketResponse(e.requestId, e.response.status, e.response.statusText)), @@ -1049,11 +1050,10 @@ export class WKPage implements PageDelegate { validFrom: responseReceivedPayload?.response.security?.certificate?.validFrom, validTo: responseReceivedPayload?.response.security?.certificate?.validUntil, }); - const { responseBodyBytesReceived, responseHeaderBytesReceived } = event.metrics || {}; - const transferSize = responseBodyBytesReceived ? responseBodyBytesReceived + (responseHeaderBytesReceived || 0) : undefined; + request.request._sizes.transferSize += response.headersSize(); if (event.metrics?.protocol) response._setHttpVersion(event.metrics.protocol); - response._requestFinished(helper.secondsToRoundishMillis(event.timestamp - request._timestamp), undefined, transferSize); + response._requestFinished(helper.secondsToRoundishMillis(event.timestamp - request._timestamp), undefined); } this._requestIdToResponseReceivedPayloadEvent.delete(request._requestId); @@ -1081,6 +1081,14 @@ export class WKPage implements PageDelegate { this._page._frameManager.requestFailed(request.request, event.errorText.includes('cancelled')); } + _onDataReceived(event: Protocol.Network.dataReceivedPayload) { + const request = this._requestIdToRequest.get(event.requestId); + if (!request) + return; + request.request._sizes.responseBodySize += event.encodedDataLength === -1 ? event.dataLength : event.encodedDataLength; + request.request._sizes.transferSize += event.encodedDataLength === -1 ? event.dataLength : event.encodedDataLength; + } + async _grantPermissions(origin: string, permissions: string[]) { const webPermissionToProtocol = new Map([ ['geolocation', 'geolocation'], diff --git a/tests/har.spec.ts b/tests/har.spec.ts index 9b818b87ad..0703e2e0a1 100644 --- a/tests/har.spec.ts +++ b/tests/har.spec.ts @@ -262,23 +262,34 @@ it('should include content', async ({ contextFactory, server }, testInfo) => { expect(log.entries[1].response.content.compression).toBe(0); }); -it('should include sizes', async ({ contextFactory, server, browserName, platform }, testInfo) => { - it.fixme(browserName === 'webkit' && platform === 'linux', 'blocked by libsoup3'); - +it('should include sizes', async ({ contextFactory, server, asset }, testInfo) => { const { page, getLog } = await pageWithHar(contextFactory, testInfo); await page.goto(server.PREFIX + '/har.html'); const log = await getLog(); - + expect(log.entries.length).toBe(2); + expect(log.entries[0].request.url.endsWith('har.html')).toBe(true); expect(log.entries[0].request.headersSize).toBeGreaterThanOrEqual(280); - expect(log.entries[0].response.bodySize).toBeGreaterThanOrEqual(96); - expect(log.entries[0].response.headersSize).toBe(198); + expect(log.entries[0].response.bodySize).toBe(fs.statSync(asset('har.html')).size); + expect(log.entries[0].response.headersSize).toBeGreaterThanOrEqual(198); expect(log.entries[0].response._transferSize).toBeGreaterThanOrEqual(294); - expect(log.entries[1].response.bodySize).toBeGreaterThanOrEqual(37); - expect(log.entries[1].response.headersSize).toBe(197); + expect(log.entries[1].request.url.endsWith('one-style.css')).toBe(true); + expect(log.entries[1].response.bodySize).toBe(fs.statSync(asset('one-style.css')).size); + expect(log.entries[1].response.headersSize).toBeGreaterThanOrEqual(197); expect(log.entries[1].response._transferSize).toBeGreaterThanOrEqual(234); }); +it('should work with gzip compression', async ({ contextFactory, server, browserName }, testInfo) => { + it.fixme(browserName === 'webkit'); + const { page, getLog } = await pageWithHar(contextFactory, testInfo); + server.enableGzip('/simple.json'); + const response = await page.goto(server.PREFIX + '/simple.json'); + expect(response.headers()['content-encoding']).toBe('gzip'); + const log = await getLog(); + expect(log.entries.length).toBe(1); + expect(log.entries[0].response.content.compression).toBe(-20); +}); + it('should calculate time', async ({ contextFactory, server }, testInfo) => { const { page, getLog } = await pageWithHar(contextFactory, testInfo); await page.goto(server.PREFIX + '/har.html'); @@ -286,15 +297,14 @@ it('should calculate time', async ({ contextFactory, server }, testInfo) => { expect(log.entries[0].time).toBeGreaterThan(0); }); -it('should report the correct _transferSize with PNG files', async ({ contextFactory, server, browserName, platform }, testInfo) => { - it.fixme(browserName === 'webkit' && platform === 'linux', 'blocked by libsoup3'); +it('should report the correct _transferSize with PNG files', async ({ contextFactory, server, asset }, testInfo) => { const { page, getLog } = await pageWithHar(contextFactory, testInfo); await page.goto(server.EMPTY_PAGE); await page.setContent(` `); const log = await getLog(); - expect(log.entries[1].response._transferSize).toBe(6323); + expect(log.entries[1].response._transferSize).toBeGreaterThan(fs.statSync(asset('pptr.png')).size); }); it('should have -1 _transferSize when its a failed request', async ({ contextFactory, server }, testInfo) => { @@ -311,14 +321,14 @@ it('should have -1 _transferSize when its a failed request', async ({ contextFac expect(log.entries[1].response._transferSize).toBe(-1); }); -it('should report the correct body size', async ({ contextFactory, server }, testInfo) => { +it('should report the correct request body size', async ({ contextFactory, server }, testInfo) => { server.setRoute('/api', (req, res) => res.end()); const { page, getLog } = await pageWithHar(contextFactory, testInfo); await page.goto(server.EMPTY_PAGE); await Promise.all([ - page.waitForResponse(server.PREFIX + '/api'), + page.waitForResponse(server.PREFIX + '/api1'), page.evaluate(() => { - fetch('/api', { + fetch('/api1', { method: 'POST', body: 'abc123' }); @@ -328,6 +338,31 @@ it('should report the correct body size', async ({ contextFactory, server }, tes expect(log.entries[1].request.bodySize).toBe(6); }); +it('should report the correct request body size when the bodySize is 0', async ({ contextFactory, server }, testInfo) => { + server.setRoute('/api', (req, res) => res.end()); + const { page, getLog } = await pageWithHar(contextFactory, testInfo); + await page.goto(server.EMPTY_PAGE); + await Promise.all([ + page.waitForResponse(server.PREFIX + '/api2'), + page.evaluate(() => { + fetch('/api2', { + method: 'POST', + body: '' + }); + }) + ]); + const log = await getLog(); + expect(log.entries[1].request.bodySize).toBe(0); +}); + +it('should report the correct response body size when the bodySize is 0', async ({ contextFactory, server }, testInfo) => { + server.setRoute('/empty.html', (req, res) => res.end('')); + const { page, getLog } = await pageWithHar(contextFactory, testInfo); + await page.goto(server.EMPTY_PAGE); + const log = await getLog(); + expect(log.entries[0].response.bodySize).toBe(0); +}); + it('should have popup requests', async ({ contextFactory, server }, testInfo) => { const { page, getLog } = await pageWithHar(contextFactory, testInfo); await page.goto(server.EMPTY_PAGE);