Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

fix(har): rewrite sizes and make transferSize work in WK/Linux (#8504)

Browse source
This commit is contained in:
Max Schmitt 2021-08-27 20:42:45 +02:00 committed by GitHub
parent 621af2c737
commit 89245de0ef
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 129 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
src/server/chromium/crNetworkManager.ts
View file

@ -361,8 +361,11 @@ export class CRNetworkManager {
// Under certain conditions we never get the Network.responseReceived
// event from protocol. @see https://crbug.com/883475
const response = request.request._existingResponse();
if (response)
response._requestFinished(helper.secondsToRoundishMillis(event.timestamp - request._timestamp), undefined, event.encodedDataLength);
if (response) {
request.request._sizes.transferSize = event.encodedDataLength;
request.request._sizes.responseBodySize = event.encodedDataLength - response?.headersSize();
response._requestFinished(helper.secondsToRoundishMillis(event.timestamp - request._timestamp), undefined);
}
this._requestIdToRequest.delete(request._requestId);
if (request._interceptionId)
this._attemptedAuthentications.delete(request._interceptionId);

6
src/server/firefox/ffNetworkManager.ts
View file

@ -116,13 +116,17 @@ export class FFNetworkManager {
if (!request)
return;
const response = request.request._existingResponse()!;
request.request._sizes.transferSize = event.transferSize;
request.request._sizes.responseBodySize = event.transferSize - response.headersSize();
// Keep redirected requests in the map for future reference as redirectedFrom.
const isRedirected = response.status() >= 300 && response.status() <= 399;
if (isRedirected) {
response._requestFinished(this._relativeTiming(event.responseEndTime), 'Response body is unavailable for redirect responses');
} else {
this._requests.delete(request._id);
response._requestFinished(this._relativeTiming(event.responseEndTime), undefined, event.transferSize);
response._requestFinished(this._relativeTiming(event.responseEndTime), undefined);
}
this._page._frameManager.requestFinished(request.request);
}

55
src/server/network.ts
View file

@ -78,6 +78,11 @@ export function stripFragmentFromUrl(url: string): string {
return url.substring(0, url.indexOf('#'));
}
type RequestSizes = {
responseBodySize: number;
transferSize: number;
};
export class Request extends SdkObject {
private _response: Response | null = null;
private _redirectedFrom: Request | null;
@ -95,6 +100,7 @@ export class Request extends SdkObject {
private _waitForResponsePromise: Promise<Response | null>;
private _waitForResponsePromiseCallback: (value: Response | null) => void = () => {};
_responseEndTiming = -1;
_sizes: RequestSizes = { responseBodySize: 0, transferSize: 0 };
constructor(frame: frames.Frame, redirectedFrom: Request | null, documentId: string | undefined,
url: string, resourceType: string, method: string, postData: Buffer | null, headers: types.HeadersArray) {
@ -193,6 +199,22 @@ export class Request extends SdkObject {
this._headersMap.set('host', host);
}
}
bodySize(): number {
return this.postDataBuffer()?.length || 0;
}
headersSize(): number {
if (!this._response)
return 0;
let headersSize = 4; // 4 = 2 spaces + 2 line breaks (GET /path \r\n)
headersSize += this.method().length;
headersSize += (new URL(this.url())).pathname.length;
headersSize += 8; // httpVersion
for (const header of this._headers)
headersSize += header.name.length + header.value.length + 4; // 4 = ': ' + '\r\n'
return headersSize;
}
}
export class Route extends SdkObject {
@ -302,8 +324,7 @@ export class Response extends SdkObject {
private _serverAddrPromiseCallback: (arg?: RemoteAddr) => void = () => {};
private _securityDetailsPromise: Promise<SecurityDetails|undefined>;
private _securityDetailsPromiseCallback: (arg?: SecurityDetails) => void = () => {};
_httpVersion: string | undefined;
_transferSize: number | undefined;
private _httpVersion: string | undefined;
constructor(request: Request, status: number, statusText: string, headers: types.HeadersArray, timing: ResourceTiming, getResponseBodyCallback: GetResponseBodyCallback, httpVersion?: string) {
super(request.frame(), 'response');
@ -337,9 +358,8 @@ export class Response extends SdkObject {
this._securityDetailsPromiseCallback(securityDetails);
}
_requestFinished(responseEndTiming: number, error?: string, transferSize?: number) {
_requestFinished(responseEndTiming: number, error?: string) {
this._request._responseEndTiming = Math.max(responseEndTiming, this._timing.responseStart);
this._transferSize = transferSize;
this._finishedPromiseCallback({ error });
}
@ -401,6 +421,33 @@ export class Response extends SdkObject {
frame(): frames.Frame {
return this._request.frame();
}
transferSize(): number | undefined {
return this._request._sizes.transferSize;
}
bodySize(): number {
return this._request._sizes.responseBodySize;
}
httpVersion(): string {
if (!this._httpVersion)
return 'HTTP/1.1';
if (this._httpVersion === 'http/1.1')
return 'HTTP/1.1';
return this._httpVersion;
}
headersSize(): number {
let headersSize = 4; // 4 = 2 spaces + 2 line breaks (HTTP/1.1 200 Ok\r\n)
headersSize += 8; // httpVersion;
headersSize += 3; // statusCode;
headersSize += this.statusText().length;
for (const header of this.headers())
headersSize += header.name.length + header.value.length + 4; // 4 = ': ' + '\r\n'
headersSize += 2; // '\r\n'
return headersSize;
}
}
export class InterceptedResponse extends SdkObject {

49
src/server/supplements/har/harTracer.ts
View file

@ -14,13 +14,11 @@
* limitations under the License.
*/
import { URL } from 'url';
import { BrowserContext } from '../../browserContext';
import { helper } from '../../helper';
import * as network from '../../network';
import { Page } from '../../page';
import * as har from './har';
import * as types from '../../types';
import { calculateSha1, monotonicTime } from '../../../utils/utils';
import { eventsHelper, RegisteredListener } from '../../../utils/eventsHelper';
import * as mime from 'mime';
@ -158,7 +156,7 @@ export class HarTracer {
queryString: [...url.searchParams].map(e => ({ name: e[0], value: e[1] })),
postData: postDataForHar(request, this._options.content),
headersSize: -1,
bodySize: calculateRequestBodySize(request) || 0,
bodySize: request.bodySize(),
},
response: {
status: -1,
@ -204,16 +202,15 @@ export class HarTracer {
if (!response)
return;
const httpVersion = normaliseHttpVersion(response._httpVersion);
const transferSize = response._transferSize || -1;
const headersSize = calculateResponseHeadersSize(httpVersion, response.status(), response.statusText(), response.headers());
const bodySize = transferSize !== -1 ? transferSize - headersSize : -1;
const httpVersion = response.httpVersion();
const transferSize = response.transferSize() || -1;
const responseHeadersSize = response.headersSize();
harEntry.request.httpVersion = httpVersion;
harEntry.response.bodySize = bodySize;
harEntry.response.headersSize = headersSize;
harEntry.response.bodySize = response.bodySize();
harEntry.response.headersSize = responseHeadersSize;
harEntry.response._transferSize = transferSize;
harEntry.request.headersSize = calculateRequestHeadersSize(request.method(), request.url(), httpVersion, request.headers());
harEntry.request.headersSize = request.headersSize();
const promise = response.body().then(buffer => {
const content = harEntry.response.content;
@ -258,7 +255,7 @@ export class HarTracer {
harEntry.response = {
status: response.status(),
statusText: response.statusText(),
httpVersion: normaliseHttpVersion(response._httpVersion),
httpVersion: response.httpVersion(),
cookies: cookiesForHar(response.headerValue('set-cookie'), '\n'),
headers: response.headers().map(header => ({ name: header.name, value: header.value })),
content: {
@ -397,33 +394,3 @@ function parseCookie(c: string): har.Cookie {
}
return cookie;
}
function calculateResponseHeadersSize(protocol: string, status: number, statusText: string , headers: types.HeadersArray) {
let rawHeaders = `${protocol} ${status} ${statusText}\r\n`;
for (const header of headers)
rawHeaders += `${header.name}: ${header.value}\r\n`;
rawHeaders += '\r\n';
return rawHeaders.length;
}
function calculateRequestHeadersSize(method: string, url: string, httpVersion: string, headers: types.HeadersArray) {
let rawHeaders = `${method} ${(new URL(url)).pathname} ${httpVersion}\r\n`;
for (const header of headers)
rawHeaders += `${header.name}: ${header.value}\r\n`;
return rawHeaders.length;
}
function normaliseHttpVersion(httpVersion?: string) {
if (!httpVersion)
return FALLBACK_HTTP_VERSION;
if (httpVersion === 'http/1.1')
return 'HTTP/1.1';
return httpVersion;
}
function calculateRequestBodySize(request: network.Request): number|undefined {
const postData = request.postDataBuffer();
if (!postData)
return;
return new TextEncoder().encode(postData.toString('utf8')).length;
}

14
src/server/webkit/wkPage.ts
View file

@ -380,6 +380,7 @@ export class WKPage implements PageDelegate {
eventsHelper.addEventListener(this._session, 'Network.responseReceived', e => this._onResponseReceived(e)),
eventsHelper.addEventListener(this._session, 'Network.loadingFinished', e => this._onLoadingFinished(e)),
eventsHelper.addEventListener(this._session, 'Network.loadingFailed', e => this._onLoadingFailed(e)),
eventsHelper.addEventListener(this._session, 'Network.dataReceived', e => this._onDataReceived(e)),
eventsHelper.addEventListener(this._session, 'Network.webSocketCreated', e => this._page._frameManager.onWebSocketCreated(e.requestId, e.url)),
eventsHelper.addEventListener(this._session, 'Network.webSocketWillSendHandshakeRequest', e => this._page._frameManager.onWebSocketRequest(e.requestId)),
eventsHelper.addEventListener(this._session, 'Network.webSocketHandshakeResponseReceived', e => this._page._frameManager.onWebSocketResponse(e.requestId, e.response.status, e.response.statusText)),
@ -1049,11 +1050,10 @@ export class WKPage implements PageDelegate {
validFrom: responseReceivedPayload?.response.security?.certificate?.validFrom,
validTo: responseReceivedPayload?.response.security?.certificate?.validUntil,
});
const { responseBodyBytesReceived, responseHeaderBytesReceived } = event.metrics || {};
const transferSize = responseBodyBytesReceived ? responseBodyBytesReceived + (responseHeaderBytesReceived || 0) : undefined;
request.request._sizes.transferSize += response.headersSize();
if (event.metrics?.protocol)
response._setHttpVersion(event.metrics.protocol);
response._requestFinished(helper.secondsToRoundishMillis(event.timestamp - request._timestamp), undefined, transferSize);
response._requestFinished(helper.secondsToRoundishMillis(event.timestamp - request._timestamp), undefined);
}
this._requestIdToResponseReceivedPayloadEvent.delete(request._requestId);
@ -1081,6 +1081,14 @@ export class WKPage implements PageDelegate {
this._page._frameManager.requestFailed(request.request, event.errorText.includes('cancelled'));
}
_onDataReceived(event: Protocol.Network.dataReceivedPayload) {
const request = this._requestIdToRequest.get(event.requestId);
if (!request)
return;
request.request._sizes.responseBodySize += event.encodedDataLength === -1 ? event.dataLength : event.encodedDataLength;
request.request._sizes.transferSize += event.encodedDataLength === -1 ? event.dataLength : event.encodedDataLength;
}
async _grantPermissions(origin: string, permissions: string[]) {
const webPermissionToProtocol = new Map<string, string>([
['geolocation', 'geolocation'],

63
tests/har.spec.ts
View file

@ -262,23 +262,34 @@ it('should include content', async ({ contextFactory, server }, testInfo) => {
expect(log.entries[1].response.content.compression).toBe(0);
});
it('should include sizes', async ({ contextFactory, server, browserName, platform }, testInfo) => {
it.fixme(browserName === 'webkit' && platform === 'linux', 'blocked by libsoup3');
it('should include sizes', async ({ contextFactory, server, asset }, testInfo) => {
const { page, getLog } = await pageWithHar(contextFactory, testInfo);
await page.goto(server.PREFIX + '/har.html');
const log = await getLog();
expect(log.entries.length).toBe(2);
expect(log.entries[0].request.url.endsWith('har.html')).toBe(true);
expect(log.entries[0].request.headersSize).toBeGreaterThanOrEqual(280);
expect(log.entries[0].response.bodySize).toBeGreaterThanOrEqual(96);
expect(log.entries[0].response.headersSize).toBe(198);
expect(log.entries[0].response.bodySize).toBe(fs.statSync(asset('har.html')).size);
expect(log.entries[0].response.headersSize).toBeGreaterThanOrEqual(198);
expect(log.entries[0].response._transferSize).toBeGreaterThanOrEqual(294);
expect(log.entries[1].response.bodySize).toBeGreaterThanOrEqual(37);
expect(log.entries[1].response.headersSize).toBe(197);
expect(log.entries[1].request.url.endsWith('one-style.css')).toBe(true);
expect(log.entries[1].response.bodySize).toBe(fs.statSync(asset('one-style.css')).size);
expect(log.entries[1].response.headersSize).toBeGreaterThanOrEqual(197);
expect(log.entries[1].response._transferSize).toBeGreaterThanOrEqual(234);
});
it('should work with gzip compression', async ({ contextFactory, server, browserName }, testInfo) => {
it.fixme(browserName === 'webkit');
const { page, getLog } = await pageWithHar(contextFactory, testInfo);
server.enableGzip('/simple.json');
const response = await page.goto(server.PREFIX + '/simple.json');
expect(response.headers()['content-encoding']).toBe('gzip');
const log = await getLog();
expect(log.entries.length).toBe(1);
expect(log.entries[0].response.content.compression).toBe(-20);
});
it('should calculate time', async ({ contextFactory, server }, testInfo) => {
const { page, getLog } = await pageWithHar(contextFactory, testInfo);
await page.goto(server.PREFIX + '/har.html');
@ -286,15 +297,14 @@ it('should calculate time', async ({ contextFactory, server }, testInfo) => {
expect(log.entries[0].time).toBeGreaterThan(0);
});
it('should report the correct _transferSize with PNG files', async ({ contextFactory, server, browserName, platform }, testInfo) => {
it.fixme(browserName === 'webkit' && platform === 'linux', 'blocked by libsoup3');
it('should report the correct _transferSize with PNG files', async ({ contextFactory, server, asset }, testInfo) => {
const { page, getLog } = await pageWithHar(contextFactory, testInfo);
await page.goto(server.EMPTY_PAGE);
await page.setContent(`
<img src="${server.PREFIX}/pptr.png" />
`);
const log = await getLog();
expect(log.entries[1].response._transferSize).toBe(6323);
expect(log.entries[1].response._transferSize).toBeGreaterThan(fs.statSync(asset('pptr.png')).size);
});
it('should have -1 _transferSize when its a failed request', async ({ contextFactory, server }, testInfo) => {
@ -311,14 +321,14 @@ it('should have -1 _transferSize when its a failed request', async ({ contextFac
expect(log.entries[1].response._transferSize).toBe(-1);
});
it('should report the correct body size', async ({ contextFactory, server }, testInfo) => {
it('should report the correct request body size', async ({ contextFactory, server }, testInfo) => {
server.setRoute('/api', (req, res) => res.end());
const { page, getLog } = await pageWithHar(contextFactory, testInfo);
await page.goto(server.EMPTY_PAGE);
await Promise.all([
page.waitForResponse(server.PREFIX + '/api'),
page.waitForResponse(server.PREFIX + '/api1'),
page.evaluate(() => {
fetch('/api', {
fetch('/api1', {
method: 'POST',
body: 'abc123'
});
@ -328,6 +338,31 @@ it('should report the correct body size', async ({ contextFactory, server }, tes
expect(log.entries[1].request.bodySize).toBe(6);
});
it('should report the correct request body size when the bodySize is 0', async ({ contextFactory, server }, testInfo) => {
server.setRoute('/api', (req, res) => res.end());
const { page, getLog } = await pageWithHar(contextFactory, testInfo);
await page.goto(server.EMPTY_PAGE);
await Promise.all([
page.waitForResponse(server.PREFIX + '/api2'),
page.evaluate(() => {
fetch('/api2', {
method: 'POST',
body: ''
});
})
]);
const log = await getLog();
expect(log.entries[1].request.bodySize).toBe(0);
});
it('should report the correct response body size when the bodySize is 0', async ({ contextFactory, server }, testInfo) => {
server.setRoute('/empty.html', (req, res) => res.end(''));
const { page, getLog } = await pageWithHar(contextFactory, testInfo);
await page.goto(server.EMPTY_PAGE);
const log = await getLog();
expect(log.entries[0].response.bodySize).toBe(0);
});
it('should have popup requests', async ({ contextFactory, server }, testInfo) => {
const { page, getLog } = await pageWithHar(contextFactory, testInfo);
await page.goto(server.EMPTY_PAGE);