From 42e44f888bffa306be806059b53b2bda6d9547c4 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Wed, 1 Sep 2021 18:28:20 -0700 Subject: [PATCH] feat(rawheaders): introduce initial plumbing (#8638) --- docs/src/api/class-headers.md | 30 ++++++++ docs/src/api/class-request.md | 7 +- docs/src/api/class-response.md | 7 +- src/client/api.ts | 1 + src/client/browserContext.ts | 6 +- src/client/network.ts | 67 +++++++++++++++++- src/client/types.ts | 3 +- src/common/types.ts | 1 + src/dispatchers/browserContextDispatcher.ts | 1 - src/dispatchers/networkDispatchers.ts | 9 ++- src/protocol/channels.ts | 14 +++- src/protocol/protocol.yml | 18 +++-- src/protocol/validator.ts | 2 + src/server/chromium/crNetworkManager.ts | 71 ++++++++++--------- src/server/frames.ts | 7 -- src/server/network.ts | 66 ++++++++--------- src/server/supplements/har/harTracer.ts | 18 +++-- src/server/webkit/wkPage.ts | 8 ++- src/utils/multimap.ts | 78 +++++++++++++++++++++ tests/page/page-network-request.spec.ts | 41 ++++++++--- types/types.d.ts | 43 +++++++++++- 21 files changed, 385 insertions(+), 113 deletions(-) create mode 100644 docs/src/api/class-headers.md create mode 100644 src/utils/multimap.ts diff --git a/docs/src/api/class-headers.md b/docs/src/api/class-headers.md new file mode 100644 index 0000000000..50525db2a0 --- /dev/null +++ b/docs/src/api/class-headers.md @@ -0,0 +1,30 @@ +# class: Headers + +HTTP request and response raw headers collection. + +## method: Headers.get +- returns: <[string|null]> +Returns header value for the given name. + +### param: Headers.get.name +- `name` <[string]> +Header name, case-insensitive. + +## method: Headers.getAll +- returns: <[Array]<[string]>> + +Returns all header values for the given header name. + +### param: Headers.getAll.name +- `name` <[string]> +Header name, case-insensitive. + +## method: Headers.headerNames +- returns: <[Array]<[string]>> + +Returns all header names in this headers collection. + +## method: Headers.headers +- returns: <[Array]<{ name: string, value: string }>> + +Returns all raw headers. diff --git a/docs/src/api/class-request.md b/docs/src/api/class-request.md index d1290849d5..137f8c6b66 100644 --- a/docs/src/api/class-request.md +++ b/docs/src/api/class-request.md @@ -54,7 +54,7 @@ Returns the [Frame] that initiated this request. ## method: Request.headers - returns: <[Object]<[string], [string]>> -An object with HTTP headers associated with the request. All header names are lower-case. +**DEPRECATED** Use [`method: Request.rawHeaders`] instead. ## method: Request.isNavigationRequest - returns: <[boolean]> @@ -85,6 +85,11 @@ Returns parsed request's body for `form-urlencoded` and JSON as a fallback if an When the response is `application/x-www-form-urlencoded` then a key/value object of the values will be returned. Otherwise it will be parsed as JSON. +## async method: Request.rawHeaders +- returns: <[Headers]> + +An object with the raw request HTTP headers associated with the request. All headers are as seen in the network stack. + ## method: Request.redirectedFrom - returns: <[null]|[Request]> diff --git a/docs/src/api/class-response.md b/docs/src/api/class-response.md index 4bf06377c4..7bff9f63ca 100644 --- a/docs/src/api/class-response.md +++ b/docs/src/api/class-response.md @@ -20,7 +20,7 @@ Returns the [Frame] that initiated this response. ## method: Response.headers - returns: <[Object]<[string], [string]>> -Returns the object with HTTP headers associated with the response. All header names are lower-case. +**DEPRECATED** Use [`method: Response.rawHeaders`] instead. ## async method: Response.json * langs: js, python @@ -43,6 +43,11 @@ This method will throw if the response body is not parsable via `JSON.parse`. Contains a boolean stating whether the response was successful (status in the range 200-299) or not. +## async method: Response.rawHeaders +- returns: <[Headers]> + +An object with the raw response HTTP headers associated with the request. All headers are as seen in the network stack. + ## method: Response.request - returns: <[Request]> diff --git a/src/client/api.ts b/src/client/api.ts index 736ddb6cb5..663659fe90 100644 --- a/src/client/api.ts +++ b/src/client/api.ts @@ -41,3 +41,4 @@ export { Video } from './video'; export { Worker } from './worker'; export { CDPSession } from './cdpSession'; export { Playwright } from './playwright'; +export { RawHeaders as Headers } from './network'; diff --git a/src/client/browserContext.ts b/src/client/browserContext.ts index 94f763ac65..538c78afa3 100644 --- a/src/client/browserContext.ts +++ b/src/client/browserContext.ts @@ -28,7 +28,7 @@ import { Events } from './events'; import { TimeoutSettings } from '../utils/timeoutSettings'; import { Waiter } from './waiter'; import { URLMatch, Headers, WaitForEventOptions, BrowserContextOptions, StorageState, LaunchOptions } from './types'; -import { isUnderTest, headersObjectToArray, mkdirIfNeeded, isString, headersArrayToObject } from '../utils/utils'; +import { isUnderTest, headersObjectToArray, mkdirIfNeeded, isString } from '../utils/utils'; import { isSafeCloseError } from '../utils/errors'; import * as api from '../../types/types'; import * as structs from '../../types/structs'; @@ -125,15 +125,13 @@ export class BrowserContext extends ChannelOwner | undefined; private _postData: Buffer | null; _timing: ResourceTiming; _sizes: RequestSizes = { requestBodySize: 0, requestHeadersSize: 0, responseBodySize: 0, responseHeadersSize: 0, responseTransferSize: 0 }; @@ -131,10 +133,26 @@ export class Request extends ChannelOwner { + if (this._rawHeadersPromise) + return this._rawHeadersPromise; + this._rawHeadersPromise = this.response().then(response => { + if (!response) + return new RawHeaders([]); + return response._wrapApiCall(async (channel: channels.ResponseChannel) => { + return new RawHeaders((await channel.rawRequestHeaders()).headers); + }); + }); + return this._rawHeadersPromise; + } + async response(): Promise { return this._wrapApiCall(async (channel: channels.RequestChannel) => { return Response.fromNullable((await channel.response()).response); @@ -183,11 +201,13 @@ export class InterceptedResponse implements api.Response { private readonly _initializer: channels.InterceptedResponse; private readonly _request: Request; private readonly _headers: Headers; + private readonly _rawHeaders: RawHeaders; constructor(route: Route, initializer: channels.InterceptedResponse) { this._route = route; this._initializer = initializer; this._headers = headersArrayToObject(initializer.headers, true /* lowerCase */); + this._rawHeaders = new RawHeaders(initializer.headers); this._request = Request.from(initializer.request); } @@ -230,6 +250,10 @@ export class InterceptedResponse implements api.Response { return { ...this._headers }; } + async rawHeaders(): Promise { + return this._rawHeaders; + } + async body(): Promise { return this._route._responseBody(); } @@ -386,6 +410,7 @@ export class Response extends ChannelOwner(); + private _rawHeadersPromise: Promise | undefined; static from(response: channels.ResponseChannel): Response { return (response as any)._object; @@ -399,7 +424,6 @@ export class Response extends ChannelOwner { + if (this._rawHeadersPromise) + return this._rawHeadersPromise; + this._rawHeadersPromise = this._wrapApiCall(async (channel: channels.ResponseChannel) => { + return new RawHeaders((await channel.rawResponseHeaders()).headers); + }); + return this._rawHeadersPromise; + } + async finished(): Promise { return this._finishedPromise.then(() => null); } @@ -600,3 +636,30 @@ export class RouteHandler { this.handledCount++; } } + +export class RawHeaders implements api.Headers { + private _headersArray: HeadersArray; + private _headersMap = new MultiMap(); + + constructor(headers: HeadersArray) { + this._headersArray = headers; + for (const header of headers) + this._headersMap.set(header.name.toLowerCase(), header.value); + } + + get(name: string): string | null { + return this.getAll(name)[0] || null; + } + + getAll(name: string): string[] { + return [...this._headersMap.get(name.toLowerCase())]; + } + + headerNames(): string[] { + return [...new Set(this._headersArray.map(h => h.name))]; + } + + headers(): HeadersArray { + return this._headersArray; + } +} diff --git a/src/client/types.ts b/src/client/types.ts index d5fa77e090..cd720adc83 100644 --- a/src/client/types.ts +++ b/src/client/types.ts @@ -16,7 +16,7 @@ */ import * as channels from '../protocol/channels'; -import type { Size } from '../common/types'; +import type { NameValue, Size } from '../common/types'; import type { ParsedStackTrace } from '../utils/stackTrace'; export { Size, Point, Rect, Quad, URLMatch, TimeoutOptions } from '../common/types'; @@ -32,6 +32,7 @@ export interface ClientSideInstrumentation { export type StrictOptions = { strict?: boolean }; export type Headers = { [key: string]: string }; +export type HeadersArray = NameValue[]; export type Env = { [key: string]: string | number | boolean | undefined }; export type WaitForEventOptions = Function | { predicate?: Function, timeout?: number }; diff --git a/src/common/types.ts b/src/common/types.ts index 6b8db6ae3c..9d361770e6 100644 --- a/src/common/types.ts +++ b/src/common/types.ts @@ -20,3 +20,4 @@ export type Rect = Size & Point; export type Quad = [ Point, Point, Point, Point ]; export type URLMatch = string | RegExp | ((url: URL) => boolean); export type TimeoutOptions = { timeout?: number }; +export type NameValue = { name: string, value: string }; diff --git a/src/dispatchers/browserContextDispatcher.ts b/src/dispatchers/browserContextDispatcher.ts index 4233ccfe90..588d71795b 100644 --- a/src/dispatchers/browserContextDispatcher.ts +++ b/src/dispatchers/browserContextDispatcher.ts @@ -86,7 +86,6 @@ export class BrowserContextDispatcher extends Dispatcher this._dispatchEvent('requestFinished', { request: RequestDispatcher.from(scope, request), response: ResponseDispatcher.fromNullable(scope, response), - responseHeaders: response?.headers(), responseEndTiming: request._responseEndTiming, requestSizes: request.sizes(), page: PageDispatcher.fromNullable(this._scope, request.frame()._page.initializedOrUndefined()), diff --git a/src/dispatchers/networkDispatchers.ts b/src/dispatchers/networkDispatchers.ts index db289a7033..d55e88fef4 100644 --- a/src/dispatchers/networkDispatchers.ts +++ b/src/dispatchers/networkDispatchers.ts @@ -67,7 +67,6 @@ export class ResponseDispatcher extends Dispatcher { return { value: await this._object.serverAddr() || undefined }; } + + async rawRequestHeaders(params?: channels.ResponseRawRequestHeadersParams, metadata?: channels.Metadata): Promise { + return { headers: await this._object.rawRequestHeaders() }; + } + + async rawResponseHeaders(params?: channels.ResponseRawResponseHeadersParams, metadata?: channels.Metadata): Promise { + return { headers: await this._object.rawResponseHeaders() }; + } } export class RouteDispatcher extends Dispatcher implements channels.RouteChannel { diff --git a/src/protocol/channels.ts b/src/protocol/channels.ts index ef1df1206f..d38e94a9b9 100644 --- a/src/protocol/channels.ts +++ b/src/protocol/channels.ts @@ -802,7 +802,6 @@ export type BrowserContextRequestFinishedEvent = { request: RequestChannel, response?: ResponseChannel, responseEndTiming: number, - responseHeaders?: NameValue[], requestSizes: RequestSizes, page?: PageChannel, }; @@ -2690,7 +2689,6 @@ export type ResponseInitializer = { url: string, status: number, statusText: string, - requestHeaders: NameValue[], headers: NameValue[], timing: ResourceTiming, }; @@ -2698,6 +2696,8 @@ export interface ResponseChannel extends Channel { body(params?: ResponseBodyParams, metadata?: Metadata): Promise; securityDetails(params?: ResponseSecurityDetailsParams, metadata?: Metadata): Promise; serverAddr(params?: ResponseServerAddrParams, metadata?: Metadata): Promise; + rawRequestHeaders(params?: ResponseRawRequestHeadersParams, metadata?: Metadata): Promise; + rawResponseHeaders(params?: ResponseRawResponseHeadersParams, metadata?: Metadata): Promise; } export type ResponseBodyParams = {}; export type ResponseBodyOptions = {}; @@ -2714,6 +2714,16 @@ export type ResponseServerAddrOptions = {}; export type ResponseServerAddrResult = { value?: RemoteAddr, }; +export type ResponseRawRequestHeadersParams = {}; +export type ResponseRawRequestHeadersOptions = {}; +export type ResponseRawRequestHeadersResult = { + headers: NameValue[], +}; +export type ResponseRawResponseHeadersParams = {}; +export type ResponseRawResponseHeadersOptions = {}; +export type ResponseRawResponseHeadersResult = { + headers: NameValue[], +}; export interface ResponseEvents { } diff --git a/src/protocol/protocol.yml b/src/protocol/protocol.yml index b108d06449..57189f3836 100644 --- a/src/protocol/protocol.yml +++ b/src/protocol/protocol.yml @@ -766,9 +766,6 @@ BrowserContext: request: Request response: Response? responseEndTiming: number - responseHeaders: - type: array? - items: NameValue requestSizes: RequestSizes page: Page? @@ -2197,9 +2194,6 @@ Response: url: string status: number statusText: string - requestHeaders: - type: array - items: NameValue headers: type: array items: NameValue @@ -2220,6 +2214,18 @@ Response: returns: value: RemoteAddr? + rawRequestHeaders: + returns: + headers: + type: array + items: NameValue + + rawResponseHeaders: + returns: + headers: + type: array + items: NameValue + SecurityDetails: type: object diff --git a/src/protocol/validator.ts b/src/protocol/validator.ts index a974813f98..59b8c9334a 100644 --- a/src/protocol/validator.ts +++ b/src/protocol/validator.ts @@ -1053,6 +1053,8 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme { scheme.ResponseBodyParams = tOptional(tObject({})); scheme.ResponseSecurityDetailsParams = tOptional(tObject({})); scheme.ResponseServerAddrParams = tOptional(tObject({})); + scheme.ResponseRawRequestHeadersParams = tOptional(tObject({})); + scheme.ResponseRawResponseHeadersParams = tOptional(tObject({})); scheme.SecurityDetails = tObject({ issuer: tOptional(tString), protocol: tOptional(tString), diff --git a/src/server/chromium/crNetworkManager.ts b/src/server/chromium/crNetworkManager.ts index 23043aa560..5c4b42c77b 100644 --- a/src/server/chromium/crNetworkManager.ts +++ b/src/server/chromium/crNetworkManager.ts @@ -38,7 +38,6 @@ export class CRNetworkManager { private _protocolRequestInterceptionEnabled = false; private _requestIdToRequestPausedEvent = new Map(); private _eventListeners: RegisteredListener[]; - private _requestIdToExtraInfo = new Map(); private _responseExtraInfoTracker = new ResponseExtraInfoTracker(); constructor(client: CRSession, page: Page, parentManager: CRNetworkManager | null) { @@ -133,19 +132,10 @@ export class CRNetworkManager { } else { this._onRequest(workerFrame, event, null); } - const extraInfo = this._requestIdToExtraInfo.get(event.requestId); - if (extraInfo) - this._onRequestWillBeSentExtraInfo(extraInfo); } _onRequestWillBeSentExtraInfo(event: Protocol.Network.requestWillBeSentExtraInfoPayload) { - const request = this._requestIdToRequest.get(event.requestId); - if (request) { - request.request.updateWithRawHeaders(headersObjectToArray(event.headers)); - this._requestIdToExtraInfo.delete(event.requestId); - } else { - this._requestIdToExtraInfo.set(event.requestId, event); - } + this._responseExtraInfoTracker.requestWillBeSentExtraInfo(event); } _onAuthRequired(event: Protocol.Fetch.authRequiredPayload) { @@ -566,6 +556,7 @@ const errorReasons: { [reason: string]: Protocol.Network.ErrorReason } = { type RequestInfo = { requestId: string, + requestWillBeSentExtraInfo: Protocol.Network.requestWillBeSentExtraInfoPayload[], responseReceivedExtraInfo: Protocol.Network.responseReceivedExtraInfoPayload[], responses: network.Response[], loadingFinished?: Protocol.Network.loadingFinishedPayload, @@ -592,26 +583,18 @@ class ResponseExtraInfoTracker { requestWillBeSent(event: Protocol.Network.requestWillBeSentPayload) { const info = this._requests.get(event.requestId); - if (info) { - // This is redirect. + if (info && event.redirectResponse) this._innerResponseReceived(info, event.redirectResponse); - } else { + else this._getOrCreateEntry(event.requestId); - } } - _getOrCreateEntry(requestId: string): RequestInfo { - let info = this._requests.get(requestId); - if (!info) { - info = { - requestId: requestId, - responseReceivedExtraInfo: [], - responses: [], - sawResponseWithoutConnectionId: false - }; - this._requests.set(requestId, info); - } - return info; + requestWillBeSentExtraInfo(event: Protocol.Network.requestWillBeSentExtraInfoPayload) { + const info = this._getOrCreateEntry(event.requestId); + if (!info) + return; + info.requestWillBeSentExtraInfo.push(event); + this._patchHeaders(info, info.requestWillBeSentExtraInfo.length - 1); } responseReceived(event: Protocol.Network.responseReceivedPayload) { @@ -621,8 +604,8 @@ class ResponseExtraInfoTracker { this._innerResponseReceived(info, event.response); } - private _innerResponseReceived(info: RequestInfo, response: Protocol.Network.Response | undefined) { - if (!response?.connectionId) { + private _innerResponseReceived(info: RequestInfo, response: Protocol.Network.Response) { + if (!response.connectionId) { // Starting with this response we no longer can guarantee that response and extra info correspond to the same index. info.sawResponseWithoutConnectionId = true; } @@ -631,7 +614,7 @@ class ResponseExtraInfoTracker { responseReceivedExtraInfo(event: Protocol.Network.responseReceivedExtraInfoPayload) { const info = this._getOrCreateEntry(event.requestId); info.responseReceivedExtraInfo.push(event); - this._patchResponseHeaders(info, info.responseReceivedExtraInfo.length - 1); + this._patchHeaders(info, info.responseReceivedExtraInfo.length - 1); this._checkFinished(info); } @@ -648,7 +631,7 @@ class ResponseExtraInfoTracker { return; response.setWillReceiveExtraHeaders(); info.responses.push(response); - this._patchResponseHeaders(info, info.responses.length - 1); + this._patchHeaders(info, info.responses.length - 1); } loadingFinished(event: Protocol.Network.loadingFinishedPayload) { @@ -667,11 +650,29 @@ class ResponseExtraInfoTracker { this._checkFinished(info); } - private _patchResponseHeaders(info: RequestInfo, index: number) { + _getOrCreateEntry(requestId: string): RequestInfo { + let info = this._requests.get(requestId); + if (!info) { + info = { + requestId: requestId, + requestWillBeSentExtraInfo: [], + responseReceivedExtraInfo: [], + responses: [], + sawResponseWithoutConnectionId: false + }; + this._requests.set(requestId, info); + } + return info; + } + + private _patchHeaders(info: RequestInfo, index: number) { const response = info.responses[index]; - const extraInfo = info.responseReceivedExtraInfo[index]; - if (response && extraInfo) - response.extraHeadersReceived(headersObjectToArray(extraInfo.headers)); + const requestExtraInfo = info.requestWillBeSentExtraInfo[index]; + if (response && requestExtraInfo) + response.setRawRequestHeaders(headersObjectToArray(requestExtraInfo.headers)); + const responseExtraInfo = info.responseReceivedExtraInfo[index]; + if (response && responseExtraInfo) + response.setRawResponseHeaders(headersObjectToArray(responseExtraInfo.headers)); } private _checkFinished(info: RequestInfo) { diff --git a/src/server/frames.ts b/src/server/frames.ts index 082caa79f1..510aa71965 100644 --- a/src/server/frames.ts +++ b/src/server/frames.ts @@ -280,13 +280,6 @@ export class FrameManager { this._inflightRequestFinished(request); if (request._isFavicon) return; - this._dispatchRequestFinished(request, response).catch(() => {}); - } - - private async _dispatchRequestFinished(request: network.Request, response: network.Response | null) { - // Avoid unnecessary microtask, we want to report finished early for regular redirects. - if (response?.willWaitForExtraHeaders()) - await response?.waitForExtraHeadersIfNeeded(); this._page._browserContext.emit(BrowserContext.Events.RequestFinished, { request, response }); } diff --git a/src/server/network.ts b/src/server/network.ts index e965613879..57b8b77bd6 100644 --- a/src/server/network.ts +++ b/src/server/network.ts @@ -19,6 +19,7 @@ import * as types from './types'; import { assert } from '../utils/utils'; import { ManualPromise } from '../utils/async'; import { SdkObject } from './instrumentation'; +import { NameValue } from '../common/types'; export function filterCookies(cookies: types.NetworkCookie[], urls: string[]): types.NetworkCookie[] { const parsedURLs = urls.map(s => new URL(s)); @@ -95,7 +96,7 @@ export class Request extends SdkObject { private _resourceType: string; private _method: string; private _postData: Buffer | null; - private _headers: types.HeadersArray; + readonly _headers: types.HeadersArray; private _headersMap = new Map(); private _frame: frames.Frame; private _waitForResponsePromise = new ManualPromise(); @@ -150,6 +151,10 @@ export class Request extends SdkObject { return this._headersMap.get(name); } + async rawHeaders(): Promise { + return this._headers; + } + response(): PromiseLike { return this._waitForResponsePromise; } @@ -187,18 +192,6 @@ export class Request extends SdkObject { }; } - updateWithRawHeaders(headers: types.HeadersArray) { - this._headers = headers; - this._headersMap.clear(); - for (const { name, value } of this._headers) - this._headersMap.set(name.toLowerCase(), value); - if (!this._headersMap.has('host')) { - const host = new URL(this._url).host; - this._headers.push({ name: 'host', value: host }); - this._headersMap.set('host', host); - } - } - bodySize(): number { return this.postDataBuffer()?.length || 0; } @@ -330,7 +323,8 @@ export class Response extends SdkObject { private _timing: ResourceTiming; private _serverAddrPromise = new ManualPromise(); private _securityDetailsPromise = new ManualPromise(); - private _extraHeadersPromise: ManualPromise | undefined; + private _rawRequestHeadersPromise: ManualPromise | undefined; + private _rawResponseHeadersPromise: ManualPromise | undefined; private _httpVersion: string | undefined; constructor(request: Request, status: number, statusText: string, headers: types.HeadersArray, timing: ResourceTiming, getResponseBodyCallback: GetResponseBodyCallback, httpVersion?: string) { @@ -365,25 +359,6 @@ export class Response extends SdkObject { this._httpVersion = httpVersion; } - setWillReceiveExtraHeaders() { - this._extraHeadersPromise = new ManualPromise(); - } - - willWaitForExtraHeaders(): boolean { - return !!this._extraHeadersPromise && !this._extraHeadersPromise.isDone(); - } - - async waitForExtraHeadersIfNeeded(): Promise { - await this._extraHeadersPromise; - } - - extraHeadersReceived(headers: types.HeadersArray) { - this._headers = headers; - for (const { name, value } of this._headers) - this._headersMap.set(name.toLowerCase(), value); - this._extraHeadersPromise?.resolve(); - } - url(): string { return this._url; } @@ -404,6 +379,31 @@ export class Response extends SdkObject { return this._headersMap.get(name); } + async rawRequestHeaders(): Promise { + return this._rawRequestHeadersPromise || Promise.resolve(this._request._headers); + } + + async rawResponseHeaders(): Promise { + return this._rawResponseHeadersPromise || Promise.resolve(this._headers); + } + + setWillReceiveExtraHeaders() { + this._rawRequestHeadersPromise = new ManualPromise(); + this._rawResponseHeadersPromise = new ManualPromise(); + } + + setRawRequestHeaders(headers: types.HeadersArray) { + if (!this._rawRequestHeadersPromise) + this._rawRequestHeadersPromise = new ManualPromise(); + this._rawRequestHeadersPromise!.resolve(headers); + } + + setRawResponseHeaders(headers: types.HeadersArray) { + if (!this._rawResponseHeadersPromise) + this._rawResponseHeadersPromise = new ManualPromise(); + this._rawResponseHeadersPromise!.resolve(headers); + } + timing(): ResourceTiming { return this._timing; } diff --git a/src/server/supplements/har/harTracer.ts b/src/server/supplements/har/harTracer.ts index 7bcc93ff44..1de0da2187 100644 --- a/src/server/supplements/har/harTracer.ts +++ b/src/server/supplements/har/harTracer.ts @@ -246,9 +246,6 @@ export class HarTracer { return; const request = response.request(); - // Rewrite provisional headers with actual - harEntry.request.headers = request.headers().map(header => ({ name: header.name, value: header.value })); - harEntry.request.cookies = cookiesForHar(request.headerValue('cookie'), ';'); harEntry.request.postData = postDataForHar(request, this._options.content); harEntry.response = { @@ -259,7 +256,7 @@ export class HarTracer { headers: response.headers().map(header => ({ name: header.name, value: header.value })), content: { size: -1, - mimeType: response.headerValue('content-type') || 'x-unknown', + mimeType: 'x-unknown', }, headersSize: -1, bodySize: -1, @@ -293,6 +290,19 @@ export class HarTracer { if (details) harEntry._securityDetails = details; })); + this._addBarrier(page, response.rawRequestHeaders().then(headers => { + for (const header of headers.filter(header => header.name.toLowerCase() === 'cookie')) + harEntry.request.cookies.push(...cookiesForHar(header.value, ';')); + harEntry.request.headers = headers; + })); + this._addBarrier(page, response.rawResponseHeaders().then(headers => { + for (const header of headers.filter(header => header.name.toLowerCase() === 'set-cookie')) + harEntry.response.cookies.push(...cookiesForHar(header.value, '\n')); + harEntry.response.headers = headers; + const contentType = headers.find(header => header.name.toLowerCase() === 'content-type'); + if (contentType) + harEntry.response.content.mimeType = contentType.value; + })); } async flush() { diff --git a/src/server/webkit/wkPage.ts b/src/server/webkit/wkPage.ts index f348b7b141..a2cb05a87b 100644 --- a/src/server/webkit/wkPage.ts +++ b/src/server/webkit/wkPage.ts @@ -1017,8 +1017,12 @@ export class WKPage implements PageDelegate { return; this._requestIdToResponseReceivedPayloadEvent.set(request._requestId, event); const response = request.createResponse(event.response); - if (event.response.requestHeaders && Object.keys(event.response.requestHeaders).length) - request.request.updateWithRawHeaders(headersObjectToArray(event.response.requestHeaders)); + if (event.response.requestHeaders && Object.keys(event.response.requestHeaders).length) { + const headers = { ...event.response.requestHeaders }; + if (!headers['host']) + headers['Host'] = new URL(request.request.url()).host; + response.setRawRequestHeaders(headersObjectToArray(headers)); + } this._page._frameManager.requestReceivedResponse(response); if (response.status() === 204) { diff --git a/src/utils/multimap.ts b/src/utils/multimap.ts new file mode 100644 index 0000000000..e32e1efbf2 --- /dev/null +++ b/src/utils/multimap.ts @@ -0,0 +1,78 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export class MultiMap { + private _map: Map>; + + constructor() { + this._map = new Map>(); + } + + set(key: K, value: V) { + let set = this._map.get(key); + if (!set) { + set = new Set(); + this._map.set(key, set); + } + set.add(value); + } + + get(key: K): Set { + return this._map.get(key) || new Set(); + } + + has(key: K): boolean { + return this._map.has(key); + } + + hasValue(key: K, value: V): boolean { + const set = this._map.get(key); + if (!set) + return false; + return set.has(value); + } + + get size(): number { + return this._map.size; + } + + delete(key: K, value: V): boolean { + const values = this.get(key); + const result = values.delete(value); + if (!values.size) + this._map.delete(key); + return result; + } + + deleteAll(key: K) { + this._map.delete(key); + } + + keys(): IterableIterator { + return this._map.keys(); + } + + values(): Iterable { + const result: V[] = []; + for (const key of this.keys()) + result.push(...Array.from(this.get(key)!)); + return result; + } + + clear() { + this._map.clear(); + } +} diff --git a/tests/page/page-network-request.spec.ts b/tests/page/page-network-request.spec.ts index b30b7bd4a6..03232951f1 100644 --- a/tests/page/page-network-request.spec.ts +++ b/tests/page/page-network-request.spec.ts @@ -83,18 +83,20 @@ it('should return headers', async ({page, server, browserName}) => { it('should get the same headers as the server', async ({ page, server, browserName, platform }) => { it.fail(browserName === 'webkit' && platform === 'win32', 'Curl does not show accept-encoding and accept-language'); - it.fixme(browserName === 'chromium', 'Flaky, see https://github.com/microsoft/playwright/issues/6690'); - let serverRequest; server.setRoute('/empty.html', (request, response) => { serverRequest = request; response.end('done'); }); const response = await page.goto(server.PREFIX + '/empty.html'); - expect(response.request().headers()).toEqual(serverRequest.headers); + const headers = await response.request().rawHeaders(); + const result = {}; + for (const header of headers.headers()) + result[header.name.toLowerCase()] = header.value; + expect(result).toEqual(serverRequest.headers); }); -it('should get the same headers as the server CORP', async ({page, server, browserName, platform}) => { +it('should get the same headers as the server CORS', async ({page, server, browserName, platform}) => { it.fail(browserName === 'webkit' && platform === 'win32', 'Curl does not show accept-encoding and accept-language'); await page.goto(server.PREFIX + '/empty.html'); @@ -109,9 +111,14 @@ it('should get the same headers as the server CORP', async ({page, server, brows const data = await fetch(url); return data.text(); }, server.CROSS_PROCESS_PREFIX + '/something'); - const response = await responsePromise; expect(text).toBe('done'); - expect(response.request().headers()).toEqual(serverRequest.headers); + const response = await responsePromise; + const headers = await response.request().rawHeaders(); + const result = {}; + for (const header of headers.headers()) + result[header.name.toLowerCase()] = header.value; + + expect(result).toEqual(serverRequest.headers); }); it('should return postData', async ({page, server, isAndroid}) => { @@ -274,7 +281,7 @@ it('should set bodySize and headersSize', async ({page, server,browserName, plat ]); await (await request.response()).finished(); expect(request.sizes().requestBodySize).toBe(5); - expect(request.sizes().requestHeadersSize).toBeGreaterThanOrEqual(300); + expect(request.sizes().requestHeadersSize).toBeGreaterThanOrEqual(250); }); it('should should set bodySize to 0 if there was no body', async ({page, server,browserName, platform}) => { @@ -285,7 +292,7 @@ it('should should set bodySize to 0 if there was no body', async ({page, server, ]); await (await request.response()).finished(); expect(request.sizes().requestBodySize).toBe(0); - expect(request.sizes().requestHeadersSize).toBeGreaterThanOrEqual(228); + expect(request.sizes().requestHeadersSize).toBeGreaterThanOrEqual(200); }); it('should should set bodySize, headersSize, and transferSize', async ({page, server, browserName, platform}) => { @@ -315,6 +322,19 @@ it('should should set bodySize to 0 when there was no response body', async ({pa expect(response.request().sizes().responseTransferSize).toBeGreaterThanOrEqual(160); }); +it('should report raw headers', async ({ page, server, browserName }) => { + const response = await page.goto(server.EMPTY_PAGE); + const requestHeaders = await response.request().rawHeaders(); + expect(requestHeaders.headerNames().map(h => h.toLowerCase())).toContain('accept'); + expect(requestHeaders.getAll('host')).toHaveLength(1); + expect(requestHeaders.get('host')).toBe(`localhost:${server.PORT}`); + + const responseHeaders = await response.rawHeaders(); + expect(responseHeaders.headerNames().map(h => h.toLowerCase())).toContain('content-type'); + expect(responseHeaders.getAll('content-type')).toHaveLength(1); + expect(responseHeaders.get('content-type')).toBe('text/html; charset=utf-8'); +}); + it('should report raw response headers in redirects', async ({ page, server, browserName }) => { it.skip(browserName === 'webkit', `WebKit won't give us raw headers for redirects`); server.setExtraHeaders('/redirect/1.html', { 'sec-test-header': '1.html' }); @@ -327,14 +347,13 @@ it('should report raw response headers in redirects', async ({ page, server, bro const expectedHeaders = ['1.html', '2.html', 'empty.html']; const response = await page.goto(server.PREFIX + '/redirect/1.html'); - await response.finished(); - const redirectChain = []; const headersChain = []; for (let req = response.request(); req; req = req.redirectedFrom()) { redirectChain.unshift(req.url()); const res = await req.response(); - headersChain.unshift(res.headers()['sec-test-header']); + const headers = await res.rawHeaders(); + headersChain.unshift(headers.get('sec-test-header')); } expect(redirectChain).toEqual(expectedUrls); diff --git a/types/types.d.ts b/types/types.d.ts index 8f1de02218..ebc837dfea 100644 --- a/types/types.d.ts +++ b/types/types.d.ts @@ -12667,6 +12667,32 @@ export interface FileChooser { }): Promise; } +/** + * HTTP request and response raw headers collection. + */ +export interface Headers { + /** + * @param name + */ + get(name: string): string|null; + + /** + * Returns all header values for the given header name. + * @param name + */ + getAll(name: string): Array; + + /** + * Returns all header names in this headers collection. + */ + headerNames(): Array; + + /** + * Returns all raw headers. + */ + headers(): Array<{ name: string, value: string }>; +} + /** * Keyboard provides an api for managing a virtual keyboard. The high level api is * [keyboard.type(text[, options])](https://playwright.dev/docs/api/class-keyboard#keyboard-type), which takes raw @@ -13022,7 +13048,8 @@ export interface Request { frame(): Frame; /** - * An object with HTTP headers associated with the request. All header names are lower-case. + * **DEPRECATED** Use [request.rawHeaders()](https://playwright.dev/docs/api/class-request#request-raw-headers) instead. + * @deprecated */ headers(): { [key: string]: string; }; @@ -13054,6 +13081,11 @@ export interface Request { */ postDataJSON(): null|any; + /** + * An object with the raw request HTTP headers associated with the request. All headers are as seen in the network stack. + */ + rawHeaders(): Promise; + /** * Request that was redirected by the server to this one, if any. * @@ -13229,7 +13261,9 @@ export interface Response { frame(): Frame; /** - * Returns the object with HTTP headers associated with the response. All header names are lower-case. + * **DEPRECATED** Use [response.rawHeaders()](https://playwright.dev/docs/api/class-response#response-raw-headers) + * instead. + * @deprecated */ headers(): { [key: string]: string; }; @@ -13245,6 +13279,11 @@ export interface Response { */ ok(): boolean; + /** + * An object with the raw response HTTP headers associated with the request. All headers are as seen in the network stack. + */ + rawHeaders(): Promise; + /** * Returns the matching [Request] object. */