diff --git a/packages/playwright-core/src/server/har/harRecorder.ts b/packages/playwright-core/src/server/har/harRecorder.ts index d4af1310fd..6f175c9243 100644 --- a/packages/playwright-core/src/server/har/harRecorder.ts +++ b/packages/playwright-core/src/server/har/harRecorder.ts @@ -45,6 +45,7 @@ export class HarRecorder { content, slimMode: options.mode === 'minimal', includeTraceInfo: false, + recordRequestOverrides: true, waitForContentOnStop: true, skipScripts: false, urlFilter: urlFilterRe ?? options.urlGlob, diff --git a/packages/playwright-core/src/server/har/harTracer.ts b/packages/playwright-core/src/server/har/harTracer.ts index ce1ba85ff8..ceab8fc4ea 100644 --- a/packages/playwright-core/src/server/har/harTracer.ts +++ b/packages/playwright-core/src/server/har/harTracer.ts @@ -30,7 +30,7 @@ import { ManualPromise } from '../../utils/manualPromise'; import { getPlaywrightVersion } from '../../common/userAgent'; import { urlMatches } from '../../common/netUtils'; import { Frame } from '../frames'; -import type { LifecycleEvent } from '../types'; +import type { HeadersArray, LifecycleEvent } from '../types'; import { isTextualMimeType } from '../../utils/mimeType'; const FALLBACK_HTTP_VERSION = 'HTTP/1.1'; @@ -45,6 +45,7 @@ type HarTracerOptions = { content: 'omit' | 'attach' | 'embed'; skipScripts: boolean; includeTraceInfo: boolean; + recordRequestOverrides: boolean; waitForContentOnStop: boolean; urlFilter?: string | RegExp; slimMode?: boolean; @@ -248,6 +249,7 @@ export class HarTracer { const harEntry = createHarEntry(request.method(), url, request.frame()?.guid, this._options); if (pageEntry) harEntry.pageref = pageEntry.id; + this._recordRequestHeadersAndCookies(harEntry, request.headers()); harEntry.request.postData = this._postDataForRequest(request, this._options.content); if (!this._options.omitSizes) harEntry.request.bodySize = request.bodySize(); @@ -261,6 +263,24 @@ export class HarTracer { this._delegate.onEntryStarted(harEntry); } + private _recordRequestHeadersAndCookies(harEntry: har.Entry, headers: HeadersArray) { + if (!this._options.omitCookies) { + harEntry.request.cookies = []; + for (const header of headers.filter(header => header.name.toLowerCase() === 'cookie')) + harEntry.request.cookies.push(...header.value.split(';').map(parseCookie)); + } + harEntry.request.headers = headers; + } + + private _recordRequestOverrides(harEntry: har.Entry, request: network.Request) { + if (!request._hasOverrides() || !this._options.recordRequestOverrides) + return; + harEntry.request.method = request.method(); + harEntry.request.url = request.url(); + harEntry.request.postData = this._postDataForRequest(request, this._options.content); + this._recordRequestHeadersAndCookies(harEntry, request.headers()); + } + private async _onRequestFinished(request: network.Request, response: network.Response | null) { if (!response) return; @@ -330,6 +350,7 @@ export class HarTracer { if (request._failureText !== null) harEntry.response._failureText = request._failureText; + this._recordRequestOverrides(harEntry, request); if (this._started) this._delegate.onEntryFinished(harEntry); } @@ -423,12 +444,9 @@ export class HarTracer { harEntry._securityDetails = details; })); } + this._recordRequestOverrides(harEntry, request); this._addBarrier(page || request.serviceWorker(), request.rawRequestHeaders().then(headers => { - if (!this._options.omitCookies) { - for (const header of headers.filter(header => header.name.toLowerCase() === 'cookie')) - harEntry.request.cookies.push(...header.value.split(';').map(parseCookie)); - } - harEntry.request.headers = headers; + this._recordRequestHeadersAndCookies(harEntry, headers); })); this._addBarrier(page || request.serviceWorker(), response.rawResponseHeaders().then(headers => { if (!this._options.omitCookies) { diff --git a/packages/playwright-core/src/server/network.ts b/packages/playwright-core/src/server/network.ts index 19604df324..1294d6878e 100644 --- a/packages/playwright-core/src/server/network.ts +++ b/packages/playwright-core/src/server/network.ts @@ -22,8 +22,9 @@ import type * as channels from '../protocol/channels'; import { assert } from '../utils'; import { ManualPromise } from '../utils/manualPromise'; import { SdkObject } from './instrumentation'; -import type { NameValue } from '../common/types'; +import type { HeadersArray, NameValue } from '../common/types'; import { APIRequestContext } from './fetch'; +import type { NormalizedContinueOverrides } from './types'; export function filterCookies(cookies: channels.NetworkCookie[], urls: string[]): channels.NetworkCookie[] { const parsedURLs = urls.map(s => new URL(s)); @@ -97,17 +98,18 @@ export class Request extends SdkObject { private _resourceType: string; private _method: string; private _postData: Buffer | null; - readonly _headers: types.HeadersArray; + readonly _headers: HeadersArray; private _headersMap = new Map(); readonly _frame: frames.Frame | null = null; readonly _serviceWorker: pages.Worker | null = null; readonly _context: contexts.BrowserContext; - private _rawRequestHeadersPromise = new ManualPromise(); + private _rawRequestHeadersPromise = new ManualPromise(); private _waitForResponsePromise = new ManualPromise(); _responseEndTiming = -1; + private _overrides: NormalizedContinueOverrides | undefined; constructor(context: contexts.BrowserContext, frame: frames.Frame | null, serviceWorker: pages.Worker | null, redirectedFrom: Request | null, documentId: string | undefined, - url: string, resourceType: string, method: string, postData: Buffer | null, headers: types.HeadersArray) { + url: string, resourceType: string, method: string, postData: Buffer | null, headers: HeadersArray) { super(frame || context, 'request'); assert(!url.startsWith('data:'), 'Data urls should not fire requests'); this._context = context; @@ -122,8 +124,7 @@ export class Request extends SdkObject { this._method = method; this._postData = postData; this._headers = headers; - for (const { name, value } of this._headers) - this._headersMap.set(name.toLowerCase(), value); + this._updateHeadersMap(); this._isFavicon = url.endsWith('/favicon.ico') || !!redirectedFrom?._isFavicon; } @@ -132,8 +133,22 @@ export class Request extends SdkObject { this._waitForResponsePromise.resolve(null); } + _setOverrides(overrides: types.NormalizedContinueOverrides) { + this._overrides = overrides; + this._updateHeadersMap(); + } + + private _updateHeadersMap() { + for (const { name, value } of this.headers()) + this._headersMap.set(name.toLowerCase(), value); + } + + _hasOverrides() { + return !!this._overrides; + } + url(): string { - return this._url; + return this._overrides?.url || this._url; } resourceType(): string { @@ -141,15 +156,15 @@ export class Request extends SdkObject { } method(): string { - return this._method; + return this._overrides?.method || this._method; } postDataBuffer(): Buffer | null { - return this._postData; + return this._overrides?.postData || this._postData; } - headers(): types.HeadersArray { - return this._headers; + headers(): HeadersArray { + return this._overrides?.headers || this._headers; } headerValue(name: string): string | undefined { @@ -157,13 +172,13 @@ export class Request extends SdkObject { } // "null" means no raw headers available - we'll use provisional headers as raw headers. - setRawRequestHeaders(headers: types.HeadersArray | null) { + setRawRequestHeaders(headers: HeadersArray | null) { if (!this._rawRequestHeadersPromise.isDone()) this._rawRequestHeadersPromise.resolve(headers || this._headers); } - async rawRequestHeaders(): Promise { - return this._rawRequestHeadersPromise; + async rawRequestHeaders(): Promise { + return this._overrides?.headers || this._rawRequestHeadersPromise; } response(): PromiseLike { @@ -303,6 +318,7 @@ export class Route extends SdkObject { if (oldUrl.protocol !== newUrl.protocol) throw new Error('New URL must have same protocol as overridden URL'); } + this._request._setOverrides(overrides); await this._delegate.continue(this._request, overrides); this._endHandling(); } @@ -360,20 +376,20 @@ export class Response extends SdkObject { private _status: number; private _statusText: string; private _url: string; - private _headers: types.HeadersArray; + private _headers: HeadersArray; private _headersMap = new Map(); private _getResponseBodyCallback: GetResponseBodyCallback; private _timing: ResourceTiming; private _serverAddrPromise = new ManualPromise(); private _securityDetailsPromise = new ManualPromise(); - private _rawResponseHeadersPromise = new ManualPromise(); + private _rawResponseHeadersPromise = new ManualPromise(); private _httpVersion: string | undefined; private _fromServiceWorker: boolean; private _encodedBodySizePromise = new ManualPromise(); private _transferSizePromise = new ManualPromise(); private _responseHeadersSizePromise = new ManualPromise(); - constructor(request: Request, status: number, statusText: string, headers: types.HeadersArray, timing: ResourceTiming, getResponseBodyCallback: GetResponseBodyCallback, fromServiceWorker: boolean, httpVersion?: string) { + constructor(request: Request, status: number, statusText: string, headers: HeadersArray, timing: ResourceTiming, getResponseBodyCallback: GetResponseBodyCallback, fromServiceWorker: boolean, httpVersion?: string) { super(request.frame() || request._context, 'response'); this._request = request; this._timing = timing; @@ -418,7 +434,7 @@ export class Response extends SdkObject { return this._statusText; } - headers(): types.HeadersArray { + headers(): HeadersArray { return this._headers; } @@ -431,7 +447,7 @@ export class Response extends SdkObject { } // "null" means no raw headers available - we'll use provisional headers as raw headers. - setRawResponseHeaders(headers: types.HeadersArray | null) { + setRawResponseHeaders(headers: HeadersArray | null) { if (!this._rawResponseHeadersPromise.isDone()) this._rawResponseHeadersPromise.resolve(headers || this._headers); } @@ -658,11 +674,11 @@ export const STATUS_TEXTS: { [status: string]: string } = { '511': 'Network Authentication Required', }; -export function singleHeader(name: string, value: string): types.HeadersArray { +export function singleHeader(name: string, value: string): HeadersArray { return [{ name, value }]; } -export function mergeHeaders(headers: (types.HeadersArray | undefined | null)[]): types.HeadersArray { +export function mergeHeaders(headers: (HeadersArray | undefined | null)[]): HeadersArray { const lowerCaseToValue = new Map(); const lowerCaseToOriginalCase = new Map(); for (const h of headers) { @@ -674,7 +690,7 @@ export function mergeHeaders(headers: (types.HeadersArray | undefined | null)[]) lowerCaseToValue.set(lower, value); } } - const result: types.HeadersArray = []; + const result: HeadersArray = []; for (const [lower, value] of lowerCaseToValue) result.push({ name: lowerCaseToOriginalCase.get(lower)!, value }); return result; diff --git a/packages/playwright-core/src/server/trace/recorder/tracing.ts b/packages/playwright-core/src/server/trace/recorder/tracing.ts index c2e64a165e..7c9ffbc2d4 100644 --- a/packages/playwright-core/src/server/trace/recorder/tracing.ts +++ b/packages/playwright-core/src/server/trace/recorder/tracing.ts @@ -87,6 +87,7 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps this._harTracer = new HarTracer(context, null, this, { content: 'attach', includeTraceInfo: true, + recordRequestOverrides: false, waitForContentOnStop: false, skipScripts: true, }); diff --git a/packages/playwright-core/src/server/trace/test/inMemorySnapshotter.ts b/packages/playwright-core/src/server/trace/test/inMemorySnapshotter.ts index 6f272a62a0..efe6c4ecaa 100644 --- a/packages/playwright-core/src/server/trace/test/inMemorySnapshotter.ts +++ b/packages/playwright-core/src/server/trace/test/inMemorySnapshotter.ts @@ -34,7 +34,7 @@ export class InMemorySnapshotter extends BaseSnapshotStorage implements Snapshot constructor(context: BrowserContext) { super(); this._snapshotter = new Snapshotter(context, this); - this._harTracer = new HarTracer(context, null, this, { content: 'attach', includeTraceInfo: true, waitForContentOnStop: false, skipScripts: true }); + this._harTracer = new HarTracer(context, null, this, { content: 'attach', includeTraceInfo: true, recordRequestOverrides: false, waitForContentOnStop: false, skipScripts: true }); } async initialize(): Promise { diff --git a/packages/playwright-core/src/server/types.ts b/packages/playwright-core/src/server/types.ts index b9a1b24fb7..9f5f9c0dfe 100644 --- a/packages/playwright-core/src/server/types.ts +++ b/packages/playwright-core/src/server/types.ts @@ -15,8 +15,8 @@ * limitations under the License. */ -import type { Size, Point, TimeoutOptions } from '../common/types'; -export type { Size, Point, Rect, Quad, URLMatch, TimeoutOptions } from '../common/types'; +import type { Size, Point, TimeoutOptions, HeadersArray } from '../common/types'; +export type { Size, Point, Rect, Quad, URLMatch, TimeoutOptions, HeadersArray } from '../common/types'; import type * as channels from '../protocol/channels'; export type StrictOptions = { @@ -129,8 +129,6 @@ export type MouseMultiClickOptions = PointerActionOptions & { export type World = 'main' | 'utility'; -export type HeadersArray = { name: string, value: string }[]; - export type GotoOptions = NavigateOptions & { referer?: string, }; diff --git a/tests/library/har.spec.ts b/tests/library/har.spec.ts index da05fc0d95..74a2ce537b 100644 --- a/tests/library/har.spec.ts +++ b/tests/library/har.spec.ts @@ -256,6 +256,32 @@ it('should include secure set-cookies', async ({ contextFactory, httpsServer }, expect(cookies[0]).toEqual({ name: 'name1', value: 'value1', secure: true }); }); +it('should record request overrides', async ({ contextFactory, server }, testInfo) => { + const { page, getLog } = await pageWithHar(contextFactory, testInfo); + page.route('**/foo', route => { + route.fallback({ + url: server.EMPTY_PAGE, + method: 'POST', + headers: { + ...route.request().headers(), + 'content-type': 'text/plain', + 'cookie': 'foo=bar', + 'custom': 'value' + }, + postData: 'Hi!' + }); + }); + + await page.goto(server.PREFIX + '/foo'); + const log = await getLog(); + const request = log.entries[0].request; + expect(request.url).toBe(server.EMPTY_PAGE); + expect(request.method).toBe('POST'); + expect(request.headers).toContainEqual({ name: 'custom', value: 'value' }); + expect(request.cookies).toContainEqual({ name: 'foo', value: 'bar' }); + expect(request.postData).toEqual({ 'mimeType': 'text/plain', 'params': [], 'text': 'Hi!' }); +}); + it('should include content @smoke', async ({ contextFactory, server }, testInfo) => { const { page, getLog } = await pageWithHar(contextFactory, testInfo); await page.goto(server.PREFIX + '/har.html'); @@ -409,7 +435,7 @@ it('should have -1 _transferSize when its a failed request', async ({ contextFac const { page, getLog } = await pageWithHar(contextFactory, testInfo); server.setRoute('/one-style.css', (req, res) => { res.setHeader('Content-Type', 'text/css'); - res.connection.destroy(); + res.socket.destroy(); }); const failedRequests = []; page.on('requestfailed', request => failedRequests.push(request)); @@ -419,6 +445,49 @@ it('should have -1 _transferSize when its a failed request', async ({ contextFac expect(log.entries[1].response._transferSize).toBe(-1); }); +it('should record failed request headers', async ({ contextFactory, server }, testInfo) => { + const { page, getLog } = await pageWithHar(contextFactory, testInfo); + server.setRoute('/har.html', (req, res) => { + res.socket.destroy(); + }); + await page.goto(server.PREFIX + '/har.html').catch(() => {}); + const log = await getLog(); + expect(log.entries[0].response._failureText).toBeTruthy(); + const request = log.entries[0].request; + expect(request.url.endsWith('/har.html')).toBe(true); + expect(request.method).toBe('GET'); + expect(request.headers).toContainEqual(expect.objectContaining({ name: 'User-Agent' })); +}); + +it('should record failed request overrides', async ({ contextFactory, server }, testInfo) => { + const { page, getLog } = await pageWithHar(contextFactory, testInfo); + server.setRoute('/empty.html', (req, res) => { + res.socket.destroy(); + }); + await page.route('**/foo', route => { + route.fallback({ + url: server.EMPTY_PAGE, + method: 'POST', + headers: { + ...route.request().headers(), + 'content-type': 'text/plain', + 'cookie': 'foo=bar', + 'custom': 'value' + }, + postData: 'Hi!' + }); + }); + await page.goto(server.PREFIX + '/foo').catch(() => {}); + const log = await getLog(); + expect(log.entries[0].response._failureText).toBeTruthy(); + const request = log.entries[0].request; + expect(request.url).toBe(server.EMPTY_PAGE); + expect(request.method).toBe('POST'); + expect(request.headers).toContainEqual({ name: 'custom', value: 'value' }); + expect(request.cookies).toContainEqual({ name: 'foo', value: 'bar' }); + expect(request.postData).toEqual({ 'mimeType': 'text/plain', 'params': [], 'text': 'Hi!' }); +}); + it('should report the correct request body size', async ({ contextFactory, server }, testInfo) => { server.setRoute('/api', (req, res) => res.end()); const { page, getLog } = await pageWithHar(contextFactory, testInfo); @@ -556,7 +625,7 @@ it('should have connection details for redirects', async ({ contextFactory, serv it('should have connection details for failed requests', async ({ contextFactory, server, browserName, platform, mode }, testInfo) => { server.setRoute('/one-style.css', (_, res) => { res.setHeader('Content-Type', 'text/css'); - res.connection.destroy(); + res.socket.destroy(); }); const { page, getLog } = await pageWithHar(contextFactory, testInfo); await page.goto(server.PREFIX + '/one-style.html'); diff --git a/tests/library/trace-viewer.spec.ts b/tests/library/trace-viewer.spec.ts index 5c148811bc..147c76f8d5 100644 --- a/tests/library/trace-viewer.spec.ts +++ b/tests/library/trace-viewer.spec.ts @@ -686,3 +686,24 @@ test('should include requestUrl in route.abort', async ({ page, runAndTrace, ser await expect(callLine.locator('text=requestUrl')).toContainText('http://test.com'); }); +test('should serve overridden request', async ({ page, runAndTrace, server }) => { + server.setRoute('/custom.css', (req, res) => { + res.writeHead(200, { + 'Content-Type': 'text/css', + }); + res.end(`body { background: red }`); + }); + await page.route('**/one-style.css', route => { + route.continue({ + url: server.PREFIX + '/custom.css' + }); + }); + const traceViewer = await runAndTrace(async () => { + await page.goto(server.PREFIX + '/one-style.html'); + }); + // Render snapshot, check expectations. + const snapshotFrame = await traceViewer.snapshotFrame('page.goto'); + const color = await snapshotFrame.locator('body').evaluate(body => getComputedStyle(body).backgroundColor); + expect(color).toBe('rgb(255, 0, 0)'); +}); + diff --git a/tests/page/page-request-continue.spec.ts b/tests/page/page-request-continue.spec.ts index 4f543e2160..1cbf42a889 100644 --- a/tests/page/page-request-continue.spec.ts +++ b/tests/page/page-request-continue.spec.ts @@ -87,17 +87,15 @@ it('should amend method', async ({ page, server }) => { }); it('should override request url', async ({ page, server }) => { - const request = server.waitForRequest('/global-var.html'); + const serverRequest = server.waitForRequest('/global-var.html'); await page.route('**/foo', route => { route.continue({ url: server.PREFIX + '/global-var.html' }); }); - const [response] = await Promise.all([ - page.waitForEvent('response'), - page.goto(server.PREFIX + '/foo'), - ]); - expect(response.url()).toBe(server.PREFIX + '/foo'); + const response = await page.goto(server.PREFIX + '/foo'); + expect(response.request().url()).toBe(server.PREFIX + '/global-var.html'); + expect(response.url()).toBe(server.PREFIX + '/global-var.html'); expect(await page.evaluate(() => window['globalVar'])).toBe(123); - expect((await request).method).toBe('GET'); + expect((await serverRequest).method).toBe('GET'); }); it('should not allow changing protocol when overriding url', async ({ page, server }) => { diff --git a/tests/page/page-request-fallback.spec.ts b/tests/page/page-request-fallback.spec.ts index f373617466..d46ac9723d 100644 --- a/tests/page/page-request-fallback.spec.ts +++ b/tests/page/page-request-fallback.spec.ts @@ -199,7 +199,7 @@ it('should amend method', async ({ page, server }) => { }); it('should override request url', async ({ page, server }) => { - const request = server.waitForRequest('/global-var.html'); + const serverRequest = server.waitForRequest('/global-var.html'); let url: string; await page.route('**/global-var.html', route => { @@ -209,14 +209,12 @@ it('should override request url', async ({ page, server }) => { await page.route('**/foo', route => route.fallback({ url: server.PREFIX + '/global-var.html' })); - const [response] = await Promise.all([ - page.waitForEvent('response'), - page.goto(server.PREFIX + '/foo'), - ]); + const response = await page.goto(server.PREFIX + '/foo'); expect(url).toBe(server.PREFIX + '/global-var.html'); - expect(response.url()).toBe(server.PREFIX + '/foo'); + expect(response.request().url()).toBe(server.PREFIX + '/global-var.html'); + expect(response.url()).toBe(server.PREFIX + '/global-var.html'); expect(await page.evaluate(() => window['globalVar'])).toBe(123); - expect((await request).method).toBe('GET'); + expect((await serverRequest).method).toBe('GET'); }); it.describe('post data', () => {