From b4ca77be231cdacfb0bcef9b66e490a4e11ebe34 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Wed, 8 Sep 2021 13:40:07 -0700 Subject: [PATCH] feat(fetch): get body lazily (#8784) --- src/client/browserContext.ts | 2 +- src/client/network.ts | 20 ++++++++++++++++---- src/dispatchers/browserContextDispatcher.ts | 11 ++++++++++- src/protocol/channels.ts | 20 +++++++++++++++++++- src/protocol/protocol.yml | 12 +++++++++++- src/protocol/validator.ts | 8 +++++++- src/server/browserContext.ts | 7 +++++++ src/server/fetch.ts | 5 +++-- tests/browsercontext-fetch.spec.ts | 19 +++++++++++++++++++ 9 files changed, 93 insertions(+), 11 deletions(-) diff --git a/src/client/browserContext.ts b/src/client/browserContext.ts index ee6aeda495..575ca60d73 100644 --- a/src/client/browserContext.ts +++ b/src/client/browserContext.ts @@ -228,7 +228,7 @@ export class BrowserContext extends ChannelOwner { - return this._body; + return this._context._wrapApiCall(async (channel: channels.BrowserContextChannel) => { + const result = await channel.fetchResponseBody({ fetchUid: this._initializer.fetchUid }); + if (!result.binary) + throw new Error('Response has been disposed'); + return Buffer.from(result.binary!, 'base64'); + }); } async text(): Promise { @@ -563,6 +569,12 @@ export class FetchResponse { const content = await this.text(); return JSON.parse(content); } + + async dispose(): Promise { + return this._context._wrapApiCall(async (channel: channels.BrowserContextChannel) => { + await channel.disposeFetchResponse({ fetchUid: this._initializer.fetchUid }); + }); + } } export class WebSocket extends ChannelOwner implements api.WebSocket { diff --git a/src/dispatchers/browserContextDispatcher.ts b/src/dispatchers/browserContextDispatcher.ts index d6ba1dc0d4..6a937d8b8f 100644 --- a/src/dispatchers/browserContextDispatcher.ts +++ b/src/dispatchers/browserContextDispatcher.ts @@ -122,12 +122,21 @@ export class BrowserContextDispatcher extends Dispatcher { + const buffer = this._context.fetchResponses.get(params.fetchUid); + return { binary: buffer ? buffer.toString('base64') : undefined }; + } + + async disposeFetchResponse(params: channels.BrowserContextDisposeFetchResponseParams): Promise { + this._context.fetchResponses.delete(params.fetchUid); + } + async newPage(params: channels.BrowserContextNewPageParams, metadata: CallMetadata): Promise { return { page: lookupDispatcher(await this._context.newPage(metadata)) }; } diff --git a/src/protocol/channels.ts b/src/protocol/channels.ts index 2f3aaa1f1b..ed54effd40 100644 --- a/src/protocol/channels.ts +++ b/src/protocol/channels.ts @@ -151,11 +151,11 @@ export type InterceptedResponse = { }; export type FetchResponse = { + fetchUid: string, url: string, status: number, statusText: string, headers: NameValue[], - body: Binary, }; // ----------- Root ----------- @@ -754,6 +754,8 @@ export interface BrowserContextChannel extends EventTargetChannel { cookies(params: BrowserContextCookiesParams, metadata?: Metadata): Promise; exposeBinding(params: BrowserContextExposeBindingParams, metadata?: Metadata): Promise; fetch(params: BrowserContextFetchParams, metadata?: Metadata): Promise; + fetchResponseBody(params: BrowserContextFetchResponseBodyParams, metadata?: Metadata): Promise; + disposeFetchResponse(params: BrowserContextDisposeFetchResponseParams, metadata?: Metadata): Promise; grantPermissions(params: BrowserContextGrantPermissionsParams, metadata?: Metadata): Promise; newPage(params?: BrowserContextNewPageParams, metadata?: Metadata): Promise; setDefaultNavigationTimeoutNoReply(params: BrowserContextSetDefaultNavigationTimeoutNoReplyParams, metadata?: Metadata): Promise; @@ -870,6 +872,22 @@ export type BrowserContextFetchResult = { response?: FetchResponse, error?: string, }; +export type BrowserContextFetchResponseBodyParams = { + fetchUid: string, +}; +export type BrowserContextFetchResponseBodyOptions = { + +}; +export type BrowserContextFetchResponseBodyResult = { + binary?: Binary, +}; +export type BrowserContextDisposeFetchResponseParams = { + fetchUid: string, +}; +export type BrowserContextDisposeFetchResponseOptions = { + +}; +export type BrowserContextDisposeFetchResponseResult = void; export type BrowserContextGrantPermissionsParams = { permissions: string[], origin?: string, diff --git a/src/protocol/protocol.yml b/src/protocol/protocol.yml index 74037fd255..d399bc62e2 100644 --- a/src/protocol/protocol.yml +++ b/src/protocol/protocol.yml @@ -220,13 +220,13 @@ InterceptedResponse: FetchResponse: type: object properties: + fetchUid: string url: string status: number statusText: string headers: type: array items: NameValue - body: binary LaunchOptions: type: mixin @@ -626,6 +626,16 @@ BrowserContext: response: FetchResponse? error: string? + fetchResponseBody: + parameters: + fetchUid: string + returns: + binary?: binary + + disposeFetchResponse: + parameters: + fetchUid: string + grantPermissions: parameters: permissions: diff --git a/src/protocol/validator.ts b/src/protocol/validator.ts index cb37b93bd9..08c7bb51ab 100644 --- a/src/protocol/validator.ts +++ b/src/protocol/validator.ts @@ -148,11 +148,11 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme { headers: tArray(tType('NameValue')), }); scheme.FetchResponse = tObject({ + fetchUid: tString, url: tString, status: tNumber, statusText: tString, headers: tArray(tType('NameValue')), - body: tBinary, }); scheme.RootInitializeParams = tObject({ sdkLanguage: tString, @@ -399,6 +399,12 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme { postData: tOptional(tBinary), timeout: tOptional(tNumber), }); + scheme.BrowserContextFetchResponseBodyParams = tObject({ + fetchUid: tString, + }); + scheme.BrowserContextDisposeFetchResponseParams = tObject({ + fetchUid: tString, + }); scheme.BrowserContextGrantPermissionsParams = tObject({ permissions: tArray(tString), origin: tOptional(tString), diff --git a/src/server/browserContext.ts b/src/server/browserContext.ts index 5d646ab3eb..989beb33e8 100644 --- a/src/server/browserContext.ts +++ b/src/server/browserContext.ts @@ -63,6 +63,7 @@ export abstract class BrowserContext extends SdkObject { private _origins = new Set(); readonly _harRecorder: HarRecorder | undefined; readonly tracing: Tracing; + readonly fetchResponses: Map = new Map(); constructor(browser: Browser, options: types.BrowserContextOptions, browserContextId: string | undefined) { super(browser, 'browser-context'); @@ -381,6 +382,12 @@ export abstract class BrowserContext extends SdkObject { this.on(BrowserContext.Events.Page, installInPage); return Promise.all(this.pages().map(installInPage)); } + + storeFetchResponseBody(body: Buffer): string { + const uid = createGuid(); + this.fetchResponses.set(uid, body); + return uid; + } } export function assertBrowserContextIsNotOwned(context: BrowserContext) { diff --git a/src/server/fetch.ts b/src/server/fetch.ts index 0d07488950..2f9fe465c4 100644 --- a/src/server/fetch.ts +++ b/src/server/fetch.ts @@ -24,7 +24,7 @@ import * as types from './types'; import { pipeline, Readable, Transform } from 'stream'; import { monotonicTime } from '../utils/utils'; -export async function playwrightFetch(context: BrowserContext, params: types.FetchOptions): Promise<{fetchResponse?: types.FetchResponse, error?: string}> { +export async function playwrightFetch(context: BrowserContext, params: types.FetchOptions): Promise<{fetchResponse?: Omit & { fetchUid: string }, error?: string}> { try { const headers: { [name: string]: string } = {}; if (params.headers) { @@ -62,7 +62,8 @@ export async function playwrightFetch(context: BrowserContext, params: types.Fet timeout, deadline }, params.postData); - return { fetchResponse }; + const fetchUid = context.storeFetchResponseBody(fetchResponse.body); + return { fetchResponse: { ...fetchResponse, fetchUid } }; } catch (e) { return { error: String(e) }; } diff --git a/tests/browsercontext-fetch.spec.ts b/tests/browsercontext-fetch.spec.ts index a4e2084650..ea68eba7a8 100644 --- a/tests/browsercontext-fetch.spec.ts +++ b/tests/browsercontext-fetch.spec.ts @@ -613,3 +613,22 @@ it('should respect timeout after redirects', async function({context, server}) { const error = await context._fetch(server.PREFIX + '/redirect').catch(e => e); expect(error.message).toContain(`Request timed out after 100ms`); }); + +it('should dispose', async function({context, server}) { + // @ts-expect-error + const response = await context._fetch(server.PREFIX + '/simple.json'); + expect(await response.json()).toEqual({ foo: 'bar' }); + await response.dispose(); + const error = await response.body().catch(e => e); + expect(error.message).toContain('Response has been disposed'); +}); + +it('should dispose when context closes', async function({context, server}) { + // @ts-expect-error + const response = await context._fetch(server.PREFIX + '/simple.json'); + expect(await response.json()).toEqual({ foo: 'bar' }); + await context.close(); + const error = await response.body().catch(e => e); + expect(error.message).toContain('Target page, context or browser has been closed'); +}); +