feat(fetch): get body lazily (#8784)

src/client/browserContext.ts

@@ -228,7 +228,7 @@ export class BrowserContext extends ChannelOwner<channels.BrowserContextChannel,
       });
       if (result.error)
         throw new Error(`Request failed: ${result.error}`);
-      return new network.FetchResponse(result.response!);
+      return new network.FetchResponse(this, result.response!);
     });
   }
 

src/client/network.ts

@@ -29,6 +29,7 @@ import { Waiter } from './waiter';
 import * as api from '../../types/types';
 import { URLMatch } from '../common/types';
 import { urlMatches } from './clientHelper';
+import { BrowserContext } from './browserContext';
 
 export type NetworkCookie = {
   name: string,
@@ -522,12 +523,12 @@ export class Response extends ChannelOwner<channels.ResponseChannel, channels.Re
 export class FetchResponse {
   private readonly _initializer: channels.FetchResponse;
   private readonly _headers: Headers;
-  private readonly _body: Buffer;
+  private readonly _context: BrowserContext;
 
-  constructor(initializer: channels.FetchResponse) {
+  constructor(context: BrowserContext, initializer: channels.FetchResponse) {
+    this._context = context;
     this._initializer = initializer;
     this._headers = headersArrayToObject(this._initializer.headers, true /* lowerCase */);
-    this._body = Buffer.from(initializer.body, 'base64');
   }
 
   ok(): boolean {
@@ -551,7 +552,12 @@ export class FetchResponse {
   }
 
   async body(): Promise<Buffer> {
-    return this._body;
+    return this._context._wrapApiCall(async (channel: channels.BrowserContextChannel) => {
+      const result = await channel.fetchResponseBody({ fetchUid: this._initializer.fetchUid });
+      if (!result.binary)
+        throw new Error('Response has been disposed');
+      return Buffer.from(result.binary!, 'base64');
+    });
   }
 
   async text(): Promise<string> {
@@ -563,6 +569,12 @@ export class FetchResponse {
     const content = await this.text();
     return JSON.parse(content);
   }
+
+  async dispose(): Promise<void> {
+    return this._context._wrapApiCall(async (channel: channels.BrowserContextChannel) => {
+      await channel.disposeFetchResponse({ fetchUid: this._initializer.fetchUid });
+    });
+  }
 }
 
 export class WebSocket extends ChannelOwner<channels.WebSocketChannel, channels.WebSocketInitializer> implements api.WebSocket {

src/dispatchers/browserContextDispatcher.ts

@@ -122,12 +122,21 @@ export class BrowserContextDispatcher extends Dispatcher<BrowserContext, channel
         status: fetchResponse.status,
         statusText: fetchResponse.statusText,
         headers: fetchResponse.headers,
-        body: fetchResponse.body.toString('base64')
+        fetchUid: fetchResponse.fetchUid
       };
     }
     return { response, error };
   }
 
+  async fetchResponseBody(params: channels.BrowserContextFetchResponseBodyParams): Promise<channels.BrowserContextFetchResponseBodyResult> {
+    const buffer = this._context.fetchResponses.get(params.fetchUid);
+    return { binary: buffer ? buffer.toString('base64') : undefined };
+  }
+
+  async disposeFetchResponse(params: channels.BrowserContextDisposeFetchResponseParams): Promise<channels.BrowserContextDisposeFetchResponseResult> {
+    this._context.fetchResponses.delete(params.fetchUid);
+  }
+
   async newPage(params: channels.BrowserContextNewPageParams, metadata: CallMetadata): Promise<channels.BrowserContextNewPageResult> {
     return { page: lookupDispatcher<PageDispatcher>(await this._context.newPage(metadata)) };
   }

src/protocol/channels.ts

@@ -151,11 +151,11 @@ export type InterceptedResponse = {
 };
 
 export type FetchResponse = {
+  fetchUid: string,
   url: string,
   status: number,
   statusText: string,
   headers: NameValue[],
-  body: Binary,
 };
 
 // ----------- Root -----------
@@ -754,6 +754,8 @@ export interface BrowserContextChannel extends EventTargetChannel {
   cookies(params: BrowserContextCookiesParams, metadata?: Metadata): Promise<BrowserContextCookiesResult>;
   exposeBinding(params: BrowserContextExposeBindingParams, metadata?: Metadata): Promise<BrowserContextExposeBindingResult>;
   fetch(params: BrowserContextFetchParams, metadata?: Metadata): Promise<BrowserContextFetchResult>;
+  fetchResponseBody(params: BrowserContextFetchResponseBodyParams, metadata?: Metadata): Promise<BrowserContextFetchResponseBodyResult>;
+  disposeFetchResponse(params: BrowserContextDisposeFetchResponseParams, metadata?: Metadata): Promise<BrowserContextDisposeFetchResponseResult>;
   grantPermissions(params: BrowserContextGrantPermissionsParams, metadata?: Metadata): Promise<BrowserContextGrantPermissionsResult>;
   newPage(params?: BrowserContextNewPageParams, metadata?: Metadata): Promise<BrowserContextNewPageResult>;
   setDefaultNavigationTimeoutNoReply(params: BrowserContextSetDefaultNavigationTimeoutNoReplyParams, metadata?: Metadata): Promise<BrowserContextSetDefaultNavigationTimeoutNoReplyResult>;
@@ -870,6 +872,22 @@ export type BrowserContextFetchResult = {
   response?: FetchResponse,
   error?: string,
 };
+export type BrowserContextFetchResponseBodyParams = {
+  fetchUid: string,
+};
+export type BrowserContextFetchResponseBodyOptions = {
+
+};
+export type BrowserContextFetchResponseBodyResult = {
+  binary?: Binary,
+};
+export type BrowserContextDisposeFetchResponseParams = {
+  fetchUid: string,
+};
+export type BrowserContextDisposeFetchResponseOptions = {
+
+};
+export type BrowserContextDisposeFetchResponseResult = void;
 export type BrowserContextGrantPermissionsParams = {
   permissions: string[],
   origin?: string,

src/protocol/protocol.yml

@@ -220,13 +220,13 @@ InterceptedResponse:
 FetchResponse:
   type: object
   properties:
+    fetchUid: string
     url: string
     status: number
     statusText: string
     headers:
       type: array
       items: NameValue
-    body: binary
 
 LaunchOptions:
   type: mixin
@@ -626,6 +626,16 @@ BrowserContext:
         response: FetchResponse?
         error: string?
 
+    fetchResponseBody:
+      parameters:
+        fetchUid: string
+      returns:
+        binary?: binary
+
+    disposeFetchResponse:
+      parameters:
+        fetchUid: string
+
     grantPermissions:
       parameters:
         permissions:

src/protocol/validator.ts

@@ -148,11 +148,11 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme {
     headers: tArray(tType('NameValue')),
   });
   scheme.FetchResponse = tObject({
+    fetchUid: tString,
     url: tString,
     status: tNumber,
     statusText: tString,
     headers: tArray(tType('NameValue')),
-    body: tBinary,
   });
   scheme.RootInitializeParams = tObject({
     sdkLanguage: tString,
@@ -399,6 +399,12 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme {
     postData: tOptional(tBinary),
     timeout: tOptional(tNumber),
   });
+  scheme.BrowserContextFetchResponseBodyParams = tObject({
+    fetchUid: tString,
+  });
+  scheme.BrowserContextDisposeFetchResponseParams = tObject({
+    fetchUid: tString,
+  });
   scheme.BrowserContextGrantPermissionsParams = tObject({
     permissions: tArray(tString),
     origin: tOptional(tString),

src/server/browserContext.ts

@@ -63,6 +63,7 @@ export abstract class BrowserContext extends SdkObject {
   private _origins = new Set<string>();
   readonly _harRecorder: HarRecorder | undefined;
   readonly tracing: Tracing;
+  readonly fetchResponses: Map<string, Buffer> = new Map();
 
   constructor(browser: Browser, options: types.BrowserContextOptions, browserContextId: string | undefined) {
     super(browser, 'browser-context');
@@ -381,6 +382,12 @@ export abstract class BrowserContext extends SdkObject {
     this.on(BrowserContext.Events.Page, installInPage);
     return Promise.all(this.pages().map(installInPage));
   }
+
+  storeFetchResponseBody(body: Buffer): string {
+    const uid = createGuid();
+    this.fetchResponses.set(uid, body);
+    return uid;
+  }
 }
 
 export function assertBrowserContextIsNotOwned(context: BrowserContext) {

src/server/fetch.ts

@@ -24,7 +24,7 @@ import * as types from './types';
 import { pipeline, Readable, Transform } from 'stream';
 import { monotonicTime } from '../utils/utils';
 
-export async function playwrightFetch(context: BrowserContext, params: types.FetchOptions): Promise<{fetchResponse?: types.FetchResponse, error?: string}> {
+export async function playwrightFetch(context: BrowserContext, params: types.FetchOptions): Promise<{fetchResponse?: Omit<types.FetchResponse, 'body'> & { fetchUid: string }, error?: string}> {
   try {
     const headers: { [name: string]: string } = {};
     if (params.headers) {
@@ -62,7 +62,8 @@ export async function playwrightFetch(context: BrowserContext, params: types.Fet
       timeout,
       deadline
     }, params.postData);
-    return { fetchResponse };
+    const fetchUid = context.storeFetchResponseBody(fetchResponse.body);
+    return { fetchResponse: { ...fetchResponse, fetchUid } };
   } catch (e) {
     return { error: String(e) };
   }

tests/browsercontext-fetch.spec.ts

@@ -613,3 +613,22 @@ it('should respect timeout after redirects', async function({context, server}) {
   const error = await context._fetch(server.PREFIX + '/redirect').catch(e => e);
   expect(error.message).toContain(`Request timed out after 100ms`);
 });
+
+it('should dispose', async function({context, server}) {
+  // @ts-expect-error
+  const response = await context._fetch(server.PREFIX + '/simple.json');
+  expect(await response.json()).toEqual({ foo: 'bar' });
+  await response.dispose();
+  const error = await response.body().catch(e => e);
+  expect(error.message).toContain('Response has been disposed');
+});
+
+it('should dispose when context closes', async function({context, server}) {
+  // @ts-expect-error
+  const response = await context._fetch(server.PREFIX + '/simple.json');
+  expect(await response.json()).toEqual({ foo: 'bar' });
+  await context.close();
+  const error = await response.body().catch(e => e);
+  expect(error.message).toContain('Target page, context or browser has been closed');
+});
+