feat(fetch): introduce FetchRequest.dispose, fulfill with global fetch (#8945)

docs/src/api/class-fetchrequest.md

@@ -6,6 +6,12 @@ environment or the service to your e2e test. When used on [Page] or a [BrowserCo
 the cookies from the corresponding [BrowserContext]. This means that if you log in using this API, your e2e test
 will be logged in and vice versa.
 
+## async method: FetchRequest.dispose
+
+All responses received through [`method: FetchRequest.fetch`], [`method: FetchRequest.get`], [`method: FetchRequest.post`]
+and other methods are stored in the memory, so that you can later call [`method: FetchResponse.body`]. This method
+discards all stored responses, and makes [`method: FetchResponse.body`] throw "Response disposed" error.
+
 ## async method: FetchRequest.fetch
 - returns: <[FetchResponse]>
 

src/client/fetch.ts

@@ -17,6 +17,7 @@
 import * as api from '../../types/types';
 import { HeadersArray } from '../common/types';
 import * as channels from '../protocol/channels';
+import { kBrowserOrContextClosedError } from '../utils/errors';
 import { assert, headersObjectToArray, isString, objectToArray } from '../utils/utils';
 import { ChannelOwner } from './channelOwner';
 import * as network from './network';
@@ -41,6 +42,12 @@ export class FetchRequest extends ChannelOwner<channels.FetchRequestChannel, cha
     super(parent, type, guid, initializer);
   }
 
+  dispose(): Promise<void> {
+    return this._wrapApiCall(async (channel: channels.FetchRequestChannel) => {
+      await channel.dispose();
+    });
+  }
+
   async get(
     urlOrRequest: string | api.Request,
     options?: {
@@ -137,10 +144,16 @@ export class FetchResponse implements api.FetchResponse {
 
   async body(): Promise<Buffer> {
     return this._request._wrapApiCall(async (channel: channels.FetchRequestChannel) => {
-      const result = await channel.fetchResponseBody({ fetchUid: this._fetchUid() });
-      if (!result.binary)
-        throw new Error('Response has been disposed');
-      return Buffer.from(result.binary!, 'base64');
+      try {
+        const result = await channel.fetchResponseBody({ fetchUid: this._fetchUid() });
+        if (!result.binary)
+          throw new Error('Response has been disposed');
+        return Buffer.from(result.binary!, 'base64');
+      } catch (e) {
+        if (e.message === kBrowserOrContextClosedError)
+          throw new Error('Response has been disposed');
+        throw e;
+      }
     });
   }
 

src/dispatchers/browserContextDispatcher.ts

@@ -15,7 +15,7 @@
  */
 
 import { BrowserContext } from '../server/browserContext';
-import { Dispatcher, DispatcherScope, existingDispatcher, lookupDispatcher } from './dispatcher';
+import { Dispatcher, DispatcherScope, lookupDispatcher } from './dispatcher';
 import { PageDispatcher, BindingCallDispatcher, WorkerDispatcher } from './pageDispatcher';
 import { FrameDispatcher } from './frameDispatcher';
 import * as channels from '../protocol/channels';
@@ -58,10 +58,6 @@ export class BrowserContextDispatcher extends Dispatcher<BrowserContext, channel
     context.on(BrowserContext.Events.Close, () => {
       this._dispatchEvent('close');
       this._dispose();
-      const fetch = existingDispatcher<FetchRequestDispatcher>(this._context.fetchRequest);
-      // FetchRequestDispatcher is created in the browser rather then context scope but its
-      // lifetime is bound to the context dispatcher, so we manually dispose it here.
-      fetch._disposeDispatcher();
     });
 
     if (context._browser.options.name === 'chromium') {

src/dispatchers/networkDispatchers.ts

@@ -171,6 +171,14 @@ export class FetchRequestDispatcher extends Dispatcher<FetchRequest, channels.Fe
 
   private constructor(scope: DispatcherScope, request: FetchRequest) {
     super(scope, request, 'FetchRequest', {}, true);
+    request.once(FetchRequest.Events.Dispose, () => {
+      if (!this._disposed)
+        super._dispose();
+    });
+  }
+
+  async dispose(params?: channels.FetchRequestDisposeParams): Promise<void> {
+    this._object.dispose();
   }
 
   async fetch(params: channels.FetchRequestFetchParams, metadata?: channels.Metadata): Promise<channels.FetchRequestFetchResult> {
@@ -204,10 +212,5 @@ export class FetchRequestDispatcher extends Dispatcher<FetchRequest, channels.Fe
   async disposeFetchResponse(params: channels.FetchRequestDisposeFetchResponseParams, metadata?: channels.Metadata): Promise<void> {
     this._object.fetchResponses.delete(params.fetchUid);
   }
-
-  _disposeDispatcher() {
-    if (!this._disposed)
-      super._dispose();
-  }
 }
 

src/protocol/channels.ts

@@ -156,6 +156,7 @@ export interface FetchRequestChannel extends Channel {
   fetch(params: FetchRequestFetchParams, metadata?: Metadata): Promise<FetchRequestFetchResult>;
   fetchResponseBody(params: FetchRequestFetchResponseBodyParams, metadata?: Metadata): Promise<FetchRequestFetchResponseBodyResult>;
   disposeFetchResponse(params: FetchRequestDisposeFetchResponseParams, metadata?: Metadata): Promise<FetchRequestDisposeFetchResponseResult>;
+  dispose(params?: FetchRequestDisposeParams, metadata?: Metadata): Promise<FetchRequestDisposeResult>;
 }
 export type FetchRequestFetchParams = {
   url: string,
@@ -194,6 +195,9 @@ export type FetchRequestDisposeFetchResponseOptions = {
 
 };
 export type FetchRequestDisposeFetchResponseResult = void;
+export type FetchRequestDisposeParams = {};
+export type FetchRequestDisposeOptions = {};
+export type FetchRequestDisposeResult = void;
 
 export interface FetchRequestEvents {
 }

src/protocol/protocol.yml

@@ -249,6 +249,8 @@ FetchRequest:
       parameters:
         fetchUid: string
 
+    dispose:
+
 
 FetchResponse:
   type: object

src/protocol/validator.ts

@@ -162,6 +162,7 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme {
   scheme.FetchRequestDisposeFetchResponseParams = tObject({
     fetchUid: tString,
   });
+  scheme.FetchRequestDisposeParams = tOptional(tObject({}));
   scheme.FetchResponse = tObject({
     fetchUid: tString,
     url: tString,

src/server/browserContext.ts

@@ -64,7 +64,7 @@ export abstract class BrowserContext extends SdkObject {
   private _origins = new Set<string>();
   readonly _harRecorder: HarRecorder | undefined;
   readonly tracing: Tracing;
-  readonly fetchRequest = new BrowserContextFetchRequest(this);
+  readonly fetchRequest: BrowserContextFetchRequest;
 
   constructor(browser: Browser, options: types.BrowserContextOptions, browserContextId: string | undefined) {
     super(browser, 'browser-context');
@@ -79,6 +79,7 @@ export abstract class BrowserContext extends SdkObject {
       this._harRecorder = new HarRecorder(this, {...this._options.recordHar, path: path.join(this._browser.options.artifactsDir, `${createGuid()}.har`)});
 
     this.tracing = new Tracing(this);
+    this.fetchRequest = new BrowserContextFetchRequest(this);
   }
 
   _setSelectors(selectors: Selectors) {
@@ -134,7 +135,6 @@ export abstract class BrowserContext extends SdkObject {
     this._closedStatus = 'closed';
     this._deleteAllDownloads();
     this._downloads.clear();
-    this.fetchRequest.dispose();
     if (this._isPersistentContext)
       this._onClosePersistent();
     this._closePromiseFulfill!(new Error('Context closed'));

src/server/fetch.ts

@@ -41,16 +41,35 @@ type FetchRequestOptions = {
 };
 
 export abstract class FetchRequest extends SdkObject {
+  static Events = {
+    Dispose: 'dispose',
+  };
+
   readonly fetchResponses: Map<string, Buffer> = new Map();
+  protected static allInstances: Set<FetchRequest> = new Set();
+
+  static findResponseBody(guid: string): Buffer | undefined {
+    for (const request of FetchRequest.allInstances) {
+      const body = request.fetchResponses.get(guid);
+      if (body)
+        return body;
+    }
+    return undefined;
+  }
 
   constructor(parent: SdkObject) {
     super(parent, 'fetchRequest');
+    FetchRequest.allInstances.add(this);
   }
 
-  dispose() {
+  protected _disposeImpl() {
+    FetchRequest.allInstances.delete(this);
     this.fetchResponses.clear();
+    this.emit(FetchRequest.Events.Dispose);
   }
 
+  abstract dispose(): void;
+
   abstract _defaultOptions(): FetchRequestOptions;
   abstract _addCookies(cookies: types.SetNetworkCookieParam[]): Promise<void>;
   abstract _cookies(url: string): Promise<types.NetworkCookie[]>;
@@ -273,6 +292,11 @@ export class BrowserContextFetchRequest extends FetchRequest {
   constructor(context: BrowserContext) {
     super(context);
     this._context = context;
+    context.once(BrowserContext.Events.Close, () => this._disposeImpl());
+  }
+
+  override dispose() {
+    this.fetchResponses.clear();
   }
 
   _defaultOptions(): FetchRequestOptions {
@@ -302,6 +326,10 @@ export class GlobalFetchRequest extends FetchRequest {
     super(playwright);
   }
 
+  override dispose() {
+    this._disposeImpl();
+  }
+
   _defaultOptions(): FetchRequestOptions {
     return {
       userAgent: '',

src/server/network.ts

@@ -20,6 +20,7 @@ import { assert } from '../utils/utils';
 import { ManualPromise } from '../utils/async';
 import { SdkObject } from './instrumentation';
 import { NameValue } from '../common/types';
+import { FetchRequest } from './fetch';
 
 export function filterCookies(cookies: types.NetworkCookie[], urls: string[]): types.NetworkCookie[] {
   const parsedURLs = urls.map(s => new URL(s));
@@ -228,7 +229,7 @@ export class Route extends SdkObject {
     if (body === undefined) {
       if (overrides.fetchResponseUid) {
         const context = this._request.frame()._page._browserContext;
-        const buffer = context.fetchRequest.fetchResponses.get(overrides.fetchResponseUid);
+        const buffer = context.fetchRequest.fetchResponses.get(overrides.fetchResponseUid) || FetchRequest.findResponseBody(overrides.fetchResponseUid);
         assert(buffer, 'Fetch response has been disposed');
         body = buffer.toString('utf8');
         isBase64 = false;

tests/browsercontext-fetch.spec.ts

@@ -670,7 +670,16 @@ it('should dispose when context closes', async function({context, server}) {
   expect(await response.json()).toEqual({ foo: 'bar' });
   await context.close();
   const error = await response.body().catch(e => e);
-  expect(error.message).toContain('Target page, context or browser has been closed');
+  expect(error.message).toContain('Response has been disposed');
+});
+
+it('should dispose global request', async function({playwright, context, server}) {
+  const request = await playwright._newRequest();
+  const response = await request.get(server.PREFIX + '/simple.json');
+  expect(await response.json()).toEqual({ foo: 'bar' });
+  await request.dispose();
+  const error = await response.body().catch(e => e);
+  expect(error.message).toContain('Response has been disposed');
 });
 
 it('should throw on invalid first argument', async function({context}) {

tests/page/page-request-fulfill.spec.ts

@@ -194,6 +194,18 @@ it('should include the origin header', async ({page, server, isAndroid}) => {
   expect(interceptedRequest.headers()['origin']).toEqual(server.PREFIX);
 });
 
+it('should fulfill with global fetch result', async ({playwright, page, server, isElectron}) => {
+  it.fixme(isElectron, 'error: Browser context management is not supported.');
+  await page.route('**/*', async route => {
+    const request = await playwright._newRequest();
+    const response = await request.get(server.PREFIX + '/simple.json');
+    route.fulfill({ response });
+  });
+  const response = await page.goto(server.EMPTY_PAGE);
+  expect(response.status()).toBe(200);
+  expect(await response.json()).toEqual({'foo': 'bar'});
+});
+
 it('should fulfill with fetch result', async ({page, server, isElectron}) => {
   it.fixme(isElectron, 'error: Browser context management is not supported.');
   await page.route('**/*', async route => {

types/types.d.ts

@@ -12625,6 +12625,19 @@ export interface Electron {
  * logged in and vice versa.
  */
 export interface FetchRequest {
+  /**
+   * All responses received through
+   * [fetchRequest.fetch(urlOrRequest[, options])](https://playwright.dev/docs/api/class-fetchrequest#fetch-request-fetch),
+   * [fetchRequest.get(urlOrRequest[, options])](https://playwright.dev/docs/api/class-fetchrequest#fetch-request-get),
+   * [fetchRequest.post(urlOrRequest[, options])](https://playwright.dev/docs/api/class-fetchrequest#fetch-request-post) and
+   * other methods are stored in the memory, so that you can later call
+   * [fetchResponse.body()](https://playwright.dev/docs/api/class-fetchresponse#fetch-response-body). This method discards
+   * all stored responses, and makes
+   * [fetchResponse.body()](https://playwright.dev/docs/api/class-fetchresponse#fetch-response-body) throw "Response
+   * disposed" error.
+   */
+  dispose(): Promise<void>;
+
   /**
    * Sends HTTP(S) fetch and returns its response. The method will populate fetch cookies from the context and update context
    * cookies from the response. The method will automatically follow redirects.