Merge 8b0a70f9bb into 7a61aa25e6

docs/src/api/class-apirequestcontext.md

@@ -897,3 +897,72 @@ Returns storage state for this request context, contains current cookies and loc
 - `indexedDB` ?<boolean>
 
 Set to `true` to include IndexedDB in the storage state snapshot.
+
+
+## event: APIRequestContext.apiRequest
+* since: v1.51
+- argument: <[Object]>
+  - `guid` <[string]> request GUID
+  - `url` <[URL]> request URL
+  - `method` <[string]> request method
+  - `headers` <[Object]<[string], [string]>> request headers
+  - `cookies` <[Array]<[Object]>>
+    - `name` <[string]>
+    - `value` <[string]>
+  - `postData` ?<[Buffer]> request post data
+
+Emitted when a request is issued from any requests created through this context.
+The [APIRequestEvent] object is read-only.
+
+## event: APIRequestContext.apiRequestFinished
+* since: v1.51
+- argument: <[Object]>
+  - `httpVersion` <[string]> HTTP version
+  - `rawHeaders` <[Array]<[string]>> raw headers
+  - `statusCode` <[int]> status code
+  - `statusMessage` <[string]> status message
+  - `body` ?<[Buffer]> response body
+  - `headers` <[IncomingHttpHeaders]>> response headers
+  - `serverIPAddress` ?<[string]> server IP address
+  - `serverPort` ?<[int]> server port
+  - `timings` <[Object]>
+    - `blocked` ?<[int]>
+    - `dns` ?<[int]>
+    - `connect` ?<[int]>
+    - `send` <[int]>
+    - `wait` <[int]>
+    - `receive` <[int]>
+    - `ssl` ?<[int]>
+    - `comment` ?<[string]>
+  - `cookies` <[Array]<[Object]>>
+    - `name` <[string]>
+    - `value` <[string]>
+    - `domain` <[string]>
+    - `path` <[string]>
+    - `expires` <[float]> Unix time in seconds.
+    - `httpOnly` <[boolean]>
+    - `secure` <[boolean]>
+    - `sameSite` <[SameSiteAttribute]<"Strict"|"Lax"|"None">>
+  - `securityDetails` ?<[Object]> security details
+    - `issuer` ?<[string]> Common Name component of the Issuer field.
+      from the certificate. This should only be used for informational purposes. Optional.
+    - `protocol` ?<[string]> The specific TLS protocol used. (e.g. `TLS 1.3`). Optional.
+    - `subjectName` ?<[string]> Common Name component of the Subject
+      field from the certificate. This should only be used for informational purposes. Optional.
+    - `validFrom` ?<[float]> Unix timestamp (in seconds) specifying
+      when this cert becomes valid. Optional.
+    - `validTo` ?<[float]> Unix timestamp (in seconds) specifying
+      when this cert becomes invalid. Optional.
+  - `requestEvent` <[Object]> request event object
+    - `guid` <[string]> request GUID
+    - `url` <[URL]> request URL
+    - `method` <[string]> request method
+    - `headers` <[Object]<[string], [string]>> request headers
+    - `cookies` <[Array]<[Object]>>
+      - `name` <[string]>
+      - `value` <[string]>
+    - `postData` ?<[Buffer]> request post data
+
+
+Emitted when a request finishes in any requests created through this context. The
+sequence of events is `apirequest` and `apirequestfinished`

docs/src/service-workers-experimental-network-events-js.md

@@ -67,6 +67,9 @@ Additionally, any network request made by the **Page** (including its sub-[Frame
 * [`method: Request.serviceWorker`] will be set to `null`, and [`method: Request.frame`] will return the [Frame]
 * [`method: Response.fromServiceWorker`] will return `true` (if a Service Worker's fetch handler was registered)
 
+Any requests made by **APIRequestContext** will have
+* [`event: APIRequestContext.apiRequest`] and its corresponding event ([`event: APIRequestContext.apiRequestFinished`])
+
 Many Service Worker implementations simply execute the request from the page (possibly with some custom caching/offline logic omitted for simplicity):
 
 ```js title="transparent-service-worker.js"

packages/playwright-core/src/client/events.ts

@@ -50,6 +50,11 @@ export const Events = {
     RequestFinished: 'requestfinished',
   },
 
+  APIRequestContext: {
+    APIRequest: 'apirequest',
+    APIRequestFinished: 'apirequestfinished',
+  },
+
   BrowserServer: {
     Close: 'close',
   },

packages/playwright-core/src/client/fetch.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+
 import { toClientCertificatesProtocol } from './browserContext';
 import { ChannelOwner } from './channelOwner';
 import { TargetClosedError, isTargetClosedError } from './errors';
@@ -23,7 +24,9 @@ import { assert } from '../utils/isomorphic/assert';
 import { mkdirIfNeeded } from './fileUtils';
 import { headersObjectToArray } from '../utils/isomorphic/headers';
 import { isString } from '../utils/isomorphic/rtti';
+import { Events } from './events';
 
+import type { APIRequestEvent, APIRequestFinishedEvent } from 'playwright-core/lib/server/fetch';
 import type { Playwright } from './playwright';
 import type { ClientCertificate, FilePayload, Headers, SetStorageState, StorageState } from './types';
 import type { Serializable } from '../../types/structs';
@@ -98,7 +101,11 @@ export class APIRequestContext extends ChannelOwner<channels.APIRequestContextCh
 
   constructor(parent: ChannelOwner, type: string, guid: string, initializer: channels.APIRequestContextInitializer) {
     super(parent, type, guid, initializer);
-    this._tracing = Tracing.from(initializer.tracing);
+    this._tracing = Tracing.from(initializer.tracing!);
+    this._channel.on('apiRequest', request => {
+      this._onRequest(request);
+    });
+    this._channel.on('apiRequestFinished', params => this._onRequestFinished(params));
   }
 
   async [Symbol.asyncDispose]() {
@@ -269,6 +276,14 @@ export class APIRequestContext extends ChannelOwner<channels.APIRequestContextCh
     }
     return state;
   }
+
+  private _onRequest(request: APIRequestEvent) {
+    this.emit(Events.APIRequestContext.APIRequest, request);
+  }
+
+  private _onRequestFinished(params: APIRequestFinishedEvent) {
+    this.emit(Events.APIRequestContext.APIRequestFinished, params);
+  }
 }
 
 async function toFormField(platform: Platform, name: string, value: string | number | boolean | fs.ReadStream | FilePayload): Promise<channels.FormField> {

packages/playwright-core/src/protocol/validator.ts

@@ -16,7 +16,7 @@
 
 // This file is generated by generate_channels.js, do not edit manually.
 
-import { scheme, tOptional, tObject, tBoolean, tNumber, tString, tAny, tEnum, tArray, tBinary, tChannel, tType } from './validatorPrimitives';
+import { scheme, tOptional, tObject, tBoolean, tNumber, tString, tAny, tEnum, tArray, tBinary, tChannel, tType, tURL } from './validatorPrimitives';
 export type { Validator, ValidatorContext } from './validatorPrimitives';
 export { ValidationError, findValidator, maybeFindValidator, createMetadataValidator } from './validatorPrimitives';
 
@@ -201,7 +201,44 @@ scheme.FormField = tObject({
   })),
 });
 scheme.APIRequestContextInitializer = tObject({
-  tracing: tChannel(['Tracing']),
+  tracing: tOptional(tChannel(['Tracing'])),
+});
+scheme.APIRequestContextApiRequestEvent = tObject({
+  guid: tString,
+  url: tURL,
+  method: tString,
+  headers: tAny,
+  cookies: tArray(tType('NameValue')),
+  postData: tOptional(tBinary),
+});
+scheme.APIRequestContextApiRequestFinishedEvent = tObject({
+  requestEvent: tObject({
+    guid: tString,
+    url: tURL,
+    method: tString,
+    headers: tAny,
+    cookies: tArray(tType('NameValue')),
+  }),
+  httpVersion: tString,
+  headers: tAny,
+  cookies: tArray(tType('NetworkCookie')),
+  rawHeaders: tArray(tString),
+  statusCode: tNumber,
+  statusMessage: tString,
+  body: tOptional(tBinary),
+  timings: tObject({
+    blocked: tOptional(tNumber),
+    dns: tOptional(tNumber),
+    connect: tOptional(tNumber),
+    send: tNumber,
+    wait: tNumber,
+    receive: tNumber,
+    ssl: tOptional(tNumber),
+    comment: tOptional(tString),
+  }),
+  serverIPAddress: tOptional(tString),
+  serverPort: tOptional(tNumber),
+  securityDetails: tOptional(tType('SecurityDetails')),
 });
 scheme.APIRequestContextFetchParams = tObject({
   url: tString,

packages/playwright-core/src/protocol/validatorPrimitives.ts

@@ -128,6 +128,13 @@ export const tEnum = (e: string[]): Validator => {
     return arg;
   };
 };
+export const tURL = (arg: any, path: string, context: ValidatorContext) => {
+  if (arg instanceof URL)
+    return arg;
+  if (typeof arg === 'string')
+    return new URL(arg);
+  throw new ValidationError(`${path}: expected URL, got ${typeof arg}`);
+};
 export const tChannel = (names: '*' | string[]): Validator => {
   return (arg: any, path: string, context: ValidatorContext) => {
     return context.tChannelImpl(names, arg, path, context);

packages/playwright-core/src/server/dispatchers/networkDispatchers.ts

@@ -19,11 +19,11 @@ import { Dispatcher, existingDispatcher } from './dispatcher';
 import { FrameDispatcher } from './frameDispatcher';
 import { WorkerDispatcher } from './pageDispatcher';
 import { TracingDispatcher } from './tracingDispatcher';
+import { APIRequestContext, APIRequestEvent, APIRequestFinishedEvent } from '../fetch';
+import { BrowserContextDispatcher } from './browserContextDispatcher';
 
-import type { APIRequestContext } from '../fetch';
 import type { CallMetadata } from '../instrumentation';
 import type { Request, Response, Route } from '../network';
-import type { BrowserContextDispatcher } from './browserContextDispatcher';
 import type { RootDispatcher } from './dispatcher';
 import type { PageDispatcher } from './pageDispatcher';
 import type * as channels from '@protocol/channels';
@@ -193,9 +193,17 @@ export class APIRequestContextDispatcher extends Dispatcher<APIRequestContext, c
       tracing,
     });
 
+
     this.adopt(tracing);
+    this.addObjectListener(APIRequestContext.Events.Request, (request: APIRequestEvent) =>  {
+      this._dispatchEvent('apiRequest', request);
+    });
+    this.addObjectListener(APIRequestContext.Events.RequestFinished, (request: APIRequestFinishedEvent) =>  {
+      this._dispatchEvent('apiRequestFinished', request);
+    });
   }
 
+
   async storageState(params: channels.APIRequestContextStorageStateParams): Promise<channels.APIRequestContextStorageStateResult> {
     return this._object.storageState(params.indexedDB);
   }

packages/playwright-core/src/server/fetch.ts

@@ -63,6 +63,7 @@ type FetchRequestOptions = {
 type HeadersObject = Readonly<{ [name: string]: string }>;
 
 export type APIRequestEvent = {
+  guid: string,
   url: URL,
   method: string,
   headers: HeadersObject,
@@ -294,6 +295,7 @@ export abstract class APIRequestContext extends SdkObject {
       return { name, value };
     }) || [];
     const requestEvent: APIRequestEvent = {
+      guid: createGuid(),
       url,
       method: options.method!,
       headers: options.headers,

packages/protocol/src/channels.d.ts

@@ -337,9 +337,11 @@ export type FormField = {
 
 // ----------- APIRequestContext -----------
 export type APIRequestContextInitializer = {
-  tracing: TracingChannel,
+  tracing?: TracingChannel,
 };
 export interface APIRequestContextEventTarget {
+  on(event: 'apiRequest', callback: (params: APIRequestContextApiRequestEvent) => void): this;
+  on(event: 'apiRequestFinished', callback: (params: APIRequestContextApiRequestFinishedEvent) => void): this;
 }
 export interface APIRequestContextChannel extends APIRequestContextEventTarget, Channel {
   _type_APIRequestContext: boolean;
@@ -350,6 +352,43 @@ export interface APIRequestContextChannel extends APIRequestContextEventTarget,
   disposeAPIResponse(params: APIRequestContextDisposeAPIResponseParams, metadata?: CallMetadata): Promise<APIRequestContextDisposeAPIResponseResult>;
   dispose(params: APIRequestContextDisposeParams, metadata?: CallMetadata): Promise<APIRequestContextDisposeResult>;
 }
+export type APIRequestContextApiRequestEvent = {
+  guid: string,
+  url: URL,
+  method: string,
+  headers: any,
+  cookies: NameValue[],
+  postData?: Binary,
+};
+export type APIRequestContextApiRequestFinishedEvent = {
+  requestEvent: {
+    guid: string,
+    url: URL,
+    method: string,
+    headers: any,
+    cookies: NameValue[],
+  },
+  httpVersion: string,
+  headers: any,
+  cookies: NetworkCookie[],
+  rawHeaders: string[],
+  statusCode: number,
+  statusMessage: string,
+  body?: Binary,
+  timings: {
+    blocked?: number,
+    dns?: number,
+    connect?: number,
+    send: number,
+    wait: number,
+    receive: number,
+    ssl?: number,
+    comment?: string,
+  },
+  serverIPAddress?: string,
+  serverPort?: number,
+  securityDetails?: SecurityDetails,
+};
 export type APIRequestContextFetchParams = {
   url: string,
   encodedParams?: string,
@@ -428,6 +467,8 @@ export type APIRequestContextDisposeOptions = {
 export type APIRequestContextDisposeResult = void;
 
 export interface APIRequestContextEvents {
+  'apiRequest': APIRequestContextApiRequestEvent;
+  'apiRequestFinished': APIRequestContextApiRequestFinishedEvent;
 }
 
 export type APIResponse = {

packages/protocol/src/protocol.yml

@@ -313,7 +313,6 @@ RecordHarOptions:
     urlRegexSource: string?
     urlRegexFlags: string?
 
-
 FormField:
   type: object
   properties:
@@ -330,7 +329,7 @@ APIRequestContext:
   type: interface
 
   initializer:
-    tracing: Tracing
+    tracing: Tracing?
 
   commands:
 
@@ -394,6 +393,54 @@ APIRequestContext:
       parameters:
         reason: string?
 
+  events:
+    apiRequest:
+      parameters:
+        guid: string
+        url: URL
+        method: string
+        headers: json
+        cookies:
+          type: array
+          items: NameValue
+        postData: binary?
+    apiRequestFinished:
+      parameters:
+        requestEvent:
+          type: object
+          properties:
+            guid: string
+            url: URL
+            method: string
+            headers: json
+            cookies:
+              type: array
+              items: NameValue
+        httpVersion: string
+        headers: json
+        cookies:
+          type: array
+          items: NetworkCookie
+        rawHeaders:
+          type: array
+          items: string
+        statusCode: number
+        statusMessage: string
+        body: binary?
+        timings:
+          type: object
+          properties:
+            blocked: number?
+            dns: number?
+            connect: number?
+            send: number
+            wait: number
+            receive: number
+            ssl: number?
+            comment: string?
+        serverIPAddress: string?
+        serverPort: number?
+        securityDetails: SecurityDetails?
 
 APIResponse:
   type: object

tests/library/apirequestcontext-network-event.spec.ts

@@ -0,0 +1,59 @@
+/**
+ * Copyright 2018 Google Inc. All rights reserved.
+ * Modifications copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { browserTest as it, expect } from '../config/browserTest';
+import { APIRequestEvent, APIRequestFinishedEvent } from 'playwright-core/src/server/fetch';
+
+it('APIRequestContext.Events.Request', async ({ context, server }) => {
+  const requests: APIRequestEvent[] = [];
+  context.request.on('apirequest', request => {
+    requests.push(request);
+  });
+  await context.request.fetch(server.EMPTY_PAGE);
+
+  await setTimeout(() => {}, 100);
+
+  const urls = requests.map(r => r.url.toString());
+  expect(urls).toEqual([
+    server.EMPTY_PAGE,
+  ]);
+});
+
+
+it('APIRequestContext.Events.RequestFinished', async ({ context, server }) => {
+
+  const finishedRequests: APIRequestFinishedEvent[] = [];
+
+  context.request.on('apirequestfinished', request => finishedRequests.push(request));
+  await context.request.fetch(server.EMPTY_PAGE);
+
+  const request = finishedRequests[0];
+
+  expect(request.requestEvent.url.toString()).toBe(server.EMPTY_PAGE);
+  expect(request.timings.send).toBeTruthy();
+});
+
+it('should fire events in proper order', async ({ context, server }) => {
+  const events: string[] = [];
+  context.request.on('apirequest', () => events.push('apirequest'));
+  context.request.on('apirequestfinished', () => events.push('apirequestfinished'));
+  await context.request.fetch(server.EMPTY_PAGE);
+  expect(events).toEqual([
+    'apirequest',
+    'apirequestfinished'
+  ]);
+});

utils/generate_channels.js

@@ -51,6 +51,9 @@ function inlineType(type, indent, wrapEnums = false) {
     }
     if (type === 'Channel')
       return { ts: `Channel`, scheme: `tChannel('*')`, optional };
+    if(type === 'URL'){
+      return { ts: 'URL', scheme: `tURL`, optional };
+    }
     return { ts: type, scheme: `tType('${type}')`, optional };
   }
   if (type.type.startsWith('array')) {
@@ -153,7 +156,7 @@ const validator_ts = [
 
 // This file is generated by ${path.basename(__filename)}, do not edit manually.
 
-import { scheme, tOptional, tObject, tBoolean, tNumber, tString, tAny, tEnum, tArray, tBinary, tChannel, tType } from './validatorPrimitives';
+import { scheme, tOptional, tObject, tBoolean, tNumber, tString, tAny, tEnum, tArray, tBinary, tChannel, tType, tURL } from './validatorPrimitives';
 export type { Validator, ValidatorContext } from './validatorPrimitives';
 export { ValidationError, findValidator, maybeFindValidator, createMetadataValidator } from './validatorPrimitives';
 `];

utils/generate_types/overrides.d.ts

@@ -16,6 +16,7 @@
 import { ChildProcess } from 'child_process';
 import { Readable } from 'stream';
 import { ReadStream } from 'fs';
+import type { IncomingHttpHeaders } from 'http';
 import { Protocol } from './protocol';
 import { Serializable, EvaluationArgument, PageFunction, PageFunctionOn, SmartHandle, ElementHandleForTag, BindingSource } from './structs';