feat(har): introduce urlFilter (#14693)

This is a glob or regex pattern that filters entries recorder in the HAR.
This commit is contained in:
Dmitry Gozman 2022-06-07 18:09:47 -07:00 committed by GitHub
parent b2d0fae3b1
commit fdcdd58d7f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 237 additions and 161 deletions

View file

@ -562,6 +562,7 @@ Logger sink for Playwright logging.
- `omitContent` ?<[boolean]> Optional setting to control whether to omit request content from the HAR. Defaults to - `omitContent` ?<[boolean]> Optional setting to control whether to omit request content from the HAR. Defaults to
`false`. `false`.
- `path` <[path]> Path on the filesystem to write the HAR file to. - `path` <[path]> Path on the filesystem to write the HAR file to.
- `urlFilter` ?<[string]|[RegExp]> A glob or regex pattern to filter requests that are stored in the HAR. When a [`option: baseURL`] via the context options was provided and the passed URL is a path, it gets merged via the [`new URL()`](https://developer.mozilla.org/en-US/docs/Web/API/URL/URL) constructor.
Enables [HAR](http://www.softwareishard.com/blog/har-12-spec) recording for all pages into `recordHar.path` file. If not Enables [HAR](http://www.softwareishard.com/blog/har-12-spec) recording for all pages into `recordHar.path` file. If not
specified, the HAR is not recorded. Make sure to await [`method: BrowserContext.close`] for the HAR to be specified, the HAR is not recorded. Make sure to await [`method: BrowserContext.close`] for the HAR to be

View file

@ -28,7 +28,7 @@ import { Events } from './events';
import { TimeoutSettings } from '../common/timeoutSettings'; import { TimeoutSettings } from '../common/timeoutSettings';
import { Waiter } from './waiter'; import { Waiter } from './waiter';
import type { URLMatch, Headers, WaitForEventOptions, BrowserContextOptions, StorageState, LaunchOptions } from './types'; import type { URLMatch, Headers, WaitForEventOptions, BrowserContextOptions, StorageState, LaunchOptions } from './types';
import { headersObjectToArray } from '../utils'; import { headersObjectToArray, isRegExp, isString } from '../utils';
import { mkdirIfNeeded } from '../utils/fileUtils'; import { mkdirIfNeeded } from '../utils/fileUtils';
import { isSafeCloseError } from '../common/errors'; import { isSafeCloseError } from '../common/errors';
import type * as api from '../../types/types'; import type * as api from '../../types/types';
@ -390,6 +390,18 @@ async function prepareStorageState(options: BrowserContextOptions): Promise<chan
} }
} }
function prepareRecordHarOptions(options: BrowserContextOptions['recordHar']): channels.RecordHarOptions | undefined {
if (!options)
return;
return {
path: options.path,
omitContent: options.omitContent,
urlGlob: isString(options.urlFilter) ? options.urlFilter : undefined,
urlRegexSource: isRegExp(options.urlFilter) ? options.urlFilter.source : undefined,
urlRegexFlags: isRegExp(options.urlFilter) ? options.urlFilter.flags : undefined,
};
}
export async function prepareBrowserContextParams(options: BrowserContextOptions): Promise<channels.BrowserNewContextParams> { export async function prepareBrowserContextParams(options: BrowserContextOptions): Promise<channels.BrowserNewContextParams> {
if (options.videoSize && !options.videosPath) if (options.videoSize && !options.videosPath)
throw new Error(`"videoSize" option requires "videosPath" to be specified`); throw new Error(`"videoSize" option requires "videosPath" to be specified`);
@ -401,6 +413,7 @@ export async function prepareBrowserContextParams(options: BrowserContextOptions
noDefaultViewport: options.viewport === null, noDefaultViewport: options.viewport === null,
extraHTTPHeaders: options.extraHTTPHeaders ? headersObjectToArray(options.extraHTTPHeaders) : undefined, extraHTTPHeaders: options.extraHTTPHeaders ? headersObjectToArray(options.extraHTTPHeaders) : undefined,
storageState: await prepareStorageState(options), storageState: await prepareStorageState(options),
recordHar: prepareRecordHarOptions(options.recordHar),
}; };
if (!contextParams.recordVideo && options.videosPath) { if (!contextParams.recordVideo && options.videosPath) {
contextParams.recordVideo = { contextParams.recordVideo = {

View file

@ -17,7 +17,7 @@
import type * as types from './types'; import type * as types from './types';
import fs from 'fs'; import fs from 'fs';
import { isString, isRegExp, constructURLBasedOnBaseURL } from '../utils'; import { isString } from '../utils';
export function envObjectToArray(env: types.Env): { name: string, value: string }[] { export function envObjectToArray(env: types.Env): { name: string, value: string }[] {
const result: { name: string, value: string }[] = []; const result: { name: string, value: string }[] = [];
@ -48,90 +48,3 @@ export async function evaluationScript(fun: Function | string | { path?: string,
} }
throw new Error('Either path or content property must be present'); throw new Error('Either path or content property must be present');
} }
export function parsedURL(url: string): URL | null {
try {
return new URL(url);
} catch (e) {
return null;
}
}
export function urlMatches(baseURL: string | undefined, urlString: string, match: types.URLMatch | undefined): boolean {
if (match === undefined || match === '')
return true;
if (isString(match) && !match.startsWith('*'))
match = constructURLBasedOnBaseURL(baseURL, match);
if (isString(match))
match = globToRegex(match);
if (isRegExp(match))
return match.test(urlString);
if (typeof match === 'string' && match === urlString)
return true;
const url = parsedURL(urlString);
if (!url)
return false;
if (typeof match === 'string')
return url.pathname === match;
if (typeof match !== 'function')
throw new Error('url parameter should be string, RegExp or function');
return match(url);
}
const escapeGlobChars = new Set(['/', '$', '^', '+', '.', '(', ')', '=', '!', '|']);
export function globToRegex(glob: string): RegExp {
const tokens = ['^'];
let inGroup;
for (let i = 0; i < glob.length; ++i) {
const c = glob[i];
if (escapeGlobChars.has(c)) {
tokens.push('\\' + c);
continue;
}
if (c === '*') {
const beforeDeep = glob[i - 1];
let starCount = 1;
while (glob[i + 1] === '*') {
starCount++;
i++;
}
const afterDeep = glob[i + 1];
const isDeep = starCount > 1 &&
(beforeDeep === '/' || beforeDeep === undefined) &&
(afterDeep === '/' || afterDeep === undefined);
if (isDeep) {
tokens.push('((?:[^/]*(?:\/|$))*)');
i++;
} else {
tokens.push('([^/]*)');
}
continue;
}
switch (c) {
case '?':
tokens.push('.');
break;
case '{':
inGroup = true;
tokens.push('(');
break;
case '}':
inGroup = false;
tokens.push(')');
break;
case ',':
if (inGroup) {
tokens.push('|');
break;
}
tokens.push('\\' + c);
break;
default:
tokens.push(c);
}
}
tokens.push('$');
return new RegExp(tokens.join(''));
}

View file

@ -29,7 +29,7 @@ import { Waiter } from './waiter';
import { Events } from './events'; import { Events } from './events';
import type { LifecycleEvent, URLMatch, SelectOption, SelectOptionOptions, FilePayload, WaitForFunctionOptions, StrictOptions } from './types'; import type { LifecycleEvent, URLMatch, SelectOption, SelectOptionOptions, FilePayload, WaitForFunctionOptions, StrictOptions } from './types';
import { kLifecycleEvents } from './types'; import { kLifecycleEvents } from './types';
import { urlMatches } from './clientHelper'; import { urlMatches } from '../common/netUtils';
import type * as api from '../../types/types'; import type * as api from '../../types/types';
import type * as structs from '../../types/structs'; import type * as structs from '../../types/structs';
import { debugLogger } from '../common/debugLogger'; import { debugLogger } from '../common/debugLogger';

View file

@ -28,7 +28,7 @@ import type { Page } from './page';
import { Waiter } from './waiter'; import { Waiter } from './waiter';
import type * as api from '../../types/types'; import type * as api from '../../types/types';
import type { HeadersArray, URLMatch } from '../common/types'; import type { HeadersArray, URLMatch } from '../common/types';
import { urlMatches } from './clientHelper'; import { urlMatches } from '../common/netUtils';
import { MultiMap } from '../utils/multimap'; import { MultiMap } from '../utils/multimap';
import { APIResponse } from './fetch'; import { APIResponse } from './fetch';

View file

@ -45,13 +45,14 @@ import type * as structs from '../../types/structs';
import fs from 'fs'; import fs from 'fs';
import path from 'path'; import path from 'path';
import type { Size, URLMatch, Headers, LifecycleEvent, WaitForEventOptions, SelectOption, SelectOptionOptions, FilePayload, WaitForFunctionOptions } from './types'; import type { Size, URLMatch, Headers, LifecycleEvent, WaitForEventOptions, SelectOption, SelectOptionOptions, FilePayload, WaitForFunctionOptions } from './types';
import { evaluationScript, urlMatches } from './clientHelper'; import { evaluationScript } from './clientHelper';
import { isString, isRegExp, isObject, headersObjectToArray } from '../utils'; import { isString, isRegExp, isObject, headersObjectToArray } from '../utils';
import { mkdirIfNeeded } from '../utils/fileUtils'; import { mkdirIfNeeded } from '../utils/fileUtils';
import { isSafeCloseError } from '../common/errors'; import { isSafeCloseError } from '../common/errors';
import { Video } from './video'; import { Video } from './video';
import { Artifact } from './artifact'; import { Artifact } from './artifact';
import type { APIRequestContext } from './fetch'; import type { APIRequestContext } from './fetch';
import { urlMatches } from '../common/netUtils';
type PDFOptions = Omit<channels.PagePdfParams, 'width' | 'height' | 'margin'> & { type PDFOptions = Omit<channels.PagePdfParams, 'width' | 'height' | 'margin'> & {
width?: string | number, width?: string | number,

View file

@ -47,13 +47,18 @@ export type SetStorageState = {
export type LifecycleEvent = channels.LifecycleEvent; export type LifecycleEvent = channels.LifecycleEvent;
export const kLifecycleEvents: Set<LifecycleEvent> = new Set(['load', 'domcontentloaded', 'networkidle', 'commit']); export const kLifecycleEvents: Set<LifecycleEvent> = new Set(['load', 'domcontentloaded', 'networkidle', 'commit']);
export type BrowserContextOptions = Omit<channels.BrowserNewContextOptions, 'viewport' | 'noDefaultViewport' | 'extraHTTPHeaders' | 'storageState'> & { export type BrowserContextOptions = Omit<channels.BrowserNewContextOptions, 'viewport' | 'noDefaultViewport' | 'extraHTTPHeaders' | 'storageState' | 'recordHar'> & {
viewport?: Size | null, viewport?: Size | null,
extraHTTPHeaders?: Headers, extraHTTPHeaders?: Headers,
logger?: Logger, logger?: Logger,
videosPath?: string, videosPath?: string,
videoSize?: Size, videoSize?: Size,
storageState?: string | SetStorageState, storageState?: string | SetStorageState,
recordHar?: {
path: string,
omitContent?: boolean,
urlFilter?: string | RegExp,
},
}; };
type LaunchOverrides = { type LaunchOverrides = {

View file

@ -20,6 +20,8 @@ import net from 'net';
import { getProxyForUrl } from '../utilsBundle'; import { getProxyForUrl } from '../utilsBundle';
import { HttpsProxyAgent } from '../utilsBundle'; import { HttpsProxyAgent } from '../utilsBundle';
import * as URL from 'url'; import * as URL from 'url';
import type { URLMatch } from './types';
import { isString, constructURLBasedOnBaseURL, isRegExp } from '../utils';
export async function createSocket(host: string, port: number): Promise<net.Socket> { export async function createSocket(host: string, port: number): Promise<net.Socket> {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
@ -101,3 +103,91 @@ export function fetchData(params: HTTPRequestParams, onError?: (params: HTTPRequ
}, reject); }, reject);
}); });
} }
export function urlMatches(baseURL: string | undefined, urlString: string, match: URLMatch | undefined): boolean {
if (match === undefined || match === '')
return true;
if (isString(match) && !match.startsWith('*'))
match = constructURLBasedOnBaseURL(baseURL, match);
if (isString(match))
match = globToRegex(match);
if (isRegExp(match))
return match.test(urlString);
if (typeof match === 'string' && match === urlString)
return true;
const url = parsedURL(urlString);
if (!url)
return false;
if (typeof match === 'string')
return url.pathname === match;
if (typeof match !== 'function')
throw new Error('url parameter should be string, RegExp or function');
return match(url);
}
function parsedURL(url: string): URL | null {
try {
return new URL.URL(url);
} catch (e) {
return null;
}
}
const escapeGlobChars = new Set(['/', '$', '^', '+', '.', '(', ')', '=', '!', '|']);
// Note: this function is exported so it can be unit-tested.
export function globToRegex(glob: string): RegExp {
const tokens = ['^'];
let inGroup;
for (let i = 0; i < glob.length; ++i) {
const c = glob[i];
if (escapeGlobChars.has(c)) {
tokens.push('\\' + c);
continue;
}
if (c === '*') {
const beforeDeep = glob[i - 1];
let starCount = 1;
while (glob[i + 1] === '*') {
starCount++;
i++;
}
const afterDeep = glob[i + 1];
const isDeep = starCount > 1 &&
(beforeDeep === '/' || beforeDeep === undefined) &&
(afterDeep === '/' || afterDeep === undefined);
if (isDeep) {
tokens.push('((?:[^/]*(?:\/|$))*)');
i++;
} else {
tokens.push('([^/]*)');
}
continue;
}
switch (c) {
case '?':
tokens.push('.');
break;
case '{':
inGroup = true;
tokens.push('(');
break;
case '}':
inGroup = false;
tokens.push(')');
break;
case ',':
if (inGroup) {
tokens.push('|');
break;
}
tokens.push('\\' + c);
break;
default:
tokens.push(c);
}
}
tokens.push('$');
return new RegExp(tokens.join(''));
}

View file

@ -263,6 +263,14 @@ export type SerializedError = {
value?: SerializedValue, value?: SerializedValue,
}; };
export type RecordHarOptions = {
omitContent?: boolean,
path: string,
urlGlob?: string,
urlRegexSource?: string,
urlRegexFlags?: string,
};
export type FormField = { export type FormField = {
name: string, name: string,
value?: string, value?: string,
@ -737,10 +745,7 @@ export type BrowserTypeLaunchPersistentContextParams = {
height: number, height: number,
}, },
}, },
recordHar?: { recordHar?: RecordHarOptions,
omitContent?: boolean,
path: string,
},
strictSelectors?: boolean, strictSelectors?: boolean,
userDataDir: string, userDataDir: string,
slowMo?: number, slowMo?: number,
@ -809,10 +814,7 @@ export type BrowserTypeLaunchPersistentContextOptions = {
height: number, height: number,
}, },
}, },
recordHar?: { recordHar?: RecordHarOptions,
omitContent?: boolean,
path: string,
},
strictSelectors?: boolean, strictSelectors?: boolean,
slowMo?: number, slowMo?: number,
}; };
@ -905,10 +907,7 @@ export type BrowserNewContextParams = {
height: number, height: number,
}, },
}, },
recordHar?: { recordHar?: RecordHarOptions,
omitContent?: boolean,
path: string,
},
strictSelectors?: boolean, strictSelectors?: boolean,
proxy?: { proxy?: {
server: string, server: string,
@ -964,10 +963,7 @@ export type BrowserNewContextOptions = {
height: number, height: number,
}, },
}, },
recordHar?: { recordHar?: RecordHarOptions,
omitContent?: boolean,
path: string,
},
strictSelectors?: boolean, strictSelectors?: boolean,
proxy?: { proxy?: {
server: string, server: string,
@ -3573,10 +3569,7 @@ export type ElectronLaunchParams = {
ignoreHTTPSErrors?: boolean, ignoreHTTPSErrors?: boolean,
locale?: string, locale?: string,
offline?: boolean, offline?: boolean,
recordHar?: { recordHar?: RecordHarOptions,
omitContent?: boolean,
path: string,
},
recordVideo?: { recordVideo?: {
dir: string, dir: string,
size?: { size?: {
@ -3609,10 +3602,7 @@ export type ElectronLaunchOptions = {
ignoreHTTPSErrors?: boolean, ignoreHTTPSErrors?: boolean,
locale?: string, locale?: string,
offline?: boolean, offline?: boolean,
recordHar?: { recordHar?: RecordHarOptions,
omitContent?: boolean,
path: string,
},
recordVideo?: { recordVideo?: {
dir: string, dir: string,
size?: { size?: {
@ -3991,10 +3981,7 @@ export type AndroidDeviceLaunchBrowserParams = {
height: number, height: number,
}, },
}, },
recordHar?: { recordHar?: RecordHarOptions,
omitContent?: boolean,
path: string,
},
strictSelectors?: boolean, strictSelectors?: boolean,
pkg?: string, pkg?: string,
proxy?: { proxy?: {
@ -4047,10 +4034,7 @@ export type AndroidDeviceLaunchBrowserOptions = {
height: number, height: number,
}, },
}, },
recordHar?: { recordHar?: RecordHarOptions,
omitContent?: boolean,
path: string,
},
strictSelectors?: boolean, strictSelectors?: boolean,
pkg?: string, pkg?: string,
proxy?: { proxy?: {

View file

@ -220,6 +220,17 @@ SerializedError:
stack: string? stack: string?
value: SerializedValue? value: SerializedValue?
RecordHarOptions:
type: object
properties:
omitContent: boolean?
path: string
urlGlob: string?
urlRegexSource: string?
urlRegexFlags: string?
FormField: FormField:
type: object type: object
properties: properties:
@ -442,11 +453,7 @@ ContextOptions:
properties: properties:
width: number width: number
height: number height: number
recordHar: recordHar: RecordHarOptions?
type: object?
properties:
omitContent: boolean?
path: string
strictSelectors: boolean? strictSelectors: boolean?
LocalUtils: LocalUtils:
@ -2786,11 +2793,7 @@ Electron:
ignoreHTTPSErrors: boolean? ignoreHTTPSErrors: boolean?
locale: string? locale: string?
offline: boolean? offline: boolean?
recordHar: recordHar: RecordHarOptions?
type: object?
properties:
omitContent: boolean?
path: string
recordVideo: recordVideo:
type: object? type: object?
properties: properties:

View file

@ -153,6 +153,13 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme {
})), })),
value: tOptional(tType('SerializedValue')), value: tOptional(tType('SerializedValue')),
}); });
scheme.RecordHarOptions = tObject({
omitContent: tOptional(tBoolean),
path: tString,
urlGlob: tOptional(tString),
urlRegexSource: tOptional(tString),
urlRegexFlags: tOptional(tString),
});
scheme.FormField = tObject({ scheme.FormField = tObject({
name: tString, name: tString,
value: tOptional(tString), value: tOptional(tString),
@ -345,10 +352,7 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme {
height: tNumber, height: tNumber,
})), })),
})), })),
recordHar: tOptional(tObject({ recordHar: tOptional(tType('RecordHarOptions')),
omitContent: tOptional(tBoolean),
path: tString,
})),
strictSelectors: tOptional(tBoolean), strictSelectors: tOptional(tBoolean),
userDataDir: tString, userDataDir: tString,
slowMo: tOptional(tNumber), slowMo: tOptional(tNumber),
@ -404,10 +408,7 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme {
height: tNumber, height: tNumber,
})), })),
})), })),
recordHar: tOptional(tObject({ recordHar: tOptional(tType('RecordHarOptions')),
omitContent: tOptional(tBoolean),
path: tString,
})),
strictSelectors: tOptional(tBoolean), strictSelectors: tOptional(tBoolean),
proxy: tOptional(tObject({ proxy: tOptional(tObject({
server: tString, server: tString,
@ -1279,10 +1280,7 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme {
ignoreHTTPSErrors: tOptional(tBoolean), ignoreHTTPSErrors: tOptional(tBoolean),
locale: tOptional(tString), locale: tOptional(tString),
offline: tOptional(tBoolean), offline: tOptional(tBoolean),
recordHar: tOptional(tObject({ recordHar: tOptional(tType('RecordHarOptions')),
omitContent: tOptional(tBoolean),
path: tString,
})),
recordVideo: tOptional(tObject({ recordVideo: tOptional(tObject({
dir: tString, dir: tString,
size: tOptional(tObject({ size: tOptional(tObject({
@ -1441,10 +1439,7 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme {
height: tNumber, height: tNumber,
})), })),
})), })),
recordHar: tOptional(tObject({ recordHar: tOptional(tType('RecordHarOptions')),
omitContent: tOptional(tBoolean),
path: tString,
})),
strictSelectors: tOptional(tBoolean), strictSelectors: tOptional(tBoolean),
pkg: tOptional(tString), pkg: tOptional(tString),
proxy: tOptional(tObject({ proxy: tOptional(tObject({

View file

@ -20,11 +20,7 @@ import { Artifact } from '../artifact';
import type { BrowserContext } from '../browserContext'; import type { BrowserContext } from '../browserContext';
import type * as har from './har'; import type * as har from './har';
import { HarTracer } from './harTracer'; import { HarTracer } from './harTracer';
import type { HarOptions } from '../types';
type HarOptions = {
path: string;
omitContent?: boolean;
};
export class HarRecorder { export class HarRecorder {
private _artifact: Artifact; private _artifact: Artifact;
@ -36,10 +32,12 @@ export class HarRecorder {
constructor(context: BrowserContext | APIRequestContext, options: HarOptions) { constructor(context: BrowserContext | APIRequestContext, options: HarOptions) {
this._artifact = new Artifact(context, options.path); this._artifact = new Artifact(context, options.path);
this._options = options; this._options = options;
const urlFilterRe = options.urlRegexSource !== undefined && options.urlRegexFlags !== undefined ? new RegExp(options.urlRegexSource, options.urlRegexFlags) : undefined;
this._tracer = new HarTracer(context, this, { this._tracer = new HarTracer(context, this, {
content: options.omitContent ? 'omit' : 'embedded', content: options.omitContent ? 'omit' : 'embedded',
waitForContentOnStop: true, waitForContentOnStop: true,
skipScripts: false, skipScripts: false,
urlFilter: urlFilterRe ?? options.urlGlob,
}); });
this._tracer.start(); this._tracer.start();
} }

View file

@ -27,6 +27,7 @@ import { eventsHelper } from '../../utils/eventsHelper';
import { mime } from '../../utilsBundle'; import { mime } from '../../utilsBundle';
import { ManualPromise } from '../../utils/manualPromise'; import { ManualPromise } from '../../utils/manualPromise';
import { getPlaywrightVersion } from '../../common/userAgent'; import { getPlaywrightVersion } from '../../common/userAgent';
import { urlMatches } from '../../common/netUtils';
const FALLBACK_HTTP_VERSION = 'HTTP/1.1'; const FALLBACK_HTTP_VERSION = 'HTTP/1.1';
@ -40,6 +41,7 @@ type HarTracerOptions = {
content: 'omit' | 'sha1' | 'embedded'; content: 'omit' | 'sha1' | 'embedded';
skipScripts: boolean; skipScripts: boolean;
waitForContentOnStop: boolean; waitForContentOnStop: boolean;
urlFilter?: string | RegExp;
}; };
export class HarTracer { export class HarTracer {
@ -51,12 +53,14 @@ export class HarTracer {
private _eventListeners: RegisteredListener[] = []; private _eventListeners: RegisteredListener[] = [];
private _started = false; private _started = false;
private _entrySymbol: symbol; private _entrySymbol: symbol;
private _baseURL: string | undefined;
constructor(context: BrowserContext | APIRequestContext, delegate: HarTracerDelegate, options: HarTracerOptions) { constructor(context: BrowserContext | APIRequestContext, delegate: HarTracerDelegate, options: HarTracerOptions) {
this._context = context; this._context = context;
this._delegate = delegate; this._delegate = delegate;
this._options = options; this._options = options;
this._entrySymbol = Symbol('requestHarEntry'); this._entrySymbol = Symbol('requestHarEntry');
this._baseURL = context instanceof APIRequestContext ? context._defaultOptions().baseURL : context._options.baseURL;
} }
start() { start() {
@ -76,7 +80,10 @@ export class HarTracer {
eventsHelper.addEventListener(this._context, BrowserContext.Events.RequestFailed, request => this._onRequestFailed(request)), eventsHelper.addEventListener(this._context, BrowserContext.Events.RequestFailed, request => this._onRequestFailed(request)),
eventsHelper.addEventListener(this._context, BrowserContext.Events.Response, (response: network.Response) => this._onResponse(response))); eventsHelper.addEventListener(this._context, BrowserContext.Events.Response, (response: network.Response) => this._onResponse(response)));
} }
}
private _shouldIncludeEntryWithUrl(urlString: string) {
return !this._options.urlFilter || urlMatches(this._baseURL, urlString, this._options.urlFilter);
} }
private _entryForRequest(request: network.Request | APIRequestEvent): har.Entry | undefined { private _entryForRequest(request: network.Request | APIRequestEvent): har.Entry | undefined {
@ -146,6 +153,8 @@ export class HarTracer {
} }
private _onAPIRequest(event: APIRequestEvent) { private _onAPIRequest(event: APIRequestEvent) {
if (!this._shouldIncludeEntryWithUrl(event.url.toString()))
return;
const harEntry = createHarEntry(event.method, event.url, '', ''); const harEntry = createHarEntry(event.method, event.url, '', '');
harEntry.request.cookies = event.cookies; harEntry.request.cookies = event.cookies;
harEntry.request.headers = Object.entries(event.headers).map(([name, value]) => ({ name, value })); harEntry.request.headers = Object.entries(event.headers).map(([name, value]) => ({ name, value }));
@ -189,6 +198,8 @@ export class HarTracer {
} }
private _onRequest(request: network.Request) { private _onRequest(request: network.Request) {
if (!this._shouldIncludeEntryWithUrl(request.url()))
return;
const page = request.frame()._page; const page = request.frame()._page;
const url = network.parsedURL(request.url()); const url = network.parsedURL(request.url());
if (!url) if (!url)

View file

@ -227,6 +227,14 @@ export type SetNetworkCookieParam = {
export type EmulatedSize = { viewport: Size, screen: Size }; export type EmulatedSize = { viewport: Size, screen: Size };
export type HarOptions = {
omitContent?: boolean,
path: string,
urlGlob?: string,
urlRegexSource?: string,
urlRegexFlags?: string,
};
export type BrowserContextOptions = { export type BrowserContextOptions = {
viewport?: Size, viewport?: Size,
screen?: Size, screen?: Size,
@ -253,10 +261,7 @@ export type BrowserContextOptions = {
dir: string, dir: string,
size?: Size, size?: Size,
}, },
recordHar?: { recordHar?: HarOptions,
omitContent?: boolean,
path: string
},
storageState?: SetStorageState, storageState?: SetStorageState,
strictSelectors?: boolean, strictSelectors?: boolean,
proxy?: ProxySettings, proxy?: ProxySettings,

View file

@ -10310,6 +10310,13 @@ export interface BrowserType<Unused = {}> {
* Path on the filesystem to write the HAR file to. * Path on the filesystem to write the HAR file to.
*/ */
path: string; path: string;
/**
* A glob or regex pattern to filter requests that are stored in the HAR. When a `baseURL` via the context options was
* provided and the passed URL is a path, it gets merged via the
* [`new URL()`](https://developer.mozilla.org/en-US/docs/Web/API/URL/URL) constructor.
*/
urlFilter?: string|RegExp;
}; };
/** /**
@ -11459,6 +11466,13 @@ export interface AndroidDevice {
* Path on the filesystem to write the HAR file to. * Path on the filesystem to write the HAR file to.
*/ */
path: string; path: string;
/**
* A glob or regex pattern to filter requests that are stored in the HAR. When a `baseURL` via the context options was
* provided and the passed URL is a path, it gets merged via the
* [`new URL()`](https://developer.mozilla.org/en-US/docs/Web/API/URL/URL) constructor.
*/
urlFilter?: string|RegExp;
}; };
/** /**
@ -12985,6 +12999,13 @@ export interface Browser extends EventEmitter {
* Path on the filesystem to write the HAR file to. * Path on the filesystem to write the HAR file to.
*/ */
path: string; path: string;
/**
* A glob or regex pattern to filter requests that are stored in the HAR. When a `baseURL` via the context options was
* provided and the passed URL is a path, it gets merged via the
* [`new URL()`](https://developer.mozilla.org/en-US/docs/Web/API/URL/URL) constructor.
*/
urlFilter?: string|RegExp;
}; };
/** /**
@ -13744,6 +13765,13 @@ export interface Electron {
* Path on the filesystem to write the HAR file to. * Path on the filesystem to write the HAR file to.
*/ */
path: string; path: string;
/**
* A glob or regex pattern to filter requests that are stored in the HAR. When a `baseURL` via the context options was
* provided and the passed URL is a path, it gets merged via the
* [`new URL()`](https://developer.mozilla.org/en-US/docs/Web/API/URL/URL) constructor.
*/
urlFilter?: string|RegExp;
}; };
/** /**
@ -15426,6 +15454,13 @@ export interface BrowserContextOptions {
* Path on the filesystem to write the HAR file to. * Path on the filesystem to write the HAR file to.
*/ */
path: string; path: string;
/**
* A glob or regex pattern to filter requests that are stored in the HAR. When a `baseURL` via the context options was
* provided and the passed URL is a path, it gets merged via the
* [`new URL()`](https://developer.mozilla.org/en-US/docs/Web/API/URL/URL) constructor.
*/
urlFilter?: string|RegExp;
}; };
/** /**

View file

@ -265,6 +265,28 @@ it('should include content @smoke', async ({ contextFactory, server }, testInfo)
expect(log.entries[1].response.content.compression).toBe(0); expect(log.entries[1].response.content.compression).toBe(0);
}); });
it('should filter by glob', async ({ contextFactory, server }, testInfo) => {
const harPath = testInfo.outputPath('test.har');
const context = await contextFactory({ baseURL: server.PREFIX, recordHar: { path: harPath, urlFilter: '/*.css' }, ignoreHTTPSErrors: true });
const page = await context.newPage();
await page.goto('/har.html');
await context.close();
const log = JSON.parse(fs.readFileSync(harPath).toString())['log'] as Log;
expect(log.entries.length).toBe(1);
expect(log.entries[0].request.url.endsWith('one-style.css')).toBe(true);
});
it('should filter by regexp', async ({ contextFactory, server }, testInfo) => {
const harPath = testInfo.outputPath('test.har');
const context = await contextFactory({ recordHar: { path: harPath, urlFilter: /HAR.X?HTML/i }, ignoreHTTPSErrors: true });
const page = await context.newPage();
await page.goto(server.PREFIX + '/har.html');
await context.close();
const log = JSON.parse(fs.readFileSync(harPath).toString())['log'] as Log;
expect(log.entries.length).toBe(1);
expect(log.entries[0].request.url.endsWith('har.html')).toBe(true);
});
it('should include sizes', async ({ contextFactory, server, asset }, testInfo) => { it('should include sizes', async ({ contextFactory, server, asset }, testInfo) => {
const { page, getLog } = await pageWithHar(contextFactory, testInfo); const { page, getLog } = await pageWithHar(contextFactory, testInfo);
await page.goto(server.PREFIX + '/har.html'); await page.goto(server.PREFIX + '/har.html');

View file

@ -16,7 +16,7 @@
*/ */
import { test as it, expect } from './pageTest'; import { test as it, expect } from './pageTest';
import { globToRegex } from '../../packages/playwright-core/lib/client/clientHelper'; import { globToRegex } from '../../packages/playwright-core/lib/common/netUtils';
import vm from 'vm'; import vm from 'vm';
it('should work with navigation @smoke', async ({ page, server }) => { it('should work with navigation @smoke', async ({ page, server }) => {