feat(coverage): export raw v8 coverage (#976)

Fixes #955
This commit is contained in:
Pavel Feldman 2020-02-13 17:39:14 -08:00 committed by GitHub
parent 7ec3bf4d94
commit cd4e9da807
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 121 additions and 154 deletions

View file

@ -21,11 +21,11 @@
- [class: Selectors](#class-selectors)
- [class: TimeoutError](#class-timeouterror)
- [class: Accessibility](#class-accessibility)
- [class: Coverage](#class-coverage)
- [class: Worker](#class-worker)
- [class: BrowserServer](#class-browserserver)
- [class: BrowserType](#class-browsertype)
- [class: ChromiumBrowser](#class-chromiumbrowser)
- [class: ChromiumCoverage](#class-chromiumcoverage)
- [class: ChromiumSession](#class-chromiumsession)
- [class: ChromiumTarget](#class-chromiumtarget)
- [class: FirefoxBrowser](#class-firefoxbrowser)
@ -821,9 +821,9 @@ Get the browser context that the page belongs to.
#### page.coverage
- returns: <[Coverage]>
- returns: <?[any]>
> **NOTE** Code coverage is currently only supported in Chromium.
Browser-specific Coverage implementation, only available for Chromium atm. See [ChromiumCoverage](#class-chromiumcoverage) for more details.
#### page.dblclick(selector[, options])
- `selector` <[string]> A selector to search for element to double click. If there are multiple elements satisfying the selector, the first will be double clicked.
@ -3261,78 +3261,6 @@ function findFocusedNode(node) {
}
```
### class: Coverage
Coverage gathers information about parts of JavaScript and CSS that were used by the page.
An example of using JavaScript and CSS coverage to get percentage of initially
executed code:
```js
// Enable both JavaScript and CSS coverage
await Promise.all([
page.coverage.startJSCoverage(),
page.coverage.startCSSCoverage()
]);
// Navigate to page
await page.goto('https://example.com');
// Disable both JavaScript and CSS coverage
const [jsCoverage, cssCoverage] = await Promise.all([
page.coverage.stopJSCoverage(),
page.coverage.stopCSSCoverage(),
]);
let totalBytes = 0;
let usedBytes = 0;
const coverage = [...jsCoverage, ...cssCoverage];
for (const entry of coverage) {
totalBytes += entry.text.length;
for (const range of entry.ranges)
usedBytes += range.end - range.start - 1;
}
console.log(`Bytes used: ${usedBytes / totalBytes * 100}%`);
```
<!-- GEN:toc -->
- [coverage.startCSSCoverage([options])](#coveragestartcsscoverageoptions)
- [coverage.startJSCoverage([options])](#coveragestartjscoverageoptions)
- [coverage.stopCSSCoverage()](#coveragestopcsscoverage)
- [coverage.stopJSCoverage()](#coveragestopjscoverage)
<!-- GEN:stop -->
#### coverage.startCSSCoverage([options])
- `options` <[Object]> Set of configurable options for coverage
- `resetOnNavigation` <[boolean]> Whether to reset coverage on every navigation. Defaults to `true`.
- returns: <[Promise]> Promise that resolves when coverage is started
#### coverage.startJSCoverage([options])
- `options` <[Object]> Set of configurable options for coverage
- `resetOnNavigation` <[boolean]> Whether to reset coverage on every navigation. Defaults to `true`.
- `reportAnonymousScripts` <[boolean]> Whether anonymous scripts generated by the page should be reported. Defaults to `false`.
- returns: <[Promise]> Promise that resolves when coverage is started
> **NOTE** Anonymous scripts are ones that don't have an associated url. These are scripts that are dynamically created on the page using `eval` or `new Function`. If `reportAnonymousScripts` is set to `true`, anonymous scripts will have `__playwright_evaluation_script__` as their URL.
#### coverage.stopCSSCoverage()
- returns: <[Promise]<[Array]<[Object]>>> Promise that resolves to the array of coverage reports for all stylesheets
- `url` <[string]> StyleSheet URL
- `text` <[string]> StyleSheet content
- `ranges` <[Array]<[Object]>> StyleSheet ranges that were used. Ranges are sorted and non-overlapping.
- `start` <[number]> A start offset in text, inclusive
- `end` <[number]> An end offset in text, exclusive
> **NOTE** CSS Coverage doesn't include dynamically injected style tags without sourceURLs.
#### coverage.stopJSCoverage()
- returns: <[Promise]<[Array]<[Object]>>> Promise that resolves to the array of coverage reports for all scripts
- `url` <[string]> Script URL
- `text` <[string]> Script content
- `ranges` <[Array]<[Object]>> Script ranges that were executed. Ranges are sorted and non-overlapping.
- `start` <[number]> A start offset in text, inclusive
- `end` <[number]> An end offset in text, exclusive
> **NOTE** JavaScript Coverage doesn't include anonymous scripts by default. However, scripts with sourceURLs are
reported.
### class: Worker
The Worker class represents a [WebWorker](https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API).
@ -3687,6 +3615,71 @@ await page.evaluate(() => window.open('https://www.example.com/'));
const newWindowTarget = await browser.chromium.waitForTarget(target => target.url() === 'https://www.example.com/');
```
### class: ChromiumCoverage
Coverage gathers information about parts of JavaScript and CSS that were used by the page.
An example of using JavaScript coverage to produce Istambul report for page load:
```js
const { chromium } = require('.');
const v8toIstanbul = require('v8-to-istanbul');
(async() => {
const browser = await chromium.launch();
const page = await browser.newPage();
await page.coverage.startJSCoverage();
await page.goto('https://chromium.org');
const coverage = await page.coverage.stopJSCoverage();
for (const entry of coverage) {
const converter = new v8toIstanbul('', 0, { source: entry.source });
await converter.load();
converter.applyCoverage(entry.functions);
console.log(JSON.stringify(converter.toIstanbul()));
}
await browser.close();
})();
```
<!-- GEN:toc -->
- [chromiumCoverage.startCSSCoverage([options])](#chromiumcoveragestartcsscoverageoptions)
- [chromiumCoverage.startJSCoverage([options])](#chromiumcoveragestartjscoverageoptions)
- [chromiumCoverage.stopCSSCoverage()](#chromiumcoveragestopcsscoverage)
- [chromiumCoverage.stopJSCoverage()](#chromiumcoveragestopjscoverage)
<!-- GEN:stop -->
#### chromiumCoverage.startCSSCoverage([options])
- `options` <[Object]> Set of configurable options for coverage
- `resetOnNavigation` <[boolean]> Whether to reset coverage on every navigation. Defaults to `true`.
- returns: <[Promise]> Promise that resolves when coverage is started
#### chromiumCoverage.startJSCoverage([options])
- `options` <[Object]> Set of configurable options for coverage
- `resetOnNavigation` <[boolean]> Whether to reset coverage on every navigation. Defaults to `true`.
- `reportAnonymousScripts` <[boolean]> Whether anonymous scripts generated by the page should be reported. Defaults to `false`.
- returns: <[Promise]> Promise that resolves when coverage is started
> **NOTE** Anonymous scripts are ones that don't have an associated url. These are scripts that are dynamically created on the page using `eval` or `new Function`. If `reportAnonymousScripts` is set to `true`, anonymous scripts will have `__playwright_evaluation_script__` as their URL.
#### chromiumCoverage.stopCSSCoverage()
- returns: <[Promise]<[Array]<[Object]>>> Promise that resolves to the array of coverage reports for all stylesheets
- `url` <[string]> StyleSheet URL
- `text` <[string]> StyleSheet content
- `ranges` <[Array]<[Object]>> StyleSheet ranges that were used. Ranges are sorted and non-overlapping.
- `start` <[number]> A start offset in text, inclusive
- `end` <[number]> An end offset in text, exclusive
> **NOTE** CSS Coverage doesn't include dynamically injected style tags without sourceURLs.
#### chromiumCoverage.stopJSCoverage()
- returns: <[Promise]<[Array]<[Object]>>> Promise that resolves to the array of coverage reports for all scripts
- `url` <[string]> Script URL
- `source` <[string]> Script content
- `functions` <[Array]<[Object]>> V8-specific coverage format.
> **NOTE** JavaScript Coverage doesn't include anonymous scripts by default. However, scripts with sourceURLs are
reported.
### class: ChromiumSession
* extends: [EventEmitter](https://nodejs.org/api/events.html#events_class_eventemitter)

View file

@ -25,10 +25,11 @@ export { Frame } from './frames';
export { Keyboard, Mouse } from './input';
export { JSHandle } from './javascript';
export { Request, Response } from './network';
export { Coverage, FileChooser, Page, Worker } from './page';
export { FileChooser, Page, Worker } from './page';
export { Selectors } from './selectors';
export { CRBrowser as ChromiumBrowser } from './chromium/crBrowser';
export { CRCoverage as ChromiumCoverage } from './chromium/crCoverage';
export { CRSession as ChromiumSession } from './chromium/crConnection';
export { CRTarget as ChromiumTarget } from './chromium/crTarget';

View file

@ -20,16 +20,33 @@ import { assert, debugError, helper, RegisteredListener } from '../helper';
import { Protocol } from './protocol';
import { EVALUATION_SCRIPT_URL } from './crExecutionContext';
import { Coverage } from '../page';
import * as types from '../types';
type CoverageEntry = {
type JSRange = {
startOffset: number,
endOffset: number,
count: number
}
type CSSCoverageEntry = {
url: string,
text: string,
ranges: {start: number, end: number}[]
text?: string,
ranges: {
start: number,
end: number
}[]
};
export class CRCoverage implements Coverage {
type JSCoverageEntry = {
url: string,
source?: string,
functions: {
functionName: string,
ranges: JSRange[]
}[]
};
export class CRCoverage {
private _jsCoverage: JSCoverage;
private _cssCoverage: CSSCoverage;
@ -42,7 +59,7 @@ export class CRCoverage implements Coverage {
return await this._jsCoverage.start(options);
}
async stopJSCoverage(): Promise<CoverageEntry[]> {
async stopJSCoverage(): Promise<JSCoverageEntry[]> {
return await this._jsCoverage.stop();
}
@ -50,7 +67,7 @@ export class CRCoverage implements Coverage {
return await this._cssCoverage.start(options);
}
async stopCSSCoverage(): Promise<CoverageEntry[]> {
async stopCSSCoverage(): Promise<CSSCoverageEntry[]> {
return await this._cssCoverage.stop();
}
}
@ -58,7 +75,7 @@ export class CRCoverage implements Coverage {
class JSCoverage {
_client: CRSession;
_enabled: boolean;
_scriptURLs: Map<string, string>;
_scriptIds: Set<string>;
_scriptSources: Map<string, string>;
_eventListeners: RegisteredListener[];
_resetOnNavigation: boolean;
@ -67,7 +84,7 @@ class JSCoverage {
constructor(client: CRSession) {
this._client = client;
this._enabled = false;
this._scriptURLs = new Map();
this._scriptIds = new Set();
this._scriptSources = new Map();
this._eventListeners = [];
this._resetOnNavigation = false;
@ -82,7 +99,7 @@ class JSCoverage {
this._resetOnNavigation = resetOnNavigation;
this._reportAnonymousScripts = reportAnonymousScripts;
this._enabled = true;
this._scriptURLs.clear();
this._scriptIds.clear();
this._scriptSources.clear();
this._eventListeners = [
helper.addEventListener(this._client, 'Debugger.scriptParsed', this._onScriptParsed.bind(this)),
@ -91,7 +108,7 @@ class JSCoverage {
this._client.on('Debugger.paused', () => this._client.send('Debugger.resume'));
await Promise.all([
this._client.send('Profiler.enable'),
this._client.send('Profiler.startPreciseCoverage', {callCount: false, detailed: true}),
this._client.send('Profiler.startPreciseCoverage', { callCount: true, detailed: true }),
this._client.send('Debugger.enable'),
this._client.send('Debugger.setSkipAllPauses', {skip: true})
]);
@ -100,7 +117,7 @@ class JSCoverage {
_onExecutionContextsCleared() {
if (!this._resetOnNavigation)
return;
this._scriptURLs.clear();
this._scriptIds.clear();
this._scriptSources.clear();
}
@ -108,12 +125,12 @@ class JSCoverage {
// Ignore playwright-injected scripts
if (event.url === EVALUATION_SCRIPT_URL)
return;
this._scriptIds.add(event.scriptId);
// Ignore other anonymous scripts unless the reportAnonymousScripts option is true.
if (!event.url && !this._reportAnonymousScripts)
return;
try {
const response = await this._client.send('Debugger.getScriptSource', {scriptId: event.scriptId});
this._scriptURLs.set(event.scriptId, event.url);
this._scriptSources.set(event.scriptId, response.scriptSource);
} catch (e) {
// This might happen if the page has already navigated away.
@ -121,7 +138,7 @@ class JSCoverage {
}
}
async stop(): Promise<CoverageEntry[]> {
async stop(): Promise<JSCoverageEntry[]> {
assert(this._enabled, 'JSCoverage is not enabled');
this._enabled = false;
const [profileResponse] = await Promise.all([
@ -132,19 +149,17 @@ class JSCoverage {
] as const);
helper.removeEventListeners(this._eventListeners);
const coverage = [];
const coverage: JSCoverageEntry[] = [];
for (const entry of profileResponse.result) {
let url = this._scriptURLs.get(entry.scriptId);
if (!url && this._reportAnonymousScripts)
url = 'debugger://VM' + entry.scriptId;
const text = this._scriptSources.get(entry.scriptId);
if (text === undefined || url === undefined)
if (!this._scriptIds.has(entry.scriptId))
continue;
const flattenRanges = [];
for (const func of entry.functions)
flattenRanges.push(...func.ranges);
const ranges = convertToDisjointRanges(flattenRanges);
coverage.push({url, ranges, text});
if (!entry.url && !this._reportAnonymousScripts)
continue;
const source = this._scriptSources.get(entry.scriptId);
if (source)
coverage.push({...entry, source});
else
coverage.push(entry);
}
return coverage;
}
@ -207,7 +222,7 @@ class CSSCoverage {
}
}
async stop(): Promise<CoverageEntry[]> {
async stop(): Promise<CSSCoverageEntry[]> {
assert(this._enabled, 'CSSCoverage is not enabled');
this._enabled = false;
const ruleTrackingResponse = await this._client.send('CSS.stopRuleUsageTracking');
@ -232,7 +247,7 @@ class CSSCoverage {
});
}
const coverage: CoverageEntry[] = [];
const coverage: CSSCoverageEntry[] = [];
for (const styleSheetId of this._stylesheetURLs.keys()) {
const url = this._stylesheetURLs.get(styleSheetId)!;
const text = this._stylesheetSources.get(styleSheetId)!;

View file

@ -23,7 +23,7 @@ import * as network from '../network';
import { CRSession, CRConnection } from './crConnection';
import { EVALUATION_SCRIPT_URL, CRExecutionContext } from './crExecutionContext';
import { CRNetworkManager } from './crNetworkManager';
import { Page, Coverage, Worker } from '../page';
import { Page, Worker } from '../page';
import { Protocol } from './protocol';
import { Events } from '../events';
import { toConsoleMessageLocation, exceptionToError, releaseObject } from './crProtocolHelper';
@ -544,7 +544,7 @@ export class CRPage implements PageDelegate {
return this._pdf.generate(options);
}
coverage(): Coverage | undefined {
coverage(): CRCoverage {
return this._coverage;
}

View file

@ -20,7 +20,7 @@ import { helper, RegisteredListener, debugError, assert } from '../helper';
import * as dom from '../dom';
import { FFSession } from './ffConnection';
import { FFExecutionContext } from './ffExecutionContext';
import { Page, PageDelegate, Coverage, Worker } from '../page';
import { Page, PageDelegate, Worker } from '../page';
import { FFNetworkManager } from './ffNetworkManager';
import { Events } from '../events';
import * as dialog from '../dialog';
@ -438,10 +438,6 @@ export class FFPage implements PageDelegate {
return getAccessibilityTree(this._session, needle);
}
coverage(): Coverage | undefined {
return undefined;
}
async getFrameElement(frame: frames.Frame): Promise<dom.ElementHandle> {
const parent = frame.parentFrame();
if (!parent)

View file

@ -73,7 +73,7 @@ export interface PageDelegate {
getAccessibilityTree(needle?: dom.ElementHandle): Promise<{tree: accessibility.AXNode, needle: accessibility.AXNode | null}>;
pdf?: (options?: types.PDFOptions) => Promise<platform.BufferType>;
coverage(): Coverage | undefined;
coverage?: () => any;
}
type PageState = {
@ -112,7 +112,7 @@ export class Page extends platform.EventEmitter {
readonly accessibility: accessibility.Accessibility;
private _workers = new Map<string, Worker>();
readonly pdf: ((options?: types.PDFOptions) => Promise<platform.BufferType>) | undefined;
readonly coverage: Coverage | undefined;
readonly coverage: any;
readonly _requestHandlers: { url: types.URLMatch, handler: (request: network.Request) => void }[] = [];
_ownedContext: BrowserContext | undefined;
@ -150,7 +150,7 @@ export class Page extends platform.EventEmitter {
this._frameManager = new frames.FrameManager(this);
if (delegate.pdf)
this.pdf = delegate.pdf.bind(delegate);
this.coverage = delegate.coverage();
this.coverage = delegate.coverage ? delegate.coverage() : null;
}
_didClose() {
@ -603,10 +603,3 @@ export class Worker {
return (await this._executionContextPromise).evaluateHandle(pageFunction, ...args as any);
}
}
export interface Coverage {
startJSCoverage(options?: types.JSCoverageOptions): Promise<void>;
stopJSCoverage(): Promise<types.CoverageEntry[]>;
startCSSCoverage(options?: types.CSSCoverageOptions): Promise<void>;
stopCSSCoverage(): Promise<types.CoverageEntry[]>;
}

View file

@ -24,7 +24,7 @@ import { Events } from '../events';
import { WKExecutionContext } from './wkExecutionContext';
import { WKInterceptableRequest } from './wkInterceptableRequest';
import { WKWorkers } from './wkWorkers';
import { Page, PageDelegate, Coverage } from '../page';
import { Page, PageDelegate } from '../page';
import { Protocol } from './protocol';
import * as dialog from '../dialog';
import { BrowserContext } from '../browserContext';
@ -599,10 +599,6 @@ export class WKPage implements PageDelegate {
return getAccessibilityTree(this._session, needle);
}
coverage(): Coverage | undefined {
return undefined;
}
async getFrameElement(frame: frames.Frame): Promise<dom.ElementHandle> {
const parent = frame.parentFrame();
if (!parent)

View file

@ -29,10 +29,7 @@ module.exports.describe = function({testRunner, expect, FFOX, CHROMIUM, WEBKIT})
const coverage = await page.coverage.stopJSCoverage();
expect(coverage.length).toBe(1);
expect(coverage[0].url).toContain('/jscoverage/simple.html');
expect(coverage[0].ranges).toEqual([
{ start: 0, end: 17 },
{ start: 35, end: 61 },
]);
expect(coverage[0].functions.find(f => f.functionName === 'foo').ranges[0].count).toEqual(1);
});
it('should report sourceURLs', async function({page, server}) {
await page.coverage.startJSCoverage();
@ -71,31 +68,6 @@ module.exports.describe = function({testRunner, expect, FFOX, CHROMIUM, WEBKIT})
expect(coverage[0].url).toContain('/jscoverage/script1.js');
expect(coverage[1].url).toContain('/jscoverage/script2.js');
});
it('should report right ranges', async function({page, server}) {
await page.coverage.startJSCoverage();
await page.goto(server.PREFIX + '/jscoverage/ranges.html');
const coverage = await page.coverage.stopJSCoverage();
expect(coverage.length).toBe(1);
const entry = coverage[0];
expect(entry.ranges.length).toBe(1);
const range = entry.ranges[0];
expect(entry.text.substring(range.start, range.end)).toBe(`console.log('used!');`);
});
it('should report scripts that have no coverage', async function({page, server}) {
await page.coverage.startJSCoverage();
await page.goto(server.PREFIX + '/jscoverage/unused.html');
const coverage = await page.coverage.stopJSCoverage();
expect(coverage.length).toBe(1);
const entry = coverage[0];
expect(entry.url).toContain('unused.html');
expect(entry.ranges.length).toBe(0);
});
it('should work with conditionals', async function({page, server}) {
await page.coverage.startJSCoverage();
await page.goto(server.PREFIX + '/jscoverage/involved.html');
const coverage = await page.coverage.stopJSCoverage();
expect(JSON.stringify(coverage, null, 2).replace(/:\d{4}\//g, ':<PORT>/')).toBeGolden('jscoverage-involved.txt');
});
describe('resetOnNavigation', function() {
it('should report scripts across navigations when disabled', async function({page, server}) {
await page.coverage.startJSCoverage({resetOnNavigation: false});
@ -210,6 +182,7 @@ module.exports.describe = function({testRunner, expect, FFOX, CHROMIUM, WEBKIT})
link.href = url;
document.head.appendChild(link);
await new Promise(x => link.onload = x);
await new Promise(f => requestAnimationFrame(f));
}, server.PREFIX + '/csscoverage/stylesheet1.css');
const coverage = await page.coverage.stopCSSCoverage();
expect(coverage.length).toBe(1);