feat(coverage): export raw v8 coverage (#976)

Fixes #955

docs/api.md

@@ -21,11 +21,11 @@
 - [class: Selectors](#class-selectors)
 - [class: TimeoutError](#class-timeouterror)
 - [class: Accessibility](#class-accessibility)
-- [class: Coverage](#class-coverage)
 - [class: Worker](#class-worker)
 - [class: BrowserServer](#class-browserserver)
 - [class: BrowserType](#class-browsertype)
 - [class: ChromiumBrowser](#class-chromiumbrowser)
+- [class: ChromiumCoverage](#class-chromiumcoverage)
 - [class: ChromiumSession](#class-chromiumsession)
 - [class: ChromiumTarget](#class-chromiumtarget)
 - [class: FirefoxBrowser](#class-firefoxbrowser)
@@ -821,9 +821,9 @@ Get the browser context that the page belongs to.
 
 #### page.coverage
 
-- returns: <[Coverage]>
+- returns: <?[any]>
 
-> **NOTE** Code coverage is currently only supported in Chromium.
+Browser-specific Coverage implementation, only available for Chromium atm. See [ChromiumCoverage](#class-chromiumcoverage) for more details.
 
 #### page.dblclick(selector[, options])
 - `selector` <[string]> A selector to search for element to double click. If there are multiple elements satisfying the selector, the first will be double clicked.
@@ -3261,78 +3261,6 @@ function findFocusedNode(node) {
 }
 ```
 
-### class: Coverage
-
-Coverage gathers information about parts of JavaScript and CSS that were used by the page.
-
-An example of using JavaScript and CSS coverage to get percentage of initially
-executed code:
-
-```js
-// Enable both JavaScript and CSS coverage
-await Promise.all([
-  page.coverage.startJSCoverage(),
-  page.coverage.startCSSCoverage()
-]);
-// Navigate to page
-await page.goto('https://example.com');
-// Disable both JavaScript and CSS coverage
-const [jsCoverage, cssCoverage] = await Promise.all([
-  page.coverage.stopJSCoverage(),
-  page.coverage.stopCSSCoverage(),
-]);
-let totalBytes = 0;
-let usedBytes = 0;
-const coverage = [...jsCoverage, ...cssCoverage];
-for (const entry of coverage) {
-  totalBytes += entry.text.length;
-  for (const range of entry.ranges)
-    usedBytes += range.end - range.start - 1;
-}
-console.log(`Bytes used: ${usedBytes / totalBytes * 100}%`);
-```
-
-<!-- GEN:toc -->
-- [coverage.startCSSCoverage([options])](#coveragestartcsscoverageoptions)
-- [coverage.startJSCoverage([options])](#coveragestartjscoverageoptions)
-- [coverage.stopCSSCoverage()](#coveragestopcsscoverage)
-- [coverage.stopJSCoverage()](#coveragestopjscoverage)
-<!-- GEN:stop -->
-
-#### coverage.startCSSCoverage([options])
-- `options` <[Object]>  Set of configurable options for coverage
-  - `resetOnNavigation` <[boolean]> Whether to reset coverage on every navigation. Defaults to `true`.
-- returns: <[Promise]> Promise that resolves when coverage is started
-
-#### coverage.startJSCoverage([options])
-- `options` <[Object]>  Set of configurable options for coverage
-  - `resetOnNavigation` <[boolean]> Whether to reset coverage on every navigation. Defaults to `true`.
-  - `reportAnonymousScripts` <[boolean]> Whether anonymous scripts generated by the page should be reported. Defaults to `false`.
-- returns: <[Promise]> Promise that resolves when coverage is started
-
-> **NOTE** Anonymous scripts are ones that don't have an associated url. These are scripts that are dynamically created on the page using `eval` or `new Function`. If `reportAnonymousScripts` is set to `true`, anonymous scripts will have `__playwright_evaluation_script__` as their URL.
-
-#### coverage.stopCSSCoverage()
-- returns: <[Promise]<[Array]<[Object]>>> Promise that resolves to the array of coverage reports for all stylesheets
-  - `url` <[string]> StyleSheet URL
-  - `text` <[string]> StyleSheet content
-  - `ranges` <[Array]<[Object]>> StyleSheet ranges that were used. Ranges are sorted and non-overlapping.
-    - `start` <[number]> A start offset in text, inclusive
-    - `end` <[number]> An end offset in text, exclusive
-
-> **NOTE** CSS Coverage doesn't include dynamically injected style tags without sourceURLs.
-
-#### coverage.stopJSCoverage()
-- returns: <[Promise]<[Array]<[Object]>>> Promise that resolves to the array of coverage reports for all scripts
-  - `url` <[string]> Script URL
-  - `text` <[string]> Script content
-  - `ranges` <[Array]<[Object]>> Script ranges that were executed. Ranges are sorted and non-overlapping.
-    - `start` <[number]> A start offset in text, inclusive
-    - `end` <[number]> An end offset in text, exclusive
-
-> **NOTE** JavaScript Coverage doesn't include anonymous scripts by default. However, scripts with sourceURLs are
-reported.
-
 ### class: Worker
 
 The Worker class represents a [WebWorker](https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API).
@@ -3687,6 +3615,71 @@ await page.evaluate(() => window.open('https://www.example.com/'));
 const newWindowTarget = await browser.chromium.waitForTarget(target => target.url() === 'https://www.example.com/');
 ```
 
+### class: ChromiumCoverage
+
+Coverage gathers information about parts of JavaScript and CSS that were used by the page.
+
+An example of using JavaScript coverage to produce Istambul report for page load:
+
+```js
+const { chromium } = require('.');
+const v8toIstanbul = require('v8-to-istanbul');
+
+(async() => {
+  const browser = await chromium.launch();
+  const page = await browser.newPage();
+  await page.coverage.startJSCoverage();
+  await page.goto('https://chromium.org');
+  const coverage = await page.coverage.stopJSCoverage();
+  for (const entry of coverage) {
+    const converter = new v8toIstanbul('', 0, { source: entry.source });
+    await converter.load();
+    converter.applyCoverage(entry.functions);
+    console.log(JSON.stringify(converter.toIstanbul()));
+  }
+  await browser.close();
+})();
+```
+
+<!-- GEN:toc -->
+- [chromiumCoverage.startCSSCoverage([options])](#chromiumcoveragestartcsscoverageoptions)
+- [chromiumCoverage.startJSCoverage([options])](#chromiumcoveragestartjscoverageoptions)
+- [chromiumCoverage.stopCSSCoverage()](#chromiumcoveragestopcsscoverage)
+- [chromiumCoverage.stopJSCoverage()](#chromiumcoveragestopjscoverage)
+<!-- GEN:stop -->
+
+#### chromiumCoverage.startCSSCoverage([options])
+- `options` <[Object]>  Set of configurable options for coverage
+  - `resetOnNavigation` <[boolean]> Whether to reset coverage on every navigation. Defaults to `true`.
+- returns: <[Promise]> Promise that resolves when coverage is started
+
+#### chromiumCoverage.startJSCoverage([options])
+- `options` <[Object]>  Set of configurable options for coverage
+  - `resetOnNavigation` <[boolean]> Whether to reset coverage on every navigation. Defaults to `true`.
+  - `reportAnonymousScripts` <[boolean]> Whether anonymous scripts generated by the page should be reported. Defaults to `false`.
+- returns: <[Promise]> Promise that resolves when coverage is started
+
+> **NOTE** Anonymous scripts are ones that don't have an associated url. These are scripts that are dynamically created on the page using `eval` or `new Function`. If `reportAnonymousScripts` is set to `true`, anonymous scripts will have `__playwright_evaluation_script__` as their URL.
+
+#### chromiumCoverage.stopCSSCoverage()
+- returns: <[Promise]<[Array]<[Object]>>> Promise that resolves to the array of coverage reports for all stylesheets
+  - `url` <[string]> StyleSheet URL
+  - `text` <[string]> StyleSheet content
+  - `ranges` <[Array]<[Object]>> StyleSheet ranges that were used. Ranges are sorted and non-overlapping.
+    - `start` <[number]> A start offset in text, inclusive
+    - `end` <[number]> An end offset in text, exclusive
+
+> **NOTE** CSS Coverage doesn't include dynamically injected style tags without sourceURLs.
+
+#### chromiumCoverage.stopJSCoverage()
+- returns: <[Promise]<[Array]<[Object]>>> Promise that resolves to the array of coverage reports for all scripts
+  - `url` <[string]> Script URL
+  - `source` <[string]> Script content
+  - `functions` <[Array]<[Object]>> V8-specific coverage format.
+
+> **NOTE** JavaScript Coverage doesn't include anonymous scripts by default. However, scripts with sourceURLs are
+reported.
+
 ### class: ChromiumSession
 
 * extends: [EventEmitter](https://nodejs.org/api/events.html#events_class_eventemitter)

src/api.ts

@@ -25,10 +25,11 @@ export { Frame } from './frames';
 export { Keyboard, Mouse } from './input';
 export { JSHandle } from './javascript';
 export { Request, Response } from './network';
-export { Coverage, FileChooser, Page, Worker } from './page';
+export { FileChooser, Page, Worker } from './page';
 export { Selectors } from './selectors';
 
 export { CRBrowser as ChromiumBrowser } from './chromium/crBrowser';
+export { CRCoverage as ChromiumCoverage } from './chromium/crCoverage';
 export { CRSession as ChromiumSession } from './chromium/crConnection';
 export { CRTarget as ChromiumTarget } from './chromium/crTarget';
 

src/chromium/crCoverage.ts

@@ -20,16 +20,33 @@ import { assert, debugError, helper, RegisteredListener } from '../helper';
 import { Protocol } from './protocol';
 
 import { EVALUATION_SCRIPT_URL } from './crExecutionContext';
-import { Coverage } from '../page';
 import * as types from '../types';
 
-type CoverageEntry = {
+type JSRange = {
+  startOffset: number,
+  endOffset: number,
+  count: number
+}
+
+type CSSCoverageEntry = {
   url: string,
-  text: string,
+  text?: string,
-  ranges: {start: number, end: number}[]
+  ranges: {
+    start: number,
+    end: number
+  }[]
 };
 
-export class CRCoverage implements Coverage {
+type JSCoverageEntry = {
+  url: string,
+  source?: string,
+  functions: {
+    functionName: string,
+    ranges: JSRange[]
+  }[]
+};
+
+export class CRCoverage {
   private _jsCoverage: JSCoverage;
   private _cssCoverage: CSSCoverage;
 
@@ -42,7 +59,7 @@ export class CRCoverage implements Coverage {
     return await this._jsCoverage.start(options);
   }
 
-  async stopJSCoverage(): Promise<CoverageEntry[]> {
+  async stopJSCoverage(): Promise<JSCoverageEntry[]> {
     return await this._jsCoverage.stop();
   }
 
@@ -50,7 +67,7 @@ export class CRCoverage implements Coverage {
     return await this._cssCoverage.start(options);
   }
 
-  async stopCSSCoverage(): Promise<CoverageEntry[]> {
+  async stopCSSCoverage(): Promise<CSSCoverageEntry[]> {
     return await this._cssCoverage.stop();
   }
 }
@@ -58,7 +75,7 @@ export class CRCoverage implements Coverage {
 class JSCoverage {
   _client: CRSession;
   _enabled: boolean;
-  _scriptURLs: Map<string, string>;
+  _scriptIds: Set<string>;
   _scriptSources: Map<string, string>;
   _eventListeners: RegisteredListener[];
   _resetOnNavigation: boolean;
@@ -67,7 +84,7 @@ class JSCoverage {
   constructor(client: CRSession) {
     this._client = client;
     this._enabled = false;
-    this._scriptURLs = new Map();
+    this._scriptIds = new Set();
     this._scriptSources = new Map();
     this._eventListeners = [];
     this._resetOnNavigation = false;
@@ -82,7 +99,7 @@ class JSCoverage {
     this._resetOnNavigation = resetOnNavigation;
     this._reportAnonymousScripts = reportAnonymousScripts;
     this._enabled = true;
-    this._scriptURLs.clear();
+    this._scriptIds.clear();
     this._scriptSources.clear();
     this._eventListeners = [
       helper.addEventListener(this._client, 'Debugger.scriptParsed', this._onScriptParsed.bind(this)),
@@ -91,7 +108,7 @@ class JSCoverage {
     this._client.on('Debugger.paused', () => this._client.send('Debugger.resume'));
     await Promise.all([
       this._client.send('Profiler.enable'),
-      this._client.send('Profiler.startPreciseCoverage', {callCount: false, detailed: true}),
+      this._client.send('Profiler.startPreciseCoverage', { callCount: true, detailed: true }),
       this._client.send('Debugger.enable'),
       this._client.send('Debugger.setSkipAllPauses', {skip: true})
     ]);
@@ -100,7 +117,7 @@ class JSCoverage {
   _onExecutionContextsCleared() {
     if (!this._resetOnNavigation)
       return;
-    this._scriptURLs.clear();
+    this._scriptIds.clear();
     this._scriptSources.clear();
   }
 
@@ -108,12 +125,12 @@ class JSCoverage {
     // Ignore playwright-injected scripts
     if (event.url === EVALUATION_SCRIPT_URL)
       return;
+    this._scriptIds.add(event.scriptId);
     // Ignore other anonymous scripts unless the reportAnonymousScripts option is true.
     if (!event.url && !this._reportAnonymousScripts)
       return;
     try {
       const response = await this._client.send('Debugger.getScriptSource', {scriptId: event.scriptId});
-      this._scriptURLs.set(event.scriptId, event.url);
       this._scriptSources.set(event.scriptId, response.scriptSource);
     } catch (e) {
       // This might happen if the page has already navigated away.
@@ -121,7 +138,7 @@ class JSCoverage {
     }
   }
 
-  async stop(): Promise<CoverageEntry[]> {
+  async stop(): Promise<JSCoverageEntry[]> {
     assert(this._enabled, 'JSCoverage is not enabled');
     this._enabled = false;
     const [profileResponse] = await Promise.all([
@@ -132,19 +149,17 @@ class JSCoverage {
     ] as const);
     helper.removeEventListeners(this._eventListeners);
 
-    const coverage = [];
+    const coverage: JSCoverageEntry[] = [];
     for (const entry of profileResponse.result) {
-      let url = this._scriptURLs.get(entry.scriptId);
+      if (!this._scriptIds.has(entry.scriptId))
-      if (!url && this._reportAnonymousScripts)
-        url = 'debugger://VM' + entry.scriptId;
-      const text = this._scriptSources.get(entry.scriptId);
-      if (text === undefined || url === undefined)
         continue;
-      const flattenRanges = [];
+      if (!entry.url && !this._reportAnonymousScripts)
-      for (const func of entry.functions)
+        continue;
-        flattenRanges.push(...func.ranges);
+      const source = this._scriptSources.get(entry.scriptId);
-      const ranges = convertToDisjointRanges(flattenRanges);
+      if (source)
-      coverage.push({url, ranges, text});
+        coverage.push({...entry, source});
+      else
+        coverage.push(entry);
     }
     return coverage;
   }
@@ -207,7 +222,7 @@ class CSSCoverage {
     }
   }
 
-  async stop(): Promise<CoverageEntry[]> {
+  async stop(): Promise<CSSCoverageEntry[]> {
     assert(this._enabled, 'CSSCoverage is not enabled');
     this._enabled = false;
     const ruleTrackingResponse = await this._client.send('CSS.stopRuleUsageTracking');
@@ -232,7 +247,7 @@ class CSSCoverage {
       });
     }
 
-    const coverage: CoverageEntry[] = [];
+    const coverage: CSSCoverageEntry[] = [];
     for (const styleSheetId of this._stylesheetURLs.keys()) {
       const url = this._stylesheetURLs.get(styleSheetId)!;
       const text = this._stylesheetSources.get(styleSheetId)!;

src/chromium/crPage.ts

@@ -23,7 +23,7 @@ import * as network from '../network';
 import { CRSession, CRConnection } from './crConnection';
 import { EVALUATION_SCRIPT_URL, CRExecutionContext } from './crExecutionContext';
 import { CRNetworkManager } from './crNetworkManager';
-import { Page, Coverage, Worker } from '../page';
+import { Page, Worker } from '../page';
 import { Protocol } from './protocol';
 import { Events } from '../events';
 import { toConsoleMessageLocation, exceptionToError, releaseObject } from './crProtocolHelper';
@@ -544,7 +544,7 @@ export class CRPage implements PageDelegate {
     return this._pdf.generate(options);
   }
 
-  coverage(): Coverage | undefined {
+  coverage(): CRCoverage {
     return this._coverage;
   }
 

src/firefox/ffPage.ts

@@ -20,7 +20,7 @@ import { helper, RegisteredListener, debugError, assert } from '../helper';
 import * as dom from '../dom';
 import { FFSession } from './ffConnection';
 import { FFExecutionContext } from './ffExecutionContext';
-import { Page, PageDelegate, Coverage, Worker } from '../page';
+import { Page, PageDelegate, Worker } from '../page';
 import { FFNetworkManager } from './ffNetworkManager';
 import { Events } from '../events';
 import * as dialog from '../dialog';
@@ -438,10 +438,6 @@ export class FFPage implements PageDelegate {
     return getAccessibilityTree(this._session, needle);
   }
 
-  coverage(): Coverage | undefined {
-    return undefined;
-  }
-
   async getFrameElement(frame: frames.Frame): Promise<dom.ElementHandle> {
     const parent = frame.parentFrame();
     if (!parent)

src/page.ts

@@ -73,7 +73,7 @@ export interface PageDelegate {
 
   getAccessibilityTree(needle?: dom.ElementHandle): Promise<{tree: accessibility.AXNode, needle: accessibility.AXNode | null}>;
   pdf?: (options?: types.PDFOptions) => Promise<platform.BufferType>;
-  coverage(): Coverage | undefined;
+  coverage?: () => any;
 }
 
 type PageState = {
@@ -112,7 +112,7 @@ export class Page extends platform.EventEmitter {
   readonly accessibility: accessibility.Accessibility;
   private _workers = new Map<string, Worker>();
   readonly pdf: ((options?: types.PDFOptions) => Promise<platform.BufferType>) | undefined;
-  readonly coverage: Coverage | undefined;
+  readonly coverage: any;
   readonly _requestHandlers: { url: types.URLMatch, handler: (request: network.Request) => void }[] = [];
   _ownedContext: BrowserContext | undefined;
 
@@ -150,7 +150,7 @@ export class Page extends platform.EventEmitter {
     this._frameManager = new frames.FrameManager(this);
     if (delegate.pdf)
       this.pdf = delegate.pdf.bind(delegate);
-    this.coverage = delegate.coverage();
+    this.coverage = delegate.coverage ? delegate.coverage() : null;
   }
 
   _didClose() {
@@ -603,10 +603,3 @@ export class Worker {
     return (await this._executionContextPromise).evaluateHandle(pageFunction, ...args as any);
   }
 }
-
-export interface Coverage {
-  startJSCoverage(options?: types.JSCoverageOptions): Promise<void>;
-  stopJSCoverage(): Promise<types.CoverageEntry[]>;
-  startCSSCoverage(options?: types.CSSCoverageOptions): Promise<void>;
-  stopCSSCoverage(): Promise<types.CoverageEntry[]>;
-}

src/webkit/wkPage.ts

@@ -24,7 +24,7 @@ import { Events } from '../events';
 import { WKExecutionContext } from './wkExecutionContext';
 import { WKInterceptableRequest } from './wkInterceptableRequest';
 import { WKWorkers } from './wkWorkers';
-import { Page, PageDelegate, Coverage } from '../page';
+import { Page, PageDelegate } from '../page';
 import { Protocol } from './protocol';
 import * as dialog from '../dialog';
 import { BrowserContext } from '../browserContext';
@@ -599,10 +599,6 @@ export class WKPage implements PageDelegate {
     return getAccessibilityTree(this._session, needle);
   }
 
-  coverage(): Coverage | undefined {
-    return undefined;
-  }
-
   async getFrameElement(frame: frames.Frame): Promise<dom.ElementHandle> {
     const parent = frame.parentFrame();
     if (!parent)

test/chromium/coverage.spec.js

@@ -29,10 +29,7 @@ module.exports.describe = function({testRunner, expect, FFOX, CHROMIUM, WEBKIT})
       const coverage = await page.coverage.stopJSCoverage();
       expect(coverage.length).toBe(1);
       expect(coverage[0].url).toContain('/jscoverage/simple.html');
-      expect(coverage[0].ranges).toEqual([
+      expect(coverage[0].functions.find(f => f.functionName === 'foo').ranges[0].count).toEqual(1);
-        { start: 0, end: 17 },
-        { start: 35, end: 61 },
-      ]);
     });
     it('should report sourceURLs', async function({page, server}) {
       await page.coverage.startJSCoverage();
@@ -71,31 +68,6 @@ module.exports.describe = function({testRunner, expect, FFOX, CHROMIUM, WEBKIT})
       expect(coverage[0].url).toContain('/jscoverage/script1.js');
       expect(coverage[1].url).toContain('/jscoverage/script2.js');
     });
-    it('should report right ranges', async function({page, server}) {
-      await page.coverage.startJSCoverage();
-      await page.goto(server.PREFIX + '/jscoverage/ranges.html');
-      const coverage = await page.coverage.stopJSCoverage();
-      expect(coverage.length).toBe(1);
-      const entry = coverage[0];
-      expect(entry.ranges.length).toBe(1);
-      const range = entry.ranges[0];
-      expect(entry.text.substring(range.start, range.end)).toBe(`console.log('used!');`);
-    });
-    it('should report scripts that have no coverage', async function({page, server}) {
-      await page.coverage.startJSCoverage();
-      await page.goto(server.PREFIX + '/jscoverage/unused.html');
-      const coverage = await page.coverage.stopJSCoverage();
-      expect(coverage.length).toBe(1);
-      const entry = coverage[0];
-      expect(entry.url).toContain('unused.html');
-      expect(entry.ranges.length).toBe(0);
-    });
-    it('should work with conditionals', async function({page, server}) {
-      await page.coverage.startJSCoverage();
-      await page.goto(server.PREFIX + '/jscoverage/involved.html');
-      const coverage = await page.coverage.stopJSCoverage();
-      expect(JSON.stringify(coverage, null, 2).replace(/:\d{4}\//g, ':<PORT>/')).toBeGolden('jscoverage-involved.txt');
-    });
     describe('resetOnNavigation', function() {
       it('should report scripts across navigations when disabled', async function({page, server}) {
         await page.coverage.startJSCoverage({resetOnNavigation: false});
@@ -210,6 +182,7 @@ module.exports.describe = function({testRunner, expect, FFOX, CHROMIUM, WEBKIT})
         link.href = url;
         document.head.appendChild(link);
         await new Promise(x => link.onload = x);
+        await new Promise(f => requestAnimationFrame(f));
       }, server.PREFIX + '/csscoverage/stylesheet1.css');
       const coverage = await page.coverage.stopCSSCoverage();
       expect(coverage.length).toBe(1);