feat(engines): introduce a css selector engine and a way to query it (#50)

.eslintignore

@@ -9,5 +9,5 @@ node6-testrunner/*
 lib/
 *.js
 src/chromium/protocol.d.ts
-src/injected/injectedSource.ts
+src/generated/*
 src/webkit/protocol.d.ts

.gitignore

@@ -14,7 +14,7 @@ package-lock.json
 yarn.lock
 /node6
 /src/chromium/protocol.d.ts
-/src/injected/injectedSource.ts
+/src/generated/*
 /src/webkit/protocol.d.ts
 /utils/browser/playwright-web.js
 /index.d.ts

package.json

@@ -23,8 +23,8 @@
     "doc": "node utils/doclint/cli.js",
     "coverage": "cross-env COVERAGE=true npm run unit",
     "tsc": "tsc -p .",
-    "build": "npx webpack --config src/injected/webpack-injected.config.js --mode='production' && tsc -p .",
+    "build": "npx webpack --config src/injected/cssSelectorEngine.webpack.config.js --mode='production' && npx webpack --config src/injected/injected.webpack.config.js --mode='production' && tsc -p .",
-    "watch": "npx webpack --config src/injected/webpack-injected.config.js --mode='development' --watch --silent | tsc -w -p .",
+    "watch": "npx webpack --config src/injected/cssSelectorEngine.webpack.config.js --mode='development' --watch --silent | npx webpack --config src/injected/injected.webpack.config.js --mode='development' --watch --silent | tsc -w -p .",
     "apply-next-version": "node utils/apply_next_version.js",
     "bundle": "npx browserify -r ./index.js:playwright -o utils/browser/playwright-web.js",
     "test-types": "node utils/doclint/generate_types && npx -p typescript@2.1 tsc -p utils/doclint/generate_types/test/",

src/chromium/ExecutionContext.ts

@@ -22,7 +22,8 @@ import { assert, helper } from '../helper';
 import { valueFromRemoteObject, getExceptionMessage } from './protocolHelper';
 import { createJSHandle, ElementHandle, JSHandle } from './JSHandle';
 import { Protocol } from './protocol';
-import { injectedSource } from '../injected/injectedSource';
+import * as injectedSource from '../generated/injectedSource';
+import * as cssSelectorEngineSource from '../generated/cssSelectorEngineSource';
 
 export const EVALUATION_SCRIPT_URL = '__playwright_evaluation_script__';
 const SOURCE_URL_REGEX = /^[\040\t]*\/\/[@#] sourceURL=\s*(\S*?)\s*$/m;
@@ -162,8 +163,15 @@ export class ExecutionContext {
   }
 
   _injected(): Promise<JSHandle> {
-    if (!this._injectedPromise)
+    if (!this._injectedPromise) {
-      this._injectedPromise = this.evaluateHandle(injectedSource);
+      const engineSources = [cssSelectorEngineSource.source];
+      const source = `
+        new (${injectedSource.source})([
+          ${engineSources.join(',\n')}
+        ])
+      `;
+      this._injectedPromise = this.evaluateHandle(source);
+    }
     return this._injectedPromise;
   }
 }

src/chromium/JSHandle.ts

@@ -25,6 +25,7 @@ import { FrameManager } from './FrameManager';
 import { Page } from './Page';
 import { Protocol } from './protocol';
 import { releaseObject, valueFromRemoteObject } from './protocolHelper';
+import Injected from '../injected/injected';
 
 type Point = {
   x: number;
@@ -430,8 +431,10 @@ export class ElementHandle extends JSHandle {
 
   async $(selector: string): Promise<ElementHandle | null> {
     const handle = await this.evaluateHandle(
-        (element, selector) => element.querySelector(selector),
+        (element, selector, injected: Injected) => {
-        selector
+          return injected.querySelector('css=' + selector, element);
+        },
+        selector, await this._context._injected()
     );
     const element = handle.asElement();
     if (element)

src/firefox/ExecutionContext.ts

@@ -1,11 +1,32 @@
+/**
+ * Copyright 2019 Google Inc. All rights reserved.
+ * Modifications copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 import {helper} from '../helper';
 import {JSHandle, createHandle} from './JSHandle';
 import { Frame } from './FrameManager';
+import * as injectedSource from '../generated/injectedSource';
+import * as cssSelectorEngineSource from '../generated/cssSelectorEngineSource';
 
 export class ExecutionContext {
   _session: any;
   _frame: Frame;
   _executionContextId: string;
+  private _injectedPromise: Promise<JSHandle> | null = null;
+
   constructor(session: any, frame: Frame | null, executionContextId: string) {
     this._session = session;
     this._frame = frame;
@@ -15,7 +36,7 @@ export class ExecutionContext {
   async evaluateHandle(pageFunction, ...args) {
     if (helper.isString(pageFunction)) {
       const payload = await this._session.send('Runtime.evaluate', {
-        expression: pageFunction,
+        expression: pageFunction.trim(),
         executionContextId: this._executionContextId,
       }).catch(rewriteError);
       return createHandle(this, payload.result, payload.exceptionDetails);
@@ -97,4 +118,16 @@ export class ExecutionContext {
     }
   }
 
+  _injected(): Promise<JSHandle> {
+    if (!this._injectedPromise) {
+      const engineSources = [cssSelectorEngineSource.source];
+      const source = `
+        new (${injectedSource.source})([
+          ${engineSources.join(',\n')}
+        ])
+      `;
+      this._injectedPromise = this.evaluateHandle(source);
+    }
+    return this._injectedPromise;
+  }
 }

src/firefox/JSHandle.ts

@@ -21,6 +21,7 @@ import {ExecutionContext} from './ExecutionContext';
 import {Frame} from './FrameManager';
 import { JugglerSession } from './Connection';
 import { MultiClickOptions, ClickOptions, selectFunction, SelectOption } from '../input';
+import Injected from '../injected/injected';
 
 export class JSHandle {
   _context: ExecutionContext;
@@ -202,8 +203,10 @@ export class ElementHandle extends JSHandle {
 
   async $(selector: string): Promise<ElementHandle | null> {
     const handle = await this._frame.evaluateHandle(
-        (element, selector) => element.querySelector(selector),
+      (element, selector, injected: Injected) => {
-        this, selector
+        return injected.querySelector('css=' + selector, element);
+      },
+      this, selector, await this._context._injected()
     );
     const element = handle.asElement();
     if (element)

src/injected/README.md

@@ -0,0 +1,5 @@
+# Injected
+
+This directory contains helper sources which are injected into the page.
+
+These sources are bundled with webpack to `src/generated` to be used as a compile-time source constants. See `*.webpack.config.js` for configs.

src/injected/cssSelectorEngine.ts

@@ -0,0 +1,78 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+import { SelectorEngine, SelectorRoot } from './selectorEngine';
+
+export const CSSEngine: SelectorEngine = {
+  name: 'css',
+
+  create(root: SelectorRoot, targetElement: Element): string | undefined {
+    const tokens: string[] = [];
+
+    function uniqueCSSSelector(prefix?: string): string | undefined {
+      const path = tokens.slice();
+      if (prefix)
+        path.unshift(prefix);
+      const selector = path.join(' > ');
+      const nodes = Array.from(root.querySelectorAll(selector));
+      return nodes[0] === targetElement ? selector : undefined;
+    }
+
+    for (let element: Element | null = targetElement; element && element !== root; element = element.parentElement) {
+      const nodeName = element.nodeName.toLowerCase();
+
+      // Element ID is the strongest signal, use it.
+      let bestTokenForLevel: string = '';
+      if (element.id) {
+        const token = /^[a-zA-Z][a-zA-Z0-9\-\_]+$/.test(element.id) ? '#' + element.id : `[id="${element.id}"]`;
+        const selector = uniqueCSSSelector(token);
+        if (selector)
+          return selector;
+        bestTokenForLevel = token;
+      }
+
+      const parent = element.parentElement;
+
+      // Combine class names until unique.
+      const classes = Array.from(element.classList);
+      for (let i = 0; i < classes.length; ++i) {
+        const token = '.' + classes.slice(0, i + 1).join('.');
+        const selector = uniqueCSSSelector(token);
+        if (selector)
+          return selector;
+        // Even if not unique, does this subset of classes uniquely identify node as a child?
+        if (!bestTokenForLevel && parent) {
+          const sameClassSiblings = parent.querySelectorAll(token);
+          if (sameClassSiblings.length === 1)
+            bestTokenForLevel = token;
+        }
+      }
+
+      // Ordinal is the weakest signal.
+      if (parent) {
+        const siblings = Array.from(parent.children);
+        const sameTagSiblings = siblings.filter(sibling => (sibling as Element).nodeName.toLowerCase() === nodeName);
+        const token = sameTagSiblings.length === 1 ? nodeName : `${nodeName}:nth-child(${1 + siblings.indexOf(element)})`;
+        const selector = uniqueCSSSelector(token);
+        if (selector)
+          return selector;
+        if (!bestTokenForLevel)
+          bestTokenForLevel = token;
+      } else if (!bestTokenForLevel) {
+        bestTokenForLevel = nodeName;
+      }
+      tokens.unshift(bestTokenForLevel);
+    }
+    return uniqueCSSSelector();
+  },
+
+  query(root: SelectorRoot, selector: string): Element | undefined {
+    return root.querySelector(selector) || undefined;
+  },
+
+  queryAll(root: SelectorRoot, selector: string): Element[] {
+    return Array.from(root.querySelectorAll(selector));
+  }
+};
+
+export default CSSEngine;

src/injected/cssSelectorEngine.webpack.config.js

@@ -0,0 +1,32 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+const path = require('path');
+const InlineSource = require('./webpack-inline-source-plugin.js');
+
+module.exports = {
+  entry: path.join(__dirname, 'cssSelectorEngine.ts'),
+  devtool: 'source-map',
+  module: {
+    rules: [
+      {
+        test: /\.tsx?$/,
+        loader: 'ts-loader',
+        options: {
+          transpileOnly: true
+        },
+        exclude: /node_modules/
+      }
+    ]
+  },
+  resolve: {
+    extensions: [ '.tsx', '.ts', '.js' ]
+  },
+  output: {
+    filename: 'cssSelectorEngineSource.js',
+    path: path.resolve(__dirname, '../../lib/injected/generated')
+  },
+  plugins: [
+    new InlineSource(path.join(__dirname, '..', 'generated', 'cssSelectorEngineSource.ts')),
+  ]
+};

src/injected/injected.ts

@@ -1,28 +1,86 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-class Utils {
+import { SelectorEngine } from './selectorEngine';
-  parentElementOrShadowHost(element: Element): Element | undefined {
+import { Utils } from './utils';
-    if (element.parentElement)
+
-      return element.parentElement;
+type ParsedSelector = { engine: SelectorEngine, selector: string }[];
-    if (!element.parentNode)
+
-      return;
+export class Injected {
-    if (element.parentNode.nodeType === Node.DOCUMENT_FRAGMENT_NODE && (element.parentNode as ShadowRoot).host)
+  readonly utils: Utils;
-      return (element.parentNode as ShadowRoot).host;
+  readonly engines: Map<string, SelectorEngine>;
+
+  constructor(engines: SelectorEngine[]) {
+    this.utils = new Utils();
+    this.engines = new Map();
+    for (const engine of engines)
+      this.engines.set(engine.name, engine);
   }
 
-  deepElementFromPoint(document: Document, x: number, y: number): Element | undefined {
+  querySelector(selector: string, root: Element): Element | undefined {
-    let container: Document | ShadowRoot | null = document;
+    const parsed = this._parseSelector(selector);
-    let element: Element | undefined;
+    let element = root;
-    while (container) {
+    for (const { engine, selector } of parsed) {
-      const innerElement = container.elementFromPoint(x, y) as Element | undefined;
+      const next = engine.query(element.shadowRoot || element, selector);
-      if (!innerElement || element === innerElement)
+      if (!next)
-        break;
+        return;
-      element = innerElement;
+      element = next;
-      container = element.shadowRoot;
     }
     return element;
   }
+
+  querySelectorAll(selector: string, root: Element): Element[] {
+    const parsed = this._parseSelector(selector);
+    let set = new Set<Element>([ root ]);
+    for (const { engine, selector } of parsed) {
+      const newSet = new Set<Element>();
+      for (const prev of set) {
+        for (const next of engine.queryAll(prev.shadowRoot || prev, selector)) {
+          if (newSet.has(next))
+            continue;
+          newSet.add(next);
+        }
+      }
+      set = newSet;
+    }
+    return Array.from(set);
+  }
+
+  private _parseSelector(selector: string): ParsedSelector {
+    let index = 0;
+    let quote: string | undefined;
+    let start = 0;
+    const result: ParsedSelector = [];
+    const append = () => {
+      const part = selector.substring(start, index);
+      const eqIndex = part.indexOf('=');
+      if (eqIndex === -1)
+        throw new Error(`Cannot parse selector ${selector}`);
+      const name = part.substring(0, eqIndex).trim();
+      const body = part.substring(eqIndex + 1);
+      const engine = this.engines.get(name.toLowerCase());
+      if (!engine)
+        throw new Error(`Unknown engine ${name} while parsing selector ${selector}`);
+      result.push({ engine, selector: body });
+    };
+    while (index < selector.length) {
+      const c = selector[index];
+      if (c === '\\' && index + 1 < selector.length) {
+        index += 2;
+      } else if (c === quote) {
+        quote = undefined;
+        index++;
+      } else if (!quote && c === '>' && selector[index + 1] === '>') {
+        append();
+        index += 2;
+        start = index;
+      } else {
+        index++;
+      }
+    }
+    append();
+    return result;
+  }
 }
 
-export const utils = new Utils();
+export default Injected;

src/injected/injected.webpack.config.js

@@ -1,19 +1,8 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-const fs = require('fs');
 const path = require('path');
-
+const InlineSource = require('./webpack-inline-source-plugin.js');
-class InlineInjectedSource {
-  apply(compiler) {
-    compiler.hooks.emit.tapAsync('InlineInjectedSource', (compilation, callback) => {
-      const source = compilation.assets['injectedSource.js'].source();
-      const newSource = 'export const injectedSource = ' + JSON.stringify(source) + ';';
-      fs.writeFileSync(path.join(__dirname, 'injectedSource.ts'), newSource);
-      callback();
-    });
-  }
-}
 
 module.exports = {
   entry: path.join(__dirname, 'injected.ts'),
@@ -35,9 +24,9 @@ module.exports = {
   },
   output: {
     filename: 'injectedSource.js',
-    path: path.resolve(__dirname, '../../lib/injected')
+    path: path.resolve(__dirname, '../../lib/injected/packed')
   },
   plugins: [
-    new InlineInjectedSource(),
+    new InlineSource(path.join(__dirname, '..', 'generated', 'injectedSource.ts')),
   ]
 };

src/injected/selectorEngine.ts

@@ -0,0 +1,12 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+export type SelectorType = 'default' | 'notext';
+export type SelectorRoot = Element | ShadowRoot | Document;
+
+export interface SelectorEngine {
+  name: string;
+  create(root: SelectorRoot, target: Element, type?: SelectorType): string | undefined;
+  query(root: SelectorRoot, selector: string): Element | undefined;
+  queryAll(root: SelectorRoot, selector: string): Element[];
+}

src/injected/utils.ts

@@ -0,0 +1,26 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+export class Utils {
+  parentElementOrShadowHost(element: Element): Element | undefined {
+    if (element.parentElement)
+      return element.parentElement;
+    if (!element.parentNode)
+      return;
+    if (element.parentNode.nodeType === Node.DOCUMENT_FRAGMENT_NODE && (element.parentNode as ShadowRoot).host)
+      return (element.parentNode as ShadowRoot).host;
+  }
+
+  deepElementFromPoint(document: Document, x: number, y: number): Element | undefined {
+    let container: Document | ShadowRoot | null = document;
+    let element: Element | undefined;
+    while (container) {
+      const innerElement = container.elementFromPoint(x, y) as Element | undefined;
+      if (!innerElement || element === innerElement)
+        break;
+      element = innerElement;
+      container = element.shadowRoot;
+    }
+    return element;
+  }
+}

src/injected/webpack-inline-source-plugin.js

@@ -0,0 +1,26 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+const fs = require('fs');
+const path = require('path');
+
+module.exports = class InlineSource {
+  constructor(outFile) {
+    this.outFile = outFile;
+  }
+
+  apply(compiler) {
+    compiler.hooks.emit.tapAsync('InlineSource', (compilation, callback) => {
+      let source = compilation.assets[path.basename(this.outFile).replace('.ts', '.js')].source();
+      const lastLine = source.split('\n').pop();
+      if (lastLine.startsWith('//# sourceMappingURL'))
+        source = source.substring(0, source.length - lastLine.length - 1);
+      if (source.endsWith(';'))
+        source = source.substring(0, source.length - 1);
+      source = '(' + source + ').default';
+      const newSource = 'export const source = ' + JSON.stringify(source) + ';';
+      fs.writeFileSync(this.outFile, newSource);
+      callback();
+    });
+  }
+};

src/webkit/ExecutionContext.ts

@@ -21,7 +21,8 @@ import { helper } from '../helper';
 import { valueFromRemoteObject } from './protocolHelper';
 import { createJSHandle, JSHandle } from './JSHandle';
 import { Protocol } from './protocol';
-
+import * as injectedSource from '../generated/injectedSource';
+import * as cssSelectorEngineSource from '../generated/cssSelectorEngineSource';
 
 export const EVALUATION_SCRIPT_URL = '__playwright_evaluation_script__';
 const SOURCE_URL_REGEX = /^[\040\t]*\/\/[@#] sourceURL=\s*(\S*?)\s*$/m;
@@ -33,6 +34,7 @@ export class ExecutionContext {
   _contextId: number;
   private _contextDestroyedCallback: any;
   private _executionContextDestroyedPromise: Promise<unknown>;
+  private _injectedPromise: Promise<JSHandle> | null = null;
 
   constructor(client: TargetSession, contextPayload: Protocol.Runtime.ExecutionContextDescription, frame: Frame | null) {
     this._session = client;
@@ -300,4 +302,17 @@ export class ExecutionContext {
     }
     return this._globalObjectId;
   }
+
+  _injected(): Promise<JSHandle> {
+    if (!this._injectedPromise) {
+      const engineSources = [cssSelectorEngineSource.source];
+      const source = `
+        new (${injectedSource.source})([
+          ${engineSources.join(',\n')}
+        ])
+      `;
+      this._injectedPromise = this.evaluateHandle(source);
+    }
+    return this._injectedPromise;
+  }
 }

src/webkit/JSHandle.ts

@@ -23,6 +23,8 @@ import { FrameManager } from './FrameManager';
 import { Page } from './Page';
 import { Protocol } from './protocol';
 import { releaseObject, valueFromRemoteObject } from './protocolHelper';
+import Injected from '../injected/injected';
+
 const writeFileAsync = helper.promisify(fs.writeFile);
 
 export function createJSHandle(context: ExecutionContext, remoteObject: Protocol.Runtime.RemoteObject) {
@@ -308,8 +310,10 @@ export class ElementHandle extends JSHandle {
 
   async $(selector: string): Promise<ElementHandle | null> {
     const handle = await this.evaluateHandle(
-        (element, selector) => element.querySelector(selector),
+      (element, selector, injected: Injected) => {
-        selector
+        return injected.querySelector('css=' + selector, element);
+      },
+      selector, await this._context._injected()
     );
     const element = handle.asElement();
     if (element)