chore(testrunner): make most modifiers external (#1581)

test/playwright.spec.js

@@ -99,6 +99,16 @@ module.exports.describe = ({testRunner, product, playwrightPath}) => {
     ASSETS_DIR,
   };
 
+  function loadTests(modulePath) {
+    const module = require(modulePath);
+    if (typeof module.describe === 'function')
+      describe('', module.describe, testOptions);
+    if (typeof module.fdescribe === 'function')
+      fdescribe('', module.fdescribe, testOptions);
+    if (typeof module.xdescribe === 'function')
+      xdescribe('', module.xdescribe, testOptions);
+  }
+
   describe('', function() {
     beforeAll(async state => {
       state.browser = await browserType.launch(defaultBrowserOptions);
@@ -154,69 +164,69 @@ module.exports.describe = ({testRunner, product, playwrightPath}) => {
 
       // Page-level tests that are given a browser, a context and a page.
       // Each test is launched in a new browser context.
-      describe('[Accessibility]', () => testRunner.loadTests(require('./accessibility.spec.js'), testOptions));
+      describe('[Accessibility]', () => loadTests('./accessibility.spec.js'));
       describe('[Driver]', () => {
-        testRunner.loadTests(require('./autowaiting.spec.js'), testOptions);
+        loadTests('./autowaiting.spec.js');
-        testRunner.loadTests(require('./click.spec.js'), testOptions);
+        loadTests('./click.spec.js');
-        testRunner.loadTests(require('./cookies.spec.js'), testOptions);
+        loadTests('./cookies.spec.js');
-        testRunner.loadTests(require('./dialog.spec.js'), testOptions);
+        loadTests('./dialog.spec.js');
-        testRunner.loadTests(require('./elementhandle.spec.js'), testOptions);
+        loadTests('./elementhandle.spec.js');
-        testRunner.loadTests(require('./emulation.spec.js'), testOptions);
+        loadTests('./emulation.spec.js');
-        testRunner.loadTests(require('./evaluation.spec.js'), testOptions);
+        loadTests('./evaluation.spec.js');
-        testRunner.loadTests(require('./frame.spec.js'), testOptions);
+        loadTests('./frame.spec.js');
-        testRunner.loadTests(require('./focus.spec.js'), testOptions);
+        loadTests('./focus.spec.js');
-        testRunner.loadTests(require('./input.spec.js'), testOptions);
+        loadTests('./input.spec.js');
-        testRunner.loadTests(require('./jshandle.spec.js'), testOptions);
+        loadTests('./jshandle.spec.js');
-        testRunner.loadTests(require('./keyboard.spec.js'), testOptions);
+        loadTests('./keyboard.spec.js');
-        testRunner.loadTests(require('./mouse.spec.js'), testOptions);
+        loadTests('./mouse.spec.js');
-        testRunner.loadTests(require('./navigation.spec.js'), testOptions);
+        loadTests('./navigation.spec.js');
-        testRunner.loadTests(require('./network.spec.js'), testOptions);
+        loadTests('./network.spec.js');
-        testRunner.loadTests(require('./page.spec.js'), testOptions);
+        loadTests('./page.spec.js');
-        testRunner.loadTests(require('./queryselector.spec.js'), testOptions);
+        loadTests('./queryselector.spec.js');
-        testRunner.loadTests(require('./screenshot.spec.js'), testOptions);
+        loadTests('./screenshot.spec.js');
-        testRunner.loadTests(require('./waittask.spec.js'), testOptions);
+        loadTests('./waittask.spec.js');
-        testRunner.loadTests(require('./interception.spec.js'), testOptions);
+        loadTests('./interception.spec.js');
-        testRunner.loadTests(require('./geolocation.spec.js'), testOptions);
+        loadTests('./geolocation.spec.js');
-        testRunner.loadTests(require('./workers.spec.js'), testOptions);
+        loadTests('./workers.spec.js');
-        testRunner.loadTests(require('./capabilities.spec.js'), testOptions);
+        loadTests('./capabilities.spec.js');
       });
       describe('[Permissions]', () => {
-        testRunner.loadTests(require('./permissions.spec.js'), testOptions);
+        loadTests('./permissions.spec.js');
       });
 
       describe.skip(!CHROMIUM)('[Chromium]', () => {
-        testRunner.loadTests(require('./chromium/chromium.spec.js'), testOptions);
+        loadTests('./chromium/chromium.spec.js');
-        testRunner.loadTests(require('./chromium/coverage.spec.js'), testOptions);
+        loadTests('./chromium/coverage.spec.js');
-        testRunner.loadTests(require('./chromium/pdf.spec.js'), testOptions);
+        loadTests('./chromium/pdf.spec.js');
-        testRunner.loadTests(require('./chromium/session.spec.js'), testOptions);
+        loadTests('./chromium/session.spec.js');
       });
     });
 
     // Browser-level tests that are given a browser.
     describe('[Driver]', () => {
-      testRunner.loadTests(require('./browser.spec.js'), testOptions);
+      loadTests('./browser.spec.js');
-      testRunner.loadTests(require('./browsercontext.spec.js'), testOptions);
+      loadTests('./browsercontext.spec.js');
-      testRunner.loadTests(require('./ignorehttpserrors.spec.js'), testOptions);
+      loadTests('./ignorehttpserrors.spec.js');
-      testRunner.loadTests(require('./popup.spec.js'), testOptions);
+      loadTests('./popup.spec.js');
     });
   });
 
   // Top-level tests that launch Browser themselves.
   describe('[Driver]', () => {
-    testRunner.loadTests(require('./defaultbrowsercontext.spec.js'), testOptions);
+    loadTests('./defaultbrowsercontext.spec.js');
-    testRunner.loadTests(require('./fixtures.spec.js'), testOptions);
+    loadTests('./fixtures.spec.js');
-    testRunner.loadTests(require('./launcher.spec.js'), testOptions);
+    loadTests('./launcher.spec.js');
-    testRunner.loadTests(require('./headful.spec.js'), testOptions);
+    loadTests('./headful.spec.js');
-    testRunner.loadTests(require('./multiclient.spec.js'), testOptions);
+    loadTests('./multiclient.spec.js');
   });
 
   describe.skip(!CHROMIUM)('[Chromium]', () => {
-    testRunner.loadTests(require('./chromium/launcher.spec.js'), testOptions);
+    loadTests('./chromium/launcher.spec.js');
-    testRunner.loadTests(require('./chromium/oopif.spec.js'), testOptions);
+    loadTests('./chromium/oopif.spec.js');
-    testRunner.loadTests(require('./chromium/tracing.spec.js'), testOptions);
+    loadTests('./chromium/tracing.spec.js');
   });
 
   describe('[Driver]', () => {
-    testRunner.loadTests(require('./web.spec.js'), testOptions);
+    loadTests('./web.spec.js');
   });
 };

test/test.js

@@ -18,6 +18,7 @@ const path = require('path');
 const {TestServer} = require('../utils/testserver/');
 const {TestRunner, Reporter} = require('../utils/testrunner/');
 const utils = require('./utils');
+const inspector = require('inspector');
 
 let parallel = 1;
 if (process.env.PW_PARALLEL_TESTS)
@@ -30,11 +31,66 @@ require('events').defaultMaxListeners *= parallel;
 let timeout = process.env.CI ? 30 * 1000 : 10 * 1000;
 if (!isNaN(process.env.TIMEOUT))
   timeout = parseInt(process.env.TIMEOUT * 1000, 10);
+const MAJOR_NODEJS_VERSION = parseInt(process.version.substring(1).split('.')[0], 10);
+if (MAJOR_NODEJS_VERSION >= 8 && inspector.url()) {
+  console.log('Detected inspector - disabling timeout to be debugger-friendly');
+  timeout = 0;
+}
+
 const testRunner = new TestRunner({
   timeout,
   parallel,
   breakOnFailure: process.argv.indexOf('--break-on-failure') !== -1,
+  installCommonHelpers: false
 });
+testRunner.testModifier('skip', (t, condition) => condition && t.setMode(t.Modes.Skip));
+testRunner.suiteModifier('skip', (s, condition) => condition && s.setMode(s.Modes.Skip));
+testRunner.testModifier('fail', (t, condition) => condition && t.setExpectation(t.Expectations.Fail));
+testRunner.suiteModifier('fail', (s, condition) => condition && s.setExpectation(s.Expectations.Fail));
+testRunner.testModifier('slow', (t, condition) => condition && t.setTimeout(t.timeout() * 3));
+testRunner.testModifier('repeat', (t, count) => t.setRepeat(count));
+testRunner.suiteModifier('repeat', (s, count) => s.setRepeat(count));
+testRunner.testAttribute('focus', t => t.setMode(t.Modes.Focus));
+testRunner.suiteAttribute('focus', s => s.setMode(s.Modes.Focus));
+testRunner.testAttribute('debug', t => {
+  t.setMode(t.Modes.Focus);
+  t.setTimeout(100000000);
+
+  let session;
+  t.before(async () => {
+    const util = require('util');
+    const fs = require('fs');
+    const url = require('url');
+    const readFileAsync = util.promisify(fs.readFile.bind(fs));
+    session = new inspector.Session();
+    session.connect();
+    const postAsync = util.promisify(session.post.bind(session));
+    await postAsync('Debugger.enable');
+    const setBreakpointCommands = [];
+    const N = t.body().toString().split('\n').length;
+    const location = t.location();
+    const lines = (await readFileAsync(location.filePath, 'utf8')).split('\n');
+    for (let line = 0; line < N; ++line) {
+      const lineNumber = line + location.lineNumber;
+      setBreakpointCommands.push(postAsync('Debugger.setBreakpointByUrl', {
+        url: url.pathToFileURL(location.filePath),
+        lineNumber,
+        condition: `console.log('${String(lineNumber + 1).padStart(6, ' ')} | ' + ${JSON.stringify(lines[lineNumber])})`,
+      }).catch(e => {}));
+    }
+    await Promise.all(setBreakpointCommands);
+  });
+
+  t.after(async () => {
+    session.disconnect();
+  });
+});
+testRunner.fdescribe = testRunner.describe.focus;
+testRunner.xdescribe = testRunner.describe.skip(true);
+testRunner.fit = testRunner.it.focus;
+testRunner.xit = testRunner.it.skip(true);
+testRunner.dit = testRunner.it.debug;
+
 const {describe, fdescribe, beforeAll, afterAll, beforeEach, afterEach} = testRunner;
 
 console.log('Testing on Node', process.version);
@@ -106,7 +162,7 @@ for (const browserConfig of BROWSER_CONFIGS) {
     continue;
   const product = browserConfig.name;
   describe(product, () => {
-    testRunner.loadTests(require('./playwright.spec.js'), {
+    testRunner.describe('', require('./playwright.spec.js').describe, {
       product,
       playwrightPath: utils.projectRoot(),
       testRunner,

test/types.d.ts

@@ -36,7 +36,7 @@ type TestRunner<STATE> = {
     fit: ItFunction<STATE>;
     dit: ItFunction<STATE>;
 
-    beforeAll, beforeEach, afterAll, afterEach, loadTests;
+    beforeAll, beforeEach, afterAll, afterEach;
 };
 
 interface TestSetup<STATE> {

utils/testrunner/Multimap.js

@@ -1,95 +0,0 @@
-/**
- * Copyright 2017 Google Inc. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-class Multimap {
-  constructor() {
-    this._map = new Map();
-  }
-
-  set(key, value) {
-    let set = this._map.get(key);
-    if (!set) {
-      set = new Set();
-      this._map.set(key, set);
-    }
-    set.add(value);
-  }
-
-  get(key) {
-    let result = this._map.get(key);
-    if (!result)
-      result = new Set();
-    return result;
-  }
-
-  has(key) {
-    return this._map.has(key);
-  }
-
-  hasValue(key, value) {
-    const set = this._map.get(key);
-    if (!set)
-      return false;
-    return set.has(value);
-  }
-
-  /**
-   * @return {number}
-   */
-  get size() {
-    return this._map.size;
-  }
-
-  delete(key, value) {
-    const values = this.get(key);
-    const result = values.delete(value);
-    if (!values.size)
-      this._map.delete(key);
-    return result;
-  }
-
-  deleteAll(key) {
-    this._map.delete(key);
-  }
-
-  firstValue(key) {
-    const set = this._map.get(key);
-    if (!set)
-      return null;
-    return set.values().next().value;
-  }
-
-  firstKey() {
-    return this._map.keys().next().value;
-  }
-
-  valuesArray() {
-    const result = [];
-    for (const key of this._map.keys())
-      result.push(...Array.from(this._map.get(key).values()));
-    return result;
-  }
-
-  keysArray() {
-    return Array.from(this._map.keys());
-  }
-
-  clear() {
-    this._map.clear();
-  }
-}
-
-module.exports = Multimap;

utils/testrunner/TestRunner.js

@@ -14,25 +14,15 @@
  * limitations under the License.
  */
 
-const util = require('util');
-const url = require('url');
-const inspector = require('inspector');
 const EventEmitter = require('events');
-const Multimap = require('./Multimap');
-const fs = require('fs');
 const {SourceMapSupport} = require('./SourceMapSupport');
 const debug = require('debug');
 const {getCallerLocation} = require('./utils');
 
 const INFINITE_TIMEOUT = 100000000;
-
-const readFileAsync = util.promisify(fs.readFile.bind(fs));
-
 const TimeoutError = new Error('Timeout');
 const TerminatedError = new Error('Terminated');
 
-const MAJOR_NODEJS_VERSION = parseInt(process.version.substring(1).split('.')[0], 10);
-
 function runUserCallback(callback, timeout, args) {
   let terminateCallback;
   let timeoutId;
@@ -609,7 +599,7 @@ class TestRunner extends EventEmitter {
       parallel = 1,
       breakOnFailure = false,
       crashIfTestsAreFocusedOnCI = true,
-      disableTimeoutWhenInspectorIsEnabled = true,
+      installCommonHelpers = true,
     } = options;
     this._crashIfTestsAreFocusedOnCI = crashIfTestsAreFocusedOnCI;
     this._sourceMapSupport = new SourceMapSupport();
@@ -626,16 +616,6 @@ class TestRunner extends EventEmitter {
     this._testModifiers = new Map();
     this._testAttributes = new Map();
 
-    if (MAJOR_NODEJS_VERSION >= 8 && disableTimeoutWhenInspectorIsEnabled) {
-      if (inspector.url()) {
-        console.log('TestRunner detected inspector; overriding certain properties to be debugger-friendly');
-        console.log('  - timeout = 0 (Infinite)');
-        this._timeout = INFINITE_TIMEOUT;
-      }
-    }
-
-    this._debuggerLogBreakpointLines = new Multimap();
-
     this.beforeAll = (callback) => this._currentSuite.beforeAll(callback);
     this.beforeEach = (callback) => this._currentSuite.beforeEach(callback);
     this.afterAll = (callback) => this._currentSuite.afterAll(callback);
@@ -644,29 +624,12 @@ class TestRunner extends EventEmitter {
     this.describe = this._suiteBuilder([]);
     this.it = this._testBuilder([]);
 
-    this.testAttribute('debug', t => {
+    if (installCommonHelpers) {
-      t.setMode(t.Modes.Focus);
+      this.fdescribe = this.describe.setup(s => s.setMode(s.Modes.Focus));
-      t.setTimeout(INFINITE_TIMEOUT);
+      this.xdescribe = this.describe.setup(s => s.setMode(s.Modes.Skip));
-      const N = t.body().toString().split('\n').length;
+      this.fit = this.it.setup(t => t.setMode(t.Modes.Focus));
-      const location = t.location();
+      this.xit = this.it.setup(t => t.setMode(t.Modes.Skip));
-      for (let line = 0; line < N; ++line)
+    }
-        this._debuggerLogBreakpointLines.set(location.filePath, line + location.lineNumber);
-    });
-
-    this.testModifier('skip', (t, condition) => condition && t.setMode(t.Modes.Skip));
-    this.suiteModifier('skip', (s, condition) => condition && s.setMode(s.Modes.Skip));
-    this.testModifier('fail', (t, condition) => condition && t.setExpectation(t.Expectations.Fail));
-    this.suiteModifier('fail', (s, condition) => condition && s.setExpectation(s.Expectations.Fail));
-    this.testModifier('slow', (t, condition) => condition && t.setTimeout(t.timeout() * 3));
-    this.testModifier('repeat', (t, count) => t.setRepeat(count));
-    this.suiteModifier('repeat', (s, count) => s.setRepeat(count));
-    this.testAttribute('focus', t => t.setMode(t.Modes.Focus));
-    this.suiteAttribute('focus', s => s.setMode(s.Modes.Focus));
-    this.fdescribe = this.describe.focus;
-    this.xdescribe = this.describe.skip(true);
-    this.fit = this.it.focus;
-    this.xit = this.it.skip(true);
-    this.dit = this.it.debug;
   }
 
   _suiteBuilder(callbacks) {
@@ -683,6 +646,8 @@ class TestRunner extends EventEmitter {
       }
     }, {
       get: (obj, prop) => {
+        if (prop === 'setup')
+          return callback => this._suiteBuilder([...callbacks, { callback, args: [] }]);
         if (this._suiteModifiers.has(prop))
           return (...args) => this._suiteBuilder([...callbacks, { callback: this._suiteModifiers.get(prop), args }]);
         if (this._suiteAttributes.has(prop))
@@ -703,6 +668,8 @@ class TestRunner extends EventEmitter {
         this._tests.push(test._clone());
     }, {
       get: (obj, prop) => {
+        if (prop === 'setup')
+          return callback => this._testBuilder([...callbacks, { callback, args: [] }]);
         if (this._testModifiers.has(prop))
           return (...args) => this._testBuilder([...callbacks, { callback: this._testModifiers.get(prop), args }]);
         if (this._testAttributes.has(prop))
@@ -728,18 +695,8 @@ class TestRunner extends EventEmitter {
     this._suiteAttributes.set(name, callback);
   }
 
-  loadTests(module, ...args) {
-    if (typeof module.describe === 'function')
-      this.describe('', module.describe, ...args);
-    if (typeof module.fdescribe === 'function')
-      this.describe.focus('', module.fdescribe, ...args);
-    if (typeof module.xdescribe === 'function')
-      this.describe.skip(true)('', module.xdescribe, ...args);
-  }
-
   async run(options = {}) {
     const { totalTimeout = 0 } = options;
-    let session = this._debuggerLogBreakpointLines.size ? await setLogBreakpoints(this._debuggerLogBreakpointLines) : null;
     const runnableTests = this.runnableTests();
     this.emit(TestRunner.Events.Started, runnableTests);
 
@@ -762,8 +719,6 @@ class TestRunner extends EventEmitter {
       }
     }
     this.emit(TestRunner.Events.Finished, result);
-    if (session)
-      session.disconnect();
     return result;
   }
 
@@ -844,36 +799,6 @@ class TestRunner extends EventEmitter {
   }
 }
 
-async function setLogBreakpoints(debuggerLogBreakpoints) {
-  const session = new inspector.Session();
-  session.connect();
-  const postAsync = util.promisify(session.post.bind(session));
-  await postAsync('Debugger.enable');
-  const setBreakpointCommands = [];
-  for (const filePath of debuggerLogBreakpoints.keysArray()) {
-    const lineNumbers = debuggerLogBreakpoints.get(filePath);
-    const lines = (await readFileAsync(filePath, 'utf8')).split('\n');
-    for (const lineNumber of lineNumbers) {
-      setBreakpointCommands.push(postAsync('Debugger.setBreakpointByUrl', {
-        url: url.pathToFileURL(filePath),
-        lineNumber,
-        condition: `console.log('${String(lineNumber + 1).padStart(6, ' ')} | ' + ${JSON.stringify(lines[lineNumber])})`,
-      }).catch(e => {}));
-    };
-  }
-  await Promise.all(setBreakpointCommands);
-  return session;
-}
-
-/**
- * @param {*} value
- * @param {string=} message
- */
-function assert(value, message) {
-  if (!value)
-    throw new Error(message);
-}
-
 TestRunner.Events = {
   Started: 'started',
   Finished: 'finished',

utils/testrunner/test/testrunner.spec.js

@@ -54,7 +54,9 @@ module.exports.addTests = function({testRunner, expect}) {
     it('should run a failed focused test', async() => {
       const t = newTestRunner();
       let run = false;
-      t.fit.fail(true)('uno', () => { run = true; throw new Error('failure'); });
+      t.fit.setup(t => t.setExpectation(t.Expectations.Fail))('uno', () => {
+        run = true; throw new Error('failure');
+      });
       expect(t.tests().length).toBe(1);
       await t.run();
       expect(run).toBe(true);
@@ -374,6 +376,8 @@ module.exports.addTests = function({testRunner, expect}) {
     });
     it('should handle repeat', async() => {
       const t = newTestRunner();
+      t.testModifier('repeat', (t, count) => t.setRepeat(count));
+      t.suiteModifier('repeat', (s, count) => s.setRepeat(count));
       let suite = 0;
       let test = 0;
       let beforeAll = 0;