chore(tests): meaningful split between test.js and playwright.spec.js (#1630)

CLI handling goes to test.js.
Everything about running tests goes to playwright.spec.js.

This will help isplaywrightready and future jest integration.

test/playwright.spec.js

@@ -16,50 +16,97 @@
  */
 const fs = require('fs');
 const path = require('path');
-const os = require('os');
 const rm = require('rimraf').sync;
 const GoldenUtils = require('./golden-utils');
 const {Matchers} = require('../utils/testrunner/');
 const readline = require('readline');
+const {TestServer} = require('../utils/testserver/');
 
 const YELLOW_COLOR = '\x1b[33m';
 const RESET_COLOR = '\x1b[0m';
 
+const BROWSER_CONFIGS = [
+  {
+    name: 'Firefox',
+    events: {
+      ...require('../lib/events').Events,
+      ...require('../lib/chromium/events').Events,
+    },
+    missingCoverage: ['browserContext.setGeolocation', 'browserContext.setOffline', 'cDPSession.send', 'cDPSession.detach'],
+  },
+  {
+    name: 'WebKit',
+    events: require('../lib/events').Events,
+    missingCoverage: ['browserContext.clearPermissions', 'cDPSession.send', 'cDPSession.detach'],
+  },
+  {
+    name: 'Chromium',
+    events: require('../lib/events').Events,
+    missingCoverage: [],
+  },
+];
+const browserNames = BROWSER_CONFIGS.map(config => config.name);
+
 /**
  * @type {TestSuite}
  */
-module.exports.describe = ({testRunner, product, playwrightPath}) => {
+module.exports.addPlaywrightTests = ({testRunner, platform, products, playwrightPath, headless, slowMo, dumpProtocolOnFailure, coverage}) => {
   const {describe, xdescribe, fdescribe} = testRunner;
-  const {it, fit, xit, dit} = testRunner;
   const {beforeAll, beforeEach, afterAll, afterEach} = testRunner;
 
+  const MAC = platform === 'darwin';
+  const LINUX = platform === 'linux';
+  const WIN = platform === 'win32';
+  const playwright = require(playwrightPath);
+
+  beforeAll(async state => {
+    const assetsPath = path.join(__dirname, 'assets');
+    const cachedPath = path.join(__dirname, 'assets', 'cached');
+
+    const port = 8907 + state.parallelIndex * 3;
+    state.server = await TestServer.create(assetsPath, port);
+    state.server.enableHTTPCache(cachedPath);
+    state.server.PORT = port;
+    state.server.PREFIX = `http://localhost:${port}`;
+    state.server.CROSS_PROCESS_PREFIX = `http://127.0.0.1:${port}`;
+    state.server.EMPTY_PAGE = `http://localhost:${port}/empty.html`;
+
+    const httpsPort = port + 1;
+    state.httpsServer = await TestServer.createHTTPS(assetsPath, httpsPort);
+    state.httpsServer.enableHTTPCache(cachedPath);
+    state.httpsServer.PORT = httpsPort;
+    state.httpsServer.PREFIX = `https://localhost:${httpsPort}`;
+    state.httpsServer.CROSS_PROCESS_PREFIX = `https://127.0.0.1:${httpsPort}`;
+    state.httpsServer.EMPTY_PAGE = `https://localhost:${httpsPort}/empty.html`;
+
+    const sourcePort = port + 2;
+    state.sourceServer = await TestServer.create(path.join(__dirname, '..'), sourcePort);
+    state.sourceServer.PORT = sourcePort;
+    state.sourceServer.PREFIX = `http://localhost:${sourcePort}`;
+  });
+
+  afterAll(async({server, sourceServer, httpsServer}) => {
+    await Promise.all([
+      server.stop(),
+      httpsServer.stop(),
+      sourceServer.stop(),
+    ]);
+  });
+
+  beforeEach(async({server, httpsServer}) => {
+    server.reset();
+    httpsServer.reset();
+  });
+
+  for (const productInfo of products) {
+    const product = productInfo.product;
+    const browserType = playwright[product.toLowerCase()];
     const CHROMIUM = product === 'Chromium';
     const FFOX = product === 'Firefox';
     const WEBKIT = product === 'WebKit';
-  const MAC = os.platform() === 'darwin';
-  const LINUX = os.platform() === 'linux';
-  const WIN = os.platform() === 'win32';
-
-  const playwright = require(playwrightPath);
-  const browserType = playwright[product.toLowerCase()];
-
-  const headless = !!valueFromEnv('HEADLESS', true);
-  const slowMo = valueFromEnv('SLOW_MO', 0);
-  const dumpProtocolOnFailure = valueFromEnv('DEBUGP', false);
-
-  function valueFromEnv(name, defaultValue) {
-    if (!(name in process.env))
-      return defaultValue;
-    return JSON.parse(process.env[name]);
-  }
-  const executablePath = {
-    'Chromium': process.env.CRPATH,
-    'Firefox': process.env.FFPATH,
-    'WebKit': process.env.WKPATH,
-  }[product];
     const defaultBrowserOptions = {
       handleSIGINT: false,
-    executablePath,
+      executablePath: productInfo.executablePath,
       slowMo,
       headless,
     };
@@ -109,6 +156,7 @@ module.exports.describe = ({testRunner, product, playwrightPath}) => {
         xdescribe('', module.xdescribe, testOptions);
     }
 
+    describe(product, () => {
       describe('', function() {
         beforeAll(async state => {
           state.browser = await browserType.launch(defaultBrowserOptions);
@@ -225,4 +273,18 @@ module.exports.describe = ({testRunner, product, playwrightPath}) => {
         loadTests('./chromium/oopif.spec.js');
         loadTests('./chromium/tracing.spec.js');
       });
+
+      if (coverage) {
+        const browserConfig = BROWSER_CONFIGS.find(config => config.name === product);
+        const api = require('../lib/api');
+        const filteredApi = {};
+        Object.keys(api).forEach(apiName => {
+          if (browserNames.some(browserName => apiName.startsWith(browserName)) && !apiName.startsWith(product))
+            return;
+          filteredApi[apiName] = api[apiName];
+        });
+        require('./utils').recordAPICoverage(testRunner, filteredApi, browserConfig.events, browserConfig.missingCoverage);
+      }
+    });
+  }
 };

test/test.js

@@ -14,11 +14,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-const path = require('path');
+
-const {TestServer} = require('../utils/testserver/');
 const {TestRunner, Reporter} = require('../utils/testrunner/');
 const utils = require('./utils');
-const inspector = require('inspector');
+const os = require('os');
 
 let parallel = 1;
 if (process.env.PW_PARALLEL_TESTS)
@@ -32,7 +31,7 @@ let timeout = process.env.CI ? 30 * 1000 : 10 * 1000;
 if (!isNaN(process.env.TIMEOUT))
   timeout = parseInt(process.env.TIMEOUT * 1000, 10);
 const MAJOR_NODEJS_VERSION = parseInt(process.version.substring(1).split('.')[0], 10);
-if (MAJOR_NODEJS_VERSION >= 8 && inspector.url()) {
+if (MAJOR_NODEJS_VERSION >= 8 && require('inspector').url()) {
   console.log('Detected inspector - disabling timeout to be debugger-friendly');
   timeout = 0;
 }
@@ -43,142 +42,38 @@ const testRunner = new TestRunner({
   breakOnFailure: process.argv.indexOf('--break-on-failure') !== -1,
   installCommonHelpers: false
 });
-testRunner.testModifier('skip', (t, condition) => condition && t.setSkipped(true));
+utils.setupTestRunner(testRunner);
-testRunner.suiteModifier('skip', (s, condition) => condition && s.setSkipped(true));
-testRunner.testModifier('fail', (t, condition) => condition && t.setExpectation(t.Expectations.Fail));
-testRunner.suiteModifier('fail', (s, condition) => condition && s.setExpectation(s.Expectations.Fail));
-testRunner.testModifier('slow', (t, condition) => condition && t.setTimeout(t.timeout() * 3));
-testRunner.testModifier('repeat', (t, count) => t.setRepeat(count));
-testRunner.suiteModifier('repeat', (s, count) => s.setRepeat(count));
-testRunner.testAttribute('focus', t => t.setFocused(true));
-testRunner.suiteAttribute('focus', s => s.setFocused(true));
-testRunner.testAttribute('debug', t => {
-  t.setFocused(true);
-  t.setTimeout(100000000);
-
-  let session;
-  t.before(async () => {
-    const util = require('util');
-    const fs = require('fs');
-    const url = require('url');
-    const readFileAsync = util.promisify(fs.readFile.bind(fs));
-    session = new inspector.Session();
-    session.connect();
-    const postAsync = util.promisify(session.post.bind(session));
-    await postAsync('Debugger.enable');
-    const setBreakpointCommands = [];
-    const N = t.body().toString().split('\n').length;
-    const location = t.location();
-    const lines = (await readFileAsync(location.filePath, 'utf8')).split('\n');
-    for (let line = 0; line < N; ++line) {
-      const lineNumber = line + location.lineNumber;
-      setBreakpointCommands.push(postAsync('Debugger.setBreakpointByUrl', {
-        url: url.pathToFileURL(location.filePath),
-        lineNumber,
-        condition: `console.log('${String(lineNumber + 1).padStart(6, ' ')} | ' + ${JSON.stringify(lines[lineNumber])})`,
-      }).catch(e => {}));
-    }
-    await Promise.all(setBreakpointCommands);
-  });
-
-  t.after(async () => {
-    session.disconnect();
-  });
-});
-testRunner.fdescribe = testRunner.describe.focus;
-testRunner.xdescribe = testRunner.describe.skip(true);
-testRunner.fit = testRunner.it.focus;
-testRunner.xit = testRunner.it.skip(true);
-testRunner.dit = testRunner.it.debug;
-
-const {describe, fdescribe, beforeAll, afterAll, beforeEach, afterEach} = testRunner;
 
 console.log('Testing on Node', process.version);
 
-beforeAll(async state => {
+const names = ['Chromium', 'Firefox', 'WebKit'].filter(name => {
-  const assetsPath = path.join(__dirname, 'assets');
+  return process.env.BROWSER === name.toLowerCase() || process.env.BROWSER === 'all';
-  const cachedPath = path.join(__dirname, 'assets', 'cached');
+});
-
+const products = names.map(name => {
-  const port = 8907 + state.parallelIndex * 3;
+  const executablePath = {
-  state.server = await TestServer.create(assetsPath, port);
+    'Chromium': process.env.CRPATH,
-  state.server.enableHTTPCache(cachedPath);
+    'Firefox': process.env.FFPATH,
-  state.server.PORT = port;
+    'WebKit': process.env.WKPATH,
-  state.server.PREFIX = `http://localhost:${port}`;
+  }[name];
-  state.server.CROSS_PROCESS_PREFIX = `http://127.0.0.1:${port}`;
+  return { product: name, executablePath };
-  state.server.EMPTY_PAGE = `http://localhost:${port}/empty.html`;
-
-  const httpsPort = port + 1;
-  state.httpsServer = await TestServer.createHTTPS(assetsPath, httpsPort);
-  state.httpsServer.enableHTTPCache(cachedPath);
-  state.httpsServer.PORT = httpsPort;
-  state.httpsServer.PREFIX = `https://localhost:${httpsPort}`;
-  state.httpsServer.CROSS_PROCESS_PREFIX = `https://127.0.0.1:${httpsPort}`;
-  state.httpsServer.EMPTY_PAGE = `https://localhost:${httpsPort}/empty.html`;
-
-  const sourcePort = port + 2;
-  state.sourceServer = await TestServer.create(path.join(__dirname, '..'), sourcePort);
-  state.sourceServer.PORT = sourcePort;
-  state.sourceServer.PREFIX = `http://localhost:${sourcePort}`;
 });
 
-afterAll(async({server, sourceServer, httpsServer}) => {
+function valueFromEnv(name, defaultValue) {
-  await Promise.all([
+  if (!(name in process.env))
-    server.stop(),
+    return defaultValue;
-    httpsServer.stop(),
+  return JSON.parse(process.env[name]);
-    sourceServer.stop(),
+}
-  ]);
-});
 
-beforeEach(async({server, httpsServer}) => {
+require('./playwright.spec.js').addPlaywrightTests({
-  server.reset();
-  httpsServer.reset();
-});
-
-const BROWSER_CONFIGS = [
-  {
-    name: 'Firefox',
-    events: {
-      ...require('../lib/events').Events,
-      ...require('../lib/chromium/events').Events,
-    },
-    missingCoverage: ['browserContext.setGeolocation', 'browserContext.setOffline', 'cDPSession.send', 'cDPSession.detach'],
-  },
-  {
-    name: 'WebKit',
-    events: require('../lib/events').Events,
-    missingCoverage: ['browserContext.clearPermissions', 'cDPSession.send', 'cDPSession.detach'],
-  },
-  {
-    name: 'Chromium',
-    events: require('../lib/events').Events,
-    missingCoverage: [],
-  },
-];
-
-const browserNames = BROWSER_CONFIGS.map(config => config.name);
-
-for (const browserConfig of BROWSER_CONFIGS) {
-  if (process.env.BROWSER !== browserConfig.name.toLowerCase() && process.env.BROWSER !== 'all')
-    continue;
-  const product = browserConfig.name;
-  describe(product, () => {
-    testRunner.describe('', require('./playwright.spec.js').describe, {
-      product,
   playwrightPath: utils.projectRoot(),
+  products,
+  platform: os.platform(),
   testRunner,
+  headless: !!valueFromEnv('HEADLESS', true),
+  slowMo: valueFromEnv('SLOW_MO', 0),
+  dumpProtocolOnFailure: valueFromEnv('DEBUGP', false),
+  coverage: process.env.COVERAGE,
 });
-    if (process.env.COVERAGE) {
-      const api = require('../lib/api');
-      const filteredApi = {};
-      Object.keys(api).forEach(apiName => {
-        if (browserNames.some(browserName => apiName.startsWith(browserName)) && !apiName.startsWith(product))
-          return;
-        filteredApi[apiName] = api[apiName];
-      });
-      utils.recordAPICoverage(testRunner, filteredApi, browserConfig.events, browserConfig.missingCoverage);
-    }
-  });
-}
 
 const filterArgIndex = process.argv.indexOf('--filter');
 if (filterArgIndex !== -1) {

test/utils.js

@@ -20,6 +20,7 @@ const path = require('path');
 const util = require('util');
 const os = require('os');
 const removeFolder = require('rimraf');
+const url = require('url');
 
 const {FlakinessDashboard} = require('../utils/flakiness-dashboard');
 const PROJECT_ROOT = fs.existsSync(path.join(__dirname, '..', 'package.json')) ? path.join(__dirname, '..') : path.join(__dirname, '..', '..');
@@ -288,5 +289,52 @@ const utils = module.exports = {
 
   removeUserDataDir: async function(dir) {
     await removeFolderAsync(dir).catch(e => {});
+  },
+
+  setupTestRunner: function(testRunner) {
+    testRunner.testModifier('skip', (t, condition) => condition && t.setSkipped(true));
+    testRunner.suiteModifier('skip', (s, condition) => condition && s.setSkipped(true));
+    testRunner.testModifier('fail', (t, condition) => condition && t.setExpectation(t.Expectations.Fail));
+    testRunner.suiteModifier('fail', (s, condition) => condition && s.setExpectation(s.Expectations.Fail));
+    testRunner.testModifier('slow', (t, condition) => condition && t.setTimeout(t.timeout() * 3));
+    testRunner.testModifier('repeat', (t, count) => t.setRepeat(count));
+    testRunner.suiteModifier('repeat', (s, count) => s.setRepeat(count));
+    testRunner.testAttribute('focus', t => t.setFocused(true));
+    testRunner.suiteAttribute('focus', s => s.setFocused(true));
+    testRunner.testAttribute('debug', t => {
+      t.setFocused(true);
+      t.setTimeout(100000000);
+
+      let session;
+      t.before(async () => {
+        const readFileAsync = util.promisify(fs.readFile.bind(fs));
+        session = new require('inspector').Session();
+        session.connect();
+        const postAsync = util.promisify(session.post.bind(session));
+        await postAsync('Debugger.enable');
+        const setBreakpointCommands = [];
+        const N = t.body().toString().split('\n').length;
+        const location = t.location();
+        const lines = (await readFileAsync(location.filePath, 'utf8')).split('\n');
+        for (let line = 0; line < N; ++line) {
+          const lineNumber = line + location.lineNumber;
+          setBreakpointCommands.push(postAsync('Debugger.setBreakpointByUrl', {
+            url: url.pathToFileURL(location.filePath),
+            lineNumber,
+            condition: `console.log('${String(lineNumber + 1).padStart(6, ' ')} | ' + ${JSON.stringify(lines[lineNumber])})`,
+          }).catch(e => {}));
         }
+        await Promise.all(setBreakpointCommands);
+      });
+
+      t.after(async () => {
+        session.disconnect();
+      });
+    });
+    testRunner.fdescribe = testRunner.describe.focus;
+    testRunner.xdescribe = testRunner.describe.skip(true);
+    testRunner.fit = testRunner.it.focus;
+    testRunner.xit = testRunner.it.skip(true);
+    testRunner.dit = testRunner.it.debug;
+  },
 };