test: restore nojest runner (#3359)

package.json

@@ -15,13 +15,14 @@
     "ctestd": "cross-env BROWSER=chromium jest --reporters=./jest/dot.js --colors",
     "ftestd": "cross-env BROWSER=firefox jest --reporters=./jest/dot.js --colors",
     "wtestd": "cross-env BROWSER=webkit jest --reporters=./jest/dot.js --colors",
+    "nojest": "cross-env BROWSER=chromium node --unhandled-rejections=strict ./test/nojest/nojest.js",
     "test": "npm run ctest && npm run ftest && npm run wtest",
     "eslint": "[ \"$CI\" = true ] && eslint --quiet -f codeframe --ext js,ts ./src || eslint --ext js,ts ./src",
     "tsc": "tsc -p .",
     "tsc-installer": "tsc -p ./src/install/tsconfig.json",
     "doc": "node utils/doclint/cli.js",
     "doc-channel": "node utils/doclint/cli.js --channel",
-    "test-infra": "node utils/doclint/check_public_api/test/test.js && node utils/doclint/preprocessor/test.js && node utils/testrunner/test/test.js",
+    "test-infra": "node utils/doclint/check_public_api/test/test.js && node utils/doclint/preprocessor/test.js",
     "lint": "npm run eslint && npm run tsc && npm run doc && npm run doc-channel && npm run check-deps && npm run generate-channels && npm run test-types && npm run test-infra",
     "debug-test": "node --inspect-brk test/test.js",
     "clean": "rimraf lib && rimraf types",

test/harness/fixturePool.js

@@ -106,6 +106,25 @@ class FixturePool {
       params[n] = this.instances.get(n).value;
     return fn(params);
   }
+
+  patchToEnableFixtures(object, name) {
+    const original = object[name];
+    object[name] = fn => {
+      return original(async () => {
+        return await this.resolveParametersAndRun(fn);
+      });
+    }
+  }
+
+  wrapTestCallback(callback) {
+    return async() => {
+      try {
+        return await this.resolveParametersAndRun(callback);
+      } finally {
+        await this.teardownScope('test');
+      }
+    };
+  }
 }
 
 function fixtureParameterNames(fn) {
@@ -117,8 +136,16 @@ function fixtureParameterNames(fn) {
   return signature.split(',').map(t => t.trim());
 }
 
-function registerFixture(name, scope, fn) {
+function innerRegisterFixture(name, scope, fn) {
   registrations.set(name, { scope, fn });
-}
+};
 
-module.exports = { FixturePool, registerFixture };
+function registerFixture(name, fn) {
+  innerRegisterFixture(name, 'test', fn);
+};
+
+function registerWorkerFixture (name, fn) {
+  innerRegisterFixture(name, 'worker', fn);
+};
+
+module.exports = { FixturePool, registerFixture, registerWorkerFixture };

test/harness/fixtures.js

@@ -18,20 +18,24 @@ const path = require('path');
 const childProcess = require('child_process');
 const playwrightImpl = require('../../index');
 
-const { TestServer } = require('../../utils/testserver/');
+const { TestServer } = require('../../utils/testserver');
 const { Connection } = require('../../lib/rpc/client/connection');
 const { Transport } = require('../../lib/rpc/transport');
 const { setupInProcess } = require('../../lib/rpc/inprocess');
 const { setUnderTest } = require('../../lib/helper');
+const { valueFromEnv } = require('./utils');
+const { registerFixture, registerWorkerFixture } = require('./fixturePool');
+
 setUnderTest();
 
 const browserName = process.env.BROWSER || 'chromium';
 
 module.exports = function registerFixtures(global) {
-  global.registerWorkerFixture('parallelIndex', async ({}, test) => {
+  registerWorkerFixture('parallelIndex', async ({}, test) => {
     await test(process.env.JEST_WORKER_ID - 1);
   });
-  global.registerWorkerFixture('http_server', async ({parallelIndex}, test) => {
+
+  registerWorkerFixture('http_server', async ({parallelIndex}, test) => {
     const assetsPath = path.join(__dirname, '..', 'assets');
     const cachedPath = path.join(__dirname, '..', 'assets', 'cached');
 
@@ -59,7 +63,7 @@ module.exports = function registerFixtures(global) {
     ]);
   });
 
-  global.registerWorkerFixture('defaultBrowserOptions', async({}, test) => {
+  registerWorkerFixture('defaultBrowserOptions', async({}, test) => {
     let executablePath = undefined;
     if (browserName === 'chromium' && process.env.CRPATH)
       executablePath = process.env.CRPATH;
@@ -77,7 +81,7 @@ module.exports = function registerFixtures(global) {
     });
   });
 
-  global.registerWorkerFixture('playwright', async({}, test) => {
+  registerWorkerFixture('playwright', async({}, test) => {
     if (process.env.PWCHANNEL === 'wire') {
       const connection = new Connection();
       const spawnedProcess = childProcess.fork(path.join(__dirname, '..', '..', 'lib', 'rpc', 'server'), [], {
@@ -108,15 +112,15 @@ module.exports = function registerFixtures(global) {
     }
   });
 
-  global.registerFixture('toImpl', async ({playwright}, test) => {
+  registerFixture('toImpl', async ({playwright}, test) => {
     await test(playwright._toImpl);
   });
 
-  global.registerWorkerFixture('browserType', async ({playwright}, test) => {
+  registerWorkerFixture('browserType', async ({playwright}, test) => {
     await test(playwright[process.env.BROWSER || 'chromium']);
   });
 
-  global.registerWorkerFixture('browser', async ({browserType, defaultBrowserOptions}, test) => {
+  registerWorkerFixture('browser', async ({browserType, defaultBrowserOptions}, test) => {
     const browser = await browserType.launch(defaultBrowserOptions);
     try {
       await test(browser);
@@ -129,7 +133,7 @@ module.exports = function registerFixtures(global) {
     }
   });
 
-  global.registerFixture('context', async ({browser}, test) => {
+  registerFixture('context', async ({browser}, test) => {
     const context = await browser.newContext();
     try {
       await test(context);
@@ -138,24 +142,18 @@ module.exports = function registerFixtures(global) {
     }
   });
 
-  global.registerFixture('page', async ({context}, test) => {
+  registerFixture('page', async ({context}, test) => {
     const page = await context.newPage();
     await test(page);
   });
 
-  global.registerFixture('server', async ({http_server}, test) => {
+  registerFixture('server', async ({http_server}, test) => {
     http_server.server.reset();
     await test(http_server.server);
   });
 
-  global.registerFixture('httpsServer', async ({http_server}, test) => {
+  registerFixture('httpsServer', async ({http_server}, test) => {
     http_server.httpsServer.reset();
     await test(http_server.httpsServer);
   });
 }
-
-function valueFromEnv(name, defaultValue) {
-  if (!(name in process.env))
-    return defaultValue;
-  return JSON.parse(process.env[name]);
-}

test/harness/testOptions.js

@@ -0,0 +1,38 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const os = require('os');
+const path = require('path');
+const { valueFromEnv } = require('./utils');
+
+const platform = process.env.REPORT_ONLY_PLATFORM || os.platform();
+const browserName = process.env.BROWSER || 'chromium';
+
+const testOptions = {};
+testOptions.MAC = platform === 'darwin';
+testOptions.LINUX = platform === 'linux';
+testOptions.WIN = platform === 'win32';
+testOptions.CHROMIUM = browserName === 'chromium';
+testOptions.FFOX = browserName === 'firefox';
+testOptions.WEBKIT = browserName === 'webkit';
+testOptions.USES_HOOKS = process.env.PWCHANNEL === 'wire';
+testOptions.CHANNEL = !!process.env.PWCHANNEL;
+testOptions.HEADLESS = !!valueFromEnv('HEADLESS', true);
+testOptions.ASSETS_DIR = path.join(__dirname, '..', 'assets');
+testOptions.GOLDEN_DIR = path.join(__dirname, '..', 'golden-' + browserName);
+testOptions.OUTPUT_DIR = path.join(__dirname, '..', 'output-' + browserName);
+
+module.exports = testOptions;

test/harness/utils.js

@@ -0,0 +1,23 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+function valueFromEnv(name, defaultValue) {
+  if (!(name in process.env))
+    return defaultValue;
+  return JSON.parse(process.env[name]);
+}
+
+module.exports = { valueFromEnv };

test/jest/playwrightEnvironment.js

@@ -14,45 +14,29 @@
  * limitations under the License.
  */
 
-const registerFixtures = require('./fixtures');
+const { FixturePool, registerFixture, registerWorkerFixture } = require('../harness/fixturePool');
-const { FixturePool, registerFixture } = require('./fixturePool');
+const registerFixtures = require('../harness/fixtures');
 const os = require('os');
 const path = require('path');
 const fs = require('fs');
 const debug = require('debug');
 const util = require('util');
-const platform = process.env.REPORT_ONLY_PLATFORM || os.platform();
 const GoldenUtils = require('../../utils/testrunner/GoldenUtils');
 const {installCoverageHooks} = require('./coverage');
-const browserName = process.env.BROWSER || 'chromium';
 const reportOnly = !!process.env.REPORT_ONLY_PLATFORM;
 const { ModuleMocker } = require('jest-mock');
 
-const testOptions = {};
+Error.stackTraceLimit = 15;
-testOptions.MAC = platform === 'darwin';
+global.testOptions = require('../harness/testOptions');
-testOptions.LINUX = platform === 'linux';
+global.registerFixture = registerFixture;
-testOptions.WIN = platform === 'win32';
+global.registerWorkerFixture = registerWorkerFixture;
-testOptions.CHROMIUM = browserName === 'chromium';
-testOptions.FFOX = browserName === 'firefox';
-testOptions.WEBKIT = browserName === 'webkit';
-testOptions.USES_HOOKS = process.env.PWCHANNEL === 'wire';
-testOptions.CHANNEL = !!process.env.PWCHANNEL;
-testOptions.HEADLESS = !!valueFromEnv('HEADLESS', true);
-testOptions.ASSETS_DIR = path.join(__dirname, '..', 'assets');
-testOptions.GOLDEN_DIR = path.join(__dirname, '..', 'golden-' + browserName);
-testOptions.OUTPUT_DIR = path.join(__dirname, '..', 'output-' + browserName);
-global.testOptions = testOptions;
-
-global.registerFixture = (name, fn) => {
-  registerFixture(name, 'test', fn);
-};
-
-global.registerWorkerFixture = (name, fn) => {
-  registerFixture(name, 'worker', fn);
-};
-
 registerFixtures(global);
 
+const browserName = process.env.BROWSER || 'chromium';
+
+const goldenPath = path.join(__dirname, '..', 'golden-' + browserName);
+const outputPath = path.join(__dirname, '..', 'output-' + browserName);
+
 let currentFixturePool = null;
 
 process.on('SIGINT', async () => {
@@ -89,7 +73,7 @@ class PlaywrightEnvironment {
     this.uninstallCoverage();
     const testRoot = path.join(__dirname, '..');
     const relativeTestPath = path.relative(testRoot, this.testPath);
-    const coveragePath = path.join(this.global.testOptions.OUTPUT_DIR, 'coverage', relativeTestPath + '.json');
+    const coveragePath = path.join(outputPath, 'coverage', relativeTestPath + '.json');
     const coverageJSON = [...this.coverage.keys()].filter(key => this.coverage.get(key));
     await fs.promises.mkdir(path.dirname(coveragePath), { recursive: true });
     await fs.promises.writeFile(coveragePath, JSON.stringify(coverageJSON, undefined, 2), 'utf8');
@@ -101,19 +85,10 @@ class PlaywrightEnvironment {
     return script.runInThisContext();
   }
 
-  patchToEnableFixtures(object, name) {
-    const original = object[name];
-    object[name] = fn => {
-      return original(async () => {
-        return await this.fixturePool.resolveParametersAndRun(fn);
-      });
-    }
-  }
-
   async handleTestEvent(event, state) {
     if (event.name === 'setup') {
-      this.patchToEnableFixtures(this.global, 'beforeEach');
+      this.fixturePool.patchToEnableFixtures(this.global, 'beforeEach');
-      this.patchToEnableFixtures(this.global, 'afterEach');
+      this.fixturePool.patchToEnableFixtures(this.global, 'afterEach');
 
       const describeSkip = this.global.describe.skip;
       this.global.describe.skip = (...args) => {
@@ -152,7 +127,7 @@ class PlaywrightEnvironment {
       function toBeGolden(received, goldenName) {
         const {snapshotState} = this;
         const updateSnapshot = snapshotState._updateSnapshot;
-        const expectedPath = path.join(testOptions.GOLDEN_DIR, goldenName);
+        const expectedPath = path.join(goldenPath, goldenName);
         const fileExists = fs.existsSync(expectedPath);
         if (updateSnapshot === 'all' || (updateSnapshot === 'new' && !fileExists)) {
           fs.writeFileSync(expectedPath, received);
@@ -166,8 +141,8 @@ class PlaywrightEnvironment {
         };
 
         const {pass, message} =  GoldenUtils.compare(received, {
-          goldenPath: testOptions.GOLDEN_DIR,
+          goldenPath,
-          outputPath: testOptions.OUTPUT_DIR,
+          outputPath,
           goldenName
         });
         if (pass)
@@ -210,12 +185,6 @@ class PlaywrightEnvironment {
   }
 }
 
-function valueFromEnv(name, defaultValue) {
-  if (!(name in process.env))
-    return defaultValue;
-  return JSON.parse(process.env[name]);
-}
-
 function testOrSuiteName(o) {
   if (o.name === 'ROOT_DESCRIBE_BLOCK')
     return '';

test/nojest/nojest.js

@@ -16,14 +16,24 @@
  */
 
 const fs = require('fs');
-const utils = require('./utils');
 const path = require('path');
+const os = require('os');
 const pirates = require('pirates');
 const babel = require('@babel/core');
-const TestRunner = require('../utils/testrunner/');
+const TestRunner = require('../../utils/testrunner');
-const { PlaywrightEnvironment, BrowserTypeEnvironment, BrowserEnvironment, PageEnvironment} = require('./environments.js');
+const { FixturePool, registerFixture, registerWorkerFixture } = require('../harness/fixturePool');
+const registerFixtures = require('../harness/fixtures');
+const testOptions = require('../harness/testOptions');
 
 Error.stackTraceLimit = 15;
+global.testOptions = require('../harness/testOptions');
+global.registerFixture = registerFixture;
+global.registerWorkerFixture = registerWorkerFixture;
+registerFixtures(global);
+process.env.JEST_WORKER_ID = 1;
+const browserName = process.env.BROWSER || 'chromium';
+const goldenPath = path.join(__dirname, '..', 'golden-' + browserName);
+const outputPath = path.join(__dirname, '..', 'output-' + browserName);
 
 function getCLIArgument(argName) {
   for (let i = 0; i < process.argv.length; ++i) {
@@ -51,14 +61,11 @@ function collect(browserNames) {
   let timeout = process.env.CI ? 30 * 1000 : 10 * 1000;
   if (!isNaN(process.env.TIMEOUT))
     timeout = parseInt(process.env.TIMEOUT * 1000, 10);
-  const MAJOR_NODEJS_VERSION = parseInt(process.version.substring(1).split('.')[0], 10);
+  if (require('inspector').url()) {
-  if (MAJOR_NODEJS_VERSION >= 8 && require('inspector').url()) {
     console.log('Detected inspector - disabling timeout to be debugger-friendly');
     timeout = 0;
   }
 
-  const config = require('./test.config');
-
   const testRunner = new TestRunner({
     timeout,
     totalTimeout: process.env.CI ? 30 * 60 * 1000 * browserNames.length : 0, // 30 minutes per browser on CI
@@ -69,93 +76,52 @@ function collect(browserNames) {
     showSlowTests: process.env.CI ? 5 : 0,
     showMarkedAsFailingTests: 10,
     lineBreak: parseInt(getCLIArgument('--line-break') || 0, 10),
+    outputPath,
+    goldenPath
   });
-  if (config.setupTestRunner)
-    config.setupTestRunner(testRunner);
 
   for (const [key, value] of Object.entries(testRunner.api()))
     global[key] = value;
 
-  // TODO: this should be a preinstalled playwright by default.
+  const collector = testRunner.collector();
-  const playwrightPath = config.playwrightPath;
+  collector.addTestModifier('skip', (t, condition) => condition && t.setSkipped(true));
-  const playwright = require('..');
+  collector.addSuiteModifier('skip', (s, condition) => condition && s.setSkipped(true));
-  const { setUnderTest } = require(require('path').join(playwrightPath, 'lib/helper.js'));
+  collector.addTestModifier('fail', (t, condition) => condition && t.setExpectation(t.Expectations.Fail));
-  setUnderTest();
+  collector.addSuiteModifier('fail', (s, condition) => condition && s.setExpectation(s.Expectations.Fail));
+  collector.addTestModifier('slow', t => t.setTimeout(t.timeout() * 3));
+  collector.addTestAttribute('debug', t => TraceTestEnvironment.enableForTest(t));
+  testRunner.api().fdescribe = testRunner.api().describe.only;
+  testRunner.api().xdescribe = testRunner.api().describe.skip(true);
+  testRunner.api().fit = testRunner.api().it.only;
+  testRunner.api().xit = testRunner.api().it.skip(true);
+  testRunner.api().dit = testRunner.api().it.only.debug;
 
-  const playwrightEnvironment = new PlaywrightEnvironment(playwright);
+  const fixturePool = new FixturePool();
-  testRunner.collector().useEnvironment(playwrightEnvironment);
+  fixturePool.patchToEnableFixtures(global, 'beforeEach');
-  for (const e of config.globalEnvironments || [])
+  fixturePool.patchToEnableFixtures(global, 'afterEach');
-    testRunner.collector().useEnvironment(e);
+  collector.addTestCallbackWrapper(callback => fixturePool.wrapTestCallback(callback));
 
-  // TODO(rpc): do not use global playwright and browserType, rely solely on environments.
+  describe('', () => {
-  global.playwright = playwright;
+    for (const name of fs.readdirSync('test')) {
-
+      const file = path.join(process.cwd(), 'test', name);
-  for (const browserName of browserNames) {
+      if (!name.includes('.spec.'))
-    const browserType = playwright[browserName];
+        continue;
-    const browserTypeEnvironment = new BrowserTypeEnvironment(browserName);
+      const revert = pirates.addHook((code, filename) => {
-
+        const result = babel.transformFileSync(filename, {
-    // TODO: maybe launch options per browser?
+          presets: [
-    const launchOptions = {
+            ['@babel/preset-env', {targets: {node: 'current'}}],
-      ...(config.launchOptions || {}),
+            '@babel/preset-typescript']
-      handleSIGINT: false,
-    };
-    if (launchOptions.executablePath)
-      launchOptions.executablePath = launchOptions.executablePath[browserName];
-    if (launchOptions.executablePath) {
-      const YELLOW_COLOR = '\x1b[33m';
-      const RESET_COLOR = '\x1b[0m';
-      console.warn(`${YELLOW_COLOR}WARN: running ${browserName} tests with ${launchOptions.executablePath}${RESET_COLOR}`);
-      browserType._executablePath = launchOptions.executablePath;
-      delete launchOptions.executablePath;
-    } else {
-      if (!fs.existsSync(browserType.executablePath()))
-        throw new Error(`Browser is not downloaded. Run 'npm install' and try to re-run tests`);
-    }
-
-    const browserEnvironment = new BrowserEnvironment(launchOptions, config.dumpLogOnFailure);
-    const pageEnvironment = new PageEnvironment();
-
-    const suiteName = { 'chromium': 'Chromium', 'firefox': 'Firefox', 'webkit': 'WebKit' }[browserName];
-    describe(suiteName, () => {
-      // In addition to state, expose these two on global so that describes can access them.
-      global.browserType = browserType;
-      global.HEADLESS = !!launchOptions.headless;
-
-      testRunner.collector().useEnvironment(browserTypeEnvironment);
-
-      for (const spec of config.specs || []) {
-        const skip = spec.browsers && !spec.browsers.includes(browserName);
-        (skip ? xdescribe : describe)(spec.title || '', () => {
-          for (const e of spec.environments || ['page']) {
-            if (e === 'page') {
-              testRunner.collector().useEnvironment(browserEnvironment);
-              testRunner.collector().useEnvironment(pageEnvironment);
-            } else {
-              testRunner.collector().useEnvironment(e);
-            }
-          }
-          for (const file of spec.files || []) {
-            const revert = pirates.addHook((code, filename) => {
-              const result = babel.transformFileSync(filename, {
-                presets: [
-                  ['@babel/preset-env', {targets: {node: 'current'}}],
-                  '@babel/preset-typescript']
-              });
-              return result.code;
-            }, {
-              exts: ['.ts']
-            });
-            require(file);
-            revert();
-            delete require.cache[require.resolve(file)];
-          }
         });
-      }
+        return result.code;
+      }, {
+        exts: ['.ts']
+      });
+      require(file);
+      revert();
+      delete require.cache[require.resolve(file)];
+    }
+  });
 
-      delete global.HEADLESS;
-      delete global.browserType;
-    });
-  }
   for (const [key, value] of Object.entries(testRunner.api())) {
     // expect is used when running tests, while the rest of api is not.
     if (key !== 'expect')

test/test.config.js

@@ -1,83 +0,0 @@
-/**
- * Copyright 2019 Google Inc. All rights reserved.
- * Modifications copyright (c) Microsoft Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-const path = require('path');
-const utils = require('./utils');
-const {DefaultBrowserOptionsEnvironment, ServerEnvironment, GoldenEnvironment, TraceTestEnvironment} = require('./environments.js');
-
-const playwrightPath = path.join(__dirname, '..');
-
-const dumpLogOnFailure = valueFromEnv('DEBUGP', false);
-const defaultBrowserOptionsEnvironment = new DefaultBrowserOptionsEnvironment({
-  handleSIGINT: false,
-  slowMo: valueFromEnv('SLOW_MO', 0),
-  headless: !!valueFromEnv('HEADLESS', true),
-}, dumpLogOnFailure, playwrightPath);
-
-const serverEnvironment = new ServerEnvironment();
-const customEnvironment = new GoldenEnvironment();
-
-function valueFromEnv(name, defaultValue) {
-  if (!(name in process.env))
-    return defaultValue;
-  return JSON.parse(process.env[name]);
-}
-
-function setupTestRunner(testRunner) {
-  const collector = testRunner.collector();
-  collector.addTestModifier('skip', (t, condition) => condition && t.setSkipped(true));
-  collector.addSuiteModifier('skip', (s, condition) => condition && s.setSkipped(true));
-  collector.addTestModifier('fail', (t, condition) => condition && t.setExpectation(t.Expectations.Fail));
-  collector.addSuiteModifier('fail', (s, condition) => condition && s.setExpectation(s.Expectations.Fail));
-  collector.addTestModifier('slow', t => t.setTimeout(t.timeout() * 3));
-  collector.addTestAttribute('debug', t => TraceTestEnvironment.enableForTest(t));
-  testRunner.api().fdescribe = testRunner.api().describe.only;
-  testRunner.api().xdescribe = testRunner.api().describe.skip(true);
-  testRunner.api().fit = testRunner.api().it.only;
-  testRunner.api().xit = testRunner.api().it.skip(true);
-  testRunner.api().dit = testRunner.api().it.only.debug;
-}
-
-module.exports = {
-  playwrightPath,
-  dumpLogOnFailure: valueFromEnv('DEBUGP', false),
-  launchOptions: {
-    executablePath: {
-      chromium: process.env.CRPATH,
-      firefox: process.env.FFPATH,
-      webkit: process.env.WKPATH,
-    },
-    slowMo: valueFromEnv('SLOW_MO', 0),
-    headless: !!valueFromEnv('HEADLESS', true),
-  },
-
-  globalEnvironments: [defaultBrowserOptionsEnvironment, serverEnvironment],
-  setupTestRunner,
-
-  specs: [
-    {
-      files: [
-      ],
-      environments: [customEnvironment],
-    },
-    {
-      files: [
-      ],
-      environments: [],
-    },
-  ],
-};

test/utils.js

@@ -189,27 +189,6 @@ const utils = module.exports = {
     await utils.removeFolderAsync(dir).catch(e => {});
   },
 
-  testOptions(browserType) {
-    const GOLDEN_DIR = path.join(__dirname, 'golden-' + browserType.name());
-    const OUTPUT_DIR = path.join(__dirname, 'output-' + browserType.name());
-    const ASSETS_DIR = path.join(__dirname, 'assets');
-    return {
-      FFOX: browserType.name() === 'firefox',
-      WEBKIT: browserType.name() === 'webkit',
-      CHROMIUM: browserType.name() === 'chromium',
-      MAC: platform === 'darwin',
-      LINUX: platform === 'linux',
-      WIN: platform === 'win32',
-      browserType,
-      GOLDEN_DIR,
-      OUTPUT_DIR,
-      ASSETS_DIR,
-      USES_HOOKS: process.env.PWCHANNEL === 'wire',
-      CHANNEL: !!process.env.PWCHANNEL,
-      HEADLESS: !!valueFromEnv('HEADLESS', true),
-    };
-  },
-
   setPlatform(p) {
     // To support isplaywrightready.
     platform = p;
@@ -263,9 +242,3 @@ const utils = module.exports = {
     }
   },
 };
-
-function valueFromEnv(name, defaultValue) {
-  if (!(name in process.env))
-    return defaultValue;
-  return JSON.parse(process.env[name]);
-}

utils/doclint/check_public_api/test/test.js

@@ -22,7 +22,10 @@ const mdBuilder = require('../MDBuilder');
 const jsBuilder = require('../JSBuilder');
 
 const TestRunner  = require('../../../testrunner/');
-const runner = new TestRunner();
+const runner = new TestRunner({
+  goldenPath: __dirname,
+  outputPath: __dirname
+});
 
 const {describe, xdescribe, fdescribe} = runner.api();
 const {it, fit, xit} = runner.api();
@@ -66,14 +69,14 @@ async function testLint(state, testRun) {
   const jsSources = await Source.readdir(dirPath, '.js');
   const messages = await checkPublicAPI(page, mdSources, jsSources.concat(tsSources));
   const errors = messages.map(message => message.text);
-  expect(errors.join('\n')).toBeGolden({goldenPath: dirPath, outputPath: dirPath, goldenName: 'result.txt'});
+  expect(errors.join('\n')).toBeGolden(path.join(testRun.test().name(), 'result.txt'));
 }
 
 async function testMDBuilder(state, testRun) {
   const dirPath = path.join(__dirname, testRun.test().name());
   const sources = await Source.readdir(dirPath, '.md');
   const {documentation} = await mdBuilder(page, sources);
-  expect(serialize(documentation)).toBeGolden({goldenPath: dirPath, outputPath: dirPath, goldenName: 'result.txt'});
+  expect(serialize(documentation)).toBeGolden(path.join(testRun.test().name(), 'result.txt'));
 }
 
 async function testJSBuilder(state, testRun) {
@@ -81,7 +84,7 @@ async function testJSBuilder(state, testRun) {
   const jsSources = await Source.readdir(dirPath, '.js');
   const tsSources = await Source.readdir(dirPath, '.ts');
   const {documentation} = await jsBuilder.checkSources(jsSources.concat(tsSources));
-  expect(serialize(documentation)).toBeGolden({goldenPath: dirPath, outputPath: dirPath, goldenName: 'result.txt'});
+  expect(serialize(documentation)).toBeGolden(path.join(testRun.test().name(), 'result.txt'));
 }
 
 /**

utils/testrunner/Matchers.js

@@ -20,15 +20,88 @@ const Diff = require('text-diff');
 const GoldenUtils = require('./GoldenUtils');
 
 class Matchers {
-  constructor(customMatchers = {}) {
+  constructor(config) {
-    this._matchers = {};
-    Object.assign(this._matchers, DefaultMatchers);
-    Object.assign(this._matchers, customMatchers);
     this.expect = this.expect.bind(this);
-  }
 
-  addMatcher(name, matcher) {
+    this._matchers = {
-    this._matchers[name] = matcher;
+      toBe: function(received, expected, message) {
+        message = message || `${received} == ${expected}`;
+        return { pass: received === expected, message, formatter: toBeFormatter.bind(null, received, expected) };
+      },
+
+      toBeFalsy: function(received, message) {
+        message = message || `${received}`;
+        return { pass: !received, message };
+      },
+    
+      toBeTruthy: function(received, message) {
+        message = message || `${received}`;
+        return { pass: !!received, message };
+      },
+    
+      toBeGreaterThan: function(received, other, message) {
+        message = message || `${received} > ${other}`;
+        return { pass: received > other, message };
+      },
+    
+      toBeGreaterThanOrEqual: function(received, other, message) {
+        message = message || `${received} >= ${other}`;
+        return { pass: received >= other, message };
+      },
+    
+      toBeLessThan: function(received, other, message) {
+        message = message || `${received} < ${other}`;
+        return { pass: received < other, message };
+      },
+    
+      toBeLessThanOrEqual: function(received, other, message) {
+        message = message || `${received} <= ${other}`;
+        return { pass: received <= other, message };
+      },
+    
+      toBeNull: function(received, message) {
+        message = message || `${received} == null`;
+        return { pass: received === null, message };
+      },
+    
+      toContain: function(received, other, message) {
+        message = message || `${received} ⊇ ${other}`;
+        return { pass: received.includes(other), message };
+      },
+    
+      toEqual: function(received, other, message) {
+        let receivedJson = stringify(received);
+        let otherJson = stringify(other);
+        let formatter = objectFormatter.bind(null, receivedJson, otherJson);
+        if (receivedJson.length < 40 && otherJson.length < 40) {
+          receivedJson = receivedJson.split('\n').map(line => line.trim()).join(' ');
+          otherJson = otherJson.split('\n').map(line => line.trim()).join(' ');
+          formatter = stringFormatter.bind(null, receivedJson, otherJson);
+        }
+        message = message || `\n${receivedJson} ≈ ${otherJson}`;
+        return { pass: receivedJson === otherJson, message, formatter };
+      },
+    
+      toBeCloseTo: function(received, other, precision, message) {
+        return {
+          pass: Math.abs(received - other) < Math.pow(10, -precision),
+          message
+        };
+      },
+    
+      toBeInstanceOf: function(received, other, message) {
+        message = message || `${received.constructor.name} instanceof ${other.name}`;
+        return { pass: received instanceof other, message };
+      },
+    
+      toBeGolden: function(received, goldenName) {
+        return GoldenUtils.compare(received, {
+          goldenPath: config.goldenPath,
+          outputPath: config.outputPath,
+          goldenName
+        });
+      }    
+    }
   }
 
   expect(received) {
@@ -155,82 +228,6 @@ function toBeFormatter(received, expected) {
   ].join('\n');
 }
 
-const DefaultMatchers = {
-  toBe: function(received, expected, message) {
-    message = message || `${received} == ${expected}`;
-    return { pass: received === expected, message, formatter: toBeFormatter.bind(null, received, expected) };
-  },
-
-  toBeFalsy: function(received, message) {
-    message = message || `${received}`;
-    return { pass: !received, message };
-  },
-
-  toBeTruthy: function(received, message) {
-    message = message || `${received}`;
-    return { pass: !!received, message };
-  },
-
-  toBeGreaterThan: function(received, other, message) {
-    message = message || `${received} > ${other}`;
-    return { pass: received > other, message };
-  },
-
-  toBeGreaterThanOrEqual: function(received, other, message) {
-    message = message || `${received} >= ${other}`;
-    return { pass: received >= other, message };
-  },
-
-  toBeLessThan: function(received, other, message) {
-    message = message || `${received} < ${other}`;
-    return { pass: received < other, message };
-  },
-
-  toBeLessThanOrEqual: function(received, other, message) {
-    message = message || `${received} <= ${other}`;
-    return { pass: received <= other, message };
-  },
-
-  toBeNull: function(received, message) {
-    message = message || `${received} == null`;
-    return { pass: received === null, message };
-  },
-
-  toContain: function(received, other, message) {
-    message = message || `${received} ⊇ ${other}`;
-    return { pass: received.includes(other), message };
-  },
-
-  toEqual: function(received, other, message) {
-    let receivedJson = stringify(received);
-    let otherJson = stringify(other);
-    let formatter = objectFormatter.bind(null, receivedJson, otherJson);
-    if (receivedJson.length < 40 && otherJson.length < 40) {
-      receivedJson = receivedJson.split('\n').map(line => line.trim()).join(' ');
-      otherJson = otherJson.split('\n').map(line => line.trim()).join(' ');
-      formatter = stringFormatter.bind(null, receivedJson, otherJson);
-    }
-    message = message || `\n${receivedJson} ≈ ${otherJson}`;
-    return { pass: receivedJson === otherJson, message, formatter };
-  },
-
-  toBeCloseTo: function(received, other, precision, message) {
-    return {
-      pass: Math.abs(received - other) < Math.pow(10, -precision),
-      message
-    };
-  },
-
-  toBeInstanceOf: function(received, other, message) {
-    message = message || `${received.constructor.name} instanceof ${other.name}`;
-    return { pass: received instanceof other, message };
-  },
-
-  toBeGolden: function(received, golden) {
-    return GoldenUtils.compare(received, golden);
-  },
-};
-
 function stringify(value) {
   function stabilize(key, object) {
     if (typeof object !== 'object' || object === undefined || object === null || Array.isArray(object))

utils/testrunner/TestCollector.js

@@ -172,6 +172,7 @@ class TestCollector {
     this._suiteAttributes = new Map();
     this._testModifiers = new Map();
     this._testAttributes = new Map();
+    this._testCallbackWrappers = [];
     this._api = {};
 
     this._currentSuite = new Suite(null, '', new Location());
@@ -189,6 +190,8 @@ class TestCollector {
     });
     this._api.it = specBuilder(this._testModifiers, this._testAttributes, (specs, name, testCallback) => {
       const location = Location.getCallerLocation();
+      for (const wrapper of this._testCallbackWrappers)
+        testCallback = wrapper(testCallback);
       const test = new Test(this._currentSuite, name, testCallback, location);
       test.setTimeout(timeout);
       for (const { callback, args } of specs)
@@ -207,6 +210,10 @@ class TestCollector {
     return this._currentSuite.addEnvironment(environment);
   }
 
+  addTestCallbackWrapper(wrapper) {
+    this._testCallbackWrappers.push(wrapper);
+  }
+
   addTestModifier(name, callback) {
     this._testModifiers.set(name, callback);
   }

utils/testrunner/index.js

@@ -40,6 +40,8 @@ class DefaultTestRunner {
       verbose,
       summary,
       lineBreak,
+      goldenPath,
+      outputPath,
     } = options;
 
     this._crashIfTestsAreFocusedOnCI = crashIfTestsAreFocusedOnCI;
@@ -61,7 +63,7 @@ class DefaultTestRunner {
 
     this._api = {
       ...this._collector.api(),
-      expect: new Matchers().expect,
+      expect: new Matchers({ goldenPath, outputPath }).expect,
     };
     this._collector.addSuiteAttribute('only', s => this._filter.focusSuite(s));
     this._collector.addSuiteAttribute('skip', s => s.setSkipped(true));