test: make mocha runner work in parallel (#3383)

package-lock.json

@@ -3154,9 +3154,9 @@
       }
     },
     "commander": {
-      "version": "2.20.3",
+      "version": "6.0.0",
-      "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
+      "resolved": "https://registry.npmjs.org/commander/-/commander-6.0.0.tgz",
-      "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==",
+      "integrity": "sha512-s7EA+hDtTYNhuXkTlhqew4txMZVdszBmKWSPEMxGr8ru8JXR7bLUFIAtPhcSuFdJQ0ILMxnJi8GkQL0yvDy/YA==",
       "dev": true
     },
     "commondir": {
@@ -10056,6 +10056,12 @@
         "source-map-support": "~0.5.12"
       },
       "dependencies": {
+        "commander": {
+          "version": "2.20.3",
+          "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
+          "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==",
+          "dev": true
+        },
         "source-map": {
           "version": "0.6.1",
           "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",

package.json

@@ -70,6 +70,7 @@
     "@typescript-eslint/eslint-plugin": "^2.6.1",
     "@typescript-eslint/parser": "^2.6.1",
     "colors": "^1.4.0",
+    "commander": "^6.0.0",
     "commonmark": "^0.28.1",
     "cross-env": "^5.0.5",
     "electron": "^9.0.0-beta.24",

test/jest/playwrightEnvironment.js

@@ -15,7 +15,6 @@
  */
 
 const { FixturePool, registerFixture, registerWorkerFixture } = require('../harness/fixturePool');
-const os = require('os');
 const path = require('path');
 const fs = require('fs');
 const debug = require('debug');

test/mocha/dotReporter.js

@@ -18,7 +18,7 @@ const Base = require('mocha/lib/reporters/base');
 const constants = require('mocha/lib/runner').constants;
 const colors = require('colors/safe');
 
-class Dot extends Base {
+class DotReporter extends Base {
   constructor(runner, options) {
     super(runner, options);
 
@@ -48,4 +48,4 @@ class Dot extends Base {
   }
 }
 
-module.exports = Dot;
+module.exports = DotReporter;

test/mocha/index.js

@@ -14,38 +14,42 @@
  * limitations under the License.
  */
 
+const builtinReporters = require('mocha/lib/reporters');
 const fs = require('fs');
 const path = require('path');
-const Mocha = require('mocha');
+const program = require('commander');
-const { fixturesUI, fixturePool } = require('./fixturesUI');
+const { Runner } = require('./runner');
-const dot = require('./dot');
+const DotRunner = require('./dotReporter');
-const { Matchers } = require('../../utils/testrunner/Matchers');
 
-const browserName = process.env.BROWSER || 'chromium';
-const goldenPath = path.join(__dirname, '..', 'golden-' + browserName);
-const outputPath = path.join(__dirname, '..', 'output-' + browserName);
-global.expect = new Matchers({ goldenPath, outputPath }).expect;
-global.testOptions = require('../harness/testOptions');
 
-const mocha = new Mocha({
+program
-	ui: fixturesUI,
+  .version('Version ' + require('../../package.json').version)
-	reporter: dot,
+  .option('--reporter <reporter>', 'reporter to use', '')
-	timeout: 10000,
+  .option('--max-workers <maxWorkers>', 'reporter to use', '')
-});
+  .action(async (command, args) => {
-const testDir = path.join(process.cwd(), 'test');
+    const testDir = path.join(process.cwd(), 'test');
+    const files = [];
+    for (const name of fs.readdirSync(testDir)) {
+      if (!name.includes('.spec.'))
+        continue;
+      if (!command.args.length) {
+        files.push(path.join(testDir, name));
+        continue;
+      }
+      for (const filter of command.args) {
+        if (name.includes(filter)) {
+          files.push(path.join(testDir, name));
+          break;
+        }
+      }
+    }
 
-const filter = process.argv[2];
+    const runner = new Runner({
+      reporter: command.reporter ? builtinReporters[command.reporter] : DotRunner,
+      maxWorkers: command.maxWorkers || Math.ceil(require('os').cpus().length / 2)
+    });
+    await runner.run(files);
+    await runner.stop();
+  });
 
-fs.readdirSync(testDir).filter(function(file) {
+program.parse(process.argv);
-  return file.includes('.spec.') && (!filter || file.includes(filter));
-}).forEach(function(file) {
-  mocha.addFile(path.join(testDir, file));
-});
-
-const runner = mocha.run((failures) => {
-  process.exitCode = failures ? 1 : 0;
-});
-const constants = Mocha.Runner.constants;
-runner.on(constants.EVENT_RUN_END, test => {
-  fixturePool.teardownScope('worker');
-});

test/mocha/runner.js

@@ -0,0 +1,146 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const child_process = require('child_process');
+const path = require('path');
+const { EventEmitter } = require('events');
+const Mocha = require('mocha');
+const { Serializer } = require('v8');
+
+const constants = Mocha.Runner.constants;
+
+class Runner extends EventEmitter {
+  constructor(options) {
+    super();
+    this._maxWorkers = options.maxWorkers;
+    this._workers = new Set();
+    this._freeWorkers = [];
+    this._callbacks = [];
+    this._workerId = 0;
+    this.stats = {
+      duration: 0,
+      failures: 0,
+      passes: 0,
+      pending: 0,
+      tests: 0,
+    };
+    this._reporter = new options.reporter(this, {});
+  }
+
+  async run(files) {
+    this.emit(constants.EVENT_RUN_BEGIN, {});
+    const result = new Promise(f => this._runCallback = f);
+    for (const file of files) {
+      const worker = await this._obtainWorker();
+      worker.send({ method: 'run', params: file });
+    }
+    await result;
+    this.emit(constants.EVENT_RUN_END, {});
+  }
+
+  async _obtainWorker() {
+    if (this._freeWorkers.length)
+      return this._freeWorkers.pop();
+
+    if (this._workers.size < this._maxWorkers) {
+      const worker = child_process.fork(path.join(__dirname, 'worker.js'), {
+        detached: false
+      });
+      let readyCallback;
+      const result = new Promise(f => readyCallback = f);
+      worker.send({ method: 'init', params: ++this._workerId });
+      worker.on('message', message => {
+        if (message.method === 'ready')
+            readyCallback();
+        this._messageFromWorker(worker, message);
+      });
+      worker.on('exit', () => {
+        this._workers.delete(worker);
+        if (!this._workers.size)
+          this._runCallback();
+      });
+      this._workers.add(worker);
+      await result;
+      return worker;
+    }
+
+    return new Promise(f => this._callbacks.push(f));
+  }
+
+  _messageFromWorker(worker, message) {
+    const { method, params } = message;
+    switch (method) {
+      case 'done': {
+        if (this._callbacks.length) {
+          const callback = this._callbacks.shift();
+          callback(worker);
+        } else {
+          this._freeWorkers.push(worker);
+          if (this._freeWorkers.length === this._workers.size) {
+            this._runCallback();
+          }
+        }
+        break;
+      }
+      case 'start':
+        break;
+      case 'test':
+        this.emit(constants.EVENT_TEST_BEGIN, this._parse(params.test));
+        break;
+      case 'pending':
+        this.emit(constants.EVENT_TEST_PENDING, this._parse(params.test));
+        break;
+      case 'pass':
+        this.emit(constants.EVENT_TEST_PASS, this._parse(params.test));
+        break;
+      case 'fail':
+        const test = this._parse(params.test);
+        this.emit(constants.EVENT_TEST_FAIL, test, params.error);
+        break;
+      case 'end':
+        this.stats.duration += params.stats.duration;
+        this.stats.failures += params.stats.failures;
+        this.stats.passes += params.stats.passes;
+        this.stats.pending += params.stats.pending;
+        this.stats.tests += params.stats.tests;
+        break;
+    }
+  }
+
+  _parse(serialized) {
+    return {
+      ...serialized,
+      currentRetry: () => serialized.currentRetry,
+      fullTitle: () => serialized.fullTitle,
+      slow: () => serialized.slow,
+      timeout: () => serialized.timeout,
+      titlePath: () => serialized.titlePath,
+      isPending: () => serialized.isPending,
+      parent: {
+        fullTitle: () => ''
+      }
+    };
+  }
+
+  async stop() {
+    const result = new Promise(f => this._stopCallback = f);
+    for (const worker of this._workers)
+      worker.send({ method: 'stop' });
+    await result;
+  }
+}
+
+module.exports = { Runner };

test/mocha/worker.js

@@ -0,0 +1,166 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const path = require('path');
+const Mocha = require('mocha');
+const { fixturesUI } = require('./fixturesUI');
+const { gracefullyCloseAll } = require('../../lib/server/processLauncher');
+const GoldenUtils = require('../../utils/testrunner/GoldenUtils');
+
+const browserName = process.env.BROWSER || 'chromium';
+const goldenPath = path.join(__dirname, '..', 'golden-' + browserName);
+const outputPath = path.join(__dirname, '..', 'output-' + browserName);
+global.expect = require('expect');
+global.testOptions = require('../harness/testOptions');
+
+extendExpects();
+
+let closed = false;
+
+process.on('message', async message => {
+  if (message.method === 'init')
+    process.env.JEST_WORKER_ID = message.params;
+  if (message.method === 'stop')
+    gracefullyCloseAndExit();
+  if (message.method === 'run')
+    await runSingleTest(message.params);
+});
+
+process.on('disconnect', gracefullyCloseAndExit);
+process.on('SIGINT',() => {});
+process.on('SIGTERM',() => {});
+sendMessageToParent('ready');
+
+async function gracefullyCloseAndExit() {
+  closed = true;
+  // Force exit after 30 seconds.
+  setTimeout(() => process.exit(0), 30000);
+  // Meanwhile, try to gracefully close all browsers.
+  await gracefullyCloseAll();
+  process.exit(0);
+}
+
+class NullReporter {}
+
+async function runSingleTest(file) {
+  const mocha = new Mocha({
+    ui: fixturesUI,
+    timeout: 10000,
+    reporter: NullReporter
+  });
+  mocha.addFile(file);
+
+  const runner = mocha.run();
+
+  const constants = Mocha.Runner.constants;
+  runner.on(constants.EVENT_RUN_BEGIN, () => {
+    sendMessageToParent('start');
+  });
+
+  runner.on(constants.EVENT_TEST_BEGIN, test => {
+    sendMessageToParent('test', { test: sanitizeTest(test) });
+  });
+
+  runner.on(constants.EVENT_TEST_PENDING, test => {
+    sendMessageToParent('pending', { test: sanitizeTest(test) });
+  });
+
+  runner.on(constants.EVENT_TEST_PASS, test => {
+    sendMessageToParent('pass', { test: sanitizeTest(test) });
+  });
+
+  runner.on(constants.EVENT_TEST_FAIL, (test, error) => {
+    sendMessageToParent('fail', {
+      test: sanitizeTest(test),
+      error: serializeError(error),
+     });
+  });
+
+  runner.once(constants.EVENT_RUN_END, async () => {
+    sendMessageToParent('end', { stats: serializeStats(runner.stats) });
+    sendMessageToParent('done');
+  });
+}
+
+function sendMessageToParent(method, params = {}) {
+  if (closed)
+    return;
+  try {
+    process.send({ method, params });
+  } catch (e) {
+    // Can throw when closing.
+  }
+}
+
+function sanitizeTest(test) {
+  return {
+    currentRetry: test.currentRetry(),
+    duration: test.duration,
+    file: test.file,
+    fullTitle: test.fullTitle(),
+    isPending: test.isPending(),
+    slow: test.slow(),
+    timeout: test.timeout(),
+    title: test.title,
+    titlePath: test.titlePath(),
+  };
+}
+
+function serializeStats(stats) {
+  return {
+    tests: stats.tests,
+    passes: stats.passes,
+    duration: stats.duration,
+    failures: stats.failures,
+    pending: stats.pending,
+  }
+}
+
+function trimCycles(obj) {
+  const cache = new Set();
+  return JSON.parse(
+    JSON.stringify(obj, function(key, value) {
+      if (typeof value === 'object' && value !== null) {
+        if (cache.has(value))
+          return '' + value;
+        cache.add(value);
+      }
+      return value;
+    })
+  );
+}
+
+function serializeError(error) {
+  if (error instanceof Error) {
+    return {
+      message: error.message,
+      stack: error.stack
+    }
+  }
+  return trimCycles(error);
+}
+
+function extendExpects() {
+  function toBeGolden(received, goldenName) {
+    const {pass, message} =  GoldenUtils.compare(received, {
+      goldenPath,
+      outputPath,
+      goldenName
+    });
+    return {pass, message: () => message};
+  };
+  global.expect.extend({ toBeGolden });
+}