test: allow overriding test fixtures, add some test runner tests (#3463)

test/runner/GoldenUtils.js

@@ -130,7 +130,7 @@ function compare(actual, expectedPath) {
 
   return {
     pass: false,
-    message: output.join('n'),
+    message: output.join('\n'),
   };
 }
 

test/runner/fixtures.js

@@ -14,10 +14,11 @@
  * limitations under the License.
  */
 
+const crypto = require('crypto');
 const debug = require('debug');
 
 const registrations = new Map();
-const filesWithRegistrations = new Set();
+const registrationsByFile = new Map();
 
 class Fixture {
   constructor(pool, name, scope, fn) {
@@ -143,10 +144,11 @@ function innerRegisterFixture(name, scope, fn) {
   const stackFrame = new Error().stack.split('\n').slice(1).filter(line => !line.includes(__filename))[0];
   const location = stackFrame.replace(/.*at Object.<anonymous> \((.*)\)/, '$1');
   const file = location.replace(/^(.+):\d+:\d+$/, '$1');
-  const registration = { scope, fn, file, location };
+  const registration = { name, scope, fn, file, location };
   registrations.set(name, registration);
-  if (scope === 'worker')
-    filesWithRegistrations.add(file);
+  if (!registrationsByFile.has(file))
+    registrationsByFile.set(file, []);
+  registrationsByFile.get(file).push(registration);
 };
 
 function registerFixture(name, fn) {
@@ -157,4 +159,46 @@ function registerWorkerFixture (name, fn) {
   innerRegisterFixture(name, 'worker', fn);
 };
 
-module.exports = { FixturePool, registerFixture, registerWorkerFixture, filesWithRegistrations };
+function collectRequires(file, result) {
+  if (result.has(file))
+    return;
+  result.add(file);
+  const cache = require.cache[file];
+  const deps = cache.children.map(m => m.id).slice().reverse();
+  for (const dep of deps)
+    collectRequires(dep, result);
+}
+
+function lookupRegistrations(file, scope) {
+  const deps = new Set();
+  collectRequires(file, deps);
+  const allDeps = [...deps].reverse();
+  let result = [];
+  for (const dep of allDeps) {
+    const registrationList = registrationsByFile.get(dep);
+    if (!registrationList)
+      continue;
+    result = result.concat(registrationList.filter(r => r.scope === scope));
+  }
+  return result;
+}
+
+function rerunRegistrations(file, scope) {
+  // When we are running several tests in the same worker, we should re-run registrations before
+  // each file. That way we erase potential fixture overrides from the previous test runs.
+  for (const registration of lookupRegistrations(file, scope))
+    registrations.set(registration.name, registration);
+}
+
+function computeWorkerHash(file) {
+  // At this point, registrationsByFile contains all the files with worker fixture registrations.
+  // For every test, build the require closure and map each file to fixtures declared in it.
+  // This collection of fixtures is the fingerprint of the worker setup, a "worker hash".
+  // Tests with the matching "worker hash" will reuse the same worker.
+  const hash = crypto.createHash('sha1');
+  for (const registration of lookupRegistrations(file, 'worker'))
+    hash.update(registration.location);
+  return hash.digest('hex');
+}
+
+module.exports = { FixturePool, registerFixture, registerWorkerFixture, computeWorkerHash, rerunRegistrations };

test/runner/fixturesUI.js

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-const { FixturePool, registerFixture, registerWorkerFixture } = require('./fixtures');
+const { FixturePool, registerFixture, registerWorkerFixture, rerunRegistrations } = require('./fixtures');
 const { Test, Suite } = require('mocha');
 const { installTransform } = require('./transform');
 const commonSuite = require('mocha/lib/interfaces/common');
@@ -139,6 +139,7 @@ function fixturesUI(trialRun, suite) {
 
   suite.on(Suite.constants.EVENT_FILE_POST_REQUIRE, function(context, file, mocha) {
     revertBabelRequire();
+    rerunRegistrations(file, 'test');
   });
 };
 

test/runner/runner.js

@@ -15,13 +15,12 @@
  */
 
 const child_process = require('child_process');
-const crypto = require('crypto');
 const path = require('path');
 const { EventEmitter } = require('events');
 const Mocha = require('mocha');
 const builtinReporters = require('mocha/lib/reporters');
 const DotRunner = require('./dotReporter');
-const { filesWithRegistrations } = require('./fixtures');
+const { computeWorkerHash } = require('./fixtures');
 
 const constants = Mocha.Runner.constants;
 // Mocha runner does not remove uncaughtException listeners.
@@ -232,30 +231,4 @@ class Worker extends EventEmitter {
   }
 }
 
-function collectRequires(file, allDeps) {
-  if (allDeps.has(file))
-    return;
-  allDeps.add(file);
-  const cache = require.cache[file];
-  const deps = cache.children.map(m => m.id);
-  for (const dep of deps)
-    collectRequires(dep, allDeps);
-}
-
-function computeWorkerHash(file) {
-  // At this point, filesWithRegistrations contains all the files with worker fixture registrations.
-  // For every test, build the require closure and map each file to fixtures declared in it.
-  // This collection of fixtures is the fingerprint of the worker setup, a "worker hash".
-  // Tests with the matching "worker hash" will reuse the same worker.
-  const deps = new Set();
-  const hash = crypto.createHash('sha1');
-  collectRequires(file, deps);
-  for (const dep of deps) {
-    if (!filesWithRegistrations.has(dep))
-      continue;
-    hash.update(dep);
-  }
-  return hash.digest('hex');
-}
-
 module.exports = { Runner };

test/test-runner-helper.ts

@@ -0,0 +1,27 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { registerFixture } from './runner/fixtures';
+
+declare global {
+  interface FixtureState {
+    helperFixture: string;
+  }
+}
+
+registerFixture('helperFixture', async ({}, test) => {
+  await test('helperFixture');
+});

test/test-runner-overrides-1.spec.ts

@@ -0,0 +1,26 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import './test-runner-helper';
+import { registerFixture } from './runner/fixtures';
+
+registerFixture('helperFixture', async ({}, test) => {
+  await test('helperFixture - overridden');
+});
+
+it('should override fixture from requires', async ({helperFixture}) => {
+  expect(helperFixture).toBe('helperFixture - overridden');
+});

test/test-runner-overrides-2.spec.ts

@@ -0,0 +1,21 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import './test-runner-helper';
+
+it('override should have no side effects', async ({helperFixture}) => {
+  expect(helperFixture).toBe('helperFixture');
+});