From 1865c5685a34a2e91ace07e08aee7ef15261406c Mon Sep 17 00:00:00 2001 From: Andrey Lushnikov Date: Wed, 24 Jun 2020 22:08:46 -0700 Subject: [PATCH] testrunner: support globalSetup and globalTeardown hooks. (#2686) --- utils/testrunner/Test.js | 10 + utils/testrunner/TestCollector.js | 2 + utils/testrunner/TestRunner.js | 251 ++++++++++++++++------- utils/testrunner/test/testrunner.spec.js | 216 +++++++++++++++++++ 4 files changed, 410 insertions(+), 69 deletions(-) diff --git a/utils/testrunner/Test.js b/utils/testrunner/Test.js index ed975a53c7..e460d5f6d5 100644 --- a/utils/testrunner/Test.js +++ b/utils/testrunner/Test.js @@ -57,6 +57,16 @@ class Environment { return this; } + globalSetup(callback) { + this._hooks.push(createHook(callback, 'globalSetup')); + return this; + } + + globalTeardown(callback) { + this._hooks.push(createHook(callback, 'globalTeardown')); + return this; + } + hooks(name) { return this._hooks.filter(hook => !name || hook.name === name); } diff --git a/utils/testrunner/TestCollector.js b/utils/testrunner/TestCollector.js index c04e91d8d0..91115b5906 100644 --- a/utils/testrunner/TestCollector.js +++ b/utils/testrunner/TestCollector.js @@ -199,6 +199,8 @@ class TestCollector { this._api.beforeEach = callback => this._currentSuite.environment().beforeEach(callback); this._api.afterAll = callback => this._currentSuite.environment().afterAll(callback); this._api.afterEach = callback => this._currentSuite.environment().afterEach(callback); + this._api.globalSetup = callback => this._currentSuite.environment().globalSetup(callback); + this._api.globalTeardown = callback => this._currentSuite.environment().globalTeardown(callback); } useEnvironment(environment) { diff --git a/utils/testrunner/TestRunner.js b/utils/testrunner/TestRunner.js index c3b76ba902..4d839cdb91 100644 --- a/utils/testrunner/TestRunner.js +++ b/utils/testrunner/TestRunner.js @@ -55,6 +55,11 @@ class TestRun { this._endTimestamp = 0; this._workerId = null; this._output = []; + + this._environments = test._environments.filter(env => !env.isEmpty()).reverse(); + for (let suite = test.suite(); suite; suite = suite.parentSuite()) + this._environments.push(...suite._environments.filter(env => !env.isEmpty()).reverse()); + this._environments.reverse(); } finished() { @@ -135,14 +140,14 @@ class Result { } class TestWorker { - constructor(testRunner, workerId, parallelIndex) { + constructor(testRunner, hookRunner, workerId, parallelIndex) { this._testRunner = testRunner; + this._hookRunner = hookRunner; this._state = { parallelIndex }; this._environmentStack = []; this._terminating = false; this._workerId = workerId; this._runningTestTerminate = null; - this._runningHookTerminate = null; this._runs = []; } @@ -150,8 +155,7 @@ class TestWorker { this._terminating = true; if (this._runningTestTerminate) this._runningTestTerminate(); - if (terminateHooks && this._runningHookTerminate) - this._runningHookTerminate(); + this._hookRunner.terminateWorker(this); } _markTerminated(testRun) { @@ -185,7 +189,7 @@ class TestWorker { return; } - const environmentStack = allTestEnvironments(test); + const environmentStack = testRun._environments; let common = 0; while (common < environmentStack.length && this._environmentStack[common] === environmentStack[common]) common++; @@ -194,20 +198,20 @@ class TestWorker { if (this._markTerminated(testRun)) return; const environment = this._environmentStack.pop(); - for (const hook of environment.hooks('afterAll')) { - if (!await this._runHook(testRun, hook, environment.name(), [this._state])) - return; - } + if (!await this._hookRunner.runAfterAll(environment, this, testRun, [this._state])) + return; + if (!await this._hookRunner.ensureGlobalTeardown(environment)) + return; } while (this._environmentStack.length < environmentStack.length) { if (this._markTerminated(testRun)) return; const environment = environmentStack[this._environmentStack.length]; this._environmentStack.push(environment); - for (const hook of environment.hooks('beforeAll')) { - if (!await this._runHook(testRun, hook, environment.name(), [this._state])) - return; - } + if (!await this._hookRunner.ensureGlobalSetup(environment)) + return; + if (!await this._hookRunner.runBeforeAll(environment, this, testRun, [this._state])) + return; } if (this._markTerminated(testRun)) @@ -218,8 +222,7 @@ class TestWorker { await this._willStartTestRun(testRun); for (const environment of this._environmentStack) { - for (const hook of environment.hooks('beforeEach')) - await this._runHook(testRun, hook, environment.name(), [this._state, testRun]); + await this._hookRunner.runBeforeEach(environment, this, testRun, [this._state, testRun]); } if (!testRun._error && !this._markTerminated(testRun)) { @@ -241,20 +244,91 @@ class TestWorker { await this._didFinishTestBody(testRun); } - for (const environment of this._environmentStack.slice().reverse()) { - for (const hook of environment.hooks('afterEach')) - await this._runHook(testRun, hook, environment.name(), [this._state, testRun]); - } + for (const environment of this._environmentStack.slice().reverse()) + await this._hookRunner.runAfterEach(environment, this, testRun, [this._state, testRun]); await this._didFinishTestRun(testRun); } - async _runHook(testRun, hook, fullName, hookArgs = []) { - await this._willStartHook(testRun, hook, fullName); + async _willStartTestRun(testRun) { + testRun._startTimestamp = Date.now(); + testRun._workerId = this._workerId; + await this._testRunner._runDelegateCallback(this._testRunner._delegate.onTestRunStarted, [testRun]); + } + + async _didFinishTestRun(testRun) { + testRun._endTimestamp = Date.now(); + testRun._workerId = this._workerId; + + this._hookRunner.markFinishedTestRun(testRun); + await this._testRunner._runDelegateCallback(this._testRunner._delegate.onTestRunFinished, [testRun]); + } + + async _willStartTestBody(testRun) { + debug('testrunner:test')(`[${this._workerId}] starting "${testRun.test().fullName()}" (${testRun.test().location()})`); + } + + async _didFinishTestBody(testRun) { + debug('testrunner:test')(`[${this._workerId}] ${testRun._result.toUpperCase()} "${testRun.test().fullName()}" (${testRun.test().location()})`); + } + + async shutdown() { + while (this._environmentStack.length > 0) { + const environment = this._environmentStack.pop(); + await this._hookRunner.runAfterAll(environment, this, null, [this._state]); + await this._hookRunner.ensureGlobalTeardown(environment); + } + } +} + +class HookRunner { + constructor(testRunner, testRuns) { + this._testRunner = testRunner; + this._runningHookTerminations = new Map(); + + this._environmentToGlobalState = new Map(); + for (const testRun of testRuns) { + for (const env of testRun._environments) { + let globalState = this._environmentToGlobalState.get(env); + if (!globalState) { + globalState = { + pendingTestRuns: new Set(), + globalSetupPromise: null, + globalTeardownPromise: null, + }; + this._environmentToGlobalState.set(env, globalState); + } + globalState.pendingTestRuns.add(testRun); + } + } + } + + terminateWorker(worker) { + let termination = this._runningHookTerminations.get(worker); + this._runningHookTerminations.delete(worker); + if (termination) + termination(); + } + + terminateAll() { + for (const termination of this._runningHookTerminations.values()) + termination(); + this._runningHookTerminations.clear(); + } + + markFinishedTestRun(testRun) { + for (const environment of testRun._environments) { + const globalState = this._environmentToGlobalState.get(environment); + globalState.pendingTestRuns.delete(testRun); + } + } + + async _runHook(worker, testRun, hook, fullName, hookArgs = []) { + await this._willStartHook(worker, testRun, hook, fullName); const timeout = this._testRunner._hookTimeout; const { promise, terminate } = runUserCallback(hook.body, timeout, hookArgs); - this._runningHookTerminate = terminate; + this._runningHookTerminations.set(worker, terminate); let error = await promise; - this._runningHookTerminate = null; + this._runningHookTerminations.delete(worker); if (error) { if (testRun && testRun._result !== TestResult.Terminated) { @@ -274,58 +348,108 @@ class TestWorker { await this._testRunner._sourceMapSupport.rewriteStackTraceWithSourceMaps(error); message = `${hook.location.toDetailedString()} - FAILED while running "${hook.name}" in suite "${fullName}": `; } - await this._didFailHook(testRun, hook, fullName, message, error); + await this._didFailHook(worker, testRun, hook, fullName, message, error); if (testRun) testRun._error = error; return false; } - await this._didCompleteHook(testRun, hook, fullName); + await this._didCompleteHook(worker, testRun, hook, fullName); return true; } - async _willStartTestRun(testRun) { - testRun._startTimestamp = Date.now(); - testRun._workerId = this._workerId; - await this._testRunner._runDelegateCallback(this._testRunner._delegate.onTestRunStarted, [testRun]); + async runAfterAll(environment, worker, testRun, hookArgs) { + for (const hook of environment.hooks('afterAll')) { + if (!await this._runHook(worker, testRun, hook, environment.name(), hookArgs)) + return false; + } + return true; } - async _didFinishTestRun(testRun) { - testRun._endTimestamp = Date.now(); - testRun._workerId = this._workerId; - await this._testRunner._runDelegateCallback(this._testRunner._delegate.onTestRunFinished, [testRun]); + async runBeforeAll(environment, worker, testRun, hookArgs) { + for (const hook of environment.hooks('beforeAll')) { + if (!await this._runHook(worker, testRun, hook, environment.name(), hookArgs)) + return false; + } + return true; } - async _willStartTestBody(testRun) { - debug('testrunner:test')(`[${this._workerId}] starting "${testRun.test().fullName()}" (${testRun.test().location()})`); + async runAfterEach(environment, worker, testRun, hookArgs) { + for (const hook of environment.hooks('afterEach')) { + if (!await this._runHook(worker, testRun, hook, environment.name(), hookArgs)) + return false; + } + return true; } - async _didFinishTestBody(testRun) { - debug('testrunner:test')(`[${this._workerId}] ${testRun._result.toUpperCase()} "${testRun.test().fullName()}" (${testRun.test().location()})`); + async runBeforeEach(environment, worker, testRun, hookArgs) { + for (const hook of environment.hooks('beforeEach')) { + if (!await this._runHook(worker, testRun, hook, environment.name(), hookArgs)) + return false; + } + return true; } - async _willStartHook(testRun, hook, fullName) { - debug('testrunner:hook')(`[${this._workerId}] "${fullName}.${hook.name}" started for "${testRun ? testRun.test().fullName() : ''}" (${hook.location})`); + async ensureGlobalSetup(environment) { + const globalState = this._environmentToGlobalState.get(environment); + if (!globalState.globalSetupPromise) { + globalState.globalSetupPromise = (async () => { + let result = true; + for (const hook of environment.hooks('globalSetup')) { + if (!await this._runHook(null /* worker */, null /* testRun */, hook, environment.name(), [])) + result = false; + } + return result; + })(); + } + if (!await globalState.globalSetupPromise) { + await this._testRunner._terminate(TestResult.Crashed, 'Global setup failed!', false, null); + return false; + } + return true; } - async _didFailHook(testRun, hook, fullName, message, error) { - debug('testrunner:hook')(`[${this._workerId}] "${fullName}.${hook.name}" FAILED for "${testRun ? testRun.test().fullName() : ''}" (${hook.location})`); + async ensureGlobalTeardown(environment) { + const globalState = this._environmentToGlobalState.get(environment); + if (!globalState.globalTeardownPromise) { + if (!globalState.pendingTestRuns.size || (this._testRunner._terminating && globalState.globalSetupPromise)) { + globalState.globalTeardownPromise = (async () => { + let result = true; + for (const hook of environment.hooks('globalTeardown')) { + if (!await this._runHook(null /* worker */, null /* testRun */, hook, environment.name(), [])) + result = false; + } + return result; + })(); + } + } + if (!globalState.globalTeardownPromise) + return true; + if (!await globalState.globalTeardownPromise) { + await this._testRunner._terminate(TestResult.Crashed, 'Global teardown failed!', false, null); + return false; + } + return true; + } + + async _willStartHook(worker, testRun, hook, fullName) { + debug('testrunner:hook')(`${workerName(worker)} "${fullName}.${hook.name}" started for "${testRun ? testRun.test().fullName() : ''}" (${hook.location})`); + } + + async _didFailHook(worker, testRun, hook, fullName, message, error) { + debug('testrunner:hook')(`${workerName(worker)} "${fullName}.${hook.name}" FAILED for "${testRun ? testRun.test().fullName() : ''}" (${hook.location})`); if (message) - this._testRunner._result.addError(message, error, this); + this._testRunner._result.addError(message, error, worker); this._testRunner._result.setResult(TestResult.Crashed, message); } - async _didCompleteHook(testRun, hook, fullName) { - debug('testrunner:hook')(`[${this._workerId}] "${fullName}.${hook.name}" OK for "${testRun ? testRun.test().fullName() : ''}" (${hook.location})`); + async _didCompleteHook(worker, testRun, hook, fullName) { + debug('testrunner:hook')(`${workerName(worker)} "${fullName}.${hook.name}" OK for "${testRun ? testRun.test().fullName() : ''}" (${hook.location})`); } +} - async shutdown() { - while (this._environmentStack.length > 0) { - const environment = this._environmentStack.pop(); - for (const hook of environment.hooks('afterAll')) - await this._runHook(null, hook, environment.name(), [this._state]); - } - } +function workerName(worker) { + return worker ? `` : `<_global_>`; } class TestRunner { @@ -335,6 +459,7 @@ class TestRunner { this._workers = []; this._terminating = false; this._result = null; + this._hookRunner = null; } async _runDelegateCallback(callback, args) { @@ -406,6 +531,8 @@ class TestRunner { } await this._runDelegateCallback(this._delegate.onStarted, [testRuns]); + this._hookRunner = new HookRunner(this, testRuns); + const workerCount = Math.min(parallel, testRuns.length); const workerPromises = []; for (let i = 0; i < workerCount; ++i) { @@ -430,7 +557,7 @@ class TestRunner { } async _runWorker(testRunIndex, testRuns, parallelIndex) { - let worker = new TestWorker(this, this._nextWorkerId++, parallelIndex); + let worker = new TestWorker(this, this._hookRunner, this._nextWorkerId++, parallelIndex); this._workers[parallelIndex] = worker; while (!this._terminating) { let skipped = 0; @@ -455,7 +582,7 @@ class TestRunner { await this._terminate(TestResult.Terminated, message, false /* force */, null /* error */); return; } - worker = new TestWorker(this, this._nextWorkerId++, parallelIndex); + worker = new TestWorker(this, this._hookRunner, this._nextWorkerId++, parallelIndex); this._workers[parallelIndex] = worker; } } @@ -467,6 +594,8 @@ class TestRunner { this._terminating = true; for (const worker of this._workers) worker.terminate(force /* terminateHooks */); + if (this._hookRunner) + this._hookRunner.terminateAll(); this._result.setResult(result, message); if (this._result.message === 'SIGINT received' && message === 'SIGTERM received') this._result.message = message; @@ -484,20 +613,4 @@ class TestRunner { } } -function allTestEnvironments(test) { - const environmentStack = []; - for (const environment of test._environments.slice().reverse()) { - if (!environment.isEmpty()) - environmentStack.push(environment); - } - for (let suite = test.suite(); suite; suite = suite.parentSuite()) { - for (const environment of suite._environments.slice().reverse()) { - if (!environment.isEmpty()) - environmentStack.push(environment); - } - } - environmentStack.reverse(); - return environmentStack; -} - module.exports = { TestRunner, TestRun, TestResult, Result }; diff --git a/utils/testrunner/test/testrunner.spec.js b/utils/testrunner/test/testrunner.spec.js index 8a5926d7c4..de5765e250 100644 --- a/utils/testrunner/test/testrunner.spec.js +++ b/utils/testrunner/test/testrunner.spec.js @@ -39,6 +39,10 @@ class Runner { return this._collector.tests(); } + parallel() { + return this._options.parallel || 1; + } + focusedTests() { return this._filter.focusedTests(this._collector.tests()); } @@ -492,6 +496,158 @@ module.exports.addTests = function({describe, fdescribe, xdescribe, it, xit, fit }); }); + describe('globalSetup & globalTeardwon', () => { + it('should run globalSetup and globalTeardown in proper order', async() => { + const t = new Runner({timeout: 10000}); + const tracer = new TestTracer(t); + tracer.traceAllHooks(); + tracer.addTest('', 'test1'); + await t.run(); + + expect(tracer.trace()).toEqual([ + 'globalSetup', + 'beforeAll', + 'beforeEach', + 'test1', + 'afterEach', + 'afterAll', + 'globalTeardown', + ]); + }); + it('should run globalSetup and globalTeardown in proper order if parallel', async() => { + const t = new Runner({timeout: 10000, parallel: 2}); + const tracer = new TestTracer(t); + tracer.traceAllHooks('', async (hookName) => { + // slowdown globalsetup to see the rest hooks awaiting this one + if (hookName === 'globalSetup') + await new Promise(x => setTimeout(x, 50)); + }); + tracer.addTest('', 'test1'); + tracer.addTest('', 'test2'); + await t.run(); + + expect(tracer.trace()).toEqual([ + '<_global_> globalSetup', + ' beforeAll', + ' beforeAll', + ' beforeEach', + ' beforeEach', + ' test1', + ' test2', + ' afterEach', + ' afterEach', + ' afterAll', + ' afterAll', + '<_global_> globalTeardown', + ]); + }); + it('should support globalSetup/globalTeardown in nested suites', async() => { + const t = new Runner({timeout: 10000, parallel: 2}); + const tracer = new TestTracer(t); + tracer.traceAllHooks(''); + t.describe('suite', () => { + tracer.traceAllHooks(' '); + tracer.addTest(' ', 'test1'); + tracer.addTest(' ', 'test2'); + tracer.addTest(' ', 'test3'); + }); + await t.run(); + + expect(tracer.trace()).toEqual([ + '<_global_> globalSetup', + ' beforeAll', + ' beforeAll', + ' <_global_> globalSetup', + ' beforeAll', + ' beforeAll', + ' beforeEach', + ' beforeEach', + ' beforeEach', + ' beforeEach', + ' test1', + ' test2', + ' afterEach', + ' afterEach', + ' afterEach', + ' afterEach', + ' afterAll', + ' beforeEach', + ' afterAll', + ' beforeEach', + ' test3', + ' afterEach', + ' afterEach', + ' afterAll', + ' <_global_> globalTeardown', + ' afterAll', + '<_global_> globalTeardown', + ]); + }); + it('should report as crashed when global hook crashes', async() => { + const t = new Runner({timeout: 10000}); + t.globalSetup(() => { throw new Error('crash!'); }); + t.it('uno', () => { }); + const result = await t.run(); + expect(result.result).toBe('crashed'); + }); + it('should terminate and unwind hooks if globalSetup fails', async() => { + const t = new Runner({timeout: 10000}); + const tracer = new TestTracer(t); + tracer.traceAllHooks(); + t.describe('suite', () => { + tracer.traceAllHooks(' ', (hookName) => { + if (hookName === 'globalSetup') { + tracer.log(' !! CRASH !!'); + throw new Error('crash'); + } + }); + tracer.addTest(' ', 'test1'); + }); + await t.run(); + expect(tracer.trace()).toEqual([ + 'globalSetup', + 'beforeAll', + ' globalSetup', + ' !! CRASH !!', + ' afterAll', + ' globalTeardown', + 'afterAll', + 'globalTeardown', + ]); + }); + it('should not run globalSetup / globalTeardown if all tests are skipped', async() => { + const t = new Runner({timeout: 10000}); + const tracer = new TestTracer(t); + tracer.traceAllHooks(); + t.describe('suite', () => { + tracer.addSkippedTest(' ', 'test1'); + }); + await t.run(); + expect(tracer.trace()).toEqual([ + ]); + }); + it('should properly run globalTeardown if some tests are not run', async() => { + const t = new Runner({timeout: 10000}); + const tracer = new TestTracer(t); + tracer.traceAllHooks(); + t.describe('suite', () => { + tracer.addSkippedTest(' ', 'test1'); + tracer.addFailingTest(' ', 'test2'); + tracer.addTest(' ', 'test3'); + }); + await t.run(); + expect(tracer.trace()).toEqual([ + 'globalSetup', + 'beforeAll', + 'beforeEach', + ' test3', + 'afterEach', + 'afterAll', + 'globalTeardown', + ]); + }); + }); + describe('TestRunner.run', () => { it('should run a test', async() => { const t = new Runner(); @@ -855,3 +1011,63 @@ module.exports.addTests = function({describe, fdescribe, xdescribe, it, xit, fit }); }; +class TestTracer { + constructor(testRunner) { + this._testRunner = testRunner; + this._trace = []; + } + + addSkippedTest(prefix, testName, callback) { + this._testRunner.it.skip(testName, async(...args) => { + if (callback) + await callback(...args); + this._trace.push(prefix + this._workerPrefix(args[0]) + testName); + }); + } + + addFailingTest(prefix, testName, callback) { + this._testRunner.it.fail(testName, async(...args) => { + if (callback) + await callback(...args); + this._trace.push(prefix + this._workerPrefix(args[0]) + testName); + }); + } + + addTest(prefix, testName, callback) { + this._testRunner.it(testName, async(...args) => { + if (callback) + await callback(...args); + this._trace.push(prefix + this._workerPrefix(args[0]) + testName); + }); + } + + traceHooks(hookNames, prefix = '', callback) { + for (const hookName of hookNames) { + this._testRunner[hookName].call(this._testRunner, async (state) => { + this._trace.push(prefix + this._workerPrefix(state) + hookName); + if (callback) + await callback(hookName); + }); + } + } + + _workerPrefix(state) { + if (this._testRunner.parallel() === 1) + return ''; + return state && (typeof state.parallelIndex !== 'undefined') ? ` ` : `<_global_> `; + + } + + traceAllHooks(prefix = '', callback) { + this.traceHooks(['globalSetup', 'globalTeardown', 'beforeAll', 'afterAll', 'beforeEach', 'afterEach'], prefix, callback); + } + + log(text) { + this._trace.push(text); + } + + trace() { + return this._trace; + } +} +