From 1605cb453ccde3bcd63dc36fea53e5e2d48808fa Mon Sep 17 00:00:00 2001 From: Andrey Lushnikov Date: Tue, 30 Jun 2020 16:51:03 -0700 Subject: [PATCH] testrunner: make environment a simple class (#2769) This patch: - makes environment a simple class with optional methods `beforeEach`, `afterEach`, `beforeAll`, `afterAll`, `globalSetup` and `globalTeardown` - removes capability to have multiple hooks of the same name inside suite - removes default environment for test. (`dit` now adds a `TraceTestEnvironment` to the test) - extracts all environments that we use in our tests in `//test/environments.js` Downsides: - we no longer know hook locations for the environments. This, however, should not be a big deal since stack traces (if any) will still point into it. - this also regresses hook locations for suites for simplicity. We can get them back, but it shouldn't be pressing since we now have only one hook of each kind in every suite. --- test/environments.js | 247 +++++++++++++++++++++++ test/test.config.js | 118 +---------- test/test.js | 66 +----- utils/testrunner/Test.js | 108 +++------- utils/testrunner/TestCollector.js | 12 +- utils/testrunner/TestRunner.js | 101 +++------ utils/testrunner/test/testrunner.spec.js | 99 ++++----- 7 files changed, 366 insertions(+), 385 deletions(-) create mode 100644 test/environments.js diff --git a/test/environments.js b/test/environments.js new file mode 100644 index 0000000000..ffc89ce1dd --- /dev/null +++ b/test/environments.js @@ -0,0 +1,247 @@ +/** + * Copyright 2019 Google Inc. All rights reserved. + * Modifications copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const utils = require('./utils'); +const fs = require('fs'); +const path = require('path'); +const rm = require('rimraf').sync; +const {TestServer} = require('../utils/testserver/'); + +class ServerEnvironment { + async beforeAll(state) { + const assetsPath = path.join(__dirname, 'assets'); + const cachedPath = path.join(__dirname, 'assets', 'cached'); + + const port = 8907 + state.parallelIndex * 2; + state.server = await TestServer.create(assetsPath, port); + state.server.enableHTTPCache(cachedPath); + state.server.PORT = port; + state.server.PREFIX = `http://localhost:${port}`; + state.server.CROSS_PROCESS_PREFIX = `http://127.0.0.1:${port}`; + state.server.EMPTY_PAGE = `http://localhost:${port}/empty.html`; + + const httpsPort = port + 1; + state.httpsServer = await TestServer.createHTTPS(assetsPath, httpsPort); + state.httpsServer.enableHTTPCache(cachedPath); + state.httpsServer.PORT = httpsPort; + state.httpsServer.PREFIX = `https://localhost:${httpsPort}`; + state.httpsServer.CROSS_PROCESS_PREFIX = `https://127.0.0.1:${httpsPort}`; + state.httpsServer.EMPTY_PAGE = `https://localhost:${httpsPort}/empty.html`; + } + + async afterAll({server, httpsServer}) { + await Promise.all([ + server.stop(), + httpsServer.stop(), + ]); + } + + async beforeEach(state) { + state.server.reset(); + state.httpsServer.reset(); + } +} + +class DefaultBrowserOptionsEnvironment { + constructor(defaultBrowserOptions, dumpLogOnFailure, playwrightPath) { + this._defaultBrowserOptions = defaultBrowserOptions; + this._dumpLogOnFailure = dumpLogOnFailure; + this._playwrightPath = playwrightPath; + this._loggerSymbol = Symbol('DefaultBrowserOptionsEnvironment.logger'); + } + + async beforeAll(state) { + state[this._loggerSymbol] = utils.createTestLogger(this._dumpLogOnFailure, null, 'extra'); + state.defaultBrowserOptions = { + ...this._defaultBrowserOptions, + logger: state[this._loggerSymbol], + }; + state.playwrightPath = this._playwrightPath; + } + + async beforeEach(state, testRun) { + state[this._loggerSymbol].setTestRun(testRun); + } + + async afterEach(state) { + state[this._loggerSymbol].setTestRun(null); + } +} + +// simulate globalSetup per browserType that happens only once regardless of TestWorker. +const hasBeenCleaned = new Set(); + +class GoldenEnvironment { + async beforeAll(state) { + const { OUTPUT_DIR, GOLDEN_DIR } = utils.testOptions(state.browserType); + if (!hasBeenCleaned.has(state.browserType)) { + hasBeenCleaned.add(state.browserType); + if (fs.existsSync(OUTPUT_DIR)) + rm(OUTPUT_DIR); + fs.mkdirSync(OUTPUT_DIR, { recursive: true }); + } + state.golden = goldenName => ({ goldenPath: GOLDEN_DIR, outputPath: OUTPUT_DIR, goldenName }); + } + + async afterAll(state) { + delete state.golden; + } + + async afterEach(state, testRun) { + if (state.browser && state.browser.contexts().length !== 0) { + if (testRun.ok()) + console.warn(`\nWARNING: test "${testRun.test().fullName()}" (${testRun.test().location()}) did not close all created contexts!\n`); + await Promise.all(state.browser.contexts().map(context => context.close())); + } + } +} + +class TraceTestEnvironment { + static enableForTest(test) { + test.setTimeout(100000000); + test.addEnvironment(new TraceTestEnvironment()); + } + + constructor() { + this._session = null; + } + + async beforeEach() { + const inspector = require('inspector'); + const fs = require('fs'); + const util = require('util'); + const url = require('url'); + const readFileAsync = util.promisify(fs.readFile.bind(fs)); + this._session = new inspector.Session(); + this._session.connect(); + const postAsync = util.promisify(this._session.post.bind(this._session)); + await postAsync('Debugger.enable'); + const setBreakpointCommands = []; + const N = t.body().toString().split('\n').length; + const location = t.location(); + const lines = (await readFileAsync(location.filePath(), 'utf8')).split('\n'); + for (let line = 0; line < N; ++line) { + const lineNumber = line + location.lineNumber(); + setBreakpointCommands.push(postAsync('Debugger.setBreakpointByUrl', { + url: url.pathToFileURL(location.filePath()), + lineNumber, + condition: `console.log('${String(lineNumber + 1).padStart(6, ' ')} | ' + ${JSON.stringify(lines[lineNumber])})`, + }).catch(e => {})); + } + await Promise.all(setBreakpointCommands); + } + + async afterEach() { + this._session.disconnect(); + } +} + +class PlaywrightEnvironment { + constructor(playwright) { + this._playwright = playwright; + } + + name() { return 'Playwright'; }; + beforeAll(state) { state.playwright = this._playwright; } + afterAll(state) { delete state.playwright; } +} + +class BrowserTypeEnvironment { + constructor(browserType) { + this._browserType = browserType; + } + + async beforeAll(state) { + // Channel substitute + let overridenBrowserType = this._browserType; + if (process.env.PWCHANNEL) { + const dispatcherScope = new DispatcherScope(); + const connection = new Connection(); + dispatcherScope.onmessage = async message => { + setImmediate(() => connection.send(message)); + }; + connection.onmessage = async message => { + const result = await dispatcherScope.send(message); + await new Promise(f => setImmediate(f)); + return result; + }; + BrowserTypeDispatcher.from(dispatcherScope, this._browserType); + overridenBrowserType = await connection.waitForObjectWithKnownName(this._browserType.name()); + } + state.browserType = overridenBrowserType; + } + + async afterAll(state) { + delete state.browserType; + } +} + +class BrowserEnvironment { + constructor(browserType, launchOptions, dumpLogOnFailure) { + this._browserType = browserType; + this._launchOptions = launchOptions; + this._dumpLogOnFailure = dumpLogOnFailure; + this._loggerSymbol = Symbol('BrowserEnvironment.logger'); + } + + name() { return this._browserType.name(); } + + async beforeAll(state) { + state[this._loggerSymbol] = utils.createTestLogger(this._dumpLogOnFailure); + state.browser = await this._browserType.launch({ + ...this._launchOptions, + logger: state[this._loggerSymbol], + }); + } + + async afterAll(state) { + await state.browser.close(); + delete state.browser; + } + + async beforeEach(state, testRun) { + state[this._loggerSymbol].setTestRun(testRun); + } + + async afterEach(state, testRun) { + state[this._loggerSymbol].setTestRun(null); + } +} + +class PageEnvironment { + async beforeEach(state) { + state.context = await state.browser.newContext(); + state.page = await state.context.newPage(); + } + + async afterEach(state) { + await state.context.close(); + state.context = null; + state.page = null; + } +} + +module.exports = { + ServerEnvironment, + GoldenEnvironment, + TraceTestEnvironment, + DefaultBrowserOptionsEnvironment, + PlaywrightEnvironment, + BrowserTypeEnvironment, + BrowserEnvironment, + PageEnvironment, +}; diff --git a/test/test.config.js b/test/test.config.js index 924d44f7d1..016578aaad 100644 --- a/test/test.config.js +++ b/test/test.config.js @@ -15,85 +15,21 @@ * limitations under the License. */ -const fs = require('fs'); const path = require('path'); -const rm = require('rimraf').sync; const utils = require('./utils'); -const {TestServer} = require('../utils/testserver/'); -const {Environment} = require('../utils/testrunner/Test'); +const {DefaultBrowserOptionsEnvironment, ServerEnvironment, GoldenEnvironment, TraceTestEnvironment} = require('./environments.js'); const playwrightPath = path.join(__dirname, '..'); -const serverEnvironment = new Environment('TestServer'); -serverEnvironment.beforeAll(async state => { - const assetsPath = path.join(__dirname, 'assets'); - const cachedPath = path.join(__dirname, 'assets', 'cached'); +const dumpLogOnFailure = valueFromEnv('DEBUGP', false); +const defaultBrowserOptionsEnvironment = new DefaultBrowserOptionsEnvironment({ + handleSIGINT: false, + slowMo: valueFromEnv('SLOW_MO', 0), + headless: !!valueFromEnv('HEADLESS', true), +}, dumpLogOnFailure, playwrightPath); - const port = 8907 + state.parallelIndex * 2; - state.server = await TestServer.create(assetsPath, port); - state.server.enableHTTPCache(cachedPath); - state.server.PORT = port; - state.server.PREFIX = `http://localhost:${port}`; - state.server.CROSS_PROCESS_PREFIX = `http://127.0.0.1:${port}`; - state.server.EMPTY_PAGE = `http://localhost:${port}/empty.html`; - - const httpsPort = port + 1; - state.httpsServer = await TestServer.createHTTPS(assetsPath, httpsPort); - state.httpsServer.enableHTTPCache(cachedPath); - state.httpsServer.PORT = httpsPort; - state.httpsServer.PREFIX = `https://localhost:${httpsPort}`; - state.httpsServer.CROSS_PROCESS_PREFIX = `https://127.0.0.1:${httpsPort}`; - state.httpsServer.EMPTY_PAGE = `https://localhost:${httpsPort}/empty.html`; - - state._extraLogger = utils.createTestLogger(valueFromEnv('DEBUGP', false), null, 'extra'); - state.defaultBrowserOptions = { - handleSIGINT: false, - slowMo: valueFromEnv('SLOW_MO', 0), - headless: !!valueFromEnv('HEADLESS', true), - logger: state._extraLogger, - }; - state.playwrightPath = playwrightPath; -}); -serverEnvironment.afterAll(async({server, httpsServer}) => { - await Promise.all([ - server.stop(), - httpsServer.stop(), - ]); -}); -serverEnvironment.beforeEach(async(state, testRun) => { - state.server.reset(); - state.httpsServer.reset(); - state._extraLogger.setTestRun(testRun); -}); -serverEnvironment.afterEach(async(state) => { - state._extraLogger.setTestRun(null); -}); - -const customEnvironment = new Environment('Golden+CheckContexts'); - -// simulate globalSetup per browserType that happens only once regardless of TestWorker. -const hasBeenCleaned = new Set(); - -customEnvironment.beforeAll(async state => { - const { OUTPUT_DIR, GOLDEN_DIR } = require('./utils').testOptions(state.browserType); - if (!hasBeenCleaned.has(state.browserType)) { - hasBeenCleaned.add(state.browserType); - if (fs.existsSync(OUTPUT_DIR)) - rm(OUTPUT_DIR); - fs.mkdirSync(OUTPUT_DIR, { recursive: true }); - } - state.golden = goldenName => ({ goldenPath: GOLDEN_DIR, outputPath: OUTPUT_DIR, goldenName }); -}); -customEnvironment.afterAll(async state => { - delete state.golden; -}); -customEnvironment.afterEach(async (state, testRun) => { - if (state.browser && state.browser.contexts().length !== 0) { - if (testRun.ok()) - console.warn(`\nWARNING: test "${testRun.test().fullName()}" (${testRun.test().location()}) did not close all created contexts!\n`); - await Promise.all(state.browser.contexts().map(context => context.close())); - } -}); +const serverEnvironment = new ServerEnvironment(); +const customEnvironment = new GoldenEnvironment(); function valueFromEnv(name, defaultValue) { if (!(name in process.env)) @@ -108,39 +44,7 @@ function setupTestRunner(testRunner) { collector.addTestModifier('fail', (t, condition) => condition && t.setExpectation(t.Expectations.Fail)); collector.addSuiteModifier('fail', (s, condition) => condition && s.setExpectation(s.Expectations.Fail)); collector.addTestModifier('slow', t => t.setTimeout(t.timeout() * 3)); - collector.addTestAttribute('debug', t => { - t.setTimeout(100000000); - - let session; - t.environment().beforeEach(async () => { - const inspector = require('inspector'); - const fs = require('fs'); - const util = require('util'); - const url = require('url'); - const readFileAsync = util.promisify(fs.readFile.bind(fs)); - session = new inspector.Session(); - session.connect(); - const postAsync = util.promisify(session.post.bind(session)); - await postAsync('Debugger.enable'); - const setBreakpointCommands = []; - const N = t.body().toString().split('\n').length; - const location = t.location(); - const lines = (await readFileAsync(location.filePath(), 'utf8')).split('\n'); - for (let line = 0; line < N; ++line) { - const lineNumber = line + location.lineNumber(); - setBreakpointCommands.push(postAsync('Debugger.setBreakpointByUrl', { - url: url.pathToFileURL(location.filePath()), - lineNumber, - condition: `console.log('${String(lineNumber + 1).padStart(6, ' ')} | ' + ${JSON.stringify(lines[lineNumber])})`, - }).catch(e => {})); - } - await Promise.all(setBreakpointCommands); - }); - - t.environment().afterEach(async () => { - session.disconnect(); - }); - }); + collector.addTestAttribute('debug', t => TraceTestEnvironment.enableForTest(t)); testRunner.api().fdescribe = testRunner.api().describe.only; testRunner.api().xdescribe = testRunner.api().describe.skip(true); testRunner.api().fit = testRunner.api().it.only; @@ -161,7 +65,7 @@ module.exports = { headless: !!valueFromEnv('HEADLESS', true), }, - globalEnvironments: [serverEnvironment], + globalEnvironments: [defaultBrowserOptionsEnvironment, serverEnvironment], setupTestRunner, specs: [ diff --git a/test/test.js b/test/test.js index 35e611eaf0..8db7b31b11 100644 --- a/test/test.js +++ b/test/test.js @@ -18,11 +18,11 @@ const fs = require('fs'); const utils = require('./utils'); const TestRunner = require('../utils/testrunner/'); -const {Environment} = require('../utils/testrunner/Test'); const { DispatcherScope } = require('../lib/rpc/dispatcher'); const { Connection } = require('../lib/rpc/connection'); const { helper } = require('../lib/helper'); const { BrowserTypeDispatcher } = require('../lib/rpc/server/browserTypeDispatcher'); +const { PlaywrightEnvironment, BrowserTypeEnvironment, BrowserEnvironment, PageEnvironment} = require('./environments.js'); Error.stackTraceLimit = 15; @@ -83,15 +83,7 @@ function collect(browserNames) { const { setUnderTest } = require(require('path').join(playwrightPath, 'lib/helper.js')); setUnderTest(); - const playwrightEnvironment = new Environment('Playwright'); - playwrightEnvironment.beforeAll(async state => { - state.playwright = playwright; - }); - playwrightEnvironment.afterAll(async state => { - delete state.playwright; - }); - - testRunner.collector().useEnvironment(playwrightEnvironment); + testRunner.collector().useEnvironment(new PlaywrightEnvironment(playwright)); for (const e of config.globalEnvironments || []) testRunner.collector().useEnvironment(e); @@ -99,30 +91,7 @@ function collect(browserNames) { for (const browserName of browserNames) { const browserType = playwright[browserName]; - - const browserTypeEnvironment = new Environment('BrowserType'); - browserTypeEnvironment.beforeAll(async state => { - // Channel substitute - let overridenBrowserType = browserType; - if (process.env.PWCHANNEL) { - const dispatcherScope = new DispatcherScope(); - const connection = new Connection(); - dispatcherScope.onmessage = async message => { - setImmediate(() => connection.send(message)); - }; - connection.onmessage = async message => { - const result = await dispatcherScope.send(message); - await new Promise(f => setImmediate(f)); - return result; - }; - BrowserTypeDispatcher.from(dispatcherScope, browserType); - overridenBrowserType = await connection.waitForObjectWithKnownName(browserType.name()); - } - state.browserType = overridenBrowserType; - }); - browserTypeEnvironment.afterAll(async state => { - delete state.browserType; - }); + const browserTypeEnvironment = new BrowserTypeEnvironment(browserType); // TODO: maybe launch options per browser? const launchOptions = { @@ -142,33 +111,8 @@ function collect(browserNames) { throw new Error(`Browser is not downloaded. Run 'npm install' and try to re-run tests`); } - const browserEnvironment = new Environment(browserName); - browserEnvironment.beforeAll(async state => { - state._logger = utils.createTestLogger(config.dumpLogOnFailure); - state.browser = await state.browserType.launch({...launchOptions, logger: state._logger}); - }); - browserEnvironment.afterAll(async state => { - await state.browser.close(); - delete state.browser; - delete state._logger; - }); - browserEnvironment.beforeEach(async(state, testRun) => { - state._logger.setTestRun(testRun); - }); - browserEnvironment.afterEach(async (state, testRun) => { - state._logger.setTestRun(null); - }); - - const pageEnvironment = new Environment('Page'); - pageEnvironment.beforeEach(async state => { - state.context = await state.browser.newContext(); - state.page = await state.context.newPage(); - }); - pageEnvironment.afterEach(async state => { - await state.context.close(); - state.context = null; - state.page = null; - }); + const browserEnvironment = new BrowserEnvironment(browserType, launchOptions, config.dumpLogOnFailure); + const pageEnvironment = new PageEnvironment(); const suiteName = { 'chromium': 'Chromium', 'firefox': 'Firefox', 'webkit': 'WebKit' }[browserName]; describe(suiteName, () => { diff --git a/utils/testrunner/Test.js b/utils/testrunner/Test.js index e460d5f6d5..d9005a05fd 100644 --- a/utils/testrunner/Test.js +++ b/utils/testrunner/Test.js @@ -22,60 +22,6 @@ const TestExpectation = { Fail: 'fail', }; -function createHook(callback, name) { - const location = Location.getCallerLocation(); - return { name, body: callback, location }; -} - -class Environment { - constructor(name) { - this._name = name; - this._hooks = []; - } - - name() { - return this._name; - } - - beforeEach(callback) { - this._hooks.push(createHook(callback, 'beforeEach')); - return this; - } - - afterEach(callback) { - this._hooks.push(createHook(callback, 'afterEach')); - return this; - } - - beforeAll(callback) { - this._hooks.push(createHook(callback, 'beforeAll')); - return this; - } - - afterAll(callback) { - this._hooks.push(createHook(callback, 'afterAll')); - return this; - } - - globalSetup(callback) { - this._hooks.push(createHook(callback, 'globalSetup')); - return this; - } - - globalTeardown(callback) { - this._hooks.push(createHook(callback, 'globalTeardown')); - return this; - } - - hooks(name) { - return this._hooks.filter(hook => !name || hook.name === name); - } - - isEmpty() { - return !this._hooks.length; - } -} - class Test { constructor(suite, name, callback, location) { this._suite = suite; @@ -86,8 +32,7 @@ class Test { this._body = callback; this._location = location; this._timeout = 100000000; - this._defaultEnvironment = new Environment(this._fullName); - this._environments = [this._defaultEnvironment]; + this._environments = []; this.Expectations = { ...TestExpectation }; } @@ -138,10 +83,6 @@ class Test { return this; } - environment() { - return this._defaultEnvironment; - } - addEnvironment(environment) { this._environments.push(environment); return this; @@ -164,49 +105,50 @@ class Suite { this._location = location; this._skipped = false; this._expectation = TestExpectation.Ok; - this._defaultEnvironment = new Environment(this._fullName); + + this._defaultEnvironment = { + name() { return this._fullName; }, + }; + this._environments = [this._defaultEnvironment]; this.Expectations = { ...TestExpectation }; } - parentSuite() { - return this._parentSuite; + _addHook(name, callback) { + if (this._defaultEnvironment[name]) + throw new Error(`ERROR: cannot re-assign hook "${name}" for suite "${this._fullName}"`); + this._defaultEnvironment[name] = callback; } - name() { - return this._name; - } + beforeEach(callback) { this._addHook('beforeEach', callback); } + afterEach(callback) { this._addHook('afterEach', callback); } + beforeAll(callback) { this._addHook('beforeAll', callback); } + afterAll(callback) { this._addHook('afterAll', callback); } + globalSetup(callback) { this._addHook('globalSetup', callback); } + globalTeardown(callback) { this._addHook('globalTeardown', callback); } - fullName() { - return this._fullName; - } + parentSuite() { return this._parentSuite; } - skipped() { - return this._skipped; - } + name() { return this._name; } + + fullName() { return this._fullName; } + + skipped() { return this._skipped; } setSkipped(skipped) { this._skipped = skipped; return this; } - location() { - return this._location; - } + location() { return this._location; } - expectation() { - return this._expectation; - } + expectation() { return this._expectation; } setExpectation(expectation) { this._expectation = expectation; return this; } - environment() { - return this._defaultEnvironment; - } - addEnvironment(environment) { this._environments.push(environment); return this; @@ -221,4 +163,4 @@ class Suite { } } -module.exports = { TestExpectation, Environment, Test, Suite }; +module.exports = { TestExpectation, Test, Suite }; diff --git a/utils/testrunner/TestCollector.js b/utils/testrunner/TestCollector.js index 91115b5906..7f95eefcf5 100644 --- a/utils/testrunner/TestCollector.js +++ b/utils/testrunner/TestCollector.js @@ -195,12 +195,12 @@ class TestCollector { callback(test, ...args); this._tests.push(test); }); - this._api.beforeAll = callback => this._currentSuite.environment().beforeAll(callback); - this._api.beforeEach = callback => this._currentSuite.environment().beforeEach(callback); - this._api.afterAll = callback => this._currentSuite.environment().afterAll(callback); - this._api.afterEach = callback => this._currentSuite.environment().afterEach(callback); - this._api.globalSetup = callback => this._currentSuite.environment().globalSetup(callback); - this._api.globalTeardown = callback => this._currentSuite.environment().globalTeardown(callback); + this._api.beforeAll = callback => this._currentSuite.beforeAll(callback); + this._api.beforeEach = callback => this._currentSuite.beforeEach(callback); + this._api.afterAll = callback => this._currentSuite.afterAll(callback); + this._api.afterEach = callback => this._currentSuite.afterEach(callback); + this._api.globalSetup = callback => this._currentSuite.globalSetup(callback); + this._api.globalTeardown = callback => this._currentSuite.globalTeardown(callback); } useEnvironment(environment) { diff --git a/utils/testrunner/TestRunner.js b/utils/testrunner/TestRunner.js index 4d839cdb91..f790d0231d 100644 --- a/utils/testrunner/TestRunner.js +++ b/utils/testrunner/TestRunner.js @@ -46,6 +46,11 @@ const TestResult = { Crashed: 'crashed', // If testrunner crashed due to this test }; +function isEmptyEnvironment(env) { + return !env.afterEach && !env.afterAll && !env.beforeEach && !env.beforeAll && + !env.globalSetup && !env.globalTeardown; +} + class TestRun { constructor(test) { this._test = test; @@ -56,9 +61,9 @@ class TestRun { this._workerId = null; this._output = []; - this._environments = test._environments.filter(env => !env.isEmpty()).reverse(); + this._environments = test._environments.filter(env => !isEmptyEnvironment(env)).reverse(); for (let suite = test.suite(); suite; suite = suite.parentSuite()) - this._environments.push(...suite._environments.filter(env => !env.isEmpty()).reverse()); + this._environments.push(...suite._environments.filter(env => !isEmptyEnvironment(env)).reverse()); this._environments.reverse(); } @@ -198,9 +203,9 @@ class TestWorker { if (this._markTerminated(testRun)) return; const environment = this._environmentStack.pop(); - if (!await this._hookRunner.runAfterAll(environment, this, testRun, [this._state])) + if (!await this._hookRunner.runHook(environment, 'afterAll', [this._state], this, testRun)) return; - if (!await this._hookRunner.ensureGlobalTeardown(environment)) + if (!await this._hookRunner.maybeRunGlobalTeardown(environment)) return; } while (this._environmentStack.length < environmentStack.length) { @@ -208,9 +213,9 @@ class TestWorker { return; const environment = environmentStack[this._environmentStack.length]; this._environmentStack.push(environment); - if (!await this._hookRunner.ensureGlobalSetup(environment)) + if (!await this._hookRunner.maybeRunGlobalSetup(environment)) return; - if (!await this._hookRunner.runBeforeAll(environment, this, testRun, [this._state])) + if (!await this._hookRunner.runHook(environment, 'beforeAll', [this._state], this, testRun)) return; } @@ -222,7 +227,7 @@ class TestWorker { await this._willStartTestRun(testRun); for (const environment of this._environmentStack) { - await this._hookRunner.runBeforeEach(environment, this, testRun, [this._state, testRun]); + await this._hookRunner.runHook(environment, 'beforeEach', [this._state, testRun], this, testRun); } if (!testRun._error && !this._markTerminated(testRun)) { @@ -245,7 +250,7 @@ class TestWorker { } for (const environment of this._environmentStack.slice().reverse()) - await this._hookRunner.runAfterEach(environment, this, testRun, [this._state, testRun]); + await this._hookRunner.runHook(environment, 'afterEach', [this._state, testRun], this, testRun); await this._didFinishTestRun(testRun); } @@ -274,8 +279,8 @@ class TestWorker { async shutdown() { while (this._environmentStack.length > 0) { const environment = this._environmentStack.pop(); - await this._hookRunner.runAfterAll(environment, this, null, [this._state]); - await this._hookRunner.ensureGlobalTeardown(environment); + await this._hookRunner.runHook(environment, 'afterAll', [this._state], this, null); + await this._hookRunner.maybeRunGlobalTeardown(environment); } } } @@ -322,7 +327,7 @@ class HookRunner { } } - async _runHook(worker, testRun, hook, fullName, hookArgs = []) { + async _runHookInternal(worker, testRun, hook, fullName, hookArgs = []) { await this._willStartHook(worker, testRun, hook, fullName); const timeout = this._testRunner._hookTimeout; const { promise, terminate } = runUserCallback(hook.body, timeout, hookArgs); @@ -337,7 +342,7 @@ class HookRunner { } let message; if (error === TimeoutError) { - message = `${hook.location.toDetailedString()} - Timeout Exceeded ${timeout}ms while running "${hook.name}" in "${fullName}"`; + message = `Timeout Exceeded ${timeout}ms while running "${hook.name}" in "${fullName}"`; error = null; } else if (error === TerminatedError) { // Do not report termination details - it's just noise. @@ -346,7 +351,7 @@ class HookRunner { } else { if (error.stack) await this._testRunner._sourceMapSupport.rewriteStackTraceWithSourceMaps(error); - message = `${hook.location.toDetailedString()} - FAILED while running "${hook.name}" in suite "${fullName}": `; + message = `FAILED while running "${hook.name}" in suite "${fullName}": `; } await this._didFailHook(worker, testRun, hook, fullName, message, error); if (testRun) @@ -358,50 +363,18 @@ class HookRunner { return true; } - async runAfterAll(environment, worker, testRun, hookArgs) { - for (const hook of environment.hooks('afterAll')) { - if (!await this._runHook(worker, testRun, hook, environment.name(), hookArgs)) - return false; - } - return true; + async runHook(environment, hookName, hookArgs, worker = null, testRun = null) { + const hookBody = environment[hookName]; + if (!hookBody) + return true; + const envName = environment.name ? environment.name() : environment.constructor.name; + return await this._runHookInternal(worker, testRun, {name: hookName, body: hookBody.bind(environment)}, envName, hookArgs); } - async runBeforeAll(environment, worker, testRun, hookArgs) { - for (const hook of environment.hooks('beforeAll')) { - if (!await this._runHook(worker, testRun, hook, environment.name(), hookArgs)) - return false; - } - return true; - } - - async runAfterEach(environment, worker, testRun, hookArgs) { - for (const hook of environment.hooks('afterEach')) { - if (!await this._runHook(worker, testRun, hook, environment.name(), hookArgs)) - return false; - } - return true; - } - - async runBeforeEach(environment, worker, testRun, hookArgs) { - for (const hook of environment.hooks('beforeEach')) { - if (!await this._runHook(worker, testRun, hook, environment.name(), hookArgs)) - return false; - } - return true; - } - - async ensureGlobalSetup(environment) { + async maybeRunGlobalSetup(environment) { const globalState = this._environmentToGlobalState.get(environment); - if (!globalState.globalSetupPromise) { - globalState.globalSetupPromise = (async () => { - let result = true; - for (const hook of environment.hooks('globalSetup')) { - if (!await this._runHook(null /* worker */, null /* testRun */, hook, environment.name(), [])) - result = false; - } - return result; - })(); - } + if (!globalState.globalSetupPromise) + globalState.globalSetupPromise = this.runHook(environment, 'globalSetup', []); if (!await globalState.globalSetupPromise) { await this._testRunner._terminate(TestResult.Crashed, 'Global setup failed!', false, null); return false; @@ -409,19 +382,11 @@ class HookRunner { return true; } - async ensureGlobalTeardown(environment) { + async maybeRunGlobalTeardown(environment) { const globalState = this._environmentToGlobalState.get(environment); if (!globalState.globalTeardownPromise) { - if (!globalState.pendingTestRuns.size || (this._testRunner._terminating && globalState.globalSetupPromise)) { - globalState.globalTeardownPromise = (async () => { - let result = true; - for (const hook of environment.hooks('globalTeardown')) { - if (!await this._runHook(null /* worker */, null /* testRun */, hook, environment.name(), [])) - result = false; - } - return result; - })(); - } + if (!globalState.pendingTestRuns.size || (this._testRunner._terminating && globalState.globalSetupPromise)) + globalState.globalTeardownPromise = this.runHook(environment, 'globalTeardown', []); } if (!globalState.globalTeardownPromise) return true; @@ -433,18 +398,18 @@ class HookRunner { } async _willStartHook(worker, testRun, hook, fullName) { - debug('testrunner:hook')(`${workerName(worker)} "${fullName}.${hook.name}" started for "${testRun ? testRun.test().fullName() : ''}" (${hook.location})`); + debug('testrunner:hook')(`${workerName(worker)} "${fullName}.${hook.name}" started for "${testRun ? testRun.test().fullName() : ''}"`); } async _didFailHook(worker, testRun, hook, fullName, message, error) { - debug('testrunner:hook')(`${workerName(worker)} "${fullName}.${hook.name}" FAILED for "${testRun ? testRun.test().fullName() : ''}" (${hook.location})`); + debug('testrunner:hook')(`${workerName(worker)} "${fullName}.${hook.name}" FAILED for "${testRun ? testRun.test().fullName() : ''}"`); if (message) this._testRunner._result.addError(message, error, worker); this._testRunner._result.setResult(TestResult.Crashed, message); } async _didCompleteHook(worker, testRun, hook, fullName) { - debug('testrunner:hook')(`${workerName(worker)} "${fullName}.${hook.name}" OK for "${testRun ? testRun.test().fullName() : ''}" (${hook.location})`); + debug('testrunner:hook')(`${workerName(worker)} "${fullName}.${hook.name}" OK for "${testRun ? testRun.test().fullName() : ''}"`); } } diff --git a/utils/testrunner/test/testrunner.spec.js b/utils/testrunner/test/testrunner.spec.js index de5765e250..3655c292ea 100644 --- a/utils/testrunner/test/testrunner.spec.js +++ b/utils/testrunner/test/testrunner.spec.js @@ -281,34 +281,27 @@ module.exports.addTests = function({describe, fdescribe, xdescribe, it, xit, fit it('should run all hooks in proper order', async() => { const log = []; const t = new Runner(); - const e = new Environment('env'); - e.beforeAll(() => log.push('env:beforeAll')); - e.afterAll(() => log.push('env:afterAll')); - e.beforeEach(() => log.push('env:beforeEach')); - e.afterEach(() => log.push('env:afterEach')); - const e2 = new Environment('env2'); - e2.beforeAll(() => log.push('env2:beforeAll')); - e2.afterAll(() => log.push('env2:afterAll')); + const e = { + name() { return 'env'; }, + beforeAll() { log.push('env:beforeAll'); }, + afterAll() { log.push('env:afterAll'); }, + beforeEach() { log.push('env:beforeEach'); }, + afterEach() { log.push('env:afterEach'); }, + }; + const e2 = { + name() { return 'env2'; }, + beforeAll() { log.push('env2:beforeAll'); }, + afterAll() { log.push('env2:afterAll'); }, + }; t.beforeAll(() => log.push('root:beforeAll')); - t.beforeEach(() => log.push('root:beforeEach1')); - t.beforeEach(() => log.push('root:beforeEach2')); + t.beforeEach(() => log.push('root:beforeEach')); t.it('uno', () => log.push('test #1')); t.describe('suite1', () => { - t.beforeAll(() => log.push('suite:beforeAll1')); - t.beforeAll(() => log.push('suite:beforeAll2')); + t.beforeAll(() => log.push('suite:beforeAll')); t.beforeEach(() => log.push('suite:beforeEach')); t.it('dos', () => log.push('test #2')); - t.tests()[t.tests().length - 1].environment().beforeEach(() => log.push('test:before1')); - t.tests()[t.tests().length - 1].environment().beforeEach(() => log.push('test:before2')); - t.tests()[t.tests().length - 1].environment().afterEach(() => log.push('test:after1')); - t.tests()[t.tests().length - 1].environment().afterEach(() => log.push('test:after2')); t.it('tres', () => log.push('test #3')); - t.tests()[t.tests().length - 1].environment().beforeEach(() => log.push('test:before1')); - t.tests()[t.tests().length - 1].environment().beforeEach(() => log.push('test:before2')); - t.tests()[t.tests().length - 1].environment().afterEach(() => log.push('test:after1')); - t.tests()[t.tests().length - 1].environment().afterEach(() => log.push('test:after2')); - t.afterEach(() => log.push('suite:afterEach1')); - t.afterEach(() => log.push('suite:afterEach2')); + t.afterEach(() => log.push('suite:afterEach')); t.afterAll(() => log.push('suite:afterAll')); }); t.it('cuatro', () => log.push('test #4')); @@ -326,41 +319,26 @@ module.exports.addTests = function({describe, fdescribe, xdescribe, it, xit, fit t.suites()[t.suites().length - 1].addEnvironment(e); t.suites()[t.suites().length - 1].addEnvironment(e2); t.afterEach(() => log.push('root:afterEach')); - t.afterAll(() => log.push('root:afterAll1')); - t.afterAll(() => log.push('root:afterAll2')); + t.afterAll(() => log.push('root:afterAll')); await t.run(); expect(log).toEqual([ 'root:beforeAll', - 'root:beforeEach1', - 'root:beforeEach2', + 'root:beforeEach', 'test #1', 'root:afterEach', - 'suite:beforeAll1', - 'suite:beforeAll2', + 'suite:beforeAll', - 'root:beforeEach1', - 'root:beforeEach2', + 'root:beforeEach', 'suite:beforeEach', - 'test:before1', - 'test:before2', 'test #2', - 'test:after1', - 'test:after2', - 'suite:afterEach1', - 'suite:afterEach2', + 'suite:afterEach', 'root:afterEach', - 'root:beforeEach1', - 'root:beforeEach2', + 'root:beforeEach', 'suite:beforeEach', - 'test:before1', - 'test:before2', 'test #3', - 'test:after1', - 'test:after2', - 'suite:afterEach1', - 'suite:afterEach2', + 'suite:afterEach', 'root:afterEach', 'suite:afterAll', @@ -368,16 +346,14 @@ module.exports.addTests = function({describe, fdescribe, xdescribe, it, xit, fit 'env:beforeAll', 'env2:beforeAll', - 'root:beforeEach1', - 'root:beforeEach2', + 'root:beforeEach', 'env:beforeEach', 'test #4', 'env:afterEach', 'root:afterEach', 'suite2:beforeAll', - 'root:beforeEach1', - 'root:beforeEach2', + 'root:beforeEach', 'env:beforeEach', 'test #5', 'env:afterEach', @@ -387,23 +363,26 @@ module.exports.addTests = function({describe, fdescribe, xdescribe, it, xit, fit 'env2:afterAll', 'env:afterAll', - 'root:afterAll1', - 'root:afterAll2', + 'root:afterAll', ]); }); it('should remove environment', async() => { const log = []; const t = new Runner(); - const e = new Environment('env'); - e.beforeAll(() => log.push('env:beforeAll')); - e.afterAll(() => log.push('env:afterAll')); - e.beforeEach(() => log.push('env:beforeEach')); - e.afterEach(() => log.push('env:afterEach')); - const e2 = new Environment('env2'); - e2.beforeAll(() => log.push('env2:beforeAll')); - e2.afterAll(() => log.push('env2:afterAll')); - e2.beforeEach(() => log.push('env2:beforeEach')); - e2.afterEach(() => log.push('env2:afterEach')); + const e = { + name() { return 'env'; }, + beforeAll() { log.push('env:beforeAll'); }, + afterAll() { log.push('env:afterAll'); }, + beforeEach() { log.push('env:beforeEach'); }, + afterEach() { log.push('env:afterEach'); }, + }; + const e2 = { + name() { return 'env2'; }, + beforeAll() { log.push('env2:beforeAll'); }, + afterAll() { log.push('env2:afterAll'); }, + beforeEach() { log.push('env2:beforeEach'); }, + afterEach() { log.push('env2:afterEach'); }, + }; t.it('uno', () => log.push('test #1')); t.tests()[0].addEnvironment(e).addEnvironment(e2).removeEnvironment(e); await t.run();