diff --git a/test/playwright.spec.js b/test/playwright.spec.js index e0af0b399f..b872538f7e 100644 --- a/test/playwright.spec.js +++ b/test/playwright.spec.js @@ -18,7 +18,7 @@ const fs = require('fs'); const path = require('path'); const rm = require('rimraf').sync; const GoldenUtils = require('./golden-utils'); -const {Matchers} = require('../utils/testrunner/'); +const {Matchers} = require('../utils/testrunner/Matchers'); const readline = require('readline'); const {TestServer} = require('../utils/testserver/'); @@ -174,14 +174,13 @@ module.exports.addPlaywrightTests = ({testRunner, platform, products, playwright state._stderr.close(); }); - beforeEach(async(state, test) => { - test.output = []; - const dumpout = data => test.output.push(`\x1b[33m[pw:stdio:out]\x1b[0m ${data}`); - const dumperr = data => test.output.push(`\x1b[31m[pw:stdio:err]\x1b[0m ${data}`); + beforeEach(async(state, testRun) => { + const dumpout = data => testRun.log(`\x1b[33m[pw:stdio:out]\x1b[0m ${data}`); + const dumperr = data => testRun.log(`\x1b[31m[pw:stdio:err]\x1b[0m ${data}`); state._stdout.on('line', dumpout); state._stderr.on('line', dumperr); if (dumpProtocolOnFailure) - state.browser._debugProtocol.log = data => test.output.push(`\x1b[32m[pw:protocol]\x1b[0m ${data}`); + state.browser._debugProtocol.log = data => testRun.log(`\x1b[32m[pw:protocol]\x1b[0m ${data}`); state.tearDown = async () => { state._stdout.off('line', dumpout); state._stderr.off('line', dumperr); diff --git a/test/test.js b/test/test.js index 3818e7fcb1..2bf761782f 100644 --- a/test/test.js +++ b/test/test.js @@ -15,7 +15,7 @@ * limitations under the License. */ -const {TestRunner, Reporter} = require('../utils/testrunner/'); +const TestRunner = require('../utils/testrunner/'); const utils = require('./utils'); const os = require('os'); @@ -38,9 +38,13 @@ if (MAJOR_NODEJS_VERSION >= 8 && require('inspector').url()) { const testRunner = new TestRunner({ timeout, + totalTimeout: process.env.CI ? 15 * 60 * 1000 : 0, parallel, breakOnFailure: process.argv.indexOf('--break-on-failure') !== -1, - installCommonHelpers: false + verbose: process.argv.includes('--verbose'), + summary: !process.argv.includes('--verbose'), + showSlowTests: process.env.CI ? 5 : 0, + showMarkedAsFailingTests: 10, }); utils.setupTestRunner(testRunner); @@ -68,7 +72,7 @@ require('./playwright.spec.js').addPlaywrightTests({ playwrightPath: utils.projectRoot(), products, platform: os.platform(), - testRunner, + testRunner: testRunner.api(), headless: !!valueFromEnv('HEADLESS', true), slowMo: valueFromEnv('SLOW_MO', 0), dumpProtocolOnFailure: valueFromEnv('DEBUGP', false), @@ -81,15 +85,5 @@ if (filterArgIndex !== -1) { testRunner.focusMatchingTests(new RegExp(filter, 'i')); } -new Reporter(testRunner, { - verbose: process.argv.includes('--verbose'), - summary: !process.argv.includes('--verbose'), - showSlowTests: process.env.CI ? 5 : 0, - showMarkedAsFailingTests: 10, -}); - // await utils.initializeFlakinessDashboardIfNeeded(testRunner); -testRunner.run({ totalTimeout: process.env.CI ? 15 * 60 * 1000 : 0 }).then(result => { - process.exit(result.exitCode); -}); - +testRunner.run(); diff --git a/test/utils.js b/test/utils.js index 9172156907..f44e88a10f 100644 --- a/test/utils.js +++ b/test/utils.js @@ -85,9 +85,8 @@ const utils = module.exports = { const ignoredMethods = new Set(ignoredMethodsArray); for (const [className, classType] of Object.entries(api)) traceAPICoverage(coverage, events, className, classType); - const focus = testRunner.hasFocusedTestsOrSuites(); - (focus ? testRunner.fdescribe : testRunner.describe)(COVERAGE_TESTSUITE_NAME, () => { - (focus ? testRunner.fit : testRunner.it)('should call all API methods', () => { + testRunner.describe(COVERAGE_TESTSUITE_NAME, () => { + testRunner.it('should call all API methods', () => { const missingMethods = []; const extraIgnoredMethods = []; for (const method of coverage.keys()) { @@ -292,34 +291,31 @@ const utils = module.exports = { }, setupTestRunner: function(testRunner) { - testRunner.testModifier('skip', (t, condition) => condition && t.setSkipped(true)); - testRunner.suiteModifier('skip', (s, condition) => condition && s.setSkipped(true)); - testRunner.testModifier('fail', (t, condition) => condition && t.setExpectation(t.Expectations.Fail)); - testRunner.suiteModifier('fail', (s, condition) => condition && s.setExpectation(s.Expectations.Fail)); - testRunner.testModifier('slow', (t, condition) => condition && t.setTimeout(t.timeout() * 3)); - testRunner.testModifier('repeat', (t, count) => t.setRepeat(count)); - testRunner.suiteModifier('repeat', (s, count) => s.setRepeat(count)); - testRunner.testAttribute('focus', t => t.setFocused(true)); - testRunner.suiteAttribute('focus', s => s.setFocused(true)); - testRunner.testAttribute('debug', t => { - t.setFocused(true); + const collector = testRunner._collector; + collector.addTestModifier('skip', (t, condition) => condition && t.setSkipped(true)); + collector.addSuiteModifier('skip', (s, condition) => condition && s.setSkipped(true)); + collector.addTestModifier('fail', (t, condition) => condition && t.setExpectation(t.Expectations.Fail)); + collector.addSuiteModifier('fail', (s, condition) => condition && s.setExpectation(s.Expectations.Fail)); + collector.addTestModifier('slow', t => t.setTimeout(t.timeout() * 3)); + collector.addTestAttribute('debug', t => { t.setTimeout(100000000); let session; - t.before(async () => { + t.environment().beforeEach(async () => { + const inspector = require('inspector'); const readFileAsync = util.promisify(fs.readFile.bind(fs)); - session = new require('inspector').Session(); + session = new inspector.Session(); session.connect(); const postAsync = util.promisify(session.post.bind(session)); await postAsync('Debugger.enable'); const setBreakpointCommands = []; const N = t.body().toString().split('\n').length; const location = t.location(); - const lines = (await readFileAsync(location.filePath, 'utf8')).split('\n'); + const lines = (await readFileAsync(location.filePath(), 'utf8')).split('\n'); for (let line = 0; line < N; ++line) { - const lineNumber = line + location.lineNumber; + const lineNumber = line + location.lineNumber(); setBreakpointCommands.push(postAsync('Debugger.setBreakpointByUrl', { - url: url.pathToFileURL(location.filePath), + url: url.pathToFileURL(location.filePath()), lineNumber, condition: `console.log('${String(lineNumber + 1).padStart(6, ' ')} | ' + ${JSON.stringify(lines[lineNumber])})`, }).catch(e => {})); @@ -327,14 +323,14 @@ const utils = module.exports = { await Promise.all(setBreakpointCommands); }); - t.after(async () => { + t.environment().afterEach(async () => { session.disconnect(); }); }); - testRunner.fdescribe = testRunner.describe.focus; - testRunner.xdescribe = testRunner.describe.skip(true); - testRunner.fit = testRunner.it.focus; - testRunner.xit = testRunner.it.skip(true); - testRunner.dit = testRunner.it.debug; + testRunner.api().fdescribe = testRunner.api().describe.only; + testRunner.api().xdescribe = testRunner.api().describe.skip(true); + testRunner.api().fit = testRunner.api().it.only; + testRunner.api().xit = testRunner.api().it.skip(true); + testRunner.api().dit = testRunner.api().it.only.debug; }, }; diff --git a/utils/doclint/check_public_api/test/test.js b/utils/doclint/check_public_api/test/test.js index 99a559e7ce..9634668eba 100644 --- a/utils/doclint/check_public_api/test/test.js +++ b/utils/doclint/check_public_api/test/test.js @@ -22,15 +22,15 @@ const mdBuilder = require('../MDBuilder'); const jsBuilder = require('../JSBuilder'); const GoldenUtils = require('../../../../test/golden-utils'); -const {TestRunner, Reporter, Matchers} = require('../../../testrunner/'); +const {Matchers} = require('../../../testrunner/Matchers'); +const TestRunner = require('../../../testrunner/'); const runner = new TestRunner(); -const reporter = new Reporter(runner); -const {describe, xdescribe, fdescribe} = runner; -const {it, fit, xit} = runner; -const {beforeAll, beforeEach, afterAll, afterEach} = runner; +const {describe, xdescribe, fdescribe} = runner.api(); +const {it, fit, xit} = runner.api(); +const {beforeAll, beforeEach, afterAll, afterEach} = runner.api(); -let browserContext; +let browser; let page; beforeAll(async function() { @@ -60,8 +60,8 @@ describe('checkPublicAPI', function() { runner.run(); -async function testLint(state, test) { - const dirPath = path.join(__dirname, test.name()); +async function testLint(state, testRun) { + const dirPath = path.join(__dirname, testRun.test().name()); const {expect} = new Matchers({ toBeGolden: GoldenUtils.compare.bind(null, dirPath, dirPath) }); @@ -74,8 +74,8 @@ async function testLint(state, test) { expect(errors.join('\n')).toBeGolden('result.txt'); } -async function testMDBuilder(state, test) { - const dirPath = path.join(__dirname, test.name()); +async function testMDBuilder(state, testRun) { + const dirPath = path.join(__dirname, testRun.test().name()); const {expect} = new Matchers({ toBeGolden: GoldenUtils.compare.bind(null, dirPath, dirPath) }); @@ -84,8 +84,8 @@ async function testMDBuilder(state, test) { expect(serialize(documentation)).toBeGolden('result.txt'); } -async function testJSBuilder(state, test) { - const dirPath = path.join(__dirname, test.name()); +async function testJSBuilder(state, testRun) { + const dirPath = path.join(__dirname, testRun.test().name()); const {expect} = new Matchers({ toBeGolden: GoldenUtils.compare.bind(null, dirPath, dirPath) }); diff --git a/utils/doclint/preprocessor/test.js b/utils/doclint/preprocessor/test.js index b28ddfcc9b..031b9a3e51 100644 --- a/utils/doclint/preprocessor/test.js +++ b/utils/doclint/preprocessor/test.js @@ -16,14 +16,13 @@ const {runCommands, ensureReleasedAPILinks} = require('.'); const Source = require('../Source'); -const {TestRunner, Reporter, Matchers} = require('../../testrunner/'); +const TestRunner = require('../../testrunner/'); const runner = new TestRunner(); -new Reporter(runner); -const {describe, xdescribe, fdescribe} = runner; -const {it, fit, xit} = runner; -const {beforeAll, beforeEach, afterAll, afterEach} = runner; -const {expect} = new Matchers(); +const {describe, xdescribe, fdescribe} = runner.api(); +const {it, fit, xit} = runner.api(); +const {beforeAll, beforeEach, afterAll, afterEach} = runner.api(); +const {expect} = runner.api(); describe('ensureReleasedAPILinks', function() { it('should work with non-release version', function() { diff --git a/utils/testrunner/Location.js b/utils/testrunner/Location.js index 1b6d5be1f7..2330f031f5 100644 --- a/utils/testrunner/Location.js +++ b/utils/testrunner/Location.js @@ -49,6 +49,8 @@ class Location { return this._fileName + ':' + this._lineNumber + ':' + this._columnNumber; } + // TODO: static getCallerLocationIn(glob) vs getCallerLocationIgnoring(glob). + static getCallerLocation(filename) { const error = new Error(); const stackFrames = error.stack.split('\n').slice(1); diff --git a/utils/testrunner/Reporter.js b/utils/testrunner/Reporter.js index 8c98c9d365..c3c862157b 100644 --- a/utils/testrunner/Reporter.js +++ b/utils/testrunner/Reporter.js @@ -19,7 +19,7 @@ const colors = require('colors/safe'); const {MatchError} = require('./Matchers.js'); class Reporter { - constructor(runner, options = {}) { + constructor(delegate, options = {}) { const { showSlowTests = 3, showMarkedAsFailingTests = Infinity, @@ -27,27 +27,25 @@ class Reporter { summary = true, } = options; this._filePathToLines = new Map(); - this._runner = runner; + this._delegate = delegate; this._showSlowTests = showSlowTests; this._showMarkedAsFailingTests = showMarkedAsFailingTests; this._verbose = verbose; this._summary = summary; this._testCounter = 0; - runner.setDelegate(this); } onStarted(testRuns) { this._testCounter = 0; this._timestamp = Date.now(); - const allTests = this._runner.tests(); - if (!this._runner.hasFocusedTestsOrSuites()) { - console.log(`Running all ${colors.yellow(testRuns.length)} tests on ${colors.yellow(this._runner.parallel())} worker${this._runner.parallel() > 1 ? 's' : ''}:\n`); + if (!this._delegate.hasFocusedTestsOrSuites()) { + console.log(`Running all ${colors.yellow(testRuns.length)} tests on ${colors.yellow(this._delegate.parallel())} worker${this._delegate.parallel() > 1 ? 's' : ''}:\n`); } else { - console.log(`Running ${colors.yellow(testRuns.length)} focused tests out of total ${colors.yellow(allTests.length)} on ${colors.yellow(this._runner.parallel())} worker${this._runner.parallel() > 1 ? 's' : ''}`); + console.log(`Running ${colors.yellow(testRuns.length)} focused tests out of total ${colors.yellow(this._delegate.testCount())} on ${colors.yellow(this._delegate.parallel())} worker${this._delegate.parallel() > 1 ? 's' : ''}`); console.log(''); const focusedEntities = [ - ...this._runner.suites().filter(suite => suite.focused()), - ...this._runner.tests().filter(test => test.focused()), + ...this._delegate.focusedSuites(), + ...this._delegate.focusedTests(), ]; if (focusedEntities.length) { console.log('Focused Suites and Tests:'); @@ -174,7 +172,7 @@ class Reporter { _printVerboseTestRunResult(resultIndex, testRun) { const test = testRun.test(); let prefix = `${resultIndex})`; - if (this._runner.parallel() > 1) + if (this._delegate.parallel() > 1) prefix += ' ' + colors.gray(`[worker = ${testRun.workerId()}]`); if (testRun.result() === 'ok') { console.log(`${prefix} ${colors.green('[OK]')} ${test.fullName()} (${formatLocation(test.location())})`); @@ -187,9 +185,10 @@ class Reporter { console.log(`${prefix} ${colors.yellow('[MARKED AS FAILING]')} ${test.fullName()} (${formatLocation(test.location())})`); } else if (testRun.result() === 'timedout') { console.log(`${prefix} ${colors.red(`[TIMEOUT ${test.timeout()}ms]`)} ${test.fullName()} (${formatLocation(test.location())})`); - if (testRun.output) { + const output = testRun.output(); + if (output.length) { console.log(' Output:'); - for (const line of testRun.output) + for (const line of output) console.log(' ' + line); } } else if (testRun.result() === 'failed') { @@ -235,9 +234,10 @@ class Reporter { console.log(padLines(stack, 4)); } } - if (testRun.output) { + const output = testRun.output(); + if (output.length) { console.log(' Output:'); - for (const line of testRun.output) + for (const line of output) console.log(' ' + line); } } diff --git a/utils/testrunner/Test.js b/utils/testrunner/Test.js new file mode 100644 index 0000000000..0a2754f9fd --- /dev/null +++ b/utils/testrunner/Test.js @@ -0,0 +1,219 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * Modifications copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const Location = require('./Location'); + +const TestExpectation = { + Ok: 'ok', + Fail: 'fail', +}; + +function createHook(callback, name) { + const location = Location.getCallerLocation(__filename); + return { name, body: callback, location }; +} + +class Environment { + constructor(name, parentEnvironment = null) { + this._parentEnvironment = parentEnvironment; + this._name = name; + this._hooks = []; + } + + parentEnvironment() { + return this._parentEnvironment; + } + + name() { + return this._name; + } + + beforeEach(callback) { + this._hooks.push(createHook(callback, 'beforeEach')); + return this; + } + + afterEach(callback) { + this._hooks.push(createHook(callback, 'afterEach')); + return this; + } + + beforeAll(callback) { + this._hooks.push(createHook(callback, 'beforeAll')); + return this; + } + + afterAll(callback) { + this._hooks.push(createHook(callback, 'afterAll')); + return this; + } + + hooks(name) { + return this._hooks.filter(hook => !name || hook.name === name); + } + + isEmpty() { + return !this._hooks.length; + } +} + +class Test { + constructor(suite, name, callback, location) { + this._suite = suite; + this._name = name; + this._fullName = (suite.fullName() + ' ' + name).trim(); + this._skipped = false; + this._expectation = TestExpectation.Ok; + this._body = callback; + this._location = location; + this._timeout = 100000000; + this._defaultEnvironment = new Environment(this._fullName); + this._environments = [this._defaultEnvironment]; + this.Expectations = { ...TestExpectation }; + } + + suite() { + return this._suite; + } + + name() { + return this._name; + } + + fullName() { + return this._fullName; + } + + location() { + return this._location; + } + + body() { + return this._body; + } + + skipped() { + return this._skipped; + } + + setSkipped(skipped) { + this._skipped = skipped; + return this; + } + + timeout() { + return this._timeout; + } + + setTimeout(timeout) { + this._timeout = timeout; + return this; + } + + expectation() { + return this._expectation; + } + + setExpectation(expectation) { + this._expectation = expectation; + return this; + } + + environment() { + return this._defaultEnvironment; + } + + addEnvironment(environment) { + this._environments.push(environment); + return this; + } + + removeEnvironment(environment) { + const index = this._environments.indexOf(environment); + if (index === -1) + throw new Error(`Environment "${environment.name()}" cannot be removed because it was not added to the suite "${this.fullName()}"`); + this._environments.splice(index, 1); + return this; + } +} + +class Suite { + constructor(parentSuite, name, location) { + this._parentSuite = parentSuite; + this._name = name; + this._fullName = (parentSuite ? parentSuite.fullName() + ' ' + name : name).trim(); + this._location = location; + this._skipped = false; + this._expectation = TestExpectation.Ok; + this._defaultEnvironment = new Environment(this._fullName); + this._environments = [this._defaultEnvironment]; + this.Expectations = { ...TestExpectation }; + } + + parentSuite() { + return this._parentSuite; + } + + name() { + return this._name; + } + + fullName() { + return this._fullName; + } + + skipped() { + return this._skipped; + } + + setSkipped(skipped) { + this._skipped = skipped; + return this; + } + + location() { + return this._location; + } + + expectation() { + return this._expectation; + } + + setExpectation(expectation) { + this._expectation = expectation; + return this; + } + + environment() { + return this._defaultEnvironment; + } + + addEnvironment(environment) { + this._environments.push(environment); + return this; + } + + removeEnvironment(environment) { + const index = this._environments.indexOf(environment); + if (index === -1) + throw new Error(`Environment "${environment.name()}" cannot be removed because it was not added to the suite "${this.fullName()}"`); + this._environments.splice(index, 1); + return this; + } +} + +module.exports = { TestExpectation, Environment, Test, Suite }; diff --git a/utils/testrunner/TestCollector.js b/utils/testrunner/TestCollector.js new file mode 100644 index 0000000000..fc6d695299 --- /dev/null +++ b/utils/testrunner/TestCollector.js @@ -0,0 +1,191 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * Modifications copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const Location = require('./Location'); +const { Test, Suite } = require('./Test'); +const { TestRun } = require('./TestRunner'); + +class FocusedFilter { + constructor() { + this._focused = new Set(); + } + + markFocused(testOrSuite) { + this._focused.add(testOrSuite); + } + + hasFocusedTestsOrSuites() { + return !!this._focused.size; + } + + focusedTests(tests) { + return tests.filter(test => this._focused.has(test)); + } + + focusedSuites(suites) { + return suites.filter(suite => this._focused.has(suite)); + } + + filter(tests) { + if (!this.hasFocusedTestsOrSuites()) + return tests; + + const ignoredSuites = new Set(); + for (const test of tests) { + if (this._focused.has(test)) { + // Focused tests should be run even if skipped. + test.setSkipped(false); + // TODO: remove next line once we run failing tests. + test.setExpectation(test.Expectations.Ok); + } + for (let suite = test.suite(); suite; suite = suite.parentSuite()) { + if (this._focused.has(suite)) { + // Focused suites should be run even if skipped. + suite.setSkipped(false); + // TODO: remove next line once we run failing tests. + suite.setExpectation(suite.Expectations.Ok); + } + // Mark parent suites of focused tests as ignored. + if (this._focused.has(test)) + ignoredSuites.add(suite); + } + } + // Pick all tests that are focused or belong to focused suites. + const result = []; + for (const test of tests) { + let focused = this._focused.has(test); + for (let suite = test.suite(); suite; suite = suite.parentSuite()) + focused = focused || (this._focused.has(suite) && !ignoredSuites.has(suite)); + if (focused) + result.push(test); + } + return result; + } +} + +class Repeater { + constructor() { + this._repeatCount = new Map(); + } + + repeat(testOrSuite, count) { + this._repeatCount.set(testOrSuite, count); + } + + _get(testOrSuite) { + const repeat = this._repeatCount.get(testOrSuite); + return repeat === undefined ? 1 : repeat; + } + + createTestRuns(tests) { + const testRuns = []; + for (const test of tests) { + let repeat = this._get(test); + for (let suite = test.suite(); suite; suite = suite.parentSuite()) + repeat *= this._get(suite); + for (let i = 0; i < repeat; i++) + testRuns.push(new TestRun(test)); + } + return testRuns; + } +} + +function specBuilder(modifiers, attributes, specCallback) { + function builder(specs) { + return new Proxy((...args) => specCallback(specs, ...args), { + get: (obj, prop) => { + if (modifiers.has(prop)) + return (...args) => builder([...specs, { callback: modifiers.get(prop), args }]); + if (attributes.has(prop)) + return builder([...specs, { callback: attributes.get(prop), args: [] }]); + return obj[prop]; + }, + }); + } + return builder([]); +} + +class TestCollector { + constructor(options = {}) { + let { timeout = 10 * 1000 } = options; + if (timeout === 0) + timeout = 100000000; // Inifinite timeout. + + this._tests = []; + this._suites = []; + this._suiteModifiers = new Map(); + this._suiteAttributes = new Map(); + this._testModifiers = new Map(); + this._testAttributes = new Map(); + this._api = {}; + + this._currentSuite = new Suite(null, '', new Location()); + + this._api.describe = specBuilder(this._suiteModifiers, this._suiteAttributes, (specs, name, suiteCallback, ...suiteArgs) => { + const location = Location.getCallerLocation(__filename); + const suite = new Suite(this._currentSuite, name, location); + for (const { callback, args } of specs) + callback(suite, ...args); + this._currentSuite = suite; + suiteCallback(...suiteArgs); + this._suites.push(suite); + this._currentSuite = suite.parentSuite(); + }); + this._api.it = specBuilder(this._testModifiers, this._testAttributes, (specs, name, testCallback) => { + const location = Location.getCallerLocation(__filename); + const test = new Test(this._currentSuite, name, testCallback, location); + test.setTimeout(timeout); + for (const { callback, args } of specs) + callback(test, ...args); + this._tests.push(test); + }); + this._api.beforeAll = callback => this._currentSuite.environment().beforeAll(callback); + this._api.beforeEach = callback => this._currentSuite.environment().beforeEach(callback); + this._api.afterAll = callback => this._currentSuite.environment().afterAll(callback); + this._api.afterEach = callback => this._currentSuite.environment().afterEach(callback); + } + + addTestModifier(name, callback) { + this._testModifiers.set(name, callback); + } + + addTestAttribute(name, callback) { + this._testAttributes.set(name, callback); + } + + addSuiteModifier(name, callback) { + this._suiteModifiers.set(name, callback); + } + + addSuiteAttribute(name, callback) { + this._suiteAttributes.set(name, callback); + } + + api() { + return this._api; + } + + tests() { + return this._tests; + } + + suites() { + return this._suites; + } +} + +module.exports = { TestCollector, specBuilder, FocusedFilter, Repeater }; diff --git a/utils/testrunner/TestRunner.js b/utils/testrunner/TestRunner.js index c42a91e9ec..55f7dc1643 100644 --- a/utils/testrunner/TestRunner.js +++ b/utils/testrunner/TestRunner.js @@ -15,11 +15,10 @@ * limitations under the License. */ -const {SourceMapSupport} = require('./SourceMapSupport'); +const { SourceMapSupport } = require('./SourceMapSupport'); const debug = require('debug'); -const Location = require('./Location'); +const { TestExpectation } = require('./Test'); -const INFINITE_TIMEOUT = 100000000; const TimeoutError = new Error('Timeout'); const TerminatedError = new Error('Terminated'); @@ -37,11 +36,6 @@ function runUserCallback(callback, timeout, args) { return { promise, terminate }; } -const TestExpectation = { - Ok: 'ok', - Fail: 'fail', -}; - const TestResult = { Ok: 'ok', MarkedAsFailing: 'markedAsFailing', // User marked as failed @@ -52,241 +46,6 @@ const TestResult = { Crashed: 'crashed', // If testrunner crashed due to this test }; -function createHook(callback, name) { - const location = Location.getCallerLocation(__filename); - return { name, body: callback, location }; -} - -class Test { - constructor(suite, name, callback, location) { - this._suite = suite; - this._name = name; - this._fullName = (suite.fullName() + ' ' + name).trim(); - this._skipped = false; - this._focused = false; - this._expectation = TestExpectation.Ok; - this._body = callback; - this._location = location; - this._timeout = INFINITE_TIMEOUT; - this._repeat = 1; - this._hooks = []; - this._environments = []; - - this.Expectations = { ...TestExpectation }; - } - - suite() { - return this._suite; - } - - name() { - return this._name; - } - - fullName() { - return this._fullName; - } - - location() { - return this._location; - } - - body() { - return this._body; - } - - skipped() { - return this._skipped; - } - - setSkipped(skipped) { - this._skipped = skipped; - return this; - } - - focused() { - return this._focused; - } - - setFocused(focused) { - this._focused = focused; - return this; - } - - timeout() { - return this._timeout; - } - - setTimeout(timeout) { - this._timeout = timeout; - return this; - } - - expectation() { - return this._expectation; - } - - setExpectation(expectation) { - this._expectation = expectation; - return this; - } - - repeat() { - return this._repeat; - } - - setRepeat(repeat) { - this._repeat = repeat; - return this; - } - - before(callback) { - this._hooks.push(createHook(callback, 'before')); - return this; - } - - after(callback) { - this._hooks.push(createHook(callback, 'after')); - return this; - } - - hooks(name) { - return this._hooks.filter(hook => !name || hook.name === name); - } - - addEnvironment(environment) { - const parents = new Set(); - for (let parent = environment; !(parent instanceof Suite); parent = parent.parentEnvironment()) - parents.add(parent); - for (const env of this._environments) { - for (let parent = env; !(parent instanceof Suite); parent = parent.parentEnvironment()) { - if (parents.has(parent)) - throw new Error(`Cannot use environments "${environment.name()}" and "${env.name()}" that share a parent environment "${parent.fullName()}" in test "${this.fullName()}"`); - } - } - const environmentParentSuite = environment.parentSuite(); - for (let suite = this.suite(); suite; suite = suite.parentSuite()) { - if (suite === environmentParentSuite) { - this._environments.push(environment); - return this; - } - } - throw new Error(`Cannot use environment "${environment.name()}" from suite "${environment.parentSuite().fullName()}" in unrelated test "${this.fullName()}"`); - } - - removeEnvironment(environment) { - const index = this._environments.indexOf(environment); - if (index === -1) - throw new Error(`Environment "${environment.name()}" cannot be removed because it was not added to the test "${test.fullName()}"`); - this._environments.splice(index, 1); - return this; - } -} - -class Environment { - constructor(parentEnvironment, name, location) { - this._parentEnvironment = parentEnvironment; - this._parentSuite = parentEnvironment; - if (parentEnvironment && !(parentEnvironment instanceof Suite)) - this._parentSuite = parentEnvironment.parentSuite(); - this._name = name; - this._fullName = (parentEnvironment ? parentEnvironment.fullName() + ' ' + name : name).trim(); - this._location = location; - this._hooks = []; - } - - parentEnvironment() { - return this._parentEnvironment; - } - - parentSuite() { - return this._parentSuite; - } - - name() { - return this._name; - } - - fullName() { - return this._fullName; - } - - beforeEach(callback) { - this._hooks.push(createHook(callback, 'beforeEach')); - return this; - } - - afterEach(callback) { - this._hooks.push(createHook(callback, 'afterEach')); - return this; - } - - beforeAll(callback) { - this._hooks.push(createHook(callback, 'beforeAll')); - return this; - } - - afterAll(callback) { - this._hooks.push(createHook(callback, 'afterAll')); - return this; - } - - hooks(name) { - return this._hooks.filter(hook => !name || hook.name === name); - } -} - -class Suite extends Environment { - constructor(parentSuite, name, location) { - super(parentSuite, name, location); - this._skipped = false; - this._focused = false; - this._expectation = TestExpectation.Ok; - this._repeat = 1; - this.Expectations = { ...TestExpectation }; - } - - skipped() { - return this._skipped; - } - - setSkipped(skipped) { - this._skipped = skipped; - return this; - } - - focused() { - return this._focused; - } - - setFocused(focused) { - this._focused = focused; - return this; - } - - location() { - return this._location; - } - - expectation() { - return this._expectation; - } - - setExpectation(expectation) { - this._expectation = expectation; - return this; - } - - repeat() { - return this._repeat; - } - - setRepeat(repeat) { - this._repeat = repeat; - return this; - } -} - class TestRun { constructor(test) { this._test = test; @@ -295,6 +54,7 @@ class TestRun { this._startTimestamp = 0; this._endTimestamp = 0; this._workerId = null; + this._output = []; } finished() { @@ -328,6 +88,14 @@ class TestRun { workerId() { return this._workerId; } + + log(log) { + this._output.push(log); + } + + output() { + return this._output; + } } class Result { @@ -397,9 +165,9 @@ class TestWorker { this._runs.push(testRun); const test = testRun.test(); - let skipped = test.skipped() && !test.focused(); + let skipped = test.skipped(); for (let suite = test.suite(); suite; suite = suite.parentSuite()) - skipped = skipped || (suite.skipped() && !suite.focused()); + skipped = skipped || suite.skipped(); if (skipped) { await this._willStartTestRun(testRun); testRun._result = TestResult.Skipped; @@ -410,7 +178,7 @@ class TestWorker { let expectedToFail = test.expectation() === TestExpectation.Fail; for (let suite = test.suite(); suite; suite = suite.parentSuite()) expectedToFail = expectedToFail || (suite.expectation() === TestExpectation.Fail); - if (expectedToFail && !test.focused()) { + if (expectedToFail) { await this._willStartTestRun(testRun); testRun._result = TestResult.MarkedAsFailing; await this._didFinishTestRun(testRun); @@ -418,15 +186,20 @@ class TestWorker { } const environmentStack = []; - for (let suite = test.suite(); suite; suite = suite.parentSuite()) - environmentStack.push(suite); - environmentStack.reverse(); - for (const environment of test._environments) { - const insert = []; - for (let parent = environment; !(parent instanceof Suite); parent = parent.parentEnvironment()) - insert.push(parent); - environmentStack.splice(environmentStack.indexOf(environment.parentSuite()) + 1, 0, ...insert.reverse()); + function appendEnvironment(e) { + while (e) { + if (!e.isEmpty()) + environmentStack.push(e); + e = e.parentEnvironment(); + } } + for (const environment of test._environments.slice().reverse()) + appendEnvironment(environment); + for (let suite = test.suite(); suite; suite = suite.parentSuite()) { + for (const environment of suite._environments.slice().reverse()) + appendEnvironment(environment); + } + environmentStack.reverse(); let common = 0; while (common < environmentStack.length && this._environmentStack[common] === environmentStack[common]) @@ -437,7 +210,7 @@ class TestWorker { return; const environment = this._environmentStack.pop(); for (const hook of environment.hooks('afterAll')) { - if (!await this._runHook(testRun, hook, environment.fullName())) + if (!await this._runHook(testRun, hook, environment.name())) return; } } @@ -447,7 +220,7 @@ class TestWorker { const environment = environmentStack[this._environmentStack.length]; this._environmentStack.push(environment); for (const hook of environment.hooks('beforeAll')) { - if (!await this._runHook(testRun, hook, environment.fullName())) + if (!await this._runHook(testRun, hook, environment.name())) return; } } @@ -461,14 +234,12 @@ class TestWorker { await this._willStartTestRun(testRun); for (const environment of this._environmentStack) { for (const hook of environment.hooks('beforeEach')) - await this._runHook(testRun, hook, environment.fullName(), true); + await this._runHook(testRun, hook, environment.name(), true); } - for (const hook of test.hooks('before')) - await this._runHook(testRun, hook, test.fullName(), true); if (!testRun._error && !this._markTerminated(testRun)) { await this._willStartTestBody(testRun); - const { promise, terminate } = runUserCallback(test.body(), test.timeout(), [this._state, test]); + const { promise, terminate } = runUserCallback(test.body(), test.timeout(), [this._state, testRun]); this._runningTestTerminate = terminate; testRun._error = await promise; this._runningTestTerminate = null; @@ -485,19 +256,17 @@ class TestWorker { await this._didFinishTestBody(testRun); } - for (const hook of test.hooks('after')) - await this._runHook(testRun, hook, test.fullName(), true); for (const environment of this._environmentStack.slice().reverse()) { for (const hook of environment.hooks('afterEach')) - await this._runHook(testRun, hook, environment.fullName(), true); + await this._runHook(testRun, hook, environment.name(), true); } await this._didFinishTestRun(testRun); } - async _runHook(testRun, hook, fullName, passTest = false) { + async _runHook(testRun, hook, fullName, passTestRun = false) { await this._willStartHook(hook, fullName); - const timeout = this._testRunner._timeout; - const { promise, terminate } = runUserCallback(hook.body, timeout, passTest ? [this._state, testRun.test()] : [this._state]); + const timeout = this._testRunner._hookTimeout; + const { promise, terminate } = runUserCallback(hook.body, timeout, passTestRun ? [this._state, testRun] : [this._state]); this._runningHookTerminate = terminate; let error = await promise; this._runningHookTerminate = null; @@ -569,198 +338,84 @@ class TestWorker { while (this._environmentStack.length > 0) { const environment = this._environmentStack.pop(); for (const hook of environment.hooks('afterAll')) - await this._runHook(null, hook, environment.fullName()); + await this._runHook(null, hook, environment.name()); } } } class TestRunner { - constructor(options = {}) { - const { - timeout = 10 * 1000, // Default timeout is 10 seconds. - parallel = 1, - breakOnFailure = false, - crashIfTestsAreFocusedOnCI = true, - installCommonHelpers = true, - } = options; - this._crashIfTestsAreFocusedOnCI = crashIfTestsAreFocusedOnCI; + constructor() { this._sourceMapSupport = new SourceMapSupport(); - this._rootSuite = new Suite(null, '', new Location()); - this._currentEnvironment = this._rootSuite; - this._tests = []; - this._suites = []; - this._timeout = timeout === 0 ? INFINITE_TIMEOUT : timeout; - this._parallel = parallel; - this._breakOnFailure = breakOnFailure; - this._suiteModifiers = new Map(); - this._suiteAttributes = new Map(); - this._testModifiers = new Map(); - this._testAttributes = new Map(); this._nextWorkerId = 1; this._workers = []; this._terminating = false; this._result = null; + } + async run(testRuns, options = {}) { + const { + parallel = 1, + breakOnFailure = false, + hookTimeout = 10 * 1000, + totalTimeout = 0, + onStarted = async (testRuns) => {}, + onFinished = async (result) => {}, + onTestRunStarted = async(testRun) => {}, + onTestRunFinished = async (testRun) => {}, + } = options; + this._breakOnFailure = breakOnFailure; + this._hookTimeout = hookTimeout; this._delegate = { - async onStarted(testRuns) {}, - async onFinished(result) {}, - async onTestRunStarted(testRun) {}, - async onTestRunFinished(testRun) {}, + onStarted, + onFinished, + onTestRunStarted, + onTestRunFinished }; - this.beforeAll = (callback) => this._currentEnvironment.beforeAll(callback); - this.beforeEach = (callback) => this._currentEnvironment.beforeEach(callback); - this.afterAll = (callback) => this._currentEnvironment.afterAll(callback); - this.afterEach = (callback) => this._currentEnvironment.afterEach(callback); - - this.describe = this._suiteBuilder([]); - this.it = this._testBuilder([]); - this.environment = (name, callback) => { - const location = Location.getCallerLocation(__filename); - const environment = new Environment(this._currentEnvironment, name, location); - this._currentEnvironment = environment; - callback(); - this._currentEnvironment = environment.parentEnvironment(); - return environment; - }; - this.Expectations = { ...TestExpectation }; - - if (installCommonHelpers) { - this.fdescribe = this._suiteBuilder([{ callback: s => s.setFocused(true), args: [] }]); - this.xdescribe = this._suiteBuilder([{ callback: s => s.setSkipped(true), args: [] }]); - this.fit = this._testBuilder([{ callback: t => t.setFocused(true), args: [] }]); - this.xit = this._testBuilder([{ callback: t => t.setSkipped(true), args: [] }]); - } - } - - _suiteBuilder(callbacks) { - return new Proxy((name, callback, ...suiteArgs) => { - if (!(this._currentEnvironment instanceof Suite)) - throw new Error(`Cannot define a suite inside an environment`); - const location = Location.getCallerLocation(__filename); - const suite = new Suite(this._currentEnvironment, name, location); - for (const { callback, args } of callbacks) - callback(suite, ...args); - this._currentEnvironment = suite; - callback(...suiteArgs); - this._suites.push(suite); - this._currentEnvironment = suite.parentSuite(); - return suite; - }, { - get: (obj, prop) => { - if (this._suiteModifiers.has(prop)) - return (...args) => this._suiteBuilder([...callbacks, { callback: this._suiteModifiers.get(prop), args }]); - if (this._suiteAttributes.has(prop)) - return this._suiteBuilder([...callbacks, { callback: this._suiteAttributes.get(prop), args: [] }]); - return obj[prop]; - }, - }); - } - - _testBuilder(callbacks) { - return new Proxy((name, callback) => { - if (!(this._currentEnvironment instanceof Suite)) - throw new Error(`Cannot define a test inside an environment`); - const location = Location.getCallerLocation(__filename); - const test = new Test(this._currentEnvironment, name, callback, location); - test.setTimeout(this._timeout); - for (const { callback, args } of callbacks) - callback(test, ...args); - this._tests.push(test); - return test; - }, { - get: (obj, prop) => { - if (this._testModifiers.has(prop)) - return (...args) => this._testBuilder([...callbacks, { callback: this._testModifiers.get(prop), args }]); - if (this._testAttributes.has(prop)) - return this._testBuilder([...callbacks, { callback: this._testAttributes.get(prop), args: [] }]); - return obj[prop]; - }, - }); - } - - testModifier(name, callback) { - this._testModifiers.set(name, callback); - } - - testAttribute(name, callback) { - this._testAttributes.set(name, callback); - } - - suiteModifier(name, callback) { - this._suiteModifiers.set(name, callback); - } - - suiteAttribute(name, callback) { - this._suiteAttributes.set(name, callback); - } - - setDelegate(delegate) { - this._delegate = delegate; - } - - async run(options = {}) { - const { totalTimeout = 0 } = options; - const testRuns = []; - for (const test of this._testsToRun()) { - let repeat = test.repeat(); - for (let suite = test.suite(); suite; suite = suite.parentSuite()) - repeat *= suite.repeat(); - for (let i = 0; i < repeat; i++) - testRuns.push(new TestRun(test)); - } - this._result = new Result(); + this._result.runs = testRuns; + await this._delegate.onStarted(testRuns); - if (this._crashIfTestsAreFocusedOnCI && process.env.CI && this.hasFocusedTestsOrSuites()) { - await this._delegate.onStarted([]); - this._result.setResult(TestResult.Crashed, '"focused" tests or suites are probitted on CI'); - await this._delegate.onFinished(this._result); - } else { - await this._delegate.onStarted(testRuns); - this._result.runs = testRuns; + let timeoutId; + if (totalTimeout) { + timeoutId = setTimeout(() => { + this._terminate(TestResult.Terminated, `Total timeout of ${totalTimeout}ms reached.`, true /* force */, null /* error */); + }, totalTimeout); + } - let timeoutId; - if (totalTimeout) { - timeoutId = setTimeout(() => { - this._terminate(TestResult.Terminated, `Total timeout of ${totalTimeout}ms reached.`, true /* force */, null /* error */); - }, totalTimeout); - } + const terminations = [ + createTermination.call(this, 'SIGINT', TestResult.Terminated, 'SIGINT received'), + createTermination.call(this, 'SIGHUP', TestResult.Terminated, 'SIGHUP received'), + createTermination.call(this, 'SIGTERM', TestResult.Terminated, 'SIGTERM received'), + createTermination.call(this, 'unhandledRejection', TestResult.Crashed, 'UNHANDLED PROMISE REJECTION'), + createTermination.call(this, 'uncaughtException', TestResult.Crashed, 'UNHANDLED ERROR'), + ]; + for (const termination of terminations) + process.on(termination.event, termination.handler); - const terminations = [ - createTermination.call(this, 'SIGINT', TestResult.Terminated, 'SIGINT received'), - createTermination.call(this, 'SIGHUP', TestResult.Terminated, 'SIGHUP received'), - createTermination.call(this, 'SIGTERM', TestResult.Terminated, 'SIGTERM received'), - createTermination.call(this, 'unhandledRejection', TestResult.Crashed, 'UNHANDLED PROMISE REJECTION'), - createTermination.call(this, 'uncaughtException', TestResult.Crashed, 'UNHANDLED ERROR'), - ]; - for (const termination of terminations) - process.on(termination.event, termination.handler); + const workerCount = Math.min(parallel, testRuns.length); + const workerPromises = []; + for (let i = 0; i < workerCount; ++i) { + const initialTestRunIndex = i * Math.floor(testRuns.length / workerCount); + workerPromises.push(this._runWorker(initialTestRunIndex, testRuns, i)); + } + await Promise.all(workerPromises); - const parallel = Math.min(this._parallel, testRuns.length); - const workerPromises = []; - for (let i = 0; i < parallel; ++i) { - const initialTestRunIndex = i * Math.floor(testRuns.length / parallel); - workerPromises.push(this._runWorker(initialTestRunIndex, testRuns, i)); - } - await Promise.all(workerPromises); + for (const termination of terminations) + process.removeListener(termination.event, termination.handler); - for (const termination of terminations) - process.removeListener(termination.event, termination.handler); + if (testRuns.some(run => run.isFailure())) + this._result.setResult(TestResult.Failed, ''); - if (testRuns.some(run => run.isFailure())) - this._result.setResult(TestResult.Failed, ''); + clearTimeout(timeoutId); + await this._delegate.onFinished(this._result); - clearTimeout(timeoutId); - await this._delegate.onFinished(this._result); - - function createTermination(event, result, message) { - return { - event, - message, - handler: error => this._terminate(result, message, event === 'SIGTERM', event.startsWith('SIG') ? null : error), - }; - } + function createTermination(event, result, message) { + return { + event, + message, + handler: error => this._terminate(result, message, event === 'SIGTERM', event.startsWith('SIG') ? null : error), + }; } const result = this._result; @@ -818,61 +473,11 @@ class TestRunner { } } - _testsToRun() { - if (!this.hasFocusedTestsOrSuites()) - return this._tests; - const notFocusedSuites = new Set(); - // Mark parent suites of focused tests as not focused. - for (const test of this._tests) { - if (test.focused()) { - for (let suite = test.suite(); suite; suite = suite.parentSuite()) - notFocusedSuites.add(suite); - } - } - // Pick all tests that are focused or belong to focused suites. - const tests = []; - for (const test of this._tests) { - let focused = test.focused(); - for (let suite = test.suite(); suite; suite = suite.parentSuite()) - focused = focused || (suite.focused() && !notFocusedSuites.has(suite)); - if (focused) - tests.push(test); - } - return tests; - } - async terminate() { if (!this._result) return; await this._terminate(TestResult.Terminated, 'Terminated with |TestRunner.terminate()| call', true /* force */, null /* error */); } - - timeout() { - return this._timeout; - } - - hasFocusedTestsOrSuites() { - return this._tests.some(test => test.focused()) || this._suites.some(suite => suite.focused()); - } - - focusMatchingTests(fullNameRegex) { - for (const test of this._tests) { - if (fullNameRegex.test(test.fullName())) - test.setFocused(true); - } - } - - tests() { - return this._tests.slice(); - } - - suites() { - return this._suites.slice(); - } - - parallel() { - return this._parallel; - } } -module.exports = TestRunner; +module.exports = { TestRunner, TestRun, TestResult, Result }; diff --git a/utils/testrunner/index.js b/utils/testrunner/index.js index 84f9410671..34ed948c6d 100644 --- a/utils/testrunner/index.js +++ b/utils/testrunner/index.js @@ -14,8 +14,125 @@ * limitations under the License. */ -const TestRunner = require('./TestRunner'); +const { TestRunner, Result, TestResult } = require('./TestRunner'); +const { TestCollector, FocusedFilter, Repeater } = require('./TestCollector'); const Reporter = require('./Reporter'); -const {Matchers} = require('./Matchers'); +const { Matchers } = require('./Matchers'); -module.exports = { TestRunner, Reporter, Matchers }; +class DefaultTestRunner { + constructor(options = {}) { + const { + // Our options. + crashIfTestsAreFocusedOnCI = true, + exit = true, + reporter = true, + // Collector options. + timeout, + // Runner options. + parallel = 1, + breakOnFailure, + totalTimeout, + hookTimeout = timeout, + // Reporting options. + showSlowTests, + showMarkedAsFailingTests, + verbose, + summary, + } = options; + + this._crashIfTestsAreFocusedOnCI = crashIfTestsAreFocusedOnCI; + this._exit = exit; + this._parallel = parallel; + this._breakOnFailure = breakOnFailure; + this._totalTimeout = totalTimeout; + this._hookTimeout = hookTimeout; + this._needReporter = reporter; + this._showSlowTests = showSlowTests; + this._showMarkedAsFailingTests = showMarkedAsFailingTests; + this._verbose = verbose; + this._summary = summary; + + this._filter = new FocusedFilter(); + this._repeater = new Repeater(); + this._collector = new TestCollector({ timeout }); + + this._api = { + ...this._collector.api(), + expect: new Matchers().expect, + }; + this._collector.addSuiteAttribute('only', s => this._filter.markFocused(s)); + this._collector.addSuiteAttribute('skip', s => s.setSkipped(true)); + this._collector.addSuiteModifier('repeat', (s, count) => this._repeater.repeat(s, count)); + this._collector.addTestAttribute('only', t => this._filter.markFocused(t)); + this._collector.addTestAttribute('skip', t => t.setSkipped(true)); + this._collector.addTestAttribute('todo', t => t.setSkipped(true)); + this._collector.addTestAttribute('slow', t => t.setTimeout(t.timeout() * 3)); + this._collector.addTestModifier('repeat', (t, count) => this._repeater.repeat(t, count)); + this._api.fdescribe = this._api.describe.only; + this._api.xdescribe = this._api.describe.skip; + this._api.fit = this._api.it.only; + this._api.xit = this._api.it.skip; + } + + api() { + return this._api; + } + + focusMatchingTests(fullNameRegex) { + for (const test of this._collector.tests()) { + if (fullNameRegex.test(test.fullName())) + this._filter.markFocused(test); + } + } + + async run() { + let reporter = null; + + if (this._needReporter) { + const reporterDelegate = { + focusedSuites: () => this._filter.focusedTests(this._collector.suites()), + focusedTests: () => this._filter.focusedSuites(this._collector.tests()), + hasFocusedTestsOrSuites: () => this._filter.hasFocusedTestsOrSuites(), + parallel: () => this._parallel, + testCount: () => this._collector.tests().length, + }; + const reporterOptions = { + showSlowTests: this._showSlowTests, + showMarkedAsFailingTests: this._showMarkedAsFailingTests, + verbose: this._verbose, + summary: this._summary, + }; + reporter = new Reporter(reporterDelegate, reporterOptions); + } + + if (this._crashIfTestsAreFocusedOnCI && process.env.CI && this._filter.hasFocusedTestsOrSuites()) { + if (reporter) + await reporter.onStarted([]); + const result = new Result(); + result.setResult(TestResult.Crashed, '"focused" tests or suites are probitted on CI'); + if (reporter) + await reporter.onFinished(result); + if (this._exit) + process.exit(result.exitCode); + return result; + } + + const testRuns = this._repeater.createTestRuns(this._filter.filter(this._collector.tests())); + const testRunner = new TestRunner(); + const result = await testRunner.run(testRuns, { + parallel: this._parallel, + breakOnFailure: this._breakOnFailure, + totalTimeout: this._totalTimeout, + hookTimeout: this._hookTimeout, + onStarted: (...args) => reporter && reporter.onStarted(...args), + onFinished: (...args) => reporter && reporter.onFinished(...args), + onTestRunStarted: (...args) => reporter && reporter.onTestRunStarted(...args), + onTestRunFinished: (...args) => reporter && reporter.onTestRunFinished(...args), + }); + if (this._exit) + process.exit(result.exitCode); + return result; + } +} + +module.exports = DefaultTestRunner; diff --git a/utils/testrunner/test/test.js b/utils/testrunner/test/test.js index f18570038c..304b847967 100644 --- a/utils/testrunner/test/test.js +++ b/utils/testrunner/test/test.js @@ -1,14 +1,4 @@ -const {TestRunner, Matchers, Reporter} = require('..'); - +const TestRunner = require('..'); const testRunner = new TestRunner(); -const {expect} = new Matchers(); - -require('./testrunner.spec.js').addTests({testRunner, expect}); - -new Reporter(testRunner, { - verbose: process.argv.includes('--verbose'), - summary: true, - showSlowTests: 0, -}); +require('./testrunner.spec.js').addTests(testRunner.api()); testRunner.run(); - diff --git a/utils/testrunner/test/testrunner.spec.js b/utils/testrunner/test/testrunner.spec.js index 092bd1f5b6..55e985c97c 100644 --- a/utils/testrunner/test/testrunner.spec.js +++ b/utils/testrunner/test/testrunner.spec.js @@ -1,25 +1,70 @@ -const {TestRunner} = require('..'); +const { TestRunner } = require('../TestRunner'); +const { TestCollector, FocusedFilter, Repeater } = require('../TestCollector'); +const { TestExpectation, Environment } = require('../Test'); -function newTestRunner(options) { - return new TestRunner({ - crashIfTestsAreFocusedOnCI: false, - ...options, - }); +class Runner { + constructor(options = {}) { + this._options = options; + this._filter = new FocusedFilter(); + this._repeater = new Repeater(); + this._collector = new TestCollector(options); + this._collector.addSuiteAttribute('only', s => this._filter.markFocused(s)); + this._collector.addTestAttribute('only', t => this._filter.markFocused(t)); + this._collector.addSuiteAttribute('skip', s => s.setSkipped(true)); + this._collector.addTestAttribute('skip', t => t.setSkipped(true)); + this._collector.addTestAttribute('fail', t => t.setExpectation(t.Expectations.Fail)); + this._collector.addSuiteModifier('repeat', (s, count) => this._repeater.repeat(s, count)); + this._collector.addTestModifier('repeat', (t, count) => this._repeater.repeat(t, count)); + + const api = this._collector.api(); + for (const [key, value] of Object.entries(api)) + this[key] = value; + this.fdescribe = api.describe.only; + this.xdescribe = api.describe.skip; + this.fit = api.it.only; + this.xit = api.it.skip; + this.Expectations = { ...TestExpectation }; + } + + createTestRuns() { + return this._repeater.createTestRuns(this._filter.filter(this._collector.tests())); + } + + run() { + this._testRunner = new TestRunner(); + return this._testRunner.run(this.createTestRuns(), this._options); + } + + tests() { + return this._collector.tests(); + } + + focusedTests() { + return this._filter.focusedTests(this._collector.tests()); + } + + suites() { + return this._collector.suites(); + } + + focusedSuites() { + return this._filter.focusedSuites(this._collector.suites()); + } + + terminate() { + this._testRunner.terminate(); + } } -module.exports.addTests = function({testRunner, expect}) { - const {describe, fdescribe, xdescribe} = testRunner; - const {it, xit, fit} = testRunner; - +module.exports.addTests = function({describe, fdescribe, xdescribe, it, xit, fit, expect}) { describe('TestRunner.it', () => { it('should declare a test', async() => { - const t = newTestRunner(); + const t = new Runner(); t.it('uno', () => {}); expect(t.tests().length).toBe(1); const test = t.tests()[0]; expect(test.name()).toBe('uno'); expect(test.fullName()).toBe('uno'); - expect(test.focused()).toBe(false); expect(test.skipped()).toBe(false); expect(test.location().filePath()).toEqual(__filename); expect(test.location().fileName()).toEqual('testrunner.spec.js'); @@ -27,7 +72,7 @@ module.exports.addTests = function({testRunner, expect}) { expect(test.location().columnNumber()).toBeTruthy(); }); it('should run a test', async() => { - const t = newTestRunner(); + const t = new Runner(); t.it('uno', () => {}); const result = await t.run(); expect(result.runs.length).toBe(1); @@ -38,17 +83,16 @@ module.exports.addTests = function({testRunner, expect}) { describe('TestRunner.xit', () => { it('should declare a skipped test', async() => { - const t = newTestRunner(); + const t = new Runner(); t.xit('uno', () => {}); expect(t.tests().length).toBe(1); const test = t.tests()[0]; expect(test.name()).toBe('uno'); expect(test.fullName()).toBe('uno'); - expect(test.focused()).toBe(false); expect(test.skipped()).toBe(true); }); it('should not run a skipped test', async() => { - const t = newTestRunner(); + const t = new Runner(); t.xit('uno', () => {}); const result = await t.run(); expect(result.runs.length).toBe(1); @@ -59,17 +103,17 @@ module.exports.addTests = function({testRunner, expect}) { describe('TestRunner.fit', () => { it('should declare a focused test', async() => { - const t = newTestRunner(); + const t = new Runner(); t.fit('uno', () => {}); expect(t.tests().length).toBe(1); const test = t.tests()[0]; expect(test.name()).toBe('uno'); expect(test.fullName()).toBe('uno'); - expect(test.focused()).toBe(true); expect(test.skipped()).toBe(false); + expect(t.focusedTests()[0]).toBe(test); }); it('should run a focused test', async() => { - const t = newTestRunner(); + const t = new Runner(); t.fit('uno', () => {}); const result = await t.run(); expect(result.runs.length).toBe(1); @@ -77,12 +121,12 @@ module.exports.addTests = function({testRunner, expect}) { expect(result.runs[0].result()).toBe('ok'); }); it('should run a failed focused test', async() => { - const t = newTestRunner(); + const t = new Runner(); let run = false; - t.it('uno', () => { + t.it.only.fail('uno', () => { run = true; throw new Error('failure'); - }).setFocused(true).setExpectation(t.Expectations.Fail); - expect(t.tests()[0].focused()).toBe(true); + }); + expect(t.focusedTests().length).toBe(1); expect(t.tests()[0].expectation()).toBe(t.Expectations.Fail); const result = await t.run(); expect(run).toBe(true); @@ -94,7 +138,7 @@ module.exports.addTests = function({testRunner, expect}) { describe('TestRunner.describe', () => { it('should declare a suite', async() => { - const t = newTestRunner(); + const t = new Runner(); t.describe('suite', () => { t.it('uno', () => {}); }); @@ -102,30 +146,26 @@ module.exports.addTests = function({testRunner, expect}) { const test = t.tests()[0]; expect(test.name()).toBe('uno'); expect(test.fullName()).toBe('suite uno'); - expect(test.focused()).toBe(false); expect(test.skipped()).toBe(false); expect(test.suite().name()).toBe('suite'); expect(test.suite().fullName()).toBe('suite'); - expect(test.suite().focused()).toBe(false); expect(test.suite().skipped()).toBe(false); }); }); describe('TestRunner.xdescribe', () => { it('should declare a skipped suite', async() => { - const t = newTestRunner(); + const t = new Runner(); t.xdescribe('suite', () => { t.it('uno', () => {}); }); expect(t.tests().length).toBe(1); const test = t.tests()[0]; - expect(test.focused()).toBe(false); expect(test.skipped()).toBe(false); - expect(test.suite().focused()).toBe(false); expect(test.suite().skipped()).toBe(true); }); it('focused tests inside a skipped suite are not run', async() => { - const t = newTestRunner(); + const t = new Runner(); let run = false; t.xdescribe('suite', () => { t.fit('uno', () => { run = true; }); @@ -140,19 +180,18 @@ module.exports.addTests = function({testRunner, expect}) { describe('TestRunner.fdescribe', () => { it('should declare a focused suite', async() => { - const t = newTestRunner(); + const t = new Runner(); t.fdescribe('suite', () => { t.it('uno', () => {}); }); expect(t.tests().length).toBe(1); const test = t.tests()[0]; - expect(test.focused()).toBe(false); expect(test.skipped()).toBe(false); - expect(test.suite().focused()).toBe(true); + expect(t.focusedSuites()[0]).toBe(test.suite()); expect(test.suite().skipped()).toBe(false); }); it('skipped tests inside a focused suite should not be run', async() => { - const t = newTestRunner(); + const t = new Runner(); t.fdescribe('suite', () => { t.xit('uno', () => {}); }); @@ -163,7 +202,7 @@ module.exports.addTests = function({testRunner, expect}) { }); it('should run all "run" tests inside a focused suite', async() => { const log = []; - const t = newTestRunner(); + const t = new Runner(); t.it('uno', () => log.push(1)); t.fdescribe('suite1', () => { t.it('dos', () => log.push(2)); @@ -175,7 +214,7 @@ module.exports.addTests = function({testRunner, expect}) { }); it('should run only "focus" tests inside a focused suite', async() => { const log = []; - const t = newTestRunner(); + const t = new Runner(); t.it('uno', () => log.push(1)); t.fdescribe('suite1', () => { t.fit('dos', () => log.push(2)); @@ -187,7 +226,7 @@ module.exports.addTests = function({testRunner, expect}) { }); it('should run both "run" tests in focused suite and non-descendant focus tests', async() => { const log = []; - const t = newTestRunner(); + const t = new Runner(); t.it('uno', () => log.push(1)); t.fdescribe('suite1', () => { t.it('dos', () => log.push(2)); @@ -201,38 +240,32 @@ module.exports.addTests = function({testRunner, expect}) { describe('TestRunner attributes', () => { it('should work', async() => { - const t = newTestRunner({timeout: 123}); + const t = new Runner({timeout: 123}); const log = []; - t.testModifier('foo', (t, ...args) => { + t._collector.addTestModifier('foo', (t, ...args) => { log.push('foo'); - expect(t.focused()).toBe(false); expect(t.skipped()).toBe(false); expect(t.Expectations.Ok).toBeTruthy(); expect(t.Expectations.Fail).toBeTruthy(); expect(t.expectation()).toBe(t.Expectations.Ok); expect(t.timeout()).toBe(123); - expect(t.repeat()).toBe(1); expect(args.length).toBe(2); expect(args[0]).toBe('uno'); expect(args[1]).toBe('dos'); - t.setFocused(true); t.setExpectation(t.Expectations.Fail); t.setTimeout(234); - t.setRepeat(42); }); - t.testAttribute('bar', t => { + t._collector.addTestAttribute('bar', t => { log.push('bar'); t.setSkipped(true); - expect(t.focused()).toBe(true); expect(t.skipped()).toBe(true); expect(t.expectation()).toBe(t.Expectations.Fail); expect(t.timeout()).toBe(234); - expect(t.repeat()).toBe(42); }); t.it.foo('uno', 'dos').bar('test', () => { }); @@ -243,18 +276,15 @@ module.exports.addTests = function({testRunner, expect}) { describe('TestRunner hooks', () => { it('should run all hooks in proper order', async() => { const log = []; - const t = newTestRunner(); - let e2; - t.environment('env', () => { - t.beforeAll(() => log.push('env:beforeAll')); - t.afterAll(() => log.push('env:afterAll')); - t.beforeEach(() => log.push('env:beforeEach')); - t.afterEach(() => log.push('env:afterEach')); - e2 = t.environment('env2', () => { - t.beforeAll(() => log.push('env2:beforeAll')); - t.afterAll(() => log.push('env2:afterAll')); - }); - }); + const t = new Runner(); + const e = new Environment('env'); + e.beforeAll(() => log.push('env:beforeAll')); + e.afterAll(() => log.push('env:afterAll')); + e.beforeEach(() => log.push('env:beforeEach')); + e.afterEach(() => log.push('env:afterEach')); + const e2 = new Environment('env2', e); + e2.beforeAll(() => log.push('env2:beforeAll')); + e2.afterAll(() => log.push('env2:afterAll')); t.beforeAll(() => log.push('root:beforeAll')); t.beforeEach(() => log.push('root:beforeEach1')); t.beforeEach(() => log.push('root:beforeEach2')); @@ -262,29 +292,33 @@ module.exports.addTests = function({testRunner, expect}) { t.describe('suite1', () => { t.beforeAll(() => log.push('suite:beforeAll1')); t.beforeAll(() => log.push('suite:beforeAll2')); - t.beforeEach((state, test) => { - log.push('suite:beforeEach'); - test.before(() => log.push('test:before1')); - test.before(() => log.push('test:before2')); - test.after(() => log.push('test:after1')); - test.after(() => log.push('test:after2')); - }); + t.beforeEach(() => log.push('suite:beforeEach')); t.it('dos', () => log.push('test #2')); + t.tests()[t.tests().length - 1].environment().beforeEach(() => log.push('test:before1')); + t.tests()[t.tests().length - 1].environment().beforeEach(() => log.push('test:before2')); + t.tests()[t.tests().length - 1].environment().afterEach(() => log.push('test:after1')); + t.tests()[t.tests().length - 1].environment().afterEach(() => log.push('test:after2')); t.it('tres', () => log.push('test #3')); + t.tests()[t.tests().length - 1].environment().beforeEach(() => log.push('test:before1')); + t.tests()[t.tests().length - 1].environment().beforeEach(() => log.push('test:before2')); + t.tests()[t.tests().length - 1].environment().afterEach(() => log.push('test:after1')); + t.tests()[t.tests().length - 1].environment().afterEach(() => log.push('test:after2')); t.afterEach(() => log.push('suite:afterEach1')); t.afterEach(() => log.push('suite:afterEach2')); t.afterAll(() => log.push('suite:afterAll')); }); - t.it('cuatro', () => log.push('test #4')).addEnvironment(e2); + t.it('cuatro', () => log.push('test #4')); + t.tests()[t.tests().length - 1].addEnvironment(e2); t.describe('no hooks suite', () => { t.describe('suite2', () => { t.beforeAll(() => log.push('suite2:beforeAll')); t.afterAll(() => log.push('suite2:afterAll')); t.describe('no hooks suite 2', () => { - t.it('cinco', () => log.push('test #5')).addEnvironment(e2); + t.it('cinco', () => log.push('test #5')); }); }); }); + t.suites()[t.suites().length - 1].addEnvironment(e2); t.afterEach(() => log.push('root:afterEach')); t.afterAll(() => log.push('root:afterAll1')); t.afterAll(() => log.push('root:afterAll2')); @@ -353,20 +387,19 @@ module.exports.addTests = function({testRunner, expect}) { }); it('should remove environment', async() => { const log = []; - const t = newTestRunner(); - const e = t.environment('env', () => { - t.beforeAll(() => log.push('env:beforeAll')); - t.afterAll(() => log.push('env:afterAll')); - t.beforeEach(() => log.push('env:beforeEach')); - t.afterEach(() => log.push('env:afterEach')); - }); - const e2 = t.environment('env2', () => { - t.beforeAll(() => log.push('env2:beforeAll')); - t.afterAll(() => log.push('env2:afterAll')); - t.beforeEach(() => log.push('env2:beforeEach')); - t.afterEach(() => log.push('env2:afterEach')); - }); - t.it('uno', () => log.push('test #1')).addEnvironment(e).addEnvironment(e2).removeEnvironment(e); + const t = new Runner(); + const e = new Environment('env'); + e.beforeAll(() => log.push('env:beforeAll')); + e.afterAll(() => log.push('env:afterAll')); + e.beforeEach(() => log.push('env:beforeEach')); + e.afterEach(() => log.push('env:afterEach')); + const e2 = new Environment('env2'); + e2.beforeAll(() => log.push('env2:beforeAll')); + e2.afterAll(() => log.push('env2:afterAll')); + e2.beforeEach(() => log.push('env2:beforeEach')); + e2.afterEach(() => log.push('env2:afterEach')); + t.it('uno', () => log.push('test #1')); + t.tests()[0].addEnvironment(e).addEnvironment(e2).removeEnvironment(e); await t.run(); expect(log).toEqual([ 'env2:beforeAll', @@ -376,49 +409,9 @@ module.exports.addTests = function({testRunner, expect}) { 'env2:afterAll', ]); }); - it('environment restrictions', async () => { - const t = newTestRunner(); - let env; - let env2; - t.describe('suite1', () => { - env = t.environment('env', () => { - env2 = t.environment('env2', () => {}); - try { - t.it('test', () => {}); - expect(true).toBe(false); - } catch (e) { - expect(e.message).toBe('Cannot define a test inside an environment'); - } - try { - t.describe('suite', () => {}); - expect(true).toBe(false); - } catch (e) { - expect(e.message).toBe('Cannot define a suite inside an environment'); - } - }); - try { - t.it('test', () => {}).addEnvironment(env).addEnvironment(env2); - expect(true).toBe(false); - } catch (e) { - expect(e.message).toBe('Cannot use environments "env2" and "env" that share a parent environment "suite1 env" in test "suite1 test"'); - } - try { - t.it('test', () => {}).addEnvironment(env2).addEnvironment(env); - expect(true).toBe(false); - } catch (e) { - expect(e.message).toBe('Cannot use environments "env" and "env2" that share a parent environment "suite1 env" in test "suite1 test"'); - } - }); - try { - t.it('test', () => {}).addEnvironment(env); - expect(true).toBe(false); - } catch (e) { - expect(e.message).toBe('Cannot use environment "env" from suite "suite1" in unrelated test "test"'); - } - }); it('should have the same state object in hooks and test', async() => { const states = []; - const t = newTestRunner(); + const t = new Runner(); t.beforeEach(state => states.push(state)); t.afterEach(state => states.push(state)); t.beforeAll(state => states.push(state)); @@ -431,7 +424,7 @@ module.exports.addTests = function({testRunner, expect}) { }); it('should unwind hooks properly when terminated', async() => { const log = []; - const t = newTestRunner({timeout: 10000}); + const t = new Runner({timeout: 10000}); t.beforeAll(() => log.push('beforeAll')); t.beforeEach(() => log.push('beforeEach')); t.afterEach(() => log.push('afterEach')); @@ -451,14 +444,14 @@ module.exports.addTests = function({testRunner, expect}) { ]); }); it('should report as terminated even when hook crashes', async() => { - const t = newTestRunner({timeout: 10000}); + const t = new Runner({timeout: 10000}); t.afterEach(() => { throw new Error('crash!'); }); t.it('uno', () => { t.terminate(); }); const result = await t.run(); expect(result.runs[0].result()).toBe('terminated'); }); it('should report as terminated when terminated during hook', async() => { - const t = newTestRunner({timeout: 10000}); + const t = new Runner({timeout: 10000}); t.afterEach(() => { t.terminate(); }); t.it('uno', () => { }); const result = await t.run(); @@ -466,7 +459,7 @@ module.exports.addTests = function({testRunner, expect}) { }); it('should unwind hooks properly when crashed', async() => { const log = []; - const t = newTestRunner({timeout: 10000}); + const t = new Runner({timeout: 10000}); t.beforeAll(() => log.push('root beforeAll')); t.beforeEach(() => log.push('root beforeEach')); t.describe('suite', () => { @@ -499,16 +492,14 @@ module.exports.addTests = function({testRunner, expect}) { describe('TestRunner.run', () => { it('should run a test', async() => { - const t = newTestRunner(); + const t = new Runner(); let ran = false; t.it('uno', () => ran = true); await t.run(); expect(ran).toBe(true); }); it('should handle repeat', async() => { - const t = newTestRunner(); - t.testModifier('repeat', (t, count) => t.setRepeat(count)); - t.suiteModifier('repeat', (s, count) => s.setRepeat(count)); + const t = new Runner(); let suite = 0; let test = 0; let beforeAll = 0; @@ -526,7 +517,7 @@ module.exports.addTests = function({testRunner, expect}) { expect(test).toBe(6); }); it('should run tests if some fail', async() => { - const t = newTestRunner(); + const t = new Runner(); const log = []; t.it('uno', () => log.push(1)); t.it('dos', () => { throw new Error('bad'); }); @@ -535,7 +526,7 @@ module.exports.addTests = function({testRunner, expect}) { expect(log.join()).toBe('1,3'); }); it('should run tests if some timeout', async() => { - const t = newTestRunner({timeout: 1}); + const t = new Runner({timeout: 1}); const log = []; t.it('uno', () => log.push(1)); t.it('dos', async() => new Promise(() => {})); @@ -545,7 +536,7 @@ module.exports.addTests = function({testRunner, expect}) { }); it('should break on first failure if configured so', async() => { const log = []; - const t = newTestRunner({breakOnFailure: true}); + const t = new Runner({breakOnFailure: true}); t.it('test#1', () => log.push('test#1')); t.it('test#2', () => log.push('test#2')); t.it('test#3', () => { throw new Error('crash'); }); @@ -558,17 +549,17 @@ module.exports.addTests = function({testRunner, expect}) { }); it('should pass a state and a test as a test parameters', async() => { const log = []; - const t = newTestRunner(); + const t = new Runner(); t.beforeEach(state => state.FOO = 42); - t.it('uno', (state, test) => { + t.it('uno', (state, testRun) => { log.push('state.FOO=' + state.FOO); - log.push('test=' + test.name()); + log.push('test=' + testRun.test().name()); }); await t.run(); expect(log.join()).toBe('state.FOO=42,test=uno'); }); it('should run async test', async() => { - const t = newTestRunner(); + const t = new Runner(); let ran = false; t.it('uno', async() => { await new Promise(x => setTimeout(x, 10)); @@ -579,7 +570,7 @@ module.exports.addTests = function({testRunner, expect}) { }); it('should run async tests in order of their declaration', async() => { const log = []; - const t = newTestRunner(); + const t = new Runner(); t.it('uno', async() => { await new Promise(x => setTimeout(x, 30)); log.push(1); @@ -597,14 +588,14 @@ module.exports.addTests = function({testRunner, expect}) { }); it('should run multiple tests', async() => { const log = []; - const t = newTestRunner(); + const t = new Runner(); t.it('uno', () => log.push(1)); t.it('dos', () => log.push(2)); await t.run(); expect(log.join()).toBe('1,2'); }); it('should NOT run a skipped test', async() => { - const t = newTestRunner(); + const t = new Runner(); let ran = false; t.xit('uno', () => ran = true); await t.run(); @@ -612,7 +603,7 @@ module.exports.addTests = function({testRunner, expect}) { }); it('should run ONLY non-skipped tests', async() => { const log = []; - const t = newTestRunner(); + const t = new Runner(); t.it('uno', () => log.push(1)); t.xit('dos', () => log.push(2)); t.it('tres', () => log.push(3)); @@ -621,7 +612,7 @@ module.exports.addTests = function({testRunner, expect}) { }); it('should run ONLY focused tests', async() => { const log = []; - const t = newTestRunner(); + const t = new Runner(); t.it('uno', () => log.push(1)); t.xit('dos', () => log.push(2)); t.fit('tres', () => log.push(3)); @@ -630,7 +621,7 @@ module.exports.addTests = function({testRunner, expect}) { }); it('should run tests in order of their declaration', async() => { const log = []; - const t = newTestRunner(); + const t = new Runner(); t.it('uno', () => log.push(1)); t.describe('suite1', () => { t.it('dos', () => log.push(2)); @@ -641,9 +632,9 @@ module.exports.addTests = function({testRunner, expect}) { expect(log.join()).toBe('1,2,3,4'); }); it('should respect total timeout', async() => { - const t = newTestRunner({timeout: 10000}); + const t = new Runner({timeout: 10000, totalTimeout: 1}); t.it('uno', async () => { await new Promise(() => {}); }); - const result = await t.run({totalTimeout: 1}); + const result = await t.run(); expect(result.runs[0].result()).toBe('terminated'); expect(result.message).toContain('Total timeout'); }); @@ -651,25 +642,25 @@ module.exports.addTests = function({testRunner, expect}) { describe('TestRunner.run result', () => { it('should return OK if all tests pass', async() => { - const t = newTestRunner(); + const t = new Runner(); t.it('uno', () => {}); const result = await t.run(); expect(result.result).toBe('ok'); }); it('should return FAIL if at least one test fails', async() => { - const t = newTestRunner(); + const t = new Runner(); t.it('uno', () => { throw new Error('woof'); }); const result = await t.run(); expect(result.result).toBe('failed'); }); it('should return FAIL if at least one test times out', async() => { - const t = newTestRunner({timeout: 1}); + const t = new Runner({timeout: 1}); t.it('uno', async() => new Promise(() => {})); const result = await t.run(); expect(result.result).toBe('failed'); }); it('should return TERMINATED if it was terminated', async() => { - const t = newTestRunner({timeout: 1}); + const t = new Runner({timeout: 1000000}); t.it('uno', async() => new Promise(() => {})); const [result] = await Promise.all([ t.run(), @@ -678,7 +669,7 @@ module.exports.addTests = function({testRunner, expect}) { expect(result.result).toBe('terminated'); }); it('should return CRASHED if it crashed', async() => { - const t = newTestRunner({timeout: 1}); + const t = new Runner({timeout: 1}); t.it('uno', async() => new Promise(() => {})); t.afterAll(() => { throw new Error('woof');}); const result = await t.run(); @@ -689,7 +680,7 @@ module.exports.addTests = function({testRunner, expect}) { describe('TestRunner parallel', () => { it('should run tests in parallel', async() => { const log = []; - const t = newTestRunner({parallel: 2}); + const t = new Runner({parallel: 2}); t.it('uno', async state => { log.push(`Worker #${state.parallelIndex} Starting: UNO`); await Promise.resolve(); @@ -712,17 +703,17 @@ module.exports.addTests = function({testRunner, expect}) { describe('TestRunner.hasFocusedTestsOrSuites', () => { it('should work', () => { - const t = newTestRunner(); + const t = new Runner(); t.it('uno', () => {}); - expect(t.hasFocusedTestsOrSuites()).toBe(false); + expect(t._filter.hasFocusedTestsOrSuites()).toBe(false); }); it('should work #2', () => { - const t = newTestRunner(); + const t = new Runner(); t.fit('uno', () => {}); - expect(t.hasFocusedTestsOrSuites()).toBe(true); + expect(t._filter.hasFocusedTestsOrSuites()).toBe(true); }); it('should work #3', () => { - const t = newTestRunner(); + const t = new Runner(); t.describe('suite #1', () => { t.fdescribe('suite #2', () => { t.describe('suite #3', () => { @@ -730,13 +721,13 @@ module.exports.addTests = function({testRunner, expect}) { }); }); }); - expect(t.hasFocusedTestsOrSuites()).toBe(true); + expect(t._filter.hasFocusedTestsOrSuites()).toBe(true); }); }); describe('TestRunner result', () => { it('should work for both throwing and timeouting tests', async() => { - const t = newTestRunner({timeout: 1}); + const t = new Runner({timeout: 1}); t.it('uno', () => { throw new Error('boo');}); t.it('dos', () => new Promise(() => {})); const result = await t.run(); @@ -744,50 +735,50 @@ module.exports.addTests = function({testRunner, expect}) { expect(result.runs[1].result()).toBe('timedout'); }); it('should report crashed tests', async() => { - const t = newTestRunner(); + const t = new Runner(); t.beforeEach(() => { throw new Error('woof');}); t.it('uno', () => {}); const result = await t.run(); expect(result.runs[0].result()).toBe('crashed'); }); it('skipped should work for both throwing and timeouting tests', async() => { - const t = newTestRunner({timeout: 1}); + const t = new Runner({timeout: 1}); t.xit('uno', () => { throw new Error('boo');}); const result = await t.run(); expect(result.runs[0].result()).toBe('skipped'); }); it('should return OK', async() => { - const t = newTestRunner(); + const t = new Runner(); t.it('uno', () => {}); const result = await t.run(); expect(result.runs[0].result()).toBe('ok'); }); it('should return TIMEDOUT', async() => { - const t = newTestRunner({timeout: 1}); + const t = new Runner({timeout: 1}); t.it('uno', async() => new Promise(() => {})); const result = await t.run(); expect(result.runs[0].result()).toBe('timedout'); }); it('should return SKIPPED', async() => { - const t = newTestRunner(); + const t = new Runner(); t.xit('uno', () => {}); const result = await t.run(); expect(result.runs[0].result()).toBe('skipped'); }); it('should return FAILED', async() => { - const t = newTestRunner(); + const t = new Runner(); t.it('uno', async() => Promise.reject('woof')); const result = await t.run(); expect(result.runs[0].result()).toBe('failed'); }); it('should return TERMINATED', async() => { - const t = newTestRunner(); + const t = new Runner(); t.it('uno', async() => t.terminate()); const result = await t.run(); expect(result.runs[0].result()).toBe('terminated'); }); it('should return CRASHED', async() => { - const t = newTestRunner(); + const t = new Runner(); t.it('uno', () => {}); t.afterEach(() => {throw new Error('foo');}); const result = await t.run(); @@ -796,20 +787,19 @@ module.exports.addTests = function({testRunner, expect}) { }); describe('TestRunner delegate', () => { - it('should call delegate methpds in proper order', async() => { + it('should call delegate methods in proper order', async() => { const log = []; - const t = newTestRunner(); - t.beforeAll(() => log.push('beforeAll')); - t.beforeEach(() => log.push('beforeEach')); - t.it('test#1', () => log.push('test#1')); - t.afterEach(() => log.push('afterEach')); - t.afterAll(() => log.push('afterAll')); - t.setDelegate({ + const t = new Runner({ onStarted: () => log.push('E:started'), onTestRunStarted: () => log.push('E:teststarted'), onTestRunFinished: () => log.push('E:testfinished'), onFinished: () => log.push('E:finished'), }); + t.beforeAll(() => log.push('beforeAll')); + t.beforeEach(() => log.push('beforeEach')); + t.it('test#1', () => log.push('test#1')); + t.afterEach(() => log.push('afterEach')); + t.afterAll(() => log.push('afterAll')); await t.run(); expect(log).toEqual([ 'E:started', @@ -824,15 +814,11 @@ module.exports.addTests = function({testRunner, expect}) { ]); }); it('should call onFinished with result', async() => { - const t = newTestRunner(); + let onFinished; + const finishedPromise = new Promise(f => onFinished = f); const [result] = await Promise.all([ - new Promise(x => t.setDelegate({ - onStarted() {}, - onFinished(result) { x(result); }, - onTestRunStarted() {}, - onTestRunFinished() {}, - })), - t.run(), + finishedPromise, + new TestRunner().run([], { onFinished }), ]); expect(result.result).toBe('ok'); });