Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

feat(testurnner): allow multiple hooks isntances and per-test hooks (#1571)

Browse source
This commit is contained in:
Dmitry Gozman 2020-03-27 22:57:22 -07:00 committed by GitHub
parent 6503c83302
commit 4bd46bafa9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 118 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

121
utils/testrunner/TestRunner.js
View file

@ -72,6 +72,11 @@ function isTestFailure(testResult) {
return testResult === TestResult.Failed || testResult === TestResult.TimedOut || testResult === TestResult.Crashed;
}
function createHook(callback, name) {
const location = getCallerLocation(__filename);
return { name, body: callback, location };
}
class Test {
constructor(suite, name, callback, location) {
this._suite = suite;
@ -83,6 +88,7 @@ class Test {
this._location = location;
this._timeout = INFINITE_TIMEOUT;
this._repeat = 1;
this._hooks = [];
// Test results. TODO: make these private.
this.result = null;
@ -100,6 +106,7 @@ class Test {
test._timeout = this._timeout;
test._mode = this._mode;
test._expectation = this._expectation;
test._hooks = this._hooks.slice();
return test;
}
@ -155,6 +162,18 @@ class Test {
setRepeat(repeat) {
this._repeat = repeat;
}
before(callback) {
this._hooks.push(createHook(callback, 'before'));
}
after(callback) {
this._hooks.push(createHook(callback, 'after'));
}
hooks(name) {
return this._hooks.filter(hook => !name || hook.name === name);
}
}
class Suite {
@ -166,12 +185,7 @@ class Suite {
this._expectation = TestExpectation.Ok;
this._location = location;
this._repeat = 1;
// TODO: make these private.
this.beforeAll = null;
this.beforeEach = null;
this.afterAll = null;
this.afterEach = null;
this._hooks = [];
this.Modes = { ...TestMode };
this.Expectations = { ...TestExpectation };
@ -225,6 +239,26 @@ class Suite {
setRepeat(repeat) {
this._repeat = repeat;
}
beforeEach(callback) {
this._hooks.push(createHook(callback, 'beforeEach'));
}
afterEach(callback) {
this._hooks.push(createHook(callback, 'afterEach'));
}
beforeAll(callback) {
this._hooks.push(createHook(callback, 'beforeAll'));
}
afterAll(callback) {
this._hooks.push(createHook(callback, 'afterAll'));
}
hooks(name) {
return this._hooks.filter(hook => !name || hook.name === name);
}
}
class Result {
@ -330,16 +364,20 @@ class TestWorker {
if (this._markTerminated(test))
return;
const suite = this._suiteStack.pop();
if (!await this._runHook(test, suite, 'afterAll'))
return;
for (const hook of suite.hooks('afterAll')) {
if (!await this._runHook(test, hook, suite.fullName()))
return;
}
}
while (this._suiteStack.length < suiteStack.length) {
if (this._markTerminated(test))
return;
const suite = suiteStack[this._suiteStack.length];
this._suiteStack.push(suite);
if (!await this._runHook(test, suite, 'beforeAll'))
return;
for (const hook of suite.hooks('beforeAll')) {
if (!await this._runHook(test, hook, suite.fullName()))
return;
}
}
if (this._markTerminated(test))
@ -349,8 +387,12 @@ class TestWorker {
// no matter what happens.
await this._testPass._willStartTest(this, test);
for (let i = 0; i < this._suiteStack.length; i++)
await this._runHook(test, this._suiteStack[i], 'beforeEach');
for (const suite of this._suiteStack) {
for (const hook of suite.hooks('beforeEach'))
await this._runHook(test, hook, suite.fullName(), true);
}
for (const hook of test.hooks('before'))
await this._runHook(test, hook, test.fullName(), true);
if (!test.error && !this._markTerminated(test)) {
await this._testPass._willStartTestBody(this, test);
@ -371,19 +413,19 @@ class TestWorker {
await this._testPass._didFinishTestBody(this, test);
}
for (let i = this._suiteStack.length - 1; i >= 0; i--)
await this._runHook(test, this._suiteStack[i], 'afterEach');
for (const hook of test.hooks('after'))
await this._runHook(test, hook, test.fullName(), true);
for (const suite of this._suiteStack.slice().reverse()) {
for (const hook of suite.hooks('afterEach'))
await this._runHook(test, hook, suite.fullName(), true);
}
await this._testPass._didFinishTest(this, test);
}
async _runHook(test, suite, hookName) {
const hook = suite[hookName];
if (!hook)
return true;
await this._testPass._willStartHook(this, suite, hook.location, hookName);
async _runHook(test, hook, fullName, passTest = false) {
await this._testPass._willStartHook(this, hook, fullName);
const timeout = this._testPass._runner._timeout;
const { promise, terminate } = runUserCallback(hook.body, timeout, [this._state, test]);
const { promise, terminate } = runUserCallback(hook.body, timeout, passTest ? [this._state, test] : [this._state]);
this._runningHookTerminate = terminate;
let error = await promise;
this._runningHookTerminate = null;
@ -396,7 +438,7 @@ class TestWorker {
}
let message;
if (error === TimeoutError) {
message = `${locationString} - Timeout Exceeded ${timeout}ms while running "${hookName}" in suite "${suite.fullName()}"`;
message = `${locationString} - Timeout Exceeded ${timeout}ms while running "${hook.name}" in "${fullName}"`;
error = null;
} else if (error === TerminatedError) {
// Do not report termination details - it's just noise.
@ -405,21 +447,22 @@ class TestWorker {
} else {
if (error.stack)
await this._testPass._runner._sourceMapSupport.rewriteStackTraceWithSourceMaps(error);
message = `${locationString} - FAILED while running "${hookName}" in suite "${suite.fullName()}": `;
message = `${locationString} - FAILED while running "${hook.name}" in suite "${fullName}": `;
}
await this._testPass._didFailHook(this, suite, hook.location, hookName, message, error);
await this._testPass._didFailHook(this, hook, fullName, message, error);
test.error = error;
return false;
}
await this._testPass._didCompleteHook(this, suite, hook.location, hookName);
await this._testPass._didCompleteHook(this, hook, fullName);
return true;
}
async shutdown() {
while (this._suiteStack.length > 0) {
const suite = this._suiteStack.pop();
await this._runHook({}, suite, 'afterAll');
for (const hook of suite.hooks('afterAll'))
await this._runHook({}, hook, suite.fullName());
}
}
}
@ -542,19 +585,19 @@ class TestPass {
debug('testrunner:test')(`[${worker._workerId}] ${test.result.toUpperCase()} "${test.fullName()}" (${test.location().fileName + ':' + test.location().lineNumber})`);
}
async _willStartHook(worker, suite, location, hookName) {
debug('testrunner:hook')(`[${worker._workerId}] "${hookName}" started for "${suite.fullName()}" (${location.fileName + ':' + location.lineNumber})`);
async _willStartHook(worker, hook, fullName) {
debug('testrunner:hook')(`[${worker._workerId}] "${hook.name}" started for "${fullName}" (${hook.location.fileName + ':' + hook.location.lineNumber})`);
}
async _didFailHook(worker, suite, location, hookName, message, error) {
debug('testrunner:hook')(`[${worker._workerId}] "${hookName}" FAILED for "${suite.fullName()}" (${location.fileName + ':' + location.lineNumber})`);
async _didFailHook(worker, hook, fullName, message, error) {
debug('testrunner:hook')(`[${worker._workerId}] "${hook.name}" FAILED for "${fullName}" (${hook.location.fileName + ':' + hook.location.lineNumber})`);
if (message)
this._result.addError(message, error, worker);
this._result.setResult(TestResult.Crashed, message);
}
async _didCompleteHook(worker, suite, location, hookName) {
debug('testrunner:hook')(`[${worker._workerId}] "${hookName}" OK for "${suite.fullName()}" (${location.fileName + ':' + location.lineNumber})`);
async _didCompleteHook(worker, hook, fullName) {
debug('testrunner:hook')(`[${worker._workerId}] "${hook.name}" OK for "${fullName}" (${hook.location.fileName + ':' + hook.location.lineNumber})`);
}
}
@ -593,10 +636,10 @@ class TestRunner extends EventEmitter {
this._debuggerLogBreakpointLines = new Multimap();
this.beforeAll = this._addHook.bind(this, 'beforeAll');
this.beforeEach = this._addHook.bind(this, 'beforeEach');
this.afterAll = this._addHook.bind(this, 'afterAll');
this.afterEach = this._addHook.bind(this, 'afterEach');
this.beforeAll = (callback) => this._currentSuite.beforeAll(callback);
this.beforeEach = (callback) => this._currentSuite.beforeEach(callback);
this.afterAll = (callback) => this._currentSuite.afterAll(callback);
this.afterEach = (callback) => this._currentSuite.afterEach(callback);
this.describe = this._suiteBuilder([]);
this.it = this._testBuilder([]);
@ -694,12 +737,6 @@ class TestRunner extends EventEmitter {
this.describe.skip(true)('', module.xdescribe, ...args);
}
_addHook(hookName, callback) {
assert(this._currentSuite[hookName] === null, `Only one ${hookName} hook available per suite`);
const location = getCallerLocation(__filename);
this._currentSuite[hookName] = { body: callback, location };
}
async run(options = {}) {
const { totalTimeout = 0 } = options;
let session = this._debuggerLogBreakpointLines.size ? await setLogBreakpoints(this._debuggerLogBreakpointLines) : null;

52
utils/testrunner/test/testrunner.spec.js
View file

@ -214,47 +214,73 @@ module.exports.addTests = function({testRunner, expect}) {
const log = [];
const t = newTestRunner();
t.beforeAll(() => log.push('root:beforeAll'));
t.beforeEach(() => log.push('root:beforeEach'));
t.beforeEach(() => log.push('root:beforeEach1'));
t.beforeEach(() => log.push('root:beforeEach2'));
t.it('uno', () => log.push('test #1'));
t.describe('suite1', () => {
t.beforeAll(() => log.push('suite:beforeAll'));
t.beforeEach(() => log.push('suite:beforeEach'));
t.beforeAll(() => log.push('suite:beforeAll1'));
t.beforeAll(() => log.push('suite:beforeAll2'));
t.beforeEach((state, test) => {
log.push('suite:beforeEach');
test.before(() => log.push('test:before1'));
test.before(() => log.push('test:before2'));
test.after(() => log.push('test:after1'));
test.after(() => log.push('test:after2'));
});
t.it('dos', () => log.push('test #2'));
t.it('tres', () => log.push('test #3'));
t.afterEach(() => log.push('suite:afterEach'));
t.afterEach(() => log.push('suite:afterEach1'));
t.afterEach(() => log.push('suite:afterEach2'));
t.afterAll(() => log.push('suite:afterAll'));
});
t.it('cuatro', () => log.push('test #4'));
t.afterEach(() => log.push('root:afterEach'));
t.afterAll(() => log.push('root:afterAll'));
t.afterAll(() => log.push('root:afterAll1'));
t.afterAll(() => log.push('root:afterAll2'));
await t.run();
expect(log).toEqual([
'root:beforeAll',
'root:beforeEach',
'root:beforeEach1',
'root:beforeEach2',
'test #1',
'root:afterEach',
'suite:beforeAll',
'suite:beforeAll1',
'suite:beforeAll2',
'root:beforeEach',
'root:beforeEach1',
'root:beforeEach2',
'suite:beforeEach',
'test:before1',
'test:before2',
'test #2',
'suite:afterEach',
'test:after1',
'test:after2',
'suite:afterEach1',
'suite:afterEach2',
'root:afterEach',
'root:beforeEach',
'root:beforeEach1',
'root:beforeEach2',
'suite:beforeEach',
'test:before1',
'test:before2',
'test #3',
'suite:afterEach',
'test:after1',
'test:after2',
'suite:afterEach1',
'suite:afterEach2',
'root:afterEach',
'suite:afterAll',
'root:beforeEach',
'root:beforeEach1',
'root:beforeEach2',
'test #4',
'root:afterEach',
'root:afterAll',
'root:afterAll1',
'root:afterAll2',
]);
});
it('should have the same state object in hooks and test', async() => {