Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

feat(testurnner): allow multiple hooks isntances and per-test hooks (#1571)

Browse source
This commit is contained in:
Dmitry Gozman 2020-03-27 22:57:22 -07:00 committed by GitHub
parent 6503c83302
commit 4bd46bafa9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 118 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

121
utils/testrunner/TestRunner.js
View file

@ -72,6 +72,11 @@ function isTestFailure(testResult) {
return testResult === TestResult.Failed || testResult === TestResult.TimedOut || testResult === TestResult.Crashed; return testResult === TestResult.Failed || testResult === TestResult.TimedOut || testResult === TestResult.Crashed;
} }
function createHook(callback, name) {
const location = getCallerLocation(__filename);
return { name, body: callback, location };
}
class Test { class Test {
constructor(suite, name, callback, location) { constructor(suite, name, callback, location) {
this._suite = suite; this._suite = suite;
@ -83,6 +88,7 @@ class Test {
this._location = location; this._location = location;
this._timeout = INFINITE_TIMEOUT; this._timeout = INFINITE_TIMEOUT;
this._repeat = 1; this._repeat = 1;
this._hooks = [];
// Test results. TODO: make these private. // Test results. TODO: make these private.
this.result = null; this.result = null;
@ -100,6 +106,7 @@ class Test {
test._timeout = this._timeout; test._timeout = this._timeout;
test._mode = this._mode; test._mode = this._mode;
test._expectation = this._expectation; test._expectation = this._expectation;
test._hooks = this._hooks.slice();
return test; return test;
} }
@ -155,6 +162,18 @@ class Test {
setRepeat(repeat) { setRepeat(repeat) {
this._repeat = repeat; this._repeat = repeat;
} }
before(callback) {
this._hooks.push(createHook(callback, 'before'));
}
after(callback) {
this._hooks.push(createHook(callback, 'after'));
}
hooks(name) {
return this._hooks.filter(hook => !name || hook.name === name);
}
} }
class Suite { class Suite {
@ -166,12 +185,7 @@ class Suite {
this._expectation = TestExpectation.Ok; this._expectation = TestExpectation.Ok;
this._location = location; this._location = location;
this._repeat = 1; this._repeat = 1;
this._hooks = [];
// TODO: make these private.
this.beforeAll = null;
this.beforeEach = null;
this.afterAll = null;
this.afterEach = null;
this.Modes = { ...TestMode }; this.Modes = { ...TestMode };
this.Expectations = { ...TestExpectation }; this.Expectations = { ...TestExpectation };
@ -225,6 +239,26 @@ class Suite {
setRepeat(repeat) { setRepeat(repeat) {
this._repeat = repeat; this._repeat = repeat;
} }
beforeEach(callback) {
this._hooks.push(createHook(callback, 'beforeEach'));
}
afterEach(callback) {
this._hooks.push(createHook(callback, 'afterEach'));
}
beforeAll(callback) {
this._hooks.push(createHook(callback, 'beforeAll'));
}
afterAll(callback) {
this._hooks.push(createHook(callback, 'afterAll'));
}
hooks(name) {
return this._hooks.filter(hook => !name || hook.name === name);
}
} }
class Result { class Result {
@ -330,16 +364,20 @@ class TestWorker {
if (this._markTerminated(test)) if (this._markTerminated(test))
return; return;
const suite = this._suiteStack.pop(); const suite = this._suiteStack.pop();
if (!await this._runHook(test, suite, 'afterAll')) for (const hook of suite.hooks('afterAll')) {
return; if (!await this._runHook(test, hook, suite.fullName()))
return;
}
} }
while (this._suiteStack.length < suiteStack.length) { while (this._suiteStack.length < suiteStack.length) {
if (this._markTerminated(test)) if (this._markTerminated(test))
return; return;
const suite = suiteStack[this._suiteStack.length]; const suite = suiteStack[this._suiteStack.length];
this._suiteStack.push(suite); this._suiteStack.push(suite);
if (!await this._runHook(test, suite, 'beforeAll')) for (const hook of suite.hooks('beforeAll')) {
return; if (!await this._runHook(test, hook, suite.fullName()))
return;
}
} }
if (this._markTerminated(test)) if (this._markTerminated(test))
@ -349,8 +387,12 @@ class TestWorker {
// no matter what happens. // no matter what happens.
await this._testPass._willStartTest(this, test); await this._testPass._willStartTest(this, test);
for (let i = 0; i < this._suiteStack.length; i++) for (const suite of this._suiteStack) {
await this._runHook(test, this._suiteStack[i], 'beforeEach'); for (const hook of suite.hooks('beforeEach'))
await this._runHook(test, hook, suite.fullName(), true);
}
for (const hook of test.hooks('before'))
await this._runHook(test, hook, test.fullName(), true);
if (!test.error && !this._markTerminated(test)) { if (!test.error && !this._markTerminated(test)) {
await this._testPass._willStartTestBody(this, test); await this._testPass._willStartTestBody(this, test);
@ -371,19 +413,19 @@ class TestWorker {
await this._testPass._didFinishTestBody(this, test); await this._testPass._didFinishTestBody(this, test);
} }
for (let i = this._suiteStack.length - 1; i >= 0; i--) for (const hook of test.hooks('after'))
await this._runHook(test, this._suiteStack[i], 'afterEach'); await this._runHook(test, hook, test.fullName(), true);
for (const suite of this._suiteStack.slice().reverse()) {
for (const hook of suite.hooks('afterEach'))
await this._runHook(test, hook, suite.fullName(), true);
}
await this._testPass._didFinishTest(this, test); await this._testPass._didFinishTest(this, test);
} }
async _runHook(test, suite, hookName) { async _runHook(test, hook, fullName, passTest = false) {
const hook = suite[hookName]; await this._testPass._willStartHook(this, hook, fullName);
if (!hook)
return true;
await this._testPass._willStartHook(this, suite, hook.location, hookName);
const timeout = this._testPass._runner._timeout; const timeout = this._testPass._runner._timeout;
const { promise, terminate } = runUserCallback(hook.body, timeout, [this._state, test]); const { promise, terminate } = runUserCallback(hook.body, timeout, passTest ? [this._state, test] : [this._state]);
this._runningHookTerminate = terminate; this._runningHookTerminate = terminate;
let error = await promise; let error = await promise;
this._runningHookTerminate = null; this._runningHookTerminate = null;
@ -396,7 +438,7 @@ class TestWorker {
} }
let message; let message;
if (error === TimeoutError) { if (error === TimeoutError) {
message = `${locationString} - Timeout Exceeded ${timeout}ms while running "${hookName}" in suite "${suite.fullName()}"`; message = `${locationString} - Timeout Exceeded ${timeout}ms while running "${hook.name}" in "${fullName}"`;
error = null; error = null;
} else if (error === TerminatedError) { } else if (error === TerminatedError) {
// Do not report termination details - it's just noise. // Do not report termination details - it's just noise.
@ -405,21 +447,22 @@ class TestWorker {
} else { } else {
if (error.stack) if (error.stack)
await this._testPass._runner._sourceMapSupport.rewriteStackTraceWithSourceMaps(error); await this._testPass._runner._sourceMapSupport.rewriteStackTraceWithSourceMaps(error);
message = `${locationString} - FAILED while running "${hookName}" in suite "${suite.fullName()}": `; message = `${locationString} - FAILED while running "${hook.name}" in suite "${fullName}": `;
} }
await this._testPass._didFailHook(this, suite, hook.location, hookName, message, error); await this._testPass._didFailHook(this, hook, fullName, message, error);
test.error = error; test.error = error;
return false; return false;
} }
await this._testPass._didCompleteHook(this, suite, hook.location, hookName); await this._testPass._didCompleteHook(this, hook, fullName);
return true; return true;
} }
async shutdown() { async shutdown() {
while (this._suiteStack.length > 0) { while (this._suiteStack.length > 0) {
const suite = this._suiteStack.pop(); const suite = this._suiteStack.pop();
await this._runHook({}, suite, 'afterAll'); for (const hook of suite.hooks('afterAll'))
await this._runHook({}, hook, suite.fullName());
} }
} }
} }
@ -542,19 +585,19 @@ class TestPass {
debug('testrunner:test')(`[${worker._workerId}] ${test.result.toUpperCase()} "${test.fullName()}" (${test.location().fileName + ':' + test.location().lineNumber})`); debug('testrunner:test')(`[${worker._workerId}] ${test.result.toUpperCase()} "${test.fullName()}" (${test.location().fileName + ':' + test.location().lineNumber})`);
} }
async _willStartHook(worker, suite, location, hookName) { async _willStartHook(worker, hook, fullName) {
debug('testrunner:hook')(`[${worker._workerId}] "${hookName}" started for "${suite.fullName()}" (${location.fileName + ':' + location.lineNumber})`); debug('testrunner:hook')(`[${worker._workerId}] "${hook.name}" started for "${fullName}" (${hook.location.fileName + ':' + hook.location.lineNumber})`);
} }
async _didFailHook(worker, suite, location, hookName, message, error) { async _didFailHook(worker, hook, fullName, message, error) {
debug('testrunner:hook')(`[${worker._workerId}] "${hookName}" FAILED for "${suite.fullName()}" (${location.fileName + ':' + location.lineNumber})`); debug('testrunner:hook')(`[${worker._workerId}] "${hook.name}" FAILED for "${fullName}" (${hook.location.fileName + ':' + hook.location.lineNumber})`);
if (message) if (message)
this._result.addError(message, error, worker); this._result.addError(message, error, worker);
this._result.setResult(TestResult.Crashed, message); this._result.setResult(TestResult.Crashed, message);
} }
async _didCompleteHook(worker, suite, location, hookName) { async _didCompleteHook(worker, hook, fullName) {
debug('testrunner:hook')(`[${worker._workerId}] "${hookName}" OK for "${suite.fullName()}" (${location.fileName + ':' + location.lineNumber})`); debug('testrunner:hook')(`[${worker._workerId}] "${hook.name}" OK for "${fullName}" (${hook.location.fileName + ':' + hook.location.lineNumber})`);
} }
} }
@ -593,10 +636,10 @@ class TestRunner extends EventEmitter {
this._debuggerLogBreakpointLines = new Multimap(); this._debuggerLogBreakpointLines = new Multimap();
this.beforeAll = this._addHook.bind(this, 'beforeAll'); this.beforeAll = (callback) => this._currentSuite.beforeAll(callback);
this.beforeEach = this._addHook.bind(this, 'beforeEach'); this.beforeEach = (callback) => this._currentSuite.beforeEach(callback);
this.afterAll = this._addHook.bind(this, 'afterAll'); this.afterAll = (callback) => this._currentSuite.afterAll(callback);
this.afterEach = this._addHook.bind(this, 'afterEach'); this.afterEach = (callback) => this._currentSuite.afterEach(callback);
this.describe = this._suiteBuilder([]); this.describe = this._suiteBuilder([]);
this.it = this._testBuilder([]); this.it = this._testBuilder([]);
@ -694,12 +737,6 @@ class TestRunner extends EventEmitter {
this.describe.skip(true)('', module.xdescribe, ...args); this.describe.skip(true)('', module.xdescribe, ...args);
} }
_addHook(hookName, callback) {
assert(this._currentSuite[hookName] === null, `Only one ${hookName} hook available per suite`);
const location = getCallerLocation(__filename);
this._currentSuite[hookName] = { body: callback, location };
}
async run(options = {}) { async run(options = {}) {
const { totalTimeout = 0 } = options; const { totalTimeout = 0 } = options;
let session = this._debuggerLogBreakpointLines.size ? await setLogBreakpoints(this._debuggerLogBreakpointLines) : null; let session = this._debuggerLogBreakpointLines.size ? await setLogBreakpoints(this._debuggerLogBreakpointLines) : null;

52
utils/testrunner/test/testrunner.spec.js
View file

@ -214,47 +214,73 @@ module.exports.addTests = function({testRunner, expect}) {
const log = []; const log = [];
const t = newTestRunner(); const t = newTestRunner();
t.beforeAll(() => log.push('root:beforeAll')); t.beforeAll(() => log.push('root:beforeAll'));
t.beforeEach(() => log.push('root:beforeEach')); t.beforeEach(() => log.push('root:beforeEach1'));
t.beforeEach(() => log.push('root:beforeEach2'));
t.it('uno', () => log.push('test #1')); t.it('uno', () => log.push('test #1'));
t.describe('suite1', () => { t.describe('suite1', () => {
t.beforeAll(() => log.push('suite:beforeAll')); t.beforeAll(() => log.push('suite:beforeAll1'));
t.beforeEach(() => log.push('suite:beforeEach')); t.beforeAll(() => log.push('suite:beforeAll2'));
t.beforeEach((state, test) => {
log.push('suite:beforeEach');
test.before(() => log.push('test:before1'));
test.before(() => log.push('test:before2'));
test.after(() => log.push('test:after1'));
test.after(() => log.push('test:after2'));
});
t.it('dos', () => log.push('test #2')); t.it('dos', () => log.push('test #2'));
t.it('tres', () => log.push('test #3')); t.it('tres', () => log.push('test #3'));
t.afterEach(() => log.push('suite:afterEach')); t.afterEach(() => log.push('suite:afterEach1'));
t.afterEach(() => log.push('suite:afterEach2'));
t.afterAll(() => log.push('suite:afterAll')); t.afterAll(() => log.push('suite:afterAll'));
}); });
t.it('cuatro', () => log.push('test #4')); t.it('cuatro', () => log.push('test #4'));
t.afterEach(() => log.push('root:afterEach')); t.afterEach(() => log.push('root:afterEach'));
t.afterAll(() => log.push('root:afterAll')); t.afterAll(() => log.push('root:afterAll1'));
t.afterAll(() => log.push('root:afterAll2'));
await t.run(); await t.run();
expect(log).toEqual([ expect(log).toEqual([
'root:beforeAll', 'root:beforeAll',
'root:beforeEach', 'root:beforeEach1',
'root:beforeEach2',
'test #1', 'test #1',
'root:afterEach', 'root:afterEach',
'suite:beforeAll', 'suite:beforeAll1',
'suite:beforeAll2',
'root:beforeEach', 'root:beforeEach1',
'root:beforeEach2',
'suite:beforeEach', 'suite:beforeEach',
'test:before1',
'test:before2',
'test #2', 'test #2',
'suite:afterEach', 'test:after1',
'test:after2',
'suite:afterEach1',
'suite:afterEach2',
'root:afterEach', 'root:afterEach',
'root:beforeEach', 'root:beforeEach1',
'root:beforeEach2',
'suite:beforeEach', 'suite:beforeEach',
'test:before1',
'test:before2',
'test #3', 'test #3',
'suite:afterEach', 'test:after1',
'test:after2',
'suite:afterEach1',
'suite:afterEach2',
'root:afterEach', 'root:afterEach',
'suite:afterAll', 'suite:afterAll',
'root:beforeEach', 'root:beforeEach1',
'root:beforeEach2',
'test #4', 'test #4',
'root:afterEach', 'root:afterEach',
'root:afterAll', 'root:afterAll1',
'root:afterAll2',
]); ]);
}); });
it('should have the same state object in hooks and test', async() => { it('should have the same state object in hooks and test', async() => {