Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

chore(testrunner): separate expectations from run mode (#1395)

Browse source 
Run/Focus/Skip is orthogonal to expect to Pass/Fail/Flake.
This change separates the two, in a preparation to run Fail/Flaky tests.
This commit is contained in:
Dmitry Gozman 2020-03-15 23:10:49 -07:00 committed by GitHub
parent 951126a58a
commit e7eeefe4c7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 46 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

97
utils/testrunner/TestRunner.js
View file

@ -72,8 +72,11 @@ const TestMode = {
Run: 'run', Run: 'run',
Skip: 'skip', Skip: 'skip',
Focus: 'focus', Focus: 'focus',
MarkAsFailing: 'markAsFailing', };
Flake: 'flake'
const TestExpectation = {
Ok: 'ok',
Fail: 'fail',
}; };
const TestResult = { const TestResult = {
@ -91,11 +94,12 @@ function isTestFailure(testResult) {
} }
class Test { class Test {
constructor(suite, name, callback, declaredMode, timeout) { constructor(suite, name, callback, declaredMode, expectation, timeout) {
this.suite = suite; this.suite = suite;
this.name = name; this.name = name;
this.fullName = (suite.fullName + ' ' + name).trim(); this.fullName = (suite.fullName + ' ' + name).trim();
this.declaredMode = declaredMode; this.declaredMode = declaredMode;
this.expectation = expectation;
this._userCallback = new UserCallback(callback, timeout); this._userCallback = new UserCallback(callback, timeout);
this.location = this._userCallback.location; this.location = this._userCallback.location;
this.timeout = timeout; this.timeout = timeout;
@ -109,11 +113,12 @@ class Test {
} }
class Suite { class Suite {
constructor(parentSuite, name, declaredMode) { constructor(parentSuite, name, declaredMode, expectation) {
this.parentSuite = parentSuite; this.parentSuite = parentSuite;
this.name = name; this.name = name;
this.fullName = (parentSuite ? parentSuite.fullName + ' ' + name : name).trim(); this.fullName = (parentSuite ? parentSuite.fullName + ' ' + name : name).trim();
this.declaredMode = declaredMode; this.declaredMode = declaredMode;
this.expectation = expectation;
/** @type {!Array<(!Test|!Suite)>} */ /** @type {!Array<(!Test|!Suite)>} */
this.children = []; this.children = [];
this.location = getCallerLocation(__filename); this.location = getCallerLocation(__filename);
@ -195,16 +200,16 @@ class TestWorker {
if (this._markTerminated(test)) if (this._markTerminated(test))
return; return;
if (test.declaredMode === TestMode.MarkAsFailing) { if (test.declaredMode === TestMode.Skip) {
await this._testPass._willStartTest(this, test); await this._testPass._willStartTest(this, test);
test.result = TestResult.MarkedAsFailing; test.result = TestResult.Skipped;
await this._testPass._didFinishTest(this, test); await this._testPass._didFinishTest(this, test);
return; return;
} }
if (test.declaredMode === TestMode.Skip) { if (test.expectation === TestExpectation.Fail) {
await this._testPass._willStartTest(this, test); await this._testPass._willStartTest(this, test);
test.result = TestResult.Skipped; test.result = TestResult.MarkedAsFailing;
await this._testPass._didFinishTest(this, test); await this._testPass._didFinishTest(this, test);
return; return;
} }
@ -443,14 +448,17 @@ class TestPass {
function specBuilder(defaultTimeout, action) { function specBuilder(defaultTimeout, action) {
let mode = TestMode.Run; let mode = TestMode.Run;
let expectation = TestExpectation.Ok;
let repeat = 1; let repeat = 1;
let timeout = defaultTimeout; let timeout = defaultTimeout;
const func = (...args) => { const func = (...args) => {
for (let i = 0; i < repeat; ++i) for (let i = 0; i < repeat; ++i)
action(mode, timeout, ...args); action(mode, expectation, timeout, ...args);
mode = TestMode.Run; mode = TestMode.Run;
expectation = TestExpectation.Ok;
repeat = 1; repeat = 1;
timeout = defaultTimeout;
}; };
func.skip = condition => { func.skip = condition => {
@ -460,12 +468,7 @@ function specBuilder(defaultTimeout, action) {
}; };
func.fail = condition => { func.fail = condition => {
if (condition) if (condition)
mode = TestMode.MarkAsFailing; expectation = TestExpectation.Fail;
return func;
};
func.flake = condition => {
if (condition)
mode = TestMode.Flake;
return func; return func;
}; };
func.slow = () => { func.slow = () => {
@ -507,14 +510,14 @@ class TestRunner extends EventEmitter {
} }
} }
this.describe = specBuilder(this._timeout, (mode, timeout, ...args) => this._addSuite(mode, ...args)); this.describe = specBuilder(this._timeout, (mode, expectation, timeout, ...args) => this._addSuite(mode, expectation, ...args));
this.fdescribe = specBuilder(this._timeout, (mode, timeout, ...args) => this._addSuite(TestMode.Focus, ...args)); this.fdescribe = specBuilder(this._timeout, (mode, expectation, timeout, ...args) => this._addSuite(TestMode.Focus, expectation, ...args));
this.xdescribe = specBuilder(this._timeout, (mode, timeout, ...args) => this._addSuite(TestMode.Skip, ...args)); this.xdescribe = specBuilder(this._timeout, (mode, expectation, timeout, ...args) => this._addSuite(TestMode.Skip, expectation, ...args));
this.it = specBuilder(this._timeout, (mode, timeout, name, callback) => this._addTest(name, callback, mode, timeout)); this.it = specBuilder(this._timeout, (mode, expectation, timeout, name, callback) => this._addTest(name, callback, mode, expectation, timeout));
this.fit = specBuilder(this._timeout, (mode, timeout, name, callback) => this._addTest(name, callback, TestMode.Focus, timeout)); this.fit = specBuilder(this._timeout, (mode, expectation, timeout, name, callback) => this._addTest(name, callback, TestMode.Focus, expectation, timeout));
this.xit = specBuilder(this._timeout, (mode, timeout, name, callback) => this._addTest(name, callback, TestMode.Skip, timeout)); this.xit = specBuilder(this._timeout, (mode, expectation, timeout, name, callback) => this._addTest(name, callback, TestMode.Skip, expectation, timeout));
this.dit = specBuilder(this._timeout, (mode, timeout, name, callback) => { this.dit = specBuilder(this._timeout, (mode, expectation, timeout, name, callback) => {
const test = this._addTest(name, callback, TestMode.Focus, INFINITE_TIMEOUT); const test = this._addTest(name, callback, TestMode.Focus, expectation, INFINITE_TIMEOUT);
const N = callback.toString().split('\n').length; const N = callback.toString().split('\n').length;
for (let i = 0; i < N; ++i) for (let i = 0; i < N; ++i)
this._debuggerLogBreakpointLines.set(test.location.filePath, i + test.location.lineNumber); this._debuggerLogBreakpointLines.set(test.location.filePath, i + test.location.lineNumber);
@ -529,37 +532,29 @@ class TestRunner extends EventEmitter {
loadTests(module, ...args) { loadTests(module, ...args) {
if (typeof module.describe === 'function') if (typeof module.describe === 'function')
this._addSuite(TestMode.Run, '', module.describe, ...args); this._addSuite(TestMode.Run, TestExpectation.Ok, '', module.describe, ...args);
if (typeof module.fdescribe === 'function') if (typeof module.fdescribe === 'function')
this._addSuite(TestMode.Focus, '', module.fdescribe, ...args); this._addSuite(TestMode.Focus, TestExpectation.Ok, '', module.fdescribe, ...args);
if (typeof module.xdescribe === 'function') if (typeof module.xdescribe === 'function')
this._addSuite(TestMode.Skip, '', module.xdescribe, ...args); this._addSuite(TestMode.Skip, TestExpectation.Ok, '', module.xdescribe, ...args);
} }
_addTest(name, callback, mode, timeout) { _addTest(name, callback, mode, expectation, timeout) {
let suite = this._currentSuite; for (let suite = this._currentSuite; suite; suite = suite.parentSuite) {
let markedAsFailing = suite.declaredMode === TestMode.MarkAsFailing; if (suite.expectation === TestExpectation.Fail)
while ((suite = suite.parentSuite)) expectation = TestExpectation.Fail;
markedAsFailing |= suite.declaredMode === TestMode.MarkAsFailing; if (suite.declaredMode === TestMode.Skip)
if (markedAsFailing) mode = TestMode.Skip;
mode = TestMode.MarkAsFailing; }
const test = new Test(this._currentSuite, name, callback, mode, expectation, timeout);
suite = this._currentSuite;
let skip = suite.declaredMode === TestMode.Skip;
while ((suite = suite.parentSuite))
skip |= suite.declaredMode === TestMode.Skip;
if (skip)
mode = TestMode.Skip;
const test = new Test(this._currentSuite, name, callback, mode, timeout);
this._currentSuite.children.push(test); this._currentSuite.children.push(test);
this._tests.push(test); this._tests.push(test);
return test; return test;
} }
_addSuite(mode, name, callback, ...args) { _addSuite(mode, expectation, name, callback, ...args) {
const oldSuite = this._currentSuite; const oldSuite = this._currentSuite;
const suite = new Suite(this._currentSuite, name, mode); const suite = new Suite(this._currentSuite, name, mode, expectation);
this._suites.push(suite); this._suites.push(suite);
this._currentSuite.children.push(suite); this._currentSuite.children.push(suite);
this._currentSuite = suite; this._currentSuite = suite;
@ -650,11 +645,11 @@ class TestRunner extends EventEmitter {
} }
focusedSuites() { focusedSuites() {
return this._suites.filter(suite => suite.declaredMode === 'focus'); return this._suites.filter(suite => suite.declaredMode === TestMode.Focus);
} }
focusedTests() { focusedTests() {
return this._tests.filter(test => test.declaredMode === 'focus'); return this._tests.filter(test => test.declaredMode === TestMode.Focus);
} }
hasFocusedTestsOrSuites() { hasFocusedTestsOrSuites() {
@ -664,7 +659,7 @@ class TestRunner extends EventEmitter {
focusMatchingTests(fullNameRegex) { focusMatchingTests(fullNameRegex) {
for (const test of this._tests) { for (const test of this._tests) {
if (fullNameRegex.test(test.fullName)) if (fullNameRegex.test(test.fullName))
test.declaredMode = 'focus'; test.declaredMode = TestMode.Focus;
} }
} }
@ -673,19 +668,19 @@ class TestRunner extends EventEmitter {
} }
failedTests() { failedTests() {
return this._tests.filter(test => test.result === 'failed' || test.result === 'timedout' || test.result === 'crashed'); return this._tests.filter(test => test.result === TestResult.Failed || test.result === TestResult.TimedOut || test.result === TestResult.Crashed);
} }
passedTests() { passedTests() {
return this._tests.filter(test => test.result === 'ok'); return this._tests.filter(test => test.result === TestResult.Ok);
} }
skippedTests() { skippedTests() {
return this._tests.filter(test => test.result === 'skipped'); return this._tests.filter(test => test.result === TestResult.Skipped);
} }
markedAsFailingTests() { markedAsFailingTests() {
return this._tests.filter(test => test.result === 'markedAsFailing'); return this._tests.filter(test => test.result === TestResult.MarkedAsFailing);
} }
parallel() { parallel() {