feat(testrunner): allow filtering by name and show all focused tests (#1354)

test/test.js

@@ -124,9 +124,10 @@ for (const browserConfig of BROWSER_CONFIGS) {
   });
 }
 
-if (process.env.CI && testRunner.hasFocusedTestsOrSuites()) {
-  console.error('ERROR: "focused" tests/suites are prohibited on bots. Remove any "fit"/"fdescribe" declarations.');
-  process.exit(1);
+const filterArgIndex = process.argv.indexOf('--filter');
+if (filterArgIndex !== -1) {
+  const filter = process.argv[filterArgIndex + 1];
+  testRunner.focusMatchingTests(new RegExp(filter, 'i'));
 }
 
 new Reporter(testRunner, {

utils/testrunner/Reporter.js

@@ -47,7 +47,18 @@ class Reporter {
     if (allTests.length === runnableTests.length) {
       console.log(`Running all ${colors.yellow(runnableTests.length)} tests on ${colors.yellow(this._runner.parallel())} worker${this._runner.parallel() > 1 ? 's' : ''}:\n`);
     } else {
-      console.log(`Running ${colors.yellow(runnableTests.length)} focused tests out of total ${colors.yellow(allTests.length)} on ${colors.yellow(this._runner.parallel())} worker${this._runner.parallel() > 1 ? 's' : ''}:\n`);
+      console.log(`Running ${colors.yellow(runnableTests.length)} focused tests out of total ${colors.yellow(allTests.length)} on ${colors.yellow(this._runner.parallel())} worker${this._runner.parallel() > 1 ? 's' : ''}`);
+      console.log('');
+      const focusedSuites = this._runner.focusedSuites();
+      const focusedTests = this._runner.focusedTests();
+      if (focusedSuites.length) {
+        console.log('Focused Suites and Tests:');
+        for (let i = 0; i < focusedSuites.length; ++i)
+          console.log(`  ${i + 1}) ${focusedSuites[i].fullName} (${formatLocation(focusedSuites[i].location)})`);
+        for (let i = 0; i < focusedTests.length; ++i)
+          console.log(`  ${i + 1 + focusedSuites.length}) ${focusedTests[i].fullName} (${formatLocation(focusedTests[i].location)})`);
+        console.log('');
+      }
     }
   }
 

utils/testrunner/TestRunner.js

@@ -116,6 +116,7 @@ class Suite {
     this.declaredMode = declaredMode;
     /** @type {!Array<(!Test|!Suite)>} */
     this.children = [];
+    this.location = getCallerLocation(__filename);
 
     this.beforeAll = null;
     this.beforeEach = null;
@@ -441,18 +442,19 @@ class TestRunner extends EventEmitter {
       timeout = 10 * 1000, // Default timeout is 10 seconds.
       parallel = 1,
       breakOnFailure = false,
+      crashIfTestsAreFocusedOnCI = true,
       disableTimeoutWhenInspectorIsEnabled = true,
     } = options;
+    this._crashIfTestsAreFocusedOnCI = crashIfTestsAreFocusedOnCI;
     this._sourceMapSupport = new SourceMapSupport();
     this._rootSuite = new Suite(null, '', TestMode.Run);
     this._currentSuite = this._rootSuite;
     this._tests = [];
+    this._suites = [];
     this._timeout = timeout === 0 ? INFINITE_TIMEOUT : timeout;
     this._parallel = parallel;
     this._breakOnFailure = breakOnFailure;
 
-    this._hasFocusedTestsOrSuites = false;
-
     if (MAJOR_NODEJS_VERSION >= 8 && disableTimeoutWhenInspectorIsEnabled) {
       if (inspector.url()) {
         console.log('TestRunner detected inspector; overriding certain properties to be debugger-friendly');
@@ -508,18 +510,17 @@ class TestRunner extends EventEmitter {
     const test = new Test(this._currentSuite, name, callback, mode, timeout);
     this._currentSuite.children.push(test);
     this._tests.push(test);
-    this._hasFocusedTestsOrSuites = this._hasFocusedTestsOrSuites || mode === TestMode.Focus;
     return test;
   }
 
   _addSuite(mode, name, callback, ...args) {
     const oldSuite = this._currentSuite;
     const suite = new Suite(this._currentSuite, name, mode);
+    this._suites.push(suite);
     this._currentSuite.children.push(suite);
     this._currentSuite = suite;
     callback(...args);
     this._currentSuite = oldSuite;
-    this._hasFocusedTestsOrSuites = this._hasFocusedTestsOrSuites || mode === TestMode.Focus;
   }
 
   _addHook(hookName, callback) {
@@ -530,15 +531,21 @@ class TestRunner extends EventEmitter {
 
   async run() {
     let session = this._debuggerLogBreakpointLines.size ? await setLogBreakpoints(this._debuggerLogBreakpointLines) : null;
-    const runnableTests = this._runnableTests();
+    const runnableTests = this.runnableTests();
     this.emit(TestRunner.Events.Started, runnableTests);
+
+    const result = {};
+    if (this._crashIfTestsAreFocusedOnCI && process.env.CI && this.hasFocusedTestsOrSuites()) {
+      result.result = TestResult.Crashed;
+      result.exitCode = 2;
+      result.terminationMessage = '"focused" tests or suites are probitted on CI';
+    } else {
       this._runningPass = new TestPass(this, this._parallel, this._breakOnFailure);
       const termination = await this._runningPass.run(runnableTests).catch(e => {
         console.error(e);
         throw e;
       });
       this._runningPass = null;
-    const result = {};
       if (termination) {
         result.result = termination.result;
         result.exitCode = 130;
@@ -553,6 +560,7 @@ class TestRunner extends EventEmitter {
           result.exitCode = 0;
         }
       }
+    }
     this.emit(TestRunner.Events.Finished, result);
     if (session)
       session.disconnect();
@@ -569,8 +577,8 @@ class TestRunner extends EventEmitter {
     return this._timeout;
   }
 
-  _runnableTests() {
-    if (!this._hasFocusedTestsOrSuites)
+  runnableTests() {
+    if (!this.hasFocusedTestsOrSuites())
       return this._tests;
 
     const tests = [];
@@ -601,8 +609,23 @@ class TestRunner extends EventEmitter {
     return tests.map(t => t.test);
   }
 
+  focusedSuites() {
+    return this._suites.filter(suite => suite.declaredMode === 'focus');
+  }
+
+  focusedTests() {
+    return this._tests.filter(test => test.declaredMode === 'focus');
+  }
+
   hasFocusedTestsOrSuites() {
-    return this._hasFocusedTestsOrSuites;
+    return !!this.focusedTests().length || !!this.focusedSuites().length;
+  }
+
+  focusMatchingTests(fullNameRegex) {
+    for (const test of this._tests) {
+      if (fullNameRegex.test(test.fullName))
+        test.declaredMode = 'focus';
+    }
   }
 
   tests() {

utils/testrunner/test/testrunner.spec.js

@@ -1,5 +1,12 @@
 const {TestRunner} = require('..');
 
+function newTestRunner(options) {
+  return new TestRunner({
+    crashIfTestsAreFocusedOnCI: false,
+    ...options,
+  });
+}
+
 module.exports.addTests = function({testRunner, expect}) {
   const {describe, fdescribe, xdescribe} = testRunner;
   const {it, xit, fit} = testRunner;