chore(test-runner): remove the notion of Spec (#7661)

We now have Suites and Tests. When running multiple projects the whole
suite is cloned for each project. Same happens for repeatEach.

This simplifies the reporters API, but there is still room for improvement.

JSON reporter continues to produce old json output.

src/test/dispatcher.ts

@@ -50,10 +50,8 @@ export class Dispatcher {
 
     this._suite = suite;
     for (const suite of this._suite.suites) {
-      for (const spec of suite._allSpecs()) {
+      for (const test of suite._allTests())
-        for (const test of spec.tests)
+        this._testById.set(test._id, { test, result: test._appendTestResult() });
-          this._testById.set(test._id, { test, result: test._appendTestResult() });
-      }
     }
 
     this._queue = this._filesSortedByWorkerHash();
@@ -83,31 +81,29 @@ export class Dispatcher {
     const entriesByWorkerHashAndFile = new Map<string, Map<string, DispatcherEntry>>();
     for (const fileSuite of this._suite.suites) {
       const file = fileSuite._requireFile;
-      for (const spec of fileSuite._allSpecs()) {
+      for (const test of fileSuite._allTests()) {
-        for (const test of spec.tests) {
+        let entriesByFile = entriesByWorkerHashAndFile.get(test._workerHash);
-          let entriesByFile = entriesByWorkerHashAndFile.get(test._workerHash);
+        if (!entriesByFile) {
-          if (!entriesByFile) {
+          entriesByFile = new Map();
-            entriesByFile = new Map();
+          entriesByWorkerHashAndFile.set(test._workerHash, entriesByFile);
-            entriesByWorkerHashAndFile.set(test._workerHash, entriesByFile);
-          }
-          let entry = entriesByFile.get(file);
-          if (!entry) {
-            entry = {
-              runPayload: {
-                entries: [],
-                file,
-              },
-              repeatEachIndex: test._repeatEachIndex,
-              projectIndex: test._projectIndex,
-              hash: test._workerHash,
-            };
-            entriesByFile.set(file, entry);
-          }
-          entry.runPayload.entries.push({
-            retry: this._testById.get(test._id)!.result.retry,
-            testId: test._id,
-          });
         }
+        let entry = entriesByFile.get(file);
+        if (!entry) {
+          entry = {
+            runPayload: {
+              entries: [],
+              file,
+            },
+            repeatEachIndex: fileSuite._repeatEachIndex,
+            projectIndex: fileSuite._projectIndex,
+            hash: test._workerHash,
+          };
+          entriesByFile.set(file, entry);
+        }
+        entry.runPayload.entries.push({
+          retry: this._testById.get(test._id)!.result.retry,
+          testId: test._id,
+        });
       }
     }
 

src/test/project.ts

@@ -15,7 +15,7 @@
  */
 
 import type { TestType, FullProject, Fixtures, FixturesWithLocation } from './types';
-import { Spec, Test } from './test';
+import { Suite, Test } from './test';
 import { FixturePool } from './fixtures';
 import { DeclaredFixtures, TestTypeImpl } from './testType';
 
@@ -24,7 +24,7 @@ export class ProjectImpl {
   private index: number;
   private defines = new Map<TestType<any, any>, Fixtures>();
   private testTypePools = new Map<TestTypeImpl, FixturePool>();
-  private specPools = new Map<Spec, FixturePool>();
+  private testPools = new Map<Test, FixturePool>();
 
   constructor(project: FullProject, index: number) {
     this.config = project;
@@ -52,51 +52,60 @@ export class ProjectImpl {
     return this.testTypePools.get(testType)!;
   }
 
-  buildPool(spec: Spec): FixturePool {
+  // TODO: we can optimize this function by building the pool inline in cloneSuite
-    if (!this.specPools.has(spec)) {
+  private buildPool(test: Test): FixturePool {
-      let pool = this.buildTestTypePool(spec._testType);
+    if (!this.testPools.has(test)) {
-      const overrides: Fixtures = spec.parent!._buildFixtureOverrides();
+      let pool = this.buildTestTypePool(test._testType);
+      const overrides: Fixtures = test.parent!._buildFixtureOverrides();
       if (Object.entries(overrides).length) {
         const overridesWithLocation = {
           fixtures: overrides,
           location: {
-            file: spec.file,
+            file: test.file,
             line: 1,  // TODO: capture location
             column: 1,  // TODO: capture location
           }
         };
         pool = new FixturePool([overridesWithLocation], pool);
       }
-      this.specPools.set(spec, pool);
+      this.testPools.set(test, pool);
 
-      pool.validateFunction(spec.fn, 'Test', true, spec);
+      pool.validateFunction(test.fn, 'Test', true, test);
-      for (let parent = spec.parent; parent; parent = parent.parent) {
+      for (let parent = test.parent; parent; parent = parent.parent) {
         for (const hook of parent._hooks)
           pool.validateFunction(hook.fn, hook.type + ' hook', hook.type === 'beforeEach' || hook.type === 'afterEach', hook.location);
         for (const modifier of parent._modifiers)
           pool.validateFunction(modifier.fn, modifier.type + ' modifier', true, modifier.location);
       }
     }
-    return this.specPools.get(spec)!;
+    return this.testPools.get(test)!;
   }
 
-  generateTests(spec: Spec, repeatEachIndex?: number) {
+  cloneSuite(suite: Suite, repeatEachIndex: number, filter: (test: Test) => boolean): Suite | undefined {
-    const digest = this.buildPool(spec).digest;
+    const result = suite._clone();
-    const min = repeatEachIndex === undefined ? 0 : repeatEachIndex;
+    result._repeatEachIndex = repeatEachIndex;
-    const max = repeatEachIndex === undefined ? this.config.repeatEach - 1 : repeatEachIndex;
+    result._projectIndex = this.index;
-    const tests: Test[] = [];
+    for (const entry of suite._entries) {
-    for (let i = min; i <= max; i++) {
+      if (entry instanceof Suite) {
-      const test = new Test(spec);
+        const cloned = this.cloneSuite(entry, repeatEachIndex, filter);
-      test.projectName = this.config.name;
+        if (cloned)
-      test.retries = this.config.retries;
+          result._addSuite(cloned);
-      test._repeatEachIndex = i;
+      } else {
-      test._projectIndex = this.index;
+        const pool = this.buildPool(entry);
-      test._workerHash = `run${this.index}-${digest}-repeat${i}`;
+        const test = entry._clone();
-      test._id = `${spec._ordinalInFile}@${spec._requireFile}#run${this.index}-repeat${i}`;
+        test.projectName = this.config.name;
-      spec.tests.push(test);
+        test.retries = this.config.retries;
-      tests.push(test);
+        test._workerHash = `run${this.index}-${pool.digest}-repeat${repeatEachIndex}`;
+        test._id = `${entry._ordinalInFile}@${entry._requireFile}#run${this.index}-repeat${repeatEachIndex}`;
+        test._pool = pool;
+        test._buildFullTitle(suite.fullTitle());
+        if (!filter(test))
+          continue;
+        result._addTest(test);
+      }
     }
-    return tests;
+    if (result._entries.length)
+      return result;
   }
 
   private resolveFixtures(testType: TestTypeImpl): FixturesWithLocation[] {

src/test/reporter.ts

@@ -23,23 +23,16 @@ export interface Suite {
   line: number;
   column: number;
   suites: Suite[];
-  specs: Spec[];
+  tests: Test[];
   findTest(fn: (test: Test) => boolean | void): boolean;
-  findSpec(fn: (spec: Spec) => boolean | void): boolean;
   totalTestCount(): number;
 }
-export interface Spec {
+export interface Test {
   suite: Suite;
   title: string;
   file: string;
   line: number;
   column: number;
-  tests: Test[];
-  fullTitle(): string;
-  ok(): boolean;
-}
-export interface Test {
-  spec: Spec;
   results: TestResult[];
   skipped: boolean;
   expectedStatus: TestStatus;
@@ -61,9 +54,9 @@ export interface TestResult {
   stdout: (string | Buffer)[];
   stderr: (string | Buffer)[];
 }
-export type FullResult = {
+export interface FullResult {
   status: 'passed' | 'failed' | 'timedout' | 'interrupted';
-};
+}
 export interface Reporter {
   onBegin(config: FullConfig, suite: Suite): void;
   onTestBegin(test: Test): void;

src/test/reporters/base.ts

@@ -21,7 +21,7 @@ import fs from 'fs';
 import milliseconds from 'ms';
 import path from 'path';
 import StackUtils from 'stack-utils';
-import { FullConfig, TestStatus, Test, Spec, Suite, TestResult, TestError, Reporter, FullResult } from '../reporter';
+import { FullConfig, TestStatus, Test, Suite, TestResult, TestError, Reporter, FullResult } from '../reporter';
 
 const stackUtils = new StackUtils();
 
@@ -56,7 +56,7 @@ export class BaseReporter implements Reporter  {
   }
 
   onTestEnd(test: Test, result: TestResult) {
-    const relativePath = relativeSpecPath(this.config, test.spec);
+    const relativePath = relativeTestPath(this.config, test);
     const fileAndProject = relativePath + (test.projectName ? ` [${test.projectName}]` : '');
     const duration = this.fileDurations.get(fileAndProject) || 0;
     this.fileDurations.set(fileAndProject, duration + result.duration);
@@ -157,14 +157,13 @@ export function formatFailure(config: FullConfig, test: Test, index?: number): s
   return tokens.join('\n');
 }
 
-function relativeSpecPath(config: FullConfig, spec: Spec): string {
+function relativeTestPath(config: FullConfig, test: Test): string {
-  return path.relative(config.rootDir, spec.file) || path.basename(spec.file);
+  return path.relative(config.rootDir, test.file) || path.basename(test.file);
 }
 
 export function formatTestTitle(config: FullConfig, test: Test): string {
-  const spec = test.spec;
+  let relativePath = relativeTestPath(config, test);
-  let relativePath = relativeSpecPath(config, spec);
+  relativePath += ':' + test.line + ':' + test.column;
-  relativePath += ':' + spec.line + ':' + spec.column;
   return `${relativePath} › ${test.fullTitle()}`;
 }
 
@@ -183,9 +182,9 @@ function formatFailedResult(test: Test, result: TestResult): string {
     tokens.push('');
     tokens.push(indent(colors.red(`Timeout of ${test.timeout}ms exceeded.`), '    '));
     if (result.error !== undefined)
-      tokens.push(indent(formatError(result.error, test.spec.file), '    '));
+      tokens.push(indent(formatError(result.error, test.file), '    '));
   } else {
-    tokens.push(indent(formatError(result.error!, test.spec.file), '    '));
+    tokens.push(indent(formatError(result.error!, test.file), '    '));
   }
   return tokens.join('\n');
 }

src/test/reporters/json.ts

@@ -17,7 +17,7 @@
 import fs from 'fs';
 import path from 'path';
 import EmptyReporter from './empty';
-import { FullConfig, Test, Suite, Spec, TestResult, TestError, FullResult, TestStatus } from '../reporter';
+import { FullConfig, Test, Suite, TestResult, TestError, FullResult, TestStatus } from '../reporter';
 
 export interface JSONReport {
   config: Omit<FullConfig, 'projects'> & {
@@ -119,13 +119,54 @@ class JSONReporter extends EmptyReporter {
           };
         })
       },
-      suites: this.suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s) as JSONReportSuite[],
+      suites: this._mergeSuites(this.suite.suites),
       errors: this._errors
     };
   }
 
+  private _mergeSuites(suites: Suite[]): JSONReportSuite[] {
+    debugger;
+    const fileSuites = new Map<string, JSONReportSuite>();
+    const result: JSONReportSuite[] = [];
+    for (const suite of suites) {
+      if (!fileSuites.has(suite.file)) {
+        const serialized = this._serializeSuite(suite);
+        if (serialized) {
+          fileSuites.set(suite.file, serialized);
+          result.push(serialized);
+        }
+      } else {
+        this._mergeTestsFromSuite(fileSuites.get(suite.file)!, suite);
+      }
+    }
+    return result;
+  }
+
+  private _mergeTestsFromSuite(to: JSONReportSuite, from: Suite) {
+    for (const fromSuite of from.suites) {
+      const toSuite = (to.suites || []).find(s => s.title === fromSuite.title && s.file === toPosixPath(path.relative(this.config.rootDir, fromSuite.file)) && s.line === fromSuite.line && s.column === fromSuite.column);
+      if (toSuite) {
+        this._mergeTestsFromSuite(toSuite, fromSuite);
+      } else {
+        const serialized = this._serializeSuite(fromSuite);
+        if (serialized) {
+          if (!to.suites)
+            to.suites = [];
+          to.suites.push(serialized);
+        }
+      }
+    }
+    for (const test of from.tests) {
+      const toSpec = to.specs.find(s => s.title === test.title && s.file === toPosixPath(path.relative(this.config.rootDir, test.file)) && s.line === test.line && s.column === test.column);
+      if (toSpec)
+        toSpec.tests.push(this._serializeTest(test));
+      else
+        to.specs.push(this._serializeTestSpec(test));
+    }
+  }
+
   private _serializeSuite(suite: Suite): null | JSONReportSuite {
-    if (!suite.findSpec(test => true))
+    if (!suite.findTest(test => true))
       return null;
     const suites = suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s) as JSONReportSuite[];
     return {
@@ -133,19 +174,19 @@ class JSONReporter extends EmptyReporter {
       file: toPosixPath(path.relative(this.config.rootDir, suite.file)),
       line: suite.line,
       column: suite.column,
-      specs: suite.specs.map(test => this._serializeTestSpec(test)),
+      specs: suite.tests.map(test => this._serializeTestSpec(test)),
       suites: suites.length ? suites : undefined,
     };
   }
 
-  private _serializeTestSpec(spec: Spec): JSONReportSpec {
+  private _serializeTestSpec(test: Test): JSONReportSpec {
     return {
-      title: spec.title,
+      title: test.title,
-      ok: spec.ok(),
+      ok: test.ok(),
-      tests: spec.tests.map(r => this._serializeTest(r)),
+      tests: [ this._serializeTest(test) ],
-      file: toPosixPath(path.relative(this.config.rootDir, spec.file)),
+      file: toPosixPath(path.relative(this.config.rootDir, test.file)),
-      line: spec.line,
+      line: test.line,
-      column: spec.column,
+      column: test.column,
     };
   }
 

src/test/reporters/junit.ts

@@ -121,7 +121,7 @@ class JUnitReporter extends EmptyReporter {
     const entry = {
       name: 'testcase',
       attributes: {
-        name: test.spec.fullTitle(),
+        name: test.fullTitle(),
         classname: formatTestTitle(this.config, test),
         time: (test.results.reduce((acc, value) => acc + value.duration, 0)) / 1000
       },
@@ -138,7 +138,7 @@ class JUnitReporter extends EmptyReporter {
       entry.children.push({
         name: 'failure',
         attributes: {
-          message: `${path.basename(test.spec.file)}:${test.spec.line}:${test.spec.column} ${test.spec.title}`,
+          message: `${path.basename(test.file)}:${test.line}:${test.column} ${test.title}`,
           type: 'FAILURE',
         },
         text: stripAscii(formatFailure(this.config, test))

src/test/runner.ts

@@ -22,7 +22,7 @@ import * as path from 'path';
 import { promisify } from 'util';
 import { Dispatcher } from './dispatcher';
 import { createMatcher, FilePatternFilter, monotonicTime, raceAgainstDeadline } from './util';
-import { Spec, Suite } from './test';
+import { Test, Suite } from './test';
 import { Loader } from './loader';
 import { Reporter } from './reporter';
 import { Multiplexer } from './reporters/multiplexer';
@@ -41,16 +41,16 @@ const removeFolderAsync = promisify(rimraf);
 const readDirAsync = promisify(fs.readdir);
 const readFileAsync = promisify(fs.readFile);
 
-type RunResultStatus = 'passed' | 'failed' | 'sigint' | 'forbid-only' | 'clashing-spec-titles' | 'no-tests' | 'timedout';
+type RunResultStatus = 'passed' | 'failed' | 'sigint' | 'forbid-only' | 'clashing-test-titles' | 'no-tests' | 'timedout';
 
 type RunResult = {
-  status: Exclude<RunResultStatus, 'forbid-only' | 'clashing-spec-titles'>;
+  status: Exclude<RunResultStatus, 'forbid-only' | 'clashing-test-titles'>;
 } | {
   status: 'forbid-only',
   locations: string[]
 } | {
-  status: 'clashing-spec-titles',
+  status: 'clashing-test-titles',
-  clashingSpecs: Map<string, Spec[]>
+  clashingTests: Map<string, Test[]>
 };
 
 export class Runner {
@@ -114,13 +114,13 @@ export class Runner {
       console.error('=================');
       console.error(' no tests found.');
       console.error('=================');
-    } else if (result?.status === 'clashing-spec-titles') {
+    } else if (result?.status === 'clashing-test-titles') {
       console.error('=================');
       console.error(' duplicate test titles are not allowed.');
-      for (const [title, specs] of result?.clashingSpecs.entries()) {
+      for (const [title, tests] of result?.clashingTests.entries()) {
         console.error(` - title: ${title}`);
-        for (const spec of specs)
+        for (const test of tests)
-          console.error(`   - ${buildItemLocation(config.rootDir, spec)}`);
+          console.error(`   - ${buildItemLocation(config.rootDir, test)}`);
         console.error('=================');
       }
     }
@@ -175,38 +175,42 @@ export class Runner {
       for (const file of allTestFiles)
         await this._loader.loadTestFile(file);
 
-      const rootSuite = new Suite('');
+      const preprocessRoot = new Suite('');
       for (const fileSuite of this._loader.fileSuites().values())
-        rootSuite._addSuite(fileSuite);
+        preprocessRoot._addSuite(fileSuite);
       if (config.forbidOnly) {
-        const onlySpecAndSuites = rootSuite._getOnlyItems();
+        const onlyTestsAndSuites = preprocessRoot._getOnlyItems();
-        if (onlySpecAndSuites.length > 0)
+        if (onlyTestsAndSuites.length > 0)
-          return { status: 'forbid-only', locations: onlySpecAndSuites.map(specOrSuite => `${buildItemLocation(config.rootDir, specOrSuite)} > ${specOrSuite.fullTitle()}`) };
+          return { status: 'forbid-only', locations: onlyTestsAndSuites.map(testOrSuite => `${buildItemLocation(config.rootDir, testOrSuite)} > ${testOrSuite.fullTitle()}`) };
       }
-      const uniqueSpecs = getUniqueSpecsPerSuite(rootSuite);
+      const clashingTests = getClashingTestsPerSuite(preprocessRoot);
-      if (uniqueSpecs.size > 0)
+      if (clashingTests.size > 0)
-        return { status: 'clashing-spec-titles', clashingSpecs: uniqueSpecs };
+        return { status: 'clashing-test-titles', clashingTests: clashingTests };
-      filterOnly(rootSuite);
+      filterOnly(preprocessRoot);
-      filterByFocusedLine(rootSuite, testFileReFilters);
+      filterByFocusedLine(preprocessRoot, testFileReFilters);
 
       const fileSuites = new Map<string, Suite>();
-      for (const fileSuite of rootSuite.suites)
+      for (const fileSuite of preprocessRoot.suites)
         fileSuites.set(fileSuite._requireFile, fileSuite);
 
       const outputDirs = new Set<string>();
       const grepMatcher = createMatcher(config.grep);
       const grepInvertMatcher = config.grepInvert ? createMatcher(config.grepInvert) : null;
+      const rootSuite = new Suite('');
       for (const project of projects) {
         for (const file of files.get(project)!) {
           const fileSuite = fileSuites.get(file);
           if (!fileSuite)
             continue;
-          for (const spec of fileSuite._allSpecs()) {
+          for (let repeatEachIndex = 0; repeatEachIndex < project.config.repeatEach; repeatEachIndex++) {
-            const fullTitle = spec._testFullTitle(project.config.name);
+            const cloned = project.cloneSuite(fileSuite, repeatEachIndex, test => {
-            if (grepInvertMatcher?.(fullTitle))
+              const fullTitle = test.fullTitle();
-              continue;
+              if (grepInvertMatcher?.(fullTitle))
-            if (grepMatcher(fullTitle))
+                return false;
-              project.generateTests(spec);
+              return grepMatcher(fullTitle);
+            });
+            if (cloned)
+              rootSuite._addSuite(cloned);
           }
         }
         outputDirs.add(project.config.outputDir);
@@ -235,7 +239,7 @@ export class Runner {
       if (process.stdout.isTTY) {
         const workers = new Set();
         rootSuite.findTest(test => {
-          workers.add(test.spec._requireFile + test._workerHash);
+          workers.add(test._requireFile + test._workerHash);
         });
         console.log();
         const jobs = Math.min(config.workers, workers.size);
@@ -259,7 +263,7 @@ export class Runner {
         return { status: 'sigint' };
       }
 
-      const failed = hasWorkerErrors || rootSuite.findSpec(spec => !spec.ok());
+      const failed = hasWorkerErrors || rootSuite.findTest(test => !test.ok());
       await this._reporter.onEnd({ status: failed ? 'failed' : 'passed' });
       return { status: failed ? 'failed' : 'passed' };
     } finally {
@@ -274,27 +278,27 @@ export class Runner {
 
 function filterOnly(suite: Suite) {
   const suiteFilter = (suite: Suite) => suite._only;
-  const specFilter = (spec: Spec) => spec._only;
+  const testFilter = (test: Test) => test._only;
-  return filterSuite(suite, suiteFilter, specFilter);
+  return filterSuite(suite, suiteFilter, testFilter);
 }
 
 function filterByFocusedLine(suite: Suite, focusedTestFileLines: FilePatternFilter[]) {
-  const testFileLineMatches = (specFileName: string, specLine: number) => focusedTestFileLines.some(({re, line}) => {
+  const testFileLineMatches = (testFileName: string, testLine: number) => focusedTestFileLines.some(({re, line}) => {
     re.lastIndex = 0;
-    return re.test(specFileName) && (line === specLine || line === null);
+    return re.test(testFileName) && (line === testLine || line === null);
   });
   const suiteFilter = (suite: Suite) => testFileLineMatches(suite.file, suite.line);
-  const specFilter = (spec: Spec) => testFileLineMatches(spec.file, spec.line);
+  const testFilter = (test: Test) => testFileLineMatches(test.file, test.line);
-  return filterSuite(suite, suiteFilter, specFilter);
+  return filterSuite(suite, suiteFilter, testFilter);
 }
 
-function filterSuite(suite: Suite, suiteFilter: (suites: Suite) => boolean, specFilter: (spec: Spec) => boolean) {
+function filterSuite(suite: Suite, suiteFilter: (suites: Suite) => boolean, testFilter: (test: Test) => boolean) {
-  const onlySuites = suite.suites.filter(child => filterSuite(child, suiteFilter, specFilter) || suiteFilter(child));
+  const onlySuites = suite.suites.filter(child => filterSuite(child, suiteFilter, testFilter) || suiteFilter(child));
-  const onlyTests = suite.specs.filter(specFilter);
+  const onlyTests = suite.tests.filter(testFilter);
   const onlyEntries = new Set([...onlySuites, ...onlyTests]);
   if (onlyEntries.size) {
     suite.suites = onlySuites;
-    suite.specs = onlyTests;
+    suite.tests = onlyTests;
     suite._entries = suite._entries.filter(e => onlyEntries.has(e)); // Preserve the order.
     return true;
   }
@@ -370,29 +374,29 @@ async function collectFiles(testDir: string): Promise<string[]> {
   return files;
 }
 
-function getUniqueSpecsPerSuite(rootSuite: Suite): Map<string, Spec[]> {
+function getClashingTestsPerSuite(rootSuite: Suite): Map<string, Test[]> {
-  function visit(suite: Suite, clashingSpecs: Map<string, Spec[]>) {
+  function visit(suite: Suite, clashingTests: Map<string, Test[]>) {
     for (const childSuite of suite.suites)
-      visit(childSuite, clashingSpecs);
+      visit(childSuite, clashingTests);
-    for (const spec of suite.specs) {
+    for (const test of suite.tests) {
-      const fullTitle = spec.fullTitle();
+      const fullTitle = test.fullTitle();
-      if (!clashingSpecs.has(fullTitle))
+      if (!clashingTests.has(fullTitle))
-        clashingSpecs.set(fullTitle, []);
+        clashingTests.set(fullTitle, []);
-      clashingSpecs.set(fullTitle, clashingSpecs.get(fullTitle)!.concat(spec));
+      clashingTests.set(fullTitle, clashingTests.get(fullTitle)!.concat(test));
     }
   }
-  const out = new Map<string, Spec[]>();
+  const out = new Map<string, Test[]>();
   for (const fileSuite of rootSuite.suites) {
-    const clashingSpecs = new Map<string, Spec[]>();
+    const clashingTests = new Map<string, Test[]>();
-    visit(fileSuite, clashingSpecs);
+    visit(fileSuite, clashingTests);
-    for (const [title, specs] of clashingSpecs.entries()) {
+    for (const [title, tests] of clashingTests.entries()) {
-      if (specs.length > 1)
+      if (tests.length > 1)
-        out.set(title, specs);
+        out.set(title, tests);
     }
   }
   return out;
 }
 
-function buildItemLocation(rootDir: string, specOrSuite: Suite | Spec) {
+function buildItemLocation(rootDir: string, testOrSuite: Suite | Test) {
-  return `${path.relative(rootDir, specOrSuite.file)}:${specOrSuite.line}`;
+  return `${path.relative(rootDir, testOrSuite.file)}:${testOrSuite.line}`;
 }

src/test/test.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+import type { FixturePool } from './fixtures';
 import * as reporterTypes from './reporter';
 import type { TestTypeImpl } from './testType';
 import { Annotations, Location } from './types';
@@ -25,6 +26,7 @@ class Base {
   column: number = 0;
   parent?: Suite;
 
+  _fullTitle: string = '';
   _only = false;
   _requireFile: string = '';
 
@@ -32,39 +34,15 @@ class Base {
     this.title = title;
   }
 
-  titlePath(): string[] {
+  _buildFullTitle(parentFullTitle: string) {
-    if (!this.parent)
+    if (this.title)
-      return [];
+      this._fullTitle = (parentFullTitle ? parentFullTitle + ' ' : '') + this.title;
-    if (!this.title)
+    else
-      return this.parent.titlePath();
+      this._fullTitle = parentFullTitle;
-    return [...this.parent.titlePath(), this.title];
   }
 
   fullTitle(): string {
-    return this.titlePath().join(' ');
+    return this._fullTitle;
-  }
-}
-
-export class Spec extends Base implements reporterTypes.Spec {
-  suite!: Suite;
-  fn: Function;
-  tests: Test[] = [];
-  _ordinalInFile: number;
-  _testType: TestTypeImpl;
-
-  constructor(title: string, fn: Function, ordinalInFile: number, testType: TestTypeImpl) {
-    super(title);
-    this.fn = fn;
-    this._ordinalInFile = ordinalInFile;
-    this._testType = testType;
-  }
-
-  ok(): boolean {
-    return !this.tests.find(r => !r.ok());
-  }
-
-  _testFullTitle(projectName: string) {
-    return (projectName ? `[${projectName}] ` : '') + this.fullTitle();
   }
 }
 
@@ -77,9 +55,9 @@ export type Modifier = {
 
 export class Suite extends Base implements reporterTypes.Suite {
   suites: Suite[] = [];
-  specs: Spec[] = [];
+  tests: Test[] = [];
   _fixtureOverrides: any = {};
-  _entries: (Suite | Spec)[] = [];
+  _entries: (Suite | Test)[] = [];
   _hooks: {
     type: 'beforeEach' | 'afterEach' | 'beforeAll' | 'afterAll',
     fn: Function,
@@ -88,12 +66,14 @@ export class Suite extends Base implements reporterTypes.Suite {
   _timeout: number | undefined;
   _annotations: Annotations = [];
   _modifiers: Modifier[] = [];
+  _repeatEachIndex = 0;
+  _projectIndex = 0;
 
-  _addSpec(spec: Spec) {
+  _addTest(test: Test) {
-    spec.parent = this;
+    test.parent = this;
-    spec.suite = this;
+    test.suite = this;
-    this.specs.push(spec);
+    this.tests.push(test);
-    this._entries.push(spec);
+    this._entries.push(test);
   }
 
   _addSuite(suite: Suite) {
@@ -107,21 +87,6 @@ export class Suite extends Base implements reporterTypes.Suite {
       if (entry instanceof Suite) {
         if (entry.findTest(fn))
           return true;
-      } else {
-        for (const test of entry.tests) {
-          if (fn(test))
-            return true;
-        }
-      }
-    }
-    return false;
-  }
-
-  findSpec(fn: (spec: Spec) => boolean | void): boolean {
-    for (const entry of this._entries) {
-      if (entry instanceof Suite) {
-        if (entry.findSpec(fn))
-          return true;
       } else {
         if (fn(entry))
           return true;
@@ -130,48 +95,53 @@ export class Suite extends Base implements reporterTypes.Suite {
     return false;
   }
 
-  findSuite(fn: (suite: Suite) => boolean | void): boolean {
-    if (fn(this))
-      return true;
-    for (const suite of this.suites) {
-      if (suite.findSuite(fn))
-        return true;
-    }
-    return false;
-  }
-
   totalTestCount(): number {
     let total = 0;
     for (const suite of this.suites)
       total += suite.totalTestCount();
-    for (const spec of this.specs)
+    total += this.tests.length;
-      total += spec.tests.length;
     return total;
   }
 
-  _allSpecs(): Spec[] {
+  _allTests(): Test[] {
-    const result: Spec[] = [];
+    const result: Test[] = [];
-    this.findSpec(test => { result.push(test); });
+    this.findTest(test => { result.push(test); });
     return result;
   }
 
-  _getOnlyItems(): (Spec | Suite)[] {
+  _getOnlyItems(): (Test | Suite)[] {
-    const items: (Spec | Suite)[] = [];
+    const items: (Test | Suite)[] = [];
     if (this._only)
       items.push(this);
     for (const suite of this.suites)
       items.push(...suite._getOnlyItems());
-    items.push(...this.specs.filter(spec => spec._only));
+    items.push(...this.tests.filter(test => test._only));
     return items;
   }
 
   _buildFixtureOverrides(): any {
     return this.parent ? { ...this.parent._buildFixtureOverrides(), ...this._fixtureOverrides } : this._fixtureOverrides;
   }
+
+  _clone(): Suite {
+    const suite = new Suite(this.title);
+    suite._only = this._only;
+    suite.file = this.file;
+    suite.line = this.line;
+    suite.column = this.column;
+    suite._requireFile = this._requireFile;
+    suite._fixtureOverrides = this._fixtureOverrides;
+    suite._hooks = this._hooks.slice();
+    suite._timeout = this._timeout;
+    suite._annotations = this._annotations.slice();
+    suite._modifiers = this._modifiers.slice();
+    return suite;
+  }
 }
 
-export class Test implements reporterTypes.Test {
+export class Test extends Base implements reporterTypes.Test {
-  spec: Spec;
+  suite!: Suite;
+  fn: Function;
   results: reporterTypes.TestResult[] = [];
 
   skipped = false;
@@ -181,13 +151,17 @@ export class Test implements reporterTypes.Test {
   projectName = '';
   retries = 0;
 
+  _ordinalInFile: number;
+  _testType: TestTypeImpl;
   _id = '';
-  _repeatEachIndex = 0;
-  _projectIndex = 0;
   _workerHash = '';
+  _pool: FixturePool | undefined;
 
-  constructor(spec: Spec) {
+  constructor(title: string, fn: Function, ordinalInFile: number, testType: TestTypeImpl) {
-    this.spec = spec;
+    super(title);
+    this.fn = fn;
+    this._ordinalInFile = ordinalInFile;
+    this._testType = testType;
   }
 
   status(): 'skipped' | 'expected' | 'unexpected' | 'flaky' {
@@ -216,8 +190,18 @@ export class Test implements reporterTypes.Test {
     return status === 'expected' || status === 'flaky' || status === 'skipped';
   }
 
+  _clone(): Test {
+    const test = new Test(this.title, this.fn, this._ordinalInFile, this._testType);
+    test._only = this._only;
+    test.file = this.file;
+    test.line = this.line;
+    test.column = this.column;
+    test._requireFile = this._requireFile;
+    return test;
+  }
+
   fullTitle(): string {
-    return this.spec._testFullTitle(this.projectName);
+    return (this.projectName ? `[${this.projectName}] ` : '') + this._fullTitle;
   }
 
   _appendTestResult(): reporterTypes.TestResult {

src/test/testType.ts

@@ -16,7 +16,7 @@
 
 import { expect } from './expect';
 import { currentlyLoadingFileSuite, currentTestInfo, setCurrentlyLoadingFileSuite } from './globals';
-import { Spec, Suite } from './test';
+import { Test, Suite } from './test';
 import { wrapFunctionWithLocation } from './transform';
 import { Fixtures, FixturesWithLocation, Location, TestType } from './types';
 
@@ -34,9 +34,9 @@ export class TestTypeImpl {
   constructor(fixtures: (FixturesWithLocation | DeclaredFixtures)[]) {
     this.fixtures = fixtures;
 
-    const test: any = wrapFunctionWithLocation(this._spec.bind(this, 'default'));
+    const test: any = wrapFunctionWithLocation(this._createTest.bind(this, 'default'));
     test.expect = expect;
-    test.only = wrapFunctionWithLocation(this._spec.bind(this, 'only'));
+    test.only = wrapFunctionWithLocation(this._createTest.bind(this, 'only'));
     test.describe = wrapFunctionWithLocation(this._describe.bind(this, 'default'));
     test.describe.only = wrapFunctionWithLocation(this._describe.bind(this, 'only'));
     test.beforeEach = wrapFunctionWithLocation(this._hook.bind(this, 'beforeEach'));
@@ -54,7 +54,7 @@ export class TestTypeImpl {
     this.test = test;
   }
 
-  private _spec(type: 'default' | 'only', location: Location, title: string, fn: Function) {
+  private _createTest(type: 'default' | 'only', location: Location, title: string, fn: Function) {
     const suite = currentlyLoadingFileSuite();
     if (!suite)
       throw new Error(`test() can only be called in a test file`);
@@ -62,15 +62,16 @@ export class TestTypeImpl {
     const ordinalInFile = countByFile.get(suite._requireFile) || 0;
     countByFile.set(suite._requireFile, ordinalInFile + 1);
 
-    const spec = new Spec(title, fn, ordinalInFile, this);
+    const test = new Test(title, fn, ordinalInFile, this);
-    spec._requireFile = suite._requireFile;
+    test._requireFile = suite._requireFile;
-    spec.file = location.file;
+    test.file = location.file;
-    spec.line = location.line;
+    test.line = location.line;
-    spec.column = location.column;
+    test.column = location.column;
-    suite._addSpec(spec);
+    suite._addTest(test);
+    test._buildFullTitle(suite.fullTitle());
 
     if (type === 'only')
-      spec._only = true;
+      test._only = true;
   }
 
   private _describe(type: 'default' | 'only', location: Location, title: string, fn: Function) {
@@ -84,6 +85,7 @@ export class TestTypeImpl {
     child.line = location.line;
     child.column = location.column;
     suite._addSuite(child);
+    child._buildFullTitle(suite.fullTitle());
 
     if (type === 'only')
       child._only = true;

src/test/workerRunner.ts

@@ -23,10 +23,10 @@ import { monotonicTime, DeadlineRunner, raceAgainstDeadline, serializeError } fr
 import { TestBeginPayload, TestEndPayload, RunPayload, TestEntry, DonePayload, WorkerInitParams } from './ipc';
 import { setCurrentTestInfo } from './globals';
 import { Loader } from './loader';
-import { Modifier, Spec, Suite, Test } from './test';
+import { Modifier, Suite, Test } from './test';
 import { Annotations, TestError, TestInfo, WorkerInfo } from './types';
 import { ProjectImpl } from './project';
-import { FixtureRunner } from './fixtures';
+import { FixturePool, FixtureRunner } from './fixtures';
 
 const removeFolderAsync = util.promisify(rimraf);
 
@@ -116,20 +116,22 @@ export class WorkerRunner extends EventEmitter {
     this._entries = new Map(runPayload.entries.map(e => [ e.testId, e ]));
 
     await this._loadIfNeeded();
+
     const fileSuite = await this._loader.loadTestFile(runPayload.file);
-    let anySpec: Spec | undefined;
+    let anyPool: FixturePool | undefined;
-    fileSuite.findSpec(spec => {
+    const suite = this._project.cloneSuite(fileSuite, this._params.repeatEachIndex, test => {
-      const test = this._project.generateTests(spec, this._params.repeatEachIndex)[0];
+      if (!this._entries.has(test._id))
-      if (this._entries.has(test._id))
+        return false;
-        anySpec = spec;
+      anyPool = test._pool;
+      return true;
     });
-    if (!anySpec) {
+
+    if (!suite || !anyPool) {
       this._reportDone();
       return;
     }
-
+    this._fixtureRunner.setPool(anyPool);
-    this._fixtureRunner.setPool(this._project.buildPool(anySpec));
+    await this._runSuite(suite, []);
-    await this._runSuite(fileSuite, []);
     if (this._isStopped)
       return;
 
@@ -156,7 +158,7 @@ export class WorkerRunner extends EventEmitter {
         annotations.push({ type: beforeAllModifier.type, description: beforeAllModifier.description });
     }
 
-    const skipHooks = !this._hasTestsToRun(suite) || annotations.some(a => a.type === 'fixme' || a.type === 'skip');
+    const skipHooks = annotations.some(a => a.type === 'fixme' || a.type === 'skip');
     for (const hook of suite._hooks) {
       if (hook.type !== 'beforeAll' || skipHooks)
         continue;
@@ -173,7 +175,7 @@ export class WorkerRunner extends EventEmitter {
       if (entry instanceof Suite)
         await this._runSuite(entry, annotations);
       else
-        await this._runSpec(entry, annotations);
+        await this._runTest(entry, annotations);
     }
     for (const hook of suite._hooks) {
       if (hook.type !== 'afterAll' || skipHooks)
@@ -189,10 +191,9 @@ export class WorkerRunner extends EventEmitter {
     }
   }
 
-  private async _runSpec(spec: Spec, annotations: Annotations) {
+  private async _runTest(test: Test, annotations: Annotations) {
     if (this._isStopped)
       return;
-    const test = spec.tests[0];
     const entry = this._entries.get(test._id);
     if (!entry)
       return;
@@ -202,9 +203,9 @@ export class WorkerRunner extends EventEmitter {
     const testId = test._id;
 
     const baseOutputDir = (() => {
-      const relativeTestFilePath = path.relative(this._project.config.testDir, spec._requireFile.replace(/\.(spec|test)\.(js|ts|mjs)$/, ''));
+      const relativeTestFilePath = path.relative(this._project.config.testDir, test._requireFile.replace(/\.(spec|test)\.(js|ts|mjs)$/, ''));
       const sanitizedRelativePath = relativeTestFilePath.replace(process.platform === 'win32' ? new RegExp('\\\\', 'g') : new RegExp('/', 'g'), '-');
-      let testOutputDir = sanitizedRelativePath + '-' + sanitizeForFilePath(spec.title);
+      let testOutputDir = sanitizedRelativePath + '-' + sanitizeForFilePath(test.title);
       if (this._uniqueProjectNamePathSegment)
         testOutputDir += '-' + this._uniqueProjectNamePathSegment;
       if (entry.retry)
@@ -216,11 +217,11 @@ export class WorkerRunner extends EventEmitter {
 
     const testInfo: TestInfo = {
       ...this._workerInfo,
-      title: spec.title,
+      title: test.title,
-      file: spec.file,
+      file: test.file,
-      line: spec.line,
+      line: test.line,
-      column: spec.column,
+      column: test.column,
-      fn: spec.fn,
+      fn: test.fn,
       repeatEachIndex: this._params.repeatEachIndex,
       retry: entry.retry,
       expectedStatus: 'passed',
@@ -248,7 +249,7 @@ export class WorkerRunner extends EventEmitter {
           snapshotName = sanitizeForFilePath(snapshotName.substring(0, snapshotName.length - ext.length)) + suffix + ext;
         else
           snapshotName = sanitizeForFilePath(snapshotName) + suffix;
-        return path.join(spec._requireFile + '-snapshots', snapshotName);
+        return path.join(test._requireFile + '-snapshots', snapshotName);
       },
       skip: (...args: [arg?: any, description?: string]) => modifier(testInfo, 'skip', args),
       fixme: (...args: [arg?: any, description?: string]) => modifier(testInfo, 'fixme', args),
@@ -262,7 +263,7 @@ export class WorkerRunner extends EventEmitter {
     };
 
     // Inherit test.setTimeout() from parent suites.
-    for (let suite = spec.parent; suite; suite = suite.parent) {
+    for (let suite = test.parent; suite; suite = suite.parent) {
       if (suite._timeout !== undefined) {
         testInfo.setTimeout(suite._timeout);
         break;
@@ -301,7 +302,7 @@ export class WorkerRunner extends EventEmitter {
     }
 
     // Update the fixture pool - it may differ between tests, but only in test-scoped fixtures.
-    this._fixtureRunner.setPool(this._project.buildPool(spec));
+    this._fixtureRunner.setPool(test._pool!);
 
     deadlineRunner = new DeadlineRunner(this._runTestWithBeforeHooks(test, testInfo), deadline());
     const result = await deadlineRunner.result;
@@ -352,7 +353,7 @@ export class WorkerRunner extends EventEmitter {
   private async _runTestWithBeforeHooks(test: Test, testInfo: TestInfo) {
     try {
       const beforeEachModifiers: Modifier[] = [];
-      for (let s = test.spec.parent; s; s = s.parent) {
+      for (let s = test.parent; s; s = s.parent) {
         const modifiers = s._modifiers.filter(modifier => !this._fixtureRunner.dependsOnWorkerFixturesOnly(modifier.fn, modifier.location));
         beforeEachModifiers.push(...modifiers.reverse());
       }
@@ -363,7 +364,7 @@ export class WorkerRunner extends EventEmitter {
         const result = await this._fixtureRunner.resolveParametersAndRunHookOrTest(modifier.fn, 'test', testInfo);
         testInfo[modifier.type](!!result, modifier.description!);
       }
-      await this._runHooks(test.spec.parent!, 'beforeEach', testInfo);
+      await this._runHooks(test.parent!, 'beforeEach', testInfo);
     } catch (error) {
       if (error instanceof SkipError) {
         if (testInfo.status === 'passed')
@@ -380,7 +381,7 @@ export class WorkerRunner extends EventEmitter {
       return;
 
     try {
-      await this._fixtureRunner.resolveParametersAndRunHookOrTest(test.spec.fn, 'test', testInfo);
+      await this._fixtureRunner.resolveParametersAndRunHookOrTest(test.fn, 'test', testInfo);
     } catch (error) {
       if (error instanceof SkipError) {
         if (testInfo.status === 'passed')
@@ -399,7 +400,7 @@ export class WorkerRunner extends EventEmitter {
 
   private async _runAfterHooks(test: Test, testInfo: TestInfo) {
     try {
-      await this._runHooks(test.spec.parent!, 'afterEach', testInfo);
+      await this._runHooks(test.parent!, 'afterEach', testInfo);
     } catch (error) {
       if (!(error instanceof SkipError)) {
         if (testInfo.status === 'passed')
@@ -458,13 +459,6 @@ export class WorkerRunner extends EventEmitter {
     this._reportDone();
     this.stop();
   }
-
-  private _hasTestsToRun(suite: Suite): boolean {
-    return suite.findSpec(spec => {
-      const entry = this._entries.get(spec.tests[0]._id);
-      return !!entry;
-    });
-  }
 }
 
 function buildTestBeginPayload(testId: string, testInfo: TestInfo): TestBeginPayload {

tests/playwright-test/config.spec.ts

@@ -321,65 +321,3 @@ test('should work with undefined values and base', async ({ runInlineTest }) =>
   expect(result.exitCode).toBe(0);
   expect(result.passed).toBe(1);
 });
-
-test('should work with custom reporter', async ({ runInlineTest }) => {
-  const result = await runInlineTest({
-    'reporter.ts': `
-      class Reporter {
-        constructor(options) {
-          this.options = options;
-        }
-        onBegin() {
-          console.log('\\n%%reporter-begin%%' + this.options.begin);
-        }
-        onTestBegin() {
-          console.log('\\n%%reporter-testbegin%%');
-        }
-        onStdOut() {
-          console.log('\\n%%reporter-stdout%%');
-        }
-        onStdErr() {
-          console.log('\\n%%reporter-stderr%%');
-        }
-        onTestEnd() {
-          console.log('\\n%%reporter-testend%%');
-        }
-        onTimeout() {
-          console.log('\\n%%reporter-timeout%%');
-        }
-        onError() {
-          console.log('\\n%%reporter-error%%');
-        }
-        async onEnd() {
-          await new Promise(f => setTimeout(f, 500));
-          console.log('\\n%%reporter-end%%' + this.options.end);
-        }
-      }
-      export default Reporter;
-    `,
-    'playwright.config.ts': `
-      module.exports = {
-        reporter: [
-          [ './reporter.ts', { begin: 'begin', end: 'end' } ]
-        ]
-      };
-    `,
-    'a.test.ts': `
-      const { test } = pwt;
-      test('pass', async ({}) => {
-        console.log('log');
-        console.error('error');
-      });
-    `
-  }, { reporter: '' });
-
-  expect(result.exitCode).toBe(0);
-  expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([
-    '%%reporter-begin%%begin',
-    '%%reporter-testbegin%%',
-    '%%reporter-stdout%%',
-    '%%reporter-stderr%%',
-    '%%reporter-testend%%',
-    '%%reporter-end%%end',
-  ]);
-});

tests/playwright-test/hooks.spec.ts

@@ -190,3 +190,27 @@ test('beforeAll from a helper file should throw', async ({ runInlineTest }) => {
   expect(result.exitCode).toBe(1);
   expect(result.output).toContain('beforeAll hook can only be called in a test file');
 });
+
+test('beforeAll hooks are skipped when no tests in the suite are run', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const { test } = pwt;
+      test.describe('suite1', () => {
+        test.beforeAll(() => {
+          console.log('\\n%%beforeAll1');
+        });
+        test('skipped', () => {});
+      });
+      test.describe('suite2', () => {
+        test.beforeAll(() => {
+          console.log('\\n%%beforeAll2');
+        });
+        test.only('passed', () => {});
+      });
+    `,
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+  expect(result.output).toContain('%%beforeAll2');
+  expect(result.output).not.toContain('%%beforeAll1');
+});

tests/playwright-test/junit-reporter.spec.ts

@@ -205,17 +205,23 @@ test('should render projects', async ({ runInlineTest }) => {
   const xml = parseXML(result.output);
   expect(xml['testsuites']['$']['tests']).toBe('2');
   expect(xml['testsuites']['$']['failures']).toBe('0');
-  expect(xml['testsuites']['testsuite'].length).toBe(1);
+  expect(xml['testsuites']['testsuite'].length).toBe(2);
+
   expect(xml['testsuites']['testsuite'][0]['$']['name']).toBe('a.test.js');
-  expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('2');
+  expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('1');
   expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0');
   expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('0');
-  expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['name']).toBe('one');
+  expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['name']).toBe('[project1] one');
   expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['classname']).toContain('[project1] one');
   expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['classname']).toContain('a.test.js:6:7');
-  expect(xml['testsuites']['testsuite'][0]['testcase'][1]['$']['name']).toBe('one');
+
-  expect(xml['testsuites']['testsuite'][0]['testcase'][1]['$']['classname']).toContain('[project2] one');
+  expect(xml['testsuites']['testsuite'][1]['$']['name']).toBe('a.test.js');
-  expect(xml['testsuites']['testsuite'][0]['testcase'][1]['$']['classname']).toContain('a.test.js:6:7');
+  expect(xml['testsuites']['testsuite'][1]['$']['tests']).toBe('1');
+  expect(xml['testsuites']['testsuite'][1]['$']['failures']).toBe('0');
+  expect(xml['testsuites']['testsuite'][1]['$']['skipped']).toBe('0');
+  expect(xml['testsuites']['testsuite'][1]['testcase'][0]['$']['name']).toBe('[project2] one');
+  expect(xml['testsuites']['testsuite'][1]['testcase'][0]['$']['classname']).toContain('[project2] one');
+  expect(xml['testsuites']['testsuite'][1]['testcase'][0]['$']['classname']).toContain('a.test.js:6:7');
   expect(result.exitCode).toBe(0);
 });
 

tests/playwright-test/match-grep.spec.ts

@@ -64,12 +64,14 @@ const files = {
 test('should grep test name', async ({ runInlineTest }) => {
   const result = await runInlineTest(files, { 'grep': 'test [A-B]' });
   expect(result.passed).toBe(6);
+  expect(result.skipped).toBe(0);
   expect(result.exitCode).toBe(0);
 });
 
 test('should grep test name with regular expression', async ({ runInlineTest }) => {
   const result = await runInlineTest(files, { 'grep': '/B$/' });
   expect(result.passed).toBe(3);
+  expect(result.skipped).toBe(0);
   expect(result.exitCode).toBe(0);
 });
 
@@ -100,5 +102,6 @@ test('should grep by project name', async ({ runInlineTest }) => {
 test('should grep invert test name', async ({ runInlineTest }) => {
   const result = await runInlineTest(files, { 'grep-invert': 'BB' });
   expect(result.passed).toBe(6);
+  expect(result.skipped).toBe(0);
   expect(result.exitCode).toBe(0);
 });

tests/playwright-test/reporter.spec.ts

@@ -0,0 +1,91 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './playwright-test-fixtures';
+
+test('should work with custom reporter', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'reporter.ts': `
+      class Reporter {
+        constructor(options) {
+          this.options = options;
+        }
+        onBegin(config, suite) {
+          console.log('\\n%%reporter-begin-' + this.options.begin + '-' + suite.suites.length + '%%');
+        }
+        onTestBegin(test) {
+          console.log('\\n%%reporter-testbegin-' + test.title + '-' + test.projectName + '%%');
+        }
+        onStdOut() {
+          console.log('\\n%%reporter-stdout%%');
+        }
+        onStdErr() {
+          console.log('\\n%%reporter-stderr%%');
+        }
+        onTestEnd(test) {
+          console.log('\\n%%reporter-testend-' + test.title + '-' + test.projectName + '%%');
+        }
+        onTimeout() {
+          console.log('\\n%%reporter-timeout%%');
+        }
+        onError() {
+          console.log('\\n%%reporter-error%%');
+        }
+        async onEnd() {
+          await new Promise(f => setTimeout(f, 500));
+          console.log('\\n%%reporter-end-' + this.options.end + '%%');
+        }
+      }
+      export default Reporter;
+    `,
+    'playwright.config.ts': `
+      module.exports = {
+        reporter: [
+          [ './reporter.ts', { begin: 'begin', end: 'end' } ]
+        ],
+        projects: [
+          { name: 'foo', repeatEach: 2 },
+          { name: 'bar' },
+        ],
+      };
+    `,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('pass', async ({}) => {
+        console.log('log');
+        console.error('error');
+      });
+    `
+  }, { reporter: '', workers: 1 });
+
+  expect(result.exitCode).toBe(0);
+  expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([
+    '%%reporter-begin-begin-3%%',
+    '%%reporter-testbegin-pass-foo%%',
+    '%%reporter-stdout%%',
+    '%%reporter-stderr%%',
+    '%%reporter-testend-pass-foo%%',
+    '%%reporter-testbegin-pass-foo%%',
+    '%%reporter-stdout%%',
+    '%%reporter-stderr%%',
+    '%%reporter-testend-pass-foo%%',
+    '%%reporter-testbegin-pass-bar%%',
+    '%%reporter-stdout%%',
+    '%%reporter-stderr%%',
+    '%%reporter-testend-pass-bar%%',
+    '%%reporter-end-end%%',
+  ]);
+});