chore: experimental oop loader (#20269)

packages/playwright-test/src/loaderHost.ts

@@ -0,0 +1,36 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { TestError } from '../reporter';
+import type { SerializedConfig } from './ipc';
+import { ProcessHost } from './processHost';
+import { Suite } from './test';
+
+export class LoaderHost extends ProcessHost<SerializedConfig> {
+  constructor() {
+    super(require.resolve('./loaderRunner.js'), 'loader');
+  }
+
+  async start(config: SerializedConfig) {
+    await this.startRunner(config, true, {});
+  }
+
+  async loadTestFiles(files: string[], loadErrors: TestError[]): Promise<Suite> {
+    const result = await this.sendMessage({ method: 'loadTestFiles', params: { files } }) as any;
+    loadErrors.push(...result.loadErrors);
+    return Suite._deepParse(result.rootSuite);
+  }
+}

packages/playwright-test/src/loaderRunner.ts

@@ -0,0 +1,48 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { SerializedConfig } from './ipc';
+import type { TestError } from '../reporter';
+import { ConfigLoader } from './configLoader';
+import { ProcessRunner } from './process';
+import { loadTestFilesInProcess } from './testLoader';
+import { setFatalErrorSink } from './globals';
+
+export class LoaderRunner extends ProcessRunner {
+  private _config: SerializedConfig;
+  private _configLoaderPromise: Promise<ConfigLoader> | undefined;
+
+  constructor(config: SerializedConfig) {
+    super();
+    this._config = config;
+  }
+
+  private _configLoader(): Promise<ConfigLoader> {
+    if (!this._configLoaderPromise)
+      this._configLoaderPromise = ConfigLoader.deserialize(this._config);
+    return this._configLoaderPromise;
+  }
+
+  async loadTestFiles(params: { files: string[] }) {
+    const loadErrors: TestError[] = [];
+    setFatalErrorSink(loadErrors);
+    const configLoader = await this._configLoader();
+    const rootSuite = await loadTestFilesInProcess(configLoader.fullConfig(), params.files, loadErrors);
+    return { rootSuite: rootSuite._deepSerialize(), loadErrors };
+  }
+}
+
+export const create = (config: SerializedConfig) => new LoaderRunner(config);

packages/playwright-test/src/process.ts

@@ -15,8 +15,7 @@
  */
 
 import type { WriteStream } from 'tty';
-import * as util from 'util';
+import type { ProcessInitParams, TeardownErrorsPayload, TtyParams } from './ipc';
-import type { ProcessInitParams, TeardownErrorsPayload, TestOutputPayload, TtyParams } from './ipc';
 import { startProfiling, stopProfiling } from './profiler';
 import type { TestInfoError } from './types';
 import { serializeError } from './util';
@@ -29,7 +28,7 @@ export type ProtocolRequest = {
 
 export type ProtocolResponse = {
   id?: number;
-  error?: string;
+  error?: TestInfoError;
   method?: string;
   params?: any;
   result?: any;
@@ -49,24 +48,6 @@ let closed = false;
 
 sendMessageToParent({ method: 'ready' });
 
-process.stdout.write = (chunk: string | Buffer) => {
-  const outPayload: TestOutputPayload = {
-    ...chunkToParams(chunk)
-  };
-  sendMessageToParent({ method: 'stdOut', params: outPayload });
-  return true;
-};
-
-if (!process.env.PW_RUNNER_DEBUG) {
-  process.stderr.write = (chunk: string | Buffer) => {
-    const outPayload: TestOutputPayload = {
-      ...chunkToParams(chunk)
-    };
-    sendMessageToParent({ method: 'stdErr', params: outPayload });
-    return true;
-  };
-}
-
 process.on('disconnect', gracefullyCloseAndExit);
 process.on('SIGINT', () => {});
 process.on('SIGTERM', () => {});
@@ -94,7 +75,7 @@ process.on('message', async message => {
       const response: ProtocolResponse = { id, result };
       sendMessageToParent({ method: '__dispatch__', params: response });
     } catch (e) {
-      const response: ProtocolResponse = { id, error: e.toString() };
+      const response: ProtocolResponse = { id, error: serializeError(e) };
       sendMessageToParent({ method: '__dispatch__', params: response });
     }
   }
@@ -132,14 +113,6 @@ function sendMessageToParent(message: { method: string, params?: any }) {
   }
 }
 
-function chunkToParams(chunk: Buffer | string):  { text?: string, buffer?: string } {
-  if (chunk instanceof Buffer)
-    return { buffer: chunk.toString('base64') };
-  if (typeof chunk !== 'string')
-    return { text: util.inspect(chunk) };
-  return { text: chunk };
-}
-
 function setTtyParams(stream: WriteStream, params: TtyParams) {
   stream.isTTY = true;
   if (params.rows)

packages/playwright-test/src/processHost.ts

@@ -16,6 +16,7 @@
 
 import child_process from 'child_process';
 import { EventEmitter } from 'events';
+import { debug } from 'playwright-core/lib/utilsBundle';
 import type { ProcessInitParams } from './ipc';
 import type { ProtocolResponse } from './process';
 
@@ -41,17 +42,11 @@ export class ProcessHost<InitParams> extends EventEmitter {
     this._processName = processName;
   }
 
-  protected async startRunner(runnerParams: InitParams) {
+  protected async startRunner(runnerParams: InitParams, inheritStdio: boolean, env: NodeJS.ProcessEnv) {
     this.process = child_process.fork(require.resolve('./process'), {
       detached: false,
-      env: {
+      env: { ...process.env, ...env },
-        FORCE_COLOR: '1',
+      stdio: inheritStdio ? ['ignore', 'inherit', 'inherit', 'ipc'] : ['ignore', 'ignore', process.env.PW_RUNNER_DEBUG ? 'inherit' : 'ignore', 'ipc'],
-        DEBUG_COLORS: '1',
-        PW_PROCESS_RUNNER_SCRIPT: this._runnerScript,
-        ...process.env
-      },
-      // Can't pipe since piping slows down termination for some reason.
-      stdio: ['ignore', 'ignore', process.env.PW_RUNNER_DEBUG ? 'inherit' : 'ignore', 'ipc']
     });
     this.process.on('exit', (code, signal) => {
       this.didExit = true;
@@ -59,15 +54,20 @@ export class ProcessHost<InitParams> extends EventEmitter {
     });
     this.process.on('error', e => {});  // do not yell at a send to dead process.
     this.process.on('message', (message: any) => {
+      if (debug.enabled('pw:test:protocol'))
+        debug('pw:test:protocol')('◀ RECV ' + JSON.stringify(message));
       if (message.method === '__dispatch__') {
         const { id, error, method, params, result } = message.params as ProtocolResponse;
         if (id && this._callbacks.has(id)) {
           const { resolve, reject } = this._callbacks.get(id)!;
           this._callbacks.delete(id);
-          if (error)
+          if (error) {
-            reject(new Error(error));
+            const errorObject = new Error(error.message);
-          else
+            errorObject.stack = error.stack;
+            reject(errorObject);
+          } else {
             resolve(result);
+          }
         } else {
           this.emit(method!, params);
         }
@@ -140,7 +140,8 @@ export class ProcessHost<InitParams> extends EventEmitter {
   }
 
   private send(message: { method: string, params?: any }) {
-    // This is a great place for debug logging.
+    if (debug.enabled('pw:test:protocol'))
+      debug('pw:test:protocol')('SEND ► ' + JSON.stringify(message));
     this.process.send(message);
   }
 }

packages/playwright-test/src/runner.ts

@@ -45,9 +45,9 @@ import type { Config, FullConfigInternal, FullProjectInternal, ReporterInternal
 import { createFileMatcher, createFileMatcherFromFilters, createTitleMatcher, serializeError } from './util';
 import type { Matcher, TestFileFilter } from './util';
 import { setFatalErrorSink } from './globals';
-import { TestLoader } from './testLoader';
+import { buildFileSuiteForProject, filterOnly, filterSuite, filterSuiteWithOnlySemantics, filterTestsRemoveEmptySuites } from './suiteUtils';
-import { buildFileSuiteForProject, filterTests } from './suiteUtils';
+import { LoaderHost } from './loaderHost';
-import { PoolBuilder } from './poolBuilder';
+import { loadTestFilesInProcess } from './testLoader';
 
 const removeFolderAsync = promisify(rimraf);
 const readDirAsync = promisify(fs.readdir);
@@ -271,27 +271,23 @@ export class Runner {
     const config = this._configLoader.fullConfig();
     const projects = this._collectProjects(options.projectFilter);
     const filesByProject = await this._collectFiles(projects, options.testFileFilters);
-    const result = await this._createFilteredRootSuite(options, filesByProject);
+    const rootSuite = await this._createFilteredRootSuite(options, filesByProject);
-    this._fatalErrors.push(...result.fatalErrors);
-    const { rootSuite } = result;
 
     const testGroups = createTestGroups(rootSuite.suites, config.workers);
     return { rootSuite, testGroups };
   }
 
-  private async _createFilteredRootSuite(options: RunOptions, filesByProject: Map<FullProjectInternal, string[]>): Promise<{rootSuite: Suite, fatalErrors: TestError[]}> {
+  private async _createFilteredRootSuite(options: RunOptions, filesByProject: Map<FullProjectInternal, string[]>): Promise<Suite> {
     const config = this._configLoader.fullConfig();
-    const fatalErrors: TestError[] = [];
     const allTestFiles = new Set<string>();
     for (const files of filesByProject.values())
       files.forEach(file => allTestFiles.add(file));
 
     // Load all tests.
-    const { rootSuite: preprocessRoot, loadErrors } = await this._loadTests(allTestFiles);
+    const preprocessRoot = await this._loadTests(allTestFiles);
-    fatalErrors.push(...loadErrors);
 
     // Complain about duplicate titles.
-    fatalErrors.push(...createDuplicateTitlesErrors(config, preprocessRoot));
+    this._fatalErrors.push(...createDuplicateTitlesErrors(config, preprocessRoot));
 
     // Filter tests to respect line/column filter.
     filterByFocusedLine(preprocessRoot, options.testFileFilters);
@@ -300,7 +296,7 @@ export class Runner {
     if (config.forbidOnly) {
       const onlyTestsAndSuites = preprocessRoot._getOnlyItems();
       if (onlyTestsAndSuites.length > 0)
-        fatalErrors.push(...createForbidOnlyErrors(config, onlyTestsAndSuites));
+        this._fatalErrors.push(...createForbidOnlyErrors(config, onlyTestsAndSuites));
     }
 
     // Filter only.
@@ -335,30 +331,26 @@ export class Runner {
           continue;
         for (let repeatEachIndex = 0; repeatEachIndex < project.repeatEach; repeatEachIndex++) {
           const builtSuite = buildFileSuiteForProject(project, fileSuite, repeatEachIndex);
-          if (!filterTests(builtSuite, titleMatcher))
+          if (!filterTestsRemoveEmptySuites(builtSuite, titleMatcher))
             continue;
           projectSuite._addSuite(builtSuite);
         }
       }
     }
-    return { rootSuite, fatalErrors };
+    return rootSuite;
   }
 
-  private async _loadTests(testFiles: Set<string>): Promise<{ rootSuite: Suite, loadErrors: TestError[] }> {
+  private async _loadTests(testFiles: Set<string>): Promise<Suite> {
-    const config = this._configLoader.fullConfig();
+    if (process.env.PWTEST_OOP_LOADER) {
-    const testLoader = new TestLoader(config);
+      const loaderHost = new LoaderHost();
-    const loadErrors: TestError[] = [];
+      await loaderHost.start(this._configLoader.serializedConfig());
-    const rootSuite = new Suite('', 'root');
+      try {
-    for (const file of testFiles) {
+        return await loaderHost.loadTestFiles([...testFiles], this._fatalErrors);
-      const fileSuite = await testLoader.loadTestFile(file, 'loader');
+      } finally {
-      if (fileSuite._loadError)
+        await loaderHost.stop();
-        loadErrors.push(fileSuite._loadError);
+      }
-      // We have to clone only if there maybe subsequent calls of this method.
-      rootSuite._addSuite(fileSuite);
     }
-    // Generate hashes.
+    return loadTestFilesInProcess(this._configLoader.fullConfig(), [...testFiles], this._fatalErrors);
-    PoolBuilder.buildForLoader(rootSuite);
-    return { rootSuite, loadErrors };
   }
 
   private _filterForCurrentShard(rootSuite: Suite, testGroups: TestGroup[]) {
@@ -404,8 +396,6 @@ export class Runner {
       // Filtering with "only semantics" does not work when we have zero tests - it leaves all the tests.
       // We need an empty suite in this case.
       rootSuite._entries = [];
-      rootSuite.suites = [];
-      rootSuite.tests = [];
     } else {
       filterSuiteWithOnlySemantics(rootSuite, () => false, test => shardTests.has(test));
     }
@@ -493,23 +483,6 @@ export class Runner {
     return 'success';
   }
 
-  private _skipTestsFromMatchingGroups(testGroups: TestGroup[], groupFilter: (g: TestGroup) => boolean): TestGroup[] {
-    const result = [];
-    for (const group of testGroups) {
-      if (groupFilter(group)) {
-        for (const test of group.tests) {
-          const result = test._appendTestResult();
-          this._reporter.onTestBegin?.(test, result);
-          result.status = 'skipped';
-          this._reporter.onTestEnd?.(test, result);
-        }
-      } else {
-        result.push(group);
-      }
-    }
-    return result;
-  }
-
   private async _removeOutputDirs(options: RunOptions): Promise<boolean> {
     const config = this._configLoader.fullConfig();
     const outputDirs = new Set<string>();
@@ -616,14 +589,6 @@ export class Runner {
   }
 }
 
-function filterOnly(suite: Suite) {
-  if (!suite._getOnlyItems().length)
-    return;
-  const suiteFilter = (suite: Suite) => suite._only;
-  const testFilter = (test: TestCase) => test._only;
-  return filterSuiteWithOnlySemantics(suite, suiteFilter, testFilter);
-}
-
 function createFileMatcherFromFilter(filter: TestFileFilter) {
   const fileMatcher = createFileMatcher(filter.re || filter.exact || '');
   return (testFileName: string, testLine: number, testColumn: number) =>
@@ -640,29 +605,6 @@ function filterByFocusedLine(suite: Suite, focusedTestFileLines: TestFileFilter[
   return filterSuite(suite, suiteFilter, testFilter);
 }
 
-function filterSuiteWithOnlySemantics(suite: Suite, suiteFilter: (suites: Suite) => boolean, testFilter: (test: TestCase) => boolean) {
-  const onlySuites = suite.suites.filter(child => filterSuiteWithOnlySemantics(child, suiteFilter, testFilter) || suiteFilter(child));
-  const onlyTests = suite.tests.filter(testFilter);
-  const onlyEntries = new Set([...onlySuites, ...onlyTests]);
-  if (onlyEntries.size) {
-    suite.suites = onlySuites;
-    suite.tests = onlyTests;
-    suite._entries = suite._entries.filter(e => onlyEntries.has(e)); // Preserve the order.
-    return true;
-  }
-  return false;
-}
-
-function filterSuite(suite: Suite, suiteFilter: (suites: Suite) => boolean, testFilter: (test: TestCase) => boolean) {
-  for (const child of suite.suites) {
-    if (!suiteFilter(child))
-      filterSuite(child, suiteFilter, testFilter);
-  }
-  suite.tests = suite.tests.filter(testFilter);
-  const entries = new Set([...suite.suites, ...suite.tests]);
-  suite._entries = suite._entries.filter(e => entries.has(e)); // Preserve the order.
-}
-
 async function collectFiles(testDir: string, respectGitIgnore: boolean): Promise<string[]> {
   if (!fs.existsSync(testDir))
     return [];

packages/playwright-test/src/suiteUtils.ts

@@ -19,10 +19,20 @@ import { calculateSha1 } from 'playwright-core/lib/utils';
 import type { Suite, TestCase } from './test';
 import type { FullProjectInternal } from './types';
 
-export function filterTests(suite: Suite, filter: (test: TestCase) => boolean): boolean {
+export function filterSuite(suite: Suite, suiteFilter: (suites: Suite) => boolean, testFilter: (test: TestCase) => boolean) {
-  suite.suites = suite.suites.filter(child => filterTests(child, filter));
+  for (const child of suite.suites) {
-  suite.tests = suite.tests.filter(filter);
+    if (!suiteFilter(child))
-  const entries = new Set([...suite.suites, ...suite.tests]);
+      filterSuite(child, suiteFilter, testFilter);
+  }
+  const filteredTests = suite.tests.filter(testFilter);
+  const entries = new Set([...suite.suites, ...filteredTests]);
+  suite._entries = suite._entries.filter(e => entries.has(e)); // Preserve the order.
+}
+
+export function filterTestsRemoveEmptySuites(suite: Suite, filter: (test: TestCase) => boolean): boolean {
+  const filteredSuites = suite.suites.filter(child => filterTestsRemoveEmptySuites(child, filter));
+  const filteredTests = suite.tests.filter(filter);
+  const entries = new Set([...filteredSuites, ...filteredTests]);
   suite._entries = suite._entries.filter(e => entries.has(e)); // Preserve the order.
   return !!suite._entries.length;
 }
@@ -59,3 +69,22 @@ export function buildFileSuiteForProject(project: FullProjectInternal, suite: Su
 
   return result;
 }
+
+export function filterOnly(suite: Suite) {
+  if (!suite._getOnlyItems().length)
+    return;
+  const suiteFilter = (suite: Suite) => suite._only;
+  const testFilter = (test: TestCase) => test._only;
+  return filterSuiteWithOnlySemantics(suite, suiteFilter, testFilter);
+}
+
+export function filterSuiteWithOnlySemantics(suite: Suite, suiteFilter: (suites: Suite) => boolean, testFilter: (test: TestCase) => boolean) {
+  const onlySuites = suite.suites.filter(child => filterSuiteWithOnlySemantics(child, suiteFilter, testFilter) || suiteFilter(child));
+  const onlyTests = suite.tests.filter(testFilter);
+  const onlyEntries = new Set([...onlySuites, ...onlyTests]);
+  if (onlyEntries.size) {
+    suite._entries = suite._entries.filter(e => onlyEntries.has(e)); // Preserve the order.
+    return true;
+  }
+  return false;
+}

packages/playwright-test/src/test.ts

@@ -17,6 +17,7 @@
 import type { FixturePool } from './fixtures';
 import type * as reporterTypes from '../types/testReporter';
 import type { TestTypeImpl } from './testType';
+import { rootTestType } from './testType';
 import type { Annotation, FixturesWithLocation, FullProject, FullProjectInternal, Location } from './types';
 
 class Base {
@@ -37,8 +38,6 @@ export type Modifier = {
 };
 
 export class Suite extends Base implements reporterTypes.Suite {
-  suites: Suite[] = [];
-  tests: TestCase[] = [];
   location?: Location;
   parent?: Suite;
   _use: FixturesWithLocation[] = [];
@@ -51,7 +50,6 @@ export class Suite extends Base implements reporterTypes.Suite {
   _modifiers: Modifier[] = [];
   _parallelMode: 'default' | 'serial' | 'parallel' = 'default';
   _projectConfig: FullProjectInternal | undefined;
-  _loadError?: reporterTypes.TestError;
   _fileId: string | undefined;
   readonly _type: 'root' | 'project' | 'file' | 'describe';
 
@@ -60,15 +58,21 @@ export class Suite extends Base implements reporterTypes.Suite {
     this._type = type;
   }
 
+  get suites(): Suite[] {
+    return this._entries.filter(entry => entry instanceof Suite) as Suite[];
+  }
+
+  get tests(): TestCase[] {
+    return this._entries.filter(entry => entry instanceof TestCase) as TestCase[];
+  }
+
   _addTest(test: TestCase) {
     test.parent = this;
-    this.tests.push(test);
     this._entries.push(test);
   }
 
   _addSuite(suite: Suite) {
     suite.parent = this;
-    this.suites.push(suite);
     this._entries.push(suite);
   }
 
@@ -115,6 +119,29 @@ export class Suite extends Base implements reporterTypes.Suite {
     return suite;
   }
 
+  _deepSerialize(): any {
+    const suite = this._serialize();
+    suite.entries = [];
+    for (const entry of this._entries) {
+      if (entry instanceof Suite)
+        suite.entries.push(entry._deepSerialize());
+      else
+        suite.entries.push(entry._serialize());
+    }
+    return suite;
+  }
+
+  static _deepParse(data: any): Suite {
+    const suite = Suite._parse(data);
+    for (const entry of data.entries) {
+      if (entry.kind === 'suite')
+        suite._addSuite(Suite._deepParse(entry));
+      else
+        suite._addTest(TestCase._parse(entry));
+    }
+    return suite;
+  }
+
   forEachTest(visitor: (test: TestCase, suite: Suite) => void) {
     for (const entry of this._entries) {
       if (entry instanceof Suite)
@@ -124,20 +151,45 @@ export class Suite extends Base implements reporterTypes.Suite {
     }
   }
 
+  _serialize(): any {
+    return {
+      kind: 'suite',
+      title: this.title,
+      type: this._type,
+      location: this.location,
+      only: this._only,
+      requireFile: this._requireFile,
+      timeout: this._timeout,
+      retries: this._retries,
+      annotations: this._annotations.slice(),
+      modifiers: this._modifiers.slice(),
+      parallelMode: this._parallelMode,
+      skipped: this._skipped,
+      hooks: this._hooks.map(h => ({ type: h.type, location: h.location })),
+    };
+  }
+
+  static _parse(data: any): Suite {
+    const suite = new Suite(data.title, data.type);
+    suite.location = data.location;
+    suite._only = data.only;
+    suite._requireFile = data.requireFile;
+    suite._timeout = data.timeout;
+    suite._retries = data.retries;
+    suite._annotations = data.annotations;
+    suite._modifiers = data.modifiers;
+    suite._parallelMode = data.parallelMode;
+    suite._skipped = data.skipped;
+    suite._hooks = data.hooks.map((h: any) => ({ type: h.type, location: h.location, fn: () => { } }));
+    return suite;
+  }
+
   _clone(): Suite {
-    const suite = new Suite(this.title, this._type);
+    const data = this._serialize();
-    suite._only = this._only;
+    const suite = Suite._parse(data);
-    suite.location = this.location;
-    suite._requireFile = this._requireFile;
     suite._use = this._use.slice();
     suite._hooks = this._hooks.slice();
-    suite._timeout = this._timeout;
-    suite._retries = this._retries;
-    suite._annotations = this._annotations.slice();
-    suite._modifiers = this._modifiers.slice();
-    suite._parallelMode = this._parallelMode;
     suite._projectConfig = this._projectConfig;
-    suite._skipped = this._skipped;
     return suite;
   }
 
@@ -197,14 +249,34 @@ export class TestCase extends Base implements reporterTypes.TestCase {
     return status === 'expected' || status === 'flaky' || status === 'skipped';
   }
 
+  _serialize(): any {
+    return {
+      kind: 'test',
+      title: this.title,
+      location: this.location,
+      only: this._only,
+      requireFile: this._requireFile,
+      poolDigest: this._poolDigest,
+      expectedStatus: this.expectedStatus,
+      annotations: this.annotations.slice(),
+    };
+  }
+
+  static _parse(data: any): TestCase {
+    const test = new TestCase(data.title, () => {}, rootTestType, data.location);
+    test._only = data.only;
+    test._requireFile = data.requireFile;
+    test._poolDigest = data.poolDigest;
+    test.expectedStatus = data.expectedStatus;
+    test.annotations = data.annotations;
+    return test;
+  }
+
   _clone(): TestCase {
-    const test = new TestCase(this.title, this.fn, this._testType, this.location);
+    const data = this._serialize();
-    test._only = this._only;
+    const test = TestCase._parse(data);
-    test._requireFile = this._requireFile;
+    test._testType = this._testType;
-    test._poolDigest = this._poolDigest;
+    test.fn = this.fn;
-    test.expectedStatus = this.expectedStatus;
-    test.annotations = this.annotations.slice();
-    test._annotateWithInheritence = this._annotateWithInheritence;
     return test;
   }
 

packages/playwright-test/src/testLoader.ts

@@ -14,11 +14,13 @@
  * limitations under the License.
  */
 
-import * as path from 'path';
+import path from 'path';
+import type { TestError } from '../reporter';
+import type { FullConfigInternal } from './types';
 import { setCurrentlyLoadingFileSuite } from './globals';
+import { PoolBuilder } from './poolBuilder';
 import { Suite } from './test';
 import { requireOrImport } from './transform';
-import type { FullConfigInternal } from './types';
 import { serializeError } from './util';
 
 export const defaultTimeout = 30000;
@@ -34,7 +36,7 @@ export class TestLoader {
     this._fullConfig = fullConfig;
   }
 
-  async loadTestFile(file: string, environment: 'loader' | 'worker'): Promise<Suite> {
+  async loadTestFile(file: string, environment: 'loader' | 'worker', loadErrors: TestError[]): Promise<Suite> {
     if (cachedFileSuites.has(file))
       return cachedFileSuites.get(file)!;
     const suite = new Suite(path.relative(this._fullConfig.rootDir, file) || path.basename(file), 'file');
@@ -48,7 +50,7 @@ export class TestLoader {
     } catch (e) {
       if (environment === 'worker')
         throw e;
-      suite._loadError = serializeError(e);
+      loadErrors.push(serializeError(e));
     } finally {
       setCurrentlyLoadingFileSuite(undefined);
     }
@@ -76,3 +78,15 @@ export class TestLoader {
     return suite;
   }
 }
+
+export async function loadTestFilesInProcess(config: FullConfigInternal, testFiles: string[], loadErrors: TestError[]): Promise<Suite> {
+  const testLoader = new TestLoader(config);
+  const rootSuite = new Suite('', 'root');
+  for (const file of testFiles) {
+    const fileSuite = await testLoader.loadTestFile(file, 'loader', loadErrors);
+    rootSuite._addSuite(fileSuite);
+  }
+  // Generate hashes.
+  PoolBuilder.buildForLoader(rootSuite);
+  return rootSuite;
+}

packages/playwright-test/src/workerHost.ts

@@ -44,7 +44,10 @@ export class WorkerHost extends ProcessHost<WorkerInitParams> {
   }
 
   async start() {
-    await this.startRunner(this._params);
+    await this.startRunner(this._params, false, {
+      FORCE_COLOR: '1',
+      DEBUG_COLORS: '1',
+    });
   }
 
   runTestGroup(runPayload: RunPayload) {

packages/playwright-test/src/workerRunner.ts

@@ -17,7 +17,7 @@
 import { colors, rimraf } from 'playwright-core/lib/utilsBundle';
 import util from 'util';
 import { debugTest, formatLocation, relativeFilePath, serializeError } from './util';
-import type { TestBeginPayload, TestEndPayload, RunPayload, DonePayload, WorkerInitParams, TeardownErrorsPayload } from './ipc';
+import type { TestBeginPayload, TestEndPayload, RunPayload, DonePayload, WorkerInitParams, TeardownErrorsPayload, TestOutputPayload } from './ipc';
 import { setCurrentTestInfo } from './globals';
 import { ConfigLoader } from './configLoader';
 import type { Suite, TestCase } from './test';
@@ -29,7 +29,7 @@ import type { TimeSlot } from './timeoutManager';
 import { TimeoutManager } from './timeoutManager';
 import { ProcessRunner } from './process';
 import { TestLoader } from './testLoader';
-import { buildFileSuiteForProject, filterTests } from './suiteUtils';
+import { buildFileSuiteForProject, filterTestsRemoveEmptySuites } from './suiteUtils';
 import { PoolBuilder } from './poolBuilder';
 
 const removeFolderAsync = util.promisify(rimraf);
@@ -76,6 +76,23 @@ export class WorkerRunner extends ProcessRunner {
 
     process.on('unhandledRejection', reason => this.unhandledError(reason));
     process.on('uncaughtException', error => this.unhandledError(error));
+    process.stdout.write = (chunk: string | Buffer) => {
+      const outPayload: TestOutputPayload = {
+        ...chunkToParams(chunk)
+      };
+      this.dispatchEvent('stdOut', outPayload);
+      return true;
+    };
+
+    if (!process.env.PW_RUNNER_DEBUG) {
+      process.stderr.write = (chunk: string | Buffer) => {
+        const outPayload: TestOutputPayload = {
+          ...chunkToParams(chunk)
+        };
+        this.dispatchEvent('stdErr', outPayload);
+        return true;
+      };
+    }
   }
 
   private _stop(): Promise<void> {
@@ -184,9 +201,9 @@ export class WorkerRunner extends ProcessRunner {
     let fatalUnknownTestIds;
     try {
       await this._loadIfNeeded();
-      const fileSuite = await this._testLoader.loadTestFile(runPayload.file, 'worker');
+      const fileSuite = await this._testLoader.loadTestFile(runPayload.file, 'worker', []);
       const suite = buildFileSuiteForProject(this._project, fileSuite, this._params.repeatEachIndex);
-      const hasEntries = filterTests(suite, test => entries.has(test.id));
+      const hasEntries = filterTestsRemoveEmptySuites(suite, test => entries.has(test.id));
       if (hasEntries) {
         this._poolBuilder.buildPools(suite);
         this._extraSuiteAnnotations = new Map();
@@ -618,4 +635,12 @@ function formatTestTitle(test: TestCase, projectName: string) {
   return `${projectTitle}${location} › ${titles.join(' › ')}`;
 }
 
+function chunkToParams(chunk: Buffer | string):  { text?: string, buffer?: string } {
+  if (chunk instanceof Buffer)
+    return { buffer: chunk.toString('base64') };
+  if (typeof chunk !== 'string')
+    return { text: util.inspect(chunk) };
+  return { text: chunk };
+}
+
 export const create = (params: WorkerInitParams) => new WorkerRunner(params);

tests/playwright-test/reporter-html.spec.ts

@@ -775,11 +775,14 @@ test.describe('gitCommitInfo plugin', () => {
 
     const result = await runInlineTest({
       'uncommitted.txt': `uncommitted file`,
-      'playwright.config.ts': `export default {};`,
+      'playwright.config.ts': `
-      'example.spec.ts': `
         import { gitCommitInfo } from '@playwright/test/lib/plugins';
         const { test, _addRunnerPlugin } = pwt;
         _addRunnerPlugin(gitCommitInfo());
+        export default {};
+      `,
+      'example.spec.ts': `
+        const { test } = pwt;
         test('sample', async ({}) => { expect(2).toBe(2); });
       `,
     }, { reporter: 'dot,html' }, { PW_TEST_HTML_REPORT_OPEN: 'never', GITHUB_REPOSITORY: 'microsoft/playwright-example-for-test', GITHUB_RUN_ID: 'example-run-id', GITHUB_SERVER_URL: 'https://playwright.dev', GITHUB_SHA: 'example-sha' }, undefined, beforeRunPlaywrightTest);
@@ -805,9 +808,6 @@ test.describe('gitCommitInfo plugin', () => {
     const result = await runInlineTest({
       'uncommitted.txt': `uncommitted file`,
       'playwright.config.ts': `
-        export default {};
-      `,
-      'example.spec.ts': `
         import { gitCommitInfo } from '@playwright/test/lib/plugins';
         const { test, _addRunnerPlugin } = pwt;
         _addRunnerPlugin(gitCommitInfo({
@@ -819,6 +819,11 @@ test.describe('gitCommitInfo plugin', () => {
             'revision.email': 'shakespeare@example.local',
           },
         }));
+        export default {};
+      `,
+      'example.spec.ts': `
+        import { gitCommitInfo } from '@playwright/test/lib/plugins';
+        const { test } = pwt;
         test('sample', async ({}) => { expect(2).toBe(2); });
       `,
     }, { reporter: 'dot,html' }, { PW_TEST_HTML_REPORT_OPEN: 'never', GITHUB_REPOSITORY: 'microsoft/playwright-example-for-test', GITHUB_RUN_ID: 'example-run-id', GITHUB_SERVER_URL: 'https://playwright.dev', GITHUB_SHA: 'example-sha' }, undefined);

tests/playwright-test/web-server.spec.ts

@@ -21,7 +21,7 @@ import { test, expect } from './playwright-test-fixtures';
 const SIMPLE_SERVER_PATH = path.join(__dirname, 'assets', 'simple-server.js');
 
 test('should create a server', async ({ runInlineTest }, { workerIndex }) => {
-  const port = workerIndex + 10500;
+  const port = workerIndex * 2 + 10500;
   const result = await runInlineTest({
     'test.spec.ts': `
       const { test } = pwt;
@@ -87,7 +87,7 @@ test('should create a server', async ({ runInlineTest }, { workerIndex }) => {
 });
 
 test('should create a server with environment variables', async ({ runInlineTest }, { workerIndex }) => {
-  const port = workerIndex + 10500;
+  const port = workerIndex * 2 + 10500;
   const result = await runInlineTest({
     'test.spec.ts': `
       const { test } = pwt;
@@ -117,7 +117,7 @@ test('should create a server with environment variables', async ({ runInlineTest
 });
 
 test('should default cwd to config directory', async ({ runInlineTest }, testInfo) => {
-  const port = testInfo.workerIndex + 10500;
+  const port = testInfo.workerIndex * 2 + 10500;
   const configDir = testInfo.outputPath('foo');
   const relativeSimpleServerPath = path.relative(configDir, SIMPLE_SERVER_PATH);
   const result = await runInlineTest({
@@ -145,7 +145,7 @@ test('should default cwd to config directory', async ({ runInlineTest }, testInf
 });
 
 test('should resolve cwd wrt config directory', async ({ runInlineTest }, testInfo) => {
-  const port = testInfo.workerIndex + 10500;
+  const port = testInfo.workerIndex * 2 + 10500;
   const testdir = testInfo.outputPath();
   const relativeSimpleServerPath = path.relative(testdir, SIMPLE_SERVER_PATH);
   const result = await runInlineTest({
@@ -175,7 +175,7 @@ test('should resolve cwd wrt config directory', async ({ runInlineTest }, testIn
 
 
 test('should create a server with url', async ({ runInlineTest }, { workerIndex }) => {
-  const port = workerIndex + 10500;
+  const port = workerIndex * 2 + 10500;
   const result = await runInlineTest({
     'test.spec.ts': `
       const { test } = pwt;
@@ -200,7 +200,7 @@ test('should create a server with url', async ({ runInlineTest }, { workerIndex
 });
 
 test('should time out waiting for a server', async ({ runInlineTest }, { workerIndex }) => {
-  const port = workerIndex + 10500;
+  const port = workerIndex * 2 + 10500;
   const result = await runInlineTest({
     'test.spec.ts': `
       const { test } = pwt;
@@ -225,7 +225,7 @@ test('should time out waiting for a server', async ({ runInlineTest }, { workerI
 });
 
 test('should time out waiting for a server with url', async ({ runInlineTest }, { workerIndex }) => {
-  const port = workerIndex + 10500;
+  const port = workerIndex * 2 + 10500;
   const result = await runInlineTest({
     'test.spec.ts': `
       const { test } = pwt;
@@ -250,7 +250,7 @@ test('should time out waiting for a server with url', async ({ runInlineTest },
 });
 
 test('should be able to specify the baseURL without the server', async ({ runInlineTest }, { workerIndex }) => {
-  const port = workerIndex + 10500;
+  const port = workerIndex * 2 + 10500;
   const server = http.createServer((req: http.IncomingMessage, res: http.ServerResponse) => {
     res.end('<html><body>hello</body></html>');
   });
@@ -313,7 +313,7 @@ test('should be able to specify a custom baseURL with the server', async ({ runI
 });
 
 test('should be able to use an existing server when reuseExistingServer:true', async ({ runInlineTest }, { workerIndex }) => {
-  const port = workerIndex + 10500;
+  const port = workerIndex * 2 + 10500;
   const server = http.createServer((req: http.IncomingMessage, res: http.ServerResponse) => {
     res.end('<html><body>hello</body></html>');
   });
@@ -346,7 +346,7 @@ test('should be able to use an existing server when reuseExistingServer:true', a
 });
 
 test('should throw when a server is already running on the given port and strict is true', async ({ runInlineTest }, { workerIndex }) => {
-  const port = workerIndex + 10500;
+  const port = workerIndex * 2 + 10500;
   const server = http.createServer((req: http.IncomingMessage, res: http.ServerResponse) => {
     res.end('<html><body>hello</body></html>');
   });
@@ -378,7 +378,7 @@ test('should throw when a server is already running on the given port and strict
 
 for (const host of ['localhost', '127.0.0.1', '0.0.0.0']) {
   test(`should detect the server if a web-server is already running on ${host}`, async ({ runInlineTest }, { workerIndex }) => {
-    const port = workerIndex + 10500;
+    const port = workerIndex * 2 + 10500;
     const server = http.createServer((req: http.IncomingMessage, res: http.ServerResponse) => {
       res.end('<html><body>hello</body></html>');
     });
@@ -581,7 +581,7 @@ test.describe('baseURL with plugins', () => {
 });
 
 test('should treat 3XX as available server', async ({ runInlineTest }, { workerIndex }) => {
-  const port = workerIndex + 10500;
+  const port = workerIndex * 2 + 10500;
   const result = await runInlineTest({
     'test.spec.ts': `
       const { test } = pwt;