chore: pass the private data to the runner via config (#29467)

packages/playwright-ct-core/.npmignore

@@ -7,4 +7,4 @@
 !types/**
 !index.d.ts
 !index.js
-!plugin.js
+

packages/playwright-ct-core/index.js

@@ -16,16 +16,26 @@
 
 const { test: baseTest, expect, devices, defineConfig: originalDefineConfig } = require('playwright/test');
 const { fixtures } = require('./lib/mount');
+const { clearCacheCommand, findRelatedTestsCommand } = require('./lib/cliOverrides');
+const { createPlugin } = require('./lib/vitePlugin');
 
-const defineConfig = (config, ...configs) => originalDefineConfig({
-  ...config,
-  build: {
-    ...config.build,
-    babelPlugins: [
-      [require.resolve('./lib/tsxTransform')]
-    ],
-  }
-}, ...configs);
+const defineConfig = (...configs) => {
+  const original = originalDefineConfig(...configs);
+  return {
+    ...original,
+    '@playwright/test': {
+      ...original['@playwright/test'],
+      plugins: [() => createPlugin()],
+      babelPlugins: [
+        [require.resolve('./lib/tsxTransform')]
+      ],
+      cli: {
+        'clear-cache': clearCacheCommand,
+        'find-related-tests': findRelatedTestsCommand,
+      },
+    }
+  };
+};
 
 const test = baseTest.extend(fixtures);
 

packages/playwright-ct-core/package.json

@@ -21,7 +21,6 @@
     },
     "./lib/mount": "./lib/mount.js",
     "./lib/program": "./lib/program.js",
-    "./plugin": "./plugin.js",
     "./types/component": {
       "types": "./types/component.d.ts"
     }

packages/playwright-ct-core/plugin.js

@@ -1,17 +0,0 @@
-#!/usr/bin/env node
-/**
- * Copyright (c) Microsoft Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-module.exports = require('./lib/vitePlugin');

packages/playwright-ct-core/src/cliOverrides.ts

@@ -0,0 +1,35 @@
+
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { removeFolder } from 'playwright/lib/program';
+import { affectedTestFiles, cacheDir } from 'playwright/lib/transform/compilationCache';
+import { buildBundle } from './vitePlugin';
+import { resolveDirs } from './viteUtils';
+import type { Suite } from 'playwright/lib/common/test';
+import type { FullConfig } from 'playwright/test';
+
+export async function clearCacheCommand(config: FullConfig, configDir: string) {
+  const dirs = await resolveDirs(configDir, config);
+  if (dirs)
+    await removeFolder(dirs.outDir);
+  await removeFolder(cacheDir);
+}
+
+export async function findRelatedTestsCommand(files: string[],  config: FullConfig, configDir: string, suite: Suite) {
+  await buildBundle(config, configDir, suite);
+  return { relatedTests: affectedTestFiles(files) };
+}

packages/playwright-ct-core/src/devServer.ts

@@ -21,14 +21,15 @@ import { loadConfigFromFile } from 'playwright/lib/common/configLoader';
 import { Runner } from 'playwright/lib/runner/runner';
 import type { PluginContext } from 'rollup';
 import { source as injectedSource } from './generated/indexSource';
-import { createConfig, populateComponentsFromTests, resolveDirs, transformIndexFile } from './viteUtils';
+import { createConfig, populateComponentsFromTests, resolveDirs, transformIndexFile, frameworkConfig } from './viteUtils';
 import type { ComponentRegistry } from './viteUtils';
 
-export async function runDevServer(configFile: string, registerSourceFile: string, frameworkPluginFactory: () => Promise<any>) {
+export async function runDevServer(configFile: string) {
   const config = await loadConfigFromFile(configFile);
   if (!config)
     return;
 
+  const { registerSourceFile, frameworkPluginFactory } = frameworkConfig(config.config);
   const runner = new Runner(config);
   await runner.loadAllTests();
   const componentRegistry: ComponentRegistry = new Map();

packages/playwright-ct-core/src/program.ts

@@ -16,56 +16,17 @@
 
 import type { Command } from 'playwright-core/lib/utilsBundle';
 
-import path from 'path';
-import { program, removeFolder, setClearCacheCommandOverride, setFindRelatedTestsCommandOverride, withRunnerAndMutedWrite } from 'playwright/lib/program';
+import { program } from 'playwright/lib/program';
 import { runDevServer } from './devServer';
-import { resolveDirs } from './viteUtils';
-import { affectedTestFiles, cacheDir } from 'playwright/lib/transform/compilationCache';
-import { loadConfigFromFile } from 'playwright/lib/common/configLoader';
-import { buildBundle } from './vitePlugin';
 export { program } from 'playwright/lib/program';
 
-let _framework: { registerSource: string, frameworkPluginFactory: () => Promise<any> };
-
-export function initializePlugin(framework: { registerSource: string, frameworkPluginFactory: () => Promise<any> }) {
-  _framework = framework;
-}
-
 function addDevServerCommand(program: Command) {
   const command = program.command('dev-server');
   command.description('start dev server');
   command.option('-c, --config <file>', `Configuration file, or a test directory with optional "playwright.config.{m,c}?{js,ts}"`);
   command.action(options => {
-    runDevServer(options.config, _framework.registerSource, _framework.frameworkPluginFactory);
+    runDevServer(options.config);
   });
 }
 
-setFindRelatedTestsCommandOverride(async (files, options) => {
-  await withRunnerAndMutedWrite(options.config, async (runner, config, configDir) => {
-    const result = await runner.loadAllTests();
-    if (result.status !== 'passed' || !result.suite)
-      return { errors: result.errors };
-    await buildBundle({
-      config,
-      configDir,
-      suite: result.suite,
-      registerSourceFile: _framework.registerSource,
-      frameworkPluginFactory: _framework.frameworkPluginFactory,
-    });
-    const resolvedFiles = (files as string[]).map(file => path.resolve(process.cwd(), file));
-    return { relatedTests: affectedTestFiles(resolvedFiles) };
-  });
-});
-
-setClearCacheCommandOverride(async options => {
-  const configFile = options.config;
-  const config = await loadConfigFromFile(configFile);
-  if (!config)
-    return;
-  const dirs = await resolveDirs(config.configDir, config.config);
-  if (dirs)
-    await removeFolder(dirs.outDir);
-  await removeFolder(cacheDir);
-});
-
 addDevServerCommand(program);

packages/playwright-ct-core/src/vitePlugin.ts

@@ -30,7 +30,7 @@ import type { TestRunnerPlugin } from '../../playwright/src/plugins';
 import { source as injectedSource } from './generated/indexSource';
 import type { ImportInfo } from './tsxTransform';
 import type { ComponentRegistry } from './viteUtils';
-import { createConfig, hasJSComponents, populateComponentsFromTests, resolveDirs, resolveEndpoint, transformIndexFile } from './viteUtils';
+import { createConfig, frameworkConfig, hasJSComponents, populateComponentsFromTests, resolveDirs, resolveEndpoint, transformIndexFile } from './viteUtils';
 import { resolveHook } from 'playwright/lib/transform/transform';
 
 const log = debug('pw:vite');
@@ -38,9 +38,7 @@ const log = debug('pw:vite');
 let stoppableServer: any;
 const playwrightVersion = getPlaywrightVersion();
 
-export function createPlugin(
-  registerSourceFile: string,
-  frameworkPluginFactory?: () => Promise<Plugin>): TestRunnerPlugin {
+export function createPlugin(): TestRunnerPlugin {
   let configDir: string;
   let config: FullConfig;
   return {
@@ -52,13 +50,7 @@ export function createPlugin(
     },
 
     begin: async (suite: Suite) => {
-      const result = await buildBundle({
-        config,
-        configDir,
-        suite,
-        registerSourceFile,
-        frameworkPluginFactory: frameworkPluginFactory,
-      });
+      const result = await buildBundle(config, configDir, suite);
       if (!result)
         return;
 
@@ -96,16 +88,11 @@ type BuildInfo = {
   }
 };
 
-export async function buildBundle(options: {
-  config: FullConfig,
-  configDir: string,
-  suite: Suite,
-  registerSourceFile: string,
-  frameworkPluginFactory?: () => Promise<Plugin>
-}): Promise<{ buildInfo: BuildInfo, viteConfig: Record<string, any> } | null> {
+export async function buildBundle(config: FullConfig, configDir: string, suite: Suite): Promise<{ buildInfo: BuildInfo, viteConfig: Record<string, any> } | null> {
+  const { registerSourceFile, frameworkPluginFactory } = frameworkConfig(config);
   {
     // Detect a running dev server and use it if available.
-    const endpoint = resolveEndpoint(options.config);
+    const endpoint = resolveEndpoint(config);
     const protocol = endpoint.https ? 'https:' : 'http:';
     const url = new URL(`${protocol}//${endpoint.host}:${endpoint.port}`);
     if (await isURLAvailable(url, true)) {
@@ -116,7 +103,7 @@ export async function buildBundle(options: {
     }
   }
 
-  const dirs = await resolveDirs(options.configDir, options.config);
+  const dirs = await resolveDirs(configDir, config);
   if (!dirs) {
     // eslint-disable-next-line no-console
     console.log(`Template file playwright/index.html is missing.`);
@@ -128,7 +115,7 @@ export async function buildBundle(options: {
   let buildExists = false;
   let buildInfo: BuildInfo;
 
-  const registerSource = injectedSource + '\n' + await fs.promises.readFile(options.registerSourceFile, 'utf-8');
+  const registerSource = injectedSource + '\n' + await fs.promises.readFile(registerSourceFile, 'utf-8');
   const registerSourceHash = calculateSha1(registerSource);
 
   const { version: viteVersion, build, mergeConfig } = await import('vite');
@@ -168,7 +155,7 @@ export async function buildBundle(options: {
   buildInfo.components = [...componentRegistry.values()];
 
   const jsxInJS = hasJSComponents(buildInfo.components);
-  const viteConfig = await createConfig(dirs, options.config, options.frameworkPluginFactory, jsxInJS);
+  const viteConfig = await createConfig(dirs, config, frameworkPluginFactory, jsxInJS);
 
   if (sourcesDirty) {
     // Only add out own plugin when we actually build / transform.
@@ -183,7 +170,7 @@ export async function buildBundle(options: {
 
   {
     // Update dependencies based on the vite build.
-    for (const projectSuite of options.suite.suites) {
+    for (const projectSuite of suite.suites) {
       for (const fileSuite of projectSuite.suites) {
         // For every test file...
         const testFile = fileSuite.location!.file;

packages/playwright-ct-core/src/viteUtils.ts

@@ -192,3 +192,7 @@ export function transformIndexFile(id: string, content: string, templateDir: str
     map: { mappings: '' }
   };
 }
+
+export function frameworkConfig(config: FullConfig): { registerSourceFile: string, frameworkPluginFactory?: () => Promise<Plugin> } {
+  return (config as any)['@playwright/experimental-ct-core'];
+}

packages/playwright-ct-react/cli.js

@@ -15,8 +15,6 @@
  * limitations under the License.
  */
 
-const { program, initializePlugin } = require('@playwright/experimental-ct-core/lib/program');
-const { _framework } = require('./index');
+const { program } = require('@playwright/experimental-ct-core/lib/program');
 
-initializePlugin(_framework);
 program.parse(process.argv);

packages/playwright-ct-react/index.js

@@ -17,14 +17,17 @@
 const { test, expect, devices, defineConfig: originalDefineConfig } = require('@playwright/experimental-ct-core');
 const path = require('path');
 
-const registerSource = path.join(__dirname, 'registerSource.mjs');
-const frameworkPluginFactory = () => import('@vitejs/plugin-react').then(plugin => plugin.default());
-
-const plugin = () => {
-  // Only fetch upon request to avoid resolution in workers.
-  const { createPlugin } = require('@playwright/experimental-ct-core/plugin');
-  return createPlugin(registerSource, frameworkPluginFactory);
+const defineConfig = (config, ...configs) => {
+  return originalDefineConfig({
+    ...config,
+    '@playwright/test': {
+      packageJSON: require.resolve('./package.json'),
+    },
+    '@playwright/experimental-ct-core': {
+      registerSourceFile: path.join(__dirname, 'registerSource.mjs'),
+      frameworkPluginFactory: () => import('@vitejs/plugin-react').then(plugin => plugin.default()),
+    },
+  }, ...configs);
 };
-const defineConfig = (config, ...configs) => originalDefineConfig({ ...config, _plugins: [plugin] }, ...configs);
 
-module.exports = { test, expect, devices, defineConfig, _framework: { registerSource, frameworkPluginFactory } };
+module.exports = { test, expect, devices, defineConfig };

packages/playwright-ct-react17/index.js

@@ -17,14 +17,17 @@
 const { test, expect, devices, defineConfig: originalDefineConfig } = require('@playwright/experimental-ct-core');
 const path = require('path');
 
-const registerSource = path.join(__dirname, 'registerSource.mjs');
-const frameworkPluginFactory = () => import('@vitejs/plugin-react').then(plugin => plugin.default());
-
-const plugin = () => {
-  // Only fetch upon request to avoid resolution in workers.
-  const { createPlugin } = require('@playwright/experimental-ct-core/plugin');
-  return createPlugin(registerSource, frameworkPluginFactory);
+const defineConfig = (config, ...configs) => {
+  return originalDefineConfig({
+    ...config,
+    '@playwright/test': {
+      packageJSON: require.resolve('./package.json'),
+    },
+    '@playwright/experimental-ct-core': {
+      registerSourceFile: path.join(__dirname, 'registerSource.mjs'),
+      frameworkPluginFactory: () => import('@vitejs/plugin-react').then(plugin => plugin.default()),
+    },
+  }, ...configs);
 };
-const defineConfig = (config, ...configs) => originalDefineConfig({ ...config, _plugins: [plugin] }, ...configs);
 
-module.exports = { test, expect, devices, defineConfig, _framework: { registerSource, frameworkPluginFactory } };
+module.exports = { test, expect, devices, defineConfig };

packages/playwright-ct-solid/index.js

@@ -17,14 +17,17 @@
 const { test, expect, devices, defineConfig: originalDefineConfig } = require('@playwright/experimental-ct-core');
 const path = require('path');
 
-const registerSource = path.join(__dirname, 'registerSource.mjs');
-const frameworkPluginFactory = () => import('vite-plugin-solid').then(plugin => plugin.default());
-
-const plugin = () => {
-  // Only fetch upon request to avoid resolution in workers.
-  const { createPlugin } = require('@playwright/experimental-ct-core/plugin');
-  return createPlugin(registerSource, frameworkPluginFactory);
+const defineConfig = (config, ...configs) => {
+  return originalDefineConfig({
+    ...config,
+    '@playwright/test': {
+      packageJSON: require.resolve('./package.json'),
+    },
+    '@playwright/experimental-ct-core': {
+      registerSourceFile: path.join(__dirname, 'registerSource.mjs'),
+      frameworkPluginFactory: () => import('vite-plugin-solid').then(plugin => plugin.default()),
+    },
+  }, ...configs);
 };
-const defineConfig = (config, ...configs) => originalDefineConfig({ ...config, _plugins: [plugin] }, ...configs);
 
-module.exports = { test, expect, devices, defineConfig, _framework: { registerSource, frameworkPluginFactory } };
+module.exports = { test, expect, devices, defineConfig };

packages/playwright-ct-svelte/index.js

@@ -17,14 +17,17 @@
 const { test, expect, devices, defineConfig: originalDefineConfig } = require('@playwright/experimental-ct-core');
 const path = require('path');
 
-const registerSource = path.join(__dirname, 'registerSource.mjs');
-const frameworkPluginFactory = () => import('@sveltejs/vite-plugin-svelte').then(plugin => plugin.svelte());
-
-const plugin = () => {
-  // Only fetch upon request to avoid resolution in workers.
-  const { createPlugin } = require('@playwright/experimental-ct-core/plugin');
-  return createPlugin(registerSource, frameworkPluginFactory);
+const defineConfig = (config, ...configs) => {
+  return originalDefineConfig({
+    ...config,
+    '@playwright/test': {
+      packageJSON: require.resolve('./package.json'),
+    },
+    '@playwright/experimental-ct-core': {
+      registerSourceFile: path.join(__dirname, 'registerSource.mjs'),
+      frameworkPluginFactory: () => import('@sveltejs/vite-plugin-svelte').then(plugin => plugin.svelte()),
+    },
+  }, ...configs);
 };
-const defineConfig = (config, ...configs) => originalDefineConfig({ ...config, _plugins: [plugin] }, ...configs);
 
-module.exports = { test, expect, devices, defineConfig, _framework: { registerSource, frameworkPluginFactory } };
+module.exports = { test, expect, devices, defineConfig };

packages/playwright-ct-vue/index.js

@@ -17,14 +17,17 @@
 const { test, expect, devices, defineConfig: originalDefineConfig } = require('@playwright/experimental-ct-core');
 const path = require('path');
 
-const registerSource = path.join(__dirname, 'registerSource.mjs');
-const frameworkPluginFactory = () => import('@vitejs/plugin-vue').then(plugin => plugin.default());
-
-const plugin = () => {
-  // Only fetch upon request to avoid resolution in workers.
-  const { createPlugin } = require('@playwright/experimental-ct-core/plugin');
-  return createPlugin(registerSource, frameworkPluginFactory);
+const defineConfig = (config, ...configs) => {
+  return originalDefineConfig({
+    ...config,
+    '@playwright/test': {
+      packageJSON: require.resolve('./package.json'),
+    },
+    '@playwright/experimental-ct-core': {
+      registerSourceFile: path.join(__dirname, 'registerSource.mjs'),
+      frameworkPluginFactory: () => import('@vitejs/plugin-vue').then(plugin => plugin.default()),
+    },
+  }, ...configs);
 };
-const defineConfig = (config, ...configs) => originalDefineConfig({ ...config, _plugins: [plugin] }, ...configs);
 
-module.exports = { test, expect, devices, defineConfig, _framework: { registerSource, frameworkPluginFactory } };
+module.exports = { test, expect, devices, defineConfig };

packages/playwright-ct-vue2/index.js

@@ -17,14 +17,17 @@
 const { test, expect, devices, defineConfig: originalDefineConfig } = require('@playwright/experimental-ct-core');
 const path = require('path');
 
-const registerSource = path.join(__dirname, 'registerSource.mjs');
-const frameworkPluginFactory = () => import('@vitejs/plugin-vue2').then(plugin => plugin.default());
-
-const plugin = () => {
-  // Only fetch upon request to avoid resolution in workers.
-  const { createPlugin } = require('@playwright/experimental-ct-core/plugin');
-  return createPlugin(registerSource, frameworkPluginFactory);
+const defineConfig = (config, ...configs) => {
+  return originalDefineConfig({
+    ...config,
+    '@playwright/test': {
+      packageJSON: require.resolve('./package.json'),
+    },
+    '@playwright/experimental-ct-core': {
+      registerSourceFile: path.join(__dirname, 'registerSource.mjs'),
+      frameworkPluginFactory: () => import('@vitejs/plugin-vue2').then(plugin => plugin.default()),
+    },
+  }, ...configs);
 };
-const defineConfig = (config, ...configs) => originalDefineConfig({ ...config, _plugins: [plugin] }, ...configs);
 
-module.exports = { test, expect, devices, defineConfig, _framework: { registerSource, frameworkPluginFactory } };
+module.exports = { test, expect, devices, defineConfig };

packages/playwright/src/common/config.ts

@@ -71,7 +71,8 @@ export class FullConfigInternal {
     this.configCLIOverrides = configCLIOverrides;
     this.globalOutputDir = takeFirst(configCLIOverrides.outputDir, pathResolve(configDir, config.outputDir), throwawayArtifactsPath, path.resolve(process.cwd()));
     this.ignoreSnapshots = takeFirst(configCLIOverrides.ignoreSnapshots, config.ignoreSnapshots, false);
-    this.plugins = ((config as any)._plugins || []).map((p: any) => ({ factory: p }));
+    const privateConfiguration = (config as any)['@playwright/test'];
+    this.plugins = (privateConfiguration?.plugins || []).map((p: any) => ({ factory: p }));
 
     this.config = {
       configFile,
@@ -96,6 +97,11 @@ export class FullConfigInternal {
       workers: 0,
       webServer: null,
     };
+    for (const key in config) {
+      if (key.startsWith('@'))
+        (this.config as any)[key] = (config as any)[key];
+    }
+
     (this.config as any)[configInternalSymbol] = this;
 
     const workers = takeFirst(configCLIOverrides.workers, config.workers, '50%');
@@ -127,7 +133,7 @@ export class FullConfigInternal {
     resolveProjectDependencies(this.projects);
     this._assignUniqueProjectIds(this.projects);
     setTransformConfig({
-      babelPlugins: (config as any).build?.babelPlugins || [],
+      babelPlugins: privateConfiguration?.babelPlugins || [],
       external: config.build?.external || [],
     });
     this.config.projects = this.projects.map(p => p.project);

packages/playwright/src/program.ts

@@ -66,19 +66,7 @@ function addListFilesCommand(program: Command) {
   command.option('-c, --config <file>', `Configuration file, or a test directory with optional "playwright.config.{m,c}?{js,ts}"`);
   command.option('--project <project-name...>', `Only run tests from the specified list of projects (default: list all projects)`);
   command.option('--project-grep <pattern>', `Only run tests from the projects matching this regular expression (default: list all projects)`);
-  command.action(async (args, opts) => {
-    try {
-      await listTestFiles(opts);
-    } catch (e) {
-      console.error(e);
-      gracefullyProcessExitDoNotHang(1);
-    }
-  });
-}
-
-let clearCacheCommandOverride: (opts: any) => Promise<void>;
-export function setClearCacheCommandOverride(body: (opts: any) => Promise<void>) {
-  clearCacheCommandOverride = body;
+  command.action(async (args, opts) => listTestFiles(opts));
 }
 
 function addClearCacheCommand(program: Command) {
@@ -86,8 +74,15 @@ function addClearCacheCommand(program: Command) {
   command.description('clears build and test caches');
   command.option('-c, --config <file>', `Configuration file, or a test directory with optional "playwright.config.{m,c}?{js,ts}"`);
   command.action(async opts => {
-    if (clearCacheCommandOverride)
-      return clearCacheCommandOverride(opts);
+    const configInternal = await loadConfigFromFile(opts.config);
+    if (!configInternal)
+      return;
+    const { config, configDir } = configInternal;
+    const override = (config as any)['@playwright/test']?.['cli']?.['clear-cache'];
+    if (override) {
+      await override(config, configDir);
+      return;
+    }
     await removeFolder(cacheDir);
   });
 }
@@ -102,23 +97,20 @@ export async function removeFolder(folder: string) {
   }
 }
 
-let findRelatedTestsCommandOverride: (files: string[], opts: any) => Promise<void>;
-export function setFindRelatedTestsCommandOverride(body: (files: string[], opts: any) => Promise<void>) {
-  findRelatedTestsCommandOverride = body;
-}
-
 function addFindRelatedTestsCommand(program: Command) {
   const command = program.command('find-related-tests [source-files...]');
   command.description('Returns the list of related tests to the given files');
   command.option('-c, --config <file>', `Configuration file, or a test directory with optional "playwright.config.{m,c}?{js,ts}"`);
   command.action(async (files, options) => {
-    if (findRelatedTestsCommandOverride)
-      return findRelatedTestsCommandOverride(files, options);
-    await withRunnerAndMutedWrite(options.config, async runner => {
+    await withRunnerAndMutedWrite(options.config, async (runner, config, configDir) => {
       const result = await runner.loadAllTests();
       if (result.status !== 'passed' || !result.suite)
         return { errors: result.errors };
+
       const resolvedFiles = (files as string[]).map(file => path.resolve(process.cwd(), file));
+      const override = (config as any)['@playwright/test']?.['cli']?.['find-related-tests'];
+      if (override)
+        return await override(resolvedFiles, config, configDir, result.suite);
       return { relatedTests: affectedTestFiles(resolvedFiles) };
     });
   });
@@ -217,7 +209,10 @@ export async function withRunnerAndMutedWrite(configFile: string | undefined, ca
 async function listTestFiles(opts: { [key: string]: any }) {
   if (opts.project && opts.projectGrep)
     throw new Error('Only one of --project and --project-grep can be specified.');
-  await withRunnerAndMutedWrite(opts.config, async runner => runner.listTestFiles(opts.project, opts.projectGrep));
+  await withRunnerAndMutedWrite(opts.config, async (runner, config) => {
+    const frameworkPackage = (config as any)['@playwright/test']?.['packageJSON'];
+    return await runner.listTestFiles(frameworkPackage, opts.project, opts.projectGrep);
+  });
 }
 
 async function mergeReports(reportDir: string | undefined, opts: { [key: string]: any }) {

packages/playwright/src/runner/runner.ts

@@ -15,6 +15,7 @@
  * limitations under the License.
  */
 
+import path from 'path';
 import { monotonicTime } from 'playwright-core/lib/utils';
 import type { FullResult, TestError } from '../../types/testReporter';
 import { webServerPluginsForConfig } from '../plugins/webServerPlugin';
@@ -39,6 +40,8 @@ type ProjectConfigWithFiles = {
 
 type ConfigListFilesReport = {
   projects: ProjectConfigWithFiles[];
+  cliEntryPoint?: string;
+  error?: TestError;
 };
 
 export class Runner {
@@ -48,10 +51,11 @@ export class Runner {
     this._config = config;
   }
 
-  async listTestFiles(projectNames: string[] | undefined, projectGrep: string | undefined): Promise<any> {
+  async listTestFiles(frameworkPackage: string | undefined, projectNames: string[] | undefined, projectGrep: string | undefined): Promise<ConfigListFilesReport> {
     const projects = filterProjects(this._config.projects, projectNames, projectGrep);
     const report: ConfigListFilesReport = {
-      projects: []
+      projects: [],
+      cliEntryPoint: frameworkPackage ? path.join(path.dirname(frameworkPackage), 'cli.js') : undefined,
     };
     for (const project of projects) {
       report.projects.push({

tests/playwright-test/config.spec.ts

@@ -656,8 +656,12 @@ test('should merge ct configs', async ({ runInlineTest }) => {
       expect(derivedConfig).toEqual(expect.objectContaining({
         use: { foo: 1, bar: 2 },
         grep: 'hi',
-        build: { babelPlugins: [expect.anything()] },
-        _plugins: [expect.anything()],
+        '@playwright/test': expect.objectContaining({
+          babelPlugins: [[expect.stringContaining('tsxTransform.js')]]        
+        }),
+        '@playwright/experimental-ct-core': expect.objectContaining({
+          registerSourceFile: expect.stringContaining('registerSource'),
+        }),
       }));
     `,
     'a.test.ts': `

tests/playwright-test/global-setup.spec.ts

@@ -324,14 +324,14 @@ test('globalSetup auth should compile', async ({ runTSC }) => {
 test('teardown order', async ({ runInlineTest }) => {
   const result = await runInlineTest({
     'playwright.config.ts': `
-      const _plugins = [];
+      const plugins = [];
       for (let i = 1; i < 4; ++i) {
-        _plugins.push(() => ({
+        plugins.push(() => ({
           setup: () => console.log('\\n%%setup ' + i),
           teardown: () => console.log('\\n%%teardown ' + i),
         }));
       }
-      export default { _plugins };
+      export default { '@playwright/test': { plugins } };
     `,
     'a.test.ts': `
       import { test, expect } from '@playwright/test';
@@ -353,9 +353,9 @@ test('teardown order', async ({ runInlineTest }) => {
 test('teardown after error', async ({ runInlineTest }) => {
   const result = await runInlineTest({
     'playwright.config.ts': `
-      const _plugins = [];
+      const plugins = [];
       for (let i = 1; i < 4; ++i) {
-        _plugins.push(() => ({
+        plugins.push(() => ({
           setup: () => console.log('\\n%%setup ' + i),
           teardown: () => {
             console.log('\\n%%teardown ' + i);
@@ -363,7 +363,7 @@ test('teardown after error', async ({ runInlineTest }) => {
           },
         }));
       }
-      export default { _plugins };
+      export default { '@playwright/test': { plugins } };
     `,
     'a.test.ts': `
       import { test, expect } from '@playwright/test';

tests/playwright-test/reporter-html.spec.ts

@@ -883,7 +883,8 @@ for (const useIntermediateMergeReport of [false, true] as const) {
           'playwright.config.ts': `
             import { gitCommitInfo } from 'playwright/lib/plugins';
             import { test, expect } from '@playwright/test';
-            export default { _plugins: [gitCommitInfo()] };
+            const plugins = [gitCommitInfo()];
+            export default { '@playwright/test': { plugins } };
           `,
           'example.spec.ts': `
             import { test, expect } from '@playwright/test';
@@ -945,7 +946,7 @@ for (const useIntermediateMergeReport of [false, true] as const) {
                 'revision.email': 'shakespeare@example.local',
               },
             });
-            export default { _plugins: [plugin] };
+            export default { '@playwright/test': { plugins: [plugin] } };
           `,
           'example.spec.ts': `
             import { gitCommitInfo } from 'playwright/lib/plugins';

tests/playwright-test/runner.spec.ts

@@ -450,8 +450,8 @@ test('sigint should stop plugins', async ({ interactWithTestRunner }) => {
 
   const testProcess = await interactWithTestRunner({
     'playwright.config.ts': `
-      const _plugins = [];
-      _plugins.push(() => ({
+      const plugins = [];
+      plugins.push(() => ({
         setup: async () => {
           console.log('Plugin1 setup');
           console.log('%%SEND-SIGINT%%');
@@ -462,7 +462,7 @@ test('sigint should stop plugins', async ({ interactWithTestRunner }) => {
         }
       }));
 
-      _plugins.push(() => ({
+      plugins.push(() => ({
         setup: async () => {
           console.log('Plugin2 setup');
         },
@@ -471,7 +471,7 @@ test('sigint should stop plugins', async ({ interactWithTestRunner }) => {
         }
       }));
       module.exports = {
-        _plugins
+        '@playwright/test': { plugins }
       };
     `,
     'a.spec.js': `
@@ -500,8 +500,8 @@ test('sigint should stop plugins 2', async ({ interactWithTestRunner }) => {
 
   const testProcess = await interactWithTestRunner({
     'playwright.config.ts': `
-      const _plugins = [];
-      _plugins.push(() => ({
+      const plugins = [];
+      plugins.push(() => ({
         setup: async () => {
           console.log('Plugin1 setup');
         },
@@ -510,7 +510,7 @@ test('sigint should stop plugins 2', async ({ interactWithTestRunner }) => {
         }
       }));
 
-      _plugins.push(() => ({
+      plugins.push(() => ({
         setup: async () => {
           console.log('Plugin2 setup');
           console.log('%%SEND-SIGINT%%');
@@ -520,7 +520,7 @@ test('sigint should stop plugins 2', async ({ interactWithTestRunner }) => {
           console.log('Plugin2 teardown');
         }
       }));
-      module.exports = { _plugins };
+      module.exports = { '@playwright/test': { plugins } };
     `,
     'a.spec.js': `
       import { test, expect } from '@playwright/test';