diff --git a/.github/workflows/tests_primary.yml b/.github/workflows/tests_primary.yml index 452293987a..7b9eb93636 100644 --- a/.github/workflows/tests_primary.yml +++ b/.github/workflows/tests_primary.yml @@ -151,6 +151,12 @@ jobs: - os: ubuntu-latest node-version: 18 shard: 2/2 + - os: ubuntu-latest + node-version: 20 + shard: 1/2 + - os: ubuntu-latest + node-version: 20 + shard: 2/2 runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 diff --git a/packages/playwright-test/package.json b/packages/playwright-test/package.json index 6afc5b894a..468a1d30f9 100644 --- a/packages/playwright-test/package.json +++ b/packages/playwright-test/package.json @@ -20,8 +20,8 @@ "./lib/cli": "./lib/cli.js", "./lib/transform/babelBundle": "./lib/transform/babelBundle.js", "./lib/transform/compilationCache": "./lib/transform/compilationCache.js", + "./lib/transform/esmLoader": "./lib/transform/esmLoader.js", "./lib/internalsForTest": "./lib/internalsForTest.js", - "./lib/experimentalLoader": "./lib/experimentalLoader.js", "./lib/plugins": "./lib/plugins/index.js", "./jsx-runtime": { "import": "./jsx-runtime.mjs", diff --git a/packages/playwright-test/src/DEPS.list b/packages/playwright-test/src/DEPS.list index 09354b5b0f..d49e6f1ac5 100644 --- a/packages/playwright-test/src/DEPS.list +++ b/packages/playwright-test/src/DEPS.list @@ -9,8 +9,5 @@ common/ [index.ts] @testIsomorphic/** -[experimentalLoader.ts] -./transform/ - [internalsForTest.ts] ** diff --git a/packages/playwright-test/src/common/configLoader.ts b/packages/playwright-test/src/common/configLoader.ts index a3e614c2df..96bde3978f 100644 --- a/packages/playwright-test/src/common/configLoader.ts +++ b/packages/playwright-test/src/common/configLoader.ts @@ -23,6 +23,8 @@ import type { Config, Project } from '../../types/test'; import { errorWithFile } from '../util'; import { setCurrentConfig } from './globals'; import { FullConfigInternal } from './config'; +import { addToCompilationCache } from '../transform/compilationCache'; +import { initializeEsmLoader } from './esmLoaderHost'; const kDefineConfigWasUsed = Symbol('defineConfigWasUsed'); export const defineConfig = (config: any) => { @@ -39,8 +41,11 @@ export class ConfigLoader { } static async deserialize(data: SerializedConfig): Promise { - const loader = new ConfigLoader(data.configCLIOverrides); setBabelPlugins(data.babelTransformPlugins); + addToCompilationCache(data.compilationCache); + await initializeEsmLoader(); + + const loader = new ConfigLoader(data.configCLIOverrides); if (data.configFile) return await loader.loadConfigFile(data.configFile); diff --git a/packages/playwright-test/src/common/esmLoaderHost.ts b/packages/playwright-test/src/common/esmLoaderHost.ts new file mode 100644 index 0000000000..14b40098c0 --- /dev/null +++ b/packages/playwright-test/src/common/esmLoaderHost.ts @@ -0,0 +1,52 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { addToCompilationCache, serializeCompilationCache } from '../transform/compilationCache'; +import { getBabelPlugins } from '../transform/transform'; +import { PortTransport } from '../transform/portTransport'; + +const port = (globalThis as any).__esmLoaderPort; + +const loaderChannel = port ? new PortTransport(port, async (method, params) => { + if (method === 'pushToCompilationCache') + addToCompilationCache(params.cache); +}) : undefined; + +export async function startCollectingFileDeps() { + if (!loaderChannel) + return; + await loaderChannel.send('startCollectingFileDeps', {}); +} + +export async function stopCollectingFileDeps(file: string) { + if (!loaderChannel) + return; + await loaderChannel.send('stopCollectingFileDeps', { file }); +} + +export async function incorporateCompilationCache() { + if (!loaderChannel) + return; + const result = await loaderChannel.send('getCompilationCache', {}); + addToCompilationCache(result.cache); +} + +export async function initializeEsmLoader() { + if (!loaderChannel) + return; + await loaderChannel.send('setBabelPlugins', { plugins: getBabelPlugins() }); + await loaderChannel.send('addToCompilationCache', { cache: serializeCompilationCache() }); +} diff --git a/packages/playwright-test/src/common/testLoader.ts b/packages/playwright-test/src/common/testLoader.ts index ddeef0f084..c0e781e552 100644 --- a/packages/playwright-test/src/common/testLoader.ts +++ b/packages/playwright-test/src/common/testLoader.ts @@ -22,6 +22,7 @@ import { Suite } from './test'; import { requireOrImport } from '../transform/transform'; import { filterStackTrace } from '../util'; import { startCollectingFileDeps, stopCollectingFileDeps } from '../transform/compilationCache'; +import * as esmLoaderHost from './esmLoaderHost'; export const defaultTimeout = 30000; @@ -37,8 +38,10 @@ export async function loadTestFile(file: string, rootDir: string, testErrors?: T suite.location = { file, line: 0, column: 0 }; setCurrentlyLoadingFileSuite(suite); - if (!isWorkerProcess()) + if (!isWorkerProcess()) { startCollectingFileDeps(); + await esmLoaderHost.startCollectingFileDeps(); + } try { await requireOrImport(file); cachedFileSuites.set(file, suite); @@ -47,8 +50,11 @@ export async function loadTestFile(file: string, rootDir: string, testErrors?: T throw e; testErrors.push(serializeLoadError(file, e)); } finally { - stopCollectingFileDeps(file); setCurrentlyLoadingFileSuite(undefined); + if (!isWorkerProcess()) { + stopCollectingFileDeps(file); + await esmLoaderHost.stopCollectingFileDeps(file); + } } { diff --git a/packages/playwright-test/src/loader/loaderMain.ts b/packages/playwright-test/src/loader/loaderMain.ts index cea150b2a8..23444ca21f 100644 --- a/packages/playwright-test/src/loader/loaderMain.ts +++ b/packages/playwright-test/src/loader/loaderMain.ts @@ -20,8 +20,9 @@ import { ProcessRunner } from '../common/process'; import type { FullConfigInternal } from '../common/config'; import { loadTestFile } from '../common/testLoader'; import type { TestError } from '../../reporter'; -import { addToCompilationCache, serializeCompilationCache } from '../transform/compilationCache'; +import { serializeCompilationCache } from '../transform/compilationCache'; import { PoolBuilder } from '../common/poolBuilder'; +import { incorporateCompilationCache } from '../common/esmLoaderHost'; export class LoaderMain extends ProcessRunner { private _serializedConfig: SerializedConfig; @@ -30,7 +31,6 @@ export class LoaderMain extends ProcessRunner { constructor(serializedConfig: SerializedConfig) { super(); - addToCompilationCache(serializedConfig.compilationCache); this._serializedConfig = serializedConfig; } @@ -48,7 +48,8 @@ export class LoaderMain extends ProcessRunner { return { fileSuite: fileSuite._deepSerialize(), testErrors }; } - async serializeCompilationCache() { + async getCompilationCacheFromLoader() { + await incorporateCompilationCache(); return serializeCompilationCache(); } } diff --git a/packages/playwright-test/src/runner/loadUtils.ts b/packages/playwright-test/src/runner/loadUtils.ts index 1af701d461..55fd9a33b3 100644 --- a/packages/playwright-test/src/runner/loadUtils.ts +++ b/packages/playwright-test/src/runner/loadUtils.ts @@ -95,6 +95,7 @@ export async function loadFileSuites(testRun: TestRun, mode: 'out-of-process' | // Load test files. const fileSuiteByFile = new Map(); const loaderHost = mode === 'out-of-process' ? new OutOfProcessLoaderHost(config) : new InProcessLoaderHost(config); + await loaderHost.start(); for (const file of allTestFiles) { const fileSuite = await loaderHost.loadTestFile(file, errors); fileSuiteByFile.set(file, fileSuite); diff --git a/packages/playwright-test/src/runner/loaderHost.ts b/packages/playwright-test/src/runner/loaderHost.ts index 060a31905f..67aab310e2 100644 --- a/packages/playwright-test/src/runner/loaderHost.ts +++ b/packages/playwright-test/src/runner/loaderHost.ts @@ -23,6 +23,7 @@ import type { FullConfigInternal } from '../common/config'; import { PoolBuilder } from '../common/poolBuilder'; import { addToCompilationCache } from '../transform/compilationCache'; import { setBabelPlugins } from '../transform/transform'; +import { incorporateCompilationCache, initializeEsmLoader } from '../common/esmLoaderHost'; export class InProcessLoaderHost { private _config: FullConfigInternal; @@ -31,10 +32,14 @@ export class InProcessLoaderHost { constructor(config: FullConfigInternal) { this._config = config; this._poolBuilder = PoolBuilder.createForLoader(); + } + + async start() { const babelTransformPlugins: [string, any?][] = []; - for (const plugin of config.plugins) + for (const plugin of this._config.plugins) babelTransformPlugins.push(...plugin.babelPlugins || []); setBabelPlugins(babelTransformPlugins); + await initializeEsmLoader(); } async loadTestFile(file: string, testErrors: TestError[]): Promise { @@ -43,28 +48,32 @@ export class InProcessLoaderHost { return result; } - async stop() {} + async stop() { + await incorporateCompilationCache(); + } } export class OutOfProcessLoaderHost { - private _startPromise: Promise; + private _config: FullConfigInternal; private _processHost: ProcessHost; constructor(config: FullConfigInternal) { + this._config = config; this._processHost = new ProcessHost(require.resolve('../loader/loaderMain.js'), 'loader', {}); - this._startPromise = this._processHost.startRunner(serializeConfig(config), true); + } + + async start() { + await this._processHost.startRunner(serializeConfig(this._config), true); } async loadTestFile(file: string, testErrors: TestError[]): Promise { - await this._startPromise; const result = await this._processHost.sendMessage({ method: 'loadTestFile', params: { file } }) as any; testErrors.push(...result.testErrors); return Suite._deepParse(result.fileSuite); } async stop() { - await this._startPromise; - const result = await this._processHost.sendMessage({ method: 'serializeCompilationCache' }) as any; + const result = await this._processHost.sendMessage({ method: 'getCompilationCacheFromLoader' }) as any; addToCompilationCache(result); await this._processHost.stop(); } diff --git a/packages/playwright-test/src/experimentalLoader.ts b/packages/playwright-test/src/transform/esmLoader.ts similarity index 64% rename from packages/playwright-test/src/experimentalLoader.ts rename to packages/playwright-test/src/transform/esmLoader.ts index b4617d918b..b8e9618442 100644 --- a/packages/playwright-test/src/experimentalLoader.ts +++ b/packages/playwright-test/src/transform/esmLoader.ts @@ -16,8 +16,9 @@ import fs from 'fs'; import url from 'url'; -import { belongsToNodeModules, currentFileDepsCollector } from './transform/compilationCache'; -import { transformHook, resolveHook } from './transform/transform'; +import { addToCompilationCache, belongsToNodeModules, currentFileDepsCollector, serializeCompilationCache, startCollectingFileDeps, stopCollectingFileDeps } from './compilationCache'; +import { transformHook, resolveHook, setBabelPlugins } from './transform'; +import { PortTransport } from './portTransport'; // Node < 18.6: defaultResolve takes 3 arguments. // Node >= 18.6: nextResolve from the chain takes 2 arguments. @@ -54,9 +55,46 @@ async function load(moduleUrl: string, context: { format?: string }, defaultLoad const code = fs.readFileSync(filename, 'utf-8'); const source = transformHook(code, filename, moduleUrl); + + // Flush the source maps to the main thread. + await transport?.send('pushToCompilationCache', { cache: serializeCompilationCache() }); + // Output format is always the same as input format, if it was unknown, we always report modules. - // shortCurcuit is required by Node >= 18.6 to designate no more loaders should be called. + // shortCircuit is required by Node >= 18.6 to designate no more loaders should be called. return { format: context.format || 'module', source, shortCircuit: true }; } -module.exports = { resolve, load }; +let transport: PortTransport | undefined; + +function globalPreload(context: { port: MessagePort }) { + transport = new PortTransport(context.port, async (method, params) => { + if (method === 'setBabelPlugins') { + setBabelPlugins(params.plugins); + return; + } + + if (method === 'addToCompilationCache') { + addToCompilationCache(params.cache); + return; + } + + if (method === 'getCompilationCache') + return { cache: serializeCompilationCache() }; + + if (method === 'startCollectingFileDeps') { + startCollectingFileDeps(); + return; + } + + if (method === 'stopCollectingFileDeps') { + stopCollectingFileDeps(params.file); + return; + } + }); + + return ` + globalThis.__esmLoaderPort = port; + `; +} + +module.exports = { resolve, load, globalPreload }; diff --git a/packages/playwright-test/src/transform/portTransport.ts b/packages/playwright-test/src/transform/portTransport.ts new file mode 100644 index 0000000000..027e6b6e2c --- /dev/null +++ b/packages/playwright-test/src/transform/portTransport.ts @@ -0,0 +1,49 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export class PortTransport { + private _lastId = 0; + private _port: MessagePort; + private _callbacks = new Map void>(); + + constructor(port: MessagePort, handler: (method: string, params: any) => Promise) { + this._port = port; + port.onmessage = async event => { + const message = event.data; + const { id, ackId, method, params, result } = message; + if (id) { + const result = await handler(method, params); + this._port.postMessage({ ackId: id, result }); + return; + } + + if (ackId) { + const callback = this._callbacks.get(ackId); + this._callbacks.delete(ackId); + callback?.(result); + return; + } + }; + } + + async send(method: string, params: any) { + return await new Promise(f => { + const id = ++this._lastId; + this._callbacks.set(id, f); + this._port.postMessage({ id, method, params }); + }); + } +} diff --git a/packages/playwright-test/src/transform/transform.ts b/packages/playwright-test/src/transform/transform.ts index c43be7c3fc..bd0ae9078b 100644 --- a/packages/playwright-test/src/transform/transform.ts +++ b/packages/playwright-test/src/transform/transform.ts @@ -41,6 +41,10 @@ export function setBabelPlugins(plugins: BabelPlugin[]) { babelPlugins = plugins; } +export function getBabelPlugins(): BabelPlugin[] { + return babelPlugins; +} + function validateTsConfig(tsconfig: TsConfigLoaderResult): ParsedTsConfigData | undefined { if (!tsconfig.tsConfigPath || !tsconfig.baseUrl) return; diff --git a/packages/playwright-test/src/util.ts b/packages/playwright-test/src/util.ts index e2d0f14c39..c5d2b6aae3 100644 --- a/packages/playwright-test/src/util.ts +++ b/packages/playwright-test/src/util.ts @@ -300,7 +300,7 @@ function folderIsModule(folder: string): boolean { } export function experimentalLoaderOption() { - return ` --no-warnings --experimental-loader=${url.pathToFileURL(require.resolve('@playwright/test/lib/experimentalLoader')).toString()}`; + return ` --no-warnings --experimental-loader=${url.pathToFileURL(require.resolve('@playwright/test/lib/transform/esmLoader')).toString()}`; } export function envWithoutExperimentalLoaderOptions(): NodeJS.ProcessEnv { diff --git a/packages/playwright-test/src/worker/workerMain.ts b/packages/playwright-test/src/worker/workerMain.ts index 3284dd3cd4..b2fc4938d5 100644 --- a/packages/playwright-test/src/worker/workerMain.ts +++ b/packages/playwright-test/src/worker/workerMain.ts @@ -30,7 +30,6 @@ import { ProcessRunner } from '../common/process'; import { loadTestFile } from '../common/testLoader'; import { buildFileSuiteForProject, filterTestsRemoveEmptySuites } from '../common/suiteUtils'; import { PoolBuilder } from '../common/poolBuilder'; -import { addToCompilationCache } from '../transform/compilationCache'; import type { TestInfoError } from '../../types/test'; const removeFolderAsync = util.promisify(rimraf); @@ -67,7 +66,6 @@ export class WorkerMain extends ProcessRunner { process.env.TEST_WORKER_INDEX = String(params.workerIndex); process.env.TEST_PARALLEL_INDEX = String(params.parallelIndex); setIsWorkerProcess(); - addToCompilationCache(params.config.compilationCache); this._params = params; this._fixtureRunner = new FixtureRunner();