diff --git a/packages/playwright-test/src/experimentalLoader.ts b/packages/playwright-test/src/experimentalLoader.ts
index ca16535a65..d309fb55a2 100644
--- a/packages/playwright-test/src/experimentalLoader.ts
+++ b/packages/playwright-test/src/experimentalLoader.ts
@@ -15,12 +15,8 @@
  */
 
 import fs from 'fs';
-import path from 'path';
-import { tsConfigLoader, TsConfigLoaderResult } from './third_party/tsconfig-loader';
 import { transformHook } from './transform';
 
-const tsConfigCache = new Map<string, TsConfigLoaderResult>();
-
 async function resolve(specifier: string, context: { parentURL: string }, defaultResolve: any) {
   if (specifier.endsWith('.js') || specifier.endsWith('.ts') || specifier.endsWith('.mjs'))
     return defaultResolve(specifier, context, defaultResolve);
@@ -36,17 +32,8 @@ async function resolve(specifier: string, context: { parentURL: string }, defaul
 async function load(url: string, context: any, defaultLoad: any) {
   if (url.endsWith('.ts') || url.endsWith('.tsx')) {
     const filename = url.substring('file://'.length);
-    const cwd = path.dirname(filename);
-    let tsconfig = tsConfigCache.get(cwd);
-    if (!tsconfig) {
-      tsconfig = tsConfigLoader({
-        getEnv: (name: string) => process.env[name],
-        cwd
-      });
-      tsConfigCache.set(cwd, tsconfig);
-    }
     const code = fs.readFileSync(filename, 'utf-8');
-    const source = transformHook(code, filename, tsconfig, true);
+    const source = transformHook(code, filename, true);
     return { format: 'module', source };
   }
   return defaultLoad(url, context, defaultLoad);
diff --git a/packages/playwright-test/src/loader.ts b/packages/playwright-test/src/loader.ts
index b4caa51850..100f9617d7 100644
--- a/packages/playwright-test/src/loader.ts
+++ b/packages/playwright-test/src/loader.ts
@@ -27,12 +27,10 @@ import { ProjectImpl } from './project';
 import { Reporter } from '../types/testReporter';
 import { BuiltInReporter, builtInReporters } from './runner';
 import { isRegExp } from 'playwright-core/lib/utils/utils';
-import { tsConfigLoader, TsConfigLoaderResult } from './third_party/tsconfig-loader';
 
 // To allow multiple loaders in the same process without clearing require cache,
 // we make these maps global.
 const cachedFileSuites = new Map<string, Suite>();
-const cachedTSConfigs = new Map<string, TsConfigLoaderResult>();
 
 export class Loader {
   private _defaultConfig: Config;
@@ -194,18 +192,7 @@ export class Loader {
 
 
   private async _requireOrImport(file: string) {
-    // Respect tsconfig paths.
-    const cwd = path.dirname(file);
-    let tsconfig = cachedTSConfigs.get(cwd);
-    if (!tsconfig) {
-      tsconfig = tsConfigLoader({
-        getEnv: (name: string) => process.env[name],
-        cwd
-      });
-      cachedTSConfigs.set(cwd, tsconfig);
-    }
-
-    const revertBabelRequire = installTransform(tsconfig);
+    const revertBabelRequire = installTransform();
 
     // Figure out if we are importing or requiring.
     let isModule: boolean;
diff --git a/packages/playwright-test/src/transform.ts b/packages/playwright-test/src/transform.ts
index 056bb5bd2c..7a498352a5 100644
--- a/packages/playwright-test/src/transform.ts
+++ b/packages/playwright-test/src/transform.ts
@@ -22,12 +22,20 @@ import * as pirates from 'pirates';
 import * as sourceMapSupport from 'source-map-support';
 import * as url from 'url';
 import type { Location } from './types';
-import { TsConfigLoaderResult } from './third_party/tsconfig-loader';
+import { tsConfigLoader, TsConfigLoaderResult } from './third_party/tsconfig-loader';
 
 const version = 6;
 const cacheDir = process.env.PWTEST_CACHE_DIR || path.join(os.tmpdir(), 'playwright-transform-cache');
 const sourceMaps: Map<string, string> = new Map();
 
+type ParsedTsConfigData = {
+  absoluteBaseUrl: string,
+  singlePath: { [key: string]: string },
+  hash: string,
+  alias: { [key: string]: string | ((s: string[]) => string) },
+};
+const cachedTSConfigs = new Map<string, ParsedTsConfigData | undefined>();
+
 const kStackTraceLimit = 15;
 Error.stackTraceLimit = kStackTraceLimit;
 
@@ -47,9 +55,9 @@ sourceMapSupport.install({
   }
 });
 
-function calculateCachePath(tsconfig: TsConfigLoaderResult, content: string, filePath: string): string {
+function calculateCachePath(tsconfigData: ParsedTsConfigData | undefined, content: string, filePath: string): string {
   const hash = crypto.createHash('sha1')
-      .update(tsconfig.serialized || '')
+      .update(tsconfigData?.hash || '')
       .update(process.env.PW_EXPERIMENTAL_TS_ESM ? 'esm' : 'no_esm')
       .update(content)
       .update(filePath)
@@ -59,10 +67,64 @@ function calculateCachePath(tsconfig: TsConfigLoaderResult, content: string, fil
   return path.join(cacheDir, hash[0] + hash[1], fileName);
 }
 
-export function transformHook(code: string, filename: string, tsconfig: TsConfigLoaderResult, isModule = false): string {
+function validateTsConfig(tsconfig: TsConfigLoaderResult): ParsedTsConfigData | undefined {
+  if (!tsconfig.tsConfigPath || !tsconfig.paths || !tsconfig.baseUrl)
+    return;
+
+  const paths = tsconfig.paths;
+  // Path that only contains "*", ".", "/" and "\" is too ambiguous.
+  const ambiguousPath = Object.keys(paths).find(key => key.match(/^[*./\\]+$/));
+  if (ambiguousPath)
+    return;
+  const multiplePath = Object.keys(paths).find(key => paths[key].length > 1);
+  if (multiplePath)
+    return;
+  // Only leave a single path mapping.
+  const singlePath = Object.fromEntries(Object.entries(paths).map(([key, values]) => ([key, values[0]])));
+  // Make 'baseUrl' absolute, because it is relative to the tsconfig.json, not to cwd.
+  const absoluteBaseUrl = path.resolve(path.dirname(tsconfig.tsConfigPath), tsconfig.baseUrl);
+  const hash = JSON.stringify({ absoluteBaseUrl, singlePath });
+
+  const alias: ParsedTsConfigData['alias'] = {};
+  for (const [key, value] of Object.entries(singlePath)) {
+    const regexKey = '^' + key.replace('*', '.*');
+    alias[regexKey] = ([name]) => {
+      let relative: string;
+      if (key.endsWith('/*'))
+        relative = value.substring(0, value.length - 1) + name.substring(key.length - 1);
+      else
+        relative = value;
+      relative = relative.replace(/\//g, path.sep);
+      return path.resolve(absoluteBaseUrl, relative);
+    };
+  }
+
+  return {
+    absoluteBaseUrl,
+    singlePath,
+    hash,
+    alias,
+  };
+}
+
+function loadAndValidateTsconfigForFile(file: string): ParsedTsConfigData | undefined {
+  const cwd = path.dirname(file);
+  if (!cachedTSConfigs.has(cwd)) {
+    const loaded = tsConfigLoader({
+      getEnv: (name: string) => process.env[name],
+      cwd
+    });
+    cachedTSConfigs.set(cwd, validateTsConfig(loaded));
+  }
+  return cachedTSConfigs.get(cwd);
+}
+
+export function transformHook(code: string, filename: string, isModule = false): string {
   if (isComponentImport(filename))
     return componentStub();
-  const cachePath = calculateCachePath(tsconfig, code, filename);
+
+  const tsconfigData = loadAndValidateTsconfigForFile(filename);
+  const cachePath = calculateCachePath(tsconfigData, code, filename);
   const codePath = cachePath + '.js';
   const sourceMapPath = cachePath + '.map';
   sourceMaps.set(filename, sourceMapPath);
@@ -73,30 +135,6 @@ export function transformHook(code: string, filename: string, tsconfig: TsConfig
   process.env.BROWSERSLIST_IGNORE_OLD_DATA = 'true';
   const babel: typeof import('@babel/core') = require('@babel/core');
 
-  const hasBaseUrl = !!tsconfig.baseUrl;
-  const extensions = ['', '.js', '.ts', '.mjs', ...(process.env.PW_COMPONENT_TESTING ? ['.tsx', '.jsx'] : [])];  const alias: { [key: string]: string | ((s: string[]) => string) } = {};
-  for (const [key, values] of Object.entries(tsconfig.paths || { '*': '*' })) {
-    const regexKey = '^' + key.replace('*', '.*');
-    alias[regexKey] = ([name]) => {
-      for (const value of values) {
-        let relative: string;
-        if (key === '*' && value === '*')
-          relative = name;
-        else if (key.endsWith('/*'))
-          relative = value.substring(0, value.length - 1) + name.substring(key.length - 1);
-        else
-          relative = value;
-        relative = relative.replace(/\//g, path.sep);
-        const result = path.resolve(tsconfig.baseUrl || '', relative);
-        for (const extension of extensions) {
-          if (fs.existsSync(result + extension))
-            return result + extension;
-        }
-      }
-      return name;
-    };
-  }
-
   const plugins = [
     [require.resolve('@babel/plugin-proposal-class-properties')],
     [require.resolve('@babel/plugin-proposal-numeric-separator')],
@@ -110,10 +148,13 @@ export function transformHook(code: string, filename: string, tsconfig: TsConfig
     [require.resolve('@babel/plugin-proposal-export-namespace-from')],
   ] as any;
 
-  if (hasBaseUrl) {
+  if (tsconfigData) {
     plugins.push([require.resolve('babel-plugin-module-resolver'), {
       root: ['./'],
-      alias
+      alias: tsconfigData.alias,
+      // Silences warning 'Could not resovle ...' that we trigger because we resolve
+      // into 'foo/bar', and not 'foo/bar.ts'.
+      loglevel: 'silent',
     }]);
   }
 
@@ -143,16 +184,13 @@ export function transformHook(code: string, filename: string, tsconfig: TsConfig
     fs.mkdirSync(path.dirname(cachePath), { recursive: true });
     if (result.map)
       fs.writeFileSync(sourceMapPath, JSON.stringify(result.map), 'utf8');
-    // Compiled files with base URL depend on the FS state during compilation,
-    // never cache them.
-    if (!hasBaseUrl)
-      fs.writeFileSync(codePath, result.code, 'utf8');
+    fs.writeFileSync(codePath, result.code, 'utf8');
   }
   return result.code || '';
 }
 
-export function installTransform(tsconfig: TsConfigLoaderResult): () => void {
-  return pirates.addHook((code: string, filename: string) => transformHook(code, filename, tsconfig), { exts: ['.ts', '.tsx'] });
+export function installTransform(): () => void {
+  return pirates.addHook((code: string, filename: string) => transformHook(code, filename), { exts: ['.ts', '.tsx'] });
 }
 
 export function wrapFunctionWithLocation<A extends any[], R>(func: (location: Location, ...args: A) => R): (...args: A) => R {
diff --git a/tests/browsertype-connect.spec.ts b/tests/browsertype-connect.spec.ts
index affe2b57ed..bd36412ff1 100644
--- a/tests/browsertype-connect.spec.ts
+++ b/tests/browsertype-connect.spec.ts
@@ -17,7 +17,7 @@
 
 import fs from 'fs';
 import * as path from 'path';
-import { getUserAgent } from 'playwright-core/lib/utils/utils';
+import { getUserAgent } from '../packages/playwright-core/lib/utils/utils';
 import WebSocket from 'ws';
 import { expect, playwrightTest as test } from './config/browserTest';
 import { parseTrace, suppressCertificateWarning } from './config/utils';
diff --git a/tests/chromium/chromium.spec.ts b/tests/chromium/chromium.spec.ts
index 5839942bc3..2d7f12ada1 100644
--- a/tests/chromium/chromium.spec.ts
+++ b/tests/chromium/chromium.spec.ts
@@ -19,7 +19,7 @@ import { contextTest as test, expect } from '../config/browserTest';
 import { playwrightTest } from '../config/browserTest';
 import http from 'http';
 import fs from 'fs';
-import { getUserAgent } from 'playwright-core/lib/utils/utils';
+import { getUserAgent } from '../../packages/playwright-core/lib/utils/utils';
 import { suppressCertificateWarning } from '../config/utils';
 
 test('should create a worker from a service worker', async ({ page, server }) => {
diff --git a/tests/config/browserTest.ts b/tests/config/browserTest.ts
index 8c2ce3090c..aedd2b1bcc 100644
--- a/tests/config/browserTest.ts
+++ b/tests/config/browserTest.ts
@@ -19,7 +19,7 @@ import * as os from 'os';
 import { PageTestFixtures, PageWorkerFixtures } from '../page/pageTestApi';
 import * as path from 'path';
 import type { BrowserContext, BrowserContextOptions, BrowserType, Page } from 'playwright-core';
-import { removeFolders } from 'playwright-core/lib/utils/utils';
+import { removeFolders } from '../../packages/playwright-core/lib/utils/utils';
 import { baseTest } from './baseTest';
 import { RemoteServer, RemoteServerOptions } from './remoteServer';
 
diff --git a/tests/global-fetch.spec.ts b/tests/global-fetch.spec.ts
index 1ca5e138cf..3d8da215cb 100644
--- a/tests/global-fetch.spec.ts
+++ b/tests/global-fetch.spec.ts
@@ -17,7 +17,7 @@
 import http from 'http';
 import os from 'os';
 import * as util from 'util';
-import { getPlaywrightVersion } from 'playwright-core/lib/utils/utils';
+import { getPlaywrightVersion } from '../packages/playwright-core/lib/utils/utils';
 import { expect, playwrightTest as it } from './config/browserTest';
 
 it.skip(({ mode }) => mode !== 'default');
diff --git a/tests/har.spec.ts b/tests/har.spec.ts
index f57c12598f..6b0d4b2733 100644
--- a/tests/har.spec.ts
+++ b/tests/har.spec.ts
@@ -21,7 +21,7 @@ import fs from 'fs';
 import http2 from 'http2';
 import type { BrowserContext, BrowserContextOptions } from 'playwright-core';
 import type { AddressInfo } from 'net';
-import type { Log } from 'playwright-core/lib/server/supplements/har/har';
+import type { Log } from '../packages/playwright-core/src/server/supplements/har/har';
 
 async function pageWithHar(contextFactory: (options?: BrowserContextOptions) => Promise<BrowserContext>, testInfo: any, outputPath: string = 'test.har') {
   const harPath = testInfo.outputPath(outputPath);
diff --git a/tests/inspector/inspectorTest.ts b/tests/inspector/inspectorTest.ts
index 4de20d9019..1bb6e48f45 100644
--- a/tests/inspector/inspectorTest.ts
+++ b/tests/inspector/inspectorTest.ts
@@ -17,7 +17,7 @@
 import { contextTest } from '../config/browserTest';
 import type { Page } from 'playwright-core';
 import * as path from 'path';
-import type { Source } from 'playwright-core/lib/server/supplements/recorder/recorderTypes';
+import type { Source } from '../../packages/playwright-core/src/server/supplements/recorder/recorderTypes';
 import { CommonFixtures, TestChildProcess } from '../config/commonFixtures';
 export { expect } from '@playwright/test';
 
diff --git a/tests/playwright-test/playwright.spec.ts b/tests/playwright-test/playwright.spec.ts
index cf7cb03a30..7cd266aced 100644
--- a/tests/playwright-test/playwright.spec.ts
+++ b/tests/playwright-test/playwright.spec.ts
@@ -18,7 +18,7 @@ import { test, expect, stripAscii } from './playwright-test-fixtures';
 import fs from 'fs';
 import path from 'path';
 import { spawnSync } from 'child_process';
-import { registry } from 'playwright-core/lib/utils/registry';
+import { registry } from '../../packages/playwright-core/lib/utils/registry';
 
 const ffmpeg = registry.findExecutable('ffmpeg')!.executablePath();
 
diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts
index 146c7f7fa8..b88c489097 100644
--- a/tests/playwright-test/reporter-html.spec.ts
+++ b/tests/playwright-test/reporter-html.spec.ts
@@ -15,7 +15,7 @@
  */
 
 import { test as baseTest, expect } from './playwright-test-fixtures';
-import { HttpServer } from 'playwright-core/lib/utils/httpServer';
+import { HttpServer } from '../../packages/playwright-core/lib/utils/httpServer';
 import { startHtmlReportServer } from '../../packages/playwright-test/lib/reporters/html';
 
 const test = baseTest.extend<{ showReport: () => Promise<void> }>({
diff --git a/tests/playwright-test/resolver.spec.ts b/tests/playwright-test/resolver.spec.ts
index ef2b508070..236689a2ef 100644
--- a/tests/playwright-test/resolver.spec.ts
+++ b/tests/playwright-test/resolver.spec.ts
@@ -17,6 +17,8 @@
 import { test, expect } from './playwright-test-fixtures';
 
 test('should respect path resolver', async ({ runInlineTest }) => {
+  test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/11656' });
+
   const result = await runInlineTest({
     'playwright.config.ts': `
       export default {
@@ -31,7 +33,7 @@ test('should respect path resolver', async ({ runInlineTest }) => {
         "baseUrl": ".",
         "paths": {
           "util/*": ["./foo/bar/util/*"],
-          "util2/*": ["./non-existent/*", "./foo/bar/util/*"],
+          "util2/*": ["./foo/bar/util/*"],
           "util3": ["./foo/bar/util/b"],
         },
       },
@@ -60,10 +62,36 @@ test('should respect path resolver', async ({ runInlineTest }) => {
     'foo/bar/util/b.ts': `
       export const foo: string = 'foo';
     `,
+    'helper.ts': `
+      export { foo } from 'util3';
+    `,
+    'dir/tsconfig.json': `{
+      "compilerOptions": {
+        "target": "ES2019",
+        "module": "commonjs",
+        "lib": ["esnext", "dom", "DOM.Iterable"],
+        "baseUrl": ".",
+        "paths": {
+          "parent-util/*": ["../foo/bar/util/*"],
+        },
+      },
+    }`,
+    'dir/inner.spec.ts': `
+      // This import should pick up <root>/dir/tsconfig
+      import { foo } from 'parent-util/b';
+      // This import should pick up <root>/tsconfig through the helper
+      import { foo as foo2 } from '../helper';
+      const { test } = pwt;
+      test('test', ({}, testInfo) => {
+        expect(testInfo.project.name).toBe(foo);
+        expect(testInfo.project.name).toBe(foo2);
+      });
+    `,
   });
 
-  expect(result.passed).toBe(3);
+  expect(result.passed).toBe(4);
   expect(result.exitCode).toBe(0);
+  expect(result.output).not.toContain(`Could not`);
 });
 
 test('should respect baseurl', async ({ runInlineTest }) => {
@@ -108,13 +136,8 @@ test('should respect baseurl', async ({ runInlineTest }) => {
   expect(result.exitCode).toBe(0);
 });
 
-test('should respect baseurl w/o paths', async ({ runInlineTest }) => {
+test('should ignore for baseurl w/o paths', async ({ runInlineTest }) => {
   const result = await runInlineTest({
-    'playwright.config.ts': `
-      export default {
-        projects: [{name: 'foo'}],
-      };
-    `,
     'tsconfig.json': `{
       "compilerOptions": {
         "target": "ES2019",
@@ -125,19 +148,60 @@ test('should respect baseurl w/o paths', async ({ runInlineTest }) => {
     }`,
     'a.test.ts': `
       import { foo } from 'foo/b';
-      const { test } = pwt;
-      test('test', ({}, testInfo) => {
-        expect(testInfo.project.name).toBe(foo);
-      });
     `,
     'foo/b.ts': `
-      export const foo: string = 'foo';
+      export const foo = 1;
     `,
   });
 
-  expect(result.output).not.toContain('Could not');
-  expect(result.passed).toBe(1);
-  expect(result.exitCode).toBe(0);
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain(`Cannot find module 'foo/b'`);
+});
+
+test('should ignore tsconfig with ambiguous paths', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'tsconfig.json': `{
+      "compilerOptions": {
+        "target": "ES2019",
+        "module": "commonjs",
+        "lib": ["esnext", "dom", "DOM.Iterable"],
+        "baseUrl": "./",
+        "paths": { "*/*": ["*/*"] }
+      },
+    }`,
+    'a.test.ts': `
+      import { foo } from 'foo/b';
+    `,
+    'foo/b.ts': `
+      export const foo = 1;
+    `,
+  });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain(`Cannot find module 'foo/b'`);
+});
+
+test('should ignore tsconfig with multi-value paths', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'tsconfig.json': `{
+      "compilerOptions": {
+        "target": "ES2019",
+        "module": "commonjs",
+        "lib": ["esnext", "dom", "DOM.Iterable"],
+        "baseUrl": "./",
+        "paths": { "foo/*": ["./foo/*", "./bar/*"] }
+      },
+    }`,
+    'a.test.ts': `
+      import { foo } from 'foo/b';
+    `,
+    'foo/b.ts': `
+      export const foo = 1;
+    `,
+  });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain(`Cannot find module 'foo/b'`);
 });
 
 test('should respect path resolver in experimental mode', async ({ runInlineTest }) => {
diff --git a/tests/tsconfig.json b/tests/tsconfig.json
index 9020456d30..47b0c03736 100644
--- a/tests/tsconfig.json
+++ b/tests/tsconfig.json
@@ -9,10 +9,6 @@
         "strictBindCallApply": true,
         "allowSyntheticDefaultImports": true,
         "useUnknownInCatchVariables": false,
-        "baseUrl": ".",
-        "paths": {
-          "playwright-core/lib/*": ["../packages/playwright-core/src/*"]
-        },
     },
     "include": ["**/*.spec.js", "**/*.ts", "index.d.ts"],
     "exclude": ["playwright-test/"]
diff --git a/tests/video.spec.ts b/tests/video.spec.ts
index 132e773a5c..9446f4df90 100644
--- a/tests/video.spec.ts
+++ b/tests/video.spec.ts
@@ -19,7 +19,7 @@ import fs from 'fs';
 import path from 'path';
 import { spawnSync } from 'child_process';
 import { PNG } from 'pngjs';
-import { registry } from 'playwright-core/lib/utils/registry';
+import { registry } from '../packages/playwright-core/lib/utils/registry';
 
 const ffmpeg = registry.findExecutable('ffmpeg')!.executablePath('javascript');
 
diff --git a/tsconfig.json b/tsconfig.json
index eb26d087a2..18af9593ce 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -5,7 +5,6 @@
     "lib": ["esnext", "dom", "DOM.Iterable"],
     "baseUrl": ".",
     "paths": {
-      "*": ["./packages/*/"],
       "playwright-core/lib/*": ["./packages/playwright-core/src/*"]
     },
     "esModuleInterop": true,