Merge branch 'main' into sharding-algorithm

.github/workflows/tests_bidi.yml

@@ -26,8 +26,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # TODO: add Firefox
+        channel: [bidi-chromium, bidi-firefox-beta]
-        channel: [bidi-chrome-stable]
     steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-node@v4
@@ -38,5 +37,8 @@ jobs:
         PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: '1'
     - run: npm run build
     - run: npx playwright install --with-deps chromium
+      if: matrix.channel == 'bidi-chromium'
+    - run: npx -y @puppeteer/browsers install firefox@beta
+      if: matrix.channel == 'bidi-firefox-beta'
     - name: Run tests
       run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run biditest -- --project=${{ matrix.channel }}*

docs/src/test-api/class-testinfoerror-matcherresult.md

@@ -1,35 +0,0 @@
-# class: TestInfoErrorMatcherResult
-* since: v1.48
-* langs: js
-
-Matcher-specific details for the error thrown during the `expect` call.
-
-## property: TestInfoErrorMatcherResult.actual
-* since: v1.48
-- type: ?<[unknown]>
-
-Actual value.
-
-## property: TestInfoErrorMatcherResult.expected
-* since: v1.48
-- type: ?<[unknown]>
-
-Expected value.
-
-## property: TestInfoErrorMatcherResult.name
-* since: v1.48
-- type: ?<[string]>
-
-Matcher name.
-
-## property: TestInfoErrorMatcherResult.pass
-* since: v1.48
-- type: <[string]>
-
-Whether the matcher passed.
-
-## property: TestInfoErrorMatcherResult.timeout
-* since: v1.48
-- type: ?<[int]>
-
-Timeout that was used during matching.

docs/src/test-api/class-testinfoerror.md

@@ -4,12 +4,6 @@
 
 Information about an error thrown during test execution.
 
-## property: TestInfoError.matcherResult
-* since: v1.48
-- type: ?<[TestInfoErrorMatcherResult]>
-
-Matcher result details.
-
 ## property: TestInfoError.message
 * since: v1.10
 - type: ?<[string]>

docs/src/test-reporter-api/class-testerror-matcherresult.md

@@ -1,35 +0,0 @@
-# class: TestErrorMatcherResult
-* since: v1.48
-* langs: js
-
-Matcher-specific details for the error thrown during the `expect` call.
-
-## property: TestErrorMatcherResult.actual
-* since: v1.48
-- type: ?<[unknown]>
-
-Actual value.
-
-## property: TestErrorMatcherResult.expected
-* since: v1.48
-- type: ?<[unknown]>
-
-Expected value.
-
-## property: TestErrorMatcherResult.name
-* since: v1.48
-- type: ?<[string]>
-
-Matcher name.
-
-## property: TestErrorMatcherResult.pass
-* since: v1.48
-- type: <[string]>
-
-Whether the matcher passed.
-
-## property: TestErrorMatcherResult.timeout
-* since: v1.48
-- type: ?<[int]>
-
-Timeout that was used during matching.

docs/src/test-reporter-api/class-testerror.md

@@ -4,12 +4,6 @@
 
 Information about an error thrown during test execution.
 
-## property: TestError.matcherResult
-* since: v1.48
-- type: ?<[TestErrorMatcherResult]>
-
-Matcher result details.
-
 ## property: TestError.message
 * since: v1.10
 - type: ?<[string]>

packages/playwright-core/src/server/registry/index.ts

@@ -354,7 +354,7 @@ function readDescriptors(browsersJSON: BrowsersJSON) {
 
 export type BrowserName = 'chromium' | 'firefox' | 'webkit' | 'bidi';
 type InternalTool = 'ffmpeg' | 'firefox-beta' | 'chromium-tip-of-tree' | 'android';
-type BidiChannel = 'bidi-firefox-stable' | 'bidi-chrome-canary' | 'bidi-chrome-stable';
+type BidiChannel = 'bidi-firefox-stable' | 'bidi-firefox-beta' | 'bidi-firefox-nightly' | 'bidi-chrome-canary' | 'bidi-chrome-stable' | 'bidi-chromium';
 type ChromiumChannel = 'chrome' | 'chrome-beta' | 'chrome-dev' | 'chrome-canary' | 'msedge' | 'msedge-beta' | 'msedge-dev' | 'msedge-canary';
 const allDownloadable = ['chromium', 'firefox', 'webkit', 'ffmpeg', 'firefox-beta', 'chromium-tip-of-tree'];
 
@@ -525,11 +525,22 @@ export class Registry {
       'win32': `\\Microsoft\\Edge SxS\\Application\\msedge.exe`,
     }));
 
-    this._executables.push(this._createBidiChannel('bidi-firefox-stable', {
+    this._executables.push(this._createBidiFirefoxChannel('bidi-firefox-stable', {
-      'linux': '/usr/bin/firefox',
+      'linux': '/firefox/firefox',
-      'darwin': '/Applications/Firefox.app/Contents/MacOS/firefox',
+      'darwin': '/Firefox.app/Contents/MacOS/firefox',
-      'win32': '\\Mozilla Firefox\\firefox.exe',
+      'win32': '\\core\\firefox.exe',
     }));
+    this._executables.push(this._createBidiFirefoxChannel('bidi-firefox-beta', {
+      'linux': '/firefox/firefox',
+      'darwin': '/Firefox.app/Contents/MacOS/firefox',
+      'win32': '\\core\\firefox.exe',
+    }));
+    this._executables.push(this._createBidiFirefoxChannel('bidi-firefox-nightly', {
+      'linux': '/firefox/firefox',
+      'darwin': '/Firefox Nightly.app/Contents/MacOS/firefox',
+      'win32': '\\firefox\\firefox.exe',
+    }));
+
     this._executables.push(this._createBidiChannel('bidi-chrome-stable', {
       'linux': '/opt/google/chrome/chrome',
       'darwin': '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
@@ -540,6 +551,21 @@ export class Registry {
       'darwin': '/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary',
       'win32': `\\Google\\Chrome SxS\\Application\\chrome.exe`,
     }));
+    this._executables.push({
+      type: 'browser',
+      name: 'bidi-chromium',
+      browserName: 'bidi',
+      directory: chromium.dir,
+      executablePath: () => chromiumExecutable,
+      executablePathOrDie: (sdkLanguage: string) => executablePathOrDie('chromium', chromiumExecutable, chromium.installByDefault, sdkLanguage),
+      installType: 'download-on-demand',
+      _validateHostRequirements: (sdkLanguage: string) => this._validateHostRequirements(sdkLanguage, 'chromium', chromium.dir, ['chrome-linux'], [], ['chrome-win']),
+      downloadURLs: this._downloadURLs(chromium),
+      browserVersion: chromium.browserVersion,
+      _install: () => this._downloadExecutable(chromium, chromiumExecutable),
+      _dependencyGroup: 'chromium',
+      _isHermeticInstallation: true,
+    });
 
     const firefox = descriptors.find(d => d.name === 'firefox')!;
     const firefoxExecutable = findExecutablePath(firefox.dir, 'firefox');
@@ -691,6 +717,48 @@ export class Registry {
     };
   }
 
+  private _createBidiFirefoxChannel(name: BidiChannel, lookAt: Record<'linux' | 'darwin' | 'win32', string>, install?: () => Promise<void>): ExecutableImpl {
+    const executablePath = (sdkLanguage: string, shouldThrow: boolean) => {
+      const suffix = lookAt[process.platform as 'linux' | 'darwin' | 'win32'];
+      if (!suffix) {
+        if (shouldThrow)
+          throw new Error(`Firefox distribution '${name}' is not supported on ${process.platform}`);
+        return undefined;
+      }
+      const folder = path.resolve('firefox');
+      let channelName = 'stable';
+      if (name.includes('beta'))
+        channelName = 'beta';
+      else if (name.includes('nightly'))
+        channelName = 'nightly';
+      const installedVersions = fs.readdirSync(folder);
+      const found = installedVersions.filter(e => e.includes(channelName));
+      if (found.length === 1)
+        return path.join(folder, found[0], suffix);
+      if (found.length > 1) {
+        if (shouldThrow)
+          throw new Error(`Multiple Firefox installations found for channel '${name}': ${found.join(', ')}`);
+        else
+          return undefined;
+      }
+      if (shouldThrow)
+        throw new Error(`Cannot find Firefox installation for channel '${name}' under ${folder}`);
+      return undefined;
+    };
+    return {
+      type: 'channel',
+      name,
+      browserName: 'bidi',
+      directory: undefined,
+      executablePath: (sdkLanguage: string) => executablePath(sdkLanguage, false),
+      executablePathOrDie: (sdkLanguage: string) => executablePath(sdkLanguage, true)!,
+      installType: 'none',
+      _validateHostRequirements: () => Promise.resolve(),
+      _isHermeticInstallation: true,
+      _install: install,
+    };
+  }
+
   private _createBidiChannel(name: BidiChannel, lookAt: Record<'linux' | 'darwin' | 'win32', string>, install?: () => Promise<void>): ExecutableImpl {
     const executablePath = (sdkLanguage: string, shouldThrow: boolean) => {
       const suffix = lookAt[process.platform as 'linux' | 'darwin' | 'win32'];

packages/playwright/src/reporters/base.ts

@@ -32,7 +32,6 @@ type Annotation = {
 type ErrorDetails = {
   message: string;
   location?: Location;
-  matcherResult?: TestError['matcherResult'];
 };
 
 type TestSummary = {
@@ -400,7 +399,6 @@ export function formatResultFailure(test: TestCase, result: TestResult, initialI
     errorDetails.push({
       message: indent(formattedError.message, initialIndent),
       location: formattedError.location,
-      matcherResult: formattedError.matcherResult
     });
   }
   return errorDetails;
@@ -491,7 +489,6 @@ export function formatError(error: TestError, highlightCode: boolean): ErrorDeta
   return {
     location,
     message: tokens.join('\n'),
-    matcherResult: error.matcherResult
   };
 }
 

packages/playwright/src/third_party/tsconfig-loader.ts

@@ -53,9 +53,13 @@ export interface LoadedTsConfig {
 }
 
 export function loadTsConfig(configPath: string): LoadedTsConfig[] {
-  const references: LoadedTsConfig[] = [];
+  try {
-  const config = innerLoadTsConfig(configPath, references);
+    const references: LoadedTsConfig[] = [];
-  return [config, ...references];
+    const config = innerLoadTsConfig(configPath, references);
+    return [config, ...references];
+  } catch (e) {
+    throw new Error(`Failed to load tsconfig file at ${configPath}:\n${e.message}`);
+  }
 }
 
 function resolveConfigFile(baseConfigFile: string, referencedConfigFile: string) {

packages/playwright/src/transform/transform.ts

@@ -24,7 +24,7 @@ import type { LoadedTsConfig } from '../third_party/tsconfig-loader';
 import { loadTsConfig } from '../third_party/tsconfig-loader';
 import Module from 'module';
 import type { BabelPlugin, BabelTransformFunction } from './babelBundle';
-import { createFileMatcher, fileIsModule, resolveImportSpecifierExtension } from '../util';
+import { createFileMatcher, fileIsModule, resolveImportSpecifierAfterMapping } from '../util';
 import type { Matcher } from '../util';
 import { getFromCompilationCache, currentFileDepsCollector, belongsToNodeModules, installSourceMapSupport } from './compilationCache';
 
@@ -136,8 +136,13 @@ export function resolveHook(filename: string, specifier: string): string | undef
     return;
 
   if (isRelativeSpecifier(specifier))
-    return resolveImportSpecifierExtension(path.resolve(path.dirname(filename), specifier));
+    return resolveImportSpecifierAfterMapping(path.resolve(path.dirname(filename), specifier), false);
 
+  /**
+   * TypeScript discourages path-mapping into node_modules:
+   * https://www.typescriptlang.org/docs/handbook/modules/reference.html#paths-should-not-point-to-monorepo-packages-or-node_modules-packages
+   * However, if path-mapping doesn't yield a result, TypeScript falls back to the default resolution through node_modules.
+   */
   const isTypeScript = filename.endsWith('.ts') || filename.endsWith('.tsx');
   const tsconfigs = loadAndValidateTsconfigsForFile(filename);
   for (const tsconfig of tsconfigs) {
@@ -179,7 +184,7 @@ export function resolveHook(filename: string, specifier: string): string | undef
         if (value.includes('*'))
           candidate = candidate.replace('*', matchedPartOfSpecifier);
         candidate = path.resolve(tsconfig.pathsBase!, candidate);
-        const existing = resolveImportSpecifierExtension(candidate);
+        const existing = resolveImportSpecifierAfterMapping(candidate, true);
         if (existing) {
           longestPrefixLength = keyPrefix.length;
           pathMatchedByLongestPrefix = existing;
@@ -193,7 +198,7 @@ export function resolveHook(filename: string, specifier: string): string | undef
   if (path.isAbsolute(specifier)) {
     // Handle absolute file paths like `import '/path/to/file'`
     // Do not handle module imports like `import 'fs'`
-    return resolveImportSpecifierExtension(specifier);
+    return resolveImportSpecifierAfterMapping(specifier, false);
   }
 }
 

packages/playwright/src/util.ts

@@ -63,20 +63,8 @@ export function filteredStackTrace(rawStack: RawStack): StackFrame[] {
 }
 
 export function serializeError(error: Error | any): TestInfoError {
-  if (error instanceof Error) {
+  if (error instanceof Error)
-    const result: TestInfoError = filterStackTrace(error);
+    return filterStackTrace(error);
-    if ('matcherResult' in error && error.matcherResult) {
-      const matcherResult = (error.matcherResult as TestInfoError['matcherResult'])!;
-      result.matcherResult = {
-        pass: matcherResult.pass,
-        name: matcherResult.name,
-        expected: matcherResult.expected,
-        actual: matcherResult.actual,
-        timeout: matcherResult.timeout,
-      };
-    }
-    return result;
-  }
   return {
     value: util.inspect(error)
   };
@@ -307,8 +295,23 @@ function folderIsModule(folder: string): boolean {
   return require(packageJsonPath).type === 'module';
 }
 
-// This follows the --moduleResolution=bundler strategy from tsc.
+const packageJsonMainFieldCache = new Map<string, string | undefined>();
-// https://devblogs.microsoft.com/typescript/announcing-typescript-5-0-beta/#moduleresolution-bundler
+
+function getMainFieldFromPackageJson(packageJsonPath: string) {
+  if (!packageJsonMainFieldCache.has(packageJsonPath)) {
+    let mainField: string | undefined;
+    try {
+      mainField = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8')).main;
+    } catch {
+    }
+    packageJsonMainFieldCache.set(packageJsonPath, mainField);
+  }
+  return packageJsonMainFieldCache.get(packageJsonPath);
+}
+
+// This method performs "file extension subsitution" to find the ts, js or similar source file
+// based on the import specifier, which might or might not have an extension. See TypeScript docs:
+// https://www.typescriptlang.org/docs/handbook/modules/reference.html#file-extension-substitution.
 const kExtLookups = new Map([
   ['.js', ['.jsx', '.ts', '.tsx']],
   ['.jsx', ['.tsx']],
@@ -316,7 +319,7 @@ const kExtLookups = new Map([
   ['.mjs', ['.mts']],
   ['', ['.js', '.ts', '.jsx', '.tsx', '.cjs', '.mjs', '.cts', '.mts']],
 ]);
-export function resolveImportSpecifierExtension(resolved: string): string | undefined {
+function resolveImportSpecifierExtension(resolved: string): string | undefined {
   if (fileExists(resolved))
     return resolved;
 
@@ -330,13 +333,45 @@ export function resolveImportSpecifierExtension(resolved: string): string | unde
     }
     break;  // Do not try '' when a more specific extension like '.jsx' matched.
   }
+}
+
+// This method resolves directory imports and performs "file extension subsitution".
+// It is intended to be called after the path mapping resolution.
+//
+// Directory imports follow the --moduleResolution=bundler strategy from tsc.
+// https://www.typescriptlang.org/docs/handbook/modules/reference.html#directory-modules-index-file-resolution
+// https://www.typescriptlang.org/docs/handbook/modules/reference.html#bundler
+//
+// See also Node.js "folder as module" behavior:
+// https://nodejs.org/dist/latest-v20.x/docs/api/modules.html#folders-as-modules.
+export function resolveImportSpecifierAfterMapping(resolved: string, afterPathMapping: boolean): string | undefined {
+  const resolvedFile = resolveImportSpecifierExtension(resolved);
+  if (resolvedFile)
+    return resolvedFile;
 
   if (dirExists(resolved)) {
+    const packageJsonPath = path.join(resolved, 'package.json');
+
+    if (afterPathMapping) {
+      // Most notably, the module resolution algorithm is not performed after the path mapping.
+      // This means no node_modules lookup or package.json#exports.
+      //
+      // Only the "folder as module" Node.js behavior is respected:
+      //  - consult `package.json#main`;
+      //  - look for `index.js` or similar.
+      const mainField = getMainFieldFromPackageJson(packageJsonPath);
+      const mainFieldResolved = mainField ? resolveImportSpecifierExtension(path.resolve(resolved, mainField)) : undefined;
+      return mainFieldResolved || resolveImportSpecifierExtension(path.join(resolved, 'index'));
+    }
+
     // If we import a package, let Node.js figure out the correct import based on package.json.
-    if (fileExists(path.join(resolved, 'package.json')))
+    // This also covers the "main" field for "folder as module".
+    if (fileExists(packageJsonPath))
       return resolved;
 
-    // Otherwise, try to find a corresponding index file.
+    // Implement the "folder as module" Node.js behavior.
+    // Note that we do not delegate to Node.js, because we support this for ESM as well,
+    // following the TypeScript "bundler" mode.
     const dirImport = path.join(resolved, 'index');
     return resolveImportSpecifierExtension(dirImport);
   }

packages/playwright/types/test.d.ts

@@ -8238,45 +8238,10 @@ export interface TestInfo {
   workerIndex: number;
 }
 
-/**
- * Matcher-specific details for the error thrown during the `expect` call.
- */
-export interface TestInfoErrorMatcherResult {
-  /**
-   * Actual value.
-   */
-  actual?: unknown;
-
-  /**
-   * Expected value.
-   */
-  expected?: unknown;
-
-  /**
-   * Matcher name.
-   */
-  name?: string;
-
-  /**
-   * Whether the matcher passed.
-   */
-  pass: string;
-
-  /**
-   * Timeout that was used during matching.
-   */
-  timeout?: number;
-}
-
 /**
  * Information about an error thrown during test execution.
  */
 export interface TestInfoError {
-  /**
-   * Matcher result details.
-   */
-  matcherResult?: TestInfoErrorMatcherResult;
-
   /**
    * Error message. Set when [Error] (or its subclass) has been thrown.
    */

packages/playwright/types/testReporter.d.ts

@@ -284,7 +284,6 @@ export interface JSONReportTest {
 export interface JSONReportError {
   message: string;
   location?: Location;
-  matcherResult?: TestErrorMatcherResult;
 }
 
 export interface JSONReportTestResult {
@@ -568,36 +567,6 @@ export interface TestCase {
   type: "test";
 }
 
-/**
- * Matcher-specific details for the error thrown during the `expect` call.
- */
-export interface TestErrorMatcherResult {
-  /**
-   * Actual value.
-   */
-  actual?: unknown;
-
-  /**
-   * Expected value.
-   */
-  expected?: unknown;
-
-  /**
-   * Matcher name.
-   */
-  name?: string;
-
-  /**
-   * Whether the matcher passed.
-   */
-  pass: string;
-
-  /**
-   * Timeout that was used during matching.
-   */
-  timeout?: number;
-}
-
 /**
  * Information about an error thrown during test execution.
  */
@@ -607,11 +576,6 @@ export interface TestError {
    */
   location?: Location;
 
-  /**
-   * Matcher result details.
-   */
-  matcherResult?: TestErrorMatcherResult;
-
   /**
    * Error message. Set when [Error] (or its subclass) has been thrown.
    */

tests/bidi/README.md

@@ -0,0 +1,23 @@
+## Running Bidi tests
+
+To run Playwright tests with Bidi:
+
+```sh
+git clone https://github.com/microsoft/playwright.git
+cd playwright
+npm run build # call `npm run watch` for watch mode
+npx playwright install chromium
+npm run biditest -- --project='bidi-firefox-beta-*'
+```
+
+To install beta channel of Firefox, run the following command in the project root:
+```sh
+npx -y @puppeteer/browsers install firefox@beta
+```
+
+You can also pass custom binary path via `BIDIPATH`:
+```sh
+BIDIPATH='/Users/myself/Downloads/chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing'
+```
+
+

tests/bidi/playwright.config.ts

@@ -63,8 +63,8 @@ if (executablePath && !process.env.TEST_WORKER_INDEX)
   console.error(`Using executable at ${executablePath}`);
 const testIgnore: RegExp[] = [];
 const browserToChannels = {
-  '_bidiChromium': ['bidi-chrome-stable'],
+  '_bidiChromium': ['bidi-chromium', 'bidi-chrome-canary', 'bidi-chrome-stable'],
-  '_bidiFirefox': ['bidi-firefox-stable'],
+  '_bidiFirefox': ['bidi-firefox-nightly', 'bidi-firefox-beta', 'bidi-firefox-stable'],
 };
 for (const [key, channels] of Object.entries(browserToChannels)) {
   const browserName: any = key;

tests/playwright-test/reporter-blob.spec.ts

@@ -2040,12 +2040,15 @@ test('project filter in report name', async ({ runInlineTest }) => {
   const reportDir = test.info().outputPath('blob-report');
 
   {
-    await runInlineTest(files, { shard: `2/2`, project: 'foo' });
+    const result = await runInlineTest(files, { shard: `2/2`, project: 'foo' });
+    expect(result.exitCode).toBe(0);
     const reportFiles = await fs.promises.readdir(reportDir);
     expect(reportFiles.sort()).toEqual(['report-foo-2.zip']);
   }
+
   {
-    await runInlineTest(files, { shard: `1/2`, project: 'foo,b*r', grep: 'smoke' });
+    const result = await runInlineTest(files, { shard: `1/2`, project: ['foo', 'b*r'], grep: 'smoke' });
+    expect(result.exitCode).toBe(0);
     const reportFiles = await fs.promises.readdir(reportDir);
     expect(reportFiles.sort()).toEqual(['report-foo-b-r-6d9d49e-1.zip']);
   }

tests/playwright-test/reporter-errors.spec.ts

@@ -1,145 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import { test, expect } from './playwright-test-fixtures';
-
-test('should report matcherResults for generic matchers', async ({ runInlineTest }) => {
-  const result = await runInlineTest({
-    'a.test.js': `
-      import { test, expect as baseExpect } from '@playwright/test';
-      const expect = baseExpect.soft;
-      test('fail', ({}) => {
-        expect(1).toBe(2);
-        expect(1).toBeCloseTo(2);
-        expect(undefined).toBeDefined();
-        expect(1).toBeFalsy();
-        expect(1).toBeGreaterThan(2);
-        expect(1).toBeGreaterThanOrEqual(2);
-        expect('a').toBeInstanceOf(Number);
-        expect(2).toBeLessThan(1);
-        expect(2).toBeLessThanOrEqual(1);
-        expect(1).toBeNaN();
-        expect(1).toBeNull();
-        expect(0).toBeTruthy();
-        expect(1).toBeUndefined();
-        expect([1]).toContain(2);
-        expect([1]).toContainEqual(2);
-        expect([1]).toEqual([2]);
-        expect([1]).toHaveLength(2);
-        expect({ a: 1 }).toHaveProperty('b');
-        expect('a').toMatch(/b/);
-        expect({ a: 1 }).toMatchObject({ b: 2 });
-        expect({ a: 1 }).toStrictEqual({ b: 2 });
-        expect(() => {}).toThrow();
-        expect(() => {}).toThrowError('a');
-      });
-    `
-  }, { });
-  expect(result.exitCode).toBe(1);
-
-  const { errors } = result.report.suites[0].specs[0].tests[0].results[0];
-  const matcherResults = errors.map(e => e.matcherResult);
-  expect(matcherResults).toEqual([
-    { name: 'toBe', pass: false, expected: 2, actual: 1 },
-    { pass: false },
-    { pass: false },
-    { pass: false },
-    { pass: false },
-    { pass: false },
-    { pass: false },
-    { pass: false },
-    { pass: false },
-    { pass: false },
-    { pass: false },
-    { pass: false },
-    { pass: false },
-    { pass: false },
-    { pass: false },
-    { name: 'toEqual', pass: false, expected: [2], actual: [1] },
-    { pass: false },
-    { pass: false },
-    { pass: false },
-    { pass: false },
-    { name: 'toStrictEqual', pass: false, expected: { b: 2 }, actual: { a: 1 } },
-    { pass: false },
-    { pass: false },
-  ]);
-});
-
-test('should report matcherResults for web matchers', async ({ runInlineTest }) => {
-  const result = await runInlineTest({
-    'a.test.js': `
-      import { test, expect as baseExpect } from '@playwright/test';
-
-      const expect = baseExpect.configure({ soft: true, timeout: 1 });
-      test('fail', async ({ page }) => {
-        await page.setContent('<span>Hello</span><div display="none">World</div><input type="checkbox"><textarea></textarea><button>Submit</button><select multiple><option value="value">Text</option></select>');
-        await expect(page.locator('input')).toBeChecked();
-        await expect(page.locator('input')).toBeDisabled();
-        await expect(page.locator('textarea')).not.toBeEditable();
-        await expect(page.locator('span')).toBeEmpty();
-        await expect(page.locator('button')).not.toBeEnabled();
-        await expect(page.locator('button')).toBeFocused();
-        await expect(page.locator('span')).toBeHidden();
-        await expect(page.locator('div')).not.toBeInViewport();
-        await expect(page.locator('div')).not.toBeVisible();
-        await expect(page.locator('span')).toContainText('World');
-        await expect(page.locator('span')).toHaveAccessibleDescription('World');
-        await expect(page.locator('span')).toHaveAccessibleName('World');
-        await expect(page.locator('span')).toHaveAttribute('name', 'value');
-        await expect(page.locator('span')).toHaveAttribute('name');
-        await expect(page.locator('span')).toHaveClass('name');
-        await expect(page.locator('span')).toHaveCount(2);
-        await expect(page.locator('span')).toHaveCSS('width', '10');
-        await expect(page.locator('span')).toHaveId('id');
-        await expect(page.locator('span')).toHaveJSProperty('name', 'value');
-        await expect(page.locator('span')).toHaveRole('role');
-        await expect(page.locator('span')).toHaveText('World');
-        await expect(page.locator('textarea')).toHaveValue('value');
-        await expect(page.locator('select')).toHaveValues(['value']);
-      });
-    `
-  }, { });
-  expect(result.exitCode).toBe(1);
-
-  const { errors } = result.report.suites[0].specs[0].tests[0].results[0];
-  const matcherResults = errors.map(e => e.matcherResult);
-  expect(matcherResults).toEqual([
-    { name: 'toBeChecked', pass: false, expected: 'checked', actual: 'unchecked', timeout: 1 },
-    { name: 'toBeDisabled', pass: false, expected: 'disabled', actual: 'enabled', timeout: 1 },
-    { name: 'toBeEditable', pass: true, expected: 'editable', actual: 'editable', timeout: 1 },
-    { name: 'toBeEmpty', pass: false, expected: 'empty', actual: 'notEmpty', timeout: 1 },
-    { name: 'toBeEnabled', pass: true, expected: 'enabled', actual: 'enabled', timeout: 1 },
-    { name: 'toBeFocused', pass: false, expected: 'focused', actual: 'inactive', timeout: 1 },
-    { name: 'toBeHidden', pass: false, expected: 'hidden', actual: 'visible', timeout: 1 },
-    { name: 'toBeInViewport', pass: true, expected: 'in viewport', actual: 'in viewport', timeout: 1 },
-    { name: 'toBeVisible', pass: true, expected: 'visible', actual: 'visible', timeout: 1 },
-    { name: 'toContainText', pass: false, expected: 'World', actual: 'Hello', timeout: 1 },
-    { name: 'toHaveAccessibleDescription', pass: false, expected: 'World', actual: '', timeout: 1 },
-    { name: 'toHaveAccessibleName', pass: false, expected: 'World', actual: '', timeout: 1 },
-    { name: 'toHaveAttribute', pass: false, expected: 'value', actual: null, timeout: 1 },
-    { name: 'toHaveAttribute', pass: false, expected: 'have attribute', actual: 'not have attribute', timeout: 1 },
-    { name: 'toHaveClass', pass: false, expected: 'name', actual: '', timeout: 1 },
-    { name: 'toHaveCount', pass: false, expected: 2, actual: 1, timeout: 1 },
-    { name: 'toHaveCSS', pass: false, expected: '10', actual: 'auto', timeout: 1 },
-    { name: 'toHaveId', pass: false, expected: 'id', actual: '', timeout: 1 },
-    { name: 'toHaveJSProperty', pass: false, expected: 'value', timeout: 1 },
-    { name: 'toHaveRole', pass: false, expected: 'role', actual: '', timeout: 1 },
-    { name: 'toHaveText', pass: false, expected: 'World', actual: 'Hello', timeout: 1 },
-    { name: 'toHaveValue', pass: false, expected: 'value', actual: '', timeout: 1 },
-    { name: 'toHaveValues', pass: false, expected: ['value'], actual: [], timeout: 1 },
-  ]);
-});

tests/playwright-test/resolver.spec.ts

@@ -16,6 +16,24 @@
 
 import { test, expect } from './playwright-test-fixtures';
 
+test('should print tsconfig parsing error', async ({ runInlineTest }) => {
+  const files = {
+    'a.spec.ts': `
+      import { test } from '@playwright/test';
+      test('pass', async () => {});
+    `,
+    'tsconfig.json': `
+      "foo": "bar"
+    `,
+  };
+
+  const result = await runInlineTest(files);
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain(`Failed to load tsconfig file at`);
+  expect(result.output).toContain(`tsconfig.json`);
+  expect(result.output).toContain(`JSON5: invalid character ':' at 2:12`);
+});
+
 test('should respect path resolver', async ({ runInlineTest }) => {
   test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/11656' });
 
@@ -569,43 +587,6 @@ test('should resolve paths relative to the originating config when extending and
   expect(result.exitCode).toBe(0);
 });
 
-test('should import packages with non-index main script through path resolver', async ({ runInlineTest }) => {
-  const result = await runInlineTest({
-    'app/pkg/main.ts': `
-      export const foo = 42;
-    `,
-    'app/pkg/package.json': `
-      { "main": "main.ts" }
-    `,
-    'package.json': `
-      { "name": "example-project" }
-    `,
-    'playwright.config.ts': `
-      export default {};
-    `,
-    'tsconfig.json': `{
-      "compilerOptions": {
-        "baseUrl": ".",
-        "paths": {
-          "app/*": ["app/*"],
-        },
-      },
-    }`,
-    'example.spec.ts': `
-      import { foo } from 'app/pkg';
-      import { test, expect } from '@playwright/test';
-      test('test', ({}) => {
-        console.log('foo=' + foo);
-      });
-    `,
-  });
-
-  expect(result.exitCode).toBe(0);
-  expect(result.passed).toBe(1);
-  expect(result.output).not.toContain(`find module`);
-  expect(result.output).toContain(`foo=42`);
-});
-
 test('should respect tsconfig project references', async ({ runInlineTest }) => {
   test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/29256' });
 
@@ -693,3 +674,426 @@ test('should respect --tsconfig option', async ({ runInlineTest }) => {
   expect(result.exitCode).toBe(0);
   expect(result.output).not.toContain(`Could not`);
 });
+
+test.describe('directory imports', () => {
+  test('should resolve index.js without path mapping in CJS', async ({ runInlineTest, runTSC }) => {
+    const files = {
+      'foo-pkg/index.js': `
+        exports.foo = 'bar';
+      `,
+      'foo-pkg/index.d.ts': `
+        export const foo: 'bar';
+      `,
+      'a.test.ts': `
+        import { test, expect } from '@playwright/test';
+        import { foo } from './foo-pkg';
+        test('pass', async () => {
+          const bar: 'bar' = foo;
+          expect(bar).toBe('bar');
+        });
+      `,
+    };
+
+    const result = await runInlineTest(files);
+    expect(result.passed).toBe(1);
+    expect(result.exitCode).toBe(0);
+
+    const tscResult = await runTSC(files);
+    expect(tscResult.exitCode).toBe(0);
+  });
+
+  test('should resolve index.js without path mapping in ESM', async ({ runInlineTest, runTSC }) => {
+    const files = {
+      'foo-pkg/index.js': `
+        export const foo = 'bar';
+      `,
+      'foo-pkg/index.d.ts': `
+        export const foo: 'bar';
+      `,
+      'package.json': `
+        { "type": "module" }
+      `,
+      'a.test.ts': `
+        import { test, expect } from '@playwright/test';
+        import { foo } from './foo-pkg';
+        test('pass', async () => {
+          const bar: 'bar' = foo;
+          expect(bar).toBe('bar');
+        });
+      `,
+    };
+
+    const result = await runInlineTest(files);
+    expect(result.passed).toBe(1);
+    expect(result.exitCode).toBe(0);
+
+    const tscResult = await runTSC(files);
+    expect(tscResult.exitCode).toBe(0);
+  });
+
+  test('should resolve index.js after path mapping in CJS', async ({ runInlineTest, runTSC }) => {
+    test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/31811' });
+
+    const files = {
+      '@acme/lib/index.js': `
+        exports.greet = () => 2;
+      `,
+      '@acme/lib/index.d.ts': `
+        export const greet: () => number;
+      `,
+      'tests/hello.test.ts': `
+        import { greet } from '@acme/lib';
+        import { test, expect } from '@playwright/test';
+        test('hello', async ({}) => {
+          const foo: number = greet();
+          expect(foo).toBe(2);
+        });
+      `,
+      'tsconfig.json': `
+        {
+          "compilerOptions": {
+            "paths": {
+              "@acme/*": ["./@acme/*"]
+            },
+            "moduleResolution": "bundler",
+            "module": "preserve",
+            "noEmit": true,
+            "noImplicitAny": true
+          }
+        }
+      `,
+    };
+
+    const result = await runInlineTest(files);
+    expect(result.exitCode).toBe(0);
+    expect(result.passed).toBe(1);
+
+    const tscResult = await runTSC(files);
+    expect(tscResult.exitCode).toBe(0);
+  });
+
+  test('should resolve index.js after path mapping in ESM', async ({ runInlineTest, runTSC }) => {
+    test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/31811' });
+
+    const files = {
+      '@acme/lib/index.js': `
+        export const greet = () => 2;
+      `,
+      '@acme/lib/index.d.ts': `
+        export const greet: () => number;
+      `,
+      'package.json': `
+        { "type": "module" }
+      `,
+      'tests/hello.test.ts': `
+        import { greet } from '@acme/lib';
+        import { test, expect } from '@playwright/test';
+        test('hello', async ({}) => {
+          const foo: number = greet();
+          expect(foo).toBe(2);
+        });
+      `,
+      'tsconfig.json': `
+        {
+          "compilerOptions": {
+            "paths": {
+              "@acme/*": ["./@acme/*"]
+            },
+            "moduleResolution": "bundler",
+            "module": "preserve",
+            "noEmit": true,
+            "noImplicitAny": true
+          }
+        }
+      `,
+    };
+
+    const result = await runInlineTest(files);
+    expect(result.exitCode).toBe(0);
+    expect(result.passed).toBe(1);
+
+    const tscResult = await runTSC(files);
+    expect(tscResult.exitCode).toBe(0);
+  });
+
+  test('should respect package.json#main after path mapping in CJS', async ({ runInlineTest, runTSC }) => {
+    const files = {
+      'app/pkg/main.ts': `
+        export const foo = 42;
+      `,
+      'app/pkg/package.json': `
+        { "main": "main.ts" }
+      `,
+      'package.json': `
+        { "name": "example-project" }
+      `,
+      'playwright.config.ts': `
+        export default {};
+      `,
+      'tsconfig.json': `
+        {
+          "compilerOptions": {
+            "baseUrl": ".",
+            "paths": {
+              "app/*": ["app/*"]
+            },
+            "moduleResolution": "bundler",
+            "module": "preserve",
+            "noEmit": true,
+            "noImplicitAny": true
+          }
+        }
+      `,
+      'example.spec.ts': `
+        import { foo } from 'app/pkg';
+        import { test, expect } from '@playwright/test';
+        test('test', ({}) => {
+          const bar: number = foo;
+          expect(bar).toBe(42);
+        });
+      `,
+    };
+
+    const result = await runInlineTest(files);
+    expect(result.exitCode).toBe(0);
+    expect(result.passed).toBe(1);
+    expect(result.output).not.toContain(`find module`);
+
+    const tscResult = await runTSC(files);
+    expect(tscResult.exitCode).toBe(0);
+  });
+
+  test('should respect package.json#main after path mapping in ESM', async ({ runInlineTest, runTSC }) => {
+    const files = {
+      'app/pkg/main.ts': `
+        export const foo = 42;
+      `,
+      'app/pkg/package.json': `
+        { "main": "main.ts", "type": "module" }
+      `,
+      'package.json': `
+        { "name": "example-project", "type": "module" }
+      `,
+      'playwright.config.ts': `
+        export default {};
+      `,
+      'tsconfig.json': `
+        {
+          "compilerOptions": {
+            "baseUrl": ".",
+            "paths": {
+              "app/*": ["app/*"]
+            },
+            "moduleResolution": "bundler",
+            "module": "preserve",
+            "noEmit": true,
+            "noImplicitAny": true
+          },
+        }
+      `,
+      'example.spec.ts': `
+        import { foo } from 'app/pkg';
+        import { test, expect } from '@playwright/test';
+        test('test', ({}) => {
+          const bar: number = foo;
+          expect(bar).toBe(42);
+        });
+      `,
+    };
+
+    const result = await runInlineTest(files);
+    expect(result.exitCode).toBe(0);
+    expect(result.passed).toBe(1);
+
+    const tscResult = await runTSC(files);
+    expect(tscResult.exitCode).toBe(0);
+  });
+
+  test('should respect package.json#exports without path mapping in CJS', async ({ runInlineTest, runTSC }) => {
+    const files = {
+      'node_modules/foo-pkg/package.json': `
+        { "name": "foo-pkg", "exports": { ".": "./foo.js" } }
+      `,
+      'node_modules/foo-pkg/foo.js': `
+        exports.foo = 'bar';
+      `,
+      'node_modules/foo-pkg/foo.d.ts': `
+        export const foo: 'bar';
+      `,
+      'package.json': `
+        { "name": "test-project" }
+      `,
+      'a.test.ts': `
+        import { test, expect } from '@playwright/test';
+        import { foo } from 'foo-pkg';
+        test('pass', async () => {
+          const bar: 'bar' = foo;
+          expect(bar).toBe('bar');
+        });
+      `,
+      'tsconfig.json': `
+        {
+          "compilerOptions": {
+            "moduleResolution": "bundler",
+            "module": "preserve",
+            "noEmit": true,
+            "noImplicitAny": true
+          },
+        }
+      `,
+    };
+
+    const result = await runInlineTest(files);
+    expect(result.passed).toBe(1);
+    expect(result.exitCode).toBe(0);
+
+    const tscResult = await runTSC(files);
+    expect(tscResult.exitCode).toBe(0);
+  });
+
+  test('should respect package.json#exports without path mapping in ESM', async ({ runInlineTest, runTSC }) => {
+    const files = {
+      'node_modules/foo-pkg/package.json': `
+        { "name": "foo-pkg", "type": "module", "exports": { "default": "./foo.js" } }
+      `,
+      'node_modules/foo-pkg/foo.js': `
+        export const foo = 'bar';
+      `,
+      'node_modules/foo-pkg/foo.d.ts': `
+        export const foo: 'bar';
+      `,
+      'package.json': `
+        { "name": "test-project", "type": "module" }
+      `,
+      'a.test.ts': `
+        import { test, expect } from '@playwright/test';
+        import { foo } from 'foo-pkg';
+        test('pass', async () => {
+          const bar: 'bar' = foo;
+          expect(bar).toBe('bar');
+        });
+      `,
+      'tsconfig.json': `
+        {
+          "compilerOptions": {
+            "moduleResolution": "bundler",
+            "module": "preserve",
+            "noEmit": true,
+            "noImplicitAny": true
+          },
+        }
+      `,
+    };
+
+    const result = await runInlineTest(files);
+    expect(result.passed).toBe(1);
+    expect(result.exitCode).toBe(0);
+
+    const tscResult = await runTSC(files);
+    expect(tscResult.exitCode).toBe(0);
+  });
+
+  test('should not respect package.json#exports after type mapping in CJS', async ({ runInlineTest, runTSC }) => {
+    const files = {
+      'app/pkg/main.ts': `
+        export const filename: 'main.ts' = 'main.ts';
+      `,
+      'app/pkg/index.js': `
+        export const filename = 'index.js';
+      `,
+      'app/pkg/index.d.ts': `
+        export const filename: 'index.js';
+      `,
+      'app/pkg/package.json': `
+        { "exports": { ".": "./main.ts" } }
+      `,
+      'package.json': `
+        { "name": "example-project" }
+      `,
+      'playwright.config.ts': `
+        export default {};
+      `,
+      'tsconfig.json': `
+        {
+          "compilerOptions": {
+            "baseUrl": ".",
+            "paths": {
+              "app/*": ["app/*"]
+            },
+            "moduleResolution": "bundler",
+            "module": "preserve",
+            "noEmit": true,
+            "noImplicitAny": true
+          }
+        }
+      `,
+      'example.spec.ts': `
+        import { filename } from 'app/pkg';
+        import { test, expect } from '@playwright/test';
+        test('test', ({}) => {
+          const foo: 'index.js' = filename;
+          expect(foo).toBe('index.js');
+        });
+      `,
+    };
+
+    const result = await runInlineTest(files);
+    expect(result.exitCode).toBe(0);
+    expect(result.passed).toBe(1);
+
+    const tscResult = await runTSC(files);
+    expect(tscResult.exitCode).toBe(0);
+  });
+
+  test('should not respect package.json#exports after type mapping in ESM', async ({ runInlineTest, runTSC }) => {
+    const files = {
+      'app/pkg/main.ts': `
+        export const filename: 'main.ts' = 'main.ts';
+      `,
+      'app/pkg/index.js': `
+        export const filename = 'index.js';
+      `,
+      'app/pkg/index.d.ts': `
+        export const filename: 'index.js';
+      `,
+      'app/pkg/package.json': `
+        { "exports": { ".": "./main.ts" }, "type": "module" }
+      `,
+      'package.json': `
+        { "name": "example-project", "type": "module" }
+      `,
+      'playwright.config.ts': `
+        export default {};
+      `,
+      'tsconfig.json': `
+        {
+          "compilerOptions": {
+            "baseUrl": ".",
+            "paths": {
+              "app/*": ["app/*"]
+            },
+            "moduleResolution": "bundler",
+            "module": "preserve",
+            "noEmit": true,
+            "noImplicitAny": true
+          }
+        }
+      `,
+      'example.spec.ts': `
+        import { filename } from 'app/pkg';
+        import { test, expect } from '@playwright/test';
+        test('test', ({}) => {
+          const foo: 'index.js' = filename;
+          expect(foo).toBe('index.js');
+        });
+      `,
+    };
+
+    const result = await runInlineTest(files);
+    expect(result.exitCode).toBe(0);
+    expect(result.passed).toBe(1);
+
+    const tscResult = await runTSC(files);
+    expect(tscResult.exitCode).toBe(0);
+  });
+});

utils/generate_types/overrides-testReporter.d.ts

@@ -105,7 +105,6 @@ export interface JSONReportTest {
 export interface JSONReportError {
   message: string;
   location?: Location;
-  matcherResult?: TestErrorMatcherResult;
 }
 
 export interface JSONReportTestResult {