diff --git a/.github/workflows/tests_bidi.yml b/.github/workflows/tests_bidi.yml index 433294dbea..b534a7b747 100644 --- a/.github/workflows/tests_bidi.yml +++ b/.github/workflows/tests_bidi.yml @@ -26,8 +26,7 @@ jobs: strategy: fail-fast: false matrix: - # TODO: add Firefox - channel: [bidi-chrome-stable] + channel: [bidi-chromium, bidi-firefox-beta] steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -38,5 +37,8 @@ jobs: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: '1' - run: npm run build - run: npx playwright install --with-deps chromium + if: matrix.channel == 'bidi-chromium' + - run: npx -y @puppeteer/browsers install firefox@beta + if: matrix.channel == 'bidi-firefox-beta' - name: Run tests run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run biditest -- --project=${{ matrix.channel }}* diff --git a/docs/src/test-api/class-testinfoerror-matcherresult.md b/docs/src/test-api/class-testinfoerror-matcherresult.md deleted file mode 100644 index f68e2db1ae..0000000000 --- a/docs/src/test-api/class-testinfoerror-matcherresult.md +++ /dev/null @@ -1,35 +0,0 @@ -# class: TestInfoErrorMatcherResult -* since: v1.48 -* langs: js - -Matcher-specific details for the error thrown during the `expect` call. - -## property: TestInfoErrorMatcherResult.actual -* since: v1.48 -- type: ?<[unknown]> - -Actual value. - -## property: TestInfoErrorMatcherResult.expected -* since: v1.48 -- type: ?<[unknown]> - -Expected value. - -## property: TestInfoErrorMatcherResult.name -* since: v1.48 -- type: ?<[string]> - -Matcher name. - -## property: TestInfoErrorMatcherResult.pass -* since: v1.48 -- type: <[string]> - -Whether the matcher passed. - -## property: TestInfoErrorMatcherResult.timeout -* since: v1.48 -- type: ?<[int]> - -Timeout that was used during matching. diff --git a/docs/src/test-api/class-testinfoerror.md b/docs/src/test-api/class-testinfoerror.md index b084aaf54b..66e78ecabd 100644 --- a/docs/src/test-api/class-testinfoerror.md +++ b/docs/src/test-api/class-testinfoerror.md @@ -4,12 +4,6 @@ Information about an error thrown during test execution. -## property: TestInfoError.matcherResult -* since: v1.48 -- type: ?<[TestInfoErrorMatcherResult]> - -Matcher result details. - ## property: TestInfoError.message * since: v1.10 - type: ?<[string]> diff --git a/docs/src/test-reporter-api/class-testerror-matcherresult.md b/docs/src/test-reporter-api/class-testerror-matcherresult.md deleted file mode 100644 index b8d0f1dce7..0000000000 --- a/docs/src/test-reporter-api/class-testerror-matcherresult.md +++ /dev/null @@ -1,35 +0,0 @@ -# class: TestErrorMatcherResult -* since: v1.48 -* langs: js - -Matcher-specific details for the error thrown during the `expect` call. - -## property: TestErrorMatcherResult.actual -* since: v1.48 -- type: ?<[unknown]> - -Actual value. - -## property: TestErrorMatcherResult.expected -* since: v1.48 -- type: ?<[unknown]> - -Expected value. - -## property: TestErrorMatcherResult.name -* since: v1.48 -- type: ?<[string]> - -Matcher name. - -## property: TestErrorMatcherResult.pass -* since: v1.48 -- type: <[string]> - -Whether the matcher passed. - -## property: TestErrorMatcherResult.timeout -* since: v1.48 -- type: ?<[int]> - -Timeout that was used during matching. diff --git a/docs/src/test-reporter-api/class-testerror.md b/docs/src/test-reporter-api/class-testerror.md index cc59c3a4c6..7a872c63fc 100644 --- a/docs/src/test-reporter-api/class-testerror.md +++ b/docs/src/test-reporter-api/class-testerror.md @@ -4,12 +4,6 @@ Information about an error thrown during test execution. -## property: TestError.matcherResult -* since: v1.48 -- type: ?<[TestErrorMatcherResult]> - -Matcher result details. - ## property: TestError.message * since: v1.10 - type: ?<[string]> diff --git a/packages/playwright-core/src/server/registry/index.ts b/packages/playwright-core/src/server/registry/index.ts index 058e0bcbc3..4e942967a4 100644 --- a/packages/playwright-core/src/server/registry/index.ts +++ b/packages/playwright-core/src/server/registry/index.ts @@ -354,7 +354,7 @@ function readDescriptors(browsersJSON: BrowsersJSON) { export type BrowserName = 'chromium' | 'firefox' | 'webkit' | 'bidi'; type InternalTool = 'ffmpeg' | 'firefox-beta' | 'chromium-tip-of-tree' | 'android'; -type BidiChannel = 'bidi-firefox-stable' | 'bidi-chrome-canary' | 'bidi-chrome-stable'; +type BidiChannel = 'bidi-firefox-stable' | 'bidi-firefox-beta' | 'bidi-firefox-nightly' | 'bidi-chrome-canary' | 'bidi-chrome-stable' | 'bidi-chromium'; type ChromiumChannel = 'chrome' | 'chrome-beta' | 'chrome-dev' | 'chrome-canary' | 'msedge' | 'msedge-beta' | 'msedge-dev' | 'msedge-canary'; const allDownloadable = ['chromium', 'firefox', 'webkit', 'ffmpeg', 'firefox-beta', 'chromium-tip-of-tree']; @@ -525,11 +525,22 @@ export class Registry { 'win32': `\\Microsoft\\Edge SxS\\Application\\msedge.exe`, })); - this._executables.push(this._createBidiChannel('bidi-firefox-stable', { - 'linux': '/usr/bin/firefox', - 'darwin': '/Applications/Firefox.app/Contents/MacOS/firefox', - 'win32': '\\Mozilla Firefox\\firefox.exe', + this._executables.push(this._createBidiFirefoxChannel('bidi-firefox-stable', { + 'linux': '/firefox/firefox', + 'darwin': '/Firefox.app/Contents/MacOS/firefox', + 'win32': '\\core\\firefox.exe', })); + this._executables.push(this._createBidiFirefoxChannel('bidi-firefox-beta', { + 'linux': '/firefox/firefox', + 'darwin': '/Firefox.app/Contents/MacOS/firefox', + 'win32': '\\core\\firefox.exe', + })); + this._executables.push(this._createBidiFirefoxChannel('bidi-firefox-nightly', { + 'linux': '/firefox/firefox', + 'darwin': '/Firefox Nightly.app/Contents/MacOS/firefox', + 'win32': '\\firefox\\firefox.exe', + })); + this._executables.push(this._createBidiChannel('bidi-chrome-stable', { 'linux': '/opt/google/chrome/chrome', 'darwin': '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', @@ -540,6 +551,21 @@ export class Registry { 'darwin': '/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary', 'win32': `\\Google\\Chrome SxS\\Application\\chrome.exe`, })); + this._executables.push({ + type: 'browser', + name: 'bidi-chromium', + browserName: 'bidi', + directory: chromium.dir, + executablePath: () => chromiumExecutable, + executablePathOrDie: (sdkLanguage: string) => executablePathOrDie('chromium', chromiumExecutable, chromium.installByDefault, sdkLanguage), + installType: 'download-on-demand', + _validateHostRequirements: (sdkLanguage: string) => this._validateHostRequirements(sdkLanguage, 'chromium', chromium.dir, ['chrome-linux'], [], ['chrome-win']), + downloadURLs: this._downloadURLs(chromium), + browserVersion: chromium.browserVersion, + _install: () => this._downloadExecutable(chromium, chromiumExecutable), + _dependencyGroup: 'chromium', + _isHermeticInstallation: true, + }); const firefox = descriptors.find(d => d.name === 'firefox')!; const firefoxExecutable = findExecutablePath(firefox.dir, 'firefox'); @@ -691,6 +717,48 @@ export class Registry { }; } + private _createBidiFirefoxChannel(name: BidiChannel, lookAt: Record<'linux' | 'darwin' | 'win32', string>, install?: () => Promise): ExecutableImpl { + const executablePath = (sdkLanguage: string, shouldThrow: boolean) => { + const suffix = lookAt[process.platform as 'linux' | 'darwin' | 'win32']; + if (!suffix) { + if (shouldThrow) + throw new Error(`Firefox distribution '${name}' is not supported on ${process.platform}`); + return undefined; + } + const folder = path.resolve('firefox'); + let channelName = 'stable'; + if (name.includes('beta')) + channelName = 'beta'; + else if (name.includes('nightly')) + channelName = 'nightly'; + const installedVersions = fs.readdirSync(folder); + const found = installedVersions.filter(e => e.includes(channelName)); + if (found.length === 1) + return path.join(folder, found[0], suffix); + if (found.length > 1) { + if (shouldThrow) + throw new Error(`Multiple Firefox installations found for channel '${name}': ${found.join(', ')}`); + else + return undefined; + } + if (shouldThrow) + throw new Error(`Cannot find Firefox installation for channel '${name}' under ${folder}`); + return undefined; + }; + return { + type: 'channel', + name, + browserName: 'bidi', + directory: undefined, + executablePath: (sdkLanguage: string) => executablePath(sdkLanguage, false), + executablePathOrDie: (sdkLanguage: string) => executablePath(sdkLanguage, true)!, + installType: 'none', + _validateHostRequirements: () => Promise.resolve(), + _isHermeticInstallation: true, + _install: install, + }; + } + private _createBidiChannel(name: BidiChannel, lookAt: Record<'linux' | 'darwin' | 'win32', string>, install?: () => Promise): ExecutableImpl { const executablePath = (sdkLanguage: string, shouldThrow: boolean) => { const suffix = lookAt[process.platform as 'linux' | 'darwin' | 'win32']; diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.ts index 22470b0954..c9ce2f7bcd 100644 --- a/packages/playwright/src/reporters/base.ts +++ b/packages/playwright/src/reporters/base.ts @@ -32,7 +32,6 @@ type Annotation = { type ErrorDetails = { message: string; location?: Location; - matcherResult?: TestError['matcherResult']; }; type TestSummary = { @@ -400,7 +399,6 @@ export function formatResultFailure(test: TestCase, result: TestResult, initialI errorDetails.push({ message: indent(formattedError.message, initialIndent), location: formattedError.location, - matcherResult: formattedError.matcherResult }); } return errorDetails; @@ -491,7 +489,6 @@ export function formatError(error: TestError, highlightCode: boolean): ErrorDeta return { location, message: tokens.join('\n'), - matcherResult: error.matcherResult }; } diff --git a/packages/playwright/src/third_party/tsconfig-loader.ts b/packages/playwright/src/third_party/tsconfig-loader.ts index 61ad11a3bd..b654a3b963 100644 --- a/packages/playwright/src/third_party/tsconfig-loader.ts +++ b/packages/playwright/src/third_party/tsconfig-loader.ts @@ -53,9 +53,13 @@ export interface LoadedTsConfig { } export function loadTsConfig(configPath: string): LoadedTsConfig[] { - const references: LoadedTsConfig[] = []; - const config = innerLoadTsConfig(configPath, references); - return [config, ...references]; + try { + const references: LoadedTsConfig[] = []; + const config = innerLoadTsConfig(configPath, references); + return [config, ...references]; + } catch (e) { + throw new Error(`Failed to load tsconfig file at ${configPath}:\n${e.message}`); + } } function resolveConfigFile(baseConfigFile: string, referencedConfigFile: string) { diff --git a/packages/playwright/src/transform/transform.ts b/packages/playwright/src/transform/transform.ts index 9ca80399fb..f70f385b5b 100644 --- a/packages/playwright/src/transform/transform.ts +++ b/packages/playwright/src/transform/transform.ts @@ -24,7 +24,7 @@ import type { LoadedTsConfig } from '../third_party/tsconfig-loader'; import { loadTsConfig } from '../third_party/tsconfig-loader'; import Module from 'module'; import type { BabelPlugin, BabelTransformFunction } from './babelBundle'; -import { createFileMatcher, fileIsModule, resolveImportSpecifierExtension } from '../util'; +import { createFileMatcher, fileIsModule, resolveImportSpecifierAfterMapping } from '../util'; import type { Matcher } from '../util'; import { getFromCompilationCache, currentFileDepsCollector, belongsToNodeModules, installSourceMapSupport } from './compilationCache'; @@ -136,8 +136,13 @@ export function resolveHook(filename: string, specifier: string): string | undef return; if (isRelativeSpecifier(specifier)) - return resolveImportSpecifierExtension(path.resolve(path.dirname(filename), specifier)); + return resolveImportSpecifierAfterMapping(path.resolve(path.dirname(filename), specifier), false); + /** + * TypeScript discourages path-mapping into node_modules: + * https://www.typescriptlang.org/docs/handbook/modules/reference.html#paths-should-not-point-to-monorepo-packages-or-node_modules-packages + * However, if path-mapping doesn't yield a result, TypeScript falls back to the default resolution through node_modules. + */ const isTypeScript = filename.endsWith('.ts') || filename.endsWith('.tsx'); const tsconfigs = loadAndValidateTsconfigsForFile(filename); for (const tsconfig of tsconfigs) { @@ -179,7 +184,7 @@ export function resolveHook(filename: string, specifier: string): string | undef if (value.includes('*')) candidate = candidate.replace('*', matchedPartOfSpecifier); candidate = path.resolve(tsconfig.pathsBase!, candidate); - const existing = resolveImportSpecifierExtension(candidate); + const existing = resolveImportSpecifierAfterMapping(candidate, true); if (existing) { longestPrefixLength = keyPrefix.length; pathMatchedByLongestPrefix = existing; @@ -193,7 +198,7 @@ export function resolveHook(filename: string, specifier: string): string | undef if (path.isAbsolute(specifier)) { // Handle absolute file paths like `import '/path/to/file'` // Do not handle module imports like `import 'fs'` - return resolveImportSpecifierExtension(specifier); + return resolveImportSpecifierAfterMapping(specifier, false); } } diff --git a/packages/playwright/src/util.ts b/packages/playwright/src/util.ts index 0531f167a0..460b3de07e 100644 --- a/packages/playwright/src/util.ts +++ b/packages/playwright/src/util.ts @@ -63,20 +63,8 @@ export function filteredStackTrace(rawStack: RawStack): StackFrame[] { } export function serializeError(error: Error | any): TestInfoError { - if (error instanceof Error) { - const result: TestInfoError = filterStackTrace(error); - if ('matcherResult' in error && error.matcherResult) { - const matcherResult = (error.matcherResult as TestInfoError['matcherResult'])!; - result.matcherResult = { - pass: matcherResult.pass, - name: matcherResult.name, - expected: matcherResult.expected, - actual: matcherResult.actual, - timeout: matcherResult.timeout, - }; - } - return result; - } + if (error instanceof Error) + return filterStackTrace(error); return { value: util.inspect(error) }; @@ -307,8 +295,23 @@ function folderIsModule(folder: string): boolean { return require(packageJsonPath).type === 'module'; } -// This follows the --moduleResolution=bundler strategy from tsc. -// https://devblogs.microsoft.com/typescript/announcing-typescript-5-0-beta/#moduleresolution-bundler +const packageJsonMainFieldCache = new Map(); + +function getMainFieldFromPackageJson(packageJsonPath: string) { + if (!packageJsonMainFieldCache.has(packageJsonPath)) { + let mainField: string | undefined; + try { + mainField = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8')).main; + } catch { + } + packageJsonMainFieldCache.set(packageJsonPath, mainField); + } + return packageJsonMainFieldCache.get(packageJsonPath); +} + +// This method performs "file extension subsitution" to find the ts, js or similar source file +// based on the import specifier, which might or might not have an extension. See TypeScript docs: +// https://www.typescriptlang.org/docs/handbook/modules/reference.html#file-extension-substitution. const kExtLookups = new Map([ ['.js', ['.jsx', '.ts', '.tsx']], ['.jsx', ['.tsx']], @@ -316,7 +319,7 @@ const kExtLookups = new Map([ ['.mjs', ['.mts']], ['', ['.js', '.ts', '.jsx', '.tsx', '.cjs', '.mjs', '.cts', '.mts']], ]); -export function resolveImportSpecifierExtension(resolved: string): string | undefined { +function resolveImportSpecifierExtension(resolved: string): string | undefined { if (fileExists(resolved)) return resolved; @@ -330,13 +333,45 @@ export function resolveImportSpecifierExtension(resolved: string): string | unde } break; // Do not try '' when a more specific extension like '.jsx' matched. } +} + +// This method resolves directory imports and performs "file extension subsitution". +// It is intended to be called after the path mapping resolution. +// +// Directory imports follow the --moduleResolution=bundler strategy from tsc. +// https://www.typescriptlang.org/docs/handbook/modules/reference.html#directory-modules-index-file-resolution +// https://www.typescriptlang.org/docs/handbook/modules/reference.html#bundler +// +// See also Node.js "folder as module" behavior: +// https://nodejs.org/dist/latest-v20.x/docs/api/modules.html#folders-as-modules. +export function resolveImportSpecifierAfterMapping(resolved: string, afterPathMapping: boolean): string | undefined { + const resolvedFile = resolveImportSpecifierExtension(resolved); + if (resolvedFile) + return resolvedFile; if (dirExists(resolved)) { + const packageJsonPath = path.join(resolved, 'package.json'); + + if (afterPathMapping) { + // Most notably, the module resolution algorithm is not performed after the path mapping. + // This means no node_modules lookup or package.json#exports. + // + // Only the "folder as module" Node.js behavior is respected: + // - consult `package.json#main`; + // - look for `index.js` or similar. + const mainField = getMainFieldFromPackageJson(packageJsonPath); + const mainFieldResolved = mainField ? resolveImportSpecifierExtension(path.resolve(resolved, mainField)) : undefined; + return mainFieldResolved || resolveImportSpecifierExtension(path.join(resolved, 'index')); + } + // If we import a package, let Node.js figure out the correct import based on package.json. - if (fileExists(path.join(resolved, 'package.json'))) + // This also covers the "main" field for "folder as module". + if (fileExists(packageJsonPath)) return resolved; - // Otherwise, try to find a corresponding index file. + // Implement the "folder as module" Node.js behavior. + // Note that we do not delegate to Node.js, because we support this for ESM as well, + // following the TypeScript "bundler" mode. const dirImport = path.join(resolved, 'index'); return resolveImportSpecifierExtension(dirImport); } diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index 999cdc1f2d..1c384aec01 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -8238,45 +8238,10 @@ export interface TestInfo { workerIndex: number; } -/** - * Matcher-specific details for the error thrown during the `expect` call. - */ -export interface TestInfoErrorMatcherResult { - /** - * Actual value. - */ - actual?: unknown; - - /** - * Expected value. - */ - expected?: unknown; - - /** - * Matcher name. - */ - name?: string; - - /** - * Whether the matcher passed. - */ - pass: string; - - /** - * Timeout that was used during matching. - */ - timeout?: number; -} - /** * Information about an error thrown during test execution. */ export interface TestInfoError { - /** - * Matcher result details. - */ - matcherResult?: TestInfoErrorMatcherResult; - /** * Error message. Set when [Error] (or its subclass) has been thrown. */ diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.ts index b600e3c801..52073066f0 100644 --- a/packages/playwright/types/testReporter.d.ts +++ b/packages/playwright/types/testReporter.d.ts @@ -284,7 +284,6 @@ export interface JSONReportTest { export interface JSONReportError { message: string; location?: Location; - matcherResult?: TestErrorMatcherResult; } export interface JSONReportTestResult { @@ -568,36 +567,6 @@ export interface TestCase { type: "test"; } -/** - * Matcher-specific details for the error thrown during the `expect` call. - */ -export interface TestErrorMatcherResult { - /** - * Actual value. - */ - actual?: unknown; - - /** - * Expected value. - */ - expected?: unknown; - - /** - * Matcher name. - */ - name?: string; - - /** - * Whether the matcher passed. - */ - pass: string; - - /** - * Timeout that was used during matching. - */ - timeout?: number; -} - /** * Information about an error thrown during test execution. */ @@ -607,11 +576,6 @@ export interface TestError { */ location?: Location; - /** - * Matcher result details. - */ - matcherResult?: TestErrorMatcherResult; - /** * Error message. Set when [Error] (or its subclass) has been thrown. */ diff --git a/tests/bidi/README.md b/tests/bidi/README.md new file mode 100644 index 0000000000..caac9288a4 --- /dev/null +++ b/tests/bidi/README.md @@ -0,0 +1,23 @@ +## Running Bidi tests + +To run Playwright tests with Bidi: + +```sh +git clone https://github.com/microsoft/playwright.git +cd playwright +npm run build # call `npm run watch` for watch mode +npx playwright install chromium +npm run biditest -- --project='bidi-firefox-beta-*' +``` + +To install beta channel of Firefox, run the following command in the project root: +```sh +npx -y @puppeteer/browsers install firefox@beta +``` + +You can also pass custom binary path via `BIDIPATH`: +```sh +BIDIPATH='/Users/myself/Downloads/chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing' +``` + + diff --git a/tests/bidi/playwright.config.ts b/tests/bidi/playwright.config.ts index bc7b29e56b..b04af32253 100644 --- a/tests/bidi/playwright.config.ts +++ b/tests/bidi/playwright.config.ts @@ -63,8 +63,8 @@ if (executablePath && !process.env.TEST_WORKER_INDEX) console.error(`Using executable at ${executablePath}`); const testIgnore: RegExp[] = []; const browserToChannels = { - '_bidiChromium': ['bidi-chrome-stable'], - '_bidiFirefox': ['bidi-firefox-stable'], + '_bidiChromium': ['bidi-chromium', 'bidi-chrome-canary', 'bidi-chrome-stable'], + '_bidiFirefox': ['bidi-firefox-nightly', 'bidi-firefox-beta', 'bidi-firefox-stable'], }; for (const [key, channels] of Object.entries(browserToChannels)) { const browserName: any = key; diff --git a/tests/playwright-test/reporter-blob.spec.ts b/tests/playwright-test/reporter-blob.spec.ts index 8f3064a073..91b32e76a2 100644 --- a/tests/playwright-test/reporter-blob.spec.ts +++ b/tests/playwright-test/reporter-blob.spec.ts @@ -2040,12 +2040,15 @@ test('project filter in report name', async ({ runInlineTest }) => { const reportDir = test.info().outputPath('blob-report'); { - await runInlineTest(files, { shard: `2/2`, project: 'foo' }); + const result = await runInlineTest(files, { shard: `2/2`, project: 'foo' }); + expect(result.exitCode).toBe(0); const reportFiles = await fs.promises.readdir(reportDir); expect(reportFiles.sort()).toEqual(['report-foo-2.zip']); } + { - await runInlineTest(files, { shard: `1/2`, project: 'foo,b*r', grep: 'smoke' }); + const result = await runInlineTest(files, { shard: `1/2`, project: ['foo', 'b*r'], grep: 'smoke' }); + expect(result.exitCode).toBe(0); const reportFiles = await fs.promises.readdir(reportDir); expect(reportFiles.sort()).toEqual(['report-foo-b-r-6d9d49e-1.zip']); } diff --git a/tests/playwright-test/reporter-errors.spec.ts b/tests/playwright-test/reporter-errors.spec.ts deleted file mode 100644 index ad09237234..0000000000 --- a/tests/playwright-test/reporter-errors.spec.ts +++ /dev/null @@ -1,145 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { test, expect } from './playwright-test-fixtures'; - -test('should report matcherResults for generic matchers', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.js': ` - import { test, expect as baseExpect } from '@playwright/test'; - const expect = baseExpect.soft; - test('fail', ({}) => { - expect(1).toBe(2); - expect(1).toBeCloseTo(2); - expect(undefined).toBeDefined(); - expect(1).toBeFalsy(); - expect(1).toBeGreaterThan(2); - expect(1).toBeGreaterThanOrEqual(2); - expect('a').toBeInstanceOf(Number); - expect(2).toBeLessThan(1); - expect(2).toBeLessThanOrEqual(1); - expect(1).toBeNaN(); - expect(1).toBeNull(); - expect(0).toBeTruthy(); - expect(1).toBeUndefined(); - expect([1]).toContain(2); - expect([1]).toContainEqual(2); - expect([1]).toEqual([2]); - expect([1]).toHaveLength(2); - expect({ a: 1 }).toHaveProperty('b'); - expect('a').toMatch(/b/); - expect({ a: 1 }).toMatchObject({ b: 2 }); - expect({ a: 1 }).toStrictEqual({ b: 2 }); - expect(() => {}).toThrow(); - expect(() => {}).toThrowError('a'); - }); - ` - }, { }); - expect(result.exitCode).toBe(1); - - const { errors } = result.report.suites[0].specs[0].tests[0].results[0]; - const matcherResults = errors.map(e => e.matcherResult); - expect(matcherResults).toEqual([ - { name: 'toBe', pass: false, expected: 2, actual: 1 }, - { pass: false }, - { pass: false }, - { pass: false }, - { pass: false }, - { pass: false }, - { pass: false }, - { pass: false }, - { pass: false }, - { pass: false }, - { pass: false }, - { pass: false }, - { pass: false }, - { pass: false }, - { pass: false }, - { name: 'toEqual', pass: false, expected: [2], actual: [1] }, - { pass: false }, - { pass: false }, - { pass: false }, - { pass: false }, - { name: 'toStrictEqual', pass: false, expected: { b: 2 }, actual: { a: 1 } }, - { pass: false }, - { pass: false }, - ]); -}); - -test('should report matcherResults for web matchers', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.js': ` - import { test, expect as baseExpect } from '@playwright/test'; - - const expect = baseExpect.configure({ soft: true, timeout: 1 }); - test('fail', async ({ page }) => { - await page.setContent('Hello
World
'); - await expect(page.locator('input')).toBeChecked(); - await expect(page.locator('input')).toBeDisabled(); - await expect(page.locator('textarea')).not.toBeEditable(); - await expect(page.locator('span')).toBeEmpty(); - await expect(page.locator('button')).not.toBeEnabled(); - await expect(page.locator('button')).toBeFocused(); - await expect(page.locator('span')).toBeHidden(); - await expect(page.locator('div')).not.toBeInViewport(); - await expect(page.locator('div')).not.toBeVisible(); - await expect(page.locator('span')).toContainText('World'); - await expect(page.locator('span')).toHaveAccessibleDescription('World'); - await expect(page.locator('span')).toHaveAccessibleName('World'); - await expect(page.locator('span')).toHaveAttribute('name', 'value'); - await expect(page.locator('span')).toHaveAttribute('name'); - await expect(page.locator('span')).toHaveClass('name'); - await expect(page.locator('span')).toHaveCount(2); - await expect(page.locator('span')).toHaveCSS('width', '10'); - await expect(page.locator('span')).toHaveId('id'); - await expect(page.locator('span')).toHaveJSProperty('name', 'value'); - await expect(page.locator('span')).toHaveRole('role'); - await expect(page.locator('span')).toHaveText('World'); - await expect(page.locator('textarea')).toHaveValue('value'); - await expect(page.locator('select')).toHaveValues(['value']); - }); - ` - }, { }); - expect(result.exitCode).toBe(1); - - const { errors } = result.report.suites[0].specs[0].tests[0].results[0]; - const matcherResults = errors.map(e => e.matcherResult); - expect(matcherResults).toEqual([ - { name: 'toBeChecked', pass: false, expected: 'checked', actual: 'unchecked', timeout: 1 }, - { name: 'toBeDisabled', pass: false, expected: 'disabled', actual: 'enabled', timeout: 1 }, - { name: 'toBeEditable', pass: true, expected: 'editable', actual: 'editable', timeout: 1 }, - { name: 'toBeEmpty', pass: false, expected: 'empty', actual: 'notEmpty', timeout: 1 }, - { name: 'toBeEnabled', pass: true, expected: 'enabled', actual: 'enabled', timeout: 1 }, - { name: 'toBeFocused', pass: false, expected: 'focused', actual: 'inactive', timeout: 1 }, - { name: 'toBeHidden', pass: false, expected: 'hidden', actual: 'visible', timeout: 1 }, - { name: 'toBeInViewport', pass: true, expected: 'in viewport', actual: 'in viewport', timeout: 1 }, - { name: 'toBeVisible', pass: true, expected: 'visible', actual: 'visible', timeout: 1 }, - { name: 'toContainText', pass: false, expected: 'World', actual: 'Hello', timeout: 1 }, - { name: 'toHaveAccessibleDescription', pass: false, expected: 'World', actual: '', timeout: 1 }, - { name: 'toHaveAccessibleName', pass: false, expected: 'World', actual: '', timeout: 1 }, - { name: 'toHaveAttribute', pass: false, expected: 'value', actual: null, timeout: 1 }, - { name: 'toHaveAttribute', pass: false, expected: 'have attribute', actual: 'not have attribute', timeout: 1 }, - { name: 'toHaveClass', pass: false, expected: 'name', actual: '', timeout: 1 }, - { name: 'toHaveCount', pass: false, expected: 2, actual: 1, timeout: 1 }, - { name: 'toHaveCSS', pass: false, expected: '10', actual: 'auto', timeout: 1 }, - { name: 'toHaveId', pass: false, expected: 'id', actual: '', timeout: 1 }, - { name: 'toHaveJSProperty', pass: false, expected: 'value', timeout: 1 }, - { name: 'toHaveRole', pass: false, expected: 'role', actual: '', timeout: 1 }, - { name: 'toHaveText', pass: false, expected: 'World', actual: 'Hello', timeout: 1 }, - { name: 'toHaveValue', pass: false, expected: 'value', actual: '', timeout: 1 }, - { name: 'toHaveValues', pass: false, expected: ['value'], actual: [], timeout: 1 }, - ]); -}); diff --git a/tests/playwright-test/resolver.spec.ts b/tests/playwright-test/resolver.spec.ts index 5a0e91b099..037ba14f22 100644 --- a/tests/playwright-test/resolver.spec.ts +++ b/tests/playwright-test/resolver.spec.ts @@ -16,6 +16,24 @@ import { test, expect } from './playwright-test-fixtures'; +test('should print tsconfig parsing error', async ({ runInlineTest }) => { + const files = { + 'a.spec.ts': ` + import { test } from '@playwright/test'; + test('pass', async () => {}); + `, + 'tsconfig.json': ` + "foo": "bar" + `, + }; + + const result = await runInlineTest(files); + expect(result.exitCode).toBe(1); + expect(result.output).toContain(`Failed to load tsconfig file at`); + expect(result.output).toContain(`tsconfig.json`); + expect(result.output).toContain(`JSON5: invalid character ':' at 2:12`); +}); + test('should respect path resolver', async ({ runInlineTest }) => { test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/11656' }); @@ -569,43 +587,6 @@ test('should resolve paths relative to the originating config when extending and expect(result.exitCode).toBe(0); }); -test('should import packages with non-index main script through path resolver', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'app/pkg/main.ts': ` - export const foo = 42; - `, - 'app/pkg/package.json': ` - { "main": "main.ts" } - `, - 'package.json': ` - { "name": "example-project" } - `, - 'playwright.config.ts': ` - export default {}; - `, - 'tsconfig.json': `{ - "compilerOptions": { - "baseUrl": ".", - "paths": { - "app/*": ["app/*"], - }, - }, - }`, - 'example.spec.ts': ` - import { foo } from 'app/pkg'; - import { test, expect } from '@playwright/test'; - test('test', ({}) => { - console.log('foo=' + foo); - }); - `, - }); - - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(1); - expect(result.output).not.toContain(`find module`); - expect(result.output).toContain(`foo=42`); -}); - test('should respect tsconfig project references', async ({ runInlineTest }) => { test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/29256' }); @@ -693,3 +674,426 @@ test('should respect --tsconfig option', async ({ runInlineTest }) => { expect(result.exitCode).toBe(0); expect(result.output).not.toContain(`Could not`); }); + +test.describe('directory imports', () => { + test('should resolve index.js without path mapping in CJS', async ({ runInlineTest, runTSC }) => { + const files = { + 'foo-pkg/index.js': ` + exports.foo = 'bar'; + `, + 'foo-pkg/index.d.ts': ` + export const foo: 'bar'; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + import { foo } from './foo-pkg'; + test('pass', async () => { + const bar: 'bar' = foo; + expect(bar).toBe('bar'); + }); + `, + }; + + const result = await runInlineTest(files); + expect(result.passed).toBe(1); + expect(result.exitCode).toBe(0); + + const tscResult = await runTSC(files); + expect(tscResult.exitCode).toBe(0); + }); + + test('should resolve index.js without path mapping in ESM', async ({ runInlineTest, runTSC }) => { + const files = { + 'foo-pkg/index.js': ` + export const foo = 'bar'; + `, + 'foo-pkg/index.d.ts': ` + export const foo: 'bar'; + `, + 'package.json': ` + { "type": "module" } + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + import { foo } from './foo-pkg'; + test('pass', async () => { + const bar: 'bar' = foo; + expect(bar).toBe('bar'); + }); + `, + }; + + const result = await runInlineTest(files); + expect(result.passed).toBe(1); + expect(result.exitCode).toBe(0); + + const tscResult = await runTSC(files); + expect(tscResult.exitCode).toBe(0); + }); + + test('should resolve index.js after path mapping in CJS', async ({ runInlineTest, runTSC }) => { + test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/31811' }); + + const files = { + '@acme/lib/index.js': ` + exports.greet = () => 2; + `, + '@acme/lib/index.d.ts': ` + export const greet: () => number; + `, + 'tests/hello.test.ts': ` + import { greet } from '@acme/lib'; + import { test, expect } from '@playwright/test'; + test('hello', async ({}) => { + const foo: number = greet(); + expect(foo).toBe(2); + }); + `, + 'tsconfig.json': ` + { + "compilerOptions": { + "paths": { + "@acme/*": ["./@acme/*"] + }, + "moduleResolution": "bundler", + "module": "preserve", + "noEmit": true, + "noImplicitAny": true + } + } + `, + }; + + const result = await runInlineTest(files); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + + const tscResult = await runTSC(files); + expect(tscResult.exitCode).toBe(0); + }); + + test('should resolve index.js after path mapping in ESM', async ({ runInlineTest, runTSC }) => { + test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/31811' }); + + const files = { + '@acme/lib/index.js': ` + export const greet = () => 2; + `, + '@acme/lib/index.d.ts': ` + export const greet: () => number; + `, + 'package.json': ` + { "type": "module" } + `, + 'tests/hello.test.ts': ` + import { greet } from '@acme/lib'; + import { test, expect } from '@playwright/test'; + test('hello', async ({}) => { + const foo: number = greet(); + expect(foo).toBe(2); + }); + `, + 'tsconfig.json': ` + { + "compilerOptions": { + "paths": { + "@acme/*": ["./@acme/*"] + }, + "moduleResolution": "bundler", + "module": "preserve", + "noEmit": true, + "noImplicitAny": true + } + } + `, + }; + + const result = await runInlineTest(files); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + + const tscResult = await runTSC(files); + expect(tscResult.exitCode).toBe(0); + }); + + test('should respect package.json#main after path mapping in CJS', async ({ runInlineTest, runTSC }) => { + const files = { + 'app/pkg/main.ts': ` + export const foo = 42; + `, + 'app/pkg/package.json': ` + { "main": "main.ts" } + `, + 'package.json': ` + { "name": "example-project" } + `, + 'playwright.config.ts': ` + export default {}; + `, + 'tsconfig.json': ` + { + "compilerOptions": { + "baseUrl": ".", + "paths": { + "app/*": ["app/*"] + }, + "moduleResolution": "bundler", + "module": "preserve", + "noEmit": true, + "noImplicitAny": true + } + } + `, + 'example.spec.ts': ` + import { foo } from 'app/pkg'; + import { test, expect } from '@playwright/test'; + test('test', ({}) => { + const bar: number = foo; + expect(bar).toBe(42); + }); + `, + }; + + const result = await runInlineTest(files); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + expect(result.output).not.toContain(`find module`); + + const tscResult = await runTSC(files); + expect(tscResult.exitCode).toBe(0); + }); + + test('should respect package.json#main after path mapping in ESM', async ({ runInlineTest, runTSC }) => { + const files = { + 'app/pkg/main.ts': ` + export const foo = 42; + `, + 'app/pkg/package.json': ` + { "main": "main.ts", "type": "module" } + `, + 'package.json': ` + { "name": "example-project", "type": "module" } + `, + 'playwright.config.ts': ` + export default {}; + `, + 'tsconfig.json': ` + { + "compilerOptions": { + "baseUrl": ".", + "paths": { + "app/*": ["app/*"] + }, + "moduleResolution": "bundler", + "module": "preserve", + "noEmit": true, + "noImplicitAny": true + }, + } + `, + 'example.spec.ts': ` + import { foo } from 'app/pkg'; + import { test, expect } from '@playwright/test'; + test('test', ({}) => { + const bar: number = foo; + expect(bar).toBe(42); + }); + `, + }; + + const result = await runInlineTest(files); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + + const tscResult = await runTSC(files); + expect(tscResult.exitCode).toBe(0); + }); + + test('should respect package.json#exports without path mapping in CJS', async ({ runInlineTest, runTSC }) => { + const files = { + 'node_modules/foo-pkg/package.json': ` + { "name": "foo-pkg", "exports": { ".": "./foo.js" } } + `, + 'node_modules/foo-pkg/foo.js': ` + exports.foo = 'bar'; + `, + 'node_modules/foo-pkg/foo.d.ts': ` + export const foo: 'bar'; + `, + 'package.json': ` + { "name": "test-project" } + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + import { foo } from 'foo-pkg'; + test('pass', async () => { + const bar: 'bar' = foo; + expect(bar).toBe('bar'); + }); + `, + 'tsconfig.json': ` + { + "compilerOptions": { + "moduleResolution": "bundler", + "module": "preserve", + "noEmit": true, + "noImplicitAny": true + }, + } + `, + }; + + const result = await runInlineTest(files); + expect(result.passed).toBe(1); + expect(result.exitCode).toBe(0); + + const tscResult = await runTSC(files); + expect(tscResult.exitCode).toBe(0); + }); + + test('should respect package.json#exports without path mapping in ESM', async ({ runInlineTest, runTSC }) => { + const files = { + 'node_modules/foo-pkg/package.json': ` + { "name": "foo-pkg", "type": "module", "exports": { "default": "./foo.js" } } + `, + 'node_modules/foo-pkg/foo.js': ` + export const foo = 'bar'; + `, + 'node_modules/foo-pkg/foo.d.ts': ` + export const foo: 'bar'; + `, + 'package.json': ` + { "name": "test-project", "type": "module" } + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + import { foo } from 'foo-pkg'; + test('pass', async () => { + const bar: 'bar' = foo; + expect(bar).toBe('bar'); + }); + `, + 'tsconfig.json': ` + { + "compilerOptions": { + "moduleResolution": "bundler", + "module": "preserve", + "noEmit": true, + "noImplicitAny": true + }, + } + `, + }; + + const result = await runInlineTest(files); + expect(result.passed).toBe(1); + expect(result.exitCode).toBe(0); + + const tscResult = await runTSC(files); + expect(tscResult.exitCode).toBe(0); + }); + + test('should not respect package.json#exports after type mapping in CJS', async ({ runInlineTest, runTSC }) => { + const files = { + 'app/pkg/main.ts': ` + export const filename: 'main.ts' = 'main.ts'; + `, + 'app/pkg/index.js': ` + export const filename = 'index.js'; + `, + 'app/pkg/index.d.ts': ` + export const filename: 'index.js'; + `, + 'app/pkg/package.json': ` + { "exports": { ".": "./main.ts" } } + `, + 'package.json': ` + { "name": "example-project" } + `, + 'playwright.config.ts': ` + export default {}; + `, + 'tsconfig.json': ` + { + "compilerOptions": { + "baseUrl": ".", + "paths": { + "app/*": ["app/*"] + }, + "moduleResolution": "bundler", + "module": "preserve", + "noEmit": true, + "noImplicitAny": true + } + } + `, + 'example.spec.ts': ` + import { filename } from 'app/pkg'; + import { test, expect } from '@playwright/test'; + test('test', ({}) => { + const foo: 'index.js' = filename; + expect(foo).toBe('index.js'); + }); + `, + }; + + const result = await runInlineTest(files); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + + const tscResult = await runTSC(files); + expect(tscResult.exitCode).toBe(0); + }); + + test('should not respect package.json#exports after type mapping in ESM', async ({ runInlineTest, runTSC }) => { + const files = { + 'app/pkg/main.ts': ` + export const filename: 'main.ts' = 'main.ts'; + `, + 'app/pkg/index.js': ` + export const filename = 'index.js'; + `, + 'app/pkg/index.d.ts': ` + export const filename: 'index.js'; + `, + 'app/pkg/package.json': ` + { "exports": { ".": "./main.ts" }, "type": "module" } + `, + 'package.json': ` + { "name": "example-project", "type": "module" } + `, + 'playwright.config.ts': ` + export default {}; + `, + 'tsconfig.json': ` + { + "compilerOptions": { + "baseUrl": ".", + "paths": { + "app/*": ["app/*"] + }, + "moduleResolution": "bundler", + "module": "preserve", + "noEmit": true, + "noImplicitAny": true + } + } + `, + 'example.spec.ts': ` + import { filename } from 'app/pkg'; + import { test, expect } from '@playwright/test'; + test('test', ({}) => { + const foo: 'index.js' = filename; + expect(foo).toBe('index.js'); + }); + `, + }; + + const result = await runInlineTest(files); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + + const tscResult = await runTSC(files); + expect(tscResult.exitCode).toBe(0); + }); +}); diff --git a/utils/generate_types/overrides-testReporter.d.ts b/utils/generate_types/overrides-testReporter.d.ts index 0b81724373..51eab7e370 100644 --- a/utils/generate_types/overrides-testReporter.d.ts +++ b/utils/generate_types/overrides-testReporter.d.ts @@ -105,7 +105,6 @@ export interface JSONReportTest { export interface JSONReportError { message: string; location?: Location; - matcherResult?: TestErrorMatcherResult; } export interface JSONReportTestResult {