From 8f3353865d8d98e9b40c15497e60d5e2583410b6 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Wed, 9 Oct 2024 17:46:54 -0700 Subject: [PATCH 1/7] fix(ui): bring back the headed param (#33030) Fixes https://github.com/microsoft/playwright/issues/33023 --- packages/playwright/src/runner/testServer.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/playwright/src/runner/testServer.ts b/packages/playwright/src/runner/testServer.ts index 50a8de366e..ff37da717c 100644 --- a/packages/playwright/src/runner/testServer.ts +++ b/packages/playwright/src/runner/testServer.ts @@ -302,10 +302,11 @@ export class TestServerDispatcher implements TestServerInterface { preserveOutputDir: true, reporter: params.reporters ? params.reporters.map(r => [r]) : undefined, use: { - ...(this._configCLIOverrides.use || {}), - trace: params.trace === 'on' ? { mode: 'on', sources: false, _live: true } : (params.trace === 'off' ? 'off' : undefined), - video: params.video === 'on' ? 'on' : (params.video === 'off' ? 'off' : undefined), - headless: params.headed ? false : undefined, + ...this._configCLIOverrides.use, + ...(params.trace === 'on' ? { trace: { mode: 'on', sources: false, _live: true } } : {}), + ...(params.trace === 'off' ? { trace: 'off' } : {}), + ...(params.video === 'on' || params.video === 'off' ? { video: params.video } : {}), + ...(params.headed !== undefined ? { headless: !params.headed } : {}), _optionContextReuseMode: params.reuseContext ? 'when-possible' : undefined, _optionConnectOptions: params.connectWsEndpoint ? { wsEndpoint: params.connectWsEndpoint } : undefined, }, From 25dd9b5cd49470215c71d77ef46f81a66960ec3d Mon Sep 17 00:00:00 2001 From: Dmitry Gozman Date: Thu, 10 Oct 2024 01:37:46 -0700 Subject: [PATCH 2/7] feat: config.build.tsconfig (#33026) Allows to specify `tsconfig` in the configuration file, which applies to test files but not the config file itself. Fixes #32808. --- docs/src/test-api/class-testconfig.md | 16 +++++ docs/src/test-typescript-js.md | 10 ++++ packages/playwright/src/common/config.ts | 2 + .../playwright/src/common/configLoader.ts | 2 + .../playwright/src/common/esmLoaderHost.ts | 1 + packages/playwright/types/test.d.ts | 19 ++++++ tests/playwright-test/resolver.spec.ts | 59 +++++++++++++++++++ 7 files changed, 109 insertions(+) diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index d013f5e4ea..0d1f4c1538 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -552,6 +552,22 @@ export default defineConfig({ }); ``` +## property: TestConfig.tsconfig +* since: v1.49 +- type: ?<[string]> + +Path to a single `tsconfig` applicable to all imported files. By default, `tsconfig` for each imported file is looked up separately. Note that `tsconfig` property has no effect while the configuration file or any of its dependencies are loaded. Ignored when `--tsconfig` command line option is specified. + +**Usage** + +```js title="playwright.config.ts" +import { defineConfig } from '@playwright/test'; + +export default defineConfig({ + tsconfig: './tsconfig.test.json', +}); +``` + ## property: TestConfig.updateSnapshots * since: v1.10 - type: ?<[UpdateSnapshots]<"all"|"none"|"missing">> diff --git a/docs/src/test-typescript-js.md b/docs/src/test-typescript-js.md index 6e18b3c615..538f5a137e 100644 --- a/docs/src/test-typescript-js.md +++ b/docs/src/test-typescript-js.md @@ -90,6 +90,16 @@ Alternatively, you can specify a single tsconfig file to use in the command line npx playwright test --tsconfig=tsconfig.test.json ``` +You can specify a single tsconfig file in the config file, that will be used for loading test files, reporters, etc. However, it will not be used while loading the playwright config itself or any files imported from it. + +```js title="playwright.config.ts" +import { defineConfig } from '@playwright/test'; + +export default defineConfig({ + tsconfig: './tsconfig.test.json', +}); +``` + ## Manually compile tests with TypeScript Sometimes, Playwright Test will not be able to transform your TypeScript code correctly, for example when you are using experimental or very recent features of TypeScript, usually configured in `tsconfig.json`. diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts index d7fb499645..a694839f81 100644 --- a/packages/playwright/src/common/config.ts +++ b/packages/playwright/src/common/config.ts @@ -45,6 +45,7 @@ export class FullConfigInternal { readonly webServers: NonNullable[]; readonly plugins: TestRunnerPluginRegistration[]; readonly projects: FullProjectInternal[] = []; + readonly singleTSConfigPath?: string; cliArgs: string[] = []; cliGrep: string | undefined; cliGrepInvert: string | undefined; @@ -69,6 +70,7 @@ export class FullConfigInternal { this.configCLIOverrides = configCLIOverrides; const privateConfiguration = (userConfig as any)['@playwright/test']; this.plugins = (privateConfiguration?.plugins || []).map((p: any) => ({ factory: p })); + this.singleTSConfigPath = pathResolve(configDir, userConfig.tsconfig); this.config = { configFile: resolvedConfigFile, diff --git a/packages/playwright/src/common/configLoader.ts b/packages/playwright/src/common/configLoader.ts index 5ed1c68ea7..eef56c4458 100644 --- a/packages/playwright/src/common/configLoader.ts +++ b/packages/playwright/src/common/configLoader.ts @@ -118,6 +118,8 @@ export async function loadConfig(location: ConfigLocation, overrides?: ConfigCLI const babelPlugins = (userConfig as any)['@playwright/test']?.babelPlugins || []; const external = userConfig.build?.external || []; setTransformConfig({ babelPlugins, external }); + if (!overrides?.tsconfig) + setSingleTSConfig(fullConfig?.singleTSConfigPath); // 4. Send transform options to ESM loader. await configureESMLoaderTransformConfig(); diff --git a/packages/playwright/src/common/esmLoaderHost.ts b/packages/playwright/src/common/esmLoaderHost.ts index 1611b0f91d..9b5b4f9b8c 100644 --- a/packages/playwright/src/common/esmLoaderHost.ts +++ b/packages/playwright/src/common/esmLoaderHost.ts @@ -77,5 +77,6 @@ export async function configureESMLoader() { export async function configureESMLoaderTransformConfig() { if (!loaderChannel) return; + await loaderChannel.send('setSingleTSConfig', { tsconfig: singleTSConfig() }); await loaderChannel.send('setTransformConfig', { config: transformConfig() }); } diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index 91fe2f2b5c..a1128c519e 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -1642,6 +1642,25 @@ interface TestConfig { */ timeout?: number; + /** + * Path to a single `tsconfig` applicable to all imported files. By default, `tsconfig` for each imported file is + * looked up separately. Note that `tsconfig` property has no effect while the configuration file or any of its + * dependencies are loaded. Ignored when `--tsconfig` command line option is specified. + * + * **Usage** + * + * ```js + * // playwright.config.ts + * import { defineConfig } from '@playwright/test'; + * + * export default defineConfig({ + * tsconfig: './tsconfig.test.json', + * }); + * ``` + * + */ + tsconfig?: string; + /** * Whether to update expected snapshots with the actual results produced by the test run. Defaults to `'missing'`. * - `'all'` - All tests that are executed will update snapshots that did not match. Matching snapshots will not be diff --git a/tests/playwright-test/resolver.spec.ts b/tests/playwright-test/resolver.spec.ts index 037ba14f22..d4dec96495 100644 --- a/tests/playwright-test/resolver.spec.ts +++ b/tests/playwright-test/resolver.spec.ts @@ -675,6 +675,65 @@ test('should respect --tsconfig option', async ({ runInlineTest }) => { expect(result.output).not.toContain(`Could not`); }); +test('should respect config.tsconfig option', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + export { configFoo } from '~/foo'; + export default { + testDir: './tests', + tsconfig: './tsconfig.tests.json', + }; + `, + 'tsconfig.json': `{ + "compilerOptions": { + "baseUrl": ".", + "paths": { + "~/*": ["./mapped-from-config/*"], + }, + }, + }`, + 'mapped-from-config/foo.ts': ` + export const configFoo = 17; + `, + 'tsconfig.tests.json': `{ + "compilerOptions": { + "baseUrl": ".", + "paths": { + "~/*": ["./mapped-from-tests/*"], + }, + }, + }`, + 'mapped-from-tests/foo.ts': ` + export const testFoo = 42; + `, + 'tests/tsconfig.json': `{ + "compilerOptions": { + "baseUrl": ".", + "paths": { + "~/*": ["../should-be-ignored/*"], + }, + }, + }`, + 'tests/a.test.ts': ` + import { testFoo } from '~/foo'; + import { configFoo } from '../playwright.config'; + import { test, expect } from '@playwright/test'; + test('test', ({}) => { + expect(testFoo).toBe(42); + expect(configFoo).toBe(17); + }); + `, + 'should-be-ignored/foo.ts': ` + export const testFoo = 43; + export const configFoo = 18; + `, + }); + + expect(result.passed).toBe(1); + expect(result.exitCode).toBe(0); + expect(result.output).not.toContain(`Could not`); +}); + test.describe('directory imports', () => { test('should resolve index.js without path mapping in CJS', async ({ runInlineTest, runTSC }) => { const files = { From 217b57df4c15913b345beb68cf1818d9387fdbba Mon Sep 17 00:00:00 2001 From: Playwright Service <89237858+playwrightmachine@users.noreply.github.com> Date: Thu, 10 Oct 2024 04:10:54 -0700 Subject: [PATCH 3/7] feat(webkit): roll to r2089 (#33039) Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- packages/playwright-core/browsers.json | 2 +- packages/playwright-core/src/server/webkit/protocol.d.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/playwright-core/browsers.json b/packages/playwright-core/browsers.json index c177343ebf..bc52be8381 100644 --- a/packages/playwright-core/browsers.json +++ b/packages/playwright-core/browsers.json @@ -27,7 +27,7 @@ }, { "name": "webkit", - "revision": "2084", + "revision": "2089", "installByDefault": true, "revisionOverrides": { "mac10.14": "1446", diff --git a/packages/playwright-core/src/server/webkit/protocol.d.ts b/packages/playwright-core/src/server/webkit/protocol.d.ts index 3ddfda4627..7c279f9e1e 100644 --- a/packages/playwright-core/src/server/webkit/protocol.d.ts +++ b/packages/playwright-core/src/server/webkit/protocol.d.ts @@ -6510,7 +6510,7 @@ the top of the viewport and Y increases as it proceeds towards the bottom of the /** * List of settings able to be overridden by WebInspector. */ - export type Setting = "PrivateClickMeasurementDebugModeEnabled"|"AuthorAndUserStylesEnabled"|"ICECandidateFilteringEnabled"|"ITPDebugModeEnabled"|"ImagesEnabled"|"MediaCaptureRequiresSecureConnection"|"MockCaptureDevicesEnabled"|"NeedsSiteSpecificQuirks"|"ScriptEnabled"|"ShowDebugBorders"|"ShowRepaintCounter"|"WebSecurityEnabled"|"DeviceOrientationEventEnabled"|"SpeechRecognitionEnabled"|"PointerLockEnabled"|"NotificationsEnabled"|"FullScreenEnabled"|"InputTypeMonthEnabled"|"InputTypeWeekEnabled"; + export type Setting = "PrivateClickMeasurementDebugModeEnabled"|"AuthorAndUserStylesEnabled"|"ICECandidateFilteringEnabled"|"ITPDebugModeEnabled"|"ImagesEnabled"|"MediaCaptureRequiresSecureConnection"|"MockCaptureDevicesEnabled"|"NeedsSiteSpecificQuirks"|"ScriptEnabled"|"ShowDebugBorders"|"ShowRepaintCounter"|"WebSecurityEnabled"|"DeviceOrientationEventEnabled"|"SpeechRecognitionEnabled"|"PointerLockEnabled"|"NotificationsEnabled"|"FullScreenEnabled"|"InputTypeMonthEnabled"|"InputTypeWeekEnabled"|"FixedBackgroundsPaintRelativeToDocument"; /** * A user preference that can be overriden by Web Inspector, like an accessibility preference. */ From 7de084b6dc2ec090fef800099cfa42db42895b04 Mon Sep 17 00:00:00 2001 From: Playwright Service <89237858+playwrightmachine@users.noreply.github.com> Date: Thu, 10 Oct 2024 07:32:00 -0700 Subject: [PATCH 4/7] feat(chromium-tip-of-tree): roll to r1268 (#33042) Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- packages/playwright-core/browsers.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/playwright-core/browsers.json b/packages/playwright-core/browsers.json index bc52be8381..1fbe52c90d 100644 --- a/packages/playwright-core/browsers.json +++ b/packages/playwright-core/browsers.json @@ -9,9 +9,9 @@ }, { "name": "chromium-tip-of-tree", - "revision": "1267", + "revision": "1268", "installByDefault": false, - "browserVersion": "131.0.6764.0" + "browserVersion": "131.0.6768.0" }, { "name": "firefox", From 82fe882004980e3709ebbcd52a46f99fa0691306 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Thu, 10 Oct 2024 14:32:27 -0700 Subject: [PATCH 5/7] fix(webkit): scroll mobile page with background-attachment: fixed (#33048) Fixes #31551 Fixes #23573 --- packages/playwright-core/src/server/webkit/wkPage.ts | 1 + tests/library/browsercontext-viewport-mobile.spec.ts | 11 +++++------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/playwright-core/src/server/webkit/wkPage.ts b/packages/playwright-core/src/server/webkit/wkPage.ts index 3ba773241a..0bd82ed338 100644 --- a/packages/playwright-core/src/server/webkit/wkPage.ts +++ b/packages/playwright-core/src/server/webkit/wkPage.ts @@ -231,6 +231,7 @@ export class WKPage implements PageDelegate { promises.push(session.send('Page.overrideSetting', { setting: 'PointerLockEnabled', value: !contextOptions.isMobile })); promises.push(session.send('Page.overrideSetting', { setting: 'InputTypeMonthEnabled', value: contextOptions.isMobile })); promises.push(session.send('Page.overrideSetting', { setting: 'InputTypeWeekEnabled', value: contextOptions.isMobile })); + promises.push(session.send('Page.overrideSetting', { setting: 'FixedBackgroundsPaintRelativeToDocument', value: contextOptions.isMobile })); await Promise.all(promises); } diff --git a/tests/library/browsercontext-viewport-mobile.spec.ts b/tests/library/browsercontext-viewport-mobile.spec.ts index ed774e383a..58c85d1a07 100644 --- a/tests/library/browsercontext-viewport-mobile.spec.ts +++ b/tests/library/browsercontext-viewport-mobile.spec.ts @@ -54,7 +54,7 @@ it.describe('mobile viewport', () => { } }); - it('should be detectable', async ({ playwright, browser, server, browserName, platform }) => { + it('should be detectable', async ({ playwright, browser }) => { const iPhone = playwright.devices['iPhone 6']; const context = await browser.newContext({ ...iPhone }); const page = await context.newPage(); @@ -62,7 +62,7 @@ it.describe('mobile viewport', () => { await context.close(); }); - it('should detect touch when applying viewport with touches', async ({ browser, server, browserName, platform }) => { + it('should detect touch when applying viewport with touches', async ({ browser, server }) => { const context = await browser.newContext({ viewport: { width: 800, height: 600 }, hasTouch: true }); const page = await context.newPage(); await page.goto(server.EMPTY_PAGE); @@ -154,7 +154,7 @@ it.describe('mobile viewport', () => { await desktopPage.close(); }); - it('mouse should work with mobile viewports and cross process navigations', async ({ browser, server, browserName }) => { + it('mouse should work with mobile viewports and cross process navigations', async ({ browser, server }) => { // @see https://crbug.com/929806 const context = await browser.newContext({ viewport: { width: 360, height: 640 }, isMobile: true }); const page = await context.newPage(); @@ -193,8 +193,7 @@ it.describe('mobile viewport', () => { { type: 'issue', description: 'https://github.com/microsoft/playwright/issues/31551' }, { type: 'issue', description: 'https://github.com/microsoft/playwright/issues/23573' }, ] - }, async ({ playwright, browser, server, browserName, isLinux, headless }) => { - it.fixme(browserName === 'webkit' && isLinux && headless, 'Fails on WPE apparently due to accelerated compositing + fixed layout'); + }, async ({ playwright, browser, server }) => { const iPhone = playwright.devices['iPhone 12']; const context = await browser.newContext({ ...iPhone }); const page = await context.newPage(); @@ -204,7 +203,7 @@ it.describe('mobile viewport', () => { await context.close(); }); - it('view scale should reset after navigation', async ({ browser, browserName }) => { + it('view scale should reset after navigation', async ({ browser }) => { it.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/26876' }); const context = await browser.newContext({ viewport: { width: 390, height: 664 }, From 10a9e1c730703a8f857b85d3338205d79805a5a8 Mon Sep 17 00:00:00 2001 From: Playwright Service <89237858+playwrightmachine@users.noreply.github.com> Date: Thu, 10 Oct 2024 15:08:03 -0700 Subject: [PATCH 6/7] feat(webkit): roll to r2090 (#33050) --- packages/playwright-core/browsers.json | 2 +- tests/page/page-goto.spec.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/playwright-core/browsers.json b/packages/playwright-core/browsers.json index 1fbe52c90d..8cfd41cd8c 100644 --- a/packages/playwright-core/browsers.json +++ b/packages/playwright-core/browsers.json @@ -27,7 +27,7 @@ }, { "name": "webkit", - "revision": "2089", + "revision": "2090", "installByDefault": true, "revisionOverrides": { "mac10.14": "1446", diff --git a/tests/page/page-goto.spec.ts b/tests/page/page-goto.spec.ts index ccab5abeba..e3ffb1f93c 100644 --- a/tests/page/page-goto.spec.ts +++ b/tests/page/page-goto.spec.ts @@ -258,7 +258,8 @@ it('should work with subframes return 204 with domcontentloaded', async ({ page, await page.goto(server.PREFIX + '/frames/one-frame.html', { waitUntil: 'domcontentloaded' }); }); -it('should fail when server returns 204', async ({ page, server, browserName }) => { +it('should fail when server returns 204', async ({ page, server, browserName, isLinux }) => { + it.fixme(browserName === 'webkit' && isLinux, 'Regressed in https://github.com/microsoft/playwright-browsers/pull/1297'); // WebKit just loads an empty page. server.setRoute('/empty.html', (req, res) => { res.statusCode = 204; From b9cce598dd9cc3af1246e7c8317ade5e3a43deb0 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Thu, 10 Oct 2024 16:49:17 -0700 Subject: [PATCH 7/7] fix(screenshot): show image diff inline in errors list (#32997) The diff is now shown inline in the errors list. There are 2 possible failures of toHaveScreenshot * Previous and actual snapshot mismatch. In this case html report will show diff between Actual/Previous and have Expected as a separate screenshot. * Actual/Previous are equal but they differ from the expected. In this case html report only contains Actual/Expected images and the diff. Reference: https://github.com/microsoft/playwright/issues/32341 image --- .gitignore | 1 + packages/html-reporter/src/testErrorView.css | 7 +- packages/html-reporter/src/testErrorView.tsx | 36 +++++++--- packages/html-reporter/src/testResultView.tsx | 40 +++++++++-- packages/playwright-core/src/server/page.ts | 9 ++- .../src/matchers/toMatchSnapshot.ts | 4 +- packages/web/src/shared/imageDiffView.tsx | 46 ++++++------ tests/playwright-test/reporter-html.spec.ts | 72 +++++++++++-------- 8 files changed, 142 insertions(+), 73 deletions(-) diff --git a/.gitignore b/.gitignore index aadc481067..1f3b9a7a72 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,4 @@ test-results .cache/ .eslintcache playwright.env +firefox diff --git a/packages/html-reporter/src/testErrorView.css b/packages/html-reporter/src/testErrorView.css index afb543a0c2..d5a4534e7e 100644 --- a/packages/html-reporter/src/testErrorView.css +++ b/packages/html-reporter/src/testErrorView.css @@ -14,9 +14,8 @@ limitations under the License. */ -.test-error-message { +.test-error-view { white-space: pre; - font-family: monospace; overflow: auto; flex: none; padding: 0; @@ -26,3 +25,7 @@ line-height: initial; margin-bottom: 6px; } + +.test-error-text { + font-family: monospace; +} diff --git a/packages/html-reporter/src/testErrorView.tsx b/packages/html-reporter/src/testErrorView.tsx index 5208158b1c..520da1fc19 100644 --- a/packages/html-reporter/src/testErrorView.tsx +++ b/packages/html-reporter/src/testErrorView.tsx @@ -17,21 +17,39 @@ import ansi2html from 'ansi-to-html'; import * as React from 'react'; import './testErrorView.css'; +import type { ImageDiff } from '@web/shared/imageDiffView'; +import { ImageDiffView } from '@web/shared/imageDiffView'; export const TestErrorView: React.FC<{ error: string; }> = ({ error }) => { - const html = React.useMemo(() => { - const config: any = { - bg: 'var(--color-canvas-subtle)', - fg: 'var(--color-fg-default)', - }; - config.colors = ansiColors; - return new ansi2html(config).toHtml(escapeHTML(error)); - }, [error]); - return
; + const html = React.useMemo(() => ansiErrorToHtml(error), [error]); + return
; }; +export const TestScreenshotErrorView: React.FC<{ + errorPrefix?: string, + diff: ImageDiff, + errorSuffix?: string, +}> = ({ errorPrefix, diff, errorSuffix }) => { + const prefixHtml = React.useMemo(() => ansiErrorToHtml(errorPrefix), [errorPrefix]); + const suffixHtml = React.useMemo(() => ansiErrorToHtml(errorSuffix), [errorSuffix]); + return
+
+ +
+
; +}; + +function ansiErrorToHtml(text?: string): string { + const config: any = { + bg: 'var(--color-canvas-subtle)', + fg: 'var(--color-fg-default)', + }; + config.colors = ansiColors; + return new ansi2html(config).toHtml(escapeHTML(text || '')); +} + const ansiColors = { 0: '#000', 1: '#C00', diff --git a/packages/html-reporter/src/testResultView.tsx b/packages/html-reporter/src/testResultView.tsx index 8ee36d0cda..273703a0c7 100644 --- a/packages/html-reporter/src/testResultView.tsx +++ b/packages/html-reporter/src/testResultView.tsx @@ -24,7 +24,7 @@ import { AttachmentLink, generateTraceUrl } from './links'; import { statusIcon } from './statusIcon'; import type { ImageDiff } from '@web/shared/imageDiffView'; import { ImageDiffView } from '@web/shared/imageDiffView'; -import { TestErrorView } from './testErrorView'; +import { TestErrorView, TestScreenshotErrorView } from './testErrorView'; import './testResultView.css'; function groupImageDiffs(screenshots: Set): ImageDiff[] { @@ -67,7 +67,7 @@ export const TestResultView: React.FC<{ anchor: 'video' | 'diff' | '', }> = ({ result, anchor }) => { - const { screenshots, videos, traces, otherAttachments, diffs, htmls } = React.useMemo(() => { + const { screenshots, videos, traces, otherAttachments, diffs, errors, htmls } = React.useMemo(() => { const attachments = result?.attachments || []; const screenshots = new Set(attachments.filter(a => a.contentType.startsWith('image/'))); const videos = attachments.filter(a => a.name === 'video'); @@ -76,7 +76,8 @@ export const TestResultView: React.FC<{ const otherAttachments = new Set(attachments); [...screenshots, ...videos, ...traces, ...htmls].forEach(a => otherAttachments.delete(a)); const diffs = groupImageDiffs(screenshots); - return { screenshots: [...screenshots], videos, traces, otherAttachments, diffs, htmls }; + const errors = classifyErrors(result.errors, diffs); + return { screenshots: [...screenshots], videos, traces, otherAttachments, diffs, errors, htmls }; }, [result]); const videoRef = React.useRef(null); @@ -94,15 +95,19 @@ export const TestResultView: React.FC<{ }, [scrolled, anchor, setScrolled, videoRef]); return
- {!!result.errors.length && - {result.errors.map((error, index) => )} + {!!errors.length && + {errors.map((error, index) => { + if (error.type === 'screenshot') + return ; + return ; + })} } {!!result.steps.length && {result.steps.map((step, i) => )} } {diffs.map((diff, index) => - + )} @@ -145,6 +150,29 @@ export const TestResultView: React.FC<{
; }; +function classifyErrors(testErrors: string[], diffs: ImageDiff[]) { + return testErrors.map(error => { + if (error.includes('Screenshot comparison failed:')) { + const matchingDiff = diffs.find(diff => { + const attachmentName = diff.actual?.attachment.name; + return attachmentName && error.includes(attachmentName); + }); + + if (matchingDiff) { + const lines = error.split('\n'); + const index = lines.findIndex(line => /Expected:|Previous:|Received:/.test(line)); + const errorPrefix = index !== -1 ? lines.slice(0, index).join('\n') : lines[0]; + + const diffIndex = lines.findIndex(line => / +Diff:/.test(line)); + const errorSuffix = diffIndex !== -1 ? lines.slice(diffIndex + 2).join('\n') : lines.slice(1).join('\n'); + + return { type: 'screenshot', diff: matchingDiff, errorPrefix, errorSuffix }; + } + } + return { type: 'regular', error }; + }); +} + const StepTreeItem: React.FC<{ step: TestStep; depth: number, diff --git a/packages/playwright-core/src/server/page.ts b/packages/playwright-core/src/server/page.ts index f7f1ef67e8..b78bd91ee1 100644 --- a/packages/playwright-core/src/server/page.ts +++ b/packages/playwright-core/src/server/page.ts @@ -43,7 +43,7 @@ import type { TimeoutOptions } from '../common/types'; import { isInvalidSelectorError } from '../utils/isomorphic/selectorParser'; import { parseEvaluationResultValue, source } from './isomorphic/utilityScriptSerializers'; import type { SerializedValue } from './isomorphic/utilityScriptSerializers'; -import { TargetClosedError } from './errors'; +import { TargetClosedError, TimeoutError } from './errors'; import { asLocator } from '../utils'; import { helper } from './helper'; @@ -662,7 +662,7 @@ export class Page extends SdkObject { return {}; } - if (areEqualScreenshots(actual, options.expected, previous)) { + if (areEqualScreenshots(actual, options.expected, undefined)) { progress.log(`screenshot matched expectation`); return {}; } @@ -672,10 +672,13 @@ export class Page extends SdkObject { // A: We want user to receive a friendly diff between actual and expected/previous. if (js.isJavaScriptErrorInEvaluate(e) || isInvalidSelectorError(e)) throw e; + let errorMessage = e.message; + if (e instanceof TimeoutError && intermediateResult?.previous) + errorMessage = `Failed to take two consecutive stable screenshots. ${e.message}`; return { log: e.message ? [...metadata.log, e.message] : metadata.log, ...intermediateResult, - errorMessage: e.message, + errorMessage, }; }); } diff --git a/packages/playwright/src/matchers/toMatchSnapshot.ts b/packages/playwright/src/matchers/toMatchSnapshot.ts index 0923a10e82..b3fa3f556e 100644 --- a/packages/playwright/src/matchers/toMatchSnapshot.ts +++ b/packages/playwright/src/matchers/toMatchSnapshot.ts @@ -423,7 +423,7 @@ export async function toHaveScreenshot( // - regular matcher (i.e. not a `.not`) // - perhaps an 'all' flag to update non-matching screenshots expectScreenshotOptions.expected = await fs.promises.readFile(helper.expectedPath); - const { actual, diff, errorMessage, log } = await page._expectScreenshot(expectScreenshotOptions); + const { actual, previous, diff, errorMessage, log } = await page._expectScreenshot(expectScreenshotOptions); if (!errorMessage) return helper.handleMatching(); @@ -436,7 +436,7 @@ export async function toHaveScreenshot( return helper.createMatcherResult(helper.expectedPath + ' running with --update-snapshots, writing actual.', true); } - return helper.handleDifferent(actual, expectScreenshotOptions.expected, undefined, diff, errorMessage, log); + return helper.handleDifferent(actual, expectScreenshotOptions.expected, previous, diff, errorMessage, log); } function writeFileSync(aPath: string, content: Buffer | string) { diff --git a/packages/web/src/shared/imageDiffView.tsx b/packages/web/src/shared/imageDiffView.tsx index ea0f1e0042..6f0028bca9 100644 --- a/packages/web/src/shared/imageDiffView.tsx +++ b/packages/web/src/shared/imageDiffView.tsx @@ -61,11 +61,13 @@ const checkerboardStyle: React.CSSProperties = { export const ImageDiffView: React.FC<{ diff: ImageDiff, noTargetBlank?: boolean, -}> = ({ diff, noTargetBlank }) => { + hideDetails?: boolean, +}> = ({ diff, noTargetBlank, hideDetails }) => { const [mode, setMode] = React.useState<'diff' | 'actual' | 'expected' | 'slider' | 'sxs'>(diff.diff ? 'diff' : 'actual'); const [showSxsDiff, setShowSxsDiff] = React.useState(false); const [expectedImage, setExpectedImage] = React.useState(null); + const [expectedImageTitle, setExpectedImageTitle] = React.useState('Expected'); const [actualImage, setActualImage] = React.useState(null); const [diffImage, setDiffImage] = React.useState(null); const [measure, ref] = useMeasure(); @@ -73,6 +75,7 @@ export const ImageDiffView: React.FC<{ React.useEffect(() => { (async () => { setExpectedImage(await loadImage(diff.expected?.attachment.path)); + setExpectedImageTitle(diff.expected?.title || 'Expected'); setActualImage(await loadImage(diff.actual?.attachment.path)); setDiffImage(await loadImage(diff.diff?.attachment.path)); })(); @@ -98,31 +101,31 @@ export const ImageDiffView: React.FC<{
{diff.diff &&
setMode('diff')}>Diff
}
setMode('actual')}>Actual
-
setMode('expected')}>Expected
+
setMode('expected')}>{expectedImageTitle}
setMode('sxs')}>Side by side
setMode('slider')}>Slider
- {diff.diff && mode === 'diff' && } - {diff.diff && mode === 'actual' && } - {diff.diff && mode === 'expected' && } - {diff.diff && mode === 'slider' && } + {diff.diff && mode === 'diff' && } + {diff.diff && mode === 'actual' && } + {diff.diff && mode === 'expected' && } + {diff.diff && mode === 'slider' && } {diff.diff && mode === 'sxs' &&
- - setShowSxsDiff(!showSxsDiff)} canvasWidth={sxsScale * imageWidth} canvasHeight={sxsScale * imageHeight} scale={sxsScale} /> + + setShowSxsDiff(!showSxsDiff)} hideSize={hideDetails} canvasWidth={sxsScale * imageWidth} canvasHeight={sxsScale * imageHeight} scale={sxsScale} />
} - {!diff.diff && mode === 'actual' && } - {!diff.diff && mode === 'expected' && } + {!diff.diff && mode === 'actual' && } + {!diff.diff && mode === 'expected' && } {!diff.diff && mode === 'sxs' &&
- +
}
- } } ; }; @@ -133,7 +136,9 @@ export const ImageDiffSlider: React.FC<{ canvasWidth: number, canvasHeight: number, scale: number, -}> = ({ expectedImage, actualImage, canvasWidth, canvasHeight, scale }) => { + expectedTitle: string, + hideSize?: boolean, +}> = ({ expectedImage, actualImage, canvasWidth, canvasHeight, scale, expectedTitle, hideSize }) => { const absoluteStyle: React.CSSProperties = { position: 'absolute', top: 0, @@ -144,7 +149,7 @@ export const ImageDiffSlider: React.FC<{ const sameSize = expectedImage.naturalWidth === actualImage.naturalWidth && expectedImage.naturalHeight === actualImage.naturalHeight; return
-
+ {!hideSize &&
{!sameSize && Expected } {expectedImage.naturalWidth} x @@ -153,7 +158,7 @@ export const ImageDiffSlider: React.FC<{ {!sameSize && {actualImage.naturalWidth}} {!sameSize && x} {!sameSize && {actualImage.naturalHeight}} -
+
}
setSlider(offsets[0])} resizerColor={'#57606a80'} resizerWidth={6}> - Expected @@ -179,18 +184,19 @@ const ImageWithSize: React.FunctionComponent<{ image: HTMLImageElement, title?: string, alt?: string, + hideSize?: boolean, canvasWidth: number, canvasHeight: number, scale: number, onClick?: () => void; -}> = ({ image, title, alt, canvasWidth, canvasHeight, scale, onClick }) => { +}> = ({ image, title, alt, hideSize, canvasWidth, canvasHeight, scale, onClick }) => { return
-
+ {!hideSize &&
{title && {title}} {image.naturalWidth} x {image.naturalHeight} -
+
}
{ - await expect(imageDiff.locator('img')).toHaveAttribute('alt', 'Diff'); - }); + for (const testId of ['test-results-image-diff', 'test-screenshot-error-view']) { + await test.step(testId, async () => { + const imageDiff = page.getByTestId(testId).getByTestId('test-result-image-mismatch'); + await test.step('Diff', async () => { + await expect(imageDiff.locator('img')).toHaveAttribute('alt', 'Diff'); + }); - await test.step('Actual', async () => { - await imageDiff.getByText('Actual', { exact: true }).click(); - await expect(imageDiff.locator('img')).toHaveAttribute('alt', 'Actual'); - }); + await test.step('Actual', async () => { + await imageDiff.getByText('Actual', { exact: true }).click(); + await expect(imageDiff.locator('img')).toHaveAttribute('alt', 'Actual'); + }); - await test.step('Expected', async () => { - await imageDiff.getByText('Expected', { exact: true }).click(); - await expect(imageDiff.locator('img')).toHaveAttribute('alt', 'Expected'); - }); + await test.step('Expected', async () => { + await imageDiff.getByText('Expected', { exact: true }).click(); + await expect(imageDiff.locator('img')).toHaveAttribute('alt', 'Expected'); + }); - await test.step('Side by side', async () => { - await imageDiff.getByText('Side by side').click(); - await expect(imageDiff.locator('img')).toHaveCount(2); - await expect(imageDiff.locator('img').first()).toHaveAttribute('alt', 'Expected'); - await expect(imageDiff.locator('img').last()).toHaveAttribute('alt', 'Actual'); - await imageDiff.locator('img').last().click(); - await expect(imageDiff.locator('img').last()).toHaveAttribute('alt', 'Diff'); - }); + await test.step('Side by side', async () => { + await imageDiff.getByText('Side by side').click(); + await expect(imageDiff.locator('img')).toHaveCount(2); + await expect(imageDiff.locator('img').first()).toHaveAttribute('alt', 'Expected'); + await expect(imageDiff.locator('img').last()).toHaveAttribute('alt', 'Actual'); + await imageDiff.locator('img').last().click(); + await expect(imageDiff.locator('img').last()).toHaveAttribute('alt', 'Diff'); + }); - await test.step('Slider', async () => { - await imageDiff.getByText('Slider', { exact: true }).click(); - await expect(imageDiff.locator('img')).toHaveCount(2); - await expect(imageDiff.locator('img').first()).toHaveAttribute('alt', 'Expected'); - await expect(imageDiff.locator('img').last()).toHaveAttribute('alt', 'Actual'); - }); + await test.step('Slider', async () => { + await imageDiff.getByText('Slider', { exact: true }).click(); + await expect(imageDiff.locator('img')).toHaveCount(2); + await expect(imageDiff.locator('img').first()).toHaveAttribute('alt', 'Expected'); + await expect(imageDiff.locator('img').last()).toHaveAttribute('alt', 'Actual'); + }); + }); + } }); test('should include multiple image diffs', async ({ runInlineTest, page, showReport }) => { @@ -285,8 +289,14 @@ for (const useIntermediateMergeReport of [false] as const) { await showReport(); await page.click('text=fails'); - await expect(page.locator('data-testid=test-result-image-mismatch')).toHaveCount(3); - await expect(page.locator('text=Image mismatch:')).toHaveText([ + await expect(page.getByTestId('test-screenshot-error-view').getByTestId('error-suffix')).toContainText([ + `> 6 | await expect.soft(screenshot).toMatchSnapshot('expected.png');`, + `> 7 | await expect.soft(screenshot).toMatchSnapshot('expected.png');`, + `> 8 | await expect.soft(screenshot).toMatchSnapshot('expected.png');`, + ]); + const imageDiffs = page.getByTestId('test-results-image-diff'); + await expect(imageDiffs.getByTestId('test-result-image-mismatch')).toHaveCount(3); + await expect(imageDiffs.getByText('Image mismatch:')).toHaveText([ 'Image mismatch: expected.png', 'Image mismatch: expected-1.png', 'Image mismatch: expected-2.png', @@ -323,7 +333,7 @@ for (const useIntermediateMergeReport of [false] as const) { await expect(page.getByTestId('test-result-image-mismatch-tabs').locator('div')).toHaveText([ 'Diff', 'Actual', - 'Expected', + 'Previous', 'Side by side', 'Slider', ]); @@ -460,7 +470,7 @@ for (const useIntermediateMergeReport of [false] as const) { await showReport(); await page.click('text=fails'); - await expect(page.locator('.test-error-message span:has-text("received")').nth(1)).toHaveCSS('color', 'rgb(204, 0, 0)'); + await expect(page.locator('.test-error-view span:has-text("received")').nth(1)).toHaveCSS('color', 'rgb(204, 0, 0)'); }); test('should show trace source', async ({ runInlineTest, page, showReport }) => {