diff --git a/packages/playwright-core/src/server/dispatchers/pageDispatcher.ts b/packages/playwright-core/src/server/dispatchers/pageDispatcher.ts index 48f042f902..70ccf16638 100644 --- a/packages/playwright-core/src/server/dispatchers/pageDispatcher.ts +++ b/packages/playwright-core/src/server/dispatchers/pageDispatcher.ts @@ -235,19 +235,19 @@ export class PageDispatcher extends Dispatcher { - await this._page.mouse.move(params.x, params.y, params); + await this._page.mouse.move(params.x, params.y, params, metadata); } async mouseDown(params: channels.PageMouseDownParams, metadata: CallMetadata): Promise { - await this._page.mouse.down(params); + await this._page.mouse.down(params, metadata); } async mouseUp(params: channels.PageMouseUpParams, metadata: CallMetadata): Promise { - await this._page.mouse.up(params); + await this._page.mouse.up(params, metadata); } async mouseClick(params: channels.PageMouseClickParams, metadata: CallMetadata): Promise { - await this._page.mouse.click(params.x, params.y, params); + await this._page.mouse.click(params.x, params.y, params, metadata); } async mouseWheel(params: channels.PageMouseWheelParams, metadata: CallMetadata): Promise { @@ -255,7 +255,7 @@ export class PageDispatcher extends Dispatcher { - await this._page.touchscreen.tap(params.x, params.y); + await this._page.touchscreen.tap(params.x, params.y, metadata); } async accessibilitySnapshot(params: channels.PageAccessibilitySnapshotParams, metadata: CallMetadata): Promise { diff --git a/packages/playwright-core/src/server/input.ts b/packages/playwright-core/src/server/input.ts index 973743f115..d6cc5bbd55 100644 --- a/packages/playwright-core/src/server/input.ts +++ b/packages/playwright-core/src/server/input.ts @@ -18,6 +18,7 @@ import { assert } from '../utils'; import * as keyboardLayout from './usKeyboardLayout'; import type * as types from './types'; import type { Page } from './page'; +import type { CallMetadata } from './instrumentation'; export const keypadLocation = keyboardLayout.keypadLocation; @@ -169,7 +170,9 @@ export class Mouse { this._keyboard = this._page.keyboard; } - async move(x: number, y: number, options: { steps?: number, forClick?: boolean } = {}) { + async move(x: number, y: number, options: { steps?: number, forClick?: boolean } = {}, metadata?: CallMetadata) { + if (metadata) + metadata.point = { x, y }; const { steps = 1 } = options; const fromX = this._x; const fromY = this._y; @@ -182,21 +185,27 @@ export class Mouse { } } - async down(options: { button?: types.MouseButton, clickCount?: number } = {}) { + async down(options: { button?: types.MouseButton, clickCount?: number } = {}, metadata?: CallMetadata) { + if (metadata) + metadata.point = { x: this._x, y: this._y }; const { button = 'left', clickCount = 1 } = options; this._lastButton = button; this._buttons.add(button); await this._raw.down(this._x, this._y, this._lastButton, this._buttons, this._keyboard._modifiers(), clickCount); } - async up(options: { button?: types.MouseButton, clickCount?: number } = {}) { + async up(options: { button?: types.MouseButton, clickCount?: number } = {}, metadata?: CallMetadata) { + if (metadata) + metadata.point = { x: this._x, y: this._y }; const { button = 'left', clickCount = 1 } = options; this._lastButton = 'none'; this._buttons.delete(button); await this._raw.up(this._x, this._y, button, this._buttons, this._keyboard._modifiers(), clickCount); } - async click(x: number, y: number, options: { delay?: number, button?: types.MouseButton, clickCount?: number } = {}) { + async click(x: number, y: number, options: { delay?: number, button?: types.MouseButton, clickCount?: number } = {}, metadata?: CallMetadata) { + if (metadata) + metadata.point = { x: this._x, y: this._y }; const { delay = null, clickCount = 1 } = options; if (delay) { this.move(x, y, { forClick: true }); @@ -300,7 +309,9 @@ export class Touchscreen { this._page = page; } - async tap(x: number, y: number) { + async tap(x: number, y: number, metadata?: CallMetadata) { + if (metadata) + metadata.point = { x, y }; if (!this._page._browserContext._options.hasTouch) throw new Error('hasTouch must be enabled on the browser context before using the touchscreen.'); await this._raw.tap(x, y, this._page.keyboard._modifiers()); diff --git a/packages/playwright-core/src/server/trace/recorder/tracing.ts b/packages/playwright-core/src/server/trace/recorder/tracing.ts index 131f87ae2c..df41894c05 100644 --- a/packages/playwright-core/src/server/trace/recorder/tracing.ts +++ b/packages/playwright-core/src/server/trace/recorder/tracing.ts @@ -565,6 +565,7 @@ function createAfterActionTraceEvent(metadata: CallMetadata): trace.AfterActionT endTime: metadata.endTime, error: metadata.error?.error, result: metadata.result, + point: metadata.point, }; } diff --git a/packages/trace-viewer/src/snapshotRenderer.ts b/packages/trace-viewer/src/snapshotRenderer.ts index 625f011138..81e62126c5 100644 --- a/packages/trace-viewer/src/snapshotRenderer.ts +++ b/packages/trace-viewer/src/snapshotRenderer.ts @@ -251,7 +251,7 @@ function snapshotScript(...targetIds: (string | undefined)[]) { if (!src) { iframe.setAttribute('src', 'data:text/html,'); } else { - // Retain query parameters to inherit name=, time=, showPoint= and other values from parent. + // Retain query parameters to inherit name=, time=, pointX=, pointY= and other values from parent. const url = new URL(unwrapPopoutUrl(window.location.href)); // We can be loading iframe from within iframe, reset base to be absolute. const index = url.pathname.lastIndexOf('/snapshot/'); @@ -305,8 +305,13 @@ function snapshotScript(...targetIds: (string | undefined)[]) { document.styleSheets[0].disabled = true; const search = new URL(window.location.href).searchParams; - if (search.get('showPoint')) { - for (const target of targetElements) { + + if (search.get('pointX') && search.get('pointY')) { + const pointX = +search.get('pointX')!; + const pointY = +search.get('pointY')!; + const hasTargetElements = targetElements.length > 0; + const roots = document.documentElement ? [document.documentElement] : []; + for (const target of (hasTargetElements ? targetElements : roots)) { const pointElement = document.createElement('x-pw-pointer'); pointElement.style.position = 'fixed'; pointElement.style.backgroundColor = '#f44336'; @@ -315,9 +320,24 @@ function snapshotScript(...targetIds: (string | undefined)[]) { pointElement.style.borderRadius = '10px'; pointElement.style.margin = '-10px 0 0 -10px'; pointElement.style.zIndex = '2147483646'; - const box = target.getBoundingClientRect(); - pointElement.style.left = (box.left + box.width / 2) + 'px'; - pointElement.style.top = (box.top + box.height / 2) + 'px'; + if (hasTargetElements) { + // Sometimes there are layout discrepancies between recording and rendering, e.g. fonts, + // that may place the point at the wrong place. To avoid confusion, we just show the + // point in the middle of the target element. + const box = target.getBoundingClientRect(); + const centerX = (box.left + box.width / 2); + const centerY = (box.top + box.height / 2); + pointElement.style.left = centerX + 'px'; + pointElement.style.top = centerY + 'px'; + // "Blue dot" to indicate that action point is not 100% correct. + if (Math.abs(centerX - pointX) >= 2 || Math.abs(centerY - pointY) >= 2) + pointElement.style.backgroundColor = '#3646f4'; + } else { + // For actions without a target element, e.g. page.mouse.move(), + // show the point at the recorder location. + pointElement.style.left = pointX + 'px'; + pointElement.style.top = pointY + 'px'; + } document.documentElement.appendChild(pointElement); } } diff --git a/packages/trace-viewer/src/traceModel.ts b/packages/trace-viewer/src/traceModel.ts index c80ab25474..3aa17c28e9 100644 --- a/packages/trace-viewer/src/traceModel.ts +++ b/packages/trace-viewer/src/traceModel.ts @@ -206,6 +206,8 @@ export class TraceModel { existing!.result = event.result; existing!.error = event.error; existing!.attachments = event.attachments; + if (event.point) + existing!.point = event.point; for (const attachment of event.attachments?.filter(a => a.sha1) || []) this._attachments.set(attachment.sha1!, attachment); break; diff --git a/packages/trace-viewer/src/ui/snapshotTab.tsx b/packages/trace-viewer/src/ui/snapshotTab.tsx index 9d0a03948b..9ae19d9adf 100644 --- a/packages/trace-viewer/src/ui/snapshotTab.tsx +++ b/packages/trace-viewer/src/ui/snapshotTab.tsx @@ -42,7 +42,7 @@ export const SnapshotTab: React.FunctionComponent<{ const [measure, ref] = useMeasure(); const [snapshotTab, setSnapshotTab] = React.useState<'action'|'before'|'after'>('action'); - type Snapshot = { action: ActionTraceEvent, snapshotName: string, showPoint?: boolean }; + type Snapshot = { action: ActionTraceEvent, snapshotName: string, point?: { x: number, y: number } }; const { snapshots } = React.useMemo(() => { if (!action) return { snapshots: {} }; @@ -55,7 +55,9 @@ export const SnapshotTab: React.FunctionComponent<{ beforeSnapshot = a?.afterSnapshot ? { action: a, snapshotName: a?.afterSnapshot } : undefined; } const afterSnapshot: Snapshot | undefined = action.afterSnapshot ? { action, snapshotName: action.afterSnapshot } : beforeSnapshot; - const actionSnapshot: Snapshot | undefined = action.inputSnapshot ? { action, snapshotName: action.inputSnapshot, showPoint: !!action.point } : afterSnapshot; + const actionSnapshot: Snapshot | undefined = action.inputSnapshot ? { action, snapshotName: action.inputSnapshot } : afterSnapshot; + if (actionSnapshot) + actionSnapshot.point = action.point; return { snapshots: { action: actionSnapshot, before: beforeSnapshot, after: afterSnapshot } }; }, [action]); @@ -67,16 +69,20 @@ export const SnapshotTab: React.FunctionComponent<{ const params = new URLSearchParams(); params.set('trace', context(snapshot.action).traceUrl); params.set('name', snapshot.snapshotName); - if (snapshot.showPoint) - params.set('showPoint', '1'); + if (snapshot.point) { + params.set('pointX', String(snapshot.point.x)); + params.set('pointY', String(snapshot.point.y)); + } const snapshotUrl = new URL(`snapshot/${snapshot.action.pageId}?${params.toString()}`, window.location.href).toString(); const snapshotInfoUrl = new URL(`snapshotInfo/${snapshot.action.pageId}?${params.toString()}`, window.location.href).toString(); const popoutParams = new URLSearchParams(); popoutParams.set('r', snapshotUrl); popoutParams.set('trace', context(snapshot.action).traceUrl); - if (snapshot.showPoint) - popoutParams.set('showPoint', '1'); + if (snapshot.point) { + popoutParams.set('pointX', String(snapshot.point.x)); + popoutParams.set('pointY', String(snapshot.point.y)); + } const popoutUrl = new URL(`snapshot.html?${popoutParams.toString()}`, window.location.href).toString(); return { snapshots, snapshotInfoUrl, snapshotUrl, popoutUrl }; }, [snapshots, snapshotTab]); diff --git a/packages/trace/src/trace.ts b/packages/trace/src/trace.ts index 651c41dc19..2f4974aae4 100644 --- a/packages/trace/src/trace.ts +++ b/packages/trace/src/trace.ts @@ -90,6 +90,7 @@ export type AfterActionTraceEvent = { error?: SerializedError['error']; attachments?: AfterActionTraceEventAttachment[]; result?: any; + point?: Point; }; export type LogTraceEvent = { diff --git a/tests/library/trace-viewer.spec.ts b/tests/library/trace-viewer.spec.ts index b9e49da647..1d126060f4 100644 --- a/tests/library/trace-viewer.spec.ts +++ b/tests/library/trace-viewer.spec.ts @@ -626,6 +626,7 @@ test('should highlight target elements', async ({ page, runAndTrace, browserName await page.locator('text=t5').innerText(); await expect(page.locator('text=t6')).toHaveText(/t6/i); await expect(page.locator('text=multi')).toHaveText(['a', 'b'], { timeout: 1000 }).catch(() => {}); + await page.mouse.move(123, 234); }); async function highlightedDivs(frameLocator: FrameLocator) { @@ -637,6 +638,14 @@ test('should highlight target elements', async ({ page, runAndTrace, browserName const framePageClick = await traceViewer.snapshotFrame('page.click'); await expect.poll(() => highlightedDivs(framePageClick)).toEqual(['t1']); + const box1 = await framePageClick.getByText('t1').boundingBox(); + const box2 = await framePageClick.locator('x-pw-pointer').boundingBox(); + const x1 = box1!.x + box1!.width / 2; + const y1 = box1!.y + box1!.height / 2; + const x2 = box2!.x + box2!.width / 2; + const y2 = box2!.y + box2!.height / 2; + expect(Math.abs(x1 - x2) < 2).toBeTruthy(); + expect(Math.abs(y1 - y2) < 2).toBeTruthy(); const framePageInnerText = await traceViewer.snapshotFrame('page.innerText'); await expect.poll(() => highlightedDivs(framePageInnerText)).toEqual(['t2']); @@ -655,6 +664,10 @@ test('should highlight target elements', async ({ page, runAndTrace, browserName const frameExpect2 = await traceViewer.snapshotFrame('expect.toHaveText', 1); await expect.poll(() => highlightedDivs(frameExpect2)).toEqual(['multi', 'multi']); + await expect(frameExpect2.locator('x-pw-pointer')).not.toBeVisible(); + + const frameMouseMove = await traceViewer.snapshotFrame('mouse.move'); + await expect(frameMouseMove.locator('x-pw-pointer')).toBeVisible(); }); test('should highlight target element in shadow dom', async ({ page, server, runAndTrace }) => {