playwright/src/server/supplements/injected/recorder.ts
Dmitry Gozman 06b0619260
fix(codegen): do not commit last action on mouse move (#6252)
On a slow page that does a lot of things before navigating upon click,
it is common to move mouse away from the click point. Previously,
we would commit the click action and record a `page.goto()` for the
navigation. Now we attribute any signals, even after accidental mouse move,
to the previous action, in the 5-seconds time window.
2021-04-20 18:45:52 -07:00

641 lines
23 KiB
TypeScript

/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type * as actions from '../recorder/recorderActions';
import type InjectedScript from '../../injected/injectedScript';
import { generateSelector, querySelector } from './selectorGenerator';
import type { Point } from '../../../common/types';
import type { UIState } from '../recorder/recorderTypes';
declare global {
interface Window {
_playwrightRecorderPerformAction: (action: actions.Action) => Promise<void>;
_playwrightRecorderRecordAction: (action: actions.Action) => Promise<void>;
_playwrightRecorderState: () => Promise<UIState>;
_playwrightRecorderSetSelector: (selector: string) => Promise<void>;
_playwrightRefreshOverlay: () => void;
}
}
const scriptSymbol = Symbol('scriptSymbol');
export class Recorder {
private _injectedScript: InjectedScript;
private _performingAction = false;
private _outerGlassPaneElement: HTMLElement;
private _glassPaneShadow: ShadowRoot;
private _innerGlassPaneElement: HTMLElement;
private _highlightElements: HTMLElement[] = [];
private _tooltipElement: HTMLElement;
private _listeners: (() => void)[] = [];
private _hoveredModel: HighlightModel | null = null;
private _hoveredElement: HTMLElement | null = null;
private _activeModel: HighlightModel | null = null;
private _expectProgrammaticKeyUp = false;
private _pollRecorderModeTimer: NodeJS.Timeout | undefined;
private _mode: 'none' | 'inspecting' | 'recording' = 'none';
private _actionPointElement: HTMLElement;
private _actionPoint: Point | undefined;
private _actionSelector: string | undefined;
private _params: { isUnderTest: boolean; };
private _snapshotIframe: HTMLIFrameElement | undefined;
private _snapshotUrl: string | undefined;
private _snapshotBaseUrl: string;
constructor(injectedScript: InjectedScript, params: { isUnderTest: boolean, snapshotBaseUrl: string }) {
this._params = params;
this._injectedScript = injectedScript;
this._outerGlassPaneElement = document.createElement('x-pw-glass');
this._outerGlassPaneElement.style.position = 'fixed';
this._outerGlassPaneElement.style.top = '0';
this._outerGlassPaneElement.style.right = '0';
this._outerGlassPaneElement.style.bottom = '0';
this._outerGlassPaneElement.style.left = '0';
this._outerGlassPaneElement.style.zIndex = '2147483647';
this._outerGlassPaneElement.style.pointerEvents = 'none';
this._outerGlassPaneElement.style.display = 'flex';
this._snapshotBaseUrl = params.snapshotBaseUrl;
this._tooltipElement = document.createElement('x-pw-tooltip');
this._actionPointElement = document.createElement('x-pw-action-point');
this._actionPointElement.setAttribute('hidden', 'true');
this._innerGlassPaneElement = document.createElement('x-pw-glass-inner');
this._innerGlassPaneElement.style.flex = 'auto';
this._innerGlassPaneElement.appendChild(this._tooltipElement);
// Use a closed shadow root to prevent selectors matching our internal previews.
this._glassPaneShadow = this._outerGlassPaneElement.attachShadow({ mode: this._params.isUnderTest ? 'open' : 'closed' });
this._glassPaneShadow.appendChild(this._innerGlassPaneElement);
this._glassPaneShadow.appendChild(this._actionPointElement);
const styleElement = document.createElement('style');
styleElement.textContent = `
x-pw-tooltip {
align-items: center;
backdrop-filter: blur(5px);
background-color: rgba(0, 0, 0, 0.7);
border-radius: 2px;
box-shadow: rgba(0, 0, 0, 0.1) 0px 3.6px 3.7px,
rgba(0, 0, 0, 0.15) 0px 12.1px 12.3px,
rgba(0, 0, 0, 0.1) 0px -2px 4px,
rgba(0, 0, 0, 0.15) 0px -12.1px 24px,
rgba(0, 0, 0, 0.25) 0px 54px 55px;
color: rgb(204, 204, 204);
display: none;
font-family: 'Dank Mono', 'Operator Mono', Inconsolata, 'Fira Mono',
'SF Mono', Monaco, 'Droid Sans Mono', 'Source Code Pro', monospace;
font-size: 12.8px;
font-weight: normal;
left: 0;
line-height: 1.5;
max-width: 600px;
padding: 3.2px 5.12px 3.2px;
position: absolute;
top: 0;
}
x-pw-action-point {
position: absolute;
width: 20px;
height: 20px;
background: red;
border-radius: 10px;
pointer-events: none;
margin: -10px 0 0 -10px;
z-index: 2;
}
*[hidden] {
display: none !important;
}
`;
this._glassPaneShadow.appendChild(styleElement);
this._refreshListenersIfNeeded();
setInterval(() => {
this._refreshListenersIfNeeded();
if ((window as any)._recorderScriptReadyForTest) {
(window as any)._recorderScriptReadyForTest();
delete (window as any)._recorderScriptReadyForTest;
}
}, 500);
window._playwrightRefreshOverlay = () => {
this._pollRecorderMode().catch(e => console.log(e)); // eslint-disable-line no-console
};
window._playwrightRefreshOverlay();
}
private _refreshListenersIfNeeded() {
if ((document.documentElement as any)[scriptSymbol])
return;
(document.documentElement as any)[scriptSymbol] = true;
removeEventListeners(this._listeners);
this._listeners = [
addEventListener(document, 'click', event => this._onClick(event as MouseEvent), true),
addEventListener(document, 'input', event => this._onInput(event), true),
addEventListener(document, 'keydown', event => this._onKeyDown(event as KeyboardEvent), true),
addEventListener(document, 'keyup', event => this._onKeyUp(event as KeyboardEvent), true),
addEventListener(document, 'mousedown', event => this._onMouseDown(event as MouseEvent), true),
addEventListener(document, 'mouseup', event => this._onMouseUp(event as MouseEvent), true),
addEventListener(document, 'mousemove', event => this._onMouseMove(event as MouseEvent), true),
addEventListener(document, 'mouseleave', event => this._onMouseLeave(event as MouseEvent), true),
addEventListener(document, 'focus', () => this._onFocus(), true),
addEventListener(document, 'scroll', () => {
this._hoveredModel = null;
this._actionPointElement.hidden = true;
this._updateHighlight();
}, true),
];
document.documentElement.appendChild(this._outerGlassPaneElement);
}
private _createSnapshotIframeIfNeeded(): HTMLIFrameElement | undefined {
if (this._snapshotIframe)
return this._snapshotIframe;
if (window.top === window) {
this._snapshotIframe = document.createElement('iframe');
this._snapshotIframe.src = this._snapshotBaseUrl;
this._snapshotIframe.style.background = '#ff000060';
this._snapshotIframe.style.position = 'fixed';
this._snapshotIframe.style.top = '0';
this._snapshotIframe.style.right = '0';
this._snapshotIframe.style.bottom = '0';
this._snapshotIframe.style.left = '0';
this._snapshotIframe.style.border = 'none';
this._snapshotIframe.style.width = '100%';
this._snapshotIframe.style.height = '100%';
this._snapshotIframe.style.zIndex = '2147483647';
this._snapshotIframe.style.visibility = 'hidden';
document.documentElement.appendChild(this._snapshotIframe);
}
return this._snapshotIframe;
}
private async _pollRecorderMode() {
const pollPeriod = 1000;
if (this._pollRecorderModeTimer)
clearTimeout(this._pollRecorderModeTimer);
const state = await window._playwrightRecorderState().catch(e => null);
if (!state) {
this._pollRecorderModeTimer = setTimeout(() => this._pollRecorderMode(), pollPeriod);
return;
}
const { mode, actionPoint, actionSelector, snapshotUrl } = state;
if (mode !== this._mode) {
this._mode = mode;
this._clearHighlight();
}
if (actionPoint && this._actionPoint && actionPoint.x === this._actionPoint.x && actionPoint.y === this._actionPoint.y) {
// All good.
} else if (!actionPoint && !this._actionPoint) {
// All good.
} else {
if (actionPoint) {
this._actionPointElement.style.top = actionPoint.y + 'px';
this._actionPointElement.style.left = actionPoint.x + 'px';
this._actionPointElement.hidden = false;
} else {
this._actionPointElement.hidden = true;
}
this._actionPoint = actionPoint;
}
// Race or scroll.
if (this._actionSelector && !this._hoveredModel?.elements.length)
this._actionSelector = undefined;
if (actionSelector !== this._actionSelector) {
this._hoveredModel = actionSelector ? querySelector(this._injectedScript, actionSelector, document) : null;
this._updateHighlight();
this._actionSelector = actionSelector;
}
if (snapshotUrl !== this._snapshotUrl) {
this._snapshotUrl = snapshotUrl;
const snapshotIframe = this._createSnapshotIframeIfNeeded();
if (snapshotIframe) {
if (!snapshotUrl) {
snapshotIframe.style.visibility = 'hidden';
} else {
snapshotIframe.style.visibility = 'visible';
snapshotIframe.contentWindow?.postMessage({ snapshotUrl }, '*');
}
}
}
this._pollRecorderModeTimer = setTimeout(() => this._pollRecorderMode(), pollPeriod);
}
private _clearHighlight() {
this._hoveredModel = null;
this._activeModel = null;
this._updateHighlight();
}
private _actionInProgress(event: Event): boolean {
// If Playwright is performing action for us, bail.
if (this._performingAction)
return true;
// Consume as the first thing.
consumeEvent(event);
return false;
}
private _consumedDueToNoModel(event: Event, model: HighlightModel | null): boolean {
if (model)
return false;
consumeEvent(event);
return true;
}
private _consumedDueWrongTarget(event: Event): boolean {
if (this._activeModel && this._activeModel.elements[0] === this._deepEventTarget(event))
return false;
consumeEvent(event);
return true;
}
private _onClick(event: MouseEvent) {
if (this._mode === 'inspecting')
window._playwrightRecorderSetSelector(this._hoveredModel ? this._hoveredModel.selector : '');
if (this._shouldIgnoreMouseEvent(event))
return;
if (this._actionInProgress(event))
return;
if (this._consumedDueToNoModel(event, this._hoveredModel))
return;
const checkbox = asCheckbox(this._deepEventTarget(event));
if (checkbox) {
// Interestingly, inputElement.checked is reversed inside this event handler.
this._performAction({
name: checkbox.checked ? 'check' : 'uncheck',
selector: this._hoveredModel!.selector,
signals: [],
});
return;
}
this._performAction({
name: 'click',
selector: this._hoveredModel!.selector,
signals: [],
button: buttonForEvent(event),
modifiers: modifiersForEvent(event),
clickCount: event.detail
});
}
private _shouldIgnoreMouseEvent(event: MouseEvent): boolean {
const target = this._deepEventTarget(event);
if (this._mode === 'none')
return true;
if (this._mode === 'inspecting') {
consumeEvent(event);
return true;
}
const nodeName = target.nodeName;
if (nodeName === 'SELECT')
return true;
if (nodeName === 'INPUT' && ['date'].includes((target as HTMLInputElement).type))
return true;
return false;
}
private _onMouseDown(event: MouseEvent) {
if (this._shouldIgnoreMouseEvent(event))
return;
if (!this._performingAction)
consumeEvent(event);
this._activeModel = this._hoveredModel;
}
private _onMouseUp(event: MouseEvent) {
if (this._shouldIgnoreMouseEvent(event))
return;
if (!this._performingAction)
consumeEvent(event);
}
private _onMouseMove(event: MouseEvent) {
if (this._mode === 'none')
return;
const target = this._deepEventTarget(event);
if (this._hoveredElement === target)
return;
this._hoveredElement = target;
this._updateModelForHoveredElement();
}
private _onMouseLeave(event: MouseEvent) {
// Leaving iframe.
if (this._deepEventTarget(event).nodeType === Node.DOCUMENT_NODE) {
this._hoveredElement = null;
this._updateModelForHoveredElement();
}
}
private _onFocus() {
const activeElement = this._deepActiveElement(document);
const result = activeElement ? generateSelector(this._injectedScript, activeElement) : null;
this._activeModel = result && result.selector ? result : null;
if ((window as any)._highlightUpdatedForTest)
(window as any)._highlightUpdatedForTest(result ? result.selector : null);
}
private _updateModelForHoveredElement() {
if (!this._hoveredElement) {
this._hoveredModel = null;
this._updateHighlight();
return;
}
const hoveredElement = this._hoveredElement;
const { selector, elements } = generateSelector(this._injectedScript, hoveredElement);
if ((this._hoveredModel && this._hoveredModel.selector === selector) || this._hoveredElement !== hoveredElement)
return;
this._hoveredModel = selector ? { selector, elements } : null;
this._updateHighlight();
if ((window as any)._highlightUpdatedForTest)
(window as any)._highlightUpdatedForTest(selector);
}
private _updateHighlight() {
const elements = this._hoveredModel ? this._hoveredModel.elements : [];
// Code below should trigger one layout and leave with the
// destroyed layout.
// Destroy the layout
this._tooltipElement.textContent = this._hoveredModel ? this._hoveredModel.selector : '';
this._tooltipElement.style.top = '0';
this._tooltipElement.style.left = '0';
this._tooltipElement.style.display = 'flex';
// Trigger layout.
const boxes = elements.map(e => e.getBoundingClientRect());
const tooltipWidth = this._tooltipElement.offsetWidth;
const tooltipHeight = this._tooltipElement.offsetHeight;
const totalWidth = this._innerGlassPaneElement.offsetWidth;
const totalHeight = this._innerGlassPaneElement.offsetHeight;
// Destroy the layout again.
if (boxes.length) {
const primaryBox = boxes[0];
let anchorLeft = primaryBox.left;
if (anchorLeft + tooltipWidth > totalWidth - 5)
anchorLeft = totalWidth - tooltipWidth - 5;
let anchorTop = primaryBox.bottom + 5;
if (anchorTop + tooltipHeight > totalHeight - 5) {
// If can't fit below, either position above...
if (primaryBox.top > tooltipHeight + 5) {
anchorTop = primaryBox.top - tooltipHeight - 5;
} else {
// Or on top in case of large element
anchorTop = totalHeight - 5 - tooltipHeight;
}
}
this._tooltipElement.style.top = anchorTop + 'px';
this._tooltipElement.style.left = anchorLeft + 'px';
} else {
this._tooltipElement.style.display = 'none';
}
const pool = this._highlightElements;
this._highlightElements = [];
for (const box of boxes) {
const highlightElement = pool.length ? pool.shift()! : this._createHighlightElement();
const color = this._mode === 'recording' ? '#dc6f6f7f' : '#6fa8dc7f';
highlightElement.style.backgroundColor = this._highlightElements.length ? '#f6b26b7f' : color;
highlightElement.style.left = box.x + 'px';
highlightElement.style.top = box.y + 'px';
highlightElement.style.width = box.width + 'px';
highlightElement.style.height = box.height + 'px';
highlightElement.style.display = 'block';
this._highlightElements.push(highlightElement);
}
for (const highlightElement of pool) {
highlightElement.style.display = 'none';
this._highlightElements.push(highlightElement);
}
}
private _createHighlightElement(): HTMLElement {
const highlightElement = document.createElement('x-pw-highlight');
highlightElement.style.position = 'absolute';
highlightElement.style.top = '0';
highlightElement.style.left = '0';
highlightElement.style.width = '0';
highlightElement.style.height = '0';
highlightElement.style.boxSizing = 'border-box';
this._glassPaneShadow.appendChild(highlightElement);
return highlightElement;
}
private _onInput(event: Event) {
if (this._mode !== 'recording')
return true;
const target = this._deepEventTarget(event);
if (['INPUT', 'TEXTAREA'].includes(target.nodeName)) {
const inputElement = target as HTMLInputElement;
const elementType = (inputElement.type || '').toLowerCase();
if (elementType === 'checkbox') {
// Checkbox is handled in click, we can't let input trigger on checkbox - that would mean we dispatched click events while recording.
return;
}
if (elementType === 'file') {
window._playwrightRecorderRecordAction({
name: 'setInputFiles',
selector: this._activeModel!.selector,
signals: [],
files: [...(inputElement.files || [])].map(file => file.name),
});
return;
}
// Non-navigating actions are simply recorded by Playwright.
if (this._consumedDueWrongTarget(event))
return;
window._playwrightRecorderRecordAction({
name: 'fill',
selector: this._activeModel!.selector,
signals: [],
text: inputElement.value,
});
}
if (target.nodeName === 'SELECT') {
const selectElement = target as HTMLSelectElement;
if (this._actionInProgress(event))
return;
this._performAction({
name: 'select',
selector: this._hoveredModel!.selector,
options: [...selectElement.selectedOptions].map(option => option.value),
signals: []
});
}
}
private _shouldGenerateKeyPressFor(event: KeyboardEvent): boolean {
// Backspace, Delete, AltGraph are changing input, will handle it there.
if (['Backspace', 'Delete', 'AltGraph'].includes(event.key))
return false;
// Ignore the QWERTZ shortcut for creating a at sign on MacOS
if (event.key === '@' && event.code === 'KeyL')
return false;
// Allow and ignore common used shortcut for pasting.
if (navigator.platform.includes('Mac')) {
if (event.key === 'v' && event.metaKey)
return false;
} else {
if (event.key === 'v' && event.ctrlKey)
return false;
if (event.key === 'Insert' && event.shiftKey)
return false;
}
if (['Shift', 'Control', 'Meta', 'Alt'].includes(event.key))
return false;
const hasModifier = event.ctrlKey || event.altKey || event.metaKey;
if (event.key.length === 1 && !hasModifier)
return !!asCheckbox(this._deepEventTarget(event));
return true;
}
private _onKeyDown(event: KeyboardEvent) {
if (this._mode === 'inspecting') {
consumeEvent(event);
return;
}
if (this._mode !== 'recording')
return true;
if (!this._shouldGenerateKeyPressFor(event))
return;
if (this._actionInProgress(event)) {
this._expectProgrammaticKeyUp = true;
return;
}
if (this._consumedDueWrongTarget(event))
return;
// Similarly to click, trigger checkbox on key event, not input.
if (event.key === ' ') {
const checkbox = asCheckbox(this._deepEventTarget(event));
if (checkbox) {
this._performAction({
name: checkbox.checked ? 'uncheck' : 'check',
selector: this._activeModel!.selector,
signals: [],
});
return;
}
}
this._performAction({
name: 'press',
selector: this._activeModel!.selector,
signals: [],
key: event.key,
modifiers: modifiersForEvent(event),
});
}
private _onKeyUp(event: KeyboardEvent) {
if (!this._shouldGenerateKeyPressFor(event))
return;
// Only allow programmatic keyups, ignore user input.
if (!this._expectProgrammaticKeyUp) {
consumeEvent(event);
return;
}
this._expectProgrammaticKeyUp = false;
}
private async _performAction(action: actions.Action) {
this._performingAction = true;
await window._playwrightRecorderPerformAction(action).catch(() => {});
this._performingAction = false;
// Action could have changed DOM, update hovered model selectors.
this._updateModelForHoveredElement();
// If that was a keyboard action, it similarly requires new selectors for active model.
this._onFocus();
if ((window as any)._actionPerformedForTest) {
(window as any)._actionPerformedForTest({
hovered: this._hoveredModel ? this._hoveredModel.selector : null,
active: this._activeModel ? this._activeModel.selector : null,
});
}
}
private _deepEventTarget(event: Event): HTMLElement {
return event.composedPath()[0] as HTMLElement;
}
private _deepActiveElement(document: Document): Element | null {
let activeElement = document.activeElement;
while (activeElement && activeElement.shadowRoot && activeElement.shadowRoot.activeElement)
activeElement = activeElement.shadowRoot.activeElement;
return activeElement;
}
}
function modifiersForEvent(event: MouseEvent | KeyboardEvent): number {
return (event.altKey ? 1 : 0) | (event.ctrlKey ? 2 : 0) | (event.metaKey ? 4 : 0) | (event.shiftKey ? 8 : 0);
}
function buttonForEvent(event: MouseEvent): 'left' | 'middle' | 'right' {
switch (event.which) {
case 1: return 'left';
case 2: return 'middle';
case 3: return 'right';
}
return 'left';
}
function consumeEvent(e: Event) {
e.preventDefault();
e.stopPropagation();
e.stopImmediatePropagation();
}
type HighlightModel = {
selector: string;
elements: Element[];
};
function asCheckbox(node: Node | null): HTMLInputElement | null {
if (!node || node.nodeName !== 'INPUT')
return null;
const inputElement = node as HTMLInputElement;
return inputElement.type === 'checkbox' ? inputElement : null;
}
function addEventListener(target: EventTarget, eventName: string, listener: EventListener, useCapture?: boolean): () => void {
target.addEventListener(eventName, listener, useCapture);
const remove = () => {
target.removeEventListener(eventName, listener, useCapture);
};
return remove;
}
function removeEventListeners(listeners: (() => void)[]) {
for (const listener of listeners)
listener();
listeners.splice(0, listeners.length);
}
export default Recorder;