fix(chromium): drag and drop works in chromium (#6207)

Waiting for #6203 to percolate to the cdn. But this all works locally.

Fixes #1094

src/server/chromium/crDragDrop.ts

@@ -0,0 +1,137 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+import { CRPage } from './crPage';
+import * as types from '../types';
+import { assert } from '../../utils/utils';
+import { Protocol } from './protocol';
+import { toModifiersMask } from './crProtocolHelper';
+
+declare global {
+  interface Window {
+    __cleanupDrag?: () => Promise<boolean>;
+  }
+}
+
+export class DragManager {
+  private _crPage: CRPage;
+  private _dragState: Protocol.Input.DragData | null = null;
+  private _lastPosition = {x: 0, y: 0};
+  constructor(page: CRPage) {
+    this._crPage = page;
+  }
+
+  async cancelDrag() {
+    if (!this._dragState)
+      return false;
+    await this._crPage._mainFrameSession._client.send('Input.dispatchDragEvent', {
+      type: 'dragCancel',
+      x: this._lastPosition.x,
+      y: this._lastPosition.y,
+      data: {
+        items: [],
+        dragOperationsMask: 0xFFFF,
+      }
+    });
+    this._dragState = null;
+    return true;
+  }
+
+  async interceptDragCausedByMove(x: number, y: number, button: types.MouseButton | 'none', buttons: Set<types.MouseButton>, modifiers: Set<types.KeyboardModifier>, moveCallback: () => Promise<void>): Promise<void> {
+    this._lastPosition = {x, y};
+    if (this._dragState) {
+      await this._crPage._mainFrameSession._client.send('Input.dispatchDragEvent', {
+        type: 'dragOver',
+        x,
+        y,
+        data: this._dragState,
+        modifiers: toModifiersMask(modifiers),
+      });
+      return;
+    }
+    if (button !== 'left')
+      return moveCallback();
+
+    const client = this._crPage._mainFrameSession._client;
+    let onDragIntercepted: (payload: Protocol.Input.dragInterceptedPayload) => void;
+    const dragInterceptedPromise = new Promise<Protocol.Input.dragInterceptedPayload>(x => onDragIntercepted = x);
+
+    await Promise.all(this._crPage._page.frames().map(async frame => {
+      await frame.nonStallingEvaluateInExistingContext((function() {
+        let didStartDrag = Promise.resolve(false);
+        let dragEvent: Event|null = null;
+        const dragListener = (event: Event) => dragEvent = event;
+        const mouseListener = () => {
+          didStartDrag = new Promise<boolean>(callback => {
+            window.addEventListener('dragstart', dragListener, {once: true, capture: true});
+            setTimeout(() => callback(dragEvent ? !dragEvent.defaultPrevented : false), 0);
+          });
+        };
+        window.addEventListener('mousemove', mouseListener, {once: true, capture: true});
+        window.__cleanupDrag = async () => {
+          const val = await didStartDrag;
+          window.removeEventListener('mousemove', mouseListener, {capture: true});
+          window.removeEventListener('dragstart', dragListener, {capture: true});
+          return val;
+        };
+      }).toString(), true, 'utility').catch(() => {});
+    }));
+
+    client.on('Input.dragIntercepted', onDragIntercepted!);
+    try {
+      await client.send('Input.setInterceptDrags', {enabled: true});
+    } catch {
+      // If Input.setInterceptDrags is not supported, just do a regular move.
+      // This can be removed once we stop supporting old Electron.
+      client.off('Input.dragIntercepted', onDragIntercepted!);
+      return moveCallback();
+    }
+    await moveCallback();
+
+    const expectingDrag = (await Promise.all(this._crPage._page.frames().map(async frame => {
+      return frame.nonStallingEvaluateInExistingContext('window.__cleanupDrag && window.__cleanupDrag()', false, 'utility').catch(() => false);
+    }))).some(x => x);
+    this._dragState = expectingDrag ? (await dragInterceptedPromise).data : null;
+    client.off('Input.dragIntercepted', onDragIntercepted!);
+    await client.send('Input.setInterceptDrags', {enabled: false});
+
+
+    if (this._dragState) {
+      await this._crPage._mainFrameSession._client.send('Input.dispatchDragEvent', {
+        type: 'dragEnter',
+        x,
+        y,
+        data: this._dragState,
+        modifiers: toModifiersMask(modifiers),
+      });
+    }
+  }
+
+  isDragging() {
+    return !!this._dragState;
+  }
+
+  async drop(x: number, y: number, modifiers: Set<types.KeyboardModifier>) {
+    assert(this._dragState, 'missing drag state');
+    await this._crPage._mainFrameSession._client.send('Input.dispatchDragEvent', {
+      type: 'drop',
+      x,
+      y,
+      data: this._dragState,
+      modifiers: toModifiersMask(modifiers),
+    });
+    this._dragState = null;
+  }
+}

src/server/chromium/crInput.ts

@@ -20,24 +20,15 @@ import * as types from '../types';
 import { CRSession } from './crConnection';
 import { macEditingCommands } from '../macEditingCommands';
 import { isString } from '../../utils/utils';
-
+import { DragManager } from './crDragDrop';
-function toModifiersMask(modifiers: Set<types.KeyboardModifier>): number {
+import { CRPage } from './crPage';
-  let mask = 0;
+import { toModifiersMask } from './crProtocolHelper';
-  if (modifiers.has('Alt'))
-    mask |= 1;
-  if (modifiers.has('Control'))
-    mask |= 2;
-  if (modifiers.has('Meta'))
-    mask |= 4;
-  if (modifiers.has('Shift'))
-    mask |= 8;
-  return mask;
-}
 
 export class RawKeyboardImpl implements input.RawKeyboard {
   constructor(
     private _client: CRSession,
     private _isMac: boolean,
+    private _dragManger: DragManager,
   ) { }
 
   _commandsForCode(code: string, modifiers: Set<types.KeyboardModifier>) {
@@ -60,6 +51,8 @@ export class RawKeyboardImpl implements input.RawKeyboard {
   }
 
   async keydown(modifiers: Set<types.KeyboardModifier>, code: string, keyCode: number, keyCodeWithoutLocation: number, key: string, location: number, autoRepeat: boolean, text: string | undefined): Promise<void> {
+    if (code === 'Escape' && await this._dragManger.cancelDrag())
+      return;
     const commands = this._commandsForCode(code, modifiers);
     await this._client.send('Input.dispatchKeyEvent', {
       type: text ? 'keyDown' : 'rawKeyDown',
@@ -94,22 +87,30 @@ export class RawKeyboardImpl implements input.RawKeyboard {
 
 export class RawMouseImpl implements input.RawMouse {
   private _client: CRSession;
+  private _page: CRPage;
+  private _dragManager: DragManager;
 
-  constructor(client: CRSession) {
+  constructor(page: CRPage, client: CRSession, dragManager: DragManager) {
+    this._page = page;
     this._client = client;
+    this._dragManager = dragManager;
   }
 
   async move(x: number, y: number, button: types.MouseButton | 'none', buttons: Set<types.MouseButton>, modifiers: Set<types.KeyboardModifier>): Promise<void> {
-    await this._client.send('Input.dispatchMouseEvent', {
+    await this._dragManager.interceptDragCausedByMove(x, y, button, buttons, modifiers, async () => {
-      type: 'mouseMoved',
+      await this._client.send('Input.dispatchMouseEvent', {
-      button,
+        type: 'mouseMoved',
-      x,
+        button,
-      y,
+        x,
-      modifiers: toModifiersMask(modifiers)
+        y,
+        modifiers: toModifiersMask(modifiers)
+      });
     });
   }
 
   async down(x: number, y: number, button: types.MouseButton, buttons: Set<types.MouseButton>, modifiers: Set<types.KeyboardModifier>, clickCount: number): Promise<void> {
+    if (this._dragManager.isDragging())
+      return;
     await this._client.send('Input.dispatchMouseEvent', {
       type: 'mousePressed',
       button,
@@ -121,6 +122,10 @@ export class RawMouseImpl implements input.RawMouse {
   }
 
   async up(x: number, y: number, button: types.MouseButton, buttons: Set<types.MouseButton>, modifiers: Set<types.KeyboardModifier>, clickCount: number): Promise<void> {
+    if (this._dragManager.isDragging()) {
+      await this._dragManager.drop(x, y, modifiers);
+      return;
+    }
     await this._client.send('Input.dispatchMouseEvent', {
       type: 'mouseReleased',
       button,

src/server/chromium/crPage.ts

@@ -39,6 +39,7 @@ import { rewriteErrorMessage } from '../../utils/stackTrace';
 import { assert, headersArrayToObject, createGuid, canAccessFile } from '../../utils/utils';
 import { VideoRecorder } from './videoRecorder';
 import { Progress } from '../progress';
+import { DragManager } from './crDragDrop';
 
 
 const UTILITY_WORLD_NAME = '__playwright_utility_world__';
@@ -76,8 +77,9 @@ export class CRPage implements PageDelegate {
     this._targetId = targetId;
     this._opener = opener;
     this._isBackgroundPage = isBackgroundPage;
-    this.rawKeyboard = new RawKeyboardImpl(client, browserContext._browser._isMac);
+    const dragManager = new DragManager(this);
-    this.rawMouse = new RawMouseImpl(client);
+    this.rawKeyboard = new RawKeyboardImpl(client, browserContext._browser._isMac, dragManager);
+    this.rawMouse = new RawMouseImpl(this, client, dragManager);
     this.rawTouchscreen = new RawTouchscreenImpl(client);
     this._pdf = new CRPDF(client);
     this._coverage = new CRCoverage(client);

src/server/chromium/crProtocolHelper.ts

@@ -89,3 +89,16 @@ export function exceptionToError(exceptionDetails: Protocol.Runtime.ExceptionDet
   err.name = name;
   return err;
 }
+
+export function toModifiersMask(modifiers: Set<types.KeyboardModifier>): number {
+  let mask = 0;
+  if (modifiers.has('Alt'))
+    mask |= 1;
+  if (modifiers.has('Control'))
+    mask |= 2;
+  if (modifiers.has('Meta'))
+    mask |= 4;
+  if (modifiers.has('Shift'))
+    mask |= 8;
+  return mask;
+}

src/server/frames.ts

@@ -495,6 +495,26 @@ export class Frame extends SdkObject {
     }
   }
 
+  async nonStallingEvaluateInExistingContext(expression: string, isFunction: boolean|undefined, world: types.World): Promise<any> {
+    if (this._pendingDocument)
+      throw new Error('Frame is currently attempting a navigation');
+    const context = this._contextData.get(world)?.context;
+    if (!context)
+      throw new Error('Frame does not yet have the execution context');
+
+    let callback = () => {};
+    const frameInvalidated = new Promise<void>((f, r) => callback = r);
+    this._nonStallingEvaluations.add(callback);
+    try {
+      return await Promise.race([
+        context.evaluateExpression(expression, isFunction),
+        frameInvalidated
+      ]);
+    } finally {
+      this._nonStallingEvaluations.delete(callback);
+    }
+  }
+
   private _recalculateLifecycle() {
     const events = new Set<types.LifecycleEvent>(this._firedLifecycleEvents);
     for (const child of this._childFrames) {

tests/page/page-drag.spec.ts

@@ -14,13 +14,13 @@
  * limitations under the License.
  */
 
-import type { ElementHandle } from '../../index';
+import type { ElementHandle, Route } from '../../index';
 import { test as it, expect } from './pageTest';
 import { attachFrame } from '../config/utils';
 
 it.describe('Drag and drop', () => {
-  it.skip(({ isAndroid }) => isAndroid);
+  it.skip(({isAndroid}) => isAndroid);
-  it.fixme(({ browserName }) => browserName === 'chromium');
+  it.skip(({browserName, browserMajorVersion}) => browserName === 'chromium' && browserMajorVersion < 91);
 
   it('should work', async ({page, server}) => {
     await page.goto(server.PREFIX + '/drag-n-drop.html');
@@ -65,10 +65,10 @@ it.describe('Drag and drop', () => {
       browserName === 'firefox' ? 'dragstart' : 'mousemove',
       browserName === 'firefox' ? 'mousemove' : 'dragstart',
       'dragenter',
-      'dragover',
+      browserName !== 'chromium' ? 'dragover' : null,
       'dragend',
       'mouseup',
-    ]);
+    ].filter(Boolean));
   });
 
   it.describe('iframe', () => {
@@ -122,7 +122,6 @@ it.describe('Drag and drop', () => {
   });
 
   it('should respect the drop effect', async ({page, browserName, platform}) => {
-    it.fixme(browserName === 'chromium', 'Chromium doesn\'t let users set dropEffect on our fake data transfer');
     it.fixme(browserName === 'webkit' && platform !== 'linux', 'WebKit doesn\'t handle the drop effect correctly outside of linux.');
     it.fixme(browserName === 'firefox');
 
@@ -174,6 +173,60 @@ it.describe('Drag and drop', () => {
       return await page.evaluate('dropped');
     }
   });
+  it('should work if the drag is canceled', async ({page, server}) => {
+    await page.goto(server.PREFIX + '/drag-n-drop.html');
+    await page.evaluate(() => {
+      document.body.addEventListener('dragstart', event => {
+        event.preventDefault();
+      }, false);
+    });
+    await page.hover('#source');
+    await page.mouse.down();
+    await page.hover('#target');
+    await page.mouse.up();
+    expect(await page.$eval('#target', target => target.contains(document.querySelector('#source')))).toBe(false);
+  });
+
+  it('should work if the drag event is captured but not canceled', async ({page, server}) => {
+    await page.goto(server.PREFIX + '/drag-n-drop.html');
+    await page.evaluate(() => {
+      document.body.addEventListener('dragstart', event => {
+        event.stopImmediatePropagation();
+      }, false);
+    });
+    await page.hover('#source');
+    await page.mouse.down();
+    await page.hover('#target');
+    await page.mouse.up();
+    expect(await page.$eval('#target', target => target.contains(document.querySelector('#source')))).toBe(true);
+  });
+
+  it('should be able to drag the mouse in a frame', async ({page, server}) => {
+    await page.goto(server.PREFIX + '/frames/one-frame.html');
+    const eventsHandle = await trackEvents(await page.frames()[1].$('html'));
+    await page.mouse.move(30, 30);
+    await page.mouse.down();
+    await page.mouse.move(60, 60);
+    await page.mouse.up();
+    expect(await eventsHandle.jsonValue()).toEqual(['mousemove', 'mousedown', 'mousemove', 'mouseup']);
+  });
+
+  it('should work if a frame is stalled', async ({page, server, toImpl}) => {
+    await page.goto(server.PREFIX + '/drag-n-drop.html');
+    let madeRequest;
+    const routePromise = new Promise<Route>(x => madeRequest = x);
+    await page.route('**/empty.html', async (route, request) => {
+      madeRequest(route);
+    });
+    attachFrame(page, 'frame', server.EMPTY_PAGE).catch(() => {});
+    const route = await routePromise;
+    await page.hover('#source');
+    await page.mouse.down();
+    await page.hover('#target');
+    await page.mouse.up();
+    route.abort();
+    expect(await page.$eval('#target', target => target.contains(document.querySelector('#source')))).toBe(true); // could not find source in target
+  })
 
   async function trackEvents(target: ElementHandle) {
     const eventsHandle = await target.evaluateHandle(target => {
@@ -189,3 +242,26 @@ it.describe('Drag and drop', () => {
     return eventsHandle;
   }
 });
+
+it('should work if not doing a drag', async ({page}) => {
+  const eventsHandle = await trackEvents(await page.$('html'));
+  await page.mouse.move(50, 50);
+  await page.mouse.down();
+  await page.mouse.move(100, 100);
+  await page.mouse.up();
+  expect(await eventsHandle.jsonValue()).toEqual(['mousemove', 'mousedown', 'mousemove', 'mouseup']);
+});
+
+async function trackEvents(target: ElementHandle) {
+  const eventsHandle = await target.evaluateHandle(target => {
+    const events: string[] = [];
+    for (const event of [
+      'mousedown', 'mousemove', 'mouseup',
+      'dragstart', 'dragend', 'dragover', 'dragenter', 'dragleave', 'dragexit',
+      'drop'
+    ])
+      target.addEventListener(event, () => events.push(event), false);
+    return events;
+  });
+  return eventsHandle;
+}