browser(firefox): basic screencast for headless (#2931)

browser_patches/firefox/BUILD_NUMBER

@@ -1,2 +1,2 @@
-1125
+1126
-Changed: yurys@chromium.org Tue Jul  7 10:43:01 PDT 2020
+Changed: yurys@chromium.org Tue Jul 14 11:16:21 PDT 2020

browser_patches/firefox/juggler/screencast/moz.build

@@ -11,6 +11,7 @@ XPIDL_SOURCES += [
 XPIDL_MODULE = 'jugglerscreencast'
 
 SOURCES += [
+    'HeadlessWindowCapturer.cpp',
     'nsScreencastService.cpp',
     'ScreencastEncoder.cpp',
 ]
@@ -26,6 +27,11 @@ LOCAL_INCLUDES += [
     '/media/webrtc/trunk/webrtc',
 ]
 
+LOCAL_INCLUDES += [
+    '/widget',
+    '/widget/headless',
+]
+
 include('/media/webrtc/webrtc.mozbuild')
 include('/ipc/chromium/chromium-config.mozbuild')
 

browser_patches/firefox/juggler/screencast/nsScreencastService.cpp

@@ -5,6 +5,8 @@
 #include "nsScreencastService.h"
 
 #include "ScreencastEncoder.h"
+#include "HeadlessWidget.h"
+#include "HeadlessWindowCapturer.h"
 #include "mozilla/ClearOnShutdown.h"
 #include "mozilla/PresShell.h"
 #include "mozilla/StaticPtr.h"
@@ -20,6 +22,8 @@
 #include "mozilla/widget/PlatformWidgetTypes.h"
 #include "video_engine/desktop_capture_impl.h"
 
+using namespace mozilla::widget;
+
 namespace mozilla {
 
 NS_IMPL_ISUPPORTS(nsScreencastService, nsIScreencastService)
@@ -28,13 +32,35 @@ namespace {
 
 StaticRefPtr<nsScreencastService> gScreencastService;
 
+rtc::scoped_refptr<webrtc::VideoCaptureModule> CreateWindowCapturer(nsIWidget* widget, int sessionId) {
+  if (gfxPlatform::IsHeadless()) {
+    HeadlessWidget* headlessWidget = static_cast<HeadlessWidget*>(widget);
+    return HeadlessWindowCapturer::Create(headlessWidget);
+  }
+#ifdef MOZ_WIDGET_GTK
+  mozilla::widget::CompositorWidgetInitData initData;
+  widget->GetCompositorWidgetInitData(&initData);
+  const mozilla::widget::GtkCompositorWidgetInitData& gtkInitData = initData.get_GtkCompositorWidgetInitData();
+  nsCString windowId;
+# ifdef MOZ_X11
+  windowId.AppendPrintf("%lu", gtkInitData.XWindow());
+  return webrtc::DesktopCaptureImpl::Create(sessionId, windowId.get(), webrtc::CaptureDeviceType::Window);
+# else
+  // TODO: support in wayland
+  fprintf(stderr, "Video capture for Wayland is not implemented\n");
+  return nullptr;
+# endif
+#else
+  fprintf(stderr, "Video capture is not implemented on this platform\n");
+  return nullptr;
+#endif
+}
 }
 
 class nsScreencastService::Session : public rtc::VideoSinkInterface<webrtc::VideoFrame> {
  public:
-  Session(int sessionId, const nsCString& windowId, RefPtr<ScreencastEncoder>&& encoder)
+  Session(rtc::scoped_refptr<webrtc::VideoCaptureModule>&& capturer, RefPtr<ScreencastEncoder>&& encoder)
-      : mCaptureModule(webrtc::DesktopCaptureImpl::Create(
+      : mCaptureModule(std::move(capturer))
-            sessionId, windowId.get(), webrtc::CaptureDeviceType::Window))
       , mEncoder(std::move(encoder)) {
   }
 
@@ -107,17 +133,11 @@ nsresult nsScreencastService::StartVideoRecording(nsIDocShell* aDocShell, const
   nsIWidget* widget = view->GetWidget();
 
 #ifdef MOZ_WIDGET_GTK
-  mozilla::widget::CompositorWidgetInitData initData;
-  widget->GetCompositorWidgetInitData(&initData);
-  const mozilla::widget::GtkCompositorWidgetInitData& gtkInitData = initData.get_GtkCompositorWidgetInitData();
-  nsCString windowId;
-# ifdef MOZ_X11
-  windowId.AppendPrintf("%lu", gtkInitData.XWindow());
-# else
-  // TODO: support in wayland
-  return NS_ERROR_NOT_IMPLEMENTED;
-# endif
   *sessionId = ++mLastSessionId;
+  rtc::scoped_refptr<webrtc::VideoCaptureModule> capturer = CreateWindowCapturer(widget, *sessionId);
+  if (!capturer)
+    return NS_ERROR_FAILURE;
+
   nsCString error;
   Maybe<double> maybeScale;
   if (scale)
@@ -128,7 +148,7 @@ nsresult nsScreencastService::StartVideoRecording(nsIDocShell* aDocShell, const
     return NS_ERROR_FAILURE;
   }
 
-  auto session = std::make_unique<Session>(*sessionId, windowId, std::move(encoder));
+  auto session = std::make_unique<Session>(std::move(capturer), std::move(encoder));
   if (!session->Start())
     return NS_ERROR_FAILURE;
 

browser_patches/firefox/patches/bootstrap.diff

@@ -1810,3 +1810,165 @@ index ea8b9b08f3e6f6e99b8a4fa3fa427beb8c5f5945..a7ec2bd3afe53d500f0cd8f800223ee2
  };
  
  /**
+diff --git a/widget/headless/HeadlessCompositorWidget.cpp b/widget/headless/HeadlessCompositorWidget.cpp
+index b31a969b7ab3d0fc80912b110d91dfdf3e5991f4..41f483959bd80aa9cc6ad9eac068503639b33887 100644
+--- a/widget/headless/HeadlessCompositorWidget.cpp
++++ b/widget/headless/HeadlessCompositorWidget.cpp
+@@ -3,6 +3,7 @@
+  * License, v. 2.0. If a copy of the MPL was not distributed with this
+  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+ 
++#include "mozilla/layers/CompositorThread.h"
+ #include "mozilla/widget/PlatformWidgetTypes.h"
+ #include "HeadlessCompositorWidget.h"
+ #include "VsyncDispatcher.h"
+@@ -17,6 +18,54 @@ HeadlessCompositorWidget::HeadlessCompositorWidget(
+   mClientSize = aInitData.InitialClientSize();
+ }
+ 
++void HeadlessCompositorWidget::SetSnapshotListener(HeadlessWidget::SnapshotListener&& listener) {
++  MOZ_ASSERT(NS_IsMainThread());
++
++  layers::CompositorThread()->Dispatch(NewRunnableMethod<HeadlessWidget::SnapshotListener&&>(
++      "HeadlessCompositorWidget::SetSnapshotListener", this,
++      &HeadlessCompositorWidget::SetSnapshotListenerOnCompositorThread,
++      std::move(listener)));
++}
++
++void HeadlessCompositorWidget::SetSnapshotListenerOnCompositorThread(HeadlessWidget::SnapshotListener&& listener) {
++  MOZ_ASSERT(NS_IsInCompositorThread());
++  mSnapshotListener = std::move(listener);
++  UpdateDrawTarget();
++}
++
++already_AddRefed<gfx::DrawTarget> HeadlessCompositorWidget::StartRemoteDrawingInRegion(
++    LayoutDeviceIntRegion& aInvalidRegion, layers::BufferMode* aBufferMode) {
++  if (!mDrawTarget)
++    return nullptr;
++
++  *aBufferMode = layers::BufferMode::BUFFER_NONE;
++  RefPtr<gfx::DrawTarget> result = mDrawTarget;
++  return result.forget();
++}
++
++void HeadlessCompositorWidget::EndRemoteDrawingInRegion(
++    gfx::DrawTarget* aDrawTarget, const LayoutDeviceIntRegion& aInvalidRegion) {
++  if (!mDrawTarget)
++    return;
++
++  if (!mSnapshotListener)
++    return;
++
++  RefPtr<gfx::SourceSurface> snapshot = mDrawTarget->Snapshot();
++  if (!snapshot) {
++    fprintf(stderr, "Failed to get snapshot of draw target\n");
++    return;
++  }
++
++  RefPtr<gfx::DataSourceSurface> dataSurface = snapshot->GetDataSurface();
++  if (!dataSurface) {
++    fprintf(stderr, "Failed to get data surface from snapshot\n");
++    return;
++  }
++
++  mSnapshotListener(std::move(dataSurface));
++}
++
+ void HeadlessCompositorWidget::ObserveVsync(VsyncObserver* aObserver) {
+   if (RefPtr<CompositorVsyncDispatcher> cvd =
+           mWidget->GetCompositorVsyncDispatcher()) {
+@@ -29,6 +78,25 @@ nsIWidget* HeadlessCompositorWidget::RealWidget() { return mWidget; }
+ void HeadlessCompositorWidget::NotifyClientSizeChanged(
+     const LayoutDeviceIntSize& aClientSize) {
+   mClientSize = aClientSize;
++  UpdateDrawTarget();
++}
++
++void HeadlessCompositorWidget::UpdateDrawTarget() {
++  if (!mSnapshotListener) {
++    mDrawTarget = nullptr;
++    return;
++  }
++
++  if (mClientSize.IsEmpty()) {
++    mDrawTarget = nullptr;
++    return;
++  }
++
++  gfx::SurfaceFormat format = gfx::SurfaceFormat::B8G8R8A8;
++  gfx::IntSize size = mClientSize.ToUnknownSize();
++  // TODO: this is called on Main thread, while Start/End drawing are on Compositor thread.
++  mDrawTarget = mozilla::gfx::Factory::CreateDrawTarget(
++      mozilla::gfx::BackendType::SKIA, size, format);
+ }
+ 
+ LayoutDeviceIntSize HeadlessCompositorWidget::GetClientSize() {
+diff --git a/widget/headless/HeadlessCompositorWidget.h b/widget/headless/HeadlessCompositorWidget.h
+index 7f91de9e67d7ffa02de3eef1d760e5cfd05e7ad6..849cd6f98982fbabc8e483c8bb8f7935225869fc 100644
+--- a/widget/headless/HeadlessCompositorWidget.h
++++ b/widget/headless/HeadlessCompositorWidget.h
+@@ -23,9 +23,16 @@ class HeadlessCompositorWidget final : public CompositorWidget,
+                            HeadlessWidget* aWindow);
+ 
+   void NotifyClientSizeChanged(const LayoutDeviceIntSize& aClientSize);
++  void SetSnapshotListener(HeadlessWidget::SnapshotListener&& listener);
+ 
+   // CompositorWidget Overrides
+ 
++  already_AddRefed<gfx::DrawTarget> StartRemoteDrawingInRegion(
++      LayoutDeviceIntRegion& aInvalidRegion, layers::BufferMode* aBufferMode) override;
++  void EndRemoteDrawingInRegion(
++      gfx::DrawTarget* aDrawTarget,
++      const LayoutDeviceIntRegion& aInvalidRegion) override;
++
+   uintptr_t GetWidgetKey() override;
+ 
+   LayoutDeviceIntSize GetClientSize() override;
+@@ -42,9 +49,15 @@ class HeadlessCompositorWidget final : public CompositorWidget,
+   }
+ 
+  private:
++  void SetSnapshotListenerOnCompositorThread(HeadlessWidget::SnapshotListener&& listener);
++  void UpdateDrawTarget();
++
+   HeadlessWidget* mWidget;
+ 
+   LayoutDeviceIntSize mClientSize;
++
++  HeadlessWidget::SnapshotListener mSnapshotListener;
++  RefPtr<gfx::DrawTarget> mDrawTarget;
+ };
+ 
+ }  // namespace widget
+diff --git a/widget/headless/HeadlessWidget.cpp b/widget/headless/HeadlessWidget.cpp
+index 7589d8a1a886dab5431e423d20f7d0aa19c2af75..19dd67a330848b6b39bfc578a6940385329fff8e 100644
+--- a/widget/headless/HeadlessWidget.cpp
++++ b/widget/headless/HeadlessWidget.cpp
+@@ -499,5 +499,13 @@ nsresult HeadlessWidget::SynthesizeNativeTouchPoint(
+   return NS_OK;
+ }
+ 
++void HeadlessWidget::SetSnapshotListener(SnapshotListener&& listener) {
++  if (!mCompositorWidget) {
++    fprintf(stderr, "Trying to set SnapshotListener without compositor widget\n");
++    return;
++  }
++  mCompositorWidget->SetSnapshotListener(std::move(listener));
++}
++
+ }  // namespace widget
+ }  // namespace mozilla
+diff --git a/widget/headless/HeadlessWidget.h b/widget/headless/HeadlessWidget.h
+index c375629d4a954f872a2abdd6983ae38dbb98f4ca..1857a4874ac9f8a3d7e402b5707a9ea58f241eb9 100644
+--- a/widget/headless/HeadlessWidget.h
++++ b/widget/headless/HeadlessWidget.h
+@@ -153,6 +153,9 @@ class HeadlessWidget : public nsBaseWidget {
+                                               uint32_t aPointerOrientation,
+                                               nsIObserver* aObserver) override;
+ 
++  using SnapshotListener = std::function<void(RefPtr<gfx::DataSourceSurface>&&)>;
++  void SetSnapshotListener(SnapshotListener&& listener);
++
+  private:
+   ~HeadlessWidget();
+   bool mEnabled;