browser(firefox): fit screencast images into given frame (#6495)

browser_patches/firefox-stable/BUILD_NUMBER

@@ -1,2 +1,2 @@
-1247
+1248
-Changed: dgozman@gmail.com Sat May  8 17:46:11 PDT 2021
+Changed: pavel.feldman@gmail.com Mon 10 May 2021 09:58:08 PM PDT

browser_patches/firefox-stable/juggler/TargetRegistry.js

@@ -514,7 +514,8 @@ class PageTarget {
         registry.emit(TargetRegistry.Events.ScreencastStopped, sessionId);
       },
     };
-    sessionId = screencastService.startVideoRecording(screencastClient, docShell, true, file, width, height, 0, devicePixelRatio * rect.top);
+    const viewport = this._viewportSize || this._browserContext.defaultViewportSize || { width: 0, height: 0 };
+    sessionId = screencastService.startVideoRecording(screencastClient, docShell, true, file, width, height, 0, viewport.width, viewport.height, devicePixelRatio * rect.top);
     this._videoRecordingInfo = { sessionId, file };
     this.emit(PageTarget.Events.ScreencastStarted);
   }
@@ -554,7 +555,8 @@ class PageTarget {
       screencastStopped() {
       },
     };
-    const screencastId = screencastService.startVideoRecording(screencastClient, docShell, false, '', width, height, quality || 90, devicePixelRatio * rect.top);
+    const viewport = this._viewportSize || this._browserContext.defaultViewportSize || { width: 0, height: 0 };
+    const screencastId = screencastService.startVideoRecording(screencastClient, docShell, false, '', width, height, quality || 90, viewport.width, viewport.height, devicePixelRatio * rect.top);
     this._screencastRecordingInfo = { screencastId };
     return { screencastId };
   }

browser_patches/firefox-stable/juggler/screencast/nsIScreencastService.idl

@@ -20,7 +20,7 @@ interface nsIScreencastServiceClient : nsISupports
 [scriptable, uuid(d8c4d9e0-9462-445e-9e43-68d3872ad1de)]
 interface nsIScreencastService : nsISupports
 {
-  AString startVideoRecording(in nsIScreencastServiceClient client, in nsIDocShell docShell, in boolean isVideo, in ACString fileName, in uint32_t width, in uint32_t height, in uint32_t quality, in int32_t offset_top);
+  AString startVideoRecording(in nsIScreencastServiceClient client, in nsIDocShell docShell, in boolean isVideo, in ACString fileName, in uint32_t width, in uint32_t height, in uint32_t quality, in uint32_t viewportWidth, in uint32_t viewportHeight, in uint32_t offset_top);
 
   /**
    * Will emit 'juggler-screencast-stopped' when the video file is saved.

browser_patches/firefox-stable/juggler/screencast/nsScreencastService.cpp

@@ -28,6 +28,7 @@
 extern "C" {
 #include "jpeglib.h"
 }
+#include <libyuv.h>
 
 using namespace mozilla::widget;
 
@@ -79,11 +80,22 @@ nsresult generateUid(nsString& uid) {
 class nsScreencastService::Session : public rtc::VideoSinkInterface<webrtc::VideoFrame>,
                                      public webrtc::RawFrameCallback {
  public:
-  Session(nsIScreencastServiceClient* client, rtc::scoped_refptr<webrtc::VideoCaptureModuleEx>&& capturer, RefPtr<ScreencastEncoder>&& encoder, gfx::IntMargin margin, uint32_t jpegQuality)
+  Session(
+    nsIScreencastServiceClient* client,
+    rtc::scoped_refptr<webrtc::VideoCaptureModuleEx>&& capturer,
+    RefPtr<ScreencastEncoder>&& encoder,
+    int width, int height,
+    int viewportWidth, int viewportHeight,
+    gfx::IntMargin margin,
+    uint32_t jpegQuality)
       : mClient(client)
       , mCaptureModule(std::move(capturer))
       , mEncoder(std::move(encoder))
       , mJpegQuality(jpegQuality)
+      , mWidth(width)
+      , mHeight(height)
+      , mViewportWidth(viewportWidth)
+      , mViewportHeight(viewportHeight)
       , mMargin(margin) {
   }
 
@@ -145,6 +157,14 @@ class nsScreencastService::Session : public rtc::VideoSinkInterface<webrtc::Vide
 
   // These callbacks end up running on the VideoCapture thread.
   void OnRawFrame(uint8_t* videoFrame, size_t videoFrameStride, const webrtc::VideoCaptureCapability& frameInfo) override {
+    int pageWidth = frameInfo.width - mMargin.LeftRight();
+    int pageHeight = frameInfo.height - mMargin.TopBottom();
+    // Headed Firefox brings sizes in sync slowly.
+    if (mViewportWidth && pageWidth > mViewportWidth)
+      pageWidth = mViewportWidth;
+    if (mViewportHeight && pageHeight > mViewportHeight)
+      pageHeight = mViewportHeight;
+
     {
       rtc::CritScope lock(&mCaptureCallbackCs);
       if (mFramesInFlight >= kMaxFramesInFlight) {
@@ -155,6 +175,36 @@ class nsScreencastService::Session : public rtc::VideoSinkInterface<webrtc::Vide
         return;
     }
 
+    int screenshotWidth = pageWidth;
+    int screenshotHeight = pageHeight;
+    int screenshotTopMargin = mMargin.TopBottom();
+    std::unique_ptr<uint8_t[]> canvas;
+    uint8_t* canvasPtr = videoFrame;
+    int canvasStride = videoFrameStride;
+
+    if (mWidth < pageWidth || mHeight < pageHeight) {
+      double scale = std::min(1., std::min((double)mWidth / pageWidth, (double)mHeight / pageHeight));
+      int canvasWidth = frameInfo.width * scale;
+      int canvasHeight = frameInfo.height * scale;
+      canvasStride = canvasWidth * 4;
+
+      screenshotWidth *= scale;
+      screenshotHeight *= scale;
+      screenshotTopMargin *= scale;
+
+      canvas.reset(new uint8_t[canvasWidth * canvasHeight * 4]);
+      canvasPtr = canvas.get();
+      libyuv::ARGBScale(videoFrame,
+                        videoFrameStride,
+                        frameInfo.width,
+                        frameInfo.height,
+                        canvasPtr,
+                        canvasStride,
+                        canvasWidth,
+                        canvasHeight,
+                        libyuv::kFilterBilinear);
+    }
+
     jpeg_compress_struct info;
     jpeg_error_mgr error;
     info.err = jpeg_std_error(&error);
@@ -164,8 +214,8 @@ class nsScreencastService::Session : public rtc::VideoSinkInterface<webrtc::Vide
     unsigned long bufferSize;
     jpeg_mem_dest(&info, &bufferPtr, &bufferSize);
 
-    info.image_width = frameInfo.width - mMargin.LeftRight();
+    info.image_width = screenshotWidth;
-    info.image_height = frameInfo.height - mMargin.TopBottom();
+    info.image_height = screenshotHeight;
 
 #if MOZ_LITTLE_ENDIAN()
     if (frameInfo.videoType == webrtc::VideoType::kARGB)
@@ -187,7 +237,7 @@ class nsScreencastService::Session : public rtc::VideoSinkInterface<webrtc::Vide
 
     jpeg_start_compress(&info, true);
     while (info.next_scanline < info.image_height) {
-      JSAMPROW row = videoFrame + (mMargin.top + info.next_scanline) * videoFrameStride + 4 * mMargin.left;
+      JSAMPROW row = canvasPtr + (screenshotTopMargin + info.next_scanline) * canvasStride;
       if (jpeg_write_scanlines(&info, &row, 1) != 1) {
         fprintf(stderr, "JPEG library failed to encode line\n");
         break;
@@ -204,13 +254,11 @@ class nsScreencastService::Session : public rtc::VideoSinkInterface<webrtc::Vide
       return;
     }
 
-    uint32_t deviceWidth = info.image_width;
-    uint32_t deviceHeight = info.image_height;
     nsIScreencastServiceClient* client = mClient.get();
     NS_DispatchToMainThread(NS_NewRunnableFunction(
-        "NotifyScreencastFrame", [client, base64, deviceWidth, deviceHeight]() -> void {
+        "NotifyScreencastFrame", [client, base64, pageWidth, pageHeight]() -> void {
           NS_ConvertUTF8toUTF16 utf16(base64);
-          client->ScreencastFrame(utf16, deviceWidth, deviceHeight);
+          client->ScreencastFrame(utf16, pageWidth, pageHeight);
         }));
   }
 
@@ -221,6 +269,10 @@ class nsScreencastService::Session : public rtc::VideoSinkInterface<webrtc::Vide
   uint32_t mJpegQuality;
   rtc::CriticalSection mCaptureCallbackCs;
   uint32_t mFramesInFlight = 0;
+  int mWidth;
+  int mHeight;
+  int mViewportWidth;
+  int mViewportHeight;
   gfx::IntMargin mMargin;
 };
 
@@ -241,7 +293,7 @@ nsScreencastService::nsScreencastService() = default;
 nsScreencastService::~nsScreencastService() {
 }
 
-nsresult nsScreencastService::StartVideoRecording(nsIScreencastServiceClient* aClient, nsIDocShell* aDocShell, bool isVideo, const nsACString& aVideoFileName, uint32_t width, uint32_t height, uint32_t quality, int32_t offsetTop, nsAString& sessionId) {
+nsresult nsScreencastService::StartVideoRecording(nsIScreencastServiceClient* aClient, nsIDocShell* aDocShell, bool isVideo, const nsACString& aVideoFileName, uint32_t width, uint32_t height, uint32_t quality, uint32_t viewportWidth, uint32_t viewportHeight, uint32_t offsetTop, nsAString& sessionId) {
   MOZ_RELEASE_ASSERT(NS_IsMainThread(), "Screencast service must be started on the Main thread.");
 
   PresShell* presShell = aDocShell->GetPresShell();
@@ -282,7 +334,7 @@ nsresult nsScreencastService::StartVideoRecording(nsIScreencastServiceClient* aC
   NS_ENSURE_SUCCESS(rv, rv);
   sessionId = uid;
 
-  auto session = std::make_unique<Session>(aClient, std::move(capturer), std::move(encoder), margin, isVideo ? 0 : quality);
+  auto session = std::make_unique<Session>(aClient, std::move(capturer), std::move(encoder), width, height, viewportWidth, viewportHeight, margin, isVideo ? 0 : quality);
   if (!session->Start())
     return NS_ERROR_FAILURE;
   mIdToSession.emplace(sessionId, std::move(session));

browser_patches/firefox/BUILD_NUMBER

@@ -1,2 +1,2 @@
-1257
+1258
-Changed: dgozman@gmail.com Sat May  8 17:46:11 PDT 2021
+Changed: pavel.feldman@gmail.com Mon 10 May 2021 09:56:40 PM PDT

browser_patches/firefox/juggler/TargetRegistry.js

@@ -514,7 +514,8 @@ class PageTarget {
         registry.emit(TargetRegistry.Events.ScreencastStopped, sessionId);
       },
     };
-    sessionId = screencastService.startVideoRecording(screencastClient, docShell, true, file, width, height, 0, devicePixelRatio * rect.top);
+    const viewport = this._viewportSize || this._browserContext.defaultViewportSize || { width: 0, height: 0 };
+    sessionId = screencastService.startVideoRecording(screencastClient, docShell, true, file, width, height, 0, viewport.width, viewport.height, devicePixelRatio * rect.top);
     this._videoRecordingInfo = { sessionId, file };
     this.emit(PageTarget.Events.ScreencastStarted);
   }
@@ -554,7 +555,8 @@ class PageTarget {
       screencastStopped() {
       },
     };
-    const screencastId = screencastService.startVideoRecording(screencastClient, docShell, false, '', width, height, quality || 90, devicePixelRatio * rect.top);
+    const viewport = this._viewportSize || this._browserContext.defaultViewportSize || { width: 0, height: 0 };
+    const screencastId = screencastService.startVideoRecording(screencastClient, docShell, false, '', width, height, quality || 90, viewport.width, viewport.height, devicePixelRatio * rect.top);
     this._screencastRecordingInfo = { screencastId };
     return { screencastId };
   }

browser_patches/firefox/juggler/screencast/nsIScreencastService.idl

@@ -20,7 +20,7 @@ interface nsIScreencastServiceClient : nsISupports
 [scriptable, uuid(d8c4d9e0-9462-445e-9e43-68d3872ad1de)]
 interface nsIScreencastService : nsISupports
 {
-  AString startVideoRecording(in nsIScreencastServiceClient client, in nsIDocShell docShell, in boolean isVideo, in ACString fileName, in uint32_t width, in uint32_t height, in uint32_t quality, in int32_t offset_top);
+  AString startVideoRecording(in nsIScreencastServiceClient client, in nsIDocShell docShell, in boolean isVideo, in ACString fileName, in uint32_t width, in uint32_t height, in uint32_t quality, in uint32_t viewportWidth, in uint32_t viewportHeight, in uint32_t offset_top);
 
   /**
    * Will emit 'juggler-screencast-stopped' when the video file is saved.

browser_patches/firefox/juggler/screencast/nsScreencastService.cpp

@@ -28,6 +28,7 @@
 extern "C" {
 #include "jpeglib.h"
 }
+#include <libyuv.h>
 
 using namespace mozilla::widget;
 
@@ -79,11 +80,22 @@ nsresult generateUid(nsString& uid) {
 class nsScreencastService::Session : public rtc::VideoSinkInterface<webrtc::VideoFrame>,
                                      public webrtc::RawFrameCallback {
  public:
-  Session(nsIScreencastServiceClient* client, rtc::scoped_refptr<webrtc::VideoCaptureModuleEx>&& capturer, RefPtr<ScreencastEncoder>&& encoder, gfx::IntMargin margin, uint32_t jpegQuality)
+  Session(
+    nsIScreencastServiceClient* client,
+    rtc::scoped_refptr<webrtc::VideoCaptureModuleEx>&& capturer,
+    RefPtr<ScreencastEncoder>&& encoder,
+    int width, int height,
+    int viewportWidth, int viewportHeight,
+    gfx::IntMargin margin,
+    uint32_t jpegQuality)
       : mClient(client)
       , mCaptureModule(std::move(capturer))
       , mEncoder(std::move(encoder))
       , mJpegQuality(jpegQuality)
+      , mWidth(width)
+      , mHeight(height)
+      , mViewportWidth(viewportWidth)
+      , mViewportHeight(viewportHeight)
       , mMargin(margin) {
   }
 
@@ -145,6 +157,14 @@ class nsScreencastService::Session : public rtc::VideoSinkInterface<webrtc::Vide
 
   // These callbacks end up running on the VideoCapture thread.
   void OnRawFrame(uint8_t* videoFrame, size_t videoFrameStride, const webrtc::VideoCaptureCapability& frameInfo) override {
+    int pageWidth = frameInfo.width - mMargin.LeftRight();
+    int pageHeight = frameInfo.height - mMargin.TopBottom();
+    // Headed Firefox brings sizes in sync slowly.
+    if (mViewportWidth && pageWidth > mViewportWidth)
+      pageWidth = mViewportWidth;
+    if (mViewportHeight && pageHeight > mViewportHeight)
+      pageHeight = mViewportHeight;
+
     {
       rtc::CritScope lock(&mCaptureCallbackCs);
       if (mFramesInFlight >= kMaxFramesInFlight) {
@@ -155,6 +175,36 @@ class nsScreencastService::Session : public rtc::VideoSinkInterface<webrtc::Vide
         return;
     }
 
+    int screenshotWidth = pageWidth;
+    int screenshotHeight = pageHeight;
+    int screenshotTopMargin = mMargin.TopBottom();
+    std::unique_ptr<uint8_t[]> canvas;
+    uint8_t* canvasPtr = videoFrame;
+    int canvasStride = videoFrameStride;
+
+    if (mWidth < pageWidth || mHeight < pageHeight) {
+      double scale = std::min(1., std::min((double)mWidth / pageWidth, (double)mHeight / pageHeight));
+      int canvasWidth = frameInfo.width * scale;
+      int canvasHeight = frameInfo.height * scale;
+      canvasStride = canvasWidth * 4;
+
+      screenshotWidth *= scale;
+      screenshotHeight *= scale;
+      screenshotTopMargin *= scale;
+
+      canvas.reset(new uint8_t[canvasWidth * canvasHeight * 4]);
+      canvasPtr = canvas.get();
+      libyuv::ARGBScale(videoFrame,
+                        videoFrameStride,
+                        frameInfo.width,
+                        frameInfo.height,
+                        canvasPtr,
+                        canvasStride,
+                        canvasWidth,
+                        canvasHeight,
+                        libyuv::kFilterBilinear);
+    }
+
     jpeg_compress_struct info;
     jpeg_error_mgr error;
     info.err = jpeg_std_error(&error);
@@ -164,8 +214,8 @@ class nsScreencastService::Session : public rtc::VideoSinkInterface<webrtc::Vide
     unsigned long bufferSize;
     jpeg_mem_dest(&info, &bufferPtr, &bufferSize);
 
-    info.image_width = frameInfo.width - mMargin.LeftRight();
+    info.image_width = screenshotWidth;
-    info.image_height = frameInfo.height - mMargin.TopBottom();
+    info.image_height = screenshotHeight;
 
 #if MOZ_LITTLE_ENDIAN()
     if (frameInfo.videoType == webrtc::VideoType::kARGB)
@@ -187,7 +237,7 @@ class nsScreencastService::Session : public rtc::VideoSinkInterface<webrtc::Vide
 
     jpeg_start_compress(&info, true);
     while (info.next_scanline < info.image_height) {
-      JSAMPROW row = videoFrame + (mMargin.top + info.next_scanline) * videoFrameStride + 4 * mMargin.left;
+      JSAMPROW row = canvasPtr + (screenshotTopMargin + info.next_scanline) * canvasStride;
       if (jpeg_write_scanlines(&info, &row, 1) != 1) {
         fprintf(stderr, "JPEG library failed to encode line\n");
         break;
@@ -204,13 +254,11 @@ class nsScreencastService::Session : public rtc::VideoSinkInterface<webrtc::Vide
       return;
     }
 
-    uint32_t deviceWidth = info.image_width;
-    uint32_t deviceHeight = info.image_height;
     nsIScreencastServiceClient* client = mClient.get();
     NS_DispatchToMainThread(NS_NewRunnableFunction(
-        "NotifyScreencastFrame", [client, base64, deviceWidth, deviceHeight]() -> void {
+        "NotifyScreencastFrame", [client, base64, pageWidth, pageHeight]() -> void {
           NS_ConvertUTF8toUTF16 utf16(base64);
-          client->ScreencastFrame(utf16, deviceWidth, deviceHeight);
+          client->ScreencastFrame(utf16, pageWidth, pageHeight);
         }));
   }
 
@@ -221,6 +269,10 @@ class nsScreencastService::Session : public rtc::VideoSinkInterface<webrtc::Vide
   uint32_t mJpegQuality;
   rtc::CriticalSection mCaptureCallbackCs;
   uint32_t mFramesInFlight = 0;
+  int mWidth;
+  int mHeight;
+  int mViewportWidth;
+  int mViewportHeight;
   gfx::IntMargin mMargin;
 };
 
@@ -241,7 +293,7 @@ nsScreencastService::nsScreencastService() = default;
 nsScreencastService::~nsScreencastService() {
 }
 
-nsresult nsScreencastService::StartVideoRecording(nsIScreencastServiceClient* aClient, nsIDocShell* aDocShell, bool isVideo, const nsACString& aVideoFileName, uint32_t width, uint32_t height, uint32_t quality, int32_t offsetTop, nsAString& sessionId) {
+nsresult nsScreencastService::StartVideoRecording(nsIScreencastServiceClient* aClient, nsIDocShell* aDocShell, bool isVideo, const nsACString& aVideoFileName, uint32_t width, uint32_t height, uint32_t quality, uint32_t viewportWidth, uint32_t viewportHeight, uint32_t offsetTop, nsAString& sessionId) {
   MOZ_RELEASE_ASSERT(NS_IsMainThread(), "Screencast service must be started on the Main thread.");
 
   PresShell* presShell = aDocShell->GetPresShell();
@@ -282,7 +334,7 @@ nsresult nsScreencastService::StartVideoRecording(nsIScreencastServiceClient* aC
   NS_ENSURE_SUCCESS(rv, rv);
   sessionId = uid;
 
-  auto session = std::make_unique<Session>(aClient, std::move(capturer), std::move(encoder), margin, isVideo ? 0 : quality);
+  auto session = std::make_unique<Session>(aClient, std::move(capturer), std::move(encoder), width, height, viewportWidth, viewportHeight, margin, isVideo ? 0 : quality);
   if (!session->Start())
     return NS_ERROR_FAILURE;
   mIdToSession.emplace(sessionId, std::move(session));