From 9640dbf2a6428cef63daacc7b9f532fc7a6624a4 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Tue, 7 Jul 2020 09:40:09 -0700 Subject: [PATCH] browser(firefox): exclude browser controls from screencast (#2855) --- browser_patches/firefox/BUILD_NUMBER | 4 +-- .../firefox/juggler/protocol/PageHandler.js | 4 ++- .../juggler/screencast/ScreencastEncoder.cpp | 35 ++++++++++--------- .../juggler/screencast/ScreencastEncoder.h | 7 ++-- .../screencast/nsIScreencastService.idl | 2 +- .../screencast/nsScreencastService.cpp | 8 ++--- 6 files changed, 30 insertions(+), 30 deletions(-) diff --git a/browser_patches/firefox/BUILD_NUMBER b/browser_patches/firefox/BUILD_NUMBER index 7cf2918a92..5f888a7790 100644 --- a/browser_patches/firefox/BUILD_NUMBER +++ b/browser_patches/firefox/BUILD_NUMBER @@ -1,2 +1,2 @@ -1123 -Changed: yurys@chromium.org Mon Jul 6 11:13:23 PDT 2020 +1124 +Changed: yurys@chromium.org Mon Jul 6 17:16:56 PDT 2020 diff --git a/browser_patches/firefox/juggler/protocol/PageHandler.js b/browser_patches/firefox/juggler/protocol/PageHandler.js index ac766251d7..96e6ad06ac 100644 --- a/browser_patches/firefox/juggler/protocol/PageHandler.js +++ b/browser_patches/firefox/juggler/protocol/PageHandler.js @@ -294,7 +294,9 @@ class PageHandler { const screencast = Cc['@mozilla.org/juggler/screencast;1'].getService(Ci.nsIScreencastService); const docShell = this._pageTarget._gBrowser.ownerGlobal.docShell; - this._videoSessionId = screencast.startVideoRecording(docShell, file, width, height, scale || 0); + // Exclude address bar and navigation control from the video. + const rect = this._pageTarget.linkedBrowser().getBoundingClientRect(); + this._videoSessionId = screencast.startVideoRecording(docShell, file, width, height, scale || 0, rect.top); } stopVideoRecording() { diff --git a/browser_patches/firefox/juggler/screencast/ScreencastEncoder.cpp b/browser_patches/firefox/juggler/screencast/ScreencastEncoder.cpp index b6e032a4e3..bc47ed57d8 100644 --- a/browser_patches/firefox/juggler/screencast/ScreencastEncoder.cpp +++ b/browser_patches/firefox/juggler/screencast/ScreencastEncoder.cpp @@ -158,9 +158,10 @@ void ivf_write_frame_header(FILE *outfile, int64_t pts, size_t frame_size) { class ScreencastEncoder::VPXFrame { public: - VPXFrame(rtc::scoped_refptr&& buffer, Maybe scale) + VPXFrame(rtc::scoped_refptr&& buffer, Maybe scale, int offsetTop) : m_frameBuffer(std::move(buffer)) , m_scale(scale) + , m_offsetTop(offsetTop) { } void setDuration(int duration) { m_duration = duration; } @@ -189,15 +190,15 @@ public: src_width *= image->w / dst_width; dst_width = image->w; } - int src_height = src->height(); + int src_height = src->height() - m_offsetTop; double dst_height = src_height * m_scale.value(); if (dst_height > image->h) { src_height *= image->h / dst_height; dst_height = image->h; } - libyuv::I420Scale(src->DataY(), src->StrideY(), - src->DataU(), src->StrideU(), - src->DataV(), src->StrideV(), + libyuv::I420Scale(src->DataY() + m_offsetTop * src->StrideY(), src->StrideY(), + src->DataU() + (m_offsetTop + 1) / 2 * src->StrideU(), src->StrideU(), + src->DataV() + (m_offsetTop + 1) / 2 * src->StrideV(), src->StrideV(), src_width, src_height, y_data, y_stride, u_data, uv_stride, @@ -206,10 +207,11 @@ public: libyuv::kFilterBilinear); } else { int width = std::min(image->w, src->width()); - int height = std::min(image->h, src->height()); - libyuv::I420Copy(src->DataY(), src->StrideY(), - src->DataU(), src->StrideU(), - src->DataV(), src->StrideV(), + int height = std::min(image->h, src->height() - m_offsetTop); + + libyuv::I420Copy(src->DataY() + m_offsetTop * src->StrideY(), src->StrideY(), + src->DataU() + (m_offsetTop + 1) / 2 * src->StrideU(), src->StrideU(), + src->DataV() + (m_offsetTop + 1) / 2 * src->StrideV(), src->StrideV(), y_data, y_stride, u_data, uv_stride, v_data, uv_stride, @@ -220,6 +222,7 @@ public: private: rtc::scoped_refptr m_frameBuffer; Maybe m_scale; + int m_offsetTop = 0; int m_duration = 0; }; @@ -325,11 +328,10 @@ private: std::unique_ptr m_image; }; -ScreencastEncoder::ScreencastEncoder(std::unique_ptr&& vpxCodec, int width, int height, Maybe scale) +ScreencastEncoder::ScreencastEncoder(std::unique_ptr&& vpxCodec, Maybe scale, int offsetTop) : m_vpxCodec(std::move(vpxCodec)) - , m_width(width) - , m_height(height) , m_scale(scale) + , m_offsetTop(offsetTop) { } @@ -338,10 +340,9 @@ ScreencastEncoder::~ScreencastEncoder() } static constexpr uint32_t vp8fourcc = 0x30385056; -static constexpr uint32_t vp9fourcc = 0x30395056; -static constexpr int fps = 30; +static constexpr int fps = 24; -RefPtr ScreencastEncoder::create(nsCString& errorString, const nsCString& filePath, int width, int height, Maybe scale) +RefPtr ScreencastEncoder::create(nsCString& errorString, const nsCString& filePath, int width, int height, Maybe scale, int offsetTop) { const uint32_t fourcc = vp8fourcc; vpx_codec_iface_t* codec_interface = vpx_codec_vp8_cx(); @@ -383,7 +384,7 @@ RefPtr ScreencastEncoder::create(nsCString& errorString, cons std::unique_ptr vpxCodec(new VPXCodec(fourcc, codec, cfg, file)); fprintf(stderr, "ScreencastEncoder initialized with: %s\n", vpx_codec_iface_name(codec_interface)); - return new ScreencastEncoder(std::move(vpxCodec), width, height, scale); + return new ScreencastEncoder(std::move(vpxCodec), scale, offsetTop); } void ScreencastEncoder::flushLastFrame() @@ -407,7 +408,7 @@ void ScreencastEncoder::encodeFrame(const webrtc::VideoFrame& videoFrame) fprintf(stderr, "ScreencastEncoder::encodeFrame\n"); flushLastFrame(); - m_lastFrame = std::make_unique(videoFrame.video_frame_buffer(), m_scale); + m_lastFrame = std::make_unique(videoFrame.video_frame_buffer(), m_scale, m_offsetTop); } void ScreencastEncoder::finish(std::function&& callback) diff --git a/browser_patches/firefox/juggler/screencast/ScreencastEncoder.h b/browser_patches/firefox/juggler/screencast/ScreencastEncoder.h index 99c3a653d1..80208f0697 100644 --- a/browser_patches/firefox/juggler/screencast/ScreencastEncoder.h +++ b/browser_patches/firefox/juggler/screencast/ScreencastEncoder.h @@ -21,10 +21,10 @@ class ScreencastEncoder { NS_INLINE_DECL_THREADSAFE_REFCOUNTING(ScreencastEncoder) public: - static RefPtr create(nsCString& errorString, const nsCString& filePath, int width, int height, Maybe scale); + static RefPtr create(nsCString& errorString, const nsCString& filePath, int width, int height, Maybe scale, int offsetTop); class VPXCodec; - ScreencastEncoder(std::unique_ptr&&, int width, int height, Maybe scale); + ScreencastEncoder(std::unique_ptr&&, Maybe scale, int offsetTop); void encodeFrame(const webrtc::VideoFrame& videoFrame); @@ -36,9 +36,8 @@ private: void flushLastFrame(); std::unique_ptr m_vpxCodec; - int m_width; - int m_height; Maybe m_scale; + int m_offsetTop; TimeStamp m_lastFrameTimestamp; class VPXFrame; std::unique_ptr m_lastFrame; diff --git a/browser_patches/firefox/juggler/screencast/nsIScreencastService.idl b/browser_patches/firefox/juggler/screencast/nsIScreencastService.idl index da5ca7d60d..7f896a6c46 100644 --- a/browser_patches/firefox/juggler/screencast/nsIScreencastService.idl +++ b/browser_patches/firefox/juggler/screencast/nsIScreencastService.idl @@ -12,6 +12,6 @@ interface nsIDocShell; [scriptable, uuid(d8c4d9e0-9462-445e-9e43-68d3872ad1de)] interface nsIScreencastService : nsISupports { - long startVideoRecording(in nsIDocShell docShell, in ACString fileName, in uint32_t width, in uint32_t height, in double scale); + long startVideoRecording(in nsIDocShell docShell, in ACString fileName, in uint32_t width, in uint32_t height, in double scale, in int32_t offset_top); void stopVideoRecording(in long sessionId); }; diff --git a/browser_patches/firefox/juggler/screencast/nsScreencastService.cpp b/browser_patches/firefox/juggler/screencast/nsScreencastService.cpp index a41921dbea..d15bbde696 100644 --- a/browser_patches/firefox/juggler/screencast/nsScreencastService.cpp +++ b/browser_patches/firefox/juggler/screencast/nsScreencastService.cpp @@ -33,8 +33,7 @@ StaticRefPtr gScreencastService; class nsScreencastService::Session : public rtc::VideoSinkInterface { public: Session(int sessionId, const nsCString& windowId, RefPtr&& encoder) - : mSessionId(sessionId) - , mCaptureModule(webrtc::DesktopCaptureImpl::Create( + : mCaptureModule(webrtc::DesktopCaptureImpl::Create( sessionId, windowId.get(), webrtc::CaptureDeviceType::Window)) , mEncoder(std::move(encoder)) { } @@ -71,7 +70,6 @@ class nsScreencastService::Session : public rtc::VideoSinkInterface mCaptureModule; RefPtr mEncoder; }; @@ -93,7 +91,7 @@ nsScreencastService::nsScreencastService() = default; nsScreencastService::~nsScreencastService() { } -nsresult nsScreencastService::StartVideoRecording(nsIDocShell* aDocShell, const nsACString& aFileName, uint32_t width, uint32_t height, double scale, int32_t* sessionId) { +nsresult nsScreencastService::StartVideoRecording(nsIDocShell* aDocShell, const nsACString& aFileName, uint32_t width, uint32_t height, double scale, int32_t offsetTop, int32_t* sessionId) { MOZ_RELEASE_ASSERT(NS_IsMainThread(), "Screencast service must be started on the Main thread."); *sessionId = -1; @@ -124,7 +122,7 @@ nsresult nsScreencastService::StartVideoRecording(nsIDocShell* aDocShell, const Maybe maybeScale; if (scale) maybeScale = Some(scale); - RefPtr encoder = ScreencastEncoder::create(error, PromiseFlatCString(aFileName), width, height, maybeScale); + RefPtr encoder = ScreencastEncoder::create(error, PromiseFlatCString(aFileName), width, height, maybeScale, offsetTop); if (!encoder) { fprintf(stderr, "Failed to create ScreencastEncoder: %s\n", error.get()); return NS_ERROR_FAILURE;