From 024cb1ddc19e50459d1ae159958e0ae2e5b25137 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Thu, 2 Jul 2020 10:43:04 -0700 Subject: [PATCH] browser(firefox): basic screencast implementation for GTK (#2818) --- browser_patches/firefox/BUILD_NUMBER | 2 +- browser_patches/firefox/juggler/moz.build | 2 +- .../firefox/juggler/protocol/PageHandler.js | 16 + .../firefox/juggler/protocol/Protocol.js | 10 + .../juggler/screencast/ScreencastEncoder.cpp | 397 ++++++++++++++++++ .../juggler/screencast/ScreencastEncoder.h | 47 +++ .../juggler/screencast/components.conf | 15 + .../firefox/juggler/screencast/moz.build | 32 ++ .../screencast/nsIScreencastService.idl | 17 + .../screencast/nsScreencastService.cpp | 148 +++++++ .../juggler/screencast/nsScreencastService.h | 30 ++ .../firefox/patches/bootstrap.diff | 8 +- 12 files changed, 719 insertions(+), 5 deletions(-) create mode 100644 browser_patches/firefox/juggler/screencast/ScreencastEncoder.cpp create mode 100644 browser_patches/firefox/juggler/screencast/ScreencastEncoder.h create mode 100644 browser_patches/firefox/juggler/screencast/components.conf create mode 100644 browser_patches/firefox/juggler/screencast/moz.build create mode 100644 browser_patches/firefox/juggler/screencast/nsIScreencastService.idl create mode 100644 browser_patches/firefox/juggler/screencast/nsScreencastService.cpp create mode 100644 browser_patches/firefox/juggler/screencast/nsScreencastService.h diff --git a/browser_patches/firefox/BUILD_NUMBER b/browser_patches/firefox/BUILD_NUMBER index 79e6287b6a..487285274c 100644 --- a/browser_patches/firefox/BUILD_NUMBER +++ b/browser_patches/firefox/BUILD_NUMBER @@ -1 +1 @@ -1118 +1119 diff --git a/browser_patches/firefox/juggler/moz.build b/browser_patches/firefox/juggler/moz.build index 1a0a3130bf..efc74e5767 100644 --- a/browser_patches/firefox/juggler/moz.build +++ b/browser_patches/firefox/juggler/moz.build @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -DIRS += ["components"] +DIRS += ["components", "screencast"] JAR_MANIFESTS += ["jar.mn"] #JS_PREFERENCE_FILES += ["prefs/marionette.js"] diff --git a/browser_patches/firefox/juggler/protocol/PageHandler.js b/browser_patches/firefox/juggler/protocol/PageHandler.js index d0b7ac4ac9..04c260d664 100644 --- a/browser_patches/firefox/juggler/protocol/PageHandler.js +++ b/browser_patches/firefox/juggler/protocol/PageHandler.js @@ -90,6 +90,7 @@ class PageHandler { this._dialogs = new Map(); this._enabled = false; + this._videoSessionId = -1; } _onWorkerCreated({workerId, frameId, url}) { @@ -284,6 +285,21 @@ class PageHandler { throw new Error('ERROR: cannot find worker with id ' + workerId); return await worker.sendMessage(JSON.parse(message)); } + + startVideoRecording({file, width, height, scale}) { + const screencast = Cc['@mozilla.org/juggler/screencast;1'].getService(Ci.nsIScreencastService); + const docShell = this._pageTarget._gBrowser.ownerGlobal.docShell; + this._videoSessionId = screencast.startVideoRecording(docShell, file); + } + + stopVideoRecording() { + if (this._videoSessionId === -1) + throw new Error('No video recording in progress'); + const videoSessionId = this._videoSessionId; + this._videoSessionId = -1; + const screencast = Cc['@mozilla.org/juggler/screencast;1'].getService(Ci.nsIScreencastService); + screencast.stopVideoRecording(videoSessionId); + } } class Dialog { diff --git a/browser_patches/firefox/juggler/protocol/Protocol.js b/browser_patches/firefox/juggler/protocol/Protocol.js index 1db1f8c835..792f28598d 100644 --- a/browser_patches/firefox/juggler/protocol/Protocol.js +++ b/browser_patches/firefox/juggler/protocol/Protocol.js @@ -825,6 +825,16 @@ const Page = { message: t.String, }, }, + 'startVideoRecording': { + params: { + file: t.String, + width: t.Number, + height: t.Number, + scale: t.Optional(t.Number), + }, + }, + 'stopVideoRecording': { + }, }, }; diff --git a/browser_patches/firefox/juggler/screencast/ScreencastEncoder.cpp b/browser_patches/firefox/juggler/screencast/ScreencastEncoder.cpp new file mode 100644 index 0000000000..34d3491369 --- /dev/null +++ b/browser_patches/firefox/juggler/screencast/ScreencastEncoder.cpp @@ -0,0 +1,397 @@ +/* + * Copyright (c) 2010, The WebM Project authors. All rights reserved. + * Copyright (c) 2013 The Chromium Authors. All rights reserved. + * Copyright (C) 2020 Microsoft Corporation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "ScreencastEncoder.h" + +#include +#include +#include +#include +#include "nsThreadUtils.h" +#include "webrtc/api/video/video_frame.h" + +namespace mozilla { + +namespace { +// Defines the dimension of a macro block. This is used to compute the active +// map for the encoder. +const int kMacroBlockSize = 16; + +void createImage(unsigned int width, unsigned int height, + std::unique_ptr& out_image, + std::unique_ptr& out_image_buffer) { + std::unique_ptr image(new vpx_image_t()); + memset(image.get(), 0, sizeof(vpx_image_t)); + + // libvpx seems to require both to be assigned. + image->d_w = width; + image->w = width; + image->d_h = height; + image->h = height; + + // I420 + image->fmt = VPX_IMG_FMT_YV12; + image->x_chroma_shift = 1; + image->y_chroma_shift = 1; + + // libyuv's fast-path requires 16-byte aligned pointers and strides, so pad + // the Y, U and V planes' strides to multiples of 16 bytes. + const int y_stride = ((image->w - 1) & ~15) + 16; + const int uv_unaligned_stride = y_stride >> image->x_chroma_shift; + const int uv_stride = ((uv_unaligned_stride - 1) & ~15) + 16; + + // libvpx accesses the source image in macro blocks, and will over-read + // if the image is not padded out to the next macroblock: crbug.com/119633. + // Pad the Y, U and V planes' height out to compensate. + // Assuming macroblocks are 16x16, aligning the planes' strides above also + // macroblock aligned them. + static_assert(kMacroBlockSize == 16, "macroblock_size_not_16"); + const int y_rows = ((image->h - 1) & ~(kMacroBlockSize-1)) + kMacroBlockSize; + const int uv_rows = y_rows >> image->y_chroma_shift; + + // Allocate a YUV buffer large enough for the aligned data & padding. + const int buffer_size = y_stride * y_rows + 2*uv_stride * uv_rows; + std::unique_ptr image_buffer(new uint8_t[buffer_size]); + + // Reset image value to 128 so we just need to fill in the y plane. + memset(image_buffer.get(), 128, buffer_size); + + // Fill in the information for |image_|. + unsigned char* uchar_buffer = + reinterpret_cast(image_buffer.get()); + image->planes[0] = uchar_buffer; + image->planes[1] = image->planes[0] + y_stride * y_rows; + image->planes[2] = image->planes[1] + uv_stride * uv_rows; + image->stride[0] = y_stride; + image->stride[1] = uv_stride; + image->stride[2] = uv_stride; + + out_image = std::move(image); + out_image_buffer = std::move(image_buffer); +} + +void mem_put_le16(void *vmem, int val) { + unsigned char *mem = (unsigned char *)vmem; + + mem[0] = (unsigned char)((val >> 0) & 0xff); + mem[1] = (unsigned char)((val >> 8) & 0xff); +} + +void mem_put_le32(void *vmem, int val) { + unsigned char *mem = (unsigned char *)vmem; + + mem[0] = (unsigned char)((val >> 0) & 0xff); + mem[1] = (unsigned char)((val >> 8) & 0xff); + mem[2] = (unsigned char)((val >> 16) & 0xff); + mem[3] = (unsigned char)((val >> 24) & 0xff); +} + +void ivf_write_file_header_with_video_info(FILE *outfile, uint32_t fourcc, + int frame_cnt, int frame_width, + int frame_height, + vpx_rational_t timebase) { + char header[32]; + + header[0] = 'D'; + header[1] = 'K'; + header[2] = 'I'; + header[3] = 'F'; + mem_put_le16(header + 4, 0); // version + mem_put_le16(header + 6, 32); // header size + mem_put_le32(header + 8, fourcc); // fourcc + mem_put_le16(header + 12, frame_width); // width + mem_put_le16(header + 14, frame_height); // height + mem_put_le32(header + 16, timebase.den); // rate + mem_put_le32(header + 20, timebase.num); // scale + mem_put_le32(header + 24, frame_cnt); // length + mem_put_le32(header + 28, 0); // unused + + fwrite(header, 1, 32, outfile); +} + +void ivf_write_file_header(FILE *outfile, const struct vpx_codec_enc_cfg *cfg, + uint32_t fourcc, int frame_cnt) { + ivf_write_file_header_with_video_info(outfile, fourcc, frame_cnt, cfg->g_w, + cfg->g_h, cfg->g_timebase); +} + +void ivf_write_frame_header(FILE *outfile, int64_t pts, size_t frame_size) { + char header[12]; + + mem_put_le32(header, (int)frame_size); + mem_put_le32(header + 4, (int)(pts & 0xFFFFFFFF)); + mem_put_le32(header + 8, (int)(pts >> 32)); + fwrite(header, 1, 12, outfile); +} + +} // namespace + +class ScreencastEncoder::VPXFrame { +public: + VPXFrame(rtc::scoped_refptr&& buffer, Maybe scale) + : m_frameBuffer(std::move(buffer)) + , m_scale(scale) + { } + + void setDuration(int duration) { m_duration = duration; } + int duration() const { return m_duration; } + + void convertToVpxImage(vpx_image_t* image) + { + if (m_frameBuffer->type() != webrtc::VideoFrameBuffer::Type::kI420) { + fprintf(stderr, "convertToVpxImage unexpected frame buffer type: %d\n", m_frameBuffer->type()); + return; + } + + auto src = m_frameBuffer->GetI420(); + + const int y_stride = image->stride[0]; + MOZ_ASSERT(image->stride[1] == image->stride[2]); + const int uv_stride = image->stride[1]; + uint8_t* y_data = image->planes[0]; + uint8_t* u_data = image->planes[1]; + uint8_t* v_data = image->planes[2]; + + libyuv::I420Copy(src->DataY(), src->StrideY(), + src->DataU(), src->StrideU(), + src->DataV(), src->StrideV(), + y_data, y_stride, + u_data, uv_stride, + v_data, uv_stride, + image->w, image->h); + } + +private: + rtc::scoped_refptr m_frameBuffer; + Maybe m_scale; + int m_duration = 0; +}; + + +class ScreencastEncoder::VPXCodec { +public: + VPXCodec(uint32_t fourcc, vpx_codec_ctx_t codec, vpx_codec_enc_cfg_t cfg, FILE* file) + : m_fourcc(fourcc) + , m_codec(codec) + , m_cfg(cfg) + , m_file(file) + { + nsresult rv = NS_NewNamedThread("Screencast enc", getter_AddRefs(m_encoderQueue)); + if (rv != NS_OK) { + fprintf(stderr, "ScreencastEncoder::VPXCodec failed to spawn thread %d\n", rv); + return; + } + + ivf_write_file_header(m_file, &m_cfg, m_fourcc, 0); + + createImage(cfg.g_w, cfg.g_h, m_image, m_imageBuffer); + } + + ~VPXCodec() { + m_encoderQueue->Shutdown(); + m_encoderQueue = nullptr; + } + + void encodeFrameAsync(std::unique_ptr&& frame) + { + m_encoderQueue->Dispatch(NS_NewRunnableFunction("VPXCodec::encodeFrameAsync", [this, frame = std::move(frame)] { + frame->convertToVpxImage(m_image.get()); + // TODO: figure out why passing duration to the codec results in much + // worse visual quality and makes video stutter. + for (int i = 0; i < frame->duration(); i++) + encodeFrame(m_image.get(), 1); + })); + } + + void finishAsync(std::function&& callback) + { + m_encoderQueue->Dispatch(NS_NewRunnableFunction("VPXCodec::finishAsync", [this, callback = std::move(callback)] { + finish(); + callback(); + })); + } + +private: + bool encodeFrame(vpx_image_t *img, int duration) + { + vpx_codec_iter_t iter = nullptr; + const vpx_codec_cx_pkt_t *pkt = nullptr; + int flags = 0; + const vpx_codec_err_t res = vpx_codec_encode(&m_codec, img, m_pts, duration, flags, VPX_DL_REALTIME); + if (res != VPX_CODEC_OK) { + fprintf(stderr, "Failed to encode frame: %s\n", vpx_codec_error(&m_codec)); + return false; + } + + bool gotPkts = false; + while ((pkt = vpx_codec_get_cx_data(&m_codec, &iter)) != nullptr) { + gotPkts = true; + + if (pkt->kind == VPX_CODEC_CX_FRAME_PKT) { + ivf_write_frame_header(m_file, m_pts, pkt->data.frame.sz); + if (fwrite(pkt->data.frame.buf, 1, pkt->data.frame.sz, m_file) != pkt->data.frame.sz) { + fprintf(stderr, "Failed to write compressed frame\n"); + return 0; + } + bool keyframe = (pkt->data.frame.flags & VPX_FRAME_IS_KEY) != 0; + ++m_frameCount; + fprintf(stderr, " #%03d %spts=%" PRId64 " sz=%zd\n", m_frameCount, keyframe ? "[K] " : "", pkt->data.frame.pts, pkt->data.frame.sz); + m_pts += pkt->data.frame.duration; + } + } + + return gotPkts; + } + + void finish() + { + // Flush encoder. + while (encodeFrame(nullptr, 1)) + ++m_frameCount; + + rewind(m_file); + // Update total frame count. + ivf_write_file_header(m_file, &m_cfg, m_fourcc, m_frameCount); + fclose(m_file); + fprintf(stderr, "ScreencastEncoder::finish %d frames\n", m_frameCount); + } + + RefPtr m_encoderQueue; + uint32_t m_fourcc { 0 }; + vpx_codec_ctx_t m_codec; + vpx_codec_enc_cfg_t m_cfg; + FILE* m_file { nullptr }; + int m_frameCount { 0 }; + int64_t m_pts { 0 }; + std::unique_ptr m_imageBuffer; + std::unique_ptr m_image; +}; + +ScreencastEncoder::ScreencastEncoder(std::unique_ptr&& vpxCodec, int width, int height, Maybe scale) + : m_vpxCodec(std::move(vpxCodec)) + , m_width(width) + , m_height(height) + , m_scale(scale) +{ +} + +ScreencastEncoder::~ScreencastEncoder() +{ +} + +static constexpr uint32_t vp8fourcc = 0x30385056; +static constexpr uint32_t vp9fourcc = 0x30395056; +static constexpr int fps = 30; + +RefPtr ScreencastEncoder::create(nsCString& errorString, const nsCString& filePath, int width, int height, Maybe scale) +{ + const uint32_t fourcc = vp8fourcc; + vpx_codec_iface_t* codec_interface = vpx_codec_vp8_cx(); + if (!codec_interface) { + errorString = "Codec not found."; + return nullptr; + } + + if (width <= 0 || height <= 0 || (width % 2) != 0 || (height % 2) != 0) { + errorString.AppendPrintf("Invalid frame size: %dx%d", width, height); + return nullptr; + } + + vpx_codec_enc_cfg_t cfg; + memset(&cfg, 0, sizeof(cfg)); + vpx_codec_err_t error = vpx_codec_enc_config_default(codec_interface, &cfg, 0); + if (error) { + errorString.AppendPrintf("Failed to get default codec config: %s", vpx_codec_err_to_string(error)); + return nullptr; + } + + cfg.g_w = width; + cfg.g_h = height; + cfg.g_timebase.num = 1; + cfg.g_timebase.den = fps; + cfg.g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT; + + vpx_codec_ctx_t codec; + if (vpx_codec_enc_init(&codec, codec_interface, &cfg, 0)) { + errorString.AppendPrintf("Failed to initialize encoder: %s", vpx_codec_error(&codec)); + return nullptr; + } + + FILE* file = fopen(filePath.get(), "wb"); + if (!file) { + errorString.AppendPrintf("Failed to open file '%s' for writing: %s", filePath.get(), strerror(errno)); + return nullptr; + } + + std::unique_ptr vpxCodec(new VPXCodec(fourcc, codec, cfg, file)); + fprintf(stderr, "ScreencastEncoder initialized with: %s\n", vpx_codec_iface_name(codec_interface)); + return new ScreencastEncoder(std::move(vpxCodec), width, height, scale); +} + +void ScreencastEncoder::flushLastFrame() +{ + TimeStamp now = TimeStamp::Now(); + if (m_lastFrameTimestamp) { + // If previous frame encoding failed for some rason leave the timestampt intact. + if (!m_lastFrame) + return; + + TimeDuration seconds = now - m_lastFrameTimestamp; + int duration = 1 + seconds.ToSeconds() * fps; // Duration in timebase units + m_lastFrame->setDuration(duration); + m_vpxCodec->encodeFrameAsync(std::move(m_lastFrame)); + } + m_lastFrameTimestamp = now; +} + +void ScreencastEncoder::encodeFrame(const webrtc::VideoFrame& videoFrame) +{ + fprintf(stderr, "ScreencastEncoder::encodeFrame\n"); + flushLastFrame(); + + m_lastFrame = std::make_unique(videoFrame.video_frame_buffer(), m_scale); +} + +void ScreencastEncoder::finish(std::function&& callback) +{ + if (!m_vpxCodec) { + callback(); + return; + } + + flushLastFrame(); + m_vpxCodec->finishAsync([callback = std::move(callback)] () mutable { + NS_DispatchToMainThread(NS_NewRunnableFunction("ScreencastEncoder::finish callback", std::move(callback))); + }); +} + + +} // namespace mozilla diff --git a/browser_patches/firefox/juggler/screencast/ScreencastEncoder.h b/browser_patches/firefox/juggler/screencast/ScreencastEncoder.h new file mode 100644 index 0000000000..99c3a653d1 --- /dev/null +++ b/browser_patches/firefox/juggler/screencast/ScreencastEncoder.h @@ -0,0 +1,47 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#pragma once + +#include +#include +#include "mozilla/Maybe.h" +#include "mozilla/TimeStamp.h" +#include "nsISupportsImpl.h" +#include "nsStringFwd.h" + +namespace webrtc { +class VideoFrame; +} + +namespace mozilla { + +class ScreencastEncoder { + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(ScreencastEncoder) +public: + + static RefPtr create(nsCString& errorString, const nsCString& filePath, int width, int height, Maybe scale); + + class VPXCodec; + ScreencastEncoder(std::unique_ptr&&, int width, int height, Maybe scale); + + void encodeFrame(const webrtc::VideoFrame& videoFrame); + + void finish(std::function&& callback); + +private: + ~ScreencastEncoder(); + + void flushLastFrame(); + + std::unique_ptr m_vpxCodec; + int m_width; + int m_height; + Maybe m_scale; + TimeStamp m_lastFrameTimestamp; + class VPXFrame; + std::unique_ptr m_lastFrame; +}; + +} // namespace mozilla diff --git a/browser_patches/firefox/juggler/screencast/components.conf b/browser_patches/firefox/juggler/screencast/components.conf new file mode 100644 index 0000000000..6298739122 --- /dev/null +++ b/browser_patches/firefox/juggler/screencast/components.conf @@ -0,0 +1,15 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +Classes = [ + { + 'cid': '{d8c4d9e0-9462-445e-9e43-68d3872ad1de}', + 'contract_ids': ['@mozilla.org/juggler/screencast;1'], + 'type': 'nsIScreencastService', + 'constructor': 'mozilla::nsScreencastService::GetSingleton', + 'headers': ['/juggler/screencast/nsScreencastService.h'], + }, +] diff --git a/browser_patches/firefox/juggler/screencast/moz.build b/browser_patches/firefox/juggler/screencast/moz.build new file mode 100644 index 0000000000..7080275abd --- /dev/null +++ b/browser_patches/firefox/juggler/screencast/moz.build @@ -0,0 +1,32 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +XPIDL_SOURCES += [ + 'nsIScreencastService.idl', +] + +XPIDL_MODULE = 'jugglerscreencast' + +SOURCES += [ + 'nsScreencastService.cpp', + 'ScreencastEncoder.cpp', +] + +XPCOM_MANIFESTS += [ + 'components.conf', +] + +LOCAL_INCLUDES += [ + '/dom/media/systemservices', + '/media/libyuv/libyuv/include', + '/media/webrtc/trunk', + '/media/webrtc/trunk/webrtc', +] + +include('/media/webrtc/webrtc.mozbuild') +include('/ipc/chromium/chromium-config.mozbuild') + +FINAL_LIBRARY = 'xul' diff --git a/browser_patches/firefox/juggler/screencast/nsIScreencastService.idl b/browser_patches/firefox/juggler/screencast/nsIScreencastService.idl new file mode 100644 index 0000000000..7f629b4c67 --- /dev/null +++ b/browser_patches/firefox/juggler/screencast/nsIScreencastService.idl @@ -0,0 +1,17 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +interface nsIDocShell; + +/** + * Service for recording window video. + */ +[scriptable, uuid(d8c4d9e0-9462-445e-9e43-68d3872ad1de)] +interface nsIScreencastService : nsISupports +{ + long startVideoRecording(in nsIDocShell docShell, in ACString fileName); + void stopVideoRecording(in long sessionId); +}; diff --git a/browser_patches/firefox/juggler/screencast/nsScreencastService.cpp b/browser_patches/firefox/juggler/screencast/nsScreencastService.cpp new file mode 100644 index 0000000000..25d342b37f --- /dev/null +++ b/browser_patches/firefox/juggler/screencast/nsScreencastService.cpp @@ -0,0 +1,148 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsScreencastService.h" + +#include "ScreencastEncoder.h" +#include "mozilla/ClearOnShutdown.h" +#include "mozilla/PresShell.h" +#include "mozilla/StaticPtr.h" +#include "nsIDocShell.h" +#include "nsThreadManager.h" +#include "nsView.h" +#include "nsViewManager.h" +#include "webrtc/modules/desktop_capture/desktop_capturer.h" +#include "webrtc/modules/desktop_capture/desktop_capture_options.h" +#include "webrtc/modules/desktop_capture/desktop_device_info.h" +#include "webrtc/modules/desktop_capture/desktop_frame.h" +#include "webrtc/modules/video_capture/video_capture.h" +#include "mozilla/widget/PlatformWidgetTypes.h" +#include "video_engine/desktop_capture_impl.h" + +namespace mozilla { + +NS_IMPL_ISUPPORTS(nsScreencastService, nsIScreencastService) + +namespace { + +StaticRefPtr gScreencastService; + +} + +class nsScreencastService::Session : public rtc::VideoSinkInterface { + public: + Session(int sessionId, const nsCString& windowId, RefPtr&& encoder) + : mSessionId(sessionId) + , mCaptureModule(webrtc::DesktopCaptureImpl::Create( + sessionId, windowId.get(), webrtc::CaptureDeviceType::Window)) + , mEncoder(std::move(encoder)) { + } + + bool Start() { + webrtc::VideoCaptureCapability capability; + // The size is ignored in fact. + capability.width = 1280; + capability.height = 960; + capability.maxFPS = 24; + capability.videoType = webrtc::VideoType::kI420; + int error = mCaptureModule->StartCapture(capability); + if (error) { + fprintf(stderr, "StartCapture error %d\n", error); + return false; + } + + mCaptureModule->RegisterCaptureDataCallback(this); + return true; + } + + void Stop() { + mCaptureModule->DeRegisterCaptureDataCallback(this); + int error = mCaptureModule->StopCapture(); + if (error) { + fprintf(stderr, "StopCapture error %d\n", error); + return; + } + } + + // These callbacks end up running on the VideoCapture thread. + void OnFrame(const webrtc::VideoFrame& videoFrame) override { + mEncoder->encodeFrame(videoFrame); + } + + private: + int mSessionId; + rtc::scoped_refptr mCaptureModule; + RefPtr mEncoder; +}; + + +// static +already_AddRefed nsScreencastService::GetSingleton() { + if (gScreencastService) { + return do_AddRef(gScreencastService); + } + + gScreencastService = new nsScreencastService(); + // ClearOnShutdown(&gScreencastService); + return do_AddRef(gScreencastService); +} + +nsScreencastService::nsScreencastService() = default; + +nsScreencastService::~nsScreencastService() { +} + +nsresult nsScreencastService::StartVideoRecording(nsIDocShell* aDocShell, const nsACString& aFileName, int32_t* sessionId) { + MOZ_RELEASE_ASSERT(NS_IsMainThread(), "Screencast service must be started on the Main thread."); + *sessionId = -1; + + PresShell* presShell = aDocShell->GetPresShell(); + if (!presShell) + return NS_ERROR_UNEXPECTED; + nsViewManager* viewManager = presShell->GetViewManager(); + if (!viewManager) + return NS_ERROR_UNEXPECTED; + nsView* view = viewManager->GetRootView(); + if (!view) + return NS_ERROR_UNEXPECTED; + nsIWidget* widget = view->GetWidget(); + +#ifdef MOZ_WIDGET_GTK + mozilla::widget::CompositorWidgetInitData initData; + widget->GetCompositorWidgetInitData(&initData); + const mozilla::widget::GtkCompositorWidgetInitData& gtkInitData = initData.get_GtkCompositorWidgetInitData(); +# ifdef MOZ_X11 + nsCString windowId; + windowId.AppendPrintf("%lu", gtkInitData.XWindow()); +# else + // TODO: support in wayland + return NS_ERROR_NOT_IMPLEMENTED; +# endif +#endif + *sessionId = ++mLastSessionId; + nsCString error; + RefPtr encoder = ScreencastEncoder::create(error, PromiseFlatCString(aFileName), 1280, 960, Nothing()); + if (!encoder) { + fprintf(stderr, "Failed to create ScreencastEncoder: %s\n", error.get()); + return NS_ERROR_FAILURE; + } + + auto session = std::make_unique(*sessionId, windowId, std::move(encoder)); + if (!session->Start()) + return NS_ERROR_FAILURE; + + mIdToSession.emplace(*sessionId, std::move(session)); + return NS_OK; +} + +nsresult nsScreencastService::StopVideoRecording(int32_t sessionId) { + auto it = mIdToSession.find(sessionId); + if (it == mIdToSession.end()) + return NS_ERROR_INVALID_ARG; + it->second->Stop(); + mIdToSession.erase(it); + return NS_OK; +} + +} // namespace mozilla diff --git a/browser_patches/firefox/juggler/screencast/nsScreencastService.h b/browser_patches/firefox/juggler/screencast/nsScreencastService.h new file mode 100644 index 0000000000..c9df1a1138 --- /dev/null +++ b/browser_patches/firefox/juggler/screencast/nsScreencastService.h @@ -0,0 +1,30 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#pragma once + +#include +#include +#include "nsIScreencastService.h" + +namespace mozilla { + +class nsScreencastService final : public nsIScreencastService { + public: + NS_DECL_ISUPPORTS + NS_DECL_NSISCREENCASTSERVICE + + static already_AddRefed GetSingleton(); + + nsScreencastService(); + + private: + ~nsScreencastService(); + + class Session; + int mLastSessionId = 0; + std::unordered_map> mIdToSession; +}; + +} // namespace mozilla diff --git a/browser_patches/firefox/patches/bootstrap.diff b/browser_patches/firefox/patches/bootstrap.diff index 1235889714..32c49f2d41 100644 --- a/browser_patches/firefox/patches/bootstrap.diff +++ b/browser_patches/firefox/patches/bootstrap.diff @@ -1619,7 +1619,7 @@ index 87701f8d2cfee8bd84acd28c62b3be4989c9474c..ae1aa85c019cb21d4f7e79c35e8afe72 + [optional] in unsigned long aFlags); }; diff --git a/uriloader/exthandler/nsExternalHelperAppService.cpp b/uriloader/exthandler/nsExternalHelperAppService.cpp -index 0808085752eab5ad43bc54d2fac8771fc6d2a1d1..cea94060a58b6c58848d6c6beb54d1108eba168e 100644 +index 0808085752eab5ad43bc54d2fac8771fc6d2a1d1..cb0e45b34e5242e0375abd4e707b9c3da55cd7f3 100644 --- a/uriloader/exthandler/nsExternalHelperAppService.cpp +++ b/uriloader/exthandler/nsExternalHelperAppService.cpp @@ -99,6 +99,7 @@ @@ -1706,7 +1706,7 @@ index 0808085752eab5ad43bc54d2fac8771fc6d2a1d1..cea94060a58b6c58848d6c6beb54d110 if (alwaysAsk) { // But we *don't* ask if this mimeInfo didn't come from // our user configuration datastore and the user has said -@@ -2043,6 +2089,15 @@ nsExternalAppHandler::OnSaveComplete(nsIBackgroundFileSaver* aSaver, +@@ -2043,6 +2089,16 @@ nsExternalAppHandler::OnSaveComplete(nsIBackgroundFileSaver* aSaver, NotifyTransfer(aStatus); } @@ -1716,13 +1716,14 @@ index 0808085752eab5ad43bc54d2fac8771fc6d2a1d1..cea94060a58b6c58848d6c6beb54d110 + nsCString noError; + nsresult rv = interceptor->OnDownloadComplete(this, noError); + MOZ_ASSERT(NS_SUCCEEDED(rv), "Failed to call nsIDowloadInterceptor.OnDownloadComplete"); ++ Unused << rv; + } + } + return NS_OK; } -@@ -2422,6 +2477,14 @@ NS_IMETHODIMP nsExternalAppHandler::Cancel(nsresult aReason) { +@@ -2422,6 +2478,15 @@ NS_IMETHODIMP nsExternalAppHandler::Cancel(nsresult aReason) { } } @@ -1732,6 +1733,7 @@ index 0808085752eab5ad43bc54d2fac8771fc6d2a1d1..cea94060a58b6c58848d6c6beb54d110 + GetErrorName(aReason, errorName); + nsresult rv = interceptor->OnDownloadComplete(this, errorName); + MOZ_ASSERT(NS_SUCCEEDED(rv), "Failed notify nsIDowloadInterceptor about cancel"); ++ Unused << rv; + } + // Break our reference cycle with the helper app dialog (set up in