From b4261ec0744361ec0d7989ca7284af3b685d705c Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Fri, 7 May 2021 21:47:40 -0700 Subject: [PATCH] browser(ff-stable): pick up screencast changes (#6464) --- browser_patches/firefox-stable/BUILD_NUMBER | 4 +- .../firefox-stable/juggler/TargetRegistry.js | 110 +++++++--- .../juggler/protocol/BrowserHandler.js | 13 +- .../juggler/protocol/PageHandler.js | 33 ++- .../juggler/protocol/Protocol.js | 27 ++- .../screencast/HeadlessWindowCapturer.cpp | 36 +++- .../screencast/HeadlessWindowCapturer.h | 9 +- .../juggler/screencast/ScreencastEncoder.cpp | 17 +- .../juggler/screencast/ScreencastEncoder.h | 5 +- .../screencast/nsIScreencastService.idl | 12 +- .../screencast/nsScreencastService.cpp | 192 +++++++++++++----- .../firefox-stable/patches/bootstrap.diff | 111 +++++++++- 12 files changed, 435 insertions(+), 134 deletions(-) diff --git a/browser_patches/firefox-stable/BUILD_NUMBER b/browser_patches/firefox-stable/BUILD_NUMBER index bc10ed2b79..e38e7ba6dc 100644 --- a/browser_patches/firefox-stable/BUILD_NUMBER +++ b/browser_patches/firefox-stable/BUILD_NUMBER @@ -1,2 +1,2 @@ -1245 -Changed: dgozman@gmail.com Tue May 4 16:09:13 PDT 2021 +1246 +Changed: pavel.feldman@gmail.com Fri 07 May 2021 09:10:22 PM PDT diff --git a/browser_patches/firefox-stable/juggler/TargetRegistry.js b/browser_patches/firefox-stable/juggler/TargetRegistry.js index 0e646c83f2..9613fc83c6 100644 --- a/browser_patches/firefox-stable/juggler/TargetRegistry.js +++ b/browser_patches/firefox-stable/juggler/TargetRegistry.js @@ -88,6 +88,8 @@ class DownloadInterceptor { } } +const screencastService = Cc['@mozilla.org/juggler/screencast;1'].getService(Ci.nsIScreencastService); + class TargetRegistry { constructor() { EventEmitter.decorate(this); @@ -159,8 +161,8 @@ class TargetRegistry { target.updateColorSchemeOverride(); if (!hasExplicitSize) target.updateViewportSize(); - if (browserContext.screencastOptions) - target._startVideoRecording(browserContext.screencastOptions); + if (browserContext.videoRecordingOptions) + target._startVideoRecording(browserContext.videoRecordingOptions); }; const onTabCloseListener = event => { @@ -333,7 +335,8 @@ class PageTarget { this._url = 'about:blank'; this._openerId = opener ? opener.id() : undefined; this._channel = SimpleChannel.createForMessageManager(`browser::page[${this._targetId}]`, this._linkedBrowser.messageManager); - this._screencastInfo = undefined; + this._videoRecordingInfo = undefined; + this._screencastRecordingInfo = undefined; this._dialogs = new Map(); const navigationListener = { @@ -488,7 +491,7 @@ class PageTarget { return await this._channel.connect('').send('hasFailedToOverrideTimezone').catch(e => true); } - async _startVideoRecording({width, height, scale, dir}) { + async _startVideoRecording({width, height, dir}) { // On Mac the window may not yet be visible when TargetCreated and its // NSWindow.windowNumber may be -1, so we wait until the window is known // to be initialized and visible. @@ -496,47 +499,86 @@ class PageTarget { const file = OS.Path.join(dir, helper.generateId() + '.webm'); if (width < 10 || width > 10000 || height < 10 || height > 10000) throw new Error("Invalid size"); - if (scale && (scale <= 0 || scale > 1)) - throw new Error("Unsupported scale"); - const screencast = Cc['@mozilla.org/juggler/screencast;1'].getService(Ci.nsIScreencastService); const docShell = this._gBrowser.ownerGlobal.docShell; // Exclude address bar and navigation control from the video. const rect = this.linkedBrowser().getBoundingClientRect(); const devicePixelRatio = this._window.devicePixelRatio; - const viewport = this._viewportSize || this._browserContext.defaultViewportSize || {width: 0, height: 0}; - const videoSessionId = screencast.startVideoRecording(docShell, file, width, height, viewport.width, viewport.height, scale || 0, devicePixelRatio * rect.top); - this._screencastInfo = { videoSessionId, file }; + let sessionId; + const registry = this._registry; + const screencastClient = { + QueryInterface: ChromeUtils.generateQI([Ci.nsIScreencastServiceClient]), + screencastFrame(data, deviceWidth, deviceHeight) { + }, + screencastStopped() { + registry.emit(TargetRegistry.Events.ScreencastStopped, sessionId); + }, + }; + sessionId = screencastService.startVideoRecording(screencastClient, docShell, true, file, width, height, 0, devicePixelRatio * rect.top); + this._videoRecordingInfo = { sessionId, file }; this.emit(PageTarget.Events.ScreencastStarted); } - async _stopVideoRecording() { - if (!this._screencastInfo) + _stopVideoRecording() { + if (!this._videoRecordingInfo) throw new Error('No video recording in progress'); - const screencastInfo = this._screencastInfo; - this._screencastInfo = undefined; - const screencast = Cc['@mozilla.org/juggler/screencast;1'].getService(Ci.nsIScreencastService); - const result = new Promise(resolve => - Services.obs.addObserver(function onStopped(subject, topic, data) { - if (screencastInfo.videoSessionId != data) - return; - - Services.obs.removeObserver(onStopped, 'juggler-screencast-stopped'); - resolve(); - }, 'juggler-screencast-stopped') - ); - screencast.stopVideoRecording(screencastInfo.videoSessionId); - return result; + const videoRecordingInfo = this._videoRecordingInfo; + this._videoRecordingInfo = undefined; + screencastService.stopVideoRecording(videoRecordingInfo.sessionId); } - screencastInfo() { - return this._screencastInfo; + videoRecordingInfo() { + return this._videoRecordingInfo; + } + + async startScreencast({ width, height, quality }) { + // On Mac the window may not yet be visible when TargetCreated and its + // NSWindow.windowNumber may be -1, so we wait until the window is known + // to be initialized and visible. + await this.windowReady(); + if (width < 10 || width > 10000 || height < 10 || height > 10000) + throw new Error("Invalid size"); + + const docShell = this._gBrowser.ownerGlobal.docShell; + // Exclude address bar and navigation control from the video. + const rect = this.linkedBrowser().getBoundingClientRect(); + const devicePixelRatio = this._window.devicePixelRatio; + + const self = this; + const screencastClient = { + QueryInterface: ChromeUtils.generateQI([Ci.nsIScreencastServiceClient]), + screencastFrame(data, deviceWidth, deviceHeight) { + if (self._screencastRecordingInfo) + self.emit(PageTarget.Events.ScreencastFrame, { data, deviceWidth, deviceHeight }); + }, + screencastStopped() { + }, + }; + const screencastId = screencastService.startVideoRecording(screencastClient, docShell, false, '', width, height, quality || 90, devicePixelRatio * rect.top); + this._screencastRecordingInfo = { screencastId }; + return { screencastId }; + } + + screencastFrameAck({ screencastId }) { + if (!this._screencastRecordingInfo || this._screencastRecordingInfo.screencastId !== screencastId) + return; + screencastService.screencastFrameAck(screencastId); + } + + stopScreencast() { + if (!this._screencastRecordingInfo) + throw new Error('No screencast in progress'); + const { screencastId } = this._screencastRecordingInfo; + this._screencastRecordingInfo = undefined; + screencastService.stopVideoRecording(screencastId); } dispose() { this._disposed = true; - if (this._screencastInfo) - this._stopVideoRecording().catch(e => dump(`stopVideoRecording failed:\n${e}\n`)); + if (this._videoRecordingInfo) + this._stopVideoRecording(); + if (this._screencastRecordingInfo) + this.stopScreencast(); this._browserContext.pages.delete(this); this._registry._browserToTarget.delete(this._linkedBrowser); this._registry._browserBrowsingContextToTarget.delete(this._linkedBrowser.browsingContext); @@ -554,6 +596,7 @@ class PageTarget { PageTarget.Events = { ScreencastStarted: Symbol('PageTarget.ScreencastStarted'), + ScreencastFrame: Symbol('PageTarget.ScreencastFrame'), Crashed: Symbol('PageTarget.Crashed'), DialogOpened: Symbol('PageTarget.DialogOpened'), DialogClosed: Symbol('PageTarget.DialogClosed'), @@ -594,7 +637,7 @@ class BrowserContext { this.defaultUserAgent = null; this.touchOverride = false; this.colorScheme = 'none'; - this.screencastOptions = undefined; + this.videoRecordingOptions = undefined; this.scriptsToEvaluateOnNewDocument = []; this.bindings = []; this.settings = {}; @@ -791,8 +834,8 @@ class BrowserContext { return result; } - async setScreencastOptions(options) { - this.screencastOptions = options; + async setVideoRecordingOptions(options) { + this.videoRecordingOptions = options; if (!options) return; const promises = []; @@ -908,6 +951,7 @@ TargetRegistry.Events = { TargetDestroyed: Symbol('TargetRegistry.Events.TargetDestroyed'), DownloadCreated: Symbol('TargetRegistry.Events.DownloadCreated'), DownloadFinished: Symbol('TargetRegistry.Events.DownloadFinished'), + ScreencastStopped: Symbol('TargetRegistry.ScreencastStopped'), }; var EXPORTED_SYMBOLS = ['TargetRegistry', 'PageTarget']; diff --git a/browser_patches/firefox-stable/juggler/protocol/BrowserHandler.js b/browser_patches/firefox-stable/juggler/protocol/BrowserHandler.js index 92dea19214..48c43ab7f3 100644 --- a/browser_patches/firefox-stable/juggler/protocol/BrowserHandler.js +++ b/browser_patches/firefox-stable/juggler/protocol/BrowserHandler.js @@ -37,14 +37,11 @@ class BrowserHandler { helper.on(this._targetRegistry, TargetRegistry.Events.TargetDestroyed, this._onTargetDestroyed.bind(this)), helper.on(this._targetRegistry, TargetRegistry.Events.DownloadCreated, this._onDownloadCreated.bind(this)), helper.on(this._targetRegistry, TargetRegistry.Events.DownloadFinished, this._onDownloadFinished.bind(this)), + helper.on(this._targetRegistry, TargetRegistry.Events.ScreencastStopped, sessionId => { + this._session.emitEvent('Browser.videoRecordingFinished', {screencastId: '' + sessionId}); + }) ]; - const onScreencastStopped = (subject, topic, data) => { - this._session.emitEvent('Browser.screencastFinished', {screencastId: '' + data}); - }; - Services.obs.addObserver(onScreencastStopped, 'juggler-screencast-stopped'); - this._eventListeners.push(() => Services.obs.removeObserver(onScreencastStopped, 'juggler-screencast-stopped')); - for (const target of this._targetRegistry.targets()) this._onTargetCreated(target); @@ -204,8 +201,8 @@ class BrowserHandler { await this._targetRegistry.browserContextForId(browserContextId).setColorScheme(nullToUndefined(colorScheme)); } - async ['Browser.setScreencastOptions']({browserContextId, dir, width, height, scale}) { - await this._targetRegistry.browserContextForId(browserContextId).setScreencastOptions({dir, width, height, scale}); + async ['Browser.setVideoRecordingOptions']({browserContextId, dir, width, height, scale}) { + await this._targetRegistry.browserContextForId(browserContextId).setVideoRecordingOptions({dir, width, height, scale}); } async ['Browser.setUserAgentOverride']({browserContextId, userAgent}) { diff --git a/browser_patches/firefox-stable/juggler/protocol/PageHandler.js b/browser_patches/firefox-stable/juggler/protocol/PageHandler.js index b5b52d106d..964742b00d 100644 --- a/browser_patches/firefox-stable/juggler/protocol/PageHandler.js +++ b/browser_patches/firefox-stable/juggler/protocol/PageHandler.js @@ -88,8 +88,8 @@ class PageHandler { // to be ignored by the protocol clients. this._isPageReady = false; - if (this._pageTarget.screencastInfo()) - this._onScreencastStarted(); + if (this._pageTarget.videoRecordingInfo()) + this._onVideoRecordingStarted(); this._eventListeners = [ helper.on(this._pageTarget, PageTarget.Events.DialogOpened, this._onDialogOpened.bind(this)), @@ -97,7 +97,8 @@ class PageHandler { helper.on(this._pageTarget, PageTarget.Events.Crashed, () => { this._session.emitEvent('Page.crashed', {}); }), - helper.on(this._pageTarget, PageTarget.Events.ScreencastStarted, this._onScreencastStarted.bind(this)), + helper.on(this._pageTarget, PageTarget.Events.ScreencastStarted, this._onVideoRecordingStarted.bind(this)), + helper.on(this._pageTarget, PageTarget.Events.ScreencastFrame, this._onScreencastFrame.bind(this)), helper.on(this._pageNetwork, PageNetwork.Events.Request, this._handleNetworkEvent.bind(this, 'Network.requestWillBeSent')), helper.on(this._pageNetwork, PageNetwork.Events.Response, this._handleNetworkEvent.bind(this, 'Network.responseReceived')), helper.on(this._pageNetwork, PageNetwork.Events.RequestFinished, this._handleNetworkEvent.bind(this, 'Network.requestFinished')), @@ -146,9 +147,13 @@ class PageHandler { helper.removeListeners(this._eventListeners); } - _onScreencastStarted() { - const info = this._pageTarget.screencastInfo(); - this._session.emitEvent('Page.screencastStarted', { screencastId: info.videoSessionId, file: info.file }); + _onVideoRecordingStarted() { + const info = this._pageTarget.videoRecordingInfo(); + this._session.emitEvent('Page.videoRecordingStarted', { screencastId: info.sessionId, file: info.file }); + } + + _onScreencastFrame(params) { + this._session.emitEvent('Page.screencastFrame', params); } _onPageReady(event) { @@ -373,16 +378,24 @@ class PageHandler { return await this._contentPage.send('setInterceptFileChooserDialog', options); } + async ['Page.startScreencast'](options) { + return await this._pageTarget.startScreencast(options); + } + + async ['Page.screencastFrameAck'](options) { + await this._pageTarget.screencastFrameAck(options); + } + + async ['Page.stopScreencast'](options) { + await this._pageTarget.stopScreencast(options); + } + async ['Page.sendMessageToWorker']({workerId, message}) { const worker = this._workers.get(workerId); if (!worker) throw new Error('ERROR: cannot find worker with id ' + workerId); return await worker.sendMessage(JSON.parse(message)); } - - async ['Page.stopVideoRecording']() { - await this._pageTarget.stopVideoRecording(); - } } var EXPORTED_SYMBOLS = ['PageHandler']; diff --git a/browser_patches/firefox-stable/juggler/protocol/Protocol.js b/browser_patches/firefox-stable/juggler/protocol/Protocol.js index c9693b162b..99b2ca5a34 100644 --- a/browser_patches/firefox-stable/juggler/protocol/Protocol.js +++ b/browser_patches/firefox-stable/juggler/protocol/Protocol.js @@ -226,7 +226,7 @@ const Browser = { canceled: t.Optional(t.Boolean), error: t.Optional(t.String), }, - 'screencastFinished': { + 'videoRecordingFinished': { screencastId: t.String, }, }, @@ -420,13 +420,12 @@ const Browser = { colorScheme: t.Nullable(t.Enum(['dark', 'light', 'no-preference'])), }, }, - 'setScreencastOptions': { + 'setVideoRecordingOptions': { params: { browserContextId: t.Optional(t.String), dir: t.String, width: t.Number, height: t.Number, - scale: t.Optional(t.Number), }, }, }, @@ -665,7 +664,7 @@ const Page = { workerId: t.String, message: t.String, }, - 'screencastStarted': { + 'videoRecordingStarted': { screencastId: t.String, file: t.String, }, @@ -697,6 +696,11 @@ const Page = { opcode: t.Number, data: t.String, }, + 'screencastFrame': { + data: t.String, + deviceWidth: t.Number, + deviceHeight: t.Number, + }, }, methods: { @@ -896,15 +900,22 @@ const Page = { message: t.String, }, }, - 'startVideoRecording': { + 'startScreencast': { params: { - file: t.String, width: t.Number, height: t.Number, - scale: t.Optional(t.Number), + quality: t.Number, + }, + returns: { + screencastId: t.String, }, }, - 'stopVideoRecording': { + 'screencastFrameAck': { + params: { + screencastId: t.String, + }, + }, + 'stopScreencast': { }, }, }; diff --git a/browser_patches/firefox-stable/juggler/screencast/HeadlessWindowCapturer.cpp b/browser_patches/firefox-stable/juggler/screencast/HeadlessWindowCapturer.cpp index 0caf74d5cb..66cb94b0c8 100644 --- a/browser_patches/firefox-stable/juggler/screencast/HeadlessWindowCapturer.cpp +++ b/browser_patches/firefox-stable/juggler/screencast/HeadlessWindowCapturer.cpp @@ -17,7 +17,7 @@ using namespace webrtc; namespace mozilla { -rtc::scoped_refptr HeadlessWindowCapturer::Create(HeadlessWidget* headlessWindow) { +rtc::scoped_refptr HeadlessWindowCapturer::Create(HeadlessWidget* headlessWindow) { return new rtc::RefCountedObject(headlessWindow); } @@ -41,6 +41,19 @@ void HeadlessWindowCapturer::DeRegisterCaptureDataCallback(rtc::VideoSinkInterfa } } +void HeadlessWindowCapturer::RegisterRawFrameCallback(webrtc::RawFrameCallback* rawFrameCallback) { + rtc::CritScope lock2(&_callBackCs); + _rawFrameCallbacks.insert(rawFrameCallback); +} + +void HeadlessWindowCapturer::DeRegisterRawFrameCallback(webrtc::RawFrameCallback* rawFrameCallback) { + rtc::CritScope lock2(&_callBackCs); + auto it = _rawFrameCallbacks.find(rawFrameCallback); + if (it != _rawFrameCallbacks.end()) { + _rawFrameCallbacks.erase(it); + } +} + void HeadlessWindowCapturer::NotifyFrameCaptured(const webrtc::VideoFrame& frame) { rtc::CritScope lock2(&_callBackCs); for (auto dataCallBack : _dataCallBacks) @@ -63,10 +76,28 @@ int32_t HeadlessWindowCapturer::StartCapture(const VideoCaptureCapability& capab } if (dataSurface->GetFormat() != gfx::SurfaceFormat::B8G8R8A8) { - fprintf(stderr, "Uexpected snapshot surface format: %hhd\n", dataSurface->GetFormat()); + fprintf(stderr, "Unexpected snapshot surface format: %hhd\n", dataSurface->GetFormat()); return; } + webrtc::VideoCaptureCapability frameInfo; + frameInfo.width = dataSurface->GetSize().width; + frameInfo.height = dataSurface->GetSize().height; +#if MOZ_LITTLE_ENDIAN() + frameInfo.videoType = VideoType::kARGB; +#else + frameInfo.videoType = VideoType::kBGRA; +#endif + + { + rtc::CritScope lock2(&_callBackCs); + for (auto rawFrameCallback : _rawFrameCallbacks) { + rawFrameCallback->OnRawFrame(dataSurface->GetData(), dataSurface->Stride(), frameInfo); + } + if (!_dataCallBacks.size()) + return; + } + int width = dataSurface->GetSize().width; int height = dataSurface->GetSize().height; rtc::scoped_refptr buffer = I420Buffer::Create(width, height); @@ -87,7 +118,6 @@ int32_t HeadlessWindowCapturer::StartCapture(const VideoCaptureCapability& capab buffer->MutableDataU(), buffer->StrideU(), buffer->MutableDataV(), buffer->StrideV(), width, height); - if (conversionResult != 0) { fprintf(stderr, "Failed to convert capture frame to I420: %d\n", conversionResult); return; diff --git a/browser_patches/firefox-stable/juggler/screencast/HeadlessWindowCapturer.h b/browser_patches/firefox-stable/juggler/screencast/HeadlessWindowCapturer.h index 86852b1e78..6e0700e288 100644 --- a/browser_patches/firefox-stable/juggler/screencast/HeadlessWindowCapturer.h +++ b/browser_patches/firefox-stable/juggler/screencast/HeadlessWindowCapturer.h @@ -10,6 +10,7 @@ #include "media/base/videosinkinterface.h" #include "modules/video_capture/video_capture.h" #include "rtc_base/criticalsection.h" +#include "video_engine/desktop_capture_impl.h" class nsIWidget; @@ -19,9 +20,9 @@ namespace widget { class HeadlessWidget; } -class HeadlessWindowCapturer : public webrtc::VideoCaptureModule { +class HeadlessWindowCapturer : public webrtc::VideoCaptureModuleEx { public: - static rtc::scoped_refptr Create(mozilla::widget::HeadlessWidget*); + static rtc::scoped_refptr Create(mozilla::widget::HeadlessWidget*); void RegisterCaptureDataCallback( rtc::VideoSinkInterface* dataCallback) override; @@ -29,6 +30,9 @@ class HeadlessWindowCapturer : public webrtc::VideoCaptureModule { rtc::VideoSinkInterface* dataCallback) override; int32_t StopCaptureIfAllClientsClose() override; + void RegisterRawFrameCallback(webrtc::RawFrameCallback* rawFrameCallback) override; + void DeRegisterRawFrameCallback(webrtc::RawFrameCallback* rawFrameCallback) override; + int32_t SetCaptureRotation(webrtc::VideoRotation) override { return -1; } bool SetApplyRotation(bool) override { return false; } bool GetApplyRotation() override { return true; } @@ -54,6 +58,7 @@ class HeadlessWindowCapturer : public webrtc::VideoCaptureModule { mozilla::widget::HeadlessWidget* mWindow = nullptr; rtc::CriticalSection _callBackCs; std::set*> _dataCallBacks; + std::set _rawFrameCallbacks; }; } // namespace mozilla diff --git a/browser_patches/firefox-stable/juggler/screencast/ScreencastEncoder.cpp b/browser_patches/firefox-stable/juggler/screencast/ScreencastEncoder.cpp index d0352f1cfa..981bbc9915 100644 --- a/browser_patches/firefox-stable/juggler/screencast/ScreencastEncoder.cpp +++ b/browser_patches/firefox-stable/juggler/screencast/ScreencastEncoder.cpp @@ -110,9 +110,8 @@ void createImage(unsigned int width, unsigned int height, class ScreencastEncoder::VPXFrame { public: - VPXFrame(rtc::scoped_refptr&& buffer, Maybe scale, const gfx::IntMargin& margin) + VPXFrame(rtc::scoped_refptr&& buffer, const gfx::IntMargin& margin) : m_frameBuffer(std::move(buffer)) - , m_scale(scale) , m_margin(margin) { } @@ -137,8 +136,8 @@ public: double src_width = src->width() - m_margin.LeftRight(); double src_height = src->height() - m_margin.top; - if (m_scale || (src_width > image->w || src_height > image->h)) { - double scale = m_scale ? m_scale.value() : std::min(image->w / src_width, image->h / src_height); + if (src_width > image->w || src_height > image->h) { + double scale = std::min(image->w / src_width, image->h / src_height); double dst_width = src_width * scale; if (dst_width > image->w) { src_width *= image->w / dst_width; @@ -174,7 +173,6 @@ public: private: rtc::scoped_refptr m_frameBuffer; - Maybe m_scale; gfx::IntMargin m_margin; TimeDuration m_duration; }; @@ -276,9 +274,8 @@ private: std::unique_ptr m_image; }; -ScreencastEncoder::ScreencastEncoder(std::unique_ptr&& vpxCodec, Maybe scale, const gfx::IntMargin& margin) +ScreencastEncoder::ScreencastEncoder(std::unique_ptr&& vpxCodec, const gfx::IntMargin& margin) : m_vpxCodec(std::move(vpxCodec)) - , m_scale(scale) , m_margin(margin) { } @@ -287,7 +284,7 @@ ScreencastEncoder::~ScreencastEncoder() { } -RefPtr ScreencastEncoder::create(nsCString& errorString, const nsCString& filePath, int width, int height, Maybe scale, const gfx::IntMargin& margin) +RefPtr ScreencastEncoder::create(nsCString& errorString, const nsCString& filePath, int width, int height, const gfx::IntMargin& margin) { vpx_codec_iface_t* codec_interface = vpx_codec_vp8_cx(); if (!codec_interface) { @@ -328,7 +325,7 @@ RefPtr ScreencastEncoder::create(nsCString& errorString, cons std::unique_ptr vpxCodec(new VPXCodec(codec, cfg, file)); // fprintf(stderr, "ScreencastEncoder initialized with: %s\n", vpx_codec_iface_name(codec_interface)); - return new ScreencastEncoder(std::move(vpxCodec), scale, margin); + return new ScreencastEncoder(std::move(vpxCodec), margin); } void ScreencastEncoder::flushLastFrame() @@ -350,7 +347,7 @@ void ScreencastEncoder::encodeFrame(const webrtc::VideoFrame& videoFrame) // fprintf(stderr, "ScreencastEncoder::encodeFrame\n"); flushLastFrame(); - m_lastFrame = std::make_unique(videoFrame.video_frame_buffer(), m_scale, m_margin); + m_lastFrame = std::make_unique(videoFrame.video_frame_buffer(), m_margin); } void ScreencastEncoder::finish(std::function&& callback) diff --git a/browser_patches/firefox-stable/juggler/screencast/ScreencastEncoder.h b/browser_patches/firefox-stable/juggler/screencast/ScreencastEncoder.h index b6827800bc..bfc9b69e76 100644 --- a/browser_patches/firefox-stable/juggler/screencast/ScreencastEncoder.h +++ b/browser_patches/firefox-stable/juggler/screencast/ScreencastEncoder.h @@ -23,10 +23,10 @@ class ScreencastEncoder { public: static constexpr int fps = 25; - static RefPtr create(nsCString& errorString, const nsCString& filePath, int width, int height, Maybe scale, const gfx::IntMargin& margin); + static RefPtr create(nsCString& errorString, const nsCString& filePath, int width, int height, const gfx::IntMargin& margin); class VPXCodec; - ScreencastEncoder(std::unique_ptr&&, Maybe scale, const gfx::IntMargin& margin); + ScreencastEncoder(std::unique_ptr&&, const gfx::IntMargin& margin); void encodeFrame(const webrtc::VideoFrame& videoFrame); @@ -38,7 +38,6 @@ private: void flushLastFrame(); std::unique_ptr m_vpxCodec; - Maybe m_scale; gfx::IntMargin m_margin; TimeStamp m_lastFrameTimestamp; class VPXFrame; diff --git a/browser_patches/firefox-stable/juggler/screencast/nsIScreencastService.idl b/browser_patches/firefox-stable/juggler/screencast/nsIScreencastService.idl index b7edfe3c1a..302a47ce7c 100644 --- a/browser_patches/firefox-stable/juggler/screencast/nsIScreencastService.idl +++ b/browser_patches/firefox-stable/juggler/screencast/nsIScreencastService.idl @@ -6,16 +6,26 @@ interface nsIDocShell; +[scriptable, uuid(0b5d32c4-aeeb-11eb-8529-0242ac130003)] +interface nsIScreencastServiceClient : nsISupports +{ + void screencastFrame(in AString frame, in uint32_t deviceWidth, in uint32_t deviceHeight); + + void screencastStopped(); +}; + /** * Service for recording window video. */ [scriptable, uuid(d8c4d9e0-9462-445e-9e43-68d3872ad1de)] interface nsIScreencastService : nsISupports { - AString startVideoRecording(in nsIDocShell docShell, in ACString fileName, in uint32_t width, in uint32_t height, in uint32_t viewportWidth, in uint32_t viewportHeight, in double scale, in int32_t offset_top); + AString startVideoRecording(in nsIScreencastServiceClient client, in nsIDocShell docShell, in boolean isVideo, in ACString fileName, in uint32_t width, in uint32_t height, in uint32_t quality, in int32_t offset_top); /** * Will emit 'juggler-screencast-stopped' when the video file is saved. */ void stopVideoRecording(in AString sessionId); + + void screencastFrameAck(in AString sessionId); }; diff --git a/browser_patches/firefox-stable/juggler/screencast/nsScreencastService.cpp b/browser_patches/firefox-stable/juggler/screencast/nsScreencastService.cpp index cbebd9fbc1..90699235e9 100644 --- a/browser_patches/firefox-stable/juggler/screencast/nsScreencastService.cpp +++ b/browser_patches/firefox-stable/juggler/screencast/nsScreencastService.cpp @@ -7,6 +7,7 @@ #include "ScreencastEncoder.h" #include "HeadlessWidget.h" #include "HeadlessWindowCapturer.h" +#include "mozilla/Base64.h" #include "mozilla/ClearOnShutdown.h" #include "mozilla/PresShell.h" #include "mozilla/StaticPtr.h" @@ -24,6 +25,9 @@ #include "webrtc/modules/video_capture/video_capture.h" #include "mozilla/widget/PlatformWidgetTypes.h" #include "video_engine/desktop_capture_impl.h" +extern "C" { +#include "jpeglib.h" +} using namespace mozilla::widget; @@ -33,9 +37,11 @@ NS_IMPL_ISUPPORTS(nsScreencastService, nsIScreencastService) namespace { +const int kMaxFramesInFlight = 1; + StaticRefPtr gScreencastService; -rtc::scoped_refptr CreateWindowCapturer(nsIWidget* widget) { +rtc::scoped_refptr CreateWindowCapturer(nsIWidget* widget) { if (gfxPlatform::IsHeadless()) { HeadlessWidget* headlessWidget = static_cast(widget); return HeadlessWindowCapturer::Create(headlessWidget); @@ -52,16 +58,6 @@ rtc::scoped_refptr CreateWindowCapturer(nsIWidget* w return webrtc::DesktopCaptureImpl::Create(++moduleId, windowId.get(), webrtc::CaptureDeviceType::Window, captureCursor); } -void NotifyScreencastStopped(const nsString& sessionId) { - nsCOMPtr observerService = mozilla::services::GetObserverService(); - if (!observerService) { - fprintf(stderr, "NotifyScreencastStopped error: no observer service\n"); - return; - } - - observerService->NotifyObservers(nullptr, "juggler-screencast-stopped", sessionId.get()); -} - nsresult generateUid(nsString& uid) { nsresult rv = NS_OK; nsCOMPtr rg = do_GetService("@mozilla.org/security/random-generator;1", &rv); @@ -80,11 +76,15 @@ nsresult generateUid(nsString& uid) { } } -class nsScreencastService::Session : public rtc::VideoSinkInterface { +class nsScreencastService::Session : public rtc::VideoSinkInterface, + public webrtc::RawFrameCallback { public: - Session(rtc::scoped_refptr&& capturer, RefPtr&& encoder) - : mCaptureModule(std::move(capturer)) - , mEncoder(std::move(encoder)) { + Session(nsIScreencastServiceClient* client, rtc::scoped_refptr&& capturer, RefPtr&& encoder, gfx::IntMargin margin, uint32_t jpegQuality) + : mClient(client) + , mCaptureModule(std::move(capturer)) + , mEncoder(std::move(encoder)) + , mJpegQuality(jpegQuality) + , mMargin(margin) { } bool Start() { @@ -100,27 +100,128 @@ class nsScreencastService::Session : public rtc::VideoSinkInterfaceRegisterCaptureDataCallback(this); + if (mEncoder) + mCaptureModule->RegisterCaptureDataCallback(this); + else + mCaptureModule->RegisterRawFrameCallback(this); return true; } - void Stop(std::function&& callback) { - mCaptureModule->DeRegisterCaptureDataCallback(this); + void Stop() { + if (mEncoder) + mCaptureModule->DeRegisterCaptureDataCallback(this); + else + mCaptureModule->DeRegisterRawFrameCallback(this); int error = mCaptureModule->StopCapture(); if (error) { fprintf(stderr, "StopCapture error %d\n", error); } - mEncoder->finish(std::move(callback)); + if (mEncoder) { + rtc::CritScope lock(&mCaptureCallbackCs); + mEncoder->finish([client = std::move(mClient)] { + NS_DispatchToMainThread(NS_NewRunnableFunction( + "NotifyScreencastStopped", [client = std::move(client)]() -> void { + client->ScreencastStopped(); + })); + }); + } else { + rtc::CritScope lock(&mCaptureCallbackCs); + mClient->ScreencastStopped(); + mClient = nullptr; + } + } + + void ScreencastFrameAck() { + rtc::CritScope lock(&mCaptureCallbackCs); + --mFramesInFlight; } // These callbacks end up running on the VideoCapture thread. void OnFrame(const webrtc::VideoFrame& videoFrame) override { + if (!mEncoder) + return; mEncoder->encodeFrame(videoFrame); } + // These callbacks end up running on the VideoCapture thread. + void OnRawFrame(uint8_t* videoFrame, size_t videoFrameStride, const webrtc::VideoCaptureCapability& frameInfo) override { + { + rtc::CritScope lock(&mCaptureCallbackCs); + if (mFramesInFlight >= kMaxFramesInFlight) { + return; + } + ++mFramesInFlight; + if (!mClient) + return; + } + + jpeg_compress_struct info; + jpeg_error_mgr error; + info.err = jpeg_std_error(&error); + jpeg_create_compress(&info); + + unsigned char* bufferPtr = nullptr; + unsigned long bufferSize; + jpeg_mem_dest(&info, &bufferPtr, &bufferSize); + + info.image_width = frameInfo.width - mMargin.LeftRight(); + info.image_height = frameInfo.height - mMargin.TopBottom(); + +#if MOZ_LITTLE_ENDIAN() + if (frameInfo.videoType == webrtc::VideoType::kARGB) + info.in_color_space = JCS_EXT_BGRA; + if (frameInfo.videoType == webrtc::VideoType::kBGRA) + info.in_color_space = JCS_EXT_ARGB; +#else + if (frameInfo.videoType == webrtc::VideoType::kARGB) + info.in_color_space = JCS_EXT_ARGB; + if (frameInfo.videoType == webrtc::VideoType::kBGRA) + info.in_color_space = JCS_EXT_BGRA; +#endif + + // # of color components in input image + info.input_components = 4; + + jpeg_set_defaults(&info); + jpeg_set_quality(&info, mJpegQuality, true); + + jpeg_start_compress(&info, true); + while (info.next_scanline < info.image_height) { + JSAMPROW row = videoFrame + (mMargin.top + info.next_scanline) * videoFrameStride + 4 * mMargin.left; + if (jpeg_write_scanlines(&info, &row, 1) != 1) { + fprintf(stderr, "JPEG library failed to encode line\n"); + break; + } + } + + jpeg_finish_compress(&info); + jpeg_destroy_compress(&info); + + nsCString base64; + nsresult rv = mozilla::Base64Encode(reinterpret_cast(bufferPtr), bufferSize, base64); + free(bufferPtr); + if (NS_WARN_IF(NS_FAILED(rv))) { + return; + } + + uint32_t deviceWidth = info.image_width; + uint32_t deviceHeight = info.image_height; + nsIScreencastServiceClient* client = mClient.get(); + NS_DispatchToMainThread(NS_NewRunnableFunction( + "NotifyScreencastFrame", [client, base64, deviceWidth, deviceHeight]() -> void { + NS_ConvertUTF8toUTF16 utf16(base64); + client->ScreencastFrame(utf16, deviceWidth, deviceHeight); + })); + } + private: - rtc::scoped_refptr mCaptureModule; + RefPtr mClient; + rtc::scoped_refptr mCaptureModule; RefPtr mEncoder; + uint32_t mJpegQuality; + rtc::CriticalSection mCaptureCallbackCs; + uint32_t mFramesInFlight = 0; + gfx::IntMargin mMargin; }; @@ -140,7 +241,7 @@ nsScreencastService::nsScreencastService() = default; nsScreencastService::~nsScreencastService() { } -nsresult nsScreencastService::StartVideoRecording(nsIDocShell* aDocShell, const nsACString& aFileName, uint32_t width, uint32_t height, uint32_t viewportWidth, uint32_t viewportHeight, double scale, int32_t offsetTop, nsAString& sessionId) { +nsresult nsScreencastService::StartVideoRecording(nsIScreencastServiceClient* aClient, nsIDocShell* aDocShell, bool isVideo, const nsACString& aVideoFileName, uint32_t width, uint32_t height, uint32_t quality, int32_t offsetTop, nsAString& sessionId) { MOZ_RELEASE_ASSERT(NS_IsMainThread(), "Screencast service must be started on the Main thread."); PresShell* presShell = aDocShell->GetPresShell(); @@ -154,42 +255,37 @@ nsresult nsScreencastService::StartVideoRecording(nsIDocShell* aDocShell, const return NS_ERROR_UNEXPECTED; nsIWidget* widget = view->GetWidget(); - rtc::scoped_refptr capturer = CreateWindowCapturer(widget); + rtc::scoped_refptr capturer = CreateWindowCapturer(widget); if (!capturer) return NS_ERROR_FAILURE; - nsCString error; - Maybe maybeScale; - if (scale) - maybeScale = Some(scale); - gfx::IntMargin margin; auto bounds = widget->GetScreenBounds().ToUnknownRect(); auto clientBounds = widget->GetClientBounds().ToUnknownRect(); - // The browser window has a minimum size, so it might be larger than the viewport size. - clientBounds.width = std::min((int)viewportWidth, clientBounds.width); - clientBounds.height = std::min((int)viewportHeight, clientBounds.height); // Crop the image to exclude frame (if any). margin = bounds - clientBounds; // Crop the image to exclude controls. margin.top += offsetTop; - RefPtr encoder = ScreencastEncoder::create(error, PromiseFlatCString(aFileName), width, height, maybeScale, margin); - if (!encoder) { - fprintf(stderr, "Failed to create ScreencastEncoder: %s\n", error.get()); - return NS_ERROR_FAILURE; + nsCString error; + RefPtr encoder; + if (isVideo) { + encoder = ScreencastEncoder::create(error, PromiseFlatCString(aVideoFileName), width, height, margin); + if (!encoder) { + fprintf(stderr, "Failed to create ScreencastEncoder: %s\n", error.get()); + return NS_ERROR_FAILURE; + } } - auto session = std::make_unique(std::move(capturer), std::move(encoder)); - if (!session->Start()) - return NS_ERROR_FAILURE; - nsString uid; nsresult rv = generateUid(uid); NS_ENSURE_SUCCESS(rv, rv); - sessionId = uid; - mIdToSession.emplace(uid, std::move(session)); + + auto session = std::make_unique(aClient, std::move(capturer), std::move(encoder), margin, isVideo ? 0 : quality); + if (!session->Start()) + return NS_ERROR_FAILURE; + mIdToSession.emplace(sessionId, std::move(session)); return NS_OK; } @@ -198,14 +294,18 @@ nsresult nsScreencastService::StopVideoRecording(const nsAString& aSessionId) { auto it = mIdToSession.find(sessionId); if (it == mIdToSession.end()) return NS_ERROR_INVALID_ARG; - it->second->Stop([sessionId] { - NS_DispatchToMainThread(NS_NewRunnableFunction( - "NotifyScreencastStopped", [sessionId]() -> void { - NotifyScreencastStopped(sessionId); - })); - }); + it->second->Stop(); mIdToSession.erase(it); return NS_OK; } +nsresult nsScreencastService::ScreencastFrameAck(const nsAString& aSessionId) { + nsString sessionId(aSessionId); + auto it = mIdToSession.find(sessionId); + if (it == mIdToSession.end()) + return NS_ERROR_INVALID_ARG; + it->second->ScreencastFrameAck(); + return NS_OK; +} + } // namespace mozilla diff --git a/browser_patches/firefox-stable/patches/bootstrap.diff b/browser_patches/firefox-stable/patches/bootstrap.diff index 5577c43efc..5cd70b1c71 100644 --- a/browser_patches/firefox-stable/patches/bootstrap.diff +++ b/browser_patches/firefox-stable/patches/bootstrap.diff @@ -1143,12 +1143,15 @@ index b3f6b63d2e52b517ca56fc88afb2bd5785717bd3..0b3c3d78468e1ebf7e1df889cbfc5273 /** Synthesize a touch event. The event types supported are: * touchstart, touchend, touchmove, and touchcancel diff --git a/dom/media/systemservices/video_engine/desktop_capture_impl.cc b/dom/media/systemservices/video_engine/desktop_capture_impl.cc -index 7bc92fe4408c2878c9d7c8bdb97a7c257258ee31..8885feebedf53c0748cef19d80ce5aa23adc900b 100644 +index 7bc92fe4408c2878c9d7c8bdb97a7c257258ee31..b20480c3c0ca96097e61d37f44e127d45ab4648a 100644 --- a/dom/media/systemservices/video_engine/desktop_capture_impl.cc +++ b/dom/media/systemservices/video_engine/desktop_capture_impl.cc -@@ -125,8 +125,9 @@ int32_t ScreenDeviceInfoImpl::GetOrientation(const char* deviceUniqueIdUTF8, +@@ -123,10 +123,11 @@ int32_t ScreenDeviceInfoImpl::GetOrientation(const char* deviceUniqueIdUTF8, + return 0; + } - VideoCaptureModule* DesktopCaptureImpl::Create(const int32_t id, +-VideoCaptureModule* DesktopCaptureImpl::Create(const int32_t id, ++VideoCaptureModuleEx* DesktopCaptureImpl::Create(const int32_t id, const char* uniqueId, - const CaptureDeviceType type) { - return new rtc::RefCountedObject(id, uniqueId, type); @@ -1199,21 +1202,93 @@ index 7bc92fe4408c2878c9d7c8bdb97a7c257258ee31..8885feebedf53c0748cef19d80ce5aa2 time_event_(EventWrapper::Create()), #if defined(_WIN32) capturer_thread_( +@@ -437,6 +444,19 @@ void DesktopCaptureImpl::DeRegisterCaptureDataCallback( + } + } + ++void DesktopCaptureImpl::RegisterRawFrameCallback(RawFrameCallback* rawFrameCallback) { ++ rtc::CritScope lock(&_apiCs); ++ _rawFrameCallbacks.insert(rawFrameCallback); ++} ++ ++void DesktopCaptureImpl::DeRegisterRawFrameCallback(RawFrameCallback* rawFrameCallback) { ++ rtc::CritScope lock(&_apiCs); ++ auto it = _rawFrameCallbacks.find(rawFrameCallback); ++ if (it != _rawFrameCallbacks.end()) { ++ _rawFrameCallbacks.erase(it); ++ } ++} ++ + int32_t DesktopCaptureImpl::StopCaptureIfAllClientsClose() { + if (_dataCallBacks.empty()) { + return StopCapture(); +@@ -644,6 +664,12 @@ void DesktopCaptureImpl::OnCaptureResult(DesktopCapturer::Result result, + frameInfo.height = frame->size().height(); + frameInfo.videoType = VideoType::kARGB; + ++ size_t videoFrameStride = ++ frameInfo.width * DesktopFrame::kBytesPerPixel; ++ for (auto rawFrameCallback : _rawFrameCallbacks) { ++ rawFrameCallback->OnRawFrame(videoFrame, videoFrameStride, frameInfo); ++ } ++ + size_t videoFrameLength = + frameInfo.width * frameInfo.height * DesktopFrame::kBytesPerPixel; + IncomingFrame(videoFrame, videoFrameLength, frameInfo); diff --git a/dom/media/systemservices/video_engine/desktop_capture_impl.h b/dom/media/systemservices/video_engine/desktop_capture_impl.h -index 75995564e5438261a2886840ecad32d2f1d7663f..dfdabcedcda4e212ed0ffd7bc4def57079218413 100644 +index 75995564e5438261a2886840ecad32d2f1d7663f..2b6564fba840c5c4cc91ca7828e72e12e0dd2741 100644 --- a/dom/media/systemservices/video_engine/desktop_capture_impl.h +++ b/dom/media/systemservices/video_engine/desktop_capture_impl.h -@@ -159,7 +159,8 @@ class DesktopCaptureImpl : public DesktopCapturer::Callback, +@@ -41,6 +41,21 @@ namespace webrtc { + + class VideoCaptureEncodeInterface; + ++class RawFrameCallback { ++ public: ++ virtual ~RawFrameCallback() {} ++ ++ virtual void OnRawFrame(uint8_t* videoFrame, size_t videoFrameLength, const VideoCaptureCapability& frameInfo) = 0; ++}; ++ ++class VideoCaptureModuleEx : public VideoCaptureModule { ++ public: ++ virtual ~VideoCaptureModuleEx() {} ++ ++ virtual void RegisterRawFrameCallback(RawFrameCallback* rawFrameCallback) = 0; ++ virtual void DeRegisterRawFrameCallback(RawFrameCallback* rawFrameCallback) = 0; ++}; ++ + // simulate deviceInfo interface for video engine, bridge screen/application and + // real screen/application device info + +@@ -153,13 +168,14 @@ class BrowserDeviceInfoImpl : public VideoCaptureModule::DeviceInfo { + // As with video, DesktopCaptureImpl is a proxy for screen sharing + // and follows the video pipeline design + class DesktopCaptureImpl : public DesktopCapturer::Callback, +- public VideoCaptureModule, ++ public VideoCaptureModuleEx, + public VideoCaptureExternal { + public: /* Create a screen capture modules object */ - static VideoCaptureModule* Create(const int32_t id, const char* uniqueId, +- static VideoCaptureModule* Create(const int32_t id, const char* uniqueId, - const CaptureDeviceType type); ++ static VideoCaptureModuleEx* Create(const int32_t id, const char* uniqueId, + const CaptureDeviceType type, + bool captureCursor = true); static VideoCaptureModule::DeviceInfo* CreateDeviceInfo( const int32_t id, const CaptureDeviceType type); -@@ -191,7 +192,7 @@ class DesktopCaptureImpl : public DesktopCapturer::Callback, +@@ -169,6 +185,8 @@ class DesktopCaptureImpl : public DesktopCapturer::Callback, + void DeRegisterCaptureDataCallback( + rtc::VideoSinkInterface* dataCallback) override; + int32_t StopCaptureIfAllClientsClose() override; ++ void RegisterRawFrameCallback(RawFrameCallback* rawFrameCallback) override; ++ void DeRegisterRawFrameCallback(RawFrameCallback* rawFrameCallback) override; + + int32_t SetCaptureRotation(VideoRotation rotation) override; + bool SetApplyRotation(bool enable) override; +@@ -191,7 +209,7 @@ class DesktopCaptureImpl : public DesktopCapturer::Callback, protected: DesktopCaptureImpl(const int32_t id, const char* uniqueId, @@ -1222,7 +1297,15 @@ index 75995564e5438261a2886840ecad32d2f1d7663f..dfdabcedcda4e212ed0ffd7bc4def570 virtual ~DesktopCaptureImpl(); int32_t DeliverCapturedFrame(webrtc::VideoFrame& captureFrame, int64_t capture_time); -@@ -239,6 +240,7 @@ class DesktopCaptureImpl : public DesktopCapturer::Callback, +@@ -214,6 +232,7 @@ class DesktopCaptureImpl : public DesktopCapturer::Callback, + rtc::CriticalSection _apiCs; + + std::set*> _dataCallBacks; ++ std::set _rawFrameCallbacks; + + int64_t _incomingFrameTimesNanos + [kFrameRateCountHistorySize]; // timestamp for local captured frames +@@ -239,6 +258,7 @@ class DesktopCaptureImpl : public DesktopCapturer::Callback, void process(); private: @@ -1584,6 +1667,18 @@ index 77b4c4ea3581e3b66b0b40dae33c807b2d5aefd8..84af4461b9e946122527ac974dc30da5 void updateTimeZone(); void internalResyncICUDefaultTimeZone(); +diff --git a/media/libjpeg/jconfig.h b/media/libjpeg/jconfig.h +index f2723e654098ff27542e1eb16a536c11ad0af617..b0b480551ff7d895dfdeb5a9800874858929c8ba 100644 +--- a/media/libjpeg/jconfig.h ++++ b/media/libjpeg/jconfig.h +@@ -17,6 +17,7 @@ + /* #undef D_ARITH_CODING_SUPPORTED */ + + /* Support in-memory source/destination managers */ ++#define MEM_SRCDST_SUPPORTED 1 + /* #undef MEM_SRCDST_SUPPORTED */ + + /* Use accelerated SIMD routines. */ diff --git a/netwerk/base/nsINetworkInterceptController.idl b/netwerk/base/nsINetworkInterceptController.idl index 64a4a71b03b28872f376aac8eee12805bebd1bd8..f6fa7d731f3b0c7c4fcb26babad3fc2cdb29aec1 100644 --- a/netwerk/base/nsINetworkInterceptController.idl