remoting: Support callback based frame capturing

Rather than doing a timer based frame capture, this change
modifies the implementation to rely on callbacks from webrtc
capturer whenever a new frame is received via pipewire.

Corresponding WebRTC change is here:
https://webrtc-review.googlesource.com/c/src/+/291080

Bug: chromium:1291247
Change-Id: Ia00edc110dfe40918e1ad0862472f70d7c849c9f
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4188923
Reviewed-by: Lambros Lambrou <lambroslambrou@chromium.org>
Commit-Queue: Salman Malik <salmanmalik@chromium.org>
Cr-Commit-Position: refs/heads/main@{#1099800}

remoting/host/desktop_and_cursor_conditional_composer.cc

@@ -5,6 +5,11 @@
 #include "remoting/host/desktop_and_cursor_conditional_composer.h"
 
 #include "base/functional/bind.h"
+#include "build/build_config.h"
+
+#if BUILDFLAG(IS_LINUX)
+#include "remoting/host/linux/wayland_utils.h"
+#endif
 
 namespace remoting {
 
@@ -90,6 +95,19 @@ bool DesktopAndCursorConditionalComposer::IsOccluded(
   return capturer_->IsOccluded(pos);
 }
 
+bool DesktopAndCursorConditionalComposer::SupportsFrameCallbacks() {
+#if BUILDFLAG(IS_LINUX)
+  return IsRunningWayland();
+#else
+  return false;
+#endif
+}
+
+void DesktopAndCursorConditionalComposer::SetMaxFrameRate(
+    uint32_t max_frame_rate) {
+  capturer_->SetMaxFrameRate(max_frame_rate);
+}
+
 #if defined(WEBRTC_USE_GIO)
 void DesktopAndCursorConditionalComposer::GetMetadataAsync(
     base::OnceCallback<void(webrtc::DesktopCaptureMetadata)> callback) {

remoting/host/desktop_and_cursor_conditional_composer.h

@@ -39,6 +39,8 @@ class DesktopAndCursorConditionalComposer : public DesktopCapturer {
   void SetMouseCursor(
       std::unique_ptr<webrtc::MouseCursor> mouse_cursor) override;
   void SetMouseCursorPosition(const webrtc::DesktopVector& position) override;
+  bool SupportsFrameCallbacks() override;
+  void SetMaxFrameRate(uint32_t max_frame_rate) override;
 #if defined(WEBRTC_USE_GIO)
   void GetMetadataAsync(base::OnceCallback<void(webrtc::DesktopCaptureMetadata)>
                             callback) override;

remoting/host/desktop_capturer_proxy.cc

@@ -34,6 +34,7 @@
 #include "base/environment.h"
 #include "base/nix/xdg_util.h"
 #include "remoting/host/linux/wayland_desktop_capturer.h"
+#include "remoting/host/linux/wayland_utils.h"
 #endif
 
 namespace remoting {
@@ -58,12 +59,14 @@ class DesktopCapturerProxy::Core : public webrtc::DesktopCapturer::Callback {
       std::unique_ptr<webrtc::SharedMemoryFactory> shared_memory_factory);
   void SelectSource(SourceId id);
   void CaptureFrame();
+  void SetMaxFrameRate(uint32_t max_frame_rate);
 #if defined(WEBRTC_USE_GIO)
   void GetAndSetMetadata();
 #endif
 
  private:
   // webrtc::DesktopCapturer::Callback implementation.
+  void OnFrameCaptureStart() override;
   void OnCaptureResult(webrtc::DesktopCapturer::Result result,
                        std::unique_ptr<webrtc::DesktopFrame> frame) override;
 
@@ -155,6 +158,14 @@ void DesktopCapturerProxy::Core::CaptureFrame() {
   }
 }
 
+void DesktopCapturerProxy::Core::SetMaxFrameRate(uint32_t max_frame_rate) {
+  DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
+
+  if (capturer_) {
+    capturer_->SetMaxFrameRate(max_frame_rate);
+  }
+}
+
 #if defined(WEBRTC_USE_GIO)
 void DesktopCapturerProxy::Core::GetAndSetMetadata() {
   DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
@@ -168,11 +179,15 @@ void DesktopCapturerProxy::Core::GetAndSetMetadata() {
 }
 #endif
 
+void DesktopCapturerProxy::Core::OnFrameCaptureStart() {
+  caller_task_runner_->PostTask(
+      FROM_HERE,
+      base::BindOnce(&DesktopCapturerProxy::OnFrameCaptureStarting, proxy_));
+}
+
 void DesktopCapturerProxy::Core::OnCaptureResult(
     webrtc::DesktopCapturer::Result result,
     std::unique_ptr<webrtc::DesktopFrame> frame) {
-  DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
-
   caller_task_runner_->PostTask(
       FROM_HERE, base::BindOnce(&DesktopCapturerProxy::OnFrameCaptured, proxy_,
                                 result, std::move(frame)));
@@ -253,6 +268,12 @@ bool DesktopCapturerProxy::SelectSource(SourceId id) {
   return false;
 }
 
+void DesktopCapturerProxy::OnFrameCaptureStarting() {
+  DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
+
+  callback_->OnFrameCaptureStart();
+}
+
 void DesktopCapturerProxy::OnFrameCaptured(
     webrtc::DesktopCapturer::Result result,
     std::unique_ptr<webrtc::DesktopFrame> frame) {
@@ -277,6 +298,24 @@ void DesktopCapturerProxy::OnMetadata(webrtc::DesktopCaptureMetadata metadata) {
 
   std::move(metadata_callback_).Run(std::move(metadata));
 }
+#endif
 
+bool DesktopCapturerProxy::SupportsFrameCallbacks() {
+  DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
+
+#if BUILDFLAG(IS_LINUX)
+  return IsRunningWayland();
+#else
+  return false;
 #endif
+}
+
+void DesktopCapturerProxy::SetMaxFrameRate(uint32_t max_frame_rate) {
+  DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
+
+  capture_task_runner_->PostTask(
+      FROM_HERE, base::BindOnce(&Core::SetMaxFrameRate,
+                                base::Unretained(core_.get()), max_frame_rate));
+}
+
 }  // namespace remoting

remoting/host/desktop_capturer_proxy.h

@@ -52,6 +52,8 @@ class DesktopCapturerProxy : public DesktopCapturer {
   void CaptureFrame() override;
   bool GetSourceList(SourceList* sources) override;
   bool SelectSource(SourceId id) override;
+  bool SupportsFrameCallbacks() override;
+  void SetMaxFrameRate(uint32_t max_frame_rate) override;
 #if defined(WEBRTC_USE_GIO)
   void GetMetadataAsync(base::OnceCallback<void(webrtc::DesktopCaptureMetadata)>
                             callback) override;
@@ -60,6 +62,7 @@ class DesktopCapturerProxy : public DesktopCapturer {
  private:
   class Core;
 
+  void OnFrameCaptureStarting();
   void OnFrameCaptured(webrtc::DesktopCapturer::Result result,
                        std::unique_ptr<webrtc::DesktopFrame> frame);
 

remoting/host/desktop_capturer_wrapper.cc

@@ -8,10 +8,15 @@
 #include <utility>
 
 #include "base/logging.h"
+#include "build/build_config.h"
 #include "third_party/webrtc/modules/desktop_capture/desktop_capture_options.h"
 #include "third_party/webrtc/modules/desktop_capture/desktop_capturer.h"
 #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
 
+#if BUILDFLAG(IS_LINUX)
+#include "remoting/host/linux/wayland_utils.h"
+#endif
+
 namespace remoting {
 
 DesktopCapturerWrapper::DesktopCapturerWrapper() {
@@ -78,6 +83,12 @@ bool DesktopCapturerWrapper::SelectSource(SourceId id) {
   return false;
 }
 
+void DesktopCapturerWrapper::OnFrameCaptureStart() {
+  DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
+
+  callback_->OnFrameCaptureStart();
+}
+
 void DesktopCapturerWrapper::OnCaptureResult(
     webrtc::DesktopCapturer::Result result,
     std::unique_ptr<webrtc::DesktopFrame> frame) {
@@ -86,6 +97,24 @@ void DesktopCapturerWrapper::OnCaptureResult(
   callback_->OnCaptureResult(result, std::move(frame));
 }
 
+bool DesktopCapturerWrapper::SupportsFrameCallbacks() {
+  DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
+
+#if BUILDFLAG(IS_LINUX)
+  return capturer_ && IsRunningWayland();
+#else
+  return false;
+#endif
+}
+
+void DesktopCapturerWrapper::SetMaxFrameRate(uint32_t max_frame_rate) {
+  DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
+
+  if (capturer_) {
+    capturer_->SetMaxFrameRate(max_frame_rate);
+  }
+}
+
 #if defined(WEBRTC_USE_GIO)
 void DesktopCapturerWrapper::GetMetadataAsync(
     base::OnceCallback<void(webrtc::DesktopCaptureMetadata)> callback) {

remoting/host/desktop_capturer_wrapper.h

@@ -37,13 +37,16 @@ class DesktopCapturerWrapper : public DesktopCapturer,
   void CaptureFrame() override;
   bool GetSourceList(SourceList* sources) override;
   bool SelectSource(SourceId id) override;
+  bool SupportsFrameCallbacks() override;
+  void SetMaxFrameRate(uint32_t max_frame_rate) override;
 #if defined(WEBRTC_USE_GIO)
   void GetMetadataAsync(base::OnceCallback<void(webrtc::DesktopCaptureMetadata)>
                             callback) override;
 #endif
 
  private:
   // webrtc::DesktopCapturer::Callback implementation.
+  void OnFrameCaptureStart() override;
   void OnCaptureResult(webrtc::DesktopCapturer::Result result,
                        std::unique_ptr<webrtc::DesktopFrame> frame) override;
 

remoting/host/linux/BUILD.gn

@@ -79,14 +79,17 @@ if (enable_me2me_host) {
   }
 }
 
-if (remoting_use_x11) {
+if (is_linux) {
   source_set("wayland") {
     sources = [
       "wayland_utils.cc",
       "wayland_utils.h",
     ]
     deps = [ "//base" ]
   }
+}
+
+if (remoting_use_x11) {
   source_set("x11") {
     sources = [
       "unicode_to_keysym.cc",

remoting/host/linux/wayland_desktop_capturer.cc

@@ -28,7 +28,9 @@ WaylandDesktopCapturer::WaylandDesktopCapturer(
                               // Note: RemoteDesktopPortal doesn't own `this`
                               std::make_unique<xdg_portal::RemoteDesktopPortal>(
                                   this,
-                                  options.prefer_cursor_embedded())) {}
+                                  options.prefer_cursor_embedded())) {
+  base_capturer_pipewire_.SendFramesImmediately(true);
+}
 
 WaylandDesktopCapturer::~WaylandDesktopCapturer() {
   WaylandManager::Get()->OnDesktopCapturerDestroyed();
@@ -58,6 +60,10 @@ void WaylandDesktopCapturer::SetScreenResolution(ScreenResolution resolution,
                                            resolution.dimensions().height());
 }
 
+void WaylandDesktopCapturer::SetMaxFrameRate(uint32_t max_frame_rate) {
+  base_capturer_pipewire_.SetMaxFrameRate(max_frame_rate);
+}
+
 #if defined(WEBRTC_USE_GIO)
 webrtc::DesktopCaptureMetadata WaylandDesktopCapturer::GetMetadata() {
   SessionDetails session_details = base_capturer_pipewire_.GetSessionDetails();
@@ -88,4 +94,8 @@ void WaylandDesktopCapturer::OnScreenCastSessionClosed() {
   base_capturer_pipewire_.OnScreenCastSessionClosed();
 }
 
+bool WaylandDesktopCapturer::SupportsFrameCallbacks() {
+  return true;
+}
+
 }  // namespace remoting

remoting/host/linux/wayland_desktop_capturer.h

@@ -7,14 +7,15 @@
 
 #include "base/memory/weak_ptr.h"
 #include "remoting/host/base/screen_resolution.h"
+#include "remoting/protocol/desktop_capturer.h"
 #include "third_party/webrtc/modules/desktop_capture/desktop_capture_metadata.h"
 #include "third_party/webrtc/modules/desktop_capture/desktop_capture_options.h"
 #include "third_party/webrtc/modules/desktop_capture/desktop_capture_types.h"
 #include "third_party/webrtc/modules/desktop_capture/linux/wayland/base_capturer_pipewire.h"
 
 namespace remoting {
 
-class WaylandDesktopCapturer : public webrtc::DesktopCapturer,
+class WaylandDesktopCapturer : public DesktopCapturer,
                                public webrtc::ScreenCastPortal::PortalNotifier {
  public:
   explicit WaylandDesktopCapturer(const webrtc::DesktopCaptureOptions& options);
@@ -27,6 +28,8 @@ class WaylandDesktopCapturer : public webrtc::DesktopCapturer,
   void CaptureFrame() override;
   bool GetSourceList(SourceList* sources) override;
   bool SelectSource(SourceId id) override;
+  bool SupportsFrameCallbacks() override;
+  void SetMaxFrameRate(uint32_t max_frame_rate) override;
 
   // PortalNotifier interface.
   void OnScreenCastRequestResult(webrtc::xdg_portal::RequestResponse result,

remoting/protocol/BUILD.gn

@@ -76,6 +76,7 @@ static_library("protocol") {
     "data_channel_manager.cc",
     "data_channel_manager.h",
     "datagram_channel_factory.h",
+    "desktop_capturer.cc",
     "desktop_capturer.h",
     "display_size.cc",
     "display_size.h",
@@ -278,6 +279,8 @@ static_library("protocol") {
       "capture_scheduler.h",
       "ice_connection_to_client.cc",
       "ice_connection_to_client.h",
+      "no_op_webrtc_frame_scheduler.cc",
+      "no_op_webrtc_frame_scheduler.h",
       "video_frame_pump.cc",
       "video_frame_pump.h",
       "video_stream_event_router.cc",

remoting/protocol/desktop_capturer.cc

@@ -0,0 +1,13 @@
+// Copyright 2023 The Chromium Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "remoting/protocol/desktop_capturer.h"
+
+namespace remoting {
+
+bool DesktopCapturer::SupportsFrameCallbacks() {
+  return false;
+}
+
+}  // namespace remoting

remoting/protocol/desktop_capturer.h

@@ -26,6 +26,10 @@ class DesktopCapturer : public webrtc::DesktopCapturer {
   // Change the position of the composed mouse cursor.
   virtual void SetMouseCursorPosition(const webrtc::DesktopVector& position) {}
 
+  // Whether capturer can notify the callback interface of the available frames
+  // immediately.
+  virtual bool SupportsFrameCallbacks();
+
 #if defined(WEBRTC_USE_GIO)
   virtual void GetMetadataAsync(
       base::OnceCallback<void(webrtc::DesktopCaptureMetadata)> callback) {}

remoting/protocol/no_op_webrtc_frame_scheduler.cc

@@ -0,0 +1,53 @@
+// Copyright 2023 The Chromium Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "remoting/protocol/no_op_webrtc_frame_scheduler.h"
+
+#include "base/notreached.h"
+
+namespace remoting::protocol {
+
+NoOpWebrtcFrameScheduler::NoOpWebrtcFrameScheduler(DesktopCapturer* capturer)
+    : capturer_(capturer) {
+  DETACH_FROM_SEQUENCE(sequence_checker_);
+}
+
+NoOpWebrtcFrameScheduler::~NoOpWebrtcFrameScheduler() {
+  DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
+}
+
+void NoOpWebrtcFrameScheduler::Start(
+    const base::RepeatingClosure& capture_callback) {
+  DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
+}
+
+void NoOpWebrtcFrameScheduler::Pause(bool pause) {
+  DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
+
+  if (pause) {
+    capturer_->SetMaxFrameRate(0u);
+  } else {
+    capturer_->SetMaxFrameRate(last_frame_rate_);
+  }
+}
+
+void NoOpWebrtcFrameScheduler::OnFrameCaptured(
+    const webrtc::DesktopFrame* frame) {
+  DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
+}
+
+void NoOpWebrtcFrameScheduler::SetMaxFramerateFps(int max_framerate_fps) {
+  DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
+
+  last_frame_rate_ = max_framerate_fps;
+  capturer_->SetMaxFrameRate(last_frame_rate_);
+}
+
+void NoOpWebrtcFrameScheduler::BoostCaptureRate(
+    base::TimeDelta capture_interval,
+    base::TimeDelta duration) {
+  NOTIMPLEMENTED() << "Boosting frame rate is not supported for wayland";
+}
+
+}  // namespace remoting::protocol