Media Foundation for Clear: Frame Server Mode Feature Flag

This change adds a new feature flag to indicate whether Media Foundation
Renderer should use Frame Server mode or not. This feature flag is not
intended to force Frame Server exclusivity.

A recent issue was discovered with dual adapter systems where the GPU
process may use a different adapter than the MF Utility process, which
results in a failure to create the frame server texture pool textures.

This revealed a need for a bit more testing control on when Frame Server
mode is available which we will use this flag for to determine whether
to start in Frame Server or DComp mode & to check future rendering mode
changes against.

Bug: 1258887
Change-Id: I89167072f1a11a4f27ee2305a96618ea70819075
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3612105
Commit-Queue: William Carr <wicarr@microsoft.com>
Reviewed-by: Dale Curtis <dalecurtis@chromium.org>
Reviewed-by: Will Harris <wfh@chromium.org>
Reviewed-by: Xiaohan Wang <xhwang@chromium.org>
Cr-Commit-Position: refs/heads/main@{#998714}

media/base/media_switches.cc

@@ -876,6 +876,42 @@ const base::Feature kD3D11HEVCDecoding{"D3D11HEVCDecoding",
 const base::Feature kD3D11Vp9kSVCHWDecoding{"D3D11Vp9kSVCHWDecoding",
                                             base::FEATURE_DISABLED_BY_DEFAULT};
 
+// The Media Foundation Rendering Strategy determines which presentation mode
+// Media Foundation Renderer should use for presenting clear content. This
+// strategy has no impact for protected content, which must always use Direct
+// Composition.
+//
+// The strategy may be one of the following options:
+// 1.) Direct Composition: Media Foundation Renderer will use a Windowsless
+//     Swapchain to present directly to a Direct Composition surface.
+// 2.) Frame Server: Media Foundation Renderer will produce Video Frames that
+//     may be passed through the Chromium video frame rendering pipeline.
+// 3.) Dynamic: Media Foundation Renderer may freely switch between Direct
+//     Composition & Frame Server mode based on the current operating
+//     conditions.
+//
+// Command line invocation:
+// --enable-features=MediaFoundationClearRendering:strategy/direct-composition
+// --enable-features=MediaFoundationClearRendering:strategy/frame-server
+// --enable-features=MediaFoundationClearRendering:strategy/dynamic
+const base::Feature kMediaFoundationClearRendering = {
+    "MediaFoundationClearRendering", base::FEATURE_ENABLED_BY_DEFAULT};
+
+constexpr base::FeatureParam<MediaFoundationClearRenderingStrategy>::Option
+    kMediaFoundationClearRenderingStrategyOptions[] = {
+        {MediaFoundationClearRenderingStrategy::kDirectComposition,
+         "direct-composition"},
+        {MediaFoundationClearRenderingStrategy::kFrameServer, "frame-server"},
+        {MediaFoundationClearRenderingStrategy::kDynamic, "dynamic"}};
+
+// TODO(crbug.com/1321817, wicarr): Media Foundation for Clear should operate in
+// dynamic mode by default. However due to a bug with dual adapters when using
+// Frame Serve mode we currently start in Direct Composition mode.
+const base::FeatureParam<MediaFoundationClearRenderingStrategy>
+    kMediaFoundationClearRenderingStrategyParam{
+        &kMediaFoundationClearRendering, "strategy",
+        MediaFoundationClearRenderingStrategy::kDirectComposition,
+        &kMediaFoundationClearRenderingStrategyOptions};
 #endif  // BUILDFLAG(IS_WIN)
 
 #if BUILDFLAG(IS_CHROMEOS)

media/base/media_switches.h

@@ -275,9 +275,31 @@ MEDIA_EXPORT extern const base::Feature kMediaFoundationVP8Decoding;
 MEDIA_EXPORT extern const base::Feature kMediaFoundationD3D11VideoCapture;
 
 MEDIA_EXPORT extern const base::Feature kMediaFoundationClearPlayback;
+MEDIA_EXPORT extern const base::Feature kAllowMediaFoundationFrameServerMode;
 MEDIA_EXPORT extern const base::Feature kWasapiRawAudioCapture;
 MEDIA_EXPORT extern const base::Feature kD3D11HEVCDecoding;
 MEDIA_EXPORT extern const base::Feature kD3D11Vp9kSVCHWDecoding;
+
+// Strategy affecting how Media Foundation Renderer determines its rendering
+// mode when used with clear video media. This strategy does not impact
+// protected media which must always use Direct Composition mode.
+enum class MediaFoundationClearRenderingStrategy {
+  // The renderer will operate in Direct Composition mode (e.g. windowless
+  // swapchain).
+  kDirectComposition,
+  // The renderer will operate in Frame Server mode.
+  kFrameServer,
+  // The renderer is allowed to switch between Direct Composition & Frame Server
+  // mode at its discretion.
+  kDynamic,
+};
+
+// Under this feature, a given MediaFoundationClearRenderingStrategy param is
+// used by the Media Foundation Renderer for Clear content scenarios.
+MEDIA_EXPORT extern const base::Feature kMediaFoundationClearRendering;
+MEDIA_EXPORT extern const base::FeatureParam<
+    MediaFoundationClearRenderingStrategy>
+    kMediaFoundationClearRenderingStrategyParam;
 #endif  // BUILDFLAG(IS_WIN)
 
 #if BUILDFLAG(IS_CHROMEOS)

media/mojo/clients/win/media_foundation_renderer_client.cc

@@ -9,6 +9,7 @@
 #include "base/callback_helpers.h"
 #include "base/task/bind_post_task.h"
 #include "media/base/media_log.h"
+#include "media/base/media_switches.h"
 #include "media/base/win/mf_feature_checks.h"
 #include "media/base/win/mf_helpers.h"
 #include "media/mojo/mojom/speech_recognition_service.mojom.h"
@@ -79,28 +80,35 @@ void MediaFoundationRendererClient::Initialize(MediaResource* media_resource,
   init_cb_ = std::move(init_cb);
 
   auto media_streams = media_resource->GetAllStreams();
-  bool start_in_dcomp_mode = false;
+
+  // Check the rendering strategy & whether we're operating on clear or
+  // protected content to determine the starting 'rendering_mode_'.
+  // If the Direct Composition strategy is specified or if we're operating on
+  // protected content then start in Direct Composition mode, else start in
+  // Frame Server mode. This behavior must match the logic in
+  // MediaFoundationRenderer::Initialize.
+  auto rendering_strategy = kMediaFoundationClearRenderingStrategyParam.Get();
+  rendering_mode_ =
+      rendering_strategy ==
+              MediaFoundationClearRenderingStrategy::kDirectComposition
+          ? MediaFoundationRenderingMode::DirectComposition
+          : MediaFoundationRenderingMode::FrameServer;
+
   // Start off at 60 fps for our render interval, however it will be updated
   // later in OnVideoFrameRateChange
   render_interval_ = base::Microseconds(16666);
   for (DemuxerStream* stream : media_streams) {
     if (stream->type() == DemuxerStream::Type::VIDEO) {
       if (stream->video_decoder_config().is_encrypted()) {
-        // If the content is clear we'll start in frame server mode
-        // and wait to be promoted to DComp.
-        // This conditional must match the conditional in
-        // MediaFoundationRenderer::Initialize
-        start_in_dcomp_mode = true;
+        // This is protected content which only supports Direct Composition
+        // mode, update 'rendering_mode_' accordingly.
+        rendering_mode_ = MediaFoundationRenderingMode::DirectComposition;
       }
       has_video_ = true;
       break;
     }
   }
 
-  if (!start_in_dcomp_mode) {
-    media_engine_in_frame_server_mode_ = true;
-  }
-
   mojo_renderer_->Initialize(
       media_resource, this,
       base::BindOnce(
@@ -131,7 +139,7 @@ void MediaFoundationRendererClient::InitializeFramePool(
 }
 
 bool MediaFoundationRendererClient::IsFrameServerMode() const {
-  return media_engine_in_frame_server_mode_;
+  return rendering_mode_ == MediaFoundationRenderingMode::FrameServer;
 }
 
 void MediaFoundationRendererClient::OnFrameAvailable(
@@ -324,13 +332,13 @@ void MediaFoundationRendererClient::OnVideoFrameRateChange(
 }
 
 // RenderCallback implementation.
-scoped_refptr<media::VideoFrame> MediaFoundationRendererClient::Render(
+scoped_refptr<VideoFrame> MediaFoundationRendererClient::Render(
     base::TimeTicks deadline_min,
     base::TimeTicks deadline_max,
     RenderingMode mode) {
   // Sends a frame request if in frame server mode, otherwise return nothing as
   // it is rendered independently by Windows Direct Composition.
-  if (!media_engine_in_frame_server_mode_) {
+  if (!IsFrameServerMode()) {
     return nullptr;
   }
 
@@ -433,7 +441,7 @@ void MediaFoundationRendererClient::OnSetOutputRectDone(
   if (output_size_updated_)
     return;
 
-  if (media_engine_in_frame_server_mode_) {
+  if (IsFrameServerMode()) {
     return;
   }
 
@@ -514,12 +522,10 @@ void MediaFoundationRendererClient::OnVideoFrameCreated(
   if (cdm_context_) {
     video_frame->metadata().protected_video = true;
   } else {
+    DCHECK(SupportMediaFoundationClearPlayback());
+    // This video frame is for clear content: setup observation of the mailbox
+    // overlay state changes.
     video_frame->metadata().wants_promotion_hint = true;
-  }
-
-  // If Media Foundation for Clear is enabled then setup observation of the
-  // mailbox for overlay state changes.
-  if (media::SupportMediaFoundationClearPlayback()) {
     ObserveMailboxForOverlayState(mailbox);
   }
 
@@ -549,7 +555,7 @@ void MediaFoundationRendererClient::SignalMediaPlayingStateChange(
   }
 
   // Only start the render loop if we are in frame server mode
-  if (media_engine_in_frame_server_mode_) {
+  if (IsFrameServerMode()) {
     if (is_playing) {
       sink_->Start(this);
     } else {
@@ -563,50 +569,59 @@ void MediaFoundationRendererClient::OnOverlayStateChanged(
     const gpu::Mailbox& mailbox,
     bool promoted) {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
-  MEDIA_LOG(INFO, media_log_) << "Overlay State promoted = " << promoted
-                              << ", starting frame server mode = "
-                              << media_engine_in_frame_server_mode_;
-  renderer_extension_->SetRenderingMode(
-      promoted ? media::RenderingMode::DirectComposition
-               : media::RenderingMode::FrameServer);
-
-  if (promoted && media_engine_in_frame_server_mode_) {
-    // Switch to DComp
-    media_engine_in_frame_server_mode_ = false;
+  MEDIA_LOG(INFO, media_log_) << "Overlay State promoted = " << promoted;
+
+  if (promoted &&
+      rendering_mode_ != MediaFoundationRenderingMode::DirectComposition) {
+    // Switch to Direct Composition mode.
+    rendering_mode_ = MediaFoundationRenderingMode::DirectComposition;
+    renderer_extension_->SetMediaFoundationRenderingMode(rendering_mode_);
     if (is_playing_) {
       sink_->Stop();
     }
     // If we don't have a DComp Visual then create one, otherwise paint
-    // DComp frame again
+    // DComp frame again.
     if (!dcomp_video_frame_) {
       InitializeDCOMPRenderingIfNeeded();
     } else {
       sink_->PaintSingleFrame(dcomp_video_frame_, true);
     }
-  } else if (!promoted && !media_engine_in_frame_server_mode_) {
-    // Switch to Frame Server if not already there
-    media_engine_in_frame_server_mode_ = true;
+  } else if (!promoted &&
+             rendering_mode_ != MediaFoundationRenderingMode::FrameServer) {
+    // Switch to Frame Server mode.
+    rendering_mode_ = MediaFoundationRenderingMode::FrameServer;
+    renderer_extension_->SetMediaFoundationRenderingMode(rendering_mode_);
     if (is_playing_) {
       sink_->Start(this);
     }
+  } else {
+    MEDIA_LOG(INFO, media_log_)
+        << "Overlay State Changed but there was no mode change.";
   }
 }
 
 void MediaFoundationRendererClient::ObserveMailboxForOverlayState(
     const gpu::Mailbox& mailbox) {
-  mailbox_ = mailbox;
-  // 'observe_overlay_state_cb_' creates a content::OverlayStateObserver to
-  // subscribe to overlay state information for the given 'mailbox' from the
-  // Viz layer in the GPU process. We hold an OverlayStateObserverSubscription
-  // since a direct dependency on a content object is not allowed. Once the
-  // OverlayStateObserverSubscription is destroyed the OnOverlayStateChanged
-  // callback will no longer be invoked, so base::Unretained(this) is safe to
-  // use.
-  observer_subscription_ = observe_overlay_state_cb_.Run(
-      mailbox,
-      base::BindRepeating(&MediaFoundationRendererClient::OnOverlayStateChanged,
-                          base::Unretained(this), mailbox));
-  DCHECK(observer_subscription_);
+  // If the rendering strategy is dynamic then setup an OverlayStateObserver to
+  // respond to promotion changes. If the rendering strategy is Direct
+  // Composition or Frame Server then we do not need to listen & respond to
+  // overlay state changes.
+  auto rendering_strategy = kMediaFoundationClearRenderingStrategyParam.Get();
+  if (rendering_strategy == MediaFoundationClearRenderingStrategy::kDynamic) {
+    mailbox_ = mailbox;
+    // 'observe_overlay_state_cb_' creates a content::OverlayStateObserver to
+    // subscribe to overlay state information for the given 'mailbox' from the
+    // Viz layer in the GPU process. We hold an OverlayStateObserverSubscription
+    // since a direct dependency on a content object is not allowed. Once the
+    // OverlayStateObserverSubscription is destroyed the OnOverlayStateChanged
+    // callback will no longer be invoked, so base::Unretained(this) is safe to
+    // use.
+    observer_subscription_ = observe_overlay_state_cb_.Run(
+        mailbox, base::BindRepeating(
+                     &MediaFoundationRendererClient::OnOverlayStateChanged,
+                     base::Unretained(this), mailbox));
+    DCHECK(observer_subscription_);
+  }
 }
 
 }  // namespace media

media/mojo/clients/win/media_foundation_renderer_client.h

@@ -20,6 +20,7 @@
 #include "media/mojo/clients/mojo_renderer.h"
 #include "media/mojo/mojom/dcomp_surface_registry.mojom.h"
 #include "media/mojo/mojom/renderer_extensions.mojom.h"
+#include "media/renderers/win/media_foundation_rendering_mode.h"
 #include "mojo/public/cpp/bindings/pending_receiver.h"
 #include "mojo/public/cpp/bindings/pending_remote.h"
 #include "mojo/public/cpp/bindings/receiver.h"
@@ -162,11 +163,14 @@ class MediaFoundationRendererClient
   bool output_size_updated_ = false;
   bool is_playing_ = false;
   bool has_video_ = false;
-  bool media_engine_in_frame_server_mode_ = false;
   scoped_refptr<VideoFrame> dcomp_video_frame_;
   scoped_refptr<VideoFrame> next_video_frame_;
   gpu::Mailbox mailbox_;
 
+  // Rendering mode the Media Engine will use.
+  MediaFoundationRenderingMode rendering_mode_ =
+      MediaFoundationRenderingMode::DirectComposition;
+
   PipelineStatusCallback init_cb_;
   raw_ptr<CdmContext> cdm_context_ = nullptr;
   CdmAttachedCB cdm_attached_cb_;

media/mojo/mojom/BUILD.gn

@@ -721,8 +721,8 @@ mojom("mojom") {
       {
         types = [
           {
-            mojom = "media.mojom.RenderingMode"
-            cpp = "::media::RenderingMode"
+            mojom = "media.mojom.MediaFoundationRenderingMode"
+            cpp = "::media::MediaFoundationRenderingMode"
           },
         ]
         traits_headers = [ "media_foundation_rendering_mode_mojom_traits.h" ]

media/mojo/mojom/media_foundation_rendering_mode_mojom_traits.h

@@ -6,35 +6,38 @@
 #define MEDIA_MOJO_MOJOM_MEDIA_FOUNDATION_RENDERING_MODE_MOJOM_TRAITS_H_
 
 #include "media/mojo/mojom/renderer_extensions.mojom-shared.h"
-#include "media/renderers/win/media_foundation_renderer_extension.h"
+#include "media/renderers/win/media_foundation_rendering_mode.h"
 
 template <>
-struct mojo::EnumTraits<media::mojom::RenderingMode, media::RenderingMode> {
+struct mojo::EnumTraits<media::mojom::MediaFoundationRenderingMode,
+                        media::MediaFoundationRenderingMode> {
  public:
-  static bool FromMojom(media::mojom::RenderingMode data,
-                        media::RenderingMode* output) {
+  static bool FromMojom(media::mojom::MediaFoundationRenderingMode data,
+                        media::MediaFoundationRenderingMode* output) {
     switch (data) {
-      case media::mojom::RenderingMode::DirectComposition:
-        *output = media::RenderingMode::DirectComposition;
+      case media::mojom::MediaFoundationRenderingMode::DirectComposition:
+        *output = media::MediaFoundationRenderingMode::DirectComposition;
         return true;
-      case media::mojom::RenderingMode::FrameServer:
-        *output = media::RenderingMode::FrameServer;
+      case media::mojom::MediaFoundationRenderingMode::FrameServer:
+        *output = media::MediaFoundationRenderingMode::FrameServer;
         return true;
     }
     NOTREACHED();
     return false;
   }
 
-  static media::mojom::RenderingMode ToMojom(media::RenderingMode data) {
+  static media::mojom::MediaFoundationRenderingMode ToMojom(
+      media::MediaFoundationRenderingMode data) {
     switch (data) {
-      case media::RenderingMode::DirectComposition:
-        return media::mojom::RenderingMode::DirectComposition;
-      case media::RenderingMode::FrameServer:
-        return media::mojom::RenderingMode::FrameServer;
+      case media::MediaFoundationRenderingMode::DirectComposition:
+        return media::mojom::MediaFoundationRenderingMode::DirectComposition;
+      case media::MediaFoundationRenderingMode::FrameServer:
+        return media::mojom::MediaFoundationRenderingMode::FrameServer;
+        break;
     }
     NOTREACHED();
-    return media::mojom::RenderingMode::DirectComposition;
+    return media::mojom::MediaFoundationRenderingMode::DirectComposition;
   }
 };
 
-#endif  // MEDIA_MOJO_MOJOM_MEDIA_FOUNDATION_RENDERING_MODE_MOJOM_TRAITS_H_
+#endif  // MEDIA_MOJO_MOJOM_MEDIA_FOUNDATION_RENDERING_MODE_MOJOM_TRAITS_H_

media/mojo/mojom/renderer_extensions.mojom

@@ -91,7 +91,7 @@ interface MediaFoundationRendererClientExtension {
 // direct composition visual, skipping the Chromium compositor.
 // FrameServer is when the media engine will render to a texture and
 // that textured is provided to the Chromium compositor.
-enum RenderingMode {
+enum MediaFoundationRenderingMode {
   DirectComposition,
   FrameServer
 };
@@ -133,7 +133,7 @@ interface MediaFoundationRendererExtension {
     mojo_base.mojom.TimeTicks deadline_max);
 
   // Notify which rendering mode to be using for future video frames.
-  SetRenderingMode(RenderingMode mode);
+  SetMediaFoundationRenderingMode(MediaFoundationRenderingMode mode);
 };
 
 // This interface is used by the browser to determine if there are any renderers

media/mojo/services/media_foundation_renderer_wrapper.cc

@@ -217,9 +217,8 @@ void MediaFoundationRendererWrapper::RequestNextFrameBetweenTimestamps(
   renderer_->RequestNextFrameBetweenTimestamps(deadline_min, deadline_max);
 }
 
-void MediaFoundationRendererWrapper::SetRenderingMode(
-    media::RenderingMode mode) {
-  // We define the media RenderingMode enum to match the mojom.
-  renderer_->SetRenderingMode(mode);
+void MediaFoundationRendererWrapper::SetMediaFoundationRenderingMode(
+    MediaFoundationRenderingMode mode) {
+  renderer_->SetMediaFoundationRenderingMode(mode);
 }
 }  // namespace media