From 3fb310da2ede0d9f890f884d8b62f34aa90546a9 Mon Sep 17 00:00:00 2001 From: Saman Sami Date: Thu, 9 Nov 2017 16:51:54 +0000 Subject: [PATCH] Implement a video detector in Viz Current video detector is broken with Viz enabled. Move parts of ash::VideoDetector that need to be run in Viz process into viz::VideoDetector and notify ash::VideoDetector over mojo. I don't expect any regressions in behaviour. One improvement is that when the video activity is happening in an occluded window, the new video detector does not report that activity, whereas the old video detector did. Test: There are unit tests. I also tried --mash, --mus,--enable-viz, and no flags manually for a chromeos build on my workstation. Also tried --mash and --mus and no flags manually on a link chromebook. BUG=775035 Change-Id: I5098492edaf5cce08f9ba6c50a5d7482eb739426 Reviewed-on: https://chromium-review.googlesource.com/702044 Reviewed-by: kylechar Reviewed-by: Scott Violet Reviewed-by: Yuzhu Shen Reviewed-by: Tom Sepez Reviewed-by: Dan Erat Commit-Queue: Saman Sami Cr-Commit-Position: refs/heads/master@{#515186} --- ash/BUILD.gn | 1 + ash/DEPS | 2 + ash/mus/manifest.json | 2 +- ash/mus/shell_port_mus.cc | 13 + ash/mus/shell_port_mus.h | 2 + ash/shell.cc | 7 +- ash/shell_port.h | 5 + ash/shell_port_classic.cc | 9 + ash/shell_port_classic.h | 2 + .../power/video_activity_notifier_unittest.cc | 3 +- ash/wm/video_detector.cc | 123 +------ ash/wm/video_detector.h | 66 +--- ash/wm/video_detector_unittest.cc | 170 +--------- .../viz/host/host_frame_sink_manager.cc | 5 + components/viz/host/host_frame_sink_manager.h | 3 + components/viz/service/BUILD.gn | 3 + .../viz/service/display/surface_aggregator.cc | 14 +- .../viz/service/display/surface_aggregator.h | 1 + .../compositor_frame_sink_support.h | 2 + .../frame_sinks/frame_sink_manager_impl.cc | 20 ++ .../frame_sinks/frame_sink_manager_impl.h | 14 + .../viz/service/frame_sinks/video_detector.cc | 167 ++++++++++ .../viz/service/frame_sinks/video_detector.h | 109 ++++++ .../frame_sinks/video_detector_unittest.cc | 315 ++++++++++++++++++ .../viz/service/surfaces/surface_manager.cc | 5 + .../viz/service/surfaces/surface_manager.h | 9 + .../viz/service/surfaces/surface_observer.h | 4 + .../viz/test/compositor_frame_helpers.cc | 4 +- .../viz/test/compositor_frame_helpers.h | 2 +- components/viz/test/test_frame_sink_manager.h | 2 + .../app/mojo/content_browser_manifest.json | 2 +- mojo/public/cpp/bindings/interface_ptr_set.h | 2 + services/ui/manifest.json | 3 + services/ui/public/interfaces/BUILD.gn | 1 + .../ui/public/interfaces/video_detector.mojom | 13 + services/ui/service.cc | 6 + services/ui/service.h | 3 + services/ui/ws/BUILD.gn | 2 + services/ui/ws/video_detector_impl.cc | 28 ++ services/ui/ws/video_detector_impl.h | 44 +++ services/ui/ws/window_server.cc | 1 + services/ui/ws/window_server.h | 5 + .../compositing/frame_sink_manager.mojom | 5 + services/viz/public/interfaces/BUILD.gn | 1 + .../compositing/video_detector_observer.mojom | 11 + 45 files changed, 875 insertions(+), 336 deletions(-) create mode 100644 components/viz/service/frame_sinks/video_detector.cc create mode 100644 components/viz/service/frame_sinks/video_detector.h create mode 100644 components/viz/service/frame_sinks/video_detector_unittest.cc create mode 100644 services/ui/public/interfaces/video_detector.mojom create mode 100644 services/ui/ws/video_detector_impl.cc create mode 100644 services/ui/ws/video_detector_impl.h create mode 100644 services/viz/public/interfaces/compositing/video_detector_observer.mojom diff --git a/ash/BUILD.gn b/ash/BUILD.gn index d682d84342ac29..66ef2dfdb4e810 100644 --- a/ash/BUILD.gn +++ b/ash/BUILD.gn @@ -972,6 +972,7 @@ component("ash") { "//components/strings", "//components/user_manager", "//components/vector_icons", + "//components/viz/host", "//components/viz/service", "//components/wallpaper", "//device/bluetooth", diff --git a/ash/DEPS b/ash/DEPS index 4be6af97d01008..7a6076f3180435 100644 --- a/ash/DEPS +++ b/ash/DEPS @@ -12,6 +12,7 @@ include_rules = [ "+components/user_manager", "+components/vector_icons", "+components/viz/common", + "+components/viz/host", "+components/wallpaper", "+gpu/config", "+media", @@ -19,6 +20,7 @@ include_rules = [ "+services/preferences/public", "+services/service_manager/public", "+services/ui/public", + "+services/viz/public", "+skia/ext", "+third_party/cros_system_api", "+third_party/icu", diff --git a/ash/mus/manifest.json b/ash/mus/manifest.json index 3f5fe8a9961dfd..7e86148843a66f 100644 --- a/ash/mus/manifest.json +++ b/ash/mus/manifest.json @@ -45,7 +45,7 @@ "*": [ "accessibility", "app" ], "ash_pref_connector": [ "pref_connector" ], "local_state": [ "pref_client" ], - "ui": [ "display_dev", "window_manager" ], + "ui": [ "display_dev", "window_manager", "video_detector" ], "touch_hud": [ "mash:launchable" ] } } diff --git a/ash/mus/shell_port_mus.cc b/ash/mus/shell_port_mus.cc index 966b5311f07aba..dbab14f88bc100 100644 --- a/ash/mus/shell_port_mus.cc +++ b/ash/mus/shell_port_mus.cc @@ -33,6 +33,7 @@ #include "ash/wm/workspace/workspace_event_handler_classic.h" #include "base/memory/ptr_util.h" #include "services/ui/public/interfaces/constants.mojom.h" +#include "services/ui/public/interfaces/video_detector.mojom.h" #include "ui/aura/env.h" #include "ui/aura/mus/focus_synchronizer.h" #include "ui/aura/mus/window_tree_client.h" @@ -262,5 +263,17 @@ ShellPortMus::CreateAcceleratorController() { accelerator_controller_delegate_.get(), nullptr); } +void ShellPortMus::AddVideoDetectorObserver( + viz::mojom::VideoDetectorObserverPtr observer) { + // We may not have access to the connector in unit tests. + if (!window_manager_->connector()) + return; + + ui::mojom::VideoDetectorPtr video_detector; + window_manager_->connector()->BindInterface(ui::mojom::kServiceName, + &video_detector); + video_detector->AddObserver(std::move(observer)); +} + } // namespace mus } // namespace ash diff --git a/ash/mus/shell_port_mus.h b/ash/mus/shell_port_mus.h index afe9c8c1676317..6fc8867e899033 100644 --- a/ash/mus/shell_port_mus.h +++ b/ash/mus/shell_port_mus.h @@ -85,6 +85,8 @@ class ShellPortMus : public ShellPort { std::unique_ptr CreateNativeDisplayDelegate() override; std::unique_ptr CreateAcceleratorController() override; + void AddVideoDetectorObserver( + viz::mojom::VideoDetectorObserverPtr observer) override; protected: WindowManager* window_manager_; diff --git a/ash/shell.cc b/ash/shell.cc index 6d78d5c3f6d711..c6863a36b8c0cb 100644 --- a/ash/shell.cc +++ b/ash/shell.cc @@ -144,6 +144,7 @@ #include "chromeos/system/devicemode.h" #include "components/prefs/pref_registry_simple.h" #include "components/prefs/pref_service.h" +#include "components/viz/host/host_frame_sink_manager.h" #include "services/preferences/public/cpp/pref_service_factory.h" #include "services/preferences/public/interfaces/preferences.mojom.h" #include "services/service_manager/public/cpp/connector.h" @@ -1068,7 +1069,11 @@ void Shell::Init(const ShellInitParams& init_params) { autoclick_controller_.reset(AutoclickController::CreateInstance()); high_contrast_controller_.reset(new HighContrastController); - video_detector_.reset(new VideoDetector); + + viz::mojom::VideoDetectorObserverPtr observer; + video_detector_ = + std::make_unique(mojo::MakeRequest(&observer)); + shell_port_->AddVideoDetectorObserver(std::move(observer)); tooltip_controller_.reset(new views::corewm::TooltipController( std::unique_ptr(new views::corewm::TooltipAura))); diff --git a/ash/shell_port.h b/ash/shell_port.h index ab37f10466c815..6a2ae8c25f2c33 100644 --- a/ash/shell_port.h +++ b/ash/shell_port.h @@ -14,6 +14,7 @@ #include "ash/wm/lock_state_observer.h" #include "base/observer_list.h" #include "base/optional.h" +#include "services/viz/public/interfaces/compositing/video_detector_observer.mojom.h" #include "ui/aura/client/window_types.h" #include "ui/base/cursor/cursor_data.h" #include "ui/base/ui_base_types.h" @@ -168,6 +169,10 @@ class ASH_EXPORT ShellPort { // sent to the server. virtual void UpdateSystemModalAndBlockingContainers() = 0; + // Adds an observer for viz::VideoDetector. + virtual void AddVideoDetectorObserver( + viz::mojom::VideoDetectorObserverPtr observer) = 0; + protected: ShellPort(); diff --git a/ash/shell_port_classic.cc b/ash/shell_port_classic.cc index 30d8d3483c76b6..eec225ef472931 100644 --- a/ash/shell_port_classic.cc +++ b/ash/shell_port_classic.cc @@ -26,6 +26,7 @@ #include "ash/wm/window_util.h" #include "ash/wm/workspace/workspace_event_handler_classic.h" #include "base/memory/ptr_util.h" +#include "components/viz/host/host_frame_sink_manager.h" #include "ui/aura/env.h" #include "ui/display/manager/chromeos/default_touch_transform_setter.h" #include "ui/display/types/native_display_delegate.h" @@ -171,4 +172,12 @@ ShellPortClassic::CreateAcceleratorController() { accelerator_controller_delegate_.get(), nullptr); } +void ShellPortClassic::AddVideoDetectorObserver( + viz::mojom::VideoDetectorObserverPtr observer) { + aura::Env::GetInstance() + ->context_factory_private() + ->GetHostFrameSinkManager() + ->AddVideoDetectorObserver(std::move(observer)); +} + } // namespace ash diff --git a/ash/shell_port_classic.h b/ash/shell_port_classic.h index ac37666f72e862..d1e86852769ed4 100644 --- a/ash/shell_port_classic.h +++ b/ash/shell_port_classic.h @@ -68,6 +68,8 @@ class ASH_EXPORT ShellPortClassic : public ShellPort { std::unique_ptr CreateNativeDisplayDelegate() override; std::unique_ptr CreateAcceleratorController() override; + void AddVideoDetectorObserver( + viz::mojom::VideoDetectorObserverPtr observer) override; private: std::unique_ptr pointer_watcher_adapter_; diff --git a/ash/system/power/video_activity_notifier_unittest.cc b/ash/system/power/video_activity_notifier_unittest.cc index dea54c5eea2176..a4eb165b3d309d 100644 --- a/ash/system/power/video_activity_notifier_unittest.cc +++ b/ash/system/power/video_activity_notifier_unittest.cc @@ -23,7 +23,8 @@ class VideoActivityNotifierTest : public AshTestBase { AshTestBase::SetUp(); power_client_ = static_cast( chromeos::DBusThreadManager::Get()->GetPowerManagerClient()); - detector_.reset(new VideoDetector()); + detector_ = std::make_unique( + viz::mojom::VideoDetectorObserverRequest()); notifier_.reset(new VideoActivityNotifier(detector_.get())); } diff --git a/ash/wm/video_detector.cc b/ash/wm/video_detector.cc index 587d4a7abb00dd..4c4626d5b88cd1 100644 --- a/ash/wm/video_detector.cc +++ b/ash/wm/video_detector.cc @@ -15,72 +15,13 @@ namespace ash { -const int VideoDetector::kMinUpdateWidth = 333; -const int VideoDetector::kMinUpdateHeight = 250; -const int VideoDetector::kMinFramesPerSecond = 15; -const int VideoDetector::kVideoTimeoutMs = 1000; -const int VideoDetector::kMinVideoDurationMs = 3000; - -// Stores information about updates to a window and determines whether it's -// likely that a video is playing in it. -class VideoDetector::WindowInfo { - public: - WindowInfo() : buffer_start_(0), buffer_size_(0) {} - - // Handles an update within a window, returning true if it appears that - // video is currently playing in the window. - bool RecordUpdateAndCheckForVideo(const gfx::Rect& region, - base::TimeTicks now) { - if (region.width() < kMinUpdateWidth || region.height() < kMinUpdateHeight) - return false; - - // If the buffer is full, drop the first timestamp. - if (buffer_size_ == static_cast(kMinFramesPerSecond)) { - buffer_start_ = (buffer_start_ + 1) % kMinFramesPerSecond; - buffer_size_--; - } - - update_times_[(buffer_start_ + buffer_size_) % kMinFramesPerSecond] = now; - buffer_size_++; - - const bool in_video = - (buffer_size_ == static_cast(kMinFramesPerSecond)) && - ((now - update_times_[buffer_start_]).InSecondsF() <= 1.0); - - if (in_video && video_start_time_.is_null()) - video_start_time_ = update_times_[buffer_start_]; - else if (!in_video && !video_start_time_.is_null()) - video_start_time_ = base::TimeTicks(); - - const base::TimeDelta elapsed = now - video_start_time_; - return in_video && - elapsed >= base::TimeDelta::FromMilliseconds(kMinVideoDurationMs); - } - - private: - // Circular buffer containing update times of the last (up to - // |kMinFramesPerSecond|) video-sized updates to this window. - base::TimeTicks update_times_[kMinFramesPerSecond]; - - // Time at which the current sequence of updates that looks like video - // started. Empty if video isn't currently playing. - base::TimeTicks video_start_time_; - - // Index into |update_times_| of the oldest update. - size_t buffer_start_; - - // Number of updates stored in |update_times_|. - size_t buffer_size_; - - DISALLOW_COPY_AND_ASSIGN(WindowInfo); -}; - -VideoDetector::VideoDetector() +VideoDetector::VideoDetector(viz::mojom::VideoDetectorObserverRequest request) : state_(State::NOT_PLAYING), video_is_playing_(false), window_observer_manager_(this), scoped_session_observer_(this), - is_shutting_down_(false) { + is_shutting_down_(false), + binding_(this, std::move(request)) { aura::Env::GetInstance()->AddObserver(this); Shell::Get()->AddShellObserver(this); } @@ -98,34 +39,10 @@ void VideoDetector::RemoveObserver(Observer* observer) { observers_.RemoveObserver(observer); } -bool VideoDetector::TriggerTimeoutForTest() { - if (!video_inactive_timer_.IsRunning()) - return false; - - video_inactive_timer_.Stop(); - HandleVideoInactive(); - return true; -} - void VideoDetector::OnWindowInitialized(aura::Window* window) { window_observer_manager_.Add(window); } -void VideoDetector::OnDelegatedFrameDamage( - aura::Window* window, - const gfx::Rect& damage_rect_in_dip) { - if (is_shutting_down_) - return; - std::unique_ptr& info = window_infos_[window]; - if (!info.get()) - info.reset(new WindowInfo); - - base::TimeTicks now = - !now_for_test_.is_null() ? now_for_test_ : base::TimeTicks::Now(); - if (info->RecordUpdateAndCheckForVideo(damage_rect_in_dip, now)) - HandleVideoActivity(window, now); -} - void VideoDetector::OnWindowDestroying(aura::Window* window) { if (fullscreen_root_windows_.count(window)) { window_observer_manager_.Remove(window); @@ -135,7 +52,6 @@ void VideoDetector::OnWindowDestroying(aura::Window* window) { } void VideoDetector::OnWindowDestroyed(aura::Window* window) { - window_infos_.erase(window); window_observer_manager_.Remove(window); } @@ -159,27 +75,6 @@ void VideoDetector::OnFullscreenStateChanged(bool is_fullscreen, } } -void VideoDetector::HandleVideoActivity(aura::Window* window, - base::TimeTicks now) { - if (!window->IsVisible()) - return; - - gfx::Rect root_bounds = window->GetRootWindow()->bounds(); - if (!window->GetBoundsInRootWindow().Intersects(root_bounds)) - return; - - video_is_playing_ = true; - video_inactive_timer_.Start( - FROM_HERE, base::TimeDelta::FromMilliseconds(kVideoTimeoutMs), this, - &VideoDetector::HandleVideoInactive); - UpdateState(); -} - -void VideoDetector::HandleVideoInactive() { - video_is_playing_ = false; - UpdateState(); -} - void VideoDetector::UpdateState() { State new_state = State::NOT_PLAYING; if (video_is_playing_) { @@ -194,4 +89,16 @@ void VideoDetector::UpdateState() { } } +void VideoDetector::OnVideoActivityStarted() { + if (is_shutting_down_) + return; + video_is_playing_ = true; + UpdateState(); +} + +void VideoDetector::OnVideoActivityEnded() { + video_is_playing_ = false; + UpdateState(); +} + } // namespace ash diff --git a/ash/wm/video_detector.h b/ash/wm/video_detector.h index 68ea70136994ae..b7349ce0806f0f 100644 --- a/ash/wm/video_detector.h +++ b/ash/wm/video_detector.h @@ -18,6 +18,8 @@ #include "base/scoped_observer.h" #include "base/time/time.h" #include "base/timer/timer.h" +#include "mojo/public/cpp/bindings/binding.h" +#include "services/viz/public/interfaces/compositing/video_detector_observer.mojom.h" #include "ui/aura/env_observer.h" #include "ui/aura/window_observer.h" @@ -25,19 +27,16 @@ namespace aura { class Window; } -namespace gfx { -class Rect; -} - namespace ash { -// Watches for updates to windows and tries to detect when a video is playing. -// We err on the side of false positives and can be fooled by things like -// continuous scrolling of a page. +// Receives notifications from viz::VideoDetector about whether it is likely +// that a video is being played on screen. If video activity is detected, this +// class will classify it as full screen or windowed. class ASH_EXPORT VideoDetector : public aura::EnvObserver, public aura::WindowObserver, public SessionObserver, - public ShellObserver { + public ShellObserver, + public viz::mojom::VideoDetectorObserver { public: // State of detected video activity. enum class State { @@ -60,42 +59,18 @@ class ASH_EXPORT VideoDetector : public aura::EnvObserver, virtual ~Observer() {} }; - // Minimum dimensions in pixels that a window update must have to be - // considered a potential video frame. - static const int kMinUpdateWidth; - static const int kMinUpdateHeight; - - // Number of video-sized updates that we must see within a second in a window - // before we assume that a video is playing. - static const int kMinFramesPerSecond; - - // Timeout after which video is no longer considered to be playing. - static const int kVideoTimeoutMs; - - // Duration video must be playing in a window before it is reported to - // observers. - static const int kMinVideoDurationMs; - - VideoDetector(); + VideoDetector(viz::mojom::VideoDetectorObserverRequest request); ~VideoDetector() override; State state() const { return state_; } - void set_now_for_test(base::TimeTicks now) { now_for_test_ = now; } - void AddObserver(Observer* observer); void RemoveObserver(Observer* observer); - // Runs HandleVideoInactive() and returns true, or returns false if - // |video_inactive_timer_| wasn't running. - bool TriggerTimeoutForTest() WARN_UNUSED_RESULT; - // EnvObserver overrides. void OnWindowInitialized(aura::Window* window) override; // aura::WindowObserver overrides. - void OnDelegatedFrameDamage(aura::Window* window, - const gfx::Rect& region) override; void OnWindowDestroyed(aura::Window* window) override; void OnWindowDestroying(aura::Window* window) override; @@ -106,14 +81,11 @@ class ASH_EXPORT VideoDetector : public aura::EnvObserver, void OnFullscreenStateChanged(bool is_fullscreen, aura::Window* root_window) override; - private: - // Called when video activity is observed in |window|. - void HandleVideoActivity(aura::Window* window, base::TimeTicks now); - - // Called by |inactive_timer_| |kVideoTimeoutMs| after the last-observed video - // activity. - void HandleVideoInactive(); + // viz::mojom::VideoDetectorObserver implementation. + void OnVideoActivityStarted() override; + void OnVideoActivityEnded() override; + private: // Updates |state_| and notifies |observers_| if it changed. void UpdateState(); @@ -126,25 +98,15 @@ class ASH_EXPORT VideoDetector : public aura::EnvObserver, // Currently-fullscreen root windows. std::set fullscreen_root_windows_; - // Maps from a window that we're tracking to information about it. - class WindowInfo; - using WindowInfoMap = std::map>; - WindowInfoMap window_infos_; - base::ObserverList observers_; - // Calls HandleVideoInactive(). - base::OneShotTimer video_inactive_timer_; - - // If set, used when the current time is needed. This can be set by tests to - // simulate the passage of time. - base::TimeTicks now_for_test_; - ScopedObserver window_observer_manager_; ScopedSessionObserver scoped_session_observer_; bool is_shutting_down_; + mojo::Binding binding_; + DISALLOW_COPY_AND_ASSIGN(VideoDetector); }; diff --git a/ash/wm/video_detector_unittest.cc b/ash/wm/video_detector_unittest.cc index e16744a02aa125..0fef97079c71e2 100644 --- a/ash/wm/video_detector_unittest.cc +++ b/ash/wm/video_detector_unittest.cc @@ -54,16 +54,7 @@ class TestObserver : public VideoDetector::Observer { class VideoDetectorTest : public AshTestBase { public: - VideoDetectorTest() - : kMinFps(VideoDetector::kMinFramesPerSecond), - kMinRect(gfx::Point(0, 0), - gfx::Size(VideoDetector::kMinUpdateWidth, - VideoDetector::kMinUpdateHeight)), - kMinDuration(base::TimeDelta::FromMilliseconds( - VideoDetector::kMinVideoDurationMs)), - kTimeout( - base::TimeDelta::FromMilliseconds(VideoDetector::kVideoTimeoutMs)), - next_window_id_(1000) {} + VideoDetectorTest() : next_window_id_(1000) {} ~VideoDetectorTest() override {} void SetUp() override { @@ -71,9 +62,6 @@ class VideoDetectorTest : public AshTestBase { observer_.reset(new TestObserver); detector_ = Shell::Get()->video_detector(); detector_->AddObserver(observer_.get()); - - now_ = base::TimeTicks::Now(); - detector_->set_now_for_test(now_); } void TearDown() override { @@ -82,48 +70,15 @@ class VideoDetectorTest : public AshTestBase { } protected: - // Move |detector_|'s idea of the current time forward by |delta|. - void AdvanceTime(base::TimeDelta delta) { - now_ += delta; - detector_->set_now_for_test(now_); - } - // Creates and returns a new window with |bounds|. std::unique_ptr CreateTestWindow(const gfx::Rect& bounds) { return std::unique_ptr( CreateTestWindowInShell(SK_ColorRED, next_window_id_++, bounds)); } - // Report updates to |window| of area |region| at a rate of - // |updates_per_second| over |duration|. The first update will be sent - // immediately and |now_| will incremented by |duration| upon returning. - void SendUpdates(aura::Window* window, - const gfx::Rect& region, - int updates_per_second, - base::TimeDelta duration) { - const base::TimeDelta time_between_updates = - base::TimeDelta::FromSecondsD(1.0 / updates_per_second); - const base::TimeTicks end_time = now_ + duration; - while (now_ < end_time) { - detector_->OnDelegatedFrameDamage(window, region); - AdvanceTime(time_between_updates); - } - now_ = end_time; - detector_->set_now_for_test(now_); - } - - // Constants placed here for convenience. - const int kMinFps; - const gfx::Rect kMinRect; - const base::TimeDelta kMinDuration; - const base::TimeDelta kTimeout; - VideoDetector* detector_; // not owned std::unique_ptr observer_; - // The current (fake) time used by |detector_|. - base::TimeTicks now_; - // Next ID to be assigned by CreateTestWindow(). int next_window_id_; @@ -131,125 +86,8 @@ class VideoDetectorTest : public AshTestBase { DISALLOW_COPY_AND_ASSIGN(VideoDetectorTest); }; -TEST_F(VideoDetectorTest, DontReportWhenRegionTooSmall) { - std::unique_ptr window = - CreateTestWindow(gfx::Rect(0, 0, 1024, 768)); - gfx::Rect rect = kMinRect; - rect.Inset(0, 0, 1, 0); - SendUpdates(window.get(), rect, 2 * kMinFps, 2 * kMinDuration); - EXPECT_TRUE(observer_->empty()); -} - -TEST_F(VideoDetectorTest, DontReportWhenFramerateTooLow) { - std::unique_ptr window = - CreateTestWindow(gfx::Rect(0, 0, 1024, 768)); - SendUpdates(window.get(), kMinRect, kMinFps - 5, 2 * kMinDuration); - EXPECT_TRUE(observer_->empty()); -} - -TEST_F(VideoDetectorTest, DontReportWhenNotPlayingLongEnough) { - std::unique_ptr window = - CreateTestWindow(gfx::Rect(0, 0, 1024, 768)); - SendUpdates(window.get(), kMinRect, 2 * kMinFps, 0.5 * kMinDuration); - EXPECT_TRUE(observer_->empty()); - - // Continue playing. - SendUpdates(window.get(), kMinRect, 2 * kMinFps, 0.6 * kMinDuration); - EXPECT_EQ(VideoDetector::State::PLAYING_WINDOWED, observer_->PopState()); - EXPECT_TRUE(observer_->empty()); -} - -TEST_F(VideoDetectorTest, DontReportWhenWindowOffscreen) { - std::unique_ptr window = - CreateTestWindow(gfx::Rect(0, 0, 1024, 768)); - window->SetBounds( - gfx::Rect(gfx::Point(Shell::GetPrimaryRootWindow()->bounds().width(), 0), - window->bounds().size())); - SendUpdates(window.get(), kMinRect, 2 * kMinFps, 2 * kMinDuration); - EXPECT_TRUE(observer_->empty()); - - // Move the window onscreen. - window->SetBounds(gfx::Rect(gfx::Point(0, 0), window->bounds().size())); - SendUpdates(window.get(), kMinRect, 2 * kMinFps, 2 * kMinDuration); - EXPECT_EQ(VideoDetector::State::PLAYING_WINDOWED, observer_->PopState()); - EXPECT_TRUE(observer_->empty()); -} - -TEST_F(VideoDetectorTest, DontReportWhenWindowHidden) { - std::unique_ptr window = - CreateTestWindow(gfx::Rect(0, 0, 1024, 768)); - // Reparent the window to the root to make sure that visibility changes aren't - // animated. - Shell::GetPrimaryRootWindow()->AddChild(window.get()); - window->Hide(); - SendUpdates(window.get(), kMinRect, kMinFps + 5, 2 * kMinDuration); - EXPECT_TRUE(observer_->empty()); - - // Make the window visible. - observer_->reset(); - AdvanceTime(kTimeout); - window->Show(); - SendUpdates(window.get(), kMinRect, kMinFps + 5, 2 * kMinDuration); - EXPECT_EQ(VideoDetector::State::PLAYING_WINDOWED, observer_->PopState()); - EXPECT_TRUE(observer_->empty()); -} - -TEST_F(VideoDetectorTest, DontReportDuringShutdown) { - std::unique_ptr window = - CreateTestWindow(gfx::Rect(0, 0, 1024, 768)); - Shell::Get()->session_controller()->NotifyChromeTerminating(); - SendUpdates(window.get(), kMinRect, kMinFps + 5, 2 * kMinDuration); - EXPECT_TRUE(observer_->empty()); -} - -TEST_F(VideoDetectorTest, ReportStartAndStop) { - const base::TimeDelta kDuration = - kMinDuration + base::TimeDelta::FromMilliseconds(100); - std::unique_ptr window = - CreateTestWindow(gfx::Rect(0, 0, 1024, 768)); - SendUpdates(window.get(), kMinRect, kMinFps + 5, kDuration); - EXPECT_EQ(VideoDetector::State::PLAYING_WINDOWED, observer_->PopState()); - EXPECT_TRUE(observer_->empty()); - - AdvanceTime(kTimeout); - EXPECT_TRUE(detector_->TriggerTimeoutForTest()); - EXPECT_EQ(VideoDetector::State::NOT_PLAYING, observer_->PopState()); - EXPECT_TRUE(observer_->empty()); - - // The timer shouldn't be running anymore. - EXPECT_FALSE(detector_->TriggerTimeoutForTest()); - - // Start playing again. - SendUpdates(window.get(), kMinRect, kMinFps + 5, kDuration); - EXPECT_EQ(VideoDetector::State::PLAYING_WINDOWED, observer_->PopState()); - EXPECT_TRUE(observer_->empty()); - - AdvanceTime(kTimeout); - EXPECT_TRUE(detector_->TriggerTimeoutForTest()); - EXPECT_EQ(VideoDetector::State::NOT_PLAYING, observer_->PopState()); - EXPECT_TRUE(observer_->empty()); -} - -TEST_F(VideoDetectorTest, ReportOnceForMultipleWindows) { - gfx::Rect kWindowBounds(gfx::Point(), gfx::Size(1024, 768)); - std::unique_ptr window1 = CreateTestWindow(kWindowBounds); - std::unique_ptr window2 = CreateTestWindow(kWindowBounds); - - // Even if there's video playing in both windows, the observer should only - // receive a single notification. - const int fps = 2 * kMinFps; - const base::TimeDelta time_between_updates = - base::TimeDelta::FromSecondsD(1.0 / fps); - const base::TimeTicks start_time = now_; - while (now_ < start_time + 2 * kMinDuration) { - detector_->OnDelegatedFrameDamage(window1.get(), kMinRect); - detector_->OnDelegatedFrameDamage(window2.get(), kMinRect); - AdvanceTime(time_between_updates); - } - EXPECT_EQ(VideoDetector::State::PLAYING_WINDOWED, observer_->PopState()); - EXPECT_TRUE(observer_->empty()); -} - +// Verify that the video detector can distinguish fullscreen and windowed video +// activity. TEST_F(VideoDetectorTest, ReportFullscreen) { UpdateDisplay("1024x768,1024x768"); @@ -260,7 +98,7 @@ TEST_F(VideoDetectorTest, ReportFullscreen) { window_state->OnWMEvent(&toggle_fullscreen_event); ASSERT_TRUE(window_state->IsFullscreen()); window->Focus(); - SendUpdates(window.get(), kMinRect, 2 * kMinFps, 2 * kMinDuration); + detector_->OnVideoActivityStarted(); EXPECT_EQ(VideoDetector::State::PLAYING_FULLSCREEN, observer_->PopState()); EXPECT_TRUE(observer_->empty()); diff --git a/components/viz/host/host_frame_sink_manager.cc b/components/viz/host/host_frame_sink_manager.cc index 2de59528ec2351..d9cb25ca5bf4bb 100644 --- a/components/viz/host/host_frame_sink_manager.cc +++ b/components/viz/host/host_frame_sink_manager.cc @@ -353,4 +353,9 @@ HostFrameSinkManager::FrameSinkData::~FrameSinkData() = default; HostFrameSinkManager::FrameSinkData& HostFrameSinkManager::FrameSinkData:: operator=(FrameSinkData&& other) = default; +void HostFrameSinkManager::AddVideoDetectorObserver( + mojom::VideoDetectorObserverPtr observer) { + frame_sink_manager_->AddVideoDetectorObserver(std::move(observer)); +} + } // namespace viz diff --git a/components/viz/host/host_frame_sink_manager.h b/components/viz/host/host_frame_sink_manager.h index 95cb73544dcec2..1dc046b611ba76 100644 --- a/components/viz/host/host_frame_sink_manager.h +++ b/components/viz/host/host_frame_sink_manager.h @@ -118,6 +118,9 @@ class VIZ_HOST_EXPORT HostFrameSinkManager const FrameSinkId& owner); void DropTemporaryReference(const SurfaceId& surface_id); + // Asks viz to send updates regarding video activity to |observer|. + void AddVideoDetectorObserver(mojom::VideoDetectorObserverPtr observer); + // CompositorFrameSinkSupportManager: std::unique_ptr CreateCompositorFrameSinkSupport( mojom::CompositorFrameSinkClient* client, diff --git a/components/viz/service/BUILD.gn b/components/viz/service/BUILD.gn index 3d37f020b76f15..2213a0214d985f 100644 --- a/components/viz/service/BUILD.gn +++ b/components/viz/service/BUILD.gn @@ -110,6 +110,8 @@ viz_component("service") { "frame_sinks/surface_resource_holder.h", "frame_sinks/surface_resource_holder_client.h", "frame_sinks/video_capture/capturable_frame_sink.h", + "frame_sinks/video_detector.cc", + "frame_sinks/video_detector.h", "gl/gpu_service_impl.cc", "gl/gpu_service_impl.h", "hit_test/hit_test_aggregator.cc", @@ -258,6 +260,7 @@ viz_source_set("unit_tests") { "frame_sinks/referenced_surface_tracker_unittest.cc", "frame_sinks/surface_references_unittest.cc", "frame_sinks/surface_synchronization_unittest.cc", + "frame_sinks/video_detector_unittest.cc", "gl/gpu_service_impl_unittest.cc", "hit_test/hit_test_aggregator_unittest.cc", "surfaces/surface_hittest_unittest.cc", diff --git a/components/viz/service/display/surface_aggregator.cc b/components/viz/service/display/surface_aggregator.cc index 7402fafe66c13b..10c4e74ecfa26b 100644 --- a/components/viz/service/display/surface_aggregator.cc +++ b/components/viz/service/display/surface_aggregator.cc @@ -769,6 +769,7 @@ void SurfaceAggregator::ProcessAddedAndRemovedSurfaces() { gfx::Rect SurfaceAggregator::PrewalkTree(Surface* surface, bool in_moved_pixel_surface, int parent_pass_id, + bool will_draw, PrewalkResult* result) { // This is for debugging a possible use after free. // TODO(jbauman): Remove this once we have enough information. @@ -781,6 +782,10 @@ gfx::Rect SurfaceAggregator::PrewalkTree(Surface* surface, contained_surfaces_[surface->surface_id()] = surface->GetActiveFrameIndex(); if (!surface->HasActiveFrame()) return gfx::Rect(); + + if (will_draw) + manager_->SurfaceWillBeDrawn(surface); + const CompositorFrame& frame = surface->GetActiveFrame(); int child_id = 0; // TODO(jbauman): hack for unit tests that don't set up rp @@ -935,7 +940,8 @@ gfx::Rect SurfaceAggregator::PrewalkTree(Surface* surface, if (surface) { surface_damage.Union(PrewalkTree(surface, surface_info.has_moved_pixels, - surface_info.parent_pass_id, result)); + surface_info.parent_pass_id, will_draw, + result)); } if (surface_damage.IsEmpty()) @@ -953,12 +959,13 @@ gfx::Rect SurfaceAggregator::PrewalkTree(Surface* surface, } CHECK(debug_weak_this.get()); + for (const auto& surface_id : frame.metadata.referenced_surfaces) { if (!contained_surfaces_.count(surface_id)) { result->undrawn_surfaces.insert(surface_id); Surface* undrawn_surface = manager_->GetSurfaceForId(surface_id); if (undrawn_surface) - PrewalkTree(undrawn_surface, false, 0, result); + PrewalkTree(undrawn_surface, false, 0, false /* will_draw */, result); } } @@ -1075,7 +1082,8 @@ CompositorFrame SurfaceAggregator::Aggregate(const SurfaceId& surface_id) { valid_surfaces_.clear(); has_cached_render_passes_ = false; PrewalkResult prewalk_result; - root_damage_rect_ = PrewalkTree(surface, false, 0, &prewalk_result); + root_damage_rect_ = + PrewalkTree(surface, false, 0, true /* will_draw */, &prewalk_result); PropagateCopyRequestPasses(); has_copy_requests_ = !copy_request_passes_.empty(); frame.metadata.may_contain_video = prewalk_result.may_contain_video; diff --git a/components/viz/service/display/surface_aggregator.h b/components/viz/service/display/surface_aggregator.h index fc7211d006086c..17e5cd488088b8 100644 --- a/components/viz/service/display/surface_aggregator.h +++ b/components/viz/service/display/surface_aggregator.h @@ -151,6 +151,7 @@ class VIZ_SERVICE_EXPORT SurfaceAggregator { gfx::Rect PrewalkTree(Surface* surface, bool in_moved_pixel_surface, int parent_pass, + bool will_draw, PrewalkResult* result); void CopyUndrawnSurfaces(PrewalkResult* prewalk); void CopyPasses(const CompositorFrame& frame, Surface* surface); diff --git a/components/viz/service/frame_sinks/compositor_frame_sink_support.h b/components/viz/service/frame_sinks/compositor_frame_sink_support.h index 3a06eec7c4cd02..a888a76255af57 100644 --- a/components/viz/service/frame_sinks/compositor_frame_sink_support.h +++ b/components/viz/service/frame_sinks/compositor_frame_sink_support.h @@ -60,6 +60,8 @@ class VIZ_SERVICE_EXPORT CompositorFrameSinkSupport const FrameSinkId& frame_sink_id() const { return frame_sink_id_; } + const SurfaceId& current_surface_id() const { return current_surface_id_; } + const LocalSurfaceId& local_surface_id() const { return current_surface_id_.local_surface_id(); } diff --git a/components/viz/service/frame_sinks/frame_sink_manager_impl.cc b/components/viz/service/frame_sinks/frame_sink_manager_impl.cc index f3e19bcaa9816e..d678774e819c67 100644 --- a/components/viz/service/frame_sinks/frame_sink_manager_impl.cc +++ b/components/viz/service/frame_sinks/frame_sink_manager_impl.cc @@ -81,6 +81,8 @@ void FrameSinkManagerImpl::RegisterFrameSinkId( const FrameSinkId& frame_sink_id) { DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); surface_manager_.RegisterFrameSinkId(frame_sink_id); + if (video_detector_) + video_detector_->OnFrameSinkIdRegistered(frame_sink_id); } void FrameSinkManagerImpl::InvalidateFrameSinkId( @@ -88,6 +90,8 @@ void FrameSinkManagerImpl::InvalidateFrameSinkId( DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); compositor_frame_sinks_.erase(frame_sink_id); surface_manager_.InvalidateFrameSinkId(frame_sink_id); + if (video_detector_) + video_detector_->OnFrameSinkIdInvalidated(frame_sink_id); } void FrameSinkManagerImpl::SetFrameSinkDebugLabel( @@ -423,4 +427,20 @@ void FrameSinkManagerImpl::SwitchActiveAggregatedHitTestRegionList( } } +void FrameSinkManagerImpl::AddVideoDetectorObserver( + mojom::VideoDetectorObserverPtr observer) { + if (!video_detector_) + video_detector_ = std::make_unique(&surface_manager_); + video_detector_->AddObserver(std::move(observer)); +} + +VideoDetector* FrameSinkManagerImpl::CreateVideoDetectorForTesting( + std::unique_ptr tick_clock, + scoped_refptr task_runner) { + DCHECK(!video_detector_); + video_detector_ = std::make_unique( + surface_manager(), std::move(tick_clock), task_runner); + return video_detector_.get(); +} + } // namespace viz diff --git a/components/viz/service/frame_sinks/frame_sink_manager_impl.h b/components/viz/service/frame_sinks/frame_sink_manager_impl.h index 5957eb106b181d..5368a28e4ede78 100644 --- a/components/viz/service/frame_sinks/frame_sink_manager_impl.h +++ b/components/viz/service/frame_sinks/frame_sink_manager_impl.h @@ -16,6 +16,7 @@ #include "base/threading/thread_checker.h" #include "components/viz/common/surfaces/frame_sink_id.h" #include "components/viz/service/frame_sinks/primary_begin_frame_source.h" +#include "components/viz/service/frame_sinks/video_detector.h" #include "components/viz/service/hit_test/hit_test_manager.h" #include "components/viz/service/surfaces/surface_manager.h" #include "components/viz/service/surfaces/surface_observer.h" @@ -23,6 +24,7 @@ #include "gpu/ipc/common/surface_handle.h" #include "mojo/public/cpp/bindings/binding.h" #include "services/viz/privileged/interfaces/compositing/frame_sink_manager.mojom.h" +#include "services/viz/public/interfaces/compositing/video_detector_observer.mojom.h" namespace cc { @@ -86,6 +88,8 @@ class VIZ_SERVICE_EXPORT FrameSinkManagerImpl : public SurfaceObserver, void AssignTemporaryReference(const SurfaceId& surface_id, const FrameSinkId& owner) override; void DropTemporaryReference(const SurfaceId& surface_id) override; + void AddVideoDetectorObserver( + mojom::VideoDetectorObserverPtr observer) override; // CompositorFrameSinkSupport, hierarchy, and BeginFrameSource can be // registered and unregistered in any order with respect to each other. @@ -154,6 +158,12 @@ class VIZ_SERVICE_EXPORT FrameSinkManagerImpl : public SurfaceObserver, // This method is virtual so the implementation can be modified in unit tests. virtual uint64_t GetActiveFrameIndex(const SurfaceId& surface_id); + // Instantiates |video_detector_| for tests where we simulate the passage of + // time. + VideoDetector* CreateVideoDetectorForTesting( + std::unique_ptr tick_clock, + scoped_refptr task_runner); + private: friend class cc::test::SurfaceSynchronizationTest; @@ -221,6 +231,10 @@ class VIZ_SERVICE_EXPORT FrameSinkManagerImpl : public SurfaceObserver, // directly connected. Use this to make function calls. mojom::FrameSinkManagerClient* client_ = nullptr; + // |video_detector_| is instantiated lazily in order to avoid overhead on + // platforms that don't need video detection. + std::unique_ptr video_detector_; + mojom::FrameSinkManagerClientPtr client_ptr_; mojo::Binding binding_; diff --git a/components/viz/service/frame_sinks/video_detector.cc b/components/viz/service/frame_sinks/video_detector.cc new file mode 100644 index 00000000000000..6bf93c22b3b3f2 --- /dev/null +++ b/components/viz/service/frame_sinks/video_detector.cc @@ -0,0 +1,167 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/viz/service/frame_sinks/video_detector.h" + +#include "base/time/time.h" +#include "components/viz/common/quads/compositor_frame.h" +#include "components/viz/service/surfaces/surface_manager.h" +#include "ui/gfx/geometry/dip_util.h" +#include "ui/gfx/geometry/rect.h" + +namespace viz { + +constexpr base::TimeDelta VideoDetector::kVideoTimeout; +constexpr base::TimeDelta VideoDetector::kMinVideoDuration; + +// Stores information about updates to a client and determines whether it's +// likely that a video is playing in it. +class VideoDetector::ClientInfo { + public: + ClientInfo() = default; + + // Called when a Surface belonging to this client is drawn. Returns true if we + // determine that video is playing in this client. + bool ReportDrawnAndCheckForVideo(Surface* surface, base::TimeTicks now) { + uint64_t frame_index = surface->GetActiveFrameIndex(); + + // If |frame_index| hasn't increased, then no new frame was submitted since + // the last draw. + if (frame_index <= last_drawn_frame_index_) + return false; + + last_drawn_frame_index_ = frame_index; + + const CompositorFrame& frame = surface->GetActiveFrame(); + + gfx::Rect damage = + gfx::ConvertRectToDIP(frame.device_scale_factor(), + frame.render_pass_list.back()->damage_rect); + + if (damage.width() < kMinDamageWidth || damage.height() < kMinDamageHeight) + return false; + + // If the buffer is full, drop the first timestamp. + if (buffer_size_ == kMinFramesPerSecond) { + buffer_start_ = (buffer_start_ + 1) % kMinFramesPerSecond; + buffer_size_--; + } + + update_times_[(buffer_start_ + buffer_size_) % kMinFramesPerSecond] = now; + buffer_size_++; + + const bool in_video = + (buffer_size_ == kMinFramesPerSecond) && + (now - update_times_[buffer_start_] <= base::TimeDelta::FromSeconds(1)); + + if (in_video && video_start_time_.is_null()) + video_start_time_ = update_times_[buffer_start_]; + else if (!in_video && !video_start_time_.is_null()) + video_start_time_ = base::TimeTicks(); + + const base::TimeDelta elapsed = now - video_start_time_; + return in_video && elapsed >= kMinVideoDuration; + } + + private: + // Circular buffer containing update times of the last (up to + // |kMinFramesPerSecond|) video-sized updates to this client. + base::TimeTicks update_times_[kMinFramesPerSecond]; + + // Time at which the current sequence of updates that looks like video + // started. Empty if video isn't currently playing. + base::TimeTicks video_start_time_; + + // Index into |update_times_| of the oldest update. + uint32_t buffer_start_ = 0; + + // Number of updates stored in |update_times_|. + uint32_t buffer_size_ = 0; + + // Frame index of the last drawn Surface. We use this number to determine + // whether a new frame was submitted since the last time the Surface was + // drawn. + uint64_t last_drawn_frame_index_ = 0; + + DISALLOW_COPY_AND_ASSIGN(ClientInfo); +}; + +VideoDetector::VideoDetector( + SurfaceManager* surface_manager, + std::unique_ptr tick_clock, + scoped_refptr task_runner) + : tick_clock_(std::move(tick_clock)), + video_inactive_timer_(tick_clock_.get()), + surface_manager_(surface_manager) { + surface_manager_->AddObserver(this); + for (auto it : surface_manager_->valid_frame_sink_labels()) + client_infos_[it.first] = std::make_unique(); + if (task_runner) + video_inactive_timer_.SetTaskRunner(task_runner); +} + +VideoDetector::~VideoDetector() { + surface_manager_->RemoveObserver(this); +} + +void VideoDetector::OnVideoActivityEnded() { + DCHECK(video_is_playing_); + video_is_playing_ = false; + observers_.ForAllPtrs([](mojom::VideoDetectorObserver* observer) { + observer->OnVideoActivityEnded(); + }); +} + +void VideoDetector::AddObserver(mojom::VideoDetectorObserverPtr observer) { + if (video_is_playing_) + observer->OnVideoActivityStarted(); + observers_.AddPtr(std::move(observer)); +} + +void VideoDetector::OnFrameSinkIdRegistered(const FrameSinkId& frame_sink_id) { + DCHECK(!client_infos_.count(frame_sink_id)); + client_infos_[frame_sink_id] = std::make_unique(); +} + +void VideoDetector::OnFrameSinkIdInvalidated(const FrameSinkId& frame_sink_id) { + client_infos_.erase(frame_sink_id); +} + +bool VideoDetector::OnSurfaceDamaged(const SurfaceId& surface_id, + const BeginFrameAck& ack) { + return false; +} + +// |surface| is scheduled to be drawn. See if it has a new frame since the +// last time it was drawn and record the damage. +void VideoDetector::OnSurfaceWillBeDrawn(Surface* surface) { + // If there is no observer, don't waste cycles detecting video activity. + if (observers_.empty()) + return; + + const FrameSinkId& frame_sink_id = surface->surface_id().frame_sink_id(); + + auto it = client_infos_.find(frame_sink_id); + + // If the corresponding entry in |client_infos_| does not exist, it means the + // FrameSinkId has been invalidated and the client's CompositorFrameSink is + // destroyed so it cannot send new frames and video activity is impossible. + if (it == client_infos_.end()) + return; + + base::TimeTicks now = tick_clock_->NowTicks(); + + if (it->second->ReportDrawnAndCheckForVideo(surface, now)) { + video_inactive_timer_.Start(FROM_HERE, kVideoTimeout, this, + &VideoDetector::OnVideoActivityEnded); + if (!video_is_playing_) { + video_is_playing_ = true; + observers_.ForAllPtrs([](mojom::VideoDetectorObserver* observer) { + observer->OnVideoActivityStarted(); + }); + } + } +} + +} // namespace viz diff --git a/components/viz/service/frame_sinks/video_detector.h b/components/viz/service/frame_sinks/video_detector.h new file mode 100644 index 00000000000000..192f579cbf98ce --- /dev/null +++ b/components/viz/service/frame_sinks/video_detector.h @@ -0,0 +1,109 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_VIZ_SERVICE_FRAME_SINKS_VIDEO_DETECTOR_H_ +#define COMPONENTS_VIZ_SERVICE_FRAME_SINKS_VIDEO_DETECTOR_H_ + +#include + +#include "base/time/default_tick_clock.h" +#include "base/timer/timer.h" +#include "components/viz/common/surfaces/frame_sink_id.h" +#include "components/viz/service/surfaces/surface_observer.h" +#include "components/viz/service/viz_service_export.h" +#include "mojo/public/cpp/bindings/interface_ptr_set.h" +#include "services/viz/public/interfaces/compositing/video_detector_observer.mojom.h" + +namespace viz { + +class SurfaceManager; +class VideoDetectorTest; + +// Watches for updates to clients and tries to detect when a video is playing. +// If a client sends CompositorFrames with damages of size at least +// |kMinDamageWidth| x |kMinDamageHeight| at the rate of at least +// |kMinFramesPerSecond| for the duration of at least |kMinVideoDuration| then +// we say it is playing video. We err on the side of false positives and can be +// fooled by things like continuous scrolling of a page. +class VIZ_SERVICE_EXPORT VideoDetector : public SurfaceObserver { + public: + VideoDetector(SurfaceManager* surface_manager, + std::unique_ptr tick_clock = + std::make_unique(), + scoped_refptr task_runner = nullptr); + virtual ~VideoDetector(); + + // Adds an observer. The observer can be removed by closing the mojo + // connection. + void AddObserver(mojom::VideoDetectorObserverPtr observer); + + // When a FrameSinkId is registered/invalidated, we need to insert/delete the + // corresponding entry in client_infos_. + void OnFrameSinkIdRegistered(const FrameSinkId& frame_sink_id); + void OnFrameSinkIdInvalidated(const FrameSinkId& frame_sink_id); + + private: + friend class VideoDetectorTest; + + class ClientInfo; + + // Minimum dimensions in pixels that damages must have to be considered a + // potential video frame. + static constexpr int kMinDamageWidth = 333; + static constexpr int kMinDamageHeight = 250; + + // Number of video-sized updates that we must see within a second in a client + // before we assume that a video is playing. + static constexpr int kMinFramesPerSecond = 15; + + // Timeout after which video is no longer considered to be playing. + static constexpr base::TimeDelta kVideoTimeout = + base::TimeDelta::FromMilliseconds(1000); + + // Duration video must be playing in a client before it is reported to + // observers. + static constexpr base::TimeDelta kMinVideoDuration = + base::TimeDelta::FromMilliseconds(3000); + + // If no video activity is detected for |kVideoTimeout|, this + // method will be called by |video_inactive_timer_|; + void OnVideoActivityEnded(); + + // SurfaceObserver implementation. + void OnFirstSurfaceActivation(const SurfaceInfo& surface_info) override {} + void OnSurfaceActivated(const SurfaceId& surface_id) override {} + void OnSurfaceDestroyed(const SurfaceId& surface_id) override {} + bool OnSurfaceDamaged(const SurfaceId& surface_id, + const BeginFrameAck& ack) override; + void OnSurfaceDiscarded(const SurfaceId& surface_id) override {} + void OnSurfaceDamageExpected(const SurfaceId& surface_id, + const BeginFrameArgs& args) override {} + void OnSurfaceSubtreeDamaged(const SurfaceId& surface_id) override {} + void OnSurfaceWillBeDrawn(Surface* surface) override; + + // True if video has been observed in the last |kVideoTimeout|. + bool video_is_playing_ = false; + + // Provides the current time. + std::unique_ptr tick_clock_; + + // Calls OnVideoActivityEnded() after |kVideoTimeout|. Uses |tick_clock_| to + // measure time. + base::OneShotTimer video_inactive_timer_; + + // Contains information used for determining video activity in each client. + base::flat_map> client_infos_; + + // Observers that are interested to know about video activity. We only detect + // video activity if there is at least one client. + mojo::InterfacePtrSet observers_; + + SurfaceManager* const surface_manager_; + + DISALLOW_COPY_AND_ASSIGN(VideoDetector); +}; + +} // namespace viz + +#endif // COMPONENTS_VIZ_SERVICE_FRAME_SINKS_VIDEO_DETECTOR_H_ diff --git a/components/viz/service/frame_sinks/video_detector_unittest.cc b/components/viz/service/frame_sinks/video_detector_unittest.cc new file mode 100644 index 00000000000000..1891b02d68186e --- /dev/null +++ b/components/viz/service/frame_sinks/video_detector_unittest.cc @@ -0,0 +1,315 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include +#include + +#include "base/compiler_specific.h" +#include "base/containers/circular_deque.h" +#include "base/test/test_mock_time_task_runner.h" +#include "base/time/tick_clock.h" +#include "base/time/time.h" +#include "components/viz/common/quads/surface_draw_quad.h" +#include "components/viz/common/surfaces/local_surface_id_allocator.h" +#include "components/viz/service/display/surface_aggregator.h" +#include "components/viz/service/frame_sinks/compositor_frame_sink_support.h" +#include "components/viz/service/frame_sinks/frame_sink_manager_impl.h" +#include "components/viz/service/frame_sinks/video_detector.h" +#include "components/viz/test/compositor_frame_helpers.h" +#include "components/viz/test/fake_compositor_frame_sink_client.h" +#include "mojo/public/cpp/bindings/binding.h" +#include "services/viz/public/interfaces/compositing/video_detector_observer.mojom.h" +#include "testing/gtest/include/gtest/gtest.h" +#include "third_party/skia/include/core/SkColor.h" +#include "ui/gfx/geometry/rect.h" + +namespace viz { + +namespace { + +// Implementation that just records video state changes. +class TestObserver : public mojom::VideoDetectorObserver { + public: + TestObserver() : binding_(this) {} + + void Bind(mojom::VideoDetectorObserverRequest request) { + binding_.Bind(std::move(request)); + } + + bool IsEmpty() { + binding_.FlushForTesting(); + return states_.empty(); + } + + void Reset() { + binding_.FlushForTesting(); + states_.clear(); + } + + // Pops and returns the earliest-received state. + bool PopState() { + binding_.FlushForTesting(); + CHECK(!states_.empty()); + uint8_t first_state = states_.front(); + states_.pop_front(); + return first_state; + } + + // mojom::VideoDetectorObserver implementation. + void OnVideoActivityStarted() override { states_.push_back(true); } + void OnVideoActivityEnded() override { states_.push_back(false); } + + private: + // States in the order they were received. + base::circular_deque states_; + + mojo::Binding binding_; + + DISALLOW_COPY_AND_ASSIGN(TestObserver); +}; + +} // namespace + +class VideoDetectorTest : public testing::Test { + public: + VideoDetectorTest() + : surface_aggregator_(frame_sink_manager_.surface_manager(), + nullptr, + false) {} + + ~VideoDetectorTest() override {} + + void SetUp() override { + mock_task_runner_ = base::MakeRefCounted(); + + detector_ = frame_sink_manager_.CreateVideoDetectorForTesting( + mock_task_runner_->GetMockTickClock(), mock_task_runner_); + + mojom::VideoDetectorObserverPtr video_detector_observer; + observer_.Bind(mojo::MakeRequest(&video_detector_observer)); + detector_->AddObserver(std::move(video_detector_observer)); + + root_frame_sink_ = CreateFrameSink(); + root_frame_sink_->SubmitCompositorFrame( + local_surface_id_allocator_.GenerateId(), test::MakeCompositorFrame()); + } + + protected: + // Constants placed here for convenience. + static constexpr int kMinFps = VideoDetector::kMinFramesPerSecond; + static constexpr gfx::Rect kMinRect = + gfx::Rect(VideoDetector::kMinDamageWidth, + VideoDetector::kMinDamageHeight); + static constexpr base::TimeDelta kMinDuration = + VideoDetector::kMinVideoDuration; + static constexpr base::TimeDelta kTimeout = VideoDetector::kVideoTimeout; + + // Move |detector_|'s idea of the current time forward by |delta|. + void AdvanceTime(base::TimeDelta delta) { + mock_task_runner_->FastForwardBy(delta); + } + + void CreateDisplayFrame() { + surface_aggregator_.Aggregate(root_frame_sink_->current_surface_id()); + } + + void EmbedClient(CompositorFrameSinkSupport* frame_sink) { + embedded_clients_.insert(frame_sink); + SubmitRootFrame(); + } + + void SubmitRootFrame() { + CompositorFrame frame = test::MakeCompositorFrame(); + RenderPass* render_pass = frame.render_pass_list.back().get(); + SharedQuadState* shared_quad_state = + render_pass->CreateAndAppendSharedQuadState(); + for (CompositorFrameSinkSupport* frame_sink : embedded_clients_) { + SurfaceDrawQuad* quad = + render_pass->CreateAndAppendDrawQuad(); + quad->SetNew(shared_quad_state, gfx::Rect(0, 0, 10, 10), + gfx::Rect(0, 0, 5, 5), frame_sink->current_surface_id(), + base::nullopt, SK_ColorMAGENTA); + } + root_frame_sink_->SubmitCompositorFrame( + root_frame_sink_->local_surface_id(), std::move(frame)); + } + + void SendUpdate(CompositorFrameSinkSupport* frame_sink, + const gfx::Rect& damage) { + LocalSurfaceId local_surface_id = + frame_sink->local_surface_id().is_valid() + ? frame_sink->local_surface_id() + : local_surface_id_allocator_.GenerateId(); + frame_sink->SubmitCompositorFrame(local_surface_id, + MakeDamagedCompositorFrame(damage)); + } + + // Report updates to |client| of area |damage| at a rate of + // |updates_per_second| over |duration|. The first update will be sent + // immediately and time will have advanced by |duration| upon returning. + void SendUpdates(CompositorFrameSinkSupport* frame_sink, + const gfx::Rect& damage, + int updates_per_second, + base::TimeDelta duration) { + const base::TimeDelta time_between_updates = + base::TimeDelta::FromSecondsD(1.0 / updates_per_second); + for (base::TimeDelta d; d < duration; d += time_between_updates) { + SendUpdate(frame_sink, damage); + CreateDisplayFrame(); + AdvanceTime(std::min(time_between_updates, duration - d)); + } + } + + std::unique_ptr CreateFrameSink() { + constexpr bool is_root = false; + constexpr bool needs_sync_points = true; + static uint32_t client_id = 1; + FrameSinkId frame_sink_id(client_id++, 0); + frame_sink_manager_.RegisterFrameSinkId(frame_sink_id); + std::unique_ptr frame_sink = + CompositorFrameSinkSupport::Create(&frame_sink_client_, + &frame_sink_manager_, frame_sink_id, + is_root, needs_sync_points); + SendUpdate(frame_sink.get(), gfx::Rect()); + return frame_sink; + } + + VideoDetector* detector_; + TestObserver observer_; + + scoped_refptr mock_task_runner_; + + private: + CompositorFrame MakeDamagedCompositorFrame(const gfx::Rect& damage) { + constexpr gfx::Size kFrameSinkSize = gfx::Size(10000, 10000); + CompositorFrame frame = test::MakeCompositorFrame(kFrameSinkSize); + frame.render_pass_list.back()->damage_rect = damage; + return frame; + } + + FrameSinkManagerImpl frame_sink_manager_; + FakeCompositorFrameSinkClient frame_sink_client_; + LocalSurfaceIdAllocator local_surface_id_allocator_; + SurfaceAggregator surface_aggregator_; + std::unique_ptr root_frame_sink_; + std::set embedded_clients_; + + DISALLOW_COPY_AND_ASSIGN(VideoDetectorTest); +}; + +constexpr gfx::Rect VideoDetectorTest::kMinRect; +constexpr base::TimeDelta VideoDetectorTest::kMinDuration; +constexpr base::TimeDelta VideoDetectorTest::kTimeout; + +// Verify that VideoDetector does not report clients with small damage rects. +TEST_F(VideoDetectorTest, DontReportWhenDamageTooSmall) { + std::unique_ptr frame_sink = CreateFrameSink(); + EmbedClient(frame_sink.get()); + + { + // Send damages with a smaller width than |kMinRect|. Make sure video + // activity isn't detected. + gfx::Rect rect = kMinRect; + rect.Inset(0, 0, 1, 0); + SendUpdates(frame_sink.get(), rect, 2 * kMinFps, 2 * kMinDuration); + EXPECT_TRUE(observer_.IsEmpty()); + } + + { + // Send damages with a smaller height than |kMinRect|. Make sure video + // activity isn't detected. + gfx::Rect rect = kMinRect; + rect.Inset(0, 0, 1, 0); + SendUpdates(frame_sink.get(), rect, 2 * kMinFps, 2 * kMinDuration); + EXPECT_TRUE(observer_.IsEmpty()); + } +} + +// Verify that VideoDetector does not report clients with a low frame rate. +TEST_F(VideoDetectorTest, DontReportWhenFramerateTooLow) { + std::unique_ptr frame_sink = CreateFrameSink(); + EmbedClient(frame_sink.get()); + SendUpdates(frame_sink.get(), kMinRect, kMinFps - 5, 2 * kMinDuration); + EXPECT_TRUE(observer_.IsEmpty()); +} + +// Verify that VideoDetector does not report clients until they have played for +// the minimum necessary duration. +TEST_F(VideoDetectorTest, DontReportWhenNotPlayingLongEnough) { + std::unique_ptr frame_sink = CreateFrameSink(); + EmbedClient(frame_sink.get()); + SendUpdates(frame_sink.get(), kMinRect, 2 * kMinFps, 0.5 * kMinDuration); + EXPECT_TRUE(observer_.IsEmpty()); + + SendUpdates(frame_sink.get(), kMinRect, 2 * kMinFps, 0.6 * kMinDuration); + EXPECT_TRUE(observer_.PopState()); + EXPECT_TRUE(observer_.IsEmpty()); +} + +// Verify that VideoDetector does not report clients that are not visible +// on screen. +TEST_F(VideoDetectorTest, DontReportWhenClientHidden) { + std::unique_ptr frame_sink = CreateFrameSink(); + + SendUpdates(frame_sink.get(), kMinRect, kMinFps + 5, 2 * kMinDuration); + EXPECT_TRUE(observer_.IsEmpty()); + + // Make the client visible. + observer_.Reset(); + AdvanceTime(kTimeout); + EmbedClient(frame_sink.get()); + SendUpdates(frame_sink.get(), kMinRect, kMinFps + 5, 2 * kMinDuration); + EXPECT_TRUE(observer_.PopState()); + EXPECT_TRUE(observer_.IsEmpty()); +} + +// Turn video activity on and off. Make sure the observers are notified +// properly. +TEST_F(VideoDetectorTest, ReportStartAndStop) { + const base::TimeDelta kDuration = + kMinDuration + base::TimeDelta::FromMilliseconds(100); + std::unique_ptr frame_sink = CreateFrameSink(); + EmbedClient(frame_sink.get()); + SendUpdates(frame_sink.get(), kMinRect, kMinFps + 5, kDuration); + EXPECT_TRUE(observer_.PopState()); + EXPECT_TRUE(observer_.IsEmpty()); + + AdvanceTime(kTimeout); + EXPECT_FALSE(observer_.PopState()); + EXPECT_TRUE(observer_.IsEmpty()); + + // Start playing again. + SendUpdates(frame_sink.get(), kMinRect, kMinFps + 5, kDuration); + EXPECT_TRUE(observer_.PopState()); + EXPECT_TRUE(observer_.IsEmpty()); + + AdvanceTime(kTimeout); + EXPECT_FALSE(observer_.PopState()); + EXPECT_TRUE(observer_.IsEmpty()); +} + +// If there are multiple clients playing video, make sure that observers only +// receive a single notification. +TEST_F(VideoDetectorTest, ReportOnceForMultipleClients) { + std::unique_ptr frame_sink1 = CreateFrameSink(); + std::unique_ptr frame_sink2 = CreateFrameSink(); + EmbedClient(frame_sink1.get()); + EmbedClient(frame_sink2.get()); + + // Even if there's video playing in both clients, the observer should only + // receive a single notification. + constexpr int fps = 2 * kMinFps; + constexpr base::TimeDelta time_between_updates = + base::TimeDelta::FromSecondsD(1.0 / fps); + for (base::TimeDelta d; d < 2 * kMinDuration; d += time_between_updates) { + SendUpdate(frame_sink1.get(), kMinRect); + SendUpdate(frame_sink2.get(), kMinRect); + AdvanceTime(time_between_updates); + CreateDisplayFrame(); + } + EXPECT_TRUE(observer_.PopState()); + EXPECT_TRUE(observer_.IsEmpty()); +} + +} // namespace viz diff --git a/components/viz/service/surfaces/surface_manager.cc b/components/viz/service/surfaces/surface_manager.cc index 52a178272f40b6..06dc61090ebf82 100644 --- a/components/viz/service/surfaces/surface_manager.cc +++ b/components/viz/service/surfaces/surface_manager.cc @@ -612,4 +612,9 @@ bool SurfaceManager::IsMarkedForDestruction(const SurfaceId& surface_id) { return surfaces_to_destroy_.count(surface_id) != 0; } +void SurfaceManager::SurfaceWillBeDrawn(Surface* surface) { + for (auto& observer : observer_list_) + observer.OnSurfaceWillBeDrawn(surface); +} + } // namespace viz diff --git a/components/viz/service/surfaces/surface_manager.h b/components/viz/service/surfaces/surface_manager.h index fb648c2f15f725..237c14f9b34d92 100644 --- a/components/viz/service/surfaces/surface_manager.h +++ b/components/viz/service/surfaces/surface_manager.h @@ -128,6 +128,11 @@ class VIZ_SERVICE_EXPORT SurfaceManager { // possibly because a renderer process has crashed. void InvalidateFrameSinkId(const FrameSinkId& frame_sink_id); + const base::flat_map& valid_frame_sink_labels() + const { + return valid_frame_sink_labels_; + } + // Set |debug_label| of the |frame_sink_id|. |frame_sink_id| must exist in // |valid_frame_sink_labels_| already when UpdateFrameSinkDebugLabel is // called. @@ -189,6 +194,10 @@ class VIZ_SERVICE_EXPORT SurfaceManager { return lifetime_type_ == LifetimeType::REFERENCES; } + // Called by SurfaceAggregator notifying us that it will use |surface| in the + // next display frame. We will notify SurfaceObservers accordingly. + void SurfaceWillBeDrawn(Surface* surface); + private: friend class test::SurfaceSynchronizationTest; friend class test::SurfaceReferencesTest; diff --git a/components/viz/service/surfaces/surface_observer.h b/components/viz/service/surfaces/surface_observer.h index 2b7846838e725e..5e7f6e84c3112b 100644 --- a/components/viz/service/surfaces/surface_observer.h +++ b/components/viz/service/surfaces/surface_observer.h @@ -7,6 +7,7 @@ namespace viz { +class Surface; class SurfaceId; class SurfaceInfo; struct BeginFrameAck; @@ -46,6 +47,9 @@ class SurfaceObserver { // TODO(crbug.com/776098): This is only used in tests. We can probably remove // it. virtual void OnSurfaceSubtreeDamaged(const SurfaceId& surface_id) = 0; + + // Called whenever |surface| will be drawn in the next display frame. + virtual void OnSurfaceWillBeDrawn(Surface* surface) {} }; } // namespace viz diff --git a/components/viz/test/compositor_frame_helpers.cc b/components/viz/test/compositor_frame_helpers.cc index 20b89f57641b7d..fbd8a16180e7aa 100644 --- a/components/viz/test/compositor_frame_helpers.cc +++ b/components/viz/test/compositor_frame_helpers.cc @@ -9,10 +9,10 @@ namespace viz { namespace test { -CompositorFrame MakeCompositorFrame() { +CompositorFrame MakeCompositorFrame(const gfx::Size& size) { CompositorFrame frame = MakeEmptyCompositorFrame(); std::unique_ptr pass = RenderPass::Create(); - pass->SetNew(1, gfx::Rect(0, 0, 20, 20), gfx::Rect(), gfx::Transform()); + pass->SetNew(1, gfx::Rect(size), gfx::Rect(), gfx::Transform()); frame.render_pass_list.push_back(std::move(pass)); return frame; } diff --git a/components/viz/test/compositor_frame_helpers.h b/components/viz/test/compositor_frame_helpers.h index 7c7300d77e17d9..a3ac800e80d4c7 100644 --- a/components/viz/test/compositor_frame_helpers.h +++ b/components/viz/test/compositor_frame_helpers.h @@ -16,7 +16,7 @@ class CompositorFrame; namespace test { // Creates a valid CompositorFrame. -CompositorFrame MakeCompositorFrame(); +CompositorFrame MakeCompositorFrame(const gfx::Size& size = gfx::Size(20, 20)); // Creates a CompositorFrame that will be valid once its render_pass_list is // initialized. diff --git a/components/viz/test/test_frame_sink_manager.h b/components/viz/test/test_frame_sink_manager.h index 8a68190d70566f..205fca4c959f0a 100644 --- a/components/viz/test/test_frame_sink_manager.h +++ b/components/viz/test/test_frame_sink_manager.h @@ -45,6 +45,8 @@ class TestFrameSinkManagerImpl : public mojom::FrameSinkManager { void AssignTemporaryReference(const SurfaceId& surface_id, const FrameSinkId& owner) override {} void DropTemporaryReference(const SurfaceId& surface_id) override {} + void AddVideoDetectorObserver( + mojom::VideoDetectorObserverPtr observer) override {} mojo::Binding binding_; diff --git a/content/public/app/mojo/content_browser_manifest.json b/content/public/app/mojo/content_browser_manifest.json index 60aa6266fca9bb..61effa581a8234 100644 --- a/content/public/app/mojo/content_browser_manifest.json +++ b/content/public/app/mojo/content_browser_manifest.json @@ -90,7 +90,7 @@ "url_loader" ], "patch_service": [ "patch_file" ], - "ui": [ "display_output_protection" ], + "ui": [ "display_output_protection", "video_detector" ], "service_manager": [ "service_manager:client_process", "service_manager:instance_name", diff --git a/mojo/public/cpp/bindings/interface_ptr_set.h b/mojo/public/cpp/bindings/interface_ptr_set.h index 09a268229dd93f..4c4f6db51a415d 100644 --- a/mojo/public/cpp/bindings/interface_ptr_set.h +++ b/mojo/public/cpp/bindings/interface_ptr_set.h @@ -49,6 +49,8 @@ class PtrSet { ptrs_.clear(); } + bool empty() const { return ptrs_.empty(); } + private: class Element { public: diff --git a/services/ui/manifest.json b/services/ui/manifest.json index 5d6c0cd8e2311d..2c578fee39d87a 100644 --- a/services/ui/manifest.json +++ b/services/ui/manifest.json @@ -18,6 +18,9 @@ "ui::mojom::InputDeviceServer", "ui::mojom::WindowTreeFactory" ], + "video_detector": [ + "ui::mojom::VideoDetector" + ], // Interfaces provided by mus-gpu for mus-ws. "ozone": [ "ui::ozone::mojom::DeviceCursor", diff --git a/services/ui/public/interfaces/BUILD.gn b/services/ui/public/interfaces/BUILD.gn index 6a1202e60c9925..c85658a14dd274 100644 --- a/services/ui/public/interfaces/BUILD.gn +++ b/services/ui/public/interfaces/BUILD.gn @@ -16,6 +16,7 @@ mojom("interfaces") { "remote_event_dispatcher.mojom", "user_access_manager.mojom", "user_activity_monitor.mojom", + "video_detector.mojom", "window_manager.mojom", "window_manager_constants.mojom", "window_manager_window_tree_factory.mojom", diff --git a/services/ui/public/interfaces/video_detector.mojom b/services/ui/public/interfaces/video_detector.mojom new file mode 100644 index 00000000000000..bdaf84bdb99bd7 --- /dev/null +++ b/services/ui/public/interfaces/video_detector.mojom @@ -0,0 +1,13 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +module ui.mojom; + +import "services/viz/public/interfaces/compositing/video_detector_observer.mojom"; + +// This is a priviledged interface allowing clients to get notified when video +// activity is detected. See viz::VideoDetector. +interface VideoDetector { + AddObserver(viz.mojom.VideoDetectorObserver observer); +}; diff --git a/services/ui/service.cc b/services/ui/service.cc index f5b71323768192..394b9079e61c43 100644 --- a/services/ui/service.cc +++ b/services/ui/service.cc @@ -325,6 +325,8 @@ void Service::OnStart() { } registry_.AddInterface(base::Bind( &Service::BindRemoteEventDispatcherRequest, base::Unretained(this))); + registry_.AddInterface( + base::Bind(&Service::BindVideoDetectorRequest, base::Unretained(this))); // On non-Linux platforms there will be no DeviceDataManager instance and no // purpose in adding the Mojo interface to connect to. @@ -542,4 +544,8 @@ void Service::BindRemoteEventDispatcherRequest( std::move(request)); } +void Service::BindVideoDetectorRequest(mojom::VideoDetectorRequest request) { + window_server_->video_detector()->AddBinding(std::move(request)); +} + } // namespace ui diff --git a/services/ui/service.h b/services/ui/service.h index 3283d16b32ed55..fed9b2d1677383 100644 --- a/services/ui/service.h +++ b/services/ui/service.h @@ -31,6 +31,7 @@ #include "services/ui/public/interfaces/remote_event_dispatcher.mojom.h" #include "services/ui/public/interfaces/user_access_manager.mojom.h" #include "services/ui/public/interfaces/user_activity_monitor.mojom.h" +#include "services/ui/public/interfaces/video_detector.mojom.h" #include "services/ui/public/interfaces/window_manager_window_tree_factory.mojom.h" #include "services/ui/public/interfaces/window_server_test.mojom.h" #include "services/ui/public/interfaces/window_tree.mojom.h" @@ -178,6 +179,8 @@ class Service : public service_manager::Service, void BindRemoteEventDispatcherRequest( mojom::RemoteEventDispatcherRequest request); + void BindVideoDetectorRequest(mojom::VideoDetectorRequest request); + std::unique_ptr window_server_; std::unique_ptr event_source_; using PendingRequests = std::vector>; diff --git a/services/ui/ws/BUILD.gn b/services/ui/ws/BUILD.gn index 02a621eebe82de..0e5bdbbf09ee73 100644 --- a/services/ui/ws/BUILD.gn +++ b/services/ui/ws/BUILD.gn @@ -95,6 +95,8 @@ static_library("lib") { "user_id_tracker.cc", "user_id_tracker.h", "user_id_tracker_observer.h", + "video_detector_impl.cc", + "video_detector_impl.h", "window_coordinate_conversions.cc", "window_coordinate_conversions.h", "window_finder.cc", diff --git a/services/ui/ws/video_detector_impl.cc b/services/ui/ws/video_detector_impl.cc new file mode 100644 index 00000000000000..e74d2e1edd793f --- /dev/null +++ b/services/ui/ws/video_detector_impl.cc @@ -0,0 +1,28 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "services/ui/ws/video_detector_impl.h" + +#include "components/viz/host/host_frame_sink_manager.h" + +namespace ui { +namespace ws { + +VideoDetectorImpl::VideoDetectorImpl( + viz::HostFrameSinkManager* host_frame_sink_manager) + : host_frame_sink_manager_(host_frame_sink_manager) {} + +VideoDetectorImpl::~VideoDetectorImpl() = default; + +void VideoDetectorImpl::AddBinding(mojom::VideoDetectorRequest request) { + binding_set_.AddBinding(this, std::move(request)); +} + +void VideoDetectorImpl::AddObserver( + viz::mojom::VideoDetectorObserverPtr observer) { + host_frame_sink_manager_->AddVideoDetectorObserver(std::move(observer)); +} + +} // namespace ws +} // namespace ui diff --git a/services/ui/ws/video_detector_impl.h b/services/ui/ws/video_detector_impl.h new file mode 100644 index 00000000000000..77170fd71281f0 --- /dev/null +++ b/services/ui/ws/video_detector_impl.h @@ -0,0 +1,44 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef SERVICES_UI_WS_VIDEO_DETECTOR_IMPL_H_ +#define SERVICES_UI_WS_VIDEO_DETECTOR_IMPL_H_ + +#include "mojo/public/cpp/bindings/binding_set.h" +#include "services/ui/public/interfaces/video_detector.mojom.h" +#include "services/viz/public/interfaces/compositing/video_detector_observer.mojom.h" + +namespace viz { +class HostFrameSinkManager; +} + +namespace ui { +namespace ws { + +// WS-side representation of viz::VideoDetector. Internally forwards observer +// requests to viz. +// TODO(crbug.com/780514): If we restart viz the observers won't receive +// notifications anymore. This needs to be fixed. +class VideoDetectorImpl : public mojom::VideoDetector { + public: + explicit VideoDetectorImpl( + viz::HostFrameSinkManager* host_frame_sink_manager); + ~VideoDetectorImpl() override; + + void AddBinding(mojom::VideoDetectorRequest request); + + // mojom::VideoDetector implementation. + void AddObserver(viz::mojom::VideoDetectorObserverPtr observer) override; + + private: + viz::HostFrameSinkManager* host_frame_sink_manager_; + mojo::BindingSet binding_set_; + + DISALLOW_COPY_AND_ASSIGN(VideoDetectorImpl); +}; + +} // namespace ws +} // namespace ui + +#endif // SERVICES_UI_WS_VIDEO_DETECTOR_IMPL_H_ diff --git a/services/ui/ws/window_server.cc b/services/ui/ws/window_server.cc index b4a43aa41724f5..9a650a95a89002 100644 --- a/services/ui/ws/window_server.cc +++ b/services/ui/ws/window_server.cc @@ -68,6 +68,7 @@ WindowServer::WindowServer(WindowServerDelegate* delegate) next_wm_change_id_(0), window_manager_window_tree_factory_set_(this, &user_id_tracker_), host_frame_sink_manager_(base::MakeUnique()), + video_detector_(host_frame_sink_manager_.get()), display_creation_config_(DisplayCreationConfig::UNKNOWN) { user_id_tracker_.AddObserver(this); OnUserIdAdded(user_id_tracker_.active_id()); diff --git a/services/ui/ws/window_server.h b/services/ui/ws/window_server.h index a3f7dae314e55a..a7d69553acc964 100644 --- a/services/ui/ws/window_server.h +++ b/services/ui/ws/window_server.h @@ -28,6 +28,7 @@ #include "services/ui/ws/user_display_manager_delegate.h" #include "services/ui/ws/user_id_tracker.h" #include "services/ui/ws/user_id_tracker_observer.h" +#include "services/ui/ws/video_detector_impl.h" #include "services/ui/ws/window_manager_window_tree_factory_set.h" namespace ui { @@ -246,6 +247,8 @@ class WindowServer : public ServerWindowDelegate, void OnNoMoreDisplays(); WindowManagerState* GetWindowManagerStateForUser(const UserId& user_id); + VideoDetectorImpl* video_detector() { return &video_detector_; } + // ServerWindowDelegate: viz::HostFrameSinkManager* GetHostFrameSinkManager() override; void OnFirstSurfaceActivation(const viz::SurfaceInfo& surface_info, @@ -410,6 +413,8 @@ class WindowServer : public ServerWindowDelegate, // Provides interfaces to create and manage FrameSinks. std::unique_ptr host_frame_sink_manager_; + VideoDetectorImpl video_detector_; + // System modal windows not attached to a display are added here. Once // attached to a display they are removed. ServerWindowTracker pending_system_modal_windows_; diff --git a/services/viz/privileged/interfaces/compositing/frame_sink_manager.mojom b/services/viz/privileged/interfaces/compositing/frame_sink_manager.mojom index 66d6c48cea7ddb..6dc4f75c883748 100644 --- a/services/viz/privileged/interfaces/compositing/frame_sink_manager.mojom +++ b/services/viz/privileged/interfaces/compositing/frame_sink_manager.mojom @@ -14,6 +14,7 @@ import "services/viz/public/interfaces/compositing/local_surface_id.mojom"; import "services/viz/public/interfaces/compositing/surface_id.mojom"; import "services/viz/public/interfaces/compositing/surface_info.mojom"; import "ui/gfx/geometry/mojo/geometry.mojom"; +import "services/viz/public/interfaces/compositing/video_detector_observer.mojom"; // The FrameSinkManager interface is a privileged interface that allows the // frame sink manager host (browser or window server) to create @@ -82,6 +83,10 @@ interface FrameSinkManager { // Drops the temporary reference for |surface_id|. This will get called when // the FrameSinkManagerClient doesn't think |surface_id| will be embedded. DropTemporaryReference(SurfaceId surface_id); + + // Requests viz to notify |observer| whenever video activity is detected in + // one of the clients. See viz::VideoDetector. + AddVideoDetectorObserver(VideoDetectorObserver observer); }; // The FrameSinkManagerClient interface is implemented by the Display diff --git a/services/viz/public/interfaces/BUILD.gn b/services/viz/public/interfaces/BUILD.gn index f43a882d10cfad..8a6b101d841dee 100644 --- a/services/viz/public/interfaces/BUILD.gn +++ b/services/viz/public/interfaces/BUILD.gn @@ -30,6 +30,7 @@ mojom("interfaces") { "compositing/texture_mailbox.mojom", "compositing/texture_mailbox_releaser.mojom", "compositing/transferable_resource.mojom", + "compositing/video_detector_observer.mojom", "constants.mojom", "hit_test/hit_test_region_list.mojom", ] diff --git a/services/viz/public/interfaces/compositing/video_detector_observer.mojom b/services/viz/public/interfaces/compositing/video_detector_observer.mojom new file mode 100644 index 00000000000000..7850fb10cfda67 --- /dev/null +++ b/services/viz/public/interfaces/compositing/video_detector_observer.mojom @@ -0,0 +1,11 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +module viz.mojom; + +// Observers of viz::VideoDetector implement this interface. +interface VideoDetectorObserver { + OnVideoActivityStarted(); + OnVideoActivityEnded(); +};