Add new timeout finch for the GPU watchdog V2

Bug:949839

Change-Id: Id84825c9b13a800c80a22ed3bf42b4056bc16343
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2084718
Commit-Queue: Maggie Chen <magchen@chromium.org>
Reviewed-by: Zhenyao Mo <zmo@chromium.org>
Cr-Commit-Position: refs/heads/master@{#748710}

gpu/config/gpu_finch_features.cc

@@ -70,6 +70,10 @@ const base::Feature kGpuWatchdogV2{"GpuWatchdogV2",
 const base::Feature kGpuWatchdogV1NewTimeout{"GpuWatchdogV1NewTimeout",
                                              base::FEATURE_ENABLED_BY_DEFAULT};
 
+// Use a different set of watchdog timeouts on V2
+const base::Feature kGpuWatchdogV2NewTimeout{"GpuWatchdogV2NewTimeout",
+                                             base::FEATURE_DISABLED_BY_DEFAULT};
+
 #if defined(OS_MACOSX)
 // Enable use of Metal for OOP rasterization.
 const base::Feature kMetal{"Metal", base::FEATURE_DISABLED_BY_DEFAULT};

gpu/config/gpu_finch_features.h

@@ -36,6 +36,8 @@ GPU_EXPORT extern const base::Feature kGpuWatchdogV2;
 
 GPU_EXPORT extern const base::Feature kGpuWatchdogV1NewTimeout;
 
+GPU_EXPORT extern const base::Feature kGpuWatchdogV2NewTimeout;
+
 #if defined(OS_MACOSX)
 GPU_EXPORT extern const base::Feature kMetal;
 #endif

gpu/ipc/service/gpu_watchdog_thread.h

@@ -53,8 +53,14 @@ enum class GpuWatchdogTimeoutEvent {
   kProgressAfterWait,
   // Just continue if it's not on the TTY of our host X11 server.
   kContinueOnNonHostServerTty,
-
-  kMaxValue = kContinueOnNonHostServerTty,
+  // Windows only: After detecting GPU hang and continuing running through
+  // OnGpuWatchdogTimeout for the max cycles, the GPU main thread still cannot
+  // get the full thread time.
+  kLessThanFullThreadTimeAfterCapped,
+  // Windows only: The GPU main thread went through the
+  // kLessThanFullThreadTimeAfterCapped stage before the process is killed.
+  kKillOnLessThreadTime,
+  kMaxValue = kKillOnLessThreadTime,
 };
 
 // A thread that intermitently sends tasks to a group of watched message loops

gpu/ipc/service/gpu_watchdog_thread_unittest.cc

@@ -20,6 +20,8 @@ namespace {
 constexpr auto kGpuWatchdogTimeoutForTesting =
     base::TimeDelta::FromMilliseconds(1000);
 
+constexpr int kMaxExtraCyclesBeforeKillForTesting = 1;
+
 // This task will run for duration_ms milliseconds.
 void SimpleTask(base::TimeDelta duration) {
   base::PlatformThread::Sleep(duration);
@@ -71,6 +73,7 @@ void GpuWatchdogTest::SetUp() {
   watchdog_thread_ = gpu::GpuWatchdogThreadImplV2::Create(
       /*start_backgrounded*/ false,
       /*timeout*/ kGpuWatchdogTimeoutForTesting,
+      /*max_extra_cycles_before_kill*/ kMaxExtraCyclesBeforeKillForTesting,
       /*test_mode*/ true);
 }
 
@@ -139,12 +142,15 @@ void GpuWatchdogPowerTest::LongTaskOnResume(
 TEST_F(GpuWatchdogTest, GpuInitializationHang) {
   // GPU init takes longer than timeout.
 #if defined(OS_WIN)
-  SimpleTask(kGpuWatchdogTimeoutForTesting * kInitFactor +
-             kGpuWatchdogTimeoutForTesting *
-                 kMaxCountOfMoreGpuThreadTimeAllowed +
-             base::TimeDelta::FromMilliseconds(3000));
+  SimpleTask(
+      kGpuWatchdogTimeoutForTesting * kInitFactor +
+      kGpuWatchdogTimeoutForTesting * kMaxCountOfMoreGpuThreadTimeAllowed +
+      kGpuWatchdogTimeoutForTesting * kMaxExtraCyclesBeforeKillForTesting +
+      base::TimeDelta::FromMilliseconds(3000));
 #else
   SimpleTask(kGpuWatchdogTimeoutForTesting * kInitFactor +
+             kGpuWatchdogTimeoutForTesting *
+                 kMaxExtraCyclesBeforeKillForTesting +
              base::TimeDelta::FromMilliseconds(3000));
 #endif
 
@@ -199,13 +205,15 @@ TEST_F(GpuWatchdogTest, GpuRunningATaskHang) {
       base::BindOnce(&SimpleTask, kGpuWatchdogTimeoutForTesting * 2 +
                                       kGpuWatchdogTimeoutForTesting *
                                           kMaxCountOfMoreGpuThreadTimeAllowed +
-
+                                      kGpuWatchdogTimeoutForTesting *
+                                          kMaxExtraCyclesBeforeKillForTesting +
                                       base::TimeDelta::FromMilliseconds(4000)));
 #else
   task_environment_.GetMainThreadTaskRunner()->PostTask(
       FROM_HERE,
       base::BindOnce(&SimpleTask, kGpuWatchdogTimeoutForTesting * 2 +
-
+                                      kGpuWatchdogTimeoutForTesting *
+                                          kMaxExtraCyclesBeforeKillForTesting +
                                       base::TimeDelta::FromMilliseconds(4000)));
 #endif
 
@@ -258,7 +266,8 @@ TEST_F(GpuWatchdogTest, GpuSwitchingToForegroundHang) {
                      /*duration*/ kGpuWatchdogTimeoutForTesting * 2 +
                          kGpuWatchdogTimeoutForTesting *
                              kMaxCountOfMoreGpuThreadTimeAllowed +
-
+                         kGpuWatchdogTimeoutForTesting *
+                             kMaxExtraCyclesBeforeKillForTesting +
                          base::TimeDelta::FromMilliseconds(4200),
                      /*time_to_switch_to_foreground*/
                      base::TimeDelta::FromMilliseconds(200)));
@@ -268,7 +277,8 @@ TEST_F(GpuWatchdogTest, GpuSwitchingToForegroundHang) {
       base::BindOnce(&GpuWatchdogTest::LongTaskFromBackgroundToForeground,
                      base::Unretained(this),
                      /*duration*/ kGpuWatchdogTimeoutForTesting * 2 +
-
+                         kGpuWatchdogTimeoutForTesting *
+                             kMaxExtraCyclesBeforeKillForTesting +
                          base::TimeDelta::FromMilliseconds(4200),
                      /*time_to_switch_to_foreground*/
                      base::TimeDelta::FromMilliseconds(200)));
@@ -301,12 +311,15 @@ TEST_F(GpuWatchdogTest, GpuInitializationPause) {
   watchdog_thread_->ResumeWatchdog();
   // The Gpu init continues for (init timeout + 4000) ms.
 #if defined(OS_WIN)
-  SimpleTask(kGpuWatchdogTimeoutForTesting * kInitFactor +
-             kGpuWatchdogTimeoutForTesting *
-                 kMaxCountOfMoreGpuThreadTimeAllowed +
-             base::TimeDelta::FromMilliseconds(4000));
+  SimpleTask(
+      kGpuWatchdogTimeoutForTesting * kInitFactor +
+      kGpuWatchdogTimeoutForTesting * kMaxCountOfMoreGpuThreadTimeAllowed +
+      kGpuWatchdogTimeoutForTesting * kMaxExtraCyclesBeforeKillForTesting +
+      base::TimeDelta::FromMilliseconds(4000));
 #else
   SimpleTask(kGpuWatchdogTimeoutForTesting * kInitFactor +
+             kGpuWatchdogTimeoutForTesting *
+                 kMaxExtraCyclesBeforeKillForTesting +
              base::TimeDelta::FromMilliseconds(4000));
 #endif
 
@@ -350,6 +363,8 @@ TEST_F(GpuWatchdogPowerTest, GpuOnResumeHang) {
           /*duration*/ kGpuWatchdogTimeoutForTesting * kRestartFactor +
               kGpuWatchdogTimeoutForTesting *
                   kMaxCountOfMoreGpuThreadTimeAllowed +
+              kGpuWatchdogTimeoutForTesting *
+                  kMaxExtraCyclesBeforeKillForTesting +
               base::TimeDelta::FromMilliseconds(4200),
           /*time_to_power_resume*/
           base::TimeDelta::FromMilliseconds(200)));
@@ -359,6 +374,8 @@ TEST_F(GpuWatchdogPowerTest, GpuOnResumeHang) {
       base::BindOnce(
           &GpuWatchdogPowerTest::LongTaskOnResume, base::Unretained(this),
           /*duration*/ kGpuWatchdogTimeoutForTesting * kRestartFactor +
+              kGpuWatchdogTimeoutForTesting *
+                  kMaxExtraCyclesBeforeKillForTesting +
               base::TimeDelta::FromMilliseconds(4200),
           /*time_to_power_resume*/
           base::TimeDelta::FromMilliseconds(200)));