Add (heavily-filtered) pixel test cq job for interactive_ui_tests

Currently, there are a number of tests which cannot be run as part of browser_tests because they flake or fail to run on some platforms and must be run as part of interactive_ui_tests. These include, e.g. a number of IPH tests, but also a number of manual tests we intend to automate as part of an ongoing test automation project. If a test for a secondary UI surface will not reliably run in browser_tests, it is currently not possible to do Skia Gold visual regression testing on that surface, as currently only browser_tests is ever run with Skia Gold support. This is done as pixel_browser_tests, in which the browser_tests suite is re-run with a very restrictive filter (basically, only tests that actually take screenshots) so that only the bare minimum of time and resources are consumed. There was no equivalent for interactive_ui_tests, even though it should be fairly straightforward to create one. This CL adds a similar variation on interactive_ui_tests called pixel_interactive_ui_tests, which invokes interactive_ui_tests with Skia Gold support. It mimics pixel_browser_tests in the following ways: - Shares the same filter, so that all pixel tests can be listed in the same place and no non-pixel tests are run - Runs on a limited set of platforms - Does not retry failed tests Note that any tests included in pixel_interactive_ui_tests will still run in interactive_ui_tests with screenshotting as a no-op, just as with pixel_browser_tests and browser_tests. This ensures that any behavioral regressions on other platforms are still caught by CQ. Initially, this only has the effect of re-enabling the IPH pixel test suite, which was moved to interactive_ui_tests because it would not run reliably in browser_tests (due to its sensitivity to window focus/activation.) Bug: 1319632 Change-Id: I4fc44b404ca66fa1c82ea0873d80fedceac97fc5 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3606835 Reviewed-by: Erik Staab <estaab@chromium.org> Commit-Queue: Dana Fried <dfried@chromium.org> Cr-Commit-Position: refs/heads/main@{#996361}
Denger-Network · Apr 26, 2022 · 86d6e51 · 86d6e51
1 parent b710cef
commit 86d6e51
Show file tree

Hide file tree

Showing 6 changed files with 204 additions and 72 deletions.
diff --git a/testing/buildbot/chromium.fyi.json b/testing/buildbot/chromium.fyi.json
@@ -9428,7 +9428,7 @@
         "args": [
           "--browser-ui-tests-verify-pixels",
           "--enable-pixel-output-in-tests",
-          "--test-launcher-filter-file=../../testing/buildbot/filters/pixel_browser_tests.filter",
+          "--test-launcher-filter-file=../../testing/buildbot/filters/pixel_tests.filter",
           "--git-revision=${got_revision}"
         ],
         "isolate_profile_data": true,
@@ -10186,6 +10186,37 @@
         "test": "interactive_ui_tests",
         "test_id_prefix": "ninja://chrome/test:interactive_ui_tests/"
       },
+      {
+        "args": [
+          "--browser-ui-tests-verify-pixels",
+          "--enable-pixel-output-in-tests",
+          "--test-launcher-filter-file=../../testing/buildbot/filters/pixel_tests.filter",
+          "--git-revision=${got_revision}"
+        ],
+        "isolate_profile_data": true,
+        "merge": {
+          "args": [],
+          "script": "//testing/merge_scripts/standard_gtest_merge.py"
+        },
+        "name": "pixel_interactive_ui_tests",
+        "precommit_args": [
+          "--gerrit-issue=${patch_issue}",
+          "--gerrit-patchset=${patch_set}",
+          "--buildbucket-id=${buildbucket_build_id}"
+        ],
+        "swarming": {
+          "can_use_on_swarming_builders": true,
+          "dimension_sets": [
+            {
+              "cpu": "x86-64",
+              "os": "Windows-11-22000"
+            }
+          ],
+          "service_account": "chromium-tester@chops-service-accounts.iam.gserviceaccount.com"
+        },
+        "test": "interactive_ui_tests",
+        "test_id_prefix": "ninja://chrome/test:interactive_ui_tests/"
+      },
       {
         "isolate_profile_data": true,
         "merge": {
@@ -104417,7 +104448,7 @@
         "args": [
           "--browser-ui-tests-verify-pixels",
           "--enable-pixel-output-in-tests",
-          "--test-launcher-filter-file=../../testing/buildbot/filters/pixel_browser_tests.filter",
+          "--test-launcher-filter-file=../../testing/buildbot/filters/pixel_tests.filter",
           "--git-revision=${got_revision}"
         ],
         "merge": {
@@ -104444,6 +104475,38 @@
         },
         "test": "browser_tests",
         "test_id_prefix": "ninja://chrome/test:browser_tests/"
+      },
+      {
+        "args": [
+          "--browser-ui-tests-verify-pixels",
+          "--enable-pixel-output-in-tests",
+          "--test-launcher-filter-file=../../testing/buildbot/filters/pixel_tests.filter",
+          "--git-revision=${got_revision}"
+        ],
+        "merge": {
+          "args": [],
+          "script": "//testing/merge_scripts/standard_gtest_merge.py"
+        },
+        "name": "pixel_interactive_ui_tests",
+        "non_precommit_args": [
+          "--test-launcher-retry-limit=0"
+        ],
+        "precommit_args": [
+          "--gerrit-issue=${patch_issue}",
+          "--gerrit-patchset=${patch_set}",
+          "--buildbucket-id=${buildbucket_build_id}"
+        ],
+        "swarming": {
+          "can_use_on_swarming_builders": true,
+          "dimension_sets": [
+            {
+              "os": "Windows-10-19042"
+            }
+          ],
+          "service_account": "chromium-tester@chops-service-accounts.iam.gserviceaccount.com"
+        },
+        "test": "interactive_ui_tests",
+        "test_id_prefix": "ninja://chrome/test:interactive_ui_tests/"
       }
     ]
   },

diff --git a/testing/buildbot/chromium.win.json b/testing/buildbot/chromium.win.json
@@ -2378,7 +2378,7 @@
         "args": [
           "--browser-ui-tests-verify-pixels",
           "--enable-pixel-output-in-tests",
-          "--test-launcher-filter-file=../../testing/buildbot/filters/pixel_browser_tests.filter",
+          "--test-launcher-filter-file=../../testing/buildbot/filters/pixel_tests.filter",
           "--git-revision=${got_revision}"
         ],
         "isolate_profile_data": true,
@@ -3138,6 +3138,37 @@
         "test": "interactive_ui_tests",
         "test_id_prefix": "ninja://chrome/test:interactive_ui_tests/"
       },
+      {
+        "args": [
+          "--browser-ui-tests-verify-pixels",
+          "--enable-pixel-output-in-tests",
+          "--test-launcher-filter-file=../../testing/buildbot/filters/pixel_tests.filter",
+          "--git-revision=${got_revision}"
+        ],
+        "isolate_profile_data": true,
+        "merge": {
+          "args": [],
+          "script": "//testing/merge_scripts/standard_gtest_merge.py"
+        },
+        "name": "pixel_interactive_ui_tests",
+        "precommit_args": [
+          "--gerrit-issue=${patch_issue}",
+          "--gerrit-patchset=${patch_set}",
+          "--buildbucket-id=${buildbucket_build_id}"
+        ],
+        "swarming": {
+          "can_use_on_swarming_builders": true,
+          "dimension_sets": [
+            {
+              "cpu": "x86-64",
+              "os": "Windows-10-19042"
+            }
+          ],
+          "service_account": "chromium-tester@chops-service-accounts.iam.gserviceaccount.com"
+        },
+        "test": "interactive_ui_tests",
+        "test_id_prefix": "ninja://chrome/test:interactive_ui_tests/"
+      },
       {
         "isolate_profile_data": true,
         "merge": {

diff --git a/testing/buildbot/filters/BUILD.gn b/testing/buildbot/filters/BUILD.gn
@@ -72,7 +72,7 @@ source_set("browser_tests_filters") {
     "//testing/buildbot/filters/mac.mac11-arm64-rel.browser_tests.filter",
     "//testing/buildbot/filters/mac.mac-rel.browser_tests.filter",
     "//testing/buildbot/filters/ozone-linux.wayland_browser_tests.filter",
-    "//testing/buildbot/filters/pixel_browser_tests.filter",
+    "//testing/buildbot/filters/pixel_tests.filter",
     "//testing/buildbot/filters/webrtc_functional.browser_tests.filter",
     "//testing/buildbot/filters/win_backuprefptr_fyi.browser_tests.filter",
   ]
@@ -300,6 +300,7 @@ source_set("interactive_ui_tests_filters") {
   data = [
     "//testing/buildbot/filters/linux-lacros.interactive_ui_tests.filter",
     "//testing/buildbot/filters/ozone-linux.interactive_ui_tests_wayland.filter",
+    "//testing/buildbot/filters/pixel_tests.filter",
   ]
 }
 

diff --git a/...ildbot/filters/pixel_browser_tests.filter → testing/buildbot/filters/pixel_tests.filter b/...ildbot/filters/pixel_browser_tests.filter → testing/buildbot/filters/pixel_tests.filter
@@ -1,67 +1,79 @@
-# This is the filter for browser_tests that supports pixel tests.
-AccuracyTipBubbleViewDialogBrowserTest.*
-AppInfoDialogBrowserTest.*
-AskGoogleForSuggestionsDialogTest.*
-BookmarkBubbleViewBrowserTest.*
-BookmarkEditorViewBrowserTest.*
-ChromeLabsUiTest.*
-ConfirmBubbleTest.*
-ContentAnalysysDialogUiTest.*
-ContentSettingBubbleDialogTest.*
-CookieControlsBubbleViewTest.*
-CryptoModulePasswordDialogTest.*
-DeepScanningFailureModalDialogTest.*
-*DiceWebSigninInterceptionBubblePixelTest.InvokeUi_*
-*DiceWebSigninInterceptionBubbleSyncPromoPixelTest.InvokeUi_*
-ExtensionInstallDialogViewInteractiveBrowserTest.*
-ExtensionUninstallDialogViewInteractiveBrowserTest.*
-*EyeDropperBrowserTest.*
-FeaturePromoDialogTest.*
-FileSystemAccessUsageBubbleViewTest.*
-FirstRunDialogTest.*
-GlobalErrorBubbleTest.*
-HatsBubbleTest.*
-HungRendererDialogViewBrowserTest.*
-ImportLockDialogViewBrowserTest.*
-InlineLoginHelperBrowserTest.InvokeUi_*
-LocalCardMigrationBrowserUiTest.*
-NewTabPageTest.*
-OneTimePermissionPromptBubbleViewBrowserTest.*
-OutdatedUpgradeBubbleTest.*
-PageInfoBubbleViewAboutThisSiteDialogBrowserTest.*
-PageInfoBubbleViewDialogBrowserTest.*
-PageInfoBubbleViewPrivacySandboxDialogBrowserTest.*
-PageInfoBubbleViewHistoryDialogBrowserTest.*
-*PasswordBubbleBrowserTest.*
-PasswordReuseModalWarningTest.*
-PermissionRequestChipDialogBrowserTest.*
-*PermissionPromptBubbleViewBrowserTest.*
-PrivacySandboxDialogViewBrowserTest.*
-PromptForScanningModalDialogTest.*
-QRCodeGeneratorBubbleBrowserTest.*
-RelaunchRecommendedBubbleViewDialogTest.*
-RelaunchRequiredDialogViewDialogTest.*
-ReopenTabPromoControllerDialogBrowserTest.*
-SafetyTipPageInfoBubbleViewDialogTest.*
-ScreenCaptureNotificationUiBrowserTest.*
-SecurePaymentConfirmationDialogViewTest.InvokeUi_*
-*SendTabToSelfBubbleTest.*
-SessionCrashedBubbleViewTest.*
-TabGroupEditorBubbleViewDialogBrowserTest.*
-TabHoverCardBubbleViewBrowserTest.*
-UpdateRecommendedDialogTest.*
-WebAppConfirmViewBrowserTest.*
-ZoomBubbleDialogTest.*
-
-# This test uses random network port and shows it on ui.
--ContentSettingBubbleDialogTest.InvokeUi_popups
--OutdatedUpgradeBubbleTest.InvokeUi_Critical
-# TODO(crbug.com/1108988): Fix this flakiness test.
--BookmarkBubbleViewBrowserTest.InvokeUi_bookmark_details_signed_in
-
-# TODO(crbug.com/2666206): Flaky
--ContentSettingBubbleDialogTest.InvokeUi_mediastream_camera
--ContentSettingBubbleDialogTest.InvokeUi_mediastream_mic_and_camera
-
-# TODO(crbug.com/1164574): Flaky
--ContentSettingBubbleDialogTest.InvokeUi_mediastream_mic
+# This is the filter for browser_tests and interactive_ui_tests that support
+# pixel tests. In order for pixel tests to work, you *must* include your test
+# here.
+#
+# Since these tests will all run in both browser_tests (or interactive_ui_tests)
+# and pixel_browser_tests (or pixel_interactive_ui_tests), avoid including
+# non-pixel tests here.
+#
+# Prefer browser_tests to interactive_ui_tests, as they are less expensive to
+# run; only use interactive_ui_tests if running your test in browser_tests
+# causes flakes (due to widget activation, occlusion, etc.)
+
+AccuracyTipBubbleViewDialogBrowserTest.*
+AppInfoDialogBrowserTest.*
+AskGoogleForSuggestionsDialogTest.*
+BookmarkBubbleViewBrowserTest.*
+BookmarkEditorViewBrowserTest.*
+ChromeLabsUiTest.*
+ConfirmBubbleTest.*
+ContentAnalysysDialogUiTest.*
+ContentSettingBubbleDialogTest.*
+CookieControlsBubbleViewTest.*
+CryptoModulePasswordDialogTest.*
+DeepScanningFailureModalDialogTest.*
+*DiceWebSigninInterceptionBubblePixelTest.InvokeUi_*
+*DiceWebSigninInterceptionBubbleSyncPromoPixelTest.InvokeUi_*
+ExtensionInstallDialogViewInteractiveBrowserTest.*
+ExtensionUninstallDialogViewInteractiveBrowserTest.*
+*EyeDropperBrowserTest.*
+FeaturePromoDialogIntentChipTest.InvokeUi_*
+FeaturePromoDialogTest.InvokeUi_*
+FileSystemAccessUsageBubbleViewTest.*
+FirstRunDialogTest.*
+GlobalErrorBubbleTest.*
+HatsBubbleTest.*
+HungRendererDialogViewBrowserTest.*
+ImportLockDialogViewBrowserTest.*
+InlineLoginHelperBrowserTest.InvokeUi_*
+LocalCardMigrationBrowserUiTest.*
+NewTabPageTest.*
+OneTimePermissionPromptBubbleViewBrowserTest.*
+OutdatedUpgradeBubbleTest.*
+PageInfoBubbleViewAboutThisSiteDialogBrowserTest.*
+PageInfoBubbleViewDialogBrowserTest.*
+PageInfoBubbleViewPrivacySandboxDialogBrowserTest.*
+PageInfoBubbleViewHistoryDialogBrowserTest.*
+*PasswordBubbleBrowserTest.*
+PasswordReuseModalWarningTest.*
+PermissionRequestChipDialogBrowserTest.*
+*PermissionPromptBubbleViewBrowserTest.*
+PrivacySandboxDialogViewBrowserTest.*
+PromptForScanningModalDialogTest.*
+QRCodeGeneratorBubbleBrowserTest.*
+RelaunchRecommendedBubbleViewDialogTest.*
+RelaunchRequiredDialogViewDialogTest.*
+ReopenTabPromoControllerDialogBrowserTest.*
+SafetyTipPageInfoBubbleViewDialogTest.*
+ScreenCaptureNotificationUiBrowserTest.*
+SecurePaymentConfirmationDialogViewTest.InvokeUi_*
+*SendTabToSelfBubbleTest.*
+SessionCrashedBubbleViewTest.*
+TabGroupEditorBubbleViewDialogBrowserTest.*
+TabHoverCardBubbleViewBrowserTest.*
+UpdateRecommendedDialogTest.*
+WebAppConfirmViewBrowserTest.*
+ZoomBubbleDialogTest.*
+
+# This test uses random network port and shows it on ui.
+-ContentSettingBubbleDialogTest.InvokeUi_popups
+-OutdatedUpgradeBubbleTest.InvokeUi_Critical
+# TODO(crbug.com/1108988): Fix this flakiness test.
+-BookmarkBubbleViewBrowserTest.InvokeUi_bookmark_details_signed_in
+
+# TODO(crbug.com/2666206): Flaky
+-ContentSettingBubbleDialogTest.InvokeUi_mediastream_camera
+-ContentSettingBubbleDialogTest.InvokeUi_mediastream_mic_and_camera
+
+# TODO(crbug.com/1164574): Flaky
+-ContentSettingBubbleDialogTest.InvokeUi_mediastream_mic
diff --git a/testing/buildbot/test_suite_exceptions.pyl b/testing/buildbot/test_suite_exceptions.pyl
@@ -2581,6 +2581,19 @@
       },
     },
   },
+  'pixel_interactive_ui_tests': {
+    'modifications': {
+      'win-pixel-tester-rel': {
+        'non_precommit_args': [
+          # Do not allow retry or it will break the bulk approval process.
+          # When retry with fail-pass pattern, the passing test will overwrite the previously
+          # seen flaky version on the trace. We can triage the image using the printed link,
+          # but it will not show on the Skia Gold search page.
+          '--test-launcher-retry-limit=0',
+        ],
+      },
+    },
+  },
   'pixel_skia_gold_passthrough_test': {
     'modifications': {
       'Android FYI Release (Pixel 4)': {

diff --git a/testing/buildbot/test_suites.pyl b/testing/buildbot/test_suites.pyl
@@ -4615,13 +4615,25 @@
         'args': [
           '--browser-ui-tests-verify-pixels',
           '--enable-pixel-output-in-tests',
-          '--test-launcher-filter-file=../../testing/buildbot/filters/pixel_browser_tests.filter',
+          '--test-launcher-filter-file=../../testing/buildbot/filters/pixel_tests.filter',
         ],
         'test': 'browser_tests',
         'mixins': [
           'skia_gold_test',
         ],
       },
+      'pixel_interactive_ui_tests': {
+        'name': 'pixel_interactive_ui_tests',
+        'args': [
+          '--browser-ui-tests-verify-pixels',
+          '--enable-pixel-output-in-tests',
+          '--test-launcher-filter-file=../../testing/buildbot/filters/pixel_tests.filter',
+        ],
+        'test': 'interactive_ui_tests',
+        'mixins': [
+          'skia_gold_test',
+        ],
+      },
     },
 
     # TODO(dpranke): These are run on the p/chromium waterfall; they should