[DataProvider] Detect additional "remote download output" events

Extend the analysis to find further events that indicate outputs were downloaded from a remote resource. This improves the detection of remotely executed events. Signed-off-by: Sara Adams <sara.e.adams@gmail.com>
EngFlow · Nov 28, 2023 · 9a52093 · 9a52093
1 parent 284b803
commit 9a52093
Show file tree

Hide file tree

Showing 4 changed files with 26 additions and 14 deletions.
diff --git a/analyzer/java/com/engflow/bazel/invocation/analyzer/bazelprofile/BazelEventsUtil.java b/analyzer/java/com/engflow/bazel/invocation/analyzer/bazelprofile/BazelEventsUtil.java
@@ -54,7 +54,13 @@ public static boolean indicatesRemoteCacheCheck(CompleteEvent event) {
    * after an action was executed remotely.
    */
   public static boolean indicatesRemoteDownloadOutputs(CompleteEvent event) {
-    return CAT_REMOTE_OUTPUT_DOWNLOAD.equals(event.category);
+    // See
+    // https://github.com/bazelbuild/bazel/blob/4a29f0851d1cde0240793cdc7a2e2cab926d31b7/src/main/java/com/google/devtools/build/lib/profiler/ProfilerTask.java#L88
+    // and
+    // https://github.com/bazelbuild/bazel/blob/b2cca31705419ba1cd3744c23be4b1d9bdb5c467/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java#L1211C79-L1211C79
+    return CAT_REMOTE_OUTPUT_DOWNLOAD.equals(event.category)
+        || CAT_GENERAL_INFORMATION.equals(event.category)
+            && BazelProfileConstants.COMPLETE_REMOTE_DOWNLOAD.equals(event.name);
   }
 
   /** The event documents uploading outputs to a remote cache. */

diff --git a/analyzer/java/com/engflow/bazel/invocation/analyzer/bazelprofile/BazelProfileConstants.java b/analyzer/java/com/engflow/bazel/invocation/analyzer/bazelprofile/BazelProfileConstants.java
@@ -93,6 +93,10 @@ public class BazelProfileConstants {
   public static final String COMPLETE_SUBPROCESS_RUN = "subprocess.run";
   public static final String COMPLETE_EXECUTE_REMOTELY = "execute remotely";
 
+  // May be written when downloading the outputs of a remotely executed action, see
+  // https://github.com/bazelbuild/bazel/blob/b2cca31705419ba1cd3744c23be4b1d9bdb5c467/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java#L1211C79-L1211C79
+  public static final String COMPLETE_REMOTE_DOWNLOAD = "Remote.download";
+
   // https://github.com/bazelbuild/bazel/blob/7d10999fc0357596824f2b6022bbbd895f245a3c/src/main/java/com/google/devtools/build/lib/remote/RemoteRepositoryRemoteExecutor.java#L160
   /**
    * For complete events of category {@link #CAT_REMOTE_EXECUTION_UPLOAD_TIME}, this name indicates

diff --git a/...er/java/com/engflow/bazel/invocation/analyzer/dataproviders/LocalActionsDataProvider.java b/...er/java/com/engflow/bazel/invocation/analyzer/dataproviders/LocalActionsDataProvider.java
@@ -14,9 +14,6 @@
 
 package com.engflow.bazel.invocation.analyzer.dataproviders;
 
-import static com.engflow.bazel.invocation.analyzer.bazelprofile.BazelProfileConstants.CAT_REMOTE_ACTION_CACHE_CHECK;
-import static com.engflow.bazel.invocation.analyzer.bazelprofile.BazelProfileConstants.CAT_REMOTE_EXECUTION_UPLOAD_TIME;
-import static com.engflow.bazel.invocation.analyzer.bazelprofile.BazelProfileConstants.CAT_REMOTE_OUTPUT_DOWNLOAD;
 import static com.engflow.bazel.invocation.analyzer.bazelprofile.ProfileThread.ofCategoryTypes;
 
 import com.engflow.bazel.invocation.analyzer.bazelprofile.BazelEventsUtil;
@@ -37,17 +34,10 @@
 import com.google.common.collect.Iterators;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 public class LocalActionsDataProvider extends DataProvider {
-  private static final Set<String> RELATED_EVENTS_CATEGORIES =
-      Set.of(
-          CAT_REMOTE_ACTION_CACHE_CHECK,
-          CAT_REMOTE_OUTPUT_DOWNLOAD,
-          CAT_REMOTE_EXECUTION_UPLOAD_TIME);
-
   @Override
   public List<DatumSupplierSpecification<?>> getSuppliers() {
     return List.of(
@@ -66,8 +56,11 @@ LocalActions derive() throws InvalidProfileException, MissingInputException, Nul
   }
 
   private static boolean isRelatedEvent(CompleteEvent event) {
-    return RELATED_EVENTS_CATEGORIES.contains(event.category)
-        || BazelEventsUtil.indicatesLocalExecution(event);
+    return BazelEventsUtil.indicatesRemoteCacheCheck(event)
+        || BazelEventsUtil.indicatesRemoteUploadOutputs(event)
+        || BazelEventsUtil.indicatesRemoteDownloadOutputs(event)
+        || BazelEventsUtil.indicatesLocalExecution(event)
+        || BazelEventsUtil.indicatesRemoteExecution(event);
   }
 
   Stream<LocalAction> coalesce(ProfileThread thread) {

diff --git a/...zer/javatests/com/engflow/bazel/invocation/analyzer/bazelprofile/BazelEventsUtilTest.java b/...zer/javatests/com/engflow/bazel/invocation/analyzer/bazelprofile/BazelEventsUtilTest.java
@@ -1,6 +1,7 @@
 package com.engflow.bazel.invocation.analyzer.bazelprofile;
 
 import static com.engflow.bazel.invocation.analyzer.bazelprofile.BazelProfileConstants.CAT_ACTION_PROCESSING;
+import static com.engflow.bazel.invocation.analyzer.bazelprofile.BazelProfileConstants.CAT_GENERAL_INFORMATION;
 import static com.engflow.bazel.invocation.analyzer.bazelprofile.BazelProfileConstants.CAT_LOCAL_ACTION_EXECUTION;
 import static com.engflow.bazel.invocation.analyzer.bazelprofile.BazelProfileConstants.CAT_REMOTE_ACTION_CACHE_CHECK;
 import static com.engflow.bazel.invocation.analyzer.bazelprofile.BazelProfileConstants.CAT_REMOTE_ACTION_EXECUTION;
@@ -72,13 +73,21 @@ public void doesNotIndicateRemoteCacheCheck() {
   }
 
   @Test
-  public void indicatesRemoteDownloadOutputs() {
+  public void indicatesRemoteDownloadOutputsCat() {
     assertThat(
             BazelEventsUtil.indicatesRemoteDownloadOutputs(
                 completeEvent("random name", CAT_REMOTE_OUTPUT_DOWNLOAD)))
         .isTrue();
   }
 
+  @Test
+  public void indicatesRemoteDownloadOutputsName() {
+    assertThat(
+            BazelEventsUtil.indicatesRemoteDownloadOutputs(
+                completeEvent("Remote.download", CAT_GENERAL_INFORMATION)))
+        .isTrue();
+  }
+
   @Test
   public void doesNotIndicateRemoteDownloadOutputs() {
     assertThat(