Instrument wait time for the dispatcher at shutdown

mozilla · May 10, 2023 · ce37542 · ce37542
1 parent 5f4b8f2
commit ce37542
Show file tree

Hide file tree

Showing 3 changed files with 45 additions and 0 deletions.
diff --git a/glean-core/metrics.yaml b/glean-core/metrics.yaml
@@ -744,6 +744,23 @@ glean.validation:
       - jrediger@mozilla.com
     expires: never
 
+  shutdown_dispatcher_wait:
+    type: timing_distribution
+    time_unit: millisecond
+    description: |
+      Time waited for the dispatcher to unblock during shutdown.
+      Most samples are expected to be below the 10s timeout used.
+    bugs:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1828066
+    data_reviews:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1828066#c7
+    data_sensitivity:
+      - technical
+    notification_emails:
+      - glean-team@mozilla.com
+      - jrediger@mozilla.com
+    expires: never
+
 glean:
   restarted:
     type: event

diff --git a/glean-core/src/internal_metrics.rs b/glean-core/src/internal_metrics.rs
@@ -24,6 +24,9 @@ pub struct AdditionalMetrics {
 
     /// Time waited for the uploader at shutdown.
     pub shutdown_wait: TimingDistributionMetric,
+
+    /// Time waited for the dispatcher to unblock during shutdown.
+    pub shutdown_dispatcher_wait: TimingDistributionMetric,
 }
 
 impl CoreMetrics {
@@ -97,6 +100,18 @@ impl AdditionalMetrics {
                 },
                 TimeUnit::Millisecond,
             ),
+
+            shutdown_dispatcher_wait: TimingDistributionMetric::new(
+                CommonMetricData {
+                    name: "shutdown_dispatcher_wait".into(),
+                    category: "glean.validation".into(),
+                    send_in_pings: vec!["metrics".into()],
+                    lifetime: Lifetime::Ping,
+                    disabled: false,
+                    dynamic_label: None,
+                },
+                TimeUnit::Millisecond,
+            ),
         }
     }
 }

diff --git a/glean-core/src/lib.rs b/glean-core/src/lib.rs
@@ -571,12 +571,25 @@ pub fn shutdown() {
     // TODO: Make the timeout configurable?
     // The default hang watchdog on Firefox waits 60s,
     // Glean's `uploader_shutdown` further below waits up to 30s.
+    let timer_id = core::with_glean(|glean| {
+        glean
+            .additional_metrics
+            .shutdown_dispatcher_wait
+            .start_sync()
+    });
     if dispatcher::block_on_queue_timeout(Duration::from_secs(10)).is_err() {
         log::error!(
             "Timeout while blocking on the dispatcher. No further shutdown cleanup will happen."
         );
         return;
     }
+    let stop_time = time::precise_time_ns();
+    core::with_glean(|glean| {
+        glean
+            .additional_metrics
+            .shutdown_dispatcher_wait
+            .set_stop_and_accumulate(glean, timer_id, stop_time);
+    });
 
     if let Err(e) = dispatcher::shutdown() {
         log::error!("Can't shutdown dispatcher thread: {:?}", e);