Precompute network programming latency percentailes by endpoint

Hopefully this should allow to easily find an offending kube-proxy \#experimental
Sebuckle · Jan 30, 2020 · e6ddd83 · e6ddd83
1 parent 89fca92
commit e6ddd83
Showing 1 changed file with 15 additions and 0 deletions.
diff --git a/clusterloader2/pkg/prometheus/manifests/prometheus-rules.yaml b/clusterloader2/pkg/prometheus/manifests/prometheus-rules.yaml
@@ -74,6 +74,21 @@ spec:
       record: kubeproxy:kubeproxy_network_programming_duration:histogram_quantile
       labels:
         quantile: "0.50"
+    - expr: |
+        histogram_quantile(0.99, sum(rate(kubeproxy_network_programming_duration_seconds_bucket[5m])) by (le, endpoint))
+      record: kubeproxy:kubeproxy_network_programming_duration:histogram_quantile_by_endpoint
+      labels:
+        quantile: "0.99"
+    - expr: |
+        histogram_quantile(0.90, sum(rate(kubeproxy_network_programming_duration_seconds_bucket[5m])) by (le, endpoint))
+      record: kubeproxy:kubeproxy_network_programming_duration:histogram_quantile_by_endpoint
+      labels:
+        quantile: "0.90"
+    - expr: |
+        histogram_quantile(0.50, sum(rate(kubeproxy_network_programming_duration_seconds_bucket[5m])) by (le, endpoint))
+      record: kubeproxy:kubeproxy_network_programming_duration:histogram_quantile_by_endpoint
+      labels:
+        quantile: "0.50"
   - name: apiserver.1m.rules
     rules:
     - expr: |