alerts: Fixed compactor alert to use correct aggregation function. (#…

…2875) max is aggregating across series. We need to aggregate something across time as well as series due to rollout. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
thanos-io · Jul 10, 2020 · 1ae66b9 · 1ae66b9
1 parent af3dc2c
commit 1ae66b9
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 3 deletions.
diff --git a/examples/alerts/alerts.md b/examples/alerts/alerts.md
@@ -54,7 +54,7 @@ rules:
 - alert: ThanosCompactHasNotRun
   annotations:
     message: Thanos Compact {{$labels.job}} has not uploaded anything for 24 hours.
-  expr: (time() - max(thanos_objstore_bucket_last_successful_upload_time{job=~"thanos-compact.*"}))
+  expr: (time() - max(max_over_time(thanos_objstore_bucket_last_successful_upload_time{job=~"thanos-compact.*"}[24h])))
     / 60 / 60 > 24
   labels:
     severity: warning

diff --git a/examples/alerts/alerts.yaml b/examples/alerts/alerts.yaml
@@ -47,7 +47,7 @@ groups:
   - alert: ThanosCompactHasNotRun
     annotations:
       message: Thanos Compact {{$labels.job}} has not uploaded anything for 24 hours.
-    expr: (time() - max(thanos_objstore_bucket_last_successful_upload_time{job=~"thanos-compact.*"}))
+    expr: (time() - max(max_over_time(thanos_objstore_bucket_last_successful_upload_time{job=~"thanos-compact.*"}[24h])))
       / 60 / 60 > 24
     labels:
       severity: warning

diff --git a/mixin/alerts/compact.libsonnet b/mixin/alerts/compact.libsonnet
@@ -73,7 +73,7 @@
             annotations: {
               message: 'Thanos Compact {{$labels.job}} has not uploaded anything for 24 hours.',
             },
-            expr: '(time() - max(thanos_objstore_bucket_last_successful_upload_time{%(selector)s})) / 60 / 60 > 24' % thanos.compact,
+            expr: '(time() - max(max_over_time(thanos_objstore_bucket_last_successful_upload_time{%(selector)s}[24h]))) / 60 / 60 > 24' % thanos.compact,
             labels: {
               severity: 'warning',
             },