Fix MimirCompactorHasNotUploadedBlocks alert false positive when Mimi…

…r is deployed in monolithic mode (#1902) Signed-off-by: Marco Pracucci <marco@pracucci.com>
grafana · May 23, 2022 · 48cae31 · 48cae31
1 parent 12929ad
commit 48cae31
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 11 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -17,6 +17,7 @@
 * [BUGFIX] Fix `container_memory_usage_bytes:sum` recording rule #1865
 * [BUGFIX] Fix `MimirGossipMembersMismatch` alerts if Mimir alertmanager is activated #1870
 * [BUGFIX] Fix `MimirRulerMissedEvaluations` to show % of missed alerts as a value between 0 and 100 instead of 0 and 1. #1895
+* [BUGFIX] Fix `MimirCompactorHasNotUploadedBlocks` alert false positive when Mimir is deployed in monolithic mode. #1901
 * [BUGFIX] Do not trigger `MimirAllocatingTooMuchMemory` alerts if no container limits are supplied. #1905
 
 ## 2.1.0-rc.0

diff --git a/operations/mimir-mixin-compiled/alerts.yaml b/operations/mimir-mixin-compiled/alerts.yaml
@@ -680,9 +680,9 @@ groups:
       message: Mimir Compactor {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
         }} has not uploaded any block in the last 24 hours.
     expr: |
-      (time() - thanos_objstore_bucket_last_successful_upload_time{job=~".+/(compactor.*|cortex|mimir)"} > 60 * 60 * 24)
+      (time() - thanos_objstore_bucket_last_successful_upload_time{component="compactor"} > 60 * 60 * 24)
       and
-      (thanos_objstore_bucket_last_successful_upload_time{job=~".+/(compactor.*|cortex|mimir)"} > 0)
+      (thanos_objstore_bucket_last_successful_upload_time{component="compactor"} > 0)
     for: 15m
     labels:
       severity: critical
@@ -691,7 +691,7 @@ groups:
       message: Mimir Compactor {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
         }} has not uploaded any block in the last 24 hours.
     expr: |
-      thanos_objstore_bucket_last_successful_upload_time{job=~".+/(compactor.*|cortex|mimir)"} == 0
+      thanos_objstore_bucket_last_successful_upload_time{component="compactor"} == 0
     for: 24h
     labels:
       severity: critical
@@ -700,7 +700,7 @@ groups:
       message: Mimir Compactor {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
         }} has found and ignored blocks with out of order chunks.
     expr: |
-      increase(cortex_compactor_blocks_marked_for_no_compaction_total{job=~".+/(compactor.*|cortex|mimir)", reason="block-index-out-of-order-chunk"}[5m]) > 0
+      increase(cortex_compactor_blocks_marked_for_no_compaction_total{component="compactor", reason="block-index-out-of-order-chunk"}[5m]) > 0
     for: 1m
     labels:
       severity: warning

diff --git a/operations/mimir-mixin/alerts/compactor.libsonnet b/operations/mimir-mixin/alerts/compactor.libsonnet
@@ -65,10 +65,10 @@
           alert: $.alertName('CompactorHasNotUploadedBlocks'),
           'for': '15m',
           expr: |||
-            (time() - thanos_objstore_bucket_last_successful_upload_time{job=~".+/(%(compactor)s)"} > 60 * 60 * 24)
+            (time() - thanos_objstore_bucket_last_successful_upload_time{component="compactor"} > 60 * 60 * 24)
             and
-            (thanos_objstore_bucket_last_successful_upload_time{job=~".+/(%(compactor)s)"} > 0)
-          ||| % $._config.job_names,
+            (thanos_objstore_bucket_last_successful_upload_time{component="compactor"} > 0)
+          |||,
           labels: {
             severity: 'critical',
           },
@@ -81,8 +81,8 @@
           alert: $.alertName('CompactorHasNotUploadedBlocks'),
           'for': '24h',
           expr: |||
-            thanos_objstore_bucket_last_successful_upload_time{job=~".+/(%(compactor)s)"} == 0
-          ||| % $._config.job_names,
+            thanos_objstore_bucket_last_successful_upload_time{component="compactor"} == 0
+          |||,
           labels: {
             severity: 'critical',
           },
@@ -95,8 +95,8 @@
           alert: $.alertName('CompactorSkippedBlocksWithOutOfOrderChunks'),
           'for': '1m',
           expr: |||
-            increase(cortex_compactor_blocks_marked_for_no_compaction_total{job=~".+/(%(compactor)s)", reason="block-index-out-of-order-chunk"}[5m]) > 0
-          ||| % $._config.job_names,
+            increase(cortex_compactor_blocks_marked_for_no_compaction_total{component="compactor", reason="block-index-out-of-order-chunk"}[5m]) > 0
+          |||,
           labels: {
             severity: 'warning',
           },