Reintroduce thanos_objstore_bucket_operation_failures_total alert

Metric was removed on PR 2002 by mistake Signed-off-by: dmaiocchi <dmaiocchi@suse.com>
cpotteck · Dec 17, 2020 · d00e8d2 · d00e8d2
1 parent 4abb51a
commit d00e8d2
Show file tree

Hide file tree

Showing 4 changed files with 37 additions and 1 deletion.
diff --git a/examples/alerts/alerts.md b/examples/alerts/alerts.md
@@ -314,6 +314,17 @@ rules:
   for: 5m
   labels:
     severity: critical
+- alert: ThanosSidecarBucketOperationsFailed
+  annotations:
+    description: Thanos Sidecar {{$labels.job}} {{$labels.pod}} bucket operations
+      are failing
+    runbook_url: https://github.com/thanos-io/thanos/tree/master/mixin/runbook.md#alert-name-thanossidecarbucketoperationsfailed
+    summary: Thanos Sidecar bucket operations are failing
+  expr: |
+    rate(thanos_objstore_bucket_operation_failures_total{job=~"thanos-sidecar.*"}[5m]) > 0
+  for: 5m
+  labels:
+    severity: critical
 - alert: ThanosSidecarUnhealthy
   annotations:
     description: Thanos Sidecar {{$labels.job}} {{$labels.pod}} is unhealthy for {{

diff --git a/examples/alerts/alerts.yaml b/examples/alerts/alerts.yaml
@@ -295,6 +295,17 @@ groups:
     for: 5m
     labels:
       severity: critical
+  - alert: ThanosSidecarBucketOperationsFailed
+    annotations:
+      description: Thanos Sidecar {{$labels.job}} {{$labels.pod}} bucket operations
+        are failing
+      runbook_url: https://github.com/thanos-io/thanos/tree/master/mixin/runbook.md#alert-name-thanossidecarbucketoperationsfailed
+      summary: Thanos Sidecar bucket operations are failing
+    expr: |
+      rate(thanos_objstore_bucket_operation_failures_total{job=~"thanos-sidecar.*"}[5m]) > 0
+    for: 5m
+    labels:
+      severity: critical
   - alert: ThanosSidecarUnhealthy
     annotations:
       description: Thanos Sidecar {{$labels.job}} {{$labels.pod}} is unhealthy for

diff --git a/mixin/alerts/sidecar.libsonnet b/mixin/alerts/sidecar.libsonnet
@@ -22,6 +22,20 @@
               severity: 'critical',
             },
           },
+          {
+            alert: 'ThanosSidecarBucketOperationsFailed',
+            annotations: {
+              description: 'Thanos Sidecar {{$labels.job}} {{$labels.pod}} bucket operations are failing',
+              summary: 'Thanos Sidecar bucket operations are failing',
+            },
+            expr: |||
+              rate(thanos_objstore_bucket_operation_failures_total{%(selector)s}[5m]) > 0
+            ||| % thanos.sidecar,
+            'for': '5m',
+            labels: {
+              severity: 'critical',
+            },
+          },
           {
             alert: 'ThanosSidecarUnhealthy',
             annotations: {

diff --git a/pkg/rules/rules_test.go b/pkg/rules/rules_test.go
@@ -82,7 +82,7 @@ func testRulesAgainstExamples(t *testing.T, dir string, server rulespb.RulesServ
 		{
 			Name:                    "thanos-sidecar",
 			File:                    filepath.Join(dir, "alerts.yaml"),
-			Rules:                   []*rulespb.Rule{someAlert, someAlert},
+			Rules:                   []*rulespb.Rule{someAlert, someAlert, someAlert},
 			Interval:                60,
 			PartialResponseStrategy: storepb.PartialResponseStrategy_ABORT,
 		},