diff --git a/examples/alerts/alerts.md b/examples/alerts/alerts.md index ce4a0700a8..f6b4a29659 100644 --- a/examples/alerts/alerts.md +++ b/examples/alerts/alerts.md @@ -314,6 +314,17 @@ rules: for: 5m labels: severity: critical +- alert: ThanosSidecarBucketOperationsFailed + annotations: + description: Thanos Sidecar {{$labels.job}} {{$labels.pod}} bucket operations + are failing + runbook_url: https://github.com/thanos-io/thanos/tree/master/mixin/runbook.md#alert-name-thanossidecarbucketoperationsfailed + summary: Thanos Sidecar bucket operations are failing + expr: | + rate(thanos_objstore_bucket_operation_failures_total{job=~"thanos-sidecar.*"}[5m]) > 0 + for: 5m + labels: + severity: critical - alert: ThanosSidecarUnhealthy annotations: description: Thanos Sidecar {{$labels.job}} {{$labels.pod}} is unhealthy for {{ diff --git a/examples/alerts/alerts.yaml b/examples/alerts/alerts.yaml index a903973d4b..8618e43e2b 100644 --- a/examples/alerts/alerts.yaml +++ b/examples/alerts/alerts.yaml @@ -295,6 +295,17 @@ groups: for: 5m labels: severity: critical + - alert: ThanosSidecarBucketOperationsFailed + annotations: + description: Thanos Sidecar {{$labels.job}} {{$labels.pod}} bucket operations + are failing + runbook_url: https://github.com/thanos-io/thanos/tree/master/mixin/runbook.md#alert-name-thanossidecarbucketoperationsfailed + summary: Thanos Sidecar bucket operations are failing + expr: | + rate(thanos_objstore_bucket_operation_failures_total{job=~"thanos-sidecar.*"}[5m]) > 0 + for: 5m + labels: + severity: critical - alert: ThanosSidecarUnhealthy annotations: description: Thanos Sidecar {{$labels.job}} {{$labels.pod}} is unhealthy for diff --git a/mixin/alerts/sidecar.libsonnet b/mixin/alerts/sidecar.libsonnet index 85cef551e5..b4826c39bb 100644 --- a/mixin/alerts/sidecar.libsonnet +++ b/mixin/alerts/sidecar.libsonnet @@ -22,6 +22,20 @@ severity: 'critical', }, }, + { + alert: 'ThanosSidecarBucketOperationsFailed', + annotations: { + description: 'Thanos Sidecar {{$labels.job}} {{$labels.pod}} bucket operations are failing', + summary: 'Thanos Sidecar bucket operations are failing', + }, + expr: ||| + rate(thanos_objstore_bucket_operation_failures_total{%(selector)s}[5m]) > 0 + ||| % thanos.sidecar, + 'for': '5m', + labels: { + severity: 'critical', + }, + }, { alert: 'ThanosSidecarUnhealthy', annotations: { diff --git a/pkg/rules/rules_test.go b/pkg/rules/rules_test.go index 621ad45071..ada7bf8ebf 100644 --- a/pkg/rules/rules_test.go +++ b/pkg/rules/rules_test.go @@ -82,7 +82,7 @@ func testRulesAgainstExamples(t *testing.T, dir string, server rulespb.RulesServ { Name: "thanos-sidecar", File: filepath.Join(dir, "alerts.yaml"), - Rules: []*rulespb.Rule{someAlert, someAlert}, + Rules: []*rulespb.Rule{someAlert, someAlert, someAlert}, Interval: 60, PartialResponseStrategy: storepb.PartialResponseStrategy_ABORT, },