Adjust and rename ThanosSidecarUnhealthy to ThanosSidecarNoConnectionToStartedPrometheus; Remove ThanosSidecarPrometheusDown alert; Remove unused thanos_sidecar_last_heartbeat_success_time_seconds metrics

CHANGELOG.md

@@ -13,6 +13,10 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 ### Added
 - [#4680](https://github.com/thanos-io/thanos/pull/4680) Query: add `exemplar.partial-response` flag to control partial response.
 
+### Fixed
+
+- [#4508](https://github.com/thanos-io/thanos/pull/4508) Adjust and rename `ThanosSidecarUnhealthy` to `ThanosSidecarNoConnectionToStartedPrometheus`; Remove `ThanosSidecarPrometheusDown` alert; Remove unused `thanos_sidecar_last_heartbeat_success_time_seconds` metrics.
+
 ## v0.23.0 - In Progress
 
 ### Added

cmd/thanos/sidecar.go

@@ -138,10 +138,6 @@ func runSidecar(
 			Name: "thanos_sidecar_prometheus_up",
 			Help: "Boolean indicator whether the sidecar can reach its Prometheus peer.",
 		})
-		lastHeartbeat := promauto.With(reg).NewGauge(prometheus.GaugeOpts{
-			Name: "thanos_sidecar_last_heartbeat_success_time_seconds",
-			Help: "Timestamp of the last successful heartbeat in seconds.",
-		})
 
 		ctx, cancel := context.WithCancel(context.Background())
 		g.Add(func() error {
@@ -191,7 +187,6 @@ func runSidecar(
 				)
 				promUp.Set(1)
 				statusProber.Ready()
-				lastHeartbeat.SetToCurrentTime()
 				return nil
 			})
 			if err != nil {
@@ -213,7 +208,6 @@ func runSidecar(
 					promUp.Set(0)
 				} else {
 					promUp.Set(1)
-					lastHeartbeat.SetToCurrentTime()
 				}
 
 				return nil

examples/alerts/alerts.md

@@ -296,16 +296,6 @@ rules:
 ```yaml mdox-exec="cat examples/tmp/thanos-sidecar.yaml"
 name: thanos-sidecar
 rules:
-- alert: ThanosSidecarPrometheusDown
-  annotations:
-    description: Thanos Sidecar {{$labels.instance}} cannot connect to Prometheus.
-    runbook_url: https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarprometheusdown
-    summary: Thanos Sidecar cannot connect to Prometheus
-  expr: |
-    thanos_sidecar_prometheus_up{job=~".*thanos-sidecar.*"} == 0
-  for: 5m
-  labels:
-    severity: critical
 - alert: ThanosSidecarBucketOperationsFailed
   annotations:
     description: Thanos Sidecar {{$labels.instance}} bucket operations are failing
@@ -316,14 +306,16 @@ rules:
   for: 5m
   labels:
     severity: critical
-- alert: ThanosSidecarUnhealthy
+- alert: ThanosSidecarNoConnectionToStartedPrometheus
   annotations:
-    description: Thanos Sidecar {{$labels.instance}} is unhealthy for more than {{$value}}
-      seconds.
-    runbook_url: https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy
-    summary: Thanos Sidecar is unhealthy.
+    description: Thanos Sidecar {{$labels.instance}} is unhealthy.
+    runbook_url: https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarnoconnectiontostartedprometheus
+    summary: Thanos Sidecar cannot access Prometheus, even though Prometheus seems
+      healthy and has reloaded WAL.
   expr: |
-    time() - max by (job, instance) (thanos_sidecar_last_heartbeat_success_time_seconds{job=~".*thanos-sidecar.*"}) >= 240
+    thanos_sidecar_prometheus_up{job=~".*thanos-sidecar.*"} == 0
+    AND on (namespace, pod)
+    prometheus_tsdb_data_replay_duration_seconds != 0
   for: 5m
   labels:
     severity: critical

examples/alerts/alerts.yaml

@@ -301,16 +301,6 @@ groups:
       severity: warning
 - name: thanos-sidecar
   rules:
-  - alert: ThanosSidecarPrometheusDown
-    annotations:
-      description: Thanos Sidecar {{$labels.instance}} cannot connect to Prometheus.
-      runbook_url: https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarprometheusdown
-      summary: Thanos Sidecar cannot connect to Prometheus
-    expr: |
-      thanos_sidecar_prometheus_up{job=~".*thanos-sidecar.*"} == 0
-    for: 5m
-    labels:
-      severity: critical
   - alert: ThanosSidecarBucketOperationsFailed
     annotations:
       description: Thanos Sidecar {{$labels.instance}} bucket operations are failing
@@ -321,14 +311,16 @@ groups:
     for: 5m
     labels:
       severity: critical
-  - alert: ThanosSidecarUnhealthy
+  - alert: ThanosSidecarNoConnectionToStartedPrometheus
     annotations:
-      description: Thanos Sidecar {{$labels.instance}} is unhealthy for more than
-        {{$value}} seconds.
-      runbook_url: https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy
-      summary: Thanos Sidecar is unhealthy.
+      description: Thanos Sidecar {{$labels.instance}} is unhealthy.
+      runbook_url: https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarnoconnectiontostartedprometheus
+      summary: Thanos Sidecar cannot access Prometheus, even though Prometheus seems
+        healthy and has reloaded WAL.
     expr: |
-      time() - max by (job, instance) (thanos_sidecar_last_heartbeat_success_time_seconds{job=~".*thanos-sidecar.*"}) >= 240
+      thanos_sidecar_prometheus_up{job=~".*thanos-sidecar.*"} == 0
+      AND on (namespace, pod)
+      prometheus_tsdb_data_replay_duration_seconds != 0
     for: 5m
     labels:
       severity: critical

examples/alerts/tests.yaml

@@ -7,127 +7,74 @@ evaluation_interval: 1m
 tests:
 - interval: 1m
   input_series:
-  - series: 'thanos_sidecar_last_heartbeat_success_time_seconds{namespace="production", job="thanos-sidecar", instance="thanos-sidecar-0"}'
-    values: '5 10 43 17 11 0 0 0'
-  - series: 'thanos_sidecar_last_heartbeat_success_time_seconds{namespace="production", job="thanos-sidecar", instance="thanos-sidecar-1"}'
-    values: '4 9 42 15 10 0 0 0'
-  promql_expr_test:
-    - expr: time()
-      eval_time: 1m
-      exp_samples:
-        - labels: '{}'
-          value: 60
-    - expr: time()
-      eval_time: 2m
-      exp_samples:
-        - labels: '{}'
-          value: 120
-    - expr: max(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"}) by (job, instance)
-      eval_time: 2m
-      exp_samples:
-        - labels: '{job="thanos-sidecar", instance="thanos-sidecar-0"}'
-          value: 43
-        - labels: '{job="thanos-sidecar", instance="thanos-sidecar-1"}'
-          value: 42
-    - expr: max(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"}) by (job, instance)
-      eval_time: 10m
-      exp_samples:
-        - labels: '{job="thanos-sidecar", instance="thanos-sidecar-0"}'
-          value: 0
-        - labels: '{job="thanos-sidecar", instance="thanos-sidecar-1"}'
-          value: 0
-    - expr: max(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"}) by (job, instance)
-      eval_time: 11m
-      exp_samples:
-        - labels: '{job="thanos-sidecar", instance="thanos-sidecar-0"}'
-          value: 0
-        - labels: '{job="thanos-sidecar", instance="thanos-sidecar-1"}'
-          value: 0
-    - expr: time() - max(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"}) by (job, instance)
-      eval_time: 10m
-      exp_samples:
-        - labels: '{job="thanos-sidecar", instance="thanos-sidecar-0"}'
-          value: 600
-        - labels: '{job="thanos-sidecar", instance="thanos-sidecar-1"}'
-          value: 600
-    - expr: time() - max(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"}) by (job, instance)
-      eval_time: 11m
-      exp_samples:
-        - labels: '{job="thanos-sidecar", instance="thanos-sidecar-0"}'
-          value: 660
-        - labels: '{job="thanos-sidecar", instance="thanos-sidecar-1"}'
-          value: 660
-    - expr: time() - max(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"}) by (job, instance) >= 600
-      eval_time: 12m
-      exp_samples:
-        - labels: '{job="thanos-sidecar", instance="thanos-sidecar-0"}'
-          value: 720
-        - labels: '{job="thanos-sidecar", instance="thanos-sidecar-1"}'
-          value: 720
+  - series: 'thanos_sidecar_prometheus_up{namespace="production", job="thanos-sidecar", instance="thanos-sidecar-0", pod="prometheus-0"}'
+    values: '1x5 0x15'
+  - series: 'thanos_sidecar_prometheus_up{namespace="production", job="thanos-sidecar", instance="thanos-sidecar-1", pod="prometheus-1"}'
+    values: '1x4 0x15'
+  - series: 'prometheus_tsdb_data_replay_duration_seconds{namespace="production", job="prometheus-k8s", instance="prometheus-k8s-0", pod="prometheus-0"}'
+    values: '4x5 0x5 5x15'
+  - series: 'prometheus_tsdb_data_replay_duration_seconds{namespace="production", job="prometheus-k8s", instance="prometheus-k8s-1", pod="prometheus-1"}'
+    values: '10x14 0x6'
   alert_rule_test:
   - eval_time: 1m
-    alertname: ThanosSidecarUnhealthy
+    alertname: ThanosSidecarNoConnectionToStartedPrometheus
   - eval_time: 2m
-    alertname: ThanosSidecarUnhealthy
+    alertname: ThanosSidecarNoConnectionToStartedPrometheus
   - eval_time: 3m
-    alertname: ThanosSidecarUnhealthy
+    alertname: ThanosSidecarNoConnectionToStartedPrometheus
   - eval_time: 10m
-    alertname: ThanosSidecarUnhealthy
+    alertname: ThanosSidecarNoConnectionToStartedPrometheus
     exp_alerts:
-    - exp_labels:
-        severity: critical
-        job: thanos-sidecar
-        instance: thanos-sidecar-0
-      exp_annotations:
-        description: 'Thanos Sidecar thanos-sidecar-0 is unhealthy for more than 600 seconds.'
-        runbook_url: 'https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy'
-        summary: 'Thanos Sidecar is unhealthy.'
     - exp_labels:
         severity: critical
         job: thanos-sidecar
         instance: thanos-sidecar-1
+        namespace: production
+        pod: prometheus-1
       exp_annotations:
-        description: 'Thanos Sidecar thanos-sidecar-1 is unhealthy for more than 600 seconds.'
-        runbook_url: 'https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy'
-        summary: 'Thanos Sidecar is unhealthy.'
+        description: 'Thanos Sidecar thanos-sidecar-1 is unhealthy.'
+        runbook_url: 'https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarnoconnectiontostartedprometheus'
+        summary: 'Thanos Sidecar cannot access Prometheus, even though Prometheus seems healthy and has reloaded WAL.'
   - eval_time: 11m
-    alertname: ThanosSidecarUnhealthy
+    alertname: ThanosSidecarNoConnectionToStartedPrometheus
     exp_alerts:
-    - exp_labels:
-        severity: critical
-        job: thanos-sidecar
-        instance: thanos-sidecar-0
-      exp_annotations:
-        description: 'Thanos Sidecar thanos-sidecar-0 is unhealthy for more than 660 seconds.'
-        runbook_url: 'https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy'
-        summary: 'Thanos Sidecar is unhealthy.'
     - exp_labels:
         severity: critical
         job: thanos-sidecar
         instance: thanos-sidecar-1
+        namespace: production
+        pod: prometheus-1
       exp_annotations:
-        description: 'Thanos Sidecar thanos-sidecar-1 is unhealthy for more than 660 seconds.'
-        runbook_url: 'https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy'
-        summary: 'Thanos Sidecar is unhealthy.'
+        description: 'Thanos Sidecar thanos-sidecar-1 is unhealthy.'
+        runbook_url: 'https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarnoconnectiontostartedprometheus'
+        summary: 'Thanos Sidecar cannot access Prometheus, even though Prometheus seems healthy and has reloaded WAL.'
   - eval_time: 12m
-    alertname: ThanosSidecarUnhealthy
+    alertname: ThanosSidecarNoConnectionToStartedPrometheus
     exp_alerts:
     - exp_labels:
         severity: critical
         job: thanos-sidecar
-        instance: thanos-sidecar-0
+        instance: thanos-sidecar-1
+        namespace: production
+        pod: prometheus-1
       exp_annotations:
-        description: 'Thanos Sidecar thanos-sidecar-0 is unhealthy for more than 720 seconds.'
-        runbook_url: 'https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy'
-        summary: 'Thanos Sidecar is unhealthy.'
+        description: 'Thanos Sidecar thanos-sidecar-1 is unhealthy.'
+        runbook_url: 'https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarnoconnectiontostartedprometheus'
+        summary: 'Thanos Sidecar cannot access Prometheus, even though Prometheus seems healthy and has reloaded WAL.'
+  - eval_time: 20m
+    alertname: ThanosSidecarNoConnectionToStartedPrometheus
+    exp_alerts:
     - exp_labels:
         severity: critical
         job: thanos-sidecar
-        instance: thanos-sidecar-1
+        instance: thanos-sidecar-0
+        namespace: production
+        pod: prometheus-0
       exp_annotations:
-        description: 'Thanos Sidecar thanos-sidecar-1 is unhealthy for more than 720 seconds.'
-        runbook_url: 'https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy'
-        summary: 'Thanos Sidecar is unhealthy.'
+        description: 'Thanos Sidecar thanos-sidecar-0 is unhealthy.'
+        runbook_url: 'https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarnoconnectiontostartedprometheus'
+        summary: 'Thanos Sidecar cannot access Prometheus, even though Prometheus seems healthy and has reloaded WAL.'
+
 - interval: 1m
   input_series:
   - series: 'prometheus_rule_evaluations_total{namespace="production", job="thanos-ruler", instance="thanos-ruler-0"}'

mixin/README.md

@@ -106,6 +106,7 @@ This project is intended to be used as a library. You can extend and customize d
   },
   sidecar+:: {
     selector: 'job=~".*thanos-sidecar.*"',
+    thanosPrometheusCommonDimensions: 'namespace, pod',
     title: '%(prefix)sSidecar' % $.dashboard.prefix,
   },
   // TODO(kakkoyun): Fix naming convention: bucketReplicate

mixin/alerts/sidecar.libsonnet

@@ -2,6 +2,7 @@
   local thanos = self,
   sidecar+:: {
     selector: error 'must provide selector for Thanos Sidecar alerts',
+    thanosPrometheusCommonDimensions: error 'must provide commonDimensions between Thanos and Prometheus metrics for Sidecar alerts',
     dimensions: std.join(', ', std.objectFields(thanos.targetGroups) + ['job', 'instance']),
   },
   prometheusAlerts+:: {
@@ -10,20 +11,6 @@
       {
         name: 'thanos-sidecar',
         rules: [
-          {
-            alert: 'ThanosSidecarPrometheusDown',
-            annotations: {
-              description: 'Thanos Sidecar {{$labels.instance}}%s cannot connect to Prometheus.' % location,
-              summary: 'Thanos Sidecar cannot connect to Prometheus',
-            },
-            expr: |||
-              thanos_sidecar_prometheus_up{%(selector)s} == 0
-            ||| % thanos.sidecar,
-            'for': '5m',
-            labels: {
-              severity: 'critical',
-            },
-          },
           {
             alert: 'ThanosSidecarBucketOperationsFailed',
             annotations: {
@@ -39,13 +26,15 @@
             },
           },
           {
-            alert: 'ThanosSidecarUnhealthy',
+            alert: 'ThanosSidecarNoConnectionToStartedPrometheus',
             annotations: {
-              description: 'Thanos Sidecar {{$labels.instance}}%s is unhealthy for more than {{$value}} seconds.' % location,
-              summary: 'Thanos Sidecar is unhealthy.',
+              description: 'Thanos Sidecar {{$labels.instance}}%s is unhealthy.' % location,
+              summary: 'Thanos Sidecar cannot access Prometheus, even though Prometheus seems healthy and has reloaded WAL.',
             },
             expr: |||
-              time() - max by (%(dimensions)s) (thanos_sidecar_last_heartbeat_success_time_seconds{%(selector)s}) >= 240
+              thanos_sidecar_prometheus_up{%(selector)s} == 0
+              AND on (%(thanosPrometheusCommonDimensions)s)
+              prometheus_tsdb_data_replay_duration_seconds != 0
             ||| % thanos.sidecar,
             'for': '5m',
             labels: {

mixin/config.libsonnet

@@ -46,6 +46,7 @@
   },
   sidecar+:: {
     selector: 'job=~".*thanos-sidecar.*"',
+    thanosPrometheusCommonDimensions: 'namespace, pod',
     title: '%(prefix)sSidecar' % $.dashboard.prefix,
   },
   // TODO(kakkoyun): Fix naming convention: bucketReplicate

mixin/runbook.md

@@ -85,9 +85,8 @@
 
 |Name|Summary|Description|Severity|Runbook|
 |---|---|---|---|---|
-|ThanosSidecarPrometheusDown|Thanos Sidecar cannot connect to Prometheus|Thanos Sidecar {{$labels.instance}} cannot connect to Prometheus.|critical|[https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarprometheusdown](https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarprometheusdown)|
 |ThanosSidecarBucketOperationsFailed|Thanos Sidecar bucket operations are failing|Thanos Sidecar {{$labels.instance}} bucket operations are failing|critical|[https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarbucketoperationsfailed](https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarbucketoperationsfailed)|
-|ThanosSidecarUnhealthy|Thanos Sidecar is unhealthy.|Thanos Sidecar {{$labels.instance}} is unhealthy for more than {{$value}} seconds.|critical|[https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy](https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy)|
+|ThanosSidecarNoConnectionToStartedPrometheus|Thanos Sidecar cannot access Prometheus, even though Prometheus seems healthy and has reloaded WAL.|Thanos Sidecar {{$labels.instance}} is unhealthy.|critical|[https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarnoconnectiontostartedprometheus](https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarnoconnectiontostartedprometheus)|
 
 ## thanos-store
 

pkg/rules/rules_test.go

@@ -82,7 +82,7 @@ func testRulesAgainstExamples(t *testing.T, dir string, server rulespb.RulesServ
 		{
 			Name:                    "thanos-sidecar",
 			File:                    filepath.Join(dir, "alerts.yaml"),
-			Rules:                   []*rulespb.Rule{someAlert, someAlert, someAlert},
+			Rules:                   []*rulespb.Rule{someAlert, someAlert},
 			Interval:                60,
 			PartialResponseStrategy: storepb.PartialResponseStrategy_ABORT,
 		},