mixin: Use sidecar's metric timestamp for healthcheck (thanos-io#3204)

During prometheus updates the alert was firing because the metric was initialized with a value of '0' before the first heartbeat was sent. As such, the evaluation of the alert results into actually taking just the value of time() into consideration which led to misleading information about the health of the sidecar. As the thanos_sidecar_last_heartbeat_success_time_seconds metric is effectively just a timestamp that resets on new deployments, we can simply wrap it around the timestamp() function which should return almost the same value of the metric itself with the added benefit that heartbeat resets will be ignored. This also refactors the relevant tests and drops the timeout to 4 minutes in order to ensure that we do not get hit by stale data if the sidecar takes longer to start. Signed-off-by: Markos Chandras <markos@chandras.me>
cpotteck · Mar 5, 2021 · 80e0257 · 80e0257
1 parent befb025
commit 80e0257
Show file tree

Hide file tree

Showing 6 changed files with 67 additions and 114 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -15,6 +15,7 @@ We use _breaking :warning:_ to mark changes that are not backward compatible (re
 ### Added
 
 ### Fixed
+- [#3204](https://github.com/thanos-io/thanos/pull/3204) Mixin: Use sidecar's metric timestamp for healthcheck.
 
 ### Changed
 

diff --git a/examples/alerts/alerts.md b/examples/alerts/alerts.md
@@ -324,12 +324,12 @@ rules:
     severity: critical
 - alert: ThanosSidecarUnhealthy
   annotations:
-    description: Thanos Sidecar {{$labels.job}} {{$labels.instance}} is unhealthy
-      for {{ $value }} seconds.
+    description: Thanos Sidecar {{$labels.job}} {{$labels.pod}} is unhealthy for more
+      than {{ $value }} seconds.
     runbook_url: https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy
     summary: Thanos Sidecar is unhealthy.
   expr: |
-    time() - max(thanos_sidecar_last_heartbeat_success_time_seconds{job=~"thanos-sidecar.*"}) by (job, instance) >= 600
+    time() - max(timestamp(thanos_sidecar_last_heartbeat_success_time_seconds{job=~"thanos-sidecar.*"})) by (job,pod) >= 240
   labels:
     severity: critical
 ```

diff --git a/examples/alerts/alerts.yaml b/examples/alerts/alerts.yaml
@@ -308,12 +308,12 @@ groups:
       severity: critical
   - alert: ThanosSidecarUnhealthy
     annotations:
-      description: Thanos Sidecar {{$labels.job}} {{$labels.instance}} is unhealthy
-        for {{ $value }} seconds.
+      description: Thanos Sidecar {{$labels.job}} {{$labels.pod}} is unhealthy for
+        more than {{ $value }} seconds.
       runbook_url: https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy
       summary: Thanos Sidecar is unhealthy.
     expr: |
-      time() - max(thanos_sidecar_last_heartbeat_success_time_seconds{job=~"thanos-sidecar.*"}) by (job, instance) >= 600
+      time() - max(timestamp(thanos_sidecar_last_heartbeat_success_time_seconds{job=~"thanos-sidecar.*"})) by (job,pod) >= 240
     labels:
       severity: critical
 - name: thanos-store

diff --git a/examples/alerts/tests.yaml b/examples/alerts/tests.yaml
@@ -7,10 +7,10 @@ evaluation_interval: 1m
 tests:
 - interval: 1m
   input_series:
-  - series: 'thanos_sidecar_last_heartbeat_success_time_seconds{namespace="production", job="thanos-sidecar", instance="thanos-sidecar-0"}'
-    values: '5 10 43 17 11 0 0 0'
-  - series: 'thanos_sidecar_last_heartbeat_success_time_seconds{namespace="production", job="thanos-sidecar", instance="thanos-sidecar-1"}'
-    values: '4 9 42 15 10 0 0 0'
+  - series: 'thanos_sidecar_last_heartbeat_success_time_seconds{namespace="production", job="thanos-sidecar", pod="thanos-sidecar-pod-0"}'
+    values: '5 10 43 17 11 _x5 0x10'
+  - series: 'thanos_sidecar_last_heartbeat_success_time_seconds{namespace="production", job="thanos-sidecar", pod="thanos-sidecar-pod-1"}'
+    values: '4 9 42 15 10 _x5 0x10'
   promql_expr_test:
     - expr: time()
       eval_time: 1m
@@ -22,109 +22,61 @@ tests:
       exp_samples:
         - labels: '{}'
           value: 120
-    - expr: max(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"}) by (job, instance)
-      eval_time: 2m
-      exp_samples:
-        - labels: '{job="thanos-sidecar", instance="thanos-sidecar-0"}'
-          value: 43
-        - labels: '{job="thanos-sidecar", instance="thanos-sidecar-1"}'
-          value: 42
-    - expr: max(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"}) by (job, instance)
-      eval_time: 10m
+    - expr: time() - max(timestamp(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"})) by (job, pod)
+      eval_time: 5m
       exp_samples:
-        - labels: '{job="thanos-sidecar", instance="thanos-sidecar-0"}'
-          value: 0
-        - labels: '{job="thanos-sidecar", instance="thanos-sidecar-1"}'
-          value: 0
-    - expr: max(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"}) by (job, instance)
-      eval_time: 11m
-      exp_samples:
-        - labels: '{job="thanos-sidecar", instance="thanos-sidecar-0"}'
-          value: 0
-        - labels: '{job="thanos-sidecar", instance="thanos-sidecar-1"}'
-          value: 0
-    - expr: time() - max(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"}) by (job, instance)
-      eval_time: 10m
+        - labels: '{job="thanos-sidecar", pod="thanos-sidecar-pod-0"}'
+          value: 60
+        - labels: '{job="thanos-sidecar", pod="thanos-sidecar-pod-1"}'
+          value: 60
+    - expr: time() - max(timestamp(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"})) by (job, pod)
+      eval_time: 6m
       exp_samples:
-        - labels: '{job="thanos-sidecar", instance="thanos-sidecar-0"}'
-          value: 600
-        - labels: '{job="thanos-sidecar", instance="thanos-sidecar-1"}'
-          value: 600
-    - expr: time() - max(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"}) by (job, instance)
-      eval_time: 11m
+        - labels: '{job="thanos-sidecar", pod="thanos-sidecar-pod-0"}'
+          value: 120
+        - labels: '{job="thanos-sidecar", pod="thanos-sidecar-pod-1"}'
+          value: 120
+    - expr: time() - max(timestamp(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"})) by (job, pod)
+      eval_time: 7m
       exp_samples:
-        - labels: '{job="thanos-sidecar", instance="thanos-sidecar-0"}'
-          value: 660
-        - labels: '{job="thanos-sidecar", instance="thanos-sidecar-1"}'
-          value: 660
-    - expr: time() - max(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"}) by (job, instance) >= 600
-      eval_time: 12m
+        - labels: '{job="thanos-sidecar", pod="thanos-sidecar-pod-0"}'
+          value: 180
+        - labels: '{job="thanos-sidecar", pod="thanos-sidecar-pod-1"}'
+          value: 180
+    - expr: time() - max(timestamp(thanos_sidecar_last_heartbeat_success_time_seconds{job="thanos-sidecar"})) by (job, pod)
+      eval_time: 8m
       exp_samples:
-        - labels: '{job="thanos-sidecar", instance="thanos-sidecar-0"}'
-          value: 720
-        - labels: '{job="thanos-sidecar", instance="thanos-sidecar-1"}'
-          value: 720
+        - labels: '{job="thanos-sidecar", pod="thanos-sidecar-pod-0"}'
+          value: 240
+        - labels: '{job="thanos-sidecar", pod="thanos-sidecar-pod-1"}'
+          value: 240
   alert_rule_test:
-  - eval_time: 1m
-    alertname: ThanosSidecarUnhealthy
-  - eval_time: 2m
-    alertname: ThanosSidecarUnhealthy
-  - eval_time: 3m
-    alertname: ThanosSidecarUnhealthy
-  - eval_time: 10m
-    alertname: ThanosSidecarUnhealthy
-    exp_alerts:
-    - exp_labels:
-        severity: critical
-        job: thanos-sidecar
-        instance: thanos-sidecar-0
-      exp_annotations:
-        description: 'Thanos Sidecar thanos-sidecar thanos-sidecar-0 is unhealthy for 600 seconds.'
-        runbook_url: 'https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy'
-        summary: 'Thanos Sidecar is unhealthy.'
-    - exp_labels:
-        severity: critical
-        job: thanos-sidecar
-        instance: thanos-sidecar-1
-      exp_annotations:
-        description: 'Thanos Sidecar thanos-sidecar thanos-sidecar-1 is unhealthy for 600 seconds.'
-        runbook_url: 'https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy'
-        summary: 'Thanos Sidecar is unhealthy.'
-  - eval_time: 11m
-    alertname: ThanosSidecarUnhealthy
-    exp_alerts:
-    - exp_labels:
-        severity: critical
-        job: thanos-sidecar
-        instance: thanos-sidecar-0
-      exp_annotations:
-        description: 'Thanos Sidecar thanos-sidecar thanos-sidecar-0 is unhealthy for 660 seconds.'
-        runbook_url: 'https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy'
-        summary: 'Thanos Sidecar is unhealthy.'
-    - exp_labels:
-        severity: critical
-        job: thanos-sidecar
-        instance: thanos-sidecar-1
-      exp_annotations:
-        description: 'Thanos Sidecar thanos-sidecar thanos-sidecar-1 is unhealthy for 660 seconds.'
-        runbook_url: 'https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy'
-        summary: 'Thanos Sidecar is unhealthy.'
-  - eval_time: 12m
-    alertname: ThanosSidecarUnhealthy
-    exp_alerts:
-    - exp_labels:
-        severity: critical
-        job: thanos-sidecar
-        instance: thanos-sidecar-0
-      exp_annotations:
-        description: 'Thanos Sidecar thanos-sidecar thanos-sidecar-0 is unhealthy for 720 seconds.'
-        runbook_url: 'https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy'
-        summary: 'Thanos Sidecar is unhealthy.'
-    - exp_labels:
-        severity: critical
-        job: thanos-sidecar
-        instance: thanos-sidecar-1
-      exp_annotations:
-        description: 'Thanos Sidecar thanos-sidecar thanos-sidecar-1 is unhealthy for 720 seconds.'
-        runbook_url: 'https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy'
-        summary: 'Thanos Sidecar is unhealthy.'
+    - eval_time: 1m
+      alertname: ThanosSidecarUnhealthy
+    - eval_time: 2m
+      alertname: ThanosSidecarUnhealthy
+    - eval_time: 3m
+      alertname: ThanosSidecarUnhealthy
+    - eval_time: 5m
+      alertname: ThanosSidecarUnhealthy
+    - eval_time: 8m
+      alertname: ThanosSidecarUnhealthy
+      exp_alerts:
+      - exp_labels:
+          severity: critical
+          job: thanos-sidecar
+          pod: thanos-sidecar-pod-0
+        exp_annotations:
+          description: 'Thanos Sidecar thanos-sidecar thanos-sidecar-pod-0 is unhealthy for more than 240 seconds.'
+          runbook_url: 'https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy'
+          summary: 'Thanos Sidecar is unhealthy.'
+      - exp_labels:
+          severity: critical
+          job: thanos-sidecar
+          pod: thanos-sidecar-pod-1
+        exp_annotations:
+          description: 'Thanos Sidecar thanos-sidecar thanos-sidecar-pod-1 is unhealthy for more than 240 seconds.'
+          runbook_url: 'https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy'
+          summary: 'Thanos Sidecar is unhealthy.'
+    - eval_time: 10m
+      alertname: ThanosSidecarUnhealthy
diff --git a/mixin/alerts/sidecar.libsonnet b/mixin/alerts/sidecar.libsonnet
@@ -39,11 +39,11 @@
           {
             alert: 'ThanosSidecarUnhealthy',
             annotations: {
-              description: 'Thanos Sidecar {{$labels.job}} {{$labels.instance}} is unhealthy for {{ $value }} seconds.',
+              description: 'Thanos Sidecar {{$labels.job}} {{$labels.pod}} is unhealthy for more than {{ $value }} seconds.',
               summary: 'Thanos Sidecar is unhealthy.',
             },
             expr: |||
-              time() - max(thanos_sidecar_last_heartbeat_success_time_seconds{%(selector)s}) by (job, instance) >= 600
+              time() - max(timestamp(thanos_sidecar_last_heartbeat_success_time_seconds{%(selector)s})) by (job,pod) >= 240
             ||| % thanos.sidecar,
             labels: {
               severity: 'critical',

diff --git a/mixin/runbook.md b/mixin/runbook.md
@@ -87,7 +87,7 @@
 |---|---|---|---|---|
 |ThanosSidecarPrometheusDown|Thanos Sidecar cannot connect to Prometheus|Thanos Sidecar {{$labels.job}} {{$labels.instance}} cannot connect to Prometheus.|critical|[https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarprometheusdown](https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarprometheusdown)|
 |ThanosSidecarBucketOperationsFailed|Thanos Sidecar bucket operations are failing|Thanos Sidecar {{$labels.job}} {{$labels.instance}} bucket operations are failing|critical|[https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarbucketoperationsfailed](https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarbucketoperationsfailed)|
-|ThanosSidecarUnhealthy|Thanos Sidecar is unhealthy.|Thanos Sidecar {{$labels.job}} {{$labels.instance}} is unhealthy for {{ $value }} seconds.|critical|[https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy](https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy)|
+|ThanosSidecarUnhealthy|Thanos Sidecar is unhealthy.|Thanos Sidecar {{$labels.job}} {{$labels.pod}} is unhealthy for more than {{ $value }} seconds.|critical|[https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy](https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanossidecarunhealthy)|
 
 ## thanos-store