Add OpenTelemetry alerts (grafana#6381)

fgouteroux · Feb 19, 2024 · 20d7865 · 20d7865
1 parent 287b1c6
commit 20d7865
Showing 1 changed file with 25 additions and 0 deletions.
diff --git a/operations/agent-flow-mixin/alerts/opentelemetry.libsonnet b/operations/agent-flow-mixin/alerts/opentelemetry.libsonnet
@@ -0,0 +1,25 @@
+local alert = import './utils/alert.jsonnet';
+
+alert.newGroup(
+  'otelcol',
+  [
+    // An otelcol.exporter component rcould not push some spans to the pipeline.
+    // This could be due to reaching a limit such as the ones
+    // imposed by otelcol.processor.memory_limiter.
+    alert.newRule(
+      'OtelcolReceiverRefusedSpans',
+      'sum(rate(receiver_refused_spans_ratio_total{}[1m])) > 0',
+      'The receiver could not push some spans to the pipeline.',
+      '5m',
+    ),
+
+    // The exporter failed to send spans to their destination.
+    // There could be an issue with the payload or with the destination endpoint.
+    alert.newRule(
+      'OtelcolExporterFailedSpans',
+      'sum(rate(exporter_send_failed_spans_ratio_total{}[1m])) > 0',
+      'The exporter failed to send spans to their destination.',
+      '5m',
+    ),
+  ]
+)