diff --git a/CHANGELOG.md b/CHANGELOG.md index eb1a9b1c6bb..5fbfb7acf89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,6 +66,7 @@ * [ENHANCEMENT] Alerts: exclude `529` and `598` status codes from failure codes in `MimirRequestsError`. #7889 * [BUGFIX] Dashboards: Fix regular expression for matching read-path gRPC ingester methods to include querying of exemplars, label-related queries, or active series queries. #7676 * [BUGFIX] Dashboards: Fix user id abbreviations and column heads for Top Tenants dashboard. #7724 +* [BUGFIX] Dashboards: fix incorrect query used for "queue length" panel on "Ruler" dashboard. #8006 ### Jsonnet diff --git a/operations/helm/tests/metamonitoring-values-generated/mimir-distributed/templates/metamonitoring/grafana-dashboards.yaml b/operations/helm/tests/metamonitoring-values-generated/mimir-distributed/templates/metamonitoring/grafana-dashboards.yaml index 9faf9c48a83..43c43e17ab7 100644 --- a/operations/helm/tests/metamonitoring-values-generated/mimir-distributed/templates/metamonitoring/grafana-dashboards.yaml +++ b/operations/helm/tests/metamonitoring-values-generated/mimir-distributed/templates/metamonitoring/grafana-dashboards.yaml @@ -29143,7 +29143,7 @@ data: "span": 4, "targets": [ { - "expr": "sum by(user) (rate(cortex_prometheus_notifications_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n /\nsum by(user) (rate(cortex_prometheus_notifications_queue_capacity{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) > 0\n", + "expr": "sum by(user) (cortex_prometheus_notifications_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"})\n /\nsum by(user) (cortex_prometheus_notifications_queue_capacity{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}) > 0\n", "format": "time_series", "legendFormat": "{{ user }}", "legendLink": null diff --git a/operations/mimir-mixin-compiled-baremetal/dashboards/mimir-ruler.json b/operations/mimir-mixin-compiled-baremetal/dashboards/mimir-ruler.json index ea915031e69..e25287f8624 100644 --- a/operations/mimir-mixin-compiled-baremetal/dashboards/mimir-ruler.json +++ b/operations/mimir-mixin-compiled-baremetal/dashboards/mimir-ruler.json @@ -1648,7 +1648,7 @@ "span": 4, "targets": [ { - "expr": "sum by(user) (rate(cortex_prometheus_notifications_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n /\nsum by(user) (rate(cortex_prometheus_notifications_queue_capacity{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) > 0\n", + "expr": "sum by(user) (cortex_prometheus_notifications_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"})\n /\nsum by(user) (cortex_prometheus_notifications_queue_capacity{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}) > 0\n", "format": "time_series", "legendFormat": "{{ user }}", "legendLink": null diff --git a/operations/mimir-mixin-compiled/dashboards/mimir-ruler.json b/operations/mimir-mixin-compiled/dashboards/mimir-ruler.json index ea915031e69..e25287f8624 100644 --- a/operations/mimir-mixin-compiled/dashboards/mimir-ruler.json +++ b/operations/mimir-mixin-compiled/dashboards/mimir-ruler.json @@ -1648,7 +1648,7 @@ "span": 4, "targets": [ { - "expr": "sum by(user) (rate(cortex_prometheus_notifications_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n /\nsum by(user) (rate(cortex_prometheus_notifications_queue_capacity{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) > 0\n", + "expr": "sum by(user) (cortex_prometheus_notifications_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"})\n /\nsum by(user) (cortex_prometheus_notifications_queue_capacity{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}) > 0\n", "format": "time_series", "legendFormat": "{{ user }}", "legendLink": null diff --git a/operations/mimir-mixin/dashboards/ruler.libsonnet b/operations/mimir-mixin/dashboards/ruler.libsonnet index 9190a413f1a..f1e80769a5b 100644 --- a/operations/mimir-mixin/dashboards/ruler.libsonnet +++ b/operations/mimir-mixin/dashboards/ruler.libsonnet @@ -162,9 +162,9 @@ local filename = 'mimir-ruler.json'; .addPanel( $.timeseriesPanel('Queue length') + $.queryPanel(||| - sum by(user) (rate(cortex_prometheus_notifications_queue_length{%s}[$__rate_interval])) + sum by(user) (cortex_prometheus_notifications_queue_length{%s}) / - sum by(user) (rate(cortex_prometheus_notifications_queue_capacity{%s}[$__rate_interval])) > 0 + sum by(user) (cortex_prometheus_notifications_queue_capacity{%s}) > 0 ||| % [$.jobMatcher($._config.job_names.ruler), $.jobMatcher($._config.job_names.ruler)], '{{ user }}') { fieldConfig+: { defaults+: { unit: 'percentunit', min: 0, max: 1 } } }, )