Remote ruler reads dashboard: allow using cortex_request_duration_sec…

…onds native histogram

Signed-off-by: Yuri Nikolic <durica.nikolic@grafana.com>

CHANGELOG.md

@@ -68,6 +68,7 @@
   * Writes dashboard: `cortex_request_duration_seconds` metric. #8757 #8791
   * Reads dashboard: `cortex_request_duration_seconds` metric. #8752
   * Rollout progress dashboard. #8779
+  * Remote ruler reads dashboard: `cortex_request_duration_seconds` metric.
 * [ENHANCEMENT] Alerts: `MimirRunningIngesterReceiveDelayTooHigh` alert has been tuned to be more reactive to high receive delay. #8538
 * [ENHANCEMENT] Dashboards: improve end-to-end latency and strong read consistency panels when experimental ingest storage is enabled. #8543
 * [ENHANCEMENT] Dashboards: Add panels for monitoring ingester autoscaling when not using ingest-storage. These panels are disabled by default, but can be enabled using the `autoscaling.ingester.enabled: true` config option. #8484

operations/helm/charts/mimir-distributed/CHANGELOG.md

@@ -34,6 +34,7 @@ Entries should include a reference to the Pull Request that introduced the chang
   * Writes dashboard: `cortex_request_duration_seconds` metric. #8757
   * Reads dashboard: `cortex_request_duration_seconds` metric. #8752
   * Rollout progress dashboard. #8779
+  * Remote ruler reads dashboard: `cortex_request_duration_seconds` metric.
 * [ENHANCEMENT] Memcached: Update to Memcached 1.6.28 and memcached-exporter 0.14.4. #8557
 * [ENHANCEMENT] Add missing fields in multiple topology spread constraints. #8533
 * [ENHANCEMENT] Add support for setting the image pull secrets, node selectors, tolerations and topology spread constraints for the Grafana Agent pods used for metamonitoring. #8670

operations/mimir-mixin/dashboards/dashboard-queries.libsonnet

@@ -82,6 +82,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
   queries:: {
     // Define the supported replacement variables in a single place. Most of them are frequently used.
     local variables = {
+      requestsPerSecondMetric: $.requests_per_second_metric,
       gatewayMatcher: $.jobMatcher($._config.job_names.gateway),
       distributorMatcher: $.jobMatcher($._config.job_names.distributor),
       ingesterMatcher: $.jobMatcher($._config.job_names.ingester),
@@ -90,29 +91,34 @@ local utils = import 'mixin-utils/utils.libsonnet';
       alertmanagerMatcher: $.jobMatcher($._config.job_names.alertmanager),
       namespaceMatcher: $.namespaceMatcher(),
       storeGatewayMatcher: $.jobMatcher($._config.job_names.store_gateway),
+      rulerQueryFrontendMatcher: $.jobMatcher($._config.job_names.ruler_query_frontend),
       writeHTTPRoutesRegex: $.queries.write_http_routes_regex,
       writeDistributorRoutesRegex: std.join('|', [$.queries.write_grpc_distributor_routes_regex, $.queries.write_http_routes_regex]),
       writeGRPCIngesterRoute: $.queries.write_grpc_ingester_route,
       readHTTPRoutesRegex: $.queries.read_http_routes_regex,
       readGRPCIngesterRoute: $.queries.read_grpc_ingester_route,
       readGRPCStoreGatewayRoute: $.queries.read_grpc_store_gateway_route,
+      rulerQueryFrontendRoutesRegex: $.queries.ruler_query_frontend_routes_regex,
       perClusterLabel: $._config.per_cluster_label,
       recordingRulePrefix: $.recordingRulePrefix($.jobSelector('any')),  // The job name does not matter here.
       groupPrefixJobs: $._config.group_prefix_jobs,
       instance: $._config.per_instance_label,
     },
 
+    requests_per_second_metric: 'cortex_request_duration_seconds',
     write_http_routes_regex: 'api_(v1|prom)_push|otlp_v1_metrics',
     write_grpc_distributor_routes_regex: '/distributor.Distributor/Push|/httpgrpc.*',
     write_grpc_ingester_route: '/cortex.Ingester/Push',
     read_http_routes_regex: '(prometheus|api_prom)_api_v1_.+',
     read_grpc_ingester_route: $._config.ingester_read_path_routes_regex,
     read_grpc_store_gateway_route: $._config.store_gateway_read_path_routes_regex,
     query_http_routes_regex: '(prometheus|api_prom)_api_v1_query(_range)?',
+    // Both support gRPC and HTTP requests. HTTP request is used when rule evaluation query requests go through the query-tee.
+    ruler_query_frontend_routes_regex: '/httpgrpc.HTTP/Handle|.*api_v1_query',
 
     gateway: {
       local p = self,
-      requestsPerSecondMetric: 'cortex_request_duration_seconds',
+      requestsPerSecondMetric: $.queries.requests_per_second_metric,
       writeRequestsPerSecondSelector: '%(gatewayMatcher)s, route=~"%(writeHTTPRoutesRegex)s"' % variables,
       readRequestsPerSecondSelector: '%(gatewayMatcher)s, route=~"%(readHTTPRoutesRegex)s"' % variables,
 
@@ -125,7 +131,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
 
     distributor: {
       local p = self,
-      requestsPerSecondMetric: 'cortex_request_duration_seconds',
+      requestsPerSecondMetric: $.queries.requests_per_second_metric,
       writeRequestsPerSecondRouteRegex: '%(writeDistributorRoutesRegex)s' % variables,
       writeRequestsPerSecondSelector: '%(distributorMatcher)s, route=~"%(writeDistributorRoutesRegex)s"' % variables,
       samplesPerSecond: 'sum(%(groupPrefixJobs)s:cortex_distributor_received_samples:rate5m{%(distributorMatcher)s})' % variables,
@@ -137,7 +143,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
 
     query_frontend: {
       local p = self,
-      requestsPerSecondMetric: 'cortex_request_duration_seconds',
+      requestsPerSecondMetric: $.queries.requests_per_second_metric,
       readRequestsPerSecondSelector: '%(queryFrontendMatcher)s, route=~"%(readHTTPRoutesRegex)s"' % variables,
       // These query routes are used in the overview and other dashboard, everythign else is considered "other" queries.
       // Has to be a list to keep the same colors as before, see overridesNonErrorColorsPalette.
@@ -196,6 +202,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
     },
 
     ruler: {
+      requestsPerSecondMetric: $.queries.requests_per_second_metric,
       evaluations: {
         successPerSecond:
           |||
@@ -289,7 +296,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
     },
 
     ingester: {
-      requestsPerSecondMetric: 'cortex_request_duration_seconds',
+      requestsPerSecondMetric: $.queries.requests_per_second_metric,
       readRequestsPerSecondSelector: '%(ingesterMatcher)s,route=~"%(readGRPCIngesterRoute)s"' % variables,
       writeRequestsPerSecondSelector: '%(ingesterMatcher)s, route="%(writeGRPCIngesterRoute)s"' % variables,
 
@@ -320,8 +327,13 @@ local utils = import 'mixin-utils/utils.libsonnet';
     },
 
     store_gateway: {
-      requestsPerSecondMetric: 'cortex_request_duration_seconds',
+      requestsPerSecondMetric: $.queries.requests_per_second_metric,
       readRequestsPerSecondSelector: '%(storeGatewayMatcher)s,route=~"%(readGRPCStoreGatewayRoute)s"' % variables,
     },
+
+    ruler_query_frontend: {
+      requestsPerSecondMetric: $.queries.requests_per_second_metric,
+      readRequestsPerSecondSelector: '%(rulerQueryFrontendMatcher)s,route=~"%(rulerQueryFrontendRoutesRegex)s"' % variables,
+    },
   },
 }

operations/mimir-mixin/dashboards/remote-ruler-reads.libsonnet

@@ -3,9 +3,6 @@ local filename = 'mimir-remote-ruler-reads.json';
 
 (import 'dashboard-utils.libsonnet') +
 (import 'dashboard-queries.libsonnet') {
-  // Both support gRPC and HTTP requests. HTTP request is used when rule evaluation query requests go through the query-tee.
-  local rulerRoutesRegex = '/httpgrpc.HTTP/Handle|.*api_v1_query',
-
   [filename]:
     assert std.md5(filename) == 'f103238f7f5ab2f1345ce650cbfbfe2f' : 'UID of the dashboard has changed, please update references to dashboard.';
     ($.dashboard('Remote ruler reads') + { uid: std.md5(filename) })
@@ -33,19 +30,7 @@ local filename = 'mimir-remote-ruler-reads.json';
        })
       .addPanel(
         $.panel('Evaluations / sec') +
-        $.statPanel(|||
-          sum(
-            rate(
-              cortex_request_duration_seconds_count{
-                %(queryFrontend)s,
-                route=~"%(rulerRoutesRegex)s"
-              }[$__rate_interval]
-            )
-          )
-        ||| % {
-          queryFrontend: $.jobMatcher($._config.job_names.ruler_query_frontend),
-          rulerRoutesRegex: rulerRoutesRegex,
-        }, format='reqps') +
+        $.statPanel(utils.ncHistogramSumBy(utils.ncHistogramCountRate($.queries.ruler_query_frontend.requestsPerSecondMetric, $.queries.ruler_query_frontend.readRequestsPerSecondSelector)), format='reqps') +
         $.panelDescription(
           'Evaluations per second',
           |||
@@ -58,7 +43,7 @@ local filename = 'mimir-remote-ruler-reads.json';
       queryFrontendJobName=$._config.job_names.ruler_query_frontend,
       querySchedulerJobName=$._config.job_names.ruler_query_scheduler,
       querierJobName=$._config.job_names.ruler_querier,
-      queryRoutesRegex=rulerRoutesRegex,
+      queryRoutesRegex=$.queries.ruler_query_frontend_routes_regex,
 
       rowTitlePrefix='Ruler-',
     ))

operations/mimir-mixin/jsonnetfile.lock.json

@@ -8,7 +8,7 @@
           "subdir": "grafana-builder"
         }
       },
-      "version": "1d877bb0651ef92176f651d0be473c06e372a8a0",
+      "version": "33fd33f2cd59a045ce4580f0b5a7e06a620e8d37",
       "sum": "udZaafkbKYMGodLqsFhEe+Oy/St2p0edrK7hiMPEey0="
     },
     {
@@ -18,7 +18,7 @@
           "subdir": "mixin-utils"
         }
       },
-      "version": "1d877bb0651ef92176f651d0be473c06e372a8a0",
+      "version": "33fd33f2cd59a045ce4580f0b5a7e06a620e8d37",
       "sum": "mzLmCv9n3ldLChVGPfyRJOVKoBw+dfK40vU9792aHIM="
     }
   ],