add HA dedupe functionality to docker dev environment

CHANGELOG.md

@@ -64,6 +64,7 @@
     * `cortex_ruler_independent_rule_evaluation_concurrency_attempts_incomplete_total`
     * `cortex_ruler_independent_rule_evaluation_concurrency_attempts_completed_total`
 * [ENHANCEMENT] Expose a new `s3.session-token` configuration option to enable using temporary security credentials. #8952
+* [ENHANCEMENT] Add HA deduplication features to the `mimir-microservices-mode` development environment. #9012
 * [ENHANCEMENT] Make `-query-frontend.additional-query-queue-dimensions-enabled` and `-query-scheduler.additional-query-queue-dimensions-enabled` non-operational flags in preparation for removal. #8984
 * [ENHANCEMENT] Add a new ingester endpoint to prepare instances to downscale. #8956
 * [BUGFIX] Ruler: add support for draining any outstanding alert notifications before shutting down. This can be enabled with the `-ruler.drain-notification-queue-on-shutdown=true` CLI flag. #8346

development/mimir-microservices-mode/config/mimir.yaml

@@ -1,6 +1,10 @@
 multitenancy_enabled: false
 
 distributor:
+  ha_tracker:
+    enable_ha_tracker: true
+    kvstore:
+      store: consul
   pool:
     health_check_ingesters: true
   ring:
@@ -171,6 +175,11 @@ limits:
   native_histograms_ingestion_enabled: true
   cardinality_analysis_enabled: true
   query_ingesters_within: 20m
+  # HA tracker configuration
+  accept_ha_samples: true
+  ha_cluster_label: ha_cluster
+  ha_replica_label: ha_replica
+  ha_max_clusters: 10
 
 runtime_config:
   file: ./config/runtime.yaml

development/mimir-microservices-mode/config/prom-ha-pair-1.yaml

@@ -0,0 +1,35 @@
+global:
+  scrape_interval: 5s
+  external_labels:
+    scraped_by: prometheus
+
+scrape_configs:
+  - job_name: ancillary-services
+    static_configs:
+      - targets:
+          - 'memcached-exporter:9150'
+          - 'load-generator:9900'
+        labels:
+          ha_cluster: 'prom-ha-pair'
+          ha_replica: 'prom-ha-pair-1'
+          namespace: 'mimir-microservices-mode'
+    relabel_configs:
+      - source_labels: ['__address__']
+        target_label: 'pod'
+        regex: '([^:]+)(:[0-9]+)?'
+        replacement: '${1}'
+      - source_labels: ['namespace', 'pod']
+        target_label: 'job'
+        separator: '/'
+        regex: '(.+?)(-\d+)?'
+        replacement: '${1}'
+      - source_labels: ['pod']
+        target_label: 'container'
+        regex: '(.+?)(-\d+)?'
+        replacement: '${1}'
+    scrape_classic_histograms: true
+
+remote_write:
+  - url: http://distributor-2:8001/api/v1/push
+    send_native_histograms: true
+    send_exemplars: true

development/mimir-microservices-mode/config/prom-ha-pair-2.yaml

@@ -0,0 +1,35 @@
+global:
+  scrape_interval: 5s
+  external_labels:
+    scraped_by: prometheus
+
+scrape_configs:
+  - job_name: ancillary-services
+    static_configs:
+      - targets:
+          - 'memcached-exporter:9150'
+          - 'load-generator:9900'
+        labels:
+          ha_cluster: 'prom-ha-pair'
+          ha_replica: 'prom-ha-pair-2'
+          namespace: 'mimir-microservices-mode'
+    relabel_configs:
+      - source_labels: ['__address__']
+        target_label: 'pod'
+        regex: '([^:]+)(:[0-9]+)?'
+        replacement: '${1}'
+      - source_labels: ['namespace', 'pod']
+        target_label: 'job'
+        separator: '/'
+        regex: '(.+?)(-\d+)?'
+        replacement: '${1}'
+      - source_labels: ['pod']
+        target_label: 'container'
+        regex: '(.+?)(-\d+)?'
+        replacement: '${1}'
+    scrape_classic_histograms: true
+
+remote_write:
+  - url: http://distributor-2:8001/api/v1/push
+    send_native_histograms: true
+    send_exemplars: true

development/mimir-microservices-mode/config/prometheus.yaml

@@ -20,8 +20,6 @@ scrape_configs:
           - 'store-gateway-1:8008'
           - 'store-gateway-2:8009'
           - 'query-scheduler:8011'
-          - 'memcached-exporter:9150'
-          - 'load-generator:9900'
         labels:
           cluster: 'docker-compose'
           namespace: 'mimir-microservices-mode'

development/mimir-microservices-mode/docker-compose.jsonnet

@@ -28,6 +28,9 @@ std.manifestYamlDoc({
     // If true, start and enable scraping by these components.
     // Note that if more than one component is enabled, the dashboards shown in Grafana may contain duplicate series or aggregates may be doubled or tripled.
     enable_grafana_agent: false,
+    // If true, start a base prometheus that scrapes the Mimir component metrics and remote writes to distributor-1.
+    // Two additional Prometheus instances are started that scrape the same memcached-exporter and load-generator
+    // targets and remote write to distributor-2.
     enable_prometheus: true,  // If Prometheus is disabled, recording rules will not be evaluated and so dashboards in Grafana that depend on these recorded series will display no data.
     enable_otel_collector: false,
 
@@ -46,12 +49,12 @@ std.manifestYamlDoc({
     self.alertmanagers(3) +
     self.nginx +
     self.minio +
-    (if $._config.enable_prometheus then self.prometheus else {}) +
+    (if $._config.enable_prometheus then self.prometheus + self.prompair1 + self.prompair2 else {}) +
     self.grafana +
     (if $._config.enable_grafana_agent then self.grafana_agent else {}) +
     (if $._config.enable_otel_collector then self.otel_collector else {}) +
     self.jaeger +
-    (if $._config.ring == 'consul' || $._config.ring == 'multi' then self.consul else {}) +
+    self.consul +
     (if $._config.cache_backend == 'redis' then self.redis else self.memcached + self.memcached_exporter) +
     (if $._config.enable_load_generator then self.load_generator else {}) +
     (if $._config.enable_query_tee then self.query_tee else {}) +
@@ -62,12 +65,14 @@ std.manifestYamlDoc({
       name: 'distributor-1',
       target: 'distributor',
       httpPort: 8000,
+      extraArguments: '-distributor.ha-tracker.consul.hostname=consul:8500',
     }),
 
     'distributor-2': mimirService({
       name: 'distributor-2',
       target: 'distributor',
       httpPort: 8001,
+      extraArguments: '-distributor.ha-tracker.consul.hostname=consul:8500',
     }),
   },
 
@@ -250,8 +255,9 @@ std.manifestYamlDoc({
   // Other services used by Mimir.
   consul:: {
     consul: {
-      image: 'consul',
-      command: ['agent', '-dev', '-client=0.0.0.0', '-log-level=info'],
+      image: 'consul:1.15',
+      command: ['agent', '-dev', '-client=0.0.0.0', '-log-level=debug'],
+      hostname: 'consul',
       ports: ['8500:8500'],
     },
   },
@@ -332,6 +338,39 @@ std.manifestYamlDoc({
     },
   },
 
+  prompair1:: {
+    prompair1: {
+      image: 'prom/prometheus:v2.51.1',
+      hostname: 'prom-ha-pair-1',
+      command: [
+        '--config.file=/etc/prometheus/prom-ha-pair-1.yaml',
+        '--enable-feature=exemplar-storage',
+        '--enable-feature=native-histograms',
+      ],
+      volumes: [
+        './config:/etc/prometheus',
+      ],
+      ports: ['9092:9090'],
+    },
+  },
+
+  prompair2:: {
+    prompair2: {
+      image: 'prom/prometheus:v2.51.1',
+      hostname: 'prom-ha-pair-2',
+      command: [
+        '--config.file=/etc/prometheus/prom-ha-pair-2.yaml',
+        '--enable-feature=exemplar-storage',
+        '--enable-feature=native-histograms',
+      ],
+      volumes: [
+        './config:/etc/prometheus',
+      ],
+      ports: ['9093:9090'],
+    },
+  },
+
+
   grafana:: {
     grafana: {
       image: 'grafana/grafana:10.4.3',

development/mimir-microservices-mode/docker-compose.yml

@@ -103,14 +103,24 @@
     "volumes":
       - "./config:/mimir/config"
       - "./activity:/activity"
+  "consul":
+    "command":
+      - "agent"
+      - "-dev"
+      - "-client=0.0.0.0"
+      - "-log-level=debug"
+    "hostname": "consul"
+    "image": "consul:1.15"
+    "ports":
+      - "8500:8500"
   "distributor-1":
     "build":
       "context": "."
       "dockerfile": "dev.dockerfile"
     "command":
       - "sh"
       - "-c"
-      - "sleep 3 && exec ./mimir -config.file=./config/mimir.yaml -target=distributor -server.http-listen-port=8000 -server.grpc-listen-port=9000 -activity-tracker.filepath=/activity/distributor-8000  -memberlist.nodename=distributor -memberlist.bind-port=10000 -ingester.ring.store=memberlist -distributor.ring.store=memberlist -compactor.ring.store=memberlist -store-gateway.sharding-ring.store=memberlist -ruler.ring.store=memberlist -alertmanager.sharding-ring.store=memberlist -blocks-storage.bucket-store.index-cache.backend=memcached -blocks-storage.bucket-store.chunks-cache.backend=memcached -blocks-storage.bucket-store.metadata-cache.backend=memcached -query-frontend.results-cache.backend=memcached -ruler-storage.cache.backend=memcached -blocks-storage.bucket-store.index-cache.memcached.addresses=dns+memcached:11211 -blocks-storage.bucket-store.chunks-cache.memcached.addresses=dns+memcached:11211 -blocks-storage.bucket-store.metadata-cache.memcached.addresses=dns+memcached:11211 -query-frontend.results-cache.memcached.addresses=dns+memcached:11211 -ruler-storage.cache.memcached.addresses=dns+memcached:11211"
+      - "sleep 3 && exec ./mimir -config.file=./config/mimir.yaml -target=distributor -server.http-listen-port=8000 -server.grpc-listen-port=9000 -activity-tracker.filepath=/activity/distributor-8000 -distributor.ha-tracker.consul.hostname=consul:8500 -memberlist.nodename=distributor -memberlist.bind-port=10000 -ingester.ring.store=memberlist -distributor.ring.store=memberlist -compactor.ring.store=memberlist -store-gateway.sharding-ring.store=memberlist -ruler.ring.store=memberlist -alertmanager.sharding-ring.store=memberlist -blocks-storage.bucket-store.index-cache.backend=memcached -blocks-storage.bucket-store.chunks-cache.backend=memcached -blocks-storage.bucket-store.metadata-cache.backend=memcached -query-frontend.results-cache.backend=memcached -ruler-storage.cache.backend=memcached -blocks-storage.bucket-store.index-cache.memcached.addresses=dns+memcached:11211 -blocks-storage.bucket-store.chunks-cache.memcached.addresses=dns+memcached:11211 -blocks-storage.bucket-store.metadata-cache.memcached.addresses=dns+memcached:11211 -query-frontend.results-cache.memcached.addresses=dns+memcached:11211 -ruler-storage.cache.memcached.addresses=dns+memcached:11211"
     "depends_on":
       - "minio"
     "environment":
@@ -135,7 +145,7 @@
     "command":
       - "sh"
       - "-c"
-      - "sleep 3 && exec ./mimir -config.file=./config/mimir.yaml -target=distributor -server.http-listen-port=8001 -server.grpc-listen-port=9001 -activity-tracker.filepath=/activity/distributor-8001  -memberlist.nodename=distributor -memberlist.bind-port=10001 -ingester.ring.store=memberlist -distributor.ring.store=memberlist -compactor.ring.store=memberlist -store-gateway.sharding-ring.store=memberlist -ruler.ring.store=memberlist -alertmanager.sharding-ring.store=memberlist -blocks-storage.bucket-store.index-cache.backend=memcached -blocks-storage.bucket-store.chunks-cache.backend=memcached -blocks-storage.bucket-store.metadata-cache.backend=memcached -query-frontend.results-cache.backend=memcached -ruler-storage.cache.backend=memcached -blocks-storage.bucket-store.index-cache.memcached.addresses=dns+memcached:11211 -blocks-storage.bucket-store.chunks-cache.memcached.addresses=dns+memcached:11211 -blocks-storage.bucket-store.metadata-cache.memcached.addresses=dns+memcached:11211 -query-frontend.results-cache.memcached.addresses=dns+memcached:11211 -ruler-storage.cache.memcached.addresses=dns+memcached:11211"
+      - "sleep 3 && exec ./mimir -config.file=./config/mimir.yaml -target=distributor -server.http-listen-port=8001 -server.grpc-listen-port=9001 -activity-tracker.filepath=/activity/distributor-8001 -distributor.ha-tracker.consul.hostname=consul:8500 -memberlist.nodename=distributor -memberlist.bind-port=10001 -ingester.ring.store=memberlist -distributor.ring.store=memberlist -compactor.ring.store=memberlist -store-gateway.sharding-ring.store=memberlist -ruler.ring.store=memberlist -alertmanager.sharding-ring.store=memberlist -blocks-storage.bucket-store.index-cache.backend=memcached -blocks-storage.bucket-store.chunks-cache.backend=memcached -blocks-storage.bucket-store.metadata-cache.backend=memcached -query-frontend.results-cache.backend=memcached -ruler-storage.cache.backend=memcached -blocks-storage.bucket-store.index-cache.memcached.addresses=dns+memcached:11211 -blocks-storage.bucket-store.chunks-cache.memcached.addresses=dns+memcached:11211 -blocks-storage.bucket-store.metadata-cache.memcached.addresses=dns+memcached:11211 -query-frontend.results-cache.memcached.addresses=dns+memcached:11211 -ruler-storage.cache.memcached.addresses=dns+memcached:11211"
     "depends_on":
       - "minio"
     "environment":
@@ -314,6 +324,28 @@
       - "./config:/etc/prometheus"
       - "../../operations/mimir-mixin-compiled/alerts.yaml:/etc/mixin/mimir-alerts.yaml"
       - "../../operations/mimir-mixin-compiled/rules.yaml:/etc/mixin/mimir-rules.yaml"
+  "prompair1":
+    "command":
+      - "--config.file=/etc/prometheus/prom-ha-pair-1.yaml"
+      - "--enable-feature=exemplar-storage"
+      - "--enable-feature=native-histograms"
+    "hostname": "prom-ha-pair-1"
+    "image": "prom/prometheus:v2.51.1"
+    "ports":
+      - "9092:9090"
+    "volumes":
+      - "./config:/etc/prometheus"
+  "prompair2":
+    "command":
+      - "--config.file=/etc/prometheus/prom-ha-pair-2.yaml"
+      - "--enable-feature=exemplar-storage"
+      - "--enable-feature=native-histograms"
+    "hostname": "prom-ha-pair-2"
+    "image": "prom/prometheus:v2.51.1"
+    "ports":
+      - "9093:9090"
+    "volumes":
+      - "./config:/etc/prometheus"
   "querier":
     "build":
       "context": "."