From 71e3e7c16d05fe2ece89a65125c27619724c0c2e Mon Sep 17 00:00:00 2001 From: Nick Pillitteri <56quarters@users.noreply.github.com> Date: Wed, 25 May 2022 04:55:48 -0400 Subject: [PATCH] Change shuffle sharding ingester lookback default config (#1921) * Change shuffle sharding ingester lookback default config Use the same default value for ingester lookback as the "query ingesters within" setting to reduce the number of things that need to be changed from their defaults. This change also removes use of the `-blocks-storage.tsdb.close-idle-tsdb-timeout` flag in jsonnet since the value being used matches the default. Follow up to #1915 Signed-off-by: Nick Pillitteri * Changelog Signed-off-by: Nick Pillitteri --- CHANGELOG.md | 5 +++-- cmd/mimir/config-descriptor.json | 4 ++-- cmd/mimir/help-all.txt.tmpl | 2 +- .../reference-configuration-parameters/index.md | 11 +++++------ operations/mimir-tests/test-defaults-generated.yaml | 2 -- .../test-disable-chunk-streaming-generated.yaml | 3 --- operations/mimir-tests/test-gossip-generated.yaml | 3 --- .../mimir-tests/test-gossip-multi-zone-generated.yaml | 5 ----- .../mimir-tests/test-gossip-multikv-generated.yaml | 3 --- ...ip-multikv-switch-primary-secondary-generated.yaml | 3 --- .../test-gossip-multikv-teardown-generated.yaml | 3 --- .../test-gossip-ruler-disabled-generated.yaml | 2 -- operations/mimir-tests/test-multi-zone-generated.yaml | 5 ----- ...t-multi-zone-with-ongoing-migration-generated.yaml | 6 ------ .../mimir-tests/test-query-sharding-generated.yaml | 3 --- .../mimir-tests/test-shuffle-sharding-generated.yaml | 5 ----- .../mimir-tests/test-storage-azure-generated.yaml | 3 --- .../mimir-tests/test-storage-gcs-generated.yaml | 3 --- operations/mimir-tests/test-storage-s3-generated.yaml | 3 --- operations/mimir/config.libsonnet | 3 --- operations/mimir/ingester.libsonnet | 3 --- operations/mimir/shuffle-sharding.libsonnet | 2 -- pkg/querier/querier.go | 9 +++++---- 23 files changed, 16 insertions(+), 75 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e17d1b4893a..75515e55c36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,9 +5,10 @@ ### Grafana Mimir * [CHANGE] Increased default configuration for `-server.grpc-max-recv-msg-size-bytes` and `-server.grpc-max-send-msg-size-bytes` from 4MB to 100MB. #1883 -* [CHANGE] Default values have changed for the following settings. This improves query performance for recent data (within 12h) by only reading from ingesters: #1909 +* [CHANGE] Default values have changed for the following settings. This improves query performance for recent data (within 12h) by only reading from ingesters: #1909 #1921 - `-blocks-storage.bucket-store.ignore-blocks-within` now defaults to `10h` (previously `0`) - `-querier.query-store-after` now defaults to `12h` (previously `0`) + - `-querier.shuffle-sharding-ingesters-lookback-period` now defaults to `13h` (previously `0`) * [ENHANCEMENT] Store-gateway: Add the experimental ability to run requests in a dedicated OS thread pool. This feature can be configured using `-store-gateway.thread-pool-size` and is disabled by default. Replaces the ability to run index header operations in a dedicated thread pool. #1660 #1812 * [ENHANCEMENT] Improved error messages to make them easier to understand and referencing a unique global identifier that can be looked up in the runbooks. #1907 * [ENHANCEMENT] Memberlist KV: incoming messages are now processed on per-key goroutine. This may reduce loss of "maintanance" packets in busy memberlist installations, but use more CPU. New `memberlist_client_received_broadcasts_dropped_total` counter tracks number of dropped per-key messages. #1912 @@ -27,7 +28,7 @@ ### Jsonnet -* [CHANGE] Remove use of `-querier.query-store-after` and `-blocks-storage.bucket-store.ignore-blocks-within` CLI flags since the values now match defaults. #1915 +* [CHANGE] Remove use of `-querier.query-store-after`, `-querier.shuffle-sharding-ingesters-lookback-period`, `-blocks-storage.bucket-store.ignore-blocks-within`, and `-blocks-storage.tsdb.close-idle-tsdb-timeout` CLI flags since the values now match defaults. #1915 #1921 ## 2.1.0-rc.0 diff --git a/cmd/mimir/config-descriptor.json b/cmd/mimir/config-descriptor.json index ac8db3fbc07..d84afca7c09 100644 --- a/cmd/mimir/config-descriptor.json +++ b/cmd/mimir/config-descriptor.json @@ -1480,9 +1480,9 @@ "kind": "field", "name": "shuffle_sharding_ingesters_lookback_period", "required": false, - "desc": "When distributor's sharding strategy is shuffle-sharding and this setting is \u003e 0, queriers fetch in-memory series from the minimum set of required ingesters, selecting only ingesters which may have received series since 'now - lookback period'. The lookback period should be greater or equal than the configured -querier.query-store-after and -querier.query-ingesters-within. If this setting is 0, queriers always query all ingesters (ingesters shuffle sharding on read path is disabled).", + "desc": "When this setting is \u003e 0, queriers fetch in-memory series from the minimum set of required ingesters, selecting only ingesters which may have received series since 'now - lookback period'. The lookback period should be greater or equal than the configured -querier.query-store-after and -querier.query-ingesters-within. If this setting is 0, queriers always query all ingesters (ingesters shuffle sharding on read path is disabled).", "fieldValue": null, - "fieldDefaultValue": 0, + "fieldDefaultValue": 46800000000000, "fieldFlag": "querier.shuffle-sharding-ingesters-lookback-period", "fieldType": "duration", "fieldCategory": "advanced" diff --git a/cmd/mimir/help-all.txt.tmpl b/cmd/mimir/help-all.txt.tmpl index 4ead10068fe..b675d93761b 100644 --- a/cmd/mimir/help-all.txt.tmpl +++ b/cmd/mimir/help-all.txt.tmpl @@ -1086,7 +1086,7 @@ Usage of ./cmd/mimir/mimir: -querier.scheduler-address string Address of the query-scheduler component, in host:port format. Only one of -querier.frontend-address or -querier.scheduler-address can be set. If neither is set, queries are only received via HTTP endpoint. -querier.shuffle-sharding-ingesters-lookback-period duration - When distributor's sharding strategy is shuffle-sharding and this setting is > 0, queriers fetch in-memory series from the minimum set of required ingesters, selecting only ingesters which may have received series since 'now - lookback period'. The lookback period should be greater or equal than the configured -querier.query-store-after and -querier.query-ingesters-within. If this setting is 0, queriers always query all ingesters (ingesters shuffle sharding on read path is disabled). + When this setting is > 0, queriers fetch in-memory series from the minimum set of required ingesters, selecting only ingesters which may have received series since 'now - lookback period'. The lookback period should be greater or equal than the configured -querier.query-store-after and -querier.query-ingesters-within. If this setting is 0, queriers always query all ingesters (ingesters shuffle sharding on read path is disabled). (default 13h0m0s) -querier.store-gateway-client.tls-ca-path string Path to the CA certificates file to validate server certificate against. If not set, the host's root CA certificates are used. -querier.store-gateway-client.tls-cert-path string diff --git a/docs/sources/operators-guide/configuring/reference-configuration-parameters/index.md b/docs/sources/operators-guide/configuring/reference-configuration-parameters/index.md index 44d3c5df070..4b09fd0fd82 100644 --- a/docs/sources/operators-guide/configuring/reference-configuration-parameters/index.md +++ b/docs/sources/operators-guide/configuring/reference-configuration-parameters/index.md @@ -835,15 +835,14 @@ store_gateway_client: # CLI flag: -querier.store-gateway-client.tls-insecure-skip-verify [tls_insecure_skip_verify: | default = false] -# (advanced) When distributor's sharding strategy is shuffle-sharding and this -# setting is > 0, queriers fetch in-memory series from the minimum set of -# required ingesters, selecting only ingesters which may have received series -# since 'now - lookback period'. The lookback period should be greater or equal -# than the configured -querier.query-store-after and +# (advanced) When this setting is > 0, queriers fetch in-memory series from the +# minimum set of required ingesters, selecting only ingesters which may have +# received series since 'now - lookback period'. The lookback period should be +# greater or equal than the configured -querier.query-store-after and # -querier.query-ingesters-within. If this setting is 0, queriers always query # all ingesters (ingesters shuffle sharding on read path is disabled). # CLI flag: -querier.shuffle-sharding-ingesters-lookback-period -[shuffle_sharding_ingesters_lookback_period: | default = 0s] +[shuffle_sharding_ingesters_lookback_period: | default = 13h] # The maximum number of concurrent queries. This config option should be set on # query-frontend too when query sharding is enabled. diff --git a/operations/mimir-tests/test-defaults-generated.yaml b/operations/mimir-tests/test-defaults-generated.yaml index a256d40fc90..9a037a5a48f 100644 --- a/operations/mimir-tests/test-defaults-generated.yaml +++ b/operations/mimir-tests/test-defaults-generated.yaml @@ -762,7 +762,6 @@ spec: - -querier.frontend-address=query-frontend-discovery.default.svc.cluster.local:9095 - -querier.frontend-client.grpc-max-send-msg-size=104857600 - -querier.max-concurrent=8 - - -querier.query-ingesters-within=13h - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true @@ -999,7 +998,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true diff --git a/operations/mimir-tests/test-disable-chunk-streaming-generated.yaml b/operations/mimir-tests/test-disable-chunk-streaming-generated.yaml index 829c12673c0..6b0b58de037 100644 --- a/operations/mimir-tests/test-disable-chunk-streaming-generated.yaml +++ b/operations/mimir-tests/test-disable-chunk-streaming-generated.yaml @@ -850,7 +850,6 @@ spec: - -mem-ballast-size-bytes=268435456 - -querier.frontend-client.grpc-max-send-msg-size=104857600 - -querier.max-concurrent=8 - - -querier.query-ingesters-within=13h - -querier.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc.keepalive.min-time-between-pings=10s @@ -1073,7 +1072,6 @@ spec: - -ingester.ring.prefix= - -ingester.ring.replication-factor=3 - -ingester.ring.store=consul - - -querier.query-ingesters-within=13h - -ruler-storage.backend=gcs - -ruler-storage.gcs.bucket-name=rules-bucket - -ruler.alertmanager-url=http://alertmanager.default.svc.cluster.local/alertmanager @@ -1328,7 +1326,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true diff --git a/operations/mimir-tests/test-gossip-generated.yaml b/operations/mimir-tests/test-gossip-generated.yaml index 7cd36a728b5..af95336873f 100644 --- a/operations/mimir-tests/test-gossip-generated.yaml +++ b/operations/mimir-tests/test-gossip-generated.yaml @@ -883,7 +883,6 @@ spec: - -memberlist.join=gossip-ring.default.svc.cluster.local:7946 - -querier.frontend-client.grpc-max-send-msg-size=104857600 - -querier.max-concurrent=8 - - -querier.query-ingesters-within=13h - -querier.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc.keepalive.min-time-between-pings=10s @@ -1110,7 +1109,6 @@ spec: - -memberlist.abort-if-join-fails=false - -memberlist.bind-port=7946 - -memberlist.join=gossip-ring.default.svc.cluster.local:7946 - - -querier.query-ingesters-within=13h - -ruler-storage.backend=gcs - -ruler-storage.gcs.bucket-name=rules-bucket - -ruler.alertmanager-url=http://alertmanager.default.svc.cluster.local/alertmanager @@ -1369,7 +1367,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true diff --git a/operations/mimir-tests/test-gossip-multi-zone-generated.yaml b/operations/mimir-tests/test-gossip-multi-zone-generated.yaml index 31766e6869f..44f2b018644 100644 --- a/operations/mimir-tests/test-gossip-multi-zone-generated.yaml +++ b/operations/mimir-tests/test-gossip-multi-zone-generated.yaml @@ -1047,7 +1047,6 @@ spec: - -memberlist.join=gossip-ring.default.svc.cluster.local:7946 - -querier.frontend-client.grpc-max-send-msg-size=104857600 - -querier.max-concurrent=8 - - -querier.query-ingesters-within=13h - -querier.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc.keepalive.min-time-between-pings=10s @@ -1321,7 +1320,6 @@ spec: - -memberlist.abort-if-join-fails=false - -memberlist.bind-port=7946 - -memberlist.join=gossip-ring.default.svc.cluster.local:7946 - - -querier.query-ingesters-within=13h - -ruler-storage.backend=gcs - -ruler-storage.gcs.bucket-name=rules-bucket - -ruler.alertmanager-url=http://alertmanager.default.svc.cluster.local/alertmanager @@ -1592,7 +1590,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true @@ -1710,7 +1707,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true @@ -1828,7 +1824,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true diff --git a/operations/mimir-tests/test-gossip-multikv-generated.yaml b/operations/mimir-tests/test-gossip-multikv-generated.yaml index fc2db5b546a..5daddccab1f 100644 --- a/operations/mimir-tests/test-gossip-multikv-generated.yaml +++ b/operations/mimir-tests/test-gossip-multikv-generated.yaml @@ -895,7 +895,6 @@ spec: - -memberlist.join=gossip-ring.default.svc.cluster.local:7946 - -querier.frontend-client.grpc-max-send-msg-size=104857600 - -querier.max-concurrent=8 - - -querier.query-ingesters-within=13h - -querier.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc.keepalive.min-time-between-pings=10s @@ -1128,7 +1127,6 @@ spec: - -memberlist.abort-if-join-fails=false - -memberlist.bind-port=7946 - -memberlist.join=gossip-ring.default.svc.cluster.local:7946 - - -querier.query-ingesters-within=13h - -ruler-storage.backend=gcs - -ruler-storage.gcs.bucket-name=rules-bucket - -ruler.alertmanager-url=http://alertmanager.default.svc.cluster.local/alertmanager @@ -1396,7 +1394,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true diff --git a/operations/mimir-tests/test-gossip-multikv-switch-primary-secondary-generated.yaml b/operations/mimir-tests/test-gossip-multikv-switch-primary-secondary-generated.yaml index e252398ed27..60bd1196a4f 100644 --- a/operations/mimir-tests/test-gossip-multikv-switch-primary-secondary-generated.yaml +++ b/operations/mimir-tests/test-gossip-multikv-switch-primary-secondary-generated.yaml @@ -895,7 +895,6 @@ spec: - -memberlist.join=gossip-ring.default.svc.cluster.local:7946 - -querier.frontend-client.grpc-max-send-msg-size=104857600 - -querier.max-concurrent=8 - - -querier.query-ingesters-within=13h - -querier.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc.keepalive.min-time-between-pings=10s @@ -1128,7 +1127,6 @@ spec: - -memberlist.abort-if-join-fails=false - -memberlist.bind-port=7946 - -memberlist.join=gossip-ring.default.svc.cluster.local:7946 - - -querier.query-ingesters-within=13h - -ruler-storage.backend=gcs - -ruler-storage.gcs.bucket-name=rules-bucket - -ruler.alertmanager-url=http://alertmanager.default.svc.cluster.local/alertmanager @@ -1396,7 +1394,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true diff --git a/operations/mimir-tests/test-gossip-multikv-teardown-generated.yaml b/operations/mimir-tests/test-gossip-multikv-teardown-generated.yaml index 100e97142c3..ff3aa8b4576 100644 --- a/operations/mimir-tests/test-gossip-multikv-teardown-generated.yaml +++ b/operations/mimir-tests/test-gossip-multikv-teardown-generated.yaml @@ -886,7 +886,6 @@ spec: - -memberlist.join=gossip-ring.default.svc.cluster.local:7946 - -querier.frontend-client.grpc-max-send-msg-size=104857600 - -querier.max-concurrent=8 - - -querier.query-ingesters-within=13h - -querier.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc.keepalive.min-time-between-pings=10s @@ -1113,7 +1112,6 @@ spec: - -memberlist.abort-if-join-fails=false - -memberlist.bind-port=7946 - -memberlist.join=gossip-ring.default.svc.cluster.local:7946 - - -querier.query-ingesters-within=13h - -ruler-storage.backend=gcs - -ruler-storage.gcs.bucket-name=rules-bucket - -ruler.alertmanager-url=http://alertmanager.default.svc.cluster.local/alertmanager @@ -1372,7 +1370,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true diff --git a/operations/mimir-tests/test-gossip-ruler-disabled-generated.yaml b/operations/mimir-tests/test-gossip-ruler-disabled-generated.yaml index e3878357657..644f721bdad 100644 --- a/operations/mimir-tests/test-gossip-ruler-disabled-generated.yaml +++ b/operations/mimir-tests/test-gossip-ruler-disabled-generated.yaml @@ -865,7 +865,6 @@ spec: - -memberlist.join=gossip-ring.default.svc.cluster.local:7946 - -querier.frontend-client.grpc-max-send-msg-size=104857600 - -querier.max-concurrent=8 - - -querier.query-ingesters-within=13h - -querier.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc.keepalive.min-time-between-pings=10s @@ -1257,7 +1256,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true diff --git a/operations/mimir-tests/test-multi-zone-generated.yaml b/operations/mimir-tests/test-multi-zone-generated.yaml index ade67c29432..74eacd0158d 100644 --- a/operations/mimir-tests/test-multi-zone-generated.yaml +++ b/operations/mimir-tests/test-multi-zone-generated.yaml @@ -1001,7 +1001,6 @@ spec: - -mem-ballast-size-bytes=268435456 - -querier.frontend-client.grpc-max-send-msg-size=104857600 - -querier.max-concurrent=8 - - -querier.query-ingesters-within=13h - -querier.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc.keepalive.min-time-between-pings=10s @@ -1271,7 +1270,6 @@ spec: - -ingester.ring.replication-factor=3 - -ingester.ring.store=consul - -ingester.ring.zone-awareness-enabled=true - - -querier.query-ingesters-within=13h - -ruler-storage.backend=gcs - -ruler-storage.gcs.bucket-name=rules-bucket - -ruler.alertmanager-url=http://alertmanager.default.svc.cluster.local/alertmanager @@ -1538,7 +1536,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true @@ -1651,7 +1648,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true @@ -1764,7 +1760,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true diff --git a/operations/mimir-tests/test-multi-zone-with-ongoing-migration-generated.yaml b/operations/mimir-tests/test-multi-zone-with-ongoing-migration-generated.yaml index 209abf90586..dff2b7ed592 100644 --- a/operations/mimir-tests/test-multi-zone-with-ongoing-migration-generated.yaml +++ b/operations/mimir-tests/test-multi-zone-with-ongoing-migration-generated.yaml @@ -1063,7 +1063,6 @@ spec: - -mem-ballast-size-bytes=268435456 - -querier.frontend-client.grpc-max-send-msg-size=104857600 - -querier.max-concurrent=8 - - -querier.query-ingesters-within=13h - -querier.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc.keepalive.min-time-between-pings=10s @@ -1333,7 +1332,6 @@ spec: - -ingester.ring.replication-factor=3 - -ingester.ring.store=consul - -ingester.ring.zone-awareness-enabled=true - - -querier.query-ingesters-within=13h - -ruler-storage.backend=gcs - -ruler-storage.gcs.bucket-name=rules-bucket - -ruler.alertmanager-url=http://alertmanager.default.svc.cluster.local/alertmanager @@ -1589,7 +1587,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true @@ -1700,7 +1697,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true @@ -1813,7 +1809,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true @@ -1926,7 +1921,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true diff --git a/operations/mimir-tests/test-query-sharding-generated.yaml b/operations/mimir-tests/test-query-sharding-generated.yaml index 54d19f0bfef..f6baabf833a 100644 --- a/operations/mimir-tests/test-query-sharding-generated.yaml +++ b/operations/mimir-tests/test-query-sharding-generated.yaml @@ -849,7 +849,6 @@ spec: - -mem-ballast-size-bytes=268435456 - -querier.frontend-client.grpc-max-send-msg-size=419430400 - -querier.max-concurrent=16 - - -querier.query-ingesters-within=13h - -querier.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc.keepalive.min-time-between-pings=10s @@ -1077,7 +1076,6 @@ spec: - -ingester.ring.prefix= - -ingester.ring.replication-factor=3 - -ingester.ring.store=consul - - -querier.query-ingesters-within=13h - -ruler-storage.backend=gcs - -ruler-storage.gcs.bucket-name=rules-bucket - -ruler.alertmanager-url=http://alertmanager.default.svc.cluster.local/alertmanager @@ -1332,7 +1330,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true diff --git a/operations/mimir-tests/test-shuffle-sharding-generated.yaml b/operations/mimir-tests/test-shuffle-sharding-generated.yaml index d24afce09a6..693378b4e71 100644 --- a/operations/mimir-tests/test-shuffle-sharding-generated.yaml +++ b/operations/mimir-tests/test-shuffle-sharding-generated.yaml @@ -851,9 +851,7 @@ spec: - -mem-ballast-size-bytes=268435456 - -querier.frontend-client.grpc-max-send-msg-size=104857600 - -querier.max-concurrent=8 - - -querier.query-ingesters-within=13h - -querier.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - - -querier.shuffle-sharding-ingesters-lookback-period=13h - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true @@ -1079,8 +1077,6 @@ spec: - -ingester.ring.prefix= - -ingester.ring.replication-factor=3 - -ingester.ring.store=consul - - -querier.query-ingesters-within=13h - - -querier.shuffle-sharding-ingesters-lookback-period=13h - -ruler-storage.backend=gcs - -ruler-storage.gcs.bucket-name=rules-bucket - -ruler.alertmanager-url=http://alertmanager.default.svc.cluster.local/alertmanager @@ -1337,7 +1333,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true diff --git a/operations/mimir-tests/test-storage-azure-generated.yaml b/operations/mimir-tests/test-storage-azure-generated.yaml index 392a0f2d30b..fbaa05e9cf3 100644 --- a/operations/mimir-tests/test-storage-azure-generated.yaml +++ b/operations/mimir-tests/test-storage-azure-generated.yaml @@ -851,7 +851,6 @@ spec: - -mem-ballast-size-bytes=268435456 - -querier.frontend-client.grpc-max-send-msg-size=104857600 - -querier.max-concurrent=8 - - -querier.query-ingesters-within=13h - -querier.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc.keepalive.min-time-between-pings=10s @@ -1076,7 +1075,6 @@ spec: - -ingester.ring.prefix= - -ingester.ring.replication-factor=3 - -ingester.ring.store=consul - - -querier.query-ingesters-within=13h - -ruler-storage.azure.account-key=rules-account-key - -ruler-storage.azure.account-name=rules-account-name - -ruler-storage.azure.container-name=rules-bucket @@ -1339,7 +1337,6 @@ spec: - -blocks-storage.azure.container-name=blocks-bucket - -blocks-storage.backend=azure - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true diff --git a/operations/mimir-tests/test-storage-gcs-generated.yaml b/operations/mimir-tests/test-storage-gcs-generated.yaml index e7d3e6bb1c3..f5eca1a0fd2 100644 --- a/operations/mimir-tests/test-storage-gcs-generated.yaml +++ b/operations/mimir-tests/test-storage-gcs-generated.yaml @@ -849,7 +849,6 @@ spec: - -mem-ballast-size-bytes=268435456 - -querier.frontend-client.grpc-max-send-msg-size=104857600 - -querier.max-concurrent=8 - - -querier.query-ingesters-within=13h - -querier.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc.keepalive.min-time-between-pings=10s @@ -1072,7 +1071,6 @@ spec: - -ingester.ring.prefix= - -ingester.ring.replication-factor=3 - -ingester.ring.store=consul - - -querier.query-ingesters-within=13h - -ruler-storage.backend=gcs - -ruler-storage.gcs.bucket-name=rules-bucket - -ruler.alertmanager-url=http://alertmanager.default.svc.cluster.local/alertmanager @@ -1327,7 +1325,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true diff --git a/operations/mimir-tests/test-storage-s3-generated.yaml b/operations/mimir-tests/test-storage-s3-generated.yaml index 413bb6b5c7c..f6db02bd429 100644 --- a/operations/mimir-tests/test-storage-s3-generated.yaml +++ b/operations/mimir-tests/test-storage-s3-generated.yaml @@ -850,7 +850,6 @@ spec: - -mem-ballast-size-bytes=268435456 - -querier.frontend-client.grpc-max-send-msg-size=104857600 - -querier.max-concurrent=8 - - -querier.query-ingesters-within=13h - -querier.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc.keepalive.min-time-between-pings=10s @@ -1074,7 +1073,6 @@ spec: - -ingester.ring.prefix= - -ingester.ring.replication-factor=3 - -ingester.ring.store=consul - - -querier.query-ingesters-within=13h - -ruler-storage.backend=s3 - -ruler-storage.s3.bucket-name=rules-bucket - -ruler-storage.s3.endpoint=s3.dualstack.eu-west-1.amazonaws.com @@ -1334,7 +1332,6 @@ spec: - -blocks-storage.s3.bucket-name=blocks-bucket - -blocks-storage.s3.endpoint=s3.dualstack.us-east-1.amazonaws.com - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true diff --git a/operations/mimir/config.libsonnet b/operations/mimir/config.libsonnet index 044b0c674a6..d1355692726 100644 --- a/operations/mimir/config.libsonnet +++ b/operations/mimir/config.libsonnet @@ -133,9 +133,6 @@ // type queries. 32 days to allow for comparision over the last month (31d) and // then some. 'store.max-query-length': '768h', - - // Ingesters don't have data older than 13h, no need to ask them. - 'querier.query-ingesters-within': '13h', }, // PromQL query engine config (shared between all services running PromQL engine, like the ruler and querier). diff --git a/operations/mimir/ingester.libsonnet b/operations/mimir/ingester.libsonnet index c215d72dfdc..ab2f1a4f863 100644 --- a/operations/mimir/ingester.libsonnet +++ b/operations/mimir/ingester.libsonnet @@ -35,9 +35,6 @@ 'blocks-storage.tsdb.block-ranges-period': '2h', 'blocks-storage.tsdb.ship-interval': '1m', - // Close idle TSDBs. - 'blocks-storage.tsdb.close-idle-tsdb-timeout': $._config.queryConfig['querier.query-ingesters-within'], - // Persist ring tokens so that when the ingester will be restarted // it will pick the same tokens 'ingester.ring.tokens-file-path': '/data/tokens', diff --git a/operations/mimir/shuffle-sharding.libsonnet b/operations/mimir/shuffle-sharding.libsonnet index 150933d8cdd..3fa96feb300 100644 --- a/operations/mimir/shuffle-sharding.libsonnet +++ b/operations/mimir/shuffle-sharding.libsonnet @@ -102,7 +102,6 @@ } ) + ( if !$._config.shuffle_sharding.ingester_read_path_enabled then {} else { - 'querier.shuffle-sharding-ingesters-lookback-period': $._config.queryConfig['querier.query-ingesters-within'], 'distributor.ingestion-tenant-shard-size': $._config.shuffle_sharding.ingester_shard_size, } ), @@ -122,7 +121,6 @@ } ) + ( if !$._config.shuffle_sharding.ingester_read_path_enabled then {} else { - 'querier.shuffle-sharding-ingesters-lookback-period': $._config.queryConfig['querier.query-ingesters-within'], 'distributor.ingestion-tenant-shard-size': $._config.shuffle_sharding.ingester_shard_size, } ) + ( diff --git a/pkg/querier/querier.go b/pkg/querier/querier.go index e382dd41f83..ee9b25fc454 100644 --- a/pkg/querier/querier.go +++ b/pkg/querier/querier.go @@ -56,13 +56,14 @@ type Config struct { } const ( - queryIngestersWithinFlag = "querier.query-ingesters-within" - queryStoreAfterFlag = "querier.query-store-after" + queryIngestersWithinFlag = "querier.query-ingesters-within" + queryStoreAfterFlag = "querier.query-store-after" + shuffleShardingIngestersLookbackPeriodFlag = "querier.shuffle-sharding-ingesters-lookback-period" ) var ( errBadLookbackConfigs = fmt.Errorf("the -%s setting must be greater than -%s otherwise queries might return partial results", queryIngestersWithinFlag, queryStoreAfterFlag) - errShuffleShardingLookbackLessThanQueryStoreAfter = errors.New("the shuffle-sharding lookback period should be greater or equal than the configured 'query store after'") + errShuffleShardingLookbackLessThanQueryStoreAfter = fmt.Errorf("the -%s setting must be greater or equal to -%s", shuffleShardingIngestersLookbackPeriodFlag, queryStoreAfterFlag) errEmptyTimeRange = errors.New("empty time range") ) @@ -74,7 +75,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.DurationVar(&cfg.QueryIngestersWithin, queryIngestersWithinFlag, 13*time.Hour, "Maximum lookback beyond which queries are not sent to ingester. 0 means all queries are sent to ingester.") f.DurationVar(&cfg.MaxQueryIntoFuture, "querier.max-query-into-future", 10*time.Minute, "Maximum duration into the future you can query. 0 to disable.") f.DurationVar(&cfg.QueryStoreAfter, queryStoreAfterFlag, 12*time.Hour, "The time after which a metric should be queried from storage and not just ingesters. 0 means all queries are sent to store. If this option is enabled, the time range of the query sent to the store-gateway will be manipulated to ensure the query end is not more recent than 'now - query-store-after'.") - f.DurationVar(&cfg.ShuffleShardingIngestersLookbackPeriod, "querier.shuffle-sharding-ingesters-lookback-period", 0, "When distributor's sharding strategy is shuffle-sharding and this setting is > 0, queriers fetch in-memory series from the minimum set of required ingesters, selecting only ingesters which may have received series since 'now - lookback period'. The lookback period should be greater or equal than the configured -querier.query-store-after and -querier.query-ingesters-within. If this setting is 0, queriers always query all ingesters (ingesters shuffle sharding on read path is disabled).") + f.DurationVar(&cfg.ShuffleShardingIngestersLookbackPeriod, shuffleShardingIngestersLookbackPeriodFlag, 13*time.Hour, "When this setting is > 0, queriers fetch in-memory series from the minimum set of required ingesters, selecting only ingesters which may have received series since 'now - lookback period'. The lookback period should be greater or equal than the configured -querier.query-store-after and -querier.query-ingesters-within. If this setting is 0, queriers always query all ingesters (ingesters shuffle sharding on read path is disabled).") cfg.EngineConfig.RegisterFlags(f) }