store-gateway: clean up chunks fetching, deprecate bucketed chunks pool

CHANGELOG.md

@@ -10,6 +10,11 @@
 * [CHANGE] Querier: `-querier.query-ingesters-within` has been moved from a global flag to a per-tenant override. #4287
 * [CHANGE] Querier: Use `-blocks-storage.tsdb.retention-period` instead of `-querier.query-ingesters-within` for calculating the lookback period for shuffle sharded ingesters. Setting `-querier.query-ingesters-within=0` no longer disables shuffle sharding on the read path. #4287
 * [CHANGE] Block upload: `/api/v1/upload/block/{block}/files` endpoint now allows file uploads with no `Content-Length`. #4956
+* [CHANGE] Store-gateway: deprecate configuration parameters for chunk pooling, they will be removed in Mimir 2.11. The following options are now also ignored: #4996
+  * `-blocks-storage.bucket-store.max-chunk-pool-bytes`
+  * `-blocks-storage.bucket-store.chunk-pool-min-bucket-size-bytes`
+  * `-blocks-storage.bucket-store.chunk-pool-max-bucket-size-bytes`
+* [CHANGE] Store-gateway: remove metrics `cortex_bucket_store_chunk_pool_requested_bytes_total` and `cortex_bucket_store_chunk_pool_returned_bytes_total`. #4996
 * [ENHANCEMENT] Add per-tenant limit `-validation.max-native-histogram-buckets` to be able to ignore native histogram samples that have too many buckets. #4765
 * [ENHANCEMENT] Store-gateway: reduce memory usage in some LabelValues calls. #4789
 * [ENHANCEMENT] Store-gateway: add a `stage` label to the metric `cortex_bucket_store_series_data_touched`. This label now applies to `data_type="chunks"` and `data_type="series"`. The `stage` label has 2 values: `processed` - the number of series that parsed - and `returned` - the number of series selected from the processed bytes to satisfy the query. #4797 #4830

cmd/mimir/config-descriptor.json

@@ -7057,7 +7057,7 @@
               "fieldDefaultValue": 2147483648,
               "fieldFlag": "blocks-storage.bucket-store.max-chunk-pool-bytes",
               "fieldType": "int",
-              "fieldCategory": "advanced"
+              "fieldCategory": "deprecated"
             },
             {
               "kind": "field",
@@ -7068,7 +7068,7 @@
               "fieldDefaultValue": 16000,
               "fieldFlag": "blocks-storage.bucket-store.chunk-pool-min-bucket-size-bytes",
               "fieldType": "int",
-              "fieldCategory": "advanced"
+              "fieldCategory": "deprecated"
             },
             {
               "kind": "field",
@@ -7079,7 +7079,7 @@
               "fieldDefaultValue": 50000000,
               "fieldFlag": "blocks-storage.bucket-store.chunk-pool-max-bucket-size-bytes",
               "fieldType": "int",
-              "fieldCategory": "advanced"
+              "fieldCategory": "deprecated"
             },
             {
               "kind": "field",

cmd/mimir/help-all.txt.tmpl

@@ -282,9 +282,9 @@ Usage of ./cmd/mimir/mimir:
   -blocks-storage.bucket-store.bucket-index.update-on-error-interval duration
     	How frequently a bucket index, which previously failed to load, should be tried to load again. This option is used only by querier. (default 1m0s)
   -blocks-storage.bucket-store.chunk-pool-max-bucket-size-bytes int
-    	Size - in bytes - of the largest chunks pool bucket. (default 50000000)
+    	[deprecated] Size - in bytes - of the largest chunks pool bucket. (default 50000000)
   -blocks-storage.bucket-store.chunk-pool-min-bucket-size-bytes int
-    	Size - in bytes - of the smallest chunks pool bucket. (default 16000)
+    	[deprecated] Size - in bytes - of the smallest chunks pool bucket. (default 16000)
   -blocks-storage.bucket-store.chunks-cache.attributes-in-memory-max-items int
     	Maximum number of object attribute items to keep in a first level in-memory LRU cache. Metadata will be stored and fetched in-memory before hitting the cache backend. 0 to disable the in-memory cache. (default 50000)
   -blocks-storage.bucket-store.chunks-cache.attributes-ttl duration
@@ -488,7 +488,7 @@ Usage of ./cmd/mimir/mimir:
   -blocks-storage.bucket-store.index-header.max-idle-file-handles uint
     	Maximum number of idle file handles the store-gateway keeps open for each index-header file. (default 1)
   -blocks-storage.bucket-store.max-chunk-pool-bytes uint
-    	Max size - in bytes - of a chunks pool, used to reduce memory allocations. The pool is shared across all tenants. 0 to disable the limit. (default 2147483648)
+    	[deprecated] Max size - in bytes - of a chunks pool, used to reduce memory allocations. The pool is shared across all tenants. 0 to disable the limit. (default 2147483648)
   -blocks-storage.bucket-store.max-concurrent int
     	Max number of concurrent queries to execute against the long-term storage. The limit is shared across all tenants. (default 100)
   -blocks-storage.bucket-store.meta-sync-concurrency int

docs/sources/mimir/configure/about-versioning.md

@@ -141,3 +141,10 @@ The following features are currently deprecated and will be **removed in Mimir 2
 
 - Ingester
   - `-blocks-storage.tsdb.max-tsdb-opening-concurrency-on-startup`
+
+The following features are currently deprecated and will be **removed in Mimir 2.11**:
+
+- Store-gateway
+  - `-blocks-storage.bucket-store.chunk-pool-min-bucket-size-bytes`
+  - `-blocks-storage.bucket-store.chunk-pool-max-bucket-size-bytes`
+  - `-blocks-storage.bucket-store.max-chunk-pool-bytes`

docs/sources/mimir/references/configuration-parameters/index.md

@@ -3199,16 +3199,16 @@ bucket_store:
   # CLI flag: -blocks-storage.bucket-store.ignore-blocks-within
   [ignore_blocks_within: <duration> | default = 10h]
 
-  # (advanced) Max size - in bytes - of a chunks pool, used to reduce memory
+  # (deprecated) Max size - in bytes - of a chunks pool, used to reduce memory
   # allocations. The pool is shared across all tenants. 0 to disable the limit.
   # CLI flag: -blocks-storage.bucket-store.max-chunk-pool-bytes
   [max_chunk_pool_bytes: <int> | default = 2147483648]
 
-  # (advanced) Size - in bytes - of the smallest chunks pool bucket.
+  # (deprecated) Size - in bytes - of the smallest chunks pool bucket.
   # CLI flag: -blocks-storage.bucket-store.chunk-pool-min-bucket-size-bytes
   [chunk_pool_min_bucket_size_bytes: <int> | default = 16000]
 
-  # (advanced) Size - in bytes - of the largest chunks pool bucket.
+  # (deprecated) Size - in bytes - of the largest chunks pool bucket.
   # CLI flag: -blocks-storage.bucket-store.chunk-pool-max-bucket-size-bytes
   [chunk_pool_max_bucket_size_bytes: <int> | default = 50000000]
 

operations/helm/charts/mimir-distributed/values.yaml

@@ -172,7 +172,6 @@ mimir:
     blocks_storage:
       backend: s3
       bucket_store:
-        max_chunk_pool_bytes: 12884901888 # 12GiB
         {{- if index .Values "chunks-cache" "enabled" }}
         chunks_cache:
           backend: memcached

operations/helm/tests/gateway-enterprise-values-generated/mimir-distributed/templates/mimir-config.yaml

@@ -47,7 +47,6 @@ data:
     blocks_storage:
       backend: s3
       bucket_store:
-        max_chunk_pool_bytes: 12884901888
         sync_dir: /data/tsdb-sync
       s3:
         access_key_id: grafana-mimir

operations/helm/tests/gateway-nginx-values-generated/mimir-distributed/templates/mimir-config.yaml

@@ -30,7 +30,6 @@ data:
     blocks_storage:
       backend: s3
       bucket_store:
-        max_chunk_pool_bytes: 12884901888
         sync_dir: /data/tsdb-sync
       s3:
         access_key_id: grafana-mimir

operations/helm/tests/graphite-enabled-values-generated/mimir-distributed/templates/mimir-config.yaml

@@ -47,7 +47,6 @@ data:
     blocks_storage:
       backend: s3
       bucket_store:
-        max_chunk_pool_bytes: 12884901888
         sync_dir: /data/tsdb-sync
       s3:
         access_key_id: grafana-mimir

operations/helm/tests/large-values-generated/mimir-distributed/templates/mimir-config.yaml

@@ -35,7 +35,6 @@ data:
             addresses: dns+large-values-mimir-index-cache.citestns.svc:11211
             max_idle_connections: 150
             max_item_size: 5242880
-        max_chunk_pool_bytes: 12884901888
         metadata_cache:
           backend: memcached
           memcached:

operations/helm/tests/metamonitoring-values-generated/mimir-distributed/templates/mimir-config.yaml

@@ -30,7 +30,6 @@ data:
     blocks_storage:
       backend: s3
       bucket_store:
-        max_chunk_pool_bytes: 12884901888
         sync_dir: /data/tsdb-sync
       s3:
         access_key_id: grafana-mimir

operations/helm/tests/openshift-values-generated/mimir-distributed/templates/mimir-config.yaml

@@ -32,7 +32,6 @@ data:
     blocks_storage:
       backend: s3
       bucket_store:
-        max_chunk_pool_bytes: 12884901888
         sync_dir: /data/tsdb-sync
       tsdb:
         dir: /data/tsdb

operations/helm/tests/scheduler-name-values-generated/mimir-distributed/templates/mimir-config.yaml

@@ -22,7 +22,6 @@ data:
     blocks_storage:
       backend: s3
       bucket_store:
-        max_chunk_pool_bytes: 12884901888
         sync_dir: /data/tsdb-sync
       tsdb:
         dir: /data/tsdb

operations/helm/tests/small-values-generated/mimir-distributed/templates/mimir-config.yaml

@@ -35,7 +35,6 @@ data:
             addresses: dns+small-values-mimir-index-cache.citestns.svc:11211
             max_idle_connections: 150
             max_item_size: 5242880
-        max_chunk_pool_bytes: 12884901888
         metadata_cache:
           backend: memcached
           memcached:

operations/helm/tests/test-enterprise-configmap-values-generated/mimir-distributed/templates/mimir-config.yaml

@@ -47,7 +47,6 @@ data:
     blocks_storage:
       backend: s3
       bucket_store:
-        max_chunk_pool_bytes: 12884901888
         sync_dir: /data/tsdb-sync
       s3:
         access_key_id: ${MINIO_ROOT_USER}

operations/helm/tests/test-enterprise-k8s-1.25-values-generated/mimir-distributed/templates/mimir-config.yaml

@@ -47,7 +47,6 @@ data:
     blocks_storage:
       backend: s3
       bucket_store:
-        max_chunk_pool_bytes: 12884901888
         sync_dir: /data/tsdb-sync
       s3:
         access_key_id: grafana-mimir