Querier: Refine error messages for per-tenant query limits (#5059)

* Querier: Refine error messages for per-tenant query limits

Refine error messages for per-tenant query limits, informing the user
of the preferred strategy for not hitting the limit, in addition to how
they may tweak the limit. Affected limits:

* max_fetched_chunks_per_query
* max_fetched_chunk_bytes_per_query
* max_fetched_series_per_query

---------

Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
Co-authored-by: Charles Korn <charleskorn@users.noreply.github.com>

CHANGELOG.md

@@ -9,6 +9,7 @@
 * [ENHANCEMENT] Cardinality API: When zone aware replication is enabled, the label values cardinality API can now tolerate single zone failure #5178
 * [ENHANCEMENT] Distributor: optimize sending requests to ingesters when incoming requests don't need to be modified. For now this feature can be disabled by setting `-timeseries-unmarshal-caching-optimization-enabled=false`. #5137
 * [ENHANCEMENT] Query-frontend: added "response_size_bytes" field to "query stats" log. #5196
+* [ENHANCEMENT] Querier: Refine error messages for per-tenant query limits, informing the user of the preferred strategy for not hitting the limit, in addition to how they may tweak the limit. #5059
 
 ### Mixin
 * [CHANGE] Dashboards: show all workloads in selected namespace on "rollout progress" dashboard. #5113

pkg/ingester/client/streaming_test.go

@@ -222,12 +222,12 @@ func TestSeriesChunksStreamReader_ChunksLimits(t *testing.T) {
 		"query selects too many chunks": {
 			maxChunks:     2,
 			maxChunkBytes: 200,
-			expectedError: "attempted to read series at index 0 from stream, but the stream has failed: the query exceeded the maximum number of chunks (limit: 2 chunks) (err-mimir-max-chunks-per-query). To adjust the related per-tenant limit, configure -querier.max-fetched-chunks-per-query, or contact your service administrator.",
+			expectedError: "attempted to read series at index 0 from stream, but the stream has failed: the query exceeded the maximum number of chunks (limit: 2 chunks) (err-mimir-max-chunks-per-query). Consider reducing the time range and/or number of series selected by the query. One way to reduce the number of selected series is to add more label matchers to the query. Otherwise, to adjust the related per-tenant limit, configure -querier.max-fetched-chunks-per-query, or contact your service administrator.",
 		},
 		"query selects too many chunk bytes": {
 			maxChunks:     4,
 			maxChunkBytes: 100,
-			expectedError: "attempted to read series at index 0 from stream, but the stream has failed: the query exceeded the aggregated chunks size limit (limit: 100 bytes) (err-mimir-max-chunks-bytes-per-query). To adjust the related per-tenant limit, configure -querier.max-fetched-chunk-bytes-per-query, or contact your service administrator.",
+			expectedError: "attempted to read series at index 0 from stream, but the stream has failed: the query exceeded the aggregated chunks size limit (limit: 100 bytes) (err-mimir-max-chunks-bytes-per-query). Consider reducing the time range and/or number of series selected by the query. One way to reduce the number of selected series is to add more label matchers to the query. Otherwise, to adjust the related per-tenant limit, configure -querier.max-fetched-chunk-bytes-per-query, or contact your service administrator.",
 		},
 	}
 

pkg/util/globalerror/errors.go

@@ -87,6 +87,15 @@ func (id ID) MessageWithPerTenantLimitConfig(msg, flag string, addFlags ...strin
 	return fmt.Sprintf("%s (%s%s). To adjust the related per-tenant limit%s, configure %s, or contact your service administrator.", msg, errPrefix, id, plural, flagsList)
 }
 
+// MessageWithStrategyAndPerTenantLimitConfig returns the provided msg, appending the error id and a
+// suggestion on which strategy to follow to try not hitting the limit, plus which configuration
+// flag(s) to otherwise change the per-tenant limit.
+func (id ID) MessageWithStrategyAndPerTenantLimitConfig(msg, strategy, flag string, addFlags ...string) string {
+	flagsList, plural := buildFlagsList(flag, addFlags...)
+	return fmt.Sprintf("%s (%s%s). %s. Otherwise, to adjust the related per-tenant limit%s, configure %s, or contact your service administrator.",
+		msg, errPrefix, id, strategy, plural, flagsList)
+}
+
 func buildFlagsList(flag string, addFlags ...string) (string, string) {
 	var sb strings.Builder
 	sb.WriteString("-")

pkg/util/limiter/query_limiter.go

@@ -19,18 +19,23 @@ import (
 
 type queryLimiterCtxKey struct{}
 
+const cardinalityStrategy = "Consider reducing the time range and/or number of series selected by the query. One way to reduce the number of selected series is to add more label matchers to the query"
+
 var (
 	ctxKey                = &queryLimiterCtxKey{}
-	MaxSeriesHitMsgFormat = globalerror.MaxSeriesPerQuery.MessageWithPerTenantLimitConfig(
+	MaxSeriesHitMsgFormat = globalerror.MaxSeriesPerQuery.MessageWithStrategyAndPerTenantLimitConfig(
 		"the query exceeded the maximum number of series (limit: %d series)",
+		cardinalityStrategy,
 		validation.MaxSeriesPerQueryFlag,
 	)
-	MaxChunkBytesHitMsgFormat = globalerror.MaxChunkBytesPerQuery.MessageWithPerTenantLimitConfig(
+	MaxChunkBytesHitMsgFormat = globalerror.MaxChunkBytesPerQuery.MessageWithStrategyAndPerTenantLimitConfig(
 		"the query exceeded the aggregated chunks size limit (limit: %d bytes)",
+		cardinalityStrategy,
 		validation.MaxChunkBytesPerQueryFlag,
 	)
-	MaxChunksPerQueryLimitMsgFormat = globalerror.MaxChunksPerQuery.MessageWithPerTenantLimitConfig(
+	MaxChunksPerQueryLimitMsgFormat = globalerror.MaxChunksPerQuery.MessageWithStrategyAndPerTenantLimitConfig(
 		"the query exceeded the maximum number of chunks (limit: %d chunks)",
+		cardinalityStrategy,
 		validation.MaxChunksPerQueryFlag,
 	)
 )