Support disabling metric relabeling per tenant (#6970)

* Support disabling metric relabeling per tenant Fixes #6968 * Update docs and changelog * Update about-versioning * Update reference-help * Update CHANGELOG.md Co-authored-by: Nick Pillitteri <56quarters@users.noreply.github.com> --------- Co-authored-by: Nick Pillitteri <56quarters@users.noreply.github.com>
grafana · Dec 19, 2023 · 14590eb · 14590eb
1 parent bcb5c34
commit 14590eb
Show file tree

Hide file tree

Showing 8 changed files with 84 additions and 36 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -11,6 +11,7 @@
 * [ENHANCEMENT] Ruler: exclude vector queries from being tracked in `cortex_ruler_queries_zero_fetched_series_total`. #6544
 * [ENHANCEMENT] Query-Frontend and Query-Scheduler: split tenant query request queues by query component with `query-frontend.additional-query-queue-dimensions-enabled` and `query-scheduler.additional-query-queue-dimensions-enabled`. #6772
 * [ENHANCEMENT] Store-gateway: include more information about lazy index-header loading in traces. #6922
+* [ENHANCEMENT] Distributor: support disabling metric relabel rules per-tenant via the flag `-distributor.metric-relabeling-enabled` or associated YAML. #6970
 * [BUGFIX] Ingester: don't ignore errors encountered while iterating through chunks or samples in response to a query request. #6451
 * [BUGFIX] Fix issue where queries can fail or omit OOO samples if OOO head compaction occurs between creating a querier and reading chunks #6766
 * [BUGFIX] Fix issue where concatenatingChunkIterator can obscure errors #6766

diff --git a/cmd/mimir/config-descriptor.json b/cmd/mimir/config-descriptor.json
@@ -3300,6 +3300,17 @@
           "fieldType": "relabel_config...",
           "fieldCategory": "experimental"
         },
+        {
+          "kind": "field",
+          "name": "metric_relabeling_enabled",
+          "required": false,
+          "desc": "Enable metric relabeling for the tenant. This configuration option can be used to forcefully disable metric relabeling on a per-tenant basis.",
+          "fieldValue": null,
+          "fieldDefaultValue": true,
+          "fieldFlag": "distributor.metric-relabeling-enabled",
+          "fieldType": "boolean",
+          "fieldCategory": "experimental"
+        },
         {
           "kind": "field",
           "name": "service_overload_status_code_on_rate_limit_enabled",

diff --git a/cmd/mimir/help-all.txt.tmpl b/cmd/mimir/help-all.txt.tmpl
@@ -1145,6 +1145,8 @@ Usage of ./cmd/mimir/mimir:
     	[experimental] Use experimental method of limiting push requests.
   -distributor.max-recv-msg-size int
     	Max message size in bytes that the distributors will accept for incoming push requests to the remote write API. If exceeded, the request will be rejected. (default 104857600)
+  -distributor.metric-relabeling-enabled
+    	[experimental] Enable metric relabeling for the tenant. This configuration option can be used to forcefully disable metric relabeling on a per-tenant basis. (default true)
   -distributor.otel-metric-suffixes-enabled
     	Whether to enable automatic suffixes to names of metrics ingested through OTLP.
   -distributor.remote-timeout duration

diff --git a/docs/sources/mimir/configure/about-versioning.md b/docs/sources/mimir/configure/about-versioning.md
@@ -57,6 +57,7 @@ The following features are currently experimental:
   - Aligning of evaluation timestamp on interval (`align_evaluation_time_on_interval`)
 - Distributor
   - Metrics relabeling
+    - `-distributor.metric-relabeling-enabled`
   - OTLP ingestion path
   - OTLP metadata storage
     - `-distributor.enable-otlp-metadata-storage`

diff --git a/docs/sources/mimir/configure/configuration-parameters/index.md b/docs/sources/mimir/configure/configuration-parameters/index.md
@@ -2981,6 +2981,12 @@ The `limits` block configures default and per-tenant limits imposed by component
 # during the relabeling phase and cleaned afterwards: __meta_tenant_id
 [metric_relabel_configs: <relabel_config...> | default = ]
 
+# (experimental) Enable metric relabeling for the tenant. This configuration
+# option can be used to forcefully disable metric relabeling on a per-tenant
+# basis.
+# CLI flag: -distributor.metric-relabeling-enabled
+[metric_relabeling_enabled: <boolean> | default = true]
+
 # (experimental) If enabled, rate limit errors will be reported to the client
 # with HTTP status code 529 (Service is overloaded). If disabled, status code
 # 429 (Too Many Requests) is used. Enabling

diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go
@@ -784,12 +784,16 @@ func (d *Distributor) prePushRelabelMiddleware(next PushFunc) PushFunc {
 			}
 		}()
 
-		req, err := pushReq.WriteRequest()
+		userID, err := tenant.TenantID(ctx)
 		if err != nil {
 			return err
 		}
 
-		userID, err := tenant.TenantID(ctx)
+		if !d.limits.MetricRelabelingEnabled(userID) {
+			return next(ctx, pushReq)
+		}
+
+		req, err := pushReq.WriteRequest()
 		if err != nil {
 			return err
 		}

diff --git a/pkg/distributor/distributor_test.go b/pkg/distributor/distributor_test.go
@@ -3227,40 +3227,53 @@ func TestRelabelMiddleware(t *testing.T) {
 	ctxWithUser := user.InjectOrgID(context.Background(), "user")
 
 	type testCase struct {
-		name           string
-		ctx            context.Context
-		relabelConfigs []*relabel.Config
-		dropLabels     []string
-		reqs           []*mimirpb.WriteRequest
-		expectedReqs   []*mimirpb.WriteRequest
-		expectErrs     []bool
+		name              string
+		ctx               context.Context
+		relabelConfigs    []*relabel.Config
+		dropLabels        []string
+		relabelingEnabled bool
+		reqs              []*mimirpb.WriteRequest
+		expectedReqs      []*mimirpb.WriteRequest
+		expectErrs        []bool
 	}
 	testCases := []testCase{
 		{
-			name:           "do nothing",
-			ctx:            ctxWithUser,
-			relabelConfigs: nil,
-			dropLabels:     nil,
-			reqs:           []*mimirpb.WriteRequest{makeWriteRequestForGenerators(5, labelSetGenForStringPairs(t, "__name__", "metric1", "label", "value_%d"), nil, nil)},
-			expectedReqs:   []*mimirpb.WriteRequest{makeWriteRequestForGenerators(5, labelSetGenForStringPairs(t, "__name__", "metric1", "label", "value_%d"), nil, nil)},
-			expectErrs:     []bool{false},
+			name:              "do nothing",
+			ctx:               ctxWithUser,
+			relabelConfigs:    nil,
+			dropLabels:        nil,
+			relabelingEnabled: true,
+			reqs:              []*mimirpb.WriteRequest{makeWriteRequestForGenerators(5, labelSetGenForStringPairs(t, "__name__", "metric1", "label", "value_%d"), nil, nil)},
+			expectedReqs:      []*mimirpb.WriteRequest{makeWriteRequestForGenerators(5, labelSetGenForStringPairs(t, "__name__", "metric1", "label", "value_%d"), nil, nil)},
+			expectErrs:        []bool{false},
 		}, {
-			name:           "no user in context",
-			ctx:            context.Background(),
-			relabelConfigs: nil,
-			dropLabels:     nil,
-			reqs:           []*mimirpb.WriteRequest{makeWriteRequestForGenerators(5, labelSetGenForStringPairs(t, "__name__", "metric1", "label", "value_%d"), nil, nil)},
-			expectedReqs:   nil,
-			expectErrs:     []bool{true},
+			name:              "no user in context",
+			ctx:               context.Background(),
+			relabelConfigs:    nil,
+			dropLabels:        nil,
+			relabelingEnabled: true,
+			reqs:              []*mimirpb.WriteRequest{makeWriteRequestForGenerators(5, labelSetGenForStringPairs(t, "__name__", "metric1", "label", "value_%d"), nil, nil)},
+			expectedReqs:      nil,
+			expectErrs:        []bool{true},
 		}, {
-			name:           "apply a relabel rule",
-			ctx:            ctxWithUser,
-			relabelConfigs: nil,
-			dropLabels:     []string{"label1", "label3"},
-			reqs:           []*mimirpb.WriteRequest{makeWriteRequestForGenerators(5, labelSetGenForStringPairs(t, "__name__", "metric1", "label1", "value1", "label2", "value2", "label3", "value3"), nil, nil)},
-			expectedReqs:   []*mimirpb.WriteRequest{makeWriteRequestForGenerators(5, labelSetGenForStringPairs(t, "__name__", "metric1", "label2", "value2"), nil, nil)},
-			expectErrs:     []bool{false},
+			name:              "apply a relabel rule",
+			ctx:               ctxWithUser,
+			relabelConfigs:    nil,
+			dropLabels:        []string{"label1", "label3"},
+			relabelingEnabled: true,
+			reqs:              []*mimirpb.WriteRequest{makeWriteRequestForGenerators(5, labelSetGenForStringPairs(t, "__name__", "metric1", "label1", "value1", "label2", "value2", "label3", "value3"), nil, nil)},
+			expectedReqs:      []*mimirpb.WriteRequest{makeWriteRequestForGenerators(5, labelSetGenForStringPairs(t, "__name__", "metric1", "label2", "value2"), nil, nil)},
+			expectErrs:        []bool{false},
 		}, {
+			name:              "relabeling disabled",
+			ctx:               ctxWithUser,
+			relabelConfigs:    nil,
+			dropLabels:        []string{"label1", "label3"},
+			relabelingEnabled: false,
+			reqs:              []*mimirpb.WriteRequest{makeWriteRequestForGenerators(5, labelSetGenForStringPairs(t, "__name__", "metric1", "label1", "value1", "label2", "value2", "label3", "value3"), nil, nil)},
+			expectedReqs:      []*mimirpb.WriteRequest{makeWriteRequestForGenerators(5, labelSetGenForStringPairs(t, "__name__", "metric1", "label1", "value1", "label2", "value2", "label3", "value3"), nil, nil)},
+			expectErrs:        []bool{false},
+		}, {}, {
 			name: "drop two out of three labels",
 			ctx:  ctxWithUser,
 			relabelConfigs: []*relabel.Config{
@@ -3272,13 +3285,15 @@ func TestRelabelMiddleware(t *testing.T) {
 					Replacement:  "prefix_$1",
 				},
 			},
-			reqs:         []*mimirpb.WriteRequest{makeWriteRequestForGenerators(5, labelSetGenForStringPairs(t, "__name__", "metric1", "label1", "value1"), nil, nil)},
-			expectedReqs: []*mimirpb.WriteRequest{makeWriteRequestForGenerators(5, labelSetGenForStringPairs(t, "__name__", "metric1", "label1", "value1", "target", "prefix_value1"), nil, nil)},
-			expectErrs:   []bool{false},
+			relabelingEnabled: true,
+			reqs:              []*mimirpb.WriteRequest{makeWriteRequestForGenerators(5, labelSetGenForStringPairs(t, "__name__", "metric1", "label1", "value1"), nil, nil)},
+			expectedReqs:      []*mimirpb.WriteRequest{makeWriteRequestForGenerators(5, labelSetGenForStringPairs(t, "__name__", "metric1", "label1", "value1", "target", "prefix_value1"), nil, nil)},
+			expectErrs:        []bool{false},
 		}, {
-			name:       "drop entire series if they have no labels",
-			ctx:        ctxWithUser,
-			dropLabels: []string{"__name__", "label2", "label3"},
+			name:              "drop entire series if they have no labels",
+			ctx:               ctxWithUser,
+			dropLabels:        []string{"__name__", "label2", "label3"},
+			relabelingEnabled: true,
 			reqs: []*mimirpb.WriteRequest{
 				makeWriteRequestForGenerators(5, labelSetGenForStringPairs(t, "__name__", "metric1", "label1", "value1"), nil, nil),
 				makeWriteRequestForGenerators(5, labelSetGenForStringPairs(t, "__name__", "metric2", "label2", "value2"), nil, nil),
@@ -3304,6 +3319,7 @@ func TestRelabelMiddleware(t *testing.T) {
 					Replacement:  "$1",
 				},
 			},
+			relabelingEnabled: true,
 			reqs: []*mimirpb.WriteRequest{{
 				Timeseries: []mimirpb.PreallocTimeseries{makeWriteRequestTimeseries(
 					[]mimirpb.LabelAdapter{
@@ -3349,6 +3365,7 @@ func TestRelabelMiddleware(t *testing.T) {
 			flagext.DefaultValues(&limits)
 			limits.MetricRelabelConfigs = tc.relabelConfigs
 			limits.DropLabels = tc.dropLabels
+			limits.MetricRelabelingEnabled = tc.relabelingEnabled
 			ds, _, _ := prepare(t, prepConfig{
 				numDistributors: 1,
 				limits:          &limits,

diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go
@@ -91,6 +91,7 @@ type Limits struct {
 	EnforceMetadataMetricName                   bool                `yaml:"enforce_metadata_metric_name" json:"enforce_metadata_metric_name" category:"advanced"`
 	IngestionTenantShardSize                    int                 `yaml:"ingestion_tenant_shard_size" json:"ingestion_tenant_shard_size"`
 	MetricRelabelConfigs                        []*relabel.Config   `yaml:"metric_relabel_configs,omitempty" json:"metric_relabel_configs,omitempty" doc:"nocli|description=List of metric relabel configurations. Note that in most situations, it is more effective to use metrics relabeling directly in the Prometheus server, e.g. remote_write.write_relabel_configs. Labels available during the relabeling phase and cleaned afterwards: __meta_tenant_id" category:"experimental"`
+	MetricRelabelingEnabled                     bool                `yaml:"metric_relabeling_enabled" json:"metric_relabeling_enabled" category:"experimental"`
 	ServiceOverloadStatusCodeOnRateLimitEnabled bool                `yaml:"service_overload_status_code_on_rate_limit_enabled" json:"service_overload_status_code_on_rate_limit_enabled" category:"experimental"`
 	// Ingester enforced limits.
 	// Series
@@ -216,6 +217,7 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) {
 	_ = l.CreationGracePeriod.Set("10m")
 	f.Var(&l.CreationGracePeriod, CreationGracePeriodFlag, "Controls how far into the future incoming samples and exemplars are accepted compared to the wall clock. Any sample or exemplar will be rejected if its timestamp is greater than '(now + grace_period)'. This configuration is enforced in the distributor, ingester and query-frontend (to avoid querying too far into the future).")
 	f.BoolVar(&l.EnforceMetadataMetricName, "validation.enforce-metadata-metric-name", true, "Enforce every metadata has a metric name.")
+	f.BoolVar(&l.MetricRelabelingEnabled, "distributor.metric-relabeling-enabled", true, "Enable metric relabeling for the tenant. This configuration option can be used to forcefully disable metric relabeling on a per-tenant basis.")
 	f.BoolVar(&l.ServiceOverloadStatusCodeOnRateLimitEnabled, "distributor.service-overload-status-code-on-rate-limit-enabled", false, "If enabled, rate limit errors will be reported to the client with HTTP status code 529 (Service is overloaded). If disabled, status code 429 (Too Many Requests) is used. Enabling -distributor.retry-after-header.enabled before utilizing this option is strongly recommended as it helps prevent premature request retries by the client.")
 	f.BoolVar(&l.OTelMetricSuffixesEnabled, "distributor.otel-metric-suffixes-enabled", false, "Whether to enable automatic suffixes to names of metrics ingested through OTLP.")
 
@@ -743,6 +745,10 @@ func (o *Overrides) MetricRelabelConfigs(userID string) []*relabel.Config {
 	return o.getOverridesForUser(userID).MetricRelabelConfigs
 }
 
+func (o *Overrides) MetricRelabelingEnabled(userID string) bool {
+	return o.getOverridesForUser(userID).MetricRelabelingEnabled
+}
+
 // NativeHistogramsIngestionEnabled returns whether to ingest native histograms in the ingester
 func (o *Overrides) NativeHistogramsIngestionEnabled(userID string) bool {
 	return o.getOverridesForUser(userID).NativeHistogramsIngestionEnabled