Make __meta_tenant_id available in metric_relabel_configs

CHANGELOG.md

@@ -5,6 +5,7 @@
 ### Grafana Mimir
 
 * [ENHANCEMENT] Improved memory limit on the in-memory cache used for regular expression matchers. #4751
+* [ENHANCEMENT] Distributor: make `__meta_tenant_id` available in `metric_relabel_configs` #4725
 
 ### Documentation
 

cmd/mimir/config-descriptor.json

@@ -2995,7 +2995,7 @@
           "kind": "field",
           "name": "metric_relabel_configs",
           "required": false,
-          "desc": "List of metric relabel configurations. Note that in most situations, it is more effective to use metrics relabeling directly in the Prometheus server, e.g. remote_write.write_relabel_configs.",
+          "desc": "List of metric relabel configurations. Note that in most situations, it is more effective to use metrics relabeling directly in the Prometheus server, e.g. remote_write.write_relabel_configs. Labels available during the relabeling phase and cleaned afterwards: __meta_tenant_id",
           "fieldValue": null,
           "fieldDefaultValue": null,
           "fieldType": "relabel_config...",

docs/sources/mimir/references/configuration-parameters/index.md

@@ -2565,7 +2565,8 @@ The `limits` block configures default and per-tenant limits imposed by component
 
 # (experimental) List of metric relabel configurations. Note that in most
 # situations, it is more effective to use metrics relabeling directly in the
-# Prometheus server, e.g. remote_write.write_relabel_configs.
+# Prometheus server, e.g. remote_write.write_relabel_configs. Labels available
+# during the relabeling phase and cleaned afterwards: __meta_tenant_id
 [metric_relabel_configs: <relabel_config...> | default = ]
 
 # The maximum number of in-memory series per tenant, across the cluster before

pkg/distributor/distributor.go

@@ -63,6 +63,9 @@ const (
 	// ringAutoForgetUnhealthyPeriods is how many consecutive timeout periods an unhealthy instance
 	// in the ring will be automatically removed after.
 	ringAutoForgetUnhealthyPeriods = 10
+
+	// metaLabelTenantID is the name of the metric_relabel_configs label with tenant ID.
+	metaLabelTenantID = model.MetaLabelPrefix + "tenant_id"
 )
 
 const (
@@ -814,16 +817,20 @@ func (d *Distributor) prePushRelabelMiddleware(next push.Func) push.Func {
 		}
 
 		var removeTsIndexes []int
+		lb := labels.NewBuilder(labels.EmptyLabels())
 		for tsIdx := 0; tsIdx < len(req.Timeseries); tsIdx++ {
 			ts := req.Timeseries[tsIdx]
 
 			if mrc := d.limits.MetricRelabelConfigs(userID); len(mrc) > 0 {
-				l, keep := relabel.Process(mimirpb.FromLabelAdaptersToLabels(ts.Labels), mrc...)
+				lb.Reset(mimirpb.FromLabelAdaptersToLabels(ts.Labels))
+				lb.Set(metaLabelTenantID, userID)
+				keep := relabel.ProcessBuilder(lb, mrc...)
 				if !keep {
 					removeTsIndexes = append(removeTsIndexes, tsIdx)
 					continue
 				}
-				ts.Labels = mimirpb.FromLabelsToLabelAdapters(l)
+				lb.Del(metaLabelTenantID)
+				ts.Labels = mimirpb.FromLabelsToLabelAdapters(lb.Labels(labels.EmptyLabels()))
 			}
 
 			for _, labelName := range d.limits.DropLabels(userID) {

pkg/distributor/distributor_test.go

@@ -1844,6 +1844,39 @@ func BenchmarkDistributor_Push(b *testing.B) {
 			},
 			expectedErr: "received a sample whose timestamp is too far in the future",
 		},
+		"all samples go to metric_relabel_configs": {
+			prepareConfig: func(limits *validation.Limits) {
+				limits.MetricRelabelConfigs = []*relabel.Config{
+					{
+						SourceLabels: []model.LabelName{"__name__"},
+						Action:       relabel.DefaultRelabelConfig.Action,
+						Regex:        relabel.DefaultRelabelConfig.Regex,
+						Replacement:  relabel.DefaultRelabelConfig.Replacement,
+						TargetLabel:  "__tmp_name",
+					},
+				}
+			},
+			prepareSeries: func() ([]labels.Labels, []mimirpb.Sample) {
+				metrics := make([]labels.Labels, numSeriesPerRequest)
+				samples := make([]mimirpb.Sample, numSeriesPerRequest)
+
+				for i := 0; i < numSeriesPerRequest; i++ {
+					lbls := labels.NewBuilder(labels.FromStrings(model.MetricNameLabel, "foo"))
+					for i := 0; i < 10; i++ {
+						lbls.Set(fmt.Sprintf("name_%d", i), fmt.Sprintf("value_%d", i))
+					}
+
+					metrics[i] = lbls.Labels(nil)
+					samples[i] = mimirpb.Sample{
+						Value:       float64(i),
+						TimestampMs: time.Now().UnixNano() / int64(time.Millisecond),
+					}
+				}
+
+				return metrics, samples
+			},
+			expectedErr: "",
+		},
 	}
 
 	for testName, testData := range tests {
@@ -3198,6 +3231,40 @@ func TestRelabelMiddleware(t *testing.T) {
 				makeWriteRequestForGenerators(5, labelSetGenForStringPairs(t, "label4", "value4"), nil, nil),
 			},
 			expectErrs: []bool{false, false, false, false},
+		}, {
+			name: metaLabelTenantID + " available and cleaned up afterwards",
+			ctx:  ctxWithUser,
+			relabelConfigs: []*relabel.Config{
+				{
+					SourceLabels: []model.LabelName{metaLabelTenantID},
+					Action:       relabel.DefaultRelabelConfig.Action,
+					Regex:        relabel.DefaultRelabelConfig.Regex,
+					TargetLabel:  "tenant_id",
+					Replacement:  "$1",
+				},
+			},
+			reqs: []*mimirpb.WriteRequest{{
+				Timeseries: []mimirpb.PreallocTimeseries{makeWriteRequestTimeseries(
+					[]mimirpb.LabelAdapter{
+						{Name: model.MetricNameLabel, Value: "metric1"},
+						{Name: "label1", Value: "value1"},
+					},
+					123,
+					1.23,
+				)},
+			}},
+			expectedReqs: []*mimirpb.WriteRequest{{
+				Timeseries: []mimirpb.PreallocTimeseries{makeWriteRequestTimeseries(
+					[]mimirpb.LabelAdapter{
+						{Name: model.MetricNameLabel, Value: "metric1"},
+						{Name: "label1", Value: "value1"},
+						{Name: "tenant_id", Value: "user"},
+					},
+					123,
+					1.23,
+				)},
+			}},
+			expectErrs: []bool{false},
 		},
 	}
 

pkg/util/validation/limits.go

@@ -89,7 +89,7 @@ type Limits struct {
 	CreationGracePeriod       model.Duration      `yaml:"creation_grace_period" json:"creation_grace_period" category:"advanced"`
 	EnforceMetadataMetricName bool                `yaml:"enforce_metadata_metric_name" json:"enforce_metadata_metric_name" category:"advanced"`
 	IngestionTenantShardSize  int                 `yaml:"ingestion_tenant_shard_size" json:"ingestion_tenant_shard_size"`
-	MetricRelabelConfigs      []*relabel.Config   `yaml:"metric_relabel_configs,omitempty" json:"metric_relabel_configs,omitempty" doc:"nocli|description=List of metric relabel configurations. Note that in most situations, it is more effective to use metrics relabeling directly in the Prometheus server, e.g. remote_write.write_relabel_configs." category:"experimental"`
+	MetricRelabelConfigs      []*relabel.Config   `yaml:"metric_relabel_configs,omitempty" json:"metric_relabel_configs,omitempty" doc:"nocli|description=List of metric relabel configurations. Note that in most situations, it is more effective to use metrics relabeling directly in the Prometheus server, e.g. remote_write.write_relabel_configs. Labels available during the relabeling phase and cleaned afterwards: __meta_tenant_id" category:"experimental"`
 
 	// Ingester enforced limits.
 	// Series