diff --git a/CHANGELOG.md b/CHANGELOG.md index 421295c7a79..2e1f91df444 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ * [ENHANCEMENT] Go: update to 1.20.3. #4773 * [ENHANCEMENT] Store-gateway: reduce memory usage in some LabelValues calls. #4789 * [ENHANCEMENT] Store-gateway: add a `stage` label to the metric `cortex_bucket_store_series_data_touched`. This label now applies to `data_type="chunks"` and `data_type="series"`. The `stage` label has 2 values: `processed` - the number of series that parsed - and `returned` - the number of series selected from the processed bytes to satisfy the query. #4797 +* [ENHANCEMENT] Distributor: make `__meta_tenant_id` label available in relabeling rules configured via `metric_relabel_configs`. #4725 * [BUGFIX] Metadata API: Mimir will now return an empty object when no metadata is available, matching Prometheus. #4782 * [BUGFIX] Store-gateway: add collision detection on expanded postings and individual postings cache keys. #4770 * [BUGFIX] Packaging: fix preremove script preventing upgrades. #4801 diff --git a/cmd/mimir/config-descriptor.json b/cmd/mimir/config-descriptor.json index 80637eb2745..da2ea4163fd 100644 --- a/cmd/mimir/config-descriptor.json +++ b/cmd/mimir/config-descriptor.json @@ -3005,7 +3005,7 @@ "kind": "field", "name": "metric_relabel_configs", "required": false, - "desc": "List of metric relabel configurations. Note that in most situations, it is more effective to use metrics relabeling directly in the Prometheus server, e.g. remote_write.write_relabel_configs.", + "desc": "List of metric relabel configurations. Note that in most situations, it is more effective to use metrics relabeling directly in the Prometheus server, e.g. remote_write.write_relabel_configs. Labels available during the relabeling phase and cleaned afterwards: __meta_tenant_id", "fieldValue": null, "fieldDefaultValue": null, "fieldType": "relabel_config...", diff --git a/docs/sources/mimir/references/configuration-parameters/index.md b/docs/sources/mimir/references/configuration-parameters/index.md index bc91b9b5480..3d02caaf21a 100644 --- a/docs/sources/mimir/references/configuration-parameters/index.md +++ b/docs/sources/mimir/references/configuration-parameters/index.md @@ -2569,7 +2569,8 @@ The `limits` block configures default and per-tenant limits imposed by component # (experimental) List of metric relabel configurations. Note that in most # situations, it is more effective to use metrics relabeling directly in the -# Prometheus server, e.g. remote_write.write_relabel_configs. +# Prometheus server, e.g. remote_write.write_relabel_configs. Labels available +# during the relabeling phase and cleaned afterwards: __meta_tenant_id [metric_relabel_configs: | default = ] # The maximum number of in-memory series per tenant, across the cluster before diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index 746bf43c7a0..01b6cb77b95 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -63,6 +63,9 @@ const ( // ringAutoForgetUnhealthyPeriods is how many consecutive timeout periods an unhealthy instance // in the ring will be automatically removed after. ringAutoForgetUnhealthyPeriods = 10 + + // metaLabelTenantID is the name of the metric_relabel_configs label with tenant ID. + metaLabelTenantID = model.MetaLabelPrefix + "tenant_id" ) const ( @@ -814,16 +817,20 @@ func (d *Distributor) prePushRelabelMiddleware(next push.Func) push.Func { } var removeTsIndexes []int + lb := labels.NewBuilder(labels.EmptyLabels()) for tsIdx := 0; tsIdx < len(req.Timeseries); tsIdx++ { ts := req.Timeseries[tsIdx] if mrc := d.limits.MetricRelabelConfigs(userID); len(mrc) > 0 { - l, keep := relabel.Process(mimirpb.FromLabelAdaptersToLabels(ts.Labels), mrc...) + lb.Reset(mimirpb.FromLabelAdaptersToLabels(ts.Labels)) + lb.Set(metaLabelTenantID, userID) + keep := relabel.ProcessBuilder(lb, mrc...) if !keep { removeTsIndexes = append(removeTsIndexes, tsIdx) continue } - ts.Labels = mimirpb.FromLabelsToLabelAdapters(l) + lb.Del(metaLabelTenantID) + ts.Labels = mimirpb.FromLabelsToLabelAdapters(lb.Labels()) } for _, labelName := range d.limits.DropLabels(userID) { diff --git a/pkg/distributor/distributor_test.go b/pkg/distributor/distributor_test.go index 4a60973b599..c18c80cb7b6 100644 --- a/pkg/distributor/distributor_test.go +++ b/pkg/distributor/distributor_test.go @@ -1861,6 +1861,39 @@ func BenchmarkDistributor_Push(b *testing.B) { }, expectedErr: "received a sample whose timestamp is too far in the future", }, + "all samples go to metric_relabel_configs": { + prepareConfig: func(limits *validation.Limits) { + limits.MetricRelabelConfigs = []*relabel.Config{ + { + SourceLabels: []model.LabelName{"__name__"}, + Action: relabel.DefaultRelabelConfig.Action, + Regex: relabel.DefaultRelabelConfig.Regex, + Replacement: relabel.DefaultRelabelConfig.Replacement, + TargetLabel: "__tmp_name", + }, + } + }, + prepareSeries: func() ([]labels.Labels, []mimirpb.Sample) { + metrics := make([]labels.Labels, numSeriesPerRequest) + samples := make([]mimirpb.Sample, numSeriesPerRequest) + + for i := 0; i < numSeriesPerRequest; i++ { + lbls := labels.NewBuilder(labels.FromStrings(model.MetricNameLabel, "foo")) + for i := 0; i < 10; i++ { + lbls.Set(fmt.Sprintf("name_%d", i), fmt.Sprintf("value_%d", i)) + } + + metrics[i] = lbls.Labels() + samples[i] = mimirpb.Sample{ + Value: float64(i), + TimestampMs: time.Now().UnixNano() / int64(time.Millisecond), + } + } + + return metrics, samples + }, + expectedErr: "", + }, } for testName, testData := range tests { @@ -3215,6 +3248,40 @@ func TestRelabelMiddleware(t *testing.T) { makeWriteRequestForGenerators(5, labelSetGenForStringPairs(t, "label4", "value4"), nil, nil), }, expectErrs: []bool{false, false, false, false}, + }, { + name: metaLabelTenantID + " available and cleaned up afterwards", + ctx: ctxWithUser, + relabelConfigs: []*relabel.Config{ + { + SourceLabels: []model.LabelName{metaLabelTenantID}, + Action: relabel.DefaultRelabelConfig.Action, + Regex: relabel.DefaultRelabelConfig.Regex, + TargetLabel: "tenant_id", + Replacement: "$1", + }, + }, + reqs: []*mimirpb.WriteRequest{{ + Timeseries: []mimirpb.PreallocTimeseries{makeWriteRequestTimeseries( + []mimirpb.LabelAdapter{ + {Name: model.MetricNameLabel, Value: "metric1"}, + {Name: "label1", Value: "value1"}, + }, + 123, + 1.23, + )}, + }}, + expectedReqs: []*mimirpb.WriteRequest{{ + Timeseries: []mimirpb.PreallocTimeseries{makeWriteRequestTimeseries( + []mimirpb.LabelAdapter{ + {Name: model.MetricNameLabel, Value: "metric1"}, + {Name: "label1", Value: "value1"}, + {Name: "tenant_id", Value: "user"}, + }, + 123, + 1.23, + )}, + }}, + expectErrs: []bool{false}, }, } diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 849f092acb9..6b45745d7cc 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -90,7 +90,7 @@ type Limits struct { CreationGracePeriod model.Duration `yaml:"creation_grace_period" json:"creation_grace_period" category:"advanced"` EnforceMetadataMetricName bool `yaml:"enforce_metadata_metric_name" json:"enforce_metadata_metric_name" category:"advanced"` IngestionTenantShardSize int `yaml:"ingestion_tenant_shard_size" json:"ingestion_tenant_shard_size"` - MetricRelabelConfigs []*relabel.Config `yaml:"metric_relabel_configs,omitempty" json:"metric_relabel_configs,omitempty" doc:"nocli|description=List of metric relabel configurations. Note that in most situations, it is more effective to use metrics relabeling directly in the Prometheus server, e.g. remote_write.write_relabel_configs." category:"experimental"` + MetricRelabelConfigs []*relabel.Config `yaml:"metric_relabel_configs,omitempty" json:"metric_relabel_configs,omitempty" doc:"nocli|description=List of metric relabel configurations. Note that in most situations, it is more effective to use metrics relabeling directly in the Prometheus server, e.g. remote_write.write_relabel_configs. Labels available during the relabeling phase and cleaned afterwards: __meta_tenant_id" category:"experimental"` // Ingester enforced limits. // Series