diff --git a/CHANGELOG.md b/CHANGELOG.md index 44fbcebba27..b05f0e5531c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ * [FEATURE] Server: added experimental [PROXY protocol support](https://www.haproxy.org/download/2.3/doc/proxy-protocol.txt). The PROXY protocol support can be enabled via `-server.proxy-protocol-enabled=true`. When enabled, the support is added both to HTTP and gRPC listening ports. #7698 * [FEATURE] mimirtool: Add `runtime-config verify` sub-command, for verifying Mimir runtime config files. #8123 * [FEATURE] Query-frontend, querier: new experimental `/cardinality/active_native_histogram_metrics` API to get active native histogram metric names with statistics about active native histogram buckets. #7982 #7986 #8008 +* [ENHANCEMENT] Distributor: add metrics `cortex_distributor_otlp_samples_per_batch` to track samples per batch in otlp request. #8265 * [ENHANCEMENT] Reduced memory allocations in functions used to propagate contextual information between gRPC calls. #7529 * [ENHANCEMENT] Distributor: add experimental limit for exemplars per series per request, enabled with `-distributor.max-exemplars-per-series-per-request`, the number of discarded exemplars are tracked with `cortex_discarded_exemplars_total{reason="too_many_exemplars_per_series_per_request"}` #7989 #8010 * [ENHANCEMENT] Store-gateway: merge series from different blocks concurrently. #7456 diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index 8e5c561f28f..b4942d7f157 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -264,8 +264,9 @@ const ( ) type PushMetrics struct { - otlpRequestCounter *prometheus.CounterVec - uncompressedBodySize *prometheus.HistogramVec + otlpRequestCounter *prometheus.CounterVec + uncompressedBodySize *prometheus.HistogramVec + otlpIncomingSamplesPerBatch *prometheus.HistogramVec } func newPushMetrics(reg prometheus.Registerer) *PushMetrics { @@ -281,6 +282,11 @@ func newPushMetrics(reg prometheus.Registerer) *PushMetrics { NativeHistogramMinResetDuration: 1 * time.Hour, NativeHistogramMaxBucketNumber: 100, }, []string{"user"}), + otlpIncomingSamplesPerBatch: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ + Name: "cortex_distributor_otlp_samples_per_batch", + Help: "Number of samples per batch in otlp request.", + Buckets: []float64{1000, 3000, 5000, 7000, 8000, 9000, 10000, 12000}, + }, []string{"user"}), } } @@ -296,9 +302,16 @@ func (m *PushMetrics) ObserveUncompressedBodySize(user string, size float64) { } } +func (m *PushMetrics) ObserveOtlpIncomingSamplesPerBatch(user string, count float64) { + if m != nil { + m.otlpIncomingSamplesPerBatch.WithLabelValues(user).Observe(count) + } +} + func (m *PushMetrics) deleteUserMetrics(user string) { m.otlpRequestCounter.DeleteLabelValues(user) m.uncompressedBodySize.DeleteLabelValues(user) + m.otlpIncomingSamplesPerBatch.DeleteLabelValues(user) } // New constructs a new Distributor diff --git a/pkg/distributor/otel.go b/pkg/distributor/otel.go index d01ce91c2b9..80ee7621de2 100644 --- a/pkg/distributor/otel.go +++ b/pkg/distributor/otel.go @@ -187,6 +187,8 @@ func OTLPHandler( "exemplar_count", exemplarCount, ) + pushMetrics.ObserveOtlpIncomingSamplesPerBatch(tenantID, float64(sampleCount)) + req.Timeseries = metrics if enableOtelMetadataStorage {