OTLP: Add metrics to track otlp request samples per batch

grafana · Jun 3, 2024 · 113aa1c · 113aa1c
1 parent 960f224
commit 113aa1c
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 2 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -23,6 +23,7 @@
 * [FEATURE] Server: added experimental [PROXY protocol support](https://www.haproxy.org/download/2.3/doc/proxy-protocol.txt). The PROXY protocol support can be enabled via `-server.proxy-protocol-enabled=true`. When enabled, the support is added both to HTTP and gRPC listening ports. #7698
 * [FEATURE] mimirtool: Add `runtime-config verify` sub-command, for verifying Mimir runtime config files. #8123
 * [FEATURE] Query-frontend, querier: new experimental `/cardinality/active_native_histogram_metrics` API to get active native histogram metric names with statistics about active native histogram buckets. #7982 #7986 #8008
+* [ENHANCEMENT] Distributor: add metrics `cortex_distributor_otlp_samples_per_batch` to track samples per batch in otlp request.
 * [ENHANCEMENT] Reduced memory allocations in functions used to propagate contextual information between gRPC calls. #7529
 * [ENHANCEMENT] Distributor: add experimental limit for exemplars per series per request, enabled with `-distributor.max-exemplars-per-series-per-request`, the number of discarded exemplars are tracked with `cortex_discarded_exemplars_total{reason="too_many_exemplars_per_series_per_request"}` #7989 #8010
 * [ENHANCEMENT] Store-gateway: merge series from different blocks concurrently. #7456

diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go
@@ -136,6 +136,7 @@ type Distributor struct {
 	dedupedSamples                   *prometheus.CounterVec
 	labelsHistogram                  prometheus.Histogram
 	sampleDelayHistogram             prometheus.Histogram
+	incomingSamplesPerBatch          prometheus.Histogram
 	latestSeenSampleTimestampPerUser *prometheus.GaugeVec
 	hashCollisionCount               prometheus.Counter
 
@@ -264,8 +265,9 @@ const (
 )
 
 type PushMetrics struct {
-	otlpRequestCounter   *prometheus.CounterVec
-	uncompressedBodySize *prometheus.HistogramVec
+	otlpRequestCounter          *prometheus.CounterVec
+	uncompressedBodySize        *prometheus.HistogramVec
+	otlpIncomingSamplesPerBatch *prometheus.HistogramVec
 }
 
 func newPushMetrics(reg prometheus.Registerer) *PushMetrics {
@@ -281,6 +283,11 @@ func newPushMetrics(reg prometheus.Registerer) *PushMetrics {
 			NativeHistogramMinResetDuration: 1 * time.Hour,
 			NativeHistogramMaxBucketNumber:  100,
 		}, []string{"user"}),
+		otlpIncomingSamplesPerBatch: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{
+			Name:    "cortex_distributor_otlp_samples_per_batch",
+			Help:    "Number of samples per batch in otlp request.",
+			Buckets: []float64{1000, 3000, 5000, 7000, 8000, 9000, 10000, 12000},
+		}, []string{"user"}),
 	}
 }
 
@@ -296,9 +303,16 @@ func (m *PushMetrics) ObserveUncompressedBodySize(user string, size float64) {
 	}
 }
 
+func (m *PushMetrics) ObserveOtlpIncomingSamplesPerBatch(user string, count float64) {
+	if m != nil {
+		m.otlpIncomingSamplesPerBatch.WithLabelValues(user).Observe(count)
+	}
+}
+
 func (m *PushMetrics) deleteUserMetrics(user string) {
 	m.otlpRequestCounter.DeleteLabelValues(user)
 	m.uncompressedBodySize.DeleteLabelValues(user)
+	m.otlpIncomingSamplesPerBatch.DeleteLabelValues(user)
 }
 
 // New constructs a new Distributor

diff --git a/pkg/distributor/otel.go b/pkg/distributor/otel.go
@@ -187,6 +187,8 @@ func OTLPHandler(
 			"exemplar_count", exemplarCount,
 		)
 
+		pushMetrics.ObserveOtlpIncomingSamplesPerBatch(tenantID, float64(sampleCount))
+
 		req.Timeseries = metrics
 
 		if enableOtelMetadataStorage {