Update distributor and compactor tests for native histograms (#4352)

Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com>
grafana · Mar 6, 2023 · 5ad1837 · 5ad1837
1 parent a033254
commit 5ad1837
Show file tree

Hide file tree

Showing 3 changed files with 291 additions and 52 deletions.
diff --git a/pkg/compactor/compactor_test.go b/pkg/compactor/compactor_test.go
@@ -1364,7 +1364,7 @@ func TestMultitenantCompactor_ShouldSkipCompactionForJobsNoMoreOwnedAfterPlannin
 	))
 }
 
-func createTSDBBlock(t *testing.T, bkt objstore.Bucket, userID string, minT, maxT int64, numSeries int, externalLabels map[string]string) ulid.ULID {
+func createCustomTSDBBlock(t *testing.T, bkt objstore.Bucket, userID string, externalLabels map[string]string, appendFunc func(*tsdb.DB)) ulid.ULID {
 	// Create a temporary dir for TSDB.
 	tempDir := t.TempDir()
 
@@ -1381,30 +1381,7 @@ func createTSDBBlock(t *testing.T, bkt objstore.Bucket, userID string, minT, max
 
 	db.DisableCompactions()
 
-	appendSample := func(seriesID int, ts int64, value float64) {
-		lbls := labels.FromStrings("series_id", strconv.Itoa(seriesID))
-
-		app := db.Appender(context.Background())
-		_, err := app.Append(0, lbls, ts, value)
-		require.NoError(t, err)
-
-		err = app.Commit()
-		require.NoError(t, err)
-	}
-
-	seriesID := 0
-
-	// Append a sample for each series, spreading it between minT and maxT-1 (both included).
-	// Since we append one more series below, here we create N-1 series.
-	if numSeries > 1 {
-		for ts := minT; ts < maxT; ts += (maxT - minT) / int64(numSeries-1) {
-			appendSample(seriesID, ts, float64(seriesID))
-			seriesID++
-		}
-	}
-
-	// Guarantee a series with a sample at time maxT-1
-	appendSample(seriesID, maxT-1, float64(seriesID))
+	appendFunc(db)
 
 	require.NoError(t, db.Compact())
 	require.NoError(t, db.Snapshot(snapshotDir, true))
@@ -1454,6 +1431,35 @@ func createTSDBBlock(t *testing.T, bkt objstore.Bucket, userID string, minT, max
 	return blockID
 }
 
+func createTSDBBlock(t *testing.T, bkt objstore.Bucket, userID string, minT, maxT int64, numSeries int, externalLabels map[string]string) ulid.ULID {
+	return createCustomTSDBBlock(t, bkt, userID, externalLabels, func(db *tsdb.DB) {
+		appendSample := func(seriesID int, ts int64, value float64) {
+			lbls := labels.FromStrings("series_id", strconv.Itoa(seriesID))
+
+			app := db.Appender(context.Background())
+			_, err := app.Append(0, lbls, ts, value)
+			require.NoError(t, err)
+
+			err = app.Commit()
+			require.NoError(t, err)
+		}
+
+		seriesID := 0
+
+		// Append a sample for each series, spreading it between minT and maxT-1 (both included).
+		// Since we append one more series below, here we create N-1 series.
+		if numSeries > 1 {
+			for ts := minT; ts < maxT; ts += (maxT - minT) / int64(numSeries-1) {
+				appendSample(seriesID, ts, float64(seriesID))
+				seriesID++
+			}
+		}
+
+		// Guarantee a series with a sample at time maxT-1
+		appendSample(seriesID, maxT-1, float64(seriesID))
+	})
+}
+
 func createDeletionMark(t *testing.T, bkt objstore.Bucket, userID string, blockID ulid.ULID, deletionTime time.Time) {
 	content := mockDeletionMarkJSON(blockID.String(), deletionTime)
 	blockPath := path.Join(userID, blockID.String())

diff --git a/pkg/compactor/split_merge_compactor_test.go b/pkg/compactor/split_merge_compactor_test.go
@@ -29,6 +29,7 @@ import (
 	mimir_tsdb "github.com/grafana/mimir/pkg/storage/tsdb"
 	"github.com/grafana/mimir/pkg/storage/tsdb/block"
 	"github.com/grafana/mimir/pkg/storage/tsdb/metadata"
+	util_test "github.com/grafana/mimir/pkg/util/test"
 )
 
 func TestMultitenantCompactor_ShouldSupportSplitAndMergeCompactor(t *testing.T) {
@@ -604,6 +605,71 @@ func TestMultitenantCompactor_ShouldSupportSplitAndMergeCompactor(t *testing.T)
 				}
 			},
 		},
+		"compaction on blocks containing native histograms": {
+			numShards: 2,
+			setup: func(t *testing.T, bkt objstore.Bucket) []metadata.Meta {
+				minT := blockRangeMillis
+				maxT := 2 * blockRangeMillis
+
+				seriesID := 0
+
+				appendHistograms := func(db *tsdb.DB) {
+					db.EnableNativeHistograms()
+
+					appendHistogram := func(seriesID int, ts int64) {
+						lbls := labels.FromStrings("series_id", strconv.Itoa(seriesID))
+
+						app := db.Appender(context.Background())
+						_, err := app.AppendHistogram(0, lbls, ts, util_test.GenerateTestHistogram(seriesID), nil)
+						require.NoError(t, err)
+
+						err = app.Commit()
+						require.NoError(t, err)
+					}
+
+					for ts := minT; ts < maxT; ts += (maxT - minT) / int64(numSeries-1) {
+						appendHistogram(seriesID, ts)
+						seriesID++
+					}
+
+					appendHistogram(seriesID, maxT-1)
+				}
+
+				block1 := createCustomTSDBBlock(t, bkt, userID, externalLabels(""), appendHistograms)
+				block2 := createCustomTSDBBlock(t, bkt, userID, externalLabels(""), appendHistograms)
+
+				return []metadata.Meta{
+					{
+						BlockMeta: tsdb.BlockMeta{
+							MinTime: 1 * blockRangeMillis,
+							MaxTime: 2 * blockRangeMillis,
+							Compaction: tsdb.BlockMetaCompaction{
+								Sources: []ulid.ULID{block1, block2},
+							},
+						},
+						Thanos: metadata.Thanos{
+							Labels: map[string]string{
+								mimir_tsdb.CompactorShardIDExternalLabel: "1_of_2",
+							},
+						},
+					},
+					{
+						BlockMeta: tsdb.BlockMeta{
+							MinTime: 1 * blockRangeMillis,
+							MaxTime: 2 * blockRangeMillis,
+							Compaction: tsdb.BlockMetaCompaction{
+								Sources: []ulid.ULID{block1, block2},
+							},
+						},
+						Thanos: metadata.Thanos{
+							Labels: map[string]string{
+								mimir_tsdb.CompactorShardIDExternalLabel: "2_of_2",
+							},
+						},
+					},
+				}
+			},
+		},
 	}
 
 	for testName, testData := range tests {