store: Binary index header is now production ready and enabled by def…

…ault.

Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>

CHANGELOG.md

@@ -34,6 +34,7 @@ Since there are no consistency guarantees provided by some Object Storage provid
 - [#2294](https://github.com/thanos-io/thanos/pull/2294) store: optimizations for fetching postings. Queries using `=~".*"` matchers or negation matchers (`!=...` or `!~...`) benefit the most.
 - [#2301](https://github.com/thanos-io/thanos/pull/2301) Ruler: initlialization fails with filepath bad pattern error and rule manager update error.
 - [#2310](https://github.com/thanos-io/thanos/pull/2310) query: Report timespan 0 to 0 when discovering no stores.
+- [#2330](https://github.com/thanos-io/thanos/pull/2330) store: index-header is no longer experimental. It is enabled by default for store Gateway. You can disable it with new hidden flag: `--store.disable-index-header`. `--experimental.enable-index-header` flag was removed.
 
 ## [v0.11.0](https://github.com/thanos-io/thanos/releases/tag/v0.11.0) - 2020.03.02
 

cmd/thanos/store.go

@@ -78,7 +78,8 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application) {
 
 	selectorRelabelConf := regSelectorRelabelFlags(cmd)
 
-	enableIndexHeader := cmd.Flag("experimental.enable-index-header", "If true, Store Gateway will recreate index-header instead of index-cache.json for each block. This will replace index-cache.json permanently once it will be out of experimental stage.").
+	// TODO(bwplotka): Remove in v0.13.0 if no issues.
+	disableIndexHeader := cmd.Flag("store.disable-index-header", "If specified, Store Gateway will user index-cache.json for each block instead of recreating binary index-header").
 		Hidden().Default("false").Bool()
 
 	enablePostingsCompression := cmd.Flag("experimental.enable-index-cache-postings-compression", "If true, Store Gateway will reencode and compress postings before storing them into cache. Compressed postings take about 10% of the original size.").
@@ -117,7 +118,7 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application) {
 			uint64(*indexCacheSize),
 			uint64(*chunkPoolSize),
 			uint64(*maxSampleCount),
-			int(*maxConcurrent),
+			*maxConcurrent,
 			component.Store,
 			debugLogging,
 			*syncInterval,
@@ -128,7 +129,7 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application) {
 			},
 			selectorRelabelConf,
 			*advertiseCompatibilityLabel,
-			*enableIndexHeader,
+			*disableIndexHeader,
 			*enablePostingsCompression,
 			time.Duration(*consistencyDelay),
 			time.Duration(*ignoreDeletionMarksDelay),
@@ -163,7 +164,7 @@ func runStore(
 	filterConf *store.FilterConfig,
 	selectorRelabelConf *extflag.PathOrContent,
 	advertiseCompatibilityLabel bool,
-	enableIndexHeader bool,
+	disableIndexHeader bool,
 	enablePostingsCompression bool,
 	consistencyDelay time.Duration,
 	ignoreDeletionMarksDelay time.Duration,
@@ -252,7 +253,7 @@ func runStore(
 		return errors.Wrap(err, "meta fetcher")
 	}
 
-	if enableIndexHeader {
+	if !disableIndexHeader {
 		level.Info(logger).Log("msg", "index-header instead of index-cache.json enabled")
 	}
 	bs, err := store.NewBucketStore(
@@ -269,7 +270,7 @@ func runStore(
 		blockSyncConcurrency,
 		filterConf,
 		advertiseCompatibilityLabel,
-		enableIndexHeader,
+		disableIndexHeader,
 		enablePostingsCompression,
 	)
 	if err != nil {

pkg/store/bucket.go

@@ -247,7 +247,7 @@ type BucketStore struct {
 	filterConfig             *FilterConfig
 	advLabelSets             []storepb.LabelSet
 	enableCompatibilityLabel bool
-	enableIndexHeader        bool
+	disableIndexHeader       bool
 
 	// Reencode postings using diff+varint+snappy when storing to cache.
 	// This makes them smaller, but takes extra CPU and memory.
@@ -271,7 +271,7 @@ func NewBucketStore(
 	blockSyncConcurrency int,
 	filterConfig *FilterConfig,
 	enableCompatibilityLabel bool,
-	enableIndexHeader bool,
+	disableIndexHeader bool,
 	enablePostingsCompression bool,
 ) (*BucketStore, error) {
 	if logger == nil {
@@ -307,7 +307,7 @@ func NewBucketStore(
 		samplesLimiter:            NewLimiter(maxSampleCount, metrics.queriesDropped),
 		partitioner:               gapBasedPartitioner{maxGapSize: partitionerMaxGapSize},
 		enableCompatibilityLabel:  enableCompatibilityLabel,
-		enableIndexHeader:         enableIndexHeader,
+		disableIndexHeader:        disableIndexHeader,
 		enablePostingsCompression: enablePostingsCompression,
 	}
 	s.metrics = metrics
@@ -462,10 +462,10 @@ func (s *BucketStore) addBlock(ctx context.Context, meta *metadata.Meta) (err er
 	h := lset.Hash()
 
 	var indexHeaderReader indexheader.Reader
-	if s.enableIndexHeader {
+	if !s.disableIndexHeader {
 		indexHeaderReader, err = indexheader.NewBinaryReader(ctx, s.logger, s.bkt, s.dir, meta.ULID)
 		if err != nil {
-			return errors.Wrap(err, "create index header reader")
+			return errors.Wrap(err, "create binary index header reader")
 		}
 	} else {
 		indexHeaderReader, err = indexheader.NewJSONReader(ctx, s.logger, s.bkt, s.dir, meta.ULID)

test/e2e/e2ethanos/services.go

@@ -322,7 +322,7 @@ func NewStoreGW(sharedDir string, name string, bucketConfig client.BucketConfig,
 	store := NewService(
 		fmt.Sprintf("store-gw-%v", name),
 		DefaultImage(),
-		e2e.NewCommand("store", append(e2e.BuildArgs(map[string]string{
+		e2e.NewCommand("store", e2e.BuildArgs(map[string]string{
 			"--debug.name":        fmt.Sprintf("store-gw-%v", name),
 			"--grpc-address":      ":9091",
 			"--grpc-grace-period": "0s",
@@ -336,7 +336,7 @@ func NewStoreGW(sharedDir string, name string, bucketConfig client.BucketConfig,
 			"--store.grpc.series-max-concurrency": "1",
 			"--selector.relabel-config":           string(relabelConfigBytes),
 			"--consistency-delay":                 "30m",
-		}), "--experimental.enable-index-header")...),
+		})...),
 		e2e.NewReadinessProbe(80, "/-/ready", 200),
 		80,
 		9091,