From e3420264cca007a0dbb6ef4ad897404a2b1d4ce4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Fri, 8 Oct 2021 11:28:16 +0300 Subject: [PATCH 1/8] store: discard unneeded information directly (#4750) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Discard unneeded data directly by calling Discard instead of copying to `io.Discard`. The latter has a `sync.Pool` underneath from which it retrieves byte slices into which data is read into, and after that it is immediately discarded. So, save some time by just discarding unneeded bytes directly. Comparison: ``` name old time/op new time/op delta BlockSeries/concurrency:_1-16 8.81ms ± 3% 8.35ms ± 7% -5.26% (p=0.000 n=69+76) BlockSeries/concurrency:_2-16 4.76ms ± 5% 4.36ms ± 5% -8.41% (p=0.000 n=80+74) BlockSeries/concurrency:_4-16 2.83ms ± 4% 2.70ms ± 6% -4.82% (p=0.000 n=77+80) BlockSeries/concurrency:_8-16 2.24ms ± 7% 2.21ms ± 5% -1.20% (p=0.002 n=80+78) BlockSeries/concurrency:_16-16 2.36ms ± 7% 2.24ms ± 8% -5.29% (p=0.000 n=78+76) BlockSeries/concurrency:_32-16 3.53ms ±10% 3.42ms ± 9% -3.23% (p=0.000 n=79+80) name old alloc/op new alloc/op delta BlockSeries/concurrency:_1-16 5.19MB ± 8% 5.17MB ± 5% ~ (p=0.243 n=79+76) BlockSeries/concurrency:_2-16 5.34MB ± 6% 5.27MB ± 8% -1.31% (p=0.006 n=79+79) BlockSeries/concurrency:_4-16 5.28MB ±10% 5.28MB ± 9% ~ (p=0.641 n=80+79) BlockSeries/concurrency:_8-16 5.33MB ±12% 5.39MB ± 8% ~ (p=0.143 n=80+77) BlockSeries/concurrency:_16-16 6.39MB ± 9% 6.16MB ±12% -3.66% (p=0.000 n=75+78) BlockSeries/concurrency:_32-16 9.20MB ±18% 9.03MB ±18% ~ (p=0.061 n=79+80) name old allocs/op new allocs/op delta BlockSeries/concurrency:_1-16 31.6k ± 4% 31.7k ± 3% ~ (p=0.325 n=80+76) BlockSeries/concurrency:_2-16 31.9k ± 2% 30.9k ± 3% -3.37% (p=0.000 n=80+75) BlockSeries/concurrency:_4-16 32.4k ± 3% 31.9k ± 4% -1.39% (p=0.000 n=80+80) BlockSeries/concurrency:_8-16 32.2k ± 6% 32.5k ± 4% +0.96% (p=0.011 n=78+80) BlockSeries/concurrency:_16-16 35.0k ± 7% 33.7k ± 8% -3.70% (p=0.000 n=78+76) BlockSeries/concurrency:_32-16 51.6k ± 8% 50.6k ±10% -1.81% (p=0.012 n=80+80) ``` Signed-off-by: Giedrius Statkevičius --- pkg/store/bucket.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index d181f04731..39d78b257f 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -2495,7 +2495,7 @@ func (r *bucketChunkReader) loadChunks(ctx context.Context, res []seriesEntry, a readOffset = int(pIdxs[0].offset) // Save a few allocations. - written int64 + written int diff uint32 chunkLen int n int @@ -2504,11 +2504,11 @@ func (r *bucketChunkReader) loadChunks(ctx context.Context, res []seriesEntry, a for i, pIdx := range pIdxs { // Fast forward range reader to the next chunk start in case of sparse (for our purposes) byte range. for readOffset < int(pIdx.offset) { - written, err = io.CopyN(ioutil.Discard, bufReader, int64(pIdx.offset)-int64(readOffset)) + written, err = bufReader.Discard(int(pIdx.offset) - int(readOffset)) if err != nil { return errors.Wrap(err, "fast forward range reader") } - readOffset += int(written) + readOffset += written } // Presume chunk length to be reasonably large for common use cases. // However, declaration for EstimatedMaxChunkSize warns us some chunks could be larger in some rare cases. From dde2cae91808adcaa31391c58f5db5a2b9f0cc01 Mon Sep 17 00:00:00 2001 From: aymericDD Date: Fri, 8 Oct 2021 19:32:08 +0200 Subject: [PATCH 2/8] store: validate --block-sync-concurrency parameter (#4753) * store: valide block sync concurrency parameter Must be equal or greater than 1 to avoid blocked program. Signed-off-by: Aymeric * docs: update docs (#4753) Signed-off-by: Aymeric Co-authored-by: Aymeric --- CHANGELOG.md | 1 + cmd/thanos/store.go | 2 +- docs/components/store.md | 1 + pkg/store/bucket.go | 17 +++++++++++++++++ pkg/store/bucket_test.go | 26 ++++++++++++++++++++++++++ 5 files changed, 46 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8d67abc40..a2695830e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re ### Fixed - [#4663](https://github.com/thanos-io/thanos/pull/4663) Fetcher: Fix discovered data races +- [#4753](https://github.com/thanos-io/thanos/pull/4753) Store: valide block sync concurrency parameter ### Added diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 19b6dbf837..25310f9ef3 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -113,7 +113,7 @@ func (sc *storeConfig) registerFlag(cmd extkingpin.FlagClause) { cmd.Flag("sync-block-duration", "Repeat interval for syncing the blocks between local and remote view."). Default("3m").DurationVar(&sc.syncInterval) - cmd.Flag("block-sync-concurrency", "Number of goroutines to use when constructing index-cache.json blocks from object storage."). + cmd.Flag("block-sync-concurrency", "Number of goroutines to use when constructing index-cache.json blocks from object storage. Must be equal or greater than 1."). Default("20").IntVar(&sc.blockSyncConcurrency) cmd.Flag("block-meta-fetch-concurrency", "Number of goroutines to use when fetching block metadata from object storage."). diff --git a/docs/components/store.md b/docs/components/store.md index 3ae037beb9..cdc8605052 100644 --- a/docs/components/store.md +++ b/docs/components/store.md @@ -34,6 +34,7 @@ Flags: --block-sync-concurrency=20 Number of goroutines to use when constructing index-cache.json blocks from object storage. + Must be equal or greater than 1. --chunk-pool-size=2GB Maximum size of concurrently allocatable bytes reserved strictly to reuse for chunks in memory. diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index 39d78b257f..3af0c179ec 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -92,6 +92,12 @@ const ( // Labels for metrics. labelEncode = "encode" labelDecode = "decode" + + minBlockSyncConcurrency = 1 +) + +var ( + errBlockSyncConcurrencyNotValid = errors.New("the block sync concurrency must be equal or greater than 1.") ) type bucketStoreMetrics struct { @@ -298,6 +304,13 @@ type BucketStore struct { enableSeriesResponseHints bool } +func (b *BucketStore) validate() error { + if b.blockSyncConcurrency < minBlockSyncConcurrency { + return errBlockSyncConcurrencyNotValid + } + return nil +} + type noopCache struct{} func (noopCache) StorePostings(context.Context, ulid.ULID, labels.Label, []byte) {} @@ -407,6 +420,10 @@ func NewBucketStore( s.indexReaderPool = indexheader.NewReaderPool(s.logger, lazyIndexReaderEnabled, lazyIndexReaderIdleTimeout, indexReaderPoolMetrics) s.metrics = newBucketStoreMetrics(s.reg) // TODO(metalmatze): Might be possible via Option too + if err := s.validate(); err != nil { + return nil, errors.Wrap(err, "validate config") + } + if err := os.MkdirAll(dir, 0750); err != nil { return nil, errors.Wrap(err, "create dir") } diff --git a/pkg/store/bucket_test.go b/pkg/store/bucket_test.go index e635ab22c1..adc5701740 100644 --- a/pkg/store/bucket_test.go +++ b/pkg/store/bucket_test.go @@ -556,6 +556,32 @@ func TestGapBasedPartitioner_Partition(t *testing.T) { } } +func TestBucketStoreConfig_validate(t *testing.T) { + tests := map[string]struct { + config *BucketStore + expected error + }{ + "should pass on valid config": { + config: &BucketStore{ + blockSyncConcurrency: 1, + }, + expected: nil, + }, + "should fail on blockSyncConcurrency < 1": { + config: &BucketStore{ + blockSyncConcurrency: 0, + }, + expected: errBlockSyncConcurrencyNotValid, + }, + } + + for testName, testData := range tests { + t.Run(testName, func(t *testing.T) { + testutil.Equals(t, testData.expected, testData.config.validate()) + }) + } +} + func TestBucketStore_Info(t *testing.T) { defer testutil.TolerantVerifyLeak(t) From d5156d8e10f8f6bfc880ccf98b0cbf37dd0b1304 Mon Sep 17 00:00:00 2001 From: ian woolf Date: Sun, 10 Oct 2021 17:27:32 +0800 Subject: [PATCH 3/8] pkg/block: childSources in addNodeBySources do not need to be assigned every time (#4758) Signed-off-by: ian woolf --- pkg/block/fetcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/block/fetcher.go b/pkg/block/fetcher.go index f9f45202d9..77091ecbff 100644 --- a/pkg/block/fetcher.go +++ b/pkg/block/fetcher.go @@ -660,9 +660,9 @@ func (f *DeduplicateFilter) DuplicateIDs() []ulid.ULID { func addNodeBySources(root, add *Node) bool { var rootNode *Node + childSources := add.Compaction.Sources for _, node := range root.Children { parentSources := node.Compaction.Sources - childSources := add.Compaction.Sources // Block exists with same sources, add as child. if contains(parentSources, childSources) && contains(childSources, parentSources) { From 7dee6fa1568256e282b1f07a9c90ef092d70ffcb Mon Sep 17 00:00:00 2001 From: Matej Gera <38492574+matej-g@users.noreply.github.com> Date: Tue, 12 Oct 2021 11:28:57 +0200 Subject: [PATCH 4/8] Tests: Attempt to fix flaky test in reloader on directories changes (#4765) * Fix by catching up on missed steps Signed-off-by: Matej Gera * Review feedback - simplify mutex unlock Signed-off-by: Matej Gera --- pkg/reloader/reloader_test.go | 165 ++++++++++++++++++---------------- 1 file changed, 87 insertions(+), 78 deletions(-) diff --git a/pkg/reloader/reloader_test.go b/pkg/reloader/reloader_test.go index 6659d20cc7..25a0af5ae9 100644 --- a/pkg/reloader/reloader_test.go +++ b/pkg/reloader/reloader_test.go @@ -247,6 +247,84 @@ func TestReloader_DirectoriesApply(t *testing.T) { testutil.Ok(t, os.Symlink(path.Join(dir2, "rule3-source.yaml"), path.Join(dir2, "rule3-001.yaml"))) testutil.Ok(t, ioutil.WriteFile(path.Join(dir2, "rule-dir", "rule4.yaml"), []byte("rule4"), os.ModePerm)) + stepFunc := func(rel int) { + t.Log("Performing step number", rel) + switch rel { + case 0: + // Create rule2.yaml. + // + // dir + // ├─ rule-dir -> dir2/rule-dir + // ├─ rule1.yaml + // └─ rule2.yaml (*) + // dir2 + // ├─ rule-dir + // │ └─ rule4.yaml + // ├─ rule3-001.yaml -> rule3-source.yaml + // └─ rule3-source.yaml + testutil.Ok(t, ioutil.WriteFile(path.Join(dir, "rule2.yaml"), []byte("rule2"), os.ModePerm)) + case 1: + // Update rule1.yaml. + // + // dir + // ├─ rule-dir -> dir2/rule-dir + // ├─ rule1.yaml (*) + // └─ rule2.yaml + // dir2 + // ├─ rule-dir + // │ └─ rule4.yaml + // ├─ rule3-001.yaml -> rule3-source.yaml + // └─ rule3-source.yaml + testutil.Ok(t, os.Rename(tempRule1File, path.Join(dir, "rule1.yaml"))) + case 2: + // Create dir/rule3.yaml (symlink to rule3-001.yaml). + // + // dir + // ├─ rule-dir -> dir2/rule-dir + // ├─ rule1.yaml + // ├─ rule2.yaml + // └─ rule3.yaml -> dir2/rule3-001.yaml (*) + // dir2 + // ├─ rule-dir + // │ └─ rule4.yaml + // ├─ rule3-001.yaml -> rule3-source.yaml + // └─ rule3-source.yaml + testutil.Ok(t, os.Symlink(path.Join(dir2, "rule3-001.yaml"), path.Join(dir2, "rule3.yaml"))) + testutil.Ok(t, os.Rename(path.Join(dir2, "rule3.yaml"), path.Join(dir, "rule3.yaml"))) + case 3: + // Update the symlinked file and replace the symlink file to trigger fsnotify. + // + // dir + // ├─ rule-dir -> dir2/rule-dir + // ├─ rule1.yaml + // ├─ rule2.yaml + // └─ rule3.yaml -> dir2/rule3-002.yaml (*) + // dir2 + // ├─ rule-dir + // │ └─ rule4.yaml + // ├─ rule3-002.yaml -> rule3-source.yaml (*) + // └─ rule3-source.yaml (*) + testutil.Ok(t, os.Rename(tempRule3File, path.Join(dir2, "rule3-source.yaml"))) + testutil.Ok(t, os.Symlink(path.Join(dir2, "rule3-source.yaml"), path.Join(dir2, "rule3-002.yaml"))) + testutil.Ok(t, os.Symlink(path.Join(dir2, "rule3-002.yaml"), path.Join(dir2, "rule3.yaml"))) + testutil.Ok(t, os.Rename(path.Join(dir2, "rule3.yaml"), path.Join(dir, "rule3.yaml"))) + testutil.Ok(t, os.Remove(path.Join(dir2, "rule3-001.yaml"))) + case 4: + // Update rule4.yaml in the symlinked directory. + // + // dir + // ├─ rule-dir -> dir2/rule-dir + // ├─ rule1.yaml + // ├─ rule2.yaml + // └─ rule3.yaml -> rule3-source.yaml + // dir2 + // ├─ rule-dir + // │ └─ rule4.yaml (*) + // └─ rule3-source.yaml + testutil.Ok(t, os.Rename(tempRule4File, path.Join(dir2, "rule-dir", "rule4.yaml"))) + } + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) g := sync.WaitGroup{} g.Add(1) @@ -267,90 +345,21 @@ func TestReloader_DirectoriesApply(t *testing.T) { reloadsMtx.Lock() rel := reloads + reloadsMtx.Unlock() if init && rel <= reloadsSeen { - reloadsMtx.Unlock() continue } - reloadsMtx.Unlock() - init = true - reloadsSeen = rel - t.Log("Performing step number", rel) - switch rel { - case 0: - // Create rule2.yaml. - // - // dir - // ├─ rule-dir -> dir2/rule-dir - // ├─ rule1.yaml - // └─ rule2.yaml (*) - // dir2 - // ├─ rule-dir - // │ └─ rule4.yaml - // ├─ rule3-001.yaml -> rule3-source.yaml - // └─ rule3-source.yaml - testutil.Ok(t, ioutil.WriteFile(path.Join(dir, "rule2.yaml"), []byte("rule2"), os.ModePerm)) - case 1: - // Update rule1.yaml. - // - // dir - // ├─ rule-dir -> dir2/rule-dir - // ├─ rule1.yaml (*) - // └─ rule2.yaml - // dir2 - // ├─ rule-dir - // │ └─ rule4.yaml - // ├─ rule3-001.yaml -> rule3-source.yaml - // └─ rule3-source.yaml - testutil.Ok(t, os.Rename(tempRule1File, path.Join(dir, "rule1.yaml"))) - case 2: - // Create dir/rule3.yaml (symlink to rule3-001.yaml). - // - // dir - // ├─ rule-dir -> dir2/rule-dir - // ├─ rule1.yaml - // ├─ rule2.yaml - // └─ rule3.yaml -> dir2/rule3-001.yaml (*) - // dir2 - // ├─ rule-dir - // │ └─ rule4.yaml - // ├─ rule3-001.yaml -> rule3-source.yaml - // └─ rule3-source.yaml - testutil.Ok(t, os.Symlink(path.Join(dir2, "rule3-001.yaml"), path.Join(dir2, "rule3.yaml"))) - testutil.Ok(t, os.Rename(path.Join(dir2, "rule3.yaml"), path.Join(dir, "rule3.yaml"))) - case 3: - // Update the symlinked file and replace the symlink file to trigger fsnotify. - // - // dir - // ├─ rule-dir -> dir2/rule-dir - // ├─ rule1.yaml - // ├─ rule2.yaml - // └─ rule3.yaml -> dir2/rule3-002.yaml (*) - // dir2 - // ├─ rule-dir - // │ └─ rule4.yaml - // ├─ rule3-002.yaml -> rule3-source.yaml (*) - // └─ rule3-source.yaml (*) - testutil.Ok(t, os.Rename(tempRule3File, path.Join(dir2, "rule3-source.yaml"))) - testutil.Ok(t, os.Symlink(path.Join(dir2, "rule3-source.yaml"), path.Join(dir2, "rule3-002.yaml"))) - testutil.Ok(t, os.Symlink(path.Join(dir2, "rule3-002.yaml"), path.Join(dir2, "rule3.yaml"))) - testutil.Ok(t, os.Rename(path.Join(dir2, "rule3.yaml"), path.Join(dir, "rule3.yaml"))) - testutil.Ok(t, os.Remove(path.Join(dir2, "rule3-001.yaml"))) - case 4: - // Update rule4.yaml in the symlinked directory. - // - // dir - // ├─ rule-dir -> dir2/rule-dir - // ├─ rule1.yaml - // ├─ rule2.yaml - // └─ rule3.yaml -> rule3-source.yaml - // dir2 - // ├─ rule-dir - // │ └─ rule4.yaml (*) - // └─ rule3-source.yaml - testutil.Ok(t, os.Rename(tempRule4File, path.Join(dir2, "rule-dir", "rule4.yaml"))) + // Catch up if reloader is step(s) ahead. + for skipped := rel - reloadsSeen - 1; skipped > 0; skipped-- { + stepFunc(rel - skipped) } + stepFunc(rel) + + init = true + reloadsSeen = rel + if rel > 4 { // All good. return From d2b5bc0500103020b0a30c45c4bec768ad65d9d2 Mon Sep 17 00:00:00 2001 From: Matej Gera <38492574+matej-g@users.noreply.github.com> Date: Wed, 13 Oct 2021 07:30:45 +0200 Subject: [PATCH 5/8] Query: Fix panic on stores endpoint (#4754) * Do not panic on missing component type - Skip an endpoint from stores if component type is nil - Simplify - pass only endpoint status func instead of endpoints Signed-off-by: Matej Gera * Add tests Signed-off-by: Matej Gera * Update CHANGELOG Signed-off-by: Matej Gera --- CHANGELOG.md | 8 ++-- cmd/thanos/query.go | 2 +- pkg/api/query/v1.go | 14 ++++--- pkg/api/query/v1_test.go | 87 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 100 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a2695830e6..87d8c82db3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,11 +10,6 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re ## Unreleased -### Fixed - -- [#4663](https://github.com/thanos-io/thanos/pull/4663) Fetcher: Fix discovered data races -- [#4753](https://github.com/thanos-io/thanos/pull/4753) Store: valide block sync concurrency parameter - ### Added - [#4680](https://github.com/thanos-io/thanos/pull/4680) Query: add `exemplar.partial-response` flag to control partial response. @@ -24,6 +19,9 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re ### Fixed - [#4508](https://github.com/thanos-io/thanos/pull/4508) Adjust and rename `ThanosSidecarUnhealthy` to `ThanosSidecarNoConnectionToStartedPrometheus`; Remove `ThanosSidecarPrometheusDown` alert; Remove unused `thanos_sidecar_last_heartbeat_success_time_seconds` metrics. +- [#4663](https://github.com/thanos-io/thanos/pull/4663) Fetcher: Fix discovered data races. +- [#4754](https://github.com/thanos-io/thanos/pull/4754) Query: Fix possible panic on stores endpoint. +- [#4753](https://github.com/thanos-io/thanos/pull/4753) Store: validate block sync concurrency parameter ## [v0.23.1](https://github.com/thanos-io/thanos/tree/release-0.23) - 2021.10.1 diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go index 8b280229b2..373cd57913 100644 --- a/cmd/thanos/query.go +++ b/cmd/thanos/query.go @@ -569,7 +569,7 @@ func runQuery( api := v1.NewQueryAPI( logger, - endpoints, + endpoints.GetEndpointStatus, engineFactory(promql.NewEngine, engineOpts, dynamicLookbackDelta), queryableCreator, // NOTE: Will share the same replica label as the query for now. diff --git a/pkg/api/query/v1.go b/pkg/api/query/v1.go index 4f3866b62b..fd9ee99b68 100644 --- a/pkg/api/query/v1.go +++ b/pkg/api/query/v1.go @@ -93,8 +93,8 @@ type QueryAPI struct { enableExemplarPartialResponse bool disableCORS bool - replicaLabels []string - endpointSet *query.EndpointSet + replicaLabels []string + endpointStatus func() []query.EndpointStatus defaultRangeQueryStep time.Duration defaultInstantQueryMaxSourceResolution time.Duration @@ -106,7 +106,7 @@ type QueryAPI struct { // NewQueryAPI returns an initialized QueryAPI type. func NewQueryAPI( logger log.Logger, - endpointSet *query.EndpointSet, + endpointStatus func() []query.EndpointStatus, qe func(int64) *promql.Engine, c query.QueryableCreator, ruleGroups rules.UnaryClient, @@ -146,7 +146,7 @@ func NewQueryAPI( enableMetricMetadataPartialResponse: enableMetricMetadataPartialResponse, enableExemplarPartialResponse: enableExemplarPartialResponse, replicaLabels: replicaLabels, - endpointSet: endpointSet, + endpointStatus: endpointStatus, defaultRangeQueryStep: defaultRangeQueryStep, defaultInstantQueryMaxSourceResolution: defaultInstantQueryMaxSourceResolution, defaultMetadataTimeRange: defaultMetadataTimeRange, @@ -715,7 +715,11 @@ func (qapi *QueryAPI) labelNames(r *http.Request) (interface{}, []error, *api.Ap func (qapi *QueryAPI) stores(_ *http.Request) (interface{}, []error, *api.ApiError) { statuses := make(map[string][]query.EndpointStatus) - for _, status := range qapi.endpointSet.GetEndpointStatus() { + for _, status := range qapi.endpointStatus() { + // Don't consider an endpoint if we cannot retrieve component type. + if status.ComponentType == nil { + continue + } statuses[status.ComponentType.String()] = append(statuses[status.ComponentType.String()], status) } return statuses, nil, nil diff --git a/pkg/api/query/v1_test.go b/pkg/api/query/v1_test.go index a9f0648f42..218498fe81 100644 --- a/pkg/api/query/v1_test.go +++ b/pkg/api/query/v1_test.go @@ -1201,6 +1201,93 @@ func TestMetadataEndpoints(t *testing.T) { } } +func TestStoresEndpoint(t *testing.T) { + apiWithNotEndpoints := &QueryAPI{ + endpointStatus: func() []query.EndpointStatus { + return []query.EndpointStatus{} + }, + } + apiWithValidEndpoints := &QueryAPI{ + endpointStatus: func() []query.EndpointStatus { + return []query.EndpointStatus{ + { + Name: "endpoint-1", + ComponentType: component.Store, + }, + { + Name: "endpoint-2", + ComponentType: component.Store, + }, + { + Name: "endpoint-3", + ComponentType: component.Sidecar, + }, + } + }, + } + apiWithInvalidEndpoint := &QueryAPI{ + endpointStatus: func() []query.EndpointStatus { + return []query.EndpointStatus{ + { + Name: "endpoint-1", + ComponentType: component.Store, + }, + { + Name: "endpoint-2", + }, + } + }, + } + + testCases := []endpointTestCase{ + { + endpoint: apiWithNotEndpoints.stores, + method: http.MethodGet, + response: map[string][]query.EndpointStatus{}, + }, + { + endpoint: apiWithValidEndpoints.stores, + method: http.MethodGet, + response: map[string][]query.EndpointStatus{ + "store": { + { + Name: "endpoint-1", + ComponentType: component.Store, + }, + { + Name: "endpoint-2", + ComponentType: component.Store, + }, + }, + "sidecar": { + { + Name: "endpoint-3", + ComponentType: component.Sidecar, + }, + }, + }, + }, + { + endpoint: apiWithInvalidEndpoint.stores, + method: http.MethodGet, + response: map[string][]query.EndpointStatus{ + "store": { + { + Name: "endpoint-1", + ComponentType: component.Store, + }, + }, + }, + }, + } + + for i, test := range testCases { + if ok := testEndpoint(t, test, strings.TrimSpace(fmt.Sprintf("#%d %s", i, test.query.Encode())), reflect.DeepEqual); !ok { + return + } + } +} + func TestParseTime(t *testing.T) { ts, err := time.Parse(time.RFC3339Nano, "2015-06-03T13:21:58.555Z") if err != nil { From 1af2000a84563ecadd265de5b4338e74cca13dd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Wed, 13 Oct 2021 16:57:04 +0300 Subject: [PATCH 6/8] store: get buf from chunk pool (#4755) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Get the `buf` from the chunkPool. With lots of concurrent `loadChunks`, the `make()` has a cost. Benchmark diff: ``` name old time/op new time/op delta BlockSeries/concurrency:_1-16 8.35ms ± 7% 7.80ms ± 6% -6.59% (p=0.000 n=76+75) BlockSeries/concurrency:_2-16 4.36ms ± 5% 4.30ms ± 4% -1.42% (p=0.000 n=74+79) BlockSeries/concurrency:_4-16 2.70ms ± 6% 2.63ms ± 5% -2.48% (p=0.000 n=80+77) BlockSeries/concurrency:_8-16 2.21ms ± 5% 2.23ms ± 7% ~ (p=0.055 n=78+78) BlockSeries/concurrency:_16-16 2.24ms ± 8% 2.22ms ± 8% ~ (p=0.265 n=76+78) BlockSeries/concurrency:_32-16 3.42ms ± 9% 3.39ms ±11% ~ (p=0.367 n=80+80) name old alloc/op new alloc/op delta BlockSeries/concurrency:_1-16 5.17MB ± 5% 4.95MB ± 7% -4.16% (p=0.000 n=76+78) BlockSeries/concurrency:_2-16 5.27MB ± 8% 5.10MB ± 7% -3.16% (p=0.000 n=79+78) BlockSeries/concurrency:_4-16 5.28MB ± 9% 4.92MB ± 8% -6.88% (p=0.000 n=79+79) BlockSeries/concurrency:_8-16 5.39MB ± 8% 5.14MB ± 9% -4.71% (p=0.000 n=77+80) BlockSeries/concurrency:_16-16 6.16MB ±12% 5.89MB ±12% -4.39% (p=0.000 n=78+78) BlockSeries/concurrency:_32-16 9.03MB ±18% 8.88MB ±18% ~ (p=0.137 n=80+80) name old allocs/op new allocs/op delta BlockSeries/concurrency:_1-16 31.7k ± 3% 31.1k ± 3% -1.93% (p=0.000 n=76+80) BlockSeries/concurrency:_2-16 30.9k ± 3% 31.3k ± 4% +1.58% (p=0.000 n=75+80) BlockSeries/concurrency:_4-16 31.9k ± 4% 31.5k ± 4% -1.21% (p=0.000 n=80+80) BlockSeries/concurrency:_8-16 32.5k ± 4% 32.5k ± 4% ~ (p=0.805 n=80+78) BlockSeries/concurrency:_16-16 33.7k ± 8% 33.9k ± 7% ~ (p=0.412 n=76+77) BlockSeries/concurrency:_32-16 50.6k ±10% 50.7k ±11% ~ (p=0.918 n=80+80) ``` Signed-off-by: Giedrius Statkevičius --- pkg/store/bucket.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index 3af0c179ec..be0e4bec97 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -2508,7 +2508,7 @@ func (r *bucketChunkReader) loadChunks(ctx context.Context, res []seriesEntry, a r.stats.chunksFetchedSizeSum += int(part.End - part.Start) var ( - buf = make([]byte, EstimatedMaxChunkSize) + buf []byte readOffset = int(pIdxs[0].offset) // Save a few allocations. @@ -2518,6 +2518,14 @@ func (r *bucketChunkReader) loadChunks(ctx context.Context, res []seriesEntry, a n int ) + bufPooled, err := r.block.chunkPool.Get(EstimatedMaxChunkSize) + if err == nil { + buf = *bufPooled + } else { + buf = make([]byte, EstimatedMaxChunkSize) + } + defer r.block.chunkPool.Put(&buf) + for i, pIdx := range pIdxs { // Fast forward range reader to the next chunk start in case of sparse (for our purposes) byte range. for readOffset < int(pIdx.offset) { From f8927b9a656d6920afb8ce7a8b6e728a8a377531 Mon Sep 17 00:00:00 2001 From: as5764 <31504606+as5764@users.noreply.github.com> Date: Wed, 13 Oct 2021 23:37:33 +0530 Subject: [PATCH 7/8] Added darwinbox logo for thanos webpage (#4771) Co-authored-by: Anurag Sharma --- website/data/adopters.yml | 5 ++++- website/static/logos/darwinbox.png | Bin 0 -> 11874 bytes 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 website/static/logos/darwinbox.png diff --git a/website/data/adopters.yml b/website/data/adopters.yml index 4274e443df..86762b0843 100644 --- a/website/data/adopters.yml +++ b/website/data/adopters.yml @@ -158,4 +158,7 @@ adopters: logo: itau-unibanco.png - name: LabyrinthLabs url: https://lablabs.io - logo: lablabs.png \ No newline at end of file + logo: lablabs.png +- name: Darwinbox Digital Solutions + url: https://darwinbox.com + logo: darwinbox.png \ No newline at end of file diff --git a/website/static/logos/darwinbox.png b/website/static/logos/darwinbox.png new file mode 100644 index 0000000000000000000000000000000000000000..02be75ac9e5bb50e2e70598e87d69ce05d3d3a13 GIT binary patch literal 11874 zcmbt)1yEhfvi4?g+=9EiyIZi}?gY2s?gR<$uEE{i-CYC0HArwLNYJ00d*t4_c~$?b zH+yQA%E%o|4tLU z39@QG`7~nm=SDBwvr2*=_N4CDcsD0C1ofT;vyWBVN5*TaiKFJ5N!ldeVJA0>Z1CZK zV)bqOP8R@NRT0KhBzM>L1yD71H02a>oA!BoEx9y6TDHNwVGfghOjbJ=nKMA6OjDcf zy00ja6f5o9Z#Dg?i1ANF>0jSGNs+Eykqn1B+$uRtJ8IX@jS{1}y$fqd?335#aaUpv z?fk7G2vwxyejWzH7r^~vXZx+>5K32s1^*3)q0$WNm=6Cq`Gmt)b*cc_O$?fObp(F9 zgYOc96nDQ>g{~s3hqT`k{sK6*$S&|7cqOm! zKK>RS2tpk>MdY5r@cDkw`|jhzjql$4Q}#%wnb^-9+O6_^;O31_wd+8!~rZl*i7mFtY5y& z&JT39;CgNo_pElt%%misxD>G6 zuf=THjmqG-HYu8fsNl1*?@7dykU{MkduS^bWa1vZX43riNI`m#^*iC1GcyS7drEwA{E?J$ApA? zKO8te*cX|EISlNAEGrI|tsZoo5|>4&(v~B65@5sD=FsZprFlp(h6+Q;*O)ambF-Tc zfp2}jVBT$44s7R@cM;Z%tc4Eggn64up9akix43w8sd?Q?trszP@vV6|o6Lo@U-L>l zf5?#anpUs4=k}E=B@ABYcT|a|hd;vPnk2X*`^zc?&KpFDWhcG-9o^BnlQ)Dk*G(yV z{EU((ldk&{YO_Jp9YQ-C08rY=&o}LNmrWX$Q&r{Y5MAG?%3c5iCwR__t=)^p+mi$H zR)KFkJU#SKY_q%o z2&*=YNQY+svgN;?Kq3mk|04cx2yb3DR7NBK7zhLe2Zwy!R{szN2ZA5~V30`cQ0N$} zC}?a7$f!ij9D?LbpNL6ccUkDyVPFuz7l2&`15xQAhh_0nW!{I@$kam)3l5z+i98s6 zF~SLe)WyMkA)4n`!-Vt@pAeI3CH^Z?cHzit!?ujG^G0d9os#Q5-5Zg11!FF15KnT- zZF}ttU|1O`+cQ%VS38cQh0NMPV;O}{NIiyTKcF0kiks4C8w_|h81&GB1%jP0N)zw+ zo|eeIa_?Z`rcWE>@qIn2W+H3-a$+>~*_7aI<4aqEshn$a=ak4 zeg)^m!qK6!eTvt$USqPVRFQ!bs$3)4gePM5!pa&GZ`m&k-KdJwqeY zSR$De)qb`v_E0y6We8JL+YGSmA3fW z>pU&(&W>(}e~^zz`3yD79nEgrBhb+}Z-Q6@J9#BdqCI(tgcH9;s>`wKaLqYwvww|P z__M3}i^{6QH9?7Qw1pJ`8+|!16c>qg${yQ6OL}k992CP1j}%>@XOpJ`c{YW)EbY1z zb`#8x@m{-oRbO zN>aSkXue3Cv2IPJ&)Rl24QbKNViGVU1GL9wo(z14;>hWd0>m)Jg+JH^!G$k5o=JB| z8BVx@6_rqmVfL5Q5L_Ztu?$JcP+U(;l#b$KhA!gBmJ<5PyFGSN&Nv0`Br;(Zn{s8k z5jB}r)4Z8&AKZ>v2RT@QDbDj!m@%JwWv&#oMR~h9P<-N3?ai%QP}H0Ph`fcVT`owe z{A5_T>^t^f0N!tqooX)eR28=jhv=|V)t4rBtZ?*jtVQb^0OAP#mooazY zkJd7xEM!8Tfe0PLBdB64~&VAUc1p{5g^D5H#_%DDwhHul) zHhLk*F8~37Bb}Aab3GD9Ir5u-`NV_LGHH47Z0@=LJ|(icX$3!6-4!py=0t3Ver-4# zzYenSrquz6GHxcjwoQMddYmvx=qO=lnr4I-cc*t#dj}Onh8h9$PG($!R768kSQ02> zFZ&%;!sKE{iV%{WL{`oEZjk=xo9VQ`@085YF92fr2MPB$qf85Z4kV^ld&Zi+$TRVW zXHrW^P4Od4D&6Cuj@D;`{dos&k*=Gi`q%v4KtpdI&NsW8X#mGkUr9v zox>=_Md$&^(zl*N9crbvR&i_NG^Px&A9vOXy#}?T7cWiNtZI&vB z0vlxMh!Y_ZB6o}}S-?qn_V(4Q41ahP8Vm%4`s)n+>CQlRzu@2Z)|=Xij*qy!Q^lpBWVdcRMFwh$wqKD+EwzbDfbR;8BbQZ zIA+Q_w+61UF4XNn^%c~Vz>-0oyDU7K&c*b6dO4P}{<}@poBr>_7WT>BltZvpjg-uA z$`NhnchJ3rtuKJF z^*W2}TE0Gvpozl!5Kq`HZM>?0wB%*L-E;O!uF8 zhrPKWT1f4Vg0^a?NYo(D$pYNP7VhvAsnwGgetA~%HN*gdfY30nZ<*krU(S8~3jjw( zLO~^FW)o6$j3psuF|c>2sqKM4BNA5d^~(`7^v~^`K4%h9a{3g%Nye^h#2R<;XYc{} z7WmvO1sgWcuVNOlHgS>G(#h(|IcT$cH?g{bg;t-R)-Yt?Z8F=qIAa(oE%0q2L}bJ| zu*n~k;N1Itj8;)DbqFRH5Pn=vtH;?SVx7vhn<4U*mvs}JsX&)R2Qrc}O((uX#jWr8 zw8v1_YMYQtq+~cp&xXjX=PnJ^E4M%yyWNQMVVBjXvv}h9K9BR^F#Nc|S*5_j&{?wI z4)(a##4&Oc#A#Dlk9G7@cs#$^tSV7hzx4=Anp@VO3F}?2sz}tJ*Hh6-$P(gLLl1uH zuP*>a40n;^7XW&NH6NYn))}Dbq>jJA{!!Bv4(;rY#GIc59rz$ZXj%e*alv03X*qSiB zRKij^c76FxQ87P!pPzF;jd?i!owQL&6<8Ftz;PFsv>CcmBp{%E5n_gGh^_cpO->gu zjb1lOXnI#+X})Rn7Srv)9+!C8O8#BA-S_ulxeCakWMy_7lgGbRY#8IavbFEGDAgo;_g%<#}NhgCL*IlybKxvtcDB zk?+6Hw=&h>yzvt`RKauIoy_7rX;h{;>0y3~ec7nim@HG-kZZR1L)Z(T;o0D!4&s=8 zwWNSrZ*smd6NB}_HM$A87Qe9nhxf$aO8xVowyb1zTg_uaCdIa~hgXfba3f4f?IU(< znA0N%S!0_A9XPT6#WHAI)VV}g4*O#%_znhY+ltjZ7}qWiIvobEMTDZ=oLt5QW@$2~ zbGNbvbBfvR?oZ;aVfUrwaXC7fBHBc3PE7y9+7#Ffk7z`#%DUIOa^l=W#Zi6_{-Y>ex~71<=lMP3^^ARGaaU0RciQM~JARCa#hYK-giMEr8Y6 zy=}eqKnQ|m=5$JpSYr>J#5tg~SZ6v9bhc&4R+b9+-wFG#)#I{KY z&P?eba$+EyEnlkzj*K07k~nw1;{5!EfCikkkR&&B9pn+B(Qr;ok2SG`0{W(Tb)B;% zDgC0NbuvoAf%;5BLh;W!vB*fD_uyld?rl)J!l_xXt(3nF!%Og_nJs zO&Q>MI&nga0F}altHKdf2TC`Wmg=bBE5{FxboqCZ_|rj5j+Zo{aYN&!c$s{;TnMEp zph}~_0)|xVIAK|noiA=p1&`ufdLVKlT>pU25*U`Y;L{$$@Nn{O*_77IRicsMXR&mj zn?>wFxolO)#FR^+o+l2ZFzyJXq%oHg*%%|ZwARz;r7$Jz&M_>dVyQU@@*5ll#is!Ea zA=8>)d|xt+r9$(d#|#!oK)(@N-4tttgP4cD!?m14V|+0Qt9+-=mccb!p^%h@CgmN^ zap+KxsO$uO#1_D|n5&SMOL0Sc`~#hSXRICX-FCE-X`Kb7Fq-(Vurk;7VcSS=e7L9l zo6=}pif6>T*Ug3kjZi4`M}!#{`wz1x$G*5xHtGGJ(H9!-KV}1Ij*QYo-P_u$fxbv9 zCF=NWOj3_}m2MWf#io(!ZIqKs47HEMXWygUIjYJHl?|hY(&u1{$V^`qGb~>~uUIVF!6yYF+9NSEaP=)@=j{r2Kxk#ynA0>q-G-@+( zJm%;##3h9xn-NBeGbFZ#onxkW;#6$r@Jt0dh+GNrrG+4t6Y_clfM}lJd7Pib!3$tV zF1j@9Ux4U~vLAE1+`PPe&VV>Rvo?Y@AL$sr?Tv6)2M6v2#M6HHk z^F%%>^RKZqLx6~wZ&ffoPSEBxM^qY;Tmg!dq2iQTnYiv|#Be()D7NW(ZIpEfHgcmJ z>U5nbem*WfR2|ez+p1$CAw335McR^BGXcJerwvOcmpVd{w>^di5H#3`fsP?PKfk=y z$VNqUSKQ&*c5Rj`ea+%51$?`(oHpL**+3?*|Dm=e4t5cM&tp)XBd)WJc{WUJ)l3su zco@t^Nb%hf&Ytgj*r3GaCw)Gd*8hYp_$jy%AQHuq2UI4lG1<)U1@t9p*e=e~Gk<4by zvkf@dj=v3_Icp8hu1evoa=kd_tIm5=}KyA;x_1Zys( zjAlT-vuN(qp8&h4aEMZ$o+F$KoTlnkVL@8_p}fa((%~5I#a@tWu5^+-D9*6`0}?-J ztx)=)T=#=c;hdBz6>xP_>p>U*{A7%zl%wF_kDbtT6WAV$sTNRdK|&Mi66YSnk5j4| z=TS4j65vF!oPTv5NsuYNo>;(jKHGIgk2e`U3d%R)F-HmjQyHUeh;!CUU@# z0mRHgibw|bzOgyyL^VAZ`qNB;3ZFLr%Hq5wegQONlp7jTrqSF(cmX?o^#rT3m21AU znGklQbfQ1lx+>Rt-nwDU2Bc(9EZY^kNBFu!?WVha3Cgx&(!U@)Wp{BW4l%sw;Plc3 zy$2G3(R>#ZedA*5b7h*oU~Vc(J{}aQPdWAW7UjM-s1LD(dTgX(d&tX{0Qbo4d_z}$ zNnt9T*3sNE2vfR|$XC~R(D2hJP}m4w=Nip7Ru_7{0v}5ursaGw$;D^Ao7c3oPrkFM z@KN?8|NT%Ck0TWm4qb4UP<$l zrRJiWv<4Tp`EXtgqM(N?w(Tc*faZ@4-;O&7B;F6bDis7cZOUhSOJ!i<;t_g6v30;i zNv07GFU%cVZ!l5Br5bS+)Y1vLmCv)TupF+0j$ggSZoi=ko}@7^Nrx(Q?xG_eG&MhD zrd@J&0dk?NmdwgYKb$nPI25{YU%*A0GU3AS9xe`<8Ovxa{7V#Yl7?o%@nGrQcK3Da z4v#cXa21O*Pz(nN9uZZp25Y_Uj^0-gajEh0lkfd!zI~W@3`%CNGEZ6Fbt-ZgiLJ#q z9`_LJCAufPx)gsHk`?et7^#xz7)K5~S8i@63;nfz|D*j^|L=A;3*EiA_fxKt5CjHh zLJCe?bz44LdyjKcKIW*XL|??f|X|3UEX2Ov-aIa~Ik%?~;{tyh5@U z0L{=hegigtOdFFn^@o{SbUahnAIm7*6g{rA-1CsOVCj!907!-@(xMCb+nU^5zbteU zxIWBsg{2Y}DoV)O#V^9WgQ$Vi%jS)oAqvh)(7Hi*-8m>?F8UdQ3E0iE+zeAi%IJC zA?`EPOSRu9h&O~2`^`2z8N@Cv?9D>&>{vA$-+r%(r9UrHnkIshG#WL7$@IG?K2xaY z=(UE_Vy0DAQDHg_6UiVU&<3ZHGdtv=(9(bYI0^P{8~J(8uFKE!3xuR-)X0xs!;guG z_t2WH90D~0NIXUwRH$eSDK7w%lGu{-AU^p91U!+9ptM7bZYiLK%uRz+iv}qf#@vaT zuqCJX^IR7Kk{yey5*_ubJrorlsWmx3RiEYsFdI2sNl$0Q6z{gxIXL#2{}HeH`@6S! zeQ)g?$~#?W9CLf#^&~!7PY5Cp8H3l$kne)N5#@%GtfX~0%-x5E1qJ9}clqQ{2F%BT zp#plec}p+Md^3ERHv$ zNtGxBczgh%7bq_Zaf^ykZA5Xh)4COq(S)v7~yX`u0)s~K0a&e zHQy&YPhAN6Dj4bCnNA!%Qio9|0OfJss)$AwsbBgmQ7(?yM~DX9edanK%$vIA6V@YW z6)Y$}K-~nY(C46$(J;Vkd9=$;qW7@+_Abrk>_k-P^fKy)%o16l6eMMoo#lY#a!t48 zJCcv{r`yFaXF{|maA{?ozHmjG$ft$SHtmkfMU4*vA?3|c8p#wm@qs3Nust+4`Sf^M zF3EjvQP$@}Y6)g7jc9O7GZjz~=W@>zov89cl`%nl2fEXa!I@AhRjFnK=Ok+h zE}NVZgVT8wpOQWo=Srollsfc<*Ei$q_Y@Kh@remb4q4oXh>?+)zkSA!lo>_Rk8yTk zoXlPgviF8(qf=-;2k##cr`oH}{Ca$EjXGsX(6px`H|h!Go&YsAKnLZjCYDe7kqSoG z?UULDeElN1$6keA*sj)kk$T)ciyM=nN=s%XlERU@KhDyuZf&H^*(6JBg@%}(>BaU$4b3}ZU}VaJ6Y}0Z6HEZ}kvwj`LaKzdY9b+Cu`~{#THzBLP&5aKV({a{aU;>*m#E@o zI1=1ia`2kipX;mj6k24(I=0G~`lB*3E}^C&2h~qG0P6G9>-qV|w`RbvWCqgJ6yN{7 zZX>Yuo%~+UWl5C75>k=;@3@m^3+&}_%4okeplDg!9)hOiw}PW@^9P`nO51-cn0Ifr z1wm8xTU?`@1wCjphitzUtYzlghL_&_?Tk~jws=?LQrf>2yy~N48lwAe*4KY*V1M|h z|H~DZJ+Cd-_7q$EB;Un>(^YiW%HQ)&fD=KpUCEQ7z_48n@lG;T91ckSt=wONXnPdT zK8h|!4m>V)mCc@OXycc{U$vWxcmU5=9)44rZ!?r|uAUiv8~$&>fA3uO;nnrMVf{~& z{(ZpzPyaP)BS!`R!GMsU*Gmn5XC0A=nFWOu4eVoy6mpRCYyK6k{Yv5@J^6XTIkL!1 zOvfsQ7!vKLG*ihqVyIjb6?kwwz4^{MwChWDS2Nx=Vie7IVRU+_u|4)EJfyQzGn9N6 z81*Olz-Tt2sLp7q;duGR`XeS--PTYOGmHKbhits@d*F_9FgRCa=> ziZsij^lmR0O1}oo!Y(6E`o|Iu60(Bra@sl}t=F8Dl#a>U++!DC?L2aqV-XbG>hy9a zrxZVMY32Z%;NmZOQAU~t*%drE<6@o-4^yv@ssDfs4i5EanEERph|HWrENBo*B&1-E zr084IGkvcAuiW*o94pdm?pje6H>(9HleQOvJzS((yU{CEEmZqs>e^x@)|01Nderx+ z?z-7FmMVl1JGyIz0bShM?X$PiHR?gf%@0#E$Euqx437J&qLciQa^thd6Tg~)_iG-Q zKESEjy@^OIxQUuFBi%k#{y+i=248R_aVZ@PlsY78@hlYgyY3%9aaZOd#rrc6{jru& z^6#1rOcyov6`k0y*OBVx{h_6?Q82gQEy@TD*vd%q_J9Jgs^kF$Ql=?;sAQju>si_` zV^SmhoyEw{({@?Sw^oBhd)@ap{h*(f{|wxLz}Jtk;6UJOUi6nc0Iy3x2Nf+`N|p1 zg^B3I`Chng89uY>uzfWYL?b~Dp-O-Kv>g5oR}N4GZZf2?LH6DwIA5;QMgx5l%NZdk z+)gFUQtQB2w+n3a`19LSO~#}5;-VL_?DDX_hwY-dO&>drwto`TS^9ml)+}O=0jav3 zb*EF@C6RO8*I`ooSso>?6RwHRS+9M<@!TcUbjF%5stN2mmuA#HR6A29*E9nyK7;iJ z4%9a`D%I3d$Hye7L!S=#usjI*CKe~x(yN7+56YBCchRcN_G?L+At(#6C=a6>#>dyS zUq{H)O!);p*E`PYowPX_Mb`EwIrLV)&)IyQ{&^H}m(qFw%BsNo!6CollB{%r0qZBE zqiI+%jOEDM4|;MA@pw<=AzG<6PNCanCUJucre#^-*Y>W{W4q+Selv>&(YxLjb$Bq293Br0 z)^}+cj2Nxt*)BO}{sVEtM$Rw9CLPTRm{hIqa&TBE{9rA-(A=2A4I9~)ucEs=VTsvFTPE17KPi@*Yg|kk;?ahpFW|A2j7Q=rWGo{X*<;{hux+E(z{2W^YKQs-Y7bT1=wKI0 z5w&6V(@bKJHkX+B_u8vi+DETw$E#S^KXy-OpCgY^s$M>dPqsv7x4ZaNT*2+!uY15d z2AXiLl4P-azurmO4$>@4P!z{8$6~B8UBhlD#MNpwc`8OzUqD4E_@FK`{;OP~6B+bz3c3>(?Drf{hAD;^k#$)g7c6{~D{pdrbCS8)#9rGE|b0N!S+y0@)xOB4T zP&*_m)H0*H@!o?7MrC~m{@gc~3+(fSN~Q!4B8h^IA?POkdok`*@lAaaj6}3P4i#$@ zQ>+(&xh^YE{oRHg{^lll4GX1QUwZYm{*r4hd8QCovJ{ZS!CG%#u_9X1DsXL|g{y*F zoXN+!h&nM`mB7R=y1pLM4W8;6Ma`{{4XZ9LI3XY;?V^UlH)C{A$Ells`pY9A>h4pb zUyWbjZBSesu3*a+A%L1DW`tM-ZEVb6T{y3va-NSk?H`U1hcGLqZle7;JnCXw1 zK{6I^O1oLkC?AL}rPMx4-^Pff6p+~fMQ(umglMGWmZ!VV2w%%Uq_uu=dH^?ST~VR zXK0g{xy%sF(D7ZkpS~t=vCHe^6J51I6Ttc$>$F=fJlhf>dJMfdewI24b?3}0W(0mE z;4$;P_XD)z-U@Q!s<#+FuraRB!)s+TiG?anpv?PF@a^8aQHS)Z8;PEp^tv;}lWw%R z259Jo9RFiypK)jS{6dLMLL#aD(Xf0M?wVj8^Ak9EbPvsD9Ck=ZmTv`b#%>Q=Xvb64 ze+D4ln}ai(!#Cv(Flq?yb%=3(WvmbA{#IbWxt`*x%tq>wYkezsYA^@YJlZV$21N=< z%Zh^!TjPd^TH^(P%l`t9UE?%eyx=@*4nyKfH-Jgbp-wignECuX&$I-LogJ9Qzjy%% z5v0ATiSCXt)6^;XDYPjViBk)KAd-#gwfGpS99TJwVss!@rd!P`X+Zk(z<6%AjLzRLW`Rk?=)Sa9lQQB?y>NZxXkx$?YLuY>e%S^ zciko9F?t(r1jKmWILK=dKg5^fuqXDlbVYKWIX*tOwN>4|02+_ery}_YAM6wIo+Ff2 zWRsdt>A@*r*-pc7vyUv9k7p2PWzG{JIeP{UJeC47Bs8MZk}ah*7mn$k+6EKyD|^?w z>i}-y$gm0sWy|#3v+?a6ZpR*V%<5hl54w}PlJ!?pc$C*SqU^F8H%|TaXNoARA5MJ3 z-xnK3h7Y7RiN_(>Ng#czhTT!zq8S_Kt(Zs48*%XvM@)#p#LFA3pCQv$)g@9>lPA_S ztnio82Je{FSn{(1N_~#OsBru6h3TjZjHfkR!T}P&ze7(bP_TV~h;C1+tu4>)gL(D? z({m4nFX#oJ0RJ^sR-^o<#RlyoUGVF{TzQ)6^>)em1#l^ZMdu$Pt3HM#NE$<`h^`@jx&qH`q=Wa0I?1+P#>4eqPWH3Dr59=a=| z@=5|pIu9c)LwfQviWd Date: Thu, 14 Oct 2021 16:50:57 +0800 Subject: [PATCH 8/8] add block-viewer.global.sync-block-timeout flag to set timetou of sync block metas (#4764) Signed-off-by: ian woolf --- CHANGELOG.md | 1 + cmd/thanos/compact.go | 7 +++++-- docs/components/compact.md | 4 ++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87d8c82db3..a843f4839b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#4680](https://github.com/thanos-io/thanos/pull/4680) Query: add `exemplar.partial-response` flag to control partial response. - [#4679](https://github.com/thanos-io/thanos/pull/4679) Added `enable-feature` flag to enable negative offsets and @ modifier, similar to Prometheus. - [#4696](https://github.com/thanos-io/thanos/pull/4696) Query: add cache name to tracing spans. +- [#4764](https://github.com/thanos-io/thanos/pull/4764) Compactor: add `block-viewer.global.sync-block-timeout` flag to set the timeout of synchronization block metas. ### Fixed diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 2ae488de6d..b4ff8f5690 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -537,14 +537,14 @@ func runCompact( } g.Add(func() error { - iterCtx, iterCancel := context.WithTimeout(ctx, conf.waitInterval) + iterCtx, iterCancel := context.WithTimeout(ctx, conf.blockViewerSyncBlockTimeout) _, _, _ = f.Fetch(iterCtx) iterCancel() // For /global state make sure to fetch periodically. return runutil.Repeat(conf.blockViewerSyncBlockInterval, ctx.Done(), func() error { return runutil.RetryWithLog(logger, time.Minute, ctx.Done(), func() error { - iterCtx, iterCancel := context.WithTimeout(ctx, conf.waitInterval) + iterCtx, iterCancel := context.WithTimeout(ctx, conf.blockViewerSyncBlockTimeout) defer iterCancel() _, _, err := f.Fetch(iterCtx) @@ -576,6 +576,7 @@ type compactConfig struct { blockSyncConcurrency int blockMetaFetchConcurrency int blockViewerSyncBlockInterval time.Duration + blockViewerSyncBlockTimeout time.Duration cleanupBlocksInterval time.Duration compactionConcurrency int downsampleConcurrency int @@ -634,6 +635,8 @@ func (cc *compactConfig) registerFlag(cmd extkingpin.FlagClause) { Default("32").IntVar(&cc.blockMetaFetchConcurrency) cmd.Flag("block-viewer.global.sync-block-interval", "Repeat interval for syncing the blocks between local and remote view for /global Block Viewer UI."). Default("1m").DurationVar(&cc.blockViewerSyncBlockInterval) + cmd.Flag("block-viewer.global.sync-block-timeout", "Maximum time for syncing the blocks between local and remote view for /global Block Viewer UI."). + Default("5m").DurationVar(&cc.blockViewerSyncBlockTimeout) cmd.Flag("compact.cleanup-interval", "How often we should clean up partially uploaded blocks and blocks with deletion mark in the background when --wait has been enabled. Setting it to \"0s\" disables it - the cleaning will only happen at the end of an iteration."). Default("5m").DurationVar(&cc.cleanupBlocksInterval) diff --git a/docs/components/compact.md b/docs/components/compact.md index ce1a796b36..8b662a88e7 100644 --- a/docs/components/compact.md +++ b/docs/components/compact.md @@ -284,6 +284,10 @@ Flags: Repeat interval for syncing the blocks between local and remote view for /global Block Viewer UI. + --block-viewer.global.sync-block-timeout=5m + Maximum time for syncing the blocks between + local and remote view for /global Block Viewer + UI. --bucket-web-label=BUCKET-WEB-LABEL Prometheus label to use as timeline title in the bucket web UI