-
Notifications
You must be signed in to change notification settings - Fork 2.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Resort store response set on internal label dedup
When deduplicating on labels which are stored internally in TSDB, the store response set needs to be resorted after replica labels are removed. In order to detect when deduplication by internal labels happens, this PR adds a bloom filter with all label names to the Info response. When a replica label is present in this bloom filter for an individual store, the proxy heap would resort a response set from that store before merging in the result with the rest of the set. Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
- Loading branch information
1 parent
6d838e7
commit e0a0529
Showing
16 changed files
with
548 additions
and
81 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,67 @@ | ||
| package bloom | ||
|
|
||
| import ( | ||
| "bytes" | ||
|
|
||
| "github.com/bits-and-blooms/bloom" | ||
| ) | ||
|
|
||
| const FilterErrorRate = 0.01 | ||
|
|
||
| type Filter interface { | ||
| Test(string) bool | ||
| Bytes() []byte | ||
| } | ||
|
|
||
| type filter struct { | ||
| bloom *bloom.BloomFilter | ||
| } | ||
|
|
||
| func NewFromBytes(bloomBytes []byte) Filter { | ||
| if bloomBytes == nil { | ||
| return NewAlwaysTrueFilter() | ||
| } | ||
|
|
||
| bloomFilter := &bloom.BloomFilter{} | ||
| byteReader := bytes.NewReader(bloomBytes) | ||
| if _, err := bloomFilter.ReadFrom(byteReader); err != nil { | ||
| return NewAlwaysTrueFilter() | ||
| } | ||
|
|
||
| return &filter{bloom: bloomFilter} | ||
| } | ||
|
|
||
| func NewFilterForStrings(items ...string) Filter { | ||
| bloomFilter := bloom.NewWithEstimates(uint(len(items)), FilterErrorRate) | ||
| for _, label := range items { | ||
| bloomFilter.AddString(label) | ||
| } | ||
|
|
||
| return &filter{bloom: bloomFilter} | ||
| } | ||
|
|
||
| func (f filter) Bytes() []byte { | ||
| var buf bytes.Buffer | ||
| if _, err := f.bloom.WriteTo(&buf); err != nil { | ||
| return nil | ||
| } | ||
| return buf.Bytes() | ||
| } | ||
|
|
||
| func (f filter) Test(s string) bool { | ||
| return f.bloom.TestString(s) | ||
| } | ||
|
|
||
| type alwaysTrueFilter struct{} | ||
|
|
||
| func NewAlwaysTrueFilter() *alwaysTrueFilter { | ||
| return &alwaysTrueFilter{} | ||
| } | ||
|
|
||
| func (e alwaysTrueFilter) Test(s string) bool { | ||
| return true | ||
| } | ||
|
|
||
| func (e alwaysTrueFilter) Bytes() []byte { | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| package infopb | ||
|
|
||
| import ( | ||
| "github.com/thanos-io/thanos/pkg/bloom" | ||
| ) | ||
|
|
||
| func NewBloomFilter(filter bloom.Filter) *BloomFilter { | ||
| return &BloomFilter{ | ||
| BloomFilterData: filter.Bytes(), | ||
| } | ||
| } |
Oops, something went wrong.