store: Add Groupcache as a cache backend

cmd/thanos/store.go

@@ -256,8 +256,11 @@ func runStore(
 	if err != nil {
 		return errors.Wrap(err, "get caching bucket configuration")
 	}
+
+	r := route.New()
+
 	if len(cachingBucketConfigYaml) > 0 {
-		bkt, err = storecache.NewCachingBucketFromYaml(cachingBucketConfigYaml, bkt, logger, reg)
+		bkt, err = storecache.NewCachingBucketFromYaml(cachingBucketConfigYaml, bkt, logger, reg, r)
 		if err != nil {
 			return errors.Wrap(err, "create caching bucket")
 		}
@@ -420,7 +423,6 @@ func runStore(
 	}
 	// Add bucket UI for loaded blocks.
 	{
-		r := route.New()
 		ins := extpromhttp.NewInstrumentationMiddleware(reg, nil)
 
 		compactorView := ui.NewBucketUI(logger, "", conf.webConfig.externalPrefix, conf.webConfig.prefixHeaderName, "/loaded", conf.component)

go.mod

@@ -28,6 +28,7 @@ require (
 	github.com/felixge/fgprof v0.9.1
 	github.com/fortytw2/leaktest v1.3.0
 	github.com/fsnotify/fsnotify v1.5.1
+	github.com/go-kit/kit v0.12.0 // indirect
 	github.com/go-kit/log v0.2.0
 	github.com/go-openapi/strfmt v0.21.0
 	github.com/go-redis/redis/v8 v8.11.4
@@ -67,6 +68,7 @@ require (
 	github.com/tencentyun/cos-go-sdk-v5 v0.7.31
 	github.com/uber/jaeger-client-go v2.29.1+incompatible
 	github.com/uber/jaeger-lib v2.4.1+incompatible
+	github.com/vimeo/galaxycache v0.0.0-20210323154928-b7e5d71c067a
 	github.com/weaveworks/common v0.0.0-20210913144402-035033b78a78
 	go.elastic.co/apm v1.11.0
 	go.elastic.co/apm/module/apmot v1.11.0
@@ -120,7 +122,6 @@ require (
 	github.com/envoyproxy/go-control-plane v0.10.1 // indirect
 	github.com/envoyproxy/protoc-gen-validate v0.6.2 // indirect
 	github.com/felixge/httpsnoop v1.0.1 // indirect
-	github.com/go-kit/kit v0.12.0 // indirect
 	github.com/go-logfmt/logfmt v0.5.1 // indirect
 	github.com/go-openapi/analysis v0.20.0 // indirect
 	github.com/go-openapi/errors v0.20.0 // indirect
@@ -196,6 +197,8 @@ replace (
 	// Using a 3rd-party branch for custom dialer - see https://github.com/bradfitz/gomemcache/pull/86.
 	// Required by Cortex https://github.com/cortexproject/cortex/pull/3051.
 	github.com/bradfitz/gomemcache => github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab
+
+	github.com/cortexproject/cortex v1.10.1-0.20211124141505-4e9fc3a2b5ab => github.com/akanshat/cortex v1.10.1-0.20211222182735-328fbeedd424
 	github.com/efficientgo/tools/core => github.com/efficientgo/tools/core v0.0.0-20210731122119-5d4a0645ce9a
 	// Update to v1.1.1 to make sure windows CI pass.
 	github.com/elastic/go-sysinfo => github.com/elastic/go-sysinfo v1.1.1
@@ -205,6 +208,8 @@ replace (
 	// Make sure Prometheus version is pinned as Prometheus semver does not include Go APIs.
 	github.com/prometheus/prometheus => github.com/prometheus/prometheus v1.8.2-0.20211119115433-692a54649ed7
 	github.com/sercand/kuberesolver => github.com/sercand/kuberesolver v2.4.0+incompatible
+
+	github.com/vimeo/galaxycache => github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e
 	google.golang.org/grpc => google.golang.org/grpc v1.40.0
 
 	// Overriding to use latest commit

go.sum

@@ -172,6 +172,8 @@ github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/
 github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
 github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
 github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
+github.com/akanshat/cortex v1.10.1-0.20211222182735-328fbeedd424 h1:XxsZ4+as6/m2fO+YpQq0BYQEoC7fDP8Nje66RWG9YCw=
+github.com/akanshat/cortex v1.10.1-0.20211222182735-328fbeedd424/go.mod h1:rOgO27ZndSaiFCRrWXYRIUHAJjeGSjk7s+fDPJU6gHg=
 github.com/alecthomas/kingpin v1.3.8-0.20210301060133-17f40c25f497 h1:aDITxVUQ/3KBhpVWX57Vo9ntGTxoRw1F0T6/x/tRzNU=
 github.com/alecthomas/kingpin v1.3.8-0.20210301060133-17f40c25f497/go.mod h1:b6br6/pDFSfMkBgC96TbpOji05q5pa+v5rIlS0Y6XtI=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -446,8 +448,6 @@ github.com/cortexproject/cortex v1.6.1-0.20210215155036-dfededd9f331/go.mod h1:8
 github.com/cortexproject/cortex v1.7.1-0.20210224085859-66d6fb5b0d42/go.mod h1:u2dxcHInYbe45wxhLoWVdlFJyDhXewsMcxtnbq/QbH4=
 github.com/cortexproject/cortex v1.7.1-0.20210316085356-3fedc1108a49/go.mod h1:/DBOW8TzYBTE/U+O7Whs7i7E2eeeZl1iRVDtIqxn5kg=
 github.com/cortexproject/cortex v1.8.1-0.20210422151339-cf1c444e0905/go.mod h1:xxm4/CLvTmDxwE7yXwtClR4dIvkG4S09o5DygPOgc1U=
-github.com/cortexproject/cortex v1.10.1-0.20211124141505-4e9fc3a2b5ab h1:THN4VQQqsZn5gNwcmQJO1GarnfZkSWfp5824ifoD9fQ=
-github.com/cortexproject/cortex v1.10.1-0.20211124141505-4e9fc3a2b5ab/go.mod h1:njSBkQ1wUNx9X4knV/j65Pi4ItlJXX4QwXRKoMflJd8=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
@@ -1640,6 +1640,8 @@ github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod
 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.194/go.mod h1:yrBKWhChnDqNz1xuXdSbWXG56XawEq0G5j1lg4VwBD4=
 github.com/tencentyun/cos-go-sdk-v5 v0.7.31 h1:NujkkOKMJ3IFs1+trCwXOKRCIPQ8qI5Lxul9JkhTg6M=
 github.com/tencentyun/cos-go-sdk-v5 v0.7.31/go.mod h1:4E4+bQ2gBVJcgEC9Cufwylio4mXOct2iu05WjgEBx1o=
+github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e h1:f1Zsv7OAU9iQhZwigp50Yl38W10g/vd5NC8Rdk1Jzng=
+github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e/go.mod h1:jXcofnrSln/cLI6/dhlBxPQZEEQHVPCcFaH75M+nSzM=
 github.com/thanos-io/thanos v0.8.1-0.20200109203923-552ffa4c1a0d/go.mod h1:usT/TxtJQ7DzinTt+G9kinDQmRS5sxwu0unVKZ9vdcw=
 github.com/thanos-io/thanos v0.13.1-0.20200731083140-69b87607decf/go.mod h1:G8caR6G7pSDreRDvFm9wFuyjEBztmr8Ag3kBYpa/fEc=
 github.com/thanos-io/thanos v0.13.1-0.20200807203500-9b578afb4763/go.mod h1:KyW0a93tsh7v4hXAwo2CVAIRYuZT1Kkf4e04gisQjAg=
@@ -2144,6 +2146,7 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

pkg/cache/caching_bucket_config.go

@@ -0,0 +1,234 @@
+// Copyright (c) The Thanos Authors.
+// Licensed under the Apache License 2.0.
+
+package cache
+
+import (
+	"time"
+
+	"github.com/thanos-io/thanos/pkg/objstore"
+)
+
+// Codec for encoding and decoding results of Iter call.
+type IterCodec interface {
+	Encode(files []string) ([]byte, error)
+	Decode(cachedData []byte) ([]string, error)
+}
+
+// CachingBucketConfig contains low-level configuration for individual bucket operations.
+// This is not exposed to the user, but it is expected that code sets up individual
+// operations based on user-provided configuration.
+type CachingBucketConfig struct {
+	get        map[string]*GetConfig
+	iter       map[string]*IterConfig
+	exists     map[string]*ExistsConfig
+	getRange   map[string]*GetRangeConfig
+	attributes map[string]*AttributesConfig
+}
+
+func NewCachingBucketConfig() *CachingBucketConfig {
+	return &CachingBucketConfig{
+		get:        map[string]*GetConfig{},
+		iter:       map[string]*IterConfig{},
+		exists:     map[string]*ExistsConfig{},
+		getRange:   map[string]*GetRangeConfig{},
+		attributes: map[string]*AttributesConfig{},
+	}
+}
+
+// SetCacheImplementation sets the value of Cache for all configurations.
+func (cfg *CachingBucketConfig) SetCacheImplementation(c Cache) {
+	if cfg.get != nil {
+		for k := range cfg.get {
+			cfg.get[k].Cache = c
+		}
+	}
+	if cfg.iter != nil {
+		for k := range cfg.iter {
+			cfg.iter[k].Cache = c
+		}
+	}
+	if cfg.exists != nil {
+		for k := range cfg.exists {
+			cfg.exists[k].Cache = c
+		}
+	}
+	if cfg.getRange != nil {
+		for k := range cfg.getRange {
+			cfg.getRange[k].Cache = c
+		}
+	}
+	if cfg.attributes != nil {
+		for k := range cfg.attributes {
+			cfg.attributes[k].Cache = c
+		}
+	}
+}
+
+// Generic config for single operation.
+type OperationConfig struct {
+	Matcher func(name string) bool
+	Cache   Cache
+}
+
+// Operation-specific configs.
+type IterConfig struct {
+	OperationConfig
+	TTL   time.Duration
+	Codec IterCodec
+}
+
+type ExistsConfig struct {
+	OperationConfig
+	ExistsTTL      time.Duration
+	DoesntExistTTL time.Duration
+}
+
+type GetConfig struct {
+	ExistsConfig
+	ContentTTL       time.Duration
+	MaxCacheableSize int
+}
+
+type GetRangeConfig struct {
+	OperationConfig
+	SubrangeSize   int64
+	MaxSubRequests int
+	AttributesTTL  time.Duration
+	SubrangeTTL    time.Duration
+}
+
+type AttributesConfig struct {
+	OperationConfig
+	TTL time.Duration
+}
+
+func newOperationConfig(cache Cache, matcher func(string) bool) OperationConfig {
+	if matcher == nil {
+		panic("matcher")
+	}
+
+	return OperationConfig{
+		Matcher: matcher,
+		Cache:   cache,
+	}
+}
+
+// CacheIter configures caching of "Iter" operation for matching directories.
+func (cfg *CachingBucketConfig) CacheIter(configName string, cache Cache, matcher func(string) bool, ttl time.Duration, codec IterCodec) {
+	cfg.iter[configName] = &IterConfig{
+		OperationConfig: newOperationConfig(cache, matcher),
+		TTL:             ttl,
+		Codec:           codec,
+	}
+}
+
+// CacheGet configures caching of "Get" operation for matching files. Content of the object is cached, as well as whether object exists or not.
+func (cfg *CachingBucketConfig) CacheGet(configName string, cache Cache, matcher func(string) bool, maxCacheableSize int, contentTTL, existsTTL, doesntExistTTL time.Duration) {
+	cfg.get[configName] = &GetConfig{
+		ExistsConfig: ExistsConfig{
+			OperationConfig: newOperationConfig(cache, matcher),
+			ExistsTTL:       existsTTL,
+			DoesntExistTTL:  doesntExistTTL,
+		},
+		ContentTTL:       contentTTL,
+		MaxCacheableSize: maxCacheableSize,
+	}
+}
+
+// CacheExists configures caching of "Exists" operation for matching files. Negative values are cached as well.
+func (cfg *CachingBucketConfig) CacheExists(configName string, cache Cache, matcher func(string) bool, existsTTL, doesntExistTTL time.Duration) {
+	cfg.exists[configName] = &ExistsConfig{
+		OperationConfig: newOperationConfig(cache, matcher),
+		ExistsTTL:       existsTTL,
+		DoesntExistTTL:  doesntExistTTL,
+	}
+}
+
+// CacheGetRange configures caching of "GetRange" operation. Subranges (aligned on subrange size) are cached individually.
+// Since caching operation needs to know the object size to compute correct subranges, object size is cached as well.
+// Single "GetRange" requests can result in multiple smaller GetRange sub-requests issued on the underlying bucket.
+// MaxSubRequests specifies how many such subrequests may be issued. Values <= 0 mean there is no limit (requests
+// for adjacent missing subranges are still merged).
+func (cfg *CachingBucketConfig) CacheGetRange(configName string, cache Cache, matcher func(string) bool, subrangeSize int64, attributesTTL, subrangeTTL time.Duration, maxSubRequests int) {
+	cfg.getRange[configName] = &GetRangeConfig{
+		OperationConfig: newOperationConfig(cache, matcher),
+		SubrangeSize:    subrangeSize,
+		AttributesTTL:   attributesTTL,
+		SubrangeTTL:     subrangeTTL,
+		MaxSubRequests:  maxSubRequests,
+	}
+}
+
+// CacheAttributes configures caching of "Attributes" operation for matching files.
+func (cfg *CachingBucketConfig) CacheAttributes(configName string, cache Cache, matcher func(name string) bool, ttl time.Duration) {
+	cfg.attributes[configName] = &AttributesConfig{
+		OperationConfig: newOperationConfig(cache, matcher),
+		TTL:             ttl,
+	}
+}
+
+func (cfg *CachingBucketConfig) AllConfigNames() map[string][]string {
+	result := map[string][]string{}
+	for n := range cfg.get {
+		result[objstore.OpGet] = append(result[objstore.OpGet], n)
+	}
+	for n := range cfg.iter {
+		result[objstore.OpIter] = append(result[objstore.OpIter], n)
+	}
+	for n := range cfg.exists {
+		result[objstore.OpExists] = append(result[objstore.OpExists], n)
+	}
+	for n := range cfg.getRange {
+		result[objstore.OpGetRange] = append(result[objstore.OpGetRange], n)
+	}
+	for n := range cfg.attributes {
+		result[objstore.OpAttributes] = append(result[objstore.OpAttributes], n)
+	}
+	return result
+}
+
+func (cfg *CachingBucketConfig) FindIterConfig(dir string) (string, *IterConfig) {
+	for n, cfg := range cfg.iter {
+		if cfg.Matcher(dir) {
+			return n, cfg
+		}
+	}
+	return "", nil
+}
+
+func (cfg *CachingBucketConfig) FindExistConfig(name string) (string, *ExistsConfig) {
+	for n, cfg := range cfg.exists {
+		if cfg.Matcher(name) {
+			return n, cfg
+		}
+	}
+	return "", nil
+}
+
+func (cfg *CachingBucketConfig) FindGetConfig(name string) (string, *GetConfig) {
+	for n, cfg := range cfg.get {
+		if cfg.Matcher(name) {
+			return n, cfg
+		}
+	}
+	return "", nil
+}
+
+func (cfg *CachingBucketConfig) FindGetRangeConfig(name string) (string, *GetRangeConfig) {
+	for n, cfg := range cfg.getRange {
+		if cfg.Matcher(name) {
+			return n, cfg
+		}
+	}
+	return "", nil
+}
+
+func (cfg *CachingBucketConfig) FindAttributesConfig(name string) (string, *AttributesConfig) {
+	for n, cfg := range cfg.attributes {
+		if cfg.Matcher(name) {
+			return n, cfg
+		}
+	}
+	return "", nil
+}