From 0ae3021b8756760b9a4b23c0b2566de2494ec741 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Fri, 18 Jun 2021 17:42:10 +0300 Subject: [PATCH 01/22] cache: add groupcache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Giedrius Statkevičius --- cmd/thanos/store.go | 6 +- go.mod | 1 + go.sum | 5 + pkg/cache/groupcache.go | 263 ++++++++++++++++++++++ pkg/store/cache/caching_bucket_factory.go | 16 +- 5 files changed, 286 insertions(+), 5 deletions(-) create mode 100644 pkg/cache/groupcache.go diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 25310f9ef3..e27cd4355f 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -252,8 +252,11 @@ func runStore( if err != nil { return errors.Wrap(err, "get caching bucket configuration") } + + r := route.New() + if len(cachingBucketConfigYaml) > 0 { - bkt, err = storecache.NewCachingBucketFromYaml(cachingBucketConfigYaml, bkt, logger, reg) + bkt, err = storecache.NewCachingBucketFromYaml(cachingBucketConfigYaml, bkt, logger, reg, r) if err != nil { return errors.Wrap(err, "create caching bucket") } @@ -407,7 +410,6 @@ func runStore( } // Add bucket UI for loaded blocks. { - r := route.New() ins := extpromhttp.NewInstrumentationMiddleware(reg, nil) compactorView := ui.NewBucketUI(logger, "", conf.webConfig.externalPrefix, conf.webConfig.prefixHeaderName, "/loaded", conf.component) diff --git a/go.mod b/go.mod index 30292e545f..799d126db1 100644 --- a/go.mod +++ b/go.mod @@ -43,6 +43,7 @@ require ( github.com/leanovate/gopter v0.2.4 github.com/lightstep/lightstep-tracer-go v0.18.1 github.com/lovoo/gcloud-opentracing v0.3.0 + github.com/mailgun/groupcache/v2 v2.2.1 github.com/miekg/dns v1.1.43 github.com/minio/minio-go/v7 v7.0.10 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f diff --git a/go.sum b/go.sum index fad7bcd931..0834c20dcf 100644 --- a/go.sum +++ b/go.sum @@ -1166,6 +1166,8 @@ github.com/lufia/iostat v1.1.0/go.mod h1:rEPNA0xXgjHQjuI5Cy05sLlS2oRcSlWHRLrvh/A github.com/lyft/protoc-gen-star v0.5.1/go.mod h1:9toiA3cC7z5uVbODF7kEQ91Xn7XNFkVUl+SrEe+ZORU= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailgun/groupcache/v2 v2.2.1 h1:OalhvLqdhiHd76CU8gn+w6UqeEa1m60ChrVwwKkmqh4= +github.com/mailgun/groupcache/v2 v2.2.1/go.mod h1:fgFJNRQar4yVloM0SzqWhOuTF83HCO5DDXVnZQVVJ58= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -1535,6 +1537,8 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUt github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/segmentio/fasthash v0.0.0-20180216231524-a72b379d632e/go.mod h1:tm/wZFQ8e24NYaBGIlnO2WGCAi67re4HHuOm0sftE/M= +github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= +github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/sercand/kuberesolver v2.4.0+incompatible h1:WE2OlRf6wjLxHwNkkFLQGaZcVLEXjMjBPjjEU5vksH8= github.com/sercand/kuberesolver v2.4.0+incompatible/go.mod h1:lWF3GL0xptCB/vCiJPl/ZshwPsX/n4Y7u0CW9E7aQIQ= @@ -2154,6 +2158,7 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/pkg/cache/groupcache.go b/pkg/cache/groupcache.go new file mode 100644 index 0000000000..e8a7840dc6 --- /dev/null +++ b/pkg/cache/groupcache.go @@ -0,0 +1,263 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package cache + +import ( + "context" + "encoding/json" + "io/ioutil" + "regexp" + "strconv" + "strings" + "time" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/mailgun/groupcache/v2" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/route" + "github.com/thanos-io/thanos/pkg/discovery/dns" + "github.com/thanos-io/thanos/pkg/extprom" + "github.com/thanos-io/thanos/pkg/model" + "github.com/thanos-io/thanos/pkg/objstore" + "github.com/thanos-io/thanos/pkg/runutil" + "gopkg.in/yaml.v2" +) + +type Groupcache struct { + dns *dns.Provider + Pool *groupcache.HTTPPool + group *groupcache.Group + logger log.Logger +} + +// GroupcacheConfig holds the in-memory cache config. +type GroupcacheConfig struct { + // Addresses of statically configured peers (repeatable). The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect store API servers through respective DNS lookups. + // Typically, you'd want something like `dns+http://thanos-store:42/`. + Peers []string `yaml:"peers"` + + // Address of ourselves in the peer list. This needs to be set to `http://external-ip:HTTP_PORT` + // of the current instance. + SelfURL string `yaml:"self_url"` + + // Maximum size of the hot in-memory cache. + MaxSize model.Bytes `yaml:"max_size"` + + // Group's name. All of the instances need to be using the same group and point to the same bucket. + GroupcacheGroup string `yaml:"groupcache_group"` + + // DNS SD resolver to use. + DNSSDResolver dns.ResolverType `yaml:"dns_sd_resolver"` + + // How often we should resolve the addresses. + DNSInterval time.Duration `yaml:"dns_interval"` +} + +var ( + DefaultGroupcacheConfig = GroupcacheConfig{ + MaxSize: 250 * 1024 * 1024, + DNSSDResolver: dns.GolangResolverType, + DNSInterval: 1 * time.Minute, + } +) + +type ExistsVerb struct { + Name string +} + +type ContentVerb struct { + Name string +} + +type IterVerb struct { + Name string +} + +type SubrangeVerb struct { + Name string + Start, End int64 +} + +type AttributesVerb struct { + Name string +} + +func ParseGroupcacheKey(key string) interface{} { + if strings.HasPrefix(key, "attrs:") { + return &AttributesVerb{Name: key[6:]} + } + if strings.HasPrefix(key, "iter:") { + return &IterVerb{Name: key[5:]} + } + if strings.HasPrefix(key, "exists:") { + return &ExistsVerb{Name: key[7:]} + } + if strings.HasPrefix(key, "content:") { + return &ContentVerb{Name: key[8:]} + } + if strings.HasPrefix(key, "subrange:") { + r := regexp.MustCompile(`subrange:(?P.+):(?P\d+):(?P\d+)`) + matches := r.FindStringSubmatch(key) + + start, _ := strconv.ParseInt(matches[2], 10, 64) + end, _ := strconv.ParseInt(matches[3], 10, 64) + + return &SubrangeVerb{Name: matches[1], Start: start, End: end} + } + + panic("unsupported verb") +} + +// parseGroupcacheConfig unmarshals a buffer into a GroupcacheConfig with default values. +func parseGroupcacheConfig(conf []byte) (GroupcacheConfig, error) { + config := DefaultGroupcacheConfig + if err := yaml.Unmarshal(conf, &config); err != nil { + return GroupcacheConfig{}, err + } + + if len(config.Peers) == 0 { + config.Peers = append(config.Peers, config.SelfURL) + } + + return config, nil +} + +// NewGroupcache creates a new Groupcache instance. +func NewGroupcache(name string, logger log.Logger, reg prometheus.Registerer, conf []byte, groupname, basepath string, r *route.Router, bucket objstore.Bucket) (*Groupcache, error) { + config, err := parseGroupcacheConfig(conf) + if err != nil { + return nil, err + } + + return NewGroupcacheWithConfig(name, logger, reg, config, groupname, basepath, r, bucket) +} + +// NewGroupcacheWithConfig creates a new Groupcache instance with the given config. +func NewGroupcacheWithConfig(name string, logger log.Logger, reg prometheus.Registerer, conf GroupcacheConfig, groupname, basepath string, r *route.Router, bucket objstore.Bucket) (*Groupcache, error) { + dnsGroupcacheProvider := dns.NewProvider( + logger, + extprom.WrapRegistererWithPrefix("thanos_store_groupcache_", reg), + dns.ResolverType(conf.DNSSDResolver), + ) + + pool := groupcache.NewHTTPPoolOpts(conf.SelfURL, &groupcache.HTTPPoolOptions{ + BasePath: basepath, + }) + + ticker := time.NewTicker(conf.DNSInterval) + + go func() { + for { + if err := dnsGroupcacheProvider.Resolve(context.Background(), conf.Peers); err != nil { + level.Error(logger).Log("msg", "failed to resolve addresses for groupcache", "err", err) + } else { + pool.Set(dnsGroupcacheProvider.Addresses()...) + } + + <-ticker.C + } + }() + + r.Get(basepath, pool.ServeHTTP) + + group := groupcache.NewGroup(conf.GroupcacheGroup, int64(conf.MaxSize), groupcache.GetterFunc( + func(ctx context.Context, id string, dest groupcache.Sink) error { + parsedData := ParseGroupcacheKey(id) + + switch v := parsedData.(type) { + case *AttributesVerb: + attrs, err := bucket.Attributes(ctx, v.Name) + if err != nil { + return err + } + + finalAttrs, err := json.Marshal(attrs) + if err != nil { + return err + } + err = dest.SetString(string(finalAttrs), time.Now().Add(5*time.Minute)) + if err != nil { + return err + } + case *IterVerb: + // Not supported. + + return nil + case *ContentVerb: + rc, err := bucket.Get(ctx, v.Name) + if err != nil { + return err + } + defer runutil.CloseWithLogOnErr(logger, rc, "closing get") + + b, err := ioutil.ReadAll(rc) + if err != nil { + return err + } + + err = dest.SetBytes(b, time.Now().Add(5*time.Minute)) + if err != nil { + return err + } + case *ExistsVerb: + exists, err := bucket.Exists(ctx, v.Name) + if err != nil { + return err + } + + err = dest.SetString(strconv.FormatBool(exists), time.Now().Add(5*time.Minute)) + if err != nil { + return err + } + case *SubrangeVerb: + rc, err := bucket.GetRange(ctx, v.Name, v.Start, v.End-v.Start) + if err != nil { + return err + } + defer runutil.CloseWithLogOnErr(logger, rc, "closing get_range") + + b, err := ioutil.ReadAll(rc) + if err != nil { + return err + } + + err = dest.SetBytes(b, time.Now().Add(5*time.Minute)) + if err != nil { + return err + } + } + + return nil + }, + )) + + return &Groupcache{ + dns: dnsGroupcacheProvider, + Pool: pool, + group: group, + logger: logger, + }, nil +} + +func (c *Groupcache) Store(ctx context.Context, data map[string][]byte, ttl time.Duration) { + // Noop since cache is already filled during fetching. +} + +func (c *Groupcache) Fetch(ctx context.Context, keys []string) map[string][]byte { + data := map[string][]byte{} + + for _, k := range keys { + var keyData []byte + + if err := c.group.Get(ctx, k, groupcache.AllocatingByteSliceSink(&keyData)); err != nil { + level.Error(c.logger).Log("msg", "failed fetching data from groupcache", "err", err, "key", k) + return nil + } + + data[k] = keyData + } + + return data +} diff --git a/pkg/store/cache/caching_bucket_factory.go b/pkg/store/cache/caching_bucket_factory.go index c9030a040a..ca9b3674a3 100644 --- a/pkg/store/cache/caching_bucket_factory.go +++ b/pkg/store/cache/caching_bucket_factory.go @@ -12,6 +12,7 @@ import ( "github.com/go-kit/kit/log/level" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/route" "gopkg.in/yaml.v2" "github.com/thanos-io/thanos/pkg/block/metadata" @@ -25,8 +26,9 @@ import ( type BucketCacheProvider string const ( - InMemoryBucketCacheProvider BucketCacheProvider = "IN-MEMORY" // In-memory cache-provider for caching bucket. - MemcachedBucketCacheProvider BucketCacheProvider = "MEMCACHED" // Memcached cache-provider for caching bucket. + InMemoryBucketCacheProvider BucketCacheProvider = "IN-MEMORY" // In-memory cache-provider for caching bucket. + MemcachedBucketCacheProvider BucketCacheProvider = "MEMCACHED" // Memcached cache-provider for caching bucket. + GroupcacheBucketCacheProvider BucketCacheProvider = "GROUPCACHE" // Groupcache cache-provider for caching bucket. ) // CachingWithBackendConfig is a configuration of caching bucket used by Store component. @@ -67,7 +69,7 @@ func (cfg *CachingWithBackendConfig) Defaults() { } // NewCachingBucketFromYaml uses YAML configuration to create new caching bucket. -func NewCachingBucketFromYaml(yamlContent []byte, bucket objstore.Bucket, logger log.Logger, reg prometheus.Registerer) (objstore.InstrumentedBucket, error) { +func NewCachingBucketFromYaml(yamlContent []byte, bucket objstore.Bucket, logger log.Logger, reg prometheus.Registerer, r *route.Router) (objstore.InstrumentedBucket, error) { level.Info(logger).Log("msg", "loading caching bucket configuration") config := &CachingWithBackendConfig{} @@ -97,6 +99,14 @@ func NewCachingBucketFromYaml(yamlContent []byte, bucket objstore.Bucket, logger if err != nil { return nil, errors.Wrapf(err, "failed to create inmemory cache") } + case string(GroupcacheBucketCacheProvider): + const basePath = "/_groupcache/" + + c, err = cache.NewGroupcache("caching-bucket", logger, reg, backendConfig, bucket.Name(), basePath, r, bucket) + if err != nil { + return nil, errors.Wrap(err, "failed to create groupcache") + } + default: return nil, errors.Errorf("unsupported cache type: %s", config.Type) } From a99bd9e64eea9ebfea521a36ddc52b53e29f113d Mon Sep 17 00:00:00 2001 From: akanshat Date: Sat, 23 Oct 2021 16:46:02 +0530 Subject: [PATCH 02/22] implement Cache interface on groupcache Signed-off-by: akanshat --- pkg/cache/groupcache.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/cache/groupcache.go b/pkg/cache/groupcache.go index e8a7840dc6..bd96c33481 100644 --- a/pkg/cache/groupcache.go +++ b/pkg/cache/groupcache.go @@ -261,3 +261,7 @@ func (c *Groupcache) Fetch(ctx context.Context, keys []string) map[string][]byte return data } + +func (c *Groupcache) Name() string { + return c.group.Name() +} From 2bf5358011cd7e7002e37ceade47c22c58b1348f Mon Sep 17 00:00:00 2001 From: akanshat Date: Sat, 30 Oct 2021 23:44:00 +0530 Subject: [PATCH 03/22] move BucketCacheKey to a new package Signed-off-by: akanshat --- pkg/cache/groupcache.go | 75 +++--------- pkg/store/cache/cachekey/cachekey.go | 97 +++++++++++++++ pkg/store/cache/cachekey/cachekey_test.go | 120 +++++++++++++++++++ pkg/store/cache/caching_bucket.go | 102 ++-------------- pkg/store/cache/caching_bucket_test.go | 138 ++-------------------- 5 files changed, 253 insertions(+), 279 deletions(-) create mode 100644 pkg/store/cache/cachekey/cachekey.go create mode 100644 pkg/store/cache/cachekey/cachekey_test.go diff --git a/pkg/cache/groupcache.go b/pkg/cache/groupcache.go index bd96c33481..2d38d07335 100644 --- a/pkg/cache/groupcache.go +++ b/pkg/cache/groupcache.go @@ -7,9 +7,7 @@ import ( "context" "encoding/json" "io/ioutil" - "regexp" "strconv" - "strings" "time" "github.com/go-kit/kit/log" @@ -22,6 +20,7 @@ import ( "github.com/thanos-io/thanos/pkg/model" "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/runutil" + "github.com/thanos-io/thanos/pkg/store/cache/cachekey" "gopkg.in/yaml.v2" ) @@ -63,53 +62,6 @@ var ( } ) -type ExistsVerb struct { - Name string -} - -type ContentVerb struct { - Name string -} - -type IterVerb struct { - Name string -} - -type SubrangeVerb struct { - Name string - Start, End int64 -} - -type AttributesVerb struct { - Name string -} - -func ParseGroupcacheKey(key string) interface{} { - if strings.HasPrefix(key, "attrs:") { - return &AttributesVerb{Name: key[6:]} - } - if strings.HasPrefix(key, "iter:") { - return &IterVerb{Name: key[5:]} - } - if strings.HasPrefix(key, "exists:") { - return &ExistsVerb{Name: key[7:]} - } - if strings.HasPrefix(key, "content:") { - return &ContentVerb{Name: key[8:]} - } - if strings.HasPrefix(key, "subrange:") { - r := regexp.MustCompile(`subrange:(?P.+):(?P\d+):(?P\d+)`) - matches := r.FindStringSubmatch(key) - - start, _ := strconv.ParseInt(matches[2], 10, 64) - end, _ := strconv.ParseInt(matches[3], 10, 64) - - return &SubrangeVerb{Name: matches[1], Start: start, End: end} - } - - panic("unsupported verb") -} - // parseGroupcacheConfig unmarshals a buffer into a GroupcacheConfig with default values. func parseGroupcacheConfig(conf []byte) (GroupcacheConfig, error) { config := DefaultGroupcacheConfig @@ -164,11 +116,14 @@ func NewGroupcacheWithConfig(name string, logger log.Logger, reg prometheus.Regi group := groupcache.NewGroup(conf.GroupcacheGroup, int64(conf.MaxSize), groupcache.GetterFunc( func(ctx context.Context, id string, dest groupcache.Sink) error { - parsedData := ParseGroupcacheKey(id) + parsedData, err := cachekey.ParseBucketCacheKey(id) + if err != nil { + return err + } - switch v := parsedData.(type) { - case *AttributesVerb: - attrs, err := bucket.Attributes(ctx, v.Name) + switch parsedData.Verb { + case cachekey.AttributesVerb: + attrs, err := bucket.Attributes(ctx, parsedData.Name) if err != nil { return err } @@ -181,12 +136,12 @@ func NewGroupcacheWithConfig(name string, logger log.Logger, reg prometheus.Regi if err != nil { return err } - case *IterVerb: + case cachekey.IterVerb: // Not supported. return nil - case *ContentVerb: - rc, err := bucket.Get(ctx, v.Name) + case cachekey.ContentVerb: + rc, err := bucket.Get(ctx, parsedData.Name) if err != nil { return err } @@ -201,8 +156,8 @@ func NewGroupcacheWithConfig(name string, logger log.Logger, reg prometheus.Regi if err != nil { return err } - case *ExistsVerb: - exists, err := bucket.Exists(ctx, v.Name) + case cachekey.ExistsVerb: + exists, err := bucket.Exists(ctx, parsedData.Name) if err != nil { return err } @@ -211,8 +166,8 @@ func NewGroupcacheWithConfig(name string, logger log.Logger, reg prometheus.Regi if err != nil { return err } - case *SubrangeVerb: - rc, err := bucket.GetRange(ctx, v.Name, v.Start, v.End-v.Start) + case cachekey.SubrangeVerb: + rc, err := bucket.GetRange(ctx, parsedData.Name, parsedData.Start, parsedData.End-parsedData.Start) if err != nil { return err } diff --git a/pkg/store/cache/cachekey/cachekey.go b/pkg/store/cache/cachekey/cachekey.go new file mode 100644 index 0000000000..642e08f59a --- /dev/null +++ b/pkg/store/cache/cachekey/cachekey.go @@ -0,0 +1,97 @@ +package cachekey + +import ( + "fmt" + "strconv" + "strings" + + "github.com/pkg/errors" +) + +var ( + ErrInvalidBucketCacheKeyFormat = errors.New("key has invalid format") + ErrInvalidBucketCacheKeyVerb = errors.New("key has invalid verb") + ErrParseKeyInt = errors.New("failed to parse integer in key") +) + +// VerbType is the type of operation whose result has been stored in the caching bucket's cache. +type VerbType string + +const ( + ExistsVerb VerbType = "exists" + ContentVerb VerbType = "content" + IterVerb VerbType = "iter" + AttributesVerb VerbType = "attrs" + SubrangeVerb VerbType = "subrange" +) + +type BucketCacheKey struct { + Verb VerbType + Name string + Start int64 + End int64 +} + +// String returns the string representation of BucketCacheKey. +func (ck BucketCacheKey) String() string { + if ck.Start == 0 && ck.End == 0 { + return fmt.Sprintf("%s:%s", ck.Verb, ck.Name) + } + + return fmt.Sprintf("%s:%s:%d:%d", ck.Verb, ck.Name, ck.Start, ck.End) +} + +// IsValidVerb checks if the VerbType matches the predefined verbs. +func IsValidVerb(v VerbType) bool { + switch v { + case + ExistsVerb, + ContentVerb, + IterVerb, + AttributesVerb, + SubrangeVerb: + return true + } + return false +} + +// ParseBucketCacheKey parses a string and returns BucketCacheKey. +func ParseBucketCacheKey(key string) (BucketCacheKey, error) { + ck := BucketCacheKey{} + slice := strings.Split(key, ":") + if len(slice) < 2 { + return ck, ErrInvalidBucketCacheKeyFormat + } + + verb := VerbType(slice[0]) + if !IsValidVerb(verb) { + return BucketCacheKey{}, ErrInvalidBucketCacheKeyVerb + } + + if verb == SubrangeVerb { + if len(slice) != 4 { + return BucketCacheKey{}, ErrInvalidBucketCacheKeyFormat + } + + start, err := strconv.ParseInt(slice[2], 10, 64) + if err != nil { + return BucketCacheKey{}, ErrParseKeyInt + } + + end, err := strconv.ParseInt(slice[3], 10, 64) + if err != nil { + return BucketCacheKey{}, ErrParseKeyInt + } + + ck.Start = start + ck.End = end + } else { + if len(slice) != 2 { + return BucketCacheKey{}, ErrInvalidBucketCacheKeyFormat + } + } + + ck.Verb = verb + ck.Name = slice[1] + return ck, nil +} diff --git a/pkg/store/cache/cachekey/cachekey_test.go b/pkg/store/cache/cachekey/cachekey_test.go new file mode 100644 index 0000000000..757be8f7d4 --- /dev/null +++ b/pkg/store/cache/cachekey/cachekey_test.go @@ -0,0 +1,120 @@ +package cachekey + +import ( + "testing" + + "github.com/thanos-io/thanos/pkg/testutil" +) + +func TestParseBucketCacheKey(t *testing.T) { + testcases := []struct { + key string + expected BucketCacheKey + expectedErr error + }{ + { + key: "exists:name", + expected: BucketCacheKey{ + Verb: ExistsVerb, + Name: "name", + Start: 0, + End: 0, + }, + expectedErr: nil, + }, + { + key: "content:name", + expected: BucketCacheKey{ + Verb: ContentVerb, + Name: "name", + Start: 0, + End: 0, + }, + expectedErr: nil, + }, + { + key: "iter:name", + expected: BucketCacheKey{ + Verb: IterVerb, + Name: "name", + Start: 0, + End: 0, + }, + expectedErr: nil, + }, + { + key: "attrs:name", + expected: BucketCacheKey{ + Verb: AttributesVerb, + Name: "name", + Start: 0, + End: 0, + }, + expectedErr: nil, + }, + { + key: "subrange:name:10:20", + expected: BucketCacheKey{ + Verb: SubrangeVerb, + Name: "name", + Start: 10, + End: 20, + }, + expectedErr: nil, + }, + // Any VerbType other than SubrangeVerb should not have a "start" and "end". + { + key: "iter:name:10:20", + expected: BucketCacheKey{}, + expectedErr: ErrInvalidBucketCacheKeyFormat, + }, + // Key must always have a name. + { + key: "iter", + expected: BucketCacheKey{}, + expectedErr: ErrInvalidBucketCacheKeyFormat, + }, + // Invalid VerbType should return an error. + { + key: "random:name", + expected: BucketCacheKey{}, + expectedErr: ErrInvalidBucketCacheKeyVerb, + }, + // Start must be an integer. + { + key: "subrange:name:random:10", + expected: BucketCacheKey{}, + expectedErr: ErrParseKeyInt, + }, + // End must be an integer. + { + key: "subrange:name:10:random", + expected: BucketCacheKey{}, + expectedErr: ErrParseKeyInt, + }, + // SubrangeVerb must have start and end. + { + key: "subrange:name", + expected: BucketCacheKey{}, + expectedErr: ErrInvalidBucketCacheKeyFormat, + }, + // SubrangeVerb must have start and end both. + { + key: "subrange:name:10", + expected: BucketCacheKey{}, + expectedErr: ErrInvalidBucketCacheKeyFormat, + }, + // Key must not be an empty string. + { + key: "", + expected: BucketCacheKey{}, + expectedErr: ErrInvalidBucketCacheKeyFormat, + }, + } + + for _, tc := range testcases { + res, err := ParseBucketCacheKey(tc.key) + testutil.Equals(t, tc.expectedErr, err) + testutil.Equals(t, tc.expected, res) + } +} diff --git a/pkg/store/cache/caching_bucket.go b/pkg/store/cache/caching_bucket.go index c29e793d08..5e72a57e1e 100644 --- a/pkg/store/cache/caching_bucket.go +++ b/pkg/store/cache/caching_bucket.go @@ -7,11 +7,9 @@ import ( "bytes" "context" "encoding/json" - "fmt" "io" "io/ioutil" "strconv" - "strings" "sync" "time" @@ -25,6 +23,7 @@ import ( "github.com/thanos-io/thanos/pkg/cache" "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/runutil" + "github.com/thanos-io/thanos/pkg/store/cache/cachekey" ) const ( @@ -33,10 +32,7 @@ const ( ) var ( - errObjNotFound = errors.Errorf("object not found") - ErrInvalidBucketCacheKeyFormat = errors.New("key has invalid format") - ErrInvalidBucketCacheKeyVerb = errors.New("key has invalid verb") - ErrParseKeyInt = errors.New("failed to parse integer in key") + errObjNotFound = errors.Errorf("object not found") ) // CachingBucket implementation that provides some caching features, based on passed configuration. @@ -136,7 +132,7 @@ func (cb *CachingBucket) Iter(ctx context.Context, dir string, f func(string) er } cb.operationRequests.WithLabelValues(objstore.OpIter, cfgName).Inc() - iterVerb := BucketCacheKey{Verb: IterVerb, Name: dir} + iterVerb := cachekey.BucketCacheKey{Verb: cachekey.IterVerb, Name: dir} key := iterVerb.String() data := cfg.cache.Fetch(ctx, []string{key}) if data[key] != nil { @@ -182,7 +178,7 @@ func (cb *CachingBucket) Exists(ctx context.Context, name string) (bool, error) cb.operationRequests.WithLabelValues(objstore.OpExists, cfgName).Inc() - existsVerb := BucketCacheKey{Verb: ExistsVerb, Name: name} + existsVerb := cachekey.BucketCacheKey{Verb: cachekey.ExistsVerb, Name: name} key := existsVerb.String() hits := cfg.cache.Fetch(ctx, []string{key}) @@ -225,9 +221,9 @@ func (cb *CachingBucket) Get(ctx context.Context, name string) (io.ReadCloser, e cb.operationRequests.WithLabelValues(objstore.OpGet, cfgName).Inc() - contentVerb := BucketCacheKey{Verb: ContentVerb, Name: name} + contentVerb := cachekey.BucketCacheKey{Verb: cachekey.ContentVerb, Name: name} contentKey := contentVerb.String() - existsVerb := BucketCacheKey{Verb: ExistsVerb, Name: name} + existsVerb := cachekey.BucketCacheKey{Verb: cachekey.ExistsVerb, Name: name} existsKey := existsVerb.String() hits := cfg.cache.Fetch(ctx, []string{contentKey, existsKey}) @@ -295,7 +291,7 @@ func (cb *CachingBucket) Attributes(ctx context.Context, name string) (objstore. } func (cb *CachingBucket) cachedAttributes(ctx context.Context, name, cfgName string, cache cache.Cache, ttl time.Duration) (objstore.ObjectAttributes, error) { - attrVerb := BucketCacheKey{Verb: AttributesVerb, Name: name} + attrVerb := cachekey.BucketCacheKey{Verb: cachekey.AttributesVerb, Name: name} key := attrVerb.String() cb.operationRequests.WithLabelValues(objstore.OpAttributes, cfgName).Inc() @@ -367,7 +363,7 @@ func (cb *CachingBucket) cachedGetRange(ctx context.Context, name string, offset end = attrs.Size } totalRequestedBytes += (end - off) - objectSubrange := BucketCacheKey{Verb: SubrangeVerb, Name: name, Start: off, End: end} + objectSubrange := cachekey.BucketCacheKey{Verb: cachekey.SubrangeVerb, Name: name, Start: off, End: end} k := objectSubrange.String() keys = append(keys, k) offsetKeys[off] = k @@ -501,88 +497,6 @@ func mergeRanges(input []rng, limit int64) []rng { return input[:last+1] } -// VerbType is the type of operation whose result has been stored in the caching bucket's cache. -type VerbType string - -const ( - ExistsVerb VerbType = "exists" - ContentVerb VerbType = "content" - IterVerb VerbType = "iter" - AttributesVerb VerbType = "attrs" - SubrangeVerb VerbType = "subrange" -) - -type BucketCacheKey struct { - Verb VerbType - Name string - Start int64 - End int64 -} - -// String returns the string representation of BucketCacheKey. -func (ck BucketCacheKey) String() string { - if ck.Start == 0 && ck.End == 0 { - return fmt.Sprintf("%s:%s", ck.Verb, ck.Name) - } - - return fmt.Sprintf("%s:%s:%d:%d", ck.Verb, ck.Name, ck.Start, ck.End) -} - -// IsValidVerb checks if the VerbType matches the predefined verbs. -func IsValidVerb(v VerbType) bool { - switch v { - case - ExistsVerb, - ContentVerb, - IterVerb, - AttributesVerb, - SubrangeVerb: - return true - } - return false -} - -// ParseBucketCacheKey parses a string and returns BucketCacheKey. -func ParseBucketCacheKey(key string) (BucketCacheKey, error) { - ck := BucketCacheKey{} - slice := strings.Split(key, ":") - if len(slice) < 2 { - return ck, ErrInvalidBucketCacheKeyFormat - } - - verb := VerbType(slice[0]) - if !IsValidVerb(verb) { - return BucketCacheKey{}, ErrInvalidBucketCacheKeyVerb - } - - if verb == SubrangeVerb { - if len(slice) != 4 { - return BucketCacheKey{}, ErrInvalidBucketCacheKeyFormat - } - - start, err := strconv.ParseInt(slice[2], 10, 64) - if err != nil { - return BucketCacheKey{}, ErrParseKeyInt - } - - end, err := strconv.ParseInt(slice[3], 10, 64) - if err != nil { - return BucketCacheKey{}, ErrParseKeyInt - } - - ck.Start = start - ck.End = end - } else { - if len(slice) != 2 { - return BucketCacheKey{}, ErrInvalidBucketCacheKeyFormat - } - } - - ck.Verb = verb - ck.Name = slice[1] - return ck, nil -} - // Reader implementation that uses in-memory subranges. type subrangesReader struct { subrangeSize int64 diff --git a/pkg/store/cache/caching_bucket_test.go b/pkg/store/cache/caching_bucket_test.go index 3587571692..cbfcb91f2d 100644 --- a/pkg/store/cache/caching_bucket_test.go +++ b/pkg/store/cache/caching_bucket_test.go @@ -21,6 +21,7 @@ import ( "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/runutil" + "github.com/thanos-io/thanos/pkg/store/cache/cachekey" "github.com/thanos-io/thanos/pkg/testutil" ) @@ -125,11 +126,11 @@ func TestChunksCaching(t *testing.T) { expectedCachedBytes: 7 * subrangeSize, init: func() { // Delete first 3 subranges. - objectSubrange := BucketCacheKey{Verb: SubrangeVerb, Name: name, Start: 0 * subrangeSize, End: 1 * subrangeSize} + objectSubrange := cachekey.BucketCacheKey{Verb: cachekey.SubrangeVerb, Name: name, Start: 0 * subrangeSize, End: 1 * subrangeSize} delete(cache.cache, objectSubrange.String()) - objectSubrange = BucketCacheKey{Verb: SubrangeVerb, Name: name, Start: 1 * subrangeSize, End: 2 * subrangeSize} + objectSubrange = cachekey.BucketCacheKey{Verb: cachekey.SubrangeVerb, Name: name, Start: 1 * subrangeSize, End: 2 * subrangeSize} delete(cache.cache, objectSubrange.String()) - objectSubrange = BucketCacheKey{Verb: SubrangeVerb, Name: name, Start: 2 * subrangeSize, End: 3 * subrangeSize} + objectSubrange = cachekey.BucketCacheKey{Verb: cachekey.SubrangeVerb, Name: name, Start: 2 * subrangeSize, End: 3 * subrangeSize} delete(cache.cache, objectSubrange.String()) }, }, @@ -143,11 +144,11 @@ func TestChunksCaching(t *testing.T) { expectedCachedBytes: 7 * subrangeSize, init: func() { // Delete last 3 subranges. - objectSubrange := BucketCacheKey{Verb: SubrangeVerb, Name: name, Start: 7 * subrangeSize, End: 8 * subrangeSize} + objectSubrange := cachekey.BucketCacheKey{Verb: cachekey.SubrangeVerb, Name: name, Start: 7 * subrangeSize, End: 8 * subrangeSize} delete(cache.cache, objectSubrange.String()) - objectSubrange = BucketCacheKey{Verb: SubrangeVerb, Name: name, Start: 8 * subrangeSize, End: 9 * subrangeSize} + objectSubrange = cachekey.BucketCacheKey{Verb: cachekey.SubrangeVerb, Name: name, Start: 8 * subrangeSize, End: 9 * subrangeSize} delete(cache.cache, objectSubrange.String()) - objectSubrange = BucketCacheKey{Verb: SubrangeVerb, Name: name, Start: 9 * subrangeSize, End: 10 * subrangeSize} + objectSubrange = cachekey.BucketCacheKey{Verb: cachekey.SubrangeVerb, Name: name, Start: 9 * subrangeSize, End: 10 * subrangeSize} delete(cache.cache, objectSubrange.String()) }, }, @@ -161,11 +162,11 @@ func TestChunksCaching(t *testing.T) { expectedCachedBytes: 7 * subrangeSize, init: func() { // Delete 3 subranges in the middle. - objectSubrange := BucketCacheKey{Verb: SubrangeVerb, Name: name, Start: 3 * subrangeSize, End: 4 * subrangeSize} + objectSubrange := cachekey.BucketCacheKey{Verb: cachekey.SubrangeVerb, Name: name, Start: 3 * subrangeSize, End: 4 * subrangeSize} delete(cache.cache, objectSubrange.String()) - objectSubrange = BucketCacheKey{Verb: SubrangeVerb, Name: name, Start: 4 * subrangeSize, End: 5 * subrangeSize} + objectSubrange = cachekey.BucketCacheKey{Verb: cachekey.SubrangeVerb, Name: name, Start: 4 * subrangeSize, End: 5 * subrangeSize} delete(cache.cache, objectSubrange.String()) - objectSubrange = BucketCacheKey{Verb: SubrangeVerb, Name: name, Start: 5 * subrangeSize, End: 6 * subrangeSize} + objectSubrange = cachekey.BucketCacheKey{Verb: cachekey.SubrangeVerb, Name: name, Start: 5 * subrangeSize, End: 6 * subrangeSize} delete(cache.cache, objectSubrange.String()) }, }, @@ -183,7 +184,7 @@ func TestChunksCaching(t *testing.T) { if i > 0 && i%3 == 0 { continue } - objectSubrange := BucketCacheKey{Verb: SubrangeVerb, Name: name, Start: i * subrangeSize, End: (i + 1) * subrangeSize} + objectSubrange := cachekey.BucketCacheKey{Verb: cachekey.SubrangeVerb, Name: name, Start: i * subrangeSize, End: (i + 1) * subrangeSize} delete(cache.cache, objectSubrange.String()) } }, @@ -204,7 +205,7 @@ func TestChunksCaching(t *testing.T) { if i == 3 || i == 5 || i == 7 { continue } - objectSubrange := BucketCacheKey{Verb: SubrangeVerb, Name: name, Start: i * subrangeSize, End: (i + 1) * subrangeSize} + objectSubrange := cachekey.BucketCacheKey{Verb: cachekey.SubrangeVerb, Name: name, Start: i * subrangeSize, End: (i + 1) * subrangeSize} delete(cache.cache, objectSubrange.String()) } }, @@ -224,7 +225,7 @@ func TestChunksCaching(t *testing.T) { if i == 5 || i == 6 || i == 7 { continue } - objectSubrange := BucketCacheKey{Verb: SubrangeVerb, Name: name, Start: i * subrangeSize, End: (i + 1) * subrangeSize} + objectSubrange := cachekey.BucketCacheKey{Verb: cachekey.SubrangeVerb, Name: name, Start: i * subrangeSize, End: (i + 1) * subrangeSize} delete(cache.cache, objectSubrange.String()) } }, @@ -673,116 +674,3 @@ func verifyObjectAttrs(t *testing.T, cb *CachingBucket, file string, expectedLen } func matchAll(string) bool { return true } - -func TestParseBucketCacheKey(t *testing.T) { - testcases := []struct { - key string - expected BucketCacheKey - expectedErr error - }{ - { - key: "exists:name", - expected: BucketCacheKey{ - Verb: ExistsVerb, - Name: "name", - Start: 0, - End: 0, - }, - expectedErr: nil, - }, - { - key: "content:name", - expected: BucketCacheKey{ - Verb: ContentVerb, - Name: "name", - Start: 0, - End: 0, - }, - expectedErr: nil, - }, - { - key: "iter:name", - expected: BucketCacheKey{ - Verb: IterVerb, - Name: "name", - Start: 0, - End: 0, - }, - expectedErr: nil, - }, - { - key: "attrs:name", - expected: BucketCacheKey{ - Verb: AttributesVerb, - Name: "name", - Start: 0, - End: 0, - }, - expectedErr: nil, - }, - { - key: "subrange:name:10:20", - expected: BucketCacheKey{ - Verb: SubrangeVerb, - Name: "name", - Start: 10, - End: 20, - }, - expectedErr: nil, - }, - // Any VerbType other than SubrangeVerb should not have a "start" and "end". - { - key: "iter:name:10:20", - expected: BucketCacheKey{}, - expectedErr: ErrInvalidBucketCacheKeyFormat, - }, - // Key must always have a name. - { - key: "iter", - expected: BucketCacheKey{}, - expectedErr: ErrInvalidBucketCacheKeyFormat, - }, - // Invalid VerbType should return an error. - { - key: "random:name", - expected: BucketCacheKey{}, - expectedErr: ErrInvalidBucketCacheKeyVerb, - }, - // Start must be an integer. - { - key: "subrange:name:random:10", - expected: BucketCacheKey{}, - expectedErr: ErrParseKeyInt, - }, - // End must be an integer. - { - key: "subrange:name:10:random", - expected: BucketCacheKey{}, - expectedErr: ErrParseKeyInt, - }, - // SubrangeVerb must have start and end. - { - key: "subrange:name", - expected: BucketCacheKey{}, - expectedErr: ErrInvalidBucketCacheKeyFormat, - }, - // SubrangeVerb must have start and end both. - { - key: "subrange:name:10", - expected: BucketCacheKey{}, - expectedErr: ErrInvalidBucketCacheKeyFormat, - }, - // Key must not be an empty string. - { - key: "", - expected: BucketCacheKey{}, - expectedErr: ErrInvalidBucketCacheKeyFormat, - }, - } - - for _, tc := range testcases { - res, err := ParseBucketCacheKey(tc.key) - testutil.Equals(t, tc.expectedErr, err) - testutil.Equals(t, tc.expected, res) - } -} From 87e8f645e001ea5cc4a7687890981511b7a8b403 Mon Sep 17 00:00:00 2001 From: akanshat Date: Sun, 31 Oct 2021 00:05:05 +0530 Subject: [PATCH 04/22] add copyright to new pkg cachekey Signed-off-by: akanshat --- pkg/store/cache/cachekey/cachekey.go | 3 +++ pkg/store/cache/cachekey/cachekey_test.go | 3 +++ 2 files changed, 6 insertions(+) diff --git a/pkg/store/cache/cachekey/cachekey.go b/pkg/store/cache/cachekey/cachekey.go index 642e08f59a..d13706a586 100644 --- a/pkg/store/cache/cachekey/cachekey.go +++ b/pkg/store/cache/cachekey/cachekey.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package cachekey import ( diff --git a/pkg/store/cache/cachekey/cachekey_test.go b/pkg/store/cache/cachekey/cachekey_test.go index 757be8f7d4..5c704ac24f 100644 --- a/pkg/store/cache/cachekey/cachekey_test.go +++ b/pkg/store/cache/cachekey/cachekey_test.go @@ -1,3 +1,6 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + package cachekey import ( From d139fef95c429d794355bfb276cbbd2a1ec7c6ba Mon Sep 17 00:00:00 2001 From: akanshat Date: Sat, 6 Nov 2021 20:59:24 +0530 Subject: [PATCH 05/22] add a fix to return partial results if Get fails Signed-off-by: akanshat --- pkg/cache/groupcache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cache/groupcache.go b/pkg/cache/groupcache.go index 2d38d07335..29f85bd9db 100644 --- a/pkg/cache/groupcache.go +++ b/pkg/cache/groupcache.go @@ -208,7 +208,7 @@ func (c *Groupcache) Fetch(ctx context.Context, keys []string) map[string][]byte if err := c.group.Get(ctx, k, groupcache.AllocatingByteSliceSink(&keyData)); err != nil { level.Error(c.logger).Log("msg", "failed fetching data from groupcache", "err", err, "key", k) - return nil + continue } data[k] = keyData From 0b7ca17bcda1ec5f65fdc394c739ed09793b02b9 Mon Sep 17 00:00:00 2001 From: akanshat Date: Wed, 10 Nov 2021 17:57:10 +0530 Subject: [PATCH 06/22] migrate from groupcache to galaxycache Signed-off-by: akanshat --- go.mod | 2 +- go.sum | 6 +-- pkg/cache/groupcache.go | 55 ++++++++++++----------- pkg/store/cache/caching_bucket_factory.go | 4 +- 4 files changed, 35 insertions(+), 32 deletions(-) diff --git a/go.mod b/go.mod index 799d126db1..9d24c19156 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,6 @@ require ( github.com/leanovate/gopter v0.2.4 github.com/lightstep/lightstep-tracer-go v0.18.1 github.com/lovoo/gcloud-opentracing v0.3.0 - github.com/mailgun/groupcache/v2 v2.2.1 github.com/miekg/dns v1.1.43 github.com/minio/minio-go/v7 v7.0.10 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f @@ -64,6 +63,7 @@ require ( github.com/tencentyun/cos-go-sdk-v5 v0.7.31 github.com/uber/jaeger-client-go v2.29.1+incompatible github.com/uber/jaeger-lib v2.4.1+incompatible + github.com/vimeo/galaxycache v0.0.0-20210323154928-b7e5d71c067a github.com/weaveworks/common v0.0.0-20210901124008-1fa3f9fa874c go.elastic.co/apm v1.11.0 go.elastic.co/apm/module/apmot v1.11.0 diff --git a/go.sum b/go.sum index 0834c20dcf..54aa29c139 100644 --- a/go.sum +++ b/go.sum @@ -1166,8 +1166,6 @@ github.com/lufia/iostat v1.1.0/go.mod h1:rEPNA0xXgjHQjuI5Cy05sLlS2oRcSlWHRLrvh/A github.com/lyft/protoc-gen-star v0.5.1/go.mod h1:9toiA3cC7z5uVbODF7kEQ91Xn7XNFkVUl+SrEe+ZORU= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mailgun/groupcache/v2 v2.2.1 h1:OalhvLqdhiHd76CU8gn+w6UqeEa1m60ChrVwwKkmqh4= -github.com/mailgun/groupcache/v2 v2.2.1/go.mod h1:fgFJNRQar4yVloM0SzqWhOuTF83HCO5DDXVnZQVVJ58= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -1537,8 +1535,6 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUt github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/segmentio/fasthash v0.0.0-20180216231524-a72b379d632e/go.mod h1:tm/wZFQ8e24NYaBGIlnO2WGCAi67re4HHuOm0sftE/M= -github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= -github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/sercand/kuberesolver v2.4.0+incompatible h1:WE2OlRf6wjLxHwNkkFLQGaZcVLEXjMjBPjjEU5vksH8= github.com/sercand/kuberesolver v2.4.0+incompatible/go.mod h1:lWF3GL0xptCB/vCiJPl/ZshwPsX/n4Y7u0CW9E7aQIQ= @@ -1681,6 +1677,8 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= +github.com/vimeo/galaxycache v0.0.0-20210323154928-b7e5d71c067a h1:5rsX36sN7yTddsMiWRv1bkPF52m6U0DvBdJDKJus7f0= +github.com/vimeo/galaxycache v0.0.0-20210323154928-b7e5d71c067a/go.mod h1:rGzH8dZnpycpywZjY5k/G1k8q5bzVUiIUM8EsTni2NU= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= diff --git a/pkg/cache/groupcache.go b/pkg/cache/groupcache.go index 29f85bd9db..1807f6f511 100644 --- a/pkg/cache/groupcache.go +++ b/pkg/cache/groupcache.go @@ -7,12 +7,12 @@ import ( "context" "encoding/json" "io/ioutil" + "net/http" "strconv" "time" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" - "github.com/mailgun/groupcache/v2" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/route" "github.com/thanos-io/thanos/pkg/discovery/dns" @@ -21,14 +21,15 @@ import ( "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/runutil" "github.com/thanos-io/thanos/pkg/store/cache/cachekey" + "github.com/vimeo/galaxycache" + galaxyhttp "github.com/vimeo/galaxycache/http" "gopkg.in/yaml.v2" ) type Groupcache struct { - dns *dns.Provider - Pool *groupcache.HTTPPool - group *groupcache.Group - logger log.Logger + galaxy *galaxycache.Galaxy + universe *galaxycache.Universe + logger log.Logger } // GroupcacheConfig holds the in-memory cache config. @@ -77,26 +78,27 @@ func parseGroupcacheConfig(conf []byte) (GroupcacheConfig, error) { } // NewGroupcache creates a new Groupcache instance. -func NewGroupcache(name string, logger log.Logger, reg prometheus.Registerer, conf []byte, groupname, basepath string, r *route.Router, bucket objstore.Bucket) (*Groupcache, error) { +func NewGroupcache(logger log.Logger, reg prometheus.Registerer, conf []byte, basepath string, r *route.Router, bucket objstore.Bucket) (*Groupcache, error) { config, err := parseGroupcacheConfig(conf) if err != nil { return nil, err } - return NewGroupcacheWithConfig(name, logger, reg, config, groupname, basepath, r, bucket) + return NewGroupcacheWithConfig(logger, reg, config, basepath, r, bucket) } // NewGroupcacheWithConfig creates a new Groupcache instance with the given config. -func NewGroupcacheWithConfig(name string, logger log.Logger, reg prometheus.Registerer, conf GroupcacheConfig, groupname, basepath string, r *route.Router, bucket objstore.Bucket) (*Groupcache, error) { +func NewGroupcacheWithConfig(logger log.Logger, reg prometheus.Registerer, conf GroupcacheConfig, basepath string, r *route.Router, bucket objstore.Bucket) (*Groupcache, error) { dnsGroupcacheProvider := dns.NewProvider( logger, extprom.WrapRegistererWithPrefix("thanos_store_groupcache_", reg), dns.ResolverType(conf.DNSSDResolver), ) - pool := groupcache.NewHTTPPoolOpts(conf.SelfURL, &groupcache.HTTPPoolOptions{ + httpProto := galaxyhttp.NewHTTPFetchProtocol(&galaxyhttp.HTTPOptions{ BasePath: basepath, }) + universe := galaxycache.NewUniverse(httpProto, conf.SelfURL) ticker := time.NewTicker(conf.DNSInterval) @@ -105,17 +107,21 @@ func NewGroupcacheWithConfig(name string, logger log.Logger, reg prometheus.Regi if err := dnsGroupcacheProvider.Resolve(context.Background(), conf.Peers); err != nil { level.Error(logger).Log("msg", "failed to resolve addresses for groupcache", "err", err) } else { - pool.Set(dnsGroupcacheProvider.Addresses()...) + universe.Set(dnsGroupcacheProvider.Addresses()...) } <-ticker.C } }() - r.Get(basepath, pool.ServeHTTP) + mux := http.NewServeMux() + galaxyhttp.RegisterHTTPHandler(universe, &galaxyhttp.HTTPOptions{ + BasePath: basepath, + }, mux) + r.Get(basepath, mux.ServeHTTP) - group := groupcache.NewGroup(conf.GroupcacheGroup, int64(conf.MaxSize), groupcache.GetterFunc( - func(ctx context.Context, id string, dest groupcache.Sink) error { + galaxy := universe.NewGalaxy(conf.GroupcacheGroup, int64(conf.MaxSize), galaxycache.GetterFunc( + func(ctx context.Context, id string, dest galaxycache.Codec) error { parsedData, err := cachekey.ParseBucketCacheKey(id) if err != nil { return err @@ -132,7 +138,7 @@ func NewGroupcacheWithConfig(name string, logger log.Logger, reg prometheus.Regi if err != nil { return err } - err = dest.SetString(string(finalAttrs), time.Now().Add(5*time.Minute)) + err = dest.UnmarshalBinary(finalAttrs) if err != nil { return err } @@ -152,7 +158,7 @@ func NewGroupcacheWithConfig(name string, logger log.Logger, reg prometheus.Regi return err } - err = dest.SetBytes(b, time.Now().Add(5*time.Minute)) + err = dest.UnmarshalBinary(b) if err != nil { return err } @@ -162,7 +168,7 @@ func NewGroupcacheWithConfig(name string, logger log.Logger, reg prometheus.Regi return err } - err = dest.SetString(strconv.FormatBool(exists), time.Now().Add(5*time.Minute)) + err = dest.UnmarshalBinary([]byte(strconv.FormatBool(exists))) if err != nil { return err } @@ -178,7 +184,7 @@ func NewGroupcacheWithConfig(name string, logger log.Logger, reg prometheus.Regi return err } - err = dest.SetBytes(b, time.Now().Add(5*time.Minute)) + err = dest.UnmarshalBinary(b) if err != nil { return err } @@ -189,10 +195,9 @@ func NewGroupcacheWithConfig(name string, logger log.Logger, reg prometheus.Regi )) return &Groupcache{ - dns: dnsGroupcacheProvider, - Pool: pool, - group: group, - logger: logger, + logger: logger, + galaxy: galaxy, + universe: universe, }, nil } @@ -204,19 +209,19 @@ func (c *Groupcache) Fetch(ctx context.Context, keys []string) map[string][]byte data := map[string][]byte{} for _, k := range keys { - var keyData []byte + codec := galaxycache.ByteCodec{} - if err := c.group.Get(ctx, k, groupcache.AllocatingByteSliceSink(&keyData)); err != nil { + if err := c.galaxy.Get(ctx, k, &codec); err != nil { level.Error(c.logger).Log("msg", "failed fetching data from groupcache", "err", err, "key", k) continue } - data[k] = keyData + data[k] = codec } return data } func (c *Groupcache) Name() string { - return c.group.Name() + return c.galaxy.Name() } diff --git a/pkg/store/cache/caching_bucket_factory.go b/pkg/store/cache/caching_bucket_factory.go index ca9b3674a3..8fe35e7e92 100644 --- a/pkg/store/cache/caching_bucket_factory.go +++ b/pkg/store/cache/caching_bucket_factory.go @@ -100,9 +100,9 @@ func NewCachingBucketFromYaml(yamlContent []byte, bucket objstore.Bucket, logger return nil, errors.Wrapf(err, "failed to create inmemory cache") } case string(GroupcacheBucketCacheProvider): - const basePath = "/_groupcache/" + const basePath = "/_galaxycache/" - c, err = cache.NewGroupcache("caching-bucket", logger, reg, backendConfig, bucket.Name(), basePath, r, bucket) + c, err = cache.NewGroupcache(logger, reg, backendConfig, basePath, r, bucket) if err != nil { return nil, errors.Wrap(err, "failed to create groupcache") } From 6bca1ff05168fffea4570327a5647d39e97eb083 Mon Sep 17 00:00:00 2001 From: akanshat Date: Sun, 14 Nov 2021 23:22:34 +0530 Subject: [PATCH 07/22] instrument metrics for galaxyCache Signed-off-by: akanshat --- pkg/cache/groupcache.go | 59 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/pkg/cache/groupcache.go b/pkg/cache/groupcache.go index 1807f6f511..1d5bc84065 100644 --- a/pkg/cache/groupcache.go +++ b/pkg/cache/groupcache.go @@ -26,6 +26,19 @@ import ( "gopkg.in/yaml.v2" ) +type CacheStatsCollector struct { + galaxy *galaxycache.Galaxy + + // GalaxyCache Metric descriptions. + gets *prometheus.Desc + loads *prometheus.Desc + peerLoads *prometheus.Desc + peerLoadErrors *prometheus.Desc + backendLoads *prometheus.Desc + backendLoadErrors *prometheus.Desc + cacheHits *prometheus.Desc +} + type Groupcache struct { galaxy *galaxycache.Galaxy universe *galaxycache.Universe @@ -107,7 +120,10 @@ func NewGroupcacheWithConfig(logger log.Logger, reg prometheus.Registerer, conf if err := dnsGroupcacheProvider.Resolve(context.Background(), conf.Peers); err != nil { level.Error(logger).Log("msg", "failed to resolve addresses for groupcache", "err", err) } else { - universe.Set(dnsGroupcacheProvider.Addresses()...) + err := universe.Set(dnsGroupcacheProvider.Addresses()...) + if err != nil { + level.Error(logger).Log("msg", "failed to set peers for groupcache", "err", err) + } } <-ticker.C @@ -194,6 +210,9 @@ func NewGroupcacheWithConfig(logger log.Logger, reg prometheus.Registerer, conf }, )) + // Register GalaxyCache stats. + RegisterCacheStatsCollector(galaxy, reg) + return &Groupcache{ logger: logger, galaxy: galaxy, @@ -225,3 +244,41 @@ func (c *Groupcache) Fetch(ctx context.Context, keys []string) map[string][]byte func (c *Groupcache) Name() string { return c.galaxy.Name() } + +func RegisterCacheStatsCollector(galaxy *galaxycache.Galaxy, reg prometheus.Registerer) { + gets := prometheus.NewDesc("thanos_cache_groupcache_get_requests_total", "Total number of get requests, including from peers.", nil, nil) + loads := prometheus.NewDesc("thanos_cache_groupcache_loads_total", "Total number of loads from backend (gets - cacheHits).", nil, nil) + peerLoads := prometheus.NewDesc("thanos_cache_groupcache_peer_loads_total", "Total number of loads from peers (remote load or remote cache hit).", nil, nil) + peerLoadErrors := prometheus.NewDesc("thanos_cache_groupcache_peer_load_errors_total", "Total number of errors from peer loads.", nil, nil) + backendLoads := prometheus.NewDesc("thanos_cache_groupcache_backend_loads_total", "Total number of direct backend loads.", nil, nil) + backendLoadErrors := prometheus.NewDesc("thanos_cache_groupcache_backend_load_errors_total", "Total number of errors on direct backend loads.", nil, nil) + cacheHits := prometheus.NewDesc("thanos_cache_groupcache_hits_total", "Total number of cache hits.", []string{"type"}, nil) + + collector := &CacheStatsCollector{ + galaxy: galaxy, + gets: gets, + loads: loads, + peerLoads: peerLoads, + peerLoadErrors: peerLoadErrors, + backendLoads: backendLoads, + backendLoadErrors: backendLoadErrors, + cacheHits: cacheHits, + } + reg.MustRegister(collector) +} + +func (s CacheStatsCollector) Collect(ch chan<- prometheus.Metric) { + stats := s.galaxy.Stats + ch <- prometheus.MustNewConstMetric(s.gets, prometheus.CounterValue, float64(stats.Gets)) + ch <- prometheus.MustNewConstMetric(s.loads, prometheus.CounterValue, float64(stats.Loads)) + ch <- prometheus.MustNewConstMetric(s.peerLoads, prometheus.CounterValue, float64(stats.PeerLoads)) + ch <- prometheus.MustNewConstMetric(s.peerLoadErrors, prometheus.CounterValue, float64(stats.PeerLoadErrors)) + ch <- prometheus.MustNewConstMetric(s.backendLoads, prometheus.CounterValue, float64(stats.BackendLoads)) + ch <- prometheus.MustNewConstMetric(s.backendLoadErrors, prometheus.CounterValue, float64(stats.BackendLoadErrors)) + ch <- prometheus.MustNewConstMetric(s.cacheHits, prometheus.CounterValue, float64(stats.MaincacheHits), galaxycache.MainCache.String()) + ch <- prometheus.MustNewConstMetric(s.cacheHits, prometheus.CounterValue, float64(stats.HotcacheHits), galaxycache.HotCache.String()) +} + +func (s CacheStatsCollector) Describe(ch chan<- *prometheus.Desc) { + prometheus.DescribeByCollect(s, ch) +} From 8f8c26161f9051eef382bcc01105c077be9422c2 Mon Sep 17 00:00:00 2001 From: akanshat Date: Sun, 14 Nov 2021 23:23:32 +0530 Subject: [PATCH 08/22] add e2e test for store with groupcache Signed-off-by: akanshat --- test/e2e/store_gateway_test.go | 155 +++++++++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) diff --git a/test/e2e/store_gateway_test.go b/test/e2e/store_gateway_test.go index 636b14cf13..355caf8f06 100644 --- a/test/e2e/store_gateway_test.go +++ b/test/e2e/store_gateway_test.go @@ -398,3 +398,158 @@ blocks_iter_ttl: 0s`, memcached.InternalEndpoint("memcached")) }) } + +func TestStoreGatewayGroupCache(t *testing.T) { + t.Parallel() + + e, err := e2e.NewDockerEnvironment("e2e_test_store_gateway_groupcache") + testutil.Ok(t, err) + t.Cleanup(e2ethanos.CleanScenario(t, e)) + + const bucket = "store_gateway_groupcache_test" + m := e2ethanos.NewMinio(e, "thanos-minio", bucket) + testutil.Ok(t, e2e.StartAndWaitReady(m)) + + groupcacheConfig := `type: GROUPCACHE +config: + self_url: http://store-gw-%d:42/ + peers: + - http://store-gw-1:42/ + - http://store-gw-2:42/ + - http://store-gw-3:42/ + groupcache_group: groupcache_test_group +blocks_iter_ttl: 0s +metafile_exists_ttl: 0s +metafile_doesnt_exist_ttl: 0s +metafile_content_ttl: 0s` + + store1, err := e2ethanos.NewStoreGW( + e, + "1", + client.BucketConfig{ + Type: client.S3, + Config: s3.Config{ + Bucket: bucket, + AccessKey: e2edb.MinioAccessKey, + SecretKey: e2edb.MinioSecretKey, + Endpoint: m.InternalEndpoint("http"), + Insecure: true, + }, + }, + fmt.Sprintf(groupcacheConfig, 1), + ) + testutil.Ok(t, err) + testutil.Ok(t, e2e.StartAndWaitReady(store1)) + + store2, err := e2ethanos.NewStoreGW( + e, + "2", + client.BucketConfig{ + Type: client.S3, + Config: s3.Config{ + Bucket: bucket, + AccessKey: e2edb.MinioAccessKey, + SecretKey: e2edb.MinioSecretKey, + Endpoint: m.InternalEndpoint("http"), + Insecure: true, + }, + }, + fmt.Sprintf(groupcacheConfig, 2), + ) + testutil.Ok(t, err) + testutil.Ok(t, e2e.StartAndWaitReady(store2)) + + store3, err := e2ethanos.NewStoreGW( + e, + "3", + client.BucketConfig{ + Type: client.S3, + Config: s3.Config{ + Bucket: bucket, + AccessKey: e2edb.MinioAccessKey, + SecretKey: e2edb.MinioSecretKey, + Endpoint: m.InternalEndpoint("http"), + Insecure: true, + }, + }, + fmt.Sprintf(groupcacheConfig, 3), + ) + + testutil.Ok(t, err) + testutil.Ok(t, e2e.StartAndWaitReady(store3)) + + q, err := e2ethanos.NewQuerierBuilder(e, "1", store1.InternalEndpoint("grpc")).Build() + testutil.Ok(t, err) + testutil.Ok(t, e2e.StartAndWaitReady(q)) + + dir := filepath.Join(e.SharedDir(), "tmp") + testutil.Ok(t, os.MkdirAll(dir, os.ModePerm)) + + series := []labels.Labels{labels.FromStrings("a", "1", "b", "2")} + extLset := labels.FromStrings("ext1", "value1", "replica", "1") + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + t.Cleanup(cancel) + + now := time.Now() + id, err := e2eutil.CreateBlockWithBlockDelay(ctx, dir, series, 10, timestamp.FromTime(now), timestamp.FromTime(now.Add(2*time.Hour)), 30*time.Minute, extLset, 0, metadata.NoneFunc) + testutil.Ok(t, err) + + l := log.NewLogfmtLogger(os.Stdout) + bkt, err := s3.NewBucketWithConfig(l, s3.Config{ + Bucket: bucket, + AccessKey: e2edb.MinioAccessKey, + SecretKey: e2edb.MinioSecretKey, + Endpoint: m.Endpoint("http"), // We need separate client config, when connecting to minio from outside. + Insecure: true, + }, "test-feed") + testutil.Ok(t, err) + + testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id.String()), id.String())) + + // Wait for store to sync blocks. + // thanos_blocks_meta_synced: 1x loadedMeta 0x labelExcludedMeta 0x TooFreshMeta. + testutil.Ok(t, store1.WaitSumMetrics(e2e.Equals(1), "thanos_blocks_meta_synced")) + testutil.Ok(t, store1.WaitSumMetrics(e2e.Equals(0), "thanos_blocks_meta_sync_failures_total")) + + testutil.Ok(t, store1.WaitSumMetrics(e2e.Equals(1), "thanos_bucket_store_blocks_loaded")) + testutil.Ok(t, store1.WaitSumMetrics(e2e.Equals(0), "thanos_bucket_store_block_drops_total")) + testutil.Ok(t, store1.WaitSumMetrics(e2e.Equals(0), "thanos_bucket_store_block_load_failures_total")) + + t.Run("query with cache miss", func(t *testing.T) { + queryAndAssertSeries(t, ctx, q.Endpoint("http"), "{a=\"1\"}", + time.Now, promclient.QueryOptions{ + Deduplicate: false, + }, + []model.Metric{ + { + "a": "1", + "b": "2", + "ext1": "value1", + "replica": "1", + }, + }, + ) + + testutil.Ok(t, store1.WaitSumMetricsWithOptions(e2e.Greater(0), []string{`thanos_cache_groupcache_loads_total`})) + testutil.Ok(t, store1.WaitSumMetricsWithOptions(e2e.Equals(0), []string{`thanos_cache_groupcache_hits_total`})) + }) + + t.Run("query with cache hit", func(t *testing.T) { + queryAndAssertSeries(t, ctx, q.Endpoint("http"), "{a=\"1\"}", + time.Now, promclient.QueryOptions{ + Deduplicate: false, + }, + []model.Metric{ + { + "a": "1", + "b": "2", + "ext1": "value1", + "replica": "1", + }, + }, + ) + + testutil.Ok(t, store1.WaitSumMetricsWithOptions(e2e.Greater(0), []string{`thanos_cache_groupcache_hits_total`})) + }) +} From 8621c68f98c69cd837194caec5b55b22595b0ed9 Mon Sep 17 00:00:00 2001 From: akanshat Date: Wed, 17 Nov 2021 17:54:32 +0530 Subject: [PATCH 09/22] fix collector interface on CacheStatsCollector Signed-off-by: akanshat --- pkg/cache/groupcache.go | 21 ++++++++++----------- test/e2e/store_gateway_test.go | 1 - 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/pkg/cache/groupcache.go b/pkg/cache/groupcache.go index 1d5bc84065..61c3ed28c2 100644 --- a/pkg/cache/groupcache.go +++ b/pkg/cache/groupcache.go @@ -267,18 +267,17 @@ func RegisterCacheStatsCollector(galaxy *galaxycache.Galaxy, reg prometheus.Regi reg.MustRegister(collector) } -func (s CacheStatsCollector) Collect(ch chan<- prometheus.Metric) { - stats := s.galaxy.Stats - ch <- prometheus.MustNewConstMetric(s.gets, prometheus.CounterValue, float64(stats.Gets)) - ch <- prometheus.MustNewConstMetric(s.loads, prometheus.CounterValue, float64(stats.Loads)) - ch <- prometheus.MustNewConstMetric(s.peerLoads, prometheus.CounterValue, float64(stats.PeerLoads)) - ch <- prometheus.MustNewConstMetric(s.peerLoadErrors, prometheus.CounterValue, float64(stats.PeerLoadErrors)) - ch <- prometheus.MustNewConstMetric(s.backendLoads, prometheus.CounterValue, float64(stats.BackendLoads)) - ch <- prometheus.MustNewConstMetric(s.backendLoadErrors, prometheus.CounterValue, float64(stats.BackendLoadErrors)) - ch <- prometheus.MustNewConstMetric(s.cacheHits, prometheus.CounterValue, float64(stats.MaincacheHits), galaxycache.MainCache.String()) - ch <- prometheus.MustNewConstMetric(s.cacheHits, prometheus.CounterValue, float64(stats.HotcacheHits), galaxycache.HotCache.String()) +func (s *CacheStatsCollector) Collect(ch chan<- prometheus.Metric) { + ch <- prometheus.MustNewConstMetric(s.gets, prometheus.CounterValue, float64(s.galaxy.Stats.Gets.Get())) + ch <- prometheus.MustNewConstMetric(s.loads, prometheus.CounterValue, float64(s.galaxy.Stats.Loads.Get())) + ch <- prometheus.MustNewConstMetric(s.peerLoads, prometheus.CounterValue, float64(s.galaxy.Stats.PeerLoads.Get())) + ch <- prometheus.MustNewConstMetric(s.peerLoadErrors, prometheus.CounterValue, float64(s.galaxy.Stats.PeerLoadErrors.Get())) + ch <- prometheus.MustNewConstMetric(s.backendLoads, prometheus.CounterValue, float64(s.galaxy.Stats.BackendLoads.Get())) + ch <- prometheus.MustNewConstMetric(s.backendLoadErrors, prometheus.CounterValue, float64(s.galaxy.Stats.BackendLoadErrors.Get())) + ch <- prometheus.MustNewConstMetric(s.cacheHits, prometheus.CounterValue, float64(s.galaxy.Stats.MaincacheHits.Get()), galaxycache.MainCache.String()) + ch <- prometheus.MustNewConstMetric(s.cacheHits, prometheus.CounterValue, float64(s.galaxy.Stats.HotcacheHits.Get()), galaxycache.HotCache.String()) } -func (s CacheStatsCollector) Describe(ch chan<- *prometheus.Desc) { +func (s *CacheStatsCollector) Describe(ch chan<- *prometheus.Desc) { prometheus.DescribeByCollect(s, ch) } diff --git a/test/e2e/store_gateway_test.go b/test/e2e/store_gateway_test.go index 355caf8f06..83c5df0def 100644 --- a/test/e2e/store_gateway_test.go +++ b/test/e2e/store_gateway_test.go @@ -532,7 +532,6 @@ metafile_content_ttl: 0s` ) testutil.Ok(t, store1.WaitSumMetricsWithOptions(e2e.Greater(0), []string{`thanos_cache_groupcache_loads_total`})) - testutil.Ok(t, store1.WaitSumMetricsWithOptions(e2e.Equals(0), []string{`thanos_cache_groupcache_hits_total`})) }) t.Run("query with cache hit", func(t *testing.T) { From a6f1945ddb3c338e239ca7c8f0ad3aceb65fc911 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Wed, 17 Nov 2021 16:26:05 +0200 Subject: [PATCH 10/22] cache: fix / clean up tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Giedrius Statkevičius --- pkg/cache/groupcache.go | 4 +++- pkg/store/cache/cache_test.go | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/cache/groupcache.go b/pkg/cache/groupcache.go index 61c3ed28c2..f78fe5fea0 100644 --- a/pkg/cache/groupcache.go +++ b/pkg/cache/groupcache.go @@ -235,7 +235,9 @@ func (c *Groupcache) Fetch(ctx context.Context, keys []string) map[string][]byte continue } - data[k] = codec + if len(codec) > 0 { + data[k] = codec + } } return data diff --git a/pkg/store/cache/cache_test.go b/pkg/store/cache/cache_test.go index 9b59f75b95..33036f9e2e 100644 --- a/pkg/store/cache/cache_test.go +++ b/pkg/store/cache/cache_test.go @@ -13,12 +13,11 @@ import ( "github.com/oklog/ulid" "github.com/prometheus/prometheus/pkg/labels" "github.com/thanos-io/thanos/pkg/testutil" - "go.uber.org/goleak" "golang.org/x/crypto/blake2b" ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + testutil.TolerantVerifyLeakMain(m) } func TestCacheKey_string(t *testing.T) { From 63b212bef7393532f542fe2ddde9a2d27943489c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Wed, 17 Nov 2021 16:50:07 +0200 Subject: [PATCH 11/22] groupcache: support IterVerb MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Giedrius Statkevičius --- pkg/cache/groupcache.go | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/pkg/cache/groupcache.go b/pkg/cache/groupcache.go index f78fe5fea0..26429b85bf 100644 --- a/pkg/cache/groupcache.go +++ b/pkg/cache/groupcache.go @@ -154,14 +154,21 @@ func NewGroupcacheWithConfig(logger log.Logger, reg prometheus.Registerer, conf if err != nil { return err } - err = dest.UnmarshalBinary(finalAttrs) - if err != nil { + return dest.UnmarshalBinary(finalAttrs) + case cachekey.IterVerb: + var list []string + if err := bucket.Iter(ctx, parsedData.Name, func(s string) error { + list = append(list, s) + return nil + }); err != nil { return err } - case cachekey.IterVerb: - // Not supported. - return nil + encodedList, err := json.Marshal(list) + if err != nil { + return err + } + return dest.UnmarshalBinary(encodedList) case cachekey.ContentVerb: rc, err := bucket.Get(ctx, parsedData.Name) if err != nil { @@ -174,20 +181,14 @@ func NewGroupcacheWithConfig(logger log.Logger, reg prometheus.Registerer, conf return err } - err = dest.UnmarshalBinary(b) - if err != nil { - return err - } + return dest.UnmarshalBinary(b) case cachekey.ExistsVerb: exists, err := bucket.Exists(ctx, parsedData.Name) if err != nil { return err } - err = dest.UnmarshalBinary([]byte(strconv.FormatBool(exists))) - if err != nil { - return err - } + return dest.UnmarshalBinary([]byte(strconv.FormatBool(exists))) case cachekey.SubrangeVerb: rc, err := bucket.GetRange(ctx, parsedData.Name, parsedData.Start, parsedData.End-parsedData.Start) if err != nil { @@ -200,10 +201,7 @@ func NewGroupcacheWithConfig(logger log.Logger, reg prometheus.Registerer, conf return err } - err = dest.UnmarshalBinary(b) - if err != nil { - return err - } + return dest.UnmarshalBinary(b) } return nil From 992e2e1b858afd404c95f0f3b333bf5dbb7dd37c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Wed, 17 Nov 2021 18:00:26 +0200 Subject: [PATCH 12/22] groupcache: changes according to comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Giedrius Statkevičius --- CHANGELOG.md | 1 + pkg/cache/groupcache.go | 39 +++++++++++++++++----------------- scripts/quickstart.sh | 15 ++++++++++++- test/e2e/store_gateway_test.go | 38 +++++++++++++++++++++++---------- 4 files changed, 61 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 40e8055db7..2cde146cb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#4710](https://github.com/thanos-io/thanos/pull/4710) Store: add metric to capture timestamp of the last loaded block. - [#4736](https://github.com/thanos-io/thanos/pull/4736) S3: Add capability to use custom AWS STS Endpoint. - [#4764](https://github.com/thanos-io/thanos/pull/4764) Compactor: add `block-viewer.global.sync-block-timeout` flag to set the timeout of synchronization block metas. +- [#4818](https://github.com/thanos-io/thanos/pull/4818) *EXPERIMENTAL* store: Add Groupcache as a cache backend. [Original issue](https://github.com/thanos-io/thanos/issues/2962). ### Fixed diff --git a/pkg/cache/groupcache.go b/pkg/cache/groupcache.go index 26429b85bf..4fab96bfc1 100644 --- a/pkg/cache/groupcache.go +++ b/pkg/cache/groupcache.go @@ -26,19 +26,6 @@ import ( "gopkg.in/yaml.v2" ) -type CacheStatsCollector struct { - galaxy *galaxycache.Galaxy - - // GalaxyCache Metric descriptions. - gets *prometheus.Desc - loads *prometheus.Desc - peerLoads *prometheus.Desc - peerLoadErrors *prometheus.Desc - backendLoads *prometheus.Desc - backendLoadErrors *prometheus.Desc - cacheHits *prometheus.Desc -} - type Groupcache struct { galaxy *galaxycache.Galaxy universe *galaxycache.Universe @@ -102,17 +89,16 @@ func NewGroupcache(logger log.Logger, reg prometheus.Registerer, conf []byte, ba // NewGroupcacheWithConfig creates a new Groupcache instance with the given config. func NewGroupcacheWithConfig(logger log.Logger, reg prometheus.Registerer, conf GroupcacheConfig, basepath string, r *route.Router, bucket objstore.Bucket) (*Groupcache, error) { - dnsGroupcacheProvider := dns.NewProvider( - logger, - extprom.WrapRegistererWithPrefix("thanos_store_groupcache_", reg), - dns.ResolverType(conf.DNSSDResolver), - ) - httpProto := galaxyhttp.NewHTTPFetchProtocol(&galaxyhttp.HTTPOptions{ BasePath: basepath, }) universe := galaxycache.NewUniverse(httpProto, conf.SelfURL) + dnsGroupcacheProvider := dns.NewProvider( + logger, + extprom.WrapRegistererWithPrefix("thanos_store_groupcache_", reg), + dns.ResolverType(conf.DNSSDResolver), + ) ticker := time.NewTicker(conf.DNSInterval) go func() { @@ -208,7 +194,6 @@ func NewGroupcacheWithConfig(logger log.Logger, reg prometheus.Registerer, conf }, )) - // Register GalaxyCache stats. RegisterCacheStatsCollector(galaxy, reg) return &Groupcache{ @@ -245,6 +230,20 @@ func (c *Groupcache) Name() string { return c.galaxy.Name() } +type CacheStatsCollector struct { + galaxy *galaxycache.Galaxy + + // GalaxyCache Metric descriptions. + gets *prometheus.Desc + loads *prometheus.Desc + peerLoads *prometheus.Desc + peerLoadErrors *prometheus.Desc + backendLoads *prometheus.Desc + backendLoadErrors *prometheus.Desc + cacheHits *prometheus.Desc +} + +// RegisterCacheStatsCollector registers a groupcache metrics collector. func RegisterCacheStatsCollector(galaxy *galaxycache.Galaxy, reg prometheus.Registerer) { gets := prometheus.NewDesc("thanos_cache_groupcache_get_requests_total", "Total number of get requests, including from peers.", nil, nil) loads := prometheus.NewDesc("thanos_cache_groupcache_loads_total", "Total number of loads from backend (gets - cacheHits).", nil, nil) diff --git a/scripts/quickstart.sh b/scripts/quickstart.sh index e99ab8fdd9..9611a1a5e8 100755 --- a/scripts/quickstart.sh +++ b/scripts/quickstart.sh @@ -46,7 +46,6 @@ if [ -n "${MINIO_ENABLED}" ]; then export S3_ENDPOINT=${MINIO_ENDPOINT} export S3_INSECURE="true" export S3_V2_SIGNATURE="true" - rm -rf data/minio mkdir -p data/minio ${MINIO_EXECUTABLE} server ./data/minio \ @@ -167,6 +166,19 @@ done sleep 0.5 if [ -n "${GCS_BUCKET}" -o -n "${S3_ENDPOINT}" ]; then +cat >groupcache.yml <<-EOF + type: GROUPCACHE +config: + self_url: http://localhost:10906/ + peers: + - http://localhost:10906/ + groupcache_group: groupcache_test_group +blocks_iter_ttl: 0s +metafile_exists_ttl: 0s +metafile_doesnt_exist_ttl: 0s +metafile_content_ttl: 0s + EOF + ${THANOS_EXECUTABLE} store \ --debug.name store \ --log.level debug \ @@ -175,6 +187,7 @@ if [ -n "${GCS_BUCKET}" -o -n "${S3_ENDPOINT}" ]; then --http-address 0.0.0.0:10906 \ --http-grace-period 1s \ --data-dir data/store \ + --store.caching-bucket.config-file=groupcache.yml \ ${OBJSTORECFG} & STORES="${STORES} --store 127.0.0.1:10905" diff --git a/test/e2e/store_gateway_test.go b/test/e2e/store_gateway_test.go index 83c5df0def..6ded0f507f 100644 --- a/test/e2e/store_gateway_test.go +++ b/test/e2e/store_gateway_test.go @@ -478,7 +478,11 @@ metafile_content_ttl: 0s` testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(store3)) - q, err := e2ethanos.NewQuerierBuilder(e, "1", store1.InternalEndpoint("grpc")).Build() + q, err := e2ethanos.NewQuerierBuilder(e, "1", + store1.InternalEndpoint("grpc"), + store2.InternalEndpoint("grpc"), + store3.InternalEndpoint("grpc"), + ).Build() testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(q)) @@ -509,14 +513,18 @@ metafile_content_ttl: 0s` // Wait for store to sync blocks. // thanos_blocks_meta_synced: 1x loadedMeta 0x labelExcludedMeta 0x TooFreshMeta. - testutil.Ok(t, store1.WaitSumMetrics(e2e.Equals(1), "thanos_blocks_meta_synced")) - testutil.Ok(t, store1.WaitSumMetrics(e2e.Equals(0), "thanos_blocks_meta_sync_failures_total")) - - testutil.Ok(t, store1.WaitSumMetrics(e2e.Equals(1), "thanos_bucket_store_blocks_loaded")) - testutil.Ok(t, store1.WaitSumMetrics(e2e.Equals(0), "thanos_bucket_store_block_drops_total")) - testutil.Ok(t, store1.WaitSumMetrics(e2e.Equals(0), "thanos_bucket_store_block_load_failures_total")) - - t.Run("query with cache miss", func(t *testing.T) { + for _, st := range []*e2e.InstrumentedRunnable{store1, store2, store3} { + t.Run(st.Name(), func(t *testing.T) { + testutil.Ok(t, st.WaitSumMetrics(e2e.Equals(1), "thanos_blocks_meta_synced")) + testutil.Ok(t, st.WaitSumMetrics(e2e.Equals(0), "thanos_blocks_meta_sync_failures_total")) + + testutil.Ok(t, st.WaitSumMetrics(e2e.Equals(1), "thanos_bucket_store_blocks_loaded")) + testutil.Ok(t, st.WaitSumMetrics(e2e.Equals(0), "thanos_bucket_store_block_drops_total")) + testutil.Ok(t, st.WaitSumMetrics(e2e.Equals(0), "thanos_bucket_store_block_load_failures_total")) + }) + } + + t.Run("query with groupcache loading from object storage", func(t *testing.T) { queryAndAssertSeries(t, ctx, q.Endpoint("http"), "{a=\"1\"}", time.Now, promclient.QueryOptions{ Deduplicate: false, @@ -531,10 +539,17 @@ metafile_content_ttl: 0s` }, ) - testutil.Ok(t, store1.WaitSumMetricsWithOptions(e2e.Greater(0), []string{`thanos_cache_groupcache_loads_total`})) + for _, st := range []*e2e.InstrumentedRunnable{store1, store2, store3} { + testutil.Ok(t, st.WaitSumMetricsWithOptions(e2e.Greater(0), []string{`thanos_cache_groupcache_loads_total`})) + testutil.Ok(t, st.WaitSumMetricsWithOptions(e2e.Greater(0), []string{`thanos_store_bucket_cache_operation_hits_total`}, e2e.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "config", "chunks")))) + } }) t.Run("query with cache hit", func(t *testing.T) { + retrievedMetrics, err := store1.SumMetrics([]string{`thanos_cache_groupcache_hits_total`, `thanos_cache_groupcache_loads_total`}) + testutil.Ok(t, err) + testutil.Assert(t, len(retrievedMetrics) == 2) + queryAndAssertSeries(t, ctx, q.Endpoint("http"), "{a=\"1\"}", time.Now, promclient.QueryOptions{ Deduplicate: false, @@ -549,6 +564,7 @@ metafile_content_ttl: 0s` }, ) - testutil.Ok(t, store1.WaitSumMetricsWithOptions(e2e.Greater(0), []string{`thanos_cache_groupcache_hits_total`})) + testutil.Ok(t, store1.WaitSumMetricsWithOptions(e2e.Greater(retrievedMetrics[0]), []string{`thanos_cache_groupcache_hits_total`})) + testutil.Ok(t, store1.WaitSumMetricsWithOptions(e2e.Equals(retrievedMetrics[1]), []string{`thanos_cache_groupcache_loads_total`})) }) } From 3c12c2ebad3ec681d843c15a5fae64db3ff8f209 Mon Sep 17 00:00:00 2001 From: akanshat Date: Wed, 8 Dec 2021 17:56:39 +0530 Subject: [PATCH 13/22] store: fix groupcache test Signed-off-by: akanshat --- test/e2e/store_gateway_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/store_gateway_test.go b/test/e2e/store_gateway_test.go index 164333f9d4..e9d75e1e22 100644 --- a/test/e2e/store_gateway_test.go +++ b/test/e2e/store_gateway_test.go @@ -526,7 +526,7 @@ metafile_content_ttl: 0s` } t.Run("query with groupcache loading from object storage", func(t *testing.T) { - queryAndAssertSeries(t, ctx, q.Endpoint("http"), "{a=\"1\"}", + queryAndAssertSeries(t, ctx, q.Endpoint("http"), func() string { return testQuery }, time.Now, promclient.QueryOptions{ Deduplicate: false, }, @@ -551,7 +551,7 @@ metafile_content_ttl: 0s` testutil.Ok(t, err) testutil.Assert(t, len(retrievedMetrics) == 2) - queryAndAssertSeries(t, ctx, q.Endpoint("http"), "{a=\"1\"}", + queryAndAssertSeries(t, ctx, q.Endpoint("http"), func() string { return testQuery }, time.Now, promclient.QueryOptions{ Deduplicate: false, }, From 088b9d338e9dc2975ed3b11be2b2e4be0d0ef200 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Mon, 22 Nov 2021 14:38:04 +0200 Subject: [PATCH 14/22] cache: add TTL support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds TTL support to galaxycache as per this pull request: https://github.com/thanos-community/galaxycache/pull/1 Signed-off-by: Giedrius Statkevičius --- go.mod | 2 + go.sum | 4 +- pkg/cache/groupcache.go | 54 ++++++++++++++++++----- pkg/store/cache/caching_bucket_factory.go | 8 +++- 4 files changed, 54 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 48ecaef2fb..8be5e20485 100644 --- a/go.mod +++ b/go.mod @@ -103,6 +103,8 @@ replace ( // Make sure Prometheus version is pinned as Prometheus semver does not include Go APIs. github.com/prometheus/prometheus => github.com/prometheus/prometheus v1.8.2-0.20211119115433-692a54649ed7 github.com/sercand/kuberesolver => github.com/sercand/kuberesolver v2.4.0+incompatible + + github.com/vimeo/galaxycache => github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e google.golang.org/grpc => google.golang.org/grpc v1.40.0 // Overriding to use latest commit diff --git a/go.sum b/go.sum index 6cb0b7ff17..99ee9cd6c8 100644 --- a/go.sum +++ b/go.sum @@ -1651,6 +1651,8 @@ github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.194/go.mod h1:yrBKWhChnDqNz1xuXdSbWXG56XawEq0G5j1lg4VwBD4= github.com/tencentyun/cos-go-sdk-v5 v0.7.31 h1:NujkkOKMJ3IFs1+trCwXOKRCIPQ8qI5Lxul9JkhTg6M= github.com/tencentyun/cos-go-sdk-v5 v0.7.31/go.mod h1:4E4+bQ2gBVJcgEC9Cufwylio4mXOct2iu05WjgEBx1o= +github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e h1:f1Zsv7OAU9iQhZwigp50Yl38W10g/vd5NC8Rdk1Jzng= +github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e/go.mod h1:jXcofnrSln/cLI6/dhlBxPQZEEQHVPCcFaH75M+nSzM= github.com/thanos-io/thanos v0.8.1-0.20200109203923-552ffa4c1a0d/go.mod h1:usT/TxtJQ7DzinTt+G9kinDQmRS5sxwu0unVKZ9vdcw= github.com/thanos-io/thanos v0.13.1-0.20200731083140-69b87607decf/go.mod h1:G8caR6G7pSDreRDvFm9wFuyjEBztmr8Ag3kBYpa/fEc= github.com/thanos-io/thanos v0.13.1-0.20200807203500-9b578afb4763/go.mod h1:KyW0a93tsh7v4hXAwo2CVAIRYuZT1Kkf4e04gisQjAg= @@ -1698,8 +1700,6 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= -github.com/vimeo/galaxycache v0.0.0-20210323154928-b7e5d71c067a h1:5rsX36sN7yTddsMiWRv1bkPF52m6U0DvBdJDKJus7f0= -github.com/vimeo/galaxycache v0.0.0-20210323154928-b7e5d71c067a/go.mod h1:rGzH8dZnpycpywZjY5k/G1k8q5bzVUiIUM8EsTni2NU= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= diff --git a/pkg/cache/groupcache.go b/pkg/cache/groupcache.go index 4fab96bfc1..66ef0264a2 100644 --- a/pkg/cache/groupcache.go +++ b/pkg/cache/groupcache.go @@ -78,17 +78,21 @@ func parseGroupcacheConfig(conf []byte) (GroupcacheConfig, error) { } // NewGroupcache creates a new Groupcache instance. -func NewGroupcache(logger log.Logger, reg prometheus.Registerer, conf []byte, basepath string, r *route.Router, bucket objstore.Bucket) (*Groupcache, error) { +func NewGroupcache(logger log.Logger, reg prometheus.Registerer, conf []byte, basepath string, r *route.Router, bucket objstore.Bucket, + isTSDBChunkFile, isMetaFile, isBlocksRootDir func(path string) bool, + MetaFileExistsTTL, MetafileDoesntExistTTL, MetafileContentTTL, ChunkObjectAttrsTTL, ChunkSubrangeTTL, BlocksIterTTL time.Duration) (*Groupcache, error) { config, err := parseGroupcacheConfig(conf) if err != nil { return nil, err } - return NewGroupcacheWithConfig(logger, reg, config, basepath, r, bucket) + return NewGroupcacheWithConfig(logger, reg, config, basepath, r, bucket, isTSDBChunkFile, isMetaFile, isBlocksRootDir, MetaFileExistsTTL, MetafileDoesntExistTTL, MetafileContentTTL, ChunkObjectAttrsTTL, ChunkSubrangeTTL, BlocksIterTTL) } // NewGroupcacheWithConfig creates a new Groupcache instance with the given config. -func NewGroupcacheWithConfig(logger log.Logger, reg prometheus.Registerer, conf GroupcacheConfig, basepath string, r *route.Router, bucket objstore.Bucket) (*Groupcache, error) { +func NewGroupcacheWithConfig(logger log.Logger, reg prometheus.Registerer, conf GroupcacheConfig, basepath string, r *route.Router, bucket objstore.Bucket, + isTSDBChunkFile, isMetaFile, isBlocksRootDir func(path string) bool, + MetaFileExistsTTL, MetafileDoesntExistTTL, MetafileContentTTL, ChunkObjectAttrsTTL, ChunkSubrangeTTL, BlocksIterTTL time.Duration) (*Groupcache, error) { httpProto := galaxyhttp.NewHTTPFetchProtocol(&galaxyhttp.HTTPOptions{ BasePath: basepath, }) @@ -140,7 +144,11 @@ func NewGroupcacheWithConfig(logger log.Logger, reg prometheus.Registerer, conf if err != nil { return err } - return dest.UnmarshalBinary(finalAttrs) + + if isTSDBChunkFile(parsedData.Name) { + return dest.UnmarshalBinary(finalAttrs, time.Now().Add(ChunkObjectAttrsTTL)) + } + panic("caching bucket layer must not call on unconfigured paths") case cachekey.IterVerb: var list []string if err := bucket.Iter(ctx, parsedData.Name, func(s string) error { @@ -154,7 +162,11 @@ func NewGroupcacheWithConfig(logger log.Logger, reg prometheus.Registerer, conf if err != nil { return err } - return dest.UnmarshalBinary(encodedList) + + if isBlocksRootDir(parsedData.Name) { + return dest.UnmarshalBinary(encodedList, time.Now().Add(BlocksIterTTL)) + } + panic("caching bucket layer must not call on unconfigured paths") case cachekey.ContentVerb: rc, err := bucket.Get(ctx, parsedData.Name) if err != nil { @@ -167,14 +179,26 @@ func NewGroupcacheWithConfig(logger log.Logger, reg prometheus.Registerer, conf return err } - return dest.UnmarshalBinary(b) + if isMetaFile(parsedData.Name) { + return dest.UnmarshalBinary(b, time.Now().Add(MetafileContentTTL)) + } + panic("caching bucket layer must not call on unconfigured paths") + case cachekey.ExistsVerb: exists, err := bucket.Exists(ctx, parsedData.Name) if err != nil { return err } - return dest.UnmarshalBinary([]byte(strconv.FormatBool(exists))) + if isMetaFile(parsedData.Name) { + if exists { + return dest.UnmarshalBinary([]byte(strconv.FormatBool(exists)), time.Now().Add(MetaFileExistsTTL)) + } else { + return dest.UnmarshalBinary([]byte(strconv.FormatBool(exists)), time.Now().Add(MetafileDoesntExistTTL)) + } + } + panic("caching bucket layer must not call on unconfigured paths") + case cachekey.SubrangeVerb: rc, err := bucket.GetRange(ctx, parsedData.Name, parsedData.Start, parsedData.End-parsedData.Start) if err != nil { @@ -187,7 +211,11 @@ func NewGroupcacheWithConfig(logger log.Logger, reg prometheus.Registerer, conf return err } - return dest.UnmarshalBinary(b) + if isTSDBChunkFile(parsedData.Name) { + return dest.UnmarshalBinary(b, time.Now().Add(ChunkSubrangeTTL)) + } + panic("caching bucket layer must not call on unconfigured paths") + } return nil @@ -218,8 +246,14 @@ func (c *Groupcache) Fetch(ctx context.Context, keys []string) map[string][]byte continue } - if len(codec) > 0 { - data[k] = codec + retrievedData, _, err := codec.MarshalBinary() + if err != nil { + level.Error(c.logger).Log("msg", "failed retrieving data", "err", err, "key", k) + continue + } + + if len(retrievedData) > 0 { + data[k] = retrievedData } } diff --git a/pkg/store/cache/caching_bucket_factory.go b/pkg/store/cache/caching_bucket_factory.go index a93c49a09b..23e4004f75 100644 --- a/pkg/store/cache/caching_bucket_factory.go +++ b/pkg/store/cache/caching_bucket_factory.go @@ -43,6 +43,8 @@ type CachingWithBackendConfig struct { // Maximum number of GetRange requests issued by this bucket for single GetRange call. Zero or negative value = unlimited. MaxChunksGetRangeRequests int `yaml:"max_chunks_get_range_requests"` + MetafileMaxSize model.Bytes `yaml:"metafile_max_size"` + // TTLs for various cache items. ChunkObjectAttrsTTL time.Duration `yaml:"chunk_object_attrs_ttl"` ChunkSubrangeTTL time.Duration `yaml:"chunk_subrange_ttl"` @@ -54,7 +56,6 @@ type CachingWithBackendConfig struct { MetafileExistsTTL time.Duration `yaml:"metafile_exists_ttl"` MetafileDoesntExistTTL time.Duration `yaml:"metafile_doesnt_exist_ttl"` MetafileContentTTL time.Duration `yaml:"metafile_content_ttl"` - MetafileMaxSize model.Bytes `yaml:"metafile_max_size"` } func (cfg *CachingWithBackendConfig) Defaults() { @@ -103,7 +104,10 @@ func NewCachingBucketFromYaml(yamlContent []byte, bucket objstore.Bucket, logger case string(GroupcacheBucketCacheProvider): const basePath = "/_galaxycache/" - c, err = cache.NewGroupcache(logger, reg, backendConfig, basePath, r, bucket) + c, err = cache.NewGroupcache(logger, reg, backendConfig, basePath, r, bucket, + isTSDBChunkFile, isMetaFile, isBlocksRootDir, + config.MetafileExistsTTL, config.MetafileDoesntExistTTL, config.MetafileContentTTL, + config.ChunkObjectAttrsTTL, config.ChunkSubrangeTTL, config.BlocksIterTTL) if err != nil { return nil, errors.Wrap(err, "failed to create groupcache") } From b15440e8d897c9bd3f4cfe9d338d4aff5ab77b27 Mon Sep 17 00:00:00 2001 From: akanshat Date: Thu, 23 Dec 2021 00:45:16 +0530 Subject: [PATCH 15/22] move caching_bucket_config to pkg cache Signed-off-by: akanshat --- Dockerfile.multi-stage | 2 +- go.mod | 2 + go.sum | 4 +- pkg/cache/caching_bucket_config.go | 205 +++++++++++++++++++++ pkg/cache/groupcache.go | 64 +++---- pkg/store/cache/caching_bucket.go | 92 +++++----- pkg/store/cache/caching_bucket_config.go | 209 ---------------------- pkg/store/cache/caching_bucket_factory.go | 17 +- pkg/store/cache/caching_bucket_test.go | 19 +- 9 files changed, 312 insertions(+), 302 deletions(-) create mode 100644 pkg/cache/caching_bucket_config.go delete mode 100644 pkg/store/cache/caching_bucket_config.go diff --git a/Dockerfile.multi-stage b/Dockerfile.multi-stage index 8f427a38df..39886e21e3 100644 --- a/Dockerfile.multi-stage +++ b/Dockerfile.multi-stage @@ -1,6 +1,6 @@ # By default we pin to amd64 sha. Use make docker to automatically adjust for arm64 versions. ARG BASE_DOCKER_SHA="14d68ca3d69fceaa6224250c83d81d935c053fb13594c811038c461194599973" -FROM golang:1.16-alpine3.12 as builder +FROM golang:1.17-alpine3.15 as builder WORKDIR $GOPATH/src/github.com/thanos-io/thanos # Change in the docker context invalidates the cache so to leverage docker diff --git a/go.mod b/go.mod index 8be5e20485..ce5ef62b4a 100644 --- a/go.mod +++ b/go.mod @@ -94,6 +94,8 @@ replace ( // Using a 3rd-party branch for custom dialer - see https://github.com/bradfitz/gomemcache/pull/86. // Required by Cortex https://github.com/cortexproject/cortex/pull/3051. github.com/bradfitz/gomemcache => github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab + + github.com/cortexproject/cortex v1.10.1-0.20211124141505-4e9fc3a2b5ab => github.com/akanshat/cortex v1.10.1-0.20211222182735-328fbeedd424 github.com/efficientgo/tools/core => github.com/efficientgo/tools/core v0.0.0-20210731122119-5d4a0645ce9a // Update to v1.1.1 to make sure windows CI pass. github.com/elastic/go-sysinfo => github.com/elastic/go-sysinfo v1.1.1 diff --git a/go.sum b/go.sum index 99ee9cd6c8..c96ee76892 100644 --- a/go.sum +++ b/go.sum @@ -173,6 +173,8 @@ github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/ github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/akanshat/cortex v1.10.1-0.20211222182735-328fbeedd424 h1:XxsZ4+as6/m2fO+YpQq0BYQEoC7fDP8Nje66RWG9YCw= +github.com/akanshat/cortex v1.10.1-0.20211222182735-328fbeedd424/go.mod h1:rOgO27ZndSaiFCRrWXYRIUHAJjeGSjk7s+fDPJU6gHg= github.com/alecthomas/kingpin v1.3.8-0.20210301060133-17f40c25f497 h1:aDITxVUQ/3KBhpVWX57Vo9ntGTxoRw1F0T6/x/tRzNU= github.com/alecthomas/kingpin v1.3.8-0.20210301060133-17f40c25f497/go.mod h1:b6br6/pDFSfMkBgC96TbpOji05q5pa+v5rIlS0Y6XtI= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -448,8 +450,6 @@ github.com/cortexproject/cortex v1.6.1-0.20210215155036-dfededd9f331/go.mod h1:8 github.com/cortexproject/cortex v1.7.1-0.20210224085859-66d6fb5b0d42/go.mod h1:u2dxcHInYbe45wxhLoWVdlFJyDhXewsMcxtnbq/QbH4= github.com/cortexproject/cortex v1.7.1-0.20210316085356-3fedc1108a49/go.mod h1:/DBOW8TzYBTE/U+O7Whs7i7E2eeeZl1iRVDtIqxn5kg= github.com/cortexproject/cortex v1.8.1-0.20210422151339-cf1c444e0905/go.mod h1:xxm4/CLvTmDxwE7yXwtClR4dIvkG4S09o5DygPOgc1U= -github.com/cortexproject/cortex v1.10.1-0.20211124141505-4e9fc3a2b5ab h1:THN4VQQqsZn5gNwcmQJO1GarnfZkSWfp5824ifoD9fQ= -github.com/cortexproject/cortex v1.10.1-0.20211124141505-4e9fc3a2b5ab/go.mod h1:njSBkQ1wUNx9X4knV/j65Pi4ItlJXX4QwXRKoMflJd8= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= diff --git a/pkg/cache/caching_bucket_config.go b/pkg/cache/caching_bucket_config.go new file mode 100644 index 0000000000..46cb584036 --- /dev/null +++ b/pkg/cache/caching_bucket_config.go @@ -0,0 +1,205 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package cache + +import ( + "time" + + "github.com/thanos-io/thanos/pkg/objstore" +) + +// Codec for encoding and decoding results of Iter call. +type IterCodec interface { + Encode(files []string) ([]byte, error) + Decode(cachedData []byte) ([]string, error) +} + +// CachingBucketConfig contains low-level configuration for individual bucket operations. +// This is not exposed to the user, but it is expected that code sets up individual +// operations based on user-provided configuration. +type CachingBucketConfig struct { + get map[string]*GetConfig + iter map[string]*IterConfig + exists map[string]*ExistsConfig + getRange map[string]*GetRangeConfig + attributes map[string]*AttributesConfig +} + +func NewCachingBucketConfig() *CachingBucketConfig { + return &CachingBucketConfig{ + get: map[string]*GetConfig{}, + iter: map[string]*IterConfig{}, + exists: map[string]*ExistsConfig{}, + getRange: map[string]*GetRangeConfig{}, + attributes: map[string]*AttributesConfig{}, + } +} + +// Generic config for single operation. +type OperationConfig struct { + Matcher func(name string) bool + Cache Cache +} + +// Operation-specific configs. +type IterConfig struct { + OperationConfig + TTL time.Duration + Codec IterCodec +} + +type ExistsConfig struct { + OperationConfig + ExistsTTL time.Duration + DoesntExistTTL time.Duration +} + +type GetConfig struct { + ExistsConfig + ContentTTL time.Duration + MaxCacheableSize int +} + +type GetRangeConfig struct { + OperationConfig + SubrangeSize int64 + MaxSubRequests int + AttributesTTL time.Duration + SubrangeTTL time.Duration +} + +type AttributesConfig struct { + OperationConfig + TTL time.Duration +} + +func newOperationConfig(cache Cache, matcher func(string) bool) OperationConfig { + if matcher == nil { + panic("matcher") + } + + return OperationConfig{ + Matcher: matcher, + Cache: cache, + } +} + +// CacheIter configures caching of "Iter" operation for matching directories. +func (cfg *CachingBucketConfig) CacheIter(configName string, cache Cache, matcher func(string) bool, ttl time.Duration, codec IterCodec) { + cfg.iter[configName] = &IterConfig{ + OperationConfig: newOperationConfig(cache, matcher), + TTL: ttl, + Codec: codec, + } +} + +// CacheGet configures caching of "Get" operation for matching files. Content of the object is cached, as well as whether object exists or not. +func (cfg *CachingBucketConfig) CacheGet(configName string, cache Cache, matcher func(string) bool, maxCacheableSize int, contentTTL, existsTTL, doesntExistTTL time.Duration) { + cfg.get[configName] = &GetConfig{ + ExistsConfig: ExistsConfig{ + OperationConfig: newOperationConfig(cache, matcher), + ExistsTTL: existsTTL, + DoesntExistTTL: doesntExistTTL, + }, + ContentTTL: contentTTL, + MaxCacheableSize: maxCacheableSize, + } +} + +// CacheExists configures caching of "Exists" operation for matching files. Negative values are cached as well. +func (cfg *CachingBucketConfig) CacheExists(configName string, cache Cache, matcher func(string) bool, existsTTL, doesntExistTTL time.Duration) { + cfg.exists[configName] = &ExistsConfig{ + OperationConfig: newOperationConfig(cache, matcher), + ExistsTTL: existsTTL, + DoesntExistTTL: doesntExistTTL, + } +} + +// CacheGetRange configures caching of "GetRange" operation. Subranges (aligned on subrange size) are cached individually. +// Since caching operation needs to know the object size to compute correct subranges, object size is cached as well. +// Single "GetRange" requests can result in multiple smaller GetRange sub-requests issued on the underlying bucket. +// MaxSubRequests specifies how many such subrequests may be issued. Values <= 0 mean there is no limit (requests +// for adjacent missing subranges are still merged). +func (cfg *CachingBucketConfig) CacheGetRange(configName string, cache Cache, matcher func(string) bool, subrangeSize int64, attributesTTL, subrangeTTL time.Duration, maxSubRequests int) { + cfg.getRange[configName] = &GetRangeConfig{ + OperationConfig: newOperationConfig(cache, matcher), + SubrangeSize: subrangeSize, + AttributesTTL: attributesTTL, + SubrangeTTL: subrangeTTL, + MaxSubRequests: maxSubRequests, + } +} + +// CacheAttributes configures caching of "Attributes" operation for matching files. +func (cfg *CachingBucketConfig) CacheAttributes(configName string, cache Cache, matcher func(name string) bool, ttl time.Duration) { + cfg.attributes[configName] = &AttributesConfig{ + OperationConfig: newOperationConfig(cache, matcher), + TTL: ttl, + } +} + +func (cfg *CachingBucketConfig) AllConfigNames() map[string][]string { + result := map[string][]string{} + for n := range cfg.get { + result[objstore.OpGet] = append(result[objstore.OpGet], n) + } + for n := range cfg.iter { + result[objstore.OpIter] = append(result[objstore.OpIter], n) + } + for n := range cfg.exists { + result[objstore.OpExists] = append(result[objstore.OpExists], n) + } + for n := range cfg.getRange { + result[objstore.OpGetRange] = append(result[objstore.OpGetRange], n) + } + for n := range cfg.attributes { + result[objstore.OpAttributes] = append(result[objstore.OpAttributes], n) + } + return result +} + +func (cfg *CachingBucketConfig) FindIterConfig(dir string) (string, *IterConfig) { + for n, cfg := range cfg.iter { + if cfg.Matcher(dir) { + return n, cfg + } + } + return "", nil +} + +func (cfg *CachingBucketConfig) FindExistConfig(name string) (string, *ExistsConfig) { + for n, cfg := range cfg.exists { + if cfg.Matcher(name) { + return n, cfg + } + } + return "", nil +} + +func (cfg *CachingBucketConfig) FindGetConfig(name string) (string, *GetConfig) { + for n, cfg := range cfg.get { + if cfg.Matcher(name) { + return n, cfg + } + } + return "", nil +} + +func (cfg *CachingBucketConfig) FindGetRangeConfig(name string) (string, *GetRangeConfig) { + for n, cfg := range cfg.getRange { + if cfg.Matcher(name) { + return n, cfg + } + } + return "", nil +} + +func (cfg *CachingBucketConfig) FindAttributesConfig(name string) (string, *AttributesConfig) { + for n, cfg := range cfg.attributes { + if cfg.Matcher(name) { + return n, cfg + } + } + return "", nil +} diff --git a/pkg/cache/groupcache.go b/pkg/cache/groupcache.go index 66ef0264a2..e97d70e626 100644 --- a/pkg/cache/groupcache.go +++ b/pkg/cache/groupcache.go @@ -78,21 +78,18 @@ func parseGroupcacheConfig(conf []byte) (GroupcacheConfig, error) { } // NewGroupcache creates a new Groupcache instance. -func NewGroupcache(logger log.Logger, reg prometheus.Registerer, conf []byte, basepath string, r *route.Router, bucket objstore.Bucket, - isTSDBChunkFile, isMetaFile, isBlocksRootDir func(path string) bool, - MetaFileExistsTTL, MetafileDoesntExistTTL, MetafileContentTTL, ChunkObjectAttrsTTL, ChunkSubrangeTTL, BlocksIterTTL time.Duration) (*Groupcache, error) { +func NewGroupcache(logger log.Logger, reg prometheus.Registerer, conf []byte, basepath string, r *route.Router, bucket objstore.Bucket, cfg *CachingBucketConfig) (*Groupcache, error) { config, err := parseGroupcacheConfig(conf) if err != nil { return nil, err } - return NewGroupcacheWithConfig(logger, reg, config, basepath, r, bucket, isTSDBChunkFile, isMetaFile, isBlocksRootDir, MetaFileExistsTTL, MetafileDoesntExistTTL, MetafileContentTTL, ChunkObjectAttrsTTL, ChunkSubrangeTTL, BlocksIterTTL) + return NewGroupcacheWithConfig(logger, reg, config, basepath, r, bucket, cfg) } // NewGroupcacheWithConfig creates a new Groupcache instance with the given config. func NewGroupcacheWithConfig(logger log.Logger, reg prometheus.Registerer, conf GroupcacheConfig, basepath string, r *route.Router, bucket objstore.Bucket, - isTSDBChunkFile, isMetaFile, isBlocksRootDir func(path string) bool, - MetaFileExistsTTL, MetafileDoesntExistTTL, MetafileContentTTL, ChunkObjectAttrsTTL, ChunkSubrangeTTL, BlocksIterTTL time.Duration) (*Groupcache, error) { + cfg *CachingBucketConfig) (*Groupcache, error) { httpProto := galaxyhttp.NewHTTPFetchProtocol(&galaxyhttp.HTTPOptions{ BasePath: basepath, }) @@ -135,6 +132,12 @@ func NewGroupcacheWithConfig(logger log.Logger, reg prometheus.Registerer, conf switch parsedData.Verb { case cachekey.AttributesVerb: + _, attrCfg := cfg.FindAttributesConfig(parsedData.Name) + if attrCfg == nil { + // TODO: Debug this. Why? Attributes get called for Chunks. + panic("caching bucket layer must not call on unconfigured paths") + } + attrs, err := bucket.Attributes(ctx, parsedData.Name) if err != nil { return err @@ -145,11 +148,13 @@ func NewGroupcacheWithConfig(logger log.Logger, reg prometheus.Registerer, conf return err } - if isTSDBChunkFile(parsedData.Name) { - return dest.UnmarshalBinary(finalAttrs, time.Now().Add(ChunkObjectAttrsTTL)) - } - panic("caching bucket layer must not call on unconfigured paths") + return dest.UnmarshalBinary(finalAttrs, time.Now().Add(attrCfg.TTL)) case cachekey.IterVerb: + _, iterCfg := cfg.FindIterConfig(parsedData.Name) + if iterCfg == nil { + panic("caching bucket layer must not call on unconfigured paths") + } + var list []string if err := bucket.Iter(ctx, parsedData.Name, func(s string) error { list = append(list, s) @@ -163,11 +168,12 @@ func NewGroupcacheWithConfig(logger log.Logger, reg prometheus.Registerer, conf return err } - if isBlocksRootDir(parsedData.Name) { - return dest.UnmarshalBinary(encodedList, time.Now().Add(BlocksIterTTL)) - } - panic("caching bucket layer must not call on unconfigured paths") + return dest.UnmarshalBinary(encodedList, time.Now().Add(iterCfg.TTL)) case cachekey.ContentVerb: + _, contentCfg := cfg.FindGetConfig(parsedData.Name) + if contentCfg == nil { + panic("caching bucket layer must not call on unconfigured paths") + } rc, err := bucket.Get(ctx, parsedData.Name) if err != nil { return err @@ -179,27 +185,28 @@ func NewGroupcacheWithConfig(logger log.Logger, reg prometheus.Registerer, conf return err } - if isMetaFile(parsedData.Name) { - return dest.UnmarshalBinary(b, time.Now().Add(MetafileContentTTL)) - } - panic("caching bucket layer must not call on unconfigured paths") - + return dest.UnmarshalBinary(b, time.Now().Add(contentCfg.ContentTTL)) case cachekey.ExistsVerb: + _, existsCfg := cfg.FindExistConfig(parsedData.Name) + if existsCfg == nil { + panic("caching bucket layer must not call on unconfigured paths") + } exists, err := bucket.Exists(ctx, parsedData.Name) if err != nil { return err } - if isMetaFile(parsedData.Name) { - if exists { - return dest.UnmarshalBinary([]byte(strconv.FormatBool(exists)), time.Now().Add(MetaFileExistsTTL)) - } else { - return dest.UnmarshalBinary([]byte(strconv.FormatBool(exists)), time.Now().Add(MetafileDoesntExistTTL)) - } + if exists { + return dest.UnmarshalBinary([]byte(strconv.FormatBool(exists)), time.Now().Add(existsCfg.ExistsTTL)) + } else { + return dest.UnmarshalBinary([]byte(strconv.FormatBool(exists)), time.Now().Add(existsCfg.DoesntExistTTL)) } - panic("caching bucket layer must not call on unconfigured paths") case cachekey.SubrangeVerb: + _, subrangeCfg := cfg.FindGetRangeConfig(parsedData.Name) + if subrangeCfg == nil { + panic("caching bucket layer must not call on unconfigured paths") + } rc, err := bucket.GetRange(ctx, parsedData.Name, parsedData.Start, parsedData.End-parsedData.Start) if err != nil { return err @@ -211,10 +218,7 @@ func NewGroupcacheWithConfig(logger log.Logger, reg prometheus.Registerer, conf return err } - if isTSDBChunkFile(parsedData.Name) { - return dest.UnmarshalBinary(b, time.Now().Add(ChunkSubrangeTTL)) - } - panic("caching bucket layer must not call on unconfigured paths") + return dest.UnmarshalBinary(b, time.Now().Add(subrangeCfg.SubrangeTTL)) } diff --git a/pkg/store/cache/caching_bucket.go b/pkg/store/cache/caching_bucket.go index d1d7d7373a..a8fbde25b2 100644 --- a/pkg/store/cache/caching_bucket.go +++ b/pkg/store/cache/caching_bucket.go @@ -39,21 +39,21 @@ var ( type CachingBucket struct { objstore.Bucket - cfg *CachingBucketConfig + cfg *cache.CachingBucketConfig logger log.Logger requestedGetRangeBytes *prometheus.CounterVec fetchedGetRangeBytes *prometheus.CounterVec refetchedGetRangeBytes *prometheus.CounterVec - operationConfigs map[string][]*operationConfig + operationConfigs map[string][]*cache.OperationConfig operationRequests *prometheus.CounterVec operationHits *prometheus.CounterVec } // NewCachingBucket creates new caching bucket with provided configuration. Configuration should not be // changed after creating caching bucket. -func NewCachingBucket(b objstore.Bucket, cfg *CachingBucketConfig, logger log.Logger, reg prometheus.Registerer) (*CachingBucket, error) { +func NewCachingBucket(b objstore.Bucket, cfg *cache.CachingBucketConfig, logger log.Logger, reg prometheus.Registerer) (*CachingBucket, error) { if b == nil { return nil, errors.New("bucket is nil") } @@ -63,7 +63,7 @@ func NewCachingBucket(b objstore.Bucket, cfg *CachingBucketConfig, logger log.Lo cfg: cfg, logger: logger, - operationConfigs: map[string][]*operationConfig{}, + operationConfigs: map[string][]*cache.OperationConfig{}, requestedGetRangeBytes: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ Name: "thanos_store_bucket_cache_getrange_requested_bytes_total", @@ -88,7 +88,7 @@ func NewCachingBucket(b objstore.Bucket, cfg *CachingBucketConfig, logger log.Lo }, []string{"operation", "config"}), } - for op, names := range cfg.allConfigNames() { + for op, names := range cfg.AllConfigNames() { for _, n := range names { cb.operationRequests.WithLabelValues(op, n) cb.operationHits.WithLabelValues(op, n) @@ -126,7 +126,7 @@ func (cb *CachingBucket) ReaderWithExpectedErrs(expectedFunc objstore.IsOpFailur } func (cb *CachingBucket) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error { - cfgName, cfg := cb.cfg.findIterConfig(dir) + cfgName, cfg := cb.cfg.FindIterConfig(dir) if cfg == nil { return cb.Bucket.Iter(ctx, dir, f, options...) } @@ -134,9 +134,9 @@ func (cb *CachingBucket) Iter(ctx context.Context, dir string, f func(string) er cb.operationRequests.WithLabelValues(objstore.OpIter, cfgName).Inc() iterVerb := cachekey.BucketCacheKey{Verb: cachekey.IterVerb, Name: dir} key := iterVerb.String() - data := cfg.cache.Fetch(ctx, []string{key}) + data := cfg.Cache.Fetch(ctx, []string{key}) if data[key] != nil { - list, err := cfg.codec.Decode(data[key]) + list, err := cfg.Codec.Decode(data[key]) if err == nil { cb.operationHits.WithLabelValues(objstore.OpIter, cfgName).Inc() for _, n := range list { @@ -158,11 +158,11 @@ func (cb *CachingBucket) Iter(ctx context.Context, dir string, f func(string) er return f(s) }, options...) - remainingTTL := cfg.ttl - time.Since(iterTime) + remainingTTL := cfg.TTL - time.Since(iterTime) if err == nil && remainingTTL > 0 { - data, encErr := cfg.codec.Encode(list) + data, encErr := cfg.Codec.Encode(list) if encErr == nil { - cfg.cache.Store(ctx, map[string][]byte{key: data}, remainingTTL) + cfg.Cache.Store(ctx, map[string][]byte{key: data}, remainingTTL) return nil } level.Warn(cb.logger).Log("msg", "failed to encode Iter result", "key", key, "err", encErr) @@ -171,7 +171,7 @@ func (cb *CachingBucket) Iter(ctx context.Context, dir string, f func(string) er } func (cb *CachingBucket) Exists(ctx context.Context, name string) (bool, error) { - cfgName, cfg := cb.cfg.findExistConfig(name) + cfgName, cfg := cb.cfg.FindExistConfig(name) if cfg == nil { return cb.Bucket.Exists(ctx, name) } @@ -180,7 +180,7 @@ func (cb *CachingBucket) Exists(ctx context.Context, name string) (bool, error) existsVerb := cachekey.BucketCacheKey{Verb: cachekey.ExistsVerb, Name: name} key := existsVerb.String() - hits := cfg.cache.Fetch(ctx, []string{key}) + hits := cfg.Cache.Fetch(ctx, []string{key}) if ex := hits[key]; ex != nil { exists, err := strconv.ParseBool(string(ex)) @@ -194,7 +194,7 @@ func (cb *CachingBucket) Exists(ctx context.Context, name string) (bool, error) existsTime := time.Now() ok, err := cb.Bucket.Exists(ctx, name) if err == nil { - storeExistsCacheEntry(ctx, key, ok, existsTime, cfg.cache, cfg.existsTTL, cfg.doesntExistTTL) + storeExistsCacheEntry(ctx, key, ok, existsTime, cfg.Cache, cfg.ExistsTTL, cfg.DoesntExistTTL) } return ok, err @@ -214,7 +214,7 @@ func storeExistsCacheEntry(ctx context.Context, cachingKey string, exists bool, } func (cb *CachingBucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { - cfgName, cfg := cb.cfg.findGetConfig(name) + cfgName, cfg := cb.cfg.FindGetConfig(name) if cfg == nil { return cb.Bucket.Get(ctx, name) } @@ -226,7 +226,7 @@ func (cb *CachingBucket) Get(ctx context.Context, name string) (io.ReadCloser, e existsVerb := cachekey.BucketCacheKey{Verb: cachekey.ExistsVerb, Name: name} existsKey := existsVerb.String() - hits := cfg.cache.Fetch(ctx, []string{contentKey, existsKey}) + hits := cfg.Cache.Fetch(ctx, []string{contentKey, existsKey}) if hits[contentKey] != nil { cb.operationHits.WithLabelValues(objstore.OpGet, cfgName).Inc() return objstore.NopCloserWithSize(bytes.NewBuffer(hits[contentKey])), nil @@ -245,22 +245,22 @@ func (cb *CachingBucket) Get(ctx context.Context, name string) (io.ReadCloser, e if err != nil { if cb.Bucket.IsObjNotFoundErr(err) { // Cache that object doesn't exist. - storeExistsCacheEntry(ctx, existsKey, false, getTime, cfg.cache, cfg.existsTTL, cfg.doesntExistTTL) + storeExistsCacheEntry(ctx, existsKey, false, getTime, cfg.Cache, cfg.ExistsTTL, cfg.DoesntExistTTL) } return nil, err } - storeExistsCacheEntry(ctx, existsKey, true, getTime, cfg.cache, cfg.existsTTL, cfg.doesntExistTTL) + storeExistsCacheEntry(ctx, existsKey, true, getTime, cfg.Cache, cfg.ExistsTTL, cfg.DoesntExistTTL) return &getReader{ - c: cfg.cache, + c: cfg.Cache, ctx: ctx, r: reader, buf: new(bytes.Buffer), startTime: getTime, - ttl: cfg.contentTTL, + ttl: cfg.ContentTTL, cacheKey: contentKey, - maxSize: cfg.maxCacheableSize, + maxSize: cfg.MaxCacheableSize, }, nil } @@ -273,7 +273,7 @@ func (cb *CachingBucket) GetRange(ctx context.Context, name string, off, length return cb.Bucket.GetRange(ctx, name, off, length) } - cfgName, cfg := cb.cfg.findGetRangeConfig(name) + cfgName, cfg := cb.cfg.FindGetRangeConfig(name) if cfg == nil { return cb.Bucket.GetRange(ctx, name, off, length) } @@ -282,12 +282,12 @@ func (cb *CachingBucket) GetRange(ctx context.Context, name string, off, length } func (cb *CachingBucket) Attributes(ctx context.Context, name string) (objstore.ObjectAttributes, error) { - cfgName, cfg := cb.cfg.findAttributesConfig(name) + cfgName, cfg := cb.cfg.FindAttributesConfig(name) if cfg == nil { return cb.Bucket.Attributes(ctx, name) } - return cb.cachedAttributes(ctx, name, cfgName, cfg.cache, cfg.ttl) + return cb.cachedAttributes(ctx, name, cfgName, cfg.Cache, cfg.TTL) } func (cb *CachingBucket) cachedAttributes(ctx context.Context, name, cfgName string, cache cache.Cache, ttl time.Duration) (objstore.ObjectAttributes, error) { @@ -322,11 +322,11 @@ func (cb *CachingBucket) cachedAttributes(ctx context.Context, name, cfgName str return attrs, nil } -func (cb *CachingBucket) cachedGetRange(ctx context.Context, name string, offset, length int64, cfgName string, cfg *getRangeConfig) (io.ReadCloser, error) { +func (cb *CachingBucket) cachedGetRange(ctx context.Context, name string, offset, length int64, cfgName string, cfg *cache.GetRangeConfig) (io.ReadCloser, error) { cb.operationRequests.WithLabelValues(objstore.OpGetRange, cfgName).Inc() cb.requestedGetRangeBytes.WithLabelValues(cfgName).Add(float64(length)) - attrs, err := cb.cachedAttributes(ctx, name, cfgName, cfg.cache, cfg.attributesTTL) + attrs, err := cb.cachedAttributes(ctx, name, cfgName, cfg.Cache, cfg.AttributesTTL) if err != nil { return nil, errors.Wrapf(err, "failed to get object attributes: %s", name) } @@ -337,28 +337,28 @@ func (cb *CachingBucket) cachedGetRange(ctx context.Context, name string, offset } // Start and end range are subrange-aligned offsets into object, that we're going to read. - startRange := (offset / cfg.subrangeSize) * cfg.subrangeSize - endRange := ((offset + length) / cfg.subrangeSize) * cfg.subrangeSize - if (offset+length)%cfg.subrangeSize > 0 { - endRange += cfg.subrangeSize + startRange := (offset / cfg.SubrangeSize) * cfg.SubrangeSize + endRange := ((offset + length) / cfg.SubrangeSize) * cfg.SubrangeSize + if (offset+length)%cfg.SubrangeSize > 0 { + endRange += cfg.SubrangeSize } // The very last subrange in the object may have length that is not divisible by subrange size. - lastSubrangeOffset := endRange - cfg.subrangeSize - lastSubrangeLength := int(cfg.subrangeSize) + lastSubrangeOffset := endRange - cfg.SubrangeSize + lastSubrangeLength := int(cfg.SubrangeSize) if endRange > attrs.Size { - lastSubrangeOffset = (attrs.Size / cfg.subrangeSize) * cfg.subrangeSize + lastSubrangeOffset = (attrs.Size / cfg.SubrangeSize) * cfg.SubrangeSize lastSubrangeLength = int(attrs.Size - lastSubrangeOffset) } - numSubranges := (endRange - startRange) / cfg.subrangeSize + numSubranges := (endRange - startRange) / cfg.SubrangeSize offsetKeys := make(map[int64]string, numSubranges) keys := make([]string, 0, numSubranges) totalRequestedBytes := int64(0) - for off := startRange; off < endRange; off += cfg.subrangeSize { - end := off + cfg.subrangeSize + for off := startRange; off < endRange; off += cfg.SubrangeSize { + end := off + cfg.SubrangeSize if end > attrs.Size { end = attrs.Size } @@ -371,7 +371,7 @@ func (cb *CachingBucket) cachedGetRange(ctx context.Context, name string, offset // Try to get all subranges from the cache. totalCachedBytes := int64(0) - hits := cfg.cache.Fetch(ctx, keys) + hits := cfg.Cache.Fetch(ctx, keys) for _, b := range hits { totalCachedBytes += int64(len(b)) } @@ -389,7 +389,7 @@ func (cb *CachingBucket) cachedGetRange(ctx context.Context, name string, offset } } - return ioutil.NopCloser(newSubrangesReader(cfg.subrangeSize, offsetKeys, hits, offset, length)), nil + return ioutil.NopCloser(newSubrangesReader(cfg.SubrangeSize, offsetKeys, hits, offset, length)), nil } type rng struct { @@ -398,19 +398,19 @@ type rng struct { // fetchMissingSubranges fetches missing subranges, stores them into "hits" map // and into cache as well (using provided cacheKeys). -func (cb *CachingBucket) fetchMissingSubranges(ctx context.Context, name string, startRange, endRange int64, cacheKeys map[int64]string, hits map[string][]byte, lastSubrangeOffset int64, lastSubrangeLength int, cfgName string, cfg *getRangeConfig) error { +func (cb *CachingBucket) fetchMissingSubranges(ctx context.Context, name string, startRange, endRange int64, cacheKeys map[int64]string, hits map[string][]byte, lastSubrangeOffset int64, lastSubrangeLength int, cfgName string, cfg *cache.GetRangeConfig) error { // Ordered list of missing sub-ranges. var missing []rng - for off := startRange; off < endRange; off += cfg.subrangeSize { + for off := startRange; off < endRange; off += cfg.SubrangeSize { if hits[cacheKeys[off]] == nil { - missing = append(missing, rng{start: off, end: off + cfg.subrangeSize}) + missing = append(missing, rng{start: off, end: off + cfg.SubrangeSize}) } } missing = mergeRanges(missing, 0) // Merge adjacent ranges. // Keep merging until we have only max number of ranges (= requests). - for limit := cfg.subrangeSize; cfg.maxSubRequests > 0 && len(missing) > cfg.maxSubRequests; limit = limit * 2 { + for limit := cfg.SubrangeSize; cfg.MaxSubRequests > 0 && len(missing) > cfg.MaxSubRequests; limit = limit * 2 { missing = mergeRanges(missing, limit) } @@ -431,7 +431,7 @@ func (cb *CachingBucket) fetchMissingSubranges(ctx context.Context, name string, if lastSubrangeOffset >= m.end { bufSize = m.end - m.start } else { - bufSize = ((m.end - m.start) - cfg.subrangeSize) + int64(lastSubrangeLength) + bufSize = ((m.end - m.start) - cfg.SubrangeSize) + int64(lastSubrangeLength) } buf := make([]byte, bufSize) @@ -440,7 +440,7 @@ func (cb *CachingBucket) fetchMissingSubranges(ctx context.Context, name string, return errors.Wrapf(err, "fetching range [%d, %d]", m.start, m.end) } - for off := m.start; off < m.end && gctx.Err() == nil; off += cfg.subrangeSize { + for off := m.start; off < m.end && gctx.Err() == nil; off += cfg.SubrangeSize { key := cacheKeys[off] if key == "" { return errors.Errorf("fetching range [%d, %d]: caching key for offset %d not found", m.start, m.end, off) @@ -453,7 +453,7 @@ func (cb *CachingBucket) fetchMissingSubranges(ctx context.Context, name string, // if object length isn't divisible by subrange size. subrangeData = buf[off-m.start : off-m.start+int64(lastSubrangeLength)] } else { - subrangeData = buf[off-m.start : off-m.start+cfg.subrangeSize] + subrangeData = buf[off-m.start : off-m.start+cfg.SubrangeSize] } storeToCache := false @@ -466,7 +466,7 @@ func (cb *CachingBucket) fetchMissingSubranges(ctx context.Context, name string, if storeToCache { cb.fetchedGetRangeBytes.WithLabelValues(originBucket, cfgName).Add(float64(len(subrangeData))) - cfg.cache.Store(gctx, map[string][]byte{key: subrangeData}, cfg.subrangeTTL) + cfg.Cache.Store(gctx, map[string][]byte{key: subrangeData}, cfg.SubrangeTTL) } else { cb.refetchedGetRangeBytes.WithLabelValues(originCache, cfgName).Add(float64(len(subrangeData))) } diff --git a/pkg/store/cache/caching_bucket_config.go b/pkg/store/cache/caching_bucket_config.go deleted file mode 100644 index d8e089d7e2..0000000000 --- a/pkg/store/cache/caching_bucket_config.go +++ /dev/null @@ -1,209 +0,0 @@ -// Copyright (c) The Thanos Authors. -// Licensed under the Apache License 2.0. - -package storecache - -import ( - "time" - - "github.com/thanos-io/thanos/pkg/cache" - "github.com/thanos-io/thanos/pkg/objstore" -) - -// Codec for encoding and decoding results of Iter call. -type IterCodec interface { - Encode(files []string) ([]byte, error) - Decode(cachedData []byte) ([]string, error) -} - -// CachingBucketConfig contains low-level configuration for individual bucket operations. -// This is not exposed to the user, but it is expected that code sets up individual -// operations based on user-provided configuration. -type CachingBucketConfig struct { - get map[string]*getConfig - iter map[string]*iterConfig - exists map[string]*existsConfig - getRange map[string]*getRangeConfig - attributes map[string]*attributesConfig -} - -func NewCachingBucketConfig() *CachingBucketConfig { - return &CachingBucketConfig{ - get: map[string]*getConfig{}, - iter: map[string]*iterConfig{}, - exists: map[string]*existsConfig{}, - getRange: map[string]*getRangeConfig{}, - attributes: map[string]*attributesConfig{}, - } -} - -// Generic config for single operation. -type operationConfig struct { - matcher func(name string) bool - cache cache.Cache -} - -// Operation-specific configs. -type iterConfig struct { - operationConfig - ttl time.Duration - codec IterCodec -} - -type existsConfig struct { - operationConfig - existsTTL time.Duration - doesntExistTTL time.Duration -} - -type getConfig struct { - existsConfig - contentTTL time.Duration - maxCacheableSize int -} - -type getRangeConfig struct { - operationConfig - subrangeSize int64 - maxSubRequests int - attributesTTL time.Duration - subrangeTTL time.Duration -} - -type attributesConfig struct { - operationConfig - ttl time.Duration -} - -func newOperationConfig(cache cache.Cache, matcher func(string) bool) operationConfig { - if cache == nil { - panic("cache") - } - if matcher == nil { - panic("matcher") - } - - return operationConfig{ - matcher: matcher, - cache: cache, - } -} - -// CacheIter configures caching of "Iter" operation for matching directories. -func (cfg *CachingBucketConfig) CacheIter(configName string, cache cache.Cache, matcher func(string) bool, ttl time.Duration, codec IterCodec) { - cfg.iter[configName] = &iterConfig{ - operationConfig: newOperationConfig(cache, matcher), - ttl: ttl, - codec: codec, - } -} - -// CacheGet configures caching of "Get" operation for matching files. Content of the object is cached, as well as whether object exists or not. -func (cfg *CachingBucketConfig) CacheGet(configName string, cache cache.Cache, matcher func(string) bool, maxCacheableSize int, contentTTL, existsTTL, doesntExistTTL time.Duration) { - cfg.get[configName] = &getConfig{ - existsConfig: existsConfig{ - operationConfig: newOperationConfig(cache, matcher), - existsTTL: existsTTL, - doesntExistTTL: doesntExistTTL, - }, - contentTTL: contentTTL, - maxCacheableSize: maxCacheableSize, - } -} - -// CacheExists configures caching of "Exists" operation for matching files. Negative values are cached as well. -func (cfg *CachingBucketConfig) CacheExists(configName string, cache cache.Cache, matcher func(string) bool, existsTTL, doesntExistTTL time.Duration) { - cfg.exists[configName] = &existsConfig{ - operationConfig: newOperationConfig(cache, matcher), - existsTTL: existsTTL, - doesntExistTTL: doesntExistTTL, - } -} - -// CacheGetRange configures caching of "GetRange" operation. Subranges (aligned on subrange size) are cached individually. -// Since caching operation needs to know the object size to compute correct subranges, object size is cached as well. -// Single "GetRange" requests can result in multiple smaller GetRange sub-requests issued on the underlying bucket. -// MaxSubRequests specifies how many such subrequests may be issued. Values <= 0 mean there is no limit (requests -// for adjacent missing subranges are still merged). -func (cfg *CachingBucketConfig) CacheGetRange(configName string, cache cache.Cache, matcher func(string) bool, subrangeSize int64, attributesTTL, subrangeTTL time.Duration, maxSubRequests int) { - cfg.getRange[configName] = &getRangeConfig{ - operationConfig: newOperationConfig(cache, matcher), - subrangeSize: subrangeSize, - attributesTTL: attributesTTL, - subrangeTTL: subrangeTTL, - maxSubRequests: maxSubRequests, - } -} - -// CacheAttributes configures caching of "Attributes" operation for matching files. -func (cfg *CachingBucketConfig) CacheAttributes(configName string, cache cache.Cache, matcher func(name string) bool, ttl time.Duration) { - cfg.attributes[configName] = &attributesConfig{ - operationConfig: newOperationConfig(cache, matcher), - ttl: ttl, - } -} - -func (cfg *CachingBucketConfig) allConfigNames() map[string][]string { - result := map[string][]string{} - for n := range cfg.get { - result[objstore.OpGet] = append(result[objstore.OpGet], n) - } - for n := range cfg.iter { - result[objstore.OpIter] = append(result[objstore.OpIter], n) - } - for n := range cfg.exists { - result[objstore.OpExists] = append(result[objstore.OpExists], n) - } - for n := range cfg.getRange { - result[objstore.OpGetRange] = append(result[objstore.OpGetRange], n) - } - for n := range cfg.attributes { - result[objstore.OpAttributes] = append(result[objstore.OpAttributes], n) - } - return result -} - -func (cfg *CachingBucketConfig) findIterConfig(dir string) (string, *iterConfig) { - for n, cfg := range cfg.iter { - if cfg.matcher(dir) { - return n, cfg - } - } - return "", nil -} - -func (cfg *CachingBucketConfig) findExistConfig(name string) (string, *existsConfig) { - for n, cfg := range cfg.exists { - if cfg.matcher(name) { - return n, cfg - } - } - return "", nil -} - -func (cfg *CachingBucketConfig) findGetConfig(name string) (string, *getConfig) { - for n, cfg := range cfg.get { - if cfg.matcher(name) { - return n, cfg - } - } - return "", nil -} - -func (cfg *CachingBucketConfig) findGetRangeConfig(name string) (string, *getRangeConfig) { - for n, cfg := range cfg.getRange { - if cfg.matcher(name) { - return n, cfg - } - } - return "", nil -} - -func (cfg *CachingBucketConfig) findAttributesConfig(name string) (string, *attributesConfig) { - for n, cfg := range cfg.attributes { - if cfg.matcher(name) { - return n, cfg - } - } - return "", nil -} diff --git a/pkg/store/cache/caching_bucket_factory.go b/pkg/store/cache/caching_bucket_factory.go index 23e4004f75..190967e623 100644 --- a/pkg/store/cache/caching_bucket_factory.go +++ b/pkg/store/cache/caching_bucket_factory.go @@ -104,10 +104,17 @@ func NewCachingBucketFromYaml(yamlContent []byte, bucket objstore.Bucket, logger case string(GroupcacheBucketCacheProvider): const basePath = "/_galaxycache/" - c, err = cache.NewGroupcache(logger, reg, backendConfig, basePath, r, bucket, - isTSDBChunkFile, isMetaFile, isBlocksRootDir, - config.MetafileExistsTTL, config.MetafileDoesntExistTTL, config.MetafileContentTTL, - config.ChunkObjectAttrsTTL, config.ChunkSubrangeTTL, config.BlocksIterTTL) + groupcacheCfg := cache.NewCachingBucketConfig() + + // Configure cache. + groupcacheCfg.CacheGetRange("chunks", c, isTSDBChunkFile, config.ChunkSubrangeSize, config.ChunkObjectAttrsTTL, config.ChunkSubrangeTTL, config.MaxChunksGetRangeRequests) + groupcacheCfg.CacheExists("meta.jsons", c, isMetaFile, config.MetafileExistsTTL, config.MetafileDoesntExistTTL) + groupcacheCfg.CacheGet("meta.jsons", c, isMetaFile, int(config.MetafileMaxSize), config.MetafileContentTTL, config.MetafileExistsTTL, config.MetafileDoesntExistTTL) + + // Cache Iter requests for root. + groupcacheCfg.CacheIter("blocks-iter", c, isBlocksRootDir, config.BlocksIterTTL, JSONIterCodec{}) + + c, err = cache.NewGroupcache(logger, reg, backendConfig, basePath, r, bucket, groupcacheCfg) if err != nil { return nil, errors.Wrap(err, "failed to create groupcache") } @@ -124,7 +131,7 @@ func NewCachingBucketFromYaml(yamlContent []byte, bucket objstore.Bucket, logger // Include interactions with cache in the traces. c = cache.NewTracingCache(c) - cfg := NewCachingBucketConfig() + cfg := cache.NewCachingBucketConfig() // Configure cache. cfg.CacheGetRange("chunks", c, isTSDBChunkFile, config.ChunkSubrangeSize, config.ChunkObjectAttrsTTL, config.ChunkSubrangeTTL, config.MaxChunksGetRangeRequests) diff --git a/pkg/store/cache/caching_bucket_test.go b/pkg/store/cache/caching_bucket_test.go index cbfcb91f2d..549afa5710 100644 --- a/pkg/store/cache/caching_bucket_test.go +++ b/pkg/store/cache/caching_bucket_test.go @@ -19,6 +19,7 @@ import ( "github.com/pkg/errors" promtest "github.com/prometheus/client_golang/prometheus/testutil" + thanoscache "github.com/thanos-io/thanos/pkg/cache" "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/runutil" "github.com/thanos-io/thanos/pkg/store/cache/cachekey" @@ -236,7 +237,7 @@ func TestChunksCaching(t *testing.T) { tc.init() } - cfg := NewCachingBucketConfig() + cfg := thanoscache.NewCachingBucketConfig() cfg.CacheGetRange("chunks", cache, isTSDBChunkFile, subrangeSize, time.Hour, time.Hour, tc.maxGetRangeRequests) cachingBucket, err := NewCachingBucket(inmem, cfg, nil, nil) @@ -354,7 +355,7 @@ func TestMergeRanges(t *testing.T) { func TestInvalidOffsetAndLength(t *testing.T) { b := &testBucket{objstore.NewInMemBucket()} - cfg := NewCachingBucketConfig() + cfg := thanoscache.NewCachingBucketConfig() cfg.CacheGetRange("chunks", newMockCache(), func(string) bool { return true }, 10000, time.Hour, time.Hour, 3) c, err := NewCachingBucket(b, cfg, nil, nil) @@ -398,7 +399,7 @@ func TestCachedIter(t *testing.T) { cache := newMockCache() const cfgName = "dirs" - cfg := NewCachingBucketConfig() + cfg := thanoscache.NewCachingBucketConfig() cfg.CacheIter(cfgName, cache, func(string) bool { return true }, 5*time.Minute, JSONIterCodec{}) cb, err := NewCachingBucket(inmem, cfg, nil, nil) @@ -460,7 +461,7 @@ func TestExists(t *testing.T) { // We reuse cache between tests (!) cache := newMockCache() - cfg := NewCachingBucketConfig() + cfg := thanoscache.NewCachingBucketConfig() const cfgName = "test" cfg.CacheExists(cfgName, cache, matchAll, 10*time.Minute, 2*time.Minute) @@ -486,7 +487,7 @@ func TestExistsCachingDisabled(t *testing.T) { // We reuse cache between tests (!) cache := newMockCache() - cfg := NewCachingBucketConfig() + cfg := thanoscache.NewCachingBucketConfig() const cfgName = "test" cfg.CacheExists(cfgName, cache, func(string) bool { return false }, 10*time.Minute, 2*time.Minute) @@ -523,7 +524,7 @@ func TestGet(t *testing.T) { // We reuse cache between tests (!) cache := newMockCache() - cfg := NewCachingBucketConfig() + cfg := thanoscache.NewCachingBucketConfig() const cfgName = "metafile" cfg.CacheGet(cfgName, cache, matchAll, 1024, 10*time.Minute, 10*time.Minute, 2*time.Minute) cfg.CacheExists(cfgName, cache, matchAll, 10*time.Minute, 2*time.Minute) @@ -554,7 +555,7 @@ func TestGetTooBigObject(t *testing.T) { // We reuse cache between tests (!) cache := newMockCache() - cfg := NewCachingBucketConfig() + cfg := thanoscache.NewCachingBucketConfig() const cfgName = "metafile" // Only allow 5 bytes to be cached. cfg.CacheGet(cfgName, cache, matchAll, 5, 10*time.Minute, 10*time.Minute, 2*time.Minute) @@ -577,7 +578,7 @@ func TestGetPartialRead(t *testing.T) { cache := newMockCache() - cfg := NewCachingBucketConfig() + cfg := thanoscache.NewCachingBucketConfig() const cfgName = "metafile" cfg.CacheGet(cfgName, cache, matchAll, 1024, 10*time.Minute, 10*time.Minute, 2*time.Minute) cfg.CacheExists(cfgName, cache, matchAll, 10*time.Minute, 2*time.Minute) @@ -636,7 +637,7 @@ func TestAttributes(t *testing.T) { // We reuse cache between tests (!) cache := newMockCache() - cfg := NewCachingBucketConfig() + cfg := thanoscache.NewCachingBucketConfig() const cfgName = "test" cfg.CacheAttributes(cfgName, cache, matchAll, time.Minute) From c7c1460162290f048c3346169e0307ae0b1e536e Mon Sep 17 00:00:00 2001 From: akanshat Date: Mon, 3 Jan 2022 13:28:11 +0530 Subject: [PATCH 16/22] modify groupcacheCfg Signed-off-by: akanshat --- pkg/cache/groupcache.go | 1 - pkg/store/cache/caching_bucket_factory.go | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cache/groupcache.go b/pkg/cache/groupcache.go index e97d70e626..59f8c5b40a 100644 --- a/pkg/cache/groupcache.go +++ b/pkg/cache/groupcache.go @@ -134,7 +134,6 @@ func NewGroupcacheWithConfig(logger log.Logger, reg prometheus.Registerer, conf case cachekey.AttributesVerb: _, attrCfg := cfg.FindAttributesConfig(parsedData.Name) if attrCfg == nil { - // TODO: Debug this. Why? Attributes get called for Chunks. panic("caching bucket layer must not call on unconfigured paths") } diff --git a/pkg/store/cache/caching_bucket_factory.go b/pkg/store/cache/caching_bucket_factory.go index 190967e623..b00d3d9e4b 100644 --- a/pkg/store/cache/caching_bucket_factory.go +++ b/pkg/store/cache/caching_bucket_factory.go @@ -107,6 +107,7 @@ func NewCachingBucketFromYaml(yamlContent []byte, bucket objstore.Bucket, logger groupcacheCfg := cache.NewCachingBucketConfig() // Configure cache. + groupcacheCfg.CacheAttributes("chunks", c, isTSDBChunkFile, config.ChunkObjectAttrsTTL) groupcacheCfg.CacheGetRange("chunks", c, isTSDBChunkFile, config.ChunkSubrangeSize, config.ChunkObjectAttrsTTL, config.ChunkSubrangeTTL, config.MaxChunksGetRangeRequests) groupcacheCfg.CacheExists("meta.jsons", c, isMetaFile, config.MetafileExistsTTL, config.MetafileDoesntExistTTL) groupcacheCfg.CacheGet("meta.jsons", c, isMetaFile, int(config.MetafileMaxSize), config.MetafileContentTTL, config.MetafileExistsTTL, config.MetafileDoesntExistTTL) From 57b1ab565e641e02312e7e8e84531c6cfb71b7a5 Mon Sep 17 00:00:00 2001 From: akanshat Date: Wed, 5 Jan 2022 01:55:55 +0530 Subject: [PATCH 17/22] remove duplicate CachingBucketConfig Signed-off-by: akanshat --- pkg/cache/caching_bucket_config.go | 29 +++++++++++++++++++++++ pkg/store/cache/caching_bucket_factory.go | 20 ++++++++-------- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/pkg/cache/caching_bucket_config.go b/pkg/cache/caching_bucket_config.go index 46cb584036..d8a22db297 100644 --- a/pkg/cache/caching_bucket_config.go +++ b/pkg/cache/caching_bucket_config.go @@ -36,6 +36,35 @@ func NewCachingBucketConfig() *CachingBucketConfig { } } +// SetCacheImplementation sets the value of Cache for all configurations. +func (cfg *CachingBucketConfig) SetCacheImplementation(c Cache) { + if cfg.get != nil { + for k := range cfg.get { + cfg.get[k].Cache = c + } + } + if cfg.iter != nil { + for k := range cfg.iter { + cfg.iter[k].Cache = c + } + } + if cfg.exists != nil { + for k := range cfg.exists { + cfg.exists[k].Cache = c + } + } + if cfg.getRange != nil { + for k := range cfg.getRange { + cfg.getRange[k].Cache = c + } + } + if cfg.attributes != nil { + for k := range cfg.attributes { + cfg.attributes[k].Cache = c + } + } +} + // Generic config for single operation. type OperationConfig struct { Matcher func(name string) bool diff --git a/pkg/store/cache/caching_bucket_factory.go b/pkg/store/cache/caching_bucket_factory.go index b00d3d9e4b..d22b31db43 100644 --- a/pkg/store/cache/caching_bucket_factory.go +++ b/pkg/store/cache/caching_bucket_factory.go @@ -87,6 +87,7 @@ func NewCachingBucketFromYaml(yamlContent []byte, bucket objstore.Bucket, logger } var c cache.Cache + cfg := cache.NewCachingBucketConfig() switch strings.ToUpper(string(config.Type)) { case string(MemcachedBucketCacheProvider): @@ -104,18 +105,17 @@ func NewCachingBucketFromYaml(yamlContent []byte, bucket objstore.Bucket, logger case string(GroupcacheBucketCacheProvider): const basePath = "/_galaxycache/" - groupcacheCfg := cache.NewCachingBucketConfig() - + cfg.SetCacheImplementation(c) // Configure cache. - groupcacheCfg.CacheAttributes("chunks", c, isTSDBChunkFile, config.ChunkObjectAttrsTTL) - groupcacheCfg.CacheGetRange("chunks", c, isTSDBChunkFile, config.ChunkSubrangeSize, config.ChunkObjectAttrsTTL, config.ChunkSubrangeTTL, config.MaxChunksGetRangeRequests) - groupcacheCfg.CacheExists("meta.jsons", c, isMetaFile, config.MetafileExistsTTL, config.MetafileDoesntExistTTL) - groupcacheCfg.CacheGet("meta.jsons", c, isMetaFile, int(config.MetafileMaxSize), config.MetafileContentTTL, config.MetafileExistsTTL, config.MetafileDoesntExistTTL) + cfg.CacheAttributes("chunks", c, isTSDBChunkFile, config.ChunkObjectAttrsTTL) + cfg.CacheGetRange("chunks", c, isTSDBChunkFile, config.ChunkSubrangeSize, config.ChunkObjectAttrsTTL, config.ChunkSubrangeTTL, config.MaxChunksGetRangeRequests) + cfg.CacheExists("meta.jsons", c, isMetaFile, config.MetafileExistsTTL, config.MetafileDoesntExistTTL) + cfg.CacheGet("meta.jsons", c, isMetaFile, int(config.MetafileMaxSize), config.MetafileContentTTL, config.MetafileExistsTTL, config.MetafileDoesntExistTTL) // Cache Iter requests for root. - groupcacheCfg.CacheIter("blocks-iter", c, isBlocksRootDir, config.BlocksIterTTL, JSONIterCodec{}) + cfg.CacheIter("blocks-iter", c, isBlocksRootDir, config.BlocksIterTTL, JSONIterCodec{}) - c, err = cache.NewGroupcache(logger, reg, backendConfig, basePath, r, bucket, groupcacheCfg) + c, err = cache.NewGroupcache(logger, reg, backendConfig, basePath, r, bucket, cfg) if err != nil { return nil, errors.Wrap(err, "failed to create groupcache") } @@ -132,8 +132,8 @@ func NewCachingBucketFromYaml(yamlContent []byte, bucket objstore.Bucket, logger // Include interactions with cache in the traces. c = cache.NewTracingCache(c) - cfg := cache.NewCachingBucketConfig() - + cfg = cache.NewCachingBucketConfig() + cfg.SetCacheImplementation(c) // Configure cache. cfg.CacheGetRange("chunks", c, isTSDBChunkFile, config.ChunkSubrangeSize, config.ChunkObjectAttrsTTL, config.ChunkSubrangeTTL, config.MaxChunksGetRangeRequests) cfg.CacheExists("meta.jsons", c, isMetaFile, config.MetafileExistsTTL, config.MetafileDoesntExistTTL) From 13eaa616e211d97ec908b6956c43a9e11d4af193 Mon Sep 17 00:00:00 2001 From: akanshat Date: Wed, 5 Jan 2022 02:02:00 +0530 Subject: [PATCH 18/22] remove duplicate calls to NewCachingBucketConfig Signed-off-by: akanshat --- pkg/store/cache/caching_bucket_factory.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/store/cache/caching_bucket_factory.go b/pkg/store/cache/caching_bucket_factory.go index d22b31db43..96bc69fa29 100644 --- a/pkg/store/cache/caching_bucket_factory.go +++ b/pkg/store/cache/caching_bucket_factory.go @@ -132,7 +132,6 @@ func NewCachingBucketFromYaml(yamlContent []byte, bucket objstore.Bucket, logger // Include interactions with cache in the traces. c = cache.NewTracingCache(c) - cfg = cache.NewCachingBucketConfig() cfg.SetCacheImplementation(c) // Configure cache. cfg.CacheGetRange("chunks", c, isTSDBChunkFile, config.ChunkSubrangeSize, config.ChunkObjectAttrsTTL, config.ChunkSubrangeTTL, config.MaxChunksGetRangeRequests) From 1e595624209b7000282713cad5f745f7f3dd6d27 Mon Sep 17 00:00:00 2001 From: akanshat Date: Wed, 5 Jan 2022 02:12:36 +0530 Subject: [PATCH 19/22] refactor cache configuration Signed-off-by: akanshat --- pkg/store/cache/caching_bucket_factory.go | 24 +++++++++-------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/pkg/store/cache/caching_bucket_factory.go b/pkg/store/cache/caching_bucket_factory.go index 96bc69fa29..ad3a01989f 100644 --- a/pkg/store/cache/caching_bucket_factory.go +++ b/pkg/store/cache/caching_bucket_factory.go @@ -89,6 +89,15 @@ func NewCachingBucketFromYaml(yamlContent []byte, bucket objstore.Bucket, logger var c cache.Cache cfg := cache.NewCachingBucketConfig() + // Configure cache. + cfg.CacheAttributes("chunks", c, isTSDBChunkFile, config.ChunkObjectAttrsTTL) + cfg.CacheGetRange("chunks", c, isTSDBChunkFile, config.ChunkSubrangeSize, config.ChunkObjectAttrsTTL, config.ChunkSubrangeTTL, config.MaxChunksGetRangeRequests) + cfg.CacheExists("meta.jsons", c, isMetaFile, config.MetafileExistsTTL, config.MetafileDoesntExistTTL) + cfg.CacheGet("meta.jsons", c, isMetaFile, int(config.MetafileMaxSize), config.MetafileContentTTL, config.MetafileExistsTTL, config.MetafileDoesntExistTTL) + + // Cache Iter requests for root. + cfg.CacheIter("blocks-iter", c, isBlocksRootDir, config.BlocksIterTTL, JSONIterCodec{}) + switch strings.ToUpper(string(config.Type)) { case string(MemcachedBucketCacheProvider): var memcached cacheutil.RemoteCacheClient @@ -106,14 +115,6 @@ func NewCachingBucketFromYaml(yamlContent []byte, bucket objstore.Bucket, logger const basePath = "/_galaxycache/" cfg.SetCacheImplementation(c) - // Configure cache. - cfg.CacheAttributes("chunks", c, isTSDBChunkFile, config.ChunkObjectAttrsTTL) - cfg.CacheGetRange("chunks", c, isTSDBChunkFile, config.ChunkSubrangeSize, config.ChunkObjectAttrsTTL, config.ChunkSubrangeTTL, config.MaxChunksGetRangeRequests) - cfg.CacheExists("meta.jsons", c, isMetaFile, config.MetafileExistsTTL, config.MetafileDoesntExistTTL) - cfg.CacheGet("meta.jsons", c, isMetaFile, int(config.MetafileMaxSize), config.MetafileContentTTL, config.MetafileExistsTTL, config.MetafileDoesntExistTTL) - - // Cache Iter requests for root. - cfg.CacheIter("blocks-iter", c, isBlocksRootDir, config.BlocksIterTTL, JSONIterCodec{}) c, err = cache.NewGroupcache(logger, reg, backendConfig, basePath, r, bucket, cfg) if err != nil { @@ -133,13 +134,6 @@ func NewCachingBucketFromYaml(yamlContent []byte, bucket objstore.Bucket, logger // Include interactions with cache in the traces. c = cache.NewTracingCache(c) cfg.SetCacheImplementation(c) - // Configure cache. - cfg.CacheGetRange("chunks", c, isTSDBChunkFile, config.ChunkSubrangeSize, config.ChunkObjectAttrsTTL, config.ChunkSubrangeTTL, config.MaxChunksGetRangeRequests) - cfg.CacheExists("meta.jsons", c, isMetaFile, config.MetafileExistsTTL, config.MetafileDoesntExistTTL) - cfg.CacheGet("meta.jsons", c, isMetaFile, int(config.MetafileMaxSize), config.MetafileContentTTL, config.MetafileExistsTTL, config.MetafileDoesntExistTTL) - - // Cache Iter requests for root. - cfg.CacheIter("blocks-iter", c, isBlocksRootDir, config.BlocksIterTTL, JSONIterCodec{}) cb, err := NewCachingBucket(bucket, cfg, logger, reg) if err != nil { From 5f2f7cfa3760543a0a432207d6b967ab2db986f8 Mon Sep 17 00:00:00 2001 From: akanshat Date: Thu, 6 Jan 2022 17:54:00 +0530 Subject: [PATCH 20/22] implement suggestions from comments Signed-off-by: akanshat --- pkg/store/cache/caching_bucket_factory.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/pkg/store/cache/caching_bucket_factory.go b/pkg/store/cache/caching_bucket_factory.go index ad3a01989f..bf75632d72 100644 --- a/pkg/store/cache/caching_bucket_factory.go +++ b/pkg/store/cache/caching_bucket_factory.go @@ -89,14 +89,14 @@ func NewCachingBucketFromYaml(yamlContent []byte, bucket objstore.Bucket, logger var c cache.Cache cfg := cache.NewCachingBucketConfig() - // Configure cache. - cfg.CacheAttributes("chunks", c, isTSDBChunkFile, config.ChunkObjectAttrsTTL) - cfg.CacheGetRange("chunks", c, isTSDBChunkFile, config.ChunkSubrangeSize, config.ChunkObjectAttrsTTL, config.ChunkSubrangeTTL, config.MaxChunksGetRangeRequests) - cfg.CacheExists("meta.jsons", c, isMetaFile, config.MetafileExistsTTL, config.MetafileDoesntExistTTL) - cfg.CacheGet("meta.jsons", c, isMetaFile, int(config.MetafileMaxSize), config.MetafileContentTTL, config.MetafileExistsTTL, config.MetafileDoesntExistTTL) + // Configure cache paths. + cfg.CacheAttributes("chunks", nil, isTSDBChunkFile, config.ChunkObjectAttrsTTL) + cfg.CacheGetRange("chunks", nil, isTSDBChunkFile, config.ChunkSubrangeSize, config.ChunkObjectAttrsTTL, config.ChunkSubrangeTTL, config.MaxChunksGetRangeRequests) + cfg.CacheExists("meta.jsons", nil, isMetaFile, config.MetafileExistsTTL, config.MetafileDoesntExistTTL) + cfg.CacheGet("meta.jsons", nil, isMetaFile, int(config.MetafileMaxSize), config.MetafileContentTTL, config.MetafileExistsTTL, config.MetafileDoesntExistTTL) // Cache Iter requests for root. - cfg.CacheIter("blocks-iter", c, isBlocksRootDir, config.BlocksIterTTL, JSONIterCodec{}) + cfg.CacheIter("blocks-iter", nil, isBlocksRootDir, config.BlocksIterTTL, JSONIterCodec{}) switch strings.ToUpper(string(config.Type)) { case string(MemcachedBucketCacheProvider): @@ -114,8 +114,6 @@ func NewCachingBucketFromYaml(yamlContent []byte, bucket objstore.Bucket, logger case string(GroupcacheBucketCacheProvider): const basePath = "/_galaxycache/" - cfg.SetCacheImplementation(c) - c, err = cache.NewGroupcache(logger, reg, backendConfig, basePath, r, bucket, cfg) if err != nil { return nil, errors.Wrap(err, "failed to create groupcache") From d61e97cccbd781f5eaa74d5427023455efaae8fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Thu, 6 Jan 2022 22:07:03 +0200 Subject: [PATCH 21/22] *: formatting changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Giedrius Statkevičius --- go.mod | 1 - scripts/quickstart.sh | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 2332ae8e63..2f499e141d 100644 --- a/go.mod +++ b/go.mod @@ -122,7 +122,6 @@ require ( github.com/envoyproxy/go-control-plane v0.10.1 // indirect github.com/envoyproxy/protoc-gen-validate v0.6.2 // indirect github.com/felixge/httpsnoop v1.0.1 // indirect - github.com/go-kit/kit v0.12.0 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/go-openapi/analysis v0.20.0 // indirect github.com/go-openapi/errors v0.20.0 // indirect diff --git a/scripts/quickstart.sh b/scripts/quickstart.sh index 79f448b791..56e41a3313 100755 --- a/scripts/quickstart.sh +++ b/scripts/quickstart.sh @@ -166,7 +166,7 @@ done sleep 0.5 if [ -n "${GCS_BUCKET}" -o -n "${S3_ENDPOINT}" ]; then -cat >groupcache.yml <<-EOF + cat >groupcache.yml <<-EOF type: GROUPCACHE config: self_url: http://localhost:10906/ From 16ac6fd481b56fea6553547f11022d48a6fcc83c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Thu, 6 Jan 2022 22:14:15 +0200 Subject: [PATCH 22/22] *: linter fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Giedrius Statkevičius --- go.mod | 2 +- pkg/cache/groupcache.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2f499e141d..17f70129da 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/felixge/fgprof v0.9.1 github.com/fortytw2/leaktest v1.3.0 github.com/fsnotify/fsnotify v1.5.1 - github.com/go-kit/kit v0.12.0 + github.com/go-kit/kit v0.12.0 // indirect github.com/go-kit/log v0.2.0 github.com/go-openapi/strfmt v0.21.0 github.com/go-redis/redis/v8 v8.11.4 diff --git a/pkg/cache/groupcache.go b/pkg/cache/groupcache.go index 59f8c5b40a..0bbf596755 100644 --- a/pkg/cache/groupcache.go +++ b/pkg/cache/groupcache.go @@ -11,8 +11,8 @@ import ( "strconv" "time" - "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" + "github.com/go-kit/log" + "github.com/go-kit/log/level" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/route" "github.com/thanos-io/thanos/pkg/discovery/dns"