diff --git a/CHANGELOG.md b/CHANGELOG.md index cf1de0fec5..40eeeab685 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,10 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#1070](https://github.com/improbable-eng/thanos/pull/1070) Downsampling works back again. Deferred closer errors are now properly captured. +### Changed + +- [#1073](https://github.com/improbable-eng/thanos/pull/1073) Store: index cache for requests. It now calculates the size properly (includes slice header), has anti-deadlock safeguard and reports more metrics. + ## [v0.4.0-rc.0](https://github.com/improbable-eng/thanos/releases/tag/v0.4.0-rc.0) - 2019.04.18 :warning: **IMPORTANT** :warning: This is the last release that supports gossip. From Thanos v0.5.0, gossip will be completely removed. diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index b75e16e58c..24b80ec5fa 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -26,6 +26,7 @@ import ( "github.com/improbable-eng/thanos/pkg/objstore" "github.com/improbable-eng/thanos/pkg/pool" "github.com/improbable-eng/thanos/pkg/runutil" + storecache "github.com/improbable-eng/thanos/pkg/store/cache" "github.com/improbable-eng/thanos/pkg/store/storepb" "github.com/improbable-eng/thanos/pkg/strutil" "github.com/improbable-eng/thanos/pkg/tracing" @@ -182,7 +183,7 @@ type BucketStore struct { metrics *bucketStoreMetrics bucket objstore.BucketReader dir string - indexCache *indexCache + indexCache *storecache.IndexCache chunkPool *pool.BytesPool // Sets of blocks that have the same labels. They are indexed by a hash over their label set. @@ -225,10 +226,17 @@ func NewBucketStore( return nil, errors.Errorf("max concurrency value cannot be lower than 0 (got %v)", maxConcurrent) } - indexCache, err := newIndexCache(reg, indexCacheSizeBytes) + // TODO(bwplotka): Add as a flag? + maxItemSizeBytes := indexCacheSizeBytes / 2 + + indexCache, err := storecache.NewIndexCache(logger, reg, storecache.Opts{ + MaxSizeBytes: indexCacheSizeBytes, + MaxItemSizeBytes: maxItemSizeBytes, + }) if err != nil { return nil, errors.Wrap(err, "create index cache") } + chunkPool, err := pool.NewBytesPool(2e5, 50e6, 2, maxChunkPoolBytes) if err != nil { return nil, errors.Wrap(err, "create chunk pool") @@ -1058,7 +1066,7 @@ type bucketBlock struct { bucket objstore.BucketReader meta *metadata.Meta dir string - indexCache *indexCache + indexCache *storecache.IndexCache chunkPool *pool.BytesPool indexVersion int @@ -1081,7 +1089,7 @@ func newBucketBlock( bkt objstore.BucketReader, id ulid.ULID, dir string, - indexCache *indexCache, + indexCache *storecache.IndexCache, chunkPool *pool.BytesPool, p partitioner, ) (b *bucketBlock, err error) { @@ -1241,13 +1249,13 @@ type bucketIndexReader struct { block *bucketBlock dec *index.Decoder stats *queryStats - cache *indexCache + cache *storecache.IndexCache mtx sync.Mutex loadedSeries map[uint64][]byte } -func newBucketIndexReader(ctx context.Context, logger log.Logger, block *bucketBlock, cache *indexCache) *bucketIndexReader { +func newBucketIndexReader(ctx context.Context, logger log.Logger, block *bucketBlock, cache *storecache.IndexCache) *bucketIndexReader { r := &bucketIndexReader{ logger: logger, ctx: ctx, @@ -1415,7 +1423,7 @@ func (r *bucketIndexReader) fetchPostings(groups []*postingGroup) error { for i, g := range groups { for j, key := range g.keys { // Get postings for the given key from cache first. - if b, ok := r.cache.postings(r.block.meta.ULID, key); ok { + if b, ok := r.cache.Postings(r.block.meta.ULID, key); ok { r.stats.postingsTouched++ r.stats.postingsTouchedSizeSum += len(b) @@ -1487,7 +1495,7 @@ func (r *bucketIndexReader) fetchPostings(groups []*postingGroup) error { // Return postings and fill LRU cache. groups[p.groupID].Fill(p.keyID, fetchedPostings) - r.cache.setPostings(r.block.meta.ULID, groups[p.groupID].keys[p.keyID], c) + r.cache.SetPostings(r.block.meta.ULID, groups[p.groupID].keys[p.keyID], c) // If we just fetched it we still have to update the stats for touched postings. r.stats.postingsTouched++ @@ -1510,7 +1518,7 @@ func (r *bucketIndexReader) PreloadSeries(ids []uint64) error { var newIDs []uint64 for _, id := range ids { - if b, ok := r.cache.series(r.block.meta.ULID, id); ok { + if b, ok := r.cache.Series(r.block.meta.ULID, id); ok { r.loadedSeries[id] = b continue } @@ -1567,7 +1575,7 @@ func (r *bucketIndexReader) loadSeries(ctx context.Context, ids []uint64, start, } c = c[n : n+int(l)] r.loadedSeries[id] = c - r.cache.setSeries(r.block.meta.ULID, id, c) + r.cache.SetSeries(r.block.meta.ULID, id, c) } return nil } diff --git a/pkg/store/bucket_e2e_test.go b/pkg/store/bucket_e2e_test.go index bc8e55e359..02ad79781a 100644 --- a/pkg/store/bucket_e2e_test.go +++ b/pkg/store/bucket_e2e_test.go @@ -15,6 +15,7 @@ import ( "github.com/improbable-eng/thanos/pkg/objstore" "github.com/improbable-eng/thanos/pkg/objstore/objtesting" "github.com/improbable-eng/thanos/pkg/runutil" + storecache "github.com/improbable-eng/thanos/pkg/store/cache" "github.com/improbable-eng/thanos/pkg/store/storepb" "github.com/improbable-eng/thanos/pkg/testutil" "github.com/pkg/errors" @@ -310,7 +311,10 @@ func testBucketStore_e2e(t testing.TB, ctx context.Context, s *storeSuite) { t.Log("Run ", i) // Always clean cache before each test. - s.store.indexCache, err = newIndexCache(nil, 100) + s.store.indexCache, err = storecache.NewIndexCache(log.NewNopLogger(), nil, storecache.Opts{ + MaxSizeBytes: 100, + MaxItemSizeBytes: 100, + }) testutil.Ok(t, err) srv := newStoreSeriesServer(ctx) diff --git a/pkg/store/cache.go b/pkg/store/cache.go deleted file mode 100644 index 3c391fe3c1..0000000000 --- a/pkg/store/cache.go +++ /dev/null @@ -1,216 +0,0 @@ -package store - -import ( - "sync" - - lru "github.com/hashicorp/golang-lru/simplelru" - "github.com/oklog/ulid" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/tsdb/labels" -) - -const ( - cacheTypePostings = "postings" - cacheTypeSeries = "series" -) - -type cacheItem struct { - block ulid.ULID - key interface{} -} - -func (c cacheItem) keyType() string { - switch c.key.(type) { - case cacheKeyPostings: - return cacheTypePostings - case cacheKeySeries: - return cacheTypeSeries - } - return "" -} - -type cacheKeyPostings labels.Label -type cacheKeySeries uint64 - -type indexCache struct { - mtx sync.Mutex - lru *lru.LRU - maxSize uint64 - curSize uint64 - - requests *prometheus.CounterVec - hits *prometheus.CounterVec - added *prometheus.CounterVec - current *prometheus.GaugeVec - currentSize *prometheus.GaugeVec - overflow *prometheus.CounterVec -} - -// newIndexCache creates a new LRU cache for index entries and ensures the total cache -// size approximately does not exceed maxBytes. -func newIndexCache(reg prometheus.Registerer, maxBytes uint64) (*indexCache, error) { - c := &indexCache{ - maxSize: maxBytes, - } - evicted := prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "thanos_store_index_cache_items_evicted_total", - Help: "Total number of items that were evicted from the index cache.", - }, []string{"item_type"}) - evicted.WithLabelValues(cacheTypePostings) - evicted.WithLabelValues(cacheTypeSeries) - - c.added = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "thanos_store_index_cache_items_added_total", - Help: "Total number of items that were added to the index cache.", - }, []string{"item_type"}) - c.added.WithLabelValues(cacheTypePostings) - c.added.WithLabelValues(cacheTypeSeries) - - c.requests = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "thanos_store_index_cache_requests_total", - Help: "Total number of requests to the cache.", - }, []string{"item_type"}) - c.requests.WithLabelValues(cacheTypePostings) - c.requests.WithLabelValues(cacheTypeSeries) - - c.overflow = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "thanos_store_index_cache_items_overflowed_total", - Help: "Total number of items that could not be added to the cache due to being too big.", - }, []string{"item_type"}) - c.overflow.WithLabelValues(cacheTypePostings) - c.overflow.WithLabelValues(cacheTypeSeries) - - c.hits = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "thanos_store_index_cache_hits_total", - Help: "Total number of requests to the cache that were a hit.", - }, []string{"item_type"}) - c.hits.WithLabelValues(cacheTypePostings) - c.hits.WithLabelValues(cacheTypeSeries) - - c.current = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "thanos_store_index_cache_items", - Help: "Current number of items in the index cache.", - }, []string{"item_type"}) - c.current.WithLabelValues(cacheTypePostings) - c.current.WithLabelValues(cacheTypeSeries) - - c.currentSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "thanos_store_index_cache_items_size_bytes", - Help: "Current byte size of items in the index cache.", - }, []string{"item_type"}) - c.currentSize.WithLabelValues(cacheTypePostings) - c.currentSize.WithLabelValues(cacheTypeSeries) - - // Initialize LRU cache with a high size limit since we will manage evictions ourselves - // based on stored size. - onEvict := func(key, val interface{}) { - k := key.(cacheItem).keyType() - v := val.([]byte) - - evicted.WithLabelValues(k).Inc() - c.current.WithLabelValues(k).Dec() - c.currentSize.WithLabelValues(k).Sub(float64(len(v))) - - c.curSize -= uint64(len(v)) - } - l, err := lru.NewLRU(1e12, onEvict) - if err != nil { - return nil, err - } - c.lru = l - - if reg != nil { - reg.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ - Name: "thanos_store_index_cache_max_size_bytes", - Help: "Maximum number of bytes to be held in the index cache.", - }, func() float64 { - return float64(maxBytes) - })) - reg.MustRegister(c.requests, c.hits, c.added, evicted, c.current, c.currentSize, c.overflow) - } - return c, nil -} - -// ensureFits tries to make sure that the passed slice will fit into the LRU cache. -// Returns true if it will fit. -func (c *indexCache) ensureFits(b []byte) bool { - if uint64(len(b)) > c.maxSize { - return false - } - for c.curSize > c.maxSize-uint64(len(b)) { - c.lru.RemoveOldest() - } - return true -} - -func (c *indexCache) setPostings(b ulid.ULID, l labels.Label, v []byte) { - c.added.WithLabelValues(cacheTypePostings).Inc() - - c.mtx.Lock() - defer c.mtx.Unlock() - - if !c.ensureFits(v) { - c.overflow.WithLabelValues(cacheTypePostings).Inc() - return - } - - // The caller may be passing in a sub-slice of a huge array. Copy the data - // to ensure we don't waste huge amounts of space for something small. - cv := make([]byte, len(v)) - copy(cv, v) - c.lru.Add(cacheItem{b, cacheKeyPostings(l)}, cv) - - c.currentSize.WithLabelValues(cacheTypePostings).Add(float64(len(v))) - c.current.WithLabelValues(cacheTypePostings).Inc() - c.curSize += uint64(len(v)) -} - -func (c *indexCache) postings(b ulid.ULID, l labels.Label) ([]byte, bool) { - c.requests.WithLabelValues(cacheTypePostings).Inc() - - c.mtx.Lock() - defer c.mtx.Unlock() - - v, ok := c.lru.Get(cacheItem{b, cacheKeyPostings(l)}) - if !ok { - return nil, false - } - c.hits.WithLabelValues(cacheTypePostings).Inc() - return v.([]byte), true -} - -func (c *indexCache) setSeries(b ulid.ULID, id uint64, v []byte) { - c.added.WithLabelValues(cacheTypeSeries).Inc() - - c.mtx.Lock() - defer c.mtx.Unlock() - - if !c.ensureFits(v) { - c.overflow.WithLabelValues(cacheTypeSeries).Inc() - return - } - - // The caller may be passing in a sub-slice of a huge array. Copy the data - // to ensure we don't waste huge amounts of space for something small. - cv := make([]byte, len(v)) - copy(cv, v) - c.lru.Add(cacheItem{b, cacheKeySeries(id)}, cv) - - c.currentSize.WithLabelValues(cacheTypeSeries).Add(float64(len(v))) - c.current.WithLabelValues(cacheTypeSeries).Inc() - c.curSize += uint64(len(v)) -} - -func (c *indexCache) series(b ulid.ULID, id uint64) ([]byte, bool) { - c.requests.WithLabelValues(cacheTypeSeries).Inc() - - c.mtx.Lock() - defer c.mtx.Unlock() - - v, ok := c.lru.Get(cacheItem{b, cacheKeySeries(id)}) - if !ok { - return nil, false - } - c.hits.WithLabelValues(cacheTypeSeries).Inc() - return v.([]byte), true -} diff --git a/pkg/store/cache/cache.go b/pkg/store/cache/cache.go new file mode 100644 index 0000000000..9703c93e85 --- /dev/null +++ b/pkg/store/cache/cache.go @@ -0,0 +1,325 @@ +package storecache + +import ( + "math" + "sync" + "time" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + lru "github.com/hashicorp/golang-lru/simplelru" + "github.com/oklog/ulid" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/tsdb/labels" +) + +const ( + cacheTypePostings string = "Postings" + cacheTypeSeries string = "Series" + + sliceHeaderSize = 16 +) + +type cacheItem struct { + block ulid.ULID + key interface{} +} + +func (c cacheItem) keyType() string { + switch c.key.(type) { + case cacheKeyPostings: + return cacheTypePostings + case cacheKeySeries: + return cacheTypeSeries + } + return "" +} + +func (c cacheItem) size() uint64 { + switch k := c.key.(type) { + case cacheKeyPostings: + // ULID + 2 slice headers + number of chars in value and name. + return sliceHeaderSize + 2*16 + uint64(len(k.Value)+len(k.Name)) + case cacheKeySeries: + return sliceHeaderSize + 8 // ULID + uint64 + } + return 0 +} + +type cacheKeyPostings labels.Label +type cacheKeySeries uint64 + +type IndexCache struct { + mtx sync.Mutex + + logger log.Logger + lru *lru.LRU + maxSizeBytes uint64 + maxItemSizeBytes uint64 + setTimeout time.Duration + + curSize uint64 + + evicted *prometheus.CounterVec + requests *prometheus.CounterVec + hits *prometheus.CounterVec + added *prometheus.CounterVec + current *prometheus.GaugeVec + currentSize *prometheus.GaugeVec + totalCurrentSize *prometheus.GaugeVec + overflow *prometheus.CounterVec +} + +type Opts struct { + // MaxSizeBytes represents overall maximum number of bytes cache can contain. + MaxSizeBytes uint64 + // MaxItemSizeBytes represents maximum size of single item. + MaxItemSizeBytes uint64 +} + +// NewIndexCache creates a new thread-safe LRU cache for index entries and ensures the total cache +// size approximately does not exceed maxBytes. +func NewIndexCache(logger log.Logger, reg prometheus.Registerer, opts Opts) (*IndexCache, error) { + if opts.MaxItemSizeBytes > opts.MaxSizeBytes { + return nil, errors.Errorf("max item size (%v) cannot be bigger than overall cache size (%v)", opts.MaxItemSizeBytes, opts.MaxSizeBytes) + } + + c := &IndexCache{ + logger: logger, + maxSizeBytes: opts.MaxSizeBytes, + maxItemSizeBytes: opts.MaxItemSizeBytes, + } + + c.evicted = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "thanos_store_index_cache_items_evicted_total", + Help: "Total number of items that were evicted from the index cache.", + }, []string{"item_type"}) + c.evicted.WithLabelValues(cacheTypePostings) + c.evicted.WithLabelValues(cacheTypeSeries) + + c.added = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "thanos_store_index_cache_items_added_total", + Help: "Total number of items that were added to the index cache.", + }, []string{"item_type"}) + c.added.WithLabelValues(cacheTypePostings) + c.added.WithLabelValues(cacheTypeSeries) + + c.requests = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "thanos_store_index_cache_requests_total", + Help: "Total number of requests to the cache.", + }, []string{"item_type"}) + c.requests.WithLabelValues(cacheTypePostings) + c.requests.WithLabelValues(cacheTypeSeries) + + c.overflow = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "thanos_store_index_cache_items_overflowed_total", + Help: "Total number of items that could not be added to the cache due to being too big.", + }, []string{"item_type"}) + c.overflow.WithLabelValues(cacheTypePostings) + c.overflow.WithLabelValues(cacheTypeSeries) + + c.hits = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "thanos_store_index_cache_hits_total", + Help: "Total number of requests to the cache that were a hit.", + }, []string{"item_type"}) + c.hits.WithLabelValues(cacheTypePostings) + c.hits.WithLabelValues(cacheTypeSeries) + + c.current = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "thanos_store_index_cache_items", + Help: "Current number of items in the index cache.", + }, []string{"item_type"}) + c.current.WithLabelValues(cacheTypePostings) + c.current.WithLabelValues(cacheTypeSeries) + + c.currentSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "thanos_store_index_cache_items_size_bytes", + Help: "Current byte size of items in the index cache.", + }, []string{"item_type"}) + c.currentSize.WithLabelValues(cacheTypePostings) + c.currentSize.WithLabelValues(cacheTypeSeries) + + c.totalCurrentSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "thanos_store_index_cache_total_size_bytes", + Help: "Current byte size of items (both value and key) in the index cache.", + }, []string{"item_type"}) + c.totalCurrentSize.WithLabelValues(cacheTypePostings) + c.totalCurrentSize.WithLabelValues(cacheTypeSeries) + + if reg != nil { + reg.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "thanos_store_index_cache_max_size_bytes", + Help: "Maximum number of bytes to be held in the index cache.", + }, func() float64 { + return float64(c.maxSizeBytes) + })) + reg.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "thanos_store_index_cache_max_item_size_bytes", + Help: "Maximum number of bytes for single entry to be held in the index cache.", + }, func() float64 { + return float64(c.maxItemSizeBytes) + })) + reg.MustRegister(c.requests, c.hits, c.added, c.evicted, c.current, c.currentSize, c.totalCurrentSize, c.overflow) + } + + // Initialize LRU cache with a high size limit since we will manage evictions ourselves + // based on stored size using `RemoveOldest` method. + l, err := lru.NewLRU(math.MaxInt64, c.onEvict) + if err != nil { + return nil, err + } + c.lru = l + + level.Info(logger).Log( + "msg", "created index cache", + "maxItemSizeBytes", c.maxItemSizeBytes, + "maxSizeBytes", c.maxSizeBytes, + "setTimeout", c.setTimeout.String(), + "maxItems", "math.MaxInt64", + ) + return c, nil +} + +func (c *IndexCache) onEvict(key, val interface{}) { + k := key.(cacheItem).keyType() + entrySize := sliceHeaderSize + uint64(len(val.([]byte))) + + c.evicted.WithLabelValues(string(k)).Inc() + c.current.WithLabelValues(string(k)).Dec() + c.currentSize.WithLabelValues(string(k)).Sub(float64(entrySize)) + c.totalCurrentSize.WithLabelValues(string(k)).Sub(float64(entrySize + key.(cacheItem).size())) + + c.curSize -= entrySize +} + +// ensureFits tries to make sure that the passed slice will fit into the LRU cache. +// Returns true if it will fit. +func (c *IndexCache) ensureFits(size uint64, typ string) bool { + const saneMaxIterations = 500 + + if size > c.maxItemSizeBytes { + level.Debug(c.logger).Log( + "msg", "item bigger than maxItemSizeBytes. Ignoring..", + "maxItemSizeBytes", c.maxItemSizeBytes, + "maxSizeBytes", c.maxSizeBytes, + "curSize", c.curSize, + "itemSize", size, + "cacheType", typ, + ) + return false + } + + for i := 0; c.curSize+size > c.maxSizeBytes; i++ { + if i >= saneMaxIterations { + level.Error(c.logger).Log( + "msg", "After max sane iterations of LRU evictions, we still cannot allocate the item. Ignoring.", + "maxItemSizeBytes", c.maxItemSizeBytes, + "maxSizeBytes", c.maxSizeBytes, + "curSize", c.curSize, + "itemSize", size, + "cacheType", typ, + "iterations", i, + ) + return false + } + + _, _, ok := c.lru.RemoveOldest() + if !ok { + level.Error(c.logger).Log( + "msg", "LRU has nothing more to evict, but we still cannot allocate the item. Ignoring.", + "maxItemSizeBytes", c.maxItemSizeBytes, + "maxSizeBytes", c.maxSizeBytes, + "curSize", c.curSize, + "itemSize", size, + "cacheType", typ, + ) + return false + } + } + return true +} + +func (c *IndexCache) SetPostings(b ulid.ULID, l labels.Label, v []byte) { + var ( + entrySize = sliceHeaderSize + uint64(len(v)) + cacheType = cacheTypePostings + ) + + c.mtx.Lock() + defer c.mtx.Unlock() + + if !c.ensureFits(entrySize, cacheType) { + c.overflow.WithLabelValues(cacheType).Inc() + return + } + + // The caller may be passing in a sub-slice of a huge array. Copy the data + // to ensure we don't waste huge amounts of space for something small. + cv := make([]byte, len(v)) + copy(cv, v) + item := cacheItem{b, cacheKeyPostings(l)} + c.lru.Add(item, cv) + + c.added.WithLabelValues(cacheType).Inc() + c.currentSize.WithLabelValues(cacheType).Add(float64(entrySize)) + c.totalCurrentSize.WithLabelValues(cacheType).Add(float64(entrySize + item.size())) + c.current.WithLabelValues(cacheType).Inc() + c.curSize += entrySize +} + +func (c *IndexCache) Postings(b ulid.ULID, l labels.Label) ([]byte, bool) { + c.requests.WithLabelValues(cacheTypePostings).Inc() + + c.mtx.Lock() + defer c.mtx.Unlock() + + v, ok := c.lru.Get(cacheItem{b, cacheKeyPostings(l)}) + if !ok { + return nil, false + } + c.hits.WithLabelValues(cacheTypePostings).Inc() + return v.([]byte), true +} + +func (c *IndexCache) SetSeries(b ulid.ULID, id uint64, v []byte) { + var ( + entrySize = 16 + uint64(len(v)) // Slice header + bytes. + cacheType = cacheTypeSeries + ) + + c.mtx.Lock() + defer c.mtx.Unlock() + + if !c.ensureFits(entrySize, cacheType) { + c.overflow.WithLabelValues(cacheType).Inc() + return + } + + // The caller may be passing in a sub-slice of a huge array. Copy the data + // to ensure we don't waste huge amounts of space for something small. + cv := make([]byte, len(v)) + copy(cv, v) + item := cacheItem{b, cacheKeySeries(id)} + c.lru.Add(item, cv) + + c.added.WithLabelValues(cacheType).Inc() + c.currentSize.WithLabelValues(cacheType).Add(float64(entrySize)) + c.totalCurrentSize.WithLabelValues(cacheType).Add(float64(entrySize + item.size())) + c.current.WithLabelValues(cacheType).Inc() + c.curSize += entrySize +} + +func (c *IndexCache) Series(b ulid.ULID, id uint64) ([]byte, bool) { + c.requests.WithLabelValues(cacheTypeSeries).Inc() + + c.mtx.Lock() + defer c.mtx.Unlock() + + v, ok := c.lru.Get(cacheItem{b, cacheKeySeries(id)}) + if !ok { + return nil, false + } + c.hits.WithLabelValues(cacheTypeSeries).Inc() + return v.([]byte), true +} diff --git a/pkg/store/cache/cache_test.go b/pkg/store/cache/cache_test.go new file mode 100644 index 0000000000..e5fe113319 --- /dev/null +++ b/pkg/store/cache/cache_test.go @@ -0,0 +1,273 @@ +// Tests out the index cache implementation. +package storecache + +import ( + "math" + "testing" + "time" + + "github.com/fortytw2/leaktest" + "github.com/go-kit/kit/log" + "github.com/hashicorp/golang-lru/simplelru" + "github.com/improbable-eng/thanos/pkg/testutil" + "github.com/oklog/ulid" + "github.com/prometheus/client_golang/prometheus" + promtest "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/prometheus/tsdb/labels" +) + +func TestIndexCache_AvoidsDeadlock(t *testing.T) { + defer leaktest.CheckTimeout(t, 10*time.Second)() + + metrics := prometheus.NewRegistry() + cache, err := NewIndexCache(log.NewNopLogger(), metrics, Opts{ + MaxItemSizeBytes: sliceHeaderSize + 5, + MaxSizeBytes: sliceHeaderSize + 5, + }) + testutil.Ok(t, err) + + l, err := simplelru.NewLRU(math.MaxInt64, func(key, val interface{}) { + cache.onEvict(key, val) + + // We hack LRU to add back entry on eviction to simulate broken evictions. + cache.lru.Add(key, val) + cache.curSize += sliceHeaderSize + uint64(len(val.([]byte))) // Slice header + bytes. + }) + testutil.Ok(t, err) + cache.lru = l + + cache.SetPostings(ulid.MustNew(0, nil), labels.Label{Name: "test2", Value: "1"}, []byte{42, 33, 14, 67, 11}) + + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.overflow.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.overflow.WithLabelValues(cacheTypeSeries))) + + // This triggers deadlock logic. + cache.SetPostings(ulid.MustNew(0, nil), labels.Label{Name: "test1", Value: "1"}, []byte{42}) + + testutil.Equals(t, float64(1), promtest.ToFloat64(cache.overflow.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.overflow.WithLabelValues(cacheTypeSeries))) +} + +// This should not happen as we hardcode math.MaxInt, but we still add test to check this out. +func TestIndexCache_MaxNumberOfItemsHit(t *testing.T) { + defer leaktest.CheckTimeout(t, 10*time.Second)() + + metrics := prometheus.NewRegistry() + cache, err := NewIndexCache(log.NewNopLogger(), metrics, Opts{ + MaxItemSizeBytes: 2*sliceHeaderSize + 10, + MaxSizeBytes: 2*sliceHeaderSize + 10, + }) + testutil.Ok(t, err) + + l, err := simplelru.NewLRU(2, cache.onEvict) + testutil.Ok(t, err) + cache.lru = l + + id := ulid.MustNew(0, nil) + + cache.SetPostings(id, labels.Label{Name: "test", Value: "123"}, []byte{42, 33}) + cache.SetPostings(id, labels.Label{Name: "test", Value: "124"}, []byte{42, 33}) + cache.SetPostings(id, labels.Label{Name: "test", Value: "125"}, []byte{42, 33}) + + testutil.Equals(t, uint64(2*sliceHeaderSize+4), cache.curSize) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.overflow.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.overflow.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(1), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(3), promtest.ToFloat64(cache.added.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.added.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.requests.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.requests.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.hits.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.hits.WithLabelValues(cacheTypeSeries))) +} + +func TestIndexCache_Eviction_WithMetrics(t *testing.T) { + defer leaktest.CheckTimeout(t, 10*time.Second)() + + metrics := prometheus.NewRegistry() + cache, err := NewIndexCache(log.NewNopLogger(), metrics, Opts{ + MaxItemSizeBytes: 2*sliceHeaderSize + 5, + MaxSizeBytes: 2*sliceHeaderSize + 5, + }) + testutil.Ok(t, err) + + id := ulid.MustNew(0, nil) + lbls := labels.Label{Name: "test", Value: "123"} + + _, ok := cache.Postings(id, lbls) + testutil.Assert(t, !ok, "no such key") + + // Add sliceHeaderSize + 2 bytes. + cache.SetPostings(id, lbls, []byte{42, 33}) + testutil.Equals(t, uint64(sliceHeaderSize+2), cache.curSize) + testutil.Equals(t, float64(1), promtest.ToFloat64(cache.current.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(sliceHeaderSize+2), promtest.ToFloat64(cache.currentSize.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(sliceHeaderSize+2+55), promtest.ToFloat64(cache.totalCurrentSize.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.current.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.currentSize.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.totalCurrentSize.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.overflow.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.overflow.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypeSeries))) + + p, ok := cache.Postings(id, lbls) + testutil.Assert(t, ok, "key exists") + testutil.Equals(t, []byte{42, 33}, p) + + _, ok = cache.Postings(ulid.MustNew(1, nil), lbls) + testutil.Assert(t, !ok, "no such key") + _, ok = cache.Postings(id, labels.Label{Name: "test", Value: "124"}) + testutil.Assert(t, !ok, "no such key") + + // Add sliceHeaderSize + 3 more bytes. + cache.SetSeries(id, 1234, []byte{222, 223, 224}) + testutil.Equals(t, uint64(2*sliceHeaderSize+5), cache.curSize) + testutil.Equals(t, float64(1), promtest.ToFloat64(cache.current.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(sliceHeaderSize+2), promtest.ToFloat64(cache.currentSize.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(sliceHeaderSize+2+55), promtest.ToFloat64(cache.totalCurrentSize.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(1), promtest.ToFloat64(cache.current.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(sliceHeaderSize+3), promtest.ToFloat64(cache.currentSize.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(sliceHeaderSize+3+24), promtest.ToFloat64(cache.totalCurrentSize.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.overflow.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.overflow.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypeSeries))) + + p, ok = cache.Series(id, 1234) + testutil.Assert(t, ok, "key exists") + testutil.Equals(t, []byte{222, 223, 224}, p) + + lbls2 := labels.Label{Name: "test", Value: "124"} + + // Add sliceHeaderSize + 5 + 16 bytes, should fully evict 2 last items. + v := []byte{42, 33, 14, 67, 11} + for i := 0; i < sliceHeaderSize; i++ { + v = append(v, 3) + } + cache.SetPostings(id, lbls2, v) + + testutil.Equals(t, uint64(2*sliceHeaderSize+5), cache.curSize) + testutil.Equals(t, float64(1), promtest.ToFloat64(cache.current.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(2*sliceHeaderSize+5), promtest.ToFloat64(cache.currentSize.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(2*sliceHeaderSize+5+55), promtest.ToFloat64(cache.totalCurrentSize.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.current.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.currentSize.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.totalCurrentSize.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.overflow.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.overflow.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(1), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypePostings))) // Eviction. + testutil.Equals(t, float64(1), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypeSeries))) // Eviction. + + // Evicted. + _, ok = cache.Postings(id, lbls) + testutil.Assert(t, !ok, "no such key") + _, ok = cache.Series(id, 1234) + testutil.Assert(t, !ok, "no such key") + + p, ok = cache.Postings(id, lbls2) + testutil.Assert(t, ok, "key exists") + testutil.Equals(t, v, p) + + // Add same item again. + // NOTE: In our caller code, we always check first hit, then we claim miss and set posting so this should not happen. + // That's why this case is not optimized and we evict + re add the item. + cache.SetPostings(id, lbls2, v) + + testutil.Equals(t, uint64(2*sliceHeaderSize+5), cache.curSize) + testutil.Equals(t, float64(1), promtest.ToFloat64(cache.current.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(2*sliceHeaderSize+5), promtest.ToFloat64(cache.currentSize.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(2*sliceHeaderSize+5+55), promtest.ToFloat64(cache.totalCurrentSize.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.current.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.currentSize.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.totalCurrentSize.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.overflow.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.overflow.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(2), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypePostings))) // Eviction. + testutil.Equals(t, float64(1), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypeSeries))) + + p, ok = cache.Postings(id, lbls2) + testutil.Assert(t, ok, "key exists") + testutil.Equals(t, v, p) + + // Add too big item. + cache.SetPostings(id, labels.Label{Name: "test", Value: "toobig"}, append(v, 5)) + testutil.Equals(t, uint64(2*sliceHeaderSize+5), cache.curSize) + testutil.Equals(t, float64(1), promtest.ToFloat64(cache.current.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(2*sliceHeaderSize+5), promtest.ToFloat64(cache.currentSize.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(2*sliceHeaderSize+5+55), promtest.ToFloat64(cache.totalCurrentSize.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.current.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.currentSize.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.totalCurrentSize.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(1), promtest.ToFloat64(cache.overflow.WithLabelValues(cacheTypePostings))) // Overflow. + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.overflow.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(2), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(1), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypeSeries))) + + _, _, ok = cache.lru.RemoveOldest() + testutil.Assert(t, ok, "something to remove") + + testutil.Equals(t, uint64(0), cache.curSize) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.current.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.currentSize.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.totalCurrentSize.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.current.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.currentSize.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.totalCurrentSize.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(1), promtest.ToFloat64(cache.overflow.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.overflow.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(3), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(1), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypeSeries))) + + _, _, ok = cache.lru.RemoveOldest() + testutil.Assert(t, !ok, "nothing to remove") + + lbls3 := labels.Label{Name: "test", Value: "124"} + + cache.SetPostings(id, lbls3, []byte{}) + + testutil.Equals(t, uint64(sliceHeaderSize), cache.curSize) + testutil.Equals(t, float64(1), promtest.ToFloat64(cache.current.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(sliceHeaderSize), promtest.ToFloat64(cache.currentSize.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(sliceHeaderSize+55), promtest.ToFloat64(cache.totalCurrentSize.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.current.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.currentSize.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.totalCurrentSize.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(1), promtest.ToFloat64(cache.overflow.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.overflow.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(3), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(1), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypeSeries))) + + p, ok = cache.Postings(id, lbls3) + testutil.Assert(t, ok, "key exists") + testutil.Equals(t, []byte{}, p) + + // nil works and still allocates empty slice. + lbls4 := labels.Label{Name: "test", Value: "125"} + cache.SetPostings(id, lbls4, []byte(nil)) + + testutil.Equals(t, 2*uint64(sliceHeaderSize), cache.curSize) + testutil.Equals(t, float64(2), promtest.ToFloat64(cache.current.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, 2*float64(sliceHeaderSize), promtest.ToFloat64(cache.currentSize.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, 2*float64(sliceHeaderSize+55), promtest.ToFloat64(cache.totalCurrentSize.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.current.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.currentSize.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.totalCurrentSize.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(1), promtest.ToFloat64(cache.overflow.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(0), promtest.ToFloat64(cache.overflow.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(3), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(1), promtest.ToFloat64(cache.evicted.WithLabelValues(cacheTypeSeries))) + + p, ok = cache.Postings(id, lbls4) + testutil.Assert(t, ok, "key exists") + testutil.Equals(t, []byte{}, p) + + // Other metrics. + testutil.Equals(t, float64(5), promtest.ToFloat64(cache.added.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(1), promtest.ToFloat64(cache.added.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(9), promtest.ToFloat64(cache.requests.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(2), promtest.ToFloat64(cache.requests.WithLabelValues(cacheTypeSeries))) + testutil.Equals(t, float64(5), promtest.ToFloat64(cache.hits.WithLabelValues(cacheTypePostings))) + testutil.Equals(t, float64(1), promtest.ToFloat64(cache.hits.WithLabelValues(cacheTypeSeries))) +} diff --git a/pkg/store/cache_test.go b/pkg/store/cache_test.go deleted file mode 100644 index b33931fd7a..0000000000 --- a/pkg/store/cache_test.go +++ /dev/null @@ -1,89 +0,0 @@ -// Tests out the index cache implementation. -package store - -import ( - "testing" - "time" - - "github.com/fortytw2/leaktest" - "github.com/improbable-eng/thanos/pkg/testutil" - "github.com/oklog/ulid" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/tsdb/labels" -) - -// TestIndexCacheEdge tests the index cache edge cases. -func TestIndexCacheEdge(t *testing.T) { - metrics := prometheus.NewRegistry() - cache, err := newIndexCache(metrics, 1) - testutil.Ok(t, err) - - fits := cache.ensureFits([]byte{42, 24}) - testutil.Equals(t, fits, false) - - fits = cache.ensureFits([]byte{42}) - testutil.Equals(t, fits, true) - - fits = cache.ensureFits([]byte{}) - testutil.Equals(t, fits, true) - - fits = cache.ensureFits([]byte{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}) - testutil.Equals(t, fits, false) - - metrics = prometheus.NewRegistry() - cache, err = newIndexCache(metrics, 0) - testutil.Ok(t, err) - - fits = cache.ensureFits([]byte{42, 24}) - testutil.Equals(t, fits, false) - - fits = cache.ensureFits([]byte{42}) - testutil.Equals(t, fits, false) - - fits = cache.ensureFits([]byte{}) - testutil.Equals(t, fits, true) - - fits = cache.ensureFits([]byte{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}) - testutil.Equals(t, fits, false) -} - -// TestIndexCacheSmoke runs the smoke tests for the index cache. -func TestIndexCacheSmoke(t *testing.T) { - defer leaktest.CheckTimeout(t, 10*time.Second)() - - metrics := prometheus.NewRegistry() - cache, err := newIndexCache(metrics, 20) - testutil.Ok(t, err) - - blid := ulid.ULID([16]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}) - labels := labels.Label{Name: "test", Value: "123"} - - cache.setPostings(blid, labels, []byte{42}) - - p, ok := cache.postings(blid, labels) - testutil.Equals(t, ok, true) - testutil.Equals(t, p, []byte{42}) - testutil.Equals(t, cache.curSize, uint64(1)) - - cache.setSeries(blid, 1234, []byte{42, 42}) - - s, ok := cache.series(blid, 1234) - testutil.Equals(t, ok, true) - testutil.Equals(t, s, []byte{42, 42}) - testutil.Equals(t, cache.curSize, uint64(3)) - - cache.lru.RemoveOldest() - testutil.Equals(t, cache.curSize, uint64(2)) - - cache.lru.RemoveOldest() - testutil.Equals(t, cache.curSize, uint64(0)) - - cache.lru.RemoveOldest() - testutil.Equals(t, cache.curSize, uint64(0)) - - cache.setSeries(blid, 1234, []byte{1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1}) - cache.setSeries(blid, 1237, []byte{1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1}) - cache.setSeries(blid, 1235, []byte{1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 5}) - testutil.Equals(t, cache.curSize, uint64(20)) - -}