metrics: more granular block cache metrics

We improve the cache metrics in two ways:
 - we break down the misses by the type of block (for the most
   important block types)
 - we also show miss rates across the last 10 minutes and the last
   hour

In follow-up work, I will attempt to plumb the LSM level as well.

Author: RaduBerinde

db_test.go

@@ -858,7 +858,7 @@ func TestMemTableReservation(t *testing.T) {
 		t.Fatalf("expected 2 refs, but found %d", refs)
 	}
 	// Verify the memtable reservation has caused our test block to be evicted.
-	if cv := tmpHandle.Peek(base.DiskFileNum(0), 0); cv != nil {
+	if cv := tmpHandle.Peek(base.DiskFileNum(0), 0, cache.CategoryBackground); cv != nil {
 		t.Fatalf("expected failure, but found success: %#v", cv)
 	}
 

internal/cache/cache.go

@@ -17,20 +17,9 @@ import (
 
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/internal/metricsutil"
 )
 
-// Metrics holds metrics for the cache.
-type Metrics struct {
-	// The number of bytes inuse by the cache.
-	Size int64
-	// The count of objects (blocks or tables) in the cache.
-	Count int64
-	// The number of cache hits.
-	Hits int64
-	// The number of cache misses.
-	Misses int64
-}
-
 // Cache implements Pebble's sharded block cache. The Clock-PRO algorithm is
 // used for page replacement
 // (http://static.usenix.org/event/usenix05/tech/general/full_papers/jiang/jiang_html/html.html). In
@@ -78,6 +67,8 @@ type Cache struct {
 	idAlloc atomic.Uint64
 	shards  []shard
 
+	metricsWindow *metricsutil.Window[HitsAndMisses]
+
 	// Traces recorded by Cache.trace. Used for debugging.
 	tr struct {
 		sync.Mutex
@@ -136,6 +127,8 @@ func NewWithShards(size int64, shards int) *Cache {
 	for i := range c.shards {
 		c.shards[i].init(size / int64(len(c.shards)))
 	}
+	c.metricsWindow = metricsutil.NewWindow[HitsAndMisses](c.hitsAndMisses)
+	c.metricsWindow.Start()
 
 	// Note: this is a no-op if invariants are disabled or race is enabled.
 	invariants.SetFinalizer(c, func(c *Cache) {
@@ -171,10 +164,19 @@ func (c *Cache) Unref() {
 	case v < 0:
 		panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
 	case v == 0:
-		for i := range c.shards {
-			c.shards[i].Free()
-		}
+		c.destroy()
+	}
+}
+
+func (c *Cache) destroy() {
+	if c.metricsWindow != nil {
+		c.metricsWindow.Stop()
+		c.metricsWindow = nil
+	}
+	for i := range c.shards {
+		c.shards[i].Free()
 	}
+	c.shards = nil
 }
 
 func (c *Cache) NewHandle() *Handle {
@@ -207,21 +209,6 @@ func (c *Cache) Reserve(n int) func() {
 	}
 }
 
-// Metrics returns the metrics for the cache.
-func (c *Cache) Metrics() Metrics {
-	var m Metrics
-	for i := range c.shards {
-		s := &c.shards[i]
-		s.mu.RLock()
-		m.Count += int64(s.blocks.Len())
-		m.Size += s.sizeHot + s.sizeCold
-		s.mu.RUnlock()
-		m.Hits += s.hits.Load()
-		m.Misses += s.misses.Load()
-	}
-	return m
-}
-
 // MaxSize returns the max size of the cache.
 func (c *Cache) MaxSize() int64 {
 	return c.maxSize
@@ -262,16 +249,16 @@ func (c *Handle) Cache() *Cache {
 // Peek retrieves the cache value for the specified file and offset, returning
 // nil if no value is present. Peek does not affect the state of the cache (it
 // does not "count" as an access as far as the cache replacement is concerned).
-func (c *Handle) Peek(fileNum base.DiskFileNum, offset uint64) *Value {
+func (c *Handle) Peek(fileNum base.DiskFileNum, offset uint64, category Category) *Value {
 	k := makeKey(c.id, fileNum, offset)
-	return c.cache.getShard(k).get(k, true /* peekOnly */)
+	return c.cache.getShard(k).get(k, category, true /* peekOnly */)
 }
 
 // Get retrieves the cache value for the specified file and offset, returning
 // nil if no value is present.
-func (c *Handle) Get(fileNum base.DiskFileNum, offset uint64) *Value {
+func (c *Handle) Get(fileNum base.DiskFileNum, offset uint64, category Category) *Value {
 	k := makeKey(c.id, fileNum, offset)
-	return c.cache.getShard(k).get(k, false /* peekOnly */)
+	return c.cache.getShard(k).get(k, category, false /* peekOnly */)
 }
 
 // GetWithReadHandle retrieves the cache value for the specified handleID, fileNum
@@ -298,7 +285,7 @@ func (c *Handle) Get(fileNum base.DiskFileNum, offset uint64) *Value {
 // While waiting, someone else may successfully read the value, which results
 // in a valid Handle being returned. This is a case where cacheHit=false.
 func (c *Handle) GetWithReadHandle(
-	ctx context.Context, fileNum base.DiskFileNum, offset uint64,
+	ctx context.Context, fileNum base.DiskFileNum, offset uint64, category Category,
 ) (
 	cv *Value,
 	rh ReadHandle,
@@ -308,7 +295,7 @@ func (c *Handle) GetWithReadHandle(
 	err error,
 ) {
 	k := makeKey(c.id, fileNum, offset)
-	cv, re := c.cache.getShard(k).getWithReadEntry(k)
+	cv, re := c.cache.getShard(k).getWithReadEntry(k, category)
 	if cv != nil {
 		return cv, ReadHandle{}, 0, 0, true, nil
 	}

internal/cache/cache_test.go

@@ -41,7 +41,7 @@ func TestCache(t *testing.T) {
 		wantHit := fields[1][0] == 'h'
 
 		var hit bool
-		cv := h.Get(base.DiskFileNum(key), 0)
+		cv := h.Get(base.DiskFileNum(key), 0, CategorySSTableData)
 		if cv == nil {
 			cv = Alloc(1)
 			cv.RawBuffer()[0] = fields[0][0]
@@ -81,14 +81,14 @@ func TestCachePeek(t *testing.T) {
 		setTestValue(h, 0, uint64(i), "a", 1)
 	}
 	for i := range size / 2 {
-		v := h.Get(base.DiskFileNum(0), uint64(i))
+		v := h.Get(base.DiskFileNum(0), uint64(i), CategoryBackground)
 		if v == nil {
 			t.Fatalf("expected to find block %d", i)
 		}
 		v.Release()
 	}
 	for i := size / 2; i < size; i++ {
-		v := h.Peek(base.DiskFileNum(0), uint64(i))
+		v := h.Peek(base.DiskFileNum(0), uint64(i), CategoryBackground)
 		if v == nil {
 			t.Fatalf("expected to find block %d", i)
 		}
@@ -100,7 +100,7 @@ func TestCachePeek(t *testing.T) {
 	}
 	// Verify that the Gets still find their values, despite the Peeks.
 	for i := range size / 2 {
-		v := h.Get(base.DiskFileNum(0), uint64(i))
+		v := h.Get(base.DiskFileNum(0), uint64(i), CategoryBackground)
 		if v == nil {
 			t.Fatalf("expected to find block %d", i)
 		}
@@ -124,12 +124,12 @@ func TestCacheDelete(t *testing.T) {
 	if expected, size := int64(10), cache.Size(); expected != size {
 		t.Fatalf("expected cache size %d, but found %d", expected, size)
 	}
-	if v := h.Get(base.DiskFileNum(0), 0); v == nil {
+	if v := h.Get(base.DiskFileNum(0), 0, CategorySSTableData); v == nil {
 		t.Fatalf("expected to find block 0/0")
 	} else {
 		v.Release()
 	}
-	if v := h.Get(base.DiskFileNum(1), 0); v != nil {
+	if v := h.Get(base.DiskFileNum(1), 0, CategorySSTableData); v != nil {
 		t.Fatalf("expected to not find block 1/0")
 	}
 	// Deleting a non-existing block does nothing.
@@ -196,11 +196,11 @@ func TestMultipleDBs(t *testing.T) {
 	if expected, size := int64(5), cache.Size(); expected != size {
 		t.Fatalf("expected cache size %d, but found %d", expected, size)
 	}
-	v := h1.Get(base.DiskFileNum(0), 0)
+	v := h1.Get(base.DiskFileNum(0), 0, CategorySSTableData)
 	if v != nil {
 		t.Fatalf("expected not present, but found %#v", v)
 	}
-	v = h2.Get(base.DiskFileNum(0), 0)
+	v = h2.Get(base.DiskFileNum(0), 0, CategorySSTableData)
 	if v := v.RawBuffer(); string(v) != "bbbbb" {
 		t.Fatalf("expected bbbbb, but found %s", v)
 	}
@@ -306,8 +306,10 @@ func BenchmarkCacheGet(b *testing.B) {
 	b.RunParallel(func(pb *testing.PB) {
 		pcg := rand.NewPCG(rand.Uint64(), rand.Uint64())
 		for pb.Next() {
-			offset := pcg.Uint64() % size
-			v := h.Get(base.DiskFileNum(0), offset)
+			randVal := pcg.Uint64()
+			offset := randVal % size
+			category := Category((randVal >> 32) % uint64(NumCategories))
+			v := h.Get(base.DiskFileNum(0), offset, category)
 			if v == nil {
 				b.Fatal("failed to look up value")
 			}

internal/cache/clockpro.go

@@ -76,9 +76,13 @@ func (k key) String() string {
 	return fmt.Sprintf("%d/%d/%d", k.id, k.fileNum, k.offset)
 }
 
-type shard struct {
+type counters [NumCategories]struct {
 	hits   atomic.Int64
 	misses atomic.Int64
+}
+
+type shard struct {
+	counters counters
 
 	mu sync.RWMutex
 
@@ -134,7 +138,7 @@ func (c *shard) init(maxSize int64) {
 //
 // If peekOnly is true, the state of the cache is not modified to reflect the
 // access.
-func (c *shard) get(k key, peekOnly bool) *Value {
+func (c *shard) get(k key, category Category, peekOnly bool) *Value {
 	c.mu.RLock()
 	if e, _ := c.blocks.Get(k); e != nil {
 		if value := e.acquireValue(); value != nil {
@@ -143,12 +147,12 @@ func (c *shard) get(k key, peekOnly bool) *Value {
 				e.referenced.Store(true)
 			}
 			c.mu.RUnlock()
-			c.hits.Add(1)
+			c.counters[category].hits.Add(1)
 			return value
 		}
 	}
 	c.mu.RUnlock()
-	c.misses.Add(1)
+	c.counters[category].misses.Add(1)
 	return nil
 }
 
@@ -157,7 +161,7 @@ func (c *shard) get(k key, peekOnly bool) *Value {
 // is not in the cache (nil Value), a non-nil readEntry is returned (in which
 // case the caller is responsible to dereference the entry, via one of
 // unrefAndTryRemoveFromMap(), setReadValue(), setReadError()).
-func (c *shard) getWithReadEntry(k key) (*Value, *readEntry) {
+func (c *shard) getWithReadEntry(k key, category Category) (*Value, *readEntry) {
 	c.mu.RLock()
 	if e, _ := c.blocks.Get(k); e != nil {
 		if value := e.acquireValue(); value != nil {
@@ -166,13 +170,13 @@ func (c *shard) getWithReadEntry(k key) (*Value, *readEntry) {
 				e.referenced.Store(true)
 			}
 			c.mu.RUnlock()
-			c.hits.Add(1)
+			c.counters[category].hits.Add(1)
 			return value, nil
 		}
 	}
 	re := c.readShard.acquireReadEntry(k)
 	c.mu.RUnlock()
-	c.misses.Add(1)
+	c.counters[category].misses.Add(1)
 	return nil, re
 }
 

internal/cache/metrics.go

@@ -0,0 +1,124 @@
+// Copyright 2025 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package cache
+
+import (
+	"fmt"
+
+	"github.com/cockroachdb/crlib/crtime"
+)
+
+// Category is used to maintain granular cache hit/miss statistics.
+type Category uint8
+
+const (
+	// CategoryBackground is used for cache accesses made by compactions or
+	// downloads.
+	CategoryBackground Category = iota
+	CategorySSTableData
+	CategorySSTableValue
+	CategoryBlobValue
+	CategoryFilter
+	// CategoryIndex includes index blocks and other metadata blocks (for both
+	// sstables and blob files).
+	CategoryIndex
+
+	// Categories can be used with the range keyword.
+	Categories
+)
+
+const NumCategories = int(Categories)
+
+func (c Category) String() string {
+	switch c {
+	case CategoryBackground:
+		return "background"
+	case CategorySSTableData:
+		return "sstdata"
+	case CategorySSTableValue:
+		return "sstval"
+	case CategoryBlobValue:
+		return "blobval"
+	case CategoryFilter:
+		return "filter"
+	case CategoryIndex:
+		return "index"
+	default:
+		return fmt.Sprintf("invalid(%d)", c)
+	}
+}
+
+// Metrics holds metrics for the cache.
+type Metrics struct {
+	// Hits and misse since the cache was created.
+	HitsAndMisses HitsAndMisses
+	// The current number of bytes inuse by the cache.
+	Size int64
+	// The current count of objects (blocks or tables) in the cache.
+	Count int64
+	// Recent contains cache hit metrics covering two recent periods (last ~10
+	// minutes and last ~1 hour).
+	Recent [2]struct {
+		HitsAndMisses
+		Since crtime.Mono
+	}
+}
+
+// HitsAndMisses contains the number of cache hits and misses across a period of
+// time.
+type HitsAndMisses [NumCategories]struct {
+	Hits   int64
+	Misses int64
+}
+
+// Aggregate returns the total hits and misses across all categories.
+func (hm *HitsAndMisses) Aggregate() (hits, misses int64) {
+	for i := range *hm {
+		hits += hm[i].Hits
+		misses += hm[i].Misses
+	}
+	return hits, misses
+}
+
+// ToRecent changes the receiver to reflect recent hits and misses, given the
+// current metrics.
+// At a high level, hm.ToRecent(current) means hm = current - hm.
+func (hm *HitsAndMisses) ToRecent(current *HitsAndMisses) {
+	for i := range *hm {
+		hm[i].Hits = current[i].Hits - hm[i].Hits
+		hm[i].Misses = current[i].Misses - hm[i].Misses
+	}
+}
+
+// Metrics returns the current metrics for the cache.
+func (c *Cache) Metrics() Metrics {
+	var m Metrics
+	m.HitsAndMisses = c.hitsAndMisses()
+	for i := range c.shards {
+		s := &c.shards[i]
+		s.mu.RLock()
+		m.Count += int64(s.blocks.Len())
+		m.Size += s.sizeHot + s.sizeCold
+		s.mu.RUnlock()
+	}
+	m.Recent[0].HitsAndMisses, m.Recent[0].Since = c.metricsWindow.TenMinutesAgo()
+	m.Recent[1].HitsAndMisses, m.Recent[1].Since = c.metricsWindow.OneHourAgo()
+	for i := range m.Recent {
+		m.Recent[i].ToRecent(&m.HitsAndMisses)
+	}
+	return m
+}
+
+func (c *Cache) hitsAndMisses() HitsAndMisses {
+	var hm HitsAndMisses
+	for i := range c.shards {
+		shardCounters := &c.shards[i].counters
+		for j := range hm {
+			hm[j].Hits += shardCounters[j].hits.Load()
+			hm[j].Misses += shardCounters[j].misses.Load()
+		}
+	}
+	return hm
+}