block: hide cache hits/misses from the level checker

We introduce a "reason" for using a `BufferPool`. We use this reason
to determine what category to pass to the cache. We introduce a
special `CategoryHidden` which doesn't update any metrics; we use this
for the level checker. Since the level checker is a testing facility
only, it only adds noise to the metrics.

Author: RaduBerinde

blob_rewrite.go

@@ -307,7 +307,7 @@ func (d *DB) runBlobFileRewriteLocked(
 	// the compaction finishes new iterators will read the new blob file, so it
 	// would be unlikely the cached blocks would be reused.
 	var bufferPool block.BufferPool
-	bufferPool.Init(4)
+	bufferPool.Init(4, block.ForBlobFileRewrite)
 	defer bufferPool.Release()
 	env := block.ReadEnv{
 		Stats:              &c.internalIteratorStats,

blob_rewrite_test.go

@@ -334,7 +334,7 @@ func TestBlobRewriteRandomized(t *testing.T) {
 	fch := fc.newHandle(ch, objStore, base.NoopLoggerAndTracer{}, sstable.ReaderOptions{}, nil)
 	defer fch.Close()
 	var bufferPool block.BufferPool
-	bufferPool.Init(4)
+	bufferPool.Init(4, block.ForBlobFileRewrite)
 	defer bufferPool.Release()
 	readEnv := block.ReadEnv{BufferPool: &bufferPool}
 

compaction.go

@@ -3379,7 +3379,7 @@ func (d *DB) compactAndWrite(
 	// a 18-buffer pool is expected to be within reason, even if all the buffers
 	// grow to the typical size of an index block (256 KiB) which would
 	// translate to 4.5 MiB per compaction.
-	c.iterationState.bufferPool.Init(18 + suggestedCacheReaders*2)
+	c.iterationState.bufferPool.Init(18+suggestedCacheReaders*2, block.ForCompaction)
 	defer c.iterationState.bufferPool.Release()
 	blockReadEnv := block.ReadEnv{
 		BufferPool: &c.iterationState.bufferPool,

external_iterator.go

@@ -92,7 +92,7 @@ func NewExternalIterWithContext(
 		},
 		seqNum: base.SeqNumMax,
 	}
-	dbi.externalIter.bufferPool.Init(2)
+	dbi.externalIter.bufferPool.Init(2, block.ForExternalIter)
 
 	if iterOpts != nil {
 		dbi.opts = *iterOpts

internal/cache/cache.go

@@ -249,11 +249,17 @@ func (c *Handle) Cache() *Cache {
 // Peek retrieves the cache value for the specified file and offset, returning
 // nil if no value is present. Peek does not affect the state of the cache (it
 // does not "count" as an access as far as the cache replacement is concerned).
+//
+// Peek supports the special CategoryHidden category, in which case the hit or
+// miss is not recorded in metrics.
 func (c *Handle) Peek(fileNum base.DiskFileNum, offset uint64, category Category) *Value {
 	k := makeKey(c.id, fileNum, offset)
 	return c.cache.getShard(k).get(k, category, true /* peekOnly */)
 }
 
+// CategoryHidden can be used with Peek to avoid recording a cache hit or miss.
+const CategoryHidden Category = -1
+
 // Get retrieves the cache value for the specified file and offset, returning
 // nil if no value is present.
 func (c *Handle) Get(fileNum base.DiskFileNum, offset uint64, category Category) *Value {

internal/cache/clockpro.go

@@ -147,12 +147,16 @@ func (c *shard) get(k key, category Category, peekOnly bool) *Value {
 				e.referenced.Store(true)
 			}
 			c.mu.RUnlock()
-			c.counters[category].hits.Add(1)
+			if category != CategoryHidden {
+				c.counters[category].hits.Add(1)
+			}
 			return value
 		}
 	}
 	c.mu.RUnlock()
-	c.counters[category].misses.Add(1)
+	if category != CategoryHidden {
+		c.counters[category].misses.Add(1)
+	}
 	return nil
 }
 

internal/cache/metrics.go

@@ -11,7 +11,7 @@ import (
 )
 
 // Category is used to maintain granular cache hit/miss statistics.
-type Category uint8
+type Category int8
 
 const (
 	// CategoryBackground is used for cache accesses made by compactions or

level_checker.go

@@ -541,7 +541,7 @@ func (d *DB) CheckLevels(stats *CheckLevelsStats) error {
 	seqNum := d.mu.versions.visibleSeqNum.Load()
 
 	bufferPool := new(block.BufferPool)
-	bufferPool.Init(10)
+	bufferPool.Init(10, block.ForLevelChecking)
 	defer bufferPool.Release()
 
 	checkConfig := &checkConfig{

sstable/blob/blob.go

@@ -666,7 +666,7 @@ func (r *FileReader) ReadProperties(ctx context.Context) (FileProperties, error)
 	// We don't want the property block to go into the block cache, so we use a
 	// buffer pool.
 	var bufferPool block.BufferPool
-	bufferPool.Init(1)
+	bufferPool.Init(1, block.ForBlobFileMetadata)
 	defer bufferPool.Release()
 	b, err := r.r.Read(
 		ctx, block.ReadEnv{BufferPool: &bufferPool}, nil /* readHandle */, r.footer.propertiesHandle,

sstable/block/block.go

@@ -269,9 +269,6 @@ type ReadEnv struct {
 
 	// BufferPool is not-nil if we read blocks into a buffer pool and not into the
 	// cache. This is used during compactions and by the level checker.
-	//
-	// When BufferPool is non-nil, any block cache accesses use the background
-	// category.
 	BufferPool *BufferPool
 
 	// ReportCorruptionFn is called with ReportCorruptionArg and the error
@@ -382,7 +379,20 @@ func (r *Reader) Read(
 	// reading a block.
 	if r.opts.CacheOpts.CacheHandle == nil || env.BufferPool != nil {
 		if r.opts.CacheOpts.CacheHandle != nil {
-			if cv := r.opts.CacheOpts.CacheHandle.Peek(r.opts.CacheOpts.FileNum, bh.Offset, cache.CategoryBackground); cv != nil {
+			var category cache.Category
+			switch env.BufferPool.Reason() {
+			case ForLevelChecking:
+				category = cache.CategoryHidden
+			case ForCompaction, ForBlobFileRewrite:
+				// We don't want background work to obscure the hit/miss metrics for
+				// "foreground" traffic; and having two variants for each category would
+				// be overkill, so compaction-related metrics all go into one
+				// "background" category.
+				category = cache.CategoryBackground
+			default:
+				category = kindToCacheCategory[kind]
+			}
+			if cv := r.opts.CacheOpts.CacheHandle.Peek(r.opts.CacheOpts.FileNum, bh.Offset, category); cv != nil {
 				recordCacheHit(ctx, env, readHandle, bh, kind)
 				return CacheBufferHandle(cv), nil
 			}