db: improve file cache metrics, fix 32-bit platform adjustment

The file cache metrics now keep track of the table and blob file
counts separately. We fix the 32-bit memory size adjustment for tests
and make it easy (and mandatory) to keep up to date.

Fixes #4831

Author: RaduBerinde

file_cache.go

@@ -34,6 +34,17 @@ import (
 	"github.com/cockroachdb/redact"
 )
 
+// FileCacheMetrics contains metrics for the file cache. Note that the file
+// cache is normally shared between all the stores on a node.
+type FileCacheMetrics struct {
+	// The number of bytes inuse by the cache.
+	Size          int64
+	TableCount    int64
+	BlobFileCount int64
+	Hits          int64
+	Misses        int64
+}
+
 var emptyIter = &errorIter{err: nil}
 var emptyKeyspanIter = &errorKeyspanIter{err: nil}
 
@@ -290,7 +301,7 @@ func (h *fileCacheHandle) SSTStatsCollector() *block.CategoryStatsCollector {
 // Metrics returns metrics for the file cache. Note that the CacheMetrics track
 // the global cache which is shared between multiple handles (stores). The
 // FilterMetrics are per-handle.
-func (h *fileCacheHandle) Metrics() (CacheMetrics, FilterMetrics) {
+func (h *fileCacheHandle) Metrics() (FileCacheMetrics, FilterMetrics) {
 	m := h.fileCache.c.Metrics()
 
 	// The generic cache maintains a count of entries, but it doesn't know which
@@ -301,10 +312,11 @@ func (h *fileCacheHandle) Metrics() (CacheMetrics, FilterMetrics) {
 	countSSTables := h.fileCache.counts.sstables.Load()
 	countBlobFiles := h.fileCache.counts.blobFiles.Load()
 
-	cm := CacheMetrics{
-		Hits:   m.Hits,
-		Misses: m.Misses,
-		Count:  countSSTables + countBlobFiles,
+	cm := FileCacheMetrics{
+		TableCount:    countSSTables,
+		BlobFileCount: countBlobFiles,
+		Hits:          m.Hits,
+		Misses:        m.Misses,
 		Size: m.Size + countSSTables*int64(unsafe.Sizeof(sstable.Reader{})) +
 			countBlobFiles*int64(unsafe.Sizeof(blob.FileReader{})),
 	}

metrics.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"math"
 	"time"
+	"unsafe"
 
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/cache"
@@ -17,6 +18,7 @@ import (
 	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache"
 	"github.com/cockroachdb/pebble/record"
 	"github.com/cockroachdb/pebble/sstable"
+	"github.com/cockroachdb/pebble/sstable/blob"
 	"github.com/cockroachdb/pebble/sstable/block"
 	"github.com/cockroachdb/pebble/wal"
 	"github.com/cockroachdb/redact"
@@ -441,7 +443,7 @@ type Metrics struct {
 		}
 	}
 
-	FileCache CacheMetrics
+	FileCache FileCacheMetrics
 
 	// Count of the number of open sstable iterators.
 	TableIters int64
@@ -811,15 +813,16 @@ func (m *Metrics) SafeFormat(w redact.SafePrinter, _ rune) {
 	}
 	w.Printf("\n")
 
-	formatCacheMetrics := func(m *CacheMetrics, name redact.SafeString) {
-		w.Printf("%s: %s entries (%s)  hit rate: %.1f%%\n",
-			name,
-			humanize.Count.Int64(m.Count),
-			humanize.Bytes.Int64(m.Size),
-			redact.Safe(hitRate(m.Hits, m.Misses)))
-	}
-	formatCacheMetrics(&m.BlockCache, "Block cache")
-	formatCacheMetrics(&m.FileCache, "Table cache")
+	w.Printf("Block cache: %s entries (%s)  hit rate: %.1f%%\n",
+		humanize.Count.Int64(m.BlockCache.Count),
+		humanize.Bytes.Int64(m.BlockCache.Size),
+		redact.Safe(hitRate(m.BlockCache.Hits, m.BlockCache.Misses)))
+
+	w.Printf("File cache: %s tables, %s blobfiles (%s)  hit rate: %.1f%%\n",
+		humanize.Count.Int64(m.FileCache.TableCount),
+		humanize.Count.Int64(m.FileCache.BlobFileCount),
+		humanize.Bytes.Int64(m.FileCache.Size),
+		redact.Safe(hitRate(m.FileCache.Hits, m.FileCache.Misses)))
 
 	formatSharedCacheMetrics := func(w redact.SafePrinter, m *SecondaryCacheMetrics, name redact.SafeString) {
 		w.Printf("%s: %s entries (%s)  hit rate: %.1f%%\n",
@@ -882,17 +885,20 @@ func percent(numerator, denominator int64) float64 {
 // provide a platform-independent result for tests.
 func (m *Metrics) StringForTests() string {
 	mCopy := *m
-	if math.MaxInt == math.MaxInt32 {
-		// README: This is the difference in Sizeof(sstable.Reader{})) + Sizeof(blob.FileReader{})
-		// between 64 and 32 bit platforms. See Metrics() in file_cache.go for more details.
-		// This magic number must be updated if the sstable.Reader or blob.FileReader struct changes.
-		// On 64-bit platforms, the size of the sstable.Reader struct is 616 bytes.
-		// On 32-bit platforms, the size of the sstable.Reader struct is 496 bytes.
-		// On 64-bit platforms, the size of the blob.FileReader struct is 88 bytes.
-		// On 32-bit platforms, the size of the blob.FileReader struct is 56 bytes.
-		// The difference is 616 - 496 + 88 - 56 = 152 bytes.
-		const tableCacheSizeAdjustment = 152
-		mCopy.FileCache.Size += mCopy.FileCache.Count * tableCacheSizeAdjustment
+
+	// We recalculate the file cache size using the 64-bit sizes, and we ignore
+	// the genericcache metadata size which is harder to adjust.
+	const sstableReaderSize64bit = 280
+	const blobFileReaderSize64bit = 96
+	mCopy.FileCache.Size = mCopy.FileCache.TableCount*sstableReaderSize64bit + mCopy.FileCache.BlobFileCount*blobFileReaderSize64bit
+	if math.MaxInt == math.MaxInt64 {
+		// Verify the 64-bit sizes, so they are kept updated.
+		if sstableReaderSize64bit != unsafe.Sizeof(sstable.Reader{}) {
+			panic(fmt.Sprintf("sstableReaderSize64bit should be updated to %d", unsafe.Sizeof(sstable.Reader{})))
+		}
+		if blobFileReaderSize64bit != unsafe.Sizeof(blob.FileReader{}) {
+			panic(fmt.Sprintf("blobFileReaderSize64bit should be updated to %d", unsafe.Sizeof(blob.FileReader{})))
+		}
 	}
 	// Don't show cgo memory statistics as they can vary based on architecture,
 	// invariants tag, etc.

metrics_test.go

@@ -76,7 +76,8 @@ func exampleMetrics() Metrics {
 	m.Table.BackingTableSize = 2 << 20
 	m.Table.ZombieCount = 16
 	m.FileCache.Size = 17
-	m.FileCache.Count = 18
+	m.FileCache.TableCount = 180
+	m.FileCache.BlobFileCount = 181
 	m.FileCache.Hits = 19
 	m.FileCache.Misses = 20
 	m.TableIters = 21
@@ -369,7 +370,7 @@ func TestMetrics(t *testing.T) {
 			// Some subset of cases show non-determinism in cache hits/misses.
 			if td.HasArg("zero-cache-hits-misses") {
 				// Avoid non-determinism.
-				m.FileCache = cache.Metrics{}
+				m.FileCache = FileCacheMetrics{}
 				m.BlockCache = cache.Metrics{}
 				// Empirically, the unknown stats are also non-deterministic.
 				if len(m.CategoryStats) > 0 && m.CategoryStats[0].Category == block.CategoryUnknown {

testdata/compaction/value_separation

@@ -160,7 +160,7 @@ Local tables size: 864B
 Compression types: snappy: 1
 Table stats: all loaded
 Block cache: 4 entries (1.6KB)  hit rate: 70.3%
-Table cache: 2 entries (808B)  hit rate: 82.2%
+File cache: 1 tables, 1 blobfiles (376B)  hit rate: 82.2%
 Range key sets: 0  Tombstones: 0  Total missized tombstones encountered: 0
 Snapshots: 0  earliest seq num: 0
 Table iters: 0

testdata/event_listener

@@ -260,7 +260,7 @@ Local tables size: 2.2KB
 Compression types: snappy: 3
 Table stats: initial load in progress
 Block cache: 2 entries (784B)  hit rate: 0.0%
-Table cache: 0 entries (0B)  hit rate: 50.0%
+File cache: 0 tables, 0 blobfiles (0B)  hit rate: 50.0%
 Range key sets: 0  Tombstones: 0  Total missized tombstones encountered: 0
 Snapshots: 0  earliest seq num: 0
 Table iters: 0
@@ -365,7 +365,7 @@ Local tables size: 4.4KB
 Compression types: snappy: 6
 Table stats: initial load in progress
 Block cache: 6 entries (2.3KB)  hit rate: 0.0%
-Table cache: 0 entries (0B)  hit rate: 50.0%
+File cache: 0 tables, 0 blobfiles (0B)  hit rate: 50.0%
 Range key sets: 0  Tombstones: 0  Total missized tombstones encountered: 0
 Snapshots: 0  earliest seq num: 0
 Table iters: 0

testdata/ingest

@@ -55,7 +55,7 @@ Local tables size: 569B
 Compression types: snappy: 1
 Table stats: all loaded
 Block cache: 3 entries (1.1KB)  hit rate: 15.4%
-Table cache: 1 entries (496B)  hit rate: 50.0%
+File cache: 1 tables, 0 blobfiles (280B)  hit rate: 50.0%
 Range key sets: 0  Tombstones: 0  Total missized tombstones encountered: 0
 Snapshots: 0  earliest seq num: 0
 Table iters: 0

testdata/metrics

@@ -24,7 +24,7 @@ Local tables size: 28B
 Compression types:
 Table stats: 31
 Block cache: 2 entries (1B)  hit rate: 42.9%
-Table cache: 18 entries (17B)  hit rate: 48.7%
+File cache: 180 tables, 181 blobfiles (17B)  hit rate: 48.7%
 Range key sets: 123  Tombstones: 456  Total missized tombstones encountered: 789
 Snapshots: 4  earliest seq num: 1024
 Table iters: 21
@@ -78,7 +78,7 @@ Local tables size: 755B
 Compression types: snappy: 1
 Table stats: all loaded
 Block cache: 2 entries (795B)  hit rate: 0.0%
-Table cache: 1 entries (496B)  hit rate: 0.0%
+File cache: 1 tables, 0 blobfiles (280B)  hit rate: 0.0%
 Range key sets: 0  Tombstones: 0  Total missized tombstones encountered: 0
 Snapshots: 0  earliest seq num: 0
 Table iters: 1
@@ -143,7 +143,7 @@ Local tables size: 1.5KB
 Compression types: snappy: 2
 Table stats: all loaded
 Block cache: 2 entries (795B)  hit rate: 33.3%
-Table cache: 2 entries (992B)  hit rate: 66.7%
+File cache: 2 tables, 0 blobfiles (560B)  hit rate: 66.7%
 Range key sets: 0  Tombstones: 0  Total missized tombstones encountered: 0
 Snapshots: 0  earliest seq num: 0
 Table iters: 2
@@ -191,7 +191,7 @@ Local tables size: 1.5KB
 Compression types: snappy: 2
 Table stats: all loaded
 Block cache: 2 entries (795B)  hit rate: 33.3%
-Table cache: 2 entries (992B)  hit rate: 66.7%
+File cache: 2 tables, 0 blobfiles (560B)  hit rate: 66.7%
 Range key sets: 0  Tombstones: 0  Total missized tombstones encountered: 0
 Snapshots: 0  earliest seq num: 0
 Table iters: 2
@@ -236,7 +236,7 @@ Local tables size: 1.5KB
 Compression types: snappy: 2
 Table stats: all loaded
 Block cache: 2 entries (795B)  hit rate: 33.3%
-Table cache: 1 entries (496B)  hit rate: 66.7%
+File cache: 1 tables, 0 blobfiles (280B)  hit rate: 66.7%
 Range key sets: 0  Tombstones: 0  Total missized tombstones encountered: 0
 Snapshots: 0  earliest seq num: 0
 Table iters: 1
@@ -284,7 +284,7 @@ Local tables size: 1.5KB
 Compression types: snappy: 2
 Table stats: all loaded
 Block cache: 0 entries (0B)  hit rate: 33.3%
-Table cache: 0 entries (0B)  hit rate: 66.7%
+File cache: 0 tables, 0 blobfiles (0B)  hit rate: 66.7%
 Range key sets: 0  Tombstones: 0  Total missized tombstones encountered: 0
 Snapshots: 0  earliest seq num: 0
 Table iters: 0
@@ -371,7 +371,7 @@ Local tables size: 7.1KB
 Compression types: snappy: 9
 Table stats: all loaded
 Block cache: 0 entries (0B)  hit rate: 33.3%
-Table cache: 0 entries (0B)  hit rate: 66.7%
+File cache: 0 tables, 0 blobfiles (0B)  hit rate: 66.7%
 Range key sets: 0  Tombstones: 0  Total missized tombstones encountered: 0
 Snapshots: 0  earliest seq num: 0
 Table iters: 0
@@ -443,7 +443,7 @@ Local tables size: 7.1KB
 Compression types: snappy: 9
 Table stats: all loaded
 Block cache: 0 entries (0B)  hit rate: 10.0%
-Table cache: 0 entries (0B)  hit rate: 55.0%
+File cache: 0 tables, 0 blobfiles (0B)  hit rate: 55.0%
 Range key sets: 0  Tombstones: 0  Total missized tombstones encountered: 0
 Snapshots: 0  earliest seq num: 0
 Table iters: 0
@@ -566,7 +566,7 @@ Local tables size: 12KB
 Compression types: snappy: 15
 Table stats: all loaded
 Block cache: 6 entries (2.3KB)  hit rate: 7.3%
-Table cache: 0 entries (0B)  hit rate: 55.0%
+File cache: 0 tables, 0 blobfiles (0B)  hit rate: 55.0%
 Range key sets: 0  Tombstones: 0  Total missized tombstones encountered: 0
 Snapshots: 0  earliest seq num: 0
 Table iters: 0
@@ -651,7 +651,7 @@ Local tables size: 17KB
 Compression types: snappy: 22
 Table stats: all loaded
 Block cache: 6 entries (2.3KB)  hit rate: 7.3%
-Table cache: 0 entries (0B)  hit rate: 55.0%
+File cache: 0 tables, 0 blobfiles (0B)  hit rate: 55.0%
 Range key sets: 0  Tombstones: 0  Total missized tombstones encountered: 0
 Snapshots: 0  earliest seq num: 0
 Table iters: 0
@@ -748,7 +748,7 @@ Local tables size: 16KB
 Compression types: snappy: 21
 Table stats: all loaded
 Block cache: 0 entries (0B)  hit rate: 0.0%
-Table cache: 0 entries (0B)  hit rate: 0.0%
+File cache: 0 tables, 0 blobfiles (0B)  hit rate: 0.0%
 Range key sets: 0  Tombstones: 0  Total missized tombstones encountered: 0
 Snapshots: 0  earliest seq num: 0
 Table iters: 0
@@ -889,7 +889,7 @@ Local tables size: 14KB
 Compression types: snappy: 18
 Table stats: all loaded
 Block cache: 0 entries (0B)  hit rate: 0.0%
-Table cache: 0 entries (0B)  hit rate: 0.0%
+File cache: 0 tables, 0 blobfiles (0B)  hit rate: 0.0%
 Range key sets: 0  Tombstones: 0  Total missized tombstones encountered: 0
 Snapshots: 0  earliest seq num: 0
 Table iters: 0
@@ -947,7 +947,7 @@ Local tables size: 2.2KB
 Compression types: snappy: 3
 Table stats: all loaded
 Block cache: 0 entries (0B)  hit rate: 0.0%
-Table cache: 0 entries (0B)  hit rate: 0.0%
+File cache: 0 tables, 0 blobfiles (0B)  hit rate: 0.0%
 Range key sets: 0  Tombstones: 0  Total missized tombstones encountered: 0
 Snapshots: 0  earliest seq num: 0
 Table iters: 0
@@ -991,7 +991,7 @@ Local tables size: 0B
 Compression types: snappy: 3
 Table stats: all loaded
 Block cache: 0 entries (0B)  hit rate: 0.0%
-Table cache: 0 entries (0B)  hit rate: 50.0%
+File cache: 0 tables, 0 blobfiles (0B)  hit rate: 50.0%
 Range key sets: 0  Tombstones: 0  Total missized tombstones encountered: 0
 Snapshots: 0  earliest seq num: 0
 Table iters: 0
@@ -1051,7 +1051,7 @@ Local tables size: 0B
 Compression types: snappy: 4
 Table stats: all loaded
 Block cache: 2 entries (787B)  hit rate: 0.0%
-Table cache: 0 entries (0B)  hit rate: 50.0%
+File cache: 0 tables, 0 blobfiles (0B)  hit rate: 50.0%
 Range key sets: 0  Tombstones: 0  Total missized tombstones encountered: 0
 Snapshots: 0  earliest seq num: 0
 Table iters: 0
@@ -1103,7 +1103,7 @@ Local tables size: 755B
 Compression types: snappy: 5
 Table stats: all loaded
 Block cache: 2 entries (787B)  hit rate: 0.0%
-Table cache: 0 entries (0B)  hit rate: 50.0%
+File cache: 0 tables, 0 blobfiles (0B)  hit rate: 50.0%
 Range key sets: 0  Tombstones: 0  Total missized tombstones encountered: 0
 Snapshots: 0  earliest seq num: 0
 Table iters: 0
@@ -1144,7 +1144,7 @@ Local tables size: 755B
 Compression types: snappy: 5
 Table stats: all loaded
 Block cache: 0 entries (0B)  hit rate: 0.0%
-Table cache: 0 entries (0B)  hit rate: 0.0%
+File cache: 0 tables, 0 blobfiles (0B)  hit rate: 0.0%
 Range key sets: 0  Tombstones: 0  Total missized tombstones encountered: 0
 Snapshots: 0  earliest seq num: 0
 Table iters: 0
@@ -1188,7 +1188,7 @@ Local tables size: 758B
 Compression types: snappy: 3
 Table stats: all loaded
 Block cache: 0 entries (0B)  hit rate: 0.0%
-Table cache: 0 entries (0B)  hit rate: 0.0%
+File cache: 0 tables, 0 blobfiles (0B)  hit rate: 0.0%
 Range key sets: 0  Tombstones: 0  Total missized tombstones encountered: 0
 Snapshots: 0  earliest seq num: 0
 Table iters: 0
@@ -1271,7 +1271,7 @@ Compression types: snappy: 5
 Garbage: point-deletions 502B range-deletions 1.5KB
 Table stats: all loaded
 Block cache: 2 entries (774B)  hit rate: 0.0%
-Table cache: 2 entries (992B)  hit rate: 0.0%
+File cache: 2 tables, 0 blobfiles (560B)  hit rate: 0.0%
 Range key sets: 0  Tombstones: 3  Total missized tombstones encountered: 0
 Snapshots: 0  earliest seq num: 0
 Table iters: 0
@@ -1313,7 +1313,7 @@ Compression types: snappy: 5
 Garbage: point-deletions 502B range-deletions 1.5KB
 Table stats: all loaded
 Block cache: 2 entries (774B)  hit rate: 0.0%
-Table cache: 2 entries (992B)  hit rate: 0.0%
+File cache: 2 tables, 0 blobfiles (560B)  hit rate: 0.0%
 Range key sets: 0  Tombstones: 3  Total missized tombstones encountered: 0
 Snapshots: 0  earliest seq num: 0
 Table iters: 0

tool/testdata/db_lsm

@@ -34,7 +34,7 @@ Local tables size: 709B
 Compression types: unknown: 1
 Table stats: <redacted>
 Block cache: 0 entries (0B)  hit rate: 0.0%
-Table cache: 0 entries (0B)  hit rate: 0.0%
+File cache: 0 tables, 0 blobfiles (0B)  hit rate: 0.0%
 Range key sets: 0  Tombstones: 0  Total missized tombstones encountered: 0
 Snapshots: 0  earliest seq num: 0
 Table iters: 0
@@ -70,7 +70,7 @@ Local tables size: 709B
 Compression types: unknown: 1
 Table stats: <redacted>
 Block cache: 0 entries (0B)  hit rate: 0.0%
-Table cache: 0 entries (0B)  hit rate: 0.0%
+File cache: 0 tables, 0 blobfiles (0B)  hit rate: 0.0%
 Range key sets: 0  Tombstones: 0  Total missized tombstones encountered: 0
 Snapshots: 0  earliest seq num: 0
 Table iters: 0