kvserver: add storage.sstable.zombie.bytes metric

Add a new timeseries metric that provides visibility into the volume of data
that exists in sstables that are not part of the most recent version of the
LSM.

Epic: none
Informs cockroachdb#121935.
Informs cockroachdb#122139.
Informs cockroachdb/pebble#3500.
Close cockroachdb#122110.
Release note (ops change): Adds a new timeseries metric
storage.sstable.zombie.bytes.

docs/generated/metrics/metrics.html

@@ -701,6 +701,7 @@
 <tr><td>STORAGE</td><td>storage.shared-storage.write</td><td>Bytes written to external storage</td><td>Bytes</td><td>GAUGE</td><td>BYTES</td><td>AVG</td><td>NONE</td></tr>
 <tr><td>STORAGE</td><td>storage.single-delete.ineffectual</td><td>Number of SingleDeletes that were ineffectual</td><td>Events</td><td>GAUGE</td><td>COUNT</td><td>AVG</td><td>NONE</td></tr>
 <tr><td>STORAGE</td><td>storage.single-delete.invariant-violation</td><td>Number of SingleDelete invariant violations</td><td>Events</td><td>GAUGE</td><td>COUNT</td><td>AVG</td><td>NONE</td></tr>
+<tr><td>STORAGE</td><td>storage.sstable.zombie.bytes</td><td>Bytes in SSTables that have been logically deleted, but can&#39;t yet be physically deleted because an open iterator may be reading them.</td><td>Bytes</td><td>GAUGE</td><td>BYTES</td><td>AVG</td><td>NONE</td></tr>
 <tr><td>STORAGE</td><td>storage.wal.bytes_in</td><td>The number of logical bytes the storage engine has written to the WAL</td><td>Events</td><td>GAUGE</td><td>COUNT</td><td>AVG</td><td>NONE</td></tr>
 <tr><td>STORAGE</td><td>storage.wal.bytes_written</td><td>The number of bytes the storage engine has written to the WAL</td><td>Events</td><td>GAUGE</td><td>COUNT</td><td>AVG</td><td>NONE</td></tr>
 <tr><td>STORAGE</td><td>storage.wal.failover.primary.duration</td><td>Cumulative time spent writing to the primary WAL directory. Only populated when WAL failover is configured</td><td>Nanoseconds</td><td>GAUGE</td><td>NANOSECONDS</td><td>AVG</td><td>NONE</td></tr>

pkg/kv/kvserver/metrics.go

@@ -942,6 +942,14 @@ bytes preserved during flushes and compactions over the lifetime of the process.
 		Measurement: "Nanoseconds",
 		Unit:        metric.Unit_NANOSECONDS,
 	}
+	metaSSTableZombieBytes = metric.Metadata{
+		Name: "storage.sstable.zombie.bytes",
+		Help: "Bytes in SSTables that have been logically deleted, " +
+			"but can't yet be physically deleted because an " +
+			"open iterator may be reading them.",
+		Measurement: "Bytes",
+		Unit:        metric.Unit_BYTES,
+	}
 )
 
 var (
@@ -2590,6 +2598,7 @@ type StoreMetrics struct {
 	BatchCommitL0StallDuration        *metric.Gauge
 	BatchCommitWALRotWaitDuration     *metric.Gauge
 	BatchCommitCommitWaitDuration     *metric.Gauge
+	SSTableZombieBytes                *metric.Gauge
 	categoryIterMetrics               pebbleCategoryIterMetricsContainer
 	categoryDiskWriteMetrics          pebbleCategoryDiskWriteMetricsContainer
 	WALBytesWritten                   *metric.Gauge
@@ -3293,6 +3302,7 @@ func newStoreMetrics(histogramWindow time.Duration) *StoreMetrics {
 		BatchCommitL0StallDuration:        metric.NewGauge(metaBatchCommitL0StallDuration),
 		BatchCommitWALRotWaitDuration:     metric.NewGauge(metaBatchCommitWALRotDuration),
 		BatchCommitCommitWaitDuration:     metric.NewGauge(metaBatchCommitCommitWaitDuration),
+		SSTableZombieBytes:                metric.NewGauge(metaSSTableZombieBytes),
 		categoryIterMetrics: pebbleCategoryIterMetricsContainer{
 			registry: storeRegistry,
 		},
@@ -3723,6 +3733,7 @@ func (sm *StoreMetrics) updateEngineMetrics(m storage.Metrics) {
 	sm.BatchCommitL0StallDuration.Update(int64(m.BatchCommitStats.L0ReadAmpWriteStallDuration))
 	sm.BatchCommitWALRotWaitDuration.Update(int64(m.BatchCommitStats.WALRotationDuration))
 	sm.BatchCommitCommitWaitDuration.Update(int64(m.BatchCommitStats.CommitWaitDuration))
+	sm.SSTableZombieBytes.Update(int64(m.Table.ZombieSize))
 	sm.categoryIterMetrics.update(m.CategoryStats)
 	sm.categoryDiskWriteMetrics.update(m.DiskWriteStats)