db: incorporate separated values when estimating compression ratios

Table stats collection estimates compression ratios for the purpose of
calculating the size of data deleted by tombstones. Previously, this
compression ratio calculation included the size of both keys and values in the
pre-compression sum but only included sstables in the post-compression sum.
This resulted in wildly overestimating the effectiveness of compression when
values were separated into external blob files.

This commit incorporates a table's references estimated physical size into the
calculation.

Informs cockroachdb/cockroach#149147.

Author: jbowens

sstable/properties.go

@@ -83,7 +83,8 @@ type CommonProperties struct {
 	NumEntries uint64 `prop:"rocksdb.num.entries"`
 	// Total raw key size.
 	RawKeySize uint64 `prop:"rocksdb.raw.key.size"`
-	// Total raw value size.
+	// Total raw value size. If values are separated, this includes the size of
+	// the separated value, NOT the value handle.
 	RawValueSize uint64 `prop:"rocksdb.raw.value.size"`
 	// Total raw key size of point deletion tombstones. This value is comparable
 	// to RawKeySize.

table_stats.go

@@ -503,15 +503,15 @@ func (d *DB) estimateSizesBeneath(
 		// resulting in a lower than expected avgValueLogicalSize. For an example of
 		// this effect see the estimate in testdata/compaction_picker_scores (search
 		// for "point-deletions-bytes-estimate: 163850").
-		fileSum    = meta.Size
+		fileSum    = meta.Size + meta.EstimatedReferenceSize()
 		entryCount = fileProps.NumEntries
 		keySum     = fileProps.RawKeySize
 		valSum     = fileProps.RawValueSize
 	)
 
 	for l := level + 1; l < numLevels; l++ {
 		for tableBeneath := range v.Overlaps(l, meta.UserKeyBounds()).All() {
-			fileSum += tableBeneath.Size
+			fileSum += tableBeneath.Size + tableBeneath.EstimatedReferenceSize()
 			if tableBeneath.StatsValid() {
 				entryCount += tableBeneath.Stats.NumEntries
 				keySum += tableBeneath.Stats.RawKeySize
@@ -727,7 +727,7 @@ func maybeSetStatsFromProperties(
 		// doesn't require any additional IO and since the number of point
 		// deletions in the file is low, the error introduced by this crude
 		// estimate is expected to be small.
-		avgValSize, compressionRatio := estimatePhysicalSizes(meta.Size, props)
+		avgValSize, compressionRatio := estimatePhysicalSizes(meta, props)
 		pointEstimate = pointDeletionsBytesEstimate(props, avgValSize, compressionRatio)
 	}
 
@@ -833,7 +833,7 @@ func pointDeletionsBytesEstimate(
 }
 
 func estimatePhysicalSizes(
-	fileSize uint64, props *sstable.CommonProperties,
+	tableMeta *manifest.TableMetadata, props *sstable.CommonProperties,
 ) (avgValLogicalSize, compressionRatio float64) {
 	// RawKeySize and RawValueSize are uncompressed totals. Scale according to
 	// the data size to account for compression, index blocks and metadata
@@ -847,8 +847,9 @@ func estimatePhysicalSizes(
 	//   -----------------------  ×  ----------
 	//   RawKeySize+RawValueSize     NumEntries
 	//
+	physicalSize := tableMeta.Size + tableMeta.EstimatedReferenceSize()
 	uncompressedSum := props.RawKeySize + props.RawValueSize
-	compressionRatio = float64(fileSize) / float64(uncompressedSum)
+	compressionRatio = float64(physicalSize) / float64(uncompressedSum)
 	if compressionRatio > 1 {
 		// We can get huge compression ratios due to the fixed overhead of files
 		// containing a tiny amount of data. By setting this to 1, we are ignoring

testdata/table_stats

@@ -978,5 +978,5 @@ wait-pending-table-stats
 num-entries: 3
 num-deletions: 3
 num-range-key-sets: 0
-point-deletions-bytes-estimate: 283
+point-deletions-bytes-estimate: 3940
 range-deletions-bytes-estimate: 0