db: tweak average logical value size estimate

When calculating the estimated space reclaimed by an unsized point tombstone,
we calculate the average logical value size of KVs in sstables overlapping a
tombstone's sstable. Previously this calculation divided the aggregate value
size by the number of entries. This undercalculated the sum by including
deletions. This commit updates this calculation to divide the aggregate value
size by the number of non-deletion keys.

Author: jbowens

table_stats.go

@@ -480,6 +480,16 @@ func (d *DB) loadTableRangeDelStats(
 	return compactionHints, rangeDeletionsBytesEstimate, nil
 }
 
+// estimateSizesBeneath calculates two statistics describing the data in the LSM
+// below the provided table metadata:
+//
+//  1. The average logical size of values: This is a precompression sum of
+//     non-tombstone values. It's helpful for estimating how much data a DEL
+//     might delete.
+//  2. The compression ratio of the data beneath the table.
+//
+// estimateSizesBeneath walks the LSM table metadata for all tables beneath meta
+// (plus the table itself), computing the above statistics.
 func (d *DB) estimateSizesBeneath(
 	ctx context.Context,
 	v *manifest.Version,
@@ -496,14 +506,11 @@ func (d *DB) estimateSizesBeneath(
 	// calculate a compression ratio of 0 which is not accurate for the file's
 	// own tombstones.
 	var (
-		// TODO(sumeer): The entryCount includes the tombstones, which can be small,
-		// resulting in a lower than expected avgValueLogicalSize. For an example of
-		// this effect see the estimate in testdata/compaction_picker_scores (search
-		// for "point-deletions-bytes-estimate: 163850").
-		fileSum    = meta.Size + meta.EstimatedReferenceSize()
-		entryCount = fileProps.NumEntries
-		keySum     = fileProps.RawKeySize
-		valSum     = fileProps.RawValueSize
+		fileSum       = meta.Size + meta.EstimatedReferenceSize()
+		entryCount    = fileProps.NumEntries
+		deletionCount = fileProps.NumDeletions
+		keySum        = fileProps.RawKeySize
+		valSum        = fileProps.RawValueSize
 	)
 
 	for l := level + 1; l < numLevels; l++ {
@@ -527,6 +534,7 @@ func (d *DB) estimateSizesBeneath(
 			}
 
 			entryCount += tableBeneath.ScaleStatistic(backingProps.NumEntries)
+			deletionCount += tableBeneath.ScaleStatistic(backingProps.NumDeletions)
 			keySum += tableBeneath.ScaleStatistic(backingProps.RawKeySize)
 			valSum += tableBeneath.ScaleStatistic(backingProps.RawValueSize)
 			continue
@@ -543,9 +551,9 @@ func (d *DB) estimateSizesBeneath(
 	//
 	//                            ↓
 	//
-	//         FileSize              RawValueSize
-	//   -----------------------  ×  ------------
-	//   RawKeySize+RawValueSize     NumEntries
+	//         FileSize                    RawValueSize
+	//   -----------------------  ×  -------------------------
+	//   RawKeySize+RawValueSize     NumEntries - NumDeletions
 	//
 	// We return the average logical value size plus the compression ratio,
 	// leaving the scaling to the caller. This allows the caller to perform
@@ -559,7 +567,9 @@ func (d *DB) estimateSizesBeneath(
 		// such overhead is not large.
 		compressionRatio = 1
 	}
-	avgValueLogicalSize = float64(valSum) / float64(entryCount)
+	// When calculating the average value size, we subtract the number of
+	// deletions from the total number of entries.
+	avgValueLogicalSize = float64(valSum) / float64(max(1, invariants.SafeSub(entryCount, deletionCount)))
 	return avgValueLogicalSize, compressionRatio, nil
 }
 
@@ -716,8 +726,7 @@ func maybeSetStatsFromProperties(meta *manifest.TableMetadata, props *sstable.Pr
 	if props.NumDeletions != 0 || props.NumRangeKeyDels != 0 {
 		return false
 	}
-	var stats manifest.TableStats
-	meta.PopulateStats(&stats)
+	meta.PopulateStats(new(manifest.TableStats))
 	return true
 }
 

testdata/compaction_picker_scores

@@ -93,7 +93,7 @@ wait-pending-table-stats
 num-entries: 5
 num-deletions: 5
 num-range-key-sets: 0
-point-deletions-bytes-estimate: 163850
+point-deletions-bytes-estimate: 327690
 range-deletions-bytes-estimate: 0
 compression: None:128
 
@@ -105,7 +105,7 @@ L1       0B       0.00     0.00           0.00
 L2       0B       0.00     0.00           0.00
 L3       0B       0.00     0.00           0.00
 L4       0B       0.00     0.00           0.00
-L5       709B     0.01     0.01           2.51
+L5       709B     0.01     0.01           5.01
 L6       321KB    1.11     1.11           1.11
 
 # Run a similar test as above, but this time the table containing the DELs is
@@ -147,7 +147,7 @@ wait-pending-table-stats
 num-entries: 5
 num-deletions: 5
 num-range-key-sets: 0
-point-deletions-bytes-estimate: 163860
+point-deletions-bytes-estimate: 327700
 range-deletions-bytes-estimate: 0
 compression: None:129
 

testdata/compaction_tombstones

@@ -83,7 +83,7 @@ wait-pending-table-stats
 num-entries: 2
 num-deletions: 1
 num-range-key-sets: 0
-point-deletions-bytes-estimate: 2
+point-deletions-bytes-estimate: 3
 range-deletions-bytes-estimate: 0
 compression: None:124
 
@@ -123,7 +123,7 @@ wait-pending-table-stats
 num-entries: 6
 num-deletions: 2
 num-range-key-sets: 0
-point-deletions-bytes-estimate: 2
+point-deletions-bytes-estimate: 3
 range-deletions-bytes-estimate: 101
 compression: None:87,Snappy:96/129
 
@@ -157,7 +157,7 @@ wait-pending-table-stats
 num-entries: 11
 num-deletions: 1
 num-range-key-sets: 0
-point-deletions-bytes-estimate: 2
+point-deletions-bytes-estimate: 3
 range-deletions-bytes-estimate: 0
 compression: None:36,Snappy:131/169
 
@@ -240,7 +240,7 @@ wait-pending-table-stats
 num-entries: 3
 num-deletions: 3
 num-range-key-sets: 0
-point-deletions-bytes-estimate: 6150
+point-deletions-bytes-estimate: 12294
 range-deletions-bytes-estimate: 0
 compression: None:128
 
@@ -371,7 +371,7 @@ wait-pending-table-stats
 num-entries: 2
 num-deletions: 1
 num-range-key-sets: 0
-point-deletions-bytes-estimate: 2459
+point-deletions-bytes-estimate: 3074
 range-deletions-bytes-estimate: 0
 compression: None:120
 
@@ -419,7 +419,7 @@ wait-pending-table-stats
 num-entries: 2
 num-deletions: 1
 num-range-key-sets: 0
-point-deletions-bytes-estimate: 2459
+point-deletions-bytes-estimate: 3074
 range-deletions-bytes-estimate: 0
 compression: None:120
 
@@ -492,7 +492,7 @@ wait-pending-table-stats
 num-entries: 5
 num-deletions: 3
 num-range-key-sets: 0
-point-deletions-bytes-estimate: 7
+point-deletions-bytes-estimate: 9
 range-deletions-bytes-estimate: 0
 compression: None:36,Snappy:95/108
 
@@ -505,7 +505,7 @@ wait-pending-table-stats force-tombstone-density-ratio=0.9
 num-entries: 5
 num-deletions: 3
 num-range-key-sets: 0
-point-deletions-bytes-estimate: 7
+point-deletions-bytes-estimate: 9
 range-deletions-bytes-estimate: 0
 tombstone-dense-blocks-ratio: 0.9
 compression: None:36,Snappy:95/108
@@ -556,7 +556,7 @@ wait-pending-table-stats
 num-entries: 5
 num-deletions: 3
 num-range-key-sets: 0
-point-deletions-bytes-estimate: 7
+point-deletions-bytes-estimate: 9
 range-deletions-bytes-estimate: 0
 compression: None:36,Snappy:95/108
 
@@ -567,7 +567,7 @@ wait-pending-table-stats force-tombstone-density-ratio=0.9
 num-entries: 5
 num-deletions: 3
 num-range-key-sets: 0
-point-deletions-bytes-estimate: 7
+point-deletions-bytes-estimate: 9
 range-deletions-bytes-estimate: 0
 tombstone-dense-blocks-ratio: 0.9
 compression: None:36,Snappy:95/108
@@ -617,7 +617,7 @@ wait-pending-table-stats
 num-entries: 5
 num-deletions: 3
 num-range-key-sets: 0
-point-deletions-bytes-estimate: 857149
+point-deletions-bytes-estimate: 1500007
 range-deletions-bytes-estimate: 0
 compression: None:36,Snappy:95/108
 
@@ -628,7 +628,7 @@ wait-pending-table-stats force-tombstone-density-ratio=0.9
 num-entries: 5
 num-deletions: 3
 num-range-key-sets: 0
-point-deletions-bytes-estimate: 857149
+point-deletions-bytes-estimate: 1500007
 range-deletions-bytes-estimate: 0
 tombstone-dense-blocks-ratio: 0.9
 compression: None:36,Snappy:95/108

testdata/table_stats

@@ -15,7 +15,7 @@ wait-pending-table-stats
 num-entries: 3
 num-deletions: 1
 num-range-key-sets: 0
-point-deletions-bytes-estimate: 2
+point-deletions-bytes-estimate: 3
 range-deletions-bytes-estimate: 0
 
 compact a-c
@@ -546,7 +546,7 @@ wait-pending-table-stats
 num-entries: 5
 num-deletions: 2
 num-range-key-sets: 0
-point-deletions-bytes-estimate: 111127
+point-deletions-bytes-estimate: 114303
 range-deletions-bytes-estimate: 0
 
 # Try a missized point tombstone. It should appear in the Metrics after the
@@ -1037,7 +1037,7 @@ wait-pending-table-stats
 num-entries: 3
 num-deletions: 3
 num-range-key-sets: 0
-point-deletions-bytes-estimate: 3908
+point-deletions-bytes-estimate: 4641
 range-deletions-bytes-estimate: 0
 compression: None:36,Snappy:73/84