pebble: materialize virtual tables only if backing contains >= 30% garbage

Virtual rewrite compactions should run more conservatively. This commit
introduces a new option to configure the garbage ratio required by a backing
table to trigger a virtual rewrite compaction. We also adjust the estimation of
the  data still referenced such that blob references are assumed to remain
within the backing's virtual tables.

Author: xinhaoz

compaction_picker.go

@@ -1718,7 +1718,12 @@ func (p *compactionPickerByScore) pickVirtualRewriteCompaction(
 	// or selecting files that aren't contiguous in a level. Successfully materializing
 	// one of the backing's virtual table will also make the backing more likely to be
 	// picked again, since the space amp will increase.
-	_, vtablesByLevel := p.latestVersionState.virtualBackings.ReplacementCandidate()
+	referencedDataPct, _, vtablesByLevel := p.latestVersionState.virtualBackings.ReplacementCandidate()
+
+	if 1-referencedDataPct < p.opts.Experimental.VirtualTableRewriteUnreferencedFraction() {
+		return nil
+	}
+
 	for level, tables := range vtablesByLevel {
 		for _, vt := range tables {
 			if vt.IsCompacting() {

internal/manifest/testdata/virtual_backings/rewrite_candidates

@@ -1,4 +1,5 @@
 # Test rewrite candidates heap with blob value sizes.
+# Blob value sizes should not be included in the unreferenced data.
 
 # Add backings with values in blob files.
 add n=1 size=100 blobValueSize=50
@@ -50,7 +51,7 @@ add-table n=1 size=50 table=1
   000003:  size=150  refBlobValueSize=75  useCount=0  protectionCount=0  virtualizedSize=0  tables: []
   000004:  size=300  refBlobValueSize=150  useCount=0  protectionCount=0  virtualizedSize=0  tables: []
   000005:  size=250  refBlobValueSize=125  useCount=0  protectionCount=0  virtualizedSize=0  tables: []
-rewrite candidates heap: 000001(33.3%) 
+rewrite candidates heap: 000001(50.0%) 
 unused virtual backings: 000002 000003 000004 000005
 
 add-table n=2 size=10 table=2
@@ -61,7 +62,7 @@ add-table n=2 size=10 table=2
   000003:  size=150  refBlobValueSize=75  useCount=0  protectionCount=0  virtualizedSize=0  tables: []
   000004:  size=300  refBlobValueSize=150  useCount=0  protectionCount=0  virtualizedSize=0  tables: []
   000005:  size=250  refBlobValueSize=125  useCount=0  protectionCount=0  virtualizedSize=0  tables: []
-rewrite candidates heap: 000002(3.3%) 000001(33.3%) 
+rewrite candidates heap: 000002(5.0%) 000001(50.0%) 
 unused virtual backings: 000003 000004 000005
 
 add-table n=3 size=90 table=3
@@ -72,7 +73,7 @@ add-table n=3 size=90 table=3
   000003:  size=150  refBlobValueSize=75  useCount=1  protectionCount=0  virtualizedSize=90  tables: [000003]
   000004:  size=300  refBlobValueSize=150  useCount=0  protectionCount=0  virtualizedSize=0  tables: []
   000005:  size=250  refBlobValueSize=125  useCount=0  protectionCount=0  virtualizedSize=0  tables: []
-rewrite candidates heap: 000002(3.3%) 000001(33.3%) 000003(40.0%) 
+rewrite candidates heap: 000002(5.0%) 000001(50.0%) 000003(60.0%) 
 unused virtual backings: 000004 000005
 
 add-table n=4 size=45 table=4
@@ -83,7 +84,7 @@ add-table n=4 size=45 table=4
   000003:  size=150  refBlobValueSize=75  useCount=1  protectionCount=0  virtualizedSize=90  tables: [000003]
   000004:  size=300  refBlobValueSize=150  useCount=1  protectionCount=0  virtualizedSize=45  tables: [000004]
   000005:  size=250  refBlobValueSize=125  useCount=0  protectionCount=0  virtualizedSize=0  tables: []
-rewrite candidates heap: 000002(3.3%) 000004(10.0%) 000003(40.0%) 000001(33.3%) 
+rewrite candidates heap: 000002(5.0%) 000004(15.0%) 000003(60.0%) 000001(50.0%) 
 unused virtual backings: 000005
 
 add-table n=5 size=100 table=5
@@ -94,7 +95,7 @@ add-table n=5 size=100 table=5
   000003:  size=150  refBlobValueSize=75  useCount=1  protectionCount=0  virtualizedSize=90  tables: [000003]
   000004:  size=300  refBlobValueSize=150  useCount=1  protectionCount=0  virtualizedSize=45  tables: [000004]
   000005:  size=250  refBlobValueSize=125  useCount=1  protectionCount=0  virtualizedSize=100  tables: [000005]
-rewrite candidates heap: 000002(3.3%) 000004(10.0%) 000003(40.0%) 000001(33.3%) 000005(26.7%) 
+rewrite candidates heap: 000002(5.0%) 000004(15.0%) 000003(60.0%) 000001(50.0%) 000005(40.0%) 
 
 add-table n=2 size=80 table=6
 ----
@@ -104,7 +105,7 @@ add-table n=2 size=80 table=6
   000003:  size=150  refBlobValueSize=75  useCount=1  protectionCount=0  virtualizedSize=90  tables: [000003]
   000004:  size=300  refBlobValueSize=150  useCount=1  protectionCount=0  virtualizedSize=45  tables: [000004]
   000005:  size=250  refBlobValueSize=125  useCount=1  protectionCount=0  virtualizedSize=100  tables: [000005]
-rewrite candidates heap: 000004(10.0%) 000005(26.7%) 000003(40.0%) 000001(33.3%) 000002(30.0%) 
+rewrite candidates heap: 000004(15.0%) 000005(40.0%) 000003(60.0%) 000001(50.0%) 000002(45.0%) 
 
 # Remove some tables to demonstrate heap updates.
 remove-table n=5 table=5
@@ -115,7 +116,7 @@ remove-table n=5 table=5
   000003:  size=150  refBlobValueSize=75  useCount=1  protectionCount=0  virtualizedSize=90  tables: [000003]
   000004:  size=300  refBlobValueSize=150  useCount=1  protectionCount=0  virtualizedSize=45  tables: [000004]
   000005:  size=250  refBlobValueSize=125  useCount=0  protectionCount=0  virtualizedSize=0  tables: []
-rewrite candidates heap: 000004(10.0%) 000002(30.0%) 000003(40.0%) 000001(33.3%) 
+rewrite candidates heap: 000004(15.0%) 000002(45.0%) 000003(60.0%) 000001(50.0%) 
 unused virtual backings: 000005
 
 # Remove backing 4's table. Should remove from heap since virtualizedSize=0.
@@ -127,5 +128,5 @@ remove-table n=4 table=4
   000003:  size=150  refBlobValueSize=75  useCount=1  protectionCount=0  virtualizedSize=90  tables: [000003]
   000004:  size=300  refBlobValueSize=150  useCount=0  protectionCount=0  virtualizedSize=0  tables: []
   000005:  size=250  refBlobValueSize=125  useCount=0  protectionCount=0  virtualizedSize=0  tables: []
-rewrite candidates heap: 000002(30.0%) 000001(33.3%) 000003(40.0%) 
+rewrite candidates heap: 000002(45.0%) 000001(50.0%) 000003(60.0%) 
 unused virtual backings: 000004 000005

internal/manifest/virtual_backings.go

@@ -71,7 +71,7 @@ import (
 type VirtualBackings struct {
 	m map[base.DiskFileNum]*backingWithMetadata
 	// rewriteCandidates is a min heap of virtual backings ordered by
-	// referencedDataPct. Used to pick a candidate for rewriting.
+	// referencedDataFraction. Used to pick a candidate for rewriting.
 	rewriteCandidates virtualBackingRewriteCandidatesHeap
 
 	// unused are all the backings in m that are not inUse(). Used for
@@ -108,9 +108,14 @@ type backingWithMetadata struct {
 	heapIndex     int
 }
 
-func (bm *backingWithMetadata) referencedDataPct() float64 {
-	return float64(bm.virtualizedSize) /
-		float64(bm.backing.Size+bm.backing.ReferencedBlobValueSizeTotal)
+// referencedDataFraction returns the percentage of data in the backing that is
+// referenced by virtual tables.
+// TODO(xinhaoz): Note that the sizes here intentionally exclude referenced
+// blob value sizes of virtual tables and the backing table. When we are able
+// to track referenced blob values for virtual tables more accurately we should
+// include those sizes in this estimation.
+func (bm *backingWithMetadata) referencedDataFraction() float64 {
+	return float64(bm.virtualizedSize) / float64(bm.backing.Size)
 }
 
 // AddAndRef adds a new backing to the set and takes a reference on it. Another
@@ -295,18 +300,21 @@ func (bv *VirtualBackings) DiskFileNums() []base.DiskFileNum {
 // referenced by virtual tables to total size, along with the list of virtual
 // tables that use the backing. If there are no backings in the set, nil is
 // returned.
-func (bv *VirtualBackings) ReplacementCandidate() (*TableBacking, [NumLevels][]*TableMetadata) {
+func (bv *VirtualBackings) ReplacementCandidate() (
+	referencedDataFraction float64,
+	backing *TableBacking,
+	tables [NumLevels][]*TableMetadata,
+) {
 	if bv.rewriteCandidates.Len() == 0 {
-		return nil, [NumLevels][]*TableMetadata{}
+		return 0, nil, [NumLevels][]*TableMetadata{}
 	}
 	v := bv.rewriteCandidates.items[0]
-	var tables [NumLevels][]*TableMetadata
 	tableNums := slices.Sorted(maps.Keys(v.virtualTables))
 	for _, t := range tableNums {
 		tl := v.virtualTables[t]
 		tables[tl.level] = append(tables[tl.level], tl.meta)
 	}
-	return v.backing, tables
+	return v.referencedDataFraction(), v.backing, tables
 }
 
 func (bv *VirtualBackings) String() string {
@@ -332,7 +340,7 @@ func (bv *VirtualBackings) String() string {
 		if len(bv.rewriteCandidates.items) > 0 {
 			fmt.Fprint(&buf, "rewrite candidates heap: ")
 			for _, v := range bv.rewriteCandidates.items {
-				fmt.Fprintf(&buf, "%s(%.1f%%) ", v.backing.DiskFileNum, v.referencedDataPct()*100)
+				fmt.Fprintf(&buf, "%s(%.1f%%) ", v.backing.DiskFileNum, v.referencedDataFraction()*100)
 			}
 			fmt.Fprintf(&buf, "\n")
 		}
@@ -382,7 +390,7 @@ func (v *virtualBackingRewriteCandidatesHeap) Len() int {
 func (v *virtualBackingRewriteCandidatesHeap) Less(i, j int) bool {
 	// We want to rewrite backings with a high percentage of garbage first,
 	// so we order the heap by ratio of data referenced in virtual tables.
-	return v.items[i].referencedDataPct() < v.items[j].referencedDataPct()
+	return v.items[i].referencedDataFraction() < v.items[j].referencedDataFraction()
 }
 
 func (v *virtualBackingRewriteCandidatesHeap) Swap(i, j int) {

options.go

@@ -38,8 +38,9 @@ import (
 )
 
 const (
-	cacheDefaultSize       = 8 << 20 // 8 MB
-	defaultLevelMultiplier = 10
+	cacheDefaultSize                        = 8 << 20 // 8 MB
+	defaultLevelMultiplier                  = 10
+	defaultVirtualTableUnreferencedFraction = 0.3
 )
 
 // FilterType exports the base.FilterType type.
@@ -782,6 +783,15 @@ type Options struct {
 
 		// SpanPolicyFunc is used to determine the SpanPolicy for a key region.
 		SpanPolicyFunc SpanPolicyFunc
+
+		// VirtualTableRewriteUnreferencedFraction configures the minimum fraction of
+		// unreferenced data in a backing table required to trigger a virtual table
+		// rewrite compaction. This is calculated as the ratio of unreferenced
+		// data size to total backing file size. A value of 0.0 triggers
+		// rewrites for any amount of unreferenced data. A value of 1.0 disables
+		// virtual table rewrite compactions entirely. The default value is 0.30
+		// (rewrite when >= 30% of backing data is unreferenced).
+		VirtualTableRewriteUnreferencedFraction func() float64
 	}
 
 	// Filters is a map from filter policy name to filter policy. It is used for
@@ -1631,6 +1641,9 @@ func (o *Options) EnsureDefaults() {
 	if o.Experimental.SpanPolicyFunc == nil {
 		o.Experimental.SpanPolicyFunc = func(startKey []byte) (SpanPolicy, []byte, error) { return SpanPolicy{}, nil, nil }
 	}
+	if o.Experimental.VirtualTableRewriteUnreferencedFraction == nil {
+		o.Experimental.VirtualTableRewriteUnreferencedFraction = func() float64 { return defaultVirtualTableUnreferencedFraction }
+	}
 	// TODO(jackson): Enable value separation by default once we have confidence
 	// in a default policy.
 

testdata/compaction/virtual_rewrite

@@ -204,3 +204,48 @@ COMPRESSION
        snappy | 296B (CR=1.23) |
 ----
 ----
+
+# Test virtual table rewrite unreferenced fraction threshold (default 30%).
+# This test creates a backing with low unreferenced ratio and verifies
+# that virtual rewrite compaction does not trigger.
+
+define auto-compactions=off
+----
+
+# Create a backing table
+batch
+set a value_a
+set b value_b  
+set c value_c
+set d value_d
+----
+
+flush
+----
+L0.0:
+  000005:[a#10,SET-d#13,SET] seqnums:[10-13] points:[a#10,SET-d#13,SET] size:695
+
+compact a-d
+----
+L6:
+  000005:[a#10,SET-d#13,SET] seqnums:[10-13] points:[a#10,SET-d#13,SET] size:695
+
+# Excise only a small portion to create low unreferenced fraction (~0.1).
+excise a.5 b.5
+----
+L6:
+  000006(000005):[a#10,SET-a#10,SET] seqnums:[10-13] points:[a#10,SET-a#10,SET] size:113(695)
+  000007(000005):[c#12,SET-d#13,SET] seqnums:[10-13] points:[c#12,SET-d#13,SET] size:113(695)
+
+virtual-backings
+----
+1 virtual backings, total size 695:
+  000005:  size=695  refBlobValueSize=0  useCount=2  protectionCount=0  virtualizedSize=226  tables: [000006 000007]
+rewrite candidates heap: 000005(32.5%) 
+
+# Should NOT trigger virtual rewrite (0.1 unreferenced fraction < default 0.3 threshold)
+run-virtual-rewrite-compaction
+----
+L6:
+  000008:[a#0,SET-a#0,SET] seqnums:[0-0] points:[a#0,SET-a#0,SET] size:658
+  000007(000005):[c#12,SET-d#13,SET] seqnums:[10-13] points:[c#12,SET-d#13,SET] size:113(695)