Skip to content

Commit 2a097e0

Browse files
committed
db: add MinimumMVCCGarbageSize to ValueSeparationPolicy struct
This patch adds a globally configurable MinimumMVCCGarbageSize to our ValueSeparationPolicy, along with some modifications around ValueStoragePolicyAdjustment that allows our setting to be persisted via a span policy. Fixes: #5377
1 parent 6ee797c commit 2a097e0

20 files changed

+113
-23
lines changed

blob_rewrite_test.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ func TestBlobRewrite(t *testing.T) {
9393
inputBlobPhysicalFiles,
9494
0, /* outputBlobReferenceDepth */
9595
0, /* minimumSize */
96+
0, /* minimum MVCC garbage size */
9697
)
9798
vs = pbr
9899
case "write-new-blob-files":
@@ -106,6 +107,7 @@ func TestBlobRewrite(t *testing.T) {
106107
},
107108
blob.FileWriterOptions{},
108109
minimumSize,
110+
0, /* minimum MVCC garbage size */
109111
valsep.WriteNewBlobFilesOptions{},
110112
)
111113
vs = newSep

compaction.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3486,6 +3486,7 @@ func (d *DB) compactAndWrite(
34863486
vSep.SetNextOutputConfig(valsep.ValueSeparationOutputConfig{
34873487
MinimumSize: spanPolicy.ValueStoragePolicy.OverrideBlobSeparationMinimumSize,
34883488
DisableValueSeparationBySuffix: spanPolicy.ValueStoragePolicy.DisableSeparationBySuffix,
3489+
MinimumMVCCGarbageSize: spanPolicy.ValueStoragePolicy.MinimumMVCCGarbageSize,
34893490
})
34903491
}
34913492
objMeta, tw, err := d.newCompactionOutputTable(jobID, c, writerOpts)

compaction_test.go

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1543,6 +1543,15 @@ func runCompactionTest(
15431543
if size == 0 {
15441544
policy.ValueStoragePolicy.DisableBlobSeparation = true
15451545
}
1546+
case "minimum-mvcc-garbage-size":
1547+
if len(parts) != 2 {
1548+
td.Fatalf(t, "expected minimum-mvcc-garbage-size=<size>, got: %s", arg)
1549+
}
1550+
size, err := strconv.ParseUint(parts[1], 10, 64)
1551+
if err != nil {
1552+
td.Fatalf(t, "parsing minimum-mvcc-garbage-size: %s", err)
1553+
}
1554+
policy.ValueStoragePolicy.MinimumMVCCGarbageSize = int(size)
15461555
default:
15471556
td.Fatalf(t, "unknown span policy arg: %s", arg)
15481557
}

compaction_value_separation.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ func (d *DB) determineCompactionValueSeparation(
4848
blobFileSet,
4949
outputBlobReferenceDepth,
5050
policy.MinimumSize,
51+
policy.MinimumMVCCGarbageSize,
5152
)
5253
}
5354

@@ -60,6 +61,7 @@ func (d *DB) determineCompactionValueSeparation(
6061
},
6162
d.makeBlobWriterOptions(c.outputLevel.level),
6263
policy.MinimumSize,
64+
policy.MinimumMVCCGarbageSize,
6365
valsep.WriteNewBlobFilesOptions{
6466
InputBlobPhysicalFiles: blobFileSet,
6567
ShortAttrExtractor: d.opts.Experimental.ShortAttributeExtractor,

data_test.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -919,6 +919,7 @@ func runDBDefineCmd(td *datadriven.TestData, opts *Options) (*DB, error) {
919919
valueSeparator.metas,
920920
0, /* outputreference depth */
921921
d.opts.Experimental.ValueSeparationPolicy().MinimumSize,
922+
d.opts.Experimental.ValueSeparationPolicy().MinimumMVCCGarbageSize,
922923
)
923924

924925
var mem *memTable
@@ -1847,6 +1848,7 @@ func parseDBOptionsArgs(opts *Options, args []datadriven.CmdArg) error {
18471848
value = arg[i+1:]
18481849
}
18491850
policy.MinimumLatencyTolerantSize = 10
1851+
policy.MinimumMVCCGarbageSize = 1
18501852
var err error
18511853
switch name {
18521854
case "enabled", "disabled":
@@ -1861,6 +1863,11 @@ func parseDBOptionsArgs(opts *Options, args []datadriven.CmdArg) error {
18611863
if err != nil {
18621864
return err
18631865
}
1866+
case "min-mvcc-garbage-size":
1867+
policy.MinimumMVCCGarbageSize, err = strconv.Atoi(value)
1868+
if err != nil {
1869+
return err
1870+
}
18641871
case "max-ref-depth":
18651872
policy.MaxBlobReferenceDepth, err = strconv.Atoi(value)
18661873
if err != nil {

event_listener_test.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -636,6 +636,7 @@ func TestBlobCorruptionEvent(t *testing.T) {
636636
Enabled: true,
637637
MinimumSize: 1,
638638
MinimumLatencyTolerantSize: 10,
639+
MinimumMVCCGarbageSize: 10,
639640
MaxBlobReferenceDepth: 10,
640641
}
641642
}

internal/compact/run.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -301,7 +301,8 @@ func (r *Runner) writeKeysToTable(
301301
}
302302

303303
valueLen := kv.V.Len()
304-
isLikelyMVCCGarbage := sstable.IsLikelyMVCCGarbage(kv.K.UserKey, prevKeyKind, kv.K.Kind(), valueLen, prefixEqual)
304+
isLikelyMVCCGarbage := valueLen > valueSeparation.OutputConfig().MinimumMVCCGarbageSize &&
305+
sstable.IsLikelyMVCCGarbage(kv.K.UserKey, prevKeyKind, kv.K.Kind(), valueLen, prefixEqual)
305306
// Add the value to the sstable, possibly separating its value into a
306307
// blob file. The ValueSeparation implementation is responsible for
307308
// writing the KV to the sstable.

iterator_test.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1423,6 +1423,7 @@ func TestIteratorValueRetrievalProfile(t *testing.T) {
14231423
Enabled: true,
14241424
MinimumSize: 1,
14251425
MinimumLatencyTolerantSize: 10,
1426+
MinimumMVCCGarbageSize: 10,
14261427
MaxBlobReferenceDepth: 5,
14271428
}
14281429
}

metamorphic/options.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -314,6 +314,7 @@ func defaultOptions(kf KeyFormat) *pebble.Options {
314314
Enabled: true,
315315
MinimumSize: 5,
316316
MinimumLatencyTolerantSize: 10,
317+
MinimumMVCCGarbageSize: 10,
317318
MaxBlobReferenceDepth: 3,
318319
RewriteMinimumAge: 50 * time.Millisecond,
319320
GarbageRatioLowPriority: 0.10, // 10% garbage
@@ -900,6 +901,7 @@ func RandomOptions(rng *rand.Rand, kf KeyFormat, cfg RandomOptionsCfg) *TestOpti
900901
Enabled: true,
901902
MinimumSize: 1 + rng.IntN(maxValueSize),
902903
MinimumLatencyTolerantSize: 5 + rng.IntN(11), // [5, 15] bytes
904+
MinimumMVCCGarbageSize: 5 + rng.IntN(11), // [5, 15] bytes
903905
MaxBlobReferenceDepth: 2 + rng.IntN(9), // 2-10
904906
RewriteMinimumAge: time.Duration(rng.IntN(90)+10) * time.Millisecond, // [10ms, 100ms)
905907
GarbageRatioLowPriority: lowPri,

metrics_test.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -272,6 +272,7 @@ func TestMetrics(t *testing.T) {
272272
Enabled: true,
273273
MinimumSize: 3,
274274
MinimumLatencyTolerantSize: 10,
275+
MinimumMVCCGarbageSize: 10,
275276
MaxBlobReferenceDepth: 5,
276277
}
277278
}

0 commit comments

Comments
 (0)