allow delete pacing knobs to be configurable

Author: darshanime

obsolete_files.go

@@ -88,8 +88,16 @@ func openCleanupManager(
 		opts:            opts,
 		objProvider:     objProvider,
 		onTableDeleteFn: onTableDeleteFn,
-		deletePacer:     newDeletionPacer(crtime.NowMono(), int64(opts.TargetByteDeletionRate), getDeletePacerInfo),
-		jobsCh:          make(chan *cleanupJob, jobsQueueDepth),
+		deletePacer: newDeletionPacer(
+			crtime.NowMono(),
+			opts.FreeSpaceThresholdBytes,
+			int64(opts.TargetByteDeletionRate),
+			opts.FreeSpaceTimeframe,
+			opts.ObsoleteBytesMaxRatio,
+			opts.ObsoleteBytesTimeframe,
+			getDeletePacerInfo,
+		),
+		jobsCh: make(chan *cleanupJob, jobsQueueDepth),
 	}
 	cm.mu.completedJobsCond.L = &cm.mu.Mutex
 	cm.waitGroup.Add(1)

options.go

@@ -1035,20 +1035,42 @@ type Options struct {
 	// changing options dynamically?
 	WALMinSyncInterval func() time.Duration
 
+	// The controls below manage deletion pacing, which slows down
+	// deletions when compactions finish or when readers close and
+	// obsolete files must be cleaned up. Rapid deletion of many
+	// files simultaneously can increase disk latency on certain
+	// SSDs, and this functionality helps protect against that.
+
 	// TargetByteDeletionRate is the rate (in bytes per second) at which sstable file
 	// deletions are limited to (under normal circumstances).
 	//
-	// Deletion pacing is used to slow down deletions when compactions finish up
-	// or readers close and newly-obsolete files need cleaning up. Deleting lots
-	// of files at once can cause disk latency to go up on some SSDs, which this
-	// functionality guards against.
-	//
 	// This value is only a best-effort target; the effective rate can be
 	// higher if deletions are falling behind or disk space is running low.
 	//
 	// Setting this to 0 disables deletion pacing, which is also the default.
 	TargetByteDeletionRate int
 
+	// FreeSpaceThresholdBytes specifies the minimum amount of free disk space that Pebble
+	// attempts to maintain. If free disk space drops below this threshold, deletions
+	// are accelerated above TargetByteDeletionRate until the threshold is restored.
+	// Default is 16GB.
+	FreeSpaceThresholdBytes uint64
+
+	// FreeSpaceTimeframe sets the duration (in seconds) within which Pebble attempts
+	// to restore the free disk space back to FreeSpaceThreshold. A lower value means
+	// more aggressive deletions. Default is 10s.
+	FreeSpaceTimeframe time.Duration
+
+	// ObsoleteBytesMaxRatio specifies the maximum allowed ratio of obsolete files to
+	// live files. If this ratio is exceeded, Pebble speeds up deletions above the
+	// TargetByteDeletionRate until the ratio is restored. Default is 0.20.
+	ObsoleteBytesMaxRatio float64
+
+	// ObsoleteBytesTimeframe sets the duration (in seconds) within which Pebble aims
+	// to restore the obsolete-to-live bytes ratio below ObsoleteBytesMaxRatio. A lower
+	// value means more aggressive deletions. Default is 300s.
+	ObsoleteBytesTimeframe time.Duration
+
 	// EnableSQLRowSpillMetrics specifies whether the Pebble instance will only be used
 	// to temporarily persist data spilled to disk for row-oriented SQL query execution.
 	EnableSQLRowSpillMetrics bool
@@ -1135,6 +1157,22 @@ func (o *Options) EnsureDefaults() {
 		o.Cleaner = DeleteCleaner{}
 	}
 
+	if o.FreeSpaceThresholdBytes == 0 {
+		o.FreeSpaceThresholdBytes = 16 << 30 // 16 GB
+	}
+
+	if o.FreeSpaceTimeframe == 0 {
+		o.FreeSpaceTimeframe = 10 * time.Second
+	}
+
+	if o.ObsoleteBytesMaxRatio == 0 {
+		o.ObsoleteBytesMaxRatio = 0.20
+	}
+
+	if o.ObsoleteBytesTimeframe == 0 {
+		o.ObsoleteBytesTimeframe = 300 * time.Second
+	}
+
 	if o.Experimental.DisableIngestAsFlushable == nil {
 		o.Experimental.DisableIngestAsFlushable = func() bool { return false }
 	}
@@ -1393,6 +1431,10 @@ func (o *Options) String() string {
 	fmt.Fprintf(&buf, "  mem_table_size=%d\n", o.MemTableSize)
 	fmt.Fprintf(&buf, "  mem_table_stop_writes_threshold=%d\n", o.MemTableStopWritesThreshold)
 	fmt.Fprintf(&buf, "  min_deletion_rate=%d\n", o.TargetByteDeletionRate)
+	fmt.Fprintf(&buf, "  free_space_threshold_bytes=%d\n", o.FreeSpaceThresholdBytes)
+	fmt.Fprintf(&buf, "  free_space_timeframe=%s\n", o.FreeSpaceTimeframe.String())
+	fmt.Fprintf(&buf, "  obsolete_bytes_max_ratio=%f\n", o.ObsoleteBytesMaxRatio)
+	fmt.Fprintf(&buf, "  obsolete_bytes_timeframe=%s\n", o.ObsoleteBytesTimeframe.String())
 	fmt.Fprintf(&buf, "  merger=%s\n", o.Merger.Name)
 	if o.Experimental.MultiLevelCompactionHeuristic != nil {
 		fmt.Fprintf(&buf, "  multilevel_compaction_heuristic=%s\n", o.Experimental.MultiLevelCompactionHeuristic.String())
@@ -1729,6 +1771,14 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error {
 				// may be meaningful again eventually.
 			case "min_deletion_rate":
 				o.TargetByteDeletionRate, err = strconv.Atoi(value)
+			case "free_space_threshold_bytes":
+				o.FreeSpaceThresholdBytes, err = strconv.ParseUint(value, 10, 64)
+			case "free_space_timeframe":
+				o.FreeSpaceTimeframe, err = time.ParseDuration(value)
+			case "obsolete_bytes_max_ratio":
+				o.ObsoleteBytesMaxRatio, err = strconv.ParseFloat(value, 64)
+			case "obsolete_bytes_timeframe":
+				o.ObsoleteBytesTimeframe, err = time.ParseDuration(value)
 			case "min_flush_rate":
 				// Do nothing; option existed in older versions of pebble, and
 				// may be meaningful again eventually.

options_test.go

@@ -103,6 +103,10 @@ func TestDefaultOptionsString(t *testing.T) {
   mem_table_size=4194304
   mem_table_stop_writes_threshold=2
   min_deletion_rate=0
+  free_space_threshold_bytes=17179869184
+  free_space_timeframe=10s
+  obsolete_bytes_max_ratio=0.200000
+  obsolete_bytes_timeframe=5m0s
   merger=pebble.concatenate
   multilevel_compaction_heuristic=wamp(0.00, false)
   read_compaction_rate=16000

pacer.go

@@ -60,18 +60,25 @@ const deletePacerHistory = 5 * time.Minute
 // normally limit deletes (when we are not falling behind or running out of
 // space). A value of 0.0 disables pacing.
 func newDeletionPacer(
-	now crtime.Mono, targetByteDeletionRate int64, getInfo func() deletionPacerInfo,
+	now crtime.Mono,
+	freeSpaceThreshold uint64,
+	targetByteDeletionRate int64,
+	freeSpaceTimeframe time.Duration,
+	obsoleteBytesMaxRatio float64,
+	obsoleteBytesTimeframe time.Duration,
+	getInfo func() deletionPacerInfo,
 ) *deletionPacer {
 	d := &deletionPacer{
-		freeSpaceThreshold: 16 << 30, // 16 GB
-		freeSpaceTimeframe: 10 * time.Second,
+		freeSpaceThreshold: freeSpaceThreshold,
+		freeSpaceTimeframe: freeSpaceTimeframe,
 
-		obsoleteBytesMaxRatio:  0.20,
-		obsoleteBytesTimeframe: 5 * time.Minute,
+		obsoleteBytesMaxRatio:  obsoleteBytesMaxRatio,
+		obsoleteBytesTimeframe: obsoleteBytesTimeframe,
 
 		targetByteDeletionRate: targetByteDeletionRate,
 		getInfo:                getInfo,
 	}
+
 	d.mu.history.Init(now, deletePacerHistory)
 	return d
 }

pacer_test.go

@@ -46,7 +46,7 @@ func TestDeletionPacer(t *testing.T) {
 			expected:      304.8,
 		},
 		// As freeBytes is 10GB below the free space threshold, rate should be
-		// increased to by 1GB/s.
+		// increased by 1GB/s.
 		{
 			freeBytes:     6 * GB,
 			obsoleteBytes: 1 * MB,
@@ -121,7 +121,17 @@ func TestDeletionPacer(t *testing.T) {
 			}
 			start := crtime.NowMono()
 			last := start
-			pacer := newDeletionPacer(start, 100*MB, getInfo)
+			var opts Options
+			opts.EnsureDefaults()
+			pacer := newDeletionPacer(
+				start,
+				opts.FreeSpaceThresholdBytes,
+				100*MB,
+				opts.FreeSpaceTimeframe,
+				opts.ObsoleteBytesMaxRatio,
+				opts.ObsoleteBytesTimeframe,
+				getInfo,
+			)
 			for _, h := range tc.history {
 				last = start + crtime.Mono(time.Second*time.Duration(h[0]))
 				pacer.ReportDeletion(last, uint64(h[1]))

replay/testdata/replay

@@ -11,7 +11,7 @@ tree
      614      000007.sst
        0      LOCK
      133      MANIFEST-000001
-    1358      OPTIONS-000003
+    1494      OPTIONS-000003
        0      marker.format-version.000001.013
        0      marker.manifest.000001.MANIFEST-000001
             simple/
@@ -21,7 +21,7 @@ tree
       25        000004.log
      586        000005.sst
       85        MANIFEST-000001
-    1358        OPTIONS-000003
+    1494        OPTIONS-000003
        0        marker.format-version.000001.013
        0        marker.manifest.000001.MANIFEST-000001
 
@@ -55,6 +55,10 @@ cat build/OPTIONS-000003
   mem_table_size=4194304
   mem_table_stop_writes_threshold=2
   min_deletion_rate=0
+  free_space_threshold_bytes=17179869184
+  free_space_timeframe=10s
+  obsolete_bytes_max_ratio=0.200000
+  obsolete_bytes_timeframe=5m0s
   merger=pebble.concatenate
   multilevel_compaction_heuristic=wamp(0.00, false)
   read_compaction_rate=16000

replay/testdata/replay_paced

@@ -14,7 +14,7 @@ tree
        0      LOCK
      133      MANIFEST-000001
      205      MANIFEST-000010
-    1358      OPTIONS-000003
+    1494      OPTIONS-000003
        0      marker.format-version.000001.013
        0      marker.manifest.000002.MANIFEST-000010
             high_read_amp/
@@ -26,7 +26,7 @@ tree
       39        000008.log
      560        000009.sst
      157        MANIFEST-000010
-    1358        OPTIONS-000003
+    1494        OPTIONS-000003
        0        marker.format-version.000001.013
        0        marker.manifest.000001.MANIFEST-000010
 

testdata/metrics

@@ -88,7 +88,7 @@ Iter category stats:
 
 disk-usage
 ----
-2.1KB
+2.3KB
 
 batch
 set b 2
@@ -149,7 +149,7 @@ Iter category stats:
 
 disk-usage
 ----
-3.7KB
+3.8KB
 
 # Closing iter a will release one of the zombie memtables.
 
@@ -238,7 +238,7 @@ Iter category stats:
 
 disk-usage
 ----
-3.0KB
+3.1KB
 
 # Closing iter b will release the last zombie sstable and the last zombie memtable.
 
@@ -284,7 +284,7 @@ Iter category stats:
 
 disk-usage
 ----
-2.3KB
+2.4KB
 
 additional-metrics
 ----