db: make iterator tracking configurable

RaduBerinde · RaduBerinde · commit 45d293d8d270 · 2025-10-02T15:40:40.000-07:00
diff --git a/db.go b/db.go
@@ -1114,7 +1114,7 @@ func (d *DB) newIter(
 		dbi.batch.batch = batch
 		dbi.batch.batchSeqNum = batch.nextSeqNum()
 	}
-	if !dbi.batchOnlyIter {
+	if !dbi.batchOnlyIter && d.iterTracker != nil {
 		dbi.tracker = d.iterTracker
 		dbi.trackerHandle = d.iterTracker.Start()
 	}
@@ -1624,8 +1624,10 @@ func (d *DB) Close() error {
 		err = firstError(err, errors.Errorf("leaked snapshots: %d open snapshots on DB %p", v, d))
 	}
 
-	d.iterTracker.Close()
-	d.iterTracker = nil
+	if d.iterTracker != nil {
+		d.iterTracker.Close()
+		d.iterTracker = nil
+	}
 
 	return err
 }
diff --git a/metamorphic/options.go b/metamorphic/options.go
@@ -934,6 +934,20 @@ func RandomOptions(rng *rand.Rand, kf KeyFormat, cfg RandomOptionsCfg) *TestOpti
 		}
 	}
 
+	switch rand.IntN(2) {
+	case 0:
+		opts.Experimental.IteratorTracking.PollInterval = 100 * time.Millisecond
+		opts.Experimental.IteratorTracking.MaxAge = 10 * time.Second
+	case 1:
+		// Disable tracking.
+		opts.Experimental.IteratorTracking.PollInterval = -1
+		opts.Experimental.IteratorTracking.MaxAge = -1
+	case 2:
+		// Default settings.
+		opts.Experimental.IteratorTracking.PollInterval = 0
+		opts.Experimental.IteratorTracking.MaxAge = 0
+	}
+
 	testOpts.seedEFOS = rng.Uint64()
 	testOpts.ingestSplit = rng.IntN(2) == 0
 	opts.Experimental.IngestSplit = func() bool { return testOpts.ingestSplit }
diff --git a/open.go b/open.go
@@ -205,11 +205,11 @@ func Open(dirname string, opts *Options) (db *DB, err error) {
 	} else {
 		d.compactionScheduler = newConcurrencyLimitScheduler(defaultTimeSource{})
 	}
-	const iterTrackerPollInterval = 5 * time.Minute
-	const iterTrackerMaxAge = 1 * time.Minute
-	d.iterTracker = inflight.NewPollingTracker(iterTrackerPollInterval, iterTrackerMaxAge, func(report string) {
-		d.opts.Logger.Infof("Long-lived iterators detected:\n%s", report)
-	})
+	if iterTrackOpts := opts.Experimental.IteratorTracking; iterTrackOpts.PollInterval > 0 && iterTrackOpts.MaxAge > 0 {
+		d.iterTracker = inflight.NewPollingTracker(iterTrackOpts.PollInterval, iterTrackOpts.MaxAge, func(report string) {
+			d.opts.Logger.Infof("Long-lived iterators detected:\n%s", report)
+		})
+	}
 
 	defer func() {
 		// If an error or panic occurs during open, attempt to release the manually
@@ -242,7 +242,10 @@ func Open(dirname string, opts *Options) (db *DB, err error) {
 			if d.mu.versions.manifestFile != nil {
 				_ = d.mu.versions.manifestFile.Close()
 			}
-			d.iterTracker.Close()
+			if d.iterTracker != nil {
+				d.iterTracker.Close()
+				d.iterTracker = nil
+			}
 			if r != nil {
 				panic(r)
 			}
diff --git a/options.go b/options.go
@@ -792,6 +792,21 @@ type Options struct {
 		// virtual table rewrite compactions entirely. The default value is 0.30
 		// (rewrite when >= 30% of backing data is unreferenced).
 		VirtualTableRewriteUnreferencedFraction func() float64
+
+		// IteratorTracking configures periodic logging of iterators held open for
+		// too long.
+		IteratorTracking struct {
+			// PollInterval is the interval at which to log a report of long-lived iterators.
+			// If negative, disables iterator tracking.
+			//
+			// The default value is 5 minutes.
+			PollInterval time.Duration
+			// MaxAge is the age above which iterators are considered long-lived. If
+			// negative, disables iterator tracking.
+			//
+			// The default value is 1 minute.
+			MaxAge time.Duration
+		}
 	}
 
 	// Filters is a map from filter policy name to filter policy. It is used for
@@ -1644,6 +1659,12 @@ func (o *Options) EnsureDefaults() {
 	if o.Experimental.VirtualTableRewriteUnreferencedFraction == nil {
 		o.Experimental.VirtualTableRewriteUnreferencedFraction = func() float64 { return defaultVirtualTableUnreferencedFraction }
 	}
+	if o.Experimental.IteratorTracking.PollInterval == 0 {
+		o.Experimental.IteratorTracking.PollInterval = 5 * time.Minute
+	}
+	if o.Experimental.IteratorTracking.MaxAge == 0 {
+		o.Experimental.IteratorTracking.MaxAge = time.Minute
+	}
 	// TODO(jackson): Enable value separation by default once we have confidence
 	// in a default policy.
 
@@ -1786,6 +1807,13 @@ func (o *Options) String() string {
 	fmt.Fprintf(&buf, "  secondary_cache_size_bytes=%d\n", o.Experimental.SecondaryCacheSizeBytes)
 	fmt.Fprintf(&buf, "  create_on_shared=%d\n", o.Experimental.CreateOnShared)
 
+	if o.Experimental.IteratorTracking.PollInterval != 0 {
+		fmt.Fprintf(&buf, "  iterator_tracking_poll_interval=%s\n", o.Experimental.IteratorTracking.PollInterval)
+	}
+	if o.Experimental.IteratorTracking.MaxAge != 0 {
+		fmt.Fprintf(&buf, "  iterator_tracking_max_age=%s\n", o.Experimental.IteratorTracking.MaxAge)
+	}
+
 	// Private options.
 	//
 	// These options are only encoded if true, because we do not want them to
@@ -2228,6 +2256,10 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error {
 				var createOnSharedInt int64
 				createOnSharedInt, err = strconv.ParseInt(value, 10, 64)
 				o.Experimental.CreateOnShared = remote.CreateOnSharedStrategy(createOnSharedInt)
+			case "iterator_tracking_poll_interval":
+				o.Experimental.IteratorTracking.PollInterval, err = time.ParseDuration(value)
+			case "iterator_tracking_max_age":
+				o.Experimental.IteratorTracking.MaxAge, err = time.ParseDuration(value)
 			default:
 				if hooks != nil && hooks.SkipUnknown != nil && hooks.SkipUnknown(section+"."+key, value) {
 					return nil
diff --git a/options_test.go b/options_test.go
@@ -140,6 +140,8 @@ func TestDefaultOptionsString(t *testing.T) {
   wal_bytes_per_sync=0
   secondary_cache_size_bytes=0
   create_on_shared=0
+  iterator_tracking_poll_interval=5m0s
+  iterator_tracking_max_age=1m0s
 
 [Level "0"]
   block_restart_interval=16
diff --git a/replay/testdata/replay b/replay/testdata/replay
@@ -11,7 +11,7 @@ tree
      508      000007.sst
        0      LOCK
      133      MANIFEST-000001
-    2769      OPTIONS-000003
+    2841      OPTIONS-000003
        0      marker.format-version.000001.013
        0      marker.manifest.000001.MANIFEST-000001
             simple/
@@ -21,7 +21,7 @@ tree
       11        000004.log
      480        000005.sst
       85        MANIFEST-000001
-    2769        OPTIONS-000003
+    2841        OPTIONS-000003
        0        marker.format-version.000001.013
        0        marker.manifest.000001.MANIFEST-000001
 
@@ -75,6 +75,8 @@ cat build/OPTIONS-000003
   wal_bytes_per_sync=0
   secondary_cache_size_bytes=0
   create_on_shared=0
+  iterator_tracking_poll_interval=5m0s
+  iterator_tracking_max_age=1m0s
 
 [Level "0"]
   block_restart_interval=16
diff --git a/replay/testdata/replay_ingest b/replay/testdata/replay_ingest
@@ -12,7 +12,7 @@ tree
        0      LOCK
      172      MANIFEST-000001
      209      MANIFEST-000008
-    2770      OPTIONS-000003
+    2842      OPTIONS-000003
        0      marker.format-version.000012.025
        0      marker.manifest.000002.MANIFEST-000008
             simple_ingest/
@@ -25,7 +25,7 @@ tree
      678        000004.sst
      661        000005.sst
      172        MANIFEST-000001
-    2770        OPTIONS-000003
+    2842        OPTIONS-000003
        0        marker.format-version.000001.025
        0        marker.manifest.000001.MANIFEST-000001
 
@@ -79,6 +79,8 @@ cat build/OPTIONS-000003
   wal_bytes_per_sync=0
   secondary_cache_size_bytes=0
   create_on_shared=0
+  iterator_tracking_poll_interval=5m0s
+  iterator_tracking_max_age=1m0s
 
 [Level "0"]
   block_restart_interval=16
diff --git a/replay/testdata/replay_paced b/replay/testdata/replay_paced
@@ -14,7 +14,7 @@ tree
        0      LOCK
      133      MANIFEST-000001
      205      MANIFEST-000010
-    2769      OPTIONS-000003
+    2841      OPTIONS-000003
        0      marker.format-version.000001.013
        0      marker.manifest.000002.MANIFEST-000010
             high_read_amp/
@@ -26,7 +26,7 @@ tree
       11        000008.log
      454        000009.sst
      157        MANIFEST-000010
-    2769        OPTIONS-000003
+    2841        OPTIONS-000003
        0        marker.format-version.000001.013
        0        marker.manifest.000001.MANIFEST-000010
 
diff --git a/replay/testdata/replay_val_sep b/replay/testdata/replay_val_sep
@@ -17,7 +17,7 @@ tree
        0      LOCK
      152      MANIFEST-000010
      250      MANIFEST-000013
-    2947      OPTIONS-000003
+    3019      OPTIONS-000003
        0      marker.format-version.000011.024
        0      marker.manifest.000003.MANIFEST-000013
             simple_val_sep/
@@ -32,7 +32,7 @@ tree
       11        000011.log
      707        000012.sst
      187        MANIFEST-000013
-    2947        OPTIONS-000003
+    3019        OPTIONS-000003
        0        marker.format-version.000001.024
        0        marker.manifest.000001.MANIFEST-000013
 
@@ -89,6 +89,8 @@ cat build/OPTIONS-000003
   wal_bytes_per_sync=0
   secondary_cache_size_bytes=0
   create_on_shared=0
+  iterator_tracking_poll_interval=5m0s
+  iterator_tracking_max_age=1m0s
 
 [Value Separation]
   enabled=true
diff --git a/scan_internal.go b/scan_internal.go
@@ -188,7 +188,6 @@ func (d *DB) newInternalIter(
 	*dbi = scanInternalIterator{
 		ctx:             ctx,
 		db:              d,
-		trackerHandle:   d.iterTracker.Start(),
 		comparer:        d.opts.Comparer,
 		merge:           d.opts.Merger.Merge,
 		readState:       readState,
@@ -199,6 +198,9 @@ func (d *DB) newInternalIter(
 		seqNum:          seqNum,
 		mergingIter:     &buf.iterAlloc.merging,
 	}
+	if d.iterTracker != nil {
+		dbi.trackerHandle = d.iterTracker.Start()
+	}
 	dbi.blobValueFetcher.Init(&vers.BlobFiles, d.fileCache, block.ReadEnv{},
 		blob.SuggestedCachedReaders(vers.MaxReadAmp()))
 
@@ -1286,7 +1288,10 @@ func (i *scanInternalIterator) error() error {
 
 // Close closes this iterator, and releases any pooled objects.
 func (i *scanInternalIterator) Close() error {
-	i.db.iterTracker.Stop(i.trackerHandle)
+	if i.trackerHandle != 0 {
+		i.db.iterTracker.Stop(i.trackerHandle)
+		i.trackerHandle = 0
+	}
 	err := i.iter.Close()
 	err = errors.CombineErrors(err, i.blobValueFetcher.Close())
 	if i.readState != nil {
diff --git a/testdata/metrics b/testdata/metrics
@@ -184,7 +184,7 @@ Iter category stats:
 
 disk-usage
 ----
-3,709B
+3,781B
 
 batch
 set b 2
@@ -287,7 +287,7 @@ Iter category stats:
 
 disk-usage
 ----
-6,002B
+6,074B
 
 # Closing iter a will release one of the zombie memtables.
 
@@ -454,7 +454,7 @@ Iter category stats:
 
 disk-usage
 ----
-5,295B
+5,367B
 
 # Closing iter b will release the last zombie sstable and the last zombie memtable.
 
@@ -539,7 +539,7 @@ Iter category stats:
 
 disk-usage
 ----
-4,588B
+4,660B
 
 additional-metrics
 ----

Original file line number	Diff line number	Diff line change
`@@ -1114,7 +1114,7 @@ func (d *DB) newIter(`
`1114`	`1114`	`dbi.batch.batch = batch`
`1115`	`1115`	`dbi.batch.batchSeqNum = batch.nextSeqNum()`
`1116`	`1116`	`}`
`1117`		`- if !dbi.batchOnlyIter {`
	`1117`	`+ if !dbi.batchOnlyIter && d.iterTracker != nil {`
`1118`	`1118`	`dbi.tracker = d.iterTracker`
`1119`	`1119`	`dbi.trackerHandle = d.iterTracker.Start()`
`1120`	`1120`	`}`
`@@ -1624,8 +1624,10 @@ func (d *DB) Close() error {`
`1624`	`1624`	`err = firstError(err, errors.Errorf("leaked snapshots: %d open snapshots on DB %p", v, d))`
`1625`	`1625`	`}`
`1626`	`1626`
`1627`		`- d.iterTracker.Close()`
`1628`		`- d.iterTracker = nil`
	`1627`	`+ if d.iterTracker != nil {`
	`1628`	`+ d.iterTracker.Close()`
	`1629`	`+ d.iterTracker = nil`
	`1630`	`+ }`
`1629`	`1631`
`1630`	`1632`	`return err`
`1631`	`1633`	`}`