storage: add cluster setting to expire sticky bits

Since we represent sticky bits as HLCs, we can provide a setting to have
them expire after a certain duration. After this duration, these
manually split ranges will be eligible for automatic merging again.

This setting is useful for cases when a user might want to split the
data as it's being loaded into tables, but after the loading is
complete, the user would want the manual splits be merged as determined
by the merge queue.

Resolves: #37696

Release note (general change): Add kv.range_merge.manual_split_ttl
                               setting

docs/generated/settings/settings.html

@@ -43,6 +43,7 @@
 <tr><td><code>kv.raft_log.disable_synchronization_unsafe</code></td><td>boolean</td><td><code>false</code></td><td>set to true to disable synchronization on Raft log writes to persistent storage. Setting to true risks data loss or data corruption on server crashes. The setting is meant for internal testing only and SHOULD NOT be used in production.</td></tr>
 <tr><td><code>kv.range.backpressure_range_size_multiplier</code></td><td>float</td><td><code>2</code></td><td>multiple of range_max_bytes that a range is allowed to grow to without splitting before writes to that range are blocked, or 0 to disable</td></tr>
 <tr><td><code>kv.range_descriptor_cache.size</code></td><td>integer</td><td><code>1000000</code></td><td>maximum number of entries in the range descriptor and leaseholder caches</td></tr>
+<tr><td><code>kv.range_merge.manual_split.ttl</code></td><td>duration</td><td><code>0s</code></td><td>if nonzero, manual splits older than this duration will be considered for automatic range merging</td></tr>
 <tr><td><code>kv.range_merge.queue_enabled</code></td><td>boolean</td><td><code>true</code></td><td>whether the automatic merge queue is enabled</td></tr>
 <tr><td><code>kv.range_merge.queue_interval</code></td><td>duration</td><td><code>1s</code></td><td>how long the merge queue waits between processing replicas (WARNING: may compromise cluster stability or correctness; do not edit without supervision)</td></tr>
 <tr><td><code>kv.range_split.by_load_enabled</code></td><td>boolean</td><td><code>true</code></td><td>allow automatic splits of ranges based on where load is concentrated</td></tr>

pkg/storage/client_merge_test.go

@@ -3277,18 +3277,24 @@ func TestMergeQueue(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 
 	ctx := context.Background()
+	manualClock := hlc.NewManualClock(123)
+	clock := hlc.NewClock(manualClock.UnixNano, time.Nanosecond)
 	storeCfg := storage.TestStoreConfig(nil)
 	storeCfg.TestingKnobs.DisableSplitQueue = true
 	storeCfg.TestingKnobs.DisableScanner = true
 	sv := &storeCfg.Settings.SV
 	storagebase.MergeQueueEnabled.Override(sv, true)
 	storage.MergeQueueInterval.Override(sv, 0) // process greedily
+	manualSplitTTL := time.Millisecond * 200
+	storage.ManualSplitTTL.Override(sv, manualSplitTTL)
 	var mtc multiTestContext
 	// This test was written before the multiTestContext started creating many
 	// system ranges at startup, and hasn't been update to take that into account.
 	mtc.startWithSingleRange = true
 
 	mtc.storeConfig = &storeCfg
+	// Inject clock for manipulation in tests
+	mtc.clock = clock
 	mtc.Start(t, 2)
 	defer mtc.Stop()
 	mtc.initGossipNetwork() // needed for the non-collocated case's rebalancing to work
@@ -3443,6 +3449,28 @@ func TestMergeQueue(t *testing.T) {
 		store.MustForceMergeScanAndProcess()
 		verifyMerged(t)
 	})
+
+	t.Run("sticky bit ttl", func(t *testing.T) {
+		reset(t)
+		store.MustForceMergeScanAndProcess()
+		verifyUnmerged(t)
+
+		// Perform manual merge and verify that no merge occurred.
+		split(t, rhsStartKey.AsRawKey(), true /* manual */)
+		clearRange(t, lhsStartKey, rhsEndKey)
+		store.MustForceMergeScanAndProcess()
+		verifyUnmerged(t)
+
+		// Sticky bit is not expired yet.
+		manualClock.Set(manualSplitTTL.Nanoseconds())
+		store.MustForceMergeScanAndProcess()
+		verifyUnmerged(t)
+
+		// Sticky bit is expired.
+		manualClock.Set(manualSplitTTL.Nanoseconds() * 2)
+		store.MustForceMergeScanAndProcess()
+		verifyMerged(t)
+	})
 }
 
 func TestInvalidSubsumeRequest(t *testing.T) {

pkg/storage/merge_queue.go

@@ -58,6 +58,17 @@ var MergeQueueInterval = func() *settings.DurationSetting {
 	return s
 }()
 
+// ManualSplitTTL is the amount of time that the merge queue will not consider
+// a manually split range for merging if it is nonzero. If ManualSplitTTL is
+// zero, then no manual splits are considered for automatic merging. If a range
+// is split in a LHS and a RHS, then the merge queue will not consider to merge
+// the RHS with the range to its left until ManualSplitTTL time has passed.
+var ManualSplitTTL = settings.RegisterNonNegativeDurationSetting(
+	"kv.range_merge.manual_split.ttl",
+	"if nonzero, manual splits older than this duration will be considered for automatic range merging",
+	0,
+)
+
 // mergeQueue manages a queue of ranges slated to be merged with their right-
 // hand neighbor.
 //
@@ -291,12 +302,17 @@ func (mq *mergeQueue) process(
 		}
 	}
 
-	// Range was manually split, so skip merging.
+	// Range was manually split and not expired, so skip merging.
 	if rhsDesc.StickyBit != nil {
-		log.VEventf(ctx, 2, "skipping merge: ranges were manually split")
-		// TODO(jeffreyxiao): Consider returning a purgatory error to avoid
-		// repeatedly processing ranges that cannot be merged.
-		return nil
+		manualSplitTTL := ManualSplitTTL.Get(&mq.store.ClusterSettings().SV)
+		manualSplitExpired := manualSplitTTL != 0 && rhsDesc.StickyBit.GoTime().Add(manualSplitTTL).Before(mq.store.Clock().PhysicalTime())
+		if !manualSplitExpired {
+			log.VEventf(ctx, 2, "skipping merge: ranges were manually split and sticky bit was not expired")
+			// TODO(jeffreyxiao): Consider returning a purgatory error to avoid
+			// repeatedly processing ranges that cannot be merged.
+			return nil
+		}
+		log.VEventf(ctx, 2, "ranges were manually split, but sticky bit was expired")
 	}
 
 	log.VEventf(ctx, 2, "merging to produce range: %s-%s", mergedDesc.StartKey, mergedDesc.EndKey)