storage: preserve raft log during up-replication

Preserve the raft log, no matter how large it grows, during
up-replication.

Fixes #10409

pkg/storage/raft_log_queue.go

@@ -142,18 +142,25 @@ func computeTruncatableIndex(
 		// the target size. If the raft log is greater than the target size we
 		// always truncate to the quorum commit index.
 		truncatableIndex = getBehindIndex(raftStatus)
-		// The pending snapshot index acts as a placeholder for a replica that is
-		// about to be added to the range. We don't want to truncate the log in a
-		// way that will require that new replica to be caught up via a Raft
-		// snapshot.
-		if pendingSnapshotIndex > 0 && truncatableIndex > pendingSnapshotIndex {
-			truncatableIndex = pendingSnapshotIndex
-		}
-		if truncatableIndex < firstIndex {
-			truncatableIndex = firstIndex
-		}
 	}
 
+	// The pending snapshot index acts as a placeholder for a replica that is
+	// about to be added to the range. We don't want to truncate the log in a
+	// way that will require that new replica to be caught up via a Raft
+	// snapshot.
+	//
+	// TODO(peter): We only need to preserve the pending snapshot index in cases
+	// where adding the new replica can knock the range out of quorum. For
+	// example, if the range currently has 1 replica and we're adding a 2nd, the
+	// range will be unable to make forward progress until the 2nd replica is
+	// brought up to date. Snapshots are throttled and not always available while
+	// Raft log replays are.
+	if pendingSnapshotIndex > 0 && truncatableIndex > pendingSnapshotIndex {
+		truncatableIndex = pendingSnapshotIndex
+	}
+	if truncatableIndex < firstIndex {
+		truncatableIndex = firstIndex
+	}
 	// Never truncate past the quorum committed index.
 	if truncatableIndex > raftStatus.Commit {
 		truncatableIndex = raftStatus.Commit

pkg/storage/raft_log_queue_test.go

@@ -87,6 +87,9 @@ func TestComputeTruncatableIndex(t *testing.T) {
 		{[]uint64{1, 2, 3, 4}, 3, 2000, 1, 0, 3},
 		{[]uint64{1, 2, 3, 4}, 3, 2000, 2, 0, 3},
 		{[]uint64{1, 2, 3, 4}, 3, 2000, 3, 0, 3},
+		// Don't truncate past the pending snapshot index
+		{[]uint64{1, 2, 3, 4}, 3, 2000, 1, 1, 1},
+		{[]uint64{1, 2, 3, 4}, 3, 2000, 1, 2, 2},
 		// Never truncate past raftStatus.Commit.
 		{[]uint64{4, 5, 6}, 3, 100, 4, 0, 3},
 	}