Merge #82677 #82797 #82816

82677: sql, pgwire: Add `SEVERITY_NONLOCALIZED` error field r=e-mbrown a=e-mbrown

Resolves #81794

Release note (sql change): We now send the Severity_Nonlocalized field
in the pgwire Notice Response.

82797: kv: extract etcd/raft utilities into raftutil library r=nvanbenschoten a=nvanbenschoten

Commit pulled from #82758.

This commit extracts some scattered logic to interpret the state of an `etcd/raft` Status into a new `raftutil` library. The library initially has two functions: `ReplicaIsBehind` and `ReplicaMayNeedSnapshot`. In the future, I expect that we'll extract more helper functions that make working with `etcd/raft` easier into this library.

82816: Upgrade Go to version 1.17.11 r=jlinder,rickystewart a=rail

* [x] Adjust version in Docker image
* [x] Adjust version in the TeamCity agent image
* [x] Rebuild and push the Docker image
* [x] Download ALL the archives (`.tar.gz`, `.zip`) for the new Go version from https://golang.org/dl/ and mirror them in the  `public-bazel-artifacts` bucket in the `Bazel artifacts` project in GCP (sub-directory `go`, next to the other Go SDK's).
* [x] Bump the version in `WORKSPACE` under `go_download_sdk`. You may need to bump [rules_go](https://github.com/bazelbuild/rules_go/releases). Also edit the filenames listed in `sdks` and update all the hashes to match what you mirrored in the step above.
* [x] Run `./dev generate bazel` to refresh `distdir_files.bzl`, then `bazel fetch `@distdir//:archives`` to ensure you've updated all hashes to the correct value.
* [x] Bump the version in `builder.sh` accordingly
* [x] Bump the default installed version of Go in `bootstrap-debian.sh`
* [ ] Update the `builder.dockerImage` parameter in the TeamCity [`Cockroach`](https://teamcity.cockroachdb.com/admin/editProject.html?projectId=Cockroach&tab=projectParams) and  [`Internal`](https://teamcity.cockroachdb.com/admin/editProject.html?projectId=Internal&tab=projectParams) projects.
* [ ] Deploy new TeamCity agent images

Partially addresses #82808

Release note: None


Co-authored-by: e-mbrown <ebsonari@gmail.com>
Co-authored-by: Nathan VanBenschoten <nvanbenschoten@gmail.com>
Co-authored-by: Rail Aliiev <rail@iqchoice.com>

WORKSPACE

@@ -158,15 +158,15 @@ load(
 go_download_sdk(
     name = "go_sdk",
     sdks = {
-        "darwin_amd64": ("go1.17.10.darwin-amd64.tar.gz", "84979d5985c70cee6f303050a7e811440aad7f304efdf28665b200f096b01945"),
-        "darwin_arm64": ("go1.17.10.darwin-arm64.tar.gz", "32098bea40117ea1ec23e7124cd188db6bdddd0ea41e2ec9bea3ba35a487e39c"),
-        "freebsd_amd64": ("go1.17.10.freebsd-amd64.tar.gz", "33794d96f58608fdc023c5114ae9baeeb4111a74720c8830ff25029debe508f0"),
-        "linux_amd64": ("go1.17.10.linux-amd64.tar.gz", "87fc728c9c731e2f74e4a999ef53cf07302d7ed3504b0839027bd9c10edaa3fd"),
-        "linux_arm64": ("go1.17.10.linux-arm64.tar.gz", "649141201efa7195403eb1301b95dc79c5b3e65968986a391da1370521701b0c"),
-        "windows_amd64": ("go1.17.10.windows-amd64.zip", "ba9198a29fa5c4f322212d21569e8507165c3b34e1ed1f1f9cf6dfb71ddcdeb2"),
+        "darwin_amd64": ("go1.17.11.darwin-amd64.tar.gz", "4f924c534230de8f0e1c7369f611c0310efd21fc2d9438b13bc2703af9dda25a"),
+        "darwin_arm64": ("go1.17.11.darwin-arm64.tar.gz", "b8e1ab009c2ff8dea462c7a1263d1f3f38e90ab5262e74c76d70e41a4db320be"),
+        "freebsd_amd64": ("go1.17.11.freebsd-amd64.tar.gz", "da78bcd5efa24cfa8ca3ccf0d222f7d66b755c4200d404869984ebdcfc7b6aa7"),
+        "linux_amd64": ("go1.17.11.linux-amd64.tar.gz", "d69a4fe2694f795d8e525c72b497ededc209cb7185f4c3b62d7a98dd6227b3fe"),
+        "linux_arm64": ("go1.17.11.linux-arm64.tar.gz", "adefa7412c6798f9cad02d1e8336fc2242f5bade30c5b32781759181e01961b7"),
+        "windows_amd64": ("go1.17.11.windows-amd64.zip", "88e60b92069d8e0932ca5d8bd8227d1693b9570fa2afbedadcc680749c428d54"),
     },
     urls = ["https://storage.googleapis.com/public-bazel-artifacts/go/{}"],
-    version = "1.17.10",
+    version = "1.17.11",
 )
 
 # To point to a local SDK path, use the following instead. We'll call the
@@ -237,7 +237,7 @@ seed_yarn_cache(name = "yarn_cache")
 
 # Install external dependencies for NPM packages in pkg/ui/ as separate bazel
 # repositories, to avoid version conflicts between those packages.
-# Unfortunately Bazel's rules_nodejs does not support yarn workspaces, so 
+# Unfortunately Bazel's rules_nodejs does not support yarn workspaces, so
 # packages have isolated dependencies and must be installed as isolated
 # Bazel repositories.
 yarn_install(

build/bazelutil/distdir_files.bzl

@@ -973,12 +973,12 @@ DISTDIR_FILES = {
     "https://storage.googleapis.com/public-bazel-artifacts/c-deps/20220520-181309/libproj_foreign.macos.20220520-181309.tar.gz": "aa618a6525c0df669ce1231c837dc46a1a789d54231ae3dcfcada599d0845b22",
     "https://storage.googleapis.com/public-bazel-artifacts/c-deps/20220520-181309/libproj_foreign.macosarm.20220520-181309.tar.gz": "e3206fa7c2544d1817103c3c61f132d342b5d4a07404d1de4582da968b622749",
     "https://storage.googleapis.com/public-bazel-artifacts/c-deps/20220520-181309/libproj_foreign.windows.20220520-181309.tar.gz": "4d935601d1b989ff020c66011355a0e2985abacf36f9d952f7f1ce34d54684ad",
-    "https://storage.googleapis.com/public-bazel-artifacts/go/go1.17.10.darwin-amd64.tar.gz": "84979d5985c70cee6f303050a7e811440aad7f304efdf28665b200f096b01945",
-    "https://storage.googleapis.com/public-bazel-artifacts/go/go1.17.10.darwin-arm64.tar.gz": "32098bea40117ea1ec23e7124cd188db6bdddd0ea41e2ec9bea3ba35a487e39c",
-    "https://storage.googleapis.com/public-bazel-artifacts/go/go1.17.10.freebsd-amd64.tar.gz": "33794d96f58608fdc023c5114ae9baeeb4111a74720c8830ff25029debe508f0",
-    "https://storage.googleapis.com/public-bazel-artifacts/go/go1.17.10.linux-amd64.tar.gz": "87fc728c9c731e2f74e4a999ef53cf07302d7ed3504b0839027bd9c10edaa3fd",
-    "https://storage.googleapis.com/public-bazel-artifacts/go/go1.17.10.linux-arm64.tar.gz": "649141201efa7195403eb1301b95dc79c5b3e65968986a391da1370521701b0c",
-    "https://storage.googleapis.com/public-bazel-artifacts/go/go1.17.10.windows-amd64.zip": "ba9198a29fa5c4f322212d21569e8507165c3b34e1ed1f1f9cf6dfb71ddcdeb2",
+    "https://storage.googleapis.com/public-bazel-artifacts/go/go1.17.11.darwin-amd64.tar.gz": "4f924c534230de8f0e1c7369f611c0310efd21fc2d9438b13bc2703af9dda25a",
+    "https://storage.googleapis.com/public-bazel-artifacts/go/go1.17.11.darwin-arm64.tar.gz": "b8e1ab009c2ff8dea462c7a1263d1f3f38e90ab5262e74c76d70e41a4db320be",
+    "https://storage.googleapis.com/public-bazel-artifacts/go/go1.17.11.freebsd-amd64.tar.gz": "da78bcd5efa24cfa8ca3ccf0d222f7d66b755c4200d404869984ebdcfc7b6aa7",
+    "https://storage.googleapis.com/public-bazel-artifacts/go/go1.17.11.linux-amd64.tar.gz": "d69a4fe2694f795d8e525c72b497ededc209cb7185f4c3b62d7a98dd6227b3fe",
+    "https://storage.googleapis.com/public-bazel-artifacts/go/go1.17.11.linux-arm64.tar.gz": "adefa7412c6798f9cad02d1e8336fc2242f5bade30c5b32781759181e01961b7",
+    "https://storage.googleapis.com/public-bazel-artifacts/go/go1.17.11.windows-amd64.zip": "88e60b92069d8e0932ca5d8bd8227d1693b9570fa2afbedadcc680749c428d54",
     "https://storage.googleapis.com/public-bazel-artifacts/gomod/github.com/bazelbuild/buildtools/v0.0.0-20200718160251-b1667ff58f71/buildtools-v0.0.0-20200718160251-b1667ff58f71.tar.gz": "a9ef5103739dfb5ed2a5b47ab1654842a89695812e4af09e57d7015a5caf97e0",
     "https://storage.googleapis.com/public-bazel-artifacts/java/railroad/rr-1.63-java8.zip": "d2791cd7a44ea5be862f33f5a9b3d40aaad9858455828ebade7007ad7113fb41",
     "https://storage.googleapis.com/public-bazel-artifacts/js/node/v16.13.0/node-v16.13.0-darwin-arm64.tar.gz": "46d83fc0bd971db5050ef1b15afc44a6665dee40bd6c1cbaec23e1b40fa49e6d",

build/bootstrap/bootstrap-debian.sh

@@ -46,9 +46,9 @@ sudo tar -C /usr --strip-components=1 -zxf /tmp/cmake.tgz && rm /tmp/cmake.tgz
 
 # Install Go.
 trap 'rm -f /tmp/go.tgz' EXIT
-curl -fsSL https://dl.google.com/go/go1.17.10.linux-amd64.tar.gz > /tmp/go.tgz
+curl -fsSL https://dl.google.com/go/go1.17.11.linux-amd64.tar.gz > /tmp/go.tgz
 sha256sum -c - <<EOF
-87fc728c9c731e2f74e4a999ef53cf07302d7ed3504b0839027bd9c10edaa3fd /tmp/go.tgz
+d69a4fe2694f795d8e525c72b497ededc209cb7185f4c3b62d7a98dd6227b3fe  /tmp/go.tgz
 EOF
 sudo tar -C /usr/local -zxf /tmp/go.tgz && rm /tmp/go.tgz
 

build/builder.sh

@@ -3,7 +3,7 @@
 set -euo pipefail
 
 image=cockroachdb/builder
-version=20220525-185738
+version=20220613-130827
 
 function init() {
   docker build --tag="${image}" "$(dirname "${0}")/builder"

build/builder/Dockerfile

@@ -114,8 +114,8 @@ RUN curl -fsSL https://github.com/Kitware/CMake/releases/download/v3.20.3/cmake-
 # releases of Go will no longer be run in CI once it is changed. Consider
 # bumping the minimum allowed version of Go in /build/go-version-check.sh.
 RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends golang \
- && curl -fsSL https://storage.googleapis.com/golang/go1.17.10.src.tar.gz -o golang.tar.gz \
- && echo '299e55af30f15691b015d8dcf8ecae72412412569e5b2ece20361753a456f2f9 golang.tar.gz' | sha256sum -c - \
+ && curl -fsSL https://storage.googleapis.com/golang/go1.17.11.src.tar.gz -o golang.tar.gz \
+ && echo 'ac2649a65944c6a5abe55054000eee3d77196880da36a3555f62e06540e8eb54  golang.tar.gz' | sha256sum -c - \
  && tar -C /usr/local -xzf golang.tar.gz \
  && rm golang.tar.gz \
  && cd /usr/local/go/src \

build/packer/teamcity-agent.sh

@@ -53,14 +53,15 @@ EOF
 tar --strip-components=1 -C /usr -xzf /tmp/cmake.tar.gz
 rm -f /tmp/cmake.tar.gz
 
-curl -fsSL https://dl.google.com/go/go1.17.10.linux-amd64.tar.gz > /tmp/go.tgz
+curl -fsSL https://dl.google.com/go/go1.17.11.linux-amd64.tar.gz > /tmp/go.tgz
 sha256sum -c - <<EOF
-87fc728c9c731e2f74e4a999ef53cf07302d7ed3504b0839027bd9c10edaa3fd /tmp/go.tgz
+d69a4fe2694f795d8e525c72b497ededc209cb7185f4c3b62d7a98dd6227b3fe  /tmp/go.tgz
 EOF
 tar -C /usr/local -zxf /tmp/go.tgz && rm /tmp/go.tgz
 
 # Install the older version in parallel in order to run the acceptance test on older branches
-# TODO: Remove this when 21.1 is EOL
+# TODO: Remove this when 21.1 is EOL (2022-11-18, according to
+# https://www.cockroachlabs.com/docs/releases/release-support-policy.html)
 curl -fsSL https://dl.google.com/go/go1.15.14.linux-amd64.tar.gz > /tmp/go_old.tgz
 sha256sum -c - <<EOF
 6f5410c113b803f437d7a1ee6f8f124100e536cc7361920f7e640fedf7add72d /tmp/go_old.tgz

pkg/BUILD.bazel

@@ -169,6 +169,7 @@ ALL_TESTS = [
     "//pkg/kv/kvserver/protectedts/ptstorage:ptstorage_test",
     "//pkg/kv/kvserver/protectedts:protectedts_test",
     "//pkg/kv/kvserver/raftentry:raftentry_test",
+    "//pkg/kv/kvserver/raftutil:raftutil_test",
     "//pkg/kv/kvserver/rangefeed:rangefeed_test",
     "//pkg/kv/kvserver/rditer:rditer_test",
     "//pkg/kv/kvserver/replicastats:replicastats_test",

pkg/cmd/roachtest/tests/go_helpers.go

@@ -43,13 +43,13 @@ func installGolang(
 	}
 
 	if err := repeatRunE(
-		ctx, t, c, node, "download go", `curl -fsSL https://dl.google.com/go/go1.17.10.linux-amd64.tar.gz > /tmp/go.tgz`,
+		ctx, t, c, node, "download go", `curl -fsSL https://dl.google.com/go/go1.17.11.linux-amd64.tar.gz > /tmp/go.tgz`,
 	); err != nil {
 		t.Fatal(err)
 	}
 	if err := repeatRunE(
 		ctx, t, c, node, "verify tarball", `sha256sum -c - <<EOF
-87fc728c9c731e2f74e4a999ef53cf07302d7ed3504b0839027bd9c10edaa3fd /tmp/go.tgz
+d69a4fe2694f795d8e525c72b497ededc209cb7185f4c3b62d7a98dd6227b3fe /tmp/go.tgz
 EOF`,
 	); err != nil {
 		t.Fatal(err)

pkg/kv/kvserver/BUILD.bazel

@@ -135,6 +135,7 @@ go_library(
         "//pkg/kv/kvserver/liveness",
         "//pkg/kv/kvserver/liveness/livenesspb",
         "//pkg/kv/kvserver/raftentry",
+        "//pkg/kv/kvserver/raftutil",
         "//pkg/kv/kvserver/rangefeed",
         "//pkg/kv/kvserver/rditer",
         "//pkg/kv/kvserver/readsummary",

pkg/kv/kvserver/allocator/allocatorimpl/BUILD.bazel

@@ -16,6 +16,7 @@ go_library(
         "//pkg/kv/kvserver/allocator/storepool",
         "//pkg/kv/kvserver/constraint",
         "//pkg/kv/kvserver/liveness/livenesspb",
+        "//pkg/kv/kvserver/raftutil",
         "//pkg/kv/kvserver/replicastats",
         "//pkg/roachpb",
         "//pkg/settings",

pkg/kv/kvserver/allocator/allocatorimpl/allocator.go

@@ -23,6 +23,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/allocator"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/allocator/storepool"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/constraint"
+	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/raftutil"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/replicastats"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/settings"
@@ -1468,8 +1469,9 @@ func (a *Allocator) ValidLeaseTargets(
 	conf roachpb.SpanConfig,
 	existing []roachpb.ReplicaDescriptor,
 	leaseRepl interface {
-		RaftStatus() *raft.Status
 		StoreID() roachpb.StoreID
+		RaftStatus() *raft.Status
+		GetFirstIndex() uint64
 	},
 	// excludeLeaseRepl dictates whether the result set can include the source
 	// replica.
@@ -1510,7 +1512,8 @@ func (a *Allocator) ValidLeaseTargets(
 		// potentially transferring the lease to a replica that may be waiting for a
 		// snapshot (which will wedge the range until the replica applies that
 		// snapshot).
-		candidates = excludeReplicasInNeedOfSnapshots(ctx, leaseRepl.RaftStatus(), candidates)
+		candidates = excludeReplicasInNeedOfSnapshots(
+			ctx, leaseRepl.RaftStatus(), leaseRepl.GetFirstIndex(), candidates)
 	}
 
 	// Determine which store(s) is preferred based on user-specified preferences.
@@ -1532,8 +1535,9 @@ func (a *Allocator) leaseholderShouldMoveDueToPreferences(
 	ctx context.Context,
 	conf roachpb.SpanConfig,
 	leaseRepl interface {
-		RaftStatus() *raft.Status
 		StoreID() roachpb.StoreID
+		RaftStatus() *raft.Status
+		GetFirstIndex() uint64
 	},
 	allExistingReplicas []roachpb.ReplicaDescriptor,
 ) bool {
@@ -1557,7 +1561,8 @@ func (a *Allocator) leaseholderShouldMoveDueToPreferences(
 	// If there are any replicas that do match lease preferences, then we check if
 	// the existing leaseholder is one of them.
 	preferred := a.PreferredLeaseholders(conf, candidates)
-	preferred = excludeReplicasInNeedOfSnapshots(ctx, leaseRepl.RaftStatus(), preferred)
+	preferred = excludeReplicasInNeedOfSnapshots(
+		ctx, leaseRepl.RaftStatus(), leaseRepl.GetFirstIndex(), preferred)
 	if len(preferred) == 0 {
 		return false
 	}
@@ -1606,9 +1611,10 @@ func (a *Allocator) TransferLeaseTarget(
 	conf roachpb.SpanConfig,
 	existing []roachpb.ReplicaDescriptor,
 	leaseRepl interface {
-		RaftStatus() *raft.Status
 		StoreID() roachpb.StoreID
 		GetRangeID() roachpb.RangeID
+		RaftStatus() *raft.Status
+		GetFirstIndex() uint64
 	},
 	stats *replicastats.ReplicaStats,
 	forceDecisionWithoutStats bool,
@@ -1854,8 +1860,9 @@ func (a *Allocator) ShouldTransferLease(
 	conf roachpb.SpanConfig,
 	existing []roachpb.ReplicaDescriptor,
 	leaseRepl interface {
-		RaftStatus() *raft.Status
 		StoreID() roachpb.StoreID
+		RaftStatus() *raft.Status
+		GetFirstIndex() uint64
 	},
 	stats *replicastats.ReplicaStats,
 ) bool {
@@ -2193,73 +2200,27 @@ func computeQuorum(nodes int) int {
 // slice. A "behind" replica is one which is not at or past the quorum commit
 // index.
 func FilterBehindReplicas(
-	ctx context.Context, raftStatus *raft.Status, replicas []roachpb.ReplicaDescriptor,
+	ctx context.Context, st *raft.Status, replicas []roachpb.ReplicaDescriptor,
 ) []roachpb.ReplicaDescriptor {
-	if raftStatus == nil || len(raftStatus.Progress) == 0 {
-		// raftStatus.Progress is only populated on the Raft leader which means we
-		// won't be able to rebalance a lease away if the lease holder is not the
-		// Raft leader. This is rare enough not to matter.
-		return nil
-	}
-	candidates := make([]roachpb.ReplicaDescriptor, 0, len(replicas))
+	var candidates []roachpb.ReplicaDescriptor
 	for _, r := range replicas {
-		if !ReplicaIsBehind(raftStatus, r.ReplicaID) {
+		if !raftutil.ReplicaIsBehind(st, r.ReplicaID) {
 			candidates = append(candidates, r)
 		}
 	}
 	return candidates
 }
 
-// ReplicaIsBehind returns whether the given replica ID is considered behind
-// according to the raft log.
-func ReplicaIsBehind(raftStatus *raft.Status, replicaID roachpb.ReplicaID) bool {
-	if raftStatus == nil || len(raftStatus.Progress) == 0 {
-		return true
-	}
-	// NB: We use raftStatus.Commit instead of getQuorumIndex() because the
-	// latter can return a value that is less than the commit index. This is
-	// useful for Raft log truncation which sometimes wishes to keep those
-	// earlier indexes, but not appropriate for determining which nodes are
-	// behind the actual commit index of the range.
-	if progress, ok := raftStatus.Progress[uint64(replicaID)]; ok {
-		if uint64(replicaID) == raftStatus.Lead ||
-			(progress.State == tracker.StateReplicate &&
-				progress.Match >= raftStatus.Commit) {
-			return false
-		}
-	}
-	return true
-}
-
-// replicaMayNeedSnapshot determines whether the replica referred to by
-// `replicaID` may be in need of a raft snapshot. If this function is called
-// with an empty or nil `raftStatus` (as will be the case when its called by a
-// replica that is not the raft leader), we pessimistically assume that
-// `replicaID` may need a snapshot.
-func replicaMayNeedSnapshot(raftStatus *raft.Status, replica roachpb.ReplicaDescriptor) bool {
-	if raftStatus == nil || len(raftStatus.Progress) == 0 {
-		return true
-	}
-	if progress, ok := raftStatus.Progress[uint64(replica.ReplicaID)]; ok {
-		// We can only reasonably assume that the follower replica is not in need of
-		// a snapshot iff it is in `StateReplicate`. However, even this is racey
-		// because we can still possibly have an ill-timed log truncation between
-		// when we make this determination and when we act on it.
-		return progress.State != tracker.StateReplicate
-	}
-	return true
-}
-
 // excludeReplicasInNeedOfSnapshots filters out the `replicas` that may be in
 // need of a raft snapshot. VOTER_INCOMING replicas are not filtered out.
 // Other replicas may be filtered out if this function is called with the
 // `raftStatus` of a non-raft leader replica.
 func excludeReplicasInNeedOfSnapshots(
-	ctx context.Context, raftStatus *raft.Status, replicas []roachpb.ReplicaDescriptor,
+	ctx context.Context, st *raft.Status, firstIndex uint64, replicas []roachpb.ReplicaDescriptor,
 ) []roachpb.ReplicaDescriptor {
 	filled := 0
 	for _, repl := range replicas {
-		if replicaMayNeedSnapshot(raftStatus, repl) {
+		if raftutil.ReplicaMayNeedSnapshot(st, firstIndex, repl.ReplicaID) != raftutil.NoSnapshotNeeded {
 			log.VEventf(
 				ctx,
 				5,

pkg/kv/kvserver/allocator/allocatorimpl/allocator_test.go

@@ -1664,6 +1664,7 @@ func (r *mockRepl) RaftStatus() *raft.Status {
 	raftStatus := &raft.Status{
 		Progress: make(map[uint64]tracker.Progress),
 	}
+	raftStatus.RaftState = raft.StateLeader
 	for i := int32(1); i <= r.replicationFactor; i++ {
 		state := tracker.StateReplicate
 		if _, ok := r.replsInNeedOfSnapshot[roachpb.ReplicaID(i)]; ok {
@@ -1674,6 +1675,10 @@ func (r *mockRepl) RaftStatus() *raft.Status {
 	return raftStatus
 }
 
+func (r *mockRepl) GetFirstIndex() uint64 {
+	return 0
+}
+
 func (r *mockRepl) StoreID() roachpb.StoreID {
 	return r.storeID
 }
@@ -7138,6 +7143,7 @@ func TestFilterBehindReplicas(t *testing.T) {
 				Progress: make(map[uint64]tracker.Progress),
 			}
 			status.Lead = c.leader
+			status.RaftState = raft.StateLeader
 			status.Commit = c.commit
 			var replicas []roachpb.ReplicaDescriptor
 			for j, v := range c.progress {
@@ -7210,6 +7216,7 @@ func TestFilterUnremovableReplicas(t *testing.T) {
 			// Use an invalid replica ID for the leader. TestFilterBehindReplicas covers
 			// valid replica IDs.
 			status.Lead = 99
+			status.RaftState = raft.StateLeader
 			status.Commit = c.commit
 			var replicas []roachpb.ReplicaDescriptor
 			for j, v := range c.progress {
@@ -7267,6 +7274,7 @@ func TestSimulateFilterUnremovableReplicas(t *testing.T) {
 			// Use an invalid replica ID for the leader. TestFilterBehindReplicas covers
 			// valid replica IDs.
 			status.Lead = 99
+			status.RaftState = raft.StateLeader
 			status.Commit = c.commit
 			var replicas []roachpb.ReplicaDescriptor
 			for j, v := range c.progress {

pkg/kv/kvserver/allocator_impl_test.go

@@ -170,6 +170,8 @@ func TestAllocatorRebalanceTarget(t *testing.T) {
 	status := &raft.Status{
 		Progress: make(map[uint64]tracker.Progress),
 	}
+	status.Lead = 1
+	status.RaftState = raft.StateLeader
 	status.Commit = 10
 	for _, replica := range replicas {
 		status.Progress[uint64(replica.ReplicaID)] = tracker.Progress{

pkg/kv/kvserver/deprecated_store_rebalancer.go

@@ -19,6 +19,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/allocator"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/allocator/allocatorimpl"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/allocator/storepool"
+	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/raftutil"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/util/hlc"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
@@ -108,7 +109,7 @@ func (sr *StoreRebalancer) deprecatedChooseLeaseToTransfer(
 			if raftStatus == nil {
 				raftStatus = sr.getRaftStatusFn(replWithStats.repl)
 			}
-			if allocatorimpl.ReplicaIsBehind(raftStatus, candidate.ReplicaID) {
+			if raftutil.ReplicaIsBehind(raftStatus, candidate.ReplicaID) {
 				log.VEventf(ctx, 3, "%v is behind or this store isn't the raft leader for r%d; raftStatus: %v",
 					candidate, desc.RangeID, raftStatus)
 				continue
@@ -297,7 +298,7 @@ func (sr *StoreRebalancer) deprecatedChooseRangeToRebalance(
 				if raftStatus == nil {
 					raftStatus = sr.getRaftStatusFn(replWithStats.repl)
 				}
-				if allocatorimpl.ReplicaIsBehind(raftStatus, replica.ReplicaID) {
+				if raftutil.ReplicaIsBehind(raftStatus, replica.ReplicaID) {
 					continue
 				}
 			}