kvcoord: use rangefeed connection class

This change switches rangefeed to use RangefeedClass for RPC traffic by
default. The corresponding cluster setting is removed since we don't
expect needing to dynamically change this. We leave an escape option:
the new env variable allows using DefaultClass instead.

This is analogous to the usage of RaftClass for raft traffic.

Release note (performance improvement): this change separates the
rangefeed traffic into its own RPC connection class. This improves
isolation and reduces interference with the foreground SQL traffic,
which reduces chances of head-of-line blocking caused by unrelated
traffic. The new COCKROACH_RANGEFEED_USE_DEFAULT_CONNECTION_CLASS env
variable can be set to use the DefaultClass instead (which was the
previous default choice for rangefeeds).

pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go

@@ -30,9 +30,9 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/server/telemetry"
 	"github.com/cockroachdb/cockroach/pkg/settings"
 	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
-	"github.com/cockroachdb/cockroach/pkg/util"
 	"github.com/cockroachdb/cockroach/pkg/util/admission/admissionpb"
 	"github.com/cockroachdb/cockroach/pkg/util/ctxgroup"
+	"github.com/cockroachdb/cockroach/pkg/util/envutil"
 	"github.com/cockroachdb/cockroach/pkg/util/grpcutil"
 	"github.com/cockroachdb/cockroach/pkg/util/hlc"
 	"github.com/cockroachdb/cockroach/pkg/util/iterutil"
@@ -55,13 +55,15 @@ type singleRangeInfo struct {
 	token      rangecache.EvictionToken
 }
 
-var useDedicatedRangefeedConnectionClass = settings.RegisterBoolSetting(
-	settings.SystemVisible,
-	"kv.rangefeed.use_dedicated_connection_class.enabled",
-	"uses dedicated connection when running rangefeeds",
-	util.ConstantWithMetamorphicTestBool(
-		"kv.rangefeed.use_dedicated_connection_class.enabled", false),
-)
+// defRangefeedConnClass is the default rpc.ConnectionClass used for rangefeed
+// traffic. Normally it is RangefeedClass, but can be flipped to DefaultClass if
+// the corresponding env variable is true.
+var defRangefeedConnClass = func() rpc.ConnectionClass {
+	if envutil.EnvOrDefaultBool("COCKROACH_RANGEFEED_USE_DEFAULT_CONNECTION_CLASS", false) {
+		return rpc.DefaultClass
+	}
+	return rpc.RangefeedClass
+}()
 
 var catchupStartupRate = settings.RegisterIntSetting(
 	settings.ApplicationLevel,
@@ -753,7 +755,7 @@ func newTransportForRange(
 		return nil, err
 	}
 	replicas.OptimizeReplicaOrder(ds.st, ds.nodeIDGetter(), ds.healthFunc, ds.latencyFunc, ds.locality)
-	opts := SendOptions{class: connectionClass(&ds.st.SV)}
+	opts := SendOptions{class: defRangefeedConnClass}
 	return ds.transportFactory(opts, replicas)
 }
 
@@ -956,13 +958,6 @@ func (ds *DistSender) singleRangeFeed(
 	}
 }
 
-func connectionClass(sv *settings.Values) rpc.ConnectionClass {
-	if useDedicatedRangefeedConnectionClass.Get(sv) {
-		return rpc.RangefeedClass
-	}
-	return rpc.DefaultClass
-}
-
 func handleStuckEvent(
 	args *kvpb.RangeFeedRequest,
 	afterCatchupScan bool,

pkg/settings/registry.go

@@ -235,6 +235,7 @@ var retiredSettings = map[InternalKey]struct{}{
 	"kv.rangefeed.catchup_scan_concurrency":                {},
 	"kv.rangefeed.scheduler.enabled":                       {},
 	"physical_replication.producer.mux_rangefeeds.enabled": {},
+	"kv.rangefeed.use_dedicated_connection_class.enabled":  {},
 }
 
 // sqlDefaultSettings is the list of "grandfathered" existing sql.defaults