From 1a87e350539a6df013b6e7427ebe0f712a9def37 Mon Sep 17 00:00:00 2001
From: Josh Black <raskchanky@users.noreply.github.com>
Date: Fri, 20 May 2022 13:49:11 -0700
Subject: [PATCH] Add autopilot automated upgrades and redundancy zones
 (#15521)

---
 api/client.go                                 |  12 +-
 api/sys_hastatus.go                           |   3 +
 api/sys_raft.go                               |  67 +++++--
 command/format.go                             |  95 +++++++---
 command/operator_members.go                   |  13 +-
 command/operator_raft_autopilot_get_config.go |   1 +
 command/operator_raft_autopilot_set_config.go |   9 +
 physical/raft/fsm.go                          |   2 +-
 physical/raft/raft.go                         |  78 +++++++-
 physical/raft/raft_autopilot.go               | 174 ++++++++++++++----
 physical/raft/raft_test.go                    |   7 +-
 physical/raft/raft_util.go                    |  14 +-
 vault/core.go                                 |   7 +
 .../raft/raft_autopilot_test.go               | 168 ++++++++---------
 vault/external_tests/raft/raft_test.go        |   9 +-
 vault/logical_system.go                       |  45 +++--
 vault/logical_system_raft.go                  |  46 +++--
 vault/raft.go                                 |  14 +-
 vault/request_forwarding.go                   |   6 +-
 vault/request_forwarding_rpc.go               |  34 +++-
 vault/request_forwarding_service.pb.go        | 171 ++++++++++-------
 vault/request_forwarding_service.proto        |   3 +
 vault/testing.go                              |  17 +-
 23 files changed, 689 insertions(+), 306 deletions(-)

diff --git a/api/client.go b/api/client.go
index efdf033af2356..feff95ae0fda6 100644
--- a/api/client.go
+++ b/api/client.go
@@ -608,7 +608,7 @@ func (c *Client) CloneConfig() *Config {
 	return newConfig
 }
 
-// Sets the address of Vault in the client. The format of address should be
+// SetAddress sets the address of Vault in the client. The format of address should be
 // "<Scheme>://<Host>:<Port>". Setting this on a client will override the
 // value of VAULT_ADDR environment variable.
 func (c *Client) SetAddress(addr string) error {
@@ -635,6 +635,16 @@ func (c *Client) Address() string {
 	return c.addr.String()
 }
 
+func (c *Client) SetCheckRedirect(f func(*http.Request, []*http.Request) error) {
+	c.modifyLock.Lock()
+	defer c.modifyLock.Unlock()
+
+	c.config.modifyLock.Lock()
+	defer c.config.modifyLock.Unlock()
+
+	c.config.HttpClient.CheckRedirect = f
+}
+
 // SetLimiter will set the rate limiter for this client.
 // This method is thread-safe.
 // rateLimit and burst are specified according to https://godoc.org/golang.org/x/time/rate#NewLimiter
diff --git a/api/sys_hastatus.go b/api/sys_hastatus.go
index 35bf40336651d..d89d59651a92f 100644
--- a/api/sys_hastatus.go
+++ b/api/sys_hastatus.go
@@ -37,4 +37,7 @@ type HANode struct {
 	ClusterAddress string     `json:"cluster_address"`
 	ActiveNode     bool       `json:"active_node"`
 	LastEcho       *time.Time `json:"last_echo"`
+	Version        string     `json:"version"`
+	UpgradeVersion string     `json:"upgrade_version,omitempty"`
+	RedundancyZone string     `json:"redundancy_zone,omitempty"`
 }
diff --git a/api/sys_raft.go b/api/sys_raft.go
index df10bf672e054..7806a1418df83 100644
--- a/api/sys_raft.go
+++ b/api/sys_raft.go
@@ -44,6 +44,7 @@ type AutopilotConfig struct {
 	MaxTrailingLogs                uint64        `json:"max_trailing_logs" mapstructure:"max_trailing_logs"`
 	MinQuorum                      uint          `json:"min_quorum" mapstructure:"min_quorum"`
 	ServerStabilizationTime        time.Duration `json:"server_stabilization_time" mapstructure:"-"`
+	DisableUpgradeMigration        bool          `json:"disable_upgrade_migration" mapstructure:"disable_upgrade_migration"`
 }
 
 // MarshalJSON makes the autopilot config fields JSON compatible
@@ -55,6 +56,7 @@ func (ac *AutopilotConfig) MarshalJSON() ([]byte, error) {
 		"max_trailing_logs":                  ac.MaxTrailingLogs,
 		"min_quorum":                         ac.MinQuorum,
 		"server_stabilization_time":          ac.ServerStabilizationTime.String(),
+		"disable_upgrade_migration":          ac.DisableUpgradeMigration,
 	})
 }
 
@@ -84,28 +86,59 @@ func (ac *AutopilotConfig) UnmarshalJSON(b []byte) error {
 
 // AutopilotState represents the response of the raft autopilot state API
 type AutopilotState struct {
-	Healthy          bool                        `mapstructure:"healthy"`
-	FailureTolerance int                         `mapstructure:"failure_tolerance"`
-	Servers          map[string]*AutopilotServer `mapstructure:"servers"`
-	Leader           string                      `mapstructure:"leader"`
-	Voters           []string                    `mapstructure:"voters"`
-	NonVoters        []string                    `mapstructure:"non_voters"`
+	Healthy                    bool                        `mapstructure:"healthy"`
+	FailureTolerance           int                         `mapstructure:"failure_tolerance"`
+	Servers                    map[string]*AutopilotServer `mapstructure:"servers"`
+	Leader                     string                      `mapstructure:"leader"`
+	Voters                     []string                    `mapstructure:"voters"`
+	NonVoters                  []string                    `mapstructure:"non_voters"`
+	RedundancyZones            map[string]AutopilotZone    `mapstructure:"redundancy_zones,omitempty"`
+	Upgrade                    *AutopilotUpgrade           `mapstructure:"upgrade_info,omitempty"`
+	OptimisticFailureTolerance int                         `mapstructure:"optimistic_failure_tolerance,omitempty"`
 }
 
 // AutopilotServer represents the server blocks in the response of the raft
 // autopilot state API.
 type AutopilotServer struct {
-	ID          string            `mapstructure:"id"`
-	Name        string            `mapstructure:"name"`
-	Address     string            `mapstructure:"address"`
-	NodeStatus  string            `mapstructure:"node_status"`
-	LastContact string            `mapstructure:"last_contact"`
-	LastTerm    uint64            `mapstructure:"last_term"`
-	LastIndex   uint64            `mapstructure:"last_index"`
-	Healthy     bool              `mapstructure:"healthy"`
-	StableSince string            `mapstructure:"stable_since"`
-	Status      string            `mapstructure:"status"`
-	Meta        map[string]string `mapstructure:"meta"`
+	ID             string `mapstructure:"id"`
+	Name           string `mapstructure:"name"`
+	Address        string `mapstructure:"address"`
+	NodeStatus     string `mapstructure:"node_status"`
+	LastContact    string `mapstructure:"last_contact"`
+	LastTerm       uint64 `mapstructure:"last_term"`
+	LastIndex      uint64 `mapstructure:"last_index"`
+	Healthy        bool   `mapstructure:"healthy"`
+	StableSince    string `mapstructure:"stable_since"`
+	Status         string `mapstructure:"status"`
+	Version        string `mapstructure:"version"`
+	UpgradeVersion string `mapstructure:"upgrade_version,omitempty"`
+	RedundancyZone string `mapstructure:"redundancy_zone,omitempty"`
+	NodeType       string `mapstructure:"node_type,omitempty"`
+}
+
+type AutopilotZone struct {
+	Servers          []string `mapstructure:"servers,omitempty"`
+	Voters           []string `mapstructure:"voters,omitempty"`
+	FailureTolerance int      `mapstructure:"failure_tolerance,omitempty"`
+}
+
+type AutopilotUpgrade struct {
+	Status                    string                                  `mapstructure:"status"`
+	TargetVersion             string                                  `mapstructure:"target_version,omitempty"`
+	TargetVersionVoters       []string                                `mapstructure:"target_version_voters,omitempty"`
+	TargetVersionNonVoters    []string                                `mapstructure:"target_version_non_voters,omitempty"`
+	TargetVersionReadReplicas []string                                `mapstructure:"target_version_read_replicas,omitempty"`
+	OtherVersionVoters        []string                                `mapstructure:"other_version_voters,omitempty"`
+	OtherVersionNonVoters     []string                                `mapstructure:"other_version_non_voters,omitempty"`
+	OtherVersionReadReplicas  []string                                `mapstructure:"other_version_read_replicas,omitempty"`
+	RedundancyZones           map[string]AutopilotZoneUpgradeVersions `mapstructure:"redundancy_zones,omitempty"`
+}
+
+type AutopilotZoneUpgradeVersions struct {
+	TargetVersionVoters    []string `mapstructure:"target_version_voters,omitempty"`
+	TargetVersionNonVoters []string `mapstructure:"target_version_non_voters,omitempty"`
+	OtherVersionVoters     []string `mapstructure:"other_version_voters,omitempty"`
+	OtherVersionNonVoters  []string `mapstructure:"other_version_non_voters,omitempty"`
 }
 
 // RaftJoin wraps RaftJoinWithContext using context.Background.
diff --git a/command/format.go b/command/format.go
index f34516067e027..6812ba4c85e2d 100644
--- a/command/format.go
+++ b/command/format.go
@@ -171,29 +171,24 @@ func formatServer(srv *api.AutopilotServer) string {
 	var buffer bytes.Buffer
 
 	buffer.WriteString(fmt.Sprintf("   %s\n", srv.ID))
-	buffer.WriteString(fmt.Sprintf("      Name:            %s\n", srv.Name))
-	buffer.WriteString(fmt.Sprintf("      Address:         %s\n", srv.Address))
-	buffer.WriteString(fmt.Sprintf("      Status:          %s\n", srv.Status))
-	buffer.WriteString(fmt.Sprintf("      Node Status:     %s\n", srv.NodeStatus))
-	buffer.WriteString(fmt.Sprintf("      Healthy:         %t\n", srv.Healthy))
-	buffer.WriteString(fmt.Sprintf("      Last Contact:    %s\n", srv.LastContact))
-	buffer.WriteString(fmt.Sprintf("      Last Term:       %d\n", srv.LastTerm))
-	buffer.WriteString(fmt.Sprintf("      Last Index:      %d\n", srv.LastIndex))
-
-	if len(srv.Meta) > 0 {
-		buffer.WriteString("      Meta\n")
-		var outputs []mapOutput
-		for k, v := range srv.Meta {
-			outputs = append(outputs, mapOutput{key: k, value: fmt.Sprintf("         %q: %q\n", k, v)})
-		}
-
-		sort.Slice(outputs, func(i, j int) bool {
-			return outputs[i].key < outputs[j].key
-		})
+	buffer.WriteString(fmt.Sprintf("      Name:              %s\n", srv.Name))
+	buffer.WriteString(fmt.Sprintf("      Address:           %s\n", srv.Address))
+	buffer.WriteString(fmt.Sprintf("      Status:            %s\n", srv.Status))
+	buffer.WriteString(fmt.Sprintf("      Node Status:       %s\n", srv.NodeStatus))
+	buffer.WriteString(fmt.Sprintf("      Healthy:           %t\n", srv.Healthy))
+	buffer.WriteString(fmt.Sprintf("      Last Contact:      %s\n", srv.LastContact))
+	buffer.WriteString(fmt.Sprintf("      Last Term:         %d\n", srv.LastTerm))
+	buffer.WriteString(fmt.Sprintf("      Last Index:        %d\n", srv.LastIndex))
+	buffer.WriteString(fmt.Sprintf("      Version:           %s\n", srv.Version))
 
-		for _, output := range outputs {
-			buffer.WriteString(output.value)
-		}
+	if srv.UpgradeVersion != "" {
+		buffer.WriteString(fmt.Sprintf("      Upgrade Version:   %s\n", srv.UpgradeVersion))
+	}
+	if srv.RedundancyZone != "" {
+		buffer.WriteString(fmt.Sprintf("      Redundancy Zone:   %s\n", srv.RedundancyZone))
+	}
+	if srv.NodeType != "" {
+		buffer.WriteString(fmt.Sprintf("      Node Type:         %s\n", srv.NodeType))
 	}
 
 	return buffer.String()
@@ -203,9 +198,9 @@ func (p PrettyFormatter) OutputAutopilotState(ui cli.Ui, data interface{}) {
 	state := data.(*api.AutopilotState)
 
 	var buffer bytes.Buffer
-	buffer.WriteString(fmt.Sprintf("Healthy:                      %t\n", state.Healthy))
-	buffer.WriteString(fmt.Sprintf("Failure Tolerance:            %d\n", state.FailureTolerance))
-	buffer.WriteString(fmt.Sprintf("Leader:                       %s\n", state.Leader))
+	buffer.WriteString(fmt.Sprintf("Healthy:                         %t\n", state.Healthy))
+	buffer.WriteString(fmt.Sprintf("Failure Tolerance:               %d\n", state.FailureTolerance))
+	buffer.WriteString(fmt.Sprintf("Leader:                          %s\n", state.Leader))
 	buffer.WriteString("Voters:\n")
 	outputStringSlice(&buffer, "   ", state.Voters)
 
@@ -214,20 +209,66 @@ func (p PrettyFormatter) OutputAutopilotState(ui cli.Ui, data interface{}) {
 		outputStringSlice(&buffer, "   ", state.NonVoters)
 	}
 
+	if state.OptimisticFailureTolerance > 0 {
+		buffer.WriteString(fmt.Sprintf("Optimistic Failure Tolerance:    %d\n", state.OptimisticFailureTolerance))
+	}
+
+	// Servers
 	buffer.WriteString("Servers:\n")
 	var outputs []mapOutput
 	for id, srv := range state.Servers {
 		outputs = append(outputs, mapOutput{key: id, value: formatServer(srv)})
 	}
-
 	sort.Slice(outputs, func(i, j int) bool {
 		return outputs[i].key < outputs[j].key
 	})
-
 	for _, output := range outputs {
 		buffer.WriteString(output.value)
 	}
 
+	// Redundancy Zones
+	if len(state.RedundancyZones) > 0 {
+		buffer.WriteString("Redundancy Zones:\n")
+		zoneList := make([]string, 0, len(state.RedundancyZones))
+		for z := range state.RedundancyZones {
+			zoneList = append(zoneList, z)
+		}
+		sort.Strings(zoneList)
+		for _, zoneName := range zoneList {
+			zone := state.RedundancyZones[zoneName]
+			servers := zone.Servers
+			voters := zone.Voters
+			sort.Strings(servers)
+			sort.Strings(voters)
+			buffer.WriteString(fmt.Sprintf("   %s\n", zoneName))
+			buffer.WriteString(fmt.Sprintf("      Servers: %s\n", strings.Join(servers, ", ")))
+			buffer.WriteString(fmt.Sprintf("      Voters: %s\n", strings.Join(voters, ", ")))
+			buffer.WriteString(fmt.Sprintf("      Failure Tolerance: %d\n", zone.FailureTolerance))
+		}
+	}
+
+	// Upgrade Info
+	if state.Upgrade != nil {
+		buffer.WriteString("Upgrade Info:\n")
+		buffer.WriteString(fmt.Sprintf("   Status: %s\n", state.Upgrade.Status))
+		buffer.WriteString(fmt.Sprintf("   Target Version: %s\n", state.Upgrade.TargetVersion))
+		buffer.WriteString(fmt.Sprintf("   Target Version Voters: %s\n", strings.Join(state.Upgrade.TargetVersionVoters, ", ")))
+		buffer.WriteString(fmt.Sprintf("   Target Version Non-Voters: %s\n", strings.Join(state.Upgrade.TargetVersionNonVoters, ", ")))
+		buffer.WriteString(fmt.Sprintf("   Other Version Voters: %s\n", strings.Join(state.Upgrade.OtherVersionVoters, ", ")))
+		buffer.WriteString(fmt.Sprintf("   Other Version Non-Voters: %s\n", strings.Join(state.Upgrade.OtherVersionNonVoters, ", ")))
+
+		if len(state.Upgrade.RedundancyZones) > 0 {
+			buffer.WriteString("   Redundancy Zones:\n")
+			for zoneName, zoneVersion := range state.Upgrade.RedundancyZones {
+				buffer.WriteString(fmt.Sprintf("      %s\n", zoneName))
+				buffer.WriteString(fmt.Sprintf("         Target Version Voters: %s\n", strings.Join(zoneVersion.TargetVersionVoters, ", ")))
+				buffer.WriteString(fmt.Sprintf("         Target Version Non-Voters: %s\n", strings.Join(zoneVersion.TargetVersionNonVoters, ", ")))
+				buffer.WriteString(fmt.Sprintf("         Other Version Voters: %s\n", strings.Join(zoneVersion.OtherVersionVoters, ", ")))
+				buffer.WriteString(fmt.Sprintf("         Other Version Non-Voters: %s\n", strings.Join(zoneVersion.OtherVersionNonVoters, ", ")))
+			}
+		}
+	}
+
 	ui.Output(buffer.String())
 }
 
diff --git a/command/operator_members.go b/command/operator_members.go
index 6b163d669f8a1..d4bd1fbe4389f 100644
--- a/command/operator_members.go
+++ b/command/operator_members.go
@@ -3,6 +3,7 @@ package command
 import (
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/mitchellh/cli"
 	"github.com/posener/complete"
@@ -70,9 +71,17 @@ func (c *OperatorMembersCommand) Run(args []string) int {
 
 	switch Format(c.UI) {
 	case "table":
-		out := []string{"Host Name | API Address | Cluster Address | ActiveNode | Last Echo"}
+		out := make([]string, 0)
+		cols := []string{"Host Name", "API Address", "Cluster Address", "Active Node", "Version", "Upgrade Version", "Redundancy Zone", "Last Echo"}
+		out = append(out, strings.Join(cols, " | "))
 		for _, node := range resp.Nodes {
-			out = append(out, fmt.Sprintf("%s | %s | %s | %t | %s", node.Hostname, node.APIAddress, node.ClusterAddress, node.ActiveNode, node.LastEcho))
+			cols := []string{node.Hostname, node.APIAddress, node.ClusterAddress, fmt.Sprintf("%t", node.ActiveNode), node.Version, node.UpgradeVersion, node.RedundancyZone}
+			if node.LastEcho != nil {
+				cols = append(cols, node.LastEcho.Format(time.RFC3339))
+			} else {
+				cols = append(cols, "")
+			}
+			out = append(out, strings.Join(cols, " | "))
 		}
 		c.UI.Output(tableOutput(out, nil))
 		return 0
diff --git a/command/operator_raft_autopilot_get_config.go b/command/operator_raft_autopilot_get_config.go
index f0a30e1e1f60a..1462e354c5590 100644
--- a/command/operator_raft_autopilot_get_config.go
+++ b/command/operator_raft_autopilot_get_config.go
@@ -88,6 +88,7 @@ func (c *OperatorRaftAutopilotGetConfigCommand) Run(args []string) int {
 	entries = append(entries, fmt.Sprintf("%s | %s", "Server Stabilization Time", config.ServerStabilizationTime.String()))
 	entries = append(entries, fmt.Sprintf("%s | %d", "Min Quorum", config.MinQuorum))
 	entries = append(entries, fmt.Sprintf("%s | %d", "Max Trailing Logs", config.MaxTrailingLogs))
+	entries = append(entries, fmt.Sprintf("%s | %t", "Disable Upgrade Migration", config.DisableUpgradeMigration))
 
 	return OutputData(c.UI, entries)
 }
diff --git a/command/operator_raft_autopilot_set_config.go b/command/operator_raft_autopilot_set_config.go
index c8e136628178b..4a839c5fae3ab 100644
--- a/command/operator_raft_autopilot_set_config.go
+++ b/command/operator_raft_autopilot_set_config.go
@@ -22,6 +22,7 @@ type OperatorRaftAutopilotSetConfigCommand struct {
 	flagMaxTrailingLogs                uint64
 	flagMinQuorum                      uint
 	flagServerStabilizationTime        time.Duration
+	flagDisableUpgradeMigration        BoolPtr
 }
 
 func (c *OperatorRaftAutopilotSetConfigCommand) Synopsis() string {
@@ -73,6 +74,11 @@ func (c *OperatorRaftAutopilotSetConfigCommand) Flags() *FlagSets {
 		Target: &c.flagServerStabilizationTime,
 	})
 
+	f.BoolPtrVar(&BoolPtrVar{
+		Name:   "disable-upgrade-migration",
+		Target: &c.flagDisableUpgradeMigration,
+	})
+
 	return set
 }
 
@@ -125,6 +131,9 @@ func (c *OperatorRaftAutopilotSetConfigCommand) Run(args []string) int {
 	if c.flagServerStabilizationTime > 0 {
 		data["server_stabilization_time"] = c.flagServerStabilizationTime.String()
 	}
+	if c.flagDisableUpgradeMigration.IsSet() {
+		data["disable_upgrade_migration"] = c.flagDisableUpgradeMigration.Get()
+	}
 
 	secret, err := client.Logical().Write("sys/storage/raft/autopilot/configuration", data)
 	if err != nil {
diff --git a/physical/raft/fsm.go b/physical/raft/fsm.go
index 1f322610b014b..29ca39a7f2a1e 100644
--- a/physical/raft/fsm.go
+++ b/physical/raft/fsm.go
@@ -14,7 +14,7 @@ import (
 	"sync/atomic"
 	"time"
 
-	metrics "github.com/armon/go-metrics"
+	"github.com/armon/go-metrics"
 	"github.com/golang/protobuf/proto"
 	log "github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/go-multierror"
diff --git a/physical/raft/raft.go b/physical/raft/raft.go
index b53d7b02a00f6..69328aee57a8c 100644
--- a/physical/raft/raft.go
+++ b/physical/raft/raft.go
@@ -30,6 +30,7 @@ import (
 	"github.com/hashicorp/vault/sdk/helper/jsonutil"
 	"github.com/hashicorp/vault/sdk/logical"
 	"github.com/hashicorp/vault/sdk/physical"
+	"github.com/hashicorp/vault/sdk/version"
 	"github.com/hashicorp/vault/vault/cluster"
 	"github.com/hashicorp/vault/vault/seal"
 	bolt "go.etcd.io/bbolt"
@@ -152,7 +153,21 @@ type RaftBackend struct {
 	// node is up and running.
 	disableAutopilot bool
 
+	// autopilotReconcileInterval is how long between rounds of performing promotions, demotions
+	// and leadership transfers.
 	autopilotReconcileInterval time.Duration
+
+	// autopilotUpdateInterval is the time between the periodic state updates. These periodic
+	// state updates take in known servers from the delegate, request Raft stats be
+	// fetched and pull in other inputs such as the Raft configuration to create
+	// an updated view of the Autopilot State.
+	autopilotUpdateInterval time.Duration
+
+	// upgradeVersion is used to override the Vault SDK version when performing an autopilot automated upgrade.
+	upgradeVersion string
+
+	// redundancyZone specifies a redundancy zone for autopilot.
+	redundancyZone string
 }
 
 // LeaderJoinInfo contains information required by a node to join itself as a
@@ -421,6 +436,29 @@ func NewRaftBackend(conf map[string]string, logger log.Logger) (physical.Backend
 		reconcileInterval = interval
 	}
 
+	var updateInterval time.Duration
+	if interval := conf["autopilot_update_interval"]; interval != "" {
+		interval, err := time.ParseDuration(interval)
+		if err != nil {
+			return nil, fmt.Errorf("autopilot_update_interval does not parse as a duration: %w", err)
+		}
+		updateInterval = interval
+	}
+
+	effectiveReconcileInterval := autopilot.DefaultReconcileInterval
+	effectiveUpdateInterval := autopilot.DefaultUpdateInterval
+
+	if reconcileInterval != 0 {
+		effectiveReconcileInterval = reconcileInterval
+	}
+	if updateInterval != 0 {
+		effectiveUpdateInterval = updateInterval
+	}
+
+	if effectiveReconcileInterval < effectiveUpdateInterval {
+		return nil, fmt.Errorf("autopilot_reconcile_interval (%v) should be larger than autopilot_update_interval (%v)", effectiveReconcileInterval, effectiveUpdateInterval)
+	}
+
 	return &RaftBackend{
 		logger:                     logger,
 		fsm:                        fsm,
@@ -435,6 +473,9 @@ func NewRaftBackend(conf map[string]string, logger log.Logger) (physical.Backend
 		maxEntrySize:               maxEntrySize,
 		followerHeartbeatTicker:    time.NewTicker(time.Second),
 		autopilotReconcileInterval: reconcileInterval,
+		autopilotUpdateInterval:    updateInterval,
+		redundancyZone:             conf["autopilot_redundancy_zone"],
+		upgradeVersion:             conf["autopilot_upgrade_version"],
 	}, nil
 }
 
@@ -486,6 +527,36 @@ func (b *RaftBackend) Close() error {
 	return nil
 }
 
+func (b *RaftBackend) RedundancyZone() string {
+	b.l.RLock()
+	defer b.l.RUnlock()
+
+	return b.redundancyZone
+}
+
+func (b *RaftBackend) EffectiveVersion() string {
+	b.l.RLock()
+	defer b.l.RUnlock()
+
+	if b.upgradeVersion != "" {
+		return b.upgradeVersion
+	}
+
+	return version.GetVersion().Version
+}
+
+// DisableUpgradeMigration returns the state of the DisableUpgradeMigration config flag and whether it was set or not
+func (b *RaftBackend) DisableUpgradeMigration() (bool, bool) {
+	b.l.RLock()
+	defer b.l.RUnlock()
+
+	if b.autopilotConfig == nil {
+		return false, false
+	}
+
+	return b.autopilotConfig.DisableUpgradeMigration, true
+}
+
 func (b *RaftBackend) CollectMetrics(sink *metricsutil.ClusterMetricSink) {
 	b.l.RLock()
 	logstoreStats := b.stableStore.(*raftboltdb.BoltStore).Stats()
@@ -636,9 +707,9 @@ func (b *RaftBackend) applyConfigSettings(config *raft.Config) error {
 			return err
 		}
 	}
-	config.ElectionTimeout = config.ElectionTimeout * time.Duration(multiplier)
-	config.HeartbeatTimeout = config.HeartbeatTimeout * time.Duration(multiplier)
-	config.LeaderLeaseTimeout = config.LeaderLeaseTimeout * time.Duration(multiplier)
+	config.ElectionTimeout *= time.Duration(multiplier)
+	config.HeartbeatTimeout *= time.Duration(multiplier)
+	config.LeaderLeaseTimeout *= time.Duration(multiplier)
 
 	snapThresholdRaw, ok := b.conf["snapshot_threshold"]
 	if ok {
@@ -1146,6 +1217,7 @@ func (b *RaftBackend) GetConfiguration(ctx context.Context) (*RaftConfigurationR
 			Voter:           server.Suffrage == raft.Voter,
 			ProtocolVersion: strconv.Itoa(raft.ProtocolVersionMax),
 		}
+
 		config.Servers = append(config.Servers, entry)
 	}
 
diff --git a/physical/raft/raft_autopilot.go b/physical/raft/raft_autopilot.go
index 0cff7aa0203be..5dbfec94c804a 100644
--- a/physical/raft/raft_autopilot.go
+++ b/physical/raft/raft_autopilot.go
@@ -11,22 +11,24 @@ import (
 	"sync"
 	"time"
 
+	"github.com/armon/go-metrics"
 	"github.com/hashicorp/go-secure-stdlib/parseutil"
 	"github.com/hashicorp/go-secure-stdlib/strutil"
-	"go.uber.org/atomic"
-
-	metrics "github.com/armon/go-metrics"
 	"github.com/hashicorp/raft"
 	autopilot "github.com/hashicorp/raft-autopilot"
+	"github.com/hashicorp/vault/sdk/version"
 	"github.com/mitchellh/mapstructure"
+	"go.uber.org/atomic"
 )
 
 type CleanupDeadServersValue int
 
 const (
-	CleanupDeadServersUnset CleanupDeadServersValue = 0
-	CleanupDeadServersTrue  CleanupDeadServersValue = 1
-	CleanupDeadServersFalse CleanupDeadServersValue = 2
+	CleanupDeadServersUnset    CleanupDeadServersValue = 0
+	CleanupDeadServersTrue     CleanupDeadServersValue = 1
+	CleanupDeadServersFalse    CleanupDeadServersValue = 2
+	AutopilotUpgradeVersionTag string                  = "upgrade_version"
+	AutopilotRedundancyZoneTag string                  = "redundancy_zone"
 )
 
 func (c CleanupDeadServersValue) Value() bool {
@@ -70,6 +72,19 @@ type AutopilotConfig struct {
 	// stable, healthy state before it can be added to the cluster. Only applicable
 	// with Raft protocol version 3 or higher.
 	ServerStabilizationTime time.Duration `mapstructure:"-"`
+
+	// (Enterprise-only) DisableUpgradeMigration will disable Autopilot's upgrade migration
+	// strategy of waiting until enough newer-versioned servers have been added to the
+	// cluster before promoting them to voters.
+	DisableUpgradeMigration bool `mapstructure:"disable_upgrade_migration"`
+
+	// (Enterprise-only) RedundancyZoneTag is the node tag to use for separating
+	// servers into zones for redundancy. If left blank, this feature will be disabled.
+	RedundancyZoneTag string `mapstructure:"redundancy_zone_tag"`
+
+	// (Enterprise-only) UpgradeVersionTag is the node tag to use for version info when
+	// performing upgrade migrations. If left blank, the Consul version will be used.
+	UpgradeVersionTag string `mapstructure:"upgrade_version_tag"`
 }
 
 // Merge combines the supplied config with the receiver. Supplied ones take
@@ -96,6 +111,10 @@ func (to *AutopilotConfig) Merge(from *AutopilotConfig) {
 	if from.ServerStabilizationTime != 0 {
 		to.ServerStabilizationTime = from.ServerStabilizationTime
 	}
+
+	// UpgradeVersionTag and RedundancyZoneTag are purposely not included here since those values aren't user
+	// controllable and should never change.
+	to.DisableUpgradeMigration = from.DisableUpgradeMigration
 }
 
 // Clone returns a duplicate instance of AutopilotConfig with the exact same values.
@@ -110,6 +129,9 @@ func (ac *AutopilotConfig) Clone() *AutopilotConfig {
 		MaxTrailingLogs:                ac.MaxTrailingLogs,
 		MinQuorum:                      ac.MinQuorum,
 		ServerStabilizationTime:        ac.ServerStabilizationTime,
+		UpgradeVersionTag:              ac.UpgradeVersionTag,
+		RedundancyZoneTag:              ac.RedundancyZoneTag,
+		DisableUpgradeMigration:        ac.DisableUpgradeMigration,
 	}
 }
 
@@ -123,6 +145,9 @@ func (ac *AutopilotConfig) MarshalJSON() ([]byte, error) {
 		"max_trailing_logs":                  ac.MaxTrailingLogs,
 		"min_quorum":                         ac.MinQuorum,
 		"server_stabilization_time":          ac.ServerStabilizationTime.String(),
+		"upgrade_version_tag":                ac.UpgradeVersionTag,
+		"redundancy_zone_tag":                ac.RedundancyZoneTag,
+		"disable_upgrade_migration":          ac.DisableUpgradeMigration,
 	})
 }
 
@@ -158,6 +183,19 @@ type FollowerState struct {
 	LastTerm        uint64
 	IsDead          *atomic.Bool
 	DesiredSuffrage string
+	Version         string
+	UpgradeVersion  string
+	RedundancyZone  string
+}
+
+// EchoRequestUpdate is here to avoid 1) the list of arguments to Update() getting huge 2) an import cycle on the vault package
+type EchoRequestUpdate struct {
+	NodeID          string
+	AppliedIndex    uint64
+	Term            uint64
+	DesiredSuffrage string
+	UpgradeVersion  string
+	RedundancyZone  string
 }
 
 // FollowerStates holds information about all the followers in the raft cluster
@@ -175,23 +213,26 @@ func NewFollowerStates() *FollowerStates {
 }
 
 // Update the peer information in the follower states
-func (s *FollowerStates) Update(nodeID string, appliedIndex uint64, term uint64, desiredSuffrage string) {
+func (s *FollowerStates) Update(req *EchoRequestUpdate) {
 	s.l.Lock()
 	defer s.l.Unlock()
 
-	state, ok := s.followers[nodeID]
+	state, ok := s.followers[req.NodeID]
 	if !ok {
 		state = &FollowerState{
 			IsDead: atomic.NewBool(false),
 		}
-		s.followers[nodeID] = state
+		s.followers[req.NodeID] = state
 	}
 
 	state.IsDead.Store(false)
-	state.AppliedIndex = appliedIndex
-	state.LastTerm = term
-	state.DesiredSuffrage = desiredSuffrage
+	state.AppliedIndex = req.AppliedIndex
+	state.LastTerm = req.Term
+	state.DesiredSuffrage = req.DesiredSuffrage
 	state.LastHeartbeat = time.Now()
+	state.Version = version.GetVersion().Version
+	state.UpgradeVersion = req.UpgradeVersion
+	state.RedundancyZone = req.RedundancyZone
 }
 
 // Clear wipes all the information regarding peers in the follower states.
@@ -360,7 +401,9 @@ func (d *Delegate) KnownServers() map[raft.ServerID]*autopilot.Server {
 			ID:          raft.ServerID(id),
 			Name:        id,
 			RaftVersion: raft.ProtocolVersionMax,
-			Ext:         d.autopilotServerExt(state.DesiredSuffrage),
+			Meta:        d.meta(state),
+			Version:     state.Version,
+			Ext:         d.autopilotServerExt(state),
 		}
 
 		switch state.IsDead.Load() {
@@ -380,8 +423,13 @@ func (d *Delegate) KnownServers() map[raft.ServerID]*autopilot.Server {
 		Name:        d.localID,
 		RaftVersion: raft.ProtocolVersionMax,
 		NodeStatus:  autopilot.NodeAlive,
-		Ext:         d.autopilotServerExt("voter"),
-		IsLeader:    true,
+		Meta: d.meta(&FollowerState{
+			UpgradeVersion: d.EffectiveVersion(),
+			RedundancyZone: d.RedundancyZone(),
+		}),
+		Version:  version.GetVersion().Version,
+		Ext:      d.autopilotServerExt(nil),
+		IsLeader: true,
 	}
 
 	return ret
@@ -455,6 +503,9 @@ func (b *RaftBackend) defaultAutopilotConfig() *AutopilotConfig {
 		DeadServerLastContactThreshold: 24 * time.Hour,
 		MaxTrailingLogs:                1000,
 		ServerStabilizationTime:        10 * time.Second,
+		DisableUpgradeMigration:        false,
+		UpgradeVersionTag:              AutopilotUpgradeVersionTag,
+		RedundancyZoneTag:              AutopilotRedundancyZoneTag,
 	}
 }
 
@@ -505,29 +556,60 @@ func (b *RaftBackend) StopAutopilot() {
 
 // AutopilotState represents the health information retrieved from autopilot.
 type AutopilotState struct {
-	Healthy          bool `json:"healthy"`
-	FailureTolerance int  `json:"failure_tolerance"`
-
-	Servers   map[string]*AutopilotServer `json:"servers"`
-	Leader    string                      `json:"leader"`
-	Voters    []string                    `json:"voters"`
-	NonVoters []string                    `json:"non_voters,omitempty"`
+	Healthy                    bool                        `json:"healthy" mapstructure:"healthy"`
+	FailureTolerance           int                         `json:"failure_tolerance" mapstructure:"failure_tolerance"`
+	Servers                    map[string]*AutopilotServer `json:"servers" mapstructure:"servers"`
+	Leader                     string                      `json:"leader" mapstructure:"leader"`
+	Voters                     []string                    `json:"voters" mapstructure:"voters"`
+	NonVoters                  []string                    `json:"non_voters,omitempty" mapstructure:"non_voters,omitempty"`
+	RedundancyZones            map[string]AutopilotZone    `json:"redundancy_zones,omitempty" mapstructure:"redundancy_zones,omitempty"`
+	Upgrade                    *AutopilotUpgrade           `json:"upgrade_info,omitempty" mapstructure:"upgrade_info,omitempty"`
+	OptimisticFailureTolerance int                         `json:"optimistic_failure_tolerance,omitempty" mapstructure:"optimistic_failure_tolerance,omitempty"`
 }
 
 // AutopilotServer represents the health information of individual server node
 // retrieved from autopilot.
 type AutopilotServer struct {
-	ID          string            `json:"id"`
-	Name        string            `json:"name"`
-	Address     string            `json:"address"`
-	NodeStatus  string            `json:"node_status"`
-	LastContact *ReadableDuration `json:"last_contact"`
-	LastTerm    uint64            `json:"last_term"`
-	LastIndex   uint64            `json:"last_index"`
-	Healthy     bool              `json:"healthy"`
-	StableSince time.Time         `json:"stable_since"`
-	Status      string            `json:"status"`
-	Meta        map[string]string `json:"meta"`
+	ID             string            `json:"id" mapstructure:"id"`
+	Name           string            `json:"name" mapstructure:"name"`
+	Address        string            `json:"address" mapstructure:"address"`
+	NodeStatus     string            `json:"node_status" mapstructure:"node_status"`
+	LastContact    *ReadableDuration `json:"last_contact" mapstructure:"last_contact"`
+	LastTerm       uint64            `json:"last_term" mapstructure:"last_term"`
+	LastIndex      uint64            `json:"last_index" mapstructure:"last_index"`
+	Healthy        bool              `json:"healthy" mapstructure:"healthy"`
+	StableSince    time.Time         `json:"stable_since" mapstructure:"stable_since"`
+	Status         string            `json:"status" mapstructure:"status"`
+	Version        string            `json:"version" mapstructure:"version"`
+	RedundancyZone string            `json:"redundancy_zone,omitempty" mapstructure:"redundancy_zone,omitempty"`
+	UpgradeVersion string            `json:"upgrade_version,omitempty" mapstructure:"upgrade_version,omitempty"`
+	ReadReplica    bool              `json:"read_replica,omitempty" mapstructure:"read_replica,omitempty"`
+	NodeType       string            `json:"node_type,omitempty" mapstructure:"node_type,omitempty"`
+}
+
+type AutopilotZone struct {
+	Servers          []string `json:"servers,omitempty" mapstructure:"servers,omitempty"`
+	Voters           []string `json:"voters,omitempty" mapstructure:"voters,omitempty"`
+	FailureTolerance int      `json:"failure_tolerance,omitempty" mapstructure:"failure_tolerance,omitempty"`
+}
+
+type AutopilotUpgrade struct {
+	Status                    string                                  `json:"status" mapstructure:"status"`
+	TargetVersion             string                                  `json:"target_version,omitempty" mapstructure:"target_version,omitempty"`
+	TargetVersionVoters       []string                                `json:"target_version_voters,omitempty" mapstructure:"target_version_voters,omitempty"`
+	TargetVersionNonVoters    []string                                `json:"target_version_non_voters,omitempty" mapstructure:"target_version_non_voters,omitempty"`
+	TargetVersionReadReplicas []string                                `json:"target_version_read_replicas,omitempty" mapstructure:"target_version_read_replicas,omitempty"`
+	OtherVersionVoters        []string                                `json:"other_version_voters,omitempty" mapstructure:"other_version_voters,omitempty"`
+	OtherVersionNonVoters     []string                                `json:"other_version_non_voters,omitempty" mapstructure:"other_version_non_voters,omitempty"`
+	OtherVersionReadReplicas  []string                                `json:"other_version_read_replicas,omitempty" mapstructure:"other_version_read_replicas,omitempty"`
+	RedundancyZones           map[string]AutopilotZoneUpgradeVersions `json:"redundancy_zones,omitempty" mapstructure:"redundancy_zones,omitempty"`
+}
+
+type AutopilotZoneUpgradeVersions struct {
+	TargetVersionVoters    []string `json:"target_version_voters,omitempty" mapstructure:"target_version_voters,omitempty"`
+	TargetVersionNonVoters []string `json:"target_version_non_voters,omitempty" mapstructure:"target_version_non_voters,omitempty"`
+	OtherVersionVoters     []string `json:"other_version_voters,omitempty" mapstructure:"other_version_voters,omitempty"`
+	OtherVersionNonVoters  []string `json:"other_version_non_voters,omitempty" mapstructure:"other_version_non_voters,omitempty"`
 }
 
 // ReadableDuration is a duration type that is serialized to JSON in human readable format.
@@ -597,13 +679,22 @@ func autopilotToAPIState(state *autopilot.State) (*AutopilotState, error) {
 	}
 
 	for id, srv := range state.Servers {
-		out.Servers[string(id)] = autopilotToAPIServer(srv)
+		aps, err := autopilotToAPIServer(srv)
+		if err != nil {
+			return nil, err
+		}
+		out.Servers[string(id)] = aps
+	}
+
+	err := autopilotToAPIStateEnterprise(state, out)
+	if err != nil {
+		return nil, err
 	}
 
 	return out, nil
 }
 
-func autopilotToAPIServer(srv *autopilot.ServerState) *AutopilotServer {
+func autopilotToAPIServer(srv *autopilot.ServerState) (*AutopilotServer, error) {
 	apiSrv := &AutopilotServer{
 		ID:          string(srv.Server.ID),
 		Name:        srv.Server.Name,
@@ -615,12 +706,16 @@ func autopilotToAPIServer(srv *autopilot.ServerState) *AutopilotServer {
 		Healthy:     srv.Health.Healthy,
 		StableSince: srv.Health.StableSince,
 		Status:      string(srv.State),
-		Meta:        srv.Server.Meta,
+		Version:     srv.Server.Version,
+		NodeType:    string(srv.Server.NodeType),
 	}
 
-	autopilotToAPIServerEnterprise(srv, apiSrv)
+	err := autopilotToAPIServerEnterprise(&srv.Server, apiSrv)
+	if err != nil {
+		return nil, err
+	}
 
-	return apiSrv
+	return apiSrv, nil
 }
 
 // GetAutopilotServerState retrieves raft cluster state from autopilot to
@@ -679,6 +774,9 @@ func (b *RaftBackend) SetupAutopilot(ctx context.Context, storageConfig *Autopil
 	if b.autopilotReconcileInterval != 0 {
 		options = append(options, autopilot.WithReconcileInterval(b.autopilotReconcileInterval))
 	}
+	if b.autopilotUpdateInterval != 0 {
+		options = append(options, autopilot.WithUpdateInterval(b.autopilotUpdateInterval))
+	}
 	b.autopilot = autopilot.New(b.raft, newDelegate(b), options...)
 	b.followerStates = followerStates
 	b.followerHeartbeatTicker = time.NewTicker(1 * time.Second)
diff --git a/physical/raft/raft_test.go b/physical/raft/raft_test.go
index 258d23bc6f27d..a9305d98518f5 100644
--- a/physical/raft/raft_test.go
+++ b/physical/raft/raft_test.go
@@ -15,12 +15,11 @@ import (
 	"testing"
 	"time"
 
-	"github.com/hashicorp/go-secure-stdlib/base62"
-
 	"github.com/go-test/deep"
 	"github.com/golang/protobuf/proto"
-	hclog "github.com/hashicorp/go-hclog"
-	uuid "github.com/hashicorp/go-uuid"
+	"github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/go-secure-stdlib/base62"
+	"github.com/hashicorp/go-uuid"
 	"github.com/hashicorp/raft"
 	"github.com/hashicorp/vault/sdk/helper/jsonutil"
 	"github.com/hashicorp/vault/sdk/physical"
diff --git a/physical/raft/raft_util.go b/physical/raft/raft_util.go
index 9761c77a7bb59..34570fba678f5 100644
--- a/physical/raft/raft_util.go
+++ b/physical/raft/raft_util.go
@@ -20,14 +20,22 @@ func (b *RaftBackend) AddNonVotingPeer(ctx context.Context, peerID, clusterAddr
 	return errors.New("adding non voting peer is not allowed")
 }
 
-func autopilotToAPIServerEnterprise(_ *autopilot.ServerState, _ *AutopilotServer) {
-	// noop in oss
+func autopilotToAPIServerEnterprise(_ *autopilot.Server, _ *AutopilotServer) error {
+	return nil
+}
+
+func autopilotToAPIStateEnterprise(_ *autopilot.State, _ *AutopilotState) error {
+	return nil
 }
 
 func (d *Delegate) autopilotConfigExt() interface{} {
 	return nil
 }
 
-func (d *Delegate) autopilotServerExt(_ string) interface{} {
+func (d *Delegate) autopilotServerExt(_ *FollowerState) interface{} {
+	return nil
+}
+
+func (d *Delegate) meta(_ *FollowerState) map[string]string {
 	return nil
 }
diff --git a/vault/core.go b/vault/core.go
index 5c7d93708bdc3..79dff2ad86ab5 100644
--- a/vault/core.go
+++ b/vault/core.go
@@ -892,6 +892,7 @@ func CreateCore(conf *CoreConfig) (*Core, error) {
 		mountMigrationTracker:          &sync.Map{},
 		disableSSCTokens:               conf.DisableSSCTokens,
 	}
+
 	c.standbyStopCh.Store(make(chan struct{}))
 	atomic.StoreUint32(c.sealed, 1)
 	c.metricSink.SetGaugeWithLabels([]string{"core", "unsealed"}, 0, nil)
@@ -3252,7 +3253,10 @@ type PeerNode struct {
 	Hostname       string    `json:"hostname"`
 	APIAddress     string    `json:"api_address"`
 	ClusterAddress string    `json:"cluster_address"`
+	Version        string    `json:"version"`
 	LastEcho       time.Time `json:"last_echo"`
+	UpgradeVersion string    `json:"upgrade_version,omitempty"`
+	RedundancyZone string    `json:"redundancy_zone,omitempty"`
 }
 
 // GetHAPeerNodesCached returns the nodes that've sent us Echo requests recently.
@@ -3265,6 +3269,9 @@ func (c *Core) GetHAPeerNodesCached() []PeerNode {
 			APIAddress:     info.nodeInfo.ApiAddr,
 			ClusterAddress: itemClusterAddr,
 			LastEcho:       info.lastHeartbeat,
+			Version:        info.version,
+			UpgradeVersion: info.upgradeVersion,
+			RedundancyZone: info.redundancyZone,
 		})
 	}
 	return nodes
diff --git a/vault/external_tests/raft/raft_autopilot_test.go b/vault/external_tests/raft/raft_autopilot_test.go
index 87daa02242d09..3759825f06271 100644
--- a/vault/external_tests/raft/raft_autopilot_test.go
+++ b/vault/external_tests/raft/raft_autopilot_test.go
@@ -30,9 +30,8 @@ func TestRaft_Autopilot_Disable(t *testing.T) {
 	})
 	defer cluster.Cleanup()
 
-	client := cluster.Cores[0].Client
-
-	state, err := client.Sys().RaftAutopilotState()
+	cli := cluster.Cores[0].Client
+	state, err := cli.Sys().RaftAutopilotState()
 	require.NoError(t, err)
 	require.Nil(t, nil, state)
 }
@@ -82,67 +81,8 @@ func TestRaft_Autopilot_Stabilization_And_State(t *testing.T) {
 	stabilizationKickOffWaitDuration := time.Duration(math.Ceil(1.1 * float64(config.ServerStabilizationTime)))
 	time.Sleep(stabilizationKickOffWaitDuration)
 
-	joinAndStabilizeFunc := func(core *vault.TestClusterCore, nodeID string, numServers int) {
-		joinFunc := func(core *vault.TestClusterCore) {
-			_, err := core.JoinRaftCluster(namespace.RootContext(context.Background()), []*raft.LeaderJoinInfo{
-				{
-					LeaderAPIAddr: client.Address(),
-					TLSConfig:     cluster.Cores[0].TLSConfig,
-					Retry:         true,
-				},
-			}, false)
-			require.NoError(t, err)
-			time.Sleep(1 * time.Second)
-			cluster.UnsealCore(t, core)
-		}
-		joinFunc(core)
-		time.Sleep(2 * time.Second)
-
-		state, err = client.Sys().RaftAutopilotState()
-		require.NoError(t, err)
-		require.Equal(t, false, state.Healthy)
-		require.Len(t, state.Servers, numServers)
-		require.Equal(t, false, state.Servers[nodeID].Healthy)
-		require.Equal(t, "alive", state.Servers[nodeID].NodeStatus)
-		require.Equal(t, "non-voter", state.Servers[nodeID].Status)
-
-		// Wait till the stabilization period is over
-		stabilizationWaitDuration := time.Duration(float64(config.ServerStabilizationTime))
-		deadline := time.Now().Add(stabilizationWaitDuration)
-		healthy := false
-		for time.Now().Before(deadline) {
-			state, err := client.Sys().RaftAutopilotState()
-			require.NoError(t, err)
-			if state.Healthy {
-				healthy = true
-			}
-			time.Sleep(1 * time.Second)
-		}
-		if !healthy {
-			t.Fatalf("cluster failed to stabilize")
-		}
-
-		// Now that the server is stable, wait for autopilot to reconcile and
-		// promotion to happen. Reconcile interval is 10 seconds. Bound it by
-		// doubling.
-		deadline = time.Now().Add(2 * autopilot.DefaultReconcileInterval)
-		failed := true
-		for time.Now().Before(deadline) {
-			state, err = client.Sys().RaftAutopilotState()
-			require.NoError(t, err)
-			if state.Servers[nodeID].Status == "voter" {
-				failed = false
-				break
-			}
-			time.Sleep(1 * time.Second)
-		}
-
-		if failed {
-			t.Fatalf("autopilot failed to promote node: id: %#v: state:%# v\n", nodeID, pretty.Formatter(state))
-		}
-	}
-	joinAndStabilizeFunc(cluster.Cores[1], "core-1", 2)
-	joinAndStabilizeFunc(cluster.Cores[2], "core-2", 3)
+	joinAndStabilizeAndPromote(t, cluster.Cores[1], client, cluster, config, "core-1", 2)
+	joinAndStabilizeAndPromote(t, cluster.Cores[2], client, cluster, config, "core-2", 3)
 	state, err = client.Sys().RaftAutopilotState()
 	require.NoError(t, err)
 	require.Equal(t, []string{"core-0", "core-1", "core-2"}, state.Voters)
@@ -313,20 +253,8 @@ func TestRaft_Autopilot_Stabilization_Delay(t *testing.T) {
 		}
 	}
 
-	joinFunc := func(core *vault.TestClusterCore) {
-		_, err := core.JoinRaftCluster(namespace.RootContext(context.Background()), []*raft.LeaderJoinInfo{
-			{
-				LeaderAPIAddr: client.Address(),
-				TLSConfig:     cluster.Cores[0].TLSConfig,
-			},
-		}, false)
-		require.NoError(t, err)
-		time.Sleep(1 * time.Second)
-		cluster.UnsealCore(t, core)
-	}
-
-	joinFunc(cluster.Cores[1])
-	joinFunc(cluster.Cores[2])
+	join(t, cluster.Cores[1], client, cluster)
+	join(t, cluster.Cores[2], client, cluster)
 
 	core2shouldBeHealthyAt := time.Now().Add(timeToHealthyCore2)
 
@@ -387,20 +315,8 @@ func TestRaft_AutoPilot_Peersets_Equivalent(t *testing.T) {
 	})
 	require.NoError(t, err)
 
-	joinFunc := func(core *vault.TestClusterCore) {
-		_, err := core.JoinRaftCluster(namespace.RootContext(context.Background()), []*raft.LeaderJoinInfo{
-			{
-				LeaderAPIAddr: client.Address(),
-				TLSConfig:     cluster.Cores[0].TLSConfig,
-			},
-		}, false)
-		require.NoError(t, err)
-		time.Sleep(1 * time.Second)
-		cluster.UnsealCore(t, core)
-	}
-
-	joinFunc(cluster.Cores[1])
-	joinFunc(cluster.Cores[2])
+	join(t, cluster.Cores[1], client, cluster)
+	join(t, cluster.Cores[2], client, cluster)
 
 	deadline := time.Now().Add(10 * time.Second)
 	var core0Peers, core1Peers, core2Peers []raft.Peer
@@ -427,3 +343,71 @@ func TestRaft_AutoPilot_Peersets_Equivalent(t *testing.T) {
 	require.Equal(t, core0Peers, core1Peers)
 	require.Equal(t, core1Peers, core2Peers)
 }
+
+func joinAndStabilizeAndPromote(t *testing.T, core *vault.TestClusterCore, client *api.Client, cluster *vault.TestCluster, config *api.AutopilotConfig, nodeID string, numServers int) {
+	joinAndStabilize(t, core, client, cluster, config, nodeID, numServers)
+
+	// Now that the server is stable, wait for autopilot to reconcile and
+	// promotion to happen. Reconcile interval is 10 seconds. Bound it by
+	// doubling.
+	deadline := time.Now().Add(2 * autopilot.DefaultReconcileInterval)
+	failed := true
+	var err error
+	var state *api.AutopilotState
+	for time.Now().Before(deadline) {
+		state, err = client.Sys().RaftAutopilotState()
+		require.NoError(t, err)
+		if state.Servers[nodeID].Status == "voter" {
+			failed = false
+			break
+		}
+		time.Sleep(1 * time.Second)
+	}
+
+	if failed {
+		t.Fatalf("autopilot failed to promote node: id: %#v: state:%# v\n", nodeID, pretty.Formatter(state))
+	}
+}
+
+func joinAndStabilize(t *testing.T, core *vault.TestClusterCore, client *api.Client, cluster *vault.TestCluster, config *api.AutopilotConfig, nodeID string, numServers int) {
+	t.Helper()
+	join(t, core, client, cluster)
+	time.Sleep(2 * time.Second)
+
+	state, err := client.Sys().RaftAutopilotState()
+	require.NoError(t, err)
+	require.Equal(t, false, state.Healthy)
+	require.Len(t, state.Servers, numServers)
+	require.Equal(t, false, state.Servers[nodeID].Healthy)
+	require.Equal(t, "alive", state.Servers[nodeID].NodeStatus)
+	require.Equal(t, "non-voter", state.Servers[nodeID].Status)
+
+	// Wait till the stabilization period is over
+	deadline := time.Now().Add(config.ServerStabilizationTime)
+	healthy := false
+	for time.Now().Before(deadline) {
+		state, err := client.Sys().RaftAutopilotState()
+		require.NoError(t, err)
+		if state.Healthy {
+			healthy = true
+		}
+		time.Sleep(1 * time.Second)
+	}
+	if !healthy {
+		t.Fatalf("cluster failed to stabilize")
+	}
+}
+
+func join(t *testing.T, core *vault.TestClusterCore, client *api.Client, cluster *vault.TestCluster) {
+	t.Helper()
+	_, err := core.JoinRaftCluster(namespace.RootContext(context.Background()), []*raft.LeaderJoinInfo{
+		{
+			LeaderAPIAddr: client.Address(),
+			TLSConfig:     cluster.Cores[0].TLSConfig,
+			Retry:         true,
+		},
+	}, false)
+	require.NoError(t, err)
+	time.Sleep(1 * time.Second)
+	cluster.UnsealCore(t, core)
+}
diff --git a/vault/external_tests/raft/raft_test.go b/vault/external_tests/raft/raft_test.go
index 31a2afff4618f..345f53f20d830 100644
--- a/vault/external_tests/raft/raft_test.go
+++ b/vault/external_tests/raft/raft_test.go
@@ -15,13 +15,11 @@ import (
 	"testing"
 	"time"
 
-	"github.com/hashicorp/vault/helper/benchhelpers"
-	vaultseal "github.com/hashicorp/vault/vault/seal"
-
 	"github.com/hashicorp/go-cleanhttp"
 	"github.com/hashicorp/go-uuid"
 	"github.com/hashicorp/vault/api"
 	credUserpass "github.com/hashicorp/vault/builtin/credential/userpass"
+	"github.com/hashicorp/vault/helper/benchhelpers"
 	"github.com/hashicorp/vault/helper/constants"
 	"github.com/hashicorp/vault/helper/namespace"
 	"github.com/hashicorp/vault/helper/testhelpers"
@@ -31,6 +29,7 @@ import (
 	"github.com/hashicorp/vault/physical/raft"
 	"github.com/hashicorp/vault/sdk/logical"
 	"github.com/hashicorp/vault/vault"
+	vaultseal "github.com/hashicorp/vault/vault/seal"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/net/http2"
 )
@@ -44,6 +43,8 @@ type RaftClusterOpts struct {
 	EnableResponseHeaderRaftNodeID bool
 	NumCores                       int
 	Seal                           vault.Seal
+	VersionMap                     map[int]string
+	RedundancyZoneMap              map[int]string
 }
 
 func raftCluster(t testing.TB, ropts *RaftClusterOpts) *vault.TestCluster {
@@ -67,6 +68,8 @@ func raftCluster(t testing.TB, ropts *RaftClusterOpts) *vault.TestCluster {
 	opts.PhysicalFactoryConfig = ropts.PhysicalFactoryConfig
 	conf.DisablePerformanceStandby = ropts.DisablePerfStandby
 	opts.NumCores = ropts.NumCores
+	opts.VersionMap = ropts.VersionMap
+	opts.RedundancyZoneMap = ropts.RedundancyZoneMap
 
 	teststorage.RaftBackendSetup(conf, &opts)
 
diff --git a/vault/logical_system.go b/vault/logical_system.go
index 7af2a737f8840..6c9644789e8d1 100644
--- a/vault/logical_system.go
+++ b/vault/logical_system.go
@@ -741,7 +741,8 @@ func (b *SystemBackend) handleRekeyRetrieve(
 	ctx context.Context,
 	req *logical.Request,
 	data *framework.FieldData,
-	recovery bool) (*logical.Response, error) {
+	recovery bool,
+) (*logical.Response, error) {
 	backup, err := b.Core.RekeyRetrieveBackup(ctx, recovery)
 	if err != nil {
 		return nil, fmt.Errorf("unable to look up backed-up keys: %w", err)
@@ -792,7 +793,8 @@ func (b *SystemBackend) handleRekeyDelete(
 	ctx context.Context,
 	req *logical.Request,
 	data *framework.FieldData,
-	recovery bool) (*logical.Response, error) {
+	recovery bool,
+) (*logical.Response, error) {
 	err := b.Core.RekeyDeleteBackup(ctx, recovery)
 	if err != nil {
 		return nil, fmt.Errorf("error during deletion of backed-up keys: %w", err)
@@ -1082,7 +1084,8 @@ func (b *SystemBackend) handleReadMount(ctx context.Context, req *logical.Reques
 
 // used to intercept an HTTPCodedError so it goes back to callee
 func handleError(
-	err error) (*logical.Response, error) {
+	err error,
+) (*logical.Response, error) {
 	if strings.Contains(err.Error(), logical.ErrReadOnly.Error()) {
 		return logical.ErrorResponse(err.Error()), err
 	}
@@ -1097,7 +1100,8 @@ func handleError(
 // Performs a similar function to handleError, but upon seeing a ReadOnlyError
 // will actually strip it out to prevent forwarding
 func handleErrorNoReadOnlyForward(
-	err error) (*logical.Response, error) {
+	err error,
+) (*logical.Response, error) {
 	if strings.Contains(err.Error(), logical.ErrReadOnly.Error()) {
 		return nil, fmt.Errorf("operation could not be completed as storage is read-only")
 	}
@@ -2007,7 +2011,8 @@ func (b *SystemBackend) handleRevokeForce(ctx context.Context, req *logical.Requ
 
 // handleRevokePrefixCommon is used to revoke a prefix with many LeaseIDs
 func (b *SystemBackend) handleRevokePrefixCommon(ctx context.Context,
-	req *logical.Request, data *framework.FieldData, force, sync bool) (*logical.Response, error) {
+	req *logical.Request, data *framework.FieldData, force, sync bool,
+) (*logical.Response, error) {
 	// Get all the options
 	prefix := data.Get("prefix").(string)
 
@@ -4318,15 +4323,21 @@ func (b *SystemBackend) handleHAStatus(ctx context.Context, req *logical.Request
 		return nil, err
 	}
 
-	nodes := []HAStatusNode{
-		{
-			Hostname:       hostname,
-			APIAddress:     b.Core.redirectAddr,
-			ClusterAddress: b.Core.ClusterAddr(),
-			ActiveNode:     true,
-		},
+	leader := HAStatusNode{
+		Hostname:       hostname,
+		APIAddress:     b.Core.redirectAddr,
+		ClusterAddress: b.Core.ClusterAddr(),
+		ActiveNode:     true,
+		Version:        version.GetVersion().Version,
+	}
+
+	if rb := b.Core.getRaftBackend(); rb != nil {
+		leader.UpgradeVersion = rb.EffectiveVersion()
+		leader.RedundancyZone = rb.RedundancyZone()
 	}
 
+	nodes := []HAStatusNode{leader}
+
 	for _, peerNode := range b.Core.GetHAPeerNodesCached() {
 		lastEcho := peerNode.LastEcho
 		nodes = append(nodes, HAStatusNode{
@@ -4334,9 +4345,16 @@ func (b *SystemBackend) handleHAStatus(ctx context.Context, req *logical.Request
 			APIAddress:     peerNode.APIAddress,
 			ClusterAddress: peerNode.ClusterAddress,
 			LastEcho:       &lastEcho,
+			Version:        peerNode.Version,
+			UpgradeVersion: peerNode.UpgradeVersion,
+			RedundancyZone: peerNode.RedundancyZone,
 		})
 	}
 
+	sort.Slice(nodes, func(i, j int) bool {
+		return nodes[i].APIAddress < nodes[j].APIAddress
+	})
+
 	return &logical.Response{
 		Data: map[string]interface{}{
 			"nodes": nodes,
@@ -4350,6 +4368,9 @@ type HAStatusNode struct {
 	ClusterAddress string     `json:"cluster_address"`
 	ActiveNode     bool       `json:"active_node"`
 	LastEcho       *time.Time `json:"last_echo"`
+	Version        string     `json:"version"`
+	UpgradeVersion string     `json:"upgrade_version,omitempty"`
+	RedundancyZone string     `json:"redundancy_zone,omitempty"`
 }
 
 func (b *SystemBackend) handleVersionHistoryList(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
diff --git a/vault/logical_system_raft.go b/vault/logical_system_raft.go
index 08d1a0d59643e..3516828cb4dcc 100644
--- a/vault/logical_system_raft.go
+++ b/vault/logical_system_raft.go
@@ -9,15 +9,16 @@ import (
 	"strings"
 	"time"
 
-	"github.com/hashicorp/vault/sdk/framework"
-	"github.com/hashicorp/vault/sdk/logical"
-	"github.com/hashicorp/vault/sdk/physical"
-
-	proto "github.com/golang/protobuf/proto"
+	"github.com/golang/protobuf/proto"
 	wrapping "github.com/hashicorp/go-kms-wrapping"
-	uuid "github.com/hashicorp/go-uuid"
+	"github.com/hashicorp/go-uuid"
+	"github.com/hashicorp/vault/helper/constants"
 	"github.com/hashicorp/vault/helper/namespace"
 	"github.com/hashicorp/vault/physical/raft"
+	"github.com/hashicorp/vault/sdk/framework"
+	"github.com/hashicorp/vault/sdk/logical"
+	"github.com/hashicorp/vault/sdk/physical"
+	"github.com/mitchellh/mapstructure"
 )
 
 // raftStoragePaths returns paths for use when raft is the storage mechanism.
@@ -188,6 +189,10 @@ func (b *SystemBackend) raftStoragePaths() []*framework.Path {
 					Type:        framework.TypeDurationSecond,
 					Description: "Minimum amount of time a server must be in a stable, healthy state before it can be added to the cluster.",
 				},
+				"disable_upgrade_migration": {
+					Type:        framework.TypeBool,
+					Description: "Whether or not to perform automated version upgrades.",
+				},
 			},
 
 			Operations: map[logical.Operation]framework.OperationHandler{
@@ -362,7 +367,10 @@ func (b *SystemBackend) handleRaftBootstrapAnswerWrite() framework.OperationFunc
 		}
 
 		if b.Core.raftFollowerStates != nil {
-			b.Core.raftFollowerStates.Update(serverID, 0, 0, desiredSuffrage)
+			b.Core.raftFollowerStates.Update(&raft.EchoRequestUpdate{
+				NodeID:          serverID,
+				DesiredSuffrage: desiredSuffrage,
+			})
 		}
 
 		peers, err := raftBackend.Peers(ctx)
@@ -417,15 +425,14 @@ func (b *SystemBackend) handleStorageRaftAutopilotState() framework.OperationFun
 			return nil, nil
 		}
 
+		data := make(map[string]interface{})
+		err = mapstructure.Decode(state, &data)
+		if err != nil {
+			return nil, err
+		}
+
 		return &logical.Response{
-			Data: map[string]interface{}{
-				"healthy":           state.Healthy,
-				"failure_tolerance": state.FailureTolerance,
-				"servers":           state.Servers,
-				"leader":            state.Leader,
-				"voters":            state.Voters,
-				"non_voters":        state.NonVoters,
-			},
+			Data: data,
 		}, nil
 	}
 }
@@ -450,6 +457,7 @@ func (b *SystemBackend) handleStorageRaftAutopilotConfigRead() framework.Operati
 				"max_trailing_logs":                  config.MaxTrailingLogs,
 				"min_quorum":                         config.MinQuorum,
 				"server_stabilization_time":          config.ServerStabilizationTime.String(),
+				"disable_upgrade_migration":          config.DisableUpgradeMigration,
 			},
 		}, nil
 	}
@@ -506,6 +514,14 @@ func (b *SystemBackend) handleStorageRaftAutopilotConfigUpdate() framework.Opera
 			config.ServerStabilizationTime = time.Duration(serverStabilizationTime.(int)) * time.Second
 			persist = true
 		}
+		disableUpgradeMigration, ok := d.GetOk("disable_upgrade_migration")
+		if ok {
+			if !constants.IsEnterprise {
+				return logical.ErrorResponse("disable_upgrade_migration is only available in Vault Enterprise"), logical.ErrInvalidRequest
+			}
+			config.DisableUpgradeMigration = disableUpgradeMigration.(bool)
+			persist = true
+		}
 
 		effectiveConf := raftBackend.AutopilotConfig()
 		effectiveConf.Merge(config)
diff --git a/vault/raft.go b/vault/raft.go
index bffab8a69a7f2..bf9f1c0af64a5 100644
--- a/vault/raft.go
+++ b/vault/raft.go
@@ -13,13 +13,13 @@ import (
 	"time"
 
 	"github.com/golang/protobuf/proto"
-	cleanhttp "github.com/hashicorp/go-cleanhttp"
+	"github.com/hashicorp/go-cleanhttp"
 	"github.com/hashicorp/go-discover"
 	discoverk8s "github.com/hashicorp/go-discover/provider/k8s"
 	"github.com/hashicorp/go-hclog"
 	wrapping "github.com/hashicorp/go-kms-wrapping"
 	"github.com/hashicorp/go-secure-stdlib/tlsutil"
-	uuid "github.com/hashicorp/go-uuid"
+	"github.com/hashicorp/go-uuid"
 	"github.com/hashicorp/vault/api"
 	"github.com/hashicorp/vault/physical/raft"
 	"github.com/hashicorp/vault/sdk/helper/jsonutil"
@@ -181,6 +181,7 @@ func (c *Core) setupRaftActiveNode(ctx context.Context) error {
 		c.logger.Error("failed to load autopilot config from storage when setting up cluster; continuing since autopilot falls back to default config", "error", err)
 	}
 	disableAutopilot := c.disableAutopilot
+
 	raftBackend.SetupAutopilot(c.activeContext, autopilotConfig, c.raftFollowerStates, disableAutopilot)
 
 	c.pendingRaftPeers = &sync.Map{}
@@ -346,7 +347,12 @@ func (c *Core) raftTLSRotatePhased(ctx context.Context, logger hclog.Logger, raf
 	}
 	for _, server := range raftConfig.Servers {
 		if server.NodeID != raftBackend.NodeID() {
-			followerStates.Update(server.NodeID, 0, 0, "voter")
+			followerStates.Update(&raft.EchoRequestUpdate{
+				NodeID:          server.NodeID,
+				AppliedIndex:    0,
+				Term:            0,
+				DesiredSuffrage: "voter",
+			})
 		}
 	}
 
@@ -1018,7 +1024,7 @@ func (c *Core) JoinRaftCluster(ctx context.Context, leaderInfos []*raft.LeaderJo
 				if err == nil {
 					return
 				}
-				c.logger.Error("failed to retry join raft cluster", "retry", "2s")
+				c.logger.Error("failed to retry join raft cluster", "retry", "2s", "err", err)
 				time.Sleep(2 * time.Second)
 			}
 		}()
diff --git a/vault/request_forwarding.go b/vault/request_forwarding.go
index b4a515f906c7e..b68b6a0d6b01e 100644
--- a/vault/request_forwarding.go
+++ b/vault/request_forwarding.go
@@ -8,7 +8,7 @@ import (
 	"crypto/x509"
 	"errors"
 	"fmt"
-	math "math"
+	"math"
 	"net/http"
 	"net/url"
 	"sync"
@@ -234,8 +234,8 @@ func (c *Core) stopForwarding() {
 // alive and that the current active address value matches the most
 // recently-known address.
 func (c *Core) refreshRequestForwardingConnection(ctx context.Context, clusterAddr string) error {
-	c.logger.Debug("refreshing forwarding connection")
-	defer c.logger.Debug("done refreshing forwarding connection")
+	c.logger.Debug("refreshing forwarding connection", "clusterAddr", clusterAddr)
+	defer c.logger.Debug("done refreshing forwarding connection", "clusterAddr", clusterAddr)
 
 	c.requestForwardingConnectionLock.Lock()
 	defer c.requestForwardingConnectionLock.Unlock()
diff --git a/vault/request_forwarding_rpc.go b/vault/request_forwarding_rpc.go
index 1c9a5deadd95d..1496c1683f6bb 100644
--- a/vault/request_forwarding_rpc.go
+++ b/vault/request_forwarding_rpc.go
@@ -8,10 +8,10 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/hashicorp/vault/sdk/helper/consts"
-
 	"github.com/hashicorp/vault/helper/forwarding"
 	"github.com/hashicorp/vault/physical/raft"
+	"github.com/hashicorp/vault/sdk/helper/consts"
+	"github.com/hashicorp/vault/sdk/version"
 	"github.com/hashicorp/vault/vault/replication"
 )
 
@@ -73,21 +73,34 @@ func (s *forwardedRequestRPCServer) ForwardRequest(ctx context.Context, freq *fo
 }
 
 type nodeHAConnectionInfo struct {
-	nodeInfo      *NodeInformation
-	lastHeartbeat time.Time
+	nodeInfo       *NodeInformation
+	lastHeartbeat  time.Time
+	version        string
+	upgradeVersion string
+	redundancyZone string
 }
 
 func (s *forwardedRequestRPCServer) Echo(ctx context.Context, in *EchoRequest) (*EchoReply, error) {
 	incomingNodeConnectionInfo := nodeHAConnectionInfo{
-		nodeInfo:      in.NodeInfo,
-		lastHeartbeat: time.Now(),
+		nodeInfo:       in.NodeInfo,
+		lastHeartbeat:  time.Now(),
+		version:        in.SdkVersion,
+		upgradeVersion: in.RaftUpgradeVersion,
+		redundancyZone: in.RaftRedundancyZone,
 	}
 	if in.ClusterAddr != "" {
 		s.core.clusterPeerClusterAddrsCache.Set(in.ClusterAddr, incomingNodeConnectionInfo, 0)
 	}
 
 	if in.RaftAppliedIndex > 0 && len(in.RaftNodeID) > 0 && s.raftFollowerStates != nil {
-		s.raftFollowerStates.Update(in.RaftNodeID, in.RaftAppliedIndex, in.RaftTerm, in.RaftDesiredSuffrage)
+		s.raftFollowerStates.Update(&raft.EchoRequestUpdate{
+			NodeID:          in.RaftNodeID,
+			AppliedIndex:    in.RaftAppliedIndex,
+			Term:            in.RaftTerm,
+			DesiredSuffrage: in.RaftDesiredSuffrage,
+			UpgradeVersion:  in.RaftUpgradeVersion,
+			RedundancyZone:  in.RaftRedundancyZone,
+		})
 	}
 
 	reply := &EchoReply{
@@ -105,9 +118,7 @@ func (s *forwardedRequestRPCServer) Echo(ctx context.Context, in *EchoRequest) (
 
 type forwardingClient struct {
 	RequestForwardingClient
-
-	core *Core
-
+	core        *Core
 	echoTicker  *time.Ticker
 	echoContext context.Context
 }
@@ -128,6 +139,7 @@ func (c *forwardingClient) startHeartbeat() {
 				Message:     "ping",
 				ClusterAddr: clusterAddr,
 				NodeInfo:    &ni,
+				SdkVersion:  version.GetVersion().Version,
 			}
 
 			if raftBackend := c.core.getRaftBackend(); raftBackend != nil {
@@ -135,6 +147,8 @@ func (c *forwardingClient) startHeartbeat() {
 				req.RaftNodeID = raftBackend.NodeID()
 				req.RaftTerm = raftBackend.Term()
 				req.RaftDesiredSuffrage = raftBackend.DesiredSuffrage()
+				req.RaftRedundancyZone = raftBackend.RedundancyZone()
+				req.RaftUpgradeVersion = raftBackend.EffectiveVersion()
 			}
 
 			ctx, cancel := context.WithTimeout(c.echoContext, 2*time.Second)
diff --git a/vault/request_forwarding_service.pb.go b/vault/request_forwarding_service.pb.go
index d16aa5d07155e..d7170ded484e2 100644
--- a/vault/request_forwarding_service.pb.go
+++ b/vault/request_forwarding_service.pb.go
@@ -38,6 +38,9 @@ type EchoRequest struct {
 	NodeInfo            *NodeInformation `protobuf:"bytes,6,opt,name=node_info,json=nodeInfo,proto3" json:"node_info,omitempty"`
 	RaftTerm            uint64           `protobuf:"varint,7,opt,name=raft_term,json=raftTerm,proto3" json:"raft_term,omitempty"`
 	RaftDesiredSuffrage string           `protobuf:"bytes,8,opt,name=raft_desired_suffrage,json=raftDesiredSuffrage,proto3" json:"raft_desired_suffrage,omitempty"`
+	RaftUpgradeVersion  string           `protobuf:"bytes,9,opt,name=raft_upgrade_version,json=raftUpgradeVersion,proto3" json:"raft_upgrade_version,omitempty"`
+	RaftRedundancyZone  string           `protobuf:"bytes,10,opt,name=raft_redundancy_zone,json=raftRedundancyZone,proto3" json:"raft_redundancy_zone,omitempty"`
+	SdkVersion          string           `protobuf:"bytes,11,opt,name=sdk_version,json=sdkVersion,proto3" json:"sdk_version,omitempty"`
 }
 
 func (x *EchoRequest) Reset() {
@@ -128,6 +131,27 @@ func (x *EchoRequest) GetRaftDesiredSuffrage() string {
 	return ""
 }
 
+func (x *EchoRequest) GetRaftUpgradeVersion() string {
+	if x != nil {
+		return x.RaftUpgradeVersion
+	}
+	return ""
+}
+
+func (x *EchoRequest) GetRaftRedundancyZone() string {
+	if x != nil {
+		return x.RaftRedundancyZone
+	}
+	return ""
+}
+
+func (x *EchoRequest) GetSdkVersion() string {
+	if x != nil {
+		return x.SdkVersion
+	}
+	return ""
+}
+
 type EchoReply struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -505,8 +529,8 @@ var file_vault_request_forwarding_service_proto_rawDesc = []byte{
 	0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69,
 	0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x05, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x1a,
 	0x1d, 0x68, 0x65, 0x6c, 0x70, 0x65, 0x72, 0x2f, 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69,
-	0x6e, 0x67, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xc5,
-	0x02, 0x0a, 0x0b, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18,
+	0x6e, 0x67, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xca,
+	0x03, 0x0a, 0x0b, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18,
 	0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
 	0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6c, 0x75, 0x73,
 	0x74, 0x65, 0x72, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b,
@@ -526,74 +550,83 @@ var file_vault_request_forwarding_service_proto_rawDesc = []byte{
 	0x72, 0x6d, 0x12, 0x32, 0x0a, 0x15, 0x72, 0x61, 0x66, 0x74, 0x5f, 0x64, 0x65, 0x73, 0x69, 0x72,
 	0x65, 0x64, 0x5f, 0x73, 0x75, 0x66, 0x66, 0x72, 0x61, 0x67, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28,
 	0x09, 0x52, 0x13, 0x72, 0x61, 0x66, 0x74, 0x44, 0x65, 0x73, 0x69, 0x72, 0x65, 0x64, 0x53, 0x75,
-	0x66, 0x66, 0x72, 0x61, 0x67, 0x65, 0x22, 0xfc, 0x01, 0x0a, 0x09, 0x45, 0x63, 0x68, 0x6f, 0x52,
-	0x65, 0x70, 0x6c, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18,
-	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x23,
-	0x0a, 0x0d, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x73, 0x18,
-	0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x41, 0x64,
-	0x64, 0x72, 0x73, 0x12, 0x2b, 0x0a, 0x11, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69,
-	0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x10,
-	0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65,
-	0x12, 0x2c, 0x0a, 0x12, 0x72, 0x61, 0x66, 0x74, 0x5f, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x64,
-	0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x10, 0x72, 0x61,
-	0x66, 0x74, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x64, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x20,
-	0x0a, 0x0c, 0x72, 0x61, 0x66, 0x74, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x05,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x72, 0x61, 0x66, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x64,
-	0x12, 0x33, 0x0a, 0x09, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x06, 0x20,
-	0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x4e, 0x6f, 0x64, 0x65,
-	0x49, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x6e, 0x6f, 0x64,
-	0x65, 0x49, 0x6e, 0x66, 0x6f, 0x22, 0xc5, 0x01, 0x0a, 0x0f, 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x6e,
-	0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6c, 0x75,
-	0x73, 0x74, 0x65, 0x72, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x0b, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x12, 0x19, 0x0a, 0x08,
-	0x61, 0x70, 0x69, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07,
-	0x61, 0x70, 0x69, 0x41, 0x64, 0x64, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x18,
-	0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x6e,
-	0x6f, 0x64, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6e, 0x6f,
-	0x64, 0x65, 0x49, 0x64, 0x12, 0x2b, 0x0a, 0x11, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74,
-	0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52,
-	0x10, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74,
-	0x65, 0x12, 0x1a, 0x0a, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x06, 0x20,
-	0x01, 0x28, 0x09, 0x52, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x49, 0x0a,
-	0x09, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79,
-	0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x0c,
-	0x0a, 0x01, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x01, 0x78, 0x12, 0x0c, 0x0a, 0x01,
-	0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x01, 0x79, 0x12, 0x0c, 0x0a, 0x01, 0x64, 0x18,
-	0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x01, 0x64, 0x22, 0x1a, 0x0a, 0x18, 0x50, 0x65, 0x72, 0x66,
-	0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79, 0x45, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49,
-	0x6e, 0x70, 0x75, 0x74, 0x22, 0xe9, 0x01, 0x0a, 0x1b, 0x50, 0x65, 0x72, 0x66, 0x53, 0x74, 0x61,
+	0x66, 0x66, 0x72, 0x61, 0x67, 0x65, 0x12, 0x30, 0x0a, 0x14, 0x72, 0x61, 0x66, 0x74, 0x5f, 0x75,
+	0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x09,
+	0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x72, 0x61, 0x66, 0x74, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64,
+	0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x30, 0x0a, 0x14, 0x72, 0x61, 0x66, 0x74,
+	0x5f, 0x72, 0x65, 0x64, 0x75, 0x6e, 0x64, 0x61, 0x6e, 0x63, 0x79, 0x5f, 0x7a, 0x6f, 0x6e, 0x65,
+	0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x72, 0x61, 0x66, 0x74, 0x52, 0x65, 0x64, 0x75,
+	0x6e, 0x64, 0x61, 0x6e, 0x63, 0x79, 0x5a, 0x6f, 0x6e, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x64,
+	0x6b, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52,
+	0x0a, 0x73, 0x64, 0x6b, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0xfc, 0x01, 0x0a, 0x09,
+	0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73,
+	0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73,
+	0x61, 0x67, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x61,
+	0x64, 0x64, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x63, 0x6c, 0x75, 0x73,
+	0x74, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x73, 0x12, 0x2b, 0x0a, 0x11, 0x72, 0x65, 0x70, 0x6c,
+	0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20,
+	0x01, 0x28, 0x0d, 0x52, 0x10, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e,
+	0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x2c, 0x0a, 0x12, 0x72, 0x61, 0x66, 0x74, 0x5f, 0x61, 0x70,
+	0x70, 0x6c, 0x69, 0x65, 0x64, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x18, 0x04, 0x20, 0x01, 0x28,
+	0x04, 0x52, 0x10, 0x72, 0x61, 0x66, 0x74, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x64, 0x49, 0x6e,
+	0x64, 0x65, 0x78, 0x12, 0x20, 0x0a, 0x0c, 0x72, 0x61, 0x66, 0x74, 0x5f, 0x6e, 0x6f, 0x64, 0x65,
+	0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x72, 0x61, 0x66, 0x74, 0x4e,
+	0x6f, 0x64, 0x65, 0x49, 0x64, 0x12, 0x33, 0x0a, 0x09, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x6e,
+	0x66, 0x6f, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74,
+	0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e,
+	0x52, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x22, 0xc5, 0x01, 0x0a, 0x0f, 0x4e,
+	0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x21,
+	0x0a, 0x0c, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x18, 0x01,
+	0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x41, 0x64, 0x64,
+	0x72, 0x12, 0x19, 0x0a, 0x08, 0x61, 0x70, 0x69, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x18, 0x02, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x70, 0x69, 0x41, 0x64, 0x64, 0x72, 0x12, 0x12, 0x0a, 0x04,
+	0x6d, 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6d, 0x6f, 0x64, 0x65,
+	0x12, 0x17, 0x0a, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x06, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x64, 0x12, 0x2b, 0x0a, 0x11, 0x72, 0x65, 0x70,
+	0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x05,
+	0x20, 0x01, 0x28, 0x0d, 0x52, 0x10, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f,
+	0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61,
+	0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61,
+	0x6d, 0x65, 0x22, 0x49, 0x0a, 0x09, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4b, 0x65, 0x79, 0x12,
+	0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74,
+	0x79, 0x70, 0x65, 0x12, 0x0c, 0x0a, 0x01, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x01,
+	0x78, 0x12, 0x0c, 0x0a, 0x01, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x01, 0x79, 0x12,
+	0x0c, 0x0a, 0x01, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x01, 0x64, 0x22, 0x1a, 0x0a,
+	0x18, 0x50, 0x65, 0x72, 0x66, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79, 0x45, 0x6c, 0x65, 0x63,
+	0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x22, 0xe9, 0x01, 0x0a, 0x1b, 0x50, 0x65,
+	0x72, 0x66, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79, 0x45, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f,
+	0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75,
+	0x73, 0x74, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63,
+	0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x30, 0x0a, 0x14, 0x70, 0x72, 0x69, 0x6d,
+	0x61, 0x72, 0x79, 0x5f, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x61, 0x64, 0x64, 0x72,
+	0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x43,
+	0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x12, 0x17, 0x0a, 0x07, 0x63, 0x61,
+	0x5f, 0x63, 0x65, 0x72, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x63, 0x61, 0x43,
+	0x65, 0x72, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x63, 0x65,
+	0x72, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74,
+	0x43, 0x65, 0x72, 0x74, 0x12, 0x2f, 0x0a, 0x0a, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x6b,
+	0x65, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74,
+	0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4b, 0x65, 0x79, 0x52, 0x09, 0x63, 0x6c, 0x69, 0x65,
+	0x6e, 0x74, 0x4b, 0x65, 0x79, 0x32, 0xf0, 0x01, 0x0a, 0x11, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
+	0x74, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x12, 0x3d, 0x0a, 0x0e, 0x46,
+	0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x13, 0x2e,
+	0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65,
+	0x73, 0x74, 0x1a, 0x14, 0x2e, 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x2e,
+	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x2e, 0x0a, 0x04, 0x45, 0x63,
+	0x68, 0x6f, 0x12, 0x12, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x52,
+	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x45,
+	0x63, 0x68, 0x6f, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x6c, 0x0a, 0x21, 0x50, 0x65,
+	0x72, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79,
+	0x45, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12,
+	0x1f, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x50, 0x65, 0x72, 0x66, 0x53, 0x74, 0x61, 0x6e,
+	0x64, 0x62, 0x79, 0x45, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x70, 0x75, 0x74,
+	0x1a, 0x22, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x50, 0x65, 0x72, 0x66, 0x53, 0x74, 0x61,
 	0x6e, 0x64, 0x62, 0x79, 0x45, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70,
-	0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
-	0x52, 0x02, 0x69, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f,
-	0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65,
-	0x72, 0x49, 0x64, 0x12, 0x30, 0x0a, 0x14, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x63,
-	0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28,
-	0x09, 0x52, 0x12, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65,
-	0x72, 0x41, 0x64, 0x64, 0x72, 0x12, 0x17, 0x0a, 0x07, 0x63, 0x61, 0x5f, 0x63, 0x65, 0x72, 0x74,
-	0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x63, 0x61, 0x43, 0x65, 0x72, 0x74, 0x12, 0x1f,
-	0x0a, 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x18, 0x05, 0x20,
-	0x01, 0x28, 0x0c, 0x52, 0x0a, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x65, 0x72, 0x74, 0x12,
-	0x2f, 0x0a, 0x0a, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x06, 0x20,
-	0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x43, 0x6c, 0x69, 0x65,
-	0x6e, 0x74, 0x4b, 0x65, 0x79, 0x52, 0x09, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4b, 0x65, 0x79,
-	0x32, 0xf0, 0x01, 0x0a, 0x11, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x46, 0x6f, 0x72, 0x77,
-	0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x12, 0x3d, 0x0a, 0x0e, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72,
-	0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x13, 0x2e, 0x66, 0x6f, 0x72, 0x77, 0x61,
-	0x72, 0x64, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e,
-	0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f,
-	0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x2e, 0x0a, 0x04, 0x45, 0x63, 0x68, 0x6f, 0x12, 0x12, 0x2e,
-	0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
-	0x74, 0x1a, 0x10, 0x2e, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65,
-	0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x6c, 0x0a, 0x21, 0x50, 0x65, 0x72, 0x66, 0x6f, 0x72, 0x6d,
-	0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79, 0x45, 0x6c, 0x65, 0x63, 0x74,
-	0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1f, 0x2e, 0x76, 0x61, 0x75,
-	0x6c, 0x74, 0x2e, 0x50, 0x65, 0x72, 0x66, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79, 0x45, 0x6c,
-	0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x1a, 0x22, 0x2e, 0x76, 0x61,
-	0x75, 0x6c, 0x74, 0x2e, 0x50, 0x65, 0x72, 0x66, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x62, 0x79, 0x45,
-	0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22,
-	0x00, 0x30, 0x01, 0x42, 0x22, 0x5a, 0x20, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
-	0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x76, 0x61, 0x75, 0x6c,
-	0x74, 0x2f, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x42, 0x22, 0x5a, 0x20, 0x67, 0x69, 0x74, 0x68,
+	0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70,
+	0x2f, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2f, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x62, 0x06, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
diff --git a/vault/request_forwarding_service.proto b/vault/request_forwarding_service.proto
index 5617242b9cc80..508248a2268fd 100644
--- a/vault/request_forwarding_service.proto
+++ b/vault/request_forwarding_service.proto
@@ -20,6 +20,9 @@ message EchoRequest {
 	NodeInformation node_info = 6;
 	uint64 raft_term = 7;
 	string raft_desired_suffrage = 8;
+	string raft_upgrade_version = 9;
+	string raft_redundancy_zone = 10;
+	string sdk_version = 11;
 }
 
 message EchoReply {
diff --git a/vault/testing.go b/vault/testing.go
index 3143a192aea93..652861c9b1f27 100644
--- a/vault/testing.go
+++ b/vault/testing.go
@@ -1150,6 +1150,10 @@ type TestClusterOptions struct {
 	PhysicalFactoryConfig map[string]interface{}
 	LicensePublicKey      ed25519.PublicKey
 	LicensePrivateKey     ed25519.PrivateKey
+
+	// this stores the vault version that should be used for each core config
+	VersionMap        map[int]string
+	RedundancyZoneMap map[int]string
 }
 
 var DefaultNumCores = 3
@@ -1411,7 +1415,6 @@ func NewTestCluster(t testing.T, base *CoreConfig, opts *TestClusterOptions) *Te
 	tlsConfigs := []*tls.Config{}
 	certGetters := []*reloadutil.CertificateGetter{}
 	for i := 0; i < numCores; i++ {
-
 		addr := &net.TCPAddr{
 			IP:   baseAddr.IP,
 			Port: 0,
@@ -1837,7 +1840,17 @@ func (testCluster *TestCluster) newCore(t testing.T, idx int, coreConfig *CoreCo
 		localConfig.Logger = testCluster.Logger.Named(fmt.Sprintf("core%d", idx))
 	}
 	if opts != nil && opts.PhysicalFactory != nil {
-		physBundle := opts.PhysicalFactory(t, idx, localConfig.Logger, opts.PhysicalFactoryConfig)
+		pfc := opts.PhysicalFactoryConfig
+		if pfc == nil {
+			pfc = make(map[string]interface{})
+		}
+		if len(opts.VersionMap) > 0 {
+			pfc["autopilot_upgrade_version"] = opts.VersionMap[idx]
+		}
+		if len(opts.RedundancyZoneMap) > 0 {
+			pfc["autopilot_redundancy_zone"] = opts.RedundancyZoneMap[idx]
+		}
+		physBundle := opts.PhysicalFactory(t, idx, localConfig.Logger, pfc)
 		switch {
 		case physBundle == nil && coreConfig.Physical != nil:
 		case physBundle == nil && coreConfig.Physical == nil: