Add autopilot automated upgrades and redundancy zones

api/client.go

@@ -608,7 +608,7 @@ func (c *Client) CloneConfig() *Config {
 	return newConfig
 }
 
-// Sets the address of Vault in the client. The format of address should be
+// SetAddress sets the address of Vault in the client. The format of address should be
 // "<Scheme>://<Host>:<Port>". Setting this on a client will override the
 // value of VAULT_ADDR environment variable.
 func (c *Client) SetAddress(addr string) error {
@@ -635,6 +635,16 @@ func (c *Client) Address() string {
 	return c.addr.String()
 }
 
+func (c *Client) SetCheckRedirect(f func(*http.Request, []*http.Request) error) {
+	c.modifyLock.Lock()
+	defer c.modifyLock.Unlock()
+
+	c.config.modifyLock.Lock()
+	defer c.config.modifyLock.Unlock()
+
+	c.config.HttpClient.CheckRedirect = f
+}
+
 // SetLimiter will set the rate limiter for this client.
 // This method is thread-safe.
 // rateLimit and burst are specified according to https://godoc.org/golang.org/x/time/rate#NewLimiter

api/sys_hastatus.go

@@ -37,4 +37,7 @@ type HANode struct {
 	ClusterAddress string     `json:"cluster_address"`
 	ActiveNode     bool       `json:"active_node"`
 	LastEcho       *time.Time `json:"last_echo"`
+	Version        string     `json:"version"`
+	UpgradeVersion string     `json:"upgrade_version,omitempty"`
+	RedundancyZone string     `json:"redundancy_zone,omitempty"`
 }

api/sys_raft.go

@@ -44,6 +44,7 @@ type AutopilotConfig struct {
 	MaxTrailingLogs                uint64        `json:"max_trailing_logs" mapstructure:"max_trailing_logs"`
 	MinQuorum                      uint          `json:"min_quorum" mapstructure:"min_quorum"`
 	ServerStabilizationTime        time.Duration `json:"server_stabilization_time" mapstructure:"-"`
+	DisableUpgradeMigration        bool          `json:"disable_upgrade_migration" mapstructure:"disable_upgrade_migration"`
 }
 
 // MarshalJSON makes the autopilot config fields JSON compatible
@@ -55,6 +56,7 @@ func (ac *AutopilotConfig) MarshalJSON() ([]byte, error) {
 		"max_trailing_logs":                  ac.MaxTrailingLogs,
 		"min_quorum":                         ac.MinQuorum,
 		"server_stabilization_time":          ac.ServerStabilizationTime.String(),
+		"disable_upgrade_migration":          ac.DisableUpgradeMigration,
 	})
 }
 
@@ -84,28 +86,59 @@ func (ac *AutopilotConfig) UnmarshalJSON(b []byte) error {
 
 // AutopilotState represents the response of the raft autopilot state API
 type AutopilotState struct {
-	Healthy          bool                        `mapstructure:"healthy"`
-	FailureTolerance int                         `mapstructure:"failure_tolerance"`
-	Servers          map[string]*AutopilotServer `mapstructure:"servers"`
-	Leader           string                      `mapstructure:"leader"`
-	Voters           []string                    `mapstructure:"voters"`
-	NonVoters        []string                    `mapstructure:"non_voters"`
+	Healthy                    bool                        `mapstructure:"healthy"`
+	FailureTolerance           int                         `mapstructure:"failure_tolerance"`
+	Servers                    map[string]*AutopilotServer `mapstructure:"servers" deep:"-"`
+	Leader                     string                      `mapstructure:"leader"`
+	Voters                     []string                    `mapstructure:"voters"`
+	NonVoters                  []string                    `mapstructure:"non_voters"`
+	RedundancyZones            map[string]AutopilotZone    `mapstructure:"redundancy_zones,omitempty"`
+	Upgrade                    *AutopilotUpgrade           `mapstructure:"upgrade_info,omitempty"`
+	OptimisticFailureTolerance int                         `mapstructure:"optimistic_failure_tolerance,omitempty"`
 }
 
 // AutopilotServer represents the server blocks in the response of the raft
 // autopilot state API.
 type AutopilotServer struct {
-	ID          string            `mapstructure:"id"`
-	Name        string            `mapstructure:"name"`
-	Address     string            `mapstructure:"address"`
-	NodeStatus  string            `mapstructure:"node_status"`
-	LastContact string            `mapstructure:"last_contact"`
-	LastTerm    uint64            `mapstructure:"last_term"`
-	LastIndex   uint64            `mapstructure:"last_index"`
-	Healthy     bool              `mapstructure:"healthy"`
-	StableSince string            `mapstructure:"stable_since"`
-	Status      string            `mapstructure:"status"`
-	Meta        map[string]string `mapstructure:"meta"`
+	ID             string `mapstructure:"id"`
+	Name           string `mapstructure:"name"`
+	Address        string `mapstructure:"address"`
+	NodeStatus     string `mapstructure:"node_status"`
+	LastContact    string `mapstructure:"last_contact"`
+	LastTerm       uint64 `mapstructure:"last_term"`
+	LastIndex      uint64 `mapstructure:"last_index"`
+	Healthy        bool   `mapstructure:"healthy"`
+	StableSince    string `mapstructure:"stable_since"`
+	Status         string `mapstructure:"status"`
+	Version        string `mapstructure:"version"`
+	UpgradeVersion string `mapstructure:"upgrade_version,omitempty"`
+	RedundancyZone string `mapstructure:"redundancy_zone,omitempty"`
+	NodeType       string `mapstructure:"node_type,omitempty"`
+}
+
+type AutopilotZone struct {
+	Servers          []string `mapstructure:"servers,omitempty"`
+	Voters           []string `mapstructure:"voters,omitempty"`
+	FailureTolerance int      `mapstructure:"failure_tolerance,omitempty"`
+}
+
+type AutopilotUpgrade struct {
+	Status                    string                                  `mapstructure:"status"`
+	TargetVersion             string                                  `mapstructure:"target_version,omitempty"`
+	TargetVersionVoters       []string                                `mapstructure:"target_version_voters,omitempty"`
+	TargetVersionNonVoters    []string                                `mapstructure:"target_version_non_voters,omitempty"`
+	TargetVersionReadReplicas []string                                `mapstructure:"target_version_read_replicas,omitempty"`
+	OtherVersionVoters        []string                                `mapstructure:"other_version_voters,omitempty"`
+	OtherVersionNonVoters     []string                                `mapstructure:"other_version_non_voters,omitempty"`
+	OtherVersionReadReplicas  []string                                `mapstructure:"other_version_read_replicas,omitempty"`
+	RedundancyZones           map[string]AutopilotZoneUpgradeVersions `mapstructure:"redundancy_zones,omitempty"`
+}
+
+type AutopilotZoneUpgradeVersions struct {
+	TargetVersionVoters    []string `mapstructure:"target_version_voters,omitempty"`
+	TargetVersionNonVoters []string `mapstructure:"target_version_non_voters,omitempty"`
+	OtherVersionVoters     []string `mapstructure:"other_version_voters,omitempty"`
+	OtherVersionNonVoters  []string `mapstructure:"other_version_non_voters,omitempty"`
 }
 
 // RaftJoin wraps RaftJoinWithContext using context.Background.

command/format.go

@@ -171,29 +171,24 @@ func formatServer(srv *api.AutopilotServer) string {
 	var buffer bytes.Buffer
 
 	buffer.WriteString(fmt.Sprintf("   %s\n", srv.ID))
-	buffer.WriteString(fmt.Sprintf("      Name:            %s\n", srv.Name))
-	buffer.WriteString(fmt.Sprintf("      Address:         %s\n", srv.Address))
-	buffer.WriteString(fmt.Sprintf("      Status:          %s\n", srv.Status))
-	buffer.WriteString(fmt.Sprintf("      Node Status:     %s\n", srv.NodeStatus))
-	buffer.WriteString(fmt.Sprintf("      Healthy:         %t\n", srv.Healthy))
-	buffer.WriteString(fmt.Sprintf("      Last Contact:    %s\n", srv.LastContact))
-	buffer.WriteString(fmt.Sprintf("      Last Term:       %d\n", srv.LastTerm))
-	buffer.WriteString(fmt.Sprintf("      Last Index:      %d\n", srv.LastIndex))
-
-	if len(srv.Meta) > 0 {
-		buffer.WriteString("      Meta\n")
-		var outputs []mapOutput
-		for k, v := range srv.Meta {
-			outputs = append(outputs, mapOutput{key: k, value: fmt.Sprintf("         %q: %q\n", k, v)})
-		}
-
-		sort.Slice(outputs, func(i, j int) bool {
-			return outputs[i].key < outputs[j].key
-		})
+	buffer.WriteString(fmt.Sprintf("      Name:              %s\n", srv.Name))
+	buffer.WriteString(fmt.Sprintf("      Address:           %s\n", srv.Address))
+	buffer.WriteString(fmt.Sprintf("      Status:            %s\n", srv.Status))
+	buffer.WriteString(fmt.Sprintf("      Node Status:       %s\n", srv.NodeStatus))
+	buffer.WriteString(fmt.Sprintf("      Healthy:           %t\n", srv.Healthy))
+	buffer.WriteString(fmt.Sprintf("      Last Contact:      %s\n", srv.LastContact))
+	buffer.WriteString(fmt.Sprintf("      Last Term:         %d\n", srv.LastTerm))
+	buffer.WriteString(fmt.Sprintf("      Last Index:        %d\n", srv.LastIndex))
+	buffer.WriteString(fmt.Sprintf("      Version:           %s\n", srv.Version))
 
-		for _, output := range outputs {
-			buffer.WriteString(output.value)
-		}
+	if srv.UpgradeVersion != "" {
+		buffer.WriteString(fmt.Sprintf("      Upgrade Version:   %s\n", srv.UpgradeVersion))
+	}
+	if srv.RedundancyZone != "" {
+		buffer.WriteString(fmt.Sprintf("      Redundancy Zone:   %s\n", srv.RedundancyZone))
+	}
+	if srv.NodeType != "" {
+		buffer.WriteString(fmt.Sprintf("      Node Type:         %s\n", srv.NodeType))
 	}
 
 	return buffer.String()
@@ -203,9 +198,9 @@ func (p PrettyFormatter) OutputAutopilotState(ui cli.Ui, data interface{}) {
 	state := data.(*api.AutopilotState)
 
 	var buffer bytes.Buffer
-	buffer.WriteString(fmt.Sprintf("Healthy:                      %t\n", state.Healthy))
-	buffer.WriteString(fmt.Sprintf("Failure Tolerance:            %d\n", state.FailureTolerance))
-	buffer.WriteString(fmt.Sprintf("Leader:                       %s\n", state.Leader))
+	buffer.WriteString(fmt.Sprintf("Healthy:                         %t\n", state.Healthy))
+	buffer.WriteString(fmt.Sprintf("Failure Tolerance:               %d\n", state.FailureTolerance))
+	buffer.WriteString(fmt.Sprintf("Leader:                          %s\n", state.Leader))
 	buffer.WriteString("Voters:\n")
 	outputStringSlice(&buffer, "   ", state.Voters)
 
@@ -214,20 +209,66 @@ func (p PrettyFormatter) OutputAutopilotState(ui cli.Ui, data interface{}) {
 		outputStringSlice(&buffer, "   ", state.NonVoters)
 	}
 
+	if state.OptimisticFailureTolerance > 0 {
+		buffer.WriteString(fmt.Sprintf("Optimistic Failure Tolerance:    %d\n", state.OptimisticFailureTolerance))
+	}
+
+	// Servers
 	buffer.WriteString("Servers:\n")
 	var outputs []mapOutput
 	for id, srv := range state.Servers {
 		outputs = append(outputs, mapOutput{key: id, value: formatServer(srv)})
 	}
-
 	sort.Slice(outputs, func(i, j int) bool {
 		return outputs[i].key < outputs[j].key
 	})
-
 	for _, output := range outputs {
 		buffer.WriteString(output.value)
 	}
 
+	// Redundancy Zones
+	if len(state.RedundancyZones) > 0 {
+		buffer.WriteString("Redundancy Zones:\n")
+		zoneList := make([]string, 0, len(state.RedundancyZones))
+		for z := range state.RedundancyZones {
+			zoneList = append(zoneList, z)
+		}
+		sort.Strings(zoneList)
+		for _, zoneName := range zoneList {
+			zone := state.RedundancyZones[zoneName]
+			servers := zone.Servers
+			voters := zone.Voters
+			sort.Strings(servers)
+			sort.Strings(voters)
+			buffer.WriteString(fmt.Sprintf("   %s\n", zoneName))
+			buffer.WriteString(fmt.Sprintf("      Servers: %s\n", strings.Join(servers, ", ")))
+			buffer.WriteString(fmt.Sprintf("      Voters: %s\n", strings.Join(voters, ", ")))
+			buffer.WriteString(fmt.Sprintf("      Failure Tolerance: %d\n", zone.FailureTolerance))
+		}
+	}
+
+	// Upgrade Info
+	if state.Upgrade != nil {
+		buffer.WriteString("Upgrade Info:\n")
+		buffer.WriteString(fmt.Sprintf("   Status: %s\n", state.Upgrade.Status))
+		buffer.WriteString(fmt.Sprintf("   Target Version: %s\n", state.Upgrade.TargetVersion))
+		buffer.WriteString(fmt.Sprintf("   Target Version Voters: %s\n", strings.Join(state.Upgrade.TargetVersionVoters, ", ")))
+		buffer.WriteString(fmt.Sprintf("   Target Version Non-Voters: %s\n", strings.Join(state.Upgrade.TargetVersionNonVoters, ", ")))
+		buffer.WriteString(fmt.Sprintf("   Other Version Voters: %s\n", strings.Join(state.Upgrade.OtherVersionVoters, ", ")))
+		buffer.WriteString(fmt.Sprintf("   Other Version Non-Voters: %s\n", strings.Join(state.Upgrade.OtherVersionNonVoters, ", ")))
+
+		if len(state.Upgrade.RedundancyZones) > 0 {
+			buffer.WriteString("   Redundancy Zones:\n")
+			for zoneName, zoneVersion := range state.Upgrade.RedundancyZones {
+				buffer.WriteString(fmt.Sprintf("      %s\n", zoneName))
+				buffer.WriteString(fmt.Sprintf("         Target Version Voters: %s\n", strings.Join(zoneVersion.TargetVersionVoters, ", ")))
+				buffer.WriteString(fmt.Sprintf("         Target Version Non-Voters: %s\n", strings.Join(zoneVersion.TargetVersionNonVoters, ", ")))
+				buffer.WriteString(fmt.Sprintf("         Other Version Voters: %s\n", strings.Join(zoneVersion.OtherVersionVoters, ", ")))
+				buffer.WriteString(fmt.Sprintf("         Other Version Non-Voters: %s\n", strings.Join(zoneVersion.OtherVersionNonVoters, ", ")))
+			}
+		}
+	}
+
 	ui.Output(buffer.String())
 }
 

command/operator_members.go

@@ -3,6 +3,7 @@ package command
 import (
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/mitchellh/cli"
 	"github.com/posener/complete"
@@ -70,9 +71,17 @@ func (c *OperatorMembersCommand) Run(args []string) int {
 
 	switch Format(c.UI) {
 	case "table":
-		out := []string{"Host Name | API Address | Cluster Address | ActiveNode | Last Echo"}
+		out := make([]string, 0)
+		cols := []string{"Host Name", "API Address", "Cluster Address", "Active Node", "Version", "Upgrade Version", "Redundancy Zone", "Last Echo"}
+		out = append(out, strings.Join(cols, " | "))
 		for _, node := range resp.Nodes {
-			out = append(out, fmt.Sprintf("%s | %s | %s | %t | %s", node.Hostname, node.APIAddress, node.ClusterAddress, node.ActiveNode, node.LastEcho))
+			cols := []string{node.Hostname, node.APIAddress, node.ClusterAddress, fmt.Sprintf("%t", node.ActiveNode), node.Version, node.UpgradeVersion, node.RedundancyZone}
+			if node.LastEcho != nil {
+				cols = append(cols, node.LastEcho.Format(time.RFC3339))
+			} else {
+				cols = append(cols, "")
+			}
+			out = append(out, strings.Join(cols, " | "))
 		}
 		c.UI.Output(tableOutput(out, nil))
 		return 0

command/operator_raft_autopilot_get_config.go

@@ -88,6 +88,7 @@ func (c *OperatorRaftAutopilotGetConfigCommand) Run(args []string) int {
 	entries = append(entries, fmt.Sprintf("%s | %s", "Server Stabilization Time", config.ServerStabilizationTime.String()))
 	entries = append(entries, fmt.Sprintf("%s | %d", "Min Quorum", config.MinQuorum))
 	entries = append(entries, fmt.Sprintf("%s | %d", "Max Trailing Logs", config.MaxTrailingLogs))
+	entries = append(entries, fmt.Sprintf("%s | %t", "Disable Upgrade Migration", config.DisableUpgradeMigration))
 
 	return OutputData(c.UI, entries)
 }

command/operator_raft_autopilot_set_config.go

@@ -22,6 +22,7 @@ type OperatorRaftAutopilotSetConfigCommand struct {
 	flagMaxTrailingLogs                uint64
 	flagMinQuorum                      uint
 	flagServerStabilizationTime        time.Duration
+	flagDisableUpgradeMigration        BoolPtr
 }
 
 func (c *OperatorRaftAutopilotSetConfigCommand) Synopsis() string {
@@ -73,6 +74,11 @@ func (c *OperatorRaftAutopilotSetConfigCommand) Flags() *FlagSets {
 		Target: &c.flagServerStabilizationTime,
 	})
 
+	f.BoolPtrVar(&BoolPtrVar{
+		Name:   "disable-upgrade-migration",
+		Target: &c.flagDisableUpgradeMigration,
+	})
+
 	return set
 }
 
@@ -125,6 +131,9 @@ func (c *OperatorRaftAutopilotSetConfigCommand) Run(args []string) int {
 	if c.flagServerStabilizationTime > 0 {
 		data["server_stabilization_time"] = c.flagServerStabilizationTime.String()
 	}
+	if c.flagDisableUpgradeMigration.IsSet() {
+		data["disable_upgrade_migration"] = c.flagDisableUpgradeMigration.Get()
+	}
 
 	secret, err := client.Logical().Write("sys/storage/raft/autopilot/configuration", data)
 	if err != nil {

physical/raft/fsm.go

@@ -14,7 +14,7 @@ import (
 	"sync/atomic"
 	"time"
 
-	metrics "github.com/armon/go-metrics"
+	"github.com/armon/go-metrics"
 	"github.com/golang/protobuf/proto"
 	log "github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/go-multierror"