Merge PR #10095: Integrated Storage Cloud Auto-Join

.circleci/config/jobs/test-go.yml

@@ -1,4 +1,4 @@
-executor: docker-env-large
+executor: docker-env-xlarge
 parallelism: 8
 steps:
   - check-branch-name

.gitignore

@@ -57,7 +57,8 @@ Vagrantfile
 
 dist/*
 
-tags
+# ignore ctags
+./tags
 
 # Editor backups
 *~
@@ -105,3 +106,6 @@ website/.cache
 website/assets/node_modules
 website/assets/public
 website/components/node_modules
+
+.buildcache/
+.releaser/

api/sys_raft.go

@@ -16,6 +16,7 @@ type RaftJoinResponse struct {
 
 // RaftJoinRequest represents the parameters consumed by the raft join API
 type RaftJoinRequest struct {
+	AutoJoin         string `json:"auto_join"`
 	LeaderAPIAddr    string `json:"leader_api_addr"`
 	LeaderCACert     string `json:"leader_ca_cert"`
 	LeaderClientCert string `json:"leader_client_cert"`

command/operator_raft_join.go

@@ -27,13 +27,18 @@ func (c *OperatorRaftJoinCommand) Synopsis() string {
 
 func (c *OperatorRaftJoinCommand) Help() string {
 	helpText := `
-Usage: vault operator raft join [options] <leader-api-addr>
+Usage: vault operator raft join [options] <leader-api-addr|auto-join-configuration>
 
   Join the current node as a peer to the Raft cluster by providing the address
   of the Raft leader node.
 
       $ vault operator raft join "http://127.0.0.2:8200"
 
+  Join the current node as a peer to the Raft cluster by providing cloud auto-join
+  configuration.
+
+      $ vault operator raft join "provider=aws region=eu-west-1 ..."
+			
   TLS certificate data can also be consumed from a file on disk by prefixing with
   the "@" symbol. For example:
 
@@ -106,14 +111,14 @@ func (c *OperatorRaftJoinCommand) Run(args []string) int {
 		return 1
 	}
 
-	leaderAPIAddr := ""
+	leaderInfo := ""
 
 	args = f.Args()
 	switch len(args) {
 	case 0:
 		// No-op: This is acceptable if we're using raft for HA-only
 	case 1:
-		leaderAPIAddr = strings.TrimSpace(args[0])
+		leaderInfo = strings.TrimSpace(args[0])
 	default:
 		c.UI.Error(fmt.Sprintf("Too many arguments (expected 0-1, got %d)", len(args)))
 		return 1
@@ -143,14 +148,21 @@ func (c *OperatorRaftJoinCommand) Run(args []string) int {
 		return 2
 	}
 
-	resp, err := client.Sys().RaftJoin(&api.RaftJoinRequest{
-		LeaderAPIAddr:    leaderAPIAddr,
+	joinReq := &api.RaftJoinRequest{
 		LeaderCACert:     leaderCACert,
 		LeaderClientCert: leaderClientCert,
 		LeaderClientKey:  leaderClientKey,
 		Retry:            c.flagRetry,
 		NonVoter:         c.flagNonVoter,
-	})
+	}
+
+	if strings.Contains(leaderInfo, "provider=") {
+		joinReq.AutoJoin = leaderInfo
+	} else {
+		joinReq.LeaderAPIAddr = leaderInfo
+	}
+
+	resp, err := client.Sys().RaftJoin(joinReq)
 	if err != nil {
 		c.UI.Error(fmt.Sprintf("Error joining the node to the Raft cluster: %s", err))
 		return 2

go.mod

@@ -10,7 +10,7 @@ require (
 	cloud.google.com/go/spanner v1.5.1
 	cloud.google.com/go/storage v1.6.0
 	github.com/Azure/azure-storage-blob-go v0.10.0
-	github.com/Azure/go-autorest/autorest v0.10.1
+	github.com/Azure/go-autorest/autorest v0.11.0
 	github.com/DataDog/zstd v1.4.5 // indirect
 	github.com/NYTimes/gziphandler v1.1.1
 	github.com/SAP/go-hdb v0.14.1
@@ -48,12 +48,12 @@ require (
 	github.com/gocql/gocql v0.0.0-20200624222514-34081eda590e
 	github.com/golang/protobuf v1.4.2
 	github.com/google/go-github v17.0.0+incompatible
-	github.com/google/go-metrics-stackdriver v0.2.0 // indirect
 	github.com/hashicorp/consul-template v0.25.1
 	github.com/hashicorp/consul/api v1.4.0
 	github.com/hashicorp/errwrap v1.0.0
 	github.com/hashicorp/go-bindata v3.0.8-0.20180209072458-bf7910af8997+incompatible
 	github.com/hashicorp/go-cleanhttp v0.5.1
+	github.com/hashicorp/go-discover v0.0.0-20200812215701-c4b85f6ed31f
 	github.com/hashicorp/go-gcp-common v0.6.0
 	github.com/hashicorp/go-hclog v0.14.1
 	github.com/hashicorp/go-kms-wrapping v0.5.16

http/sys_raft.go

@@ -67,6 +67,7 @@ func handleSysRaftJoinPost(core *vault.Core, w http.ResponseWriter, r *http.Requ
 
 	leaderInfos := []*raft.LeaderJoinInfo{
 		{
+			AutoJoin:      req.AutoJoin,
 			LeaderAPIAddr: req.LeaderAPIAddr,
 			TLSConfig:     tlsConfig,
 			Retry:         req.Retry,
@@ -90,6 +91,7 @@ type JoinResponse struct {
 }
 
 type JoinRequest struct {
+	AutoJoin         string `json:"auto_join"`
 	LeaderAPIAddr    string `json:"leader_api_addr"`
 	LeaderCACert     string `json:"leader_ca_cert"`
 	LeaderClientCert string `json:"leader_client_cert"`

physical/raft/raft.go

@@ -125,6 +125,11 @@ type RaftBackend struct {
 // LeaderJoinInfo contains information required by a node to join itself as a
 // follower to an existing raft cluster
 type LeaderJoinInfo struct {
+	// AutoJoin defines any cloud auto-join metadata. If supplied, Vault will
+	// attempt to automatically discover peers in addition to what can be provided
+	// via 'leader_api_addr'.
+	AutoJoin string `json:"auto_join"`
+
 	// LeaderAPIAddr is the address of the leader node to connect to
 	LeaderAPIAddr string `json:"leader_api_addr"`
 
@@ -178,11 +183,15 @@ func (b *RaftBackend) JoinConfig() ([]*LeaderJoinInfo, error) {
 		return nil, errors.New("invalid retry_join config")
 	}
 
-	for _, info := range leaderInfos {
+	for i, info := range leaderInfos {
+		if len(info.AutoJoin) != 0 && len(info.LeaderAPIAddr) != 0 {
+			return nil, errors.New("cannot provide both a leader_api_addr and auto_join")
+		}
+
 		info.Retry = true
 		info.TLSConfig, err = parseTLSInfo(info)
 		if err != nil {
-			return nil, errwrap.Wrapf(fmt.Sprintf("failed to create tls config to communicate with leader node %q: {{err}}", info.LeaderAPIAddr), err)
+			return nil, errwrap.Wrapf(fmt.Sprintf("failed to create tls config to communicate with leader node (retry_join index: %d): {{err}}", i), err)
 		}
 	}
 

vault/external_tests/raft/raft_test.go

@@ -23,6 +23,7 @@ import (
 	"github.com/hashicorp/vault/physical/raft"
 	"github.com/hashicorp/vault/sdk/logical"
 	"github.com/hashicorp/vault/vault"
+	"github.com/stretchr/testify/require"
 	"golang.org/x/net/http2"
 )
 
@@ -40,6 +41,56 @@ func raftCluster(t testing.TB) *vault.TestCluster {
 	return cluster
 }
 
+func TestRaft_RetryAutoJoin(t *testing.T) {
+	t.Parallel()
+
+	var (
+		conf vault.CoreConfig
+
+		opts = vault.TestClusterOptions{HandlerFunc: vaulthttp.Handler}
+	)
+
+	teststorage.RaftBackendSetup(&conf, &opts)
+
+	opts.SetupFunc = nil
+	cluster := vault.NewTestCluster(t, &conf, &opts)
+
+	cluster.Start()
+	defer cluster.Cleanup()
+
+	addressProvider := &testhelpers.TestRaftServerAddressProvider{Cluster: cluster}
+	leaderCore := cluster.Cores[0]
+	atomic.StoreUint32(&vault.TestingUpdateClusterAddr, 1)
+
+	{
+		testhelpers.EnsureCoreSealed(t, leaderCore)
+		leaderCore.UnderlyingRawStorage.(*raft.RaftBackend).SetServerAddressProvider(addressProvider)
+		cluster.UnsealCore(t, leaderCore)
+		vault.TestWaitActive(t, leaderCore.Core)
+	}
+
+	leaderInfos := []*raft.LeaderJoinInfo{
+		{
+			AutoJoin:  "provider=aws region=eu-west-1 tag_key=consul tag_value=tag access_key_id=a secret_access_key=a",
+			TLSConfig: leaderCore.TLSConfig,
+			Retry:     true,
+		},
+	}
+
+	{
+		// expected to pass but not join as we're not actually discovering leader addresses
+		core := cluster.Cores[1]
+		core.UnderlyingRawStorage.(*raft.RaftBackend).SetServerAddressProvider(addressProvider)
+
+		_, err := core.JoinRaftCluster(namespace.RootContext(context.Background()), leaderInfos, false)
+		require.NoError(t, err)
+	}
+
+	testhelpers.VerifyRaftPeers(t, cluster.Cores[0].Client, map[string]bool{
+		"core-0": true,
+	})
+}
+
 func TestRaft_Retry_Join(t *testing.T) {
 	t.Parallel()
 	var conf vault.CoreConfig

vault/raft.go

@@ -16,6 +16,7 @@ import (
 	"github.com/golang/protobuf/proto"
 	"github.com/hashicorp/errwrap"
 	cleanhttp "github.com/hashicorp/go-cleanhttp"
+	"github.com/hashicorp/go-discover"
 	"github.com/hashicorp/go-hclog"
 	wrapping "github.com/hashicorp/go-kms-wrapping"
 	uuid "github.com/hashicorp/go-uuid"
@@ -739,14 +740,14 @@ func (c *Core) JoinRaftCluster(ctx context.Context, leaderInfos []*raft.LeaderJo
 		return false, errors.New("node must be unsealed before joining")
 	}
 
-	// Disallow leader API address to be provided if we're using raft for HA-only
+	// Disallow leader API address to be provided if we're using raft for HA-only.
 	// The leader API address is obtained directly through storage. This serves
 	// as a form of verification that this node is sharing the same physical
 	// storage as the leader node.
 	if isRaftHAOnly {
 		for _, info := range leaderInfos {
-			if info.LeaderAPIAddr != "" {
-				return false, errors.New("leader API address must be unset when raft is used exclusively for HA")
+			if info.LeaderAPIAddr != "" || info.AutoJoin != "" {
+				return false, errors.New("leader API address and auto-join metadata must be unset when raft is used exclusively for HA")
 			}
 		}
 
@@ -778,12 +779,17 @@ func (c *Core) JoinRaftCluster(ctx context.Context, leaderInfos []*raft.LeaderJo
 		leaderInfos[0].LeaderAPIAddr = adv.RedirectAddr
 	}
 
+	disco, err := newDiscover()
+	if err != nil {
+		return false, errwrap.Wrapf("failed to create auto-join discovery: {{err}}", err)
+	}
+
 	join := func(retry bool) error {
-		joinLeader := func(leaderInfo *raft.LeaderJoinInfo) error {
+		joinLeader := func(leaderInfo *raft.LeaderJoinInfo, leaderAddr string) error {
 			if leaderInfo == nil {
 				return errors.New("raft leader information is nil")
 			}
-			if len(leaderInfo.LeaderAPIAddr) == 0 {
+			if len(leaderAddr) == 0 {
 				return errors.New("raft leader address not provided")
 			}
 
@@ -797,7 +803,7 @@ func (c *Core) JoinRaftCluster(ctx context.Context, leaderInfos []*raft.LeaderJo
 				return nil
 			}
 
-			c.logger.Info("attempting to join possible raft leader node", "leader_addr", leaderInfo.LeaderAPIAddr)
+			c.logger.Info("attempting to join possible raft leader node", "leader_addr", leaderAddr)
 
 			// Create an API client to interact with the leader node
 			transport := cleanhttp.DefaultPooledTransport()
@@ -819,13 +825,16 @@ func (c *Core) JoinRaftCluster(ctx context.Context, leaderInfos []*raft.LeaderJo
 			client := &http.Client{
 				Transport: transport,
 			}
+
 			config := api.DefaultConfig()
 			if config.Error != nil {
 				return errwrap.Wrapf("failed to create api client: {{err}}", config.Error)
 			}
-			config.Address = leaderInfo.LeaderAPIAddr
+
+			config.Address = leaderAddr
 			config.HttpClient = client
 			config.MaxRetries = 0
+
 			apiClient, err := api.NewClient(config)
 			if err != nil {
 				return errwrap.Wrapf("failed to create api client: {{err}}", err)
@@ -865,6 +874,7 @@ func (c *Core) JoinRaftCluster(ctx context.Context, leaderInfos []*raft.LeaderJo
 			if err := proto.Unmarshal(challengeRaw, eBlob); err != nil {
 				return errwrap.Wrapf("error decoding raft bootstrap challenge: {{err}}", err)
 			}
+
 			raftInfo := &raftInformation{
 				challenge:           eBlob,
 				leaderClient:        apiClient,
@@ -911,11 +921,36 @@ func (c *Core) JoinRaftCluster(ctx context.Context, leaderInfos []*raft.LeaderJo
 		// Each join try goes through all the possible leader nodes and attempts to join
 		// them, until one of the attempt succeeds.
 		for _, leaderInfo := range leaderInfos {
-			err = joinLeader(leaderInfo)
-			if err == nil {
-				return nil
+			switch {
+			case leaderInfo.LeaderAPIAddr != "" && leaderInfo.AutoJoin != "":
+				c.logger.Info("join attempt failed", "error", errors.New("cannot provide both leader address and auto-join metadata"))
+
+			case leaderInfo.LeaderAPIAddr != "":
+				if err := joinLeader(leaderInfo, leaderInfo.LeaderAPIAddr); err != nil {
+					c.logger.Info("join attempt failed", "error", err)
+				} else {
+					// successfully joined leader
+					return nil
+				}
+
+			case leaderInfo.AutoJoin != "":
+				addrs, err := disco.Addrs(leaderInfo.AutoJoin, c.logger.StandardLogger(nil))
+				if err != nil {
+					c.logger.Info("failed to parse addresses from auto-join metadata", "error", err)
+				}
+
+				for _, addr := range addrs {
+					if err := joinLeader(leaderInfo, addr); err != nil {
+						c.logger.Info("join attempt failed", "error", err)
+					} else {
+						// successfully joined leader
+						return nil
+					}
+				}
+
+			default:
+				c.logger.Info("join attempt failed", "error", errors.New("must provide leader address or auto-join metadata"))
 			}
-			c.logger.Info("join attempt failed", "error", err)
 		}
 
 		return errors.New("failed to join any raft leader node")
@@ -1116,3 +1151,14 @@ type answerResp struct {
 	Peers      []raft.Peer      `json:"peers"`
 	TLSKeyring *raft.TLSKeyring `json:"tls_keyring"`
 }
+
+func newDiscover() (*discover.Discover, error) {
+	providers := make(map[string]discover.Provider)
+	for k, v := range discover.Providers {
+		providers[k] = v
+	}
+
+	return discover.New(
+		discover.WithProviders(providers),
+	)
+}