vault: Initial wan fed support (tls and gossip only)

acceptance/framework/consul/cli_cluster.go

@@ -74,8 +74,8 @@ func NewCLICluster(
 	require.NoError(t, err)
 
 	// Merge all helm values
-	MergeMaps(values, valuesFromConfig)
-	MergeMaps(values, helmValues)
+	helpers.MergeMaps(values, valuesFromConfig)
+	helpers.MergeMaps(values, helmValues)
 
 	logger := terratestLogger.New(logger.TestLogger{})
 
@@ -179,7 +179,7 @@ func (h *CLICluster) Destroy(t *testing.T) {
 func (h *CLICluster) Upgrade(t *testing.T, helmValues map[string]string) {
 	t.Helper()
 
-	MergeMaps(h.helmOptions.SetValues, helmValues)
+	helpers.MergeMaps(h.helmOptions.SetValues, helmValues)
 	helm.Upgrade(t, h.helmOptions, config.HelmChartPath, h.releaseName)
 	helpers.WaitForAllPodsToBeReady(t, h.kubernetesClient, h.helmOptions.KubectlOptions.Namespace, fmt.Sprintf("release=%s", h.releaseName))
 }

acceptance/framework/consul/consul_cluster.go

@@ -75,8 +75,8 @@ func NewHelmCluster(
 	require.NoError(t, err)
 
 	// Merge all helm values
-	MergeMaps(values, valuesFromConfig)
-	MergeMaps(values, helmValues)
+	helpers.MergeMaps(values, valuesFromConfig)
+	helpers.MergeMaps(values, helmValues)
 
 	logger := terratestLogger.New(logger.TestLogger{})
 
@@ -213,7 +213,7 @@ func (h *HelmCluster) Destroy(t *testing.T) {
 func (h *HelmCluster) Upgrade(t *testing.T, helmValues map[string]string) {
 	t.Helper()
 
-	MergeMaps(h.helmOptions.SetValues, helmValues)
+	helpers.MergeMaps(h.helmOptions.SetValues, helmValues)
 	helm.Upgrade(t, h.helmOptions, config.HelmChartPath, h.releaseName)
 	helpers.WaitForAllPodsToBeReady(t, h.kubernetesClient, h.helmOptions.KubectlOptions.Namespace, fmt.Sprintf("release=%s", h.releaseName))
 }
@@ -472,11 +472,3 @@ func defaultValues() map[string]string {
 	}
 	return values
 }
-
-// MergeMaps will merge the values in b with values in a and save in a.
-// If there are conflicts, the values in b will overwrite the values in a.
-func MergeMaps(a, b map[string]string) {
-	for k, v := range b {
-		a[k] = v
-	}
-}

acceptance/framework/helpers/helpers.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/gruntwork-io/terratest/modules/helm"
+	"github.com/hashicorp/consul/api"
 
 	terratestk8s "github.com/gruntwork-io/terratest/modules/k8s"
 	"github.com/gruntwork-io/terratest/modules/random"
@@ -204,3 +205,46 @@ func IsReady(pod corev1.Pod) bool {
 
 	return false
 }
+
+// VerifyFederation checks that the WAN federation between servers is successful
+// by first checking members are alive from the perspective of both servers.
+// If secure is true, it will also check that the ACL replication is running on the secondary server.
+func VerifyFederation(t *testing.T, primaryClient, secondaryClient *api.Client, releaseName string, secure bool) {
+	retrier := &retry.Timer{Timeout: 5 * time.Minute, Wait: 1 * time.Second}
+	start := time.Now()
+
+	// Check that server in dc1 is healthy from the perspective of the server in dc2, and vice versa.
+	// We're calling the Consul health API, as opposed to checking serf membership status,
+	// because we need to make sure that the federated servers can make API calls and forward requests
+	// from one server to another. From running tests in CI for a while and using serf membership status before,
+	// we've noticed that the status could be "alive" as soon as the server in the secondary cluster joins the primary
+	// and then switch to "failed". This would require us to check that the status is "alive" is showing consistently for
+	// some amount of time, which could be quite flakey. Calling the API in another datacenter allows us to check that
+	// each server can forward calls to another, which is what we need for connect.
+	retry.RunWith(retrier, t, func(r *retry.R) {
+		secondaryServerHealth, _, err := primaryClient.Health().Node(fmt.Sprintf("%s-consul-server-0", releaseName), &api.QueryOptions{Datacenter: "dc2"})
+		require.NoError(r, err)
+		require.Equal(r, secondaryServerHealth.AggregatedStatus(), api.HealthPassing)
+
+		primaryServerHealth, _, err := secondaryClient.Health().Node(fmt.Sprintf("%s-consul-server-0", releaseName), &api.QueryOptions{Datacenter: "dc1"})
+		require.NoError(r, err)
+		require.Equal(r, primaryServerHealth.AggregatedStatus(), api.HealthPassing)
+
+		if secure {
+			replicationStatus, _, err := secondaryClient.ACL().Replication(nil)
+			require.NoError(r, err)
+			require.True(r, replicationStatus.Enabled)
+			require.True(r, replicationStatus.Running)
+		}
+	})
+
+	logger.Logf(t, "Took %s to verify federation", time.Since(start))
+}
+
+// MergeMaps will merge the values in b with values in a and save in a.
+// If there are conflicts, the values in b will overwrite the values in a.
+func MergeMaps(a, b map[string]string) {
+	for k, v := range b {
+		a[k] = v
+	}
+}

acceptance/framework/vault/vault_cluster.go

@@ -45,20 +45,24 @@ type VaultCluster struct {
 }
 
 // NewVaultCluster creates a VaultCluster which will be used to install Vault using Helm.
-func NewVaultCluster(t *testing.T, ctx environment.TestContext, cfg *config.TestConfig, releaseName string) *VaultCluster {
+func NewVaultCluster(t *testing.T, ctx environment.TestContext, cfg *config.TestConfig, releaseName string, helmValues map[string]string) *VaultCluster {
 
 	logger := terratestLogger.New(logger.TestLogger{})
 
 	kopts := ctx.KubectlOptions(t)
 
+	values := defaultHelmValues(releaseName)
+	if cfg.EnablePodSecurityPolicies {
+		values["global.psp.enable"] = "true"
+	}
+
+	helpers.MergeMaps(values, helmValues)
 	vaultHelmOpts := &helm.Options{
-		SetValues:      defaultHelmValues(releaseName),
+		SetValues:      values,
 		KubectlOptions: kopts,
 		Logger:         logger,
 	}
-	if cfg.EnablePodSecurityPolicies {
-		vaultHelmOpts.SetValues["global.psp.enable"] = "true"
-	}
+
 	helm.AddRepo(t, vaultHelmOpts, "hashicorp", "https://helm.releases.hashicorp.com")
 	// Ignoring the error from `helm repo update` as it could fail due to stale cache or unreachable servers and we're
 	// asserting a chart version on Install which would fail in an obvious way should this not succeed.
@@ -80,7 +84,7 @@ func NewVaultCluster(t *testing.T, ctx environment.TestContext, cfg *config.Test
 }
 
 // VaultClient returns the vault client.
-func (v *VaultCluster) VaultClient(t *testing.T) *vapi.Client { return v.vaultClient }
+func (v *VaultCluster) VaultClient(*testing.T) *vapi.Client { return v.vaultClient }
 
 // SetupVaultClient sets up and returns a Vault Client.
 func (v *VaultCluster) SetupVaultClient(t *testing.T) *vapi.Client {
@@ -125,8 +129,11 @@ func (v *VaultCluster) SetupVaultClient(t *testing.T) *vapi.Client {
 }
 
 // bootstrap sets up Kubernetes auth method and enables secrets engines.
-func (v *VaultCluster) bootstrap(t *testing.T, ctx environment.TestContext) {
-
+func (v *VaultCluster) bootstrap(t *testing.T) {
+	if !v.serverEnabled() {
+		v.logger.Logf(t, "skipping bootstrapping Vault because Vault server is not enabled")
+		return
+	}
 	v.vaultClient = v.SetupVaultClient(t)
 
 	// Enable the KV-V2 Secrets engine.
@@ -160,13 +167,41 @@ func (v *VaultCluster) bootstrap(t *testing.T, ctx environment.TestContext) {
 	tokenSecret, err := v.kubernetesClient.CoreV1().Secrets(namespace).Get(context.Background(), sa.Secrets[0].Name, metav1.GetOptions{})
 	require.NoError(t, err)
 	_, err = v.vaultClient.Logical().Write("auth/kubernetes/config", map[string]interface{}{
-		"token_reviewer_jwt": tokenSecret.StringData["token"],
-		"kubernetes_ca_cert": tokenSecret.StringData["ca.crt"],
+		"token_reviewer_jwt": string(tokenSecret.Data["token"]),
+		"kubernetes_ca_cert": string(tokenSecret.Data["ca.crt"]),
 		"kubernetes_host":    "https://kubernetes.default.svc",
 	})
 	require.NoError(t, err)
 }
 
+// ConfigureAuthMethod configures the auth method in Vault from the provided service account name and namespace,
+// kubernetes host and auth path.
+func (v *VaultCluster) ConfigureAuthMethod(t *testing.T, vaultClient *vapi.Client, authPath, k8sHost, saName, saNS string) {
+	v.logger.Logf(t, "enabling kubernetes auth method on %s path", authPath)
+	err := vaultClient.Sys().EnableAuthWithOptions(authPath, &vapi.EnableAuthOptions{
+		Type: "kubernetes",
+	})
+	require.NoError(t, err)
+
+	// To configure the auth method, we need to read the token and the ca cert from the auth method's
+	// service account token.
+	var sa *corev1.ServiceAccount
+	retry.Run(t, func(r *retry.R) {
+		sa, err = v.kubernetesClient.CoreV1().ServiceAccounts(saNS).Get(context.Background(), saName, metav1.GetOptions{})
+		require.NoError(t, err)
+		require.Len(t, sa.Secrets, 1)
+	})
+
+	tokenSecret, err := v.kubernetesClient.CoreV1().Secrets(saNS).Get(context.Background(), sa.Secrets[0].Name, metav1.GetOptions{})
+	require.NoError(t, err)
+	_, err = vaultClient.Logical().Write(fmt.Sprintf("auth/%s/config", authPath), map[string]interface{}{
+		"token_reviewer_jwt": string(tokenSecret.Data["token"]),
+		"kubernetes_ca_cert": string(tokenSecret.Data["ca.crt"]),
+		"kubernetes_host":    k8sHost,
+	})
+	require.NoError(t, err)
+}
+
 // Create installs Vault via Helm and then calls bootstrap to initialize it.
 func (v *VaultCluster) Create(t *testing.T, ctx environment.TestContext) {
 	t.Helper()
@@ -191,7 +226,7 @@ func (v *VaultCluster) Create(t *testing.T, ctx environment.TestContext) {
 	helpers.WaitForAllPodsToBeReady(t, v.kubernetesClient, v.helmOptions.KubectlOptions.Namespace, v.releaseLabelSelector())
 
 	// Now call bootstrap().
-	v.bootstrap(t, ctx)
+	v.bootstrap(t)
 }
 
 // Destroy issues a helm delete and deletes the PVC + any helm secrets related to the release that are leftover.
@@ -261,10 +296,20 @@ func (v *VaultCluster) releaseLabelSelector() string {
 	return fmt.Sprintf("%s=%s", releaseLabel, v.releaseName)
 }
 
+func (v *VaultCluster) serverEnabled() bool {
+	serverEnabled, ok := v.helmOptions.SetValues["server.enabled"]
+	return !ok || serverEnabled != "false"
+}
+
 // createTLSCerts generates a self-signed CA and uses it to generate
 // certificate and key  for the Vault server. It then saves those as
 // Kubernetes secrets.
 func (v *VaultCluster) createTLSCerts(t *testing.T) {
+	if !v.serverEnabled() {
+		v.logger.Logf(t, "skipping generating Vault TLS certificates because Vault server is not enabled")
+		return
+	}
+
 	v.logger.Logf(t, "generating Vault TLS certificates")
 
 	namespace := v.helmOptions.KubectlOptions.Namespace
@@ -318,8 +363,12 @@ func (v *VaultCluster) createTLSCerts(t *testing.T) {
 // initAndUnseal initializes and unseals Vault.
 // Once initialized, it saves the Vault root token into a Kubernetes secret.
 func (v *VaultCluster) initAndUnseal(t *testing.T) {
-	v.logger.Logf(t, "initializing and unsealing Vault")
+	if !v.serverEnabled() {
+		v.logger.Logf(t, "skipping initializing and unsealing Vault because Vault server is not enabled")
+		return
+	}
 
+	v.logger.Logf(t, "initializing and unsealing Vault")
 	namespace := v.helmOptions.KubectlOptions.Namespace
 	retrier := &retry.Timer{Timeout: 2 * time.Minute, Wait: 1 * time.Second}
 	retry.RunWith(retrier, t, func(r *retry.R) {

acceptance/tests/mesh-gateway/mesh_gateway_test.go

@@ -4,15 +4,13 @@ import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
 	"github.com/hashicorp/consul-k8s/acceptance/framework/consul"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/environment"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/helpers"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/k8s"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
 	"github.com/hashicorp/consul/api"
-	"github.com/hashicorp/consul/sdk/testutil/retry"
 	"github.com/stretchr/testify/require"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -108,7 +106,7 @@ func TestMeshGatewayDefault(t *testing.T) {
 
 	// Verify federation between servers
 	logger.Log(t, "verifying federation was successful")
-	verifyFederation(t, primaryClient, secondaryClient, releaseName, false)
+	helpers.VerifyFederation(t, primaryClient, secondaryClient, releaseName, false)
 
 	// Create a ProxyDefaults resource to configure services to use the mesh
 	// gateways.
@@ -244,7 +242,7 @@ func TestMeshGatewaySecure(t *testing.T) {
 
 			// Verify federation between servers
 			logger.Log(t, "verifying federation was successful")
-			verifyFederation(t, primaryClient, secondaryClient, releaseName, true)
+			helpers.VerifyFederation(t, primaryClient, secondaryClient, releaseName, true)
 
 			// Create a ProxyDefaults resource to configure services to use the mesh
 			// gateways.
@@ -280,38 +278,3 @@ func TestMeshGatewaySecure(t *testing.T) {
 		})
 	}
 }
-
-// verifyFederation checks that the WAN federation between servers is successful
-// by first checking members are alive from the perspective of both servers.
-// If secure is true, it will also check that the ACL replication is running on the secondary server.
-func verifyFederation(t *testing.T, primaryClient, secondaryClient *api.Client, releaseName string, secure bool) {
-	retrier := &retry.Timer{Timeout: 5 * time.Minute, Wait: 1 * time.Second}
-	start := time.Now()
-
-	// Check that server in dc1 is healthy from the perspective of the server in dc2, and vice versa.
-	// We're calling the Consul health API, as opposed to checking serf membership status,
-	// because we need to make sure that the federated servers can make API calls and forward requests
-	// from one server to another. From running tests in CI for a while and using serf membership status before,
-	// we've noticed that the status could be "alive" as soon as the server in the secondary cluster joins the primary
-	// and then switch to "failed". This would require us to check that the status is "alive" is showing consistently for
-	// some amount of time, which could be quite flakey. Calling the API in another datacenter allows us to check that
-	// each server can forward calls to another, which is what we need for connect.
-	retry.RunWith(retrier, t, func(r *retry.R) {
-		secondaryServerHealth, _, err := primaryClient.Health().Node(fmt.Sprintf("%s-consul-server-0", releaseName), &api.QueryOptions{Datacenter: "dc2"})
-		require.NoError(r, err)
-		require.Equal(r, secondaryServerHealth.AggregatedStatus(), api.HealthPassing)
-
-		primaryServerHealth, _, err := secondaryClient.Health().Node(fmt.Sprintf("%s-consul-server-0", releaseName), &api.QueryOptions{Datacenter: "dc1"})
-		require.NoError(r, err)
-		require.Equal(r, primaryServerHealth.AggregatedStatus(), api.HealthPassing)
-
-		if secure {
-			replicationStatus, _, err := secondaryClient.ACL().Replication(nil)
-			require.NoError(r, err)
-			require.True(r, replicationStatus.Enabled)
-			require.True(r, replicationStatus.Running)
-		}
-	})
-
-	logger.Logf(t, "Took %s to verify federation", time.Since(start))
-}

acceptance/tests/partitions/partitions_test.go

@@ -128,7 +128,7 @@ func TestPartitions(t *testing.T) {
 
 			releaseName := helpers.RandomName()
 
-			consul.MergeMaps(serverHelmValues, commonHelmValues)
+			helpers.MergeMaps(serverHelmValues, commonHelmValues)
 
 			// Install the consul cluster with servers in the default kubernetes context.
 			serverConsulCluster := consul.NewHelmCluster(t, serverHelmValues, serverClusterContext, cfg, releaseName)
@@ -229,7 +229,7 @@ func TestPartitions(t *testing.T) {
 				clientHelmValues["meshGateway.service.nodePort"] = "30100"
 			}
 
-			consul.MergeMaps(clientHelmValues, commonHelmValues)
+			helpers.MergeMaps(clientHelmValues, commonHelmValues)
 
 			// Install the consul cluster without servers in the client cluster kubernetes context.
 			clientConsulCluster := consul.NewHelmCluster(t, clientHelmValues, clientClusterContext, cfg, releaseName)