vault: add support for wan federation when ACLs are enabled

hashicorp · Feb 17, 2022 · 43f6307 · 43f6307
1 parent 4f5c935
commit 43f6307
Show file tree

Hide file tree

Showing 18 changed files with 421 additions and 186 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -70,7 +70,7 @@ commands:
         type: string
       consul-k8s-image:
         type: string
-        default: "docker.mirror.hashicorp.services/hashicorpdev/consul-k8s-control-plane:latest"
+        default: "ishustava/consul-k8s-dev:02-08-2022-d21554d8"
       go-path:
         type: string
         default: "/home/circleci/.go_workspace"

diff --git a/.github/workflows/golangci-lint-cli.yml b/.github/workflows/golangci-lint-cli.yml
@@ -1,4 +1,4 @@
-name: golangci-lint-acceptance
+name: golangci-lint-cli
 on:
   push:
     tags:

diff --git a/acceptance/framework/consul/cli_cluster.go b/acceptance/framework/consul/cli_cluster.go
@@ -51,7 +51,7 @@ func NewCLICluster(
 	ctx environment.TestContext,
 	cfg *config.TestConfig,
 	releaseName string,
-) Cluster {
+) *CLICluster {
 
 	// Create the namespace so the PSPs, SCCs, and enterprise secret can be created in the right namespace.
 	createOrUpdateNamespace(t, ctx.KubernetesClient(t), consulNS)

diff --git a/acceptance/framework/consul/consul_cluster.go b/acceptance/framework/consul/consul_cluster.go
@@ -40,6 +40,11 @@ type Cluster interface {
 // HelmCluster implements Cluster and uses Helm
 // to create, destroy, and upgrade consul
 type HelmCluster struct {
+	// ACLToken is an optional ACL token that will be used to create
+	// a Consul API client. If not provided, we will attempt to read
+	// a bootstrap token from a Kubernetes secret stored in the cluster.
+	ACLToken string
+
 	ctx                environment.TestContext
 	helmOptions        *helm.Options
 	releaseName        string
@@ -55,7 +60,7 @@ func NewHelmCluster(
 	ctx environment.TestContext,
 	cfg *config.TestConfig,
 	releaseName string,
-) Cluster {
+) *HelmCluster {
 
 	if cfg.EnablePodSecurityPolicies {
 		configurePodSecurityPolicies(t, ctx.KubernetesClient(t), cfg, ctx.KubectlOptions(t).Namespace)
@@ -234,21 +239,26 @@ func (h *HelmCluster) SetupConsulClient(t *testing.T, secure bool) *api.Client {
 		config.TLSConfig.InsecureSkipVerify = true
 		config.Scheme = "https"
 
-		// Get the ACL token. First, attempt to read it from the bootstrap token (this will be true in primary Consul servers).
-		// If the bootstrap token doesn't exist, it means we are running against a secondary cluster
-		// and will try to read the replication token from the federation secret.
-		// In secondary servers, we don't create a bootstrap token since ACLs are only bootstrapped in the primary.
-		// Instead, we provide a replication token that serves the role of the bootstrap token.
-		aclSecret, err := h.kubernetesClient.CoreV1().Secrets(namespace).Get(context.Background(), h.releaseName+"-consul-bootstrap-acl-token", metav1.GetOptions{})
-		if err != nil && errors.IsNotFound(err) {
-			federationSecret := fmt.Sprintf("%s-consul-federation", h.releaseName)
-			aclSecret, err = h.kubernetesClient.CoreV1().Secrets(namespace).Get(context.Background(), federationSecret, metav1.GetOptions{})
-			require.NoError(t, err)
-			config.Token = string(aclSecret.Data["replicationToken"])
-		} else if err == nil {
-			config.Token = string(aclSecret.Data["token"])
+		// If an ACL token is provided, we'll use that instead of trying to find it.
+		if h.ACLToken != "" {
+			config.Token = h.ACLToken
 		} else {
-			require.NoError(t, err)
+			// Get the ACL token. First, attempt to read it from the bootstrap token (this will be true in primary Consul servers).
+			// If the bootstrap token doesn't exist, it means we are running against a secondary cluster
+			// and will try to read the replication token from the federation secret.
+			// In secondary servers, we don't create a bootstrap token since ACLs are only bootstrapped in the primary.
+			// Instead, we provide a replication token that serves the role of the bootstrap token.
+			aclSecret, err := h.kubernetesClient.CoreV1().Secrets(namespace).Get(context.Background(), h.releaseName+"-consul-bootstrap-acl-token", metav1.GetOptions{})
+			if err != nil && errors.IsNotFound(err) {
+				federationSecret := fmt.Sprintf("%s-consul-federation", h.releaseName)
+				aclSecret, err = h.kubernetesClient.CoreV1().Secrets(namespace).Get(context.Background(), federationSecret, metav1.GetOptions{})
+				require.NoError(t, err)
+				config.Token = string(aclSecret.Data["replicationToken"])
+			} else if err == nil {
+				config.Token = string(aclSecret.Data["token"])
+			} else {
+				require.NoError(t, err)
+			}
 		}
 	}
 

diff --git a/acceptance/framework/consul/consul_cluster_test.go b/acceptance/framework/consul/consul_cluster_test.go
@@ -65,7 +65,7 @@ func TestNewHelmCluster(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			cluster := NewHelmCluster(t, tt.helmValues, &ctx{}, &config.TestConfig{ConsulImage: "test-config-image"}, "test")
-			require.Equal(t, cluster.(*HelmCluster).helmOptions.SetValues, tt.want)
+			require.Equal(t, cluster.helmOptions.SetValues, tt.want)
 		})
 	}
 }

diff --git a/acceptance/framework/helpers/helpers.go b/acceptance/framework/helpers/helpers.go
@@ -148,46 +148,3 @@ func MergeMaps(a, b map[string]string) {
 		a[k] = v
 	}
 }
-
-// VerifyFederation checks that the WAN federation between servers is successful
-// by first checking members are alive from the perspective of both servers.
-// If secure is true, it will also check that the ACL replication is running on the secondary server.
-func VerifyFederation(t *testing.T, primaryClient, secondaryClient *api.Client, releaseName string, secure bool) {
-	retrier := &retry.Timer{Timeout: 5 * time.Minute, Wait: 1 * time.Second}
-	start := time.Now()
-
-	// Check that server in dc1 is healthy from the perspective of the server in dc2, and vice versa.
-	// We're calling the Consul health API, as opposed to checking serf membership status,
-	// because we need to make sure that the federated servers can make API calls and forward requests
-	// from one server to another. From running tests in CI for a while and using serf membership status before,
-	// we've noticed that the status could be "alive" as soon as the server in the secondary cluster joins the primary
-	// and then switch to "failed". This would require us to check that the status is "alive" is showing consistently for
-	// some amount of time, which could be quite flakey. Calling the API in another datacenter allows us to check that
-	// each server can forward calls to another, which is what we need for connect.
-	retry.RunWith(retrier, t, func(r *retry.R) {
-		secondaryServerHealth, _, err := primaryClient.Health().Node(fmt.Sprintf("%s-consul-server-0", releaseName), &api.QueryOptions{Datacenter: "dc2"})
-		require.NoError(r, err)
-		require.Equal(r, secondaryServerHealth.AggregatedStatus(), api.HealthPassing)
-
-		primaryServerHealth, _, err := secondaryClient.Health().Node(fmt.Sprintf("%s-consul-server-0", releaseName), &api.QueryOptions{Datacenter: "dc1"})
-		require.NoError(r, err)
-		require.Equal(r, primaryServerHealth.AggregatedStatus(), api.HealthPassing)
-
-		if secure {
-			replicationStatus, _, err := secondaryClient.ACL().Replication(nil)
-			require.NoError(r, err)
-			require.True(r, replicationStatus.Enabled)
-			require.True(r, replicationStatus.Running)
-		}
-	})
-
-	logger.Logf(t, "Took %s to verify federation", time.Since(start))
-}
-
-// MergeMaps will merge the values in b with values in a and save in a.
-// If there are conflicts, the values in b will overwrite the values in a.
-func MergeMaps(a, b map[string]string) {
-	for k, v := range b {
-		a[k] = v
-	}
-}
diff --git a/acceptance/tests/vault/helpers.go b/acceptance/tests/vault/helpers.go
@@ -7,6 +7,7 @@ import (
 	"testing"
 
 	"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
+	"github.com/hashicorp/go-uuid"
 	vapi "github.com/hashicorp/vault/api"
 	"github.com/stretchr/testify/require"
 )
@@ -16,6 +17,10 @@ const (
 path "consul/data/secret/gossip" {
   capabilities = ["read"]
 }`
+	replicationTokenPolicy = `
+path "consul/data/secret/replication" {
+  capabilities = ["read", "update"]
+}`
 
 	// connectCAPolicy allows Consul to bootstrap all certificates for the service mesh in Vault.
 	// Adapted from https://www.consul.io/docs/connect/ca/vault#consul-managed-pki-paths.
@@ -113,7 +118,7 @@ func configureKubernetesAuthRoles(t *testing.T, vaultClient *vapi.Client, consul
 	params = map[string]interface{}{
 		"bound_service_account_names":      consulServerServiceAccountName,
 		"bound_service_account_namespaces": ns,
-		"policies":                         fmt.Sprintf("consul-gossip,connect-ca,consul-server-%s", datacenter),
+		"policies":                         fmt.Sprintf("consul-gossip,connect-ca,consul-server-%s,consul-replication-token", datacenter),
 		"ttl":                              "24h",
 	}
 	_, err = vaultClient.Logical().Write(fmt.Sprintf("auth/%s/role/consul-server", authPath), params)
@@ -178,3 +183,42 @@ path %q {
 
 	return certificateIssuePath
 }
+
+// configureReplicationTokenVaultSecret generates a replication token secret ID,
+// stores it in vault as a secret and configures a policy to access it.
+func configureReplicationTokenVaultSecret(t *testing.T, vaultClient *vapi.Client, consulReleaseName, ns string, authMethodPaths ...string) string {
+	// Create the Vault Policy for the replication token.
+	logger.Log(t, "Creating replication token policy")
+	err := vaultClient.Sys().PutPolicy("consul-replication-token", replicationTokenPolicy)
+	require.NoError(t, err)
+
+	// Generate the token secret.
+	token, err := uuid.GenerateUUID()
+	require.NoError(t, err)
+
+	// Create the replication token secret.
+	logger.Log(t, "Creating the replication token secret")
+	params := map[string]interface{}{
+		"data": map[string]interface{}{
+			"replication": token,
+		},
+	}
+	_, err = vaultClient.Logical().Write("consul/data/secret/replication", params)
+	require.NoError(t, err)
+
+	logger.Log(t, "Creating kubernetes auth role for the server-acl-init job")
+	serverACLInitSAName := fmt.Sprintf("%s-consul-server-acl-init", consulReleaseName)
+	params = map[string]interface{}{
+		"bound_service_account_names":      serverACLInitSAName,
+		"bound_service_account_namespaces": ns,
+		"policies":                         "consul-replication-token",
+		"ttl":                              "24h",
+	}
+
+	for _, authMethodPath := range authMethodPaths {
+		_, err := vaultClient.Logical().Write(fmt.Sprintf("auth/%s/role/server-acl-init", authMethodPath), params)
+		require.NoError(t, err)
+	}
+
+	return token
+}
diff --git a/acceptance/tests/vault/vault_wan_fed_test.go b/acceptance/tests/vault/vault_wan_fed_test.go
@@ -12,6 +12,7 @@ import (
 	"github.com/hashicorp/consul-k8s/acceptance/framework/k8s"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/vault"
+	"github.com/hashicorp/consul/api"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
@@ -115,6 +116,8 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 	primaryCertPath := configurePKICertificates(t, vaultClient, consulReleaseName, ns, "dc1")
 	secondaryCertPath := configurePKICertificates(t, vaultClient, consulReleaseName, ns, "dc2")
 
+	replicationToken := configureReplicationTokenVaultSecret(t, vaultClient, consulReleaseName, ns, "kubernetes", "kubernetes-dc2")
+
 	// Move Vault CA secret from primary to secondary so that we can mount it to pods in the
 	// secondary cluster.
 	vaultCASecretName := vault.CASecretName(vaultReleaseName)
@@ -135,7 +138,6 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 
 		// TLS config.
 		"global.tls.enabled":           "true",
-		"global.tls.httpsOnly":         "false",
 		"global.tls.enableAutoEncrypt": "true",
 		"global.tls.caCert.secretName": "pki/cert/ca",
 		"server.serverCert.secretName": primaryCertPath,
@@ -144,6 +146,12 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 		"global.gossipEncryption.secretName": "consul/data/secret/gossip",
 		"global.gossipEncryption.secretKey":  "gossip",
 
+		// ACL config.
+		"global.acls.manageSystemACLs":            "true",
+		"global.acls.createReplicationToken":      "true",
+		"global.acls.replicationToken.secretName": "consul/data/secret/replication",
+		"global.acls.replicationToken.secretKey":  "replication",
+
 		// Mesh config.
 		"connectInject.enabled": "true",
 		"controller.enabled":    "true",
@@ -156,12 +164,13 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 		"server.extraVolumes[0].load": "false",
 
 		// Vault config.
-		"global.secretsBackend.vault.enabled":          "true",
-		"global.secretsBackend.vault.consulServerRole": "consul-server",
-		"global.secretsBackend.vault.consulClientRole": "consul-client",
-		"global.secretsBackend.vault.consulCARole":     "consul-ca",
-		"global.secretsBackend.vault.ca.secretName":    vaultCASecretName,
-		"global.secretsBackend.vault.ca.secretKey":     "tls.crt",
+		"global.secretsBackend.vault.enabled":              "true",
+		"global.secretsBackend.vault.consulServerRole":     "consul-server",
+		"global.secretsBackend.vault.consulClientRole":     "consul-client",
+		"global.secretsBackend.vault.consulCARole":         "consul-ca",
+		"global.secretsBackend.vault.manageSystemACLsRole": "server-acl-init",
+		"global.secretsBackend.vault.ca.secretName":        vaultCASecretName,
+		"global.secretsBackend.vault.ca.secretKey":         "tls.crt",
 	}
 
 	if cfg.UseKind {
@@ -182,7 +191,6 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 
 		// TLS config.
 		"global.tls.enabled":           "true",
-		"global.tls.httpsOnly":         "false",
 		"global.tls.enableAutoEncrypt": "true",
 		"global.tls.caCert.secretName": "pki/cert/ca",
 		"server.serverCert.secretName": secondaryCertPath,
@@ -191,6 +199,11 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 		"global.gossipEncryption.secretName": "consul/data/secret/gossip",
 		"global.gossipEncryption.secretKey":  "gossip",
 
+		// ACL config.
+		"global.acls.manageSystemACLs":            "true",
+		"global.acls.replicationToken.secretName": "consul/data/secret/replication",
+		"global.acls.replicationToken.secretKey":  "replication",
+
 		// Mesh config.
 		"connectInject.enabled": "true",
 		"meshGateway.enabled":   "true",
@@ -203,13 +216,14 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 		"server.extraConfig":          serverExtraConfig,
 
 		// Vault config.
-		"global.secretsBackend.vault.enabled":          "true",
-		"global.secretsBackend.vault.consulServerRole": "consul-server",
-		"global.secretsBackend.vault.consulClientRole": "consul-client",
-		"global.secretsBackend.vault.consulCARole":     "consul-ca",
-		"global.secretsBackend.vault.ca.secretName":    vaultCASecretName,
-		"global.secretsBackend.vault.ca.secretKey":     "tls.crt",
-		"global.secretsBackend.vault.agentAnnotations": fmt.Sprintf("vault.hashicorp.com/tls-server-name: %s-vault", vaultReleaseName),
+		"global.secretsBackend.vault.enabled":              "true",
+		"global.secretsBackend.vault.consulServerRole":     "consul-server",
+		"global.secretsBackend.vault.consulClientRole":     "consul-client",
+		"global.secretsBackend.vault.consulCARole":         "consul-ca",
+		"global.secretsBackend.vault.manageSystemACLsRole": "server-acl-init",
+		"global.secretsBackend.vault.ca.secretName":        vaultCASecretName,
+		"global.secretsBackend.vault.ca.secretKey":         "tls.crt",
+		"global.secretsBackend.vault.agentAnnotations":     fmt.Sprintf("vault.hashicorp.com/tls-server-name: %s-vault", vaultReleaseName),
 	}
 
 	if cfg.UseKind {
@@ -223,9 +237,10 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 
 	// Verify federation between servers.
 	logger.Log(t, "verifying federation was successful")
-	primaryClient := primaryConsulCluster.SetupConsulClient(t, false)
-	secondaryClient := secondaryConsulCluster.SetupConsulClient(t, false)
-	helpers.VerifyFederation(t, primaryClient, secondaryClient, consulReleaseName, false)
+	primaryClient := primaryConsulCluster.SetupConsulClient(t, true)
+	secondaryConsulCluster.ACLToken = replicationToken
+	secondaryClient := secondaryConsulCluster.SetupConsulClient(t, true)
+	helpers.VerifyFederation(t, primaryClient, secondaryClient, consulReleaseName, true)
 
 	// Create a ProxyDefaults resource to configure services to use the mesh
 	// gateways.
@@ -243,6 +258,19 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 	logger.Log(t, "creating static-client in dc1")
 	k8s.DeployKustomize(t, primaryCtx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-multi-dc")
 
+	logger.Log(t, "creating intention")
+	_, _, err = primaryClient.ConfigEntries().Set(&api.ServiceIntentionsConfigEntry{
+		Kind: api.ServiceIntentions,
+		Name: "static-server",
+		Sources: []*api.SourceIntention{
+			{
+				Name:   "static-client",
+				Action: api.IntentionActionAllow,
+			},
+		},
+	}, nil)
+	require.NoError(t, err)
+
 	logger.Log(t, "checking that connection is successful")
 	k8s.CheckStaticServerConnectionSuccessful(t, primaryCtx.KubectlOptions(t), "http://localhost:1234")
 }

diff --git a/charts/consul/templates/_helpers.tpl b/charts/consul/templates/_helpers.tpl
@@ -59,6 +59,20 @@ as well as the global.name setting.
 {{- if .Values.global.tls -}}{{- if .Values.global.tls.serverAdditionalIPSANs -}}{{- range $ipsan := .Values.global.tls.serverAdditionalIPSANs }},{{ $ipsan }} {{- end -}}{{- end -}}{{- end -}}
 {{- end -}}
 
+{{- define "consul.vaultReplicationTokenTemplate" -}}
+|
+          {{ "{{" }}- with secret "{{ .Values.global.acls.replicationToken.secretName }}" -{{ "}}" }}
+          {{ "{{" }}- {{ printf ".Data.data.%s" .Values.global.acls.replicationToken.secretKey }} -{{ "}}" }}
+          {{ "{{" }}- end -{{ "}}" }}
+{{- end -}}
+
+{{- define "consul.vaultReplicationTokenConfigTemplate" -}}
+|
+          {{ "{{" }}- with secret "{{ .Values.global.acls.replicationToken.secretName }}" -{{ "}}" }}
+          acl { tokens { agent = "{{ "{{" }}- {{ printf ".Data.data.%s" .Values.global.acls.replicationToken.secretKey }} -{{ "}}" }}", replication = "{{ "{{" }}- {{ printf ".Data.data.%s" .Values.global.acls.replicationToken.secretKey }} -{{ "}}" }}" }}
+          {{ "{{" }}- end -{{ "}}" }}
+{{- end -}}
+
 {{/*
 Sets up the extra-from-values config file passed to consul and then uses sed to do any necessary
 substitution for HOST_IP/POD_IP/HOSTNAME. Useful for dogstats telemetry. The output file