vault: add support for wan federation when ACLs are enabled

hashicorp · Feb 9, 2022 · 9ae1d33 · 9ae1d33
1 parent b342d8c
commit 9ae1d33
Show file tree

Hide file tree

Showing 17 changed files with 420 additions and 145 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -70,7 +70,7 @@ commands:
         type: string
       consul-k8s-image:
         type: string
-        default: "docker.mirror.hashicorp.services/hashicorpdev/consul-k8s-control-plane:latest"
+        default: "ishustava/consul-k8s-dev:02-08-2022-d21554d8"
       go-path:
         type: string
         default: "/home/circleci/.go_workspace"

diff --git a/.github/workflows/golangci-lint-cli.yml b/.github/workflows/golangci-lint-cli.yml
@@ -1,4 +1,4 @@
-name: golangci-lint-acceptance
+name: golangci-lint-cli
 on:
   push:
     tags:

diff --git a/acceptance/framework/consul/cli_cluster.go b/acceptance/framework/consul/cli_cluster.go
@@ -51,7 +51,7 @@ func NewCLICluster(
 	ctx environment.TestContext,
 	cfg *config.TestConfig,
 	releaseName string,
-) Cluster {
+) *CLICluster {
 
 	// Create the namespace so the PSPs, SCCs, and enterprise secret can be created in the right namespace.
 	createOrUpdateNamespace(t, ctx.KubernetesClient(t), consulNS)

diff --git a/acceptance/framework/consul/consul_cluster.go b/acceptance/framework/consul/consul_cluster.go
@@ -47,6 +47,8 @@ type HelmCluster struct {
 	noCleanupOnFailure bool
 	debugDirectory     string
 	logger             terratestLogger.TestLogger
+
+	ACLToken string
 }
 
 func NewHelmCluster(
@@ -55,7 +57,7 @@ func NewHelmCluster(
 	ctx environment.TestContext,
 	cfg *config.TestConfig,
 	releaseName string,
-) Cluster {
+) *HelmCluster {
 
 	if cfg.EnablePodSecurityPolicies {
 		configurePodSecurityPolicies(t, ctx.KubernetesClient(t), cfg, ctx.KubectlOptions(t).Namespace)
@@ -234,21 +236,26 @@ func (h *HelmCluster) SetupConsulClient(t *testing.T, secure bool) *api.Client {
 		config.TLSConfig.InsecureSkipVerify = true
 		config.Scheme = "https"
 
-		// Get the ACL token. First, attempt to read it from the bootstrap token (this will be true in primary Consul servers).
-		// If the bootstrap token doesn't exist, it means we are running against a secondary cluster
-		// and will try to read the replication token from the federation secret.
-		// In secondary servers, we don't create a bootstrap token since ACLs are only bootstrapped in the primary.
-		// Instead, we provide a replication token that serves the role of the bootstrap token.
-		aclSecret, err := h.kubernetesClient.CoreV1().Secrets(namespace).Get(context.Background(), h.releaseName+"-consul-bootstrap-acl-token", metav1.GetOptions{})
-		if err != nil && errors.IsNotFound(err) {
-			federationSecret := fmt.Sprintf("%s-consul-federation", h.releaseName)
-			aclSecret, err = h.kubernetesClient.CoreV1().Secrets(namespace).Get(context.Background(), federationSecret, metav1.GetOptions{})
-			require.NoError(t, err)
-			config.Token = string(aclSecret.Data["replicationToken"])
-		} else if err == nil {
-			config.Token = string(aclSecret.Data["token"])
+		// If an ACL token is provided, we'll use that instead of trying to find it.
+		if h.ACLToken != "" {
+			config.Token = h.ACLToken
 		} else {
-			require.NoError(t, err)
+			// Get the ACL token. First, attempt to read it from the bootstrap token (this will be true in primary Consul servers).
+			// If the bootstrap token doesn't exist, it means we are running against a secondary cluster
+			// and will try to read the replication token from the federation secret.
+			// In secondary servers, we don't create a bootstrap token since ACLs are only bootstrapped in the primary.
+			// Instead, we provide a replication token that serves the role of the bootstrap token.
+			aclSecret, err := h.kubernetesClient.CoreV1().Secrets(namespace).Get(context.Background(), h.releaseName+"-consul-bootstrap-acl-token", metav1.GetOptions{})
+			if err != nil && errors.IsNotFound(err) {
+				federationSecret := fmt.Sprintf("%s-consul-federation", h.releaseName)
+				aclSecret, err = h.kubernetesClient.CoreV1().Secrets(namespace).Get(context.Background(), federationSecret, metav1.GetOptions{})
+				require.NoError(t, err)
+				config.Token = string(aclSecret.Data["replicationToken"])
+			} else if err == nil {
+				config.Token = string(aclSecret.Data["token"])
+			} else {
+				require.NoError(t, err)
+			}
 		}
 	}
 

diff --git a/acceptance/framework/consul/consul_cluster_test.go b/acceptance/framework/consul/consul_cluster_test.go
@@ -65,7 +65,7 @@ func TestNewHelmCluster(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			cluster := NewHelmCluster(t, tt.helmValues, &ctx{}, &config.TestConfig{ConsulImage: "test-config-image"}, "test")
-			require.Equal(t, cluster.(*HelmCluster).helmOptions.SetValues, tt.want)
+			require.Equal(t, cluster.helmOptions.SetValues, tt.want)
 		})
 	}
 }

diff --git a/acceptance/tests/vault/helpers.go b/acceptance/tests/vault/helpers.go
@@ -7,6 +7,7 @@ import (
 	"testing"
 
 	"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
+	"github.com/hashicorp/go-uuid"
 	vapi "github.com/hashicorp/vault/api"
 	"github.com/stretchr/testify/require"
 )
@@ -16,6 +17,10 @@ const (
 path "consul/data/secret/gossip" {
   capabilities = ["read"]
 }`
+	replicationTokenPolicy = `
+path "consul/data/secret/replication" {
+  capabilities = ["read", "update"]
+}`
 
 	// connectCAPolicy allows Consul to bootstrap all certificates for the service mesh in Vault.
 	// Adapted from https://www.consul.io/docs/connect/ca/vault#consul-managed-pki-paths.
@@ -113,7 +118,7 @@ func configureKubernetesAuthRoles(t *testing.T, vaultClient *vapi.Client, consul
 	params = map[string]interface{}{
 		"bound_service_account_names":      consulServerServiceAccountName,
 		"bound_service_account_namespaces": ns,
-		"policies":                         fmt.Sprintf("consul-gossip,connect-ca,consul-server-%s", datacenter),
+		"policies":                         fmt.Sprintf("consul-gossip,connect-ca,consul-server-%s,consul-replication-token", datacenter),
 		"ttl":                              "24h",
 	}
 	_, err = vaultClient.Logical().Write(fmt.Sprintf("auth/%s/role/consul-server", authPath), params)
@@ -161,20 +166,59 @@ func configurePKICertificates(t *testing.T, vaultClient *vapi.Client, consulRele
 		"max_ttl":            "1h",
 	}
 
-	pkiName := fmt.Sprintf("consul-server-%s", datacenter)
+	pkiRoleName := fmt.Sprintf("consul-server-%s", datacenter)
 
-	_, err := vaultClient.Logical().Write(fmt.Sprintf("pki/roles/%s", pkiName), params)
+	_, err := vaultClient.Logical().Write(fmt.Sprintf("pki/roles/%s", pkiRoleName), params)
 	require.NoError(t, err)
 
-	certificateIssuePath := fmt.Sprintf("pki/issue/%s", pkiName)
+	certificateIssuePath := fmt.Sprintf("pki/issue/%s", pkiRoleName)
 	serverTLSPolicy := fmt.Sprintf(`
 path %q {
   capabilities = ["create", "update"]
 }`, certificateIssuePath)
 
 	// Create the server policy.
-	err = vaultClient.Sys().PutPolicy(pkiName, serverTLSPolicy)
+	err = vaultClient.Sys().PutPolicy(pkiRoleName, serverTLSPolicy)
 	require.NoError(t, err)
 
 	return certificateIssuePath
 }
+
+// configureReplicationTokenVaultSecret generates a replication token secret ID,
+// stores it in vault as a secret and configures a policy to access it.
+func configureReplicationTokenVaultSecret(t *testing.T, vaultClient *vapi.Client, consulReleaseName, ns string, authMethodPaths ...string) string {
+	// Create the Vault Policy for the replication token.
+	logger.Log(t, "Creating replication token policy")
+	err := vaultClient.Sys().PutPolicy("consul-replication-token", replicationTokenPolicy)
+	require.NoError(t, err)
+
+	// Generate the token secret.
+	token, err := uuid.GenerateUUID()
+	require.NoError(t, err)
+
+	// Create the replication token secret.
+	logger.Log(t, "Creating the replication token secret")
+	params := map[string]interface{}{
+		"data": map[string]interface{}{
+			"replication": token,
+		},
+	}
+	_, err = vaultClient.Logical().Write("consul/data/secret/replication", params)
+	require.NoError(t, err)
+
+	logger.Log(t, "Creating kubernetes auth role for the server-acl-init job")
+	serverACLInitSAName := fmt.Sprintf("%s-consul-server-acl-init", consulReleaseName)
+	params = map[string]interface{}{
+		"bound_service_account_names":      serverACLInitSAName,
+		"bound_service_account_namespaces": ns,
+		"policies":                         "consul-replication-token",
+		"ttl":                              "24h",
+	}
+
+	for _, authMethodPath := range authMethodPaths {
+		_, err := vaultClient.Logical().Write(fmt.Sprintf("auth/%s/role/server-acl-init", authMethodPath), params)
+		require.NoError(t, err)
+	}
+
+	return token
+}
diff --git a/acceptance/tests/vault/vault_wan_fed_test.go b/acceptance/tests/vault/vault_wan_fed_test.go
@@ -12,6 +12,7 @@ import (
 	"github.com/hashicorp/consul-k8s/acceptance/framework/k8s"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/vault"
+	"github.com/hashicorp/consul/api"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
@@ -115,6 +116,8 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 	primaryCertPath := configurePKICertificates(t, vaultClient, consulReleaseName, ns, "dc1")
 	secondaryCertPath := configurePKICertificates(t, vaultClient, consulReleaseName, ns, "dc2")
 
+	replicationToken := configureReplicationTokenVaultSecret(t, vaultClient, consulReleaseName, ns, "kubernetes", "kubernetes-dc2")
+
 	// Move Vault CA secret from primary to secondary so that we can mount it to pods in the
 	// secondary cluster.
 	vaultCASecretName := vault.CASecretName(vaultReleaseName)
@@ -135,7 +138,6 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 
 		// TLS config.
 		"global.tls.enabled":           "true",
-		"global.tls.httpsOnly":         "false",
 		"global.tls.enableAutoEncrypt": "true",
 		"global.tls.caCert.secretName": "pki/cert/ca",
 		"server.serverCert.secretName": primaryCertPath,
@@ -144,6 +146,12 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 		"global.gossipEncryption.secretName": "consul/data/secret/gossip",
 		"global.gossipEncryption.secretKey":  "gossip",
 
+		// ACL config.
+		"global.acls.manageSystemACLs":            "true",
+		"global.acls.createReplicationToken":      "true",
+		"global.acls.replicationToken.secretName": "consul/data/secret/replication",
+		"global.acls.replicationToken.secretKey":  "replication",
+
 		// Mesh config.
 		"connectInject.enabled": "true",
 		"controller.enabled":    "true",
@@ -156,12 +164,13 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 		"server.extraVolumes[0].load": "false",
 
 		// Vault config.
-		"global.secretsBackend.vault.enabled":          "true",
-		"global.secretsBackend.vault.consulServerRole": "consul-server",
-		"global.secretsBackend.vault.consulClientRole": "consul-client",
-		"global.secretsBackend.vault.consulCARole":     "consul-ca",
-		"global.secretsBackend.vault.ca.secretName":    vaultCASecretName,
-		"global.secretsBackend.vault.ca.secretKey":     "tls.crt",
+		"global.secretsBackend.vault.enabled":              "true",
+		"global.secretsBackend.vault.consulServerRole":     "consul-server",
+		"global.secretsBackend.vault.consulClientRole":     "consul-client",
+		"global.secretsBackend.vault.consulCARole":         "consul-ca",
+		"global.secretsBackend.vault.manageSystemACLsRole": "server-acl-init",
+		"global.secretsBackend.vault.ca.secretName":        vaultCASecretName,
+		"global.secretsBackend.vault.ca.secretKey":         "tls.crt",
 	}
 
 	if cfg.UseKind {
@@ -182,7 +191,6 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 
 		// TLS config.
 		"global.tls.enabled":           "true",
-		"global.tls.httpsOnly":         "false",
 		"global.tls.enableAutoEncrypt": "true",
 		"global.tls.caCert.secretName": "pki/cert/ca",
 		"server.serverCert.secretName": secondaryCertPath,
@@ -191,6 +199,11 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 		"global.gossipEncryption.secretName": "consul/data/secret/gossip",
 		"global.gossipEncryption.secretKey":  "gossip",
 
+		// ACL config.
+		"global.acls.manageSystemACLs":            "true",
+		"global.acls.replicationToken.secretName": "consul/data/secret/replication",
+		"global.acls.replicationToken.secretKey":  "replication",
+
 		// Mesh config.
 		"connectInject.enabled": "true",
 		"meshGateway.enabled":   "true",
@@ -203,13 +216,14 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 		"server.extraConfig":          serverExtraConfig,
 
 		// Vault config.
-		"global.secretsBackend.vault.enabled":          "true",
-		"global.secretsBackend.vault.consulServerRole": "consul-server",
-		"global.secretsBackend.vault.consulClientRole": "consul-client",
-		"global.secretsBackend.vault.consulCARole":     "consul-ca",
-		"global.secretsBackend.vault.ca.secretName":    vaultCASecretName,
-		"global.secretsBackend.vault.ca.secretKey":     "tls.crt",
-		"global.secretsBackend.vault.agentAnnotations": fmt.Sprintf("vault.hashicorp.com/tls-server-name: %s-vault", vaultReleaseName),
+		"global.secretsBackend.vault.enabled":              "true",
+		"global.secretsBackend.vault.consulServerRole":     "consul-server",
+		"global.secretsBackend.vault.consulClientRole":     "consul-client",
+		"global.secretsBackend.vault.consulCARole":         "consul-ca",
+		"global.secretsBackend.vault.manageSystemACLsRole": "server-acl-init",
+		"global.secretsBackend.vault.ca.secretName":        vaultCASecretName,
+		"global.secretsBackend.vault.ca.secretKey":         "tls.crt",
+		"global.secretsBackend.vault.agentAnnotations":     fmt.Sprintf("vault.hashicorp.com/tls-server-name: %s-vault", vaultReleaseName),
 	}
 
 	if cfg.UseKind {
@@ -223,9 +237,10 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 
 	// Verify federation between servers.
 	logger.Log(t, "verifying federation was successful")
-	primaryClient := primaryConsulCluster.SetupConsulClient(t, false)
-	secondaryClient := secondaryConsulCluster.SetupConsulClient(t, false)
-	helpers.VerifyFederation(t, primaryClient, secondaryClient, consulReleaseName, false)
+	primaryClient := primaryConsulCluster.SetupConsulClient(t, true)
+	secondaryConsulCluster.ACLToken = replicationToken
+	secondaryClient := secondaryConsulCluster.SetupConsulClient(t, true)
+	helpers.VerifyFederation(t, primaryClient, secondaryClient, consulReleaseName, true)
 
 	// Create a ProxyDefaults resource to configure services to use the mesh
 	// gateways.
@@ -243,6 +258,19 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 	logger.Log(t, "creating static-client in dc1")
 	k8s.DeployKustomize(t, primaryCtx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-multi-dc")
 
+	logger.Log(t, "creating intention")
+	_, _, err = primaryClient.ConfigEntries().Set(&api.ServiceIntentionsConfigEntry{
+		Kind: api.ServiceIntentions,
+		Name: "static-server",
+		Sources: []*api.SourceIntention{
+			{
+				Name:   "static-client",
+				Action: api.IntentionActionAllow,
+			},
+		},
+	}, nil)
+	require.NoError(t, err)
+
 	logger.Log(t, "checking that connection is successful")
 	k8s.CheckStaticServerConnectionSuccessful(t, primaryCtx.KubectlOptions(t), "http://localhost:1234")
 }

diff --git a/charts/consul/templates/_helpers.tpl b/charts/consul/templates/_helpers.tpl
@@ -51,6 +51,20 @@ as well as the global.name setting.
 {{ printf "localhost,%s-server,*.%s-server,*.%s-server.%s,*.%s-server.%s.svc,*.server.%s.%s" $name $name $name $ns $name $ns (.Values.global.datacenter ) (.Values.global.domain) }}
 {{- end -}}
 
+{{- define "consul.vaultReplicationTokenTemplate" -}}
+|
+          {{ "{{" }}- with secret "{{ .Values.global.acls.replicationToken.secretName }}" -{{ "}}" }}
+          {{ "{{" }}- {{ printf ".Data.data.%s" .Values.global.acls.replicationToken.secretKey }} -{{ "}}" }}
+          {{ "{{" }}- end -{{ "}}" }}
+{{- end -}}
+
+{{- define "consul.vaultReplicationTokenConfigTemplate" -}}
+|
+          {{ "{{" }}- with secret "{{ .Values.global.acls.replicationToken.secretName }}" -{{ "}}" }}
+          acl { tokens { agent = "{{ "{{" }}- {{ printf ".Data.data.%s" .Values.global.acls.replicationToken.secretKey }} -{{ "}}" }}", replication = "{{ "{{" }}- {{ printf ".Data.data.%s" .Values.global.acls.replicationToken.secretKey }} -{{ "}}" }}" }}
+          {{ "{{" }}- end -{{ "}}" }}
+{{- end -}}
+
 {{/*
 Sets up the extra-from-values config file passed to consul and then uses sed to do any necessary
 substitution for HOST_IP/POD_IP/HOSTNAME. Useful for dogstats telemetry. The output file