vault: add support for wan federation with Vault when ACLs are enabled

.github/workflows/golangci-lint-cli.yml

@@ -1,4 +1,4 @@
-name: golangci-lint-acceptance
+name: golangci-lint-cli
 on:
   push:
     tags:

acceptance/framework/consul/cli_cluster.go

@@ -51,7 +51,7 @@ func NewCLICluster(
 	ctx environment.TestContext,
 	cfg *config.TestConfig,
 	releaseName string,
-) Cluster {
+) *CLICluster {
 
 	// Create the namespace so the PSPs, SCCs, and enterprise secret can be created in the right namespace.
 	createOrUpdateNamespace(t, ctx.KubernetesClient(t), consulNS)

acceptance/framework/consul/consul_cluster.go

@@ -40,6 +40,11 @@ type Cluster interface {
 // HelmCluster implements Cluster and uses Helm
 // to create, destroy, and upgrade consul
 type HelmCluster struct {
+	// ACLToken is an optional ACL token that will be used to create
+	// a Consul API client. If not provided, we will attempt to read
+	// a bootstrap token from a Kubernetes secret stored in the cluster.
+	ACLToken string
+
 	ctx                environment.TestContext
 	helmOptions        *helm.Options
 	releaseName        string
@@ -55,7 +60,7 @@ func NewHelmCluster(
 	ctx environment.TestContext,
 	cfg *config.TestConfig,
 	releaseName string,
-) Cluster {
+) *HelmCluster {
 
 	if cfg.EnablePodSecurityPolicies {
 		configurePodSecurityPolicies(t, ctx.KubernetesClient(t), cfg, ctx.KubectlOptions(t).Namespace)
@@ -234,21 +239,26 @@ func (h *HelmCluster) SetupConsulClient(t *testing.T, secure bool) *api.Client {
 		config.TLSConfig.InsecureSkipVerify = true
 		config.Scheme = "https"
 
-		// Get the ACL token. First, attempt to read it from the bootstrap token (this will be true in primary Consul servers).
-		// If the bootstrap token doesn't exist, it means we are running against a secondary cluster
-		// and will try to read the replication token from the federation secret.
-		// In secondary servers, we don't create a bootstrap token since ACLs are only bootstrapped in the primary.
-		// Instead, we provide a replication token that serves the role of the bootstrap token.
-		aclSecret, err := h.kubernetesClient.CoreV1().Secrets(namespace).Get(context.Background(), h.releaseName+"-consul-bootstrap-acl-token", metav1.GetOptions{})
-		if err != nil && errors.IsNotFound(err) {
-			federationSecret := fmt.Sprintf("%s-consul-federation", h.releaseName)
-			aclSecret, err = h.kubernetesClient.CoreV1().Secrets(namespace).Get(context.Background(), federationSecret, metav1.GetOptions{})
-			require.NoError(t, err)
-			config.Token = string(aclSecret.Data["replicationToken"])
-		} else if err == nil {
-			config.Token = string(aclSecret.Data["token"])
+		// If an ACL token is provided, we'll use that instead of trying to find it.
+		if h.ACLToken != "" {
+			config.Token = h.ACLToken
 		} else {
-			require.NoError(t, err)
+			// Get the ACL token. First, attempt to read it from the bootstrap token (this will be true in primary Consul servers).
+			// If the bootstrap token doesn't exist, it means we are running against a secondary cluster
+			// and will try to read the replication token from the federation secret.
+			// In secondary servers, we don't create a bootstrap token since ACLs are only bootstrapped in the primary.
+			// Instead, we provide a replication token that serves the role of the bootstrap token.
+			aclSecret, err := h.kubernetesClient.CoreV1().Secrets(namespace).Get(context.Background(), h.releaseName+"-consul-bootstrap-acl-token", metav1.GetOptions{})
+			if err != nil && errors.IsNotFound(err) {
+				federationSecret := fmt.Sprintf("%s-consul-federation", h.releaseName)
+				aclSecret, err = h.kubernetesClient.CoreV1().Secrets(namespace).Get(context.Background(), federationSecret, metav1.GetOptions{})
+				require.NoError(t, err)
+				config.Token = string(aclSecret.Data["replicationToken"])
+			} else if err == nil {
+				config.Token = string(aclSecret.Data["token"])
+			} else {
+				require.NoError(t, err)
+			}
 		}
 	}
 

acceptance/framework/consul/consul_cluster_test.go

@@ -65,7 +65,7 @@ func TestNewHelmCluster(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			cluster := NewHelmCluster(t, tt.helmValues, &ctx{}, &config.TestConfig{ConsulImage: "test-config-image"}, "test")
-			require.Equal(t, cluster.(*HelmCluster).helmOptions.SetValues, tt.want)
+			require.Equal(t, cluster.helmOptions.SetValues, tt.want)
 		})
 	}
 }

acceptance/tests/vault/helpers.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 
 	"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
+	"github.com/hashicorp/go-uuid"
 	vapi "github.com/hashicorp/vault/api"
 	"github.com/stretchr/testify/require"
 )
@@ -16,6 +17,10 @@ const (
 path "consul/data/secret/gossip" {
   capabilities = ["read"]
 }`
+	replicationTokenPolicy = `
+path "consul/data/secret/replication" {
+  capabilities = ["read", "update"]
+}`
 
 	// connectCAPolicy allows Consul to bootstrap all certificates for the service mesh in Vault.
 	// Adapted from https://www.consul.io/docs/connect/ca/vault#consul-managed-pki-paths.
@@ -113,7 +118,7 @@ func configureKubernetesAuthRoles(t *testing.T, vaultClient *vapi.Client, consul
 	params = map[string]interface{}{
 		"bound_service_account_names":      consulServerServiceAccountName,
 		"bound_service_account_namespaces": ns,
-		"policies":                         fmt.Sprintf("consul-gossip,connect-ca,consul-server-%s", datacenter),
+		"policies":                         fmt.Sprintf("consul-gossip,connect-ca,consul-server-%s,consul-replication-token", datacenter),
 		"ttl":                              "24h",
 	}
 	_, err = vaultClient.Logical().Write(fmt.Sprintf("auth/%s/role/consul-server", authPath), params)
@@ -178,3 +183,42 @@ path %q {
 
 	return certificateIssuePath
 }
+
+// configureReplicationTokenVaultSecret generates a replication token secret ID,
+// stores it in vault as a secret and configures a policy to access it.
+func configureReplicationTokenVaultSecret(t *testing.T, vaultClient *vapi.Client, consulReleaseName, ns string, authMethodPaths ...string) string {
+	// Create the Vault Policy for the replication token.
+	logger.Log(t, "Creating replication token policy")
+	err := vaultClient.Sys().PutPolicy("consul-replication-token", replicationTokenPolicy)
+	require.NoError(t, err)
+
+	// Generate the token secret.
+	token, err := uuid.GenerateUUID()
+	require.NoError(t, err)
+
+	// Create the replication token secret.
+	logger.Log(t, "Creating the replication token secret")
+	params := map[string]interface{}{
+		"data": map[string]interface{}{
+			"replication": token,
+		},
+	}
+	_, err = vaultClient.Logical().Write("consul/data/secret/replication", params)
+	require.NoError(t, err)
+
+	logger.Log(t, "Creating kubernetes auth role for the server-acl-init job")
+	serverACLInitSAName := fmt.Sprintf("%s-consul-server-acl-init", consulReleaseName)
+	params = map[string]interface{}{
+		"bound_service_account_names":      serverACLInitSAName,
+		"bound_service_account_namespaces": ns,
+		"policies":                         "consul-replication-token",
+		"ttl":                              "24h",
+	}
+
+	for _, authMethodPath := range authMethodPaths {
+		_, err := vaultClient.Logical().Write(fmt.Sprintf("auth/%s/role/server-acl-init", authMethodPath), params)
+		require.NoError(t, err)
+	}
+
+	return token
+}

acceptance/tests/vault/vault_wan_fed_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/hashicorp/consul-k8s/acceptance/framework/k8s"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/vault"
+	"github.com/hashicorp/consul/api"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
@@ -115,6 +116,8 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 	primaryCertPath := configurePKICertificates(t, vaultClient, consulReleaseName, ns, "dc1")
 	secondaryCertPath := configurePKICertificates(t, vaultClient, consulReleaseName, ns, "dc2")
 
+	replicationToken := configureReplicationTokenVaultSecret(t, vaultClient, consulReleaseName, ns, "kubernetes", "kubernetes-dc2")
+
 	// Move Vault CA secret from primary to secondary so that we can mount it to pods in the
 	// secondary cluster.
 	vaultCASecretName := vault.CASecretName(vaultReleaseName)
@@ -135,7 +138,6 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 
 		// TLS config.
 		"global.tls.enabled":           "true",
-		"global.tls.httpsOnly":         "false",
 		"global.tls.enableAutoEncrypt": "true",
 		"global.tls.caCert.secretName": "pki/cert/ca",
 		"server.serverCert.secretName": primaryCertPath,
@@ -144,6 +146,12 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 		"global.gossipEncryption.secretName": "consul/data/secret/gossip",
 		"global.gossipEncryption.secretKey":  "gossip",
 
+		// ACL config.
+		"global.acls.manageSystemACLs":            "true",
+		"global.acls.createReplicationToken":      "true",
+		"global.acls.replicationToken.secretName": "consul/data/secret/replication",
+		"global.acls.replicationToken.secretKey":  "replication",
+
 		// Mesh config.
 		"connectInject.enabled": "true",
 		"controller.enabled":    "true",
@@ -156,12 +164,13 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 		"server.extraVolumes[0].load": "false",
 
 		// Vault config.
-		"global.secretsBackend.vault.enabled":          "true",
-		"global.secretsBackend.vault.consulServerRole": "consul-server",
-		"global.secretsBackend.vault.consulClientRole": "consul-client",
-		"global.secretsBackend.vault.consulCARole":     "consul-ca",
-		"global.secretsBackend.vault.ca.secretName":    vaultCASecretName,
-		"global.secretsBackend.vault.ca.secretKey":     "tls.crt",
+		"global.secretsBackend.vault.enabled":              "true",
+		"global.secretsBackend.vault.consulServerRole":     "consul-server",
+		"global.secretsBackend.vault.consulClientRole":     "consul-client",
+		"global.secretsBackend.vault.consulCARole":         "consul-ca",
+		"global.secretsBackend.vault.manageSystemACLsRole": "server-acl-init",
+		"global.secretsBackend.vault.ca.secretName":        vaultCASecretName,
+		"global.secretsBackend.vault.ca.secretKey":         "tls.crt",
 	}
 
 	if cfg.UseKind {
@@ -182,7 +191,6 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 
 		// TLS config.
 		"global.tls.enabled":           "true",
-		"global.tls.httpsOnly":         "false",
 		"global.tls.enableAutoEncrypt": "true",
 		"global.tls.caCert.secretName": "pki/cert/ca",
 		"server.serverCert.secretName": secondaryCertPath,
@@ -191,6 +199,11 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 		"global.gossipEncryption.secretName": "consul/data/secret/gossip",
 		"global.gossipEncryption.secretKey":  "gossip",
 
+		// ACL config.
+		"global.acls.manageSystemACLs":            "true",
+		"global.acls.replicationToken.secretName": "consul/data/secret/replication",
+		"global.acls.replicationToken.secretKey":  "replication",
+
 		// Mesh config.
 		"connectInject.enabled": "true",
 		"meshGateway.enabled":   "true",
@@ -203,13 +216,14 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 		"server.extraConfig":          serverExtraConfig,
 
 		// Vault config.
-		"global.secretsBackend.vault.enabled":          "true",
-		"global.secretsBackend.vault.consulServerRole": "consul-server",
-		"global.secretsBackend.vault.consulClientRole": "consul-client",
-		"global.secretsBackend.vault.consulCARole":     "consul-ca",
-		"global.secretsBackend.vault.ca.secretName":    vaultCASecretName,
-		"global.secretsBackend.vault.ca.secretKey":     "tls.crt",
-		"global.secretsBackend.vault.agentAnnotations": fmt.Sprintf("vault.hashicorp.com/tls-server-name: %s-vault", vaultReleaseName),
+		"global.secretsBackend.vault.enabled":              "true",
+		"global.secretsBackend.vault.consulServerRole":     "consul-server",
+		"global.secretsBackend.vault.consulClientRole":     "consul-client",
+		"global.secretsBackend.vault.consulCARole":         "consul-ca",
+		"global.secretsBackend.vault.manageSystemACLsRole": "server-acl-init",
+		"global.secretsBackend.vault.ca.secretName":        vaultCASecretName,
+		"global.secretsBackend.vault.ca.secretKey":         "tls.crt",
+		"global.secretsBackend.vault.agentAnnotations":     fmt.Sprintf("vault.hashicorp.com/tls-server-name: %s-vault", vaultReleaseName),
 	}
 
 	if cfg.UseKind {
@@ -223,9 +237,10 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 
 	// Verify federation between servers.
 	logger.Log(t, "verifying federation was successful")
-	primaryClient := primaryConsulCluster.SetupConsulClient(t, false)
-	secondaryClient := secondaryConsulCluster.SetupConsulClient(t, false)
-	helpers.VerifyFederation(t, primaryClient, secondaryClient, consulReleaseName, false)
+	primaryClient := primaryConsulCluster.SetupConsulClient(t, true)
+	secondaryConsulCluster.ACLToken = replicationToken
+	secondaryClient := secondaryConsulCluster.SetupConsulClient(t, true)
+	helpers.VerifyFederation(t, primaryClient, secondaryClient, consulReleaseName, true)
 
 	// Create a ProxyDefaults resource to configure services to use the mesh
 	// gateways.
@@ -243,6 +258,19 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 	logger.Log(t, "creating static-client in dc1")
 	k8s.DeployKustomize(t, primaryCtx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-multi-dc")
 
+	logger.Log(t, "creating intention")
+	_, _, err = primaryClient.ConfigEntries().Set(&api.ServiceIntentionsConfigEntry{
+		Kind: api.ServiceIntentions,
+		Name: "static-server",
+		Sources: []*api.SourceIntention{
+			{
+				Name:   "static-client",
+				Action: api.IntentionActionAllow,
+			},
+		},
+	}, nil)
+	require.NoError(t, err)
+
 	logger.Log(t, "checking that connection is successful")
 	k8s.CheckStaticServerConnectionSuccessful(t, primaryCtx.KubectlOptions(t), "http://localhost:1234")
 }

charts/consul/templates/_helpers.tpl

@@ -59,6 +59,20 @@ as well as the global.name setting.
 {{- if .Values.global.tls -}}{{- if .Values.global.tls.serverAdditionalIPSANs -}}{{- range $ipsan := .Values.global.tls.serverAdditionalIPSANs }},{{ $ipsan }} {{- end -}}{{- end -}}{{- end -}}
 {{- end -}}
 
+{{- define "consul.vaultReplicationTokenTemplate" -}}
+|
+          {{ "{{" }}- with secret "{{ .Values.global.acls.replicationToken.secretName }}" -{{ "}}" }}
+          {{ "{{" }}- {{ printf ".Data.data.%s" .Values.global.acls.replicationToken.secretKey }} -{{ "}}" }}
+          {{ "{{" }}- end -{{ "}}" }}
+{{- end -}}
+
+{{- define "consul.vaultReplicationTokenConfigTemplate" -}}
+|
+          {{ "{{" }}- with secret "{{ .Values.global.acls.replicationToken.secretName }}" -{{ "}}" }}
+          acl { tokens { agent = "{{ "{{" }}- {{ printf ".Data.data.%s" .Values.global.acls.replicationToken.secretKey }} -{{ "}}" }}", replication = "{{ "{{" }}- {{ printf ".Data.data.%s" .Values.global.acls.replicationToken.secretKey }} -{{ "}}" }}" }}
+          {{ "{{" }}- end -{{ "}}" }}
+{{- end -}}
+
 {{/*
 Sets up the extra-from-values config file passed to consul and then uses sed to do any necessary
 substitution for HOST_IP/POD_IP/HOSTNAME. Useful for dogstats telemetry. The output file