-
Notifications
You must be signed in to change notification settings - Fork 316
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add support for ACLs for sync catalog and partitions. #1180
Merged
+584
−71
Merged
Changes from all commits
Commits
Show all changes
2 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,297 @@ | ||
| package partitions | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "strconv" | ||
| "testing" | ||
| "time" | ||
|
|
||
| terratestk8s "github.com/gruntwork-io/terratest/modules/k8s" | ||
| "github.com/hashicorp/consul-k8s/acceptance/framework/consul" | ||
| "github.com/hashicorp/consul-k8s/acceptance/framework/environment" | ||
| "github.com/hashicorp/consul-k8s/acceptance/framework/helpers" | ||
| "github.com/hashicorp/consul-k8s/acceptance/framework/k8s" | ||
| "github.com/hashicorp/consul-k8s/acceptance/framework/logger" | ||
| "github.com/hashicorp/consul/api" | ||
| "github.com/hashicorp/consul/sdk/testutil/retry" | ||
| "github.com/stretchr/testify/require" | ||
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| ) | ||
|
|
||
| // Test that Sync Catalog works in a default and ACLsAndAutoEncryptEnabled installations for partitions. | ||
| func TestPartitions_Sync(t *testing.T) { | ||
| env := suite.Environment() | ||
| cfg := suite.Config() | ||
|
|
||
| if !cfg.EnableEnterprise { | ||
| t.Skipf("skipping this test because -enable-enterprise is not set") | ||
| } | ||
|
|
||
| const defaultPartition = "default" | ||
| const secondaryPartition = "secondary" | ||
| const defaultNamespace = "default" | ||
| cases := []struct { | ||
| name string | ||
| destinationNamespace string | ||
| mirrorK8S bool | ||
| ACLsAndAutoEncryptEnabled bool | ||
| }{ | ||
| { | ||
| "default destination namespace", | ||
| defaultNamespace, | ||
| false, | ||
| false, | ||
| }, | ||
| { | ||
| "default destination namespace; ACLs and auto-encrypt enabled", | ||
| defaultNamespace, | ||
| false, | ||
| true, | ||
| }, | ||
| { | ||
| "single destination namespace", | ||
| staticServerNamespace, | ||
| false, | ||
| false, | ||
| }, | ||
| { | ||
| "single destination namespace; ACLs and auto-encrypt enabled", | ||
| staticServerNamespace, | ||
| false, | ||
| true, | ||
| }, | ||
| { | ||
| "mirror k8s namespaces", | ||
| staticServerNamespace, | ||
| true, | ||
| false, | ||
| }, | ||
| { | ||
| "mirror k8s namespaces; ACLs and auto-encrypt enabled", | ||
| staticServerNamespace, | ||
| true, | ||
| true, | ||
| }, | ||
| } | ||
|
|
||
| for _, c := range cases { | ||
| t.Run(c.name, func(t *testing.T) { | ||
| primaryClusterContext := env.DefaultContext(t) | ||
| secondaryClusterContext := env.Context(t, environment.SecondaryContextName) | ||
|
|
||
| ctx := context.Background() | ||
|
|
||
| commonHelmValues := map[string]string{ | ||
| "global.adminPartitions.enabled": "true", | ||
|
|
||
| "global.enableConsulNamespaces": "true", | ||
|
|
||
| "global.tls.enabled": "true", | ||
| "global.tls.httpsOnly": strconv.FormatBool(c.ACLsAndAutoEncryptEnabled), | ||
| "global.tls.enableAutoEncrypt": strconv.FormatBool(c.ACLsAndAutoEncryptEnabled), | ||
|
|
||
| "global.acls.manageSystemACLs": strconv.FormatBool(c.ACLsAndAutoEncryptEnabled), | ||
|
|
||
| "syncCatalog.enabled": "true", | ||
| // When mirroringK8S is set, this setting is ignored. | ||
| "syncCatalog.consulNamespaces.consulDestinationNamespace": c.destinationNamespace, | ||
| "syncCatalog.consulNamespaces.mirroringK8S": strconv.FormatBool(c.mirrorK8S), | ||
| "syncCatalog.addK8SNamespaceSuffix": "false", | ||
|
|
||
| "controller.enabled": "true", | ||
|
|
||
| "dns.enabled": "true", | ||
| "dns.enableRedirection": strconv.FormatBool(cfg.EnableTransparentProxy), | ||
| } | ||
|
|
||
| serverHelmValues := map[string]string{ | ||
| "server.exposeGossipAndRPCPorts": "true", | ||
| } | ||
|
|
||
| // On Kind, there are no load balancers but since all clusters | ||
| // share the same node network (docker bridge), we can use | ||
| // a NodePort service so that we can access node(s) in a different Kind cluster. | ||
| if cfg.UseKind { | ||
| serverHelmValues["global.adminPartitions.service.type"] = "NodePort" | ||
| serverHelmValues["global.adminPartitions.service.nodePort.https"] = "30000" | ||
| } | ||
|
|
||
| releaseName := helpers.RandomName() | ||
|
|
||
| helpers.MergeMaps(serverHelmValues, commonHelmValues) | ||
|
|
||
| // Install the consul cluster with servers in the default kubernetes context. | ||
| primaryConsulCluster := consul.NewHelmCluster(t, serverHelmValues, primaryClusterContext, cfg, releaseName) | ||
| primaryConsulCluster.Create(t) | ||
|
|
||
| // Get the TLS CA certificate and key secret from the server cluster and apply it to the client cluster. | ||
| caCertSecretName := fmt.Sprintf("%s-consul-ca-cert", releaseName) | ||
| caKeySecretName := fmt.Sprintf("%s-consul-ca-key", releaseName) | ||
|
|
||
| logger.Logf(t, "retrieving ca cert secret %s from the server cluster and applying to the client cluster", caCertSecretName) | ||
| copySecret(t, primaryClusterContext, secondaryClusterContext, caCertSecretName) | ||
|
|
||
| if !c.ACLsAndAutoEncryptEnabled { | ||
| // When auto-encrypt is disabled, we need both | ||
| // the CA cert and CA key to be available in the clients cluster to generate client certificates and keys. | ||
| logger.Logf(t, "retrieving ca key secret %s from the server cluster and applying to the client cluster", caKeySecretName) | ||
| copySecret(t, primaryClusterContext, secondaryClusterContext, caKeySecretName) | ||
| } | ||
|
|
||
| partitionToken := fmt.Sprintf("%s-consul-partitions-acl-token", releaseName) | ||
| if c.ACLsAndAutoEncryptEnabled { | ||
| logger.Logf(t, "retrieving partition token secret %s from the server cluster and applying to the client cluster", partitionToken) | ||
| copySecret(t, primaryClusterContext, secondaryClusterContext, partitionToken) | ||
| } | ||
|
|
||
| partitionServiceName := fmt.Sprintf("%s-consul-partition", releaseName) | ||
| partitionSvcAddress := k8s.ServiceHost(t, cfg, primaryClusterContext, partitionServiceName) | ||
|
|
||
| k8sAuthMethodHost := k8s.KubernetesAPIServerHost(t, cfg, secondaryClusterContext) | ||
|
|
||
| // Create client cluster. | ||
| clientHelmValues := map[string]string{ | ||
| "global.enabled": "false", | ||
|
|
||
| "global.adminPartitions.name": secondaryPartition, | ||
|
|
||
| "global.tls.caCert.secretName": caCertSecretName, | ||
| "global.tls.caCert.secretKey": "tls.crt", | ||
|
|
||
| "externalServers.enabled": "true", | ||
| "externalServers.hosts[0]": partitionSvcAddress, | ||
| "externalServers.tlsServerName": "server.dc1.consul", | ||
|
|
||
| "client.enabled": "true", | ||
| "client.exposeGossipPorts": "true", | ||
| "client.join[0]": partitionSvcAddress, | ||
| } | ||
|
|
||
| if c.ACLsAndAutoEncryptEnabled { | ||
| // Setup partition token and auth method host if ACLs enabled. | ||
| clientHelmValues["global.acls.bootstrapToken.secretName"] = partitionToken | ||
| clientHelmValues["global.acls.bootstrapToken.secretKey"] = "token" | ||
| clientHelmValues["externalServers.k8sAuthMethodHost"] = k8sAuthMethodHost | ||
| } else { | ||
| // Provide CA key when auto-encrypt is disabled. | ||
| clientHelmValues["global.tls.caKey.secretName"] = caKeySecretName | ||
| clientHelmValues["global.tls.caKey.secretKey"] = "tls.key" | ||
| } | ||
|
|
||
| if cfg.UseKind { | ||
| clientHelmValues["externalServers.httpsPort"] = "30000" | ||
| } | ||
|
|
||
| helpers.MergeMaps(clientHelmValues, commonHelmValues) | ||
|
|
||
| // Install the consul cluster without servers in the client cluster kubernetes context. | ||
| secondaryConsulCluster := consul.NewHelmCluster(t, clientHelmValues, secondaryClusterContext, cfg, releaseName) | ||
| secondaryConsulCluster.Create(t) | ||
|
|
||
| // Ensure consul clients are created. | ||
| agentPodList, err := secondaryClusterContext.KubernetesClient(t).CoreV1().Pods(secondaryClusterContext.KubectlOptions(t).Namespace).List(ctx, metav1.ListOptions{LabelSelector: "app=consul,component=client"}) | ||
| require.NoError(t, err) | ||
| require.NotEmpty(t, agentPodList.Items) | ||
|
|
||
| output, err := k8s.RunKubectlAndGetOutputE(t, secondaryClusterContext.KubectlOptions(t), "logs", agentPodList.Items[0].Name, "-n", secondaryClusterContext.KubectlOptions(t).Namespace) | ||
| require.NoError(t, err) | ||
| require.Contains(t, output, "Partition: 'secondary'") | ||
|
|
||
| primaryStaticServerOpts := &terratestk8s.KubectlOptions{ | ||
| ContextName: primaryClusterContext.KubectlOptions(t).ContextName, | ||
| ConfigPath: primaryClusterContext.KubectlOptions(t).ConfigPath, | ||
| Namespace: staticServerNamespace, | ||
| } | ||
| secondaryStaticServerOpts := &terratestk8s.KubectlOptions{ | ||
| ContextName: secondaryClusterContext.KubectlOptions(t).ContextName, | ||
| ConfigPath: secondaryClusterContext.KubectlOptions(t).ConfigPath, | ||
| Namespace: staticServerNamespace, | ||
| } | ||
|
|
||
| logger.Logf(t, "creating namespaces %s in servers cluster", staticServerNamespace) | ||
| k8s.RunKubectl(t, primaryClusterContext.KubectlOptions(t), "create", "ns", staticServerNamespace) | ||
| helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { | ||
| k8s.RunKubectl(t, primaryClusterContext.KubectlOptions(t), "delete", "ns", staticServerNamespace) | ||
| }) | ||
|
|
||
| logger.Logf(t, "creating namespaces %s in clients cluster", staticServerNamespace) | ||
| k8s.RunKubectl(t, secondaryClusterContext.KubectlOptions(t), "create", "ns", staticServerNamespace) | ||
| helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { | ||
| k8s.RunKubectl(t, secondaryClusterContext.KubectlOptions(t), "delete", "ns", staticServerNamespace) | ||
| }) | ||
|
|
||
| consulClient, _ := primaryConsulCluster.SetupConsulClient(t, c.ACLsAndAutoEncryptEnabled) | ||
|
|
||
| defaultPartitionQueryOpts := &api.QueryOptions{Namespace: staticServerNamespace, Partition: defaultPartition} | ||
| secondaryPartitionQueryOpts := &api.QueryOptions{Namespace: staticServerNamespace, Partition: secondaryPartition} | ||
|
|
||
| if !c.mirrorK8S { | ||
| defaultPartitionQueryOpts = &api.QueryOptions{Namespace: c.destinationNamespace, Partition: defaultPartition} | ||
| secondaryPartitionQueryOpts = &api.QueryOptions{Namespace: c.destinationNamespace, Partition: secondaryPartition} | ||
| } | ||
|
|
||
| // Check that the ACL token is deleted. | ||
| if c.ACLsAndAutoEncryptEnabled { | ||
| // We need to register the cleanup function before we create the deployments | ||
| // because golang will execute them in reverse order i.e. the last registered | ||
| // cleanup function will be executed first. | ||
| t.Cleanup(func() { | ||
| if c.ACLsAndAutoEncryptEnabled { | ||
| retry.Run(t, func(r *retry.R) { | ||
| tokens, _, err := consulClient.ACL().TokenList(defaultPartitionQueryOpts) | ||
| require.NoError(r, err) | ||
| for _, token := range tokens { | ||
| require.NotContains(r, token.Description, staticServerName) | ||
| } | ||
|
|
||
| tokens, _, err = consulClient.ACL().TokenList(secondaryPartitionQueryOpts) | ||
| require.NoError(r, err) | ||
| for _, token := range tokens { | ||
| require.NotContains(r, token.Description, staticServerName) | ||
| } | ||
| }) | ||
| } | ||
| }) | ||
| } | ||
|
|
||
| logger.Log(t, "creating a static-server with a service") | ||
| // create service in default partition. | ||
| k8s.DeployKustomize(t, primaryStaticServerOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/bases/static-server") | ||
| // create service in secondary partition. | ||
| k8s.DeployKustomize(t, secondaryStaticServerOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/bases/static-server") | ||
|
|
||
| logger.Log(t, "checking that the service has been synced to Consul") | ||
| var services map[string][]string | ||
| counter := &retry.Counter{Count: 20, Wait: 30 * time.Second} | ||
| retry.RunWith(counter, t, func(r *retry.R) { | ||
| var err error | ||
| // list services in default partition catalog. | ||
| services, _, err = consulClient.Catalog().Services(defaultPartitionQueryOpts) | ||
| require.NoError(r, err) | ||
| require.Contains(r, services, staticServerName) | ||
| if _, ok := services[staticServerName]; !ok { | ||
| r.Errorf("service '%s' is not in Consul's list of services %s in the default partition", staticServerName, services) | ||
| } | ||
| // list services in secondary partition catalog. | ||
| services, _, err = consulClient.Catalog().Services(secondaryPartitionQueryOpts) | ||
| require.NoError(r, err) | ||
| require.Contains(r, services, staticServerName) | ||
| if _, ok := services[staticServerName]; !ok { | ||
| r.Errorf("service '%s' is not in Consul's list of services %s in the secondary partition", staticServerName, services) | ||
| } | ||
| }) | ||
| // validate service in the default partition. | ||
| service, _, err := consulClient.Catalog().Service(staticServerName, "", defaultPartitionQueryOpts) | ||
| require.NoError(t, err) | ||
| require.Equal(t, 1, len(service)) | ||
| require.Equal(t, []string{"k8s"}, service[0].ServiceTags) | ||
| // validate service in the secondary partition. | ||
| service, _, err = consulClient.Catalog().Service(staticServerName, "", secondaryPartitionQueryOpts) | ||
| require.NoError(t, err) | ||
| require.Equal(t, 1, len(service)) | ||
| require.Equal(t, []string{"k8s"}, service[0].ServiceTags) | ||
|
|
||
| }) | ||
| } | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is a great clarifying comment. 👍🏻