From f9bb04b504c27d924d751a53bb65007cee492dcd Mon Sep 17 00:00:00 2001 From: Iryna Shustava Date: Thu, 1 Oct 2020 13:37:57 -0700 Subject: [PATCH] Add support for OpenShift --- .circleci/config.yml | 72 +++++++++++++- templates/client-role.yaml | 10 +- .../client-securitycontextconstraints.yaml | 54 ++++++++++ templates/create-federation-secret-job.yaml | 4 +- templates/ingress-gateways-deployment.yaml | 4 +- templates/server-acl-init-cleanup-job.yaml | 4 +- templates/server-acl-init-job.yaml | 4 +- templates/server-statefulset.yaml | 3 +- .../terminating-gateways-deployment.yaml | 4 +- templates/tls-init-cleanup-job.yaml | 4 +- templates/tls-init-job.yaml | 5 +- test/acceptance/framework/config.go | 6 ++ test/acceptance/framework/config_test.go | 9 ++ test/acceptance/framework/flags.go | 7 ++ test/acceptance/helpers/helpers.go | 6 +- test/acceptance/tests/basic/basic_test.go | 33 +++---- .../tests/consul-dns/consul_dns_test.go | 44 +++++---- .../terminating_gateway_namespaces_test.go | 2 +- .../terminating_gateway_test.go | 13 ++- test/docker/Test.dockerfile | 9 ++ test/unit/client-role.bats | 73 +++++++++++++- .../client-securitycontextconstraints.bats | 98 +++++++++++++++++++ test/unit/server-statefulset.bats | 16 ++- values.yaml | 11 ++- 24 files changed, 418 insertions(+), 77 deletions(-) create mode 100644 templates/client-securitycontextconstraints.yaml create mode 100644 test/unit/client-securitycontextconstraints.bats diff --git a/.circleci/config.yml b/.circleci/config.yml index e1dc45a43..514385462 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,7 +22,7 @@ jobs: command: bats ./test/unit unit-helm3: docker: - - image: hashicorpdev/consul-helm-test:0.5.0 + - image: hashicorpdev/consul-helm-test:0.6.0 steps: - checkout @@ -153,6 +153,58 @@ jobs: terraform destroy -var project=${CLOUDSDK_CORE_PROJECT} -auto-approve when: always + acceptance-openshift: + environment: + - TEST_RESULTS: /tmp/test-results + - OC_PRIMARY_NAME: consul-helm-test-2757871175 + - OC_SECONDARY_NAME: consul-helm-test-3737660519 + docker: + # This image is build from test/docker/Test.dockerfile + - image: hashicorpdev/consul-helm-test:0.6.0 + + steps: + - checkout + + - run: + name: openshift login + command: | + az login --service-principal -u "$ARM_CLIENT_ID" -p "$ARM_CLIENT_SECRET" --tenant "$ARM_TENANT_ID" > /dev/null + + for cluster_name in "$OC_PRIMARY_NAME" "$OC_SECONDARY_NAME"; do + apiServer=$(az aro show -g "$cluster_name" -n "$cluster_name" --query apiserverProfile.url -o tsv) + kubeUser=$(az aro list-credentials -g "$cluster_name" -n "$cluster_name" | jq -r .kubeadminUsername) + kubePassword=$(az aro list-credentials -g "$cluster_name" -n "$cluster_name" | jq -r .kubeadminPassword) + + KUBECONFIG="$HOME/.kube/$cluster_name" oc login "$apiServer" -u "$kubeUser" -p "$kubePassword" + KUBECONFIG="$HOME/.kube/$cluster_name" oc project consul + done + + # Restore go module cache if there is one + - restore_cache: + keys: + - consul-helm-modcache-v1-{{ checksum "test/acceptance/go.mod" }} + + - run: mkdir -p $TEST_RESULTS + + - run: + name: Run acceptance tests + working_directory: test/acceptance/tests + no_output_timeout: 30m + command: | + gotestsum --junitfile "$TEST_RESULTS/gotestsum-report.xml" -- ./... -p 1 -timeout 30m -failfast \ + -enable-openshift \ + -enable-enterprise \ + -enable-multi-cluster \ + -kubeconfig="$HOME/.kube/$OC_PRIMARY_NAME" \ + -secondary-kubeconfig="$HOME/.kube/$OC_SECONDARY_NAME" \ + -debug-directory="$TEST_RESULTS/debug" \ + -consul-k8s-image=hashicorpdev/consul-k8s:latest + + - store_test_results: + path: /tmp/test-results + - store_artifacts: + path: /tmp/test-results + update-helm-charts-index: docker: - image: circleci/golang:latest @@ -191,11 +243,27 @@ workflows: - go-fmt-and-vet - unit-helm2 - unit-helm3 - - acceptance: +# - acceptance: +# requires: +# - unit-helm2 +# - unit-helm3 +# - unit-acceptance-framework + - acceptance-openshift: requires: - unit-helm2 - unit-helm3 - unit-acceptance-framework +# todo: uncomment before merging the PR +# nightly-acceptance-tests: +# triggers: +# - schedule: +# cron: "0 0 * * *" +# filters: +# branches: +# only: +# - master +# jobs: +# - acceptance-openshift update-helm-charts-index: jobs: - update-helm-charts-index: diff --git a/templates/client-role.yaml b/templates/client-role.yaml index 5a9f90ef2..8295a5d1f 100644 --- a/templates/client-role.yaml +++ b/templates/client-role.yaml @@ -9,7 +9,7 @@ metadata: chart: {{ template "consul.chart" . }} heritage: {{ .Release.Service }} release: {{ .Release.Name }} -{{- if (or .Values.global.acls.manageSystemACLs .Values.global.enablePodSecurityPolicies) }} +{{- if (or .Values.global.acls.manageSystemACLs .Values.global.enablePodSecurityPolicies .Values.global.openshift.enabled) }} rules: {{- if .Values.global.enablePodSecurityPolicies }} - apiGroups: ["policy"] @@ -28,6 +28,14 @@ rules: verbs: - get {{- end }} +{{- if .Values.global.openshift.enabled}} + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + resourceNames: + - {{ template "consul.fullname" . }}-client + verbs: + - use +{{- end}} {{- else}} rules: [] {{- end }} diff --git a/templates/client-securitycontextconstraints.yaml b/templates/client-securitycontextconstraints.yaml new file mode 100644 index 000000000..08bdd2bf4 --- /dev/null +++ b/templates/client-securitycontextconstraints.yaml @@ -0,0 +1,54 @@ +{{- if (and .Values.global.openshift.enabled (or (and (ne (.Values.client.enabled | toString) "-") .Values.client.enabled) (and (eq (.Values.client.enabled | toString) "-") .Values.global.enabled))) }} +apiVersion: security.openshift.io/v1 +kind: SecurityContextConstraints +metadata: + name: {{ template "consul.fullname" . }}-client + labels: + app: {{ template "consul.name" . }} + chart: {{ template "consul.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + annotations: + kubernetes.io/description: {{ template "consul.fullname" . }}-client are the security context constraints required + to run the consul client. +{{- if .Values.client.dataDirectoryHostPath }} +allowHostDirVolumePlugin: true +{{- else }} +allowHostDirVolumePlugin: false +{{- end}} +allowHostIPC: false +allowHostNetwork: {{ .Values.client.hostNetwork }} +allowHostPID: false +allowHostPorts: true +allowPrivilegeEscalation: true +allowPrivilegedContainer: false +allowedCapabilities: null +defaultAddCapabilities: null +fsGroup: + type: MustRunAs +groups: [] +priority: null +readOnlyRootFilesystem: false +requiredDropCapabilities: +- KILL +- MKNOD +- SETUID +- SETGID +runAsUser: + type: MustRunAsRange +seLinuxContext: + type: MustRunAs +supplementalGroups: + type: MustRunAs +users: [] +volumes: +- configMap +- downwardAPI +- emptyDir +- persistentVolumeClaim +- projected +- secret +{{- if .Values.client.dataDirectoryHostPath }} +- hostPath +{{- end }} +{{- end}} \ No newline at end of file diff --git a/templates/create-federation-secret-job.yaml b/templates/create-federation-secret-job.yaml index b16d54c11..5b0af8fa5 100644 --- a/templates/create-federation-secret-job.yaml +++ b/templates/create-federation-secret-job.yaml @@ -129,9 +129,9 @@ spec: -server-ca-key-file=/consul/tls/server/ca/tls.key resources: requests: - memory: "25Mi" + memory: "50Mi" cpu: "50m" limits: - memory: "25Mi" + memory: "50Mi" cpu: "50m" {{- end }} diff --git a/templates/ingress-gateways-deployment.yaml b/templates/ingress-gateways-deployment.yaml index 7f74782a7..cb4fbfa4a 100644 --- a/templates/ingress-gateways-deployment.yaml +++ b/templates/ingress-gateways-deployment.yaml @@ -264,10 +264,10 @@ spec: {{- end }} resources: requests: - memory: "25Mi" + memory: "50Mi" cpu: "50m" limits: - memory: "25Mi" + memory: "50Mi" cpu: "50m" containers: - name: ingress-gateway diff --git a/templates/server-acl-init-cleanup-job.yaml b/templates/server-acl-init-cleanup-job.yaml index 6d8f33bfc..4f13e4412 100644 --- a/templates/server-acl-init-cleanup-job.yaml +++ b/templates/server-acl-init-cleanup-job.yaml @@ -54,10 +54,10 @@ spec: - {{ template "consul.fullname" . }}-server-acl-init resources: requests: - memory: "25Mi" + memory: "50Mi" cpu: "50m" limits: - memory: "25Mi" + memory: "50Mi" cpu: "50m" {{- end }} {{- end }} diff --git a/templates/server-acl-init-job.yaml b/templates/server-acl-init-job.yaml index e5980a33c..d70e1a70b 100644 --- a/templates/server-acl-init-job.yaml +++ b/templates/server-acl-init-job.yaml @@ -238,10 +238,10 @@ spec: {{- end }} resources: requests: - memory: "25Mi" + memory: "50Mi" cpu: "50m" limits: - memory: "25Mi" + memory: "50Mi" cpu: "50m" {{- end }} {{- end }} diff --git a/templates/server-statefulset.yaml b/templates/server-statefulset.yaml index 036f7e9c3..7ff5f2235 100644 --- a/templates/server-statefulset.yaml +++ b/templates/server-statefulset.yaml @@ -1,6 +1,7 @@ {{- if (or (and (ne (.Values.server.enabled | toString) "-") .Values.server.enabled) (and (eq (.Values.server.enabled | toString) "-") .Values.global.enabled)) }} {{- if and .Values.global.federation.enabled (not .Values.global.tls.enabled) }}{{ fail "If global.federation.enabled is true, global.tls.enabled must be true because federation is only supported with TLS enabled" }}{{ end }} {{- if and .Values.global.federation.enabled (not .Values.meshGateway.enabled) }}{{ fail "If global.federation.enabled is true, meshGateway.enabled must be true because mesh gateways are required for federation" }}{{ end }} +{{- if .Values.server.disableFsGroupSecurityContext }}{{ fail "server.disableFsGroupSecurityContext has been removed. Please use global.openshift.enabled instead." }}{{ end }} # StatefulSet to run the actual Consul server cluster. apiVersion: apps/v1 kind: StatefulSet @@ -58,7 +59,7 @@ spec: {{- end }} terminationGracePeriodSeconds: 30 serviceAccountName: {{ template "consul.fullname" . }}-server - {{- if not .Values.server.disableFsGroupSecurityContext }} + {{- if not .Values.global.openshift.enabled}} securityContext: fsGroup: 1000 {{- end }} diff --git a/templates/terminating-gateways-deployment.yaml b/templates/terminating-gateways-deployment.yaml index 52ebd6cba..ec32041c4 100644 --- a/templates/terminating-gateways-deployment.yaml +++ b/templates/terminating-gateways-deployment.yaml @@ -211,10 +211,10 @@ spec: {{- end }} resources: requests: - memory: "25Mi" + memory: "50Mi" cpu: "50m" limits: - memory: "25Mi" + memory: "50Mi" cpu: "50m" containers: - name: terminating-gateway diff --git a/templates/tls-init-cleanup-job.yaml b/templates/tls-init-cleanup-job.yaml index 396df0263..dc0d3aaea 100644 --- a/templates/tls-init-cleanup-job.yaml +++ b/templates/tls-init-cleanup-job.yaml @@ -54,10 +54,10 @@ spec: -H "Authorization: Bearer $( cat /var/run/secrets/kubernetes.io/serviceaccount/token )" resources: requests: - memory: "25Mi" + memory: "50Mi" cpu: "50m" limits: - memory: "25Mi" + memory: "50Mi" cpu: "50m" {{- end }} {{- end }} diff --git a/templates/tls-init-job.yaml b/templates/tls-init-job.yaml index 0f83643e0..7e55c83dd 100644 --- a/templates/tls-init-job.yaml +++ b/templates/tls-init-job.yaml @@ -56,6 +56,7 @@ spec: # Note that in the subsequent runs of the job, POST requests will # return a 409 because these secrets would already exist; # we are ignoring these response codes. + workingDir: /tmp command: - "/bin/sh" - "-ec" @@ -116,10 +117,10 @@ spec: {{- end }} resources: requests: - memory: "25Mi" + memory: "50Mi" cpu: "50m" limits: - memory: "25Mi" + memory: "50Mi" cpu: "50m" {{- end }} {{- end }} diff --git a/test/acceptance/framework/config.go b/test/acceptance/framework/config.go index 2ae717618..97e1743ff 100644 --- a/test/acceptance/framework/config.go +++ b/test/acceptance/framework/config.go @@ -23,6 +23,8 @@ type TestConfig struct { EnterpriseLicenseSecretName string EnterpriseLicenseSecretKey string + EnableOpenshift bool + ConsulImage string ConsulK8SImage string @@ -52,6 +54,10 @@ func (t *TestConfig) HelmValuesFromConfig() (map[string]string, error) { setIfNotEmpty(helmValues, "server.enterpriseLicense.secretKey", t.EnterpriseLicenseSecretKey) } + if t.EnableOpenshift { + setIfNotEmpty(helmValues, "global.openshift.enabled", "true") + } + setIfNotEmpty(helmValues, "global.image", t.ConsulImage) setIfNotEmpty(helmValues, "global.imageK8S", t.ConsulK8SImage) diff --git a/test/acceptance/framework/config_test.go b/test/acceptance/framework/config_test.go index 463fc4d81..3445cab60 100644 --- a/test/acceptance/framework/config_test.go +++ b/test/acceptance/framework/config_test.go @@ -67,6 +67,15 @@ func TestConfig_HelmValuesFromConfig(t *testing.T) { }, map[string]string{}, }, + { + "sets openshift value when EnableOpenshift is set", + TestConfig{ + EnableOpenshift: true, + }, + map[string]string{ + "global.openshift.enabled": "true", + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/test/acceptance/framework/flags.go b/test/acceptance/framework/flags.go index 3750a7a94..cb7fc4d94 100644 --- a/test/acceptance/framework/flags.go +++ b/test/acceptance/framework/flags.go @@ -20,6 +20,8 @@ type TestFlags struct { flagEnterpriseLicenseSecretName string flagEnterpriseLicenseSecretKey string + flagEnableOpenshift bool + flagConsulImage string flagConsulK8sImage string @@ -64,6 +66,9 @@ func (t *TestFlags) init() { flag.StringVar(&t.flagEnterpriseLicenseSecretKey, "enterprise-license-secret-key", "", "The key of the Kubernetes secret containing the enterprise license.") + flag.BoolVar(&t.flagEnableOpenshift, "enable-openshift", false, + "If true, the tests will automatically add Openshift Helm value for each Helm install.") + flag.BoolVar(&t.flagNoCleanupOnFailure, "no-cleanup-on-failure", false, "If true, the tests will not cleanup Kubernetes resources they create when they finish running."+ "Note this flag must be run with -failfast flag, otherwise subsequent tests will fail.") @@ -105,6 +110,8 @@ func (t *TestFlags) testConfigFromFlags() *TestConfig { EnterpriseLicenseSecretName: t.flagEnterpriseLicenseSecretName, EnterpriseLicenseSecretKey: t.flagEnterpriseLicenseSecretKey, + EnableOpenshift: t.flagEnableOpenshift, + ConsulImage: t.flagConsulImage, ConsulK8SImage: t.flagConsulK8sImage, diff --git a/test/acceptance/helpers/helpers.go b/test/acceptance/helpers/helpers.go index 9d8d5d2e0..020fdf3f7 100644 --- a/test/acceptance/helpers/helpers.go +++ b/test/acceptance/helpers/helpers.go @@ -38,8 +38,8 @@ func WaitForAllPodsToBeReady(t *testing.T, client kubernetes.Interface, namespac t.Log("Waiting for pods to be ready.") - // Wait up to 3m. - counter := &retry.Counter{Count: 36, Wait: 5 * time.Second} + // Wait up to 5m. + counter := &retry.Counter{Count: 60, Wait: 5 * time.Second} retry.RunWith(counter, t, func(r *retry.R) { pods, err := client.CoreV1().Pods(namespace).List(context.Background(), metav1.ListOptions{LabelSelector: podLabelSelector}) require.NoError(r, err) @@ -105,7 +105,7 @@ func DeployKustomize(t *testing.T, options *k8s.KubectlOptions, noCleanupOnFailu KubectlDeleteK(t, options, kustomizeDir) }) - RunKubectl(t, options, "wait", "--for=condition=available", fmt.Sprintf("deploy/%s", deployment.Name)) + RunKubectl(t, options, "wait", "--for=condition=available", "--timeout=1m", fmt.Sprintf("deploy/%s", deployment.Name)) } // CheckStaticServerConnection execs into a pod of the deployment given by deploymentName diff --git a/test/acceptance/tests/basic/basic_test.go b/test/acceptance/tests/basic/basic_test.go index e2c9b10bf..5ced61f70 100644 --- a/test/acceptance/tests/basic/basic_test.go +++ b/test/acceptance/tests/basic/basic_test.go @@ -1,6 +1,8 @@ package basic import ( + "fmt" + "strconv" "testing" "github.com/hashicorp/consul-helm/test/acceptance/framework" @@ -14,38 +16,33 @@ import ( // and subsequently reading it from Consul. func TestBasicInstallation(t *testing.T) { cases := []struct { - name string - helmValues map[string]string - secure bool + secure bool + autoEncrypt bool }{ { - "Default installation", - nil, + false, false, }, { - "Secure installation (with TLS and ACLs enabled)", - map[string]string{ - "global.tls.enabled": "true", - "global.acls.manageSystemACLs": "true", - }, true, + false, }, { - "Secure installation (with TLS with auto-encrypt and ACLs enabled)", - map[string]string{ - "global.tls.enabled": "true", - "global.tls.enableAutoEncrypt": "true", - "global.acls.manageSystemACLs": "true", - }, + true, true, }, } for _, c := range cases { - t.Run(c.name, func(t *testing.T) { + name := fmt.Sprintf("secure: %t, auto-encrypt: %t", c.secure, c.autoEncrypt) + t.Run(name, func(t *testing.T) { releaseName := helpers.RandomName() - consulCluster := framework.NewHelmCluster(t, c.helmValues, suite.Environment().DefaultContext(t), suite.Config(), releaseName) + helmValues := map[string]string{ + "global.acls.manageSystemACLs": strconv.FormatBool(c.secure), + "global.tls.enabled": strconv.FormatBool(c.secure), + "global.tls.enableAutoEncrypt": strconv.FormatBool(c.autoEncrypt), + } + consulCluster := framework.NewHelmCluster(t, helmValues, suite.Environment().DefaultContext(t), suite.Config(), releaseName) consulCluster.Create(t) diff --git a/test/acceptance/tests/consul-dns/consul_dns_test.go b/test/acceptance/tests/consul-dns/consul_dns_test.go index f211eb6f5..26db11517 100644 --- a/test/acceptance/tests/consul-dns/consul_dns_test.go +++ b/test/acceptance/tests/consul-dns/consul_dns_test.go @@ -7,6 +7,7 @@ import ( "github.com/hashicorp/consul-helm/test/acceptance/framework" "github.com/hashicorp/consul-helm/test/acceptance/helpers" + "github.com/hashicorp/consul/sdk/testutil/retry" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -59,10 +60,8 @@ func TestConsulDNS(t *testing.T) { } dnsTestPodArgs := []string{ - "run", "-it", podName, "--restart", "Never", "--image", "anubhavmishra/tiny-tools", "--", "dig", fmt.Sprintf("@%s-consul-dns", releaseName), "consul.service.consul", + "run", "-i", podName, "--restart", "Never", "--image", "anubhavmishra/tiny-tools", "--", "dig", fmt.Sprintf("@%s-consul-dns", releaseName), "consul.service.consul", } - logs, err := helpers.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), dnsTestPodArgs...) - require.NoError(t, err) helpers.Cleanup(t, suite.Config().NoCleanupOnFailure, func() { // Note: this delete command won't wait for pods to be fully terminated. @@ -72,23 +71,28 @@ func TestConsulDNS(t *testing.T) { helpers.RunKubectl(t, ctx.KubectlOptions(t), "delete", "pod", podName) }) - // When the `dig` request is successful, a section of it's response looks like the following: - // - // ;; ANSWER SECTION: - // consul.service.consul. 0 IN A - // - // ;; Query time: 2 msec - // ;; SERVER: #() - // ;; WHEN: Mon Aug 10 15:02:40 UTC 2020 - // ;; MSG SIZE rcvd: 98 - // - // We assert on the existence of the ANSWER SECTION, The consul-server IPs being present in the ANSWER SECTION and the the DNS IP mentioned in the SERVER: field - - require.Contains(t, logs, fmt.Sprintf("SERVER: %s", dnsIP)) - require.Contains(t, logs, "ANSWER SECTION:") - for _, ip := range serverIPs { - require.Contains(t, logs, fmt.Sprintf("consul.service.consul.\t0\tIN\tA\t%s", ip)) - } + retry.Run(t, func(r *retry.R) { + logs, err := helpers.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), dnsTestPodArgs...) + require.NoError(r, err) + + // When the `dig` request is successful, a section of it's response looks like the following: + // + // ;; ANSWER SECTION: + // consul.service.consul. 0 IN A + // + // ;; Query time: 2 msec + // ;; SERVER: #() + // ;; WHEN: Mon Aug 10 15:02:40 UTC 2020 + // ;; MSG SIZE rcvd: 98 + // + // We assert on the existence of the ANSWER SECTION, The consul-server IPs being present in the ANSWER SECTION and the the DNS IP mentioned in the SERVER: field + + require.Contains(r, logs, fmt.Sprintf("SERVER: %s", dnsIP)) + require.Contains(r, logs, "ANSWER SECTION:") + for _, ip := range serverIPs { + require.Contains(r, logs, fmt.Sprintf("consul.service.consul.\t0\tIN\tA\t%s", ip)) + } + }) }) } } diff --git a/test/acceptance/tests/terminating-gateway/terminating_gateway_namespaces_test.go b/test/acceptance/tests/terminating-gateway/terminating_gateway_namespaces_test.go index dfd6d2f18..8a149b1ad 100644 --- a/test/acceptance/tests/terminating-gateway/terminating_gateway_namespaces_test.go +++ b/test/acceptance/tests/terminating-gateway/terminating_gateway_namespaces_test.go @@ -211,7 +211,7 @@ func TestTerminatingGatewayNamespaceMirroring(t *testing.T) { } // Create the config entry for the terminating gateway - createTerminatingGatewayConfigEntry(t, consulClient, ctx.KubectlOptions(t).Namespace, testNamespace) + createTerminatingGatewayConfigEntry(t, consulClient, "", testNamespace) // Deploy the static client t.Log("deploying static client") diff --git a/test/acceptance/tests/terminating-gateway/terminating_gateway_test.go b/test/acceptance/tests/terminating-gateway/terminating_gateway_test.go index 02c43d4f5..197796a2e 100644 --- a/test/acceptance/tests/terminating-gateway/terminating_gateway_test.go +++ b/test/acceptance/tests/terminating-gateway/terminating_gateway_test.go @@ -16,7 +16,7 @@ import ( const staticClientName = "static-client" const staticServerName = "static-server" -// Test that terminating gateways work in a default installation. +// Test that terminating gateways work in a default and secure installations. func TestTerminatingGateway(t *testing.T) { cases := []struct { secure bool @@ -57,7 +57,7 @@ func TestTerminatingGateway(t *testing.T) { consulCluster := framework.NewHelmCluster(t, helmValues, ctx, cfg, releaseName) consulCluster.Create(t) - // Deploy a static-server that will play the role of an external service + // Deploy a static-server that will play the role of an external service. t.Log("creating static-server deployment") helpers.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/bases/static-server") @@ -74,7 +74,7 @@ func TestTerminatingGateway(t *testing.T) { updateTerminatingGatewayToken(t, consulClient, staticServerPolicyRules) } - // Create the config entry for the terminating gateway + // Create the config entry for the terminating gateway. createTerminatingGatewayConfigEntry(t, consulClient, "", "") // Deploy the static client @@ -89,7 +89,7 @@ func TestTerminatingGateway(t *testing.T) { assertNoConnectionAndAddIntention(t, consulClient, ctx.KubectlOptions(t), "", "") } - // Test that we can make a call to the terminating gateway + // Test that we can make a call to the terminating gateway. t.Log("trying calls to terminating gateway") helpers.CheckStaticServerConnection(t, ctx.KubectlOptions(t), true, staticClientName, "http://localhost:1234") }) @@ -130,14 +130,14 @@ func registerExternalService(t *testing.T, consulClient *api.Client, namespace s } func updateTerminatingGatewayToken(t *testing.T, consulClient *api.Client, rules string) { - // Create a write policy for the static-server + // Create a write policy for the static-server. _, _, err := consulClient.ACL().PolicyCreate(&api.ACLPolicy{ Name: "static-server-write-policy", Rules: rules, }, nil) require.NoError(t, err) - // Get the terminating gateway token + // Get the terminating gateway token. tokens, _, err := consulClient.ACL().TokenList(nil) require.NoError(t, err) var termGwTokenID string @@ -183,7 +183,6 @@ func assertNoConnectionAndAddIntention(t *testing.T, consulClient *api.Client, k t.Log("testing intentions prevent connections through the terminating gateway") helpers.CheckStaticServerConnection(t, k8sOptions, false, staticClientName, "http://localhost:1234") - // Now we create the allow intention. t.Log("creating static-client => static-server intention") _, _, err := consulClient.Connect().IntentionCreate(&api.Intention{ SourceName: staticClientName, diff --git a/test/docker/Test.dockerfile b/test/docker/Test.dockerfile index 3310d5b7f..7ef817af4 100644 --- a/test/docker/Test.dockerfile +++ b/test/docker/Test.dockerfile @@ -45,5 +45,14 @@ RUN curl -sSL https://github.com/bats-core/bats-core/archive/v${BATS_VERSION}.ta && tar -zxf /tmp/bats.tgz -C /tmp \ && /bin/bash /tmp/bats-core-${BATS_VERSION}/install.sh /usr/local +# Azure CLI +RUN curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + +# OpenShift CLI +# https://docs.microsoft.com/en-us/azure/openshift/tutorial-connect-cluster +RUN curl -sSL https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/openshift-client-linux.tar.gz -o /tmp/oc.tar.gz \ + && tar -zxvf /tmp/oc.tar.gz -C /tmp \ + && mv /tmp/oc /usr/local/bin/oc + # change the user back to what circleci/golang image has USER circleci \ No newline at end of file diff --git a/test/unit/client-role.bats b/test/unit/client-role.bats index 3fe488f31..066e4ad98 100644 --- a/test/unit/client-role.bats +++ b/test/unit/client-role.bats @@ -76,7 +76,7 @@ load _helpers #-------------------------------------------------------------------- # global.acls.manageSystemACLs -@test "client/Role: allows secret access with global.bootsrapACLs=true" { +@test "client/Role: allows secret access with global.acls.manageSystemACLs=true" { cd `chart_dir` local actual=$(helm template \ -s templates/client-role.yaml \ @@ -87,7 +87,7 @@ load _helpers [ "${actual}" = "secrets" ] } -@test "client/Role: allows secret access with global.bootsrapACLs=true and global.enablePodSecurityPolicies=true" { +@test "client/Role: allows secret access with global.acls.manageSystemACLs=true and global.enablePodSecurityPolicies=true" { cd `chart_dir` local actual=$(helm template \ -s templates/client-role.yaml \ @@ -98,3 +98,72 @@ load _helpers yq -r '.rules[1].resources[0]' | tee /dev/stderr) [ "${actual}" = "secrets" ] } + +#-------------------------------------------------------------------- +# global.openshift.enabled + +@test "client/Role: allows securitycontextconstraints access with global.openshift.enabled=true" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/client-role.yaml \ + --set 'client.enabled=true' \ + --set 'global.openshift.enabled=true' \ + . | tee /dev/stderr | + yq -r '.rules[] | select(.resources==["securitycontextconstraints"]) | .resourceNames[0]' | tee /dev/stderr) + [ "${actual}" = "release-name-consul-client" ] +} + +@test "client/Role: allows securitycontextconstraints and acl secret access with global.openshift.enabled=true and global.acls.manageSystemACLs=true" { + cd `chart_dir` + local rules=$(helm template \ + -s templates/client-role.yaml \ + --set 'client.enabled=true' \ + --set 'global.acls.manageSystemACLs=true' \ + --set 'global.openshift.enabled=true' \ + . | tee /dev/stderr | + yq -r '.rules[]' | tee /dev/stderr) + + local scc_resource=$(echo $rules | jq -r '. | select(.resources==["securitycontextconstraints"]) | .resourceNames[0]') + [ "${scc_resource}" = "release-name-consul-client" ] + + local secrets_resource=$(echo $rules | jq -r '. | select(.resources==["secrets"]) | .resourceNames[0]') + [ "${secrets_resource}" = "release-name-consul-client-acl-token" ] +} + +@test "client/Role: allows securitycontextconstraints and psp access with global.openshift.enabled=true and global.enablePodSecurityPolices=true" { + cd `chart_dir` + local rules=$(helm template \ + -s templates/client-role.yaml \ + --set 'client.enabled=true' \ + --set 'global.enablePodSecurityPolicies=true' \ + --set 'global.openshift.enabled=true' \ + . | tee /dev/stderr | + yq -r '.rules[]' | tee /dev/stderr) + + local scc_resource=$(echo $rules | jq -r '. | select(.resources==["securitycontextconstraints"]) | .resourceNames[0]') + [ "${scc_resource}" = "release-name-consul-client" ] + + local psp_resource=$(echo $rules | jq -r '. | select(.resources==["podsecuritypolicies"]) | .resourceNames[0]') + [ "${psp_resource}" = "release-name-consul-client" ] +} + +@test "client/Role: allows securitycontextconstraints, acl secret, and psp access when all global.openshift.enabled, global.enablePodSecurityPolices, and global.acls.manageSystemACLs are true " { + cd `chart_dir` + local rules=$(helm template \ + -s templates/client-role.yaml \ + --set 'client.enabled=true' \ + --set 'global.acls.manageSystemACLs=true' \ + --set 'global.enablePodSecurityPolicies=true' \ + --set 'global.openshift.enabled=true' \ + . | tee /dev/stderr | + yq -r '.rules[]' | tee /dev/stderr) + + local scc_resource=$(echo $rules | jq -r '. | select(.resources==["securitycontextconstraints"]) | .resourceNames[0]') + [ "${scc_resource}" = "release-name-consul-client" ] + + local secrets_resource=$(echo $rules | jq -r '. | select(.resources==["secrets"]) | .resourceNames[0]') + [ "${secrets_resource}" = "release-name-consul-client-acl-token" ] + + local psp_resource=$(echo $rules | jq -r '. | select(.resources==["podsecuritypolicies"]) | .resourceNames[0]') + [ "${psp_resource}" = "release-name-consul-client" ] +} diff --git a/test/unit/client-securitycontextconstraints.bats b/test/unit/client-securitycontextconstraints.bats new file mode 100644 index 000000000..c8901f7e4 --- /dev/null +++ b/test/unit/client-securitycontextconstraints.bats @@ -0,0 +1,98 @@ +#!/usr/bin/env bats + +load _helpers + +@test "client/SecurityContextConstraints: disabled by default" { + cd `chart_dir` + assert_empty helm template \ + -s templates/client-securitycontextconstraints.yaml \ + . +} + +@test "client/SecurityContextConstraints: disabled with client disabled and global.openshift.enabled=true" { + cd `chart_dir` + assert_empty helm template \ + -s templates/client-securitycontextconstraints.yaml \ + --set 'client.enabled=false' \ + --set 'global.openshift.enabled=true' \ + . +} + +@test "client/SecurityContextConstraints: enabled with global.openshift.enabled=true" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/client-securitycontextconstraints.yaml \ + --set 'global.openshift.enabled=true' \ + . | tee /dev/stderr | + yq -s 'length > 0' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "client/SecurityContextConstraints: host ports are allowed by default" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/client-securitycontextconstraints.yaml \ + --set 'global.openshift.enabled=true' \ + . | tee /dev/stderr | + yq -c '.allowHostPorts' | tee /dev/stderr) + [ "${actual}" = 'true' ] +} + + +#-------------------------------------------------------------------- +# client.dataDirectoryHostPath + +@test "client/SecurityContextConstraints: disallows hostPath volume by default" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/client-securitycontextconstraints.yaml \ + --set 'global.openshift.enabled=true' \ + . | tee /dev/stderr | + yq '.volumes | any(contains("hostPath"))' | tee /dev/stderr) + [ "${actual}" = 'false' ] +} + +@test "client/SecurityContextConstraints: allows hostPath volume when dataDirectoryHostPath is set" { + cd `chart_dir` + # Test that hostPath is an allowed volume type. + local actual=$(helm template \ + -s templates/client-securitycontextconstraints.yaml \ + --set 'global.openshift.enabled=true' \ + --set 'client.dataDirectoryHostPath=/opt/consul' \ + . | tee /dev/stderr | + yq '.volumes | any(contains("hostPath"))' | tee /dev/stderr) + [ "${actual}" = 'true' ] + + # Test that the path we're allowed to write to host path. + local actual=$(helm template \ + -s templates/client-securitycontextconstraints.yaml \ + --set 'global.openshift.enabled=true' \ + --set 'client.dataDirectoryHostPath=/opt/consul' \ + . | tee /dev/stderr | + yq -r '.allowHostDirVolumePlugin' | tee /dev/stderr) + [ "${actual}" = 'true' ] +} + +#-------------------------------------------------------------------- +# client.hostNetwork + +@test "client/SecurityContextConstraints: enabled with global.openshift.enabled=true and hostNetwork=true" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/client-securitycontextconstraints.yaml \ + --set 'global.openshift.enabled=true' \ + --set 'client.hostNetwork=true' \ + . | tee /dev/stderr | + yq '.allowHostNetwork == true' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "client/SecurityContextConstraints: enabled with global.openshift.enabled=true and default hostNetwork=false" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/client-securitycontextconstraints.yaml \ + --set 'global.openshift.enabled=true' \ + . | tee /dev/stderr | + yq '.allowHostNetwork == false' | tee /dev/stderr) + [ "${actual}" = "true" ] +} diff --git a/test/unit/server-statefulset.bats b/test/unit/server-statefulset.bats index b575d3067..bebe27af4 100755 --- a/test/unit/server-statefulset.bats +++ b/test/unit/server-statefulset.bats @@ -446,13 +446,22 @@ load _helpers } #-------------------------------------------------------------------- -# server.disableFsGroupSecurityContext +# global.openshift.enabled -@test "server/StatefulSet: can disable fsGroup security context settings" { +@test "server/StatefulSet: setting server.disableFsGroupSecurityContext fails" { + cd `chart_dir` + run helm template \ + -s templates/server-statefulset.yaml \ + --set 'server.disableFsGroupSecurityContext=true' . + [ "$status" -eq 1 ] + [[ "$output" =~ "server.disableFsGroupSecurityContext has been removed. Please use global.openshift.enabled instead." ]] +} + +@test "server/StatefulSet: fsGroup is not set when global.openshift.enabled=true" { cd `chart_dir` local actual=$(helm template \ -s templates/server-statefulset.yaml \ - --set 'server.disableFsGroupSecurityContext=true' \ + --set 'global.openshift.enabled=true' \ . | tee /dev/stderr | yq -r '.spec.template.spec.securityContext' | tee /dev/stderr) [ "${actual}" = "null" ] @@ -462,7 +471,6 @@ load _helpers cd `chart_dir` local actual=$(helm template \ -s templates/server-statefulset.yaml \ - --set 'server.disableFsGroupSecurityContext=false' \ . | tee /dev/stderr | yq -r '.spec.template.spec.securityContext.fsGroup' | tee /dev/stderr) [ "${actual}" = "1000" ] diff --git a/values.yaml b/values.yaml index 42f8d8af1..c139233e3 100644 --- a/values.yaml +++ b/values.yaml @@ -233,6 +233,13 @@ global: # See https://www.consul.io/docs/connect/proxies/envoy for full compatibility matrix between Consul and Envoy. imageEnvoy: "envoyproxy/envoy-alpine:v1.14.2" + # openshift contains configuration for running this Helm chart + # on the Red Hat OpenShift platform. + openshift: + # If true, the Helm chart will create necessary configuration for running its components + # on OpenShift. + enabled: false + # Server, when enabled, configures a server cluster to run. This should # be disabled if you plan on connecting to a Consul cluster external to # the Kube cluster. @@ -366,10 +373,6 @@ server: # https_proxy: http://localhost:3128, # no_proxy: internal.domain.com - # disableFsGroupSecurityContext disables setting the fsGroup securityContext for the server statefulset, - # this is required when using the OpenShift platform as fsGroup is automatically set to an arbitrary gid. - disableFsGroupSecurityContext : false - # Configuration for Consul servers when the servers are running outside of Kubernetes. # When running external servers, configuring these values is recommended # if setting global.tls.enableAutoEncrypt to true (requires consul-k8s >= 0.13.0)