Openshift support

.circleci/config.yml

@@ -22,7 +22,7 @@ jobs:
           command: bats ./test/unit
   unit-helm3:
     docker:
-      - image: hashicorpdev/consul-helm-test:0.5.0
+      - image: hashicorpdev/consul-helm-test:0.6.0
 
     steps:
       - checkout
@@ -153,6 +153,58 @@ jobs:
             terraform destroy -var project=${CLOUDSDK_CORE_PROJECT} -auto-approve
           when: always
 
+  acceptance-openshift:
+    environment:
+      - TEST_RESULTS: /tmp/test-results
+      - OC_PRIMARY_NAME: consul-helm-test-2757871175
+      - OC_SECONDARY_NAME: consul-helm-test-3737660519
+    docker:
+      # This image is build from test/docker/Test.dockerfile
+      - image: hashicorpdev/consul-helm-test:0.6.0
+
+    steps:
+      - checkout
+
+      - run:
+          name: openshift login
+          command: |
+            az login --service-principal -u "$ARM_CLIENT_ID" -p "$ARM_CLIENT_SECRET" --tenant "$ARM_TENANT_ID" > /dev/null
+
+            for cluster_name in "$OC_PRIMARY_NAME" "$OC_SECONDARY_NAME"; do
+              apiServer=$(az aro show -g "$cluster_name" -n "$cluster_name" --query apiserverProfile.url -o tsv)
+              kubeUser=$(az aro list-credentials -g "$cluster_name" -n "$cluster_name" | jq -r .kubeadminUsername)
+              kubePassword=$(az aro list-credentials -g "$cluster_name" -n "$cluster_name" | jq -r .kubeadminPassword)
+
+              KUBECONFIG="$HOME/.kube/$cluster_name" oc login "$apiServer" -u "$kubeUser" -p "$kubePassword"
+              KUBECONFIG="$HOME/.kube/$cluster_name" oc project consul
+            done
+
+      # Restore go module cache if there is one
+      - restore_cache:
+          keys:
+            - consul-helm-modcache-v1-{{ checksum "test/acceptance/go.mod" }}
+
+      - run: mkdir -p $TEST_RESULTS
+
+      - run:
+          name: Run acceptance tests
+          working_directory: test/acceptance/tests
+          no_output_timeout: 30m
+          command: |
+            gotestsum --junitfile "$TEST_RESULTS/gotestsum-report.xml" -- ./... -p 1 -timeout 30m -failfast \
+              -enable-openshift \
+              -enable-enterprise \
+              -enable-multi-cluster \
+              -kubeconfig="$HOME/.kube/$OC_PRIMARY_NAME" \
+              -secondary-kubeconfig="$HOME/.kube/$OC_SECONDARY_NAME" \
+              -debug-directory="$TEST_RESULTS/debug" \
+              -consul-k8s-image=hashicorpdev/consul-k8s:latest
+
+      - store_test_results:
+          path: /tmp/test-results
+      - store_artifacts:
+          path: /tmp/test-results
+
   update-helm-charts-index:
     docker:
       - image: circleci/golang:latest
@@ -191,11 +243,27 @@ workflows:
             - go-fmt-and-vet
       - unit-helm2
       - unit-helm3
-      - acceptance:
+#      - acceptance:
+#          requires:
+#            - unit-helm2
+#            - unit-helm3
+#            - unit-acceptance-framework
+      - acceptance-openshift:
           requires:
             - unit-helm2
             - unit-helm3
             - unit-acceptance-framework
+# todo: uncomment before merging the PR
+#  nightly-acceptance-tests:
+#    triggers:
+#      - schedule:
+#          cron: "0 0 * * *"
+#          filters:
+#            branches:
+#              only:
+#                - master
+#    jobs:
+#      - acceptance-openshift
   update-helm-charts-index:
     jobs:
       - update-helm-charts-index:

templates/client-role.yaml

@@ -9,7 +9,7 @@ metadata:
     chart: {{ template "consul.chart" . }}
     heritage: {{ .Release.Service }}
     release: {{ .Release.Name }}
-{{- if (or .Values.global.acls.manageSystemACLs .Values.global.enablePodSecurityPolicies) }}
+{{- if (or .Values.global.acls.manageSystemACLs .Values.global.enablePodSecurityPolicies .Values.global.openshift.enabled) }}
 rules:
 {{- if .Values.global.enablePodSecurityPolicies }}
   - apiGroups: ["policy"]
@@ -28,6 +28,14 @@ rules:
     verbs:
       - get
 {{- end }}
+{{- if .Values.global.openshift.enabled}}
+  - apiGroups: ["security.openshift.io"]
+    resources: ["securitycontextconstraints"]
+    resourceNames:
+      - {{ template "consul.fullname" . }}-client
+    verbs:
+      - use
+{{- end}}
 {{- else}}
 rules: []
 {{- end }}

templates/client-securitycontextconstraints.yaml

@@ -0,0 +1,54 @@
+{{- if (and .Values.global.openshift.enabled (or (and (ne (.Values.client.enabled | toString) "-") .Values.client.enabled) (and (eq (.Values.client.enabled | toString) "-") .Values.global.enabled))) }}
+apiVersion: security.openshift.io/v1
+kind: SecurityContextConstraints
+metadata:
+  name: {{ template "consul.fullname" . }}-client
+  labels:
+    app: {{ template "consul.name" . }}
+    chart: {{ template "consul.chart" . }}
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+  annotations:
+    kubernetes.io/description: {{ template "consul.fullname" . }}-client are the security context constraints required
+      to run the consul client.
+{{- if .Values.client.dataDirectoryHostPath }}
+allowHostDirVolumePlugin: true
+{{- else }}
+allowHostDirVolumePlugin: false
+{{- end}}
+allowHostIPC: false
+allowHostNetwork: {{ .Values.client.hostNetwork }}
+allowHostPID: false
+allowHostPorts: true
+allowPrivilegeEscalation: true
+allowPrivilegedContainer: false
+allowedCapabilities: null
+defaultAddCapabilities: null
+fsGroup:
+  type: MustRunAs
+groups: []
+priority: null
+readOnlyRootFilesystem: false
+requiredDropCapabilities:
+- KILL
+- MKNOD
+- SETUID
+- SETGID
+runAsUser:
+  type: MustRunAsRange
+seLinuxContext:
+  type: MustRunAs
+supplementalGroups:
+  type: MustRunAs
+users: []
+volumes:
+- configMap
+- downwardAPI
+- emptyDir
+- persistentVolumeClaim
+- projected
+- secret
+{{- if .Values.client.dataDirectoryHostPath }}
+- hostPath
+{{- end }}
+{{- end}}

templates/create-federation-secret-job.yaml

@@ -129,9 +129,9 @@ spec:
                   -server-ca-key-file=/consul/tls/server/ca/tls.key
           resources:
             requests:
-              memory: "25Mi"
+              memory: "50Mi"
               cpu: "50m"
             limits:
-              memory: "25Mi"
+              memory: "50Mi"
               cpu: "50m"
 {{- end }}

templates/ingress-gateways-deployment.yaml

@@ -264,10 +264,10 @@ spec:
             {{- end }}
           resources:
             requests:
-              memory: "25Mi"
+              memory: "50Mi"
               cpu: "50m"
             limits:
-              memory: "25Mi"
+              memory: "50Mi"
               cpu: "50m"
       containers:
         - name: ingress-gateway

templates/server-acl-init-cleanup-job.yaml

@@ -54,10 +54,10 @@ spec:
             - {{ template "consul.fullname" . }}-server-acl-init
           resources:
             requests:
-              memory: "25Mi"
+              memory: "50Mi"
               cpu: "50m"
             limits:
-              memory: "25Mi"
+              memory: "50Mi"
               cpu: "50m"
   {{- end }}
   {{- end }}

templates/server-acl-init-job.yaml

@@ -238,10 +238,10 @@ spec:
                 {{- end }}
           resources:
             requests:
-              memory: "25Mi"
+              memory: "50Mi"
               cpu: "50m"
             limits:
-              memory: "25Mi"
+              memory: "50Mi"
               cpu: "50m"
 {{- end }}
 {{- end }}

templates/server-statefulset.yaml

@@ -1,6 +1,7 @@
 {{- if (or (and (ne (.Values.server.enabled | toString) "-") .Values.server.enabled) (and (eq (.Values.server.enabled | toString) "-") .Values.global.enabled)) }}
 {{- if and .Values.global.federation.enabled (not .Values.global.tls.enabled) }}{{ fail "If global.federation.enabled is true, global.tls.enabled must be true because federation is only supported with TLS enabled" }}{{ end }}
 {{- if and .Values.global.federation.enabled (not .Values.meshGateway.enabled) }}{{ fail "If global.federation.enabled is true, meshGateway.enabled must be true because mesh gateways are required for federation" }}{{ end }}
+{{- if .Values.server.disableFsGroupSecurityContext }}{{ fail "server.disableFsGroupSecurityContext has been removed. Please use global.openshift.enabled instead." }}{{ end }}
 # StatefulSet to run the actual Consul server cluster.
 apiVersion: apps/v1
 kind: StatefulSet
@@ -58,7 +59,7 @@ spec:
     {{- end }}
       terminationGracePeriodSeconds: 30
       serviceAccountName: {{ template "consul.fullname" . }}-server
-      {{- if not .Values.server.disableFsGroupSecurityContext }}
+      {{- if not .Values.global.openshift.enabled}}
       securityContext:
         fsGroup: 1000
       {{- end }}

templates/terminating-gateways-deployment.yaml

@@ -211,10 +211,10 @@ spec:
             {{- end }}
           resources:
             requests:
-              memory: "25Mi"
+              memory: "50Mi"
               cpu: "50m"
             limits:
-              memory: "25Mi"
+              memory: "50Mi"
               cpu: "50m"
       containers:
         - name: terminating-gateway

templates/tls-init-cleanup-job.yaml

@@ -54,10 +54,10 @@ spec:
                 -H "Authorization: Bearer $( cat /var/run/secrets/kubernetes.io/serviceaccount/token )"
           resources:
             requests:
-              memory: "25Mi"
+              memory: "50Mi"
               cpu: "50m"
             limits:
-              memory: "25Mi"
+              memory: "50Mi"
               cpu: "50m"
 {{- end }}
 {{- end }}

templates/tls-init-job.yaml

@@ -56,6 +56,7 @@ spec:
           # Note that in the subsequent runs of the job, POST requests will
           # return a 409 because these secrets would already exist;
           # we are ignoring these response codes.
+          workingDir: /tmp
           command:
             - "/bin/sh"
             - "-ec"
@@ -116,10 +117,10 @@ spec:
           {{- end }}
           resources:
             requests:
-              memory: "25Mi"
+              memory: "50Mi"
               cpu: "50m"
             limits:
-              memory: "25Mi"
+              memory: "50Mi"
               cpu: "50m"
 {{- end }}
 {{- end }}

test/acceptance/framework/config.go

@@ -23,6 +23,8 @@ type TestConfig struct {
 	EnterpriseLicenseSecretName string
 	EnterpriseLicenseSecretKey  string
 
+	EnableOpenshift bool
+
 	ConsulImage    string
 	ConsulK8SImage string
 
@@ -52,6 +54,10 @@ func (t *TestConfig) HelmValuesFromConfig() (map[string]string, error) {
 		setIfNotEmpty(helmValues, "server.enterpriseLicense.secretKey", t.EnterpriseLicenseSecretKey)
 	}
 
+	if t.EnableOpenshift {
+		setIfNotEmpty(helmValues, "global.openshift.enabled", "true")
+	}
+
 	setIfNotEmpty(helmValues, "global.image", t.ConsulImage)
 	setIfNotEmpty(helmValues, "global.imageK8S", t.ConsulK8SImage)
 

test/acceptance/framework/config_test.go

@@ -67,6 +67,15 @@ func TestConfig_HelmValuesFromConfig(t *testing.T) {
 			},
 			map[string]string{},
 		},
+		{
+			"sets openshift value when EnableOpenshift is set",
+			TestConfig{
+				EnableOpenshift: true,
+			},
+			map[string]string{
+				"global.openshift.enabled": "true",
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

test/acceptance/framework/flags.go

@@ -20,6 +20,8 @@ type TestFlags struct {
 	flagEnterpriseLicenseSecretName string
 	flagEnterpriseLicenseSecretKey  string
 
+	flagEnableOpenshift bool
+
 	flagConsulImage    string
 	flagConsulK8sImage string
 
@@ -64,6 +66,9 @@ func (t *TestFlags) init() {
 	flag.StringVar(&t.flagEnterpriseLicenseSecretKey, "enterprise-license-secret-key", "",
 		"The key of the Kubernetes secret containing the enterprise license.")
 
+	flag.BoolVar(&t.flagEnableOpenshift, "enable-openshift", false,
+		"If true, the tests will automatically add Openshift Helm value for each Helm install.")
+
 	flag.BoolVar(&t.flagNoCleanupOnFailure, "no-cleanup-on-failure", false,
 		"If true, the tests will not cleanup Kubernetes resources they create when they finish running."+
 			"Note this flag must be run with -failfast flag, otherwise subsequent tests will fail.")
@@ -105,6 +110,8 @@ func (t *TestFlags) testConfigFromFlags() *TestConfig {
 		EnterpriseLicenseSecretName: t.flagEnterpriseLicenseSecretName,
 		EnterpriseLicenseSecretKey:  t.flagEnterpriseLicenseSecretKey,
 
+		EnableOpenshift: t.flagEnableOpenshift,
+
 		ConsulImage:    t.flagConsulImage,
 		ConsulK8SImage: t.flagConsulK8sImage,
 

test/acceptance/helpers/helpers.go

@@ -38,8 +38,8 @@ func WaitForAllPodsToBeReady(t *testing.T, client kubernetes.Interface, namespac
 
 	t.Log("Waiting for pods to be ready.")
 
-	// Wait up to 3m.
-	counter := &retry.Counter{Count: 36, Wait: 5 * time.Second}
+	// Wait up to 5m.
+	counter := &retry.Counter{Count: 60, Wait: 5 * time.Second}
 	retry.RunWith(counter, t, func(r *retry.R) {
 		pods, err := client.CoreV1().Pods(namespace).List(context.Background(), metav1.ListOptions{LabelSelector: podLabelSelector})
 		require.NoError(r, err)
@@ -105,7 +105,7 @@ func DeployKustomize(t *testing.T, options *k8s.KubectlOptions, noCleanupOnFailu
 		KubectlDeleteK(t, options, kustomizeDir)
 	})
 
-	RunKubectl(t, options, "wait", "--for=condition=available", fmt.Sprintf("deploy/%s", deployment.Name))
+	RunKubectl(t, options, "wait", "--for=condition=available", "--timeout=1m", fmt.Sprintf("deploy/%s", deployment.Name))
 }
 
 // CheckStaticServerConnection execs into a pod of the deployment given by deploymentName