Adds support for running in Istio environments (#21)

* adds istio sidecar ready/quit logic to crd commands

this may be the wrong place to put this logic. ideally, istio fixes
their shit and we no longer have to (a) wait for the sidecar to be ready
and (b) send an envoy quit signal after a job pod is done.

alas, this is not the case right now and wrapping this logic around the
actual command will require us to migrate off of a "distroless" base
docker image.

poop or vomit? pick one.

* modifies helm chart

- adds support for '--istio-enabled' crd cmd flag
- adds psp use permissions to hooks
- adds custom psp for operator
- adds new values to control rendering

* adds network policy

exposed ports are health, metrics, and webhook. not sure we need to
block access to these but maybe i'm wrong

* disables codecov patch status check

not very important yet annoying

.codecov.yml

@@ -4,5 +4,6 @@ coverage:
       default:
         target: 75%
         threshold: 5%
+    patch: off
 ignore:
   - "api/**/zz_generated.deepcopy.go"

cmd/crdapply.go

@@ -17,11 +17,12 @@ Apply Rules:
   - When a definition is is missing, it will be created
   - If a definition is already present, then it will be updated
   - Updating definitions that have not changed results in a no-op`,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		return crd.Apply(context.Background())
-	},
+	RunE: processIstioFlag(func(enabled bool) error {
+		return crd.Apply(context.Background(), enabled)
+	}),
 }
 
 func init() {
+	addIstioFlag(crdApplyCmd)
 	rootCmd.AddCommand(crdApplyCmd)
 }

cmd/crddelete.go

@@ -16,11 +16,12 @@ var crdDeleteCmd = &cobra.Command{
 Any running distributed compute resources will be decommissioned when this 
 operation runs (i.e. your deployments will be deleted immediately). This will
 only attempt to remove definitions that are already present in Kubernetes.`,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		return crd.Delete(context.Background())
-	},
+	RunE: processIstioFlag(func(enabled bool) error {
+		return crd.Delete(context.Background(), enabled)
+	}),
 }
 
 func init() {
+	addIstioFlag(crdDeleteCmd)
 	rootCmd.AddCommand(crdDeleteCmd)
 }

cmd/root.go

@@ -21,6 +21,21 @@ func Execute() {
 	}
 }
 
+func addIstioFlag(cmd *cobra.Command) {
+	cmd.Flags().BoolP("istio-enabled", "i", false, "Enable support for Istio sidecar container")
+}
+
+func processIstioFlag(op func(enabled bool) error) func(cmd *cobra.Command, args []string) error {
+	return func(cmd *cobra.Command, args []string) error {
+		istioEnabled, err := cmd.Flags().GetBool("istio-enabled")
+		if err != nil {
+			return err
+		}
+
+		return op(istioEnabled)
+	}
+}
+
 func init() {
 	// NOTE: required until https://github.com/spf13/cobra/issues/587
 	rootCmd.SetHelpCommand(&cobra.Command{Hidden: true})

deploy/helm/distributed-compute-operator/templates/hooks.yaml

@@ -22,15 +22,21 @@ metadata:
     "helm.sh/hook-weight": "-2"
     "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
 rules:
-  - apiGroups:
-    - apiextensions.k8s.io
-    resources:
-    - customresourcedefinitions
-    verbs:
-    - get
-    - create
-    - update
-    - delete
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - get
+  - create
+  - update
+  - delete
+- apiGroups:
+  - policy
+  resources:
+  - podsecuritypolicies
+  verbs:
+  - use
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1
@@ -83,7 +89,11 @@ spec:
             {{- toYaml .Values.securityContext | nindent 12 }}
           image: {{ include "dco.image" . }}
           imagePullPolicy: {{ .Values.image.pullPolicy }}
-          args: ["crd-apply"]
+          args:
+            - crd-apply
+            {{- if .Values.istio.enabled }}
+            - --istio-enabled
+            {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}
@@ -106,7 +116,7 @@ metadata:
     {{- include "common.labels.standard" . | nindent 4 }}
   annotations:
     "helm.sh/hook": post-delete
-    "helm.sh/hook-delete-policy": hook-succeeded
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
 spec:
   backoffLimit: 0
   template:
@@ -128,7 +138,11 @@ spec:
             {{- toYaml .Values.securityContext | nindent 12 }}
           image: {{ include "dco.image" . }}
           imagePullPolicy: {{ .Values.image.pullPolicy }}
-          args: ["crd-delete"]
+          args:
+            - crd-delete
+            {{- if .Values.istio.enabled }}
+            - --istio-enabled
+            {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

deploy/helm/distributed-compute-operator/templates/networkpolicy.yaml

@@ -0,0 +1,22 @@
+{{- if .Values.networkPolicy.enabled }}
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: {{ include "common.names.fullname" . }}
+  labels:
+    {{- include "common.labels.standard" . | nindent 4 }}
+spec:
+  podSelector:
+    matchLabels:
+      {{- include "common.labels.matchLabels" . | nindent 6 }}
+  policyTypes:
+  - Ingress
+  ingress:
+  - ports:
+    - port: 9443
+      protocol: TCP
+    - port: {{ .Values.config.healthProbePort }}
+      protocol: TCP
+    - port: {{ .Values.config.metricsPort }}
+      protocol: TCP
+{{- end }}

deploy/helm/distributed-compute-operator/templates/podsecuritypolicy.yaml

@@ -0,0 +1,50 @@
+{{- if .Values.podSecurityPolicy.enabled }}
+{{- $elevatePermissions := and .Values.istio.enabled (not .Values.istio.cniPluginInstalled) -}}
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: {{ include "common.names.fullname" . }}
+  labels:
+    {{- include "common.labels.standard" . | nindent 4 }}
+  annotations:
+    seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'docker/default,runtime/default'
+    apparmor.security.beta.kubernetes.io/allowedProfileNames: 'unconfined,runtime/default'
+    seccomp.security.alpha.kubernetes.io/defaultProfileName:  'runtime/default'
+    apparmor.security.beta.kubernetes.io/defaultProfileName:  'unconfined'
+spec:
+  privileged: false
+  allowPrivilegeEscalation: false
+  {{- if $elevatePermissions }}
+  allowedCapabilities:
+    - NET_ADMIN
+    - NET_RAW
+  {{- else }}
+  requiredDropCapabilities:
+    - ALL
+  {{- end }}
+  volumes:
+    - configMap
+    - emptyDir
+    - projected
+    - secret
+    - downwardAPI
+    - persistentVolumeClaim
+  hostNetwork: false
+  hostIPC: false
+  hostPID: false
+  runAsUser:
+    rule: {{ if $elevatePermissions }}RunAsAny{{ else }}MustRunAsNonRoot{{ end }}
+  seLinux:
+    rule: RunAsAny
+  supplementalGroups:
+    rule: MustRunAs
+    ranges:
+      - min: 1
+        max: 65535
+  fsGroup:
+    rule: MustRunAs
+    ranges:
+      - min: 1
+        max: 65535
+  readOnlyRootFilesystem: false
+{{- end }}

deploy/helm/distributed-compute-operator/templates/rbac.yaml

@@ -10,7 +10,7 @@ rules:
   resources:
   - podsecuritypolicies
   verbs:
-  - use # required to grant RBAC permission "use" to CR clusters
+  - use
   - list
   - watch
 

deploy/helm/distributed-compute-operator/values.yaml

@@ -5,7 +5,7 @@
 # Number of controller instances to run
 replicaCount: 1
 
-# If true, CRD resources will be installed/uninstalled as part of the Helm chart release
+# If true, CRD resources will be installed/uninstalled as part of the Helm chart release.
 # Uninstalling CRD resources will DELETE all related custom resources.
 installCRDs: true
 
@@ -27,6 +27,21 @@ config:
   # Development mode enables debug logging, console output and stacktraces suitable for troubleshooting
   logDevelopmentMode: false
 
+istio:
+  # Enable support for environments with Istio installed
+  enabled: false
+  # Elevate pod execution permissions so that Istio's init container can modify
+  # network settings when CNI plugin is NOT installed
+  cniPluginInstalled: true
+
+podSecurityPolicy:
+  # Create custom PSP for operator
+  enabled: false
+
+networkPolicy:
+  # Restrict network ingress to operator pods
+  enabled: false
+
 image:
   registry: ghcr.io
   repository: dominodatalab/distributed-compute-operator

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/banzaicloud/k8s-objectmatcher v1.5.1
 	github.com/docker/distribution v2.7.1+incompatible
 	github.com/go-logr/logr v0.3.0
+	github.com/hashicorp/go-retryablehttp v0.6.8
 	github.com/onsi/ginkgo v1.14.2
 	github.com/onsi/gomega v1.10.5
 	github.com/opencontainers/go-digest v1.0.0 // indirect

go.sum

@@ -224,10 +224,15 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t
 github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
 github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM=
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
+github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT+TFdCxsPyWs=
+github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
 github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=

pkg/crd/crd.go

@@ -23,7 +23,16 @@ var (
 
 // Apply will create or update all project CRDs inside a Kubernetes cluster.
 // The latest available version of the CRD will be used to perform this operation.
-func Apply(ctx context.Context) error {
+func Apply(ctx context.Context, istioEnabled bool) error {
+	if istioEnabled {
+		quit, err := waitForIstioSidecar()
+		if err != nil {
+			return err
+		}
+
+		defer quit()
+	}
+
 	apply := func(client apixv1client.CustomResourceDefinitionInterface, crd *apixv1.CustomResourceDefinition) error {
 		found, err := client.Get(ctx, crd.Name, metav1.GetOptions{})
 
@@ -43,7 +52,16 @@ func Apply(ctx context.Context) error {
 }
 
 // Delete will remove all project CRDs from a Kubernetes cluster.
-func Delete(ctx context.Context) error {
+func Delete(ctx context.Context, istioEnabled bool) error {
+	if istioEnabled {
+		quit, err := waitForIstioSidecar()
+		if err != nil {
+			return err
+		}
+
+		defer quit()
+	}
+
 	deleteFn := func(client apixv1client.CustomResourceDefinitionInterface, crd *apixv1.CustomResourceDefinition) error {
 		log.Info("Deleting CRD", "Name", crd.Name)
 		err := client.Delete(ctx, crd.Name, metav1.DeleteOptions{})
@@ -73,12 +91,12 @@ func processCRDs(processor func(client apixv1client.CustomResourceDefinitionInte
 	}
 
 	for _, def := range definitions {
-		crd, err := loadCRD(def)
+		customResourceDefinition, err := loadCRD(def)
 		if err != nil {
 			return err
 		}
 
-		if err := processor(client, crd); err != nil {
+		if err := processor(client, customResourceDefinition); err != nil {
 			return err
 		}
 	}
@@ -93,12 +111,12 @@ func loadCRD(bs []byte) (*apixv1.CustomResourceDefinition, error) {
 		return nil, err
 	}
 
-	crd := new(apixv1.CustomResourceDefinition)
-	if err := json.Unmarshal(bs, crd); err != nil {
+	resource := new(apixv1.CustomResourceDefinition)
+	if err := json.Unmarshal(bs, resource); err != nil {
 		return nil, err
 	}
 
-	return crd, nil
+	return resource, nil
 }
 
 // getCRDClient returns a client configured to work with custom resource definitions.

pkg/crd/crd_test.go

@@ -34,7 +34,7 @@ func TestApply(t *testing.T) {
 
 		t.Cleanup(overrideCRDClient(fakeClient))
 
-		require.NoError(t, Apply(context.Background()))
+		require.NoError(t, Apply(context.Background(), false))
 		assert.True(t, created, "New CRD was not created")
 	})
 
@@ -60,7 +60,7 @@ func TestApply(t *testing.T) {
 
 		t.Cleanup(overrideCRDClient(fakeClient))
 
-		require.NoError(t, Apply(context.Background()))
+		require.NoError(t, Apply(context.Background(), false))
 		assert.True(t, updated, "Existing CRD was not updated")
 	})
 
@@ -74,7 +74,7 @@ func TestApply(t *testing.T) {
 
 		t.Cleanup(overrideCRDClient(fakeClient))
 
-		err := Apply(context.Background())
+		err := Apply(context.Background(), false)
 		assert.Equalf(t, expected, err, "Received error %v did not match %v", err, expected)
 	})
 }
@@ -95,7 +95,7 @@ func TestDelete(t *testing.T) {
 
 		t.Cleanup(overrideCRDClient(fakeClient))
 
-		require.NoError(t, Delete(context.Background()))
+		require.NoError(t, Delete(context.Background(), false))
 		assert.True(t, deleted, "Existing CRD was not deleted")
 	})
 
@@ -107,7 +107,7 @@ func TestDelete(t *testing.T) {
 
 		t.Cleanup(overrideCRDClient(fakeClient))
 
-		assert.NoError(t, Delete(context.Background()), "Delete failed when CRD not found")
+		assert.NoError(t, Delete(context.Background(), false), "Delete failed when CRD not found")
 	})
 
 	t.Run("error", func(t *testing.T) {
@@ -119,7 +119,7 @@ func TestDelete(t *testing.T) {
 
 		t.Cleanup(overrideCRDClient(fakeClient))
 
-		err := Delete(context.Background())
+		err := Delete(context.Background(), false)
 		assert.Equalf(t, expected, err, "Received error %v did not match %v", err, expected)
 	})
 }