diff --git a/assets/controller.yaml b/assets/controller.yaml index 9f5d09ee6..878c8ab04 100644 --- a/assets/controller.yaml +++ b/assets/controller.yaml @@ -54,10 +54,6 @@ spec: value: '1' - name: AWS_CONFIG_FILE value: /var/run/secrets/aws/credentials - {{- if .CABundleConfigMap}} - - name: AWS_CA_BUNDLE - value: /etc/ca/ca-bundle.pem - {{- end}} ports: - name: healthz # Due to hostNetwork, this port is open on a node! @@ -70,11 +66,6 @@ spec: - name: bound-sa-token mountPath: /var/run/secrets/openshift/serviceaccount readOnly: true - {{- if .CABundleConfigMap}} - - name: ca-bundle - mountPath: /etc/ca - readOnly: true - {{- end}} - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ resources: @@ -170,10 +161,5 @@ spec: - serviceAccountToken: path: token audience: openshift - {{- if .CABundleConfigMap}} - - name: ca-bundle - configMap: - name: {{.CABundleConfigMap}} - {{- end}} - name: socket-dir emptyDir: {} diff --git a/pkg/generated/bindata.go b/pkg/generated/bindata.go index 9890923c3..dba9a002c 100644 --- a/pkg/generated/bindata.go +++ b/pkg/generated/bindata.go @@ -127,10 +127,6 @@ spec: value: '1' - name: AWS_CONFIG_FILE value: /var/run/secrets/aws/credentials - {{- if .CABundleConfigMap}} - - name: AWS_CA_BUNDLE - value: /etc/ca/ca-bundle.pem - {{- end}} ports: - name: healthz # Due to hostNetwork, this port is open on a node! @@ -143,11 +139,6 @@ spec: - name: bound-sa-token mountPath: /var/run/secrets/openshift/serviceaccount readOnly: true - {{- if .CABundleConfigMap}} - - name: ca-bundle - mountPath: /etc/ca - readOnly: true - {{- end}} - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ resources: @@ -243,11 +234,6 @@ spec: - serviceAccountToken: path: token audience: openshift - {{- if .CABundleConfigMap}} - - name: ca-bundle - configMap: - name: {{.CABundleConfigMap}} - {{- end}} - name: socket-dir emptyDir: {} `) diff --git a/pkg/operator/starter.go b/pkg/operator/starter.go index d116d3e1d..5b6f40bbd 100644 --- a/pkg/operator/starter.go +++ b/pkg/operator/starter.go @@ -1,17 +1,17 @@ package operator import ( - "bytes" "context" "fmt" - "text/template" "time" "github.com/openshift/library-go/pkg/controller/factory" "github.com/openshift/library-go/pkg/operator/events" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeclient "k8s.io/client-go/kubernetes" + corev1listers "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/rest" "k8s.io/klog/v2" @@ -52,6 +52,11 @@ func RunOperator(ctx context.Context, controllerConfig *controllercmd.Controller configClient := configclient.NewForConfigOrDie(rest.AddUserAgent(controllerConfig.KubeConfig, operatorName)) configInformers := configinformers.NewSharedInformerFactory(configClient, 20*time.Minute) + // Create informer for the ConfigMaps in the openshift-config-managed namespace. This is used to get the custom CA + // bundle to use when accessing the AWS API. + cloudConfigInformer := kubeInformersForNamespaces.InformersFor(cloudConfigNamespace).Core().V1().ConfigMaps() + cloudConfigLister := cloudConfigInformer.Lister().ConfigMaps(cloudConfigNamespace) + // Create GenericOperatorclient. This is used by the library-go controllers created down below gvr := opv1.SchemeGroupVersion.WithResource("clustercsidrivers") operatorClient, dynamicInformers, err := goc.NewClusterScopedOperatorClientWithConfigName(controllerConfig.KubeConfig, gvr, instanceName) @@ -92,13 +97,14 @@ func RunOperator(ctx context.Context, controllerConfig *controllercmd.Controller configInformers, ).WithCSIDriverControllerService( "AWSEBSDriverControllerServiceController", - withCustomCABundle(generated.MustAsset, kubeClient), + generated.MustAsset, "controller.yaml", kubeClient, kubeInformersForNamespaces.InformersFor(defaultNamespace), configInformers, csidrivercontrollerservicecontroller.WithSecretHashAnnotationHook(defaultNamespace, secretName, secretInformer), csidrivercontrollerservicecontroller.WithObservedProxyDeploymentHook(), + withCustomCABundle(cloudConfigLister), ).WithCSIDriverNodeService( "AWSEBSDriverNodeServiceController", generated.MustAsset, @@ -106,7 +112,10 @@ func RunOperator(ctx context.Context, controllerConfig *controllercmd.Controller kubeClient, kubeInformersForNamespaces.InformersFor(defaultNamespace), csidrivernodeservicecontroller.WithObservedProxyDaemonSetHook(), - ).WithExtraInformers(secretInformer.Informer()) + ).WithExtraInformers( + secretInformer.Informer(), + cloudConfigInformer.Informer(), + ) if err != nil { return err @@ -143,22 +152,39 @@ type controllerTemplateData struct { // withCustomCABundle executes the asset as a template to fill out the parts required when using a custom CA bundle. // The `caBundleConfigMap` parameter specifies the name of the ConfigMap containing the custom CA bundle. If the // argument supplied is empty, then no custom CA bundle will be used. -func withCustomCABundle(assetFunc func(string) []byte, kubeClient kubeclient.Interface) func(string) []byte { - templateData := controllerTemplateData{} - switch used, err := isCustomCABundleUsed(kubeClient); { - case err != nil: - klog.Fatalf("could not determine if a custom CA bundle is in use: %v", err) - case used: - templateData.CABundleConfigMap = cloudConfigName - } - return func(name string) []byte { - asset := assetFunc(name) - template := template.Must(template.New("template").Parse(string(asset))) - buf := &bytes.Buffer{} - if err := template.Execute(buf, templateData); err != nil { - klog.Fatalf("Failed to execute ") +func withCustomCABundle(cloudConfigLister corev1listers.ConfigMapNamespaceLister) csidrivercontrollerservicecontroller.DeploymentHookFunc { + return func(_ *opv1.OperatorSpec, deployment *appsv1.Deployment) error { + switch used, err := isCustomCABundleUsed(cloudConfigLister); { + case err != nil: + return fmt.Errorf("could not determine if a custom CA bundle is in use: %w", err) + case !used: + return nil + } + deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: "ca-bundle", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: cloudConfigName}, + }, + }, + }) + for i := range deployment.Spec.Template.Spec.Containers { + container := &deployment.Spec.Template.Spec.Containers[i] + if container.Name != "csi-driver" { + continue + } + container.Env = append(container.Env, corev1.EnvVar{ + Name: "AWS_CA_BUNDLE", + Value: "/etc/ca/ca-bundle.pem", + }) + container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ + Name: "ca-bundle", + MountPath: "/etc/ca", + ReadOnly: true, + }) + return nil } - return buf.Bytes() + return fmt.Errorf("could not use custom CA bundle because the csi-driver container is missing from the deployment") } } @@ -192,10 +218,8 @@ func newCustomCABundleSyncer( } // isCustomCABundleUsed returns true if the cloud config ConfigMap exists and contains a custom CA bundle. -func isCustomCABundleUsed(kubeClient kubeclient.Interface) (bool, error) { - cloudConfigCM, err := kubeClient.CoreV1(). - ConfigMaps(cloudConfigNamespace). - Get(context.Background(), cloudConfigName, metav1.GetOptions{}) +func isCustomCABundleUsed(cloudConfigLister corev1listers.ConfigMapNamespaceLister) (bool, error) { + cloudConfigCM, err := cloudConfigLister.Get(cloudConfigName) if errors.IsNotFound(err) { // no cloud config ConfigMap so there is no CA bundle return false, nil diff --git a/pkg/operator/starter_test.go b/pkg/operator/starter_test.go index ecf9d8fb0..f6d6722d4 100644 --- a/pkg/operator/starter_test.go +++ b/pkg/operator/starter_test.go @@ -2,366 +2,49 @@ package operator import ( "testing" + "time" + "github.com/openshift/library-go/pkg/operator/v1helpers" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes/fake" - - "github.com/openshift/aws-ebs-csi-driver-operator/pkg/generated" ) -const controllerWithoutCABundle = `kind: Deployment -apiVersion: apps/v1 -metadata: - name: aws-ebs-csi-driver-controller - namespace: openshift-cluster-csi-drivers - annotations: - config.openshift.io/inject-proxy: csi-driver -spec: - selector: - matchLabels: - app: aws-ebs-csi-driver-controller - serviceName: aws-ebs-csi-driver-controller - replicas: 1 - template: - metadata: - labels: - app: aws-ebs-csi-driver-controller - spec: - hostNetwork: true - serviceAccount: aws-ebs-csi-driver-controller-sa - priorityClassName: system-cluster-critical - nodeSelector: - node-role.kubernetes.io/master: "" - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - key: node-role.kubernetes.io/master - operator: Exists - effect: "NoSchedule" - containers: - - name: csi-driver - image: ${DRIVER_IMAGE} - args: - - --endpoint=$(CSI_ENDPOINT) - - --k8s-tag-cluster-id=${CLUSTER_ID} - - --logtostderr - - --v=${LOG_LEVEL} - env: - - name: CSI_ENDPOINT - value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: ebs-cloud-credentials - key: aws_access_key_id - optional: true - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: ebs-cloud-credentials - key: aws_secret_access_key - optional: true - - name: AWS_SDK_LOAD_CONFIG - value: '1' - - name: AWS_CONFIG_FILE - value: /var/run/secrets/aws/credentials - ports: - - name: healthz - # Due to hostNetwork, this port is open on a node! - containerPort: 10301 - protocol: TCP - volumeMounts: - - name: aws-credentials - mountPath: /var/run/secrets/aws - readOnly: true - - name: bound-sa-token - mountPath: /var/run/secrets/openshift/serviceaccount - readOnly: true - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - resources: - requests: - memory: 50Mi - cpu: 10m - - name: csi-provisioner - image: ${PROVISIONER_IMAGE} - args: - - --csi-address=$(ADDRESS) - - --default-fstype=ext4 - - --feature-gates=Topology=true - - --extra-create-metadata=true - - --v=${LOG_LEVEL} - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - resources: - requests: - memory: 50Mi - cpu: 10m - - name: csi-attacher - image: ${ATTACHER_IMAGE} - args: - - --csi-address=$(ADDRESS) - - --v=${LOG_LEVEL} - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - resources: - requests: - memory: 50Mi - cpu: 10m - - name: csi-resizer - image: ${RESIZER_IMAGE} - args: - - --csi-address=$(ADDRESS) - - --timeout=300s - - --v=${LOG_LEVEL} - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - resources: - requests: - memory: 50Mi - cpu: 10m - - name: csi-snapshotter - image: ${SNAPSHOTTER_IMAGE} - args: - - --csi-address=$(ADDRESS) - - --v=${LOG_LEVEL} - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - volumeMounts: - - mountPath: /var/lib/csi/sockets/pluginproxy/ - name: socket-dir - resources: - requests: - memory: 50Mi - cpu: 10m - - name: csi-liveness-probe - image: ${LIVENESS_PROBE_IMAGE} - args: - - --csi-address=/csi/csi.sock - - --probe-timeout=3s - - --health-port=10301 - volumeMounts: - - name: socket-dir - mountPath: /csi - resources: - requests: - memory: 50Mi - cpu: 10m - volumes: - - name: aws-credentials - secret: - secretName: ebs-cloud-credentials - # This service account token can be used to provide identity outside the cluster. - # For example, this token can be used with AssumeRoleWithWebIdentity to authenticate with AWS using IAM OIDC provider and STS. - - name: bound-sa-token - projected: - sources: - - serviceAccountToken: - path: token - audience: openshift - - name: socket-dir - emptyDir: {} -` - -const controllerWithCABundle = `kind: Deployment -apiVersion: apps/v1 -metadata: - name: aws-ebs-csi-driver-controller - namespace: openshift-cluster-csi-drivers - annotations: - config.openshift.io/inject-proxy: csi-driver -spec: - selector: - matchLabels: - app: aws-ebs-csi-driver-controller - serviceName: aws-ebs-csi-driver-controller - replicas: 1 - template: - metadata: - labels: - app: aws-ebs-csi-driver-controller - spec: - hostNetwork: true - serviceAccount: aws-ebs-csi-driver-controller-sa - priorityClassName: system-cluster-critical - nodeSelector: - node-role.kubernetes.io/master: "" - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - key: node-role.kubernetes.io/master - operator: Exists - effect: "NoSchedule" - containers: - - name: csi-driver - image: ${DRIVER_IMAGE} - args: - - --endpoint=$(CSI_ENDPOINT) - - --k8s-tag-cluster-id=${CLUSTER_ID} - - --logtostderr - - --v=${LOG_LEVEL} - env: - - name: CSI_ENDPOINT - value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: ebs-cloud-credentials - key: aws_access_key_id - optional: true - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: ebs-cloud-credentials - key: aws_secret_access_key - optional: true - - name: AWS_SDK_LOAD_CONFIG - value: '1' - - name: AWS_CONFIG_FILE - value: /var/run/secrets/aws/credentials - - name: AWS_CA_BUNDLE - value: /etc/ca/ca-bundle.pem - ports: - - name: healthz - # Due to hostNetwork, this port is open on a node! - containerPort: 10301 - protocol: TCP - volumeMounts: - - name: aws-credentials - mountPath: /var/run/secrets/aws - readOnly: true - - name: bound-sa-token - mountPath: /var/run/secrets/openshift/serviceaccount - readOnly: true - - name: ca-bundle - mountPath: /etc/ca - readOnly: true - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - resources: - requests: - memory: 50Mi - cpu: 10m - - name: csi-provisioner - image: ${PROVISIONER_IMAGE} - args: - - --csi-address=$(ADDRESS) - - --default-fstype=ext4 - - --feature-gates=Topology=true - - --extra-create-metadata=true - - --v=${LOG_LEVEL} - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - resources: - requests: - memory: 50Mi - cpu: 10m - - name: csi-attacher - image: ${ATTACHER_IMAGE} - args: - - --csi-address=$(ADDRESS) - - --v=${LOG_LEVEL} - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - resources: - requests: - memory: 50Mi - cpu: 10m - - name: csi-resizer - image: ${RESIZER_IMAGE} - args: - - --csi-address=$(ADDRESS) - - --timeout=300s - - --v=${LOG_LEVEL} - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - resources: - requests: - memory: 50Mi - cpu: 10m - - name: csi-snapshotter - image: ${SNAPSHOTTER_IMAGE} - args: - - --csi-address=$(ADDRESS) - - --v=${LOG_LEVEL} - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - volumeMounts: - - mountPath: /var/lib/csi/sockets/pluginproxy/ - name: socket-dir - resources: - requests: - memory: 50Mi - cpu: 10m - - name: csi-liveness-probe - image: ${LIVENESS_PROBE_IMAGE} - args: - - --csi-address=/csi/csi.sock - - --probe-timeout=3s - - --health-port=10301 - volumeMounts: - - name: socket-dir - mountPath: /csi - resources: - requests: - memory: 50Mi - cpu: 10m - volumes: - - name: aws-credentials - secret: - secretName: ebs-cloud-credentials - # This service account token can be used to provide identity outside the cluster. - # For example, this token can be used with AssumeRoleWithWebIdentity to authenticate with AWS using IAM OIDC provider and STS. - - name: bound-sa-token - projected: - sources: - - serviceAccountToken: - path: token - audience: openshift - - name: ca-bundle - configMap: - name: kube-cloud-config - - name: socket-dir - emptyDir: {} -` - func TestWithCustomCABundle(t *testing.T) { cases := []struct { - name string - cm *corev1.ConfigMap - expected string + name string + cm *corev1.ConfigMap + inDeployment *appsv1.Deployment + expected *appsv1.Deployment }{ { - name: "no configmap", - expected: controllerWithoutCABundle, + name: "no configmap", + inDeployment: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "csi-driver", + }}, + }, + }, + }, + }, + expected: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "csi-driver", + }}, + }, + }, + }, + }, }, { name: "no CA bundle in configmap", @@ -374,7 +57,28 @@ func TestWithCustomCABundle(t *testing.T) { "other-key": "other-data", }, }, - expected: controllerWithoutCABundle, + inDeployment: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "csi-driver", + }}, + }, + }, + }, + }, + expected: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "csi-driver", + }}, + }, + }, + }, + }, }, { name: "custom CA bundle", @@ -387,7 +91,45 @@ func TestWithCustomCABundle(t *testing.T) { "ca-bundle.pem": "a custom bundle", }, }, - expected: controllerWithCABundle, + inDeployment: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "csi-driver", + }}, + }, + }, + }, + }, + expected: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "csi-driver", + Env: []corev1.EnvVar{{ + Name: "AWS_CA_BUNDLE", + Value: "/etc/ca/ca-bundle.pem", + }}, + VolumeMounts: []corev1.VolumeMount{{ + Name: "ca-bundle", + MountPath: "/etc/ca", + ReadOnly: true, + }}, + }}, + Volumes: []corev1.Volume{{ + Name: "ca-bundle", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: cloudConfigName}, + }, + }, + }}, + }, + }, + }, + }, }, } for _, tc := range cases { @@ -397,9 +139,22 @@ func TestWithCustomCABundle(t *testing.T) { resources = append(resources, tc.cm) } kubeClient := fake.NewSimpleClientset(resources...) - actual := string(withCustomCABundle(generated.MustAsset, kubeClient)("controller.yaml")) - if e, a := tc.expected, actual; e != a { - t.Errorf("unexpected controller asset\nexpected:\n%s\ngot:\n%s", e, a) + kubeInformersForNamespaces := v1helpers.NewKubeInformersForNamespaces(kubeClient, cloudConfigNamespace) + cloudConfigInformer := kubeInformersForNamespaces.InformersFor(cloudConfigNamespace).Core().V1().ConfigMaps() + cloudConfigLister := cloudConfigInformer.Lister().ConfigMaps(cloudConfigNamespace) + stopCh := make(chan struct{}) + go kubeInformersForNamespaces.Start(stopCh) + defer close(stopCh) + wait.Poll(100*time.Millisecond, 30*time.Second, func() (bool, error) { + return cloudConfigInformer.Informer().HasSynced(), nil + }) + deployment := tc.inDeployment.DeepCopy() + err := withCustomCABundle(cloudConfigLister)(nil, deployment) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if e, a := tc.expected, deployment; !equality.Semantic.DeepEqual(e, a) { + t.Errorf("unexpected deployment\nwant=%#v\ngot= %#v", e, a) } }) }