diff --git a/cmd/install/assets/hypershift_operator.go b/cmd/install/assets/hypershift_operator.go index 43c704a110..265c55574b 100644 --- a/cmd/install/assets/hypershift_operator.go +++ b/cmd/install/assets/hypershift_operator.go @@ -835,6 +835,16 @@ func (o HyperShiftOperatorClusterRole) Build() *rbacv1.ClusterRole { Resources: []string{"virtualmachineinstances", "virtualmachines"}, Verbs: []string{"*"}, }, + { // This allows the kubevirt csi driver to hotplug volumes to KubeVirt VMs. + APIGroups: []string{"subresources.kubevirt.io"}, + Resources: []string{"virtualmachineinstances/addvolume", "virtualmachineinstances/removevolume"}, + Verbs: []string{"*"}, + }, + { // This allows the kubevirt csi driver to mirror guest PVCs to the mgmt/infra cluster + APIGroups: []string{"cdi.kubevirt.io"}, + Resources: []string{"datavolumes"}, + Verbs: []string{"*"}, + }, { // This allows hypershift operator to grant RBAC permissions for agents, clusterDeployments and agentClusterInstalls to the capi-provider-agent APIGroups: []string{"agent-install.openshift.io"}, Resources: []string{"agents"}, diff --git a/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/files/controller.yaml b/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/files/controller.yaml new file mode 100644 index 0000000000..b4bb087aeb --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/files/controller.yaml @@ -0,0 +1,109 @@ +# sourced from https://github.com/kubevirt/csi-driver/tree/main/deploy/split-infra-tenant +kind: Deployment +apiVersion: apps/v1 +metadata: + name: kubevirt-csi-controller +spec: + replicas: 1 + selector: + matchLabels: + app: kubevirt-csi-driver + template: + metadata: + labels: + app: kubevirt-csi-driver + spec: + serviceAccount: kubevirt-csi + priorityClassName: hypershift-control-plane + containers: + - name: csi-driver + imagePullPolicy: Always + image: quay.io/dvossel/kubevirt-csi-driver:latest + args: + - "--endpoint=$(CSI_ENDPOINT)" + - "--infra-cluster-namespace=$(INFRACLUSTER_NAMESPACE)" + - "--tenant-cluster-kubeconfig=/var/run/secrets/tenantcluster/kubeconfig" + - "--infra-cluster-labels=$(INFRACLUSTER_LABELS)" + - "--run-node-service=false" + - "--run-controller-service=true" + - --v=5 + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: INFRACLUSTER_NAMESPACE + valueFrom: + configMapKeyRef: + name: driver-config + key: infraClusterNamespace + - name: INFRACLUSTER_LABELS + valueFrom: + configMapKeyRef: + name: driver-config + key: infraClusterLabels + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: tenantcluster + mountPath: "/var/run/secrets/tenantcluster" + resources: + requests: + memory: 50Mi + cpu: 10m + - name: csi-provisioner + image: quay.io/openshift/origin-csi-external-provisioner:latest + args: + - --csi-address=$(ADDRESS) + - --default-fstype=ext4 + - --v=5 + - "--kubeconfig=/var/run/secrets/tenantcluster/kubeconfig" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: tenantcluster + mountPath: "/var/run/secrets/tenantcluster" + - name: csi-attacher + image: quay.io/openshift/origin-csi-external-attacher:latest + args: + - --csi-address=$(ADDRESS) + - --v=5 + - "--kubeconfig=/var/run/secrets/tenantcluster/kubeconfig" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: tenantcluster + mountPath: "/var/run/secrets/tenantcluster" + resources: + requests: + memory: 50Mi + cpu: 10m + - name: csi-liveness-probe + image: quay.io/openshift/origin-csi-livenessprobe:latest + args: + - --csi-address=/csi/csi.sock + - --probe-timeout=3s + - --health-port=10301 + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: tenantcluster + mountPath: "/var/run/secrets/tenantcluster" + resources: + requests: + memory: 50Mi + cpu: 10m + volumes: + - name: socket-dir + emptyDir: {} + - name: tenantcluster + secret: + secretName: kubevirt-csi-controller-tenant-kubeconfig diff --git a/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/files/daemonset.yaml b/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/files/daemonset.yaml new file mode 100644 index 0000000000..b53b88ce3a --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/files/daemonset.yaml @@ -0,0 +1,123 @@ +# sourced from https://github.com/kubevirt/csi-driver/tree/main/deploy/split-infra-tenant +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: kubevirt-csi-node +spec: + selector: + matchLabels: + app: kubevirt-csi-driver + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app: kubevirt-csi-driver + spec: + serviceAccount: kubevirt-csi-node-sa + priorityClassName: system-node-critical + tolerations: + - operator: Exists + containers: + - name: csi-driver + securityContext: + privileged: true + allowPrivilegeEscalation: true + imagePullPolicy: Always + image: quay.io/dvossel/kubevirt-csi-driver:latest + args: + - "--endpoint=unix:/csi/csi.sock" + - "--node-name=$(KUBE_NODE_NAME)" + - "--run-node-service=true" + - "--run-controller-service=false" + - --v=5 + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: kubelet-dir + mountPath: /var/lib/kubelet + mountPropagation: "Bidirectional" + - name: plugin-dir + mountPath: /csi + - name: device-dir + mountPath: /dev + - name: udev + mountPath: /run/udev + ports: + - name: healthz + containerPort: 10300 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + resources: + requests: + memory: 50Mi + cpu: 10m + - name: csi-node-driver-registrar + securityContext: + privileged: true + image: quay.io/openshift/origin-csi-node-driver-registrar:latest + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v=5 + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/csi.kubevirt.io-reg.sock /csi/csi.sock"] + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/csi.kubevirt.io/csi.sock + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + resources: + requests: + memory: 20Mi + cpu: 5m + - name: csi-liveness-probe + image: quay.io/openshift/origin-csi-livenessprobe:latest + args: + - --csi-address=/csi/csi.sock + - --probe-timeout=3s + - --health-port=10300 + volumeMounts: + - name: plugin-dir + mountPath: /csi + resources: + requests: + memory: 20Mi + cpu: 5m + volumes: + - name: kubelet-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/csi.kubevirt.io/ + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: device-dir + hostPath: + path: /dev + type: Directory + - name: udev + hostPath: + path: /run/udev diff --git a/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/files/infra_role.yaml b/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/files/infra_role.yaml new file mode 100644 index 0000000000..38dd51e32d --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/files/infra_role.yaml @@ -0,0 +1,15 @@ +# sourced from https://github.com/kubevirt/csi-driver/tree/main/deploy/split-infra-tenant +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kubevirt-csi +rules: +- apiGroups: ["cdi.kubevirt.io"] + resources: ["datavolumes"] + verbs: ["get", "create", "delete"] +- apiGroups: ["kubevirt.io"] + resources: ["virtualmachineinstances"] + verbs: ["list"] +- apiGroups: ["subresources.kubevirt.io"] + resources: ["virtualmachineinstances/addvolume", "virtualmachineinstances/removevolume"] + verbs: ["update"] diff --git a/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/files/infra_rolebinding.yaml b/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/files/infra_rolebinding.yaml new file mode 100644 index 0000000000..28792d4296 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/files/infra_rolebinding.yaml @@ -0,0 +1,12 @@ +# sourced from https://github.com/kubevirt/csi-driver/tree/main/deploy/split-infra-tenant +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kubevirt-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kubevirt-csi +subjects: +- kind: ServiceAccount + name: kubevirt-csi diff --git a/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/files/tenant_controller_clusterrole.yaml b/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/files/tenant_controller_clusterrole.yaml new file mode 100644 index 0000000000..6a3f8c194a --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/files/tenant_controller_clusterrole.yaml @@ -0,0 +1,55 @@ +# sourced from https://github.com/kubevirt/csi-driver/tree/main/deploy/split-infra-tenant +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubevirt-csi-controller-cr +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["create", "delete", "get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csidrivers"] + verbs: ["get", "list", "watch", "update", "create", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + verbs: ["use"] + resourceNames: ["privileged"] diff --git a/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/files/tenant_controller_clusterrolebinding.yaml b/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/files/tenant_controller_clusterrolebinding.yaml new file mode 100644 index 0000000000..026ee35d18 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/files/tenant_controller_clusterrolebinding.yaml @@ -0,0 +1,12 @@ +# sourced from https://github.com/kubevirt/csi-driver/tree/main/deploy/split-infra-tenant +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kubevirt-csi-controller-binding +subjects: + - kind: ServiceAccount + name: kubevirt-csi-controller-sa +roleRef: + kind: ClusterRole + name: kubevirt-csi-controller-cr + apiGroup: rbac.authorization.k8s.io diff --git a/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/files/tenant_node_clusterrole.yaml b/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/files/tenant_node_clusterrole.yaml new file mode 100644 index 0000000000..016ebeaaa0 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/files/tenant_node_clusterrole.yaml @@ -0,0 +1,37 @@ +# sourced from https://github.com/kubevirt/csi-driver/tree/main/deploy/split-infra-tenant +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubevirt-csi-node-cr +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "create", "delete", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + verbs: ["use"] + resourceNames: ["privileged"] diff --git a/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/files/tenant_node_clusterrolebinding.yaml b/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/files/tenant_node_clusterrolebinding.yaml new file mode 100644 index 0000000000..f6478b4fbd --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/files/tenant_node_clusterrolebinding.yaml @@ -0,0 +1,12 @@ +# sourced from https://github.com/kubevirt/csi-driver/tree/main/deploy/split-infra-tenant +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kubevirt-csi-node-binding +subjects: + - kind: ServiceAccount + name: kubevirt-csi-node-sa +roleRef: + kind: ClusterRole + name: kubevirt-csi-node-cr + apiGroup: rbac.authorization.k8s.io diff --git a/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/kubevirt.go b/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/kubevirt.go new file mode 100644 index 0000000000..6fd0001483 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/kubevirt.go @@ -0,0 +1,397 @@ +package kubevirt + +import ( + "bytes" + "context" + "embed" + "fmt" + "io" + + hyperv1 "github.com/openshift/hypershift/api/v1alpha1" + "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests" + "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/pki" + "github.com/openshift/hypershift/support/config" + "github.com/openshift/hypershift/support/upsert" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/util/yaml" + utilpointer "k8s.io/utils/pointer" + crclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +//go:embed files/* +var resources embed.FS + +var ( + controllerDeployment = mustDeployment("controller.yaml") + infraRole = mustRole("infra_role.yaml") + infraRoleBinding = mustRoleBinding("infra_rolebinding.yaml") + tenantControllerClusterRole = mustClusterRole("tenant_controller_clusterrole.yaml") + tenantControllerClusterRoleBinding = mustClusterRoleBinding("tenant_controller_clusterrolebinding.yaml") + + tenantNodeClusterRole = mustClusterRole("tenant_node_clusterrole.yaml") + tenantNodeClusterRoleBinding = mustClusterRoleBinding("tenant_node_clusterrolebinding.yaml") + + daemonset = mustDaemonSet("daemonset.yaml") +) + +func mustDeployment(file string) *appsv1.Deployment { + + controllerBytes := getContentsOrDie(file) + controller := &appsv1.Deployment{} + if err := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(controllerBytes), 500).Decode(&controller); err != nil { + panic(err) + } + + return controller +} + +func mustDaemonSet(file string) *appsv1.DaemonSet { + b := getContentsOrDie(file) + obj := &appsv1.DaemonSet{} + if err := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(b), 500).Decode(&obj); err != nil { + panic(err) + } + + return obj +} + +func mustClusterRole(file string) *rbacv1.ClusterRole { + b := getContentsOrDie(file) + obj := &rbacv1.ClusterRole{} + if err := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(b), 500).Decode(&obj); err != nil { + panic(err) + } + + return obj +} + +func mustClusterRoleBinding(file string) *rbacv1.ClusterRoleBinding { + b := getContentsOrDie(file) + obj := &rbacv1.ClusterRoleBinding{} + if err := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(b), 500).Decode(&obj); err != nil { + panic(err) + } + + return obj +} + +func mustRole(file string) *rbacv1.Role { + b := getContentsOrDie(file) + obj := &rbacv1.Role{} + if err := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(b), 500).Decode(&obj); err != nil { + panic(err) + } + + return obj +} + +func mustRoleBinding(file string) *rbacv1.RoleBinding { + b := getContentsOrDie(file) + obj := &rbacv1.RoleBinding{} + if err := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(b), 500).Decode(&obj); err != nil { + panic(err) + } + + return obj +} + +func getContentsOrDie(file string) []byte { + f, err := resources.Open("files/" + file) + if err != nil { + panic(err) + } + defer func() { + if err := f.Close(); err != nil { + panic(err) + } + }() + b, err := io.ReadAll(f) + if err != nil { + panic(err) + } + return b +} + +func reconcileInfraConfigMap(cm *corev1.ConfigMap) error { + cm.Data = map[string]string{ + "infraClusterNamespace": cm.Namespace, + "infraClusterLabels": "", + } + return nil +} + +func reconcileController(controller *appsv1.Deployment, componentImages map[string]string, deploymentConfig *config.DeploymentConfig) error { + controller.Spec = *controllerDeployment.Spec.DeepCopy() + + csiDriverImage, exists := componentImages["kubevirt-csi-driver"] + if !exists { + return fmt.Errorf("unable to detect kubevirt-csi-driver image from release payload") + } + + csiProvisionerImage, exists := componentImages["csi-external-provisioner"] + if !exists { + return fmt.Errorf("unable to detect csi-external-provisioner image from release payload") + } + + csiAttacherImage, exists := componentImages["csi-external-attacher"] + if !exists { + return fmt.Errorf("unable to detect csi-external-attacher image from release payload") + } + + csiLivenessProbeImage, exists := componentImages["csi-livenessprobe"] + if !exists { + return fmt.Errorf("unable to detect csi-livenessprobe image from release payload") + } + + for i, container := range controller.Spec.Template.Spec.Containers { + switch container.Name { + case "csi-driver": + controller.Spec.Template.Spec.Containers[i].Image = csiDriverImage + case "csi-provisioner": + controller.Spec.Template.Spec.Containers[i].Image = csiProvisionerImage + case "csi-attacher": + controller.Spec.Template.Spec.Containers[i].Image = csiAttacherImage + case "csi-liveness-probe": + controller.Spec.Template.Spec.Containers[i].Image = csiLivenessProbeImage + } + } + + deploymentConfig.ApplyTo(controller) + + return nil +} + +func reconcileInfraSA(sa *corev1.ServiceAccount) error { + return nil +} + +func reconcileInfraRole(role *rbacv1.Role) error { + role.Rules = infraRole.DeepCopy().Rules + return nil +} + +func reconcileInfraRoleBinding(roleBinding *rbacv1.RoleBinding) error { + dc := infraRoleBinding.DeepCopy() + + roleBinding.RoleRef = dc.RoleRef + roleBinding.Subjects = dc.Subjects + + for i := range roleBinding.Subjects { + roleBinding.Subjects[i].Namespace = roleBinding.Namespace + } + return nil +} + +func reconcileTenantControllerSA(sa *corev1.ServiceAccount) error { + return nil +} + +func reconcileTenantControllerClusterRole(cr *rbacv1.ClusterRole) error { + cr.Rules = tenantControllerClusterRole.DeepCopy().Rules + return nil +} + +func reconcileTenantControllerClusterRoleBinding(crb *rbacv1.ClusterRoleBinding, saNamespace string) error { + dc := tenantControllerClusterRoleBinding.DeepCopy() + + crb.RoleRef = dc.RoleRef + crb.Subjects = dc.Subjects + + for i := range crb.Subjects { + crb.Subjects[i].Namespace = saNamespace + } + return nil +} + +func reconcileTenantNodeSA(sa *corev1.ServiceAccount) error { + return nil +} + +func reconcileTenantNodeClusterRole(cr *rbacv1.ClusterRole) error { + cr.Rules = tenantNodeClusterRole.DeepCopy().Rules + return nil +} + +func reconcileTenantNodeClusterRoleBinding(crb *rbacv1.ClusterRoleBinding, saNamespace string) error { + dc := tenantNodeClusterRoleBinding.DeepCopy() + + crb.RoleRef = dc.RoleRef + crb.Subjects = dc.Subjects + + for i := range crb.Subjects { + crb.Subjects[i].Namespace = saNamespace + } + return nil +} + +func reconcileTenantDaemonset(ds *appsv1.DaemonSet, componentImages map[string]string) error { + ds.Spec = *daemonset.Spec.DeepCopy() + + csiDriverImage, exists := componentImages["kubevirt-csi-driver"] + if !exists { + return fmt.Errorf("unable to detect kubevirt-csi-driver image from release payload") + } + + csiNodeDriverRegistrarImage, exists := componentImages["csi-node-driver-registrar"] + if !exists { + return fmt.Errorf("unable to detect csi-node-driver-registrar image from release payload") + } + + csiLivenessProbeImage, exists := componentImages["csi-livenessprobe"] + if !exists { + return fmt.Errorf("unable to detect csi-livenessprobe image from release payload") + } + + for i, container := range ds.Spec.Template.Spec.Containers { + switch container.Name { + case "csi-driver": + ds.Spec.Template.Spec.Containers[i].Image = csiDriverImage + case "csi-node-driver-registrar": + ds.Spec.Template.Spec.Containers[i].Image = csiNodeDriverRegistrarImage + case "csi-liveness-probe": + ds.Spec.Template.Spec.Containers[i].Image = csiLivenessProbeImage + } + } + + return nil +} + +func ReconcileTenant(client crclient.Client, hcp *hyperv1.HostedControlPlane, ctx context.Context, createOrUpdate upsert.CreateOrUpdateFN, componentImages map[string]string) error { + + tenantNamespace := manifests.KubevirtCSIDriverTenantNamespaceStr + + ns := manifests.KubevirtCSIDriverTenantNamespace(tenantNamespace) + _, err := createOrUpdate(ctx, client, ns, func() error { return nil }) + if err != nil { + return err + } + + tenantNodeServiceAccount := manifests.KubevirtCSIDriverTenantNodeSA(tenantNamespace) + _, err = createOrUpdate(ctx, client, tenantNodeServiceAccount, func() error { + return reconcileTenantNodeSA(tenantNodeServiceAccount) + }) + if err != nil { + return err + } + + tenantNodeClusterRole := manifests.KubevirtCSIDriverTenantNodeClusterRole(tenantNamespace) + _, err = createOrUpdate(ctx, client, tenantNodeClusterRole, func() error { + return reconcileTenantNodeClusterRole(tenantNodeClusterRole) + }) + if err != nil { + return err + } + + tenantNodeClusterRoleBinding := manifests.KubevirtCSIDriverTenantNodeClusterRoleBinding(tenantNamespace) + _, err = createOrUpdate(ctx, client, tenantNodeClusterRoleBinding, func() error { + return reconcileTenantNodeClusterRoleBinding(tenantNodeClusterRoleBinding, tenantNamespace) + }) + if err != nil { + return err + } + + tenantControllerClusterRoleBinding := manifests.KubevirtCSIDriverTenantControllerClusterRoleBinding(tenantNamespace) + _, err = createOrUpdate(ctx, client, tenantControllerClusterRoleBinding, func() error { + return reconcileTenantControllerClusterRoleBinding(tenantControllerClusterRoleBinding, tenantNamespace) + }) + if err != nil { + return err + } + + tenantControllerClusterRole := manifests.KubevirtCSIDriverTenantControllerClusterRole(tenantNamespace) + _, err = createOrUpdate(ctx, client, tenantControllerClusterRole, func() error { + return reconcileTenantControllerClusterRole(tenantControllerClusterRole) + }) + if err != nil { + return err + } + + tenantControllerServiceAccount := manifests.KubevirtCSIDriverTenantControllerSA(tenantNamespace) + _, err = createOrUpdate(ctx, client, tenantControllerServiceAccount, func() error { + return reconcileTenantControllerSA(tenantControllerServiceAccount) + }) + if err != nil { + return err + } + + daemonSet := manifests.KubevirtCSIDriverDaemonSet(tenantNamespace) + _, err = createOrUpdate(ctx, client, daemonSet, func() error { + return reconcileTenantDaemonset(daemonSet, componentImages) + }) + if err != nil { + return err + } + + return nil +} + +// ReconcileInfra reconciles the csi driver controller on the underlying infra/Mgmt cluster +// that is hosting the KubeVirt VMs. +func ReconcileInfra(client crclient.Client, hcp *hyperv1.HostedControlPlane, ctx context.Context, createOrUpdate upsert.CreateOrUpdateFN, componentImages map[string]string) error { + + deploymentConfig := &config.DeploymentConfig{} + deploymentConfig.Scheduling.PriorityClass = config.DefaultPriorityClass + deploymentConfig.SetRestartAnnotation(hcp.ObjectMeta) + deploymentConfig.SetDefaults(hcp, nil, utilpointer.IntPtr(1)) + + infraNamespace := hcp.Namespace + + infraServiceAccount := manifests.KubevirtCSIDriverInfraSA(infraNamespace) + _, err := createOrUpdate(ctx, client, infraServiceAccount, func() error { + return reconcileInfraSA(infraServiceAccount) + }) + if err != nil { + return err + } + + infraRole := manifests.KubevirtCSIDriverInfraRole(infraNamespace) + _, err = createOrUpdate(ctx, client, infraRole, func() error { + return reconcileInfraRole(infraRole) + }) + if err != nil { + return err + } + + infraRoleBinding := manifests.KubevirtCSIDriverInfraRoleBinding(infraNamespace) + _, err = createOrUpdate(ctx, client, infraRoleBinding, func() error { + return reconcileInfraRoleBinding(infraRoleBinding) + }) + if err != nil { + return err + } + + rootCA := manifests.RootCASecret(hcp.Namespace) + if err := client.Get(ctx, crclient.ObjectKeyFromObject(rootCA), rootCA); err != nil { + return fmt.Errorf("failed to get root ca cert secret: %w", err) + } + if err != nil { + return err + } + tenantControllerKubeconfigSecret := manifests.KubevirtCSIDriverTenantKubeConfig(infraNamespace) + _, err = createOrUpdate(ctx, client, tenantControllerKubeconfigSecret, func() error { + return pki.ReconcileServiceAccountKubeconfig(tenantControllerKubeconfigSecret, rootCA, hcp, manifests.KubevirtCSIDriverTenantNamespaceStr, "kubevirt-csi-controller-sa") + }) + if err != nil { + return err + } + + infraConfigMap := manifests.KubevirtCSIDriverInfraConfigMap(infraNamespace) + _, err = createOrUpdate(ctx, client, infraConfigMap, func() error { + return reconcileInfraConfigMap(infraConfigMap) + }) + if err != nil { + return err + } + + controller := manifests.KubevirtCSIDriverController(infraNamespace) + _, err = createOrUpdate(ctx, client, controller, func() error { + return reconcileController(controller, componentImages, deploymentConfig) + }) + if err != nil { + return err + } + + return nil +} diff --git a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go index 66a2278f55..d24f988edc 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go +++ b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go @@ -23,6 +23,7 @@ import ( "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/cno" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/common" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/configoperator" + kubevirtcsi "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/cvo" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/dnsoperator" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/etcd" @@ -83,6 +84,8 @@ const ( ImageStreamClusterMachineApproverImage = "cluster-machine-approver" ) +var NoopReconcile controllerutil.MutateFn = func() error { return nil } + type InfrastructureStatus struct { APIHost string APIPort int32 @@ -825,6 +828,12 @@ func (r *HostedControlPlaneReconciler) reconcile(ctx context.Context, hostedCont return fmt.Errorf("failed to ensure control plane: %w", err) } + // Reconcile cloud csi driver + r.Log.Info("Reconciling CSI Driver") + if err := r.reconcileCSIDriver(ctx, hostedControlPlane, releaseImage, createOrUpdate); err != nil { + return fmt.Errorf("failed to reconcile csi driver: %w", err) + } + return nil } @@ -1429,6 +1438,20 @@ func (r *HostedControlPlaneReconciler) reconcilePKI(ctx context.Context, hcp *hy return nil } +func (r *HostedControlPlaneReconciler) reconcileCSIDriver(ctx context.Context, hcp *hyperv1.HostedControlPlane, releaseImage *releaseinfo.ReleaseImage, createOrUpdate upsert.CreateOrUpdateFN) error { + switch hcp.Spec.Platform.Type { + // Most csi drivers should be laid down by the Cluster Storage Operator (CSO) instead of + // the hcp operator. Only KubeVirt is unique at the moment. + case hyperv1.KubevirtPlatform: + err := kubevirtcsi.ReconcileInfra(r.Client, hcp, ctx, createOrUpdate, releaseImage.ComponentImages()) + if err != nil { + return err + } + } + + return nil +} + func (r *HostedControlPlaneReconciler) reconcileCloudProviderConfig(ctx context.Context, hcp *hyperv1.HostedControlPlane, createOrUpdate upsert.CreateOrUpdateFN) error { switch hcp.Spec.Platform.Type { case hyperv1.AWSPlatform: diff --git a/control-plane-operator/controllers/hostedcontrolplane/manifests/kubevirt.go b/control-plane-operator/controllers/hostedcontrolplane/manifests/kubevirt.go new file mode 100644 index 0000000000..4984ec4d84 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/manifests/kubevirt.go @@ -0,0 +1,135 @@ +package manifests + +import ( + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const KubevirtCSIDriverTenantNamespaceStr = "openshift-cluster-csi-drivers" + +func KubevirtCSIDriverController(ns string) *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kubevirt-csi-controller", + Namespace: ns, + }, + } +} + +func KubevirtCSIDriverInfraConfigMap(ns string) *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "driver-config", + Namespace: ns, + }, + } +} + +func KubevirtCSIDriverTenantKubeConfig(ns string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kubevirt-csi-controller-tenant-kubeconfig", + Namespace: ns, + }, + } +} + +func KubevirtCSIDriverInfraSA(ns string) *corev1.ServiceAccount { + return &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kubevirt-csi", + Namespace: ns, + }, + } +} + +func KubevirtCSIDriverInfraRole(ns string) *rbacv1.Role { + return &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kubevirt-csi", + Namespace: ns, + }, + } +} + +func KubevirtCSIDriverInfraRoleBinding(ns string) *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kubevirt-csi", + Namespace: ns, + }, + } +} + +func KubevirtCSIDriverTenantControllerSA(ns string) *corev1.ServiceAccount { + return &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kubevirt-csi-controller-sa", + Namespace: ns, + }, + } +} + +func KubevirtCSIDriverTenantControllerClusterRole(ns string) *rbacv1.ClusterRole { + return &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kubevirt-csi-controller-cr", + Namespace: ns, + }, + } +} + +func KubevirtCSIDriverTenantControllerClusterRoleBinding(ns string) *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kubevirt-csi-controller-binding", + Namespace: ns, + }, + } +} + +func KubevirtCSIDriverTenantNodeSA(ns string) *corev1.ServiceAccount { + return &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kubevirt-csi-node-sa", + Namespace: ns, + }, + } +} + +func KubevirtCSIDriverTenantNodeClusterRole(ns string) *rbacv1.ClusterRole { + return &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kubevirt-csi-node-cr", + Namespace: ns, + }, + } +} + +func KubevirtCSIDriverTenantNodeClusterRoleBinding(ns string) *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kubevirt-csi-node-binding", + Namespace: ns, + }, + } +} + +func KubevirtCSIDriverDaemonSet(ns string) *appsv1.DaemonSet { + return &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kubevirt-csi-node", + Namespace: ns, + }, + } +} + +func KubevirtCSIDriverTenantNamespace(ns string) *corev1.Namespace { + return &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: ns, + }, + } +} diff --git a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go index 8864f288f5..9581016a72 100644 --- a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go +++ b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go @@ -34,6 +34,7 @@ import ( operatorv1 "github.com/openshift/api/operator/v1" hyperv1 "github.com/openshift/hypershift/api/v1alpha1" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/cloud/azure" + kubevirtcsi "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt" cpomanifests "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests" alerts "github.com/openshift/hypershift/control-plane-operator/hostedclusterconfigoperator/controllers/resources/alerts" "github.com/openshift/hypershift/control-plane-operator/hostedclusterconfigoperator/controllers/resources/crd" @@ -468,6 +469,11 @@ func (r *reconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result log.Info("reconciling observed configuration") errs = append(errs, r.reconcileObservedConfiguration(ctx, hcp)...) + log.Info("reconciling node level csi configuration") + if err := r.reconcileCSIDriver(ctx, hcp, releaseImage); err != nil { + errs = append(errs, r.reconcileObservedConfiguration(ctx, hcp)...) + } + // Delete the DNS operator deployment in the hosted cluster, if it is // present there. A separate DNS operator deployment runs as part of // the hosted control-plane, but an upgraded cluster might still have @@ -491,6 +497,20 @@ func (r *reconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result return ctrl.Result{}, errors.NewAggregate(errs) } +func (r *reconciler) reconcileCSIDriver(ctx context.Context, hcp *hyperv1.HostedControlPlane, releaseImage *releaseinfo.ReleaseImage) error { + switch hcp.Spec.Platform.Type { + case hyperv1.KubevirtPlatform: + // Most csi drivers should be laid down by the Cluster Storage Operator (CSO) instead of + // the hcco operator. Only KubeVirt is unique at the moment. + err := kubevirtcsi.ReconcileTenant(r.client, hcp, ctx, r.CreateOrUpdate, releaseImage.ComponentImages()) + if err != nil { + return err + } + } + + return nil +} + func (r *reconciler) reconcileCRDs(ctx context.Context) error { var errs []error diff --git a/hack/app-sre/saas_template.yaml b/hack/app-sre/saas_template.yaml index 2dc0b12ace..8ddff50ef7 100644 --- a/hack/app-sre/saas_template.yaml +++ b/hack/app-sre/saas_template.yaml @@ -205,6 +205,19 @@ objects: - virtualmachines verbs: - '*' + - apiGroups: + - subresources.kubevirt.io + resources: + - virtualmachineinstances/addvolume + - virtualmachineinstances/removevolume + verbs: + - '*' + - apiGroups: + - cdi.kubevirt.io + resources: + - datavolumes + verbs: + - '*' - apiGroups: - agent-install.openshift.io resources: diff --git a/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go b/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go index 47b154fef0..95a5dae9cc 100644 --- a/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go +++ b/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go @@ -2259,6 +2259,31 @@ func reconcileControlPlaneOperatorRole(role *rbacv1.Role) error { Resources: []string{"virtualmachines", "virtualmachineinstances"}, Verbs: []string{rbacv1.VerbAll}, }, + { + APIGroups: []string{ + "cdi.kubevirt.io", + }, + Resources: []string{ + "datavolumes", + }, + Verbs: []string{ + "get", + "create", + "delete", + }, + }, + { + APIGroups: []string{ + "subresources.kubevirt.io", + }, + Resources: []string{ + "virtualmachineinstances/addvolume", + "virtualmachineinstances/removevolume", + }, + Verbs: []string{ + "update", + }, + }, } return nil }