diff --git a/pkg/operator/ceph/csi/controller.go b/pkg/operator/ceph/csi/controller.go index a905f1ddd9af..ba0bd36a6281 100644 --- a/pkg/operator/ceph/csi/controller.go +++ b/pkg/operator/ceph/csi/controller.go @@ -106,12 +106,23 @@ func (r *ReconcileCSI) Reconcile(context context.Context, request reconcile.Requ } func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, error) { + serverVersion, err := r.context.Clientset.Discovery().ServerVersion() + if err != nil { + return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to get server version") + } + // See if there is a CephCluster cephClusters := &cephv1.CephClusterList{} - err := r.client.List(r.opManagerContext, cephClusters, &client.ListOptions{}) + err = r.client.List(r.opManagerContext, cephClusters, &client.ListOptions{}) if err != nil { if kerrors.IsNotFound(err) { logger.Debug("no ceph cluster found not deploying ceph csi driver") + EnableRBD, EnableCephFS = false, false + err = r.stopDrivers(serverVersion) + if err != nil { + return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to stop Drivers") + } + return reconcile.Result{}, nil } // Error reading the object - requeue the request. @@ -121,6 +132,12 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e // // Do not nothing if no ceph cluster is present if len(cephClusters.Items) == 0 { logger.Debug("no ceph cluster found not deploying ceph csi driver") + EnableRBD, EnableCephFS = false, false + err = r.stopDrivers(serverVersion) + if err != nil { + return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to stop Drivers") + } + return reconcile.Result{}, nil } else { for _, cluster := range cephClusters.Items { @@ -155,11 +172,6 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e r.opConfig.Parameters = opConfig.Data } - serverVersion, err := r.context.Clientset.Discovery().ServerVersion() - if err != nil { - return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to get server version") - } - ownerRef, err := k8sutil.GetDeploymentOwnerReference(r.opManagerContext, r.context.Clientset, os.Getenv(k8sutil.PodNameEnvVar), r.opConfig.OperatorNamespace) if err != nil { logger.Warningf("could not find deployment owner reference to assign to csi drivers. %v", err) @@ -184,7 +196,7 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e err = r.validateAndConfigureDrivers(serverVersion, ownerInfo) if err != nil { - return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed configure ceph csi") + return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to configure ceph csi") } return reconcile.Result{}, nil diff --git a/pkg/operator/ceph/csi/csi.go b/pkg/operator/ceph/csi/csi.go index 17ce6c466bf6..726c48b87f9f 100644 --- a/pkg/operator/ceph/csi/csi.go +++ b/pkg/operator/ceph/csi/csi.go @@ -47,21 +47,13 @@ func (r *ReconcileCSI) validateAndConfigureDrivers(serverVersion *version.Info, } if CSIEnabled() { - maxRetries := 3 - for i := 0; i < maxRetries; i++ { - if err = r.startDrivers(serverVersion, ownerInfo, v); err != nil { - logger.Errorf("failed to start Ceph csi drivers, will retry starting csi drivers %d more times. %v", maxRetries-i-1, err) - } else { - break - } + if err = r.startDrivers(serverVersion, ownerInfo, v); err != nil { + return errors.Wrap(err, "failed to start ceph csi drivers") } - return errors.Wrap(err, "failed to start ceph csi drivers") } // Check whether RBD or CephFS needs to be disabled - r.stopDrivers(serverVersion) - - return nil + return r.stopDrivers(serverVersion) } func (r *ReconcileCSI) setParams() error { diff --git a/pkg/operator/ceph/csi/predicate.go b/pkg/operator/ceph/csi/predicate.go index bfe4055e8aed..8962a96e64a9 100644 --- a/pkg/operator/ceph/csi/predicate.go +++ b/pkg/operator/ceph/csi/predicate.go @@ -94,6 +94,13 @@ func predicateController(ctx context.Context, c client.Client, opNamespace strin if cm, ok := e.Object.(*v1.ConfigMap); ok { return cm.Name == opcontroller.OperatorSettingConfigMapName } + + // if cephCluster is deleted, trigger reconcile to cleanup the csi driver resources + // if zero cephClusters exist. + if _, ok := e.Object.(*cephv1.CephCluster); ok { + return true + } + return false }, diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index b5203f3a4bf8..b89987e489b9 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -221,11 +221,8 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI Param: CSIParam, Namespace: r.opConfig.OperatorNamespace, } - // if the user didn't specify a custom DriverNamePrefix use - // the namespace (and a dot). - if tp.DriverNamePrefix == "" { - tp.DriverNamePrefix = fmt.Sprintf("%s.", r.opConfig.OperatorNamespace) - } + + tp.DriverNamePrefix = fmt.Sprintf("%s.", r.opConfig.OperatorNamespace) CephFSDriverName = tp.DriverNamePrefix + "cephfs.csi.ceph.com" RBDDriverName = tp.DriverNamePrefix + "rbd.csi.ceph.com" @@ -561,58 +558,56 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI return nil } -func (r *ReconcileCSI) stopDrivers(ver *version.Info) { +func (r *ReconcileCSI) stopDrivers(ver *version.Info) error { + RBDDriverName = fmt.Sprintf("%s.rbd.csi.ceph.com", r.opConfig.OperatorNamespace) + CephFSDriverName = fmt.Sprintf("%s.cephfs.csi.ceph.com", r.opConfig.OperatorNamespace) + if !EnableRBD { logger.Info("CSI Ceph RBD driver disabled") - succeeded := r.deleteCSIDriverResources(ver, csiRBDPlugin, csiRBDProvisioner, "csi-rbdplugin-metrics", RBDDriverName) - if succeeded { - logger.Info("successfully removed CSI Ceph RBD driver") - } else { - logger.Error("failed to remove CSI Ceph RBD driver") + err := r.deleteCSIDriverResources(ver, csiRBDPlugin, csiRBDProvisioner, "csi-rbdplugin-metrics", RBDDriverName) + if err != nil { + return errors.Wrap(err, "failed to remove CSI Ceph RBD driver") } + logger.Info("successfully removed CSI Ceph RBD driver") } if !EnableCephFS { logger.Info("CSI CephFS driver disabled") - succeeded := r.deleteCSIDriverResources(ver, csiCephFSPlugin, csiCephFSProvisioner, "csi-cephfsplugin-metrics", CephFSDriverName) - if succeeded { - logger.Info("successfully removed CSI CephFS driver") - } else { - logger.Error("failed to remove CSI CephFS driver") + err := r.deleteCSIDriverResources(ver, csiCephFSPlugin, csiCephFSProvisioner, "csi-cephfsplugin-metrics", CephFSDriverName) + if err != nil { + return errors.Wrap(err, "failed to remove CSI CephFS driver") } + logger.Info("successfully removed CSI CephFS driver") } + + return nil } -func (r *ReconcileCSI) deleteCSIDriverResources(ver *version.Info, daemonset, deployment, service, driverName string) bool { - succeeded := true +func (r *ReconcileCSI) deleteCSIDriverResources(ver *version.Info, daemonset, deployment, service, driverName string) error { csiDriverobj = beta1CsiDriver{} if ver.Major > KubeMinMajor || ver.Major == KubeMinMajor && ver.Minor >= kubeMinVerForV1csiDriver { csiDriverobj = v1CsiDriver{} } err := k8sutil.DeleteDaemonset(r.opManagerContext, r.context.Clientset, r.opConfig.OperatorNamespace, daemonset) if err != nil { - logger.Errorf("failed to delete the %q. %v", daemonset, err) - succeeded = false + return errors.Wrapf(err, "failed to delete the %q", daemonset) } err = k8sutil.DeleteDeployment(r.opManagerContext, r.context.Clientset, r.opConfig.OperatorNamespace, deployment) if err != nil { - logger.Errorf("failed to delete the %q. %v", deployment, err) - succeeded = false + return errors.Wrapf(err, "failed to delete the %q", deployment) } err = k8sutil.DeleteService(r.opManagerContext, r.context.Clientset, r.opConfig.OperatorNamespace, service) if err != nil { - logger.Errorf("failed to delete the %q. %v", service, err) - succeeded = false + return errors.Wrapf(err, "failed to delete the %q", service) } err = csiDriverobj.deleteCSIDriverInfo(r.opManagerContext, r.context.Clientset, driverName) if err != nil { - logger.Errorf("failed to delete %q Driver Info. %v", driverName, err) - succeeded = false + return errors.Wrapf(err, "failed to delete %q Driver Info", driverName) } - return succeeded + return nil } func (r *ReconcileCSI) applyCephClusterNetworkConfig(ctx context.Context, objectMeta *metav1.ObjectMeta) (bool, error) {