ceph: make provisioner replicas configurable

added new option to set the provisioner replicas. with this new option the user/admin can choose how many replicas he want for provisioner pod if number of nodes is greater than 1. fixes #8153 Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>
rook · Sep 22, 2021 · ed5f281 · ed5f281
1 parent 6bcb569
commit ed5f281
Show file tree

Hide file tree

Showing 6 changed files with 28 additions and 3 deletions.
diff --git a/Documentation/helm-operator.md b/Documentation/helm-operator.md
@@ -108,6 +108,7 @@ The following tables lists the configurable parameters of the rook-operator char
 | `csi.rbdFSGroupPolicy`              | Policy for modifying a volume's ownership or permissions when the RBD PVC is being mounted                                  | ReadWriteOnceWithFSType                                   |
 | `csi.cephFSFSGroupPolicy`           | Policy for modifying a volume's ownership or permissions when the CephFS PVC is being mounted                               | `None`                                                    |
 | `csi.logLevel`                      | Set logging level for csi containers. Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity. | `0`                                                       |
+| `csi.provisionerReplicas`           | Set replicas for csi provisioner deployment.                                                                                | `2`                                                       |
 | `csi.enableGrpcMetrics`             | Enable Ceph CSI GRPC Metrics.                                                                                               | `false`                                                   |
 | `csi.enableCSIHostNetwork`          | Enable Host Networking for Ceph CSI nodeplugins.                                                                            | `false`                                                   |
 | `csi.provisionerTolerations`        | Array of tolerations in YAML format which will be added to CSI provisioner deployment.                                      | <none>                                                    |

diff --git a/cluster/charts/rook-ceph/templates/deployment.yaml b/cluster/charts/rook-ceph/templates/deployment.yaml
@@ -275,6 +275,10 @@ spec:
         - name: CSI_LOG_LEVEL
           value: {{ .Values.csi.logLevel | quote }}
 {{- end }}
+{{- if .Values.csi.provisionerReplicas }}
+        - name: CSI_PROVISIONER_REPLICAS
+          value: {{ .Values.csi.provisionerReplicas | quote }}
+{{- end }}
 {{- if .Values.csi.csiRBDProvisionerResource }}
         - name: CSI_RBD_PROVISIONER_RESOURCE
           value: {{ .Values.csi.csiRBDProvisionerResource | quote }}

diff --git a/cluster/charts/rook-ceph/values.yaml b/cluster/charts/rook-ceph/values.yaml
@@ -87,6 +87,9 @@ csi:
   # sidecar with CSI provisioner pod, to enable set it to true.
   enableOMAPGenerator: false
 
+  # Set replicas for csi provisioner deployment.
+  provisionerReplicas: 2
+
   # Set logging level for csi containers.
   # Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity.
   #logLevel: 0

diff --git a/cluster/examples/kubernetes/ceph/operator-openshift.yaml b/cluster/examples/kubernetes/ceph/operator-openshift.yaml
@@ -120,6 +120,9 @@ data:
   # Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity.
   # CSI_LOG_LEVEL: "0"
 
+  # Set replicas for csi provisioner deployment.
+  CSI_PROVISIONER_REPLICAS: "2"
+
   # OMAP generator generates the omap mapping between the PV name and the RBD image
   # which helps CSI to identify the rbd images for CSI operations.
   # CSI_ENABLE_OMAP_GENERATOR need to be enabled when we are using rbd mirroring feature.

diff --git a/cluster/examples/kubernetes/ceph/operator.yaml b/cluster/examples/kubernetes/ceph/operator.yaml
@@ -41,6 +41,9 @@ data:
   # Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity.
   # CSI_LOG_LEVEL: "0"
 
+  # Set replicas for csi provisioner deployment.
+  CSI_PROVISIONER_REPLICAS: "2"
+
   # OMAP generator will generate the omap mapping between the PV name and the RBD image.
   # CSI_ENABLE_OMAP_GENERATOR need to be enabled when we are using rbd mirroring feature.
   # By default OMAP generator sidecar is deployed with CSI provisioner pod, to disable

diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go
@@ -61,7 +61,7 @@ type Param struct {
 	CephFSLivenessMetricsPort      uint16
 	RBDGRPCMetricsPort             uint16
 	RBDLivenessMetricsPort         uint16
-	ProvisionerReplicas            uint8
+	ProvisionerReplicas            int32
 	CSICephFSPodLabels             map[string]string
 	CSIRBDPodLabels                map[string]string
 }
@@ -169,6 +169,9 @@ const (
 	// default log level for csi containers
 	defaultLogLevel uint8 = 0
 
+	// default provisioner replicas
+	defaultProvisionerReplicas int32 = 2
+
 	// update strategy
 	rollingUpdate = "RollingUpdate"
 	onDelete      = "OnDelete"
@@ -345,14 +348,22 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
 		}
 	}
 
-	tp.ProvisionerReplicas = 2
+	tp.ProvisionerReplicas = defaultProvisionerReplicas
 	nodes, err := r.context.Clientset.CoreV1().Nodes().List(r.opManagerContext, metav1.ListOptions{})
 	if err == nil {
 		if len(nodes.Items) == 1 {
 			tp.ProvisionerReplicas = 1
+		} else {
+			replicas := k8sutil.GetValue(r.opConfig.Parameters, "CSI_PROVISIONER_REPLICAS", "2")
+			r, err := strconv.ParseInt(replicas, 10, 32)
+			if err != nil {
+				logger.Errorf("failed to parse CSI_PROVISIONER_REPLICAS. Defaulting to %d. %v", defaultProvisionerReplicas, err)
+			} else {
+				tp.ProvisionerReplicas = int32(r)
+			}
 		}
 	} else {
-		logger.Errorf("failed to get nodes. Defaulting the number of replicas of provisioner pods to 2. %v", err)
+		logger.Errorf("failed to get nodes. Defaulting the number of replicas of provisioner pods to %d. %v", tp.ProvisionerReplicas, err)
 	}
 
 	if EnableRBD {