csi: add log rotation for csi pod containers

1) Make the csi container logs persisted in a file 2) Add log rotation to rotate the log file 3) Provide api specs to configure the log rotate 4) Add a sidecar log collector container closes: #12809 Signed-off-by: parth-gr <partharora1010@gmail.com>
rook · Jun 20, 2024 · ac75b2a · ac75b2a
1 parent c4e99c1
commit ac75b2a
Show file tree

Hide file tree

Showing 8 changed files with 175 additions and 19 deletions.
diff --git a/pkg/operator/ceph/controller/spec.go b/pkg/operator/ceph/controller/spec.go
@@ -757,9 +757,7 @@ func PrivilegedContext(runAsRoot bool) *v1.SecurityContext {
 	return sec
 }
 
-// LogCollectorContainer rotate logs
-func LogCollectorContainer(daemonID, ns string, c cephv1.ClusterSpec) *v1.Container {
-
+func GetLogRotateConfig(c cephv1.ClusterSpec) (resource.Quantity, string) {
 	var maxLogSize resource.Quantity
 	if c.LogCollector.MaxLogSize != nil {
 		size := c.LogCollector.MaxLogSize.Value() / 1000 / 1000
@@ -771,20 +769,28 @@ func LogCollectorContainer(daemonID, ns string, c cephv1.ClusterSpec) *v1.Contai
 		maxLogSize = resource.MustParse(fmt.Sprintf("%dM", size))
 	}
 
-	rotation := "7"
-	if strings.Contains(daemonID, "-client.rbd-mirror") {
-		rotation = "28"
-	}
-
 	var periodicity string
-	if c.LogCollector.Periodicity == "1h" || c.LogCollector.Periodicity == "hourly" {
+	switch c.LogCollector.Periodicity {
+	case "1h", "hourly":
 		periodicity = "hourly"
-	} else if c.LogCollector.Periodicity == "weekly" || c.LogCollector.Periodicity == "monthly" {
+	case "weekly", "monthly":
 		periodicity = c.LogCollector.Periodicity
-	} else {
+	default:
 		periodicity = "daily"
 	}
 
+	return maxLogSize, periodicity
+}
+
+// LogCollectorContainer rotate logs
+func LogCollectorContainer(daemonID, ns string, c cephv1.ClusterSpec) *v1.Container {
+	maxLogSize, periodicity := GetLogRotateConfig(c)
+
+	rotation := "7"
+	if strings.Contains(daemonID, "-client.rbd-mirror") {
+		rotation = "28"
+	}
+
 	logger.Debugf("setting periodicity to %q. Supported periodicity are hourly, daily, weekly and monthly", periodicity)
 
 	return &v1.Container{

diff --git a/pkg/operator/ceph/csi/controller.go b/pkg/operator/ceph/csi/controller.go
@@ -205,7 +205,7 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e
 		return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to list ceph clusters")
 	}
 
-	// // Do not nothing if no ceph cluster is present
+	// Do nothing if no ceph cluster is present
 	if len(cephClusters.Items) == 0 {
 		logger.Debug("no ceph cluster found not deploying ceph csi driver")
 		EnableRBD, EnableCephFS, EnableNFS = false, false, false
@@ -217,6 +217,10 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e
 		return reconcile.Result{}, nil
 	}
 
+	// if at least one cephcluster is present update the csi lograte sidecar
+	// with the first listed ceph cluster specs with logrotate enabled
+	setCSILogrotateParams(cephClusters.Items)
+
 	err = peermap.CreateOrUpdateConfig(r.opManagerContext, r.context, &peermap.PeerIDMappings{})
 	if err != nil {
 		return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to create pool ID mapping config map")
@@ -299,3 +303,21 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e
 
 	return reconcileResult, nil
 }
+
+func setCSILogrotateParams(cephClustersItems []cephv1.CephCluster) {
+	logger.Debug("set logrotate values in csi param")
+	spec := cephClustersItems[0].Spec
+	for _, cluster := range cephClustersItems {
+		if cluster.Spec.LogCollector.Enabled {
+			spec = cluster.Spec
+			break
+		}
+	}
+
+	CSIParam.CSILogRotation = spec.LogCollector.Enabled
+	if spec.LogCollector.Enabled {
+		maxSize, period := opcontroller.GetLogRotateConfig(spec)
+		CSIParam.CSILogRotationMaxSize = maxSize.String()
+		CSIParam.CSILogRotationPeriod = period
+	}
+}
diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go
@@ -97,6 +97,10 @@ type Param struct {
 	CSICephFSPodLabels                       map[string]string
 	CSINFSPodLabels                          map[string]string
 	CSIRBDPodLabels                          map[string]string
+	CSILogRotation                           bool
+	CSILogRotationMaxSize                    string
+	CSILogRotationPeriod                     string
+	CSILogFolder                             string
 }
 
 type templateParam struct {
@@ -178,6 +182,9 @@ var (
 	//go:embed template/nfs/csi-nfsplugin-holder.yaml
 	NFSPluginHolderTemplatePath string
 
+	//go:embed template/csi-logrotate-sidecar.yaml
+	LogrotateTemplatePath string
+
 	holderEnabled bool
 )
 
@@ -349,11 +356,19 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
 		if err != nil {
 			return errors.Wrap(err, "failed to load rbdplugin template")
 		}
+		if tp.CSILogRotation {
+			tp.CSILogFolder = "rbd-plugin"
+			applyLogrotateSidecar(&rbdPlugin.Spec.Template, "csi-rbd-daemonset-log-collector", LogrotateTemplatePath, tp)
+		}
 
 		rbdProvisionerDeployment, err = templateToDeployment("rbd-provisioner", RBDProvisionerDepTemplatePath, tp)
 		if err != nil {
 			return errors.Wrap(err, "failed to load rbd provisioner deployment template")
 		}
+		if tp.CSILogRotation {
+			tp.CSILogFolder = "rbd-provisioner"
+			applyLogrotateSidecar(&rbdProvisionerDeployment.Spec.Template, "csi-rbd-deployment-log-collector", LogrotateTemplatePath, tp)
+		}
 
 		// Create service if either liveness or GRPC metrics are enabled.
 		if CSIParam.EnableLiveness {

diff --git a/pkg/operator/ceph/csi/template/csi-logrotate-sidecar.yaml b/pkg/operator/ceph/csi/template/csi-logrotate-sidecar.yaml
@@ -0,0 +1,35 @@
+args:
+  - |
+    echo "Starting the csi-logrotate-sidecar"
+    mkdir -p /var/lib/cephcsi/logrotate-config/{{ .CSILogFolder }};
+    echo '/var/lib/cephcsi/log/{{ .CSILogFolder }}/*.log {
+        {{ .CSILogRotationPeriod }}
+        missingok
+        rotate 7
+        compress
+        copytruncate
+        notifempty
+    }' >  /var/lib/cephcsi/logrotate-config/{{ .CSILogFolder }}/csi;
+    echo "File creation container completed";
+
+    LOG_ROTATE_CEPH_CSI_FILE=/var/lib/cephcsi/logrotate-config/{{ .CSILogFolder }}/csi
+    LOG_MAX_SIZE={{ .CSILogRotationMaxSize }}
+    if [ "$LOG_MAX_SIZE" != "0" ]; then
+      sed --in-place "4i \ \ \ \ maxsize $LOG_MAX_SIZE" "$LOG_ROTATE_CEPH_CSI_FILE"
+    fi
+
+    while true; do
+      logrotate --verbose "$LOG_ROTATE_CEPH_CSI_FILE"
+      sleep 15m
+    done
+command:
+  - /bin/sh
+  - -c
+image: { { .CSIPluginImage } }
+imagePullPolicy: IfNotPresent
+name: log-collector
+volumeMounts:
+  - mountPath: /var/lib/cephcsi/logrotate-config/{{ .CSILogFolder }}
+    name: csi-logs-logrotate
+  - mountPath: /var/lib/cephcsi/log/{{ .CSILogFolder }}
+    name: csi-log
diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml
@@ -38,6 +38,9 @@ spec:
             - "--extra-create-metadata=true"
             - "--prevent-volume-mode-conversion=true"
             - "--feature-gates=HonorPVReclaimPolicy=true"
+            - "--logtostderr=false"
+            - "--alsologtostderr=true"
+            - "--log_file=/var/lib/cephcsi/log/rbd-provisioner/csi-provisioner.log"
             {{- if .EnableCSITopology }}
             - "--feature-gates=Topology=true"
             {{- end }}
@@ -48,6 +51,8 @@ spec:
           volumeMounts:
             - name: socket-dir
               mountPath: /csi
+            - mountPath: /var/lib/cephcsi/log/rbd-provisioner
+              name: csi-log
         - name: csi-resizer
           image: {{ .ResizerImage }}
           args:
@@ -81,13 +86,18 @@ spec:
             - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}"
             - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}"
             - "--default-fstype=ext4"
+            - "--logtostderr=false"
+            - "--alsologtostderr=true"
+            - "--log_file=/var/lib/cephcsi/log/rbd-provisioner/csi-attacher.log"
           env:
             - name: ADDRESS
               value: /csi/csi-provisioner.sock
           imagePullPolicy: {{ .ImagePullPolicy }}
           volumeMounts:
             - name: socket-dir
               mountPath: /csi
+            - mountPath: /var/lib/cephcsi/log/rbd-provisioner
+              name: csi-log
         {{ end }}
         {{ if .EnableRBDSnapshotter }}
         - name: csi-snapshotter
@@ -102,6 +112,9 @@ spec:
             - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}"
             - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}"
             - "--extra-create-metadata=true"
+            - "--logtostderr=false"
+            - "--alsologtostderr=true"
+            - "--log_file=/var/lib/cephcsi/log/rbd-provisioner/csi-snapshotter.log"
             {{ if .VolumeGroupSnapshotSupported }}
             - "--enable-volume-group-snapshots={{ .EnableVolumeGroupSnapshot }}"
             {{ end }}
@@ -112,15 +125,20 @@ spec:
           volumeMounts:
             - name: socket-dir
               mountPath: /csi
+            - mountPath: /var/lib/cephcsi/log/rbd-provisioner
+              name: csi-log
         {{ end }}
         {{ if .EnableOMAPGenerator }}
         - name: csi-omap-generator
           image: {{ .CSIPluginImage }}
-          args :
+          args:
             - "--type=controller"
             - "--drivernamespace=$(DRIVER_NAMESPACE)"
             - "--v={{ .LogLevel }}"
             - "--drivername={{ .DriverNamePrefix }}rbd.csi.ceph.com"
+            - "--logtostderr=false"
+            - "--alsologtostderr=true"
+            - "--log_file=/var/lib/cephcsi/log/rbd-provisioner/csi-omap-generator.log"
             {{ if .CSIEnableMetadata }}
             - "--setmetadata={{ .CSIEnableMetadata }}"
             {{ end }}
@@ -138,11 +156,13 @@ spec:
               mountPath: /etc/ceph-csi-config/
             - name: keys-tmp-dir
               mountPath: /tmp/csi/keys
+            - mountPath: /var/lib/cephcsi/log/rbd-provisioner
+              name: csi-log
         {{ end }}
         {{ if .EnableCSIAddonsSideCar }}
         - name: csi-addons
           image: {{ .CSIAddonsImage }}
-          args :
+          args:
             - "--node-id=$(NODE_ID)"
             - "--v={{ .LogLevel }}"
             - "--csi-addons-address=$(CSIADDONS_ENDPOINT)"
@@ -155,6 +175,9 @@ spec:
             - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}"
             - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}"
             - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}"
+            - "--logtostderr=false"
+            - "--alsologtostderr=true"
+            - "--log_file=/var/lib/cephcsi/log/rbd-provisioner/csi-addons.log"
           ports:
             - containerPort: {{ .CSIAddonsPort }}
           env:
@@ -180,17 +203,22 @@ spec:
           volumeMounts:
             - name: socket-dir
               mountPath: /csi
+            - mountPath: /var/lib/cephcsi/log/rbd-provisioner
+              name: csi-log
         {{ end }}
         - name: csi-rbdplugin
           image: {{ .CSIPluginImage }}
-          args :
+          args:
             - "--nodeid=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"
             - "--v={{ .LogLevel }}"
             - "--type=rbd"
             - "--controllerserver=true"
             - "--drivername={{ .DriverNamePrefix }}rbd.csi.ceph.com"
             - "--pidlimit=-1"
+            - "--logtostderr=false"
+            - "--alsologtostderr=true"
+            - "--log_file=/var/lib/cephcsi/log/rbd-provisioner/csi-rbdplugin.log"
             {{ if .EnableCSIAddonsSideCar }}
             - "--csi-addons-endpoint=$(CSIADDONS_ENDPOINT)"
             {{ end }}
@@ -225,6 +253,8 @@ spec:
               mountPath: /csi
             - mountPath: /dev
               name: host-dev
+            - mountPath: /var/lib/cephcsi/log/rbd-provisioner
+              name: csi-log
             - mountPath: /sys
               name: host-sys
             - mountPath: /lib/modules
@@ -256,6 +286,9 @@ spec:
             - "--metricspath=/metrics"
             - "--polltime=60s"
             - "--timeout=3s"
+            - "--logtostderr=false"
+            - "--alsologtostderr=true"
+            - "--log_file=/var/lib/cephcsi/log/rbd-provisioner/liveness-prometheus.log"
           env:
             - name: CSI_ENDPOINT
               value: unix:///csi/csi-provisioner.sock
@@ -266,12 +299,21 @@ spec:
           volumeMounts:
             - name: socket-dir
               mountPath: /csi
+            - mountPath: /var/lib/cephcsi/log/rbd-provisioner
+              name: csi-log
           imagePullPolicy: {{ .ImagePullPolicy }}
         {{ end }}
       volumes:
         - name: host-dev
           hostPath:
             path: /dev
+        - name: csi-log
+          hostPath:
+            path: /var/lib/cephcsi/log/rbd-provisioner
+            type: DirectoryOrCreate
+        - name: csi-logs-logrotate
+          emptyDir:
+            type: DirectoryOrCreate
         - name: host-sys
           hostPath:
             path: /sys

diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml
@@ -73,6 +73,9 @@ spec:
             - "--drivername={{ .DriverNamePrefix }}rbd.csi.ceph.com"
             - "--pidlimit=-1"
             - "--stagingpath={{ .KubeletDirPath }}/plugins/kubernetes.io/csi/"
+            - "--logtostderr=false"
+            - "--alsologtostderr=true"
+            - "--log_file=/var/lib/cephcsi/log/rbd-plugin/csi-rbdplugin.log"
             {{ if .EnableCSIAddonsSideCar }}
             - "--csi-addons-endpoint=$(CSIADDONS_ENDPOINT)"
             {{ end }}
@@ -121,6 +124,8 @@ spec:
               mountPath: /tmp/csi/keys
             - name: host-run-mount
               mountPath: /run/mount
+            - mountPath: /var/lib/cephcsi/log/rbd-plugin
+              name: csi-log
             {{ if .EnablePluginSelinuxHostMount }}
             - name: etc-selinux
               mountPath: /etc/selinux
@@ -154,6 +159,9 @@ spec:
             - "--namespace=$(POD_NAMESPACE)"
             - "--pod-uid=$(POD_UID)"
             - "--stagingpath={{ .KubeletDirPath }}/plugins/kubernetes.io/csi/"
+            - "--logtostderr=false"
+            - "--alsologtostderr=true"
+            - "--log_file=/var/lib/cephcsi/log/rbd-plugin/csi-addons.log"
           ports:
             - containerPort: {{ .CSIAddonsPort }}
           env:
@@ -179,6 +187,8 @@ spec:
           volumeMounts:
             - name: plugin-dir
               mountPath: /csi
+            - mountPath: /var/lib/cephcsi/log/rbd-plugin
+              name: csi-log
         {{ end }}
         {{ if .EnableLiveness }}
         - name: liveness-prometheus
@@ -194,6 +204,9 @@ spec:
             - "--metricspath=/metrics"
             - "--polltime=60s"
             - "--timeout=3s"
+            - "--logtostderr=false"
+            - "--alsologtostderr=true"
+            - "--log_file=/var/lib/cephcsi/log/rbd-plugin/liveness-prometheus.log"
           env:
             - name: CSI_ENDPOINT
               value: unix:///csi/csi.sock
@@ -204,13 +217,22 @@ spec:
           volumeMounts:
             - name: plugin-dir
               mountPath: /csi
+            - mountPath: /var/lib/cephcsi/log/rbd-plugin
+              name: csi-log
           imagePullPolicy: {{ .ImagePullPolicy }}
         {{ end }}
       volumes:
         - name: plugin-dir
           hostPath:
             path: "{{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}rbd.csi.ceph.com"
             type: DirectoryOrCreate
+        - name: csi-log
+          hostPath:
+            path: /var/lib/cephcsi/log/rbd-plugin
+            type: DirectoryOrCreate
+        - name: csi-logs-logrotate
+          emptyDir:
+            type: DirectoryOrCreate
         - name: plugin-mount-dir
           hostPath:
             path: "{{ .KubeletDirPath }}/plugins"