csi: add log rotation for csi cephfs nfs pod containers

Use the same logrotate flow of rbd, as merged in rook#14305 1) Make the csi cephfs and nfs container logs persisted in a file 2) Use the cephcluster api specs to configure the log rotate 3) Add log rotation to rotate the log file and Add a sidecar log collector container And some fixes on the implementation: 1) Add a operator namespace in the log file path 2) Only add volume to the container if logrotate is enabled Closes: rook#12809, rook#14429 Signed-off-by: parth-gr <partharora1010@gmail.com>
parth-gr · Jul 10, 2024 · d9104e1 · d9104e1
1 parent 0837c15
commit d9104e1
Show file tree

Hide file tree

Showing 12 changed files with 152 additions and 25 deletions.
diff --git a/deploy/examples/images.txt b/deploy/examples/images.txt
@@ -3,9 +3,9 @@
  quay.io/ceph/cosi:v0.1.1
  quay.io/cephcsi/cephcsi:v3.11.0
  quay.io/csiaddons/k8s-sidecar:v0.8.0
+ quay.io/paarora/rook-ceph:v558
  registry.k8s.io/sig-storage/csi-attacher:v4.5.1
  registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.10.1
  registry.k8s.io/sig-storage/csi-provisioner:v4.0.1
  registry.k8s.io/sig-storage/csi-resizer:v1.10.1
  registry.k8s.io/sig-storage/csi-snapshotter:v7.0.2
- rook/ceph:master
diff --git a/deploy/examples/operator.yaml b/deploy/examples/operator.yaml
@@ -610,7 +610,7 @@ spec:
             - name: ROOK_CURRENT_NAMESPACE_ONLY
               value: "false"
 
-            # Whether to start pods as privileged that mount a host path, which includes the Ceph mon and osd pods.
+            # Whether to start pods as privileged that mount a host path, which includes the Ceph mon, osd pods and csi provisioners(if logrotation is on).
             # Set this to true if SELinux is enabled (e.g. OpenShift) to workaround the anyuid issues.
             # For more details see https://github.com/rook/rook/issues/1314#issuecomment-355799641
             - name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED

diff --git a/pkg/operator/ceph/csi/controller.go b/pkg/operator/ceph/csi/controller.go
@@ -219,7 +219,7 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e
 
 	// if at least one cephcluster is present update the csi lograte sidecar
 	// with the first listed ceph cluster specs with logrotate enabled
-	setCSILogrotateParams(cephClusters.Items)
+	r.setCSILogrotateParams(cephClusters.Items)
 
 	err = peermap.CreateOrUpdateConfig(r.opManagerContext, r.context, &peermap.PeerIDMappings{})
 	if err != nil {
@@ -304,7 +304,7 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e
 	return reconcileResult, nil
 }
 
-func setCSILogrotateParams(cephClustersItems []cephv1.CephCluster) {
+func (r *ReconcileCSI) setCSILogrotateParams(cephClustersItems []cephv1.CephCluster) {
 	logger.Debug("set logrotate values in csi param")
 	spec := cephClustersItems[0].Spec
 	for _, cluster := range cephClustersItems {

diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go
@@ -358,7 +358,8 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
 			return errors.Wrap(err, "failed to load rbdplugin template")
 		}
 		if tp.CSILogRotation {
-			tp.CSILogFolder = "rbd-plugin"
+			tp.CSILogFolder = "node-plugin"
+			tp.CsiLogDirPath = tp.CsiLogDirPath + "/" + RBDDriverName
 			applyLogrotateSidecar(&rbdPlugin.Spec.Template, "csi-rbd-daemonset-log-collector", LogrotateTemplatePath, tp)
 		}
 
@@ -367,7 +368,8 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
 			return errors.Wrap(err, "failed to load rbd provisioner deployment template")
 		}
 		if tp.CSILogRotation {
-			tp.CSILogFolder = "rbd-provisioner"
+			tp.CSILogFolder = "controller-plugin"
+			tp.CsiLogDirPath = tp.CsiLogDirPath + "/" + RBDDriverName
 			applyLogrotateSidecar(&rbdProvisionerDeployment.Spec.Template, "csi-rbd-deployment-log-collector", LogrotateTemplatePath, tp)
 		}
 
@@ -393,11 +395,22 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
 		if err != nil {
 			return errors.Wrap(err, "failed to load CephFS plugin template")
 		}
+		if tp.CSILogRotation {
+			tp.CSILogFolder = "node-plugin"
+			tp.CsiLogDirPath = tp.CsiLogDirPath + "/" + CephFSDriverName
+			applyLogrotateSidecar(&cephfsPlugin.Spec.Template, "csi-cephfs-daemonset-log-collector", LogrotateTemplatePath, tp)
+		}
 
 		cephfsProvisionerDeployment, err = templateToDeployment("cephfs-provisioner", CephFSProvisionerDepTemplatePath, tp)
 		if err != nil {
 			return errors.Wrap(err, "failed to load rbd provisioner deployment template")
 		}
+		if tp.CSILogRotation {
+			tp.CSILogFolder = "controller-plugin"
+			tp.CsiLogDirPath = tp.CsiLogDirPath + "/" + CephFSDriverName
+			applyLogrotateSidecar(&cephfsProvisionerDeployment.Spec.Template, "csi-cephfs-deployment-log-collector", LogrotateTemplatePath, tp)
+		}
+
 		// Create service if either liveness or GRPC metrics are enabled.
 		if CSIParam.EnableLiveness {
 			cephfsService, err = templateToService("cephfs-service", CephFSPluginServiceTemplatePath, tp)
@@ -421,11 +434,22 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
 		if err != nil {
 			return errors.Wrap(err, "failed to load nfs plugin template")
 		}
+		if tp.CSILogRotation {
+			tp.CSILogFolder = "node-plugin"
+			tp.CsiLogDirPath = tp.CsiLogDirPath + "/" + NFSDriverName
+			applyLogrotateSidecar(&nfsPlugin.Spec.Template, "csi-nfs-daemonset-log-collector", LogrotateTemplatePath, tp)
+		}
 
 		nfsProvisionerDeployment, err = templateToDeployment("nfs-provisioner", NFSProvisionerDepTemplatePath, tp)
 		if err != nil {
 			return errors.Wrap(err, "failed to load nfs provisioner deployment template")
 		}
+		if tp.CSILogRotation {
+			tp.CSILogFolder = "controller-plugin"
+			tp.CsiLogDirPath = tp.CsiLogDirPath + "/" + NFSDriverName
+			applyLogrotateSidecar(&nfsProvisionerDeployment.Spec.Template, "csi-nfs-deployment-log-collector", LogrotateTemplatePath, tp)
+		}
+
 		enabledDrivers = append(enabledDrivers, driverDetails{
 			name:           NFSDriverShortName,
 			fullName:       NFSDriverName,

diff --git a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml
@@ -119,6 +119,11 @@ spec:
             - "--drivername={{ .DriverNamePrefix }}cephfs.csi.ceph.com"
             - "--pidlimit=-1"
             - "--forcecephkernelclient={{ .ForceCephFSKernelClient }}"
+            {{ if .CSILogRotation }}
+            - "--logtostderr=false"
+            - "--alsologtostderr=true"
+            - "--log_file={{ .CsiLogDirPath }}/log/{{ .CSILogFolder }}/csi-cephfsplugin.log"
+            {{ end }}
             {{ if .CSIEnableMetadata }}
             - "--setmetadata={{ .CSIEnableMetadata }}"
             {{ end }}
@@ -162,6 +167,10 @@ spec:
               mountPath: /etc/ceph-csi-config/
             - name: keys-tmp-dir
               mountPath: /tmp/csi/keys
+            {{ if .CSILogRotation }}
+            - mountPath: {{ .CsiLogDirPath }}/log/{{ .CSILogFolder }}
+              name: csi-log
+            {{ end }}
             {{ if .MountCustomCephConf }}
             - name: ceph-config
               mountPath: /etc/ceph/ceph.conf
@@ -187,6 +196,11 @@ spec:
             - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}"
             - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}"
             - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}"
+            {{ if .CSILogRotation }}
+            - "--logtostderr=false"
+            - "--alsologtostderr=true"
+            - "--log_file={{ .CsiLogDirPath }}/log/{{ .CSILogFolder }}/csi-addons.log"
+            {{ end }}
           ports:
             - containerPort: {{ .CSIAddonsPort }}
           env:
@@ -212,6 +226,10 @@ spec:
           volumeMounts:
             - name: socket-dir
               mountPath: /csi
+            {{ if .CSILogRotation }}
+            - mountPath: {{ .CsiLogDirPath }}/log/{{ .CSILogFolder }}
+              name: csi-log
+            {{ end }}
         {{ end }}
         {{ if .EnableLiveness }}
         - name: liveness-prometheus
@@ -259,6 +277,15 @@ spec:
           emptyDir: {
             medium: "Memory"
           }
+        {{ if .CSILogRotation }}
+        - name: csi-log
+          hostPath:
+            path: {{ .CsiLogDirPath }}/log/{{ .CSILogFolder }}
+            type: DirectoryOrCreate
+        - name: csi-logs-logrotate
+          emptyDir:
+            type: DirectoryOrCreate
+        {{ end }}
         {{ if .MountCustomCephConf }}
         - name: ceph-config
           configMap:

diff --git a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml
@@ -72,6 +72,11 @@ spec:
             - "--drivername={{ .DriverNamePrefix }}cephfs.csi.ceph.com"
             - "--pidlimit=-1"
             - "--forcecephkernelclient={{ .ForceCephFSKernelClient }}"
+            {{ if .CSILogRotation }}
+            - "--logtostderr=false"
+            - "--alsologtostderr=true"
+            - "--log_file={{ .CsiLogDirPath }}/log/{{ .CSILogFolder }}/csi-cephfsplugin.log"
+            {{ end }}
             {{ if .CephFSKernelMountOptions }}
             - "--kernelmountoptions={{ .CephFSKernelMountOptions }}"
             {{ end }}
@@ -115,6 +120,10 @@ spec:
               mountPath: /tmp/csi/keys
             - name: host-run-mount
               mountPath: /run/mount
+            {{ if .CSILogRotation }}
+            - mountPath: {{ .CsiLogDirPath }}/log/{{ .CSILogFolder }}
+              name: csi-log
+            {{ end }}
             {{ if .EnablePluginSelinuxHostMount }}
             -  name: etc-selinux
                mountPath: /etc/selinux
@@ -198,6 +207,15 @@ spec:
         - name: host-run-mount
           hostPath:
             path: /run/mount
+        {{ if .CSILogRotation }}
+        - name: csi-log
+          hostPath:
+            path: {{ .CsiLogDirPath }}/log/{{ .CSILogFolder }}
+            type: DirectoryOrCreate
+        - name: csi-logs-logrotate
+          emptyDir:
+            type: DirectoryOrCreate
+        {{ end }}
         {{ if .EnablePluginSelinuxHostMount }}
         - name: etc-selinux
           hostPath:

diff --git a/pkg/operator/ceph/csi/template/csi-logrotate-sidecar.yaml b/pkg/operator/ceph/csi/template/csi-logrotate-sidecar.yaml
@@ -1,18 +1,18 @@
 args:
   - |
     echo "Starting the csi-logrotate-sidecar"
-    mkdir -p {{ .CsiLogDirPath }}/cephcsi/logrotate-config/{{ .CSILogFolder }};
-    echo '{{ .CsiLogDirPath }}cephcsi/log/{{ .CSILogFolder }}/*.log {
+    mkdir -p {{ .CsiLogDirPath }}/logrotate-config/{{ .CSILogFolder }}
+    echo '{{ .CsiLogDirPath }}/log/{{ .CSILogFolder }}/*.log {
         {{ .CSILogRotationPeriod }}
         missingok
         rotate 7
         compress
         copytruncate
         notifempty
-    }' >  {{ .CsiLogDirPath }}/cephcsi/logrotate-config/{{ .CSILogFolder }}/csi;
-    echo "File creation container completed";
+    }' >  {{ .CsiLogDirPath }}/logrotate-config/{{ .CSILogFolder }}/csi
+    echo "File creation container completed"
 
-    LOG_ROTATE_CEPH_CSI_FILE={{ .CsiLogDirPath }}/cephcsi/logrotate-config/{{ .CSILogFolder }}/csi
+    LOG_ROTATE_CEPH_CSI_FILE={{ .CsiLogDirPath }}/logrotate-config/{{ .CSILogFolder }}/csi
     LOG_MAX_SIZE={{ .CSILogRotationMaxSize }}
     if [ "$LOG_MAX_SIZE" != "0" ]; then
       sed --in-place "4i \ \ \ \ maxsize $LOG_MAX_SIZE" "$LOG_ROTATE_CEPH_CSI_FILE"
@@ -29,7 +29,7 @@ image: {{ .CSIPluginImage }}
 imagePullPolicy: IfNotPresent
 name: log-collector
 volumeMounts:
-  - mountPath: {{ .CsiLogDirPath }}/cephcsi/logrotate-config/{{ .CSILogFolder }}
+  - mountPath: {{ .CsiLogDirPath }}/logrotate-config/{{ .CSILogFolder }}
     name: csi-logs-logrotate
-  - mountPath: {{ .CsiLogDirPath }}/cephcsi/log/{{ .CSILogFolder }}
+  - mountPath: {{ .CsiLogDirPath }}/log/{{ .CSILogFolder }}
     name: csi-log
diff --git a/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin-provisioner-dep.yaml
@@ -113,6 +113,11 @@ spec:
             - "--controllerserver=true"
             - "--drivername={{ .DriverNamePrefix }}nfs.csi.ceph.com"
             - "--pidlimit=-1"
+            {{ if .CSILogRotation }}
+            - "--logtostderr=false"
+            - "--alsologtostderr=true"
+            - "--log_file={{ .CsiLogDirPath }}/log/{{ .CSILogFolder }}/csi-nfsplugin.log"
+            {{ end }}
           env:
             - name: POD_IP
               valueFrom:
@@ -143,6 +148,10 @@ spec:
               mountPath: /etc/ceph-csi-config/
             - name: keys-tmp-dir
               mountPath: /tmp/csi/keys
+            {{ if .CSILogRotation }}
+            - mountPath: {{ .CsiLogDirPath }}/log/{{ .CSILogFolder }}
+              name: csi-log
+            {{ end }}
             {{ if .MountCustomCephConf }}
             - name: ceph-config
               mountPath: /etc/ceph/ceph.conf
@@ -172,6 +181,15 @@ spec:
           emptyDir: {
             medium: "Memory"
           }
+        {{ if .CSILogRotation }}
+        - name: csi-log
+          hostPath:
+            path: {{ .CsiLogDirPath }}/log/{{ .CSILogFolder }}
+            type: DirectoryOrCreate
+        - name: csi-logs-logrotate
+          emptyDir:
+            type: DirectoryOrCreate
+        {{ end }}
         {{ if .MountCustomCephConf }}
         - name: ceph-config
           configMap:

diff --git a/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin.yaml b/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin.yaml
@@ -66,6 +66,11 @@ spec:
             - "--nodeserver=true"
             - "--drivername={{ .DriverNamePrefix }}nfs.csi.ceph.com"
             - "--pidlimit=-1"
+            {{ if .CSILogRotation }}
+            - "--logtostderr=false"
+            - "--alsologtostderr=true"
+            - "--log_file={{ .CsiLogDirPath }}/log/{{ .CSILogFolder }}/csi-nfsplugin.log"
+            {{ end }}
           env:
           - name: NODE_ID
             valueFrom:
@@ -98,6 +103,10 @@ spec:
               mountPath: /tmp/csi/keys
             - name: host-run-mount
               mountPath: /run/mount
+            {{ if .CSILogRotation }}
+            - mountPath: {{ .CsiLogDirPath }}/log/{{ .CSILogFolder }}
+              name: csi-log
+            {{ end }}
             {{ if .EnablePluginSelinuxHostMount }}
             -  name: etc-selinux
                mountPath: /etc/selinux
@@ -147,6 +156,15 @@ spec:
         - name: host-run-mount
           hostPath:
             path: /run/mount
+        {{ if .CSILogRotation }}
+        - name: csi-log
+          hostPath:
+            path: {{ .CsiLogDirPath }}/log/{{ .CSILogFolder }}
+            type: DirectoryOrCreate
+        - name: csi-logs-logrotate
+          emptyDir:
+            type: DirectoryOrCreate
+        {{ end }}
         {{ if .EnablePluginSelinuxHostMount }}
         - name: etc-selinux
           hostPath:

diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml
@@ -158,7 +158,7 @@ spec:
             {{ if .CSILogRotation }}
             - "--logtostderr=false"
             - "--alsologtostderr=true"
-            - "--log_file={{ .CsiLogDirPath }}/cephcsi/log/rbd-provisioner/csi-addons.log"
+            - "--log_file={{ .CsiLogDirPath }}/log/{{ .CSILogFolder }}/csi-addons.log"
             {{ end }}
           ports:
             - containerPort: {{ .CSIAddonsPort }}
@@ -186,7 +186,7 @@ spec:
             - name: socket-dir
               mountPath: /csi
             {{ if .CSILogRotation }}
-            - mountPath: {{ .CsiLogDirPath }}/cephcsi/log/rbd-provisioner
+            - mountPath: {{ .CsiLogDirPath }}/log/{{ .CSILogFolder }}
               name: csi-log
             {{ end }}
         {{ end }}
@@ -203,7 +203,7 @@ spec:
             {{ if .CSILogRotation }}
             - "--logtostderr=false"
             - "--alsologtostderr=true"
-            - "--log_file={{ .CsiLogDirPath }}/cephcsi/log/rbd-provisioner/csi-rbdplugin.log"
+            - "--log_file={{ .CsiLogDirPath }}/log/{{ .CSILogFolder }}/csi-rbdplugin.log"
             {{ end }}
             {{ if .EnableCSIAddonsSideCar }}
             - "--csi-addons-endpoint=$(CSIADDONS_ENDPOINT)"
@@ -240,7 +240,7 @@ spec:
             - mountPath: /dev
               name: host-dev
             {{ if .CSILogRotation }}
-            - mountPath: {{ .CsiLogDirPath }}/cephcsi/log/rbd-provisioner
+            - mountPath: {{ .CsiLogDirPath }}/log/{{ .CSILogFolder }}
               name: csi-log
             {{ end }}
             - mountPath: /sys
@@ -290,13 +290,15 @@ spec:
         - name: host-dev
           hostPath:
             path: /dev
+        {{ if .CSILogRotation }}
         - name: csi-log
           hostPath:
-            path: {{ .CsiLogDirPath }}/cephcsi/log/rbd-provisioner
+            path: {{ .CsiLogDirPath }}/log/{{ .CSILogFolder }}
             type: DirectoryOrCreate
         - name: csi-logs-logrotate
           emptyDir:
             type: DirectoryOrCreate
+        {{ end }}
         - name: host-sys
           hostPath:
             path: /sys