Supporting user-defined kubelet directory (#2893)

Co-authored-by: yuanliu <yuanliu@cmss.chinamobile.com>
kubeovn · Jun 5, 2023 · 35aa8b4 · 35aa8b4
1 parent 3883a74
commit 35aa8b4
Show file tree

Hide file tree

Showing 7 changed files with 13 additions and 8 deletions.
diff --git a/charts/templates/ovncni-ds.yaml b/charts/templates/ovncni-ds.yaml
@@ -77,6 +77,7 @@ spec:
           - --log_file=/var/log/kube-ovn/kube-ovn-cni.log
           - --log_file_max_size=0
           - --enable-metrics={{- .Values.networking.ENABLE_METRICS }}
+          - --kubelet-dir={{ .Values.kubelet_conf.KUBELET_DIR }}
         securityContext:
           runAsUser: 0
           privileged: true
@@ -108,7 +109,7 @@ spec:
             mountPath: /lib/modules
             readOnly: true
           - name: shared-dir
-            mountPath: /var/lib/kubelet/pods
+            mountPath: {{ .Values.kubelet_conf.KUBELET_DIR }}/pods
           - mountPath: /etc/openvswitch
             name: systemid
             readOnly: true

diff --git a/charts/templates/ovsovn-ds.yaml b/charts/templates/ovsovn-ds.yaml
@@ -119,7 +119,7 @@ spec:
             {{- end }}
             {{- if .Values.HYBRID_DPDK }}
             - name: shareddir
-              mountPath: /var/lib/kubelet/pods
+              mountPath: {{ .Values.kubelet_conf.KUBELET_DIR }}/pods
             {{- end }}
           readinessProbe:
             exec:

diff --git a/dist/images/install.sh b/dist/images/install.sh
@@ -3955,7 +3955,7 @@ spec:
             - mountPath: /opt/ovs-config
               name: host-config-ovs
             - name: shareddir
-              mountPath: /var/lib/kubelet/pods
+              mountPath: $KUBELET_DIR/pods
             - name: hugepage
               mountPath: /dev/hugepages
             - mountPath: /lib/modules
@@ -4289,6 +4289,7 @@ spec:
           - --alsologtostderr=true
           - --log_file=/var/log/kube-ovn/kube-ovn-cni.log
           - --log_file_max_size=0
+          - --kubelet-dir=$KUBELET_DIR
         securityContext:
           runAsUser: 0
           privileged: true
@@ -4320,7 +4321,7 @@ spec:
             mountPath: /lib/modules
             readOnly: true
           - name: shared-dir
-            mountPath: /var/lib/kubelet/pods
+            mountPath: $KUBELET_DIR/pods
           - mountPath: /etc/openvswitch
             name: systemid
             readOnly: true

diff --git a/pkg/daemon/config.go b/pkg/daemon/config.go
@@ -59,6 +59,7 @@ type Configuration struct {
 	ExternalGatewaySwitch     string // provider network underlay vlan subnet
 	EnableMetrics             bool
 	EnableArpDetectIPConflict bool
+	KubeletDir                string
 }
 
 // ParseFlags will parse cmd args then init kubeClient and configuration
@@ -92,6 +93,7 @@ func ParseFlags() *Configuration {
 		argExternalGatewaySwitch     = pflag.String("external-gateway-switch", "external", "The name of the external gateway switch which is a ovs bridge to provide external network, default: external")
 		argEnableMetrics             = pflag.Bool("enable-metrics", true, "Whether to support metrics query")
 		argEnableArpDetectIPConflict = pflag.Bool("enable-arp-detect-ip-conflict", true, "Whether to support arp detect ip conflict in vlan network")
+		argKubeletDir                = pflag.String("kubelet-dir", "/var/lib/kubelet", "Path of the kubelet dir, default: /var/lib/kubelet")
 	)
 
 	// mute info log for ipset lib
@@ -142,6 +144,7 @@ func ParseFlags() *Configuration {
 		ExternalGatewaySwitch:     *argExternalGatewaySwitch,
 		EnableMetrics:             *argEnableMetrics,
 		EnableArpDetectIPConflict: *argEnableArpDetectIPConflict,
+		KubeletDir:                *argKubeletDir,
 	}
 	return config
 }

diff --git a/pkg/daemon/handler.go b/pkg/daemon/handler.go
@@ -147,7 +147,7 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon
 			nicType = util.OffloadType
 		} else if podRequest.VhostUserSocketVolumeName != "" {
 			nicType = util.DpdkType
-			if err = createShortSharedDir(pod, podRequest.VhostUserSocketVolumeName); err != nil {
+			if err = createShortSharedDir(pod, podRequest.VhostUserSocketVolumeName, csh.Config.KubeletDir); err != nil {
 				klog.Error(err.Error())
 				if err = resp.WriteHeaderAndEntity(http.StatusInternalServerError, request.CniResponse{Err: err.Error()}); err != nil {
 					klog.Errorf("failed to write response: %v", err)

diff --git a/pkg/daemon/handler_linux.go b/pkg/daemon/handler_linux.go
@@ -22,7 +22,7 @@ func (csh cniServerHandler) validatePodRequest(req *request.CniRequest) error {
 	return nil
 }
 
-func createShortSharedDir(pod *v1.Pod, volumeName string) (err error) {
+func createShortSharedDir(pod *v1.Pod, volumeName string, kubeletDir string) (err error) {
 	var volume *v1.Volume
 	for index, v := range pod.Spec.Volumes {
 		if v.Name == volumeName {
@@ -36,7 +36,7 @@ func createShortSharedDir(pod *v1.Pod, volumeName string) (err error) {
 	if volume.EmptyDir == nil {
 		return fmt.Errorf("volume %s is not empty dir", volume.Name)
 	}
-	originSharedDir := fmt.Sprintf("/var/lib/kubelet/pods/%s/volumes/kubernetes.io~empty-dir/%s", pod.UID, volumeName)
+	originSharedDir := fmt.Sprintf("%s/pods/%s/volumes/kubernetes.io~empty-dir/%s", kubeletDir, pod.UID, volumeName)
 	newSharedDir := getShortSharedDir(pod.UID, volumeName)
 	// set vhostuser dir 777 for qemu has the permission to create sock
 	mask := syscall.Umask(0)

diff --git a/pkg/daemon/handler_windows.go b/pkg/daemon/handler_windows.go
@@ -19,7 +19,7 @@ func (csh cniServerHandler) validatePodRequest(req *request.CniRequest) error {
 	return nil
 }
 
-func createShortSharedDir(pod *v1.Pod, volumeName string) error {
+func createShortSharedDir(pod *v1.Pod, volumeName string, kubeletDir string) error {
 	// nothing to do on Windows
 	return nil
 }