Automated cherry pick of #33616 #35566

pkg/kubelet/kubelet_getters.go

@@ -239,9 +239,9 @@ func (kl *Kubelet) GetExtraSupplementalGroupsForPod(pod *api.Pod) []int64 {
 	return kl.volumeManager.GetExtraSupplementalGroupsForPod(pod)
 }
 
-// getPodVolumeNameListFromDisk returns a list of the volume names by reading the
+// getPodVolumePathListFromDisk returns a list of the volume paths by reading the
 // volume directories for the given pod from the disk.
-func (kl *Kubelet) getPodVolumeNameListFromDisk(podUID types.UID) ([]string, error) {
+func (kl *Kubelet) getPodVolumePathListFromDisk(podUID types.UID) ([]string, error) {
 	volumes := []string{}
 	podVolDir := kl.getPodVolumesDir(podUID)
 	volumePluginDirs, err := ioutil.ReadDir(podVolDir)
@@ -258,11 +258,10 @@ func (kl *Kubelet) getPodVolumeNameListFromDisk(podUID types.UID) ([]string, err
 		}
 		for i, volumeDir := range volumeDirs {
 			if volumeDir != nil {
-				volumes = append(volumes, volumeDir.Name())
+				volumes = append(volumes, path.Join(volumePluginPath, volumeDir.Name()))
 				continue
 			}
 			glog.Errorf("Could not read directory %s: %v", podVolDir, volumeDirsStatErrs[i])
-
 		}
 	}
 	return volumes, nil

pkg/kubelet/kubelet_node_status.go

@@ -341,6 +341,8 @@ func (kl *Kubelet) tryUpdateNodeStatus() error {
 	}
 	// Update the current status on the API server
 	updatedNode, err := kl.kubeClient.Core().Nodes().UpdateStatus(node)
+	// If update finishes sucessfully, mark the volumeInUse as reportedInUse to indicate
+	// those volumes are already updated in the node's status
 	if err == nil {
 		kl.volumeManager.MarkVolumesAsReportedInUse(
 			updatedNode.Status.VolumesInUse)
@@ -801,9 +803,13 @@ func (kl *Kubelet) recordNodeSchedulableEvent(node *api.Node) {
 	}
 }
 
-// Update VolumesInUse field in Node Status
+// Update VolumesInUse field in Node Status only after states are synced up at least once
+// in volume reconciler.
 func (kl *Kubelet) setNodeVolumesInUseStatus(node *api.Node) {
-	node.Status.VolumesInUse = kl.volumeManager.GetVolumesInUse()
+	// Make sure to only update node status after reconciler starts syncing up states
+	if kl.volumeManager.ReconcilerStatesHasBeenSynced() {
+		node.Status.VolumesInUse = kl.volumeManager.GetVolumesInUse()
+	}
 }
 
 // setNodeStatus fills in the Status fields of the given Node, overwriting

pkg/kubelet/kubelet_volumes.go

@@ -27,6 +27,7 @@ import (
 	"k8s.io/kubernetes/pkg/securitycontext"
 	"k8s.io/kubernetes/pkg/types"
 	utilerrors "k8s.io/kubernetes/pkg/util/errors"
+	"k8s.io/kubernetes/pkg/util/mount"
 	"k8s.io/kubernetes/pkg/util/selinux"
 	"k8s.io/kubernetes/pkg/util/sets"
 	"k8s.io/kubernetes/pkg/volume"
@@ -153,8 +154,20 @@ func (kl *Kubelet) cleanupOrphanedPodDirs(
 			continue
 		}
 		// Check whether volume is still mounted on disk. If so, do not delete directory
-		if volumeNames, err := kl.getPodVolumeNameListFromDisk(uid); err != nil || len(volumeNames) != 0 {
-			glog.V(3).Infof("Orphaned pod %q found, but volumes are still mounted; err: %v, volumes: %v ", uid, err, volumeNames)
+		volumePaths, err := kl.getPodVolumePathListFromDisk(uid)
+		if err != nil {
+			glog.Errorf("Orphaned pod %q found, but error %v occured during reading volume dir from disk", uid, err)
+			continue
+		} else if len(volumePaths) > 0 {
+			for _, path := range volumePaths {
+				notMount, err := mount.IsNotMountPoint(path)
+				if err == nil && notMount {
+					glog.V(2).Infof("Volume path %q is no longer mounted, remove it", path)
+					os.Remove(path)
+				} else {
+					glog.Errorf("Orphaned pod %q found, but it might still mounted with error %v", uid, err)
+				}
+			}
 			continue
 		}
 

pkg/kubelet/volumemanager/cache/actual_state_of_world.go

@@ -193,6 +193,7 @@ type actualStateOfWorld struct {
 	// The key in this map is the name of the volume and the value is an object
 	// containing more information about the attached volume.
 	attachedVolumes map[api.UniqueVolumeName]attachedVolume
+
 	// volumePluginMgr is the volume plugin manager used to create volume
 	// plugin objects.
 	volumePluginMgr *volume.VolumePluginMgr

pkg/kubelet/volumemanager/cache/desired_state_of_world.go

@@ -58,7 +58,8 @@ type DesiredStateOfWorld interface {
 	// ReportedInUse value is reset to false. The default ReportedInUse value
 	// for a newly created volume is false.
 	// When set to true this value indicates that the volume was successfully
-	// added to the VolumesInUse field in the node's status.
+	// added to the VolumesInUse field in the node's status. Mount operation needs
+	// to check this value before issuing the operation.
 	// If a volume in the reportedVolumes list does not exist in the list of
 	// volumes that should be attached to this node, it is skipped without error.
 	MarkVolumesReportedInUse(reportedVolumes []api.UniqueVolumeName)