Merge pull request #62460 from mlmhl/volume-online-resize

Automatic merge from submit-queue (batch tested with PRs 62460, 64480, 63774, 64540, 64337). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. Implement kubelet side online file system resize for volume **What this PR does / why we need it**: Implement kubelet side online file system resize. xref - [kubernetes/feature#531](kubernetes/enhancements#531) proposal - [kubernetes/community#1535](kubernetes/community#1535) **Release note**: ```release-note Implement kubelet side online file system resizing ```
kubernetes · May 31, 2018 · 7303776 · 7303776
2 parents bb869d8 + ca12c73
commit 7303776
Show file tree

Hide file tree

Showing 13 changed files with 697 additions and 45 deletions.
diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go
@@ -99,6 +99,11 @@ const (
 	// Ability to Expand persistent volumes
 	ExpandPersistentVolumes utilfeature.Feature = "ExpandPersistentVolumes"
 
+	// owner: @mlmhl
+	// alpha: v1.11
+	// Ability to expand persistent volumes' file system without unmounting volumes.
+	ExpandPersistentVolumesFSWithoutUnmounting utilfeature.Feature = "ExpandPersistentVolumesFSWithoutUnmounting"
+
 	// owner: @verb
 	// alpha: v1.10
 	//
@@ -328,28 +333,29 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
 	MountPropagation:                            {Default: true, PreRelease: utilfeature.Beta},
 	QOSReserved:                                 {Default: false, PreRelease: utilfeature.Alpha},
 	ExpandPersistentVolumes:                     {Default: true, PreRelease: utilfeature.Beta},
-	CPUManager:                                  {Default: true, PreRelease: utilfeature.Beta},
-	ServiceNodeExclusion:                        {Default: false, PreRelease: utilfeature.Alpha},
-	MountContainers:                             {Default: false, PreRelease: utilfeature.Alpha},
-	VolumeScheduling:                            {Default: true, PreRelease: utilfeature.Beta},
-	CSIPersistentVolume:                         {Default: true, PreRelease: utilfeature.Beta},
-	CustomPodDNS:                                {Default: true, PreRelease: utilfeature.Beta},
-	BlockVolume:                                 {Default: false, PreRelease: utilfeature.Alpha},
-	StorageObjectInUseProtection:                {Default: true, PreRelease: utilfeature.GA},
-	ResourceLimitsPriorityFunction:              {Default: false, PreRelease: utilfeature.Alpha},
-	SupportIPVSProxyMode:                        {Default: true, PreRelease: utilfeature.GA},
-	SupportPodPidsLimit:                         {Default: false, PreRelease: utilfeature.Alpha},
-	HyperVContainer:                             {Default: false, PreRelease: utilfeature.Alpha},
-	ScheduleDaemonSetPods:                       {Default: false, PreRelease: utilfeature.Alpha},
-	TokenRequest:                                {Default: false, PreRelease: utilfeature.Alpha},
-	TokenRequestProjection:                      {Default: false, PreRelease: utilfeature.Alpha},
-	CRIContainerLogRotation:                     {Default: true, PreRelease: utilfeature.Beta},
-	GCERegionalPersistentDisk:                   {Default: true, PreRelease: utilfeature.Beta},
-	RunAsGroup:                                  {Default: false, PreRelease: utilfeature.Alpha},
-	VolumeSubpath:                               {Default: true, PreRelease: utilfeature.GA},
-	BalanceAttachedNodeVolumes:                  {Default: false, PreRelease: utilfeature.Alpha},
-	DynamicProvisioningScheduling:               {Default: false, PreRelease: utilfeature.Alpha},
-	VolumeSubpathEnvExpansion:                   {Default: false, PreRelease: utilfeature.Alpha},
+	ExpandPersistentVolumesFSWithoutUnmounting:  {Default: false, PreRelease: utilfeature.Alpha},
+	CPUManager:                     {Default: true, PreRelease: utilfeature.Beta},
+	ServiceNodeExclusion:           {Default: false, PreRelease: utilfeature.Alpha},
+	MountContainers:                {Default: false, PreRelease: utilfeature.Alpha},
+	VolumeScheduling:               {Default: true, PreRelease: utilfeature.Beta},
+	CSIPersistentVolume:            {Default: true, PreRelease: utilfeature.Beta},
+	CustomPodDNS:                   {Default: true, PreRelease: utilfeature.Beta},
+	BlockVolume:                    {Default: false, PreRelease: utilfeature.Alpha},
+	StorageObjectInUseProtection:   {Default: true, PreRelease: utilfeature.GA},
+	ResourceLimitsPriorityFunction: {Default: false, PreRelease: utilfeature.Alpha},
+	SupportIPVSProxyMode:           {Default: true, PreRelease: utilfeature.GA},
+	SupportPodPidsLimit:            {Default: false, PreRelease: utilfeature.Alpha},
+	HyperVContainer:                {Default: false, PreRelease: utilfeature.Alpha},
+	ScheduleDaemonSetPods:          {Default: false, PreRelease: utilfeature.Alpha},
+	TokenRequest:                   {Default: false, PreRelease: utilfeature.Alpha},
+	TokenRequestProjection:         {Default: false, PreRelease: utilfeature.Alpha},
+	CRIContainerLogRotation:        {Default: true, PreRelease: utilfeature.Beta},
+	GCERegionalPersistentDisk:      {Default: true, PreRelease: utilfeature.Beta},
+	RunAsGroup:                     {Default: false, PreRelease: utilfeature.Alpha},
+	VolumeSubpath:                  {Default: true, PreRelease: utilfeature.GA},
+	BalanceAttachedNodeVolumes:     {Default: false, PreRelease: utilfeature.Alpha},
+	DynamicProvisioningScheduling:  {Default: false, PreRelease: utilfeature.Alpha},
+	VolumeSubpathEnvExpansion:      {Default: false, PreRelease: utilfeature.Alpha},
 
 	// inherited features from generic apiserver, relisted here to get a conflict if it is changed
 	// unintentionally on either side:

diff --git a/pkg/kubelet/volumemanager/cache/BUILD b/pkg/kubelet/volumemanager/cache/BUILD
@@ -14,13 +14,15 @@ go_library(
     ],
     importpath = "k8s.io/kubernetes/pkg/kubelet/volumemanager/cache",
     deps = [
+        "//pkg/features:go_default_library",
         "//pkg/volume:go_default_library",
         "//pkg/volume/util:go_default_library",
         "//pkg/volume/util/operationexecutor:go_default_library",
         "//pkg/volume/util/types:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
     ],
 )
 

diff --git a/pkg/kubelet/volumemanager/cache/actual_state_of_world.go b/pkg/kubelet/volumemanager/cache/actual_state_of_world.go
@@ -28,6 +28,8 @@ import (
 
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/volume"
 	"k8s.io/kubernetes/pkg/volume/util"
 	"k8s.io/kubernetes/pkg/volume/util/operationexecutor"
@@ -148,6 +150,11 @@ type ActualStateOfWorld interface {
 	// with pod's unique name. This map can be used to determine which pod is currently
 	// in actual state of world.
 	GetPods() map[volumetypes.UniquePodName]bool
+
+	// MarkFSResizeRequired marks each volume that is successfully attached and
+	// mounted for the specified pod as requiring file system resize (if the plugin for the
+	// volume indicates it requires file system resize).
+	MarkFSResizeRequired(volumeName v1.UniqueVolumeName, podName volumetypes.UniquePodName)
 }
 
 // MountedVolume represents a volume that has successfully been mounted to a pod.
@@ -291,6 +298,10 @@ type mountedPod struct {
 
 	// volumeGidValue contains the value of the GID annotation, if present.
 	volumeGidValue string
+
+	// fsResizeRequired indicates the underlying volume has been successfully
+	// mounted to this pod but its size has been expanded after that.
+	fsResizeRequired bool
 }
 
 func (asw *actualStateOfWorld) MarkVolumeAsAttached(
@@ -444,6 +455,34 @@ func (asw *actualStateOfWorld) AddPodToVolume(
 	return nil
 }
 
+func (asw *actualStateOfWorld) MarkVolumeAsResized(
+	podName volumetypes.UniquePodName,
+	volumeName v1.UniqueVolumeName) error {
+	asw.Lock()
+	defer asw.Unlock()
+
+	volumeObj, volumeExists := asw.attachedVolumes[volumeName]
+	if !volumeExists {
+		return fmt.Errorf(
+			"no volume with the name %q exists in the list of attached volumes",
+			volumeName)
+	}
+
+	podObj, podExists := volumeObj.mountedPods[podName]
+	if !podExists {
+		return fmt.Errorf(
+			"no pod with the name %q exists in the mounted pods list of volume %s",
+			podName,
+			volumeName)
+	}
+
+	glog.V(5).Infof("Volume %s(OuterVolumeSpecName %s) of pod %s has been resized",
+		volumeName, podObj.outerVolumeSpecName, podName)
+	podObj.fsResizeRequired = false
+	asw.attachedVolumes[volumeName].mountedPods[podName] = podObj
+	return nil
+}
+
 func (asw *actualStateOfWorld) MarkRemountRequired(
 	podName volumetypes.UniquePodName) {
 	asw.Lock()
@@ -475,6 +514,46 @@ func (asw *actualStateOfWorld) MarkRemountRequired(
 	}
 }
 
+func (asw *actualStateOfWorld) MarkFSResizeRequired(
+	volumeName v1.UniqueVolumeName,
+	podName volumetypes.UniquePodName) {
+	asw.Lock()
+	defer asw.Unlock()
+	volumeObj, exist := asw.attachedVolumes[volumeName]
+	if !exist {
+		glog.Warningf("MarkFSResizeRequired for volume %s failed as volume not exist", volumeName)
+		return
+	}
+
+	podObj, exist := volumeObj.mountedPods[podName]
+	if !exist {
+		glog.Warningf("MarkFSResizeRequired for volume %s failed "+
+			"as pod(%s) not exist", volumeName, podName)
+		return
+	}
+
+	volumePlugin, err :=
+		asw.volumePluginMgr.FindExpandablePluginBySpec(podObj.volumeSpec)
+	if err != nil || volumePlugin == nil {
+		// Log and continue processing
+		glog.Errorf(
+			"MarkFSResizeRequired failed to find expandable plugin for pod %q volume: %q (volSpecName: %q)",
+			podObj.podName,
+			volumeObj.volumeName,
+			podObj.volumeSpec.Name())
+		return
+	}
+
+	if volumePlugin.RequiresFSResize() {
+		if !podObj.fsResizeRequired {
+			glog.V(3).Infof("PVC volume %s(OuterVolumeSpecName %s) of pod %s requires file system resize",
+				volumeName, podObj.outerVolumeSpecName, podName)
+			podObj.fsResizeRequired = true
+		}
+		asw.attachedVolumes[volumeName].mountedPods[podName] = podObj
+	}
+}
+
 func (asw *actualStateOfWorld) SetVolumeGloballyMounted(
 	volumeName v1.UniqueVolumeName, globallyMounted bool, devicePath, deviceMountPath string) error {
 	asw.Lock()
@@ -546,8 +625,14 @@ func (asw *actualStateOfWorld) PodExistsInVolume(
 	}
 
 	podObj, podExists := volumeObj.mountedPods[podName]
-	if podExists && podObj.remountRequired {
-		return true, volumeObj.devicePath, newRemountRequiredError(volumeObj.volumeName, podObj.podName)
+	if podExists {
+		if podObj.remountRequired {
+			return true, volumeObj.devicePath, newRemountRequiredError(volumeObj.volumeName, podObj.podName)
+		}
+		if podObj.fsResizeRequired &&
+			utilfeature.DefaultFeatureGate.Enabled(features.ExpandPersistentVolumesFSWithoutUnmounting) {
+			return true, volumeObj.devicePath, newFsResizeRequiredError(volumeObj.volumeName, podObj.podName)
+		}
 	}
 
 	return podExists, volumeObj.devicePath, nil
@@ -716,6 +801,35 @@ func newRemountRequiredError(
 	}
 }
 
+// fsResizeRequiredError is an error returned when PodExistsInVolume() found
+// volume/pod attached/mounted but fsResizeRequired was true, indicating the
+// given volume receives an resize request after attached/mounted.
+type fsResizeRequiredError struct {
+	volumeName v1.UniqueVolumeName
+	podName    volumetypes.UniquePodName
+}
+
+func (err fsResizeRequiredError) Error() string {
+	return fmt.Sprintf(
+		"volumeName %q mounted to %q needs to resize file system",
+		err.volumeName, err.podName)
+}
+
+func newFsResizeRequiredError(
+	volumeName v1.UniqueVolumeName, podName volumetypes.UniquePodName) error {
+	return fsResizeRequiredError{
+		volumeName: volumeName,
+		podName:    podName,
+	}
+}
+
+// IsFSResizeRequiredError returns true if the specified error is a
+// fsResizeRequiredError.
+func IsFSResizeRequiredError(err error) bool {
+	_, ok := err.(fsResizeRequiredError)
+	return ok
+}
+
 // getMountedVolume constructs and returns a MountedVolume object from the given
 // mountedPod and attachedVolume objects.
 func getMountedVolume(

diff --git a/pkg/kubelet/volumemanager/populator/BUILD b/pkg/kubelet/volumemanager/populator/BUILD
@@ -25,6 +25,7 @@ go_library(
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",
@@ -49,6 +50,7 @@ go_test(
     srcs = ["desired_state_of_world_populator_test.go"],
     embed = [":go_default_library"],
     deps = [
+        "//pkg/features:go_default_library",
         "//pkg/kubelet/configmap:go_default_library",
         "//pkg/kubelet/container/testing:go_default_library",
         "//pkg/kubelet/pod:go_default_library",
@@ -61,6 +63,7 @@ go_test(
         "//pkg/volume/util:go_default_library",
         "//pkg/volume/util/types:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",