From e22270f431333846c3fc5a71b73331dae50339ce Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Mon, 14 Jun 2021 10:21:39 +0200 Subject: [PATCH] UPSTREAM: 102059: Retry reading /proc/mounts when unable to get a consistent read --- .../kubernetes/pkg/util/io/consistentread.go | 26 +++++++++++++++++- .../kubernetes/pkg/volume/fc/attacher.go | 2 +- vendor/k8s.io/kubernetes/pkg/volume/fc/fc.go | 10 +++++-- .../kubernetes/pkg/volume/iscsi/iscsi.go | 9 ++++++- .../k8s.io/kubernetes/pkg/volume/util/util.go | 27 +++++++++++++++++++ 5 files changed, 69 insertions(+), 5 deletions(-) diff --git a/vendor/k8s.io/kubernetes/pkg/util/io/consistentread.go b/vendor/k8s.io/kubernetes/pkg/util/io/consistentread.go index 6e1f17b09855..9a4c5393bcea 100644 --- a/vendor/k8s.io/kubernetes/pkg/util/io/consistentread.go +++ b/vendor/k8s.io/kubernetes/pkg/util/io/consistentread.go @@ -25,6 +25,9 @@ import ( // ConsistentRead repeatedly reads a file until it gets the same content twice. // This is useful when reading files in /proc that are larger than page size // and kernel may modify them between individual read() syscalls. +// It returns InconsistentReadError when it cannot get a consistent read in +// given nr. of attempts. Caller should retry, kernel is probably under heavy +// mount/unmount load. func ConsistentRead(filename string, attempts int) ([]byte, error) { oldContent, err := ioutil.ReadFile(filename) if err != nil { @@ -41,5 +44,26 @@ func ConsistentRead(filename string, attempts int) ([]byte, error) { // Files are different, continue reading oldContent = newContent } - return nil, fmt.Errorf("could not get consistent content of %s after %d attempts", filename, attempts) + return nil, InconsistentReadError{filename, attempts} +} + +// InconsistentReadError is returned from ConsistentRead when it cannot get +// a consistent read in given nr. of attempts. Caller should retry, kernel is +// probably under heavy mount/unmount load. +type InconsistentReadError struct { + filename string + attempts int +} + +func (i InconsistentReadError) Error() string { + return fmt.Sprintf("could not get consistent content of %s after %d attempts", i.filename, i.attempts) +} + +var _ error = InconsistentReadError{} + +func IsInconsistentReadError(err error) bool { + if _, ok := err.(InconsistentReadError); ok { + return true + } + return false } diff --git a/vendor/k8s.io/kubernetes/pkg/volume/fc/attacher.go b/vendor/k8s.io/kubernetes/pkg/volume/fc/attacher.go index 8436135c4637..088813f30e92 100644 --- a/vendor/k8s.io/kubernetes/pkg/volume/fc/attacher.go +++ b/vendor/k8s.io/kubernetes/pkg/volume/fc/attacher.go @@ -26,8 +26,8 @@ import ( "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" - utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/apimachinery/pkg/util/wait" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" diff --git a/vendor/k8s.io/kubernetes/pkg/volume/fc/fc.go b/vendor/k8s.io/kubernetes/pkg/volume/fc/fc.go index 80b864175bf7..0ea8ae995f95 100644 --- a/vendor/k8s.io/kubernetes/pkg/volume/fc/fc.go +++ b/vendor/k8s.io/kubernetes/pkg/volume/fc/fc.go @@ -28,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/types" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/features" + "k8s.io/kubernetes/pkg/util/io" "k8s.io/kubernetes/pkg/util/mount" utilstrings "k8s.io/kubernetes/pkg/util/strings" "k8s.io/kubernetes/pkg/volume" @@ -196,7 +197,7 @@ func (plugin *fcPlugin) newBlockVolumeMapperInternal(spec *volume.Spec, podUID t func (plugin *fcPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) { // Inject real implementations here, test through the internal function. - return plugin.newUnmounterInternal(volName, podUID, &FCUtil{}, plugin.host.GetMounter(plugin.GetPluginName()),plugin.host.GetExec(plugin.GetPluginName())) + return plugin.newUnmounterInternal(volName, podUID, &FCUtil{}, plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetExec(plugin.GetPluginName())) } func (plugin *fcPlugin) newUnmounterInternal(volName string, podUID types.UID, manager diskManager, mounter mount.Interface, exec mount.Exec) (volume.Unmounter, error) { @@ -239,7 +240,12 @@ func (plugin *fcPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volu // globalPDPath : plugins/kubernetes.io/fc/50060e801049cfd1-lun-0 var globalPDPath string mounter := plugin.host.GetMounter(plugin.GetPluginName()) - paths, err := mount.GetMountRefs(mounter, mountPath) + paths, err := util.GetReliableMountRefs(mounter, mountPath) + if io.IsInconsistentReadError(err) { + glog.Errorf("Failed to read mount refs from /proc/mounts for %s: %s", mountPath, err) + glog.Errorf("Kubelet cannot unmount volume at %s, please unmount it manually", mountPath) + return nil, err + } if err != nil { return nil, err } diff --git a/vendor/k8s.io/kubernetes/pkg/volume/iscsi/iscsi.go b/vendor/k8s.io/kubernetes/pkg/volume/iscsi/iscsi.go index 9bcbf7baa6fa..e1becc228710 100644 --- a/vendor/k8s.io/kubernetes/pkg/volume/iscsi/iscsi.go +++ b/vendor/k8s.io/kubernetes/pkg/volume/iscsi/iscsi.go @@ -27,10 +27,12 @@ import ( "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/kubernetes/pkg/util/io" "k8s.io/kubernetes/pkg/util/keymutex" "k8s.io/kubernetes/pkg/util/mount" utilstrings "k8s.io/kubernetes/pkg/util/strings" "k8s.io/kubernetes/pkg/volume" + "k8s.io/kubernetes/pkg/volume/util" ioutil "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/volumepathhandler" ) @@ -207,7 +209,12 @@ func (plugin *iscsiPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*v // Find globalPDPath from pod volume directory(mountPath) var globalPDPath string mounter := plugin.host.GetMounter(plugin.GetPluginName()) - paths, err := mount.GetMountRefs(mounter, mountPath) + paths, err := util.GetReliableMountRefs(mounter, mountPath) + if io.IsInconsistentReadError(err) { + glog.Errorf("Failed to read mount refs from /proc/mounts for %s: %s", mountPath, err) + glog.Errorf("Kubelet cannot unmount volume at %s, please unmount it and all mounts of the same device manually.", mountPath) + return nil, err + } if err != nil { return nil, err } diff --git a/vendor/k8s.io/kubernetes/pkg/volume/util/util.go b/vendor/k8s.io/kubernetes/pkg/volume/util/util.go index abf3db2230b7..2eaa30399e05 100644 --- a/vendor/k8s.io/kubernetes/pkg/volume/util/util.go +++ b/vendor/k8s.io/kubernetes/pkg/volume/util/util.go @@ -24,6 +24,7 @@ import ( "path/filepath" "strings" "syscall" + "time" "github.com/golang/glog" "k8s.io/api/core/v1" @@ -32,12 +33,14 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" utilfeature "k8s.io/apiserver/pkg/util/feature" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/pkg/api/legacyscheme" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" "k8s.io/kubernetes/pkg/features" kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" + "k8s.io/kubernetes/pkg/util/io" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" @@ -800,3 +803,27 @@ func MapBlockVolume( return nil } + +// GetReliableMountRefs calls mounter.GetMountRefs and retries on IsInconsistentReadError. +// To be used in volume reconstruction of volume plugins that don't have any protection +// against mounting a single volume on multiple nodes (such as attach/detach). +func GetReliableMountRefs(mounter mount.Interface, mountPath string) ([]string, error) { + var paths []string + var lastErr error + err := wait.PollImmediate(10*time.Millisecond, time.Minute, func() (bool, error) { + var err error + paths, err = mounter.GetMountRefs(mountPath) + if io.IsInconsistentReadError(err) { + lastErr = err + return false, nil + } + if err != nil { + return false, err + } + return true, nil + }) + if err == wait.ErrWaitTimeout { + return nil, lastErr + } + return paths, err +}