Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: corrupted mount point in csi driver node stage/publish #88569

Merged
merged 1 commit into from Mar 2, 2020
Merged

fix: corrupted mount point in csi driver node stage/publish #88569

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 2 additions & 0 deletions pkg/volume/csi/BUILD
View file
Open in desktop
Expand Up @@ -42,6 +42,7 @@ go_library(
"//vendor/google.golang.org/grpc/codes:go_default_library",
"//vendor/google.golang.org/grpc/status:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
"//vendor/k8s.io/utils/mount:go_default_library",
"//vendor/k8s.io/utils/strings:go_default_library",
],
)
Expand Down Expand Up @@ -86,6 +87,7 @@ go_test(
"//staging/src/k8s.io/client-go/util/testing:go_default_library",
"//staging/src/k8s.io/component-base/featuregate/testing:go_default_library",
"//vendor/github.com/container-storage-interface/spec/lib/go/csi:go_default_library",
"//vendor/github.com/stretchr/testify/assert:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
],
)
Expand Down
14 changes: 10 additions & 4 deletions pkg/volume/csi/csi_attacher.go
View file
Open in desktop
Expand Up @@ -28,7 +28,7 @@ import (

"k8s.io/klog"

"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
storage "k8s.io/api/storage/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand Down Expand Up @@ -228,13 +228,19 @@ func (c *csiAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMo
return errors.New(log("attacher.MountDevice failed, deviceMountPath is empty"))
}

corruptedDir := false
mounted, err := isDirMounted(c.plugin, deviceMountPath)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think actually we may want to skip this check altogether and just always call StagePublish. @gnufied @jsafrane do you see any problems with that? Ref #86784

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes, we don't even need to do os.MkdirAll(deviceMountPath, 0750), all leave csi driver to handle that logic:

kubernetes/pkg/volume/csi/csi_attacher.go

Lines 290 to 293 in e4a5012

if err = os.MkdirAll(deviceMountPath, 0750); err != nil {
return errors.New(log("attacher.MountDevice failed to create dir %#v: %v", deviceMountPath, err))
}
klog.V(4).Info(log("created target path successfully [%s]", deviceMountPath))

this looks like a big behavior change, is it ok to do this in this PR?

Copy link
Member

@gnufied gnufied Feb 27, 2020 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am okay with removing the mounted check, but it requires uncertain mount fix to work reliably, so we won't be able to backport that change to versions without uncertain fix

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would like to back port this fix to old release, so shall we go with this PR now?
about removing the mounted check, I could work out another PR that won't be back ported, is that ok? @gnufied thanks.

if err != nil {
klog.Error(log("attacher.MountDevice failed while checking mount status for dir [%s]", deviceMountPath))
return err
if isCorruptedDir(deviceMountPath) {
corruptedDir = true // leave to CSI driver to handle corrupted mount
klog.Warning(log("attacher.MountDevice detected corrupted mount for dir [%s]", deviceMountPath))
} else {
return err
}
}

if mounted {
if mounted && !corruptedDir {
klog.V(4).Info(log("attacher.MountDevice skipping mount, dir already mounted [%s]", deviceMountPath))
return nil
}
Expand Down Expand Up @@ -287,7 +293,7 @@ func (c *csiAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMo

// Store volume metadata for UnmountDevice. Keep it around even if the
// driver does not support NodeStage, UnmountDevice still needs it.
if err = os.MkdirAll(deviceMountPath, 0750); err != nil {
if err = os.MkdirAll(deviceMountPath, 0750); err != nil && !corruptedDir {
return errors.New(log("attacher.MountDevice failed to create dir %#v: %v", deviceMountPath, err))
}
klog.V(4).Info(log("created target path successfully [%s]", deviceMountPath))
Expand Down
18 changes: 15 additions & 3 deletions pkg/volume/csi/csi_mounter.go
View file
Open in desktop
Expand Up @@ -36,6 +36,7 @@ import (
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/volume"
volumetypes "k8s.io/kubernetes/pkg/volume/util/types"
"k8s.io/utils/mount"
utilstrings "k8s.io/utils/strings"
)

Expand Down Expand Up @@ -105,12 +106,18 @@ func (c *csiMountMgr) SetUp(mounterArgs volume.MounterArgs) error {
func (c *csiMountMgr) SetUpAt(dir string, mounterArgs volume.MounterArgs) error {
klog.V(4).Infof(log("Mounter.SetUpAt(%s)", dir))

corruptedDir := false
mounted, err := isDirMounted(c.plugin, dir)
if err != nil {
return errors.New(log("mounter.SetUpAt failed while checking mount status for dir [%s]: %v", dir, err))
if isCorruptedDir(dir) {
corruptedDir = true // leave to CSI driver to handle corrupted mount
klog.Warning(log("mounter.SetUpAt detected corrupted mount for dir [%s]", dir))
} else {
return errors.New(log("mounter.SetUpAt failed while checking mount status for dir [%s]: %v", dir, err))
}
}

if mounted {
if mounted && !corruptedDir {
klog.V(4).Info(log("mounter.SetUpAt skipping mount, dir already mounted [%s]", dir))
return nil
}
Expand Down Expand Up @@ -211,7 +218,7 @@ func (c *csiMountMgr) SetUpAt(dir string, mounterArgs volume.MounterArgs) error
}

// create target_dir before call to NodePublish
if err := os.MkdirAll(dir, 0750); err != nil {
if err := os.MkdirAll(dir, 0750); err != nil && !corruptedDir {
return errors.New(log("mounter.SetUpAt failed to create dir %#v: %v", dir, err))
}
klog.V(4).Info(log("created target path successfully [%s]", dir))
Expand Down Expand Up @@ -417,6 +424,11 @@ func isDirMounted(plug *csiPlugin, dir string) (bool, error) {
return !notMnt, nil
}

func isCorruptedDir(dir string) bool {
_, pathErr := mount.PathExists(dir)
return pathErr != nil && mount.IsCorruptedMnt(pathErr)
}

// removeMountDir cleans the mount dir when dir is not mounted and removed the volume data file in dir
func removeMountDir(plug *csiPlugin, mountPath string) error {
klog.V(4).Info(log("removing mount path [%s]", mountPath))
Expand Down
33 changes: 33 additions & 0 deletions pkg/volume/csi/csi_mounter_test.go
View file
Open in desktop
Expand Up @@ -19,6 +19,7 @@ package csi
import (
"context"
"fmt"
"io/ioutil"
"math/rand"
"os"
"path"
Expand All @@ -27,6 +28,8 @@ import (

"reflect"

"github.com/stretchr/testify/assert"

api "k8s.io/api/core/v1"
storage "k8s.io/api/storage/v1"
storagev1beta1 "k8s.io/api/storage/v1beta1"
Expand Down Expand Up @@ -825,3 +828,33 @@ func TestUnmounterTeardown(t *testing.T) {
}

}

func TestIsCorruptedDir(t *testing.T) {
existingMountPath, err := ioutil.TempDir(os.TempDir(), "blobfuse-csi-mount-test")
if err != nil {
t.Fatalf("failed to create tmp dir: %v", err)
}
defer os.RemoveAll(existingMountPath)

tests := []struct {
desc string
dir string
expectedResult bool
}{
{
desc: "NotExist dir",
dir: "/tmp/NotExist",
expectedResult: false,
},
{
desc: "Existing dir",
dir: existingMountPath,
expectedResult: false,
},
}

for i, test := range tests {
isCorruptedDir := isCorruptedDir(test.dir)
assert.Equal(t, test.expectedResult, isCorruptedDir, "TestCase[%d]: %s", i, test.desc)
}
}