Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

force rbd image unlock if the image is not used #41597

Merged
merged 1 commit into from
Mar 1, 2017
Merged

force rbd image unlock if the image is not used #41597

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
83 changes: 54 additions & 29 deletions pkg/volume/rbd/rbd_util.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ import (

const (
imageWatcherStr = "watcher="
kubeLockMagic = "kubelet_lock_magic_"
)

// search /sys/bus for rbd device that matches given pool and image
Expand Down Expand Up @@ -114,8 +115,15 @@ func (util *RBDUtil) rbdLock(b rbdMounter, lock bool) error {
} else {
secret_opt = []string{"-k", b.Keyring}
}
if len(b.adminId) == 0 {
b.adminId = b.Id
}
if len(b.adminSecret) == 0 {
b.adminSecret = b.Secret
}

// construct lock id using host name and a magic prefix
lock_id := "kubelet_lock_magic_" + node.GetHostname("")
lock_id := kubeLockMagic + node.GetHostname("")

l := len(b.Mon)
// avoid mount storm, pick a host randomly
Expand All @@ -130,7 +138,7 @@ func (util *RBDUtil) rbdLock(b rbdMounter, lock bool) error {
cmd, err = b.plugin.execCommand("rbd",
append([]string{"lock", "list", b.Image, "--pool", b.Pool, "--id", b.Id, "-m", mon}, secret_opt...))
output = string(cmd)

glog.Infof("lock list output %q", output)
if err != nil {
continue
}
Expand All @@ -142,6 +150,23 @@ func (util *RBDUtil) rbdLock(b rbdMounter, lock bool) error {
glog.V(1).Infof("rbd: lock already held for %s", lock_id)
return nil
}
// clean up orphaned lock if no watcher on the image
used, statusErr := util.rbdStatus(&b)
if statusErr == nil && !used {
re := regexp.MustCompile("client.* " + kubeLockMagic + ".*")
locks := re.FindAllStringSubmatch(output, -1)
for _, v := range locks {
if len(v) > 0 {
lockInfo := strings.Split(v[0], " ")
if len(lockInfo) > 2 {
cmd, err = b.plugin.execCommand("rbd",
append([]string{"lock", "remove", b.Image, lockInfo[1], lockInfo[0], "--pool", b.Pool, "--id", b.Id, "-m", mon}, secret_opt...))
glog.Infof("remove orphaned locker %s from client %s: err %v, output: %s", lockInfo[1], lockInfo[0], err, string(cmd))
}
}
}
}

// hold a lock: rbd lock add
cmd, err = b.plugin.execCommand("rbd",
append([]string{"lock", "add", b.Image, lock_id, "--pool", b.Pool, "--id", b.Id, "-m", mon}, secret_opt...))
Expand Down Expand Up @@ -220,13 +245,39 @@ func (util *RBDUtil) AttachDisk(b rbdMounter) error {
var err error
var output []byte

// create mount point
globalPDPath := b.manager.MakeGlobalPDName(*b.rbd)
notMnt, err := b.mounter.IsLikelyNotMountPoint(globalPDPath)
// in the first time, the path shouldn't exist and IsLikelyNotMountPoint is expected to get NotExist
if err != nil && !os.IsNotExist(err) {
return fmt.Errorf("rbd: %s failed to check mountpoint", globalPDPath)
}
if !notMnt {
return nil
}
if err = os.MkdirAll(globalPDPath, 0750); err != nil {
return fmt.Errorf("rbd: failed to mkdir %s, error", globalPDPath)
}

devicePath, found := waitForPath(b.Pool, b.Image, 1)
if !found {
// modprobe
_, err = b.plugin.execCommand("modprobe", []string{"rbd"})
if err != nil {
return fmt.Errorf("rbd: failed to modprobe rbd error:%v", err)
}

// fence off other mappers
if err = util.fencing(b); err != nil {
return fmt.Errorf("rbd: image %s is locked by other nodes", b.Image)
}
// rbd lock remove needs ceph and image config
// but kubelet doesn't get them from apiserver during teardown
// so persit rbd config so upon disk detach, rbd lock can be removed
// since rbd json is persisted in the same local directory that is used as rbd mountpoint later,
// the json file remains invisible during rbd mount and thus won't be removed accidentally.
util.persistRBD(b, globalPDPath)

// rbd map
l := len(b.Mon)
// avoid mount storm, pick a host randomly
Expand Down Expand Up @@ -255,34 +306,8 @@ func (util *RBDUtil) AttachDisk(b rbdMounter) error {
return errors.New("Could not map image: Timeout after 10s")
}
}
// mount it
globalPDPath := b.manager.MakeGlobalPDName(*b.rbd)
notMnt, err := b.mounter.IsLikelyNotMountPoint(globalPDPath)
// in the first time, the path shouldn't exist and IsLikelyNotMountPoint is expected to get NotExist
if err != nil && !os.IsNotExist(err) {
return fmt.Errorf("rbd: %s failed to check mountpoint", globalPDPath)
}
if !notMnt {
return nil
}

if err := os.MkdirAll(globalPDPath, 0750); err != nil {
return fmt.Errorf("rbd: failed to mkdir %s, error", globalPDPath)
}

// fence off other mappers
if err := util.fencing(b); err != nil {
// rbd unmap before exit
b.plugin.execCommand("rbd", []string{"unmap", devicePath})
return fmt.Errorf("rbd: image %s is locked by other nodes", b.Image)
}
// rbd lock remove needs ceph and image config
// but kubelet doesn't get them from apiserver during teardown
// so persit rbd config so upon disk detach, rbd lock can be removed
// since rbd json is persisted in the same local directory that is used as rbd mountpoint later,
// the json file remains invisible during rbd mount and thus won't be removed accidentally.
util.persistRBD(b, globalPDPath)

// mount it
if err = b.mounter.FormatAndMount(devicePath, globalPDPath, b.fsType, nil); err != nil {
err = fmt.Errorf("rbd: failed to mount rbd volume %s [%s] to %s, error %v", devicePath, b.fsType, globalPDPath, err)
}
Expand Down