From 481f274f4af86db8ed332df9ebbec29d10271d01 Mon Sep 17 00:00:00 2001 From: Huamin Chen Date: Wed, 16 Sep 2015 08:52:22 -0400 Subject: [PATCH] rbd: don't use /dev/rbd/pool/image; use sysfs to find rbd device instead rbd: if rbd image is not formatted, format it to the designated filesystem type rbd: update example README.md and include instructions to get base64 encoded Ceph secret if rbd fails to lock image, unmap the image before exiting Signed-off-by: Huamin Chen --- examples/rbd/README.md | 11 +++++-- pkg/volume/rbd/rbd.go | 2 +- pkg/volume/rbd/rbd_util.go | 60 +++++++++++++++++++++++++++++--------- 3 files changed, 57 insertions(+), 16 deletions(-) diff --git a/examples/rbd/README.md b/examples/rbd/README.md index 799d0b6bfa7f..2cda5aab7de8 100644 --- a/examples/rbd/README.md +++ b/examples/rbd/README.md @@ -7,7 +7,7 @@ Install Ceph on the Kubernetes host. For example, on Fedora 21 - # yum -y install ceph + # yum -y install ceph-common If you don't have a Ceph cluster, you can set up a [containerized Ceph cluster](https://github.com/rootfs/docker-ceph) @@ -26,7 +26,14 @@ Once you have installed Ceph and new Kubernetes, you can create a pod based on m # Use Ceph Authentication Secret -If Ceph authentication secret is provided, the secret should be first be base64 encoded, then encoded string is placed in a secret yaml. An example yaml is provided [here](secret/ceph-secret.yaml). Then post the secret through ```kubectl``` in the following command. +If Ceph authentication secret is provided, the secret should be first be *base64 encoded*, then encoded string is placed in a secret yaml. For example, getting Ceph user `kube`'s base64 encoded secret can use the following command: + +```console + # grep key /etc/ceph/ceph.client.kube.keyring |awk '{printf "%s", $NF}'|base64 +QVFBTWdYaFZ3QkNlRGhBQTlubFBhRnlmVVNhdEdENGRyRldEdlE9PQ== +``` + +An example yaml is provided [here](secret/ceph-secret.yaml). Then post the secret through ```kubectl``` in the following command. ```console # kubectl create -f examples/rbd/secret/ceph-secret.yaml diff --git a/pkg/volume/rbd/rbd.go b/pkg/volume/rbd/rbd.go index 55b53b0eaa1e..431f920da767 100644 --- a/pkg/volume/rbd/rbd.go +++ b/pkg/volume/rbd/rbd.go @@ -131,7 +131,7 @@ func (plugin *rbdPlugin) newBuilderInternal(spec *volume.Spec, podUID types.UID, Pool: pool, ReadOnly: readOnly, manager: manager, - mounter: mounter, + mounter: &mount.SafeFormatAndMount{mounter, exec.New()}, plugin: plugin, }, Mon: source.CephMonitors, diff --git a/pkg/volume/rbd/rbd_util.go b/pkg/volume/rbd/rbd_util.go index f7f36d36ff02..3b2c66cec429 100644 --- a/pkg/volume/rbd/rbd_util.go +++ b/pkg/volume/rbd/rbd_util.go @@ -25,31 +25,63 @@ import ( "encoding/json" "errors" "fmt" + "io/ioutil" "math/rand" "os" "path" + "regexp" "strings" "time" "github.com/golang/glog" + "k8s.io/kubernetes/pkg/util/exec" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/util/node" "k8s.io/kubernetes/pkg/volume" ) +// search /sys/bus for rbd device that matches given pool and image +func getDevFromImageAndPool(pool, image string) (string, bool) { + // /sys/bus/rbd/devices/X/name and /sys/bus/rbd/devices/X/pool + sys_path := "/sys/bus/rbd/devices" + if dirs, err := ioutil.ReadDir(sys_path); err == nil { + for _, f := range dirs { + // pool and name format: + // see rbd_pool_show() and rbd_name_show() at + // https://github.com/torvalds/linux/blob/master/drivers/block/rbd.c + name := f.Name() + // first match pool, then match name + po := path.Join(sys_path, name, "pool") + img := path.Join(sys_path, name, "name") + exe := exec.New() + out, err := exe.Command("cat", po, img).CombinedOutput() + if err != nil { + continue + } + matched, err := regexp.MatchString("^"+pool+"\n"+image+"\n$", string(out)) + if err != nil || !matched { + continue + } + // found a match, check if device exists + devicePath := "/dev/rbd" + name + if _, err := os.Lstat(devicePath); err == nil { + return devicePath, true + } + } + } + return "", false +} + // stat a path, if not exists, retry maxRetries times -func waitForPathToExist(devicePath string, maxRetries int) bool { +func waitForPath(pool, image string, maxRetries int) (string, bool) { for i := 0; i < maxRetries; i++ { - _, err := os.Stat(devicePath) - if err == nil { - return true - } - if err != nil && !os.IsNotExist(err) { - return false + devicePath, found := getDevFromImageAndPool(pool, image) + if found { + return devicePath, true } time.Sleep(time.Second) } - return false + return "", false } // make a directory like /var/lib/kubelet/plugins/kubernetes.io/pod/rbd/pool-image-image @@ -178,9 +210,9 @@ func (util *RBDUtil) defencing(c rbdCleaner) error { func (util *RBDUtil) AttachDisk(b rbdBuilder) error { var err error - devicePath := strings.Join([]string{"/dev/rbd", b.Pool, b.Image}, "/") - exist := waitForPathToExist(devicePath, 1) - if !exist { + + devicePath, found := waitForPath(b.Pool, b.Image, 1) + if !found { // modprobe _, err = b.plugin.execCommand("modprobe", []string{"rbd"}) if err != nil { @@ -209,8 +241,8 @@ func (util *RBDUtil) AttachDisk(b rbdBuilder) error { if err != nil { return err } - exist = waitForPathToExist(devicePath, 10) - if !exist { + devicePath, found = waitForPath(b.Pool, b.Image, 10) + if !found { return errors.New("Could not map image: Timeout after 10s") } // mount it @@ -230,6 +262,8 @@ func (util *RBDUtil) AttachDisk(b rbdBuilder) error { // fence off other mappers if err := util.fencing(b); err != nil { + // rbd unmap before exit + b.plugin.execCommand("rbd", []string{"unmap", devicePath}) return fmt.Errorf("rbd: image %s is locked by other nodes", b.Image) } // rbd lock remove needs ceph and image config