Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix for Premature iSCSI logout #39202. #41196

Merged
merged 3 commits into from
Feb 15, 2017
Merged

Fix for Premature iSCSI logout #39202. #41196

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b0d285c
Fix for Premature iSCSI logout #39202.
Dec 23, 2016
2aaeefe
Updated TestExtractDeviceAndPrefix and added TestExtractIface to refl…
Feb 9, 2017
b23b475
Implemented suggestions for #39202 fix to facilitate kubelet upgrade.…
Feb 14, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
59 changes: 45 additions & 14 deletions pkg/volume/iscsi/iscsi_util.go
100644 → 100755
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,30 +90,39 @@ func getDevicePrefixRefCount(mounter mount.Interface, deviceNamePrefix string) (
return refCount, nil
}

// make a directory like /var/lib/kubelet/plugins/kubernetes.io/iscsi/portal-some_iqn-lun-lun_id
func makePDNameInternal(host volume.VolumeHost, portal string, iqn string, lun string) string {
return path.Join(host.GetPluginDir(iscsiPluginName), portal+"-"+iqn+"-lun-"+lun)
// make a directory like /var/lib/kubelet/plugins/kubernetes.io/iscsi/iface_name/portal-some_iqn-lun-lun_id
func makePDNameInternal(host volume.VolumeHost, portal string, iqn string, lun string, iface string) string {
return path.Join(host.GetPluginDir(iscsiPluginName), "iface-"+iface, portal+"-"+iqn+"-lun-"+lun)
}

type ISCSIUtil struct{}

func (util *ISCSIUtil) MakeGlobalPDName(iscsi iscsiDisk) string {
return makePDNameInternal(iscsi.plugin.host, iscsi.portals[0], iscsi.iqn, iscsi.lun)
return makePDNameInternal(iscsi.plugin.host, iscsi.portals[0], iscsi.iqn, iscsi.lun, iscsi.iface)
}

func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) error {
var devicePath string
var devicePaths []string
var iscsiTransport string

out, err := b.plugin.execCommand("iscsiadm", []string{"-m", "iface", "-I", b.iface, "-o", "show"})
if err != nil {
glog.Errorf("iscsi: could not read iface %s error: %s", b.iface, string(out))
return err
}

iscsiTransport = extractTransportname(string(out))

bkpPortal := b.portals
for _, tp := range bkpPortal {
// Rescan sessions to discover newly mapped LUNs. Do not specify the interface when rescanning
// to avoid establishing additional sessions to the same target.
out, err := b.plugin.execCommand("iscsiadm", []string{"-m", "node", "-p", tp, "-T", b.iqn, "-R"})
if err != nil {
glog.Errorf("iscsi: failed to rescan session with error: %s (%v)", string(out), err)
}

if iscsiTransport == "" {
glog.Errorf("iscsi: could not find transport name in iface %s", b.iface)
return errors.New(fmt.Sprintf("Could not parse iface file for %s", b.iface))
Expand Down Expand Up @@ -207,18 +216,29 @@ func (util *ISCSIUtil) DetachDisk(c iscsiDiskUnmounter, mntPath string) error {
return err
}
refCount, err := getDevicePrefixRefCount(c.mounter, prefix)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

also see this #41041 (comment)

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I thought about changing the HasPrefix call to Contains but I wanted to account for the iface as well and log out only if there are no other pods that use that specific iface. I know it's not foul proof especially when there are multiple ifaces using the same transport to the same target as the devices that get mounted are not necessarily the ones that were imported on the given iface, but I could not find a better solution until now.

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This seems like a solid solution to me, at least for the moment. In the long-term, I'm concerned about cases where sessions that exist independently of Kubernetes get logged off because a pod used a volume connected to the target using the same interface (I think this is the same concern @cristianpop cited just now, but I'm not totally sure). Solving that might be doable with something like resolving all of the symlinks in /dev/disk/by-path to their devices and then checking whether any of the devices that belong to the session are still mounted; I don't know if this would behave well with multipathing, though.

At any rate, this is a better fix than just changing HasPrefix to Contains.


if err == nil && refCount == 0 {
// this portal/iqn are no longer referenced, log out
// extract portal and iqn from device path
// This portal/iqn/iface is no longer referenced, log out.
// Extract the portal and iqn from device path.
portal, iqn, err := extractPortalAndIqn(device)
if err != nil {
return err
}
glog.Infof("iscsi: log out target %s iqn %s", portal, iqn)
out, err := c.plugin.execCommand("iscsiadm", []string{"-m", "node", "-p", portal, "-T", iqn, "--logout"})
if err != nil {
glog.Errorf("iscsi: failed to detach disk Error: %s", string(out))
// Extract the iface from the mountPath and use it to log out. If the iface
// is not found, maintain the previous behavior to facilitate kubelet upgrade.
// Logout may fail as no session may exist for the portal/IQN on the specified interface.
iface, found := extractIface(mntPath)
if found {
glog.Infof("iscsi: log out target %s iqn %s iface %s", portal, iqn, iface)
out, err := c.plugin.execCommand("iscsiadm", []string{"-m", "node", "-p", portal, "-T", iqn, "-I", iface, "--logout"})
if err != nil {
glog.Errorf("iscsi: failed to detach disk Error: %s", string(out))
}
} else {
glog.Infof("iscsi: log out target %s iqn %s", portal, iqn)
out, err := c.plugin.execCommand("iscsiadm", []string{"-m", "node", "-p", portal, "-T", iqn, "--logout"})
if err != nil {
glog.Errorf("iscsi: failed to detach disk Error: %s", string(out))
}
}
}
}
Expand Down Expand Up @@ -248,15 +268,26 @@ func extractDeviceAndPrefix(mntPath string) (string, string, error) {
return "", "", fmt.Errorf("iscsi detach disk: malformatted mnt path: %s", mntPath)
}
device := mntPath[(ind + 1):]
// strip -lun- from device path
ind = strings.LastIndex(device, "-lun-")
// strip -lun- from mount path
ind = strings.LastIndex(mntPath, "-lun-")
if ind < 0 {
return "", "", fmt.Errorf("iscsi detach disk: malformatted mnt path: %s", mntPath)
}
prefix := device[:ind]
prefix := mntPath[:ind]
return device, prefix, nil
}

func extractIface(mntPath string) (string, bool) {
re := regexp.MustCompile(`.+/iface-([^/]+)/.+`)

re_output := re.FindStringSubmatch(mntPath)
if re_output != nil {
return re_output[1], true
}

return "", false
}

func extractPortalAndIqn(device string) (string, string, error) {
ind1 := strings.Index(device, "-")
if ind1 < 0 {
Expand Down
20 changes: 17 additions & 3 deletions pkg/volume/iscsi/iscsi_util_test.go
100644 → 100755
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,10 +57,24 @@ func TestGetDevicePrefixRefCount(t *testing.T) {

func TestExtractDeviceAndPrefix(t *testing.T) {
devicePath := "127.0.0.1:3260-iqn.2014-12.com.example:test.tgt00"
mountPrefix := "/var/lib/kubelet/plugins/kubernetes.io/iscsi/iface-default/" + devicePath
lun := "-lun-0"
device, prefix, err := extractDeviceAndPrefix("/var/lib/kubelet/plugins/kubernetes.io/iscsi/" + devicePath + lun)
if err != nil || device != (devicePath+lun) || prefix != devicePath {
t.Errorf("extractDeviceAndPrefix: expected %s and %s, got %v %s and %s", devicePath+lun, devicePath, err, device, prefix)
device, prefix, err := extractDeviceAndPrefix(mountPrefix + lun)
if err != nil || device != (devicePath+lun) || prefix != mountPrefix {
t.Errorf("extractDeviceAndPrefix: expected %s and %s, got %v %s and %s", devicePath+lun, mountPrefix, err, device, prefix)
}
}

func TestExtractIface(t *testing.T) {
ifaceName := "default"
devicePath := "127.0.0.1:3260-iqn.2014-12.com.example:test.tgt00-lun-0"
iface, found := extractIface("/var/lib/kubelet/plugins/kubernetes.io/iscsi/iface-" + ifaceName + "/" + devicePath)
if !found || iface != ifaceName {
t.Errorf("extractIface: expected %s and %t, got %s and %t", ifaceName, true, iface, found)
}
iface, found = extractIface("/var/lib/kubelet/plugins/kubernetes.io/iscsi/" + devicePath)
if found || iface != "" {
t.Errorf("extractIface: expected %s and %t, got %s and %t", "", false, iface, found)
}
}

Expand Down