Merge branch 'CP-9983'

Merge from Creedence.

Signed-off-by: Thanos Makatos <thanos.makatos@citrix.com>
Reviewed-by: Siddharth Vinothkumar <siddharth.vinothkumar@citrix.com>

GitHub: closes #232 on xapi-project/sm

INSTALL

@@ -1 +1,3 @@
-Currently building this package outside of the XenServer build environment is somewhat broken. The goal is to replace it with a simple "make"/"make install" process.
+Currently building this package outside of the XenServer build environment is
+somewhat broken. The goal is to replace it with a simple "make"/"make install"
+process.

Makefile

@@ -56,6 +56,7 @@ SM_LIBS += lcache
 SM_LIBS += resetvdis
 SM_LIBS += B_util
 SM_LIBS += wwid_conf
+SM_LIBS += trim_util
 
 UDEV_RULES = 40-multipath
 MPATH_DAEMON = sm-multipath
@@ -158,6 +159,8 @@ install: precheck
 	install -m 755 drivers/tapdisk-pause $(SM_STAGING)$(PLUGIN_SCRIPT_DEST)
 	install -m 755 drivers/vss_control $(SM_STAGING)$(PLUGIN_SCRIPT_DEST)
 	install -m 755 drivers/intellicache-clean $(SM_STAGING)$(PLUGIN_SCRIPT_DEST)
+	install -m 755 drivers/enable-borehamwood $(SM_STAGING)$(SM_DEST)
+	install -m 755 drivers/trim $(SM_STAGING)$(PLUGIN_SCRIPT_DEST)
 	ln -sf $(PLUGIN_SCRIPT_DEST)vss_control $(SM_STAGING)$(SM_DEST)
 	install -m 755 drivers/iscsilib.py $(SM_STAGING)$(SM_DEST)
 	mkdir -p $(SM_STAGING)$(LIBEXEC)

README.md

@@ -3,6 +3,6 @@ Storage Manager for XenServer
 [![Build Status](https://travis-ci.org/xapi-project/sm.svg?branch=master)](https://travis-ci.org/xapi-project/sm)
 [![Coverage Status](https://coveralls.io/repos/xapi-project/sm/badge.png?branch=master)](https://coveralls.io/r/xapi-project/sm?branch=master)
 
-This repository contains the code which forms the Storage Management layer for XenSever. It consists of a series of "plug-ins" to xapi (the Xen management layer) which are written primarily in python.
-
-
+This repository contains the code which forms the Storage Management layer for
+XenSever. It consists of a series of "plug-ins" to xapi (the Xen management
+layer) which are written primarily in python.

drivers/EXTSR.py

@@ -70,6 +70,8 @@ def load(self, sr_uuid):
         self.remotepath = os.path.join("/dev",self.vgname,sr_uuid)
         self.attached = self._checkmount()
 
+        self._check_o_direct()
+
     def delete(self, sr_uuid):
         super(EXTSR, self).delete(sr_uuid)
 

drivers/FileSR.py

@@ -75,6 +75,14 @@ def handles(srtype):
         return srtype == 'file'
     handles = staticmethod(handles)
 
+    def _check_o_direct(self):
+        if self.sr_ref and self.session is not None:
+            other_config = self.session.xenapi.SR.get_other_config(self.sr_ref)
+            o_direct = other_config.get("o_direct")
+            self.o_direct = o_direct is not None and o_direct == "true"
+        else:
+            self.o_direct = True
+
     def load(self, sr_uuid):
         self.ops_exclusive = OPS_EXCLUSIVE
         self.lock = Lock(vhdutil.LOCK_TYPE_SR, self.uuid)
@@ -85,6 +93,8 @@ def load(self, sr_uuid):
         self.path = os.path.join(SR.MOUNT_BASE, sr_uuid)
         self.attached = False
 
+        self._check_o_direct()
+
     def create(self, sr_uuid, size):
         """ Create the SR.  The path must not already exist, or if it does, 
         it must be empty.  (This accounts for the case where the user has
@@ -381,6 +391,8 @@ class FileVDI(VDI.VDI):
     def load(self, vdi_uuid):
         self.lock = self.sr.lock
 
+        self.sr.srcmd.params['o_direct'] = self.sr.o_direct
+
         if self.sr.srcmd.cmd == "vdi_create":
             self.vdi_type = vhdutil.VDI_TYPE_VHD
             if self.sr.srcmd.params.has_key("vdi_sm_config") and \

drivers/HBASR.py

@@ -41,8 +41,6 @@
     'configuration': CONFIGURATION
     }
 
-HBA_CLI      = "/opt/Citrix/StorageLink/bin/csl_hbascan"
-
 class HBASR(SR.SR):
     """HBA storage repository"""
     def handles(type):
@@ -82,7 +80,7 @@ def _init_hba_hostname(self):
                 nodewwnval = str(nodewwn[0].firstChild.nodeValue)
                 break
         except:
-            raise xs_errors.XenError('CSLGXMLParse', opterr='HBA Host WWN scanning failed')
+            raise xs_errors.XenError('XMLParse', opterr='HBA Host WWN scanning failed')
         return nodewwnval
 
     def _init_hbas(self):
@@ -104,7 +102,7 @@ def _init_hbas(self):
                     devpath = str(devnames[0].firstChild.nodeValue).split('/')[-1]
                     adt[devpath] = portval.split()[0]
         except:
-            raise xs_errors.XenError('CSLGXMLParse', \
+            raise xs_errors.XenError('XMLParse', \
                                      opterr='HBA scanning failed')
         return adt
 
@@ -166,7 +164,7 @@ def _probe_hba(self):
 
             return dom.toxml()
         except:
-            raise xs_errors.XenError('CSLGXMLParse', \
+            raise xs_errors.XenError('XMLParse', \
                                      opterr='HBA probe failed')
 
     def attach(self, sr_uuid):

drivers/ISCSISR.py

@@ -626,13 +626,17 @@ def _getLUNbySMconfig(self, sm_config):
     def print_LUNs(self):
         self.LUNs = {}
         if os.path.exists(self.path):
+            dom0_disks = util.dom0_disks()
             for file in util.listdir(self.path):
                 if file.find("LUN") != -1 and file.find("_") == -1:
                     vdi_path = os.path.join(self.path,file)
-                    LUNid = file.replace("LUN","")
-                    obj = self.vdi(self.uuid)
-                    obj._query(vdi_path, LUNid)
-                    self.LUNs[obj.uuid] = obj
+                    if os.path.realpath(vdi_path) in dom0_disks:
+                        util.SMlog("Hide dom0 boot disk LUN")
+                    else:
+                        LUNid = file.replace("LUN","")
+                        obj = self.vdi(self.uuid)
+                        obj._query(vdi_path, LUNid)
+                        self.LUNs[obj.uuid] = obj
 
     def print_entries(self, map):
         dom = xml.dom.minidom.Document()

drivers/ISOSR.py

@@ -31,8 +31,9 @@
     [ [ 'location', 'path to mount (required) (e.g. server:/path)' ], 
       [ 'options', 
         'extra options to pass to mount (deprecated) (e.g. \'-o ro\')' ],
-      [ 'type','cifs or nfs'] ]
-
+      [ 'type','cifs or nfs'],
+      nfs.NFS_VERSION]
+
 DRIVER_INFO = {
     'name': 'ISO',
     'description': 'Handles CD images stored as files in iso format',
@@ -209,6 +210,9 @@ def load(self, sr_uuid):
         else:
             self.path = self.mountpoint
 
+        # Handle optional dconf attributes
+        self.nfsversion = nfs.validate_nfsversion(self.dconf.get('nfsversion'))
+
         # Some info we need:
         self.sr_vditype = 'phy'
         self.credentials = None
@@ -261,7 +265,8 @@ def attach(self, sr_uuid):
             # to the process waiting.
             if self.dconf.has_key('type') and self.dconf['type']!='cifs':
                 serv_path = location.split(':')
-                nfs.soft_mount(self.mountpoint, serv_path[0], serv_path[1], 'tcp')
+                nfs.soft_mount(self.mountpoint, serv_path[0], serv_path[1],
+                               'tcp', nfsversion=self.nfsversion)
             else:
                 util.pread(mountcmd, True)
         except util.CommandException, inst:

drivers/LVHDSR.py

@@ -52,7 +52,7 @@
 DEV_MAPPER_ROOT = os.path.join('/dev/mapper', lvhdutil.VG_PREFIX)
 
 geneology = {}
-CAPABILITIES = ["SR_PROBE","SR_UPDATE",
+CAPABILITIES = ["SR_PROBE","SR_UPDATE", "SR_TRIM",
         "VDI_CREATE","VDI_DELETE","VDI_ATTACH", "VDI_DETACH",
         "VDI_CLONE", "VDI_SNAPSHOT", "VDI_RESIZE", "ATOMIC_PAUSE",
         "VDI_RESET_ON_BOOT/2", "VDI_UPDATE"]
@@ -542,11 +542,48 @@ def delete(self, uuid):
             raise xs_errors.XenError('LVMMaster')
         cleanup.gc_force(self.session, self.uuid)
 
+        success = True
+        for fileName in \
+            filter(lambda x: util.extractSRFromDevMapper(x) == self.uuid, \
+                glob.glob(DEV_MAPPER_ROOT + '*')):
+            if util.doesFileHaveOpenHandles(fileName):
+                util.SMlog("LVHDSR.delete: The dev mapper entry %s has open " \
+                           "handles" % fileName)
+                success = False
+                continue
+
+            # Now attempt to remove the dev mapper entry
+            if not lvutil.removeDevMapperEntry(fileName):
+                success = False
+                continue
+
+            try:
+                lvname = os.path.basename(fileName.replace('-','/').\
+                                          replace('//', '-'))
+                os.unlink(os.path.join(self.path, lvname))
+            except Exception, e:
+                util.SMlog("LVHDSR.delete: failed to remove the symlink for " \
+                           "file %s. Error: %s" % (fileName, str(e)))
+                success = False
+
+        if success:
+            try:
+                if util.pathexists(self.path):
+                    os.rmdir(self.path)
+            except Exception, e:
+                util.SMlog("LVHDSR.delete: failed to remove the symlink " \
+                           "directory %s. Error: %s" % (self.path, str(e)))
+                success = False
+
         self._removeMetadataVolume()
         self.lvmCache.refresh()
         if len(lvhdutil.getLVInfo(self.lvmCache)) > 0:
             raise xs_errors.XenError('SRNotEmpty')
 
+        if not success:
+            raise Exception("LVHDSR delete failed, please refer to the log " \
+                            "for details.")
+
         lvutil.removeVG(self.root, self.vgname)
         self._cleanup()
 
@@ -1122,8 +1159,9 @@ def _undoAllInflateJournals(self):
                 lvhdutil.deflate(self.lvmCache, vdi.lvname, int(val))
                 if vdi.readonly:
                     self.lvmCache.setReadonly(vdi.lvname, True)
-                lvhdutil.lvRefreshOnAllSlaves(self.session, self.uuid,
-                        self.vgname, vdi.lvname, uuid)
+                if "true" == self.session.xenapi.SR.get_shared(self.sr_ref):
+                    lvhdutil.lvRefreshOnAllSlaves(self.session, self.uuid,
+                            self.vgname, vdi.lvname, uuid)
             self.journaler.remove(lvhdutil.JRN_INFLATE, uuid)
         delattr(self,"vdiInfo")
         delattr(self,"allVDIs")
@@ -1313,13 +1351,18 @@ def create(self, sr_uuid, vdi_uuid, size):
         self.sm_config = self.sr.srcmd.params["vdi_sm_config"]
         self.sr._ensureSpaceAvailable(lvSize)
 
-        self.sr.lvmCache.create(self.lvname, lvSize)
-        if self.vdi_type == vhdutil.VDI_TYPE_RAW:
-            self.size = self.sr.lvmCache.getSize(self.lvname)
-        else:
-            vhdutil.create(self.path, long(size), False, lvhdutil.MSIZE_MB)
-            self.size = vhdutil.getSizeVirt(self.path)
-        self.sr.lvmCache.deactivateNoRefcount(self.lvname)
+        try:
+            self.sr.lvmCache.create(self.lvname, lvSize)
+            if self.vdi_type == vhdutil.VDI_TYPE_RAW:
+                self.size = self.sr.lvmCache.getSize(self.lvname)
+            else:
+               vhdutil.create(self.path, long(size), False, lvhdutil.MSIZE_MB)
+               self.size = vhdutil.getSizeVirt(self.path)
+            self.sr.lvmCache.deactivateNoRefcount(self.lvname)
+        except util.CommandException, e:
+            util.SMlog("Unable to create VDI");
+            self.sr.lvmCache.remove(self.lvname)
+            raise xs_errors.XenError('VDICreate', opterr='error %d' % e.code)
 
         self.utilisation = lvSize
         self.sm_config["vdi_type"] = self.vdi_type
@@ -2083,12 +2126,7 @@ def update(self, sr_uuid, vdi_uuid):
             self.session.xenapi.VDI.get_metadata_of_pool(vdi_ref)
         LVMMetadataHandler(self.sr.mdpath).updateMetadata(update_map)
 
-try:
-    if __name__ == '__main__':
-        SRCommand.run(LVHDSR, DRIVER_INFO)
-    else:
-        SR.registerSR(LVHDSR)
-except Exception:
-    util.logException("LVHD")
-    raise
-
+if __name__ == '__main__':
+    SRCommand.run(LVHDSR, DRIVER_INFO)
+else:
+    SR.registerSR(LVHDSR)

drivers/LVHDoHBASR.py

@@ -31,10 +31,10 @@
 import glob
 import mpp_luncheck
 
-CAPABILITIES = ["SR_PROBE", "SR_UPDATE", "SR_METADATA", "VDI_CREATE", 
-                "VDI_DELETE", "VDI_ATTACH", "VDI_DETACH", 
-                "VDI_GENERATE_CONFIG", "VDI_SNAPSHOT", "VDI_CLONE", 
-                "VDI_RESIZE", "ATOMIC_PAUSE", "VDI_RESET_ON_BOOT/2", 
+CAPABILITIES = ["SR_PROBE", "SR_UPDATE", "SR_METADATA", "SR_TRIM",
+                "VDI_CREATE", "VDI_DELETE", "VDI_ATTACH", "VDI_DETACH",
+                "VDI_GENERATE_CONFIG", "VDI_SNAPSHOT", "VDI_CLONE",
+                "VDI_RESIZE", "ATOMIC_PAUSE", "VDI_RESET_ON_BOOT/2",
                 "VDI_UPDATE"]
 
 CONFIGURATION = [ [ 'SCSIid', 'The scsi_id of the destination LUN' ], \

drivers/LVHDoISCSISR.py

@@ -29,8 +29,8 @@
 import scsiutil
 import xml.dom.minidom
 
-CAPABILITIES = ["SR_PROBE", "SR_UPDATE", "SR_METADATA", "VDI_CREATE",
-                "VDI_DELETE", "VDI_ATTACH", "VDI_DETACH",
+CAPABILITIES = ["SR_PROBE", "SR_UPDATE", "SR_METADATA", "SR_TRIM",
+                "VDI_CREATE", "VDI_DELETE", "VDI_ATTACH", "VDI_DETACH",
                 "VDI_GENERATE_CONFIG", "VDI_CLONE", "VDI_SNAPSHOT",
                 "VDI_RESIZE", "ATOMIC_PAUSE", "VDI_RESET_ON_BOOT/2",
                 "VDI_UPDATE"]
@@ -320,12 +320,8 @@ def print_LUNs_XML(self):
         dom = xml.dom.minidom.Document()
         element = dom.createElement("iscsi-target")
         dom.appendChild(element)
-        # Omit the scsi-id used by iSL
-        isl_scsiids = util.get_isl_scsiids(self.session)
         for uuid in self.LUNs:
             val = self.LUNs[uuid]
-            if getattr(val,'SCSIid') in isl_scsiids:
-                continue
             entry = dom.createElement('LUN')
             element.appendChild(entry)
 

drivers/NFSSR.py

@@ -35,8 +35,9 @@
                 "VDI_GENERATE_CONFIG",
                 "VDI_RESET_ON_BOOT/2", "ATOMIC_PAUSE"]
 
-CONFIGURATION = [ [ 'server', 'hostname or IP address of NFS server (required)' ], \
-                  [ 'serverpath', 'path on remote server (required)' ] ]
+CONFIGURATION = [['server', 'hostname or IP address of NFS server (required)'],
+                 ['serverpath', 'path on remote server (required)'],
+                 nfs.NFS_VERSION]
 
 
 DRIVER_INFO = {
@@ -83,13 +84,16 @@ def load(self, sr_uuid):
         self.nosubdir = self.sm_config.get('nosubdir') == "true"
         if self.dconf.has_key('serverpath'):
             self.remotepath = os.path.join(self.dconf['serverpath'],
-                                           not self.nosubdir and sr_uuid or "")
+                    not self.nosubdir and sr_uuid or "").encode('utf-8')
         self.path = os.path.join(SR.MOUNT_BASE, sr_uuid)
 
-        # Test for the optional 'nfsoptions' dconf attribute
+        # Handle optional dconf attributes
         self.transport = DEFAULT_TRANSPORT
         if self.dconf.has_key('useUDP') and self.dconf['useUDP'] == 'true':
             self.transport = "udp"
+        self.nfsversion = nfs.validate_nfsversion(self.dconf.get('nfsversion'))
+
+        self._check_o_direct()
 
 
     def validate_remotepath(self, scan):
@@ -106,16 +110,17 @@ def validate_remotepath(self, scan):
 
     def check_server(self):
         try:
-            nfs.check_server_tcp(self.remoteserver)
+            nfs.check_server_tcp(self.remoteserver, self.nfsversion)
         except nfs.NfsException, exc:
             raise xs_errors.XenError('NFSVersion',
                                      opterr=exc.errstr)
 
 
     def mount(self, mountpoint, remotepath, timeout = 0):
         try:
-            nfs.soft_mount(mountpoint, self.remoteserver, remotepath,
-                    self.transport, timeout)
+            nfs.soft_mount(
+                mountpoint, self.remoteserver, remotepath, self.transport,
+                timeout=timeout, nfsversion=self.nfsversion)
         except nfs.NfsException, exc:
             raise xs_errors.XenError('NFSMount', opterr=exc.errstr)
 
@@ -180,7 +185,7 @@ def create(self, sr_uuid, size):
 
         # Set the target path temporarily to the base dir
         # so that we can create the target SR directory
-        self.remotepath = self.dconf['serverpath']
+        self.remotepath = self.dconf['serverpath'].encode('utf-8')
         try:
             self.mount_remotepath(sr_uuid)
         except Exception, exn:
@@ -216,7 +221,7 @@ def delete(self, sr_uuid):
 
             # Set the target path temporarily to the base dir
             # so that we can remove the target SR directory
-            self.remotepath = self.dconf['serverpath']
+            self.remotepath = self.dconf['serverpath'].encode('utf-8')
             self.mount_remotepath(sr_uuid)
             if not self.nosubdir:
                 newpath = os.path.join(self.path, sr_uuid)

drivers/RawHBASR.py

@@ -334,6 +334,8 @@ def _query(self, path, id, uuid=None, scsi_id=None):
         sm_config = util.default(self, "sm_config", lambda: {})
         sm_config['LUNid'] = str(self.LUNid)
         sm_config['SCSIid'] = self.SCSIid
+        # Make sure to use kernel blkback (not blktap3) for raw LUNs
+        sm_config['backend-kind'] = 'vbd'
         self.sm_config = sm_config
 
     def attach(self, sr_uuid, vdi_uuid):