Add persistent volumes for tgtd.

Currently if you restart the server running nova-volume
or restart tgt, you will loose your iscsi targets that
have been created. This is not good.

In order for iscsi targets to be persistent across
reboots or restarts, one has to have the target's configuration
information in /etc/tgt/targets.conf or /etc/tgt/conf.d.
So when tgtd is restarted then the iscsi targets will be there
as expected.

This patch will add a configuration file to $state_path/volumes
when the volume is created. The configuration file is identified by
the volume uuid. It creates a logicalunit when the volume is created
as well. The iscsi target and configuration file
will be removed once the volume has been removed as well.

In order to use this, you have to include the following in
your /etc/tgt/targets.conf

include $state_path/volumes/*

For upgrades, it will just re-create the volumes
already in the volumes table.

Fixes LP: #1011159

Change-Id: I38fc096ab881ccb52cb688ae46d9d36b0a7b3a45
Signed-off-by: Chuck Short <chuck.short@canonical.com>

etc/nova/rootwrap.d/volume.filters

@@ -5,6 +5,7 @@
 # nova/volume/iscsi.py: iscsi_helper '--op' ...
 ietadm: CommandFilter, /usr/sbin/ietadm, root
 tgtadm: CommandFilter, /usr/sbin/tgtadm, root
+tgt-admin: CommandFilter, /usr/sbin/tgt-admin, root
 
 # nova/volume/driver.py: 'vgs', '--noheadings', '-o', 'name'
 vgs: CommandFilter, /sbin/vgs, root

nova/tests/test_iscsi.py

@@ -28,7 +28,8 @@ def setUp(self):
         self.tid = 1
         self.target_name = 'iqn.2011-09.org.foo.bar:blaa'
         self.lun = 10
-        self.path = '/foo/bar/blaa'
+        self.path = '/foo'
+        self.vol_id = 'blaa'
 
         self.script_template = None
 
@@ -65,11 +66,10 @@ def verify(self):
     def run_commands(self):
         tgtadm = iscsi.get_target_admin()
         tgtadm.set_execute(self.fake_execute)
-        tgtadm.new_target(self.target_name, self.tid)
+        tgtadm.create_iscsi_target(self.target_name, self.tid,
+                self.lun, self.path)
         tgtadm.show_target(self.tid)
-        tgtadm.new_logicalunit(self.tid, self.lun, self.path)
-        tgtadm.delete_logicalunit(self.tid, self.lun)
-        tgtadm.delete_target(self.tid)
+        tgtadm.remove_iscsi_target(self.tid, self.lun, self.vol_id)
 
     def test_target_admin(self):
         self.clear_cmds()
@@ -83,22 +83,11 @@ def setUp(self):
         super(TgtAdmTestCase, self).setUp()
         TargetAdminTestCase.setUp(self)
         self.flags(iscsi_helper='tgtadm')
+        self.flags(volumes_dir="./")
         self.script_template = "\n".join([
-        "tgtadm --op new --lld=iscsi --mode=target --tid=%(tid)s "
-                "--targetname=%(target_name)s",
-        "tgtadm --op bind --lld=iscsi --mode=target --initiator-address=ALL "
-                "--tid=%(tid)s",
-        "tgtadm --op show --lld=iscsi --mode=target --tid=%(tid)s",
-        "tgtadm --op new --lld=iscsi --mode=logicalunit --tid=%(tid)s "
-                "--lun=%(lun)d --backing-store=%(path)s",
-        "tgtadm --op delete --lld=iscsi --mode=logicalunit --tid=%(tid)s "
-                "--lun=%(lun)d",
-        "tgtadm --op delete --lld=iscsi --mode=target --tid=%(tid)s"])
-
-    def get_script_params(self):
-        params = super(TgtAdmTestCase, self).get_script_params()
-        params['lun'] += 1
-        return params
+        "/usr/sbin/tgt-admin --conf ./blaa --update blaa",
+        "tgtadm --op show --lld=iscsi --mode=target --tid=1",
+        "/usr/bin/tgt-admin --conf ./blaa --delete blaa"])
 
 
 class IetAdmTestCase(test.TestCase, TargetAdminTestCase):
@@ -109,8 +98,8 @@ def setUp(self):
         self.flags(iscsi_helper='ietadm')
         self.script_template = "\n".join([
         "ietadm --op new --tid=%(tid)s --params Name=%(target_name)s",
-        "ietadm --op show --tid=%(tid)s",
-        "ietadm --op new --tid=%(tid)s --lun=%(lun)d "
+        "ietadm --op new --tid=%(tid)s --lun=%(lun)s "
                 "--params Path=%(path)s,Type=fileio",
-        "ietadm --op delete --tid=%(tid)s --lun=%(lun)d",
-        "ietadm --op delete --tid=%(tid)s"])
+        "ietadm --op show --tid=%(tid)s",
+        "ietadm --op delete --tid=%(tid)s",
+        "ietadm --op delete --tid=%(tid)s --lun=%(lun)s"])

nova/volume/driver.py

@@ -274,9 +274,8 @@ def ensure_export(self, context, volume):
         iscsi_name = "%s%s" % (FLAGS.iscsi_target_prefix, volume['name'])
         volume_path = "/dev/%s/%s" % (FLAGS.volume_group, volume['name'])
 
-        self.tgtadm.new_target(iscsi_name, iscsi_target, check_exit_code=False)
-        self.tgtadm.new_logicalunit(iscsi_target, 0, volume_path,
-                                    check_exit_code=False)
+        self.tgtadm.create_iscsi_target(iscsi_name, iscsi_target,
+                0, volume_path, check_exit_code=False)
 
     def _ensure_iscsi_targets(self, context, host):
         """Ensure that target ids have been created in datastore."""
@@ -297,8 +296,8 @@ def create_export(self, context, volume):
         iscsi_name = "%s%s" % (FLAGS.iscsi_target_prefix, volume['name'])
         volume_path = "/dev/%s/%s" % (FLAGS.volume_group, volume['name'])
 
-        self.tgtadm.new_target(iscsi_name, iscsi_target)
-        self.tgtadm.new_logicalunit(iscsi_target, 0, volume_path)
+        self.tgtadm.create_iscsi_target(iscsi_name, iscsi_target,
+                0, volume_path)
 
         model_update = {}
         if FLAGS.iscsi_helper == 'tgtadm':
@@ -328,8 +327,7 @@ def remove_export(self, context, volume):
                        "is presently exported for volume: %s"), volume['id'])
             return
 
-        self.tgtadm.delete_logicalunit(iscsi_target, 0)
-        self.tgtadm.delete_target(iscsi_target)
+        self.tgtadm.remove_iscsi_target(iscsi_target, 0, volume['id'])
 
     def _do_iscsi_discovery(self, volume):
         #TODO(justinsb): Deprecate discovery and use stored info

nova/volume/iscsi.py

@@ -19,18 +19,27 @@
 Helper code for the iSCSI volume driver.
 
 """
+import os
 
+from nova import exception
 from nova import flags
 from nova.openstack.common import cfg
+from nova.openstack.common import log as logging
 from nova import utils
 
+LOG = logging.getLogger(__name__)
 
-iscsi_helper_opt = cfg.StrOpt('iscsi_helper',
-                              default='tgtadm',
-                              help='iscsi target user-land tool to use')
+iscsi_helper_opt = [
+        cfg.StrOpt('iscsi_helper',
+                    default='tgtadm',
+                    help='iscsi target user-land tool to use'),
+        cfg.StrOpt('volumes_dir',
+                   default='$state_path/volumes',
+                   help='Volume configfuration file storage directory'),
+]
 
 FLAGS = flags.FLAGS
-FLAGS.register_opt(iscsi_helper_opt)
+FLAGS.register_opts(iscsi_helper_opt)
 
 
 class TargetAdmin(object):
@@ -50,23 +59,31 @@ def set_execute(self, execute):
     def _run(self, *args, **kwargs):
         self._execute(self._cmd, *args, run_as_root=True, **kwargs)
 
-    def new_target(self, name, tid, **kwargs):
+    def create_iscsi_target(self, name, tid, lun, path, **kwargs):
+        """Create a iSCSI target and logical unit"""
+        raise NotImplementedError()
+
+    def remove_iscsi_target(self, tid, lun, vol_id, **kwargs):
+        """Remove a iSCSI target and logical unit"""
+        raise NotImplementedError()
+
+    def _new_target(self, name, tid, **kwargs):
         """Create a new iSCSI target."""
         raise NotImplementedError()
 
-    def delete_target(self, tid, **kwargs):
+    def _delete_target(self, tid, **kwargs):
         """Delete a target."""
         raise NotImplementedError()
 
     def show_target(self, tid, **kwargs):
         """Query the given target ID."""
         raise NotImplementedError()
 
-    def new_logicalunit(self, tid, lun, path, **kwargs):
+    def _new_logicalunit(self, tid, lun, path, **kwargs):
         """Create a new LUN on a target using the supplied path."""
         raise NotImplementedError()
 
-    def delete_logicalunit(self, tid, lun, **kwargs):
+    def _delete_logicalunit(self, tid, lun, **kwargs):
         """Delete a logical unit from a target."""
         raise NotImplementedError()
 
@@ -77,59 +94,77 @@ class TgtAdm(TargetAdmin):
     def __init__(self, execute=utils.execute):
         super(TgtAdm, self).__init__('tgtadm', execute)
 
-    def new_target(self, name, tid, **kwargs):
-        self._run('--op', 'new',
-                  '--lld=iscsi', '--mode=target',
-                  '--tid=%s' % tid,
-                  '--targetname=%s' % name,
-                  **kwargs)
-        self._run('--op', 'bind',
-                  '--lld=iscsi', '--mode=target',
-                  '--initiator-address=ALL',
-                  '--tid=%s' % tid,
-                  **kwargs)
-
-    def delete_target(self, tid, **kwargs):
-        self._run('--op', 'delete',
-                  '--lld=iscsi', '--mode=target',
-                  '--tid=%s' % tid,
-                  **kwargs)
+    def create_iscsi_target(self, name, tid, lun, path, **kwargs):
+        try:
+            if not os.path.exists(FLAGS.volumes_dir):
+                os.makedirs(FLAGS.volumes_dir)
+
+            # grab the volume id
+            vol_id = name.split(':')[1]
+
+            volume_conf = """
+                <target %s>
+                    backing-store %s
+                </target>
+            """ % (name, path)
+
+            LOG.info(_('Creating volume: %s') % vol_id)
+            volume_path = os.path.join(FLAGS.volumes_dir, vol_id)
+            if not os.path.isfile(volume_path):
+                f = open(volume_path, 'w+')
+                f.write(volume_conf)
+                f.close()
+
+            self._execute('/usr/sbin/tgt-admin', '--conf %s' % volume_path,
+                        '--update %s' % vol_id, run_as_root=True)
+
+        except Exception as ex:
+            LOG.exception(ex)
+            raise exception.NovaException(_('Failed to create volume: %s')
+                % vol_id)
+
+    def remove_iscsi_target(self, tid, lun, vol_id, **kwargs):
+        try:
+            LOG.info(_('Removing volume: %s') % vol_id)
+            volume_path = os.path.join(FLAGS.volumes_dir, vol_id)
+            if os.path.isfile(volume_path):
+                self._execute('/usr/bin/tgt-admin', '--conf %s' % volume_path,
+                    '--delete %s' % vol_id, run_as_root_root=True)
+                os.unlink(volume_path)
+        except Exception as ex:
+            LOG.exception(ex)
+            raise exception.NovaException(_('Failed to remove volume: %s')
+                    % vol_id)
 
     def show_target(self, tid, **kwargs):
         self._run('--op', 'show',
                   '--lld=iscsi', '--mode=target',
                   '--tid=%s' % tid,
                   **kwargs)
 
-    def new_logicalunit(self, tid, lun, path, **kwargs):
-        self._run('--op', 'new',
-                  '--lld=iscsi', '--mode=logicalunit',
-                  '--tid=%s' % tid,
-                  '--lun=%d' % (lun + 1),  # lun0 is reserved
-                  '--backing-store=%s' % path,
-                  **kwargs)
-
-    def delete_logicalunit(self, tid, lun, **kwargs):
-        self._run('--op', 'delete',
-                  '--lld=iscsi', '--mode=logicalunit',
-                  '--tid=%s' % tid,
-                  '--lun=%d' % (lun + 1),
-                  **kwargs)
-
 
 class IetAdm(TargetAdmin):
     """iSCSI target administration using ietadm."""
 
     def __init__(self, execute=utils.execute):
         super(IetAdm, self).__init__('ietadm', execute)
 
-    def new_target(self, name, tid, **kwargs):
+    def create_iscsi_target(self, name, tid, lun, path, **kwargs):
+        self._new_target(name, tid, **kwargs)
+        self._new_logicalunit(tid, lun, path, **kwargs)
+
+    def remove_iscsi_target(self, tid, lun, vol_id, **kwargs):
+        LOG.info(_('Removing volume: %s') % vol_id)
+        self._delete_target(tid, **kwargs)
+        self._delete_logicalunit(tid, lun, **kwargs)
+
+    def _new_target(self, name, tid, **kwargs):
         self._run('--op', 'new',
                   '--tid=%s' % tid,
                   '--params', 'Name=%s' % name,
                   **kwargs)
 
-    def delete_target(self, tid, **kwargs):
+    def _delete_target(self, tid, **kwargs):
         self._run('--op', 'delete',
                   '--tid=%s' % tid,
                   **kwargs)
@@ -139,14 +174,14 @@ def show_target(self, tid, **kwargs):
                   '--tid=%s' % tid,
                   **kwargs)
 
-    def new_logicalunit(self, tid, lun, path, **kwargs):
+    def _new_logicalunit(self, tid, lun, path, **kwargs):
         self._run('--op', 'new',
                   '--tid=%s' % tid,
                   '--lun=%d' % lun,
                   '--params', 'Path=%s,Type=fileio' % path,
                   **kwargs)
 
-    def delete_logicalunit(self, tid, lun, **kwargs):
+    def _delete_logicalunit(self, tid, lun, **kwargs):
         self._run('--op', 'delete',
                   '--tid=%s' % tid,
                   '--lun=%d' % lun,