Consistency Group Support for the Generic Driver

The Generic Driver in Manila has no concept of consistency
groups. This patch proposes to implement a loose
consistency group support mechanism for the Generic Driver,
using existing logic to snapshot shares and create shares
from snapshots.

The actions concerning creating and deleting a consistency
group are handled by the API and the Generic Driver does not
need to remember any relevant information persistently.

The actions concerning consistency group snapshots are not
simultaneous, but are rather let to run in succession.

DocImpact
Partially implements bp manila-consistency-groups

Change-Id: Ie5e51205a1154786f9057691aa82bffcafe04de7

contrib/ci/post_test_hook.sh

@@ -88,7 +88,7 @@ fi
 set +o errexit
 cd $BASE/new/tempest
 
-export MANILA_TEMPEST_CONCURRENCY=${MANILA_TEMPEST_CONCURRENCY:-12}
+export MANILA_TEMPEST_CONCURRENCY=${MANILA_TEMPEST_CONCURRENCY:-6}
 export MANILA_TESTS=${MANILA_TESTS:-'manila_tempest_tests.tests.api'}
 
 if [[ "$JOB_NAME" =~ "scenario" ]]; then

manila/share/drivers/generic.py

@@ -108,7 +108,7 @@ def wrap(self, context, *args, **kwargs):
                 raise exception.ManilaException(
                     _("Share server handling is not available. "
                       "But 'share_server' was provided. '%s'. "
-                      "Share network should not be used.") % server['id'])
+                      "Share network should not be used.") % server.get('id'))
         elif not server:
             raise exception.ManilaException(
                 _("Share server handling is enabled. But 'share_server' "
@@ -556,7 +556,9 @@ def _update_share_stats(self):
         data = dict(
             share_backend_name=self.backend_name,
             storage_protocol='NFS_CIFS',
-            reserved_percentage=(self.configuration.reserved_share_percentage))
+            reserved_percentage=self.configuration.reserved_share_percentage,
+            consistency_group_support='pool',
+        )
         super(GenericShareDriver, self)._update_share_stats(data)
 
     @ensure_server
@@ -942,6 +944,180 @@ def _get_consumed_space(self, mount_path, server_details):
 
         return size
 
+    @ensure_server
+    def create_consistency_group(self, context, cg_dict, share_server=None):
+        """Creates a consistency group.
+
+        Since we are faking the CG object, apart from verifying if the
+        share_server is valid, we do nothing else here.
+        """
+
+        LOG.debug('Created a Consistency Group with ID: %s.', cg_dict['id'])
+
+        msg = _LW('The Generic driver has no means to guarantee consistency '
+                  'group snapshots are actually consistent. This '
+                  'implementation is for reference and testing purposes only.')
+        LOG.warning(msg)
+
+    def delete_consistency_group(self, context, cg_dict, share_server=None):
+        """Deletes a consistency group.
+
+        Since we are faking the CG object we do nothing here.
+        """
+
+        LOG.debug('Deleted the consistency group with ID %s.', cg_dict['id'])
+
+    def _cleanup_cg_share_snapshot(self, context, share_snapshot,
+                                   share_server):
+        """Deletes the snapshot of a share belonging to a consistency group."""
+
+        try:
+            self.delete_snapshot(context, share_snapshot, share_server)
+        except exception.ManilaException:
+            msg = _LE('Could not delete CG Snapshot %(snap)s '
+                      'for share %(share)s.')
+            LOG.error(msg % {
+                'snap': share_snapshot['id'],
+                'share': share_snapshot['share_id'],
+            })
+            raise
+
+    @ensure_server
+    def create_cgsnapshot(self, context, snap_dict, share_server=None):
+        """Creates a consistency group snapshot one or more shares."""
+
+        LOG.debug('Attempting to create a CG snapshot %s.' % snap_dict['id'])
+
+        msg = _LW('The Consistency Group Snapshot being created is '
+                  'not expected to be consistent. This implementation is '
+                  'for reference and testing purposes only.')
+        LOG.warning(msg)
+
+        cg_members = snap_dict.get('cgsnapshot_members', [])
+        if not cg_members:
+            LOG.warning(_LW('No shares in Consistency Group to Create CG '
+                            'snapshot.'))
+        else:
+            share_snapshots = []
+            for member in cg_members:
+                share_snapshot = {
+                    'share_id': member['share_id'],
+                    'id': member['id'],
+                }
+                try:
+                    self.create_snapshot(context, share_snapshot, share_server)
+                    share_snapshots.append(share_snapshot)
+                except exception.ManilaException as e:
+                    msg = _LE('Could not create CG Snapshot. Failed '
+                              'to create share snapshot %(snap)s for '
+                              'share %(share)s.')
+                    LOG.exception(msg % {
+                        'snap': share_snapshot['id'],
+                        'share': share_snapshot['share_id']
+                    })
+
+                    # clean up any share snapshots previously created
+                    LOG.debug('Attempting to clean up snapshots due to '
+                              'failure...')
+                    for share_snapshot in share_snapshots:
+                        self._cleanup_cg_share_snapshot(context,
+                                                        share_snapshot,
+                                                        share_server)
+                    raise e
+
+            LOG.debug('Successfully created CG snapshot %s.' % snap_dict['id'])
+
+        return None, None
+
+    @ensure_server
+    def delete_cgsnapshot(self, context, snap_dict, share_server=None):
+        """Deletes a consistency group snapshot."""
+
+        cg_members = snap_dict.get('cgsnapshot_members', [])
+
+        LOG.debug('Deleting CG snapshot %s.' % snap_dict['id'])
+
+        for member in cg_members:
+            share_snapshot = {
+                'share_id': member['share_id'],
+                'id': member['id'],
+            }
+
+            self._cleanup_cg_share_snapshot(context,
+                                            share_snapshot,
+                                            share_server)
+
+        LOG.debug('Deleted CG snapshot %s.' % snap_dict['id'])
+
+        return None, None
+
+    @ensure_server
+    def create_consistency_group_from_cgsnapshot(self, context, cg_dict,
+                                                 cgsnapshot_dict,
+                                                 share_server=None):
+        """Creates a consistency group from an existing CG snapshot."""
+
+        # Ensure that the consistency group snapshot has members
+        if not cgsnapshot_dict['cgsnapshot_members']:
+            return None, None
+
+        clone_list = self._collate_cg_snapshot_info(cg_dict, cgsnapshot_dict)
+        share_update_list = list()
+
+        LOG.debug('Creating consistency group from CG snapshot %s.',
+                  cgsnapshot_dict['id'])
+
+        for clone in clone_list:
+
+            kwargs = {}
+            if self.driver_handles_share_servers:
+                kwargs['share_server'] = share_server
+            export_location = (
+                self.create_share_from_snapshot(
+                    context,
+                    clone['share'],
+                    clone['snapshot'],
+                    **kwargs))
+
+            share_update_list.append({
+                'id': clone['share']['id'],
+                'export_locations': export_location,
+            })
+
+        return None, share_update_list
+
+    def _collate_cg_snapshot_info(self, cg_dict, cgsnapshot_dict):
+        """Collate the data for a clone of the CG snapshot.
+
+        Given two data structures, a CG snapshot (cgsnapshot_dict) and a new
+        CG to be cloned from the snapshot (cg_dict), match up both
+        structures into a list of dicts (share & snapshot) suitable for use
+        by existing method that clones individual share snapshots.
+        """
+
+        clone_list = list()
+
+        for share in cg_dict['shares']:
+
+            clone_info = {'share': share}
+
+            for cgsnapshot_member in cgsnapshot_dict['cgsnapshot_members']:
+                if (share['source_cgsnapshot_member_id'] ==
+                        cgsnapshot_member['id']):
+                    clone_info['snapshot'] = {
+                        'id': cgsnapshot_member['id'],
+                    }
+                    break
+
+            if len(clone_info) != 2:
+                msg = _("Invalid data supplied for creating consistency "
+                        "group from CG snapshot %s.") % cgsnapshot_dict['id']
+                raise exception.InvalidConsistencyGroup(reason=msg)
+
+            clone_list.append(clone_info)
+
+        return clone_list
+
 
 class NASHelperBase(object):
     """Interface to work with share."""

manila/share/manager.py

@@ -1269,7 +1269,7 @@ def create_consistency_group(self, context, cg_id):
         share_network_id = group_ref.get('share_network_id', None)
         share_server = None
 
-        if parent_share_server_id:
+        if parent_share_server_id and self.driver.driver_handles_share_servers:
             share_server = self.db.share_server_get(context,
                                                     parent_share_server_id)
             share_network_id = share_server['share_network_id']