Porting Cinder Backup tests to Zaza

Porting the Amulet tests from Cinder Backup to the Zaza framework.

The Amulet tests can be found here:
https://opendev.org/openstack/charm-cinder-backup/src/commit/4273738b94b22cca187487af2bad982be49fdd69/tests/basic_deployment.py

zaza/openstack/charm_tests/cinder_backup/__init__.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+
+# Copyright 2019 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Collection of code for setting up and testing cinder-backup."""

zaza/openstack/charm_tests/cinder_backup/tests.py

@@ -0,0 +1,181 @@
+#!/usr/bin/env python3
+#
+# Copyright 2019 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Encapsulate cinder-backup testing."""
+import logging
+
+import zaza.model
+import zaza.openstack.charm_tests.test_utils as test_utils
+import zaza.openstack.utilities.ceph as ceph_utils
+import zaza.openstack.utilities.openstack as openstack_utils
+
+
+class CinderBackupTest(test_utils.OpenStackBaseTest):
+    """Encapsulate Cinder Backup tests."""
+
+    RESOURCE_PREFIX = 'zaza-cinderbackuptests'
+
+    @classmethod
+    def setUpClass(cls):
+        """Run class setup for running Cinder Backup tests."""
+        super(CinderBackupTest, cls).setUpClass()
+        cls.cinder_client = openstack_utils.get_cinder_session_client(
+            cls.keystone_session)
+
+    @property
+    def services(self):
+        """Return a list services for Openstack Release."""
+        current_release = openstack_utils.get_os_release()
+        services = ['cinder-scheduler', 'cinder-volume']
+        if (current_release >=
+                openstack_utils.get_os_release('xenial_ocata')):
+            services.append('apache2')
+        else:
+            services.append('cinder-api')
+        return services
+
+    def test_100_volume_create_extend_delete(self):
+        """Test creating, extending a volume."""
+        vol_new = openstack_utils.create_volume(
+            self.cinder_client,
+            name='{}-100-vol'.format(self.RESOURCE_PREFIX),
+            size=1)
+        self.cinder_client.volumes.extend(
+            vol_new.id,
+            '2')
+        openstack_utils.resource_reaches_status(
+            self.cinder_client.volumes,
+            vol_new.id,
+            expected_status="available",
+            msg="Volume status wait")
+
+    def test_410_cinder_vol_create_backup_delete_restore_pool_inspect(self):
+        """Create, backup, delete, restore a ceph-backed cinder volume.
+
+        Create, backup, delete, restore a ceph-backed cinder volume, and
+        inspect ceph cinder pool object count as the volume is created
+        and deleted.
+        """
+        unit_name = zaza.model.get_lead_unit_name('ceph-mon')
+        obj_count_samples = []
+        pool_size_samples = []
+        pools = ceph_utils.get_ceph_pools(unit_name)
+        expected_pool = 'cinder-ceph'
+        cinder_ceph_pool = pools[expected_pool]
+
+        # Check ceph cinder pool object count, disk space usage and pool name
+        logging.info('Checking ceph cinder pool original samples...')
+        pool_name, obj_count, kb_used = ceph_utils.get_ceph_pool_sample(
+            unit_name, cinder_ceph_pool)
+
+        obj_count_samples.append(obj_count)
+        pool_size_samples.append(kb_used)
+
+        self.assertEqual(pool_name, expected_pool)
+
+        # Create ceph-backed cinder volume
+        cinder_vol = openstack_utils.create_volume(
+            self.cinder_client,
+            name='{}-410-vol'.format(self.RESOURCE_PREFIX),
+            size=1,
+            wait_iteration_max_time=180)
+        # Backup the volume
+        vol_backup = openstack_utils.create_volume_backup(
+            self.cinder_client,
+            cinder_vol.id,
+            name='{}-410-backup-vol'.format(self.RESOURCE_PREFIX),
+            wait_iteration_max_time=180)
+        # Delete the volume
+        openstack_utils.delete_volume(self.cinder_client, cinder_vol.id)
+        # Restore the volume
+        self.cinder_client.restores.restore(vol_backup.id)
+        openstack_utils.resource_reaches_status(
+            self.cinder_client.backups,
+            vol_backup.id,
+            wait_iteration_max_time=180,
+            stop_after_attempt=15,
+            expected_status='available',
+            msg='Backup status wait')
+        # Delete the backup
+        openstack_utils.delete_volume_backup(
+            self.cinder_client,
+            vol_backup.id)
+        openstack_utils.resource_removed(
+            self.cinder_client.volumes,
+            vol_backup.id,
+            wait_iteration_max_time=180,
+            msg="Backup volume")
+
+        # Re-check ceph cinder pool object count and disk usage
+        logging.info('Checking ceph cinder pool samples '
+                     'after volume create...')
+        pool_name, obj_count, kb_used = ceph_utils.get_ceph_pool_sample(
+            unit_name, cinder_ceph_pool, self.model_name)
+
+        obj_count_samples.append(obj_count)
+        pool_size_samples.append(kb_used)
+
+        name = '{}-410-vol'.format(self.RESOURCE_PREFIX)
+        vols = self.cinder_client.volumes.list()
+        try:
+            cinder_vols = [v for v in vols if v.name == name]
+        except AttributeError:
+            cinder_vols = [v for v in vols if v.display_name == name]
+        if not cinder_vols:
+            # NOTE(hopem): it appears that at some point cinder-backup stopped
+            # restoring volume metadata properly so revert to default name if
+            # original is not found
+            name = "restore_backup_{}".format(vol_backup.id)
+            try:
+                cinder_vols = [v for v in vols if v.name == name]
+            except AttributeError:
+                cinder_vols = [v for v in vols if v.display_name == name]
+
+        self.assertTrue(cinder_vols)
+
+        cinder_vol = cinder_vols[0]
+
+        # Delete restored cinder volume
+        openstack_utils.delete_volume(self.cinder_client, cinder_vol.id)
+        openstack_utils.resource_removed(
+            self.cinder_client.volumes,
+            cinder_vol.id,
+            wait_iteration_max_time=180,
+            msg="Volume")
+
+        # Final check, ceph cinder pool object count and disk usage
+        logging.info('Checking ceph cinder pool after volume delete...')
+        pool_name, obj_count, kb_used = ceph_utils.get_ceph_pool_sample(
+            unit_name, cinder_ceph_pool, self.model_name)
+
+        obj_count_samples.append(obj_count)
+        pool_size_samples.append(kb_used)
+
+        # Validate ceph cinder pool object count samples over time
+        original, created, deleted = range(3)
+        self.assertFalse(obj_count_samples[created] <=
+                         obj_count_samples[original])
+        self.assertFalse(obj_count_samples[deleted] >=
+                         obj_count_samples[created])
+
+        # Luminous (pike) ceph seems more efficient at disk usage so we cannot
+        # grantee the ordering of kb_used
+        if (openstack_utils.get_os_release() <
+                openstack_utils.get_os_release('xenial_mitaka')):
+            self.assertFalse(pool_size_samples[created] <=
+                             pool_size_samples[original])
+            self.assertFalse(pool_size_samples[deleted] >=
+                             pool_size_samples[created])

zaza/openstack/utilities/ceph.py

@@ -1,5 +1,5 @@
 """Module containing Ceph related utilities."""
-
+import json
 import logging
 
 import zaza.openstack.utilities.openstack as openstack_utils
@@ -97,6 +97,54 @@ def get_ceph_pools(unit_name, model_name=None):
     return pools
 
 
+def get_ceph_df(unit_name, model_name=None):
+    """Return dict of ceph df json output, including ceph pool state.
+
+    :param unit_name: Name of the unit to get ceph df
+    :type unit_name: string
+    :param model_name: Name of model to operate in
+    :type model_name: str
+    :returns: Dict of ceph df output
+    :rtype: dict
+    :raise: zaza.model.CommandRunFailed
+    """
+    cmd = 'sudo ceph df --format=json'
+    result = zaza_model.run_on_unit(unit_name, cmd, model_name=model_name)
+    if result.get('Code') != '0':
+        raise zaza_model.CommandRunFailed(cmd, result)
+    return json.loads(result.get('Stdout'))
+
+
+def get_ceph_pool_sample(unit_name, pool_id=0, model_name=None):
+    """Return list of ceph pool attributes.
+
+    Take a sample of attributes of a ceph pool, returning ceph
+    pool name, object count and disk space used for the specified
+    pool ID number.
+
+    :param unit_name: Name of the unit to get the pool sample
+    :type unit_name: string
+    :param pool_id: Ceph pool ID
+    :type pool_id: int
+    :param model_name: Name of model to operate in
+    :type model_name: str
+    :returns: List of pool name, object count, kb disk space used
+    :rtype: list
+    :raises: zaza.model.CommandRunFailed
+    """
+    df = get_ceph_df(unit_name, model_name)
+    for pool in df['pools']:
+        if pool['id'] == pool_id:
+            pool_name = pool['name']
+            obj_count = pool['stats']['objects']
+            kb_used = pool['stats']['kb_used']
+
+    logging.debug('Ceph {} pool (ID {}): {} objects, '
+                  '{} kb used'.format(pool_name, pool_id,
+                                      obj_count, kb_used))
+    return pool_name, obj_count, kb_used
+
+
 def get_rbd_hash(unit_name, pool, image, model_name=None):
     """Get SHA512 hash of RBD image.
 

zaza/openstack/utilities/openstack.py

@@ -1895,7 +1895,8 @@ def create_image(glance, image_url, image_name, image_cache_dir=None, tags=[]):
     return image
 
 
-def create_volume(cinder, size, name=None, image=None):
+def create_volume(cinder, size, name=None, image=None,
+                  wait_iteration_max_time=60):
     """Create cinder volume.
 
     :param cinder: Authenticated cinderclient
@@ -1906,6 +1907,9 @@ def create_volume(cinder, size, name=None, image=None):
     :type name: Option[str, None]
     :param image: Image to download to volume.
     :type image: Option[str, None]
+    :param wait_iteration_max_time: Wait a max of wait_iteration_max_time
+                                    between retries.
+    :type wait_iteration_max_time: int
     :returns: cinder volume pointer
     :rtype: cinderclient.common.utils.RequestIdProxy
     """
@@ -1919,12 +1923,14 @@ def create_volume(cinder, size, name=None, image=None):
     resource_reaches_status(
         cinder.volumes,
         volume.id,
+        wait_iteration_max_time=wait_iteration_max_time,
         expected_status='available',
         msg='Volume status wait')
     return volume
 
 
-def create_volume_backup(cinder, volume_id, name=None):
+def create_volume_backup(cinder, volume_id, name=None,
+                         wait_iteration_max_time=60):
     """Create cinder volume backup.
 
     :param cinder: Authenticated cinderclient
@@ -1933,6 +1939,9 @@ def create_volume_backup(cinder, volume_id, name=None):
     :type volume_id: str
     :param name: display name for new volume backup
     :type name: Option[str, None]
+    :param wait_iteration_max_time: Wait a max of wait_iteration_max_time
+                                    between retries.
+    :type wait_iteration_max_time: int
     :returns: cinder volume backup pointer
     :rtype: cinderclient.common.utils.RequestIdProxy
     """
@@ -1945,6 +1954,7 @@ def create_volume_backup(cinder, volume_id, name=None):
     resource_reaches_status(
         cinder.backups,
         volume_backup.id,
+        wait_iteration_max_time=wait_iteration_max_time,
         expected_status='available',
         msg='Volume status wait')
     return volume_backup