From fbdab856f2853e45a10522683fecca307f3ed59d Mon Sep 17 00:00:00 2001 From: Natalia Litvinova Date: Mon, 30 Sep 2019 11:17:45 +0300 Subject: [PATCH] Porting Cinder Backup tests to Zaza Porting the Amulet tests from Cinder Backup to the Zaza framework. The Amulet tests can be found here: https://opendev.org/openstack/charm-cinder-backup/src/commit/4273738b94b22cca187487af2bad982be49fdd69/tests/basic_deployment.py --- .../charm_tests/cinder_backup/__init__.py | 17 ++ .../charm_tests/cinder_backup/tests.py | 181 ++++++++++++++++++ zaza/openstack/utilities/ceph.py | 50 ++++- zaza/openstack/utilities/openstack.py | 14 +- 4 files changed, 259 insertions(+), 3 deletions(-) create mode 100644 zaza/openstack/charm_tests/cinder_backup/__init__.py create mode 100644 zaza/openstack/charm_tests/cinder_backup/tests.py diff --git a/zaza/openstack/charm_tests/cinder_backup/__init__.py b/zaza/openstack/charm_tests/cinder_backup/__init__.py new file mode 100644 index 000000000..6501e55bb --- /dev/null +++ b/zaza/openstack/charm_tests/cinder_backup/__init__.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 + +# Copyright 2019 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and testing cinder-backup.""" diff --git a/zaza/openstack/charm_tests/cinder_backup/tests.py b/zaza/openstack/charm_tests/cinder_backup/tests.py new file mode 100644 index 000000000..aea0183c9 --- /dev/null +++ b/zaza/openstack/charm_tests/cinder_backup/tests.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +# +# Copyright 2019 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulate cinder-backup testing.""" +import logging + +import zaza.model +import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.ceph as ceph_utils +import zaza.openstack.utilities.openstack as openstack_utils + + +class CinderBackupTest(test_utils.OpenStackBaseTest): + """Encapsulate Cinder Backup tests.""" + + RESOURCE_PREFIX = 'zaza-cinderbackuptests' + + @classmethod + def setUpClass(cls): + """Run class setup for running Cinder Backup tests.""" + super(CinderBackupTest, cls).setUpClass() + cls.cinder_client = openstack_utils.get_cinder_session_client( + cls.keystone_session) + + @property + def services(self): + """Return a list services for Openstack Release.""" + current_release = openstack_utils.get_os_release() + services = ['cinder-scheduler', 'cinder-volume'] + if (current_release >= + openstack_utils.get_os_release('xenial_ocata')): + services.append('apache2') + else: + services.append('cinder-api') + return services + + def test_100_volume_create_extend_delete(self): + """Test creating, extending a volume.""" + vol_new = openstack_utils.create_volume( + self.cinder_client, + name='{}-100-vol'.format(self.RESOURCE_PREFIX), + size=1) + self.cinder_client.volumes.extend( + vol_new.id, + '2') + openstack_utils.resource_reaches_status( + self.cinder_client.volumes, + vol_new.id, + expected_status="available", + msg="Volume status wait") + + def test_410_cinder_vol_create_backup_delete_restore_pool_inspect(self): + """Create, backup, delete, restore a ceph-backed cinder volume. + + Create, backup, delete, restore a ceph-backed cinder volume, and + inspect ceph cinder pool object count as the volume is created + and deleted. + """ + unit_name = zaza.model.get_lead_unit_name('ceph-mon') + obj_count_samples = [] + pool_size_samples = [] + pools = ceph_utils.get_ceph_pools(unit_name) + expected_pool = 'cinder-ceph' + cinder_ceph_pool = pools[expected_pool] + + # Check ceph cinder pool object count, disk space usage and pool name + logging.info('Checking ceph cinder pool original samples...') + pool_name, obj_count, kb_used = ceph_utils.get_ceph_pool_sample( + unit_name, cinder_ceph_pool) + + obj_count_samples.append(obj_count) + pool_size_samples.append(kb_used) + + self.assertEqual(pool_name, expected_pool) + + # Create ceph-backed cinder volume + cinder_vol = openstack_utils.create_volume( + self.cinder_client, + name='{}-410-vol'.format(self.RESOURCE_PREFIX), + size=1, + wait_iteration_max_time=180) + # Backup the volume + vol_backup = openstack_utils.create_volume_backup( + self.cinder_client, + cinder_vol.id, + name='{}-410-backup-vol'.format(self.RESOURCE_PREFIX), + wait_iteration_max_time=180) + # Delete the volume + openstack_utils.delete_volume(self.cinder_client, cinder_vol.id) + # Restore the volume + self.cinder_client.restores.restore(vol_backup.id) + openstack_utils.resource_reaches_status( + self.cinder_client.backups, + vol_backup.id, + wait_iteration_max_time=180, + stop_after_attempt=15, + expected_status='available', + msg='Backup status wait') + # Delete the backup + openstack_utils.delete_volume_backup( + self.cinder_client, + vol_backup.id) + openstack_utils.resource_removed( + self.cinder_client.volumes, + vol_backup.id, + wait_iteration_max_time=180, + msg="Backup volume") + + # Re-check ceph cinder pool object count and disk usage + logging.info('Checking ceph cinder pool samples ' + 'after volume create...') + pool_name, obj_count, kb_used = ceph_utils.get_ceph_pool_sample( + unit_name, cinder_ceph_pool, self.model_name) + + obj_count_samples.append(obj_count) + pool_size_samples.append(kb_used) + + name = '{}-410-vol'.format(self.RESOURCE_PREFIX) + vols = self.cinder_client.volumes.list() + try: + cinder_vols = [v for v in vols if v.name == name] + except AttributeError: + cinder_vols = [v for v in vols if v.display_name == name] + if not cinder_vols: + # NOTE(hopem): it appears that at some point cinder-backup stopped + # restoring volume metadata properly so revert to default name if + # original is not found + name = "restore_backup_{}".format(vol_backup.id) + try: + cinder_vols = [v for v in vols if v.name == name] + except AttributeError: + cinder_vols = [v for v in vols if v.display_name == name] + + self.assertTrue(cinder_vols) + + cinder_vol = cinder_vols[0] + + # Delete restored cinder volume + openstack_utils.delete_volume(self.cinder_client, cinder_vol.id) + openstack_utils.resource_removed( + self.cinder_client.volumes, + cinder_vol.id, + wait_iteration_max_time=180, + msg="Volume") + + # Final check, ceph cinder pool object count and disk usage + logging.info('Checking ceph cinder pool after volume delete...') + pool_name, obj_count, kb_used = ceph_utils.get_ceph_pool_sample( + unit_name, cinder_ceph_pool, self.model_name) + + obj_count_samples.append(obj_count) + pool_size_samples.append(kb_used) + + # Validate ceph cinder pool object count samples over time + original, created, deleted = range(3) + self.assertFalse(obj_count_samples[created] <= + obj_count_samples[original]) + self.assertFalse(obj_count_samples[deleted] >= + obj_count_samples[created]) + + # Luminous (pike) ceph seems more efficient at disk usage so we cannot + # grantee the ordering of kb_used + if (openstack_utils.get_os_release() < + openstack_utils.get_os_release('xenial_mitaka')): + self.assertFalse(pool_size_samples[created] <= + pool_size_samples[original]) + self.assertFalse(pool_size_samples[deleted] >= + pool_size_samples[created]) diff --git a/zaza/openstack/utilities/ceph.py b/zaza/openstack/utilities/ceph.py index 2c54512d8..61d7d6633 100644 --- a/zaza/openstack/utilities/ceph.py +++ b/zaza/openstack/utilities/ceph.py @@ -1,5 +1,5 @@ """Module containing Ceph related utilities.""" - +import json import logging import zaza.openstack.utilities.openstack as openstack_utils @@ -97,6 +97,54 @@ def get_ceph_pools(unit_name, model_name=None): return pools +def get_ceph_df(unit_name, model_name=None): + """Return dict of ceph df json output, including ceph pool state. + + :param unit_name: Name of the unit to get ceph df + :type unit_name: string + :param model_name: Name of model to operate in + :type model_name: str + :returns: Dict of ceph df output + :rtype: dict + :raise: zaza.model.CommandRunFailed + """ + cmd = 'sudo ceph df --format=json' + result = zaza_model.run_on_unit(unit_name, cmd, model_name=model_name) + if result.get('Code') != '0': + raise zaza_model.CommandRunFailed(cmd, result) + return json.loads(result.get('Stdout')) + + +def get_ceph_pool_sample(unit_name, pool_id=0, model_name=None): + """Return list of ceph pool attributes. + + Take a sample of attributes of a ceph pool, returning ceph + pool name, object count and disk space used for the specified + pool ID number. + + :param unit_name: Name of the unit to get the pool sample + :type unit_name: string + :param pool_id: Ceph pool ID + :type pool_id: int + :param model_name: Name of model to operate in + :type model_name: str + :returns: List of pool name, object count, kb disk space used + :rtype: list + :raises: zaza.model.CommandRunFailed + """ + df = get_ceph_df(unit_name, model_name) + for pool in df['pools']: + if pool['id'] == pool_id: + pool_name = pool['name'] + obj_count = pool['stats']['objects'] + kb_used = pool['stats']['kb_used'] + + logging.debug('Ceph {} pool (ID {}): {} objects, ' + '{} kb used'.format(pool_name, pool_id, + obj_count, kb_used)) + return pool_name, obj_count, kb_used + + def get_rbd_hash(unit_name, pool, image, model_name=None): """Get SHA512 hash of RBD image. diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 87b09347e..88c7e23a5 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -1895,7 +1895,8 @@ def create_image(glance, image_url, image_name, image_cache_dir=None, tags=[]): return image -def create_volume(cinder, size, name=None, image=None): +def create_volume(cinder, size, name=None, image=None, + wait_iteration_max_time=60): """Create cinder volume. :param cinder: Authenticated cinderclient @@ -1906,6 +1907,9 @@ def create_volume(cinder, size, name=None, image=None): :type name: Option[str, None] :param image: Image to download to volume. :type image: Option[str, None] + :param wait_iteration_max_time: Wait a max of wait_iteration_max_time + between retries. + :type wait_iteration_max_time: int :returns: cinder volume pointer :rtype: cinderclient.common.utils.RequestIdProxy """ @@ -1919,12 +1923,14 @@ def create_volume(cinder, size, name=None, image=None): resource_reaches_status( cinder.volumes, volume.id, + wait_iteration_max_time=wait_iteration_max_time, expected_status='available', msg='Volume status wait') return volume -def create_volume_backup(cinder, volume_id, name=None): +def create_volume_backup(cinder, volume_id, name=None, + wait_iteration_max_time=60): """Create cinder volume backup. :param cinder: Authenticated cinderclient @@ -1933,6 +1939,9 @@ def create_volume_backup(cinder, volume_id, name=None): :type volume_id: str :param name: display name for new volume backup :type name: Option[str, None] + :param wait_iteration_max_time: Wait a max of wait_iteration_max_time + between retries. + :type wait_iteration_max_time: int :returns: cinder volume backup pointer :rtype: cinderclient.common.utils.RequestIdProxy """ @@ -1945,6 +1954,7 @@ def create_volume_backup(cinder, volume_id, name=None): resource_reaches_status( cinder.backups, volume_backup.id, + wait_iteration_max_time=wait_iteration_max_time, expected_status='available', msg='Volume status wait') return volume_backup