Migrate 499 cinder-ceph test to zaza

zaza/openstack/charm_tests/ceph/mon/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2018 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Collection of code for setting up and testing ceph-mon for cinder-ceph."""

zaza/openstack/charm_tests/ceph/mon/tests.py

@@ -0,0 +1,200 @@
+# Copyright 2020 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Ceph-mon Testing for cinder-ceph."""
+
+import logging
+
+import zaza.model
+
+from zaza.openstack.utilities import (
+    generic as generic_utils,
+    openstack as openstack_utils,
+    exceptions as zaza_exceptions
+)
+import zaza.openstack.charm_tests.test_utils as test_utils
+
+
+class CinderCephMonTest(test_utils.OpenStackBaseTest):
+    """Verify that the ceph mon units are healthy."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Run class setup for running ceph mon tests with cinder."""
+        super().setUpClass()
+
+    # ported from the cinder-ceph Amulet test
+    def test_499_ceph_cmds_exit_zero(self):
+        """Verify expected state with security-checklist."""
+        logging.info("Checking exit values are 0 on ceph commands.")
+
+        units = zaza.model.get_units("ceph-mon", model_name=self.model_name)
+        current_release = openstack_utils.get_os_release()
+        bionic_train = openstack_utils.get_os_release('bionic_train')
+        if current_release < bionic_train:
+            units.extend(zaza.model.get_units("cinder-ceph",
+                                              model_name=self.model_name))
+
+        commands = [
+            'sudo ceph health',
+            'sudo ceph mds stat',
+            'sudo ceph pg stat',
+            'sudo ceph osd stat',
+            'sudo ceph mon stat',
+        ]
+
+        for unit in units:
+            run_commands(unit.name, commands)
+
+    # ported from the cinder-ceph Amulet test
+    def test_500_ceph_alternatives_cleanup(self):
+        """Check ceph alternatives removed when ceph-mon relation is broken."""
+        # Skip this test if release is less than xenial_ocata as in that case
+        # cinder HAS a relation with ceph directly and this test would fail
+        current_release = openstack_utils.get_os_release()
+        xenial_ocata = openstack_utils.get_os_release('xenial_ocata')
+        if current_release < xenial_ocata:
+            logging.info("Skipping test as release < xenial-ocata")
+            return
+
+        units = zaza.model.get_units("cinder-ceph",
+                                     model_name=self.model_name)
+
+        # check each unit prior to breaking relation
+        for unit in units:
+            dir_list = directory_listing(unit.name, "/etc/ceph")
+            if 'ceph.conf' in dir_list:
+                logging.debug(
+                    "/etc/ceph/ceph.conf exists BEFORE relation-broken")
+            else:
+                raise zaza_exceptions.CephGenericError(
+                    "unit: {} - /etc/ceph/ceph.conf does not exist "
+                    "BEFORE relation-broken".format(unit.name))
+
+        # remove the relation so that /etc/ceph/ceph.conf is removed
+        logging.info("Removing ceph-mon:client <-> cinder-ceph:ceph relation")
+        zaza.model.remove_relation(
+            "ceph-mon", "ceph-mon:client", "cinder-ceph:ceph")
+        # zaza.model.wait_for_agent_status()
+        logging.info("Wait till relation is removed...")
+        ceph_mon_units = zaza.model.get_units("ceph-mon",
+                                              model_name=self.model_name)
+        conditions = [
+            invert_condition(
+                does_relation_exist(
+                    u.name, "ceph-mon", "cinder-ceph", "ceph",
+                    self.model_name))
+            for u in ceph_mon_units]
+        zaza.model.block_until(*conditions)
+
+        logging.info("Checking each unit after breaking relation...")
+        for unit in units:
+            dir_list = directory_listing(unit.name, "/etc/ceph")
+            if 'ceph.conf' not in dir_list:
+                logging.debug(
+                    "/etc/ceph/ceph.conf removed AFTER relation-broken")
+            else:
+                raise zaza_exceptions.CephGenericError(
+                    "unit: {} - /etc/ceph/ceph.conf still exists "
+                    "AFTER relation-broken".format(unit.name))
+
+        # Restore cinder-ceph and ceph-mon relation to keep tests idempotent
+        logging.info("Restoring ceph-mon:client <-> cinder-ceph:ceph relation")
+        zaza.model.add_relation(
+            "ceph-mon", "ceph-mon:client", "cinder-ceph:ceph")
+        conditions = [
+            does_relation_exist(
+                u.name, "ceph-mon", "cinder-ceph", "ceph", self.model_name)
+            for u in ceph_mon_units]
+        logging.info("Wait till model is idle ...")
+        zaza.model.block_until(*conditions)
+        zaza.model.block_until_all_units_idle()
+        logging.info("... Done.")
+
+
+def does_relation_exist(unit_name,
+                        application_name,
+                        remote_application_name,
+                        remote_interface_name,
+                        model_name):
+    """For use in async blocking function, return True if it exists.
+
+    :param unit_name: the unit (by name) that to check on.
+    :type unit_name: str
+    :param application_name: Name of application on this side of relation
+    :type application_name: str
+    :param remote_application_name: the relation name at that unit to check for
+    :type relation_application_name: str
+    :param remote_interface_name: the interface name at that unit to check for
+    :type relation_interface_name: str
+    :param model_name: the model to check on
+    :type model_name: str
+    :returns: Corouting that returns True if the relation was found
+    :rtype: Coroutine[[], boolean]
+    """
+    async def _async_does_relation_exist_closure():
+        async with zaza.model.run_in_model(model_name) as model:
+            spec = "{}:{}".format(
+                remote_application_name, remote_interface_name)
+            for rel in model.applications[application_name].relations:
+                if rel.matches(spec):
+                    return True
+            return False
+    return _async_does_relation_exist_closure
+
+
+def invert_condition(async_condition):
+    """Invert the condition provided so it can be provided to the blocking fn.
+
+    :param async_condition: the async callable that is the test
+    :type async_condition: Callable[]
+    :returns: Corouting that returns not of the result of a the callable
+    :rtype: Coroutine[[], bool]
+    """
+    async def _async_invert_condition_closure():
+        return not(await async_condition())
+    return _async_invert_condition_closure
+
+
+def run_commands(unit_name, commands):
+    """Run commands on unit.
+
+    Apply context to commands until all variables have been replaced, then
+    run the command on the given unit.
+    """
+    errors = []
+    for cmd in commands:
+        try:
+            generic_utils.assertRemoteRunOK(zaza.model.run_on_unit(
+                unit_name,
+                cmd))
+        except Exception as e:
+            errors.append("unit: {}, command: {}, error: {}"
+                          .format(unit_name, cmd, str(e)))
+    if errors:
+        raise zaza_exceptions.CephGenericError("\n".join(errors))
+
+
+def directory_listing(unit_name, directory):
+    """Return a list of files/directories from a directory on a unit.
+
+    :param unit_name: the unit to fetch the directory listing from
+    :type unit_name: str
+    :param directory: the directory to fetch the listing from
+    :type directory: str
+    :returns: A listing using "ls -1" on the unit
+    :rtype: List[str]
+    """
+    result = zaza.model.run_on_unit(unit_name, "ls -1 {}".format(directory))
+    return result['Stdout'].splitlines()

zaza/openstack/charm_tests/cinder/tests.py

@@ -23,6 +23,12 @@
 import zaza.openstack.utilities.openstack as openstack_utils
 import zaza.openstack.charm_tests.glance.setup as glance_setup
 
+from tenacity import (
+    Retrying,
+    stop_after_attempt,
+    wait_exponential,
+)
+
 
 class CinderTests(test_utils.OpenStackBaseTest):
     """Encapsulate Cinder tests."""
@@ -32,7 +38,10 @@ class CinderTests(test_utils.OpenStackBaseTest):
     @classmethod
     def setUpClass(cls):
         """Run class setup for running tests."""
-        super(CinderTests, cls).setUpClass()
+        super(CinderTests, cls).setUpClass(application_name='cinder')
+        cls.application_name = 'cinder'
+        cls.lead_unit = zaza.model.get_lead_unit_name(
+            "cinder", model_name=cls.model_name)
         cls.cinder_client = openstack_utils.get_cinder_session_client(
             cls.keystone_session)
         cls.nova_client = openstack_utils.get_nova_session_client(
@@ -42,18 +51,66 @@ def setUpClass(cls):
     def tearDown(cls):
         """Remove test resources."""
         logging.info('Running teardown')
-        for snapshot in cls.cinder_client.volume_snapshots.list():
+        for attempt in Retrying(
+                stop=stop_after_attempt(8),
+                wait=wait_exponential(multiplier=1, min=2, max=60)):
+            with attempt:
+                volumes = list(cls.cinder_client.volumes.list())
+                snapped_volumes = [v for v in volumes
+                                   if v.name.endswith("-from-snap")]
+                if snapped_volumes:
+                    logging.info("Removing volumes from snapshot")
+                    cls._remove_volumes(snapped_volumes)
+                    volumes = list(cls.cinder_client.volumes.list())
+
+                snapshots = list(cls.cinder_client.volume_snapshots.list())
+                if snapshots:
+                    logging.info("tearDown - snapshots: {}".format(
+                        ", ".join(s.name for s in snapshots)))
+                    cls._remove_snapshots(snapshots)
+
+                if volumes:
+                    logging.info("tearDown - volumes: {}".format(
+                        ", ".join(v.name for v in volumes)))
+                    cls._remove_volumes(volumes)
+
+    @classmethod
+    def _remove_snapshots(cls, snapshots):
+        """Remove snapshots passed as param.
+
+        :param volumes: the snapshots to delete
+        :type volumes: List[snapshot objects]
+        """
+        for snapshot in snapshots:
             if snapshot.name.startswith(cls.RESOURCE_PREFIX):
-                openstack_utils.delete_resource(
-                    cls.cinder_client.volume_snapshots,
-                    snapshot.id,
-                    msg="snapshot")
-        for volume in cls.cinder_client.volumes.list():
+                logging.info("removing snapshot: {}".format(snapshot.name))
+                try:
+                    openstack_utils.delete_resource(
+                        cls.cinder_client.volume_snapshots,
+                        snapshot.id,
+                        msg="snapshot")
+                except Exception as e:
+                    logging.error("error removing snapshot: {}".format(str(e)))
+                    raise
+
+    @classmethod
+    def _remove_volumes(cls, volumes):
+        """Remove volumes passed as param.
+
+        :param volumes: the volumes to delete
+        :type volumes: List[volume objects]
+        """
+        for volume in volumes:
             if volume.name.startswith(cls.RESOURCE_PREFIX):
-                openstack_utils.delete_resource(
-                    cls.cinder_client.volumes,
-                    volume.id,
-                    msg="volume")
+                logging.info("removing volume: {}".format(volume.name))
+                try:
+                    openstack_utils.delete_resource(
+                        cls.cinder_client.volumes,
+                        volume.id,
+                        msg="volume")
+                except Exception as e:
+                    logging.error("error removing volume: {}".format(str(e)))
+                    raise
 
     def test_100_volume_create_extend_delete(self):
         """Test creating, extending a volume."""
@@ -80,12 +137,18 @@ def test_100_volume_create_extend_delete(self):
 
     def test_105_volume_create_from_img(self):
         """Test creating a volume from an image."""
+        logging.debug("finding image {} ..."
+                      .format(glance_setup.LTS_IMAGE_NAME))
         image = self.nova_client.glance.find_image(
             glance_setup.LTS_IMAGE_NAME)
+        logging.debug("using cinder_client to create volume from image {}"
+                      .format(image.id))
         vol_img = self.cinder_client.volumes.create(
             name='{}-105-vol-from-img'.format(self.RESOURCE_PREFIX),
             size=3,
             imageRef=image.id)
+        logging.debug("now waiting for volume {} to reach available"
+                      .format(vol_img.id))
         openstack_utils.resource_reaches_status(
             self.cinder_client.volumes,
             vol_img.id,

zaza/openstack/utilities/exceptions.py

@@ -168,6 +168,12 @@ class CephPoolNotConfigured(Exception):
     pass
 
 
+class CephGenericError(Exception):
+    """A generic/other Ceph error occurred."""
+
+    pass
+
+
 class NovaGuestMigrationFailed(Exception):
     """Nova guest migration failed."""