Add test_services functional test

* Add functional test
* Add helper functions
* Add unit tests

unit_tests/utilities/test_zaza_utilities_ceph.py

@@ -0,0 +1,17 @@
+# import mock
+import unit_tests.utils as ut_utils
+import zaza.utilities.ceph as ceph_utils
+
+
+class TestCephUtils(ut_utils.BaseTestCase):
+
+    def setUp(self):
+        super(TestCephUtils, self).setUp()
+
+    def test_get_ceph_osd_id_cmd(self):
+        osd_id = 1
+        expected = ("`initctl list | grep 'ceph-osd ' | "
+                    "awk 'NR=={} {{ print $2 }}' | "
+                    "grep -o '[0-9]*'`".format(osd_id + 1))
+        actual = ceph_utils.get_ceph_osd_id_cmd(osd_id)
+        self.assertEqual(expected, actual)

unit_tests/utilities/test_zaza_utilities_generic.py

@@ -263,7 +263,7 @@ def test_validate_unit_process_ids(self):
             },
             "unit/0": {
                 "pr1": ["1", "2"],
-                # 3 PID insted of [1, 2] expected
+                # 3 PID instead of [1, 2] expected
                 "pr2": ["1", "2", "3"]
             }
         }
@@ -282,3 +282,88 @@ def test_validate_unit_process_ids(self):
         }
         ret = generic_utils.validate_unit_process_ids(expected, actual)
         self.assertTrue(ret)
+
+    def test_get_ubuntu_release(self):
+        # Normal case
+        expected = 0
+        actual = generic_utils.get_ubuntu_release('oneiric')
+        self.assertEqual(expected, actual)
+
+        # Ubuntu release doesn't exist
+        bad_name = 'bad_name'
+        with self.assertRaises(zaza_exceptions.UbuntuReleaseNotFound):
+            generic_utils.get_ubuntu_release(bad_name)
+
+    def test_validate_services_by_name(self):
+        self.patch(
+            'zaza.utilities.juju.get_machine_series',
+            new_callable=mock.MagicMock,
+            name='_get_machine_series'
+        )
+        self.patch(
+            'zaza.model.run_on_unit',
+            new_callable=mock.MagicMock,
+            name='_run_on_unit'
+        )
+        self._get_machine_series.return_value = 'xenial'
+
+        # Systemd service not found
+        unit_services = {
+            'unit/0': [
+                'broken-service'
+            ]
+        }
+        cmd_returns = {
+            'Code': '1',
+            'Stdout': '',
+            'Stderr': 'broken-service: unrecognized service'
+        }
+        self._run_on_unit.return_value = cmd_returns
+        with self.assertRaises(zaza_exceptions.ServiceNotFound):
+            generic_utils.validate_services_by_name(unit_services)
+
+        # Valid systemd service
+        unit_services = {
+            'unit/0': [
+                'valid-service'
+            ]
+        }
+        cmd_returns = {
+            'Code': '0',
+            'Stdout': 'valid-service running',
+            'Stderr': ''
+        }
+        self._run_on_unit.return_value = cmd_returns
+        actual = generic_utils.validate_services_by_name(unit_services)
+        self.assertTrue(actual)
+
+        self._get_machine_series.return_value = 'utopic'
+        # Upstart service not found
+        unit_services = {
+            'unit/0': [
+                'broken-service'
+            ]
+        }
+        cmd_returns = {
+            'Code': '1',
+            'Stdout': '',
+            'Stderr': 'broken-service: unrecognized service'
+        }
+        self._run_on_unit.return_value = cmd_returns
+        with self.assertRaises(zaza_exceptions.ServiceNotFound):
+            generic_utils.validate_services_by_name(unit_services)
+
+        # Valid upstart service
+        unit_services = {
+            'unit/0': [
+                'valid-service'
+            ]
+        }
+        cmd_returns = {
+            'Code': '0',
+            'Stdout': 'valid-service start/running',
+            'Stderr': ''
+        }
+        self._run_on_unit.return_value = cmd_returns
+        actual = generic_utils.validate_services_by_name(unit_services)
+        self.assertTrue(actual)

zaza/charm_tests/ceph/tests.py

@@ -2,6 +2,7 @@
 
 import zaza.charm_tests.test_utils as test_utils
 import zaza.utilities.generic as zaza_utils
+import zaza.utilities.openstack as zaza_openstack
 
 
 class CephOsdLowLevelTest(test_utils.OpenStackBaseTest):
@@ -41,3 +42,48 @@ def test_ceph_processes(self):
         ret = zaza_utils.validate_unit_process_ids(expected_processes,
                                                    actual_pids)
         self.assertTrue(ret)
+
+    def test_services(self):
+        """Verify the ceph services.
+
+        Verify the expected services are running on the service units.
+        """
+        services = {
+            "glance/0": ['glance-registry',
+                         'glance-api'],
+            "cinder/0": ['cinder-scheduler',
+                         'cinder-volume'],
+        }
+
+        current_release = zaza_openstack.get_os_release()
+        xenial_ocata = zaza_openstack.get_os_release('xenial_ocata')
+
+        if current_release < xenial_ocata:
+            services["cinder/0"].append('cinder-api')
+        else:
+            services["cinder/0"].append('apache2')
+
+        xenial_mitaka = zaza_openstack.get_os_release('xenial_mitaka')
+
+        if current_release < xenial_mitaka:
+            # For upstart systems only.  Ceph services under systemd
+            # are checked by process name instead.
+            ceph_services = [
+                'ceph-mon-all',
+                'ceph-mon id=`hostname`',
+            ]
+            services["ceph-mon/0"] = ceph_services
+            services["ceph-mon/1"] = ceph_services
+            services["ceph-mon/2"] = ceph_services
+            services["ceph-osd/0"] = [
+                'ceph-osd-all',
+                'ceph-osd id={}'.format(zaza_utils.get_ceph_osd_id_cmd(0)),
+                'ceph-osd id={}'.format(zaza_utils.get_ceph_osd_id_cmd(1))
+            ]
+
+        trusty_liberty = zaza_openstack.get_os_release('trusty_liberty')
+        if current_release >= trusty_liberty:
+            services["keystone/0"] = ['apache2']
+
+        ret = zaza_utils.validate_services_by_name(services)
+        self.assertTrue(ret)

zaza/utilities/ceph.py

@@ -0,0 +1,13 @@
+"""Module containing Ceph related utilities."""
+
+
+def get_ceph_osd_id_cmd(osd_id):
+    """Get ceph OSD command.
+
+    Produce a shell command that will return a ceph-osd id.
+    :returns: Command for ceph OSD.
+    :rtype: string
+    """
+    return ("`initctl list | grep 'ceph-osd ' | "
+            "awk 'NR=={} {{ print $2 }}' | "
+            "grep -o '[0-9]*'`".format(osd_id + 1))

zaza/utilities/exceptions.py

@@ -98,3 +98,15 @@ class UnitCountMismatch(Exception):
     """Count of unit doesn't match."""
 
     pass
+
+
+class UbuntuReleaseNotFound(Exception):
+    """Ubuntu release not found in list."""
+
+    pass
+
+
+class ServiceNotFound(Exception):
+    """Service not found on unit."""
+
+    pass

zaza/utilities/generic.py

@@ -7,6 +7,7 @@
 from zaza import model
 from zaza.utilities import juju as juju_utils
 from zaza.utilities import exceptions as zaza_exceptions
+from zaza.utilities.os_versions import UBUNTU_OPENSTACK_RELEASE
 
 
 def dict_to_yaml(dict_data):
@@ -279,3 +280,81 @@ def validate_unit_process_ids(expected, actual):
                               '{}'.format(e_unit_name, e_proc_name,
                                           e_pids, a_pids))
     return True
+
+
+def get_ubuntu_release(ubuntu_name):
+    """Get index of Ubuntu release.
+
+    Returns the index of the name of the Ubuntu release in
+        UBUNTU_OPENSTACK_RELEASE.
+
+    :param ubuntu_name: Name of the Ubuntu release.
+    :type ubuntu_name: string
+    :returns: Index of the Ubuntu release
+    :rtype: integer
+    :raises: zaza_exceptions.UbuntuReleaseNotFound
+    """
+    ubuntu_releases = list(UBUNTU_OPENSTACK_RELEASE.keys())
+    try:
+        index = ubuntu_releases.index(ubuntu_name)
+    except ValueError:
+        msg = ('Could not find Ubuntu release {} in {}'.
+               format(ubuntu_name, UBUNTU_OPENSTACK_RELEASE))
+        raise zaza_exceptions.UbuntuReleaseNotFound(msg)
+    return index
+
+
+def validate_services_by_name(unit_services):
+    """Validate system services.
+
+    Validate system service status by service name, automatically
+        detecting init system based on Ubuntu release codename.
+
+    :param unit_services: dict with unit keys and svc list values
+    :returns: True if successful, exceptions with string message otherwise
+    """
+    logging.debug('Checking status of system services...')
+
+    # Point at which systemd became a thing
+    systemd_switch = get_ubuntu_release('vivid')
+
+    for unit_name, services_list in unit_services.items():
+        # Get lsb_release codename from unit
+        release = juju_utils.get_machine_series(unit_name)
+
+        for service_name in services_list:
+            if (get_ubuntu_release(release) >= systemd_switch or
+                    service_name in ['rabbitmq-server', 'apache2',
+                                     'memcached']):
+                # init is systemd (or regular sysv)
+                cmd = 'sudo service {} status'.format(service_name)
+                results = model.run_on_unit(unit_name=unit_name, command=cmd)
+                code = results.get('Code')
+                try:
+                    code = int(code)
+                except ValueError:
+                    code = 1
+                output = results.get('Stdout')
+                error = results.get('Stderr')
+                service_running = code == 0
+
+            elif get_ubuntu_release(release) < systemd_switch:
+                # init is upstart
+                cmd = 'sudo status {}'.format(service_name)
+                results = model.run_on_unit(unit_name=unit_name, command=cmd)
+                code = results.get('Code')
+                try:
+                    code = int(code)
+                except ValueError:
+                    code = 1
+                output = results.get('Stdout')
+                error = results.get('Stderr')
+                service_running = code == 0 and "start/running" in output
+
+            logging.debug('{} `{}` returned '
+                          '{}'.format(unit_name, cmd, code))
+            if not service_running:
+                msg = ("command `{}` returned {} {} with error {}".
+                       format(cmd, output, str(code), error))
+                raise zaza_exceptions.ServiceNotFound(msg)
+    return True