Skip to content

Commit

Permalink
mgr/test_orchestrator: refactor listing services/daemons
Browse files Browse the repository at this point in the history 
- Refactor list_daemons: return ceph-xxx processes on host.
- Refactor describe_service: return services by grouping ceph-xxx
  processes on host.
- Add dummy data.

Signed-off-by: Kiefer Chang <kiefer.chang@suse.com>
  • Loading branch information
@bk201
bk201 committed Feb 27, 2020
1 parent b6fb2ae commit a56eb3f
Show file tree
Hide file tree
Showing 2 changed files with 185 additions and 23 deletions.
115 changes: 114 additions & 1 deletion src/pybind/mgr/test_orchestrator/dummy_data.json
View file
Expand Up @@ -145,11 +145,124 @@
]
}
],
"services": [
{
"container_image_id": "d457abd6b7ef89251b9ba7b1ae2edbebffd3bf0e0b9069b59d48c7f38b11c944",
"container_image_name": "docker.io/ceph/daemon-base:latest-master-devel",
"service_name": "crash",
"size": 1,
"running": 0,
"last_refresh": "2020-02-26T07:23:56.501157"
},
{
"container_image_id": "d457abd6b7ef89251b9ba7b1ae2edbebffd3bf0e0b9069b59d48c7f38b11c944",
"container_image_name": "docker.io/ceph/daemon-base:latest-master-devel",
"service_name": "mgr",
"size": 1,
"running": 1,
"last_refresh": "2020-02-26T07:23:56.501087"
},
{
"container_image_id": "d457abd6b7ef89251b9ba7b1ae2edbebffd3bf0e0b9069b59d48c7f38b11c944",
"container_image_name": "docker.io/ceph/daemon-base:latest-master-devel",
"service_name": "mon",
"size": 1,
"running": 1,
"last_refresh": "2020-02-26T07:23:56.501013"
},
{
"container_image_id": "d457abd6b7ef89251b9ba7b1ae2edbebffd3bf0e0b9069b59d48c7f38b11c944",
"container_image_name": "docker.io/ceph/daemon-base:latest-master-devel",
"service_name": "osd",
"size": 4,
"running": 4,
"last_refresh": "2020-02-26T07:23:56.500960"
}
],
"daemons": [
{
"hostname": "host0",
"container_id": "a8ad2d6ceb5d5b8a308207ea58963b9295d52d3c282a054534b1ed52e5e77d31",
"container_image_id": "d457abd6b7ef89251b9ba7b1ae2edbebffd3bf0e0b9069b59d48c7f38b11c944",
"container_image_name": "docker.io/ceph/daemon-base:latest-master-devel",
"daemon_id": "host0",
"daemon_type": "crash",
"version": "15.1.0-1240-ge5841ce",
"status": -1,
"status_desc": "unknown",
"last_refresh": "2020-02-26T07:23:56.501157"
},
{
"hostname": "host0",
"container_id": "10f182e93c9c9110b7223e7b8e0337445fded77a317c649fc54b188f22c6e4cb",
"container_image_id": "d457abd6b7ef89251b9ba7b1ae2edbebffd3bf0e0b9069b59d48c7f38b11c944",
"container_image_name": "docker.io/ceph/daemon-base:latest-master-devel",
"daemon_id": "host0.kjapjg",
"daemon_type": "mgr",
"version": "15.1.0-1240-ge5841ce",
"status": 1,
"status_desc": "running",
"last_refresh": "2020-02-26T07:23:56.501087"
},
{
"hostname": "host0",
"container_id": "27f178927ca4dbbd8d73380662f15828f780604f97baae4cac4d3f984e2c32af",
"container_image_id": "d457abd6b7ef89251b9ba7b1ae2edbebffd3bf0e0b9069b59d48c7f38b11c944",
"container_image_name": "docker.io/ceph/daemon-base:latest-master-devel",
"daemon_id": "host0",
"daemon_type": "mon",
"version": "15.1.0-1240-ge5841ce",
"status": 1,
"status_desc": "running",
"last_refresh": "2020-02-26T07:23:56.501013"
},
{
"hostname": "host0",
"container_id": "71803dddfb9f736c26cc2127e684c66486034292ea94bc560d00132c29fe8b16",
"container_image_id": "d457abd6b7ef89251b9ba7b1ae2edbebffd3bf0e0b9069b59d48c7f38b11c944",
"container_image_name": "docker.io/ceph/daemon-base:latest-master-devel",
"daemon_id": "0",
"daemon_type": "osd",
"version": "15.1.0-1240-ge5841ce",
"status": 1,
"status_desc": "running",
"last_refresh": "2020-02-26T07:23:56.501050"
},
{
"hostname": "host0",
"container_id": "3ca816209df390e8de3cdff78d95f69efd664b4017036b4d7c47563731e1fe37",
"container_image_id": "d457abd6b7ef89251b9ba7b1ae2edbebffd3bf0e0b9069b59d48c7f38b11c944",
"container_image_name": "docker.io/ceph/daemon-base:latest-master-devel",
"daemon_id": "1",
"daemon_type": "osd",
"version": "15.1.0-1240-ge5841ce",
"status": 1,
"status_desc": "running",
"last_refresh": "2020-02-26T07:23:56.500960"
},
{
"hostname": "host0",
"container_id": "fcbcc88f023f3734f9015fcdae09277ed4c5e6b0ed1590275e4538d897588acb",
"container_image_id": "d457abd6b7ef89251b9ba7b1ae2edbebffd3bf0e0b9069b59d48c7f38b11c944",
"container_image_name": "docker.io/ceph/daemon-base:latest-master-devel",
"daemon_id": "2",
"daemon_type": "osd",
"version": "15.1.0-1240-ge5841ce",
"status": 1,
"status_desc": "running",
"last_refresh": "2020-02-26T07:23:56.501123"
},
{
"hostname": "host0",
"container_id": "55f3894fffa52854e22354c7629cc4c58db46903ac0d977988b81ed5ba4ad759",
"container_image_id": "d457abd6b7ef89251b9ba7b1ae2edbebffd3bf0e0b9069b59d48c7f38b11c944",
"container_image_name": "docker.io/ceph/daemon-base:latest-master-devel",
"daemon_id": "3",
"daemon_type": "osd",
"daemon_id": "1"
"version": "15.1.0-1240-ge5841ce",
"status": 1,
"status_desc": "running",
"last_refresh": "2020-02-26T07:23:56.501193"
}
]
}
93 changes: 71 additions & 22 deletions src/pybind/mgr/test_orchestrator/module.py
View file
Expand Up @@ -4,6 +4,7 @@
import os
import threading
import functools
import itertools
from subprocess import check_output, CalledProcessError
try:
from typing import Callable, List, Tuple
Expand Down Expand Up @@ -117,8 +118,10 @@ def serve(self):
def _init_data(self, data=None):
self._inventory = [orchestrator.InventoryHost.from_json(inventory_host)
for inventory_host in data.get('inventory', [])]
self._daemons = [orchestrator.DaemonDescription.from_json(service)
for service in data.get('daemons', [])]
self._services = [orchestrator.ServiceDescription.from_json(service)
for service in data.get('services', [])]
self._daemons = [orchestrator.DaemonDescription.from_json(daemon)
for daemon in data.get('daemons', [])]

@deferred_read
def get_inventory(self, host_filter=None, refresh=False):
Expand Down Expand Up @@ -153,36 +156,82 @@ def get_inventory(self, host_filter=None, refresh=False):
self.log.error('c-v failed: ' + str(c_v_out))
raise Exception('c-v failed')

def _get_ceph_daemons(self):
# type: () -> List[orchestrator.DaemonDescription]
""" Return ceph daemons on the running host."""
types = ("mds", "osd", "mon", "rgw", "mgr")
out = map(str, check_output(['ps', 'aux']).splitlines())
processes = [p for p in out if any(
[('ceph-' + t in p) for t in types])]

daemons = []
for p in processes:
daemon = orchestrator.DaemonDescription()
# parse daemon type
m = re.search('ceph-([^ ]+)', p)
if m:
_daemon_type = m.group(1)
else:
raise AssertionError('Fail to determine daemon type from {}'.format(p))

# parse daemon ID. Possible options: `-i <id>`, `--id=<id>`, `--id <id>`
patterns = ['-i\s(\w+)', '--id[\s=](\w+)']
daemon_id = None
for pattern in patterns:
m = re.search(pattern, p)
if m:
daemon_id = m.group(1)
break
else:
raise AssertionError('Fail to determine daemon ID from {}'.format(p))
daemon = orchestrator.DaemonDescription(
daemon_type=_daemon_type, daemon_id=daemon_id, hostname='localhost')
daemons.append(daemon)
return daemons

@deferred_read
def list_daemons(self, daemon_type=None, daemon_id=None, host_name=None, refresh=False):
def describe_service(self, service_type=None, service_name=None, refresh=False):
if self._services:
# Dummy data
services = self._services
else:
# Deduce services from daemons running on localhost
all_daemons = self._get_ceph_daemons()
services = []
for daemon_type, daemons in itertools.groupby(all_daemons, key=lambda d: d.daemon_type):
daemon_size = len(list(daemons))
services.append(orchestrator.ServiceDescription(
service_name=daemon_type, size=daemon_size, running=daemon_size))

def _filter_func(svc):
if service_type is not None and service_type != svc.service_name:
return False
return True

return list(filter(_filter_func, services))

@deferred_read
def list_daemons(self, daemon_type=None, daemon_id=None, host=None, refresh=False):
"""
There is no guarantee which daemons are returned by describe_service, except that
it returns the mgr we're running in.
"""
if daemon_type:
daemon_types = ("mds", "osd", "mon", "rgw", "mgr", "iscsi")
daemon_types = ("mds", "osd", "mon", "rgw", "mgr", "iscsi", "crash")
assert daemon_type in daemon_types, daemon_type + " unsupported"

if self._daemons:
if host_name:
return list(filter(lambda svc: svc.hostname == host_name, self._daemons))
return self._daemons
daemons = self._daemons if self._daemons else self._get_ceph_daemons()

out = map(str, check_output(['ps', 'aux']).splitlines())
types = (daemon_type, ) if daemon_type else ("mds", "osd", "mon", "rgw", "mgr")
assert isinstance(types, tuple)
processes = [p for p in out if any([('ceph-' + t in p) for t in types])]
def _filter_func(d):
if daemon_type is not None and daemon_type != d.daemon_type:
return False
if daemon_id is not None and daemon_id != d.daemon_id:
return False
if host is not None and host != d.hostname:
return False
return True

result = []
for p in processes:
sd = orchestrator.DaemonDescription()
sd.hostname = 'localhost'
res = re.search('ceph-[^ ]+', p)
assert res
sd.daemon_id = res.group()
result.append(sd)

return result
return list(filter(_filter_func, daemons))

def create_osds(self, drive_groups):
# type: (List[DriveGroupSpec]) -> TestCompletion
Expand Down

0 comments on commit a56eb3f

Please sign in to comment.