From 24e5b91e87e61847a89d140fe495c9a4c928f012 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Fri, 23 Feb 2018 09:41:27 +0100 Subject: [PATCH] mgr/dashboard_v2: Refactor `send_command` into single method mgr/dashboard_v2: Add create() to Pool controller mgr/dashboard_v2: Add minimalistic HTML debug representation of the API * Add API test * Fixed @RESTController.args_from_json Signed-off-by: Sebastian Wagner --- qa/tasks/mgr/dashboard_v2/helper.py | 3 +- qa/tasks/mgr/dashboard_v2/test_pool.py | 32 ++++++ .../mgr/dashboard_v2/controllers/cephfs.py | 16 +-- .../mgr/dashboard_v2/controllers/dashboard.py | 25 +---- .../mgr/dashboard_v2/controllers/pool.py | 39 +++++++ .../mgr/dashboard_v2/services/ceph_service.py | 43 ++++++- .../mgr/dashboard_v2/tests/test_tools.py | 32 +++++- src/pybind/mgr/dashboard_v2/tools.py | 106 +++++++++++++++--- 8 files changed, 244 insertions(+), 52 deletions(-) diff --git a/qa/tasks/mgr/dashboard_v2/helper.py b/qa/tasks/mgr/dashboard_v2/helper.py index f43970f110c4fb..5d93c15a1abce6 100644 --- a/qa/tasks/mgr/dashboard_v2/helper.py +++ b/qa/tasks/mgr/dashboard_v2/helper.py @@ -85,7 +85,8 @@ def _request(self, url, method, data=None): self._resp = self._session.delete(url, json=data) elif method == 'PUT': self._resp = self._session.put(url, json=data) - return None + else: + assert False def _get(self, url): return self._request(url, 'GET') diff --git a/qa/tasks/mgr/dashboard_v2/test_pool.py b/qa/tasks/mgr/dashboard_v2/test_pool.py index 6852ddbb631a6d..6543de63221cc4 100644 --- a/qa/tasks/mgr/dashboard_v2/test_pool.py +++ b/qa/tasks/mgr/dashboard_v2/test_pool.py @@ -5,6 +5,13 @@ class DashboardTest(DashboardTestCase): + @classmethod + def tearDownClass(cls): + super(DashboardTest, cls).tearDownClass() + cls._ceph_cmd(['osd', 'pool', 'delete', 'dashboard_pool', 'dashboard_pool', + '--yes-i-really-really-mean-it']) + + @authenticate def test_pool_list(self): data = self._get("/api/pool") @@ -60,3 +67,28 @@ def test_pool_get(self): self.assertIn('flags', pool) self.assertIn('stats', pool) self.assertNotIn('flags_names', pool) + + @authenticate + def test_pool_create(self): + data = { + 'pool': 'dashboard_pool', + 'pg_num': '10', + 'pool_type': 'replicated', + 'application_metadata': '{"rbd": {}}' + } + self._post('/api/pool/', data) + self.assertStatus(201) + + pool = self._get("/api/pool/dashboard_pool") + self.assertStatus(200) + for k, v in data.items(): + if k == 'pool_type': + self.assertEqual(pool['type'], 1) + elif k == 'pg_num': + self.assertEqual(pool[k], int(v), '{}: {} != {}'.format(k, pool[k], v)) + elif k == 'application_metadata': + self.assertEqual(pool[k], {"rbd": {}}) + elif k == 'pool': + self.assertEqual(pool['pool_name'], v) + else: + self.assertEqual(pool[k], v, '{}: {} != {}'.format(k, pool[k], v)) diff --git a/src/pybind/mgr/dashboard_v2/controllers/cephfs.py b/src/pybind/mgr/dashboard_v2/controllers/cephfs.py index c4786cebaf92fc..1036ff4a8add93 100644 --- a/src/pybind/mgr/dashboard_v2/controllers/cephfs.py +++ b/src/pybind/mgr/dashboard_v2/controllers/cephfs.py @@ -2,10 +2,10 @@ from __future__ import absolute_import from collections import defaultdict -import json import cherrypy -from mgr_module import CommandResult + +from ..services.ceph_service import CephService from .. import mgr from ..tools import ApiController, AuthRequired, BaseController, ViewCache @@ -264,6 +264,7 @@ def _clients(self, fs_id): except AttributeError: raise cherrypy.HTTPError(404, "No cephfs with id {0}".format(fs_id)) + if clients is None: raise cherrypy.HTTPError(404, "No cephfs with id {0}".format(fs_id)) @@ -305,14 +306,5 @@ def __init__(self, module_inst, fscid): # pylint: disable=unused-variable @ViewCache() def get(self): - mds_spec = "{0}:0".format(self.fscid) - result = CommandResult("") - self._module.send_command(result, "mds", mds_spec, - json.dumps({ - "prefix": "session ls", - }), - "") - r, outb, outs = result.wait() # TODO handle nonzero returns, e.g. when rank isn't active - assert r == 0 - return json.loads(outb) + return CephService.send_command('mds', 'session ls', srv_spec='{0}:0'.format(self.fscid)) diff --git a/src/pybind/mgr/dashboard_v2/controllers/dashboard.py b/src/pybind/mgr/dashboard_v2/controllers/dashboard.py index 3457c2f14db5dc..882f5072eb167f 100644 --- a/src/pybind/mgr/dashboard_v2/controllers/dashboard.py +++ b/src/pybind/mgr/dashboard_v2/controllers/dashboard.py @@ -5,7 +5,6 @@ import json import cherrypy -from mgr_module import CommandResult from .. import mgr from ..services.ceph_service import CephService @@ -33,26 +32,10 @@ def append_log(self, log_struct): self.log_buffer.appendleft(log_struct) def load_buffer(self, buf, channel_name): - result = CommandResult("") - mgr.send_command(result, "mon", "", json.dumps({ - "prefix": "log last", - "format": "json", - "channel": channel_name, - "num": LOG_BUFFER_SIZE - }), "") - r, outb, outs = result.wait() - if r != 0: - # Oh well. We won't let this stop us though. - self.log.error("Error fetching log history (r={0}, \"{1}\")".format( - r, outs)) - else: - try: - lines = json.loads(outb) - except ValueError: - self.log.error("Error decoding log history") - else: - for l in lines: - buf.appendleft(l) + lines = CephService.send_command('mon', 'log last', channel=channel_name, + num=LOG_BUFFER_SIZE) + for l in lines: + buf.appendleft(l) # pylint: disable=R0914 @cherrypy.expose diff --git a/src/pybind/mgr/dashboard_v2/controllers/pool.py b/src/pybind/mgr/dashboard_v2/controllers/pool.py index 2eac9f50f8818a..2d281bb1f36aec 100644 --- a/src/pybind/mgr/dashboard_v2/controllers/pool.py +++ b/src/pybind/mgr/dashboard_v2/controllers/pool.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import +import json + from ..services.ceph_service import CephService from ..tools import ApiController, RESTController, AuthRequired @@ -47,3 +49,40 @@ def list(self, attrs=None, stats=False): def get(self, pool_name, attrs=None, stats=False): pools = self.list(attrs, stats) return [pool for pool in pools if pool['pool_name'] == pool_name][0] + + # pylint: disable=too-many-arguments, too-many-locals + @RESTController.args_from_json + def create(self, pool, pg_num, pool_type, erasure_code_profile=None, cache_mode=None, + tier_of=None, read_tier=None, flags=None, compression_required_ratio=None, + application_metadata=None, crush_rule=None, rule_name=None, **kwargs): + ecp = erasure_code_profile if erasure_code_profile else None + CephService.send_command('mon', 'osd pool create', pool=pool, pg_num=int(pg_num), pgp_num=int(pg_num), + pool_type=pool_type, erasure_code_profile=ecp) + + if cache_mode: + CephService.send_command('mon', 'osd tier cache-mode', pool=pool, mode=cache_mode) + if tier_of: + CephService.send_command('mon', 'osd tier add', tierpool=tier_of, pool=pool) + if read_tier: + CephService.send_command('mon', 'osd tier set-overlay', pool=pool, + overlaypool=read_tier) + if flags and 'ec_overwrites' in flags: + CephService.send_command('mon', 'osd pool set', pool=pool, var='allow_ec_overwrites', + val='true') + if compression_required_ratio: + CephService.send_command('mon', 'osd pool set', pool=pool, + var='compression_required_ratio', + val=str(compression_required_ratio)) + if application_metadata: + for app in set(json.loads(application_metadata)): + CephService.send_command('mon', 'osd pool application enable', pool=pool, app=app) + if crush_rule: + CephService.send_command('mon', 'osd pool set', pool=pool, var='crush_rule', + val=rule_name) + for key, value in kwargs.items(): + if type == 'replicated' and key not in \ + ['name', 'erasure_code_profile_id'] and value is not None: + CephService.send_command('mon', 'osd pool set', pool=pool, var=key, val=value) + elif self.type == 'erasure' and key not in ['name', 'size', 'min_size'] \ + and value is not None: + CephService.send_command('mon', 'osd pool set', pool=pool, var=key, val=value) diff --git a/src/pybind/mgr/dashboard_v2/services/ceph_service.py b/src/pybind/mgr/dashboard_v2/services/ceph_service.py index cb27e1ea6e9b0f..443e28e57a4716 100644 --- a/src/pybind/mgr/dashboard_v2/services/ceph_service.py +++ b/src/pybind/mgr/dashboard_v2/services/ceph_service.py @@ -4,8 +4,10 @@ import time import collections from collections import defaultdict +import json -from .. import mgr +from mgr_module import CommandResult +from .. import logger, mgr class CephService(object): @@ -101,3 +103,42 @@ def get_rate(series): pool['stats'] = s pools_w_stats.append(pool) return pools_w_stats + + @classmethod + def send_command(cls, srv_type, prefix, srv_spec='', **kwargs): + """ + :type prefix: str + :param srv_type: mon | + :param kwargs: will be added to argdict + :param srv_spec: typically empty. or something like ":0" + + :raises PermissionError: See rados.make_ex + :raises ObjectNotFound: See rados.make_ex + :raises IOError: See rados.make_ex + :raises NoSpace: See rados.make_ex + :raises ObjectExists: See rados.make_ex + :raises ObjectBusy: See rados.make_ex + :raises NoData: See rados.make_ex + :raises InterruptedOrTimeoutError: See rados.make_ex + :raises TimedOut: See rados.make_ex + :raises ValueError: return code != 0 + """ + argdict = { + "prefix": prefix, + "format": "json", + } + argdict.update({k: v for k, v in kwargs.items() if v}) + + result = CommandResult("") + mgr.send_command(result, srv_type, srv_spec, json.dumps(argdict), "") + r, outb, outs = result.wait() + if r != 0: + # Oh well. We won't let this stop us though. + msg = "send_command '{}' failed. (r={}, \"{}\")".format(prefix, r, outs) + logger.error(msg) + raise ValueError(msg) + else: + try: + return json.loads(outb) + except Exception: # pylint: disable=broad-except + return outb diff --git a/src/pybind/mgr/dashboard_v2/tests/test_tools.py b/src/pybind/mgr/dashboard_v2/tests/test_tools.py index ca4d9040c3c084..93a845f3349fb5 100644 --- a/src/pybind/mgr/dashboard_v2/tests/test_tools.py +++ b/src/pybind/mgr/dashboard_v2/tests/test_tools.py @@ -6,10 +6,11 @@ from mock import patch from .helper import ControllerTestCase -from ..tools import RESTController +from ..tools import RESTController, ApiController # pylint: disable=W0613 +@ApiController('foo') class FooResource(RESTController): elems = [] @@ -38,8 +39,8 @@ def set(self, data, key): class FooArgs(RESTController): @RESTController.args_from_json - def set(self, code, name): - return {'code': code, 'name': name} + def set(self, code, name, opt1=None, opt2=None): + return {'code': code, 'name': name, 'opt1': opt1, 'opt2': opt2} # pylint: disable=C0102 @@ -93,7 +94,13 @@ def test_not_implemented(self): def test_args_from_json(self): self._put("/fooargs/hello", {'name': 'world'}) - self.assertJsonBody({'code': 'hello', 'name': 'world'}) + self.assertJsonBody({'code': 'hello', 'name': 'world', 'opt1': None, 'opt2': None}) + + self._put("/fooargs/hello", {'name': 'world', 'opt1': 'opt1'}) + self.assertJsonBody({'code': 'hello', 'name': 'world', 'opt1': 'opt1', 'opt2': None}) + + self._put("/fooargs/hello", {'name': 'world', 'opt2': 'opt2'}) + self.assertJsonBody({'code': 'hello', 'name': 'world', 'opt1': None, 'opt2': 'opt2'}) def test_detail_route(self): self._get('/foo/1/detail') @@ -101,3 +108,20 @@ def test_detail_route(self): self._post('/foo/1/detail', 'post-data') self.assertStatus(405) + + def test_developer_page(self): + self.getPage('/foo', headers=[('Accept', 'text/html')]) + assert '

GET foo/

' in self.body.decode('utf-8') + assert 'Content-Type: text/html' in self.body.decode('utf-8') + assert '
' in self.body.decode('utf-8') + assert '' in self.body.decode('utf-8') + + def test_developer_exception_page(self): + self.getPage('/foo', + headers=[('Accept', 'text/html'), ('Content-Length', '0')], + method='put') + assert '

PUT foo/

' in self.body.decode('utf-8') + assert 'Exception' in self.body.decode('utf-8') + assert 'Content-Type: text/html' in self.body.decode('utf-8') + assert '' in self.body.decode('utf-8') + assert '' in self.body.decode('utf-8') diff --git a/src/pybind/mgr/dashboard_v2/tools.py b/src/pybind/mgr/dashboard_v2/tools.py index f8013706ba3ea7..f26194f46eb9dd 100644 --- a/src/pybind/mgr/dashboard_v2/tools.py +++ b/src/pybind/mgr/dashboard_v2/tools.py @@ -268,6 +268,80 @@ def wrapper(*args, **kwargs): return wrapper +# pylint: disable=too-many-locals +def debug_html_page(meth): + def wrapper(self, *vpath, **kwargs): + assert isinstance(self, RESTController) + if 'text/html' not in cherrypy.request.headers.get('Accept', ''): + return meth(self, *vpath, **kwargs) + + template = """ + +

{method} {path}/{vpath}

+

Status: {status_code}

+ 

{reponse_headers}
+ 
{data}
+ {form} + """ + form_template = """ +

+ {fields}
+ + Create +

+ """ + + if '_method' in kwargs: + cherrypy.request.method = kwargs.pop('_method').upper() + + try: + data = meth(self, *vpath, **kwargs) + except Exception as e: # pylint: disable=broad-except + except_template = """ +

Exception: {etype}: {tostr}

+ 
{trace}
+ Params: {kwargs} + """ + import traceback + tb = sys.exc_info()[2] + cherrypy.response.headers['Content-Type'] = 'text/html' + data = except_template.format( + etype=e.__class__.__name__, + tostr=str(e), + trace='\n'.join(traceback.format_tb(tb)), + kwargs=kwargs + ) + + if cherrypy.response.headers['Content-Type'] == 'application/json': + data = json.dumps(json.loads(data), indent=2, sort_keys=True) + + try: + create = getattr(self, 'create') + f_args = RESTController._function_args(create) + input_fields = ['{name}:'.format(name=name) for name in + f_args] + form = form_template.format( + fields='
'.join(input_fields), + path=self._cp_path_, + vpath='/'.join(vpath) + ) + except AttributeError: + form = '' + + cherrypy.response.headers['Content-Type'] = 'text/html' + return template.format( + method=cherrypy.request.method, + path=self._cp_path_, + vpath='/'.join(vpath), + status_code=cherrypy.response.status, + reponse_headers='\n'.join( + '{}: {}'.format(k, v) for k, v in cherrypy.response.headers.items()), + data=data, + form=form + ) + return wrapper + + class RESTController(BaseController): """ Base class for providing a RESTful interface to a resource. @@ -325,6 +399,7 @@ def _get_method(self, vpath): return method, status_code @cherrypy.expose + @debug_html_page def default(self, *vpath, **params): method, status_code = self._get_method(vpath) @@ -343,10 +418,24 @@ def args_from_json(func): func._args_from_json_ = True return func + @staticmethod + def _function_args(func): + if sys.version_info > (3, 0): # pylint: disable=no-else-return + return list(inspect.signature(func).parameters.keys()) + else: + return inspect.getargspec(func).args[1:] # pylint: disable=deprecated-method + # pylint: disable=W1505 @staticmethod def _takes_json(func): def inner(*args, **kwargs): + if cherrypy.request.headers.get('Content-Type', + '') == 'application/x-www-form-urlencoded': + if hasattr(func, '_args_from_json_'): # pylint: disable=no-else-return + return func(*args, **kwargs) + else: + return func(kwargs) + content_length = int(cherrypy.request.headers['Content-Length']) body = cherrypy.request.body.read(content_length) if not body: @@ -358,19 +447,10 @@ def inner(*args, **kwargs): raise cherrypy.HTTPError(400, 'Failed to decode JSON: {}' .format(str(e))) if hasattr(func, '_args_from_json_'): - if sys.version_info > (3, 0): - f_args = list(inspect.signature(func).parameters.keys()) - else: - f_args = inspect.getargspec(func).args[1:] - n_args = [] - for arg in args: - n_args.append(arg) - for arg in f_args: - if arg in data: - n_args.append(data[arg]) - data.pop(arg) - kwargs.update(data) - return func(*n_args, **kwargs) + f_args = RESTController._function_args(func) + n_kwargs = {k: v for k, v in data.items() if k in f_args} + kwargs.update(n_kwargs) + return func(*args, **kwargs) return func(data, *args, **kwargs) return inner