mgr/dashboard_v2: Refactor send_command into single method

mgr/dashboard_v2: Add create() to Pool controller

mgr/dashboard_v2: Add minimalistic HTML debug representation of the API

* Add API test
* Fixed  @RESTController.args_from_json

Signed-off-by: Sebastian Wagner <sebastian.wagner@suse.com>

qa/tasks/mgr/dashboard_v2/helper.py

@@ -85,7 +85,8 @@ def _request(self, url, method, data=None):
             self._resp = self._session.delete(url, json=data)
         elif method == 'PUT':
             self._resp = self._session.put(url, json=data)
-        return None
+        else:
+            assert False
 
     def _get(self, url):
         return self._request(url, 'GET')

qa/tasks/mgr/dashboard_v2/test_pool.py

@@ -5,6 +5,13 @@
 
 
 class DashboardTest(DashboardTestCase):
+    @classmethod
+    def tearDownClass(cls):
+        super(DashboardTest, cls).tearDownClass()
+        cls._ceph_cmd(['osd', 'pool', 'delete', 'dashboard_pool', 'dashboard_pool',
+                       '--yes-i-really-really-mean-it'])
+
+
     @authenticate
     def test_pool_list(self):
         data = self._get("/api/pool")
@@ -60,3 +67,28 @@ def test_pool_get(self):
         self.assertIn('flags', pool)
         self.assertIn('stats', pool)
         self.assertNotIn('flags_names', pool)
+
+    @authenticate
+    def test_pool_create(self):
+        data = {
+            'pool': 'dashboard_pool',
+            'pg_num': '10',
+            'pool_type': 'replicated',
+            'application_metadata': '{"rbd": {}}'
+        }
+        self._post('/api/pool/', data)
+        self.assertStatus(201)
+
+        pool = self._get("/api/pool/dashboard_pool")
+        self.assertStatus(200)
+        for k, v in data.items():
+            if k == 'pool_type':
+                self.assertEqual(pool['type'], 1)
+            elif k == 'pg_num':
+                self.assertEqual(pool[k], int(v), '{}: {} != {}'.format(k, pool[k], v))
+            elif k == 'application_metadata':
+                self.assertEqual(pool[k], {"rbd": {}})
+            elif k == 'pool':
+                self.assertEqual(pool['pool_name'], v)
+            else:
+                self.assertEqual(pool[k], v, '{}: {} != {}'.format(k, pool[k], v))

src/pybind/mgr/dashboard_v2/controllers/cephfs.py

@@ -2,10 +2,10 @@
 from __future__ import absolute_import
 
 from collections import defaultdict
-import json
 
 import cherrypy
-from mgr_module import CommandResult
+
+from ..services.ceph_service import CephService
 
 from .. import mgr
 from ..tools import ApiController, AuthRequired, BaseController, ViewCache
@@ -264,6 +264,7 @@ def _clients(self, fs_id):
         except AttributeError:
             raise cherrypy.HTTPError(404,
                                      "No cephfs with id {0}".format(fs_id))
+
         if clients is None:
             raise cherrypy.HTTPError(404,
                                      "No cephfs with id {0}".format(fs_id))
@@ -305,14 +306,5 @@ def __init__(self, module_inst, fscid):
     # pylint: disable=unused-variable
     @ViewCache()
     def get(self):
-        mds_spec = "{0}:0".format(self.fscid)
-        result = CommandResult("")
-        self._module.send_command(result, "mds", mds_spec,
-                                  json.dumps({
-                                      "prefix": "session ls",
-                                  }),
-                                  "")
-        r, outb, outs = result.wait()
         # TODO handle nonzero returns, e.g. when rank isn't active
-        assert r == 0
-        return json.loads(outb)
+        return CephService.send_command('mds', 'session ls', srv_spec='{0}:0'.format(self.fscid))

src/pybind/mgr/dashboard_v2/controllers/dashboard.py

@@ -5,7 +5,6 @@
 import json
 
 import cherrypy
-from mgr_module import CommandResult
 
 from .. import mgr
 from ..services.ceph_service import CephService
@@ -33,26 +32,10 @@ def append_log(self, log_struct):
             self.log_buffer.appendleft(log_struct)
 
     def load_buffer(self, buf, channel_name):
-        result = CommandResult("")
-        mgr.send_command(result, "mon", "", json.dumps({
-            "prefix": "log last",
-            "format": "json",
-            "channel": channel_name,
-            "num": LOG_BUFFER_SIZE
-        }), "")
-        r, outb, outs = result.wait()
-        if r != 0:
-            # Oh well. We won't let this stop us though.
-            self.log.error("Error fetching log history (r={0}, \"{1}\")".format(
-                r, outs))
-        else:
-            try:
-                lines = json.loads(outb)
-            except ValueError:
-                self.log.error("Error decoding log history")
-            else:
-                for l in lines:
-                    buf.appendleft(l)
+        lines = CephService.send_command('mon', 'log last', channel=channel_name,
+                                         num=LOG_BUFFER_SIZE)
+        for l in lines:
+            buf.appendleft(l)
 
     # pylint: disable=R0914
     @cherrypy.expose

src/pybind/mgr/dashboard_v2/controllers/pool.py

@@ -1,6 +1,8 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
 
+import json
+
 from ..services.ceph_service import CephService
 from ..tools import ApiController, RESTController, AuthRequired
 
@@ -47,3 +49,40 @@ def list(self, attrs=None, stats=False):
     def get(self, pool_name, attrs=None, stats=False):
         pools = self.list(attrs, stats)
         return [pool for pool in pools if pool['pool_name'] == pool_name][0]
+
+    # pylint: disable=too-many-arguments, too-many-locals
+    @RESTController.args_from_json
+    def create(self, pool, pg_num, pool_type, erasure_code_profile=None, cache_mode=None,
+               tier_of=None, read_tier=None, flags=None, compression_required_ratio=None,
+               application_metadata=None, crush_rule=None, rule_name=None, **kwargs):
+        ecp = erasure_code_profile if erasure_code_profile else None
+        CephService.send_command('mon', 'osd pool create', pool=pool, pg_num=int(pg_num), pgp_num=int(pg_num),
+                                 pool_type=pool_type, erasure_code_profile=ecp)
+
+        if cache_mode:
+            CephService.send_command('mon', 'osd tier cache-mode', pool=pool, mode=cache_mode)
+        if tier_of:
+            CephService.send_command('mon', 'osd tier add', tierpool=tier_of, pool=pool)
+        if read_tier:
+            CephService.send_command('mon', 'osd tier set-overlay', pool=pool,
+                                     overlaypool=read_tier)
+        if flags and 'ec_overwrites' in flags:
+            CephService.send_command('mon', 'osd pool set', pool=pool, var='allow_ec_overwrites',
+                                     val='true')
+        if compression_required_ratio:
+            CephService.send_command('mon', 'osd pool set', pool=pool,
+                                     var='compression_required_ratio',
+                                     val=str(compression_required_ratio))
+        if application_metadata:
+            for app in set(json.loads(application_metadata)):
+                CephService.send_command('mon', 'osd pool application enable', pool=pool, app=app)
+        if crush_rule:
+            CephService.send_command('mon', 'osd pool set', pool=pool, var='crush_rule',
+                                     val=rule_name)
+        for key, value in kwargs.items():
+            if type == 'replicated' and key not in \
+                    ['name', 'erasure_code_profile_id'] and value is not None:
+                CephService.send_command('mon', 'osd pool set', pool=pool, var=key, val=value)
+            elif self.type == 'erasure' and key not in ['name', 'size', 'min_size'] \
+                    and value is not None:
+                CephService.send_command('mon', 'osd pool set', pool=pool, var=key, val=value)

src/pybind/mgr/dashboard_v2/services/ceph_service.py

@@ -4,8 +4,10 @@
 import time
 import collections
 from collections import defaultdict
+import json
 
-from .. import mgr
+from mgr_module import CommandResult
+from .. import logger, mgr
 
 
 class CephService(object):
@@ -101,3 +103,42 @@ def get_rate(series):
             pool['stats'] = s
             pools_w_stats.append(pool)
         return pools_w_stats
+
+    @classmethod
+    def send_command(cls, srv_type, prefix, srv_spec='', **kwargs):
+        """
+        :type prefix: str
+        :param srv_type: mon |
+        :param kwargs: will be added to argdict
+        :param srv_spec: typically empty. or something like "<fs_id>:0"
+
+        :raises PermissionError: See rados.make_ex
+        :raises ObjectNotFound: See rados.make_ex
+        :raises IOError: See rados.make_ex
+        :raises NoSpace: See rados.make_ex
+        :raises ObjectExists: See rados.make_ex
+        :raises ObjectBusy: See rados.make_ex
+        :raises NoData: See rados.make_ex
+        :raises InterruptedOrTimeoutError: See rados.make_ex
+        :raises TimedOut: See rados.make_ex
+        :raises ValueError: return code != 0
+        """
+        argdict = {
+            "prefix": prefix,
+            "format": "json",
+        }
+        argdict.update({k: v for k, v in kwargs.items() if v})
+
+        result = CommandResult("")
+        mgr.send_command(result, srv_type, srv_spec, json.dumps(argdict), "")
+        r, outb, outs = result.wait()
+        if r != 0:
+            # Oh well. We won't let this stop us though.
+            msg = "send_command '{}' failed. (r={}, \"{}\")".format(prefix, r, outs)
+            logger.error(msg)
+            raise ValueError(msg)
+        else:
+            try:
+                return json.loads(outb)
+            except Exception:  # pylint: disable=broad-except
+                return outb

src/pybind/mgr/dashboard_v2/tests/test_tools.py

@@ -6,10 +6,11 @@
 from mock import patch
 
 from .helper import ControllerTestCase
-from ..tools import RESTController
+from ..tools import RESTController, ApiController
 
 
 # pylint: disable=W0613
+@ApiController('foo')
 class FooResource(RESTController):
     elems = []
 
@@ -38,8 +39,8 @@ def set(self, data, key):
 
 class FooArgs(RESTController):
     @RESTController.args_from_json
-    def set(self, code, name):
-        return {'code': code, 'name': name}
+    def set(self, code, name, opt1=None, opt2=None):
+        return {'code': code, 'name': name, 'opt1': opt1, 'opt2': opt2}
 
 
 # pylint: disable=C0102
@@ -93,11 +94,34 @@ def test_not_implemented(self):
 
     def test_args_from_json(self):
         self._put("/fooargs/hello", {'name': 'world'})
-        self.assertJsonBody({'code': 'hello', 'name': 'world'})
+        self.assertJsonBody({'code': 'hello', 'name': 'world', 'opt1': None, 'opt2': None})
+
+        self._put("/fooargs/hello", {'name': 'world', 'opt1': 'opt1'})
+        self.assertJsonBody({'code': 'hello', 'name': 'world', 'opt1': 'opt1', 'opt2': None})
+
+        self._put("/fooargs/hello", {'name': 'world', 'opt2': 'opt2'})
+        self.assertJsonBody({'code': 'hello', 'name': 'world', 'opt1': None, 'opt2': 'opt2'})
 
     def test_detail_route(self):
         self._get('/foo/1/detail')
         self.assertJsonBody({'detail': ['1', ['detail']]})
 
         self._post('/foo/1/detail', 'post-data')
         self.assertStatus(405)
+
+    def test_developer_page(self):
+        self.getPage('/foo', headers=[('Accept', 'text/html')])
+        assert '<p>GET foo/</p>' in self.body.decode('utf-8')
+        assert 'Content-Type: text/html' in self.body.decode('utf-8')
+        assert '<form action="/api/foo/" method="post">' in self.body.decode('utf-8')
+        assert '<input type="hidden" name="_method" value="post" />' in self.body.decode('utf-8')
+
+    def test_developer_exception_page(self):
+        self.getPage('/foo',
+                     headers=[('Accept', 'text/html'), ('Content-Length', '0')],
+                     method='put')
+        assert '<p>PUT foo/</p>' in self.body.decode('utf-8')
+        assert 'Exception' in self.body.decode('utf-8')
+        assert 'Content-Type: text/html' in self.body.decode('utf-8')
+        assert '<form action="/api/foo/" method="post">' in self.body.decode('utf-8')
+        assert '<input type="hidden" name="_method" value="post" />' in self.body.decode('utf-8')