Merge pull request #20823 from rjfd/wip-dashboard-pools

mgr/dashboard_v2: Pool controller

Reviewed-by: Ricardo Marques <rimarques@suse.com>
Reviewed-by: Sebastian Wagner <sebastian.wagner@suse.com>

qa/suites/rados/mgr/tasks/dashboard_v2.yaml

@@ -31,3 +31,4 @@ tasks:
         - tasks.mgr.dashboard_v2.test_summary
         - tasks.mgr.dashboard_v2.test_rgw
         - tasks.mgr.dashboard_v2.test_rbd
+        - tasks.mgr.dashboard_v2.test_pool

qa/tasks/mgr/dashboard_v2/test_pool.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
+from .helper import DashboardTestCase, authenticate
+
+
+class DashboardTest(DashboardTestCase):
+    @authenticate
+    def test_pool_list(self):
+        data = self._get("/api/pool")
+        self.assertStatus(200)
+
+        cluster_pools = self.ceph_cluster.mon_manager.list_pools()
+        self.assertEqual(len(cluster_pools), len(data))
+        for pool in data:
+            self.assertIn('pool_name', pool)
+            self.assertIn('type', pool)
+            self.assertIn('flags', pool)
+            self.assertIn('flags_names', pool)
+            self.assertNotIn('stats', pool)
+            self.assertIn(pool['pool_name'], cluster_pools)
+
+    @authenticate
+    def test_pool_list_attrs(self):
+        data = self._get("/api/pool?attrs=type,flags")
+        self.assertStatus(200)
+
+        cluster_pools = self.ceph_cluster.mon_manager.list_pools()
+        self.assertEqual(len(cluster_pools), len(data))
+        for pool in data:
+            self.assertIn('pool_name', pool)
+            self.assertIn('type', pool)
+            self.assertIn('flags', pool)
+            self.assertNotIn('flags_names', pool)
+            self.assertNotIn('stats', pool)
+            self.assertIn(pool['pool_name'], cluster_pools)
+
+    @authenticate
+    def test_pool_list_stats(self):
+        data = self._get("/api/pool?stats=true")
+        self.assertStatus(200)
+
+        cluster_pools = self.ceph_cluster.mon_manager.list_pools()
+        self.assertEqual(len(cluster_pools), len(data))
+        for pool in data:
+            self.assertIn('pool_name', pool)
+            self.assertIn('type', pool)
+            self.assertIn('flags', pool)
+            self.assertIn('stats', pool)
+            self.assertIn('flags_names', pool)
+            self.assertIn(pool['pool_name'], cluster_pools)
+
+    @authenticate
+    def test_pool_get(self):
+        cluster_pools = self.ceph_cluster.mon_manager.list_pools()
+        pool = self._get("/api/pool/{}?stats=true&attrs=type,flags,stats"
+                         .format(cluster_pools[0]))
+        self.assertEqual(pool['pool_name'], cluster_pools[0])
+        self.assertIn('type', pool)
+        self.assertIn('flags', pool)
+        self.assertIn('stats', pool)
+        self.assertNotIn('flags_names', pool)

src/pybind/mgr/dashboard_v2/controllers/dashboard.py

@@ -2,14 +2,13 @@
 from __future__ import absolute_import
 
 import collections
-from collections import defaultdict
 import json
-import time
 
 import cherrypy
 from mgr_module import CommandResult
 
 from .. import mgr
+from ..services.ceph_service import CephService
 from ..tools import ApiController, AuthRequired, BaseController, NotificationQueue
 
 
@@ -72,44 +71,13 @@ def health(self):
 
         osd_map = self.osd_map()
 
-        pg_summary = mgr.get("pg_summary")
-
-        pools = []
-
-        pool_stats = defaultdict(lambda: defaultdict(
-            lambda: collections.deque(maxlen=10)))
-
-        df = mgr.get("df")
-        pool_stats_dict = dict([(p['id'], p['stats']) for p in df['pools']])
-        now = time.time()
-        for pool_id, stats in pool_stats_dict.items():
-            for stat_name, stat_val in stats.items():
-                pool_stats[pool_id][stat_name].appendleft((now, stat_val))
-
-        for pool in osd_map['pools']:
-            pool['pg_status'] = pg_summary['by_pool'][pool['pool'].__str__()]
-            stats = pool_stats[pool['pool']]
-            s = {}
-
-            def get_rate(series):
-                if len(series) >= 2:
-                    return (float(series[0][1]) - float(series[1][1])) / \
-                        (float(series[0][0]) - float(series[1][0]))
-                return 0
-
-            for stat_name, stat_series in stats.items():
-                s[stat_name] = {
-                    'latest': stat_series[0][1],
-                    'rate': get_rate(stat_series),
-                    'series': [i for i in stat_series]
-                }
-            pool['stats'] = s
-            pools.append(pool)
+        pools = CephService.get_pool_list_with_stats()
 
         # Not needed, skip the effort of transmitting this
         # to UI
         del osd_map['pg_temp']
 
+        df = mgr.get("df")
         df['stats']['total_objects'] = sum(
             [p['stats']['objects'] for p in df['pools']])
 

src/pybind/mgr/dashboard_v2/controllers/pool.py

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
+from ..services.ceph_service import CephService
+from ..tools import ApiController, RESTController, AuthRequired
+
+
+@ApiController('pool')
+@AuthRequired()
+class Pool(RESTController):
+
+    @classmethod
+    def _serialize_pool(cls, pool, attrs):
+        if not attrs or not isinstance(attrs, list):
+            return pool
+
+        res = {}
+        for attr in attrs:
+            if attr not in pool:
+                continue
+            if attr == 'type':
+                res[attr] = {1: 'replicated', 3: 'erasure'}[pool[attr]]
+            else:
+                res[attr] = pool[attr]
+
+        # pool_name is mandatory
+        res['pool_name'] = pool['pool_name']
+        return res
+
+    @staticmethod
+    def _str_to_bool(var):
+        if isinstance(var, bool):
+            return var
+        return var.lower() in ("true", "yes", "1", 1)
+
+    def list(self, attrs=None, stats=False):
+        if attrs:
+            attrs = attrs.split(',')
+
+        if self._str_to_bool(stats):
+            pools = CephService.get_pool_list_with_stats()
+        else:
+            pools = CephService.get_pool_list()
+
+        return [self._serialize_pool(pool, attrs) for pool in pools]
+
+    def get(self, pool_name, attrs=None, stats=False):
+        pools = self.list(attrs, stats)
+        return [pool for pool in pools if pool['pool_name'] == pool_name][0]

src/pybind/mgr/dashboard_v2/run-backend-api-tests.sh

@@ -80,12 +80,20 @@ sleep 10
 source $TEMP_DIR/venv/bin/activate
 BUILD_DIR=`pwd`
 
-TEST_CASES=`for i in \`ls $BUILD_DIR/../qa/tasks/mgr/dashboard_v2/test_*\`; do F=$(basename $i); M="${F%.*}"; echo -n " tasks.mgr.dashboard_v2.$M"; done`
+if [ "$#" -gt 0 ]; then
+  TEST_CASES=""
+  for t in "$@"; do
+    TEST_CASES="$TESTS_CASES $t"
+  done
+else
+  TEST_CASES=`for i in \`ls $BUILD_DIR/../qa/tasks/mgr/dashboard_v2/test_*\`; do F=$(basename $i); M="${F%.*}"; echo -n " tasks.mgr.dashboard_v2.$M"; done`
+  TEST_CASES="tasks.mgr.test_dashboard_v2 $TEST_CASES"
+fi
 
 export PATH=$BUILD_DIR/bin:$PATH
 export LD_LIBRARY_PATH=$BUILD_DIR/lib/cython_modules/lib.${CEPH_PY_VERSION_MAJOR}/:$BUILD_DIR/lib
 export PYTHONPATH=$TEMP_DIR/teuthology:$BUILD_DIR/../qa:$BUILD_DIR/lib/cython_modules/lib.${CEPH_PY_VERSION_MAJOR}/
-eval python ../qa/tasks/vstart_runner.py tasks.mgr.test_dashboard_v2 $TEST_CASES
+eval python ../qa/tasks/vstart_runner.py $TEST_CASES
 
 deactivate
 killall ceph-mgr

src/pybind/mgr/dashboard_v2/services/ceph_service.py

@@ -1,6 +1,10 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
 
+import time
+import collections
+from collections import defaultdict
+
 from .. import mgr
 
 
@@ -58,3 +62,42 @@ def get_pool_list(cls, application=None):
             return osd_map['pools']
         return [pool for pool in osd_map['pools']
                 if application in pool.get('application_metadata', {})]
+
+    @classmethod
+    def get_pool_list_with_stats(cls, application=None):
+        # pylint: disable=too-many-locals
+        pools = cls.get_pool_list(application)
+
+        pools_w_stats = []
+
+        pg_summary = mgr.get("pg_summary")
+        pool_stats = defaultdict(lambda: defaultdict(
+            lambda: collections.deque(maxlen=10)))
+
+        df = mgr.get("df")
+        pool_stats_dict = dict([(p['id'], p['stats']) for p in df['pools']])
+        now = time.time()
+        for pool_id, stats in pool_stats_dict.items():
+            for stat_name, stat_val in stats.items():
+                pool_stats[pool_id][stat_name].appendleft((now, stat_val))
+
+        for pool in pools:
+            pool['pg_status'] = pg_summary['by_pool'][pool['pool'].__str__()]
+            stats = pool_stats[pool['pool']]
+            s = {}
+
+            def get_rate(series):
+                if len(series) >= 2:
+                    return (float(series[0][1]) - float(series[1][1])) / \
+                        (float(series[0][0]) - float(series[1][0]))
+                return 0
+
+            for stat_name, stat_series in stats.items():
+                s[stat_name] = {
+                    'latest': stat_series[0][1],
+                    'rate': get_rate(stat_series),
+                    'series': [i for i in stat_series]
+                }
+            pool['stats'] = s
+            pools_w_stats.append(pool)
+        return pools_w_stats