mgr/restful: fix py got exception when get osd info

when we create pool by device class, then get osd info by restful,
such as https://192.7.7.36:8003/osd
browser display error, and found excetion in /var/log/ceph/ceph-mgr****.log

2018-03-30 16:07:52.756560 7feef9f17700  0 mgr[restful] Traceback (most recent call last):
  File "/usr/lib/python2.7/site-packages/pecan/core.py", line 570, in __call__
    self.handle_request(req, resp)
  File "/usr/lib/python2.7/site-packages/pecan/core.py", line 508, in handle_request
    result = controller(*args, **kwargs)
  File "/usr/lib64/ceph/mgr/restful/decorators.py", line 33, in decorated
    return f(*args, **kwargs)
  File "/usr/lib64/ceph/mgr/restful/api/osd.py", line 130, in get
    return module.instance.get_osds(pool_id)
  File "/usr/lib64/ceph/mgr/restful/module.py", line 543, in get_osds
    pools_map = self.get_osd_pools()
  File "/usr/lib64/ceph/mgr/restful/module.py", line 516, in get_osd_pools
    pool_osds = common.crush_rule_osds(self.get('osd_map_tree')['nodes'], rule)
  File "/usr/lib64/ceph/mgr/restful/common.py", line 149, in crush_rule_osds
    osds |= _gather_osds(nodes_by_id[step['item']], rule['steps'][i + 1:])
KeyError: -8L

buckets in osd_map_crush has more infomation than nodes in osd_map_tree
so we can use buckets instead op nodes to get rule osds

Signed-off-by: zouaiguo <zou.aiguo@zte.com.cn>
(cherry picked from commit 23b6c90)

Conflicts:
        src/pybind/mgr/restful/api/crush.py: Use 'module' instead of
	'context'

src/pybind/mgr/restful/api/crush.py

@@ -14,11 +14,11 @@ def get(self, **kwargs):
         """
         Show crush rules
         """
-        rules = module.instance.get('osd_map_crush')['rules']
-        nodes = module.instance.get('osd_map_tree')['nodes']
+        crush = module.instance.get('osd_map_crush')
+        rules = crush['rules']
 
         for rule in rules:
-            rule['osd_count'] = len(common.crush_rule_osds(nodes, rule))
+            rule['osd_count'] = len(common.crush_rule_osds(crush['buckets'], rule))
 
         return rules
 

src/pybind/mgr/restful/common.py

@@ -88,31 +88,34 @@ def pool_update_commands(pool_name, args):
 
     return commands
 
-
-def crush_rule_osds(nodes, rule):
-    nodes_by_id = dict((n['id'], n) for n in nodes)
+def crush_rule_osds(node_buckets, rule):
+    nodes_by_id = dict((b['id'], b) for b in node_buckets)
 
     def _gather_leaf_ids(node):
         if node['id'] >= 0:
             return set([node['id']])
 
         result = set()
-        for child_id in node['children']:
-            if child_id >= 0:
-                result.add(child_id)
+        for item in node['items']:
+            if item['id'] >= 0:
+                result.add(item['id'])
             else:
-                result |= _gather_leaf_ids(nodes_by_id[child_id])
+                result |= _gather_leaf_ids(nodes_by_id[item['id']])
 
         return result
 
     def _gather_descendent_ids(node, typ):
         result = set()
-        for child_id in node['children']:
-            child_node = nodes_by_id[child_id]
-            if child_node['type'] == typ:
-                result.add(child_node['id'])
-            elif 'children' in child_node:
-                result |= _gather_descendent_ids(child_node, typ)
+        for item in node['items']:
+            if item['id'] >= 0:
+                if typ == "osd":
+                    result.add(item['id'])
+            else:
+                child_node = nodes_by_id[item['id']]
+                if child_node['type_name'] == typ:
+                    result.add(child_node['id'])
+                elif 'items' in child_node:
+                    result |= _gather_descendent_ids(child_node, typ)
 
         return result
 
@@ -124,17 +127,26 @@ def _gather_osds(root, steps):
         step = steps[0]
         if step['op'] == 'choose_firstn':
             # Choose all descendents of the current node of type 'type'
-            d = _gather_descendent_ids(root, step['type'])
-            for desc_node in [nodes_by_id[i] for i in d]:
-                osds |= _gather_osds(desc_node, steps[1:])
+            descendent_ids = _gather_descendent_ids(root, step['type'])
+            for node_id in descendent_ids:
+                if node_id >= 0:
+                    osds.add(node_id)
+                else:
+                    for desc_node in nodes_by_id[node_id]:
+                        osds |= _gather_osds(desc_node, steps[1:])
         elif step['op'] == 'chooseleaf_firstn':
             # Choose all descendents of the current node of type 'type',
             # and select all leaves beneath those
-            for desc_node in [nodes_by_id[i] for i in _gather_descendent_ids(root, step['type'])]:
-                # Short circuit another iteration to find the emit
-                # and assume anything we've done a chooseleaf on
-                # is going to be part of the selected set of osds
-                osds |= _gather_leaf_ids(desc_node)
+            descendent_ids = _gather_descendent_ids(root, step['type'])
+            for node_id in descendent_ids:
+                if node_id >= 0:
+                    osds.add(node_id)
+                else:
+                    for desc_node in nodes_by_id[node_id]:
+                        # Short circuit another iteration to find the emit
+                        # and assume anything we've done a chooseleaf on
+                        # is going to be part of the selected set of osds
+                        osds |= _gather_leaf_ids(desc_node)
         elif step['op'] == 'emit':
             if root['id'] >= 0:
                 osds |= root['id']

src/pybind/mgr/restful/module.py

@@ -503,14 +503,15 @@ def get_mons(self):
     def get_osd_pools(self):
         osds = dict(map(lambda x: (x['osd'], []), self.get('osd_map')['osds']))
         pools = dict(map(lambda x: (x['pool'], x), self.get('osd_map')['pools']))
-        crush_rules = self.get('osd_map_crush')['rules']
+        crush = self.get('osd_map_crush')
+        crush_rules = crush['rules']
 
         osds_by_pool = {}
         for pool_id, pool in pools.items():
             pool_osds = None
             for rule in [r for r in crush_rules if r['rule_id'] == pool['crush_rule']]:
                 if rule['min_size'] <= pool['size'] <= rule['max_size']:
-                    pool_osds = common.crush_rule_osds(self.get('osd_map_tree')['nodes'], rule)
+                    pool_osds = common.crush_rule_osds(crush['buckets'], rule)
 
             osds_by_pool[pool_id] = pool_osds