Merge pull request #37309 from rhcs-dashboard/wip-47579-nautilus

nautilus: mgr/dashboard: fix pool usage calculation Reviewed-by: Alfonso Martínez <almartin@redhat.com>
ceph · Sep 24, 2020 · 64b007d · 64b007d
2 parents b64e55b + cc20723
commit 64b007d
Show file tree

Hide file tree

Showing 10 changed files with 56 additions and 15 deletions.
diff --git a/qa/tasks/mgr/dashboard/helper.py b/qa/tasks/mgr/dashboard/helper.py
@@ -18,6 +18,9 @@
 
 
 class DashboardTestCase(MgrTestCase):
+    # Display full error diffs
+    maxDiff = None
+
     # Increased x3 (20 -> 60)
     TIMEOUT_HEALTH_CLEAR = 60
 
@@ -433,6 +436,7 @@ def __new__(cls, typ, none=False):
 
 JTuple = namedtuple('JList', ['elem_typs'])
 
+JUnion = namedtuple('JUnion', ['elem_typs'])
 
 class JObj(namedtuple('JObj', ['sub_elems', 'allow_unknown', 'none', 'unknown_schema'])):
     def __new__(cls, sub_elems, allow_unknown=False, none=False, unknown_schema=None):
@@ -462,6 +466,10 @@ def _validate_json(val, schema, path=[]):
     ... ds = JObj({'a': int, 'b': str, 'c': JList(int)})
     ... _validate_json(d, ds)
     True
+    >>> _validate_json({'num': 1}, JObj({'num': JUnion([int,float])}))
+    True
+    >>> _validate_json({'num': 'a'}, JObj({'num': JUnion([int,float])}))
+    False
     """
     if isinstance(schema, JAny):
         if not schema.none and val is None:
@@ -480,6 +488,14 @@ def _validate_json(val, schema, path=[]):
     if isinstance(schema, JTuple):
         return all(_validate_json(val[i], typ, path + [i])
                    for i, typ in enumerate(schema.elem_typs))
+    if isinstance(schema, JUnion):
+        for typ in schema.elem_typs:
+            try:
+                if _validate_json(val, typ, path):
+                    return True
+            except _ValError:
+                pass
+        return False
     if isinstance(schema, JObj):
         if val is None and schema.none:
             return True

diff --git a/qa/tasks/mgr/dashboard/test_health.py b/qa/tasks/mgr/dashboard/test_health.py
@@ -161,7 +161,8 @@ def test_full_health(self):
                         'wr_bytes': int,
                         'compress_bytes_used': int,
                         'compress_under_bytes': int,
-                        'stored_raw': int
+                        'stored_raw': int,
+                        'avail_raw': int
                     }),
                     'name': str,
                     'id': int

diff --git a/qa/tasks/mgr/dashboard/test_pool.py b/qa/tasks/mgr/dashboard/test_pool.py
@@ -6,7 +6,7 @@
 import time
 from contextlib import contextmanager
 
-from .helper import DashboardTestCase, JAny, JList, JObj
+from .helper import DashboardTestCase, JAny, JList, JObj, JUnion
 
 log = logging.getLogger(__name__)
 
@@ -23,14 +23,16 @@ class PoolTest(DashboardTestCase):
     }, allow_unknown=True)
 
     pool_list_stat_schema = JObj(sub_elems={
-        'latest': int,
+        'latest': JUnion([int,float]),
         'rate': float,
         'rates': JList(JAny(none=False)),
     })
 
     pool_list_stats_schema = JObj(sub_elems={
+        'avail_raw': pool_list_stat_schema,
         'bytes_used': pool_list_stat_schema,
         'max_avail': pool_list_stat_schema,
+        'percent_used': pool_list_stat_schema,
         'rd_bytes': pool_list_stat_schema,
         'wr_bytes': pool_list_stat_schema,
         'rd': pool_list_stat_schema,

diff --git a/src/mon/PGMap.cc b/src/mon/PGMap.cc
@@ -936,6 +936,7 @@ void PGMapDigest::dump_object_stat_sum(
       f->dump_int("compress_under_bytes", statfs.data_compressed_original);
       // Stored by user amplified by replication
       f->dump_int("stored_raw", pool_stat.get_user_bytes(1.0, per_pool));
+      f->dump_unsigned("avail_raw", avail);
     }
   } else {
     tbl << stringify(byte_u_t(stored_normalized));
@@ -1146,7 +1147,7 @@ void PGMap::apply_incremental(CephContext *cct, const Incremental& inc)
 
     auto pool_statfs_iter =
       pool_statfs.find(std::make_pair(update_pool, update_osd));
-    if (pg_pool_sum.count(update_pool)) { 
+    if (pg_pool_sum.count(update_pool)) {
       pool_stat_t &pool_sum_ref = pg_pool_sum[update_pool];
       if (pool_statfs_iter == pool_statfs.end()) {
         pool_statfs.emplace(std::make_pair(update_pool, update_osd), statfs_inc);

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.html
@@ -23,9 +23,10 @@
 
     <ng-template #poolUsageTpl
                  let-row="row">
-      <cd-usage-bar *ngIf="row.stats?.max_avail?.latest"
-                    [totalBytes]="row.stats.bytes_used.latest + row.stats.max_avail.latest"
-                    [usedBytes]="row.stats.bytes_used.latest">
+      <cd-usage-bar *ngIf="row.stats?.avail_raw?.latest"
+                    [totalBytes]="row.stats.bytes_used.latest + row.stats.avail_raw.latest"
+                    [usedBytes]="row.stats.bytes_used.latest"
+                    decimals="2">
       </cd-usage-bar>
     </ng-template>
   </tab>

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.spec.ts
@@ -302,6 +302,8 @@ describe('PoolListComponent', () => {
           stats: {
             bytes_used: { latest: 0, rate: 0, rates: [] },
             max_avail: { latest: 0, rate: 0, rates: [] },
+            avail_raw: { latest: 0, rate: 0, rates: [] },
+            percent_used: { latest: 0, rate: 0, rates: [] },
             rd: { latest: 0, rate: 0, rates: [] },
             rd_bytes: { latest: 0, rate: 0, rates: [] },
             wr: { latest: 0, rate: 0, rates: [] },
@@ -321,8 +323,13 @@ describe('PoolListComponent', () => {
       pool = _.merge(pool, {
         stats: {
           bytes_used: { latest: 5, rate: 0, rates: [] },
-          max_avail: { latest: 15, rate: 0, rates: [] },
-          rd_bytes: { latest: 6, rate: 4, rates: [[0, 2], [1, 6]] }
+          avail_raw: { latest: 15, rate: 0, rates: [] },
+          percent_used: { latest: 0.25, rate: 0, rates: [] },
+          rd_bytes: {
+            latest: 6,
+            rate: 4,
+            rates: [[0, 2], [1, 6]]
+          }
         },
         pg_status: { 'active+clean': 8, down: 2 }
       });
@@ -331,7 +338,8 @@ describe('PoolListComponent', () => {
           pg_status: '8 active+clean, 2 down',
           stats: {
             bytes_used: { latest: 5, rate: 0, rates: [] },
-            max_avail: { latest: 15, rate: 0, rates: [] },
+            avail_raw: { latest: 15, rate: 0, rates: [] },
+            percent_used: { latest: 0.25, rate: 0, rates: [] },
             rd_bytes: { latest: 6, rate: 4, rates: [2, 6] }
           },
           usage: 0.25

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.ts
@@ -239,7 +239,16 @@ export class PoolListComponent implements OnInit {
   }
 
   transformPoolsData(pools: any) {
-    const requiredStats = ['bytes_used', 'max_avail', 'rd_bytes', 'wr_bytes', 'rd', 'wr'];
+    const requiredStats = [
+      'bytes_used',
+      'max_avail',
+      'avail_raw',
+      'percent_used',
+      'rd_bytes',
+      'wr_bytes',
+      'rd',
+      'wr'
+    ];
     const emptyStat = { latest: 0, rate: 0, rates: [] };
 
     _.forEach(pools, (pool: Pool) => {
@@ -249,8 +258,7 @@ export class PoolListComponent implements OnInit {
         stats[stat] = pool.stats && pool.stats[stat] ? pool.stats[stat] : emptyStat;
       });
       pool['stats'] = stats;
-      const avail = stats.bytes_used.latest + stats.max_avail.latest;
-      pool['usage'] = avail > 0 ? stats.bytes_used.latest / avail : avail;
+      pool['usage'] = stats.percent_used.latest;
 
       if (
         !pool.cdExecuting &&

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-stat.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-stat.ts
@@ -7,6 +7,8 @@ export class PoolStat {
 export class PoolStats {
   bytes_used?: PoolStat;
   max_avail?: PoolStat;
+  avail_raw?: PoolStat;
+  percent_used?: PoolStat;
   rd_bytes?: PoolStat;
   wr_bytes?: PoolStat;
   rd?: PoolStat;

diff --git a/...ybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.html b/...ybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.html
@@ -17,7 +17,7 @@
   <div class="progress-bar progress-bar-info"
        role="progressbar"
        [style.width]="usedPercentage + '%'">
-    <span>{{ usedPercentage }}%</span>
+    <span>{{ usedPercentage | number: '1.0-' + decimals }}%</span>
   </div>
   <div class="progress-bar progress-bar-freespace"
        role="progressbar"

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.ts
@@ -10,6 +10,8 @@ export class UsageBarComponent implements OnChanges {
   totalBytes: number;
   @Input()
   usedBytes: number;
+  @Input()
+  decimals = 0;
 
   usedPercentage: number;
   freePercentage: number;
@@ -18,7 +20,7 @@ export class UsageBarComponent implements OnChanges {
   constructor() {}
 
   ngOnChanges() {
-    this.usedPercentage = Math.round((this.usedBytes / this.totalBytes) * 100);
+    this.usedPercentage = this.totalBytes > 0 ? (this.usedBytes / this.totalBytes) * 100 : 0;
     this.freePercentage = 100 - this.usedPercentage;
     this.freeBytes = this.totalBytes - this.usedBytes;
   }