Merge pull request #1625 from MFlyer/issue#1412_Shares_usage_enhancem…

…ents

Share Usage reporting right sizes

src/rockstor/fs/btrfs.py

@@ -731,45 +731,60 @@ def update_quota(pool, qgroup, size_bytes):
     return run_command(cmd, log=True)
 
 
-def share_usage(pool, share_id):
+def volume_usage(pool, volume_id, pvolume_id=None):
+
     """
-    Return the sum of the qgroup sizes of this share and any child subvolumes
-    N.B. qgroupid defaults to a unique identifier of the form 0/<subvolume id>
+    New function to collect volumes rusage and eusage instead of share_usage
+    plus parent rusage and eusage (2015/* qgroup)
     """
-    # Obtain path to share in pool
+    # Obtain path to share in pool, this preserved because
+    # granting pool exists
     root_pool_mnt = mount_root(pool)
     cmd = [BTRFS, 'subvolume', 'list', root_pool_mnt]
     out, err, rc = run_command(cmd, log=True)
-    short_id = share_id.split('/')[1]
-    share_dir = ''
+    short_id = volume_id.split('/')[1]
+    volume_dir = ''
+
     for line in out:
         fields = line.split()
         if (len(fields) > 0 and short_id in fields[1]):
-            share_dir = root_pool_mnt + '/' + fields[8]
+            volume_dir = root_pool_mnt + '/' + fields[8]
             break
 
-    # Obtain list of child subvolume qgroups
-    cmd = [BTRFS, 'subvolume', 'list', '-o', share_dir]
-    out, err, rc = run_command(cmd, log=True)
-    qgroups = [short_id]
-    for line in out:
-        fields = line.split()
-        if (len(fields) > 0):
-            qgroups.append(fields[1])
+    """
+    Rockstor volume/subvolume hierarchy is not standard
+    and Snapshots actually not always under Share but on Pool,
+    so btrf sub list -o deprecated because won't always return
+    expected data; volumes (shares & snapshots) sizes got via qgroups.
+    Rockstor structure has default share qgroup 0/* becoming child of
+    2015/* new qgroup and share snapshots 0/*+1 qgroups assigned to new
+    Rockstor 2015/*.
+    Original 0/* qgroup returns current share content size,
+    2015/* qgroup returns 'real' share size considering snapshots sizes too
+    Note: 2015/* rfer and excl sizes are always equal so to compute
+    current real size we can indistinctly use one of them.
+    """
 
-    # Sum qgroup sizes
-    cmd = [BTRFS, 'qgroup', 'show', share_dir]
+    cmd = [BTRFS, 'qgroup', 'show', volume_dir]
     out, err, rc = run_command(cmd, log=True)
     rusage = eusage = 0
+    pqgroup_rusage = pqgroup_eusage = 0
+    share_sizes = []
+
     for line in out:
         fields = line.split()
-        qgroup = []
         if (len(fields) > 0 and '/' in fields[0]):
-            qgroup = fields[0].split('/')
-        if (len(qgroup) > 0 and qgroup[1] in qgroups):
-            rusage += convert_to_kib(fields[1])
-            eusage += convert_to_kib(fields[2])
-    return (rusage, eusage)
+            qgroup = fields[0]
+            if (qgroup == volume_id):
+                rusage = convert_to_kib(fields[1])
+                eusage = convert_to_kib(fields[2])
+                share_sizes.extend((rusage, eusage))
+            if (pvolume_id is not None and qgroup == pvolume_id):
+                pqgroup_rusage = convert_to_kib(fields[1])
+                pqgroup_eusage = convert_to_kib(fields[2])
+                share_sizes.extend((pqgroup_rusage, pqgroup_eusage))
+
+    return share_sizes
 
 
 def shares_usage(pool, share_map, snap_map):

src/rockstor/fs/tests/test_btrfs.py

@@ -1,5 +1,5 @@
 """
-Copyright (c) 2012-2013 RockStor, Inc. <http://rockstor.com>
+Copyright (c) 2012-2017 RockStor, Inc. <http://rockstor.com>
 This file is part of RockStor.
 RockStor is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published
@@ -14,7 +14,7 @@
 """
 
 import unittest
-from fs.btrfs import (pool_raid, is_subvol, share_usage, balance_status,
+from fs.btrfs import (pool_raid, is_subvol, volume_usage, balance_status,
                       share_id)
 from mock import patch
 
@@ -185,17 +185,17 @@ def test_is_subvol_nonexistent(self):
     #     self.assertFalse(is_subvol(mount_point),
     #                  msg='Did NOT return False for exception')
 
-    def test_share_usage(self):
+    def test_volume_usage(self):
         """
-        Moc the return value of "btrfs qgroup show pool_mount_point" to assess
-        test_share's parsing capabilities to extract rfer and excl subvol usage
-        information.
+        Moc the return value of "btrfs qgroup show share_mount_pt" to assess
+        to extract rfer and excl usage for original 0/* qgroup and Rockstor
+        ad hoc 2015/* qgroup.
         :return:
         """
-        # share_usage() CALLED WITH pool name of=test-pool and share_id=0/285
+        # volume_usage() called with pool name of=test-pool volume_id=0/261
+        # and new Rockstor qgroup pvolume_id=2015/4
         # mount_root(Pool object) returned /mnt2/test-pool
-        # share_usage cmd=['/sbin/btrfs', 'qgroup', 'show', u'/mnt2/test-pool']
-        # share_usage returning rusage=461404 and eusage=3512
+        # cmd=['/sbin/btrfs', 'qgroup', 'show', u'/mnt2/test-pool']
         #
         # Setup our calling variables and mock the root pool as mounted.
         o = ['qgroupid         rfer         excl ',
@@ -230,24 +230,33 @@ def test_share_usage(self):
              '2015/1          0.00B        0.00B ',
              '2015/2        2.04MiB      2.04MiB ',
              '2015/3        7.37GiB      7.37GiB ',
-             '2015/4       63.65MiB     63.65MiB ', '']
+             '2015/4       63.00MiB     63.00MiB ', '']
         e = ['']
         rc = 0
         # is_mounted returning True avoids mount command calls in mount_root()
-        mount_point = '/mnt2/test-mount'
+        mount_point = '/mnt2/test-pool'
         self.mock_mount_root.return_value = mount_point
         # setup the return values from our run_command wrapper
         # examples of output from /mnt2/test-pool from a real system install
         self.mock_run_command.return_value = (o, e, rc)
         # create a fake pool object
         pool = Pool(raid='raid0', name='test-pool')
-        # and fake share_id / qgroupid
-        share_id = '0/285'
-        # As share_usage uses convert_to_kib() everything is converted to KiB
+        # fake volume_id / qgroupid
+        volume_id = '0/261'
+        # and fake pvolume_id
+        pvolume_id = '2015/4'
+        # As volume_usage uses convert_to_kib() everything is converted to KiB
         # here we convert 450.59MiB and 3.43MiB to their KiB equivalent (x1024)
-        expected_results = (461404, 3512)
-        self.assertEqual(share_usage(pool, share_id), expected_results,
-                         msg='Failed to retrieve expected rfer and excl usage')
+        expected_results_share = [65177, 65177, 64512, 64512]
+        self.assertEqual(volume_usage(pool, volume_id, pvolume_id),
+                         expected_results_share,
+                         msg='Failed to retrieve share rfer and excl usage')
+        # We perform a test with snapshots volumes to, having pqgroup None
+        pvolume_id2 = None
+        expected_results_snapshot = [65177, 65177]
+        self.assertEqual(volume_usage(pool, volume_id, pvolume_id2),
+                         expected_results_snapshot,
+                         msg='Failed to retrieve snapshot rfer and excl usage')
 
 
 # TODO: add test_balance_status_finished

src/rockstor/storageadmin/migrations/0003_auto_20170114_1332.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('storageadmin', '0002_auto_20161125_0051'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='share',
+            name='pqgroup_eusage',
+            field=models.BigIntegerField(default=0),
+        ),
+        migrations.AddField(
+            model_name='share',
+            name='pqgroup_rusage',
+            field=models.BigIntegerField(default=0),
+        ),
+    ]

src/rockstor/storageadmin/models/share.py

@@ -1,5 +1,5 @@
 """
-Copyright (c) 2012-2013 RockStor, Inc. <http://rockstor.com>
+Copyright (c) 2012-2017 RockStor, Inc. <http://rockstor.com>
 This file is part of RockStor.
 
 RockStor is free software; you can redistribute it and/or modify
@@ -42,8 +42,15 @@ class Share(models.Model):
     subvol_name = models.CharField(max_length=4096)
     replica = models.BooleanField(default=False)
     compression_algo = models.CharField(max_length=1024, null=True)
+    # rusage and eusage reports original 0/x qgroup size
+    # and this has only current share content without snapshots
     rusage = models.BigIntegerField(default=0)
     eusage = models.BigIntegerField(default=0)
+    # Having Rockstor vol/subvols overriding btrfs standards
+    # with snapshots(subvols) not under their vols, we use qgroup sizes
+    # to report correct real vol sizes
+    pqgroup_rusage = models.BigIntegerField(default=0)
+    pqgroup_eusage = models.BigIntegerField(default=0)
 
     @property
     def size_gb(self):

src/rockstor/storageadmin/static/storageadmin/js/templates/share/shares_table.jst

@@ -13,8 +13,9 @@
         <th>Name</th>
         <th>Size</th>
         <th>Pool</th>
-        <th>Usage</th>
-	      <th>Compression algorithm</th>
+        <th>Usage <i class="fa fa-info-circle" title="Current share content"></th>
+		<th>Usage (Btrfs) <i class="fa fa-info-circle" title="Current share content considering snapshots too"></th>
+	    <th>Compression algorithm</th>
         <th>Actions</th>
       </tr>
     </thead>
@@ -24,7 +25,8 @@
         <td><a href="#shares/{{this.name}}"><i class="glyphicon glyphicon-folder-open"></i>  {{this.name}}</a></td>
         <td>{{humanize_size this.size}}</td>
         <td>{{this.pool.name}}</td>
-        <td>{{humanize_size this.eusage}}
+        <td>{{humanize_size this.rusage}}</td>
+		<td>{{humanize_size this.pqgroup_rusage}} {{checkUsage this.size this.pqgroup_rusage}}</td>
         <td>
         {{displayCompressionAlgo this.compression_algo this.name}}
         </td>

src/rockstor/storageadmin/static/storageadmin/js/views/dashboard/top_shares.js

@@ -3,7 +3,7 @@
  * @licstart  The following is the entire license notice for the
  * JavaScript code in this page.
  *
- * Copyright (c) 2012-2016 RockStor, Inc. <http://rockstor.com>
+ * Copyright (c) 2012-2017 RockStor, Inc. <http://rockstor.com>
  * This file is part of RockStor.
  *
  * RockStor is free software; you can redistribute it and/or modify
@@ -62,12 +62,15 @@ TopSharesWidget = RockStorWidgetView.extend({
 
         var _this = this;
         _this.data = this.shares.sortBy(function(s) {
-            return ((s.get('rusage') / s.get('size')) * 100);
+            return ((s.get('pqgroup_rusage') / s.get('size')) * 100);
         }).reverse().slice(0, this.numTop);
         _this.data.map(function(d) {
             d.set({
                 'pUsed': ((d.get('rusage') / d.get('size')) * 100)
             });
+            d.set({
+                'pOverUsed': (((d.get('pqgroup_rusage') - d.get('rusage')) / d.get('size')) * 100)
+            });
         });
     },
 
@@ -89,20 +92,25 @@ TopSharesWidget = RockStorWidgetView.extend({
         });
 
         this.$('.pused').each(function(index) {
-            $(this).text(_this.data[index].get('pUsed').toFixed(2) + '%');
+            var btrfs_size = (_this.data[index].get('pUsed') + _this.data[index].get('pOverUsed')).toFixed(2)
+            $(this).text(btrfs_size + '%');
         });
-		var truncate = _this.maximized ? 100 : 12;
-        this.$('.progress-animate').each(function(index) {
+        var truncate = _this.maximized ? 100 : 12;
+        this.$('.progress-animate').not('.progress-bar-info').each(function(index) {
             $(this).find('span')
                 .text(humanize.truncatechars(_this.data[index].get('name'), truncate) +
-                    '(' + humanize.filesize(_this.data[index].get('rusage') * 1024) +
+                    '(' + humanize.filesize(_this.data[index].get('pqgroup_rusage') * 1024) +
                     '/' + humanize.filesize(_this.data[index].get('size') * 1024) +
                     ')');
             $(this).animate({
                 width: _this.data[index].get('pUsed').toFixed(2) + '%'
             }, 1000);
         });
-
+        this.$('.progress-bar-info').each(function(index) {
+            $(this).animate({
+                width: _this.data[index].get('pOverUsed').toFixed(2) + '%'
+            }, 1000);
+        });
     },
 
     buildTitle: function() {
@@ -134,8 +142,11 @@ TopSharesWidget = RockStorWidgetView.extend({
         var html = '<div style="display: table;"><div class="' + percent_div['class'] + '" style="' + percent_div['style'] + '"></div>';
         html += '<div class="' + progressbar_container['class'] + '" style="' + progressbar_container['style'] + '">';
         html += '<div class="' + progressbars_defaults['class'] + '" style="' + progressbars_defaults['style'] + '" ';
-        html += 'role="' + progressbars_defaults['role'] + '">';
-        html += '<span style="' + progressbar_span['style'] + '"></span></div></div></div>';
+        html += 'role="' + progressbars_defaults['role'] + '">'
+        html += '<span style="' + progressbar_span['style'] + '"></span></div>';
+        html += '<div class="' + progressbars_defaults['class'] + ' progress-bar-info" style="' + progressbars_defaults['style'] + '" ';
+        html += 'role="' + progressbars_defaults['role'] + '"></div>';
+        html += '</div></div>';
 
         return html;
     },

src/rockstor/storageadmin/static/storageadmin/js/views/shares.js

@@ -3,7 +3,7 @@
  * @licstart  The following is the entire license notice for the
  * JavaScript code in this page.
  *
- * Copyright (c) 2012-2013 RockStor, Inc. <http://rockstor.com>
+ * Copyright (c) 2012-2017 RockStor, Inc. <http://rockstor.com>
  * This file is part of RockStor.
  *
  * RockStor is free software; you can redistribute it and/or modify
@@ -111,7 +111,7 @@ SharesView = RockstorLayoutView.extend({
 				enableButton(button);
 				_this.$('#delete-share-modal').modal('hide');
 				$('.modal-backdrop').remove();
-				app_router.navigate('shares', {trigger: true})
+				app_router.navigate('shares', {trigger: true});
 			},
 			error: function(xhr, status, error) {
 				enableButton(button);
@@ -120,7 +120,7 @@ SharesView = RockstorLayoutView.extend({
 	},
 	cancel: function(event) {
 		if (event) event.preventDefault();
-		app_router.navigate('shares', {trigger: true})
+		app_router.navigate('shares', {trigger: true});
 	},
 
 	initHandlebarHelpers: function(){
@@ -133,7 +133,7 @@ SharesView = RockstorLayoutView.extend({
 			var html = '';
 			if(shareCompression && shareCompression != 'no'){
 				html += shareCompression;
-			}else{
+			} else {
 				html += 'None(defaults to pool level compression, if any)   ' +
 				'<a href="#shares/' + shareName + '/?cView=edit"><i class="glyphicon glyphicon-pencil"></i></a>';
 			}
@@ -150,5 +150,25 @@ SharesView = RockstorLayoutView.extend({
             }
             return false;
         });
-	}
+
+        Handlebars.registerHelper('checkUsage', function(size, btrfs_usage) {
+
+            // We don't have share size enforcement with btrfs qgroup limit
+            // but with this we help users to start gettting used to it.
+            // Current warning levels are btrfs usage > 70% warning
+            // btrfs usage > 80% alert
+            var html, warning = '';
+            var usage = (btrfs_usage / size).toFixed(4);
+            if (usage >= 0.8) {
+                warning = 'text-danger';
+            } else if (usage >= 0.7){
+                warning = 'text-warning';
+            }
+            if (warning !=='') {
+                html = '<i class="fa fa-warning fa-lg ' + warning;
+                html += '" title="Usage is ' + usage * 100 + '% of share size">';
+                return new Handlebars.SafeString(html);
+            }
+        });
+    }
 });

src/rockstor/storageadmin/views/share.py

@@ -23,7 +23,7 @@
 from storageadmin.models import (Share, Pool, Snapshot, NFSExport, SambaShare,
                                  SFTP)
 from smart_manager.models import Replica
-from fs.btrfs import (add_share, remove_share, update_quota, share_usage,
+from fs.btrfs import (add_share, remove_share, update_quota, volume_usage,
                       set_property, mount_share, qgroup_id, qgroup_create)
 from system.osi import is_share_mounted
 from system.services import systemctl
@@ -210,7 +210,7 @@ def put(self, request, sname):
             if ('size' in request.data):
                 new_size = self._validate_share_size(request, share.pool)
                 qid = qgroup_id(share.pool, share.subvol_name)
-                cur_rusage, cur_eusage = share_usage(share.pool, qid)
+                cur_rusage, cur_eusage = volume_usage(share.pool, qid)
                 if (new_size < cur_rusage):
                     e_msg = ('Unable to resize because requested new '
                              'size(%dKB) is less than current usage(%dKB)'