[feature] Added organization filter to timeseries charts #532

- Added option to define "summary_query" for metrics
- Added functionality to handle "GROUP by tags" clause to InfluxDB
  client

Related to #532

openwisp_monitoring/db/backends/influxdb/client.py

@@ -256,7 +256,24 @@ def read(self, key, fields, tags, **kwargs):
         return list(self.query(q, precision='s').get_points())
 
     def get_list_query(self, query, precision='s'):
-        return list(self.query(query, precision=precision).get_points())
+        result = self.query(query, precision=precision)
+        if not len(result.keys()) or result.keys()[0][1] is None:
+            return list(result.get_points())
+        # Handles query which contains "GROUP BY TAG" clause
+        result_points = {}
+        for (measurement, tag), group_points in result.items():
+            tag_suffix = '_'.join(tag.values())
+            for point in group_points:
+                values = {}
+                for key, value in point.items():
+                    if key != 'time':
+                        values[f'{tag_suffix}'] = value
+                values['time'] = point['time']
+                try:
+                    result_points[values['time']].update(values)
+                except KeyError:
+                    result_points[values['time']] = values
+        return list(result_points.values())
 
     @retry
     def get_list_retention_policies(self):
@@ -329,7 +346,12 @@ def get_query(
             query = f'{query} LIMIT 1'
         return f"{query} tz('{timezone}')"
 
-    _group_by_regex = re.compile(r'GROUP BY time\(\w+\)', flags=re.IGNORECASE)
+    _group_by_time_tag_regex = re.compile(
+        r'GROUP BY ((time\(\w+\))(?:,\s+\w+)?)', flags=re.IGNORECASE
+    )
+    _group_by_time_regex = re.compile(r'GROUP BY time\(\w+\)\s?', flags=re.IGNORECASE)
+    _time_regex = re.compile(r'time\(\w+\)\s?', flags=re.IGNORECASE)
+    _time_comma_regex = re.compile(r'time\(\w+\),\s?', flags=re.IGNORECASE)
 
     def _group_by(self, query, time, chart_type, group_map, strip=False):
         if not self.validate_query(query):
@@ -343,7 +365,27 @@ def _group_by(self, query, time, chart_type, group_map, strip=False):
         if 'GROUP BY' not in query.upper():
             query = f'{query} {group_by}'
         else:
-            query = re.sub(self._group_by_regex, group_by, query)
+            # The query could have GROUP BY clause for a TAG
+            if group_by:
+                # The query already contains "GROUP BY", therefore
+                # we remove it from the "group_by" to avoid duplicating
+                # "GROUP BY"
+                group_by = group_by.replace('GROUP BY ', '')
+                # We only need to substitute the time function.
+                # The resulting query would be "GROUP BY time(<group_by>), <tag>"
+                query = re.sub(self._time_regex, group_by, query)
+            else:
+                # The query should not include the "GROUP by time()"
+                matches = re.search(self._group_by_time_tag_regex, query)
+                group_by_fields = matches.group(1)
+                if len(group_by_fields.split(',')) > 1:
+                    # If the query has "GROUP BY time(), tag",
+                    # then return "GROUP BY tag"
+                    query = re.sub(self._time_comma_regex, '', query)
+                else:
+                    # If the query has only has "GROUP BY time()",
+                    # then remove the "GROUP BY" clause
+                    query = re.sub(self._group_by_time_regex, '', query)
         return query
 
     _fields_regex = re.compile(

openwisp_monitoring/db/backends/influxdb/tests.py

@@ -190,6 +190,48 @@ def test_get_query_30d(self):
         self.assertIn(str(last30d)[0:10], q)
         self.assertIn('group by time(24h)', q.lower())
 
+    def test_group_by_tags(self):
+        self.assertEqual(
+            timeseries_db._group_by(
+                'SELECT COUNT(item) FROM measurement GROUP BY time(1d)',
+                time='30d',
+                chart_type='stackedbar+lines',
+                group_map={'30d': '30d'},
+                strip=False,
+            ),
+            'SELECT COUNT(item) FROM measurement GROUP BY time(30d)',
+        )
+        self.assertEqual(
+            timeseries_db._group_by(
+                'SELECT COUNT(item) FROM measurement GROUP BY time(1d)',
+                time='30d',
+                chart_type='stackedbar+lines',
+                group_map={'30d': '30d'},
+                strip=True,
+            ),
+            'SELECT COUNT(item) FROM measurement ',
+        )
+        self.assertEqual(
+            timeseries_db._group_by(
+                'SELECT COUNT(item) FROM measurement GROUP BY time(1d), tag',
+                time='30d',
+                chart_type='stackedbar+lines',
+                group_map={'30d': '30d'},
+                strip=False,
+            ),
+            'SELECT COUNT(item) FROM measurement GROUP BY time(30d), tag',
+        )
+        self.assertEqual(
+            timeseries_db._group_by(
+                'SELECT COUNT(item) FROM measurement GROUP BY time(1d), tag',
+                time='30d',
+                chart_type='stackedbar+lines',
+                group_map={'30d': '30d'},
+                strip=True,
+            ),
+            'SELECT COUNT(item) FROM measurement GROUP BY tag',
+        )
+
     def test_retention_policy(self):
         manage_short_retention_policy()
         manage_default_retention_policy()

openwisp_monitoring/device/apps.py

@@ -411,8 +411,11 @@ def register_dashboard_items(self):
                     'monitoring/css/percircle.min.css',
                     'monitoring/css/chart.css',
                     'monitoring/css/dashboard-chart.css',
+                    'admin/css/vendor/select2/select2.min.css',
+                    'admin/css/autocomplete.css',
                 ),
                 'js': (
+                    'admin/js/vendor/select2/select2.full.min.js',
                     'monitoring/js/lib/moment.min.js',
                     'monitoring/js/lib/daterangepicker.min.js',
                     'monitoring/js/lib/percircle.min.js',

openwisp_monitoring/monitoring/api/views.py

@@ -166,5 +166,33 @@ def invalidate_cache(cls, instance, *args, **kwargs):
                 return
         cls._get_charts.invalidate()
 
+    def _get_user_managed_orgs(self, request):
+        """
+        Return list of dictionary containing organization name and slug
+        in select2 compatible format.
+        """
+        orgs = []
+        qs = Organization.objects.only('slug', 'name')
+        if not request.user.is_superuser:
+            if len(request.user.organizations_managed) > 1:
+                qs = qs.filter(pk__in=request.user.organizations_managed)
+            else:
+                return orgs
+        for org in qs.iterator():
+            orgs.append({'id': org.slug, 'text': org.name})
+        if len(orgs) < 2:
+            # Handles scenarios for superuser when the project has only
+            # one organization.
+            return []
+        return orgs
+
+    def get(self, request, *args, **kwargs):
+        response = super().get(request, *args, **kwargs)
+        if not request.GET.get('csv'):
+            user_managed_orgs = self._get_user_managed_orgs(request)
+            if user_managed_orgs:
+                response.data['organizations'] = user_managed_orgs
+        return response
+
 
 dashboard_timeseries = DashboardTimeseriesView.as_view()

openwisp_monitoring/monitoring/base/models.py

@@ -593,6 +593,12 @@ def query(self):
             return query[timeseries_db.backend_name]
         return self._default_query
 
+    @property
+    def summary_query(self):
+        query = self.config_dict.get('summary_query', None)
+        if query:
+            return query[timeseries_db.backend_name]
+
     @property
     def top_fields(self):
         return self.config_dict.get('top_fields', None)
@@ -651,6 +657,8 @@ def get_query(
         additional_params=None,
     ):
         query = query or self.query
+        if summary and self.summary_query:
+            query = self.summary_query
         additional_params = additional_params or {}
         params = self._get_query_params(time, start_date, end_date)
         params.update(additional_params)

openwisp_monitoring/monitoring/static/monitoring/js/chart-utils.js

@@ -194,6 +194,10 @@ django.jQuery(function ($) {
           var timezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
           url = `${apiUrl}?timezone=${timezone}&start=${startDate}&end=${endDate}`;
         }
+        if ($('#org-selector').val()) {
+          var orgSlug = $('#org-selector').val();
+          url = `${url}&organization_slug=${orgSlug}`;
+        }
         return url;
       },
       createCharts = function (data){
@@ -210,6 +214,22 @@ django.jQuery(function ($) {
           createChart(chart, data.x, htmlId, chart.title, chart.type, chartQuickLink);
         });
       },
+      addOrganizationSelector = function (data) {
+        var orgSelector = $('#org-selector');
+        if (data.organizations === undefined) {
+          orgSelector.hide();
+          return;
+        }
+        if (orgSelector.data('select2-id') === 'org-selector') {
+          return;
+        }
+        orgSelector.select2({
+          data: data.organizations,
+          allowClear: true,
+          placeholder: gettext('Organization Filter')
+        });
+        orgSelector.show();
+      },
       loadCharts = function (time, showLoading) {
         $.ajax(getChartFetchUrl(time), {
           dataType: 'json',
@@ -230,6 +250,7 @@ django.jQuery(function ($) {
               fallback.show();
             }
             createCharts(data);
+            addOrganizationSelector(data);
           },
           error: function () {
             alert('Something went wrong while loading the charts');
@@ -255,7 +276,7 @@ django.jQuery(function ($) {
       var range = localStorage.getItem(timeRangeKey) || defaultTimeRange;
       var startLabel = localStorage.getItem(startDayKey) || moment().format('MMMM D, YYYY');
       var endLabel = localStorage.getItem(endDayKey) || moment().format('MMMM D, YYYY');
-      
+
       // Disable the zoom chart and scrolling when we refresh the page
       localStorage.setItem(isChartZoomScroll, false);
       localStorage.setItem(isChartZoomed, false);
@@ -314,20 +335,27 @@ django.jQuery(function ($) {
     });
     // bind export button
     $('#ow-chart-time a.export').click(function () {
-      var time = localStorage.getItem(timeRangeKey);
-      location.href = baseUrl + time + '&csv=1';
+      var queryString,
+        queryParams = {'csv': 1};
+        queryParams.time = localStorage.getItem(timeRangeKey);
       // If custom or pickerChosenLabelKey is 'Custom Range', pass pickerEndDate and pickerStartDate to csv url
       if (localStorage.getItem(isCustomDateRange) === 'true' || localStorage.getItem(pickerChosenLabelKey) === customDateRangeLabel) {
-      var startDate = localStorage.getItem(startDateTimeKey);
-      var endDate = localStorage.getItem(endDateTimeKey);
-      if (localStorage.getItem(isChartZoomed) === 'true') {
-        time = localStorage.getItem(zoomtimeRangeKey);
-        endDate = localStorage.getItem(zoomEndDateTimeKey);
-        startDate = localStorage.getItem(zoomStartDateTimeKey);
+        queryParams.start = localStorage.getItem(startDateTimeKey);
+        queryParams.end = localStorage.getItem(endDateTimeKey);
+        if (localStorage.getItem(isChartZoomed) === 'true') {
+          queryParams.time = localStorage.getItem(zoomtimeRangeKey);
+          queryParams.end = localStorage.getItem(zoomEndDateTimeKey);
+          queryParams.start = localStorage.getItem(zoomStartDateTimeKey);
+        }
+        queryParams.timezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
       }
-      var timezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
-      location.href = `${apiUrl}?timezone=${timezone}&start=${startDate}&end=${endDate}&csv=1`;
+      if ($('#org-selector').val()) {
+        queryParams.organization_slug = $('#org-selector').val();
       }
+      queryString = Object.keys(queryParams)
+        .map(key => `${encodeURIComponent(key)}=${encodeURIComponent(queryParams[key])}`)
+        .join('&');
+      location.href = `${apiUrl}?${queryString}`;
     });
     // fetch chart data and replace the old charts with the new ones
     function loadFetchedCharts(time){
@@ -344,5 +372,12 @@ django.jQuery(function ($) {
         },
       });
     }
+
+    $('#org-selector').change(function(){
+      loadCharts(
+        localStorage.getItem(timeRangeKey) || defaultTimeRange,
+        true
+      );
+    });
   });
 }(django.jQuery));

openwisp_monitoring/monitoring/templates/monitoring/paritals/chart.html

@@ -12,6 +12,9 @@
     <a id="daterangepicker-widget">
         <span></span>
     </a>
+    <select name="org-selector" id="org-selector" class="hide">
+        <option></option>
+    </select>
     <div id="chart-loading-overlay"><div class="ow-loading-spinner"></div></div>
     <div id="ow-chart-contents"></div>
     <div id="ow-chart-fallback" class="form-row">

openwisp_monitoring/monitoring/tests/__init__.py

@@ -136,6 +136,25 @@
             )
         },
     },
+    'group_by_tag': {
+        'type': 'stackedbars',
+        'title': 'Group by tag',
+        'description': 'Query is groupped by tag along with time',
+        'unit': 'n.',
+        'order': 999,
+        'query': {
+            'influxdb': (
+                "SELECT CUMULATIVE_SUM(SUM({field_name})) FROM {key} WHERE time >= '{time}'"
+                " GROUP BY time(1d), metric_num"
+            )
+        },
+        'summary_query': {
+            'influxdb': (
+                "SELECT SUM({field_name}) FROM {key} WHERE time >= '{time}'"
+                " GROUP BY time(30d), metric_num"
+            )
+        },
+    },
     'mean_test': {
         'type': 'line',
         'title': 'Mean test',