Merge pull request #3060 from SEED-platform/feat/inventory-column-fil…

…tering

Feat/inventory column filtering and sorting

seed/search.py

@@ -7,14 +7,16 @@
 Search methods pertaining to buildings.
 
 """
+from datetime import datetime
 import json
 import logging
 import operator
+from typing import Callable, Union
 
 from functools import reduce
 
 from django.db.models import Q
-from django.http.request import RawPostDataException
+from django.http.request import RawPostDataException, QueryDict
 from past.builtins import basestring
 
 from seed.lib.superperms.orgs.models import Organization
@@ -301,3 +303,146 @@ def inventory_search_filter_sort(inventory_type, params, user):
         )
 
     return inventory
+
+
+class FilterException(Exception):
+    pass
+
+
+def _parse_view_filter(filter_expression: str, filter_value: str, columns_by_name: dict[str, dict]) -> Q:
+    """Parse a filter expression into a Q object
+
+    :param filter_expression: should be a valid Column.column_name, with an optional
+                              Django field lookup suffix (e.g. `__gt`, `__icontains`, etc)
+                              https://docs.djangoproject.com/en/4.0/topics/db/queries/#field-lookups
+                              One custom field lookup suffix is allowed, `__ne`,
+                              which negates the expression (i.e. column_name != filter_value)
+    :param filter_value: the value evaluated against the filter_expression
+    :param columns_by_name: mapping of Column.column_name to dict representation of Column
+    :return: query object
+    """
+    data_type_parsers: dict[str, Callable] = {
+        'number': float,
+        'float': float,
+        'integer': int,
+        'string': str,
+        'geometry': str,
+        'datetime': datetime.fromisoformat,
+        'date': datetime.fromisoformat,
+        'boolean': lambda v: v.lower() == 'true',
+        'area': float,
+        'eui': float,
+    }
+
+    column_name, _, _ = filter_expression.partition('__')
+    column = columns_by_name.get(column_name)
+    if column is None:
+        return Q()
+
+    # users indicate negation with a trailing `__ne` (which is not a real Django filter)
+    # so we need to remove it if found
+    is_negated = filter_expression.endswith('__ne')
+    if is_negated:
+        filter_expression, _, _ = filter_expression.rpartition('__ne')
+
+    new_filter_expression = None
+    if column_name == 'campus':
+        # campus is the only column found on the canonical property (TaxLots don't have this column)
+        # all other columns are found in the state
+        new_filter_expression = f'property__{filter_expression}'
+    elif column['is_extra_data']:
+        new_filter_expression = f'state__extra_data__{filter_expression}'
+    else:
+        new_filter_expression = f'state__{filter_expression}'
+
+    parser = data_type_parsers.get(column['data_type'], str)
+    try:
+        new_filter_value = parser(filter_value)
+    except Exception:
+        raise FilterException(f'Invalid data type for "{column_name}". Expected a valid {column["data_type"]} value.')
+
+    new_filter_dict = {new_filter_expression: new_filter_value}
+    if is_negated:
+        return ~Q(**new_filter_dict)
+    else:
+        return Q(**new_filter_dict)
+
+
+def _parse_view_sort(sort_expression: str, columns_by_name: dict[str, dict]) -> Union[None, str]:
+    """Parse a sort expression
+
+    :param sort_expression: should be a valid Column.column_name. Optionally prefixed
+                            with '-' to indicate descending order.
+    :param columns_by_name: mapping of Column.column_name to dict representation of Column
+    :return: the parsed sort expression or None if not valid
+    """
+    column_name = sort_expression.lstrip('-')
+    direction = '-' if sort_expression.startswith('-') else ''
+    if column_name == 'id':
+        return sort_expression
+    elif column_name == 'campus':
+        # campus is the only column which is found exclusively on the Property, not the state
+        return f'property__{sort_expression}'
+    elif column_name in columns_by_name:
+        column = columns_by_name[column_name]
+        if column['is_extra_data']:
+            return f'{direction}state__extra_data__{column_name}'
+        else:
+            return f'{direction}state__{column_name}'
+    else:
+        return None
+
+
+def build_view_filters_and_sorts(filters: QueryDict, columns: list[dict]) -> tuple[Q, list[str]]:
+    """Build a query object usable for `*View.filter(...)` as well as a list of
+    column names for usable for `*View.order_by(...)`.
+
+    Filters are specified in a similar format as Django queries, as `column_name`
+    or `column_name__lookup`, where `column_name` is a valid Column.column_name,
+    and `__lookup` (which is optional) is any valid Django field lookup:
+      https://docs.djangoproject.com/en/4.0/topics/db/queries/#field-lookups
+
+    One special lookup which is not provided by Django is `__ne` which negates
+    the filter expression.
+
+    Query string examples:
+    - `?city=Denver` - inventory where City is Denver
+    - `?city__ne=Denver` - inventory where City is NOT Denver
+    - `?site_eui__gte=100` - inventory where Site EUI >= 100
+    - `?city=Denver&site_eui__gte=100` - inventory where City is Denver AND Site EUI >= 100
+    - `?my_custom_column__lt=1000` - inventory where the extra data field `my_custom_column` < 1000
+
+    Sorts are specified with the `order_by` parameter, with any valid Column.column_name
+    as the value. By default the column is sorted in ascending order, columns prefixed
+    with `-` will be sorted in descending order.
+
+    Query string examples:
+    - `?order_by=site_eui` - sort by Site EUI in ascending order
+    - `?order_by=-site_eui` - sort by Site EUI in descending order
+    - `?order_by=city&order_by=site_eui` - sort by City, then Site EUI
+
+    This function basically does the following:
+    - Ignore any filter/sort that doesn't have a corresponding column
+    - Handle cases for extra data
+    - Convert filtering values into their proper types (e.g. str -> int)
+
+    :param filters: QueryDict from a request
+    :param columns: list of all valid Columns in dict format
+    :return: filters and sorts
+    """
+    columns_by_name = {
+        c['column_name']: c
+        for c in columns
+    }
+
+    new_filters = Q()
+    for filter_expression, filter_value in filters.items():
+        new_filters &= _parse_view_filter(filter_expression, filter_value, columns_by_name)
+
+    order_by = []
+    for sort_expression in filters.getlist('order_by', ['id']):
+        parsed_sort = _parse_view_sort(sort_expression, columns_by_name)
+        if parsed_sort is not None:
+            order_by.append(parsed_sort)
+
+    return new_filters, order_by

seed/static/seed/js/controllers/inventory_list_beta_controller.js

@@ -118,6 +118,10 @@ angular.module('BE.seed.controller.inventory_list_beta', [])
 
       $scope.restoring = false;
 
+      // stores columns that have filtering and/or sorting applied
+      $scope.column_filters = []
+      $scope.column_sorts = []
+
       // Find labels that should be displayed and organize by applied inventory id
       $scope.show_labels_by_inventory_id = {};
       $scope.build_labels = function() {
@@ -599,20 +603,10 @@ angular.module('BE.seed.controller.inventory_list_beta', [])
         // Modify misc
         if (col.data_type === 'datetime') {
           options.cellFilter = 'date:\'yyyy-MM-dd h:mm a\'';
-          options.filter = inventory_service.dateFilter();
-        } else if (col.data_type === 'date') {
-          options.filter = inventory_service.dateFilter();
         } else if (col.data_type === 'eui' || col.data_type === 'area') {
-          options.filter = inventory_service.combinedFilter();
           options.cellFilter = 'number: ' + $scope.organization.display_decimal_places;
-          options.sortingAlgorithm = naturalSort;
         } else if (col.data_type === 'float' || col.is_derived_column) {
-          options.filter = inventory_service.combinedFilter();
           options.cellFilter = 'number: ' + $scope.organization.display_decimal_places;
-          options.sortingAlgorithm = naturalSort;
-        } else {
-          options.filter = inventory_service.combinedFilter();
-          options.sortingAlgorithm = naturalSort;
         }
 
         if (col.column_name === 'number_properties' && col.related) options.treeAggregationType = 'total';
@@ -771,6 +765,7 @@ angular.module('BE.seed.controller.inventory_list_beta', [])
         } else if ($scope.inventory_type === 'taxlots') {
           fn = inventory_service.get_taxlots;
         }
+
         return fn(
           page,
           chunk,
@@ -780,6 +775,8 @@ angular.module('BE.seed.controller.inventory_list_beta', [])
           true,
           $scope.organization.id,
           false,
+          $scope.column_filters,
+          $scope.column_sorts,
         ).then(function (data) {
           return data;
         });
@@ -832,14 +829,20 @@ angular.module('BE.seed.controller.inventory_list_beta', [])
       $scope.load_inventory = function (page) {
         const page_size = 100;
         spinner_utility.show()
-        return fetch(page, page_size).then(function (data) {
-          $scope.inventory_pagination = data.pagination;
-          processData(data.results);
-          $scope.gridApi.core.notifyDataChange(uiGridConstants.dataChange.EDIT);
-          modalInstance.close();
-          evaluateDerivedColumns();
-          spinner_utility.hide()
-        });
+        return fetch(page, page_size)
+          .then(function (data) {
+            if (data.status === 'error') {
+              Notification.error(data.message);
+              spinner_utility.hide();
+              return;
+            }
+            $scope.inventory_pagination = data.pagination;
+            processData(data.results);
+            $scope.gridApi.core.notifyDataChange(uiGridConstants.dataChange.EDIT);
+            modalInstance.close();
+            evaluateDerivedColumns();
+            spinner_utility.hide()
+          });
       };
 
       $scope.update_cycle = function (cycle) {
@@ -1133,14 +1136,104 @@ angular.module('BE.seed.controller.inventory_list_beta', [])
         }
       };
 
-      var saveGridSettings = function () {
+      // https://regexr.com/6cka2
+      const combinedRegex = /^(!?)=\s*(-?\d+(?:\\\.\d+)?)$|^(!?)=?\s*"((?:[^"]|\\")*)"$|^(<=?|>=?)\s*((-?\d+(?:\\\.\d+)?)|(\d{4}-\d{2}-\d{2}))$/;
+      const parseFilter = function (expression) {
+        // parses an expression string into an object containing operator and value
+        const filterData = expression.match(combinedRegex);
+        if (filterData) {
+          if (!_.isUndefined(filterData[2])) {
+            // Numeric Equality
+            operator = filterData[1];
+            value = Number(filterData[2].replace('\\.', '.'));
+            if (operator === '!') {
+              return {operator: 'ne', value};
+            } else {
+              return {operator: 'exact', value};
+            }
+          } else if (!_.isUndefined(filterData[4])) {
+            // Text Equality
+            operator = filterData[3];
+            value = filterData[4];
+            if (operator === '!') {
+              return {operator: 'ne', value};
+            } else {
+              return {operator: 'exact', value};
+            }
+          } else if (!_.isUndefined(filterData[7])) {
+            // Numeric Comparison
+            operator = filterData[5];
+            value = Number(filterData[6].replace('\\.', '.'));
+            switch (operator) {
+              case '<':
+                return {operator: 'lt', value};
+              case '<=':
+                return {operator: 'lte', value};
+              case '>':
+                return {operator: 'gt', value};
+              case '>=':
+                return {operator: 'gte', value};
+            }
+          } else {
+            // Date Comparison
+            operator = filterData[5];
+            value = filterData[8];
+            switch (operator) {
+              case '<':
+                return {operator: 'lt', value};
+              case '<=':
+                return {operator: 'lte', value};
+              case '>':
+                return {operator: 'gt', value};
+              case '>=':
+                return {operator: 'gte', value};
+            }
+          }
+        } else {
+          // Case-insensitive Contains
+          return {operator: 'icontains', value: expression}
+        }
+      }
+
+      var updateColumnFilterSort = function () {
         if (!$scope.restoring) {
           var columns = _.filter($scope.gridApi.saveState.save().columns, function (col) {
-            return _.keys(col.sort).length + (_.get(col, 'filters[0].term', '') || '').length > 0;
+            return _.keys(col.sort).filter(key => key != 'ignoreSort').length + (_.get(col, 'filters[0].term', '') || '').length > 0;
           });
+
           inventory_service.saveGridSettings(localStorageKey + '.sort', {
             columns: columns
           });
+
+          $scope.column_filters = []
+          $scope.column_sorts = []
+          // parse the filters and sorts
+          for (const column of columns) {
+            const {name, filters, sort} = column;
+            // remove the column id at the end of the name
+            const column_name = name.split("_").slice(0, -1).join("_");
+
+            for (const filter of filters) {
+              if (_.isEmpty(filter)) {
+                continue
+              }
+
+              // a filter can contain many comma-separated filters
+              const subFilters = _.map(_.split(filter.term, ','), _.trim);
+              for (const subFilter of subFilters) {
+                if (subFilter) {
+                  const {operator, value} = parseFilter(subFilter)
+                  $scope.column_filters.push({column_name, operator, value})
+                }
+              }
+            }
+
+            if (sort.direction) {
+              // remove the column id at the end of the name
+              const column_name = name.split("_").slice(0, -1).join("_");
+              $scope.column_sorts.push({column_name, direction: sort.direction});
+            }
+          }
         }
       };
 
@@ -1177,6 +1270,8 @@ angular.module('BE.seed.controller.inventory_list_beta', [])
         saveTreeView: false,
         saveVisible: false,
         saveWidths: false,
+        useExternalFiltering: true,
+        useExternalSorting: true,
         columnDefs: $scope.columns,
         onRegisterApi: function (gridApi) {
           $scope.gridApi = gridApi;
@@ -1213,8 +1308,14 @@ angular.module('BE.seed.controller.inventory_list_beta', [])
             saveSettings();
           });
           gridApi.core.on.columnVisibilityChanged($scope, saveSettings);
-          gridApi.core.on.filterChanged($scope, _.debounce(saveGridSettings, 150));
-          gridApi.core.on.sortChanged($scope, _.debounce(saveGridSettings, 150));
+          gridApi.core.on.filterChanged($scope, _.debounce(() => {
+            updateColumnFilterSort();
+            $scope.load_inventory(1);
+          }, 1000));
+          gridApi.core.on.sortChanged($scope, _.debounce(() => {
+            updateColumnFilterSort();
+            $scope.load_inventory(1);
+          }, 1000));
           gridApi.pinning.on.columnPinned($scope, saveSettings);
 
           var selectionChanged = function () {