Merge pull request #3198 from SEED-platform/Add-occupied-hours-filtering

Add occupied hours filtering

seed/data_importer/tasks.py

@@ -6,6 +6,7 @@
 """
 
 from __future__ import absolute_import
+from bisect import bisect_left
 
 import collections
 import copy
@@ -21,6 +22,7 @@
 from math import ceil
 import zipfile
 import tempfile
+from dateutil import parser
 
 from celery import chord, shared_task, group
 from celery import chain as celery_chain
@@ -919,13 +921,17 @@ def _save_sensor_readings_task(readings_tuples, data_logger_id, sensor_column_na
     else:
         try:
             with transaction.atomic():
-                reading_strings = [
-                    f"({sensor.id}, '{timestamp}', '{value}')"
-                    for timestamp, value
-                    in readings_tuples
-                ]
+                is_occupied_data = DataLogger.objects.get(id=data_logger_id).is_occupied_data
+                [occupied_timestamps, is_occupied_arr] = list(zip(*is_occupied_data))
+                occupied_timestamps = [datetime.fromisoformat(t) for t in occupied_timestamps]
+
+                reading_strings = []
+                for timestamp, value in readings_tuples:
+                    is_occupied = is_occupied_arr[bisect_left(occupied_timestamps, parser.parse(timestamp)) - 1]
+                    reading_strings.append(f"({sensor.id}, '{timestamp}', '{value}', '{is_occupied}')")
+
                 sql = (
-                    'INSERT INTO seed_sensorreading(sensor_id, timestamp, reading)' +
+                    'INSERT INTO seed_sensorreading(sensor_id, timestamp, reading, is_occupied)' +
                     ' VALUES ' + ', '.join(reading_strings) +
                     ' ON CONFLICT (sensor_id, timestamp)' +
                     ' DO UPDATE SET reading = EXCLUDED.reading' +

seed/migrations/0162_auto_20220418_2257.py

@@ -0,0 +1,24 @@
+# Generated by Django 3.2.12 on 2022-04-18 22:57
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('seed', '0161_alter_inventorydocument_file_type'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='datalogger',
+            name='is_occupied_data',
+            field=models.JSONField(default=dict),
+        ),
+        migrations.AddField(
+            model_name='sensorreading',
+            name='is_occupied',
+            field=models.BooleanField(default=False),
+            preserve_default=False,
+        ),
+    ]

seed/models/sensors.py

@@ -15,6 +15,7 @@ class DataLogger(models.Model):
 
     display_name = models.CharField(max_length=255)
     location_identifier = models.CharField(max_length=2047, default="")
+    is_occupied_data = models.JSONField(null=False, default=dict)
 
     class Meta:
         unique_together = ('property', 'display_name')
@@ -49,6 +50,7 @@ class SensorReading(models.Model):
         on_delete=models.CASCADE,
         related_name='sensor_readings',
     )
+    is_occupied = models.BooleanField(null=False)
 
     class Meta:
         unique_together = ('timestamp', 'sensor')

seed/static/seed/js/controllers/inventory_detail_sensors_controller.js

@@ -38,6 +38,7 @@ angular.module('BE.seed.controller.inventory_detail_sensors', [])
       $scope.organization = organization_payload.organization;
       $scope.property_sensor_usage = property_sensor_usage;
       $scope.filler_cycle = cycles.cycles[0].id;
+      $scope.showOnlyOccupiedReadings = false;
 
       $scope.inventory = {
         view_id: $stateParams.view_id
@@ -240,6 +241,11 @@ angular.module('BE.seed.controller.inventory_detail_sensors', [])
         selected: 'Exact'
       };
 
+      $scope.toggled_show_only_occupied_reading = function (b) {
+        $scope.showOnlyOccupiedReadings = b
+        $scope.refresh_readings();
+      };
+
       // given a list of sensor labels, it returns the filtered readings and column defs
       // This is used by the primary filterBy... functions
       var filterBySensorLabels = function filterBySensorLabels (readings, columnDefs, sensorLabels) {
@@ -303,6 +309,7 @@ angular.module('BE.seed.controller.inventory_detail_sensors', [])
           $scope.inventory.view_id,
           $scope.organization.id,
           $scope.interval.selected,
+          $scope.showOnlyOccupiedReadings,
           [] // Not excluding any sensors from the query
         ).then(function (usage) {
           // update the base data and reset filters

seed/static/seed/js/services/sensor_service.js

@@ -35,13 +35,14 @@ angular.module('BE.seed.service.sensor', [])
         });
       };
 
-      sensor_factory.property_sensor_usage = function (property_view_id, organization_id, interval, excluded_sensor_ids) {
+      sensor_factory.property_sensor_usage = function (property_view_id, organization_id, interval, showOnlyOccupiedReadings, excluded_sensor_ids) {
         if (_.isUndefined(excluded_sensor_ids)) excluded_sensor_ids = [];
         return $http.post(
           '/api/v3/properties/' + property_view_id + '/sensor_usage/?organization_id=' + organization_id,
           {
             interval: interval,
-            excluded_sensor_ids: excluded_sensor_ids
+            excluded_sensor_ids: excluded_sensor_ids,
+            showOnlyOccupiedReadings: showOnlyOccupiedReadings,
           }
         ).then(function (response) {
           return response.data;

seed/static/seed/partials/inventory_detail_sensors.html

@@ -78,7 +78,7 @@ <h4 translate>SENSORS</h4>
                 </select>
             </div>
         </div>
-        <div style="display: flex;">
+        <div style="display: flex; align-items: baseline;">
             <div class="form-group" uib-dropdown is-open="data_loggers_options.isopen" auto-close="outsideClick" on-toggle="data_logger_selection_toggled(open)" style="margin-top: 15px; margin-right: 15px;">
                 <button type="button" class="btn btn-default" uib-dropdown-toggle>
                     Filter Data Logger <span class="caret"></span>
@@ -107,6 +107,10 @@ <h4 translate>SENSORS</h4>
                     </li>
                 </ul>
             </div>
+            <div style="margin: 15px;">
+                <input ng-click="toggled_show_only_occupied_reading(!showOnlyOccupiedReadings)" type="checkbox" name="showOnlyOccupiedReadingsCheckbox">
+                <label for="showOnlyOccupiedReadingsCheckbox"> Only Show Readings for Occupied Hours</label><br>
+            </div>
         </div>
         <div class="section_content" ng-show="has_sensor_readings">
             <div class="sensor-header-container">

seed/tests/test_property_views.py

@@ -1389,12 +1389,14 @@ def test_property_sensor_usage_returns_sensor_readings(self):
             SensorReading.objects.create(**{
                 "reading": s1_reading,
                 "timestamp": timestamp,
-                "sensor": s1
+                "sensor": s1,
+                "is_occupied": False
             })
             SensorReading.objects.create(**{
                 "reading": s2_reading,
                 "timestamp": timestamp,
-                "sensor": s2
+                "sensor": s2,
+                "is_occupied": False
             })
             except_results.append({
                 "timestamp": str(timestamp.replace(tzinfo=None)),

seed/utils/sensors.py

@@ -26,12 +26,13 @@ class PropertySensorReadingsExporter():
     settings are considered/used when returning actual reading magnitudes.
     """
 
-    def __init__(self, property_id, org_id, excluded_sensor_ids):
+    def __init__(self, property_id, org_id, excluded_sensor_ids, showOnlyOccupiedReadings):
         self._cache_factors = None
         self._cache_org_country = None
 
         self.sensors = Sensor.objects.select_related('data_logger').filter(data_logger__property_id=property_id).exclude(pk__in=excluded_sensor_ids)
         self.org_id = org_id
+        self.showOnlyOccupiedReadings = showOnlyOccupiedReadings
         self.tz = timezone(TIME_ZONE)
 
     def readings_and_column_defs(self, interval):
@@ -64,7 +65,11 @@ def _usages_by_exact_times(self):
         for sensor in self.sensors:
             field_name = self._build_column_def(sensor, column_defs)
 
-            for sensor_reading in sensor.sensor_readings.all():
+            sensor_readings = sensor.sensor_readings
+            if self.showOnlyOccupiedReadings:
+                sensor_readings = sensor_readings.filter(is_occupied=True)
+
+            for sensor_reading in sensor_readings.all():
                 timestamp = sensor_reading.timestamp.astimezone(tz=self.tz).strftime(time_format)
                 times_key = str(timestamp)
 
@@ -95,8 +100,12 @@ def _usages_by_month(self):
         for sensor in self.sensors:
             field_name = self._build_column_def(sensor, column_defs)
 
+            sensor_readings = sensor.sensor_readings
+            if self.showOnlyOccupiedReadings:
+                sensor_readings = sensor_readings.filter(is_occupied=True)
+
             # group by month and avg readings
-            readings_avg_by_month = sensor.sensor_readings \
+            readings_avg_by_month = sensor_readings \
                 .annotate(month=TruncMonth('timestamp')) \
                 .values('month').order_by('month') \
                 .annotate(avg=Avg('reading')) \
@@ -132,7 +141,11 @@ def _usages_by_year(self):
         for sensor in self.sensors:
             field_name = self._build_column_def(sensor, column_defs)
 
-            readings_avg_by_year = sensor.sensor_readings \
+            sensor_readings = sensor.sensor_readings
+            if self.showOnlyOccupiedReadings:
+                sensor_readings = sensor_readings.filter(is_occupied=True)
+
+            readings_avg_by_year = sensor_readings \
                 .annotate(year=TruncYear('timestamp')) \
                 .values('year').order_by('year') \
                 .annotate(avg=Avg('reading')) \

seed/views/v3/data_logger.py

@@ -6,11 +6,15 @@
 from seed.decorators import ajax_request_class
 from seed.lib.superperms.orgs.decorators import has_perm_class
 from seed.models import (PropertyView,
-                         DataLogger)
+                         DataLogger,
+                         )
 from seed.models.sensors import Sensor
 from seed.utils.api_schema import swagger_auto_schema_org_query_param
 from seed.utils.api import OrgMixin
 from django.db.utils import IntegrityError
+from datetime import datetime, timedelta
+from config.settings.common import TIME_ZONE
+from pytz import timezone as pytztimezone
 
 
 class DataLoggerViewSet(viewsets.ViewSet, OrgMixin):
@@ -68,6 +72,29 @@ def create(self, request):
             location_identifier=location_identifier
         )
 
+        # for every weekday from 2020-2023, mark as occupied from 8-5
+        tz_obj = pytztimezone(TIME_ZONE)
+        start_time = datetime(2020, 1, 1, 0, 0, tzinfo=tz_obj)
+        end_time = datetime(2023, 1, 1, 0, 0, tzinfo=tz_obj)
+
+        day = start_time
+        is_occupied_data = []
+        while day < end_time:
+            if day.weekday() <= 4:
+                open_time = day + timedelta(hours=8)
+                is_occupied_data.append(
+                    (open_time.isoformat(), True)
+                )
+
+                close_time = day + timedelta(hours=17)
+                is_occupied_data.append(
+                    (close_time.isoformat(), False)
+                )
+
+            day += timedelta(days=1)
+
+        data_logger.is_occupied_data = is_occupied_data
+
         try:
             data_logger.save()
         except IntegrityError:

seed/views/v3/properties.py

@@ -231,6 +231,7 @@ def sensor_usage(self, request, pk):
         body = dict(request.data)
         interval = body['interval']
         excluded_sensor_ids = body['excluded_sensor_ids']
+        showOnlyOccupiedReadings = body.get('showOnlyOccupiedReadings', False)
         org_id = self.get_organization(request)
 
         property_view = PropertyView.objects.get(
@@ -239,7 +240,7 @@ def sensor_usage(self, request, pk):
         )
         property_id = property_view.property.id
 
-        exporter = PropertySensorReadingsExporter(property_id, org_id, excluded_sensor_ids)
+        exporter = PropertySensorReadingsExporter(property_id, org_id, excluded_sensor_ids, showOnlyOccupiedReadings)
 
         return exporter.readings_and_column_defs(interval)