Skip to content

Commit

Permalink
Merge pull request #3215 from SEED-platform/2685-bug/meter_data_by_mo…
Browse files Browse the repository at this point in the history 
…nth_by_second

meter usage by month down to the second
  • Loading branch information
@perryr16
perryr16 committed Apr 27, 2022
2 parents 63dc9c9 + 1a1504a commit 29be230
Show file tree
Hide file tree
Showing 2 changed files with 159 additions and 31 deletions.
126 changes: 121 additions & 5 deletions seed/tests/test_property_views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import os
import json
import unittest
from unittest import skip

from config.settings.common import TIME_ZONE

Expand Down Expand Up @@ -1828,22 +1829,22 @@ def test_property_meter_usage_can_return_monthly_meter_readings_and_column_defs_
'readings': [
{
'month': 'January 2016',
'Electric - Grid - PM - 5766973-0': 597478.9 / 3.41,
'Electric - Grid - PM - 5766973-0': round(597478.9 / 3.41, 2),
'Natural Gas - PM - 5766973-1': 576000.2,
},
{
'month': 'February 2016',
'Electric - Grid - PM - 5766973-0': 548603.7 / 3.41,
'Electric - Grid - PM - 5766973-0': round(548603.7 / 3.41, 2),
'Natural Gas - PM - 5766973-1': 488000.1,
},
{
'month': 'March 2016',
'Electric - Grid - PM - 5766973-0': 100 / 3.41,
'Electric - Grid - PM - 5766973-0': round(100 / 3.41, 2),
'Natural Gas - PM - 5766973-1': 100,
},
{
'month': 'May 2016',
'Electric - Grid - PM - 5766973-0': 200 / 3.41,
'Electric - Grid - PM - 5766973-0': round(200 / 3.41, 2),
'Natural Gas - PM - 5766973-1': 200,
},
],
Expand Down Expand Up @@ -1941,6 +1942,7 @@ def test_property_meter_usage_can_return_monthly_meter_readings_and_column_defs_
self.assertCountEqual(result_dict['readings'], expectation['readings'])
self.assertCountEqual(result_dict['column_defs'], expectation['column_defs'])

@skip('Overlapping data is not valid through ESPM. This test is no longer valid')
def test_property_meter_usage_can_return_monthly_meter_readings_and_column_defs_of_overlapping_submonthly_data_aggregating_monthly_data_to_maximize_total(self):
# add initial meters and readings
save_raw_data(self.import_file.id)
Expand Down Expand Up @@ -2057,7 +2059,6 @@ def test_property_meter_usage_can_return_monthly_meter_readings_and_column_defs_
},
]
}

self.assertCountEqual(result_dict['readings'], expectation['readings'])
self.assertCountEqual(result_dict['column_defs'], expectation['column_defs'])

Expand Down Expand Up @@ -2132,3 +2133,118 @@ def test_property_meter_usage_can_return_annual_meter_readings_and_column_defs_w

self.assertCountEqual(result_dict['readings'], expectation['readings'])
self.assertCountEqual(result_dict['column_defs'], expectation['column_defs'])

def test_property_meter_usage_can_filter_when_usages_span_a_single_month(self):
save_raw_data(self.import_file.id)

# add additional entries for the Electricity meter
tz_obj = timezone(TIME_ZONE)
meter = Meter.objects.get(property_id=self.property_view_1.property.id, type=Meter.type_lookup['Electric - Grid'])

# 2020 January-February reading has 1 full day in January 1 full day in February.
# The reading should be split 1/2 January (50) and 1/2 February (50)
reading_details = {
'meter_id': meter.id,
'start_time': make_aware(datetime(2020, 1, 31, 0, 0, 0), timezone=tz_obj),
'end_time': make_aware(datetime(2020, 2, 2, 0, 0, 0), timezone=tz_obj),
'reading': 100 * 3.41,
'source_unit': 'kBtu (thousand Btu)',
'conversion_factor': 1
}
MeterReading.objects.create(**reading_details)

# 2020 March to April reading has 1 day in march, and 2 days in april.
# The reading should be split 1/3 march (100) and 2/3 april (200)
reading_details = {
'meter_id': meter.id,
'start_time': make_aware(datetime(2020, 3, 31, 0, 0, 0), timezone=tz_obj),
'end_time': make_aware(datetime(2020, 4, 3, 0, 0, 0), timezone=tz_obj),
'reading': 300 * 3.41,
'source_unit': 'kBtu (thousand Btu)',
'conversion_factor': 1
}
MeterReading.objects.create(**reading_details)

# 2020 May to July shows readings can span multiple months.
# The reading should be split 1/32 May (10), 30/32 June (300), 1/32 July (10)
reading_details = {
'meter_id': meter.id,
'start_time': make_aware(datetime(2020, 5, 31, 0, 0, 0), timezone=tz_obj),
'end_time': make_aware(datetime(2020, 7, 2, 0, 0, 0), timezone=tz_obj),
'reading': 320 * 3.41,
'source_unit': 'kBtu (thousand Btu)',
'conversion_factor': 1
}
MeterReading.objects.create(**reading_details)

url = reverse('api:v3:properties-meter-usage', kwargs={'pk': self.property_view_1.id})
url += f'?organization_id={self.org.pk}'

post_params = json.dumps({
'interval': 'Month',
'excluded_meter_ids': [],
})
result = self.client.post(url, post_params, content_type="application/json")
result_dict = ast.literal_eval(result.content.decode("utf-8"))

expectation = {
'readings': [
{
'Electric - Grid - PM - 5766973-0': 175213.75,
'Natural Gas - PM - 5766973-1': 576000.2,
'month': 'January 2016'
},
{
'Electric - Grid - PM - 5766973-0': 160880.85,
'Natural Gas - PM - 5766973-1': 488000.1,
'month': 'February 2016'
},
{
'month': 'January 2020',
'Electric - Grid - PM - 5766973-0': 50,
},
{
'month': 'February 2020',
'Electric - Grid - PM - 5766973-0': 50,
},
{
'month': 'March 2020',
'Electric - Grid - PM - 5766973-0': 100,
},
{
'month': 'April 2020',
'Electric - Grid - PM - 5766973-0': 200,
},
{
'month': 'May 2020',
'Electric - Grid - PM - 5766973-0': 10,
},
{
'month': 'June 2020',
'Electric - Grid - PM - 5766973-0': 300,
},
{
'month': 'July 2020',
'Electric - Grid - PM - 5766973-0': 10,
},
],
'column_defs': [
{
'field': 'month',
'_filter_type': 'datetime',
},
{
'field': 'Electric - Grid - PM - 5766973-0',
'displayName': 'Electric - Grid - PM - 5766973-0 (kWh (thousand Watt-hours))',
'_filter_type': 'reading',
},
{
'field': 'Natural Gas - PM - 5766973-1',
'displayName': 'Natural Gas - PM - 5766973-1 (kBtu (thousand Btu))',
'_filter_type': 'reading',
},
]
}

self.assertCountEqual(result_dict['readings'], expectation['readings'])
self.assertCountEqual(result_dict['column_defs'], expectation['column_defs'])
64 changes: 38 additions & 26 deletions seed/utils/meters.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@

from calendar import (
monthrange,
month_name,
)

from collections import defaultdict
Expand All @@ -13,6 +12,7 @@
from datetime import (
datetime,
timedelta,
time,
)

from django.db.models import Q
Expand Down Expand Up @@ -119,9 +119,8 @@ def _usages_by_month(self):
At a high-level, following algorithm is used to acccomplish this:
- Identify the first start time and last end time
- For each month between, aggregate the readings found in that month
- The highest possible reading total without overlapping times is found
- For more details how that monthly aggregation occurs, see _max_reading_total()
- Define a range of dates between start and end time that fall within a month
- For each month in the date range, aggregate the readings found in that month using a linear relationship down to the second.
"""
# Used to consolidate different readings (types) within the same month
monthly_readings = defaultdict(lambda: {})
Expand All @@ -137,35 +136,48 @@ def _usages_by_month(self):
for meter in self.meters:
field_name, conversion_factor = self._build_column_def(meter, column_defs)

min_time = meter.meter_readings.earliest('start_time').start_time.astimezone(tz=self.tz)
max_time = meter.meter_readings.latest('end_time').end_time.astimezone(tz=self.tz)

# Iterate through months
current_month_time = min_time
while current_month_time < max_time:
_weekday, days_in_month = monthrange(current_month_time.year, current_month_time.month)
# iterate through each usage and assingn to accumulator
for usage in meter.meter_readings.values():
st, et = usage['start_time'], usage['end_time']
total_seconds = round((et - st).total_seconds())
ranges = self._get_month_ranges(st, et)

unaware_end = datetime(current_month_time.year, current_month_time.month, days_in_month, 23, 59, 59) + timedelta(seconds=1)
end_of_month = make_aware(unaware_end, timezone=self.tz)
# partial usages of the full usage are calculated from a linear relationship between the range_seconds to the total_seconds
for range in ranges:
range_seconds = round((range[1] - range[0]).total_seconds())
month_key = range[1].strftime('%B %Y')
reading = usage['reading'] / total_seconds * range_seconds / conversion_factor
if reading:
monthly_readings[month_key] = monthly_readings.get(month_key, {'month': month_key})
monthly_readings[month_key][field_name] = round(monthly_readings[month_key].get(field_name, 0) + reading, 2)

# Find all meters fully contained within this month (second-level granularity)
interval_readings = meter.meter_readings.filter(start_time__range=(current_month_time, end_of_month), end_time__range=(current_month_time, end_of_month))
if interval_readings.exists():
readings_list = list(interval_readings.order_by('end_time'))
reading_month_total = self._max_reading_total(readings_list)

if reading_month_total > 0:
month_year = '{} {}'.format(month_name[current_month_time.month], current_month_time.year)
monthly_readings[month_year]['month'] = month_year
monthly_readings[month_year][field_name] = reading_month_total / conversion_factor

current_month_time = end_of_month
sorted_readings = sorted(list(monthly_readings.values()), key=lambda reading: datetime.strptime(reading['month'], '%B %Y'))

return {
'readings': list(monthly_readings.values()),
'readings': sorted_readings,
'column_defs': list(column_defs.values())
}

def _get_month_ranges(self, st, et):
"""
Given two dates start time (st) and end date time (et)
return a list of date ranges that are within a single month
ex:
st = may 15th 2020
et = july 10th 2020
ranges = [[may 15, may 31], [june 1, june 30], [july 1, july 10]]
"""
month_count = (et.year - st.year) * 12 + et.month - st.month + 1
start = st
ranges = []
for idx in range(0, month_count):
end_of_month = make_aware(datetime.combine(start.replace(day=monthrange(start.year, start.month)[1]), time.max), timezone=self.tz)
if end_of_month >= et:
end_of_month = et
ranges.append([start, end_of_month])
start = end_of_month + timedelta(microseconds=1)
return ranges

def _usages_by_year(self):
"""
Similarly to _usages_by_month, this returns readings and column definitions
Expand Down

0 comments on commit 29be230

Please sign in to comment.