Dagtotalen herberekenen op basis van uurtotalen #924

docs/changelog.rst

@@ -9,6 +9,7 @@ v3.7.0 - 2020-xx-xx
 - [`#774 <https://github.com/dennissiemensma/dsmr-reader/issues/774>`_] Retentie omzetten naar geplande taak
 - [`#565 <https://github.com/dennissiemensma/dsmr-reader/issues/565>`_] Melding bij onvolledige vergelijking
 - [`#923 <https://github.com/dennissiemensma/dsmr-reader/issues/923>`_] Backups compressie level configureerbaar maken
+- [`#924 <https://github.com/dennissiemensma/dsmr-reader/issues/924>`_] Dagtotalen herberekenen op basis van uurtotalen
 
 
 ----

dsmr_stats/management/commands/dsmr_stats_reconstruct_missing_day_statistics.py

@@ -0,0 +1,14 @@
+from django.core.management.base import BaseCommand
+from django.utils.translation import gettext as _
+
+import dsmr_stats.services
+
+
+class Command(BaseCommand):
+    help = _('Reconstructs missing day statistics by using available hour statistics')
+
+    def handle(self, **options):
+        dsmr_stats.services.reconstruct_missing_day_statistics()
+
+        print()
+        print('To recalculate prices as well, execute:   ./manage.py dsmr_stats_recalculate_prices')

dsmr_stats/services.py

@@ -8,6 +8,7 @@
 from django.db import transaction, connection, models
 from django.db.models.aggregates import Avg, Sum, Min, Max
 from django.core.cache import cache
+from django.db.models.functions import TruncDate
 from django.utils import timezone
 from django.conf import settings
 
@@ -331,18 +332,64 @@ def update_electricity_statistics(reading):
 def recalculate_prices():
     """ Retroactively sets the prices for all statistics. E.g.: When the user has altered the prices. """
     for current_day in DayStatistics.objects.all():
-        prices = dsmr_consumption.services.get_day_prices(day=current_day.day)
+        print(' - Recalculating:', current_day.day)
 
+        prices = dsmr_consumption.services.get_day_prices(day=current_day.day)
         current_day.electricity1_cost = dsmr_consumption.services.round_decimal(
             current_day.electricity1 * prices.electricity_delivered_1_price
         )
         current_day.electricity2_cost = dsmr_consumption.services.round_decimal(
             current_day.electricity2 * prices.electricity_delivered_2_price
         )
-        current_day.gas_cost = dsmr_consumption.services.round_decimal(
-            current_day.gas * prices.gas_price
-        )
+
+        if current_day.gas is not None:
+            current_day.gas_cost = dsmr_consumption.services.round_decimal(
+                current_day.gas * prices.gas_price
+            )
+
         current_day.total_cost = dsmr_consumption.services.round_decimal(
             current_day.electricity1_cost + current_day.electricity2_cost + current_day.gas_cost
         )
         current_day.save()
+
+
+def reconstruct_missing_day_statistics():
+    """ Reconstructs missing day statistics by using available hour statistics. """
+    dates_to_generate = HourStatistics.objects.all().annotate(
+        truncated_date=TruncDate('hour_start')
+    ).exclude(
+        # Skip existing days.
+        truncated_date__in=DayStatistics.objects.all().values_list('day', flat=True)
+    ).order_by('truncated_date').values_list(
+        'truncated_date', flat=True
+    )
+
+    # Budget distinct and sorting.
+    dates_to_generate = sorted(list(set(dates_to_generate)))
+
+    print('Found {} day(s) to reconstruct'.format(len(dates_to_generate)))
+
+    for current_day in dates_to_generate:
+        print(' - Reconstructing:', current_day)
+
+        day_totals = HourStatistics.objects.filter(
+            hour_start__date=current_day
+        ).aggregate(
+            electricity1_sum=Sum('electricity1'),
+            electricity2_sum=Sum('electricity2'),
+            electricity1_returned_sum=Sum('electricity1_returned'),
+            electricity2_returned_sum=Sum('electricity2_returned'),
+            gas_sum=Sum('gas'),
+        )
+
+        DayStatistics.objects.create(
+            day=current_day,
+            electricity1=day_totals['electricity1_sum'],
+            electricity2=day_totals['electricity2_sum'],
+            electricity1_returned=day_totals['electricity1_returned_sum'],
+            electricity2_returned=day_totals['electricity2_returned_sum'],
+            gas=day_totals['gas_sum'],
+            total_cost=0,
+            electricity1_cost=0,
+            electricity2_cost=0,
+        )

dsmr_stats/tests/test_services.py

@@ -787,7 +787,18 @@ def test_recalculate_prices(self):
             electricity1_returned=0,
             electricity2_returned=0,
         )
-
+        DayStatistics.objects.create(
+            day=target_day.date() + timezone.timedelta(days=1),
+            total_cost=0,
+            electricity1=1,
+            electricity2=2,
+            electricity1_cost=0,
+            electricity2_cost=0,
+            gas=None,  # Omit gas
+            gas_cost=0,
+            electricity1_returned=0,
+            electricity2_returned=0,
+        )
         day_totals = DayStatistics.objects.all()[0]
         self.assertEqual(day_totals.electricity1_cost, 15)
         self.assertEqual(day_totals.electricity2_cost, 30)
@@ -802,6 +813,62 @@ def test_recalculate_prices(self):
         self.assertEqual(day_totals.gas_cost, 5)
         self.assertEqual(day_totals.total_cost, 55)
 
+    def test_reconstruct_missing_day_statistics(self):
+        # Existing day should be ignored.
+        DayStatistics.objects.create(
+            day=timezone.make_aware(timezone.datetime(2020, 1, 1)).date(),
+            total_cost=12345,
+            electricity1=1,
+            electricity2=2,
+            electricity1_returned=3,
+            electricity2_returned=4,
+            gas=5,
+            electricity1_cost=0,
+            electricity2_cost=0,
+            gas_cost=0,
+        )
+        HourStatistics.objects.create(
+            hour_start=timezone.make_aware(timezone.datetime(2020, 1, 1, 12)),
+            electricity1=0.1,
+            electricity2=0.2,
+            electricity1_returned=0.3,
+            electricity2_returned=0.4,
+            gas=0.5,
+        )
+
+        self.assertEqual(DayStatistics.objects.all().count(), 1)
+
+        dsmr_stats.services.reconstruct_missing_day_statistics()
+        self.assertEqual(DayStatistics.objects.all().count(), 1)
+
+        # Now add hours for missing day.
+        HourStatistics.objects.create(
+            hour_start=timezone.make_aware(timezone.datetime(2020, 1, 2, 12)),
+            electricity1=0.11,
+            electricity2=0.22,
+            electricity1_returned=0.33,
+            electricity2_returned=0.44,
+            gas=0.55,
+        )
+        HourStatistics.objects.create(
+            hour_start=timezone.make_aware(timezone.datetime(2020, 1, 2, 13)),
+            electricity1=1,
+            electricity2=2,
+            electricity1_returned=3,
+            electricity2_returned=4,
+            gas=5,
+        )
+
+        dsmr_stats.services.reconstruct_missing_day_statistics()
+        self.assertEqual(DayStatistics.objects.all().count(), 2)
+
+        latest = DayStatistics.objects.all().order_by('-pk')[0]
+        self.assertEqual(latest.electricity1, Decimal('1.11'))
+        self.assertEqual(latest.electricity2, Decimal('2.22'))
+        self.assertEqual(latest.electricity1_returned, Decimal('3.33'))
+        self.assertEqual(latest.electricity2_returned, Decimal('4.44'))
+        self.assertEqual(latest.gas, Decimal('5.55'))
+
 
 class TestServicesWithoutGas(TestServices):
     fixtures = ['dsmr_stats/electricity-consumption.json']

dsmrreader/locales/nl/LC_MESSAGES/django.po

@@ -2264,6 +2264,9 @@ msgstr "Verandert alle bestaande statistieken. NIET GEBRUIKEN IN PRODUCTIE! Word
 msgid "Recalculates day statistics prices"
 msgstr "Herberekent prijzen in dagtotalen"
 
+msgid "Reconstructs missing day statistics by using available hour statistics"
+msgstr "Reconstrueert missende dagstatistieken door beschikbare uurstatistieken te gebruiken"
+
 msgid "Note"
 msgstr "Notitie"
 

dsmrreader/provisioning/downgrade/v3.7.sh

@@ -3,7 +3,7 @@
 # Dump for DSMR-reader v3.7
 ./manage.py migrate dsmr_api 0003_create_api_user
 ./manage.py migrate dsmr_backend 0014_verbose_field_translations
-./manage.py migrate dsmr_backup 0008_verbose_field_translations
+./manage.py migrate dsmr_backup 0009_compression_level
 ./manage.py migrate dsmr_consumption 0015_track_power_current
 ./manage.py migrate dsmr_datalogger 0021_schedule_retention_data_rotation
 ./manage.py migrate dsmr_frontend 0034_mysql_timezone_support