Improve datalogger for DSMR v5.0 #212

docs/api.rst

@@ -68,7 +68,7 @@ Example
 (using the ``requests`` library available on PIP)::
 
     import requests  # Tested with requests==2.9.1
-    
+
     # Fake buffer.
     telegram_string = ''.join([
         "/KFM5KAIFA-METER\r\n",
@@ -95,14 +95,14 @@ Example
         "1-0:22.7.0(00.000*kW)\r\n",
         "!74B0\n",
     ])
-    
+
     # Register telegram by simply sending it to the application with a POST request.
     response = requests.post(
         'http://YOUR-DSMR-URL/api/v1/datalogger/dsmrreading',
         headers={'X-AUTHKEY': 'YOUR-DSMR-API-AUTHKEY'},
         data={'telegram': telegram_string},
     )
-       
+
     # You will receive a status 200 when successful.
     if response.status_code != 200:
         # Or you will find the error (hint) in the reponse body on failure.
@@ -140,29 +140,29 @@ Client file in ``/home/dsmr/dsmr_datalogger_api_client.py``::
     from serial.serialutil import SerialException
     import requests
     import serial
-    
-    
+
+
     API_SERVERS = (
         ('http://HOST-OR-IP-ONE/api/v1/datalogger/dsmrreading', 'APIKEY-BLABLABLA-ABCDEFGHI'),
     ###    ('http://HOST-OR-IP-TWO/api/v1/datalogger/dsmrreading', 'APIKEY-BLABLABLA-JKLMNOPQR'),
     )
-    
-    
+
+
     def main():
         print ('Starting...')
-    
+
         while True:
             telegram = read_telegram()
             print('Read telegram', telegram)
-    
+
             for current_server in API_SERVERS:
                 api_url, api_key = current_server
                 send_telegram(telegram, api_url, api_key)
                 print('Sent telegram to:', api_url)
-    
+
             sleep(1)
-    
-    
+
+
     def read_telegram():
         """ Reads the serial port until we can create a reading point. """
         serial_handle = serial.Serial()
@@ -174,13 +174,13 @@ Client file in ``/home/dsmr/dsmr_datalogger_api_client.py``::
         serial_handle.xonxoff = 1
         serial_handle.rtscts = 0
         serial_handle.timeout = 20
-    
+
         # This might fail, but nothing we can do so just let it crash.
         serial_handle.open()
-    
+
         telegram_start_seen = False
         telegram = ''
-    
+
         # Just keep fetching data until we got what we were looking for.
         while True:
             try:
@@ -189,41 +189,44 @@ Client file in ``/home/dsmr/dsmr_datalogger_api_client.py``::
                 # Something else and unexpected failed.
                 print('Serial connection failed:', error)
                 return
-    
+
             try:
                 # Make sure weird characters are converted properly.
                 data = str(data, 'utf-8')
             except TypeError:
                 pass
-    
+
             # This guarantees we will only parse complete telegrams. (issue #74)
             if data.startswith('/'):
                 telegram_start_seen = True
-    
+
+                # But make sure to RESET any data collected as well! (issue #212)
+                buffer = ''
+
             # Delay any logging until we've seen the start of a telegram.
             if telegram_start_seen:
                 telegram += data
-    
+
             # Telegrams ends with '!' AND we saw the start. We should have a complete telegram now.
             if data.startswith('!') and telegram_start_seen:
                 serial_handle.close()
                 return telegram
-    
-    
+
+
     def send_telegram(telegram, api_url, api_key):
         # Register telegram by simply sending it to the application with a POST request.
         response = requests.post(
             api_url,
             headers={'X-AUTHKEY': api_key},
             data={'telegram': telegram},
         )
-    
+
         # You will receive a status 200 when successful.
         if response.status_code != 200:
             # Or you will find the error (hint) in the response body on failure.
             print('[!] Error: {}'.format(response.text))
-    
-    
+
+
     if __name__ == '__main__':
         main()
 

docs/changelog.rst

@@ -18,12 +18,12 @@ Please make sure you have a fresh **database backup** before upgrading! Upgradin
 
 
 
-v1.6.0 - 2017-xx-xx
+v1.5.4 - 2017-01-12
 ^^^^^^^^^^^^^^^^^^^
 
 **Tickets resolved in this release:**
 
-- XXXXX (`#YYY <https://github.com/dennissiemensma/dsmr-reader/issues/YYY>`_).
+- Improve datalogger for DSMR v5.0 (`#212 <https://github.com/dennissiemensma/dsmr-reader/issues/212>`_).
 
 
 

dsmr_datalogger/services.py

@@ -86,6 +86,9 @@ def read_telegram():
         if data.startswith('/'):
             telegram_start_seen = True
 
+            # But make sure to RESET any data collected as well! (issue #212)
+            buffer = ''
+
         # Delay any logging until we've seen the start of a telegram.
         if telegram_start_seen:
             buffer += data

dsmr_datalogger/tests/datalogger/test_error.py

@@ -1,7 +1,10 @@
+from decimal import Decimal
 from unittest import mock
 
 from serial.serialutil import SerialException
+from django.utils import timezone
 from django.test import TestCase
+import pytz
 
 from dsmr_backend.tests.mixins import InterceptStdoutMixin
 from dsmr_datalogger.models.reading import DsmrReading
@@ -141,7 +144,7 @@ def test_okay(self, serial_readline_mock, serial_open_mock):
         serial_open_mock.return_value = None
         serial_readline_mock.side_effect = self._dsmr_dummy_data()
 
-        # The big difference here is that CRc verification should be off.
+        # The big difference here is that CRC verification should be off.
         datalogger_settings = DataloggerSettings.get_solo()
         datalogger_settings.verify_telegram_crc = False
         datalogger_settings.save()
@@ -151,3 +154,100 @@ def test_okay(self, serial_readline_mock, serial_open_mock):
         self._intercept_command_stdout('dsmr_datalogger')
 
         self.assertTrue(DsmrReading.objects.exists())
+
+
+class TestDataloggerDuplicateData(InterceptStdoutMixin, TestCase):
+    """ Test Iskra meter, DSMR v5.0, with somewhat duplicate data. """
+    def _dsmr_dummy_data(self):
+        return [
+            "/ISK5\\2M550T-1011\r\n",
+            "\r\n",
+            "1-3:0.2.8(50)\r\n",
+            "0-0:1.0.0(170110204056W)\r\n",
+            "0-0:96.1.1(xxx)\r\n",
+            "1-0:1.8.1(000012.345*kWh)\r\n",
+            "1-0:1.8.2(000067.890*kWh)\r\n",
+            "1-0:2.8.1(000123.456*kWh)\r\n",
+            "1-0:2.8.2(000789.012*kWh)\r\n",
+            "0-0:96.14.0(0002)\r\n",
+            "1-0:1.7.0(00.321*k\n",
+            "/ISK5\\2M550T-1011\r\n",
+            "\r\n",
+            "1-3:0.2.8(50)\r\n",
+            "0-0:1.0.0(170110204057W)\r\n",
+            "0-0:96.1.1(xxx)\r\n",
+            "1-0:1.8.1(009012.345*kWh)\r\n",
+            "1-0:1.8.2(009067.890*kWh)\r\n",
+            "1-0:2.8.1(009123.456*kWh)\r\n",
+            "1-0:2.8.2(009789.012*kWh)\r\n",
+            "0-0:96.14.0(0002)\r\n",
+            "1-0:1.7.0(00.320*kW)\r\n",
+            "1-0:2.7.0(00.000*kW)\r\n",
+            "0-0:96.7.21(00005)\r\n",
+            "0-0:96.7.9(00002)\r\n",
+            "1-0:99.97.0()\r\n",
+            "1-0:32.32.0(00000)\r\n",
+            "1-0:52.32.0(00000)\r\n",
+            "1-0:72.32.0(00000)\r\n",
+            "1-0:32.36.0(00001)\r\n",
+            "1-0:52.36.0(00001)\r\n",
+            "1-0:72.36.0(00001)\r\n",
+            "0-0:96.13.0()\r\n",
+            "1-0:32.7.0(227.0*V)\r\n",
+            "1-0:52.7.0(228.3*V)\r\n",
+            "1-0:72.7.0(230.4*V)\r\n",
+            "1-0:31.7.0(000*A)\r\n",
+            "1-0:51.7.0(000*A)\r\n",
+            "1-0:71.7.0(000*A)\r\n",
+            "1-0:21.7.0(00.152*kW)\r\n",
+            "1-0:41.7.0(00.052*kW)\r\n",
+            "1-0:61.7.0(00.118*kW)\r\n",
+            "1-0:22.7.0(00.000*kW)\r\n",
+            "1-0:42.7.0(00.000*kW)\r\n",
+            "1-0:62.7.0(00.000*kW)\r\n",
+            "0-1:24.1.0(003)\r\n",
+            "0-1:96.1.0(xxx)\r\n",
+            "0-1:24.2.1(170110204009W)(00123.456*m3)\r\n",
+            "!469F\r\n"
+        ]
+
+    @mock.patch('serial.Serial.open')
+    @mock.patch('serial.Serial.readline')
+    def _fake_dsmr_reading(self, serial_readline_mock, serial_open_mock):
+        """ Fake & process an DSMR vX telegram reading. """
+        serial_open_mock.return_value = None
+        serial_readline_mock.side_effect = self._dsmr_dummy_data()
+
+        self.assertFalse(DsmrReading.objects.exists())
+        self._intercept_command_stdout('dsmr_datalogger')
+        self.assertTrue(DsmrReading.objects.exists())
+
+    def test_reading_creation(self):
+        """ Test whether dsmr_datalogger can insert a reading. """
+        self.assertFalse(DsmrReading.objects.exists())
+        self._fake_dsmr_reading()
+        self.assertTrue(DsmrReading.objects.exists())
+
+    @mock.patch('django.utils.timezone.now')
+    def test_reading_values(self, now_mock):
+        """ Test whether dsmr_datalogger reads the correct values. """
+        now_mock.return_value = timezone.make_aware(timezone.datetime(2016, 4, 10, hour=14, minute=30, second=15))
+
+        self._fake_dsmr_reading()
+        self.assertTrue(DsmrReading.objects.exists())
+        reading = DsmrReading.objects.get()
+        self.assertEqual(
+            reading.timestamp,
+            timezone.datetime(2017, 1, 10, 19, 40, 57, tzinfo=pytz.UTC)
+        )
+        self.assertEqual(reading.electricity_delivered_1, Decimal('9012.345'))
+        self.assertEqual(reading.electricity_returned_1, Decimal('9123.456'))
+        self.assertEqual(reading.electricity_delivered_2, Decimal('9067.890'))
+        self.assertEqual(reading.electricity_returned_2, Decimal('9789.012'))
+        self.assertEqual(reading.electricity_currently_delivered, Decimal('0.320'))
+        self.assertEqual(reading.electricity_currently_returned, Decimal('0'))
+        self.assertEqual(
+            reading.extra_device_timestamp,
+            timezone.datetime(2017, 1, 10, 19, 40, 9, tzinfo=pytz.UTC)
+        )
+        self.assertEqual(reading.extra_device_delivered, Decimal('123.456'))

dsmrreader/__init__.py

@@ -17,6 +17,6 @@
 from django.utils.version import get_version
 
 
-VERSION = (1, 6, 0, 'beta', 0)
+VERSION = (1, 5, 4, 'beta', 1)
 
 __version__ = get_version(VERSION)