Handle nonzero status in addUPI telemetry (fixes #478)

enhydris/telemetry/tests/test_types/test_addupi.py

@@ -3,6 +3,7 @@
 from unittest.mock import patch
 
 from freezegun import freeze_time
+from parameterized import parameterized
 from requests import Timeout
 
 from enhydris.telemetry import TelemetryError
@@ -180,11 +181,6 @@ def test_request_when_no_start_date(self, mock_requests_get):
             verify=False,
         )
 
-    def test_raises_when_s_attribute_is_nonzero(self, mock_requests_get):
-        self._set_nonzero_s_request_result(mock_requests_get)
-        with self.assertRaisesRegex(TelemetryError, "not supported"):
-            self.telemetry_api_client.get_measurements(8231, None)
-
     def _set_successful_request_result(self, mock_requests_get):
         mock_requests_get.return_value.__enter__.return_value.content = (
             "<response>"
@@ -195,11 +191,52 @@ def _set_successful_request_result(self, mock_requests_get):
             "</response>"
         ).encode()
 
-    def _set_nonzero_s_request_result(self, mock_requests_get):
+
+@patch("enhydris.telemetry.types.addupi.requests.get")
+@freeze_time("2022-06-14 08:35")
+class ReturnedStatusTestCase(LoggedOnTestCaseBase):
+    @parameterized.expand(
+        [
+            (3, 'invalid status value .s="3".'),
+            (-100, 'invalid status value .s="-100".'),
+            ("hello", 'invalid status value .s="hello".'),
+        ]
+    )
+    def test_error_handling(self, mock_requests_get, s, error_msg):
+        self._set_s_request_result(s, mock_requests_get)
+        with self.assertRaisesRegex(TelemetryError, error_msg):
+            self.telemetry_api_client.get_measurements(8231, None)
+
+    @parameterized.expand(
+        [
+            (0, ""),
+            (1, "INVALID"),
+            (2, "INVALID"),
+            (-99, "MISSING"),
+            (-1, "MISSING"),
+        ]
+    )
+    def test_flag(self, mock_requests_get, s, flag):
+        self._set_s_request_result(s, mock_requests_get)
+        result = self.telemetry_api_client.get_measurements(8231, None)
+        self.assertEqual(result.getvalue(), f"2022-06-14T08:10:00,1.42,{flag}\n")
+
+    def _set_s_request_result(self, s, mock_requests_get):
         mock_requests_get.return_value.__enter__.return_value.content = (
             "<response>"
             "    <node>"
-            "        <v t='20220614T08:10:00' s='1'>1.42</v>"
+            f"        <v t='20220614T08:10:00' s='{s}'>1.42</v>"
             "    </node>"
             "</response>"
         ).encode()
+
+    def test_s_attribute_missing(self, mock_requests_get):
+        mock_requests_get.return_value.__enter__.return_value.content = (
+            "<response>"
+            "    <node>"
+            "        <v t='20220614T08:10:00'>1.42</v>"
+            "    </node>"
+            "</response>"
+        ).encode()
+        with self.assertRaisesRegex(TelemetryError, 'invalid status value .s="None".'):
+            self.telemetry_api_client.get_measurements(8231, None)

enhydris/telemetry/types/addupi.py

@@ -41,8 +41,6 @@ def get_sensors(self):
         return {x.attrib["id"]: x.attrib["name"] for x in sensors}
 
     def get_measurements(self, sensor_id, timeseries_end_date):
-        from enhydris.telemetry import TelemetryError
-
         if timeseries_end_date is None:
             timeseries_end_date = dt.datetime(1990, 1, 1)
         xmlroot = self._make_request(
@@ -59,19 +57,36 @@ def get_measurements(self, sensor_id, timeseries_end_date):
                 timestamp = prev_timestamp + dt.timedelta(seconds=int(timestamp))
             else:
                 timestamp = dt.datetime.strptime(timestamp, "%Y%m%dT%H:%M:%S")
-            s = record.attrib["s"]
-            if s != "0":
-                raise TelemetryError(
-                    f"The record with timestamp {timestamp} has a non zero s "
-                    f'attribute (s="{s}"). This is probably normal, however it is '
-                    "currently not supported by the Enhydris addupi driver. Please "
-                    "ask for the driver to be fixed."
-                )
-            prev_timestamp = timestamp
             value = float(record.text)
-            result += f"{timestamp.isoformat()},{value},\n"
+            flags = self._get_flags(record, timestamp)
+            result += f"{timestamp.isoformat()},{value},{flags}\n"
+            prev_timestamp = timestamp
         return StringIO(result)
 
+    def _get_flags(self, record, timestamp):
+        s = self._get_s_attribute(record, timestamp)
+        if s in (1, 2):
+            return "INVALID"
+        if s < 0:
+            return "MISSING"
+        return ""
+
+    def _get_s_attribute(self, record, timestamp):
+        try:
+            s = None
+            s = record.attrib["s"]
+            si = int(s)
+            if si < -99 or si > 2:
+                raise ValueError()
+            return si
+        except (KeyError, ValueError):
+            from enhydris.telemetry import TelemetryError
+
+            raise TelemetryError(
+                f"The record with timestamp {timestamp} has an invalid status "
+                f'value (s="{s}")'
+            )
+
     def _make_request(self, query_string):
         from enhydris.telemetry import TelemetryError
 

requirements-dev.txt

@@ -5,3 +5,4 @@ django-selenium-clean>=1,<2
 beautifulsoup4>=4.4
 celery>=4,<5
 freezegun>=1.1,<2
+parameterized>=0.9.0,<1