tdf: support TDF_DIFF_ARRAY

pyproject.toml

@@ -44,6 +44,10 @@ content-type = "text/markdown"
 [project.scripts]
 infuse = "infuse_iot.app.main:main"
 
+[project.urls]
+Homepage = "https://github.com/Embeint/python-tools"
+Issues = "https://github.com/Embeint/python-tools/issues"
+
 [tool.setuptools]
 package-dir = {"" = "src"}
 zip-safe = false

src/infuse_iot/tdf.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import copy
 import ctypes
 import enum
 from collections.abc import Generator
@@ -29,8 +30,14 @@ class flags(enum.IntEnum):
         TIMESTAMP_EXTENDED_RELATIVE = 0xC000
         TIMESTAMP_MASK = 0xC000
         TIME_ARRAY = 0x1000
+        DIFF_ARRAY = 0x2000
         ID_MASK = 0x0FFF
 
+    class DiffType(enum.IntEnum):
+        DIFF_16_8 = 1
+        DIFF_32_8 = 2
+        DIFF_32_16 = 3
+
     class CoreHeader(ctypes.LittleEndianStructure):
         _fields_ = [
             ("id_flags", ctypes.c_uint16),
@@ -88,13 +95,56 @@ def _buffer_pull(buffer: bytes, ctype: type[ctypes.LittleEndianStructure]):
         b = buffer[ctypes.sizeof(ctype) :]
         return v, b
 
+    @classmethod
+    def _diff_expand(cls, buffer: bytes, tdf_len: int, diff_type: DiffType, diff_num: int) -> tuple[int, bytes]:
+        t_in: type[ctypes._SimpleCData]
+        t_diff: type[ctypes._SimpleCData]
+        if diff_type == cls.DiffType.DIFF_16_8:
+            t_in = ctypes.c_uint16
+            t_diff = ctypes.c_int8
+        elif diff_type == cls.DiffType.DIFF_32_8:
+            t_in = ctypes.c_uint32
+            t_diff = ctypes.c_int8
+        elif diff_type == cls.DiffType.DIFF_32_16:
+            t_in = ctypes.c_uint32
+            t_diff = ctypes.c_int16
+        else:
+            raise RuntimeError(f"Unknown diff type {diff_type}")
+        num_fields = tdf_len // ctypes.sizeof(t_in)
+
+        class _tdf(ctypes.LittleEndianStructure):
+            _fields_ = [("data", num_fields * t_in)]
+            _pack_ = 1
+
+        class _diff(ctypes.LittleEndianStructure):
+            _fields_ = [("data", num_fields * t_diff)]
+            _pack_ = 1
+
+        class _complete(ctypes.LittleEndianStructure):
+            _fields_ = [("base", _tdf), ("diffs", diff_num * _diff)]
+            _pack_ = 1
+
+        raw = _complete.from_buffer_copy(buffer)
+        out: list[ctypes.LittleEndianStructure] = [_tdf.from_buffer_copy(buffer)]
+        for idx in range(diff_num):
+            next = copy.copy(out[-1])
+            for f in range(num_fields):
+                next.data[f] += raw.diffs[idx].data[f]
+            out.append(next)
+
+        expanded = b"".join([bytes(b) for b in out])
+        return ctypes.sizeof(raw), expanded
+
     def decode(self, buffer: bytes) -> Generator[Reading, None, None]:
         buffer_time = None
 
         while len(buffer) > 3:
             header, buffer = self._buffer_pull(buffer, self.CoreHeader)
             time_flags = header.id_flags & self.flags.TIMESTAMP_MASK
 
+            if header.id_flags in [0x0000, 0xFFFF]:
+                break
+
             tdf_id = header.id_flags & 0x0FFF
             try:
                 id_type = tdf_definitions.id_type_mapping[tdf_id]
@@ -119,7 +169,19 @@ def decode(self, buffer: bytes) -> Generator[Reading, None, None]:
                 raise RuntimeError("Unreachable time option")
 
             array_header = None
-            if header.id_flags & self.flags.TIME_ARRAY:
+            if header.id_flags & self.flags.DIFF_ARRAY:
+                array_header, buffer = self._buffer_pull(buffer, self.ArrayHeader)
+                diff_type = array_header.num >> 6
+                diff_num = array_header.num & 0x3F
+
+                total_len, expanded = self._diff_expand(buffer, header.len, self.DiffType(diff_type), diff_num)
+                buffer = buffer[total_len:]
+                assert buffer_time is not None
+                time = InfuseTime.unix_time_from_epoch(buffer_time)
+                data = [
+                    id_type.from_buffer_consume(expanded[x : x + header.len]) for x in range(0, total_len, header.len)
+                ]
+            elif header.id_flags & self.flags.TIME_ARRAY:
                 array_header, buffer = self._buffer_pull(buffer, self.ArrayHeader)
                 total_len = array_header.num * header.len
                 total_data = buffer[:total_len]

tests/tdf/test_tdf.py

@@ -0,0 +1,28 @@
+import os
+
+from infuse_iot.tdf import TDF
+
+# assert "TOXTEMPDIR" in os.environ, "you must run these tests using tox"
+
+TESTDATA_FILENAME = os.path.join(os.path.dirname(__file__), "tdf_example.bin")
+
+
+def test_tdf():
+    with open(TESTDATA_FILENAME, "rb") as f:
+        test_data = f.read(-1)
+
+    test_blocks = [test_data[i : (i + 512)] for i in range(0, len(test_data), 512)]
+
+    decoder = TDF()
+    total_tdfs = 0
+
+    # Iterate over each block
+    for block in test_blocks:
+        assert len(block) % 512 == 0
+        # Iterate over each TDF in the block
+        for tdf in decoder.decode(block):
+            assert isinstance(tdf, TDF.Reading)
+            total_tdfs += 1
+
+    # Number of TDFs on the example block should never change
+    assert total_tdfs == 53