diff --git a/pyproject.toml b/pyproject.toml index 0c54f09..d0af345 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,10 @@ content-type = "text/markdown" [project.scripts] infuse = "infuse_iot.app.main:main" +[project.urls] +Homepage = "https://github.com/Embeint/python-tools" +Issues = "https://github.com/Embeint/python-tools/issues" + [tool.setuptools] package-dir = {"" = "src"} zip-safe = false diff --git a/src/infuse_iot/tdf.py b/src/infuse_iot/tdf.py index 5e15ec9..22645ec 100644 --- a/src/infuse_iot/tdf.py +++ b/src/infuse_iot/tdf.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import copy import ctypes import enum from collections.abc import Generator @@ -29,8 +30,14 @@ class flags(enum.IntEnum): TIMESTAMP_EXTENDED_RELATIVE = 0xC000 TIMESTAMP_MASK = 0xC000 TIME_ARRAY = 0x1000 + DIFF_ARRAY = 0x2000 ID_MASK = 0x0FFF + class DiffType(enum.IntEnum): + DIFF_16_8 = 1 + DIFF_32_8 = 2 + DIFF_32_16 = 3 + class CoreHeader(ctypes.LittleEndianStructure): _fields_ = [ ("id_flags", ctypes.c_uint16), @@ -88,6 +95,46 @@ def _buffer_pull(buffer: bytes, ctype: type[ctypes.LittleEndianStructure]): b = buffer[ctypes.sizeof(ctype) :] return v, b + @classmethod + def _diff_expand(cls, buffer: bytes, tdf_len: int, diff_type: DiffType, diff_num: int) -> tuple[int, bytes]: + t_in: type[ctypes._SimpleCData] + t_diff: type[ctypes._SimpleCData] + if diff_type == cls.DiffType.DIFF_16_8: + t_in = ctypes.c_uint16 + t_diff = ctypes.c_int8 + elif diff_type == cls.DiffType.DIFF_32_8: + t_in = ctypes.c_uint32 + t_diff = ctypes.c_int8 + elif diff_type == cls.DiffType.DIFF_32_16: + t_in = ctypes.c_uint32 + t_diff = ctypes.c_int16 + else: + raise RuntimeError(f"Unknown diff type {diff_type}") + num_fields = tdf_len // ctypes.sizeof(t_in) + + class _tdf(ctypes.LittleEndianStructure): + _fields_ = [("data", num_fields * t_in)] + _pack_ = 1 + + class _diff(ctypes.LittleEndianStructure): + _fields_ = [("data", num_fields * t_diff)] + _pack_ = 1 + + class _complete(ctypes.LittleEndianStructure): + _fields_ = [("base", _tdf), ("diffs", diff_num * _diff)] + _pack_ = 1 + + raw = _complete.from_buffer_copy(buffer) + out: list[ctypes.LittleEndianStructure] = [_tdf.from_buffer_copy(buffer)] + for idx in range(diff_num): + next = copy.copy(out[-1]) + for f in range(num_fields): + next.data[f] += raw.diffs[idx].data[f] + out.append(next) + + expanded = b"".join([bytes(b) for b in out]) + return ctypes.sizeof(raw), expanded + def decode(self, buffer: bytes) -> Generator[Reading, None, None]: buffer_time = None @@ -95,6 +142,9 @@ def decode(self, buffer: bytes) -> Generator[Reading, None, None]: header, buffer = self._buffer_pull(buffer, self.CoreHeader) time_flags = header.id_flags & self.flags.TIMESTAMP_MASK + if header.id_flags in [0x0000, 0xFFFF]: + break + tdf_id = header.id_flags & 0x0FFF try: id_type = tdf_definitions.id_type_mapping[tdf_id] @@ -119,7 +169,19 @@ def decode(self, buffer: bytes) -> Generator[Reading, None, None]: raise RuntimeError("Unreachable time option") array_header = None - if header.id_flags & self.flags.TIME_ARRAY: + if header.id_flags & self.flags.DIFF_ARRAY: + array_header, buffer = self._buffer_pull(buffer, self.ArrayHeader) + diff_type = array_header.num >> 6 + diff_num = array_header.num & 0x3F + + total_len, expanded = self._diff_expand(buffer, header.len, self.DiffType(diff_type), diff_num) + buffer = buffer[total_len:] + assert buffer_time is not None + time = InfuseTime.unix_time_from_epoch(buffer_time) + data = [ + id_type.from_buffer_consume(expanded[x : x + header.len]) for x in range(0, total_len, header.len) + ] + elif header.id_flags & self.flags.TIME_ARRAY: array_header, buffer = self._buffer_pull(buffer, self.ArrayHeader) total_len = array_header.num * header.len total_data = buffer[:total_len] diff --git a/tests/tdf/tdf_example.bin b/tests/tdf/tdf_example.bin new file mode 100644 index 0000000..bf627c4 Binary files /dev/null and b/tests/tdf/tdf_example.bin differ diff --git a/tests/tdf/test_tdf.py b/tests/tdf/test_tdf.py new file mode 100644 index 0000000..d81e9b7 --- /dev/null +++ b/tests/tdf/test_tdf.py @@ -0,0 +1,28 @@ +import os + +from infuse_iot.tdf import TDF + +# assert "TOXTEMPDIR" in os.environ, "you must run these tests using tox" + +TESTDATA_FILENAME = os.path.join(os.path.dirname(__file__), "tdf_example.bin") + + +def test_tdf(): + with open(TESTDATA_FILENAME, "rb") as f: + test_data = f.read(-1) + + test_blocks = [test_data[i : (i + 512)] for i in range(0, len(test_data), 512)] + + decoder = TDF() + total_tdfs = 0 + + # Iterate over each block + for block in test_blocks: + assert len(block) % 512 == 0 + # Iterate over each TDF in the block + for tdf in decoder.decode(block): + assert isinstance(tdf, TDF.Reading) + total_tdfs += 1 + + # Number of TDFs on the example block should never change + assert total_tdfs == 53