diff --git a/src/infuse_iot/generated/tdf_definitions.py b/src/infuse_iot/generated/tdf_definitions.py index 3a4951f..c9b673b 100644 --- a/src/infuse_iot/generated/tdf_definitions.py +++ b/src/infuse_iot/generated/tdf_definitions.py @@ -1298,6 +1298,42 @@ class lora_tx(TdfReadingBase): "payload": "{}", } + class idx_array_freq(TdfReadingBase): + """Sample frequency metadata for a TDF_DATA_FORMAT_IDX_ARRAY array""" + + name = "IDX_ARRAY_FREQ" + _fields_ = [ + ("tdf_id", ctypes.c_uint16), + ("frequency", ctypes.c_uint32), + ] + _pack_ = 1 + _postfix_ = { + "tdf_id": "", + "frequency": "", + } + _display_fmt_ = { + "tdf_id": "{}", + "frequency": "{}", + } + + class idx_array_period(TdfReadingBase): + """Sample frequency metadata for a TDF_DATA_FORMAT_IDX_ARRAY array""" + + name = "IDX_ARRAY_PERIOD" + _fields_ = [ + ("tdf_id", ctypes.c_uint16), + ("period", ctypes.c_uint32), + ] + _pack_ = 1 + _postfix_ = { + "tdf_id": "", + "period": "", + } + _display_fmt_ = { + "tdf_id": "{}", + "period": "{}", + } + class array_type(TdfReadingBase): """Example array type""" @@ -1357,5 +1393,7 @@ class array_type(TdfReadingBase): 43: readings.annotation, 44: readings.lora_rx, 45: readings.lora_tx, + 46: readings.idx_array_freq, + 47: readings.idx_array_period, 100: readings.array_type, } diff --git a/src/infuse_iot/tdf.py b/src/infuse_iot/tdf.py index 6e1cd26..fc5c109 100644 --- a/src/infuse_iot/tdf.py +++ b/src/infuse_iot/tdf.py @@ -3,6 +3,7 @@ import copy import ctypes import enum +import time from collections.abc import Generator from infuse_iot.generated import tdf_base, tdf_definitions @@ -31,6 +32,8 @@ class flags(enum.IntEnum): TIMESTAMP_MASK = 0xC000 TIME_ARRAY = 0x1000 DIFF_ARRAY = 0x2000 + IDX_ARRAY = 0x3000 + ARRAY_MASK = 0x3000 ID_MASK = 0x0FFF class DiffType(enum.IntEnum): @@ -79,13 +82,15 @@ class Reading: def __init__( self, tdf_id: int, - time: None | float, + reading_time: None | float, period: None | float, + base_idx: None | int, data: list[tdf_base.TdfReadingBase], ): self.id = tdf_id - self.time = time + self.time = reading_time self.period = period + self.base_idx = base_idx self.data = data def __init__(self): @@ -137,7 +142,7 @@ class _complete(ctypes.LittleEndianStructure): expanded = b"".join([bytes(b) for b in out]) return ctypes.sizeof(raw), expanded - def decode(self, buffer: bytes) -> Generator[Reading, None, None]: + def decode(self, buffer: bytes, no_defs: bool = False) -> Generator[Reading, None, None]: buffer_time = None while len(buffer) > 3: @@ -148,49 +153,73 @@ def decode(self, buffer: bytes) -> Generator[Reading, None, None]: break tdf_id = header.id_flags & 0x0FFF - try: - id_type = tdf_definitions.id_type_mapping[tdf_id] - except KeyError: + if no_defs: id_type = unknown_tdf_factory(tdf_id, header.len) + else: + try: + id_type = tdf_definitions.id_type_mapping[tdf_id] + except KeyError: + id_type = unknown_tdf_factory(tdf_id, header.len) if time_flags == self.flags.TIMESTAMP_NONE: - time = None + reading_time = None elif time_flags == self.flags.TIMESTAMP_ABSOLUTE: t, buffer = self._buffer_pull(buffer, self.AbsoluteTime) buffer_time = t.seconds * 65536 + t.subseconds - time = InfuseTime.unix_time_from_epoch(buffer_time) + reading_time = InfuseTime.unix_time_from_epoch(buffer_time) elif time_flags == self.flags.TIMESTAMP_RELATIVE: t, buffer = self._buffer_pull(buffer, self.RelativeTime) buffer_time += t.offset - time = InfuseTime.unix_time_from_epoch(buffer_time) + reading_time = InfuseTime.unix_time_from_epoch(buffer_time) elif time_flags == self.flags.TIMESTAMP_EXTENDED_RELATIVE: t, buffer = self._buffer_pull(buffer, self.ExtendedRelativeTime) buffer_time += t.offset - time = InfuseTime.unix_time_from_epoch(buffer_time) + reading_time = InfuseTime.unix_time_from_epoch(buffer_time) else: raise RuntimeError("Unreachable time option") array_header = None - if header.id_flags & self.flags.DIFF_ARRAY: + base_idx = None + array_type = header.id_flags & self.flags.ARRAY_MASK + if array_type == self.flags.DIFF_ARRAY: array_header, buffer = self._buffer_pull(buffer, self.ArrayHeader) diff_type = array_header.num >> 6 diff_num = array_header.num & 0x3F total_len, expanded = self._diff_expand(buffer, header.len, self.DiffType(diff_type), diff_num) buffer = buffer[total_len:] - assert buffer_time is not None - time = InfuseTime.unix_time_from_epoch(buffer_time) + if buffer_time is None: + t_now = int(time.time() * 65536) + reading_time = InfuseTime.unix_time_from_epoch(t_now) + else: + reading_time = InfuseTime.unix_time_from_epoch(buffer_time) data = [ id_type.from_buffer_consume(expanded[x : x + header.len]) for x in range(0, total_len, header.len) ] - elif header.id_flags & self.flags.TIME_ARRAY: + elif array_type == self.flags.TIME_ARRAY: + array_header, buffer = self._buffer_pull(buffer, self.ArrayHeader) + total_len = array_header.num * header.len + total_data = buffer[:total_len] + buffer = buffer[total_len:] + + if buffer_time is None: + t_now = int(time.time() * 65536) + reading_time = InfuseTime.unix_time_from_epoch(t_now) + else: + reading_time = InfuseTime.unix_time_from_epoch(buffer_time) + data = [ + id_type.from_buffer_consume(total_data[x : x + header.len]) for x in range(0, total_len, header.len) + ] + elif array_type == self.flags.IDX_ARRAY: array_header, buffer = self._buffer_pull(buffer, self.ArrayHeader) total_len = array_header.num * header.len total_data = buffer[:total_len] buffer = buffer[total_len:] - assert buffer_time is not None - time = InfuseTime.unix_time_from_epoch(buffer_time) + if time_flags != self.flags.TIMESTAMP_NONE: + assert buffer_time is not None + reading_time = InfuseTime.unix_time_from_epoch(buffer_time) + base_idx = array_header.period data = [ id_type.from_buffer_consume(total_data[x : x + header.len]) for x in range(0, total_len, header.len) ] @@ -201,7 +230,29 @@ def decode(self, buffer: bytes) -> Generator[Reading, None, None]: data = [id_type.from_buffer_consume(data_bytes)] period = None - if array_header is not None: + if array_header is not None and base_idx is None: period = array_header.period / 65536 - yield self.Reading(tdf_id, time, period, data) + yield self.Reading(tdf_id, reading_time, period, base_idx, data) + + +if __name__ == "__main__": + import sys + + if len(sys.argv) != 2: + sys.exit("Expected `python -m infuse_iot.tdf /path/to/tdf.bin`") + + decoder = TDF() + + with open(sys.argv[1], "rb") as f: + tdf_binary_blob = f.read(-1) + + # Check data is in the form we expect (single block, 2 byte prefix) + if tdf_binary_blob[1] != 0x02: + sys.exit("Expected second byte to be 0x02 (INFUSE_TDF)") + + try: + for tdf in decoder.decode(tdf_binary_blob[2:]): + print(f"TDF {tdf.id} @ t={tdf.time}: {tdf.data}") + except Exception: + pass diff --git a/src/infuse_iot/tools/tdf_csv.py b/src/infuse_iot/tools/tdf_csv.py index 9969ab2..a8a0c64 100644 --- a/src/infuse_iot/tools/tdf_csv.py +++ b/src/infuse_iot/tools/tdf_csv.py @@ -54,13 +54,18 @@ def run(self): # Construct reading strings lines = [] reading_time = tdf.time - for reading in tdf.data: + for idx, reading in enumerate(tdf.data): if self.args.unix: time_func = _to_str else: time_func = InfuseTime.utc_time_string_log - if reading_time is None: + if tdf.base_idx is not None: + if reading_time is None or idx > 0: + time_str = f"{tdf.base_idx + idx}" + else: + time_str = time_func(reading_time) + elif reading_time is None: # Log with local time time_str = time_func(time.time()) else: diff --git a/src/infuse_iot/tools/tdf_list.py b/src/infuse_iot/tools/tdf_list.py index 4886456..7ef4caf 100644 --- a/src/infuse_iot/tools/tdf_list.py +++ b/src/infuse_iot/tools/tdf_list.py @@ -59,7 +59,10 @@ def run(self) -> None: offset = (len(tdf.data) - 1) * tdf.period time_str = InfuseTime.utc_time_string(tdf.time + offset) else: - time_str = InfuseTime.utc_time_string(time.time()) + if tdf.base_idx is not None: + time_str = f"IDX {tdf.base_idx}" + else: + time_str = InfuseTime.utc_time_string(time.time()) for field in t.iter_fields(): if isinstance(field.val, list): diff --git a/tests/tdf/test_tdf.py b/tests/tdf/test_tdf.py index d81e9b7..a88716e 100644 --- a/tests/tdf/test_tdf.py +++ b/tests/tdf/test_tdf.py @@ -26,3 +26,38 @@ def test_tdf(): # Number of TDFs on the example block should never change assert total_tdfs == 53 + + +def test_buffers(): + # Valid buffers generated by `infuse-sdk/tests/subsys/tdf` + hex_strings = [ + "6410040264003bdb31ccb6d508aa641004020a003bdb31ccb6d508aa", + "a00f103bdb31ccb6d508aad2e1be87e47b88ae", + "d0071d3bdb31ccb6d508aad2e1be87e47b88aeb19de6889efb2d975016f72831", + "6410040210803bdb31ccb6d508aa6410040210803bdb31ccb6d508aa", + "6400043bdb31cc64400440420f0000003bdb31cc", + "6430040100003bdb31cc64700440420f0000000101003bdb31cc", + "64400440420f0000003bdb31cc6400043bdb31cc", + "64700440420f0000000100003bdb31cc6430040101003bdb31cc", + "6410040696003bdb31ccb6d508aad2e1be87e47b88aeb19de6889efb2d97", + "6e400440420f0000003bdb31cc6fc004a086013bdb31cc", + "6e700440420f0000000100003bdb31cc6ff004a086010101003bdb31cc", + "32400640420f0000003bdb31ccb6d537c004ffffff3bdb31cc", + "32700640420f0000000100003bdb31ccb6d537f004ffffff0101003bdb31cc", + "14400440420f0000003bdb31cc1340043f421000ffff3bdb31cc", + "64400440420f0000003bdb31cc13400440420e0001003bdb31cc", + "64400440420f0000003bdb31cc648004ffff3bdb31cc648004ffff3bdb31cc", + "6400043bdb31cc6400043bdb31cc", + "6430040100003bdb31cc6430040101003bdb31cc", + "65400440420f0000003bdb31cc66800400003bdb31cc67800464003bdb31cc", + ] + decoder = TDF() + + for hex_string in hex_strings: + total_tdfs = 0 + + # Iterate over each TDF in the buffer (not necessarily valid definition matches) + for tdf in decoder.decode(bytes.fromhex(hex_string), no_defs=True): + assert isinstance(tdf, TDF.Reading) + total_tdfs += 1 + assert total_tdfs > 0