From 19b2c55f6de3ea3433f416790320783080ef4a40 Mon Sep 17 00:00:00 2001 From: Jordan Yates Date: Sun, 29 Jun 2025 20:40:35 +1000 Subject: [PATCH 1/6] infuse_iot: generated: regenerate Regenerate the base TDF definitions. Signed-off-by: Jordan Yates --- src/infuse_iot/generated/tdf_definitions.py | 38 +++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/infuse_iot/generated/tdf_definitions.py b/src/infuse_iot/generated/tdf_definitions.py index 3a4951f..c9b673b 100644 --- a/src/infuse_iot/generated/tdf_definitions.py +++ b/src/infuse_iot/generated/tdf_definitions.py @@ -1298,6 +1298,42 @@ class lora_tx(TdfReadingBase): "payload": "{}", } + class idx_array_freq(TdfReadingBase): + """Sample frequency metadata for a TDF_DATA_FORMAT_IDX_ARRAY array""" + + name = "IDX_ARRAY_FREQ" + _fields_ = [ + ("tdf_id", ctypes.c_uint16), + ("frequency", ctypes.c_uint32), + ] + _pack_ = 1 + _postfix_ = { + "tdf_id": "", + "frequency": "", + } + _display_fmt_ = { + "tdf_id": "{}", + "frequency": "{}", + } + + class idx_array_period(TdfReadingBase): + """Sample frequency metadata for a TDF_DATA_FORMAT_IDX_ARRAY array""" + + name = "IDX_ARRAY_PERIOD" + _fields_ = [ + ("tdf_id", ctypes.c_uint16), + ("period", ctypes.c_uint32), + ] + _pack_ = 1 + _postfix_ = { + "tdf_id": "", + "period": "", + } + _display_fmt_ = { + "tdf_id": "{}", + "period": "{}", + } + class array_type(TdfReadingBase): """Example array type""" @@ -1357,5 +1393,7 @@ class array_type(TdfReadingBase): 43: readings.annotation, 44: readings.lora_rx, 45: readings.lora_tx, + 46: readings.idx_array_freq, + 47: readings.idx_array_period, 100: readings.array_type, } From dc46e37b3a6991a758e751cb1429e59a17fb5331 Mon Sep 17 00:00:00 2001 From: Jordan Yates Date: Sun, 29 Jun 2025 20:42:28 +1000 Subject: [PATCH 2/6] infuse_iot: tdf: support index arrays Add support for decoding index arrays. Signed-off-by: Jordan Yates --- src/infuse_iot/tdf.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/infuse_iot/tdf.py b/src/infuse_iot/tdf.py index 6e1cd26..250a900 100644 --- a/src/infuse_iot/tdf.py +++ b/src/infuse_iot/tdf.py @@ -31,6 +31,8 @@ class flags(enum.IntEnum): TIMESTAMP_MASK = 0xC000 TIME_ARRAY = 0x1000 DIFF_ARRAY = 0x2000 + IDX_ARRAY = 0x3000 + ARRAY_MASK = 0x3000 ID_MASK = 0x0FFF class DiffType(enum.IntEnum): @@ -81,11 +83,13 @@ def __init__( tdf_id: int, time: None | float, period: None | float, + base_idx: None | int, data: list[tdf_base.TdfReadingBase], ): self.id = tdf_id self.time = time self.period = period + self.base_idx = base_idx self.data = data def __init__(self): @@ -171,7 +175,9 @@ def decode(self, buffer: bytes) -> Generator[Reading, None, None]: raise RuntimeError("Unreachable time option") array_header = None - if header.id_flags & self.flags.DIFF_ARRAY: + base_idx = None + array_type = header.id_flags & self.flags.ARRAY_MASK + if array_type == self.flags.DIFF_ARRAY: array_header, buffer = self._buffer_pull(buffer, self.ArrayHeader) diff_type = array_header.num >> 6 diff_num = array_header.num & 0x3F @@ -183,7 +189,7 @@ def decode(self, buffer: bytes) -> Generator[Reading, None, None]: data = [ id_type.from_buffer_consume(expanded[x : x + header.len]) for x in range(0, total_len, header.len) ] - elif header.id_flags & self.flags.TIME_ARRAY: + elif array_type == self.flags.TIME_ARRAY: array_header, buffer = self._buffer_pull(buffer, self.ArrayHeader) total_len = array_header.num * header.len total_data = buffer[:total_len] @@ -194,6 +200,19 @@ def decode(self, buffer: bytes) -> Generator[Reading, None, None]: data = [ id_type.from_buffer_consume(total_data[x : x + header.len]) for x in range(0, total_len, header.len) ] + elif array_type == self.flags.IDX_ARRAY: + array_header, buffer = self._buffer_pull(buffer, self.ArrayHeader) + total_len = array_header.num * header.len + total_data = buffer[:total_len] + buffer = buffer[total_len:] + + if time_flags != self.flags.TIMESTAMP_NONE: + assert buffer_time is not None + time = InfuseTime.unix_time_from_epoch(buffer_time) + base_idx = array_header.period + data = [ + id_type.from_buffer_consume(total_data[x : x + header.len]) for x in range(0, total_len, header.len) + ] else: data_bytes = buffer[: header.len] buffer = buffer[header.len :] @@ -201,7 +220,7 @@ def decode(self, buffer: bytes) -> Generator[Reading, None, None]: data = [id_type.from_buffer_consume(data_bytes)] period = None - if array_header is not None: + if array_header is not None and base_idx is None: period = array_header.period / 65536 - yield self.Reading(tdf_id, time, period, data) + yield self.Reading(tdf_id, time, period, base_idx, data) From 689cd36ae2c6c24fb1ebb3b9fd3ce7889cd0af4c Mon Sep 17 00:00:00 2001 From: Jordan Yates Date: Sun, 29 Jun 2025 20:43:08 +1000 Subject: [PATCH 3/6] infuse_iot: tools: support TDF index arrays Add support for TDF index arrays to `tdf_list` and `tdf_csv`. Signed-off-by: Jordan Yates --- src/infuse_iot/tools/tdf_csv.py | 9 +++++++-- src/infuse_iot/tools/tdf_list.py | 5 ++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/infuse_iot/tools/tdf_csv.py b/src/infuse_iot/tools/tdf_csv.py index 9969ab2..a8a0c64 100644 --- a/src/infuse_iot/tools/tdf_csv.py +++ b/src/infuse_iot/tools/tdf_csv.py @@ -54,13 +54,18 @@ def run(self): # Construct reading strings lines = [] reading_time = tdf.time - for reading in tdf.data: + for idx, reading in enumerate(tdf.data): if self.args.unix: time_func = _to_str else: time_func = InfuseTime.utc_time_string_log - if reading_time is None: + if tdf.base_idx is not None: + if reading_time is None or idx > 0: + time_str = f"{tdf.base_idx + idx}" + else: + time_str = time_func(reading_time) + elif reading_time is None: # Log with local time time_str = time_func(time.time()) else: diff --git a/src/infuse_iot/tools/tdf_list.py b/src/infuse_iot/tools/tdf_list.py index 4886456..7ef4caf 100644 --- a/src/infuse_iot/tools/tdf_list.py +++ b/src/infuse_iot/tools/tdf_list.py @@ -59,7 +59,10 @@ def run(self) -> None: offset = (len(tdf.data) - 1) * tdf.period time_str = InfuseTime.utc_time_string(tdf.time + offset) else: - time_str = InfuseTime.utc_time_string(time.time()) + if tdf.base_idx is not None: + time_str = f"IDX {tdf.base_idx}" + else: + time_str = InfuseTime.utc_time_string(time.time()) for field in t.iter_fields(): if isinstance(field.val, list): From 751afbfdc454be831506fc460ca71a15e8c90ddc Mon Sep 17 00:00:00 2001 From: Jordan Yates Date: Sun, 29 Jun 2025 20:44:47 +1000 Subject: [PATCH 4/6] infuse_iot: tdf: runnable from command line Allow `infuse_iot.tdf` to be run from the command line, if given a path to a binary file. Signed-off-by: Jordan Yates --- src/infuse_iot/tdf.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/infuse_iot/tdf.py b/src/infuse_iot/tdf.py index 250a900..34b81cc 100644 --- a/src/infuse_iot/tdf.py +++ b/src/infuse_iot/tdf.py @@ -224,3 +224,25 @@ def decode(self, buffer: bytes) -> Generator[Reading, None, None]: period = array_header.period / 65536 yield self.Reading(tdf_id, time, period, base_idx, data) + + +if __name__ == "__main__": + import sys + + if len(sys.argv) != 2: + sys.exit("Expected `python -m infuse_iot.tdf /path/to/tdf.bin`") + + decoder = TDF() + + with open(sys.argv[1], "rb") as f: + tdf_binary_blob = f.read(-1) + + # Check data is in the form we expect (single block, 2 byte prefix) + if tdf_binary_blob[1] != 0x02: + sys.exit("Expected second byte to be 0x02 (INFUSE_TDF)") + + try: + for tdf in decoder.decode(tdf_binary_blob[2:]): + print(f"TDF {tdf.id} @ t={tdf.time}: {tdf.data}") + except Exception: + pass From 4004ecddc554b338fd9aac7b16ebcd8805a680b1 Mon Sep 17 00:00:00 2001 From: Jordan Yates Date: Sun, 29 Jun 2025 21:04:59 +1000 Subject: [PATCH 5/6] infuse_iot: tdf: more permissive timestamping Allow array types without a base timestamp for decoding. Signed-off-by: Jordan Yates --- src/infuse_iot/tdf.py | 42 ++++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/src/infuse_iot/tdf.py b/src/infuse_iot/tdf.py index 34b81cc..fc5c109 100644 --- a/src/infuse_iot/tdf.py +++ b/src/infuse_iot/tdf.py @@ -3,6 +3,7 @@ import copy import ctypes import enum +import time from collections.abc import Generator from infuse_iot.generated import tdf_base, tdf_definitions @@ -81,13 +82,13 @@ class Reading: def __init__( self, tdf_id: int, - time: None | float, + reading_time: None | float, period: None | float, base_idx: None | int, data: list[tdf_base.TdfReadingBase], ): self.id = tdf_id - self.time = time + self.time = reading_time self.period = period self.base_idx = base_idx self.data = data @@ -141,7 +142,7 @@ class _complete(ctypes.LittleEndianStructure): expanded = b"".join([bytes(b) for b in out]) return ctypes.sizeof(raw), expanded - def decode(self, buffer: bytes) -> Generator[Reading, None, None]: + def decode(self, buffer: bytes, no_defs: bool = False) -> Generator[Reading, None, None]: buffer_time = None while len(buffer) > 3: @@ -152,25 +153,28 @@ def decode(self, buffer: bytes) -> Generator[Reading, None, None]: break tdf_id = header.id_flags & 0x0FFF - try: - id_type = tdf_definitions.id_type_mapping[tdf_id] - except KeyError: + if no_defs: id_type = unknown_tdf_factory(tdf_id, header.len) + else: + try: + id_type = tdf_definitions.id_type_mapping[tdf_id] + except KeyError: + id_type = unknown_tdf_factory(tdf_id, header.len) if time_flags == self.flags.TIMESTAMP_NONE: - time = None + reading_time = None elif time_flags == self.flags.TIMESTAMP_ABSOLUTE: t, buffer = self._buffer_pull(buffer, self.AbsoluteTime) buffer_time = t.seconds * 65536 + t.subseconds - time = InfuseTime.unix_time_from_epoch(buffer_time) + reading_time = InfuseTime.unix_time_from_epoch(buffer_time) elif time_flags == self.flags.TIMESTAMP_RELATIVE: t, buffer = self._buffer_pull(buffer, self.RelativeTime) buffer_time += t.offset - time = InfuseTime.unix_time_from_epoch(buffer_time) + reading_time = InfuseTime.unix_time_from_epoch(buffer_time) elif time_flags == self.flags.TIMESTAMP_EXTENDED_RELATIVE: t, buffer = self._buffer_pull(buffer, self.ExtendedRelativeTime) buffer_time += t.offset - time = InfuseTime.unix_time_from_epoch(buffer_time) + reading_time = InfuseTime.unix_time_from_epoch(buffer_time) else: raise RuntimeError("Unreachable time option") @@ -184,8 +188,11 @@ def decode(self, buffer: bytes) -> Generator[Reading, None, None]: total_len, expanded = self._diff_expand(buffer, header.len, self.DiffType(diff_type), diff_num) buffer = buffer[total_len:] - assert buffer_time is not None - time = InfuseTime.unix_time_from_epoch(buffer_time) + if buffer_time is None: + t_now = int(time.time() * 65536) + reading_time = InfuseTime.unix_time_from_epoch(t_now) + else: + reading_time = InfuseTime.unix_time_from_epoch(buffer_time) data = [ id_type.from_buffer_consume(expanded[x : x + header.len]) for x in range(0, total_len, header.len) ] @@ -195,8 +202,11 @@ def decode(self, buffer: bytes) -> Generator[Reading, None, None]: total_data = buffer[:total_len] buffer = buffer[total_len:] - assert buffer_time is not None - time = InfuseTime.unix_time_from_epoch(buffer_time) + if buffer_time is None: + t_now = int(time.time() * 65536) + reading_time = InfuseTime.unix_time_from_epoch(t_now) + else: + reading_time = InfuseTime.unix_time_from_epoch(buffer_time) data = [ id_type.from_buffer_consume(total_data[x : x + header.len]) for x in range(0, total_len, header.len) ] @@ -208,7 +218,7 @@ def decode(self, buffer: bytes) -> Generator[Reading, None, None]: if time_flags != self.flags.TIMESTAMP_NONE: assert buffer_time is not None - time = InfuseTime.unix_time_from_epoch(buffer_time) + reading_time = InfuseTime.unix_time_from_epoch(buffer_time) base_idx = array_header.period data = [ id_type.from_buffer_consume(total_data[x : x + header.len]) for x in range(0, total_len, header.len) @@ -223,7 +233,7 @@ def decode(self, buffer: bytes) -> Generator[Reading, None, None]: if array_header is not None and base_idx is None: period = array_header.period / 65536 - yield self.Reading(tdf_id, time, period, base_idx, data) + yield self.Reading(tdf_id, reading_time, period, base_idx, data) if __name__ == "__main__": From 9a8715081194fd5def19996c7b179c25d0e50822 Mon Sep 17 00:00:00 2001 From: Jordan Yates Date: Sun, 29 Jun 2025 21:05:57 +1000 Subject: [PATCH 6/6] tests: tdf: test more data buffers Increase the code paths tested by running the decoder over a range of valid TDF buffers created by the embedded test cases. Signed-off-by: Jordan Yates --- tests/tdf/test_tdf.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/tdf/test_tdf.py b/tests/tdf/test_tdf.py index d81e9b7..a88716e 100644 --- a/tests/tdf/test_tdf.py +++ b/tests/tdf/test_tdf.py @@ -26,3 +26,38 @@ def test_tdf(): # Number of TDFs on the example block should never change assert total_tdfs == 53 + + +def test_buffers(): + # Valid buffers generated by `infuse-sdk/tests/subsys/tdf` + hex_strings = [ + "6410040264003bdb31ccb6d508aa641004020a003bdb31ccb6d508aa", + "a00f103bdb31ccb6d508aad2e1be87e47b88ae", + "d0071d3bdb31ccb6d508aad2e1be87e47b88aeb19de6889efb2d975016f72831", + "6410040210803bdb31ccb6d508aa6410040210803bdb31ccb6d508aa", + "6400043bdb31cc64400440420f0000003bdb31cc", + "6430040100003bdb31cc64700440420f0000000101003bdb31cc", + "64400440420f0000003bdb31cc6400043bdb31cc", + "64700440420f0000000100003bdb31cc6430040101003bdb31cc", + "6410040696003bdb31ccb6d508aad2e1be87e47b88aeb19de6889efb2d97", + "6e400440420f0000003bdb31cc6fc004a086013bdb31cc", + "6e700440420f0000000100003bdb31cc6ff004a086010101003bdb31cc", + "32400640420f0000003bdb31ccb6d537c004ffffff3bdb31cc", + "32700640420f0000000100003bdb31ccb6d537f004ffffff0101003bdb31cc", + "14400440420f0000003bdb31cc1340043f421000ffff3bdb31cc", + "64400440420f0000003bdb31cc13400440420e0001003bdb31cc", + "64400440420f0000003bdb31cc648004ffff3bdb31cc648004ffff3bdb31cc", + "6400043bdb31cc6400043bdb31cc", + "6430040100003bdb31cc6430040101003bdb31cc", + "65400440420f0000003bdb31cc66800400003bdb31cc67800464003bdb31cc", + ] + decoder = TDF() + + for hex_string in hex_strings: + total_tdfs = 0 + + # Iterate over each TDF in the buffer (not necessarily valid definition matches) + for tdf in decoder.decode(bytes.fromhex(hex_string), no_defs=True): + assert isinstance(tdf, TDF.Reading) + total_tdfs += 1 + assert total_tdfs > 0