From 320b517ee4454cc16f9d5fda8d72b005e6d1abf2 Mon Sep 17 00:00:00 2001 From: Jordan Yates Date: Sun, 19 Jan 2025 11:47:33 +1000 Subject: [PATCH 1/4] pyproject.toml: add project URLs Add links to the Github repository in the package metadata. Signed-off-by: Jordan Yates --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 0c54f09..d0af345 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,10 @@ content-type = "text/markdown" [project.scripts] infuse = "infuse_iot.app.main:main" +[project.urls] +Homepage = "https://github.com/Embeint/python-tools" +Issues = "https://github.com/Embeint/python-tools/issues" + [tool.setuptools] package-dir = {"" = "src"} zip-safe = false From 501f83c9b0bfad2648650ba9962bc3d291448e47 Mon Sep 17 00:00:00 2001 From: Jordan Yates Date: Sun, 19 Jan 2025 12:16:38 +1000 Subject: [PATCH 2/4] tdf: detect early end of data Skip block decoding when the header bits are all '0' or '1'. Signed-off-by: Jordan Yates --- src/infuse_iot/tdf.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/infuse_iot/tdf.py b/src/infuse_iot/tdf.py index 5e15ec9..d6341fc 100644 --- a/src/infuse_iot/tdf.py +++ b/src/infuse_iot/tdf.py @@ -95,6 +95,9 @@ def decode(self, buffer: bytes) -> Generator[Reading, None, None]: header, buffer = self._buffer_pull(buffer, self.CoreHeader) time_flags = header.id_flags & self.flags.TIMESTAMP_MASK + if header.id_flags in [0x0000, 0xFFFF]: + break + tdf_id = header.id_flags & 0x0FFF try: id_type = tdf_definitions.id_type_mapping[tdf_id] From eb2df58c616e1a98341a08c2b41882a49cb10694 Mon Sep 17 00:00:00 2001 From: Jordan Yates Date: Sun, 19 Jan 2025 11:41:09 +1000 Subject: [PATCH 3/4] tdf: support `TDF_DIFF_ARRAY` Add support for decoding `TDF_DIFF_ARRAY` data. Signed-off-by: Jordan Yates --- src/infuse_iot/tdf.py | 61 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/src/infuse_iot/tdf.py b/src/infuse_iot/tdf.py index d6341fc..22645ec 100644 --- a/src/infuse_iot/tdf.py +++ b/src/infuse_iot/tdf.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import copy import ctypes import enum from collections.abc import Generator @@ -29,8 +30,14 @@ class flags(enum.IntEnum): TIMESTAMP_EXTENDED_RELATIVE = 0xC000 TIMESTAMP_MASK = 0xC000 TIME_ARRAY = 0x1000 + DIFF_ARRAY = 0x2000 ID_MASK = 0x0FFF + class DiffType(enum.IntEnum): + DIFF_16_8 = 1 + DIFF_32_8 = 2 + DIFF_32_16 = 3 + class CoreHeader(ctypes.LittleEndianStructure): _fields_ = [ ("id_flags", ctypes.c_uint16), @@ -88,6 +95,46 @@ def _buffer_pull(buffer: bytes, ctype: type[ctypes.LittleEndianStructure]): b = buffer[ctypes.sizeof(ctype) :] return v, b + @classmethod + def _diff_expand(cls, buffer: bytes, tdf_len: int, diff_type: DiffType, diff_num: int) -> tuple[int, bytes]: + t_in: type[ctypes._SimpleCData] + t_diff: type[ctypes._SimpleCData] + if diff_type == cls.DiffType.DIFF_16_8: + t_in = ctypes.c_uint16 + t_diff = ctypes.c_int8 + elif diff_type == cls.DiffType.DIFF_32_8: + t_in = ctypes.c_uint32 + t_diff = ctypes.c_int8 + elif diff_type == cls.DiffType.DIFF_32_16: + t_in = ctypes.c_uint32 + t_diff = ctypes.c_int16 + else: + raise RuntimeError(f"Unknown diff type {diff_type}") + num_fields = tdf_len // ctypes.sizeof(t_in) + + class _tdf(ctypes.LittleEndianStructure): + _fields_ = [("data", num_fields * t_in)] + _pack_ = 1 + + class _diff(ctypes.LittleEndianStructure): + _fields_ = [("data", num_fields * t_diff)] + _pack_ = 1 + + class _complete(ctypes.LittleEndianStructure): + _fields_ = [("base", _tdf), ("diffs", diff_num * _diff)] + _pack_ = 1 + + raw = _complete.from_buffer_copy(buffer) + out: list[ctypes.LittleEndianStructure] = [_tdf.from_buffer_copy(buffer)] + for idx in range(diff_num): + next = copy.copy(out[-1]) + for f in range(num_fields): + next.data[f] += raw.diffs[idx].data[f] + out.append(next) + + expanded = b"".join([bytes(b) for b in out]) + return ctypes.sizeof(raw), expanded + def decode(self, buffer: bytes) -> Generator[Reading, None, None]: buffer_time = None @@ -122,7 +169,19 @@ def decode(self, buffer: bytes) -> Generator[Reading, None, None]: raise RuntimeError("Unreachable time option") array_header = None - if header.id_flags & self.flags.TIME_ARRAY: + if header.id_flags & self.flags.DIFF_ARRAY: + array_header, buffer = self._buffer_pull(buffer, self.ArrayHeader) + diff_type = array_header.num >> 6 + diff_num = array_header.num & 0x3F + + total_len, expanded = self._diff_expand(buffer, header.len, self.DiffType(diff_type), diff_num) + buffer = buffer[total_len:] + assert buffer_time is not None + time = InfuseTime.unix_time_from_epoch(buffer_time) + data = [ + id_type.from_buffer_consume(expanded[x : x + header.len]) for x in range(0, total_len, header.len) + ] + elif header.id_flags & self.flags.TIME_ARRAY: array_header, buffer = self._buffer_pull(buffer, self.ArrayHeader) total_len = array_header.num * header.len total_data = buffer[:total_len] From 6525970a451471cadc58eaf67165c27def813c7d Mon Sep 17 00:00:00 2001 From: Jordan Yates Date: Sun, 19 Jan 2025 12:00:03 +1000 Subject: [PATCH 4/4] tests: tdf: add simple parsing check Add a basic check that parsing works as expected. Signed-off-by: Jordan Yates --- tests/tdf/tdf_example.bin | Bin 0 -> 5120 bytes tests/tdf/test_tdf.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 tests/tdf/tdf_example.bin create mode 100644 tests/tdf/test_tdf.py diff --git a/tests/tdf/tdf_example.bin b/tests/tdf/tdf_example.bin new file mode 100644 index 0000000000000000000000000000000000000000..bf627c4ddaea863171ec80810be07e83e15cd360 GIT binary patch literal 5120 zcmeHLYnW8UnXal+buQgK-90_`>FGY*GmE(3A{bqWCW4?*1Scp+z>GLgG>8Te)QDmt z6QYCwDoPwKDnU^|0nr#ilmNPlqJSDeW>EnFxeUWhUuL@dT&ilnGc22ZeDH_)wLi9d z=A6@~>eQ+4d%ySl>aEg24I2_0MxlW%k$cfe4VMr#)_=}nf~w;}B6J>l{pf{1pB`J* zv3`M+J-LumE*}ytcM117ET>bqz(TY=W2-(NPuVy~ zTsOQ-MF=M7M-1hmv-fU^kt4Cd1ZKenQHbC!(NvLl2}2TGfhj@~ni1j%DG0%IyDy<% zf+J!JmB}3VM><5M;4$h*;aLJMl2WmKm|Cjqa8+>N0)!|o%D%Q%vhD7L@~ z^9Y2XA9zI?9B`81M)gg=zpVsjNg?=ik`U~`JP1TEOwBukLpcO(cenx{Ah>}N5YgR1 zD zyQp6rK?5epI2!D3N5d85>opajC37%}RQr)?u$g$RS%ONNn%>>sv@rH3uix1zl%dF( z$Nq7YP2iUQJ0IY`hyRs_hj_W=VZ6?STx$pXV)se^e6=auCqci&jdGq@EgSJSweQec zGQqqqSce;GKA>OVq2`ZSNO>`#vn$9h^E@^f-(;?*kCUep`;-aFmL7w&W-%)M3Hw-9 znM2h-kgJnOTR|R6UaVO-Svyy~K|YY&phoy0)70Lw)YvoH!vzu>tqm`rSU)vY_$;=V zC9G-jh_<@$=lCWqoj2mQs?%~4;wkl1*V@>J^pY+SJ74M7xg&N`>DR?;J|Qo5Hpjlf zGdkTfiVMZ?D{wwlMi%FjP4V4ofD#6JeHjvJs~d5HblpW0h#$VW4WH*7u&`c zr2iVLmrWU2bA=e0T@?$79hpbt3q)OZS?p^0Y<6gD34bM9ABW`D)FHuV3Q~~;AuZDx zs%kv!`K%^EH=-v|mphxxB{TEqTURS0$aU&b!C zLM3C=m6>6Ii*y+tp>Fi>A0GjrJKi+5`QB*T9=JO&JoQE>>OeL&W=0#$P6-$zlN_NH7F-$;$N#*xyLXKrW}>rUet^}RhZvLe@=F* zb!?X$#rA6-ksj<;BdJ2}z2Lz7_*q6Lx*0v^f^##v@8Ih~hn9XnM*BZi`%IJ0zA{?x8dgsp|vzGR_LWRI&IZ~x5Qm?(0$h~6>H-63v|{-@Jl zv_!-9m-4oFOX2VMKN7>NHDqqG#2$c?$$xXdKzEw&@U6&iu7R@F&rCT!_GRKFyM?KB zFF8oC(g#d>tENx?-fJnw@f|wm;q%J*z)q4ng~jy~%6wXrr=rqB38*DJhl9 zwF}H1Y@EhpPtd6TllT$#jPag%TGO=M<`u>W_L2F#QN#XX-fS#qubXch6Vx^F8~h>7 zjConeKSF&A@PFuT1@eDj5WBTzgMXy*TC}og8JQ3bf2t-fmlq{y3A<4{sy=Om=ysI{ zFJl2;eIRNa_V>_w8?*E%YcO`uNxm127R3xcK!+*Cfp(`KHG?y)IySxZFkeWgl`X}0 z(s9KX;OFSazSnq?HtD0B3L5q8;{(*mp+yuMN6KfbeT>R-P2QoeD!MD*qTb@WEB6E| z^{=;PDb0cL_B)uCw&X5jPgid4{K)8tROgtl#DBfhpiT??N$w?!!^fnBR~2_U*C)3Xs*8lLaN%DR)>h#5Okl2;Auw(Jpb#m|Nua9lu&t+d~{lZ<74W5jmt5WwI`AE5`ZR^2nmFAQAulL|RCubkK zTdqESYbz41hwIxaBxy2cW_JE}y%?%A`9?&%it%Tt82PI2C^SF#pF;DG2hS9mGmk-` zIfKZz3e8z&{L_V|bWp?FfdAxy_t&1~VzQfy2l{;n7u&B3O-OxjF3uk4h0({s8R+`x zZ$pm(CZ&J+`RG*T^VFK^YUQhrsnuVR zaj72F4X8GIPxM1^Aos_bV*Hxbx8@Oiv-NT86k2Z`HaFu3?2D4BveQb|K2E0=CMTz= zwbmT-RkhOEYktK(um>bIvGva2fEkPJU-XmuPr|@*Vv&x+In}{ z<@BDDZP7dNeaBj>Hly6pUlb3(`&&1u?|Z8>FaLY||9n$jfO<=KIv1%&60Mo%Nbji9ag7{bv8jC?f2w>=dVzC&$@`fbt>*COnN*G!{~`N$ z{^HP-&KsYNx0(_%9Q?^nrYP z{Be*&x5PR77ctUoVz-NV@wu#8_KWRQC!zO~lT<_bb&osQv*^C~I`$V-W7;4Z%*4Gk zjrSxzpiQ_o0g7zv4+ke`Z&pE?&AM1(-4GS-H3k|JG{T<5kC=OUT^2rdx)6i~m7r2_ z^13c^P5@~IZ21`m*;$apg(a^d3N8kn2P<&hR#Z>MI1cC`Df5m14GR(p`YFtHG0PRY zid7#h-cua1^P!E>NYEYHZUu=w@(s3Jy7(uw1WjWfl14e!HyK-Ee^F1gP$bK`#BcaB zp&@LwyFyM^Uj^}S2LEpU->oO;ZOX66HZf7HVpE{XTdn?vwbV}SB;2-YMDFy3J)gn4Q$^K#>PXR}mo|GBVBacTc^ zXQ}X#`cUeMA>T^_z%Io#UwLZn1YM2Sn?BNM6Q2q@3Lq{6ROYE-bBfXq4vM^VXdozC zGXcG*AT13kTxFJ%_K~1)GSsIF7lbwlEl_)&Aq?Bx+o8Y?2+<(JS;wb=Hg!QDIk1Z$ z9R+C$>@sjs1cDdYIl$R*B{Ps0IYK2w1>NfF-lh22nMJ29HR9$WdO)&(Od_BMVI*&V z!S&>r_s^gOHY7OG2*^uM!NI9p{{kF@dygQ{f17~+s~#Ggpaf~KmIZSG8i+X|WcLOK zJS7>!R<}6EE+{omn{mL(L$n((Zwd(P@ETy(1>Wur0X@#CsxFD2RaM_v>@sv6e*N33 T>MW@Lbg}D>`2RorZx8$z7i