From c035615dd2c7200e9d8ec1618083983b304fe942 Mon Sep 17 00:00:00 2001 From: Balakrishna Avulapati Date: Thu, 9 Jul 2026 23:41:36 +0530 Subject: [PATCH 1/5] Add fixed-width sized uint codecs --- src/compact_encoding/__init__.py | 8 ++++- src/compact_encoding/integer.py | 31 ++++++++++++++++++ tests/test_integer.py | 56 ++++++++++++++++++++++++++++++++ tests/test_roundtrip.py | 6 ++++ 4 files changed, 100 insertions(+), 1 deletion(-) diff --git a/src/compact_encoding/__init__.py b/src/compact_encoding/__init__.py index a46c02d..ebc93ba 100644 --- a/src/compact_encoding/__init__.py +++ b/src/compact_encoding/__init__.py @@ -4,7 +4,7 @@ from .buffer import buffer from .codec import Codec, CompactError, OutOfBounds, decode, encode from .integer import int_codec as int -from .integer import uint, uint32 +from .integer import uint, uint8, uint16, uint24, uint32, uint40, uint48, uint56 from .state import State from .string import utf8 from .string import utf8 as string @@ -21,6 +21,12 @@ "int", "string", "uint", + "uint8", + "uint16", + "uint24", "uint32", + "uint40", + "uint48", + "uint56", "utf8", ] diff --git a/src/compact_encoding/integer.py b/src/compact_encoding/integer.py index 99213ce..3a571e5 100644 --- a/src/compact_encoding/integer.py +++ b/src/compact_encoding/integer.py @@ -85,6 +85,30 @@ def decode(self, state): return _read_le(state, 4) +class _SizedUInt: + def __init__(self, nbytes): + self._n = nbytes + self._bits = nbytes * 8 + + def _check(self, n): + _validate_uint(n) # n >= 0 and n <= MAX_SAFE_INTEGER (JS validateUint) + if n >= (1 << self._bits): + raise ValueError(f"uint{self._bits} is out of range") + + def preencode(self, state, n): + self._check(n) + state.end += self._n + + def encode(self, state, n): + self._check(n) + s = state.start + state.buffer[s : s + self._n] = n.to_bytes(self._n, "little") + state.start = s + self._n + + def decode(self, state): + return _read_le(state, self._n) + + def _zigzag(n): if n < 0: return 2 * -n - 1 @@ -113,3 +137,10 @@ def decode(self, state): uint = _UInt() uint32 = _UInt32() int_codec = _Int() + +uint8 = _SizedUInt(1) +uint16 = _SizedUInt(2) +uint24 = _SizedUInt(3) +uint40 = _SizedUInt(5) +uint48 = _SizedUInt(6) +uint56 = _SizedUInt(7) diff --git a/tests/test_integer.py b/tests/test_integer.py index d785efb..af4375f 100644 --- a/tests/test_integer.py +++ b/tests/test_integer.py @@ -117,3 +117,59 @@ def test_int_rejects_beyond_range(): cenc.encode(cenc.int, 2**52) # zigzag -> 2**53 > MAX_SAFE with pytest.raises(ValueError): cenc.encode(cenc.int, -(2**52) - 1) + + +SIZED_UINT_CASES = [ + ("uint8", 0, "00"), + ("uint8", 1, "01"), + ("uint8", 255, "ff"), + ("uint16", 0, "0000"), + ("uint16", 1, "0100"), + ("uint16", 65535, "ffff"), + ("uint24", 0, "000000"), + ("uint24", 66051, "030201"), + ("uint24", 16777215, "ffffff"), + ("uint40", 0, "0000000000"), + ("uint40", 1099511627775, "ffffffffff"), + ("uint48", 0, "000000000000"), + ("uint48", 4294967295, "ffffffff0000"), + ("uint48", 281474976710655, "ffffffffffff"), + ("uint56", 0, "00000000000000"), + ("uint56", MAX_SAFE, "ffffffffffff1f"), # MAX_SAFE ceiling binds for uint56 +] + + +@pytest.mark.parametrize("name,value,hexbytes", SIZED_UINT_CASES) +def test_sized_uint_encode(name, value, hexbytes): + codec = getattr(cenc, name) + assert cenc.encode(codec, value).hex() == hexbytes + + +@pytest.mark.parametrize("name,value,hexbytes", SIZED_UINT_CASES) +def test_sized_uint_decode(name, value, hexbytes): + codec = getattr(cenc, name) + assert cenc.decode(codec, bytes.fromhex(hexbytes)) == value + + +@pytest.mark.parametrize( + "name,bad", + [ + ("uint8", 256), + ("uint16", 65536), + ("uint24", 16777216), + ("uint40", 2**40), + ("uint48", 2**48), + ("uint56", 2**53), # exceeds MAX_SAFE_INTEGER + ("uint8", -1), + ], +) +def test_sized_uint_rejects_out_of_range(name, bad): + with pytest.raises(ValueError): + cenc.encode(getattr(cenc, name), bad) + + +def test_sized_uint_decode_truncated_raises(): + from compact_encoding.codec import OutOfBounds + + with pytest.raises(OutOfBounds): + cenc.decode(cenc.uint24, bytes.fromhex("0102")) # 2 of 3 bytes diff --git a/tests/test_roundtrip.py b/tests/test_roundtrip.py index 88db1fb..acd4740 100644 --- a/tests/test_roundtrip.py +++ b/tests/test_roundtrip.py @@ -11,6 +11,12 @@ (cenc.int, -5), (cenc.int, 2**52 - 1), (cenc.uint32, 0xDEADBEEF), + (cenc.uint8, 200), + (cenc.uint16, 40000), + (cenc.uint24, 16000000), + (cenc.uint40, 2**39), + (cenc.uint48, 2**47), + (cenc.uint56, 2**52), (cenc.utf8, "hello 世界"), (cenc.bool, True), (cenc.bool, False), From b4f70bb4da01133ddac5b881374a172642fa7133 Mon Sep 17 00:00:00 2001 From: Balakrishna Avulapati Date: Thu, 9 Jul 2026 23:50:45 +0530 Subject: [PATCH 2/5] Add fixed-width sized int codecs --- src/compact_encoding/__init__.py | 19 ++++++++++++- src/compact_encoding/integer.py | 19 +++++++++++++ tests/test_integer.py | 49 ++++++++++++++++++++++++++++++++ tests/test_roundtrip.py | 4 +++ 4 files changed, 90 insertions(+), 1 deletion(-) diff --git a/src/compact_encoding/__init__.py b/src/compact_encoding/__init__.py index ebc93ba..94e532d 100644 --- a/src/compact_encoding/__init__.py +++ b/src/compact_encoding/__init__.py @@ -3,8 +3,21 @@ from .bool import bool_codec as bool from .buffer import buffer from .codec import Codec, CompactError, OutOfBounds, decode, encode +from .integer import ( + int24, + int40, + int48, + int56, + uint, + uint8, + uint16, + uint24, + uint32, + uint40, + uint48, + uint56, +) from .integer import int_codec as int -from .integer import uint, uint8, uint16, uint24, uint32, uint40, uint48, uint56 from .state import State from .string import utf8 from .string import utf8 as string @@ -19,6 +32,10 @@ "decode", "encode", "int", + "int24", + "int40", + "int48", + "int56", "string", "uint", "uint8", diff --git a/src/compact_encoding/integer.py b/src/compact_encoding/integer.py index 3a571e5..d426716 100644 --- a/src/compact_encoding/integer.py +++ b/src/compact_encoding/integer.py @@ -109,6 +109,20 @@ def decode(self, state): return _read_le(state, self._n) +class _SizedInt: + def __init__(self, uint_codec): + self._u = uint_codec + + def preencode(self, state, n): + self._u.preencode(state, _zigzag(n)) + + def encode(self, state, n): + self._u.encode(state, _zigzag(n)) + + def decode(self, state): + return _unzigzag(self._u.decode(state)) + + def _zigzag(n): if n < 0: return 2 * -n - 1 @@ -144,3 +158,8 @@ def decode(self, state): uint40 = _SizedUInt(5) uint48 = _SizedUInt(6) uint56 = _SizedUInt(7) + +int24 = _SizedInt(uint24) +int40 = _SizedInt(uint40) +int48 = _SizedInt(uint48) +int56 = _SizedInt(uint56) diff --git a/tests/test_integer.py b/tests/test_integer.py index af4375f..e81629a 100644 --- a/tests/test_integer.py +++ b/tests/test_integer.py @@ -173,3 +173,52 @@ def test_sized_uint_decode_truncated_raises(): with pytest.raises(OutOfBounds): cenc.decode(cenc.uint24, bytes.fromhex("0102")) # 2 of 3 bytes + + +SIZED_INT_CASES = [ + ("int24", 0, "000000"), + ("int24", -1, "010000"), + ("int24", 1, "020000"), + ("int24", 8388607, "feffff"), # 2**23 - 1 + ("int24", -8388608, "ffffff"), # -2**23 + ("int40", -1, "0100000000"), + ("int40", 549755813887, "feffffffff"), # 2**39 - 1 + ("int40", -549755813888, "ffffffffff"), # -2**39 + ("int48", -1, "010000000000"), + ("int48", 140737488355327, "feffffffffff"), # 2**47 - 1 + ("int48", -140737488355328, "ffffffffffff"), # -2**47 + ("int56", -1, "01000000000000"), + ("int56", 4503599627370495, "feffffffffff1f"), # 2**52 - 1 (fixture 42) + ("int56", -4503599627370496, "ffffffffffff1f"), # -2**52 (fixture 42) +] + + +@pytest.mark.parametrize("name,value,hexbytes", SIZED_INT_CASES) +def test_sized_int_encode(name, value, hexbytes): + assert cenc.encode(getattr(cenc, name), value).hex() == hexbytes + + +@pytest.mark.parametrize("name,value,hexbytes", SIZED_INT_CASES) +def test_sized_int_decode(name, value, hexbytes): + assert cenc.decode(getattr(cenc, name), bytes.fromhex(hexbytes)) == value + + +@pytest.mark.parametrize( + "name,bad", + [ + ("int24", 8388608), # 2**23; zigzag -> 2**24, overflows uint24 + ("int24", -8388609), # -(2**23) - 1; zigzag overflows uint24 + ("int56", 2**52), # zigzag -> 2**53 > MAX_SAFE + ("int56", -(2**52) - 1), + ], +) +def test_sized_int_rejects_out_of_range(name, bad): + with pytest.raises(ValueError): + cenc.encode(getattr(cenc, name), bad) + + +def test_sized_int_decode_truncated_raises(): + from compact_encoding.codec import OutOfBounds + + with pytest.raises(OutOfBounds): + cenc.decode(cenc.int40, bytes.fromhex("010203")) # 3 of 5 bytes diff --git a/tests/test_roundtrip.py b/tests/test_roundtrip.py index acd4740..a15d35b 100644 --- a/tests/test_roundtrip.py +++ b/tests/test_roundtrip.py @@ -17,6 +17,10 @@ (cenc.uint40, 2**39), (cenc.uint48, 2**47), (cenc.uint56, 2**52), + (cenc.int24, -8000000), + (cenc.int40, 2**38), + (cenc.int48, -(2**46)), + (cenc.int56, 2**51), (cenc.utf8, "hello 世界"), (cenc.bool, True), (cenc.bool, False), From 433570e3e87c8103f38e085c4813629fd7209ea1 Mon Sep 17 00:00:00 2001 From: Balakrishna Avulapati Date: Thu, 9 Jul 2026 23:55:28 +0530 Subject: [PATCH 3/5] Add float32 and float64 codecs --- src/compact_encoding/__init__.py | 3 +++ src/compact_encoding/float.py | 29 +++++++++++++++++++++++++ tests/test_float.py | 36 ++++++++++++++++++++++++++++++++ tests/test_roundtrip.py | 3 +++ 4 files changed, 71 insertions(+) create mode 100644 src/compact_encoding/float.py create mode 100644 tests/test_float.py diff --git a/src/compact_encoding/__init__.py b/src/compact_encoding/__init__.py index 94e532d..35891bf 100644 --- a/src/compact_encoding/__init__.py +++ b/src/compact_encoding/__init__.py @@ -3,6 +3,7 @@ from .bool import bool_codec as bool from .buffer import buffer from .codec import Codec, CompactError, OutOfBounds, decode, encode +from .float import float32, float64 from .integer import ( int24, int40, @@ -31,6 +32,8 @@ "buffer", "decode", "encode", + "float32", + "float64", "int", "int24", "int40", diff --git a/src/compact_encoding/float.py b/src/compact_encoding/float.py new file mode 100644 index 0000000..caca9fa --- /dev/null +++ b/src/compact_encoding/float.py @@ -0,0 +1,29 @@ +import struct + +from .codec import OutOfBounds + + +class _Float: + def __init__(self, fmt, nbytes): + self._fmt = fmt + self._n = nbytes + + def preencode(self, state, n): + state.end += self._n + + def encode(self, state, n): + s = state.start + state.buffer[s : s + self._n] = struct.pack(self._fmt, n) + state.start = s + self._n + + def decode(self, state): + if state.remaining < self._n: + raise OutOfBounds("Out of bounds") + s = state.start + value = struct.unpack(self._fmt, bytes(state.buffer[s : s + self._n]))[0] + state.start = s + self._n + return value + + +float32 = _Float(" Date: Fri, 10 Jul 2026 00:15:57 +0530 Subject: [PATCH 4/5] Add fixed-width byte block codecs --- src/compact_encoding/__init__.py | 4 +++ src/compact_encoding/fixed.py | 34 ++++++++++++++++++++++ tests/test_fixed.py | 49 ++++++++++++++++++++++++++++++++ tests/test_roundtrip.py | 2 ++ 4 files changed, 89 insertions(+) create mode 100644 src/compact_encoding/fixed.py create mode 100644 tests/test_fixed.py diff --git a/src/compact_encoding/__init__.py b/src/compact_encoding/__init__.py index 35891bf..97e19d0 100644 --- a/src/compact_encoding/__init__.py +++ b/src/compact_encoding/__init__.py @@ -3,6 +3,7 @@ from .bool import bool_codec as bool from .buffer import buffer from .codec import Codec, CompactError, OutOfBounds, decode, encode +from .fixed import fixed, fixed32, fixed64 from .float import float32, float64 from .integer import ( int24, @@ -32,6 +33,9 @@ "buffer", "decode", "encode", + "fixed", + "fixed32", + "fixed64", "float32", "float64", "int", diff --git a/src/compact_encoding/fixed.py b/src/compact_encoding/fixed.py new file mode 100644 index 0000000..2713442 --- /dev/null +++ b/src/compact_encoding/fixed.py @@ -0,0 +1,34 @@ +from .codec import OutOfBounds + + +class _Fixed: + def __init__(self, n): + self._n = n + + def preencode(self, state, s): + if len(s) != self._n: + raise ValueError("Incorrect buffer size") + state.end += self._n + + def encode(self, state, s): + if len(s) != self._n: + raise ValueError("Incorrect buffer size") + start = state.start + state.buffer[start : start + self._n] = s + state.start = start + self._n + + def decode(self, state): + if state.remaining < self._n: + raise OutOfBounds("Out of bounds") + start = state.start + value = bytes(state.buffer[start : start + self._n]) + state.start = start + self._n + return value + + +def fixed(n): + return _Fixed(n) + + +fixed32 = fixed(32) +fixed64 = fixed(64) diff --git a/tests/test_fixed.py b/tests/test_fixed.py new file mode 100644 index 0000000..468bbcf --- /dev/null +++ b/tests/test_fixed.py @@ -0,0 +1,49 @@ +import pytest + +import compact_encoding as cenc + +BUF32 = bytes(range(32)) +BUF64 = bytes(range(64)) + + +def test_fixed32_encode_decode(): + encoded = cenc.encode(cenc.fixed32, BUF32) + assert encoded == BUF32 # no length prefix, raw bytes + assert cenc.decode(cenc.fixed32, encoded) == BUF32 + + +def test_fixed64_encode_decode(): + encoded = cenc.encode(cenc.fixed64, BUF64) + assert encoded == BUF64 + assert cenc.decode(cenc.fixed64, encoded) == BUF64 + + +def test_fixed_decode_returns_a_copy(): + encoded = cenc.encode(cenc.fixed32, BUF32) + out = cenc.decode(cenc.fixed32, encoded) + assert isinstance(out, bytes) # a copy, not a view into the buffer + + +def test_fixed_rejects_wrong_length_via_oneshot(): + with pytest.raises(ValueError): + cenc.encode(cenc.fixed32, bytes(31)) + with pytest.raises(ValueError): + cenc.encode(cenc.fixed32, bytes(33)) + + +def test_fixed_rejects_wrong_length_via_direct_encode(): + from compact_encoding.state import State + + state = State() + state.buffer = bytearray(32) + with pytest.raises(ValueError): + cenc.fixed32.encode( + state, bytes(31) + ) # encode must validate, not silently resize + + +def test_fixed_decode_truncated_raises(): + from compact_encoding.codec import OutOfBounds + + with pytest.raises(OutOfBounds): + cenc.decode(cenc.fixed32, bytes(31)) # fewer than 32 bytes diff --git a/tests/test_roundtrip.py b/tests/test_roundtrip.py index 96a3951..535ec77 100644 --- a/tests/test_roundtrip.py +++ b/tests/test_roundtrip.py @@ -28,6 +28,8 @@ (cenc.float32, 0.5), (cenc.float32, -2.0), (cenc.float64, 3.141592653589793), + (cenc.fixed32, bytes(range(32))), + (cenc.fixed64, bytes(range(64))), ] From 74f9e2b2758ff9435d2a4ef0323bf2db22794e0c Mon Sep 17 00:00:00 2001 From: Balakrishna Avulapati Date: Fri, 10 Jul 2026 00:19:18 +0530 Subject: [PATCH 5/5] Drop redundant bytes copy in float decode --- src/compact_encoding/float.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compact_encoding/float.py b/src/compact_encoding/float.py index caca9fa..c112abd 100644 --- a/src/compact_encoding/float.py +++ b/src/compact_encoding/float.py @@ -20,7 +20,7 @@ def decode(self, state): if state.remaining < self._n: raise OutOfBounds("Out of bounds") s = state.start - value = struct.unpack(self._fmt, bytes(state.buffer[s : s + self._n]))[0] + value = struct.unpack(self._fmt, state.buffer[s : s + self._n])[0] state.start = s + self._n return value