diff --git a/src/compact_encoding/__init__.py b/src/compact_encoding/__init__.py index a46c02d..97e19d0 100644 --- a/src/compact_encoding/__init__.py +++ b/src/compact_encoding/__init__.py @@ -3,8 +3,23 @@ from .bool import bool_codec as bool from .buffer import buffer from .codec import Codec, CompactError, OutOfBounds, decode, encode +from .fixed import fixed, fixed32, fixed64 +from .float import float32, float64 +from .integer import ( + int24, + int40, + int48, + int56, + uint, + uint8, + uint16, + uint24, + uint32, + uint40, + uint48, + uint56, +) from .integer import int_codec as int -from .integer import uint, uint32 from .state import State from .string import utf8 from .string import utf8 as string @@ -18,9 +33,24 @@ "buffer", "decode", "encode", + "fixed", + "fixed32", + "fixed64", + "float32", + "float64", "int", + "int24", + "int40", + "int48", + "int56", "string", "uint", + "uint8", + "uint16", + "uint24", "uint32", + "uint40", + "uint48", + "uint56", "utf8", ] diff --git a/src/compact_encoding/fixed.py b/src/compact_encoding/fixed.py new file mode 100644 index 0000000..2713442 --- /dev/null +++ b/src/compact_encoding/fixed.py @@ -0,0 +1,34 @@ +from .codec import OutOfBounds + + +class _Fixed: + def __init__(self, n): + self._n = n + + def preencode(self, state, s): + if len(s) != self._n: + raise ValueError("Incorrect buffer size") + state.end += self._n + + def encode(self, state, s): + if len(s) != self._n: + raise ValueError("Incorrect buffer size") + start = state.start + state.buffer[start : start + self._n] = s + state.start = start + self._n + + def decode(self, state): + if state.remaining < self._n: + raise OutOfBounds("Out of bounds") + start = state.start + value = bytes(state.buffer[start : start + self._n]) + state.start = start + self._n + return value + + +def fixed(n): + return _Fixed(n) + + +fixed32 = fixed(32) +fixed64 = fixed(64) diff --git a/src/compact_encoding/float.py b/src/compact_encoding/float.py new file mode 100644 index 0000000..c112abd --- /dev/null +++ b/src/compact_encoding/float.py @@ -0,0 +1,29 @@ +import struct + +from .codec import OutOfBounds + + +class _Float: + def __init__(self, fmt, nbytes): + self._fmt = fmt + self._n = nbytes + + def preencode(self, state, n): + state.end += self._n + + def encode(self, state, n): + s = state.start + state.buffer[s : s + self._n] = struct.pack(self._fmt, n) + state.start = s + self._n + + def decode(self, state): + if state.remaining < self._n: + raise OutOfBounds("Out of bounds") + s = state.start + value = struct.unpack(self._fmt, state.buffer[s : s + self._n])[0] + state.start = s + self._n + return value + + +float32 = _Float("= 0 and n <= MAX_SAFE_INTEGER (JS validateUint) + if n >= (1 << self._bits): + raise ValueError(f"uint{self._bits} is out of range") + + def preencode(self, state, n): + self._check(n) + state.end += self._n + + def encode(self, state, n): + self._check(n) + s = state.start + state.buffer[s : s + self._n] = n.to_bytes(self._n, "little") + state.start = s + self._n + + def decode(self, state): + return _read_le(state, self._n) + + +class _SizedInt: + def __init__(self, uint_codec): + self._u = uint_codec + + def preencode(self, state, n): + self._u.preencode(state, _zigzag(n)) + + def encode(self, state, n): + self._u.encode(state, _zigzag(n)) + + def decode(self, state): + return _unzigzag(self._u.decode(state)) + + def _zigzag(n): if n < 0: return 2 * -n - 1 @@ -113,3 +151,15 @@ def decode(self, state): uint = _UInt() uint32 = _UInt32() int_codec = _Int() + +uint8 = _SizedUInt(1) +uint16 = _SizedUInt(2) +uint24 = _SizedUInt(3) +uint40 = _SizedUInt(5) +uint48 = _SizedUInt(6) +uint56 = _SizedUInt(7) + +int24 = _SizedInt(uint24) +int40 = _SizedInt(uint40) +int48 = _SizedInt(uint48) +int56 = _SizedInt(uint56) diff --git a/tests/test_fixed.py b/tests/test_fixed.py new file mode 100644 index 0000000..468bbcf --- /dev/null +++ b/tests/test_fixed.py @@ -0,0 +1,49 @@ +import pytest + +import compact_encoding as cenc + +BUF32 = bytes(range(32)) +BUF64 = bytes(range(64)) + + +def test_fixed32_encode_decode(): + encoded = cenc.encode(cenc.fixed32, BUF32) + assert encoded == BUF32 # no length prefix, raw bytes + assert cenc.decode(cenc.fixed32, encoded) == BUF32 + + +def test_fixed64_encode_decode(): + encoded = cenc.encode(cenc.fixed64, BUF64) + assert encoded == BUF64 + assert cenc.decode(cenc.fixed64, encoded) == BUF64 + + +def test_fixed_decode_returns_a_copy(): + encoded = cenc.encode(cenc.fixed32, BUF32) + out = cenc.decode(cenc.fixed32, encoded) + assert isinstance(out, bytes) # a copy, not a view into the buffer + + +def test_fixed_rejects_wrong_length_via_oneshot(): + with pytest.raises(ValueError): + cenc.encode(cenc.fixed32, bytes(31)) + with pytest.raises(ValueError): + cenc.encode(cenc.fixed32, bytes(33)) + + +def test_fixed_rejects_wrong_length_via_direct_encode(): + from compact_encoding.state import State + + state = State() + state.buffer = bytearray(32) + with pytest.raises(ValueError): + cenc.fixed32.encode( + state, bytes(31) + ) # encode must validate, not silently resize + + +def test_fixed_decode_truncated_raises(): + from compact_encoding.codec import OutOfBounds + + with pytest.raises(OutOfBounds): + cenc.decode(cenc.fixed32, bytes(31)) # fewer than 32 bytes diff --git a/tests/test_float.py b/tests/test_float.py new file mode 100644 index 0000000..d02a2d3 --- /dev/null +++ b/tests/test_float.py @@ -0,0 +1,36 @@ +import pytest + +import compact_encoding as cenc + +FLOAT32_CASES = [ + (0.5, "0000003f"), + (1.5, "0000c03f"), + (-2.0, "000000c0"), +] + +FLOAT64_CASES = [ + (0.5, "000000000000e03f"), + (1.5, "000000000000f83f"), + (-2.0, "00000000000000c0"), +] + + +@pytest.mark.parametrize("value,hexbytes", FLOAT32_CASES) +def test_float32(value, hexbytes): + assert cenc.encode(cenc.float32, value).hex() == hexbytes + assert cenc.decode(cenc.float32, bytes.fromhex(hexbytes)) == value + + +@pytest.mark.parametrize("value,hexbytes", FLOAT64_CASES) +def test_float64(value, hexbytes): + assert cenc.encode(cenc.float64, value).hex() == hexbytes + assert cenc.decode(cenc.float64, bytes.fromhex(hexbytes)) == value + + +def test_float_decode_truncated_raises(): + from compact_encoding.codec import OutOfBounds + + with pytest.raises(OutOfBounds): + cenc.decode(cenc.float32, bytes.fromhex("010203")) # 3 of 4 + with pytest.raises(OutOfBounds): + cenc.decode(cenc.float64, bytes.fromhex("01020304050607")) # 7 of 8 diff --git a/tests/test_integer.py b/tests/test_integer.py index d785efb..e81629a 100644 --- a/tests/test_integer.py +++ b/tests/test_integer.py @@ -117,3 +117,108 @@ def test_int_rejects_beyond_range(): cenc.encode(cenc.int, 2**52) # zigzag -> 2**53 > MAX_SAFE with pytest.raises(ValueError): cenc.encode(cenc.int, -(2**52) - 1) + + +SIZED_UINT_CASES = [ + ("uint8", 0, "00"), + ("uint8", 1, "01"), + ("uint8", 255, "ff"), + ("uint16", 0, "0000"), + ("uint16", 1, "0100"), + ("uint16", 65535, "ffff"), + ("uint24", 0, "000000"), + ("uint24", 66051, "030201"), + ("uint24", 16777215, "ffffff"), + ("uint40", 0, "0000000000"), + ("uint40", 1099511627775, "ffffffffff"), + ("uint48", 0, "000000000000"), + ("uint48", 4294967295, "ffffffff0000"), + ("uint48", 281474976710655, "ffffffffffff"), + ("uint56", 0, "00000000000000"), + ("uint56", MAX_SAFE, "ffffffffffff1f"), # MAX_SAFE ceiling binds for uint56 +] + + +@pytest.mark.parametrize("name,value,hexbytes", SIZED_UINT_CASES) +def test_sized_uint_encode(name, value, hexbytes): + codec = getattr(cenc, name) + assert cenc.encode(codec, value).hex() == hexbytes + + +@pytest.mark.parametrize("name,value,hexbytes", SIZED_UINT_CASES) +def test_sized_uint_decode(name, value, hexbytes): + codec = getattr(cenc, name) + assert cenc.decode(codec, bytes.fromhex(hexbytes)) == value + + +@pytest.mark.parametrize( + "name,bad", + [ + ("uint8", 256), + ("uint16", 65536), + ("uint24", 16777216), + ("uint40", 2**40), + ("uint48", 2**48), + ("uint56", 2**53), # exceeds MAX_SAFE_INTEGER + ("uint8", -1), + ], +) +def test_sized_uint_rejects_out_of_range(name, bad): + with pytest.raises(ValueError): + cenc.encode(getattr(cenc, name), bad) + + +def test_sized_uint_decode_truncated_raises(): + from compact_encoding.codec import OutOfBounds + + with pytest.raises(OutOfBounds): + cenc.decode(cenc.uint24, bytes.fromhex("0102")) # 2 of 3 bytes + + +SIZED_INT_CASES = [ + ("int24", 0, "000000"), + ("int24", -1, "010000"), + ("int24", 1, "020000"), + ("int24", 8388607, "feffff"), # 2**23 - 1 + ("int24", -8388608, "ffffff"), # -2**23 + ("int40", -1, "0100000000"), + ("int40", 549755813887, "feffffffff"), # 2**39 - 1 + ("int40", -549755813888, "ffffffffff"), # -2**39 + ("int48", -1, "010000000000"), + ("int48", 140737488355327, "feffffffffff"), # 2**47 - 1 + ("int48", -140737488355328, "ffffffffffff"), # -2**47 + ("int56", -1, "01000000000000"), + ("int56", 4503599627370495, "feffffffffff1f"), # 2**52 - 1 (fixture 42) + ("int56", -4503599627370496, "ffffffffffff1f"), # -2**52 (fixture 42) +] + + +@pytest.mark.parametrize("name,value,hexbytes", SIZED_INT_CASES) +def test_sized_int_encode(name, value, hexbytes): + assert cenc.encode(getattr(cenc, name), value).hex() == hexbytes + + +@pytest.mark.parametrize("name,value,hexbytes", SIZED_INT_CASES) +def test_sized_int_decode(name, value, hexbytes): + assert cenc.decode(getattr(cenc, name), bytes.fromhex(hexbytes)) == value + + +@pytest.mark.parametrize( + "name,bad", + [ + ("int24", 8388608), # 2**23; zigzag -> 2**24, overflows uint24 + ("int24", -8388609), # -(2**23) - 1; zigzag overflows uint24 + ("int56", 2**52), # zigzag -> 2**53 > MAX_SAFE + ("int56", -(2**52) - 1), + ], +) +def test_sized_int_rejects_out_of_range(name, bad): + with pytest.raises(ValueError): + cenc.encode(getattr(cenc, name), bad) + + +def test_sized_int_decode_truncated_raises(): + from compact_encoding.codec import OutOfBounds + + with pytest.raises(OutOfBounds): + cenc.decode(cenc.int40, bytes.fromhex("010203")) # 3 of 5 bytes diff --git a/tests/test_roundtrip.py b/tests/test_roundtrip.py index 88db1fb..535ec77 100644 --- a/tests/test_roundtrip.py +++ b/tests/test_roundtrip.py @@ -11,10 +11,25 @@ (cenc.int, -5), (cenc.int, 2**52 - 1), (cenc.uint32, 0xDEADBEEF), + (cenc.uint8, 200), + (cenc.uint16, 40000), + (cenc.uint24, 16000000), + (cenc.uint40, 2**39), + (cenc.uint48, 2**47), + (cenc.uint56, 2**52), + (cenc.int24, -8000000), + (cenc.int40, 2**38), + (cenc.int48, -(2**46)), + (cenc.int56, 2**51), (cenc.utf8, "hello 世界"), (cenc.bool, True), (cenc.bool, False), (cenc.buffer, b"\x00\x01\x02\xfe\xff"), + (cenc.float32, 0.5), + (cenc.float32, -2.0), + (cenc.float64, 3.141592653589793), + (cenc.fixed32, bytes(range(32))), + (cenc.fixed64, bytes(range(64))), ]