[#469] Add binding to C++ runtime using pybind11, bind BitStreamReader

compiler/extensions/python/runtime/src/zserio/bitbuffer.py

@@ -8,6 +8,7 @@
 from zserio.hashcode import HASH_SEED
 from zserio.hashcode import calc_hashcode_int32
 from zserio.bitposition import bitsize_to_bytesize
+from zserio.cppbind import import_cpp_class
 
 class BitBuffer:
     """
@@ -90,3 +91,5 @@ def _masked_last_byte(self) -> int:
 
         return (self._buffer[rounded_bytesize - 1] if last_byte_bits == 0 else
                 self._buffer[rounded_bytesize] & (0xFF << (8 - last_byte_bits)))
+
+BitBuffer = import_cpp_class("BitBuffer") or BitBuffer # type: ignore

compiler/extensions/python/runtime/src/zserio/bitreader.py

@@ -8,6 +8,7 @@
 from zserio.limits import INT64_MIN
 from zserio.exception import PythonRuntimeException
 from zserio.float import uint16_to_float, uint32_to_float, uint64_to_float
+from zserio.cppbind import import_cpp_class
 
 class BitStreamReader:
     """
@@ -603,3 +604,12 @@ def buffer_bitsize(self) -> int:
 VARUINT_BYTE = 0x7f
 VARUINT_HAS_NEXT = 0x80
 VARSIZE_MAX_VALUE = (1 << 31) - 1
+
+_BitStreamReaderCpp = import_cpp_class("BitStreamReader")
+if _BitStreamReaderCpp is not None:
+    BitStreamReader = _BitStreamReaderCpp # type: ignore
+
+    def _bitstreamreader_fromfile(filename: str) -> 'BitStreamReader':
+        with open(filename, 'rb') as file:
+            return BitStreamReader(file.read())
+    BitStreamReader.from_file = _bitstreamreader_fromfile

compiler/extensions/python/runtime/src/zserio_cpp/ZserioCpp.cpp

@@ -0,0 +1,153 @@
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+#include <pybind11/operators.h>
+
+#include "zserio/BitPositionUtil.h"
+#include "zserio/BitStreamReader.h"
+
+namespace py = pybind11;
+
+namespace
+{
+    std::unique_ptr<zserio::BitStreamReader> bitStreamReaderInit(const py::buffer_info& bufferInfo,
+            std::optional<py::int_> optionalBitSize)
+    {
+        const uint8_t* buffer = static_cast<const uint8_t*>(bufferInfo.ptr);
+        if (optionalBitSize)
+        {
+            const size_t bitSize = *optionalBitSize;
+            if (bitSize < 0)
+                throw zserio::CppRuntimeException("BitStreamReader: Bit size cannot be negative!");
+            if ((bitSize + 7) / 8 > static_cast<size_t>(bufferInfo.size))
+                throw zserio::CppRuntimeException("BitStreamReader: Bit size out of range for given buffer!");
+
+            return std::make_unique<zserio::BitStreamReader>(buffer, bitSize, zserio::BitsTag());
+        }
+        else
+        {
+            return std::make_unique<zserio::BitStreamReader>(buffer, bufferInfo.size);
+        }
+    }
+
+    zserio::BitBuffer bitBufferInit(const py::buffer_info& bufferInfo, std::optional<py::int_> optionalBitSize)
+    {
+        uint8_t* buffer = static_cast<uint8_t*>(bufferInfo.ptr);
+        if (optionalBitSize)
+        {
+            const size_t bitSize = *optionalBitSize;
+            if (bitSize < 0)
+                throw zserio::CppRuntimeException("BitBuffer: Bit size cannot be negative!");
+            if ((bitSize + 7) / 8 > static_cast<size_t>(bufferInfo.size))
+                throw zserio::CppRuntimeException("BitBuffer: Bit size out of range for given buffer!");
+
+            return zserio::BitBuffer(buffer, bitSize);
+        }
+        else
+        {
+            return zserio::BitBuffer(buffer, bufferInfo.size * 8);
+        }
+    }
+} // namespace
+
+PYBIND11_MODULE(zserio_cpp, m)
+{
+    py::class_<zserio::BitStreamReader>(m, "BitStreamReader")
+            // constructor from bytes
+            .def(py::init([](const py::bytes& buffer, std::optional<py::int_> bitsize) {
+                return bitStreamReaderInit(py::buffer(buffer).request(), bitsize);
+            }), py::arg("buffer"), py::arg("bitsize") = py::none(), py::keep_alive<1, 2>())
+            // constructor from bytearray
+            .def(py::init([](const py::bytearray& buffer, std::optional<py::int_> bitsize) {
+                return bitStreamReaderInit(py::buffer(buffer).request(), bitsize);
+            }), py::arg("buffer"), py::arg("bitsize") = py::none(), py::keep_alive<1, 2>())
+            // constructor from memoryview
+            .def(py::init([](const py::memoryview& buffer, std::optional<py::int_> bitsize) {
+                return bitStreamReaderInit(py::buffer(buffer).request(), bitsize);
+            }), py::arg("buffer"), py::arg("bitsize") = py::none(), py::keep_alive<1, 2>())
+            .def_static("from_bitbuffer", [](const zserio::BitBuffer& bitBuffer){
+                return std::make_unique<zserio::BitStreamReader>(bitBuffer);
+            }, py::keep_alive<0, 1>())
+            .def("read_bits", [](zserio::BitStreamReader& self, py::int_ numbits){
+                if (numbits < 0)
+                    throw zserio::CppRuntimeException("BitStreamReader: Reading negative number of bits!");
+                return self.readBits64(numbits);
+            })
+            .def("read_signed_bits", [](zserio::BitStreamReader& self, py::int_ numbits){
+                if (numbits < 0)
+                    throw zserio::CppRuntimeException("BitStreamReader: Reading negative number of bits!");
+                return self.readSignedBits64(numbits);
+            })
+            .def("read_bits_unchecked", [](zserio::BitStreamReader& self, py::int_ numbits){
+                if (numbits < 0)
+                    throw zserio::CppRuntimeException("BitStreamReader: Reading negative number of bits!");
+                return self.readBits64(numbits);
+            })
+            .def("read_signed_bits_unchecked", [](zserio::BitStreamReader& self, py::int_ numbits){
+                if (numbits < 0)
+                    throw zserio::CppRuntimeException("BitStreamReader: Reading negative number of bits!");
+                return self.readSignedBits64(numbits);
+            })
+            .def("read_varint16", &zserio::BitStreamReader::readVarInt16)
+            .def("read_varint32", &zserio::BitStreamReader::readVarInt32)
+            .def("read_varint64", &zserio::BitStreamReader::readVarInt64)
+            .def("read_varint", &zserio::BitStreamReader::readVarInt)
+            .def("read_varuint16", &zserio::BitStreamReader::readVarUInt16)
+            .def("read_varuint32", &zserio::BitStreamReader::readVarUInt32)
+            .def("read_varuint64", &zserio::BitStreamReader::readVarUInt64)
+            .def("read_varuint", &zserio::BitStreamReader::readVarUInt)
+            .def("read_varsize", &zserio::BitStreamReader::readVarSize)
+            .def("read_float16", &zserio::BitStreamReader::readFloat16)
+            .def("read_float32", &zserio::BitStreamReader::readFloat32)
+            .def("read_float64", &zserio::BitStreamReader::readFloat64)
+            .def("read_bytes", [](zserio::BitStreamReader& self){
+                auto bytes = self.readBytes();
+                return py::bytearray(reinterpret_cast<char*>(bytes.data()), bytes.size());
+            })
+            .def("read_string", [](zserio::BitStreamReader& self){
+                return self.readString();
+            })
+            .def("read_bool", &zserio::BitStreamReader::readBool)
+            .def("read_bitbuffer", [](zserio::BitStreamReader& self) {
+                return self.readBitBuffer();
+            })
+            .def("alignto", &zserio::BitStreamReader::alignTo)
+            .def_property("bitposition",
+                    &zserio::BitStreamReader::getBitPosition,
+                    [](zserio::BitStreamReader& self, py::int_ bitposition) {
+                        if (bitposition < 0)
+                        {
+                            throw zserio::CppRuntimeException(
+                                    "BitStreamReader: Cannot set negative bit position!");
+                        }
+                        self.setBitPosition(bitposition);
+                    })
+            .def_property_readonly("buffer_bitsize",
+                    &zserio::BitStreamReader::getBufferBitSize)
+            ;
+
+    py::class_<zserio::BitBuffer>(m, "BitBuffer")
+            // constructor from bytes
+            .def(py::init([](const py::bytes& buffer, std::optional<py::int_> bitsize) {
+                return bitBufferInit(py::buffer(buffer).request(), bitsize);
+            }), py::arg("buffer"), py::arg("bitsize") = py::none())
+            // constructor from bytearray
+            .def(py::init([](const py::bytearray& buffer, std::optional<py::int_> bitsize) {
+                return bitBufferInit(py::buffer(buffer).request(), bitsize);
+            }), py::arg("buffer"), py::arg("bitsize") = py::none())
+            // constructor from memoryview
+            .def(py::init([](const py::memoryview& buffer, std::optional<py::int_> bitsize) {
+                return bitBufferInit(py::buffer(buffer).request(), bitsize);
+            }), py::arg("buffer"), py::arg("bitsize") = py::none())
+            .def_property_readonly("buffer", [](zserio::BitBuffer& self){
+                constexpr bool readOnly = false;
+                return py::memoryview::from_memory(self.getBuffer(), self.getByteSize(), readOnly);
+            }, py::keep_alive<0, 1>())
+            .def_property_readonly("bitsize", &zserio::BitBuffer::getBitSize)
+            .def(py::self == py::self)
+            .def("__hash__", &zserio::BitBuffer::hashCode)
+            ;
+
+    // must be at the end to prevent cyclic imports!
+    py::register_local_exception<zserio::CppRuntimeException>(m, "CppRuntimeException",
+            py::module_::import("zserio.exception").attr("PythonRuntimeException"));
+}

compiler/extensions/python/runtime/src/zserio_cpp/setup.py

@@ -0,0 +1,69 @@
+import sys
+import os
+from pathlib import Path
+from pybind11.setup_helpers import Pybind11Extension, ParallelCompile, build_ext
+from setuptools import setup
+from distutils.command.build import build
+
+os.chdir(Path(__file__).parent.resolve())
+
+OPTIONS = [
+    ('cpp-runtime-dir=', None, 'Directory containing C++ runtime soruces.')
+]
+
+class ZserioBuild(build):
+    user_options = build.user_options + OPTIONS
+
+    def initialize_options(self):
+        build.initialize_options(self)
+        self.cpp_runtime_dir = None
+
+    def finalize_options(self):
+        build.finalize_options(self)
+        if not self.cpp_runtime_dir:
+            raise Exception("Parameter '--cpp-runtime-dir' is missing!")
+        if not os.path.exists(Path(self.cpp_runtime_dir) / 'zserio'):
+            raise Exception("Parameter '--cpp-runtime-dir' does not point to Zserio C++ runitme soruces!")
+
+class ZserioBuildExt(build_ext):
+    user_options = build_ext.user_options + OPTIONS
+
+    def initialize_options(self):
+        build_ext.initialize_options(self)
+        self.cpp_runtime_dir = None
+
+    def finalize_options(self):
+        build_ext.finalize_options(self)
+        self.set_undefined_options('build', ('cpp_runtime_dir', 'cpp_runtime_dir'))
+
+        zserio_cpp = self.extensions[0]
+
+        zserio_cpp.sources.extend((str(filename) for filename in Path('.').rglob('*.cpp')))
+        zserio_cpp.sources.extend(
+            (str(filename) for filename in Path(self.cpp_runtime_dir).resolve().rglob('*.cpp'))
+        )
+
+        zserio_cpp.include_dirs.append(self.cpp_runtime_dir)
+
+        zserio_cpp.extra_compile_args.extend(['-O3'])
+
+        if sys.maxsize > 2**32:
+            zserio_cpp.define_macros.append(('ZSERIO_RUNTIME_64BIT', None))
+
+ParallelCompile(default=0).install()
+
+setup(
+    name='zserio_cpp',
+    version=0.1,
+    url="https://github.com/ndsev/zserio",
+    author='Navigation Data Standard e.V.',
+    author_email='support@nds-association.org',
+    description='Zserio C++ runtime binding to Python',
+
+    ext_modules=[Pybind11Extension('zserio_cpp', sources=[])], # will be set-up in ZserioBuildExt
+    cmdclass={"build": ZserioBuild, "build_ext": ZserioBuildExt},
+
+    python_requires='>=3.8',
+
+    license = "BSD-3 Clause",
+)