From 1e5abb33c3d4b6659daeae907ca0384faf62b28b Mon Sep 17 00:00:00 2001 From: Milan Kriz Date: Tue, 17 Jan 2023 11:37:05 +0100 Subject: [PATCH] [#469] Add binding to C++ runtime using pybind11, bind BitStreamReader --- .../python/runtime/src/zserio/bitbuffer.py | 3 + .../python/runtime/src/zserio/bitreader.py | 10 ++ .../runtime/src/zserio_cpp/ZserioCpp.cpp | 153 ++++++++++++++++++ .../python/runtime/src/zserio_cpp/setup.py | 69 ++++++++ 4 files changed, 235 insertions(+) create mode 100644 compiler/extensions/python/runtime/src/zserio_cpp/ZserioCpp.cpp create mode 100644 compiler/extensions/python/runtime/src/zserio_cpp/setup.py diff --git a/compiler/extensions/python/runtime/src/zserio/bitbuffer.py b/compiler/extensions/python/runtime/src/zserio/bitbuffer.py index 6b50c7973..77166dce3 100644 --- a/compiler/extensions/python/runtime/src/zserio/bitbuffer.py +++ b/compiler/extensions/python/runtime/src/zserio/bitbuffer.py @@ -8,6 +8,7 @@ from zserio.hashcode import HASH_SEED from zserio.hashcode import calc_hashcode_int32 from zserio.bitposition import bitsize_to_bytesize +from zserio.cppbind import import_cpp_class class BitBuffer: """ @@ -90,3 +91,5 @@ def _masked_last_byte(self) -> int: return (self._buffer[rounded_bytesize - 1] if last_byte_bits == 0 else self._buffer[rounded_bytesize] & (0xFF << (8 - last_byte_bits))) + +BitBuffer = import_cpp_class("BitBuffer") or BitBuffer # type: ignore diff --git a/compiler/extensions/python/runtime/src/zserio/bitreader.py b/compiler/extensions/python/runtime/src/zserio/bitreader.py index 4e0572cf5..03475f735 100644 --- a/compiler/extensions/python/runtime/src/zserio/bitreader.py +++ b/compiler/extensions/python/runtime/src/zserio/bitreader.py @@ -8,6 +8,7 @@ from zserio.limits import INT64_MIN from zserio.exception import PythonRuntimeException from zserio.float import uint16_to_float, uint32_to_float, uint64_to_float +from zserio.cppbind import import_cpp_class class BitStreamReader: """ @@ -603,3 +604,12 @@ def buffer_bitsize(self) -> int: VARUINT_BYTE = 0x7f VARUINT_HAS_NEXT = 0x80 VARSIZE_MAX_VALUE = (1 << 31) - 1 + +_BitStreamReaderCpp = import_cpp_class("BitStreamReader") +if _BitStreamReaderCpp is not None: + BitStreamReader = _BitStreamReaderCpp # type: ignore + + def _bitstreamreader_fromfile(filename: str) -> 'BitStreamReader': + with open(filename, 'rb') as file: + return BitStreamReader(file.read()) + BitStreamReader.from_file = _bitstreamreader_fromfile diff --git a/compiler/extensions/python/runtime/src/zserio_cpp/ZserioCpp.cpp b/compiler/extensions/python/runtime/src/zserio_cpp/ZserioCpp.cpp new file mode 100644 index 000000000..75324f10b --- /dev/null +++ b/compiler/extensions/python/runtime/src/zserio_cpp/ZserioCpp.cpp @@ -0,0 +1,153 @@ +#include +#include +#include + +#include "zserio/BitPositionUtil.h" +#include "zserio/BitStreamReader.h" + +namespace py = pybind11; + +namespace +{ + std::unique_ptr bitStreamReaderInit(const py::buffer_info& bufferInfo, + std::optional optionalBitSize) + { + const uint8_t* buffer = static_cast(bufferInfo.ptr); + if (optionalBitSize) + { + const size_t bitSize = *optionalBitSize; + if (bitSize < 0) + throw zserio::CppRuntimeException("BitStreamReader: Bit size cannot be negative!"); + if ((bitSize + 7) / 8 > static_cast(bufferInfo.size)) + throw zserio::CppRuntimeException("BitStreamReader: Bit size out of range for given buffer!"); + + return std::make_unique(buffer, bitSize, zserio::BitsTag()); + } + else + { + return std::make_unique(buffer, bufferInfo.size); + } + } + + zserio::BitBuffer bitBufferInit(const py::buffer_info& bufferInfo, std::optional optionalBitSize) + { + uint8_t* buffer = static_cast(bufferInfo.ptr); + if (optionalBitSize) + { + const size_t bitSize = *optionalBitSize; + if (bitSize < 0) + throw zserio::CppRuntimeException("BitBuffer: Bit size cannot be negative!"); + if ((bitSize + 7) / 8 > static_cast(bufferInfo.size)) + throw zserio::CppRuntimeException("BitBuffer: Bit size out of range for given buffer!"); + + return zserio::BitBuffer(buffer, bitSize); + } + else + { + return zserio::BitBuffer(buffer, bufferInfo.size * 8); + } + } +} // namespace + +PYBIND11_MODULE(zserio_cpp, m) +{ + py::class_(m, "BitStreamReader") + // constructor from bytes + .def(py::init([](const py::bytes& buffer, std::optional bitsize) { + return bitStreamReaderInit(py::buffer(buffer).request(), bitsize); + }), py::arg("buffer"), py::arg("bitsize") = py::none(), py::keep_alive<1, 2>()) + // constructor from bytearray + .def(py::init([](const py::bytearray& buffer, std::optional bitsize) { + return bitStreamReaderInit(py::buffer(buffer).request(), bitsize); + }), py::arg("buffer"), py::arg("bitsize") = py::none(), py::keep_alive<1, 2>()) + // constructor from memoryview + .def(py::init([](const py::memoryview& buffer, std::optional bitsize) { + return bitStreamReaderInit(py::buffer(buffer).request(), bitsize); + }), py::arg("buffer"), py::arg("bitsize") = py::none(), py::keep_alive<1, 2>()) + .def_static("from_bitbuffer", [](const zserio::BitBuffer& bitBuffer){ + return std::make_unique(bitBuffer); + }, py::keep_alive<0, 1>()) + .def("read_bits", [](zserio::BitStreamReader& self, py::int_ numbits){ + if (numbits < 0) + throw zserio::CppRuntimeException("BitStreamReader: Reading negative number of bits!"); + return self.readBits64(numbits); + }) + .def("read_signed_bits", [](zserio::BitStreamReader& self, py::int_ numbits){ + if (numbits < 0) + throw zserio::CppRuntimeException("BitStreamReader: Reading negative number of bits!"); + return self.readSignedBits64(numbits); + }) + .def("read_bits_unchecked", [](zserio::BitStreamReader& self, py::int_ numbits){ + if (numbits < 0) + throw zserio::CppRuntimeException("BitStreamReader: Reading negative number of bits!"); + return self.readBits64(numbits); + }) + .def("read_signed_bits_unchecked", [](zserio::BitStreamReader& self, py::int_ numbits){ + if (numbits < 0) + throw zserio::CppRuntimeException("BitStreamReader: Reading negative number of bits!"); + return self.readSignedBits64(numbits); + }) + .def("read_varint16", &zserio::BitStreamReader::readVarInt16) + .def("read_varint32", &zserio::BitStreamReader::readVarInt32) + .def("read_varint64", &zserio::BitStreamReader::readVarInt64) + .def("read_varint", &zserio::BitStreamReader::readVarInt) + .def("read_varuint16", &zserio::BitStreamReader::readVarUInt16) + .def("read_varuint32", &zserio::BitStreamReader::readVarUInt32) + .def("read_varuint64", &zserio::BitStreamReader::readVarUInt64) + .def("read_varuint", &zserio::BitStreamReader::readVarUInt) + .def("read_varsize", &zserio::BitStreamReader::readVarSize) + .def("read_float16", &zserio::BitStreamReader::readFloat16) + .def("read_float32", &zserio::BitStreamReader::readFloat32) + .def("read_float64", &zserio::BitStreamReader::readFloat64) + .def("read_bytes", [](zserio::BitStreamReader& self){ + auto bytes = self.readBytes(); + return py::bytearray(reinterpret_cast(bytes.data()), bytes.size()); + }) + .def("read_string", [](zserio::BitStreamReader& self){ + return self.readString(); + }) + .def("read_bool", &zserio::BitStreamReader::readBool) + .def("read_bitbuffer", [](zserio::BitStreamReader& self) { + return self.readBitBuffer(); + }) + .def("alignto", &zserio::BitStreamReader::alignTo) + .def_property("bitposition", + &zserio::BitStreamReader::getBitPosition, + [](zserio::BitStreamReader& self, py::int_ bitposition) { + if (bitposition < 0) + { + throw zserio::CppRuntimeException( + "BitStreamReader: Cannot set negative bit position!"); + } + self.setBitPosition(bitposition); + }) + .def_property_readonly("buffer_bitsize", + &zserio::BitStreamReader::getBufferBitSize) + ; + + py::class_(m, "BitBuffer") + // constructor from bytes + .def(py::init([](const py::bytes& buffer, std::optional bitsize) { + return bitBufferInit(py::buffer(buffer).request(), bitsize); + }), py::arg("buffer"), py::arg("bitsize") = py::none()) + // constructor from bytearray + .def(py::init([](const py::bytearray& buffer, std::optional bitsize) { + return bitBufferInit(py::buffer(buffer).request(), bitsize); + }), py::arg("buffer"), py::arg("bitsize") = py::none()) + // constructor from memoryview + .def(py::init([](const py::memoryview& buffer, std::optional bitsize) { + return bitBufferInit(py::buffer(buffer).request(), bitsize); + }), py::arg("buffer"), py::arg("bitsize") = py::none()) + .def_property_readonly("buffer", [](zserio::BitBuffer& self){ + constexpr bool readOnly = false; + return py::memoryview::from_memory(self.getBuffer(), self.getByteSize(), readOnly); + }, py::keep_alive<0, 1>()) + .def_property_readonly("bitsize", &zserio::BitBuffer::getBitSize) + .def(py::self == py::self) + .def("__hash__", &zserio::BitBuffer::hashCode) + ; + + // must be at the end to prevent cyclic imports! + py::register_local_exception(m, "CppRuntimeException", + py::module_::import("zserio.exception").attr("PythonRuntimeException")); +} diff --git a/compiler/extensions/python/runtime/src/zserio_cpp/setup.py b/compiler/extensions/python/runtime/src/zserio_cpp/setup.py new file mode 100644 index 000000000..f6c87ef47 --- /dev/null +++ b/compiler/extensions/python/runtime/src/zserio_cpp/setup.py @@ -0,0 +1,69 @@ +import sys +import os +from pathlib import Path +from pybind11.setup_helpers import Pybind11Extension, ParallelCompile, build_ext +from setuptools import setup +from distutils.command.build import build + +os.chdir(Path(__file__).parent.resolve()) + +OPTIONS = [ + ('cpp-runtime-dir=', None, 'Directory containing C++ runtime soruces.') +] + +class ZserioBuild(build): + user_options = build.user_options + OPTIONS + + def initialize_options(self): + build.initialize_options(self) + self.cpp_runtime_dir = None + + def finalize_options(self): + build.finalize_options(self) + if not self.cpp_runtime_dir: + raise Exception("Parameter '--cpp-runtime-dir' is missing!") + if not os.path.exists(Path(self.cpp_runtime_dir) / 'zserio'): + raise Exception("Parameter '--cpp-runtime-dir' does not point to Zserio C++ runitme soruces!") + +class ZserioBuildExt(build_ext): + user_options = build_ext.user_options + OPTIONS + + def initialize_options(self): + build_ext.initialize_options(self) + self.cpp_runtime_dir = None + + def finalize_options(self): + build_ext.finalize_options(self) + self.set_undefined_options('build', ('cpp_runtime_dir', 'cpp_runtime_dir')) + + zserio_cpp = self.extensions[0] + + zserio_cpp.sources.extend((str(filename) for filename in Path('.').rglob('*.cpp'))) + zserio_cpp.sources.extend( + (str(filename) for filename in Path(self.cpp_runtime_dir).resolve().rglob('*.cpp')) + ) + + zserio_cpp.include_dirs.append(self.cpp_runtime_dir) + + zserio_cpp.extra_compile_args.extend(['-O3']) + + if sys.maxsize > 2**32: + zserio_cpp.define_macros.append(('ZSERIO_RUNTIME_64BIT', None)) + +ParallelCompile(default=0).install() + +setup( + name='zserio_cpp', + version=0.1, + url="https://github.com/ndsev/zserio", + author='Navigation Data Standard e.V.', + author_email='support@nds-association.org', + description='Zserio C++ runtime binding to Python', + + ext_modules=[Pybind11Extension('zserio_cpp', sources=[])], # will be set-up in ZserioBuildExt + cmdclass={"build": ZserioBuild, "build_ext": ZserioBuildExt}, + + python_requires='>=3.8', + + license = "BSD-3 Clause", +)