[#469] Run python runtime tests also with C++ optimizations

compiler/extensions/python/runtime/tests/test_bitbuffer.py

@@ -25,7 +25,7 @@ def test_buffer_bitsize_constructor(self):
 
         out_of_range_bitsize = 9
         with self.assertRaises(PythonRuntimeException):
-            BitBuffer([1], out_of_range_bitsize) # throws!
+            BitBuffer(bytes([1]), out_of_range_bitsize) # throws!
 
     def test_eq(self):
         bitsize = 11

compiler/extensions/python/runtime/tests/test_bitsizeof.py

@@ -280,20 +280,20 @@ def test_bitsizeof_string(self):
         self.assertEqual((2 + 2 * test_string_length) * 8, bitsizeof_string(test_string))
 
     def test_bitsizeof_bitbuffer(self):
-        test_bitbuffer1 = BitBuffer([0xAB, 0x03], 8)
+        test_bitbuffer1 = BitBuffer(bytes([0xAB, 0x03]), 8)
         self.assertEqual(8 + 8, bitsizeof_bitbuffer(test_bitbuffer1))
 
-        test_bitbuffer2 = BitBuffer([0xAB, 0x03], 11)
+        test_bitbuffer2 = BitBuffer(bytes([0xAB, 0x03]), 11)
         self.assertEqual(8 + 11, bitsizeof_bitbuffer(test_bitbuffer2))
 
-        test_bitbuffer3 = BitBuffer([0xAB, 0xCD], 16)
+        test_bitbuffer3 = BitBuffer(bytes([0xAB, 0xCD]), 16)
         self.assertEqual(8 + 16, bitsizeof_bitbuffer(test_bitbuffer3))
 
-        test_bitbuffer4 = BitBuffer([0xAB, 0xCD])
+        test_bitbuffer4 = BitBuffer(bytes([0xAB, 0xCD]))
         self.assertEqual(8 + 16, bitsizeof_bitbuffer(test_bitbuffer4))
 
-        test_bitbuffer5 = BitBuffer(16 * [1], 127)
+        test_bitbuffer5 = BitBuffer(bytes(16 * [1]), 127)
         self.assertEqual(8 + 15 * 8 + 7, bitsizeof_bitbuffer(test_bitbuffer5))
 
-        test_bitbuffer6 = BitBuffer(16 * [1], 128)
+        test_bitbuffer6 = BitBuffer(bytes(16 * [1]), 128)
         self.assertEqual(16 + 16 * 8, bitsizeof_bitbuffer(test_bitbuffer6))

compiler/extensions/python/runtime/tests/test_bitstream.py

@@ -476,13 +476,19 @@ def _test_bits_impl(self, write_method, read_method, values, numbits):
     def _test_impl(self, write_method, read_method, values, max_start_bit_pos):
         for bit_pos in range(max_start_bit_pos):
             writer = BitStreamWriter()
-            if bit_pos > 0:
+            if bit_pos > 64:
+                writer.write_bits(0, 64)
+                writer.write_bits(0, bit_pos - 64)
+            elif bit_pos > 0:
                 writer.write_bits(0, bit_pos)
             for value in values:
                 write_method(writer, value)
 
             reader = BitStreamReader(buffer=writer.byte_array)
-            if bit_pos > 0:
+            if bit_pos > 64:
+                reader.read_bits(64)
+                reader.read_bits(bit_pos - 64)
+            elif bit_pos > 0:
                 reader.read_bits(bit_pos)
             for value in values:
                 self.assertEqual(value, read_method(reader), f"[bit_pos={bit_pos}]")

compiler/extensions/python/runtime/tests/test_cppbind.py

@@ -12,19 +12,17 @@ class CppBindTest(unittest.TestCase):
     def test_invalid_env(self):
         os.environ["ZSERIO_PYTHON_IMPLEMENTATION"] = "invalid"
         with self.assertRaises(PythonRuntimeException):
-            import_cpp_class(MissingCppClass)
+            import_cpp_class("MissingCppClass")
 
     def test_missing_cpp_default(self):
         os.environ.pop("ZSERIO_PYTHON_IMPLEMENTATION")
-        imported = import_cpp_class(MissingCppClass)
-        self.assertEqual(MissingCppClass, imported)
+        self.assertIsNone(import_cpp_class("MissingCppClass"))
 
     def test_missing_cpp_python(self):
         os.environ["ZSERIO_PYTHON_IMPLEMENTATION"] = "python"
-        imported = import_cpp_class(MissingCppClass)
-        self.assertEqual(MissingCppClass, imported)
+        self.assertIsNone(import_cpp_class("MissingCppClass"))
 
     def test_missing_cpp_cpp(self):
         os.environ["ZSERIO_PYTHON_IMPLEMENTATION"] = "cpp"
         with self.assertRaises(PythonRuntimeException):
-            import_cpp_class(MissingCppClass)
+            import_cpp_class("MissingCppClass")

compiler/extensions/python/runtime/tests/test_creator.py

@@ -34,7 +34,7 @@ def test_create_object_full(self):
         creator.begin_compound("nested")
         creator.set_value("value", 10)
         creator.set_value("text", "nested")
-        creator.set_value("externData", BitBuffer([0x3c], 6))
+        creator.set_value("externData", BitBuffer(bytes([0x3c]), 6))
         creator.set_value("bytesData", bytearray([0xff]))
         creator.set_value("creatorEnum", CreatorEnum.ONE)
         creator.set_value("creatorBitmask", CreatorBitmask.Values.WRITE)
@@ -54,7 +54,7 @@ def test_create_object_full(self):
         creator.add_value_element("array")
         creator.end_array()
         creator.begin_array("externArray")
-        creator.add_value_element(BitBuffer([0x0f], 4))
+        creator.add_value_element(BitBuffer(bytes([0x0f]), 4))
         creator.end_array()
         creator.begin_array("bytesArray")
         creator.add_value_element(bytearray([0xca, 0xfe]))
@@ -70,7 +70,7 @@ def test_create_object_full(self):
         self.assertEqual(13, obj.nested.param)
         self.assertEqual(10, obj.nested.value)
         self.assertEqual("nested", obj.nested.text)
-        self.assertEqual([0x3c], obj.nested.extern_data.buffer)
+        self.assertEqual(bytes([0x3c]), obj.nested.extern_data.buffer)
         self.assertEqual(bytes([0xff]), obj.nested.bytes_data)
         self.assertEqual(6, obj.nested.extern_data.bitsize)
         self.assertEqual(CreatorEnum.ONE, obj.nested.creator_enum)
@@ -82,7 +82,7 @@ def test_create_object_full(self):
         self.assertEqual(CreatorBitmask.Values.READ, obj.nested_array[0].creator_bitmask)
         self.assertEqual(["this", "is", "text", "array"], obj.text_array)
         self.assertEqual(1, len(obj.extern_array))
-        self.assertEqual([0x0f], obj.extern_array[0].buffer)
+        self.assertEqual(bytes([0x0f]), obj.extern_array[0].buffer)
         self.assertEqual(4, obj.extern_array[0].bitsize)
         self.assertEqual(1, len(obj.bytes_array))
         self.assertEqual(bytes([0xca, 0xfe]), obj.bytes_array[0])

compiler/extensions/python/runtime/tests/test_hashcode.py

@@ -108,7 +108,7 @@ def test_string_type(self):
 
     def test_bitbuffer_type(self):
         hash_seed = 1
-        bitbuffer_value = BitBuffer([])
+        bitbuffer_value = BitBuffer(bytes())
         self.assertEqual(HASH_PRIME_NUMBER + HASH_SEED, calc_hashcode_object(hash_seed, bitbuffer_value))
 
     def test_enum_type(self):
@@ -200,7 +200,7 @@ def test_str_array_type(self):
 
     def test_bitbuffer_array_type(self):
         hash_seed = 1
-        bitbuffer_array_value = [ BitBuffer([]) ]
+        bitbuffer_array_value = [ BitBuffer(bytes()) ]
         self.assertEqual(HASH_PRIME_NUMBER + HASH_SEED,
                          calc_hashcode_object_array(hash_seed, bitbuffer_array_value))
 

scripts/build.sh

@@ -19,8 +19,9 @@ test_python_runtime()
     echo "Running python runtime unit tests."
     echo
 
+    ZSERIO_PYTHON_IMPLEMENTATION="python" \
     PYTHONPATH="${SOURCES_DIR}" python \
-            -m coverage run --source "${PYTHON_RUNTIME_ROOT}/" \
+            -m coverage run --source "${PYTHON_RUNTIME_ROOT}/" --data-file=coverage_python.data \
             -m unittest discover -s "${TESTS_DIR}" -v
     local PYTHON_RESULT=$?
     if [ ${PYTHON_RESULT} -ne 0 ] ; then
@@ -30,11 +31,34 @@ test_python_runtime()
     fi
     echo
 
+    echo "Running python runtime unit tests with C++ optimizations."
+    echo
+
+    local ZSERIO_CPP_DIR
+    ZSERIO_CPP_DIR=$(ls -d1 "${BUILD_DIR}/zserio_cpp/lib"*)
+    if [ $? -ne 0 ] ; then
+        stderr_echo "Failed to locate C++ runtime binding to Python!"
+        return 1
+    fi
+
+    ZSERIO_PYTHON_IMPLEMENTATION="cpp" \
+    PYTHONPATH="${SOURCES_DIR}:${ZSERIO_CPP_DIR}" python \
+            -m coverage run --source "${PYTHON_RUNTIME_ROOT}/" --data-file=coverage_cpp.data \
+            -m unittest discover -s "${TESTS_DIR}" -v
+    local PYTHON_RESULT=$?
+    if [ ${PYTHON_RESULT} -ne 0 ] ; then
+        stderr_echo "Running python unit tests with C++ optimizations failed with return code ${PYTHON_RESULT}!"
+        popd > /dev/null
+        return 1
+    fi
+    echo
+
     echo "Running python coverage report."
     echo
 
-    python -m coverage html --directory="coverage" --fail-under=100 --omit="*test_object*" \
-            --title="Zserio Python Runtime Library"
+    python -m coverage combine --keep coverage_cpp.data coverage_python.data
+    python -m coverage html --directory="coverage" --fail-under=100 \
+            --omit="*test_object*,*cppbind*" --title="Zserio Python Runtime Library"
     local COVERAGE_RESULT=$?
     if [ ${COVERAGE_RESULT} -ne 0 ] ; then
         stderr_echo "Running python coverage report failed with return code ${COVERAGE_RESULT}!"
@@ -49,7 +73,7 @@ test_python_runtime()
 
     local PYLINT_RCFILE="${PYTHON_RUNTIME_ROOT}/pylintrc.txt"
     local PYLINT_ARGS=("--disable=too-few-public-methods")
-    run_pylint "${PYLINT_RCFILE}" PYLINT_ARGS[@] "${SOURCES_DIR}"/*
+    PYTHONPATH="${ZSERIO_CPP_DIR}" run_pylint "${PYLINT_RCFILE}" PYLINT_ARGS[@] "${SOURCES_DIR}"/zserio
     if [ $? -ne 0 ]; then
         return 1
     fi
@@ -67,7 +91,7 @@ test_python_runtime()
 
     local MYPY_CONFIG_FILE="${PYTHON_RUNTIME_ROOT}/mypy.ini"
     local MYPY_ARGS=()
-    run_mypy "${BUILD_DIR}" "${MYPY_CONFIG_FILE}" MYPY_ARGS[@] "${SOURCES_DIR}"/*
+    run_mypy "${BUILD_DIR}" "${MYPY_CONFIG_FILE}" MYPY_ARGS[@] "${SOURCES_DIR}"/zserio
     if [ $? -ne 0 ]; then
         return 1
     fi
@@ -121,10 +145,10 @@ install_python_runtime()
     rm -rf "${PYTHON_RUNTIME_DISTR_DIR}/"
     mkdir -p "${PYTHON_RUNTIME_DISTR_DIR}"
 
-    # install sources and doc
+    # install sources
     pushd "${PYTHON_RUNTIME_SOURCES}" > /dev/null
 
-    "${FIND}" . -name "*.py" | while read -r SOURCE ; do
+    "${FIND}" . -name "*.py" -o -name "*.cpp" | while read -r SOURCE ; do
         echo "Installing ${SOURCE}"
         cp --parents "${SOURCE}" "${PYTHON_RUNTIME_DISTR_DIR}"
         if [ $? -ne 0 ] ; then
@@ -134,15 +158,16 @@ install_python_runtime()
         fi
     done
 
+    popd > /dev/null
+
+    # install doc
     echo "Installing API documentation"
     cp -r "${PYTHON_RUNTIME_DOC_BUILD_DIR}/zserio_doc" "${PYTHON_RUNTIME_DISTR_DIR}/"
     if [ $? -ne 0 ] ; then
         stderr_echo "Failed to install documentation!"
-        popd > /dev/null
         return 1
     fi
 
-    popd > /dev/null
     return 0
 }
 
@@ -586,8 +611,18 @@ main()
                 return 1
             fi
 
+            local CPP_RUNTIME_DIR="${ZSERIO_PROJECT_ROOT}/compiler/extensions/cpp/runtime/src"
             local PYTHON_RUNTIME_ROOT="${ZSERIO_PROJECT_ROOT}/compiler/extensions/python/runtime"
 
+            # compile C++ runtime binding to Python
+            python ${PYTHON_RUNTIME_ROOT}/src/zserio_cpp/setup.py build \
+                    --build-base="${PYTHON_RUNTIME_BUILD_DIR}/zserio_cpp" \
+                    --cpp-runtime-dir="${CPP_RUNTIME_DIR}"
+            if [ $? -ne 0 ] ; then
+                stderr_echo "Failed to build C++ runtime binding to Python!"
+                return 1
+            fi
+
             test_python_runtime "${PYTHON_RUNTIME_ROOT}" "${PYTHON_RUNTIME_BUILD_DIR}"
             if [ $? -ne 0 ] ; then
                 return 1

scripts/common_tools.sh

@@ -285,6 +285,7 @@ activate_python_virtualenv()
         "setuptools==59.8.0"
         "coverage==6.5.0" "sphinx"
         "astroid==2.9.3" "pylint==2.12.2" "mypy==0.931"
+        "pybind11>=2.10.0"
     )
     local APSW_REQUIREMENTS=("apsw")
 
@@ -369,7 +370,7 @@ Uses the following environment variables for building:
                            "Eclipse CDT4 - Unix Makefiles".
     MSVC_CMAKE_GENERATOR   CMake generator to use with MSVC compiler. Default is
                            "Visual Studio 17 2022". Note that CMake option "-A x64"
-                           is added automatically for windows64-mscv target. 
+                           is added automatically for windows64-mscv target.
     MSVC_CMAKE_TOOLSET     MSVC toolset specification for CMake generator.
                            Default is "v141". Note that "v141" is for VS 2017,
                            "v142" is for VS 2019.