Skip to content

Commit

Permalink
Optimized writing big bytes object into array
Browse files Browse the repository at this point in the history
when running ts.void.alloc_ptr() on very big bytes object
headlock got very slow. This is optimized now.
  • Loading branch information
mrh1997 committed Feb 1, 2023
1 parent 2f1b366 commit fce340b
Show file tree
Hide file tree
Showing 2 changed files with 34 additions and 18 deletions.
17 changes: 3 additions & 14 deletions headlock/c_data_model/array.py
Expand Up @@ -69,6 +69,9 @@ def convert_to_c_repr(self, py_val):
try:
return super().convert_to_c_repr(py_val)
except NotImplementedError:
if isinstance(py_val, (bytes, bytearray)) and \
self.base_type.sizeof == 1:
return py_val + (b'\x00' * (self.sizeof - len(py_val)))
if isinstance(py_val, Iterable):
payload = b''.join(map(self.base_type.convert_to_c_repr,py_val))
return payload + (b'\x00' * (self.sizeof - len(payload)))
Expand Down Expand Up @@ -135,20 +138,6 @@ def _get_len(cls):
def __iter__(self):
return (self[ndx] for ndx in range(self.element_count))

@property
def val(self):
return [self[ndx].val for ndx in range(self.element_count)]

@val.setter
def val(self, new_val):
if isinstance(new_val, str):
new_val = map_unicode_to_list(new_val, self.base_type)
ndx = 0
for ndx, val in enumerate(new_val):
self[ndx].val = val
for ndx2 in range(ndx+1, self.element_count):
self[ndx2].val = self.base_type.null_val

@property
def c_str(self):
val = self.val
Expand Down
35 changes: 31 additions & 4 deletions tests/test_c_data_model/test_array.py
Expand Up @@ -3,7 +3,7 @@

import headlock.c_data_model as cdm
from headlock.address_space.virtual import VirtualAddressSpace

from time import time


@pytest.fixture
Expand Down Expand Up @@ -111,10 +111,14 @@ class TestCArray:

def create_int_carray_obj(self, bits, init_val):
cint_type = cdm.CIntType('i'+str(bits), bits, False, cdm.ENDIANESS)
content = b''.join(map(cint_type.convert_to_c_repr, init_val))
if isinstance(init_val, int):
content = b'\00' * (bits//8 * init_val)
size = init_val
else:
content = b''.join(map(cint_type.convert_to_c_repr, init_val))
size = len(init_val)
addrspace = VirtualAddressSpace(content)
carray_type = cdm.CArrayType(cint_type.bind(addrspace), len(init_val),
addrspace)
carray_type = cdm.CArrayType(cint_type.bind(addrspace), size, addrspace)
return cdm.CArray(carray_type, 0)

def test_str_returnsStringWithZeros(self):
Expand Down Expand Up @@ -147,6 +151,29 @@ def test_setUnicodeStr_onPyStr_changesArrayToZeroTerminatedString(self):
carray_obj.unicode_str = '\u1234\x56\0\x78'
assert carray_obj.val == [0x1234, 0x56, 0, 0x78, 0, 0]

def test_init_onVeryBigBytesObject_providesOptimizedImplementation(self):
big_array_size = 1000000
array = self.create_int_carray_obj(8, big_array_size+1)
start_timestamp = time()
array.ctype(b'\x00' * big_array_size)
assert time() - start_timestamp < 0.050

def test_setVal_onComplexStructure_convertsBaseType(self):
array = self.create_int_carray_obj(32, 2)
array.val = [0x01234567, 0x89ABCDEF]
assert array.val == [0x01234567, 0x89ABCDEF]

def test_setVal_onVeryBigBytesObject_providesOptimizedImplementation(self):
big_array_size = 1000000
array = self.create_int_carray_obj(8, big_array_size+1)
start_timestamp = time()
array.val = b'\x00' * big_array_size
assert time() - start_timestamp < 0.050

def test_getVal_onComplexStructure_convertsBaseType(self):
array = self.create_int_carray_obj(32, [0x01234567, 0x89ABCDEF])
assert array.val == [0x01234567, 0x89ABCDEF]

def test_getItem_returnsObjectAtNdx(self):
carray_obj = self.create_int_carray_obj(16, [1, 2, 3, 4])
assert carray_obj[2].__address__ \
Expand Down

0 comments on commit fce340b

Please sign in to comment.