Skip to content

Commit

Permalink
encoding rewrite
Browse files Browse the repository at this point in the history
  • Loading branch information
@TingDaoK
TingDaoK committed Apr 22, 2024
1 parent da13a7f commit eefd3f0
Show file tree
Hide file tree
Showing 6 changed files with 284 additions and 116 deletions.
100 changes: 59 additions & 41 deletions awscrt/cbor.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ class AwsCborEncoder(NativeResource):

def __init__(self):
super().__init__()
self._binding = _awscrt.cbor_encoder_new()
self._binding = _awscrt.cbor_encoder_new(self)

def get_encoded_data(self) -> bytes:
"""Return the current encoded data as bytes
Expand Down Expand Up @@ -193,6 +193,24 @@ def write_list(self, val: list):
for data_item in val:
self.write_data_item(data_item)

def write_data_item_2(self, data_item: Any):
"""Generic API to write any type of an data_item as cbor formatted.
TODO: timestamp <-> datetime?? Decimal fraction <-> decimal??
Args:
data_item (Any): any type of data_item. If the type is not supported to be converted to cbor format, ValueError will be raised.
"""
return _awscrt.cbor_encoder_write_data_item(self._binding, data_item)

def print_key(self, key):
print(f"key: {key}")


def print_value(self, value):
print(f"value: {value}")

def print_length(self, value):
print(f"length: {value}")

class AwsCborDecoder(NativeResource):
""" Decoder for CBOR """
Expand Down Expand Up @@ -283,48 +301,48 @@ def pop_next_inf_str(self) -> bytes:
return result

def pop_next_list(self) -> list:
return _awscrt.cbor_decoder_pop_next_py_list(self._binding)
# type = _awscrt.cbor_decoder_peek_type(self._binding)
# return_val = []
# if type == AwsCborElementType.InfArray:
# # Consume the inf_array
# self.consume_next_element()
# while type != AwsCborElementType.Break:
# return_val.append(self.pop_next_data_item())
# type = _awscrt.cbor_decoder_peek_type(self._binding)
# # Consume the break
# self.consume_next_element()
# return return_val
# elif type == AwsCborElementType.ArrayStart:
# number_elements = self.pop_next_array_start()
# for i in range(number_elements):
# return_val.append(self.pop_next_data_item())
# return return_val
# else:
# raise ValueError("the cbor src is not a list to decode")
# return _awscrt.cbor_decoder_pop_next_py_list(self._binding)
type = _awscrt.cbor_decoder_peek_type(self._binding)
return_val = []
if type == AwsCborElementType.InfArray:
# Consume the inf_array
self.consume_next_element()
while type != AwsCborElementType.Break:
return_val.append(self.pop_next_data_item())
type = _awscrt.cbor_decoder_peek_type(self._binding)
# Consume the break
self.consume_next_element()
return return_val
elif type == AwsCborElementType.ArrayStart:
number_elements = self.pop_next_array_start()
for i in range(number_elements):
return_val.append(self.pop_next_data_item())
return return_val
else:
raise ValueError("the cbor src is not a list to decode")

def pop_next_map(self) -> dict:
return _awscrt.cbor_decoder_pop_next_py_dict(self._binding)
# type = _awscrt.cbor_decoder_peek_type(self._binding)
# return_val = {}
# if type == AwsCborElementType.InfMap:
# # Consume the inf_map
# self.consume_next_element()
# while type != AwsCborElementType.Break:
# return_val[self.pop_next_data_item()] = self.pop_next_data_item()
# type = _awscrt.cbor_decoder_peek_type(self._binding)
# # Consume the break
# self.consume_next_element()
# return return_val
# elif type == AwsCborElementType.MapStart:
# number_elements = self.pop_next_map_start()
# for i in range(number_elements):
# key = self.pop_next_data_item()
# value = self.pop_next_data_item()
# return_val[key] = value
# return return_val
# else:
# raise ValueError("the cbor src is not a map to decode")
# return _awscrt.cbor_decoder_pop_next_py_dict(self._binding)
type = _awscrt.cbor_decoder_peek_type(self._binding)
return_val = {}
if type == AwsCborElementType.InfMap:
# Consume the inf_map
self.consume_next_element()
while type != AwsCborElementType.Break:
return_val[self.pop_next_data_item()] = self.pop_next_data_item()
type = _awscrt.cbor_decoder_peek_type(self._binding)
# Consume the break
self.consume_next_element()
return return_val
elif type == AwsCborElementType.MapStart:
number_elements = self.pop_next_map_start()
for i in range(number_elements):
key = self.pop_next_data_item()
value = self.pop_next_data_item()
return_val[key] = value
return return_val
else:
raise ValueError("the cbor src is not a map to decode")

def pop_next_data_item(self) -> Any:
# TODO: timestamp, decimal fraction
Expand Down
24 changes: 23 additions & 1 deletion benchmark_cbor.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ def random_number(r, n):
t = TestData.test_hash(100000)


# print(t)

print("cbor2 -- encode")
run_start_ns = time.perf_counter_ns()
cbor2_encoded = cbor2.dumps(t)
Expand All @@ -52,6 +54,21 @@ def random_number(r, n):
print(f"time passed: {run_secs} secs")


print("CRT -- encode 2")
encoder_2 = AwsCborEncoder()

run_start_ns = time.perf_counter_ns()
try:
encoder_2.write_data_item_2(t)
encoded_2 = encoder_2.get_encoded_data()
except Exception as e:
print(e)

run_secs = ns_to_secs(time.perf_counter_ns() - run_start_ns)
print(f"encoded MB: {bytes_to_MiB(len(encoded_2))}")
print(f"time passed: {run_secs} secs")


print("CRT -- encode")
encoder = AwsCborEncoder()

Expand All @@ -62,7 +79,9 @@ def random_number(r, n):
print(f"encoded MB: {bytes_to_MiB(len(encoded))}")
print(f"time passed: {run_secs} secs")


print(cbor2_encoded == encoded)
print(cbor2_encoded == encoded_2)

print("cbor2 -- decode")
run_start_ns = time.perf_counter_ns()
Expand All @@ -82,10 +101,13 @@ def random_number(r, n):
print("CRT -- decode 2")
run_start_ns = time.perf_counter_ns()
decoder_2 = AwsCborDecoder(encoded)
crt_decoded = decoder_2.pop_next_data_item_2()
crt_decoded_2 = decoder_2.pop_next_data_item_2()

run_secs = ns_to_secs(time.perf_counter_ns() - run_start_ns)
print(f"time passed: {run_secs} secs")

print(crt_decoded == t)
print(crt_decoded == decoded)

print(crt_decoded_2 == t)
print(crt_decoded_2 == decoded)
Loading

0 comments on commit eefd3f0

Please sign in to comment.