From ea8a4fb542b6fe374ec9cfef72fbb2f31601288e Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 6 Feb 2019 17:30:20 +0800 Subject: [PATCH 1/3] Add encode cache --- ssz/codec.py | 19 ++++++- ssz/sedes/serializable.py | 2 + tests/sedes/test_speed.py | 113 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 tests/sedes/test_speed.py diff --git a/ssz/codec.py b/ssz/codec.py index 78d5cbe1..c578df24 100644 --- a/ssz/codec.py +++ b/ssz/codec.py @@ -3,6 +3,7 @@ ) from ssz.sedes import ( + Serializable, sedes_by_name, ) from ssz.sedes.base import ( @@ -13,7 +14,7 @@ ) -def encode(value, sedes=None): +def encode(value, sedes=None, cache=True): """ Encode object in SSZ format. `sedes` needs to be explicitly mentioned for encode/decode @@ -21,6 +22,18 @@ def encode(value, sedes=None): `sedes` parameter could be given as a string or as the actual sedes object itself. """ + if isinstance(value, Serializable): + cached_ssz = value._cached_ssz + if sedes is None and cached_ssz: + return cached_ssz + else: + really_cache = ( + cache and + sedes is None + ) + else: + really_cache = False + if sedes is not None: if sedes in sedes_by_name: # Get the actual sedes object from string representation @@ -35,6 +48,10 @@ def encode(value, sedes=None): sedes_obj = infer_sedes(value) serialized_obj = sedes_obj.serialize(value) + + if really_cache: + value._cached_ssz = serialized_obj + return serialized_obj diff --git a/ssz/sedes/serializable.py b/ssz/sedes/serializable.py index d7f2cb54..ad7de65e 100644 --- a/ssz/sedes/serializable.py +++ b/ssz/sedes/serializable.py @@ -75,6 +75,8 @@ def merge_args_to_kwargs(args, kwargs, arg_names): class BaseSerializable(collections.Sequence): + _cached_ssz = None + def __init__(self, *args, **kwargs): validate_args_and_kwargs(args, kwargs, self._meta.field_names) field_values = merge_kwargs_to_args(args, kwargs, self._meta.field_names) diff --git a/tests/sedes/test_speed.py b/tests/sedes/test_speed.py new file mode 100644 index 00000000..10378174 --- /dev/null +++ b/tests/sedes/test_speed.py @@ -0,0 +1,113 @@ +import time + +from ssz import ( + encode, +) +from ssz.sedes import ( + List, + Serializable, + bytes32, + uint24, + uint64, + uint384, +) + + +class ValidatorRecord(Serializable): + fields = [ + ('pubkey', uint384), + ('withdrawal_credentials', bytes32), + ('randao_commitment', bytes32), + ('randao_layers', uint64), + ('status', uint64), + ('latest_status_change_slot', uint64), + ('exit_count', uint64), + ('poc_commitment', bytes32), + ('last_poc_change_slot', uint64), + ('second_last_poc_change_slot', uint64), + ] + + +class CrosslinkRecord(Serializable): + fields = [ + ('slot', uint64), + ('shard_block_root', bytes32), + ] + + +class ShardCommittee(Serializable): + fields = [ + ('shard', uint64), + ('committee', List(uint24)), + ('total_validator_count', uint64), + ] + + +class State(Serializable): + fields = [ + ('validator_registry', List(ValidatorRecord)), + ('shard_and_committee_for_slots', List(List(ShardCommittee))), + ('latest_crosslinks', List(CrosslinkRecord)), + ] + + +validator_record = ValidatorRecord( + pubkey=123, + withdrawal_credentials=b'\x56' * 32, + randao_commitment=b'\x56' * 32, + randao_layers=123, + status=123, + latest_status_change_slot=123, + exit_count=123, + poc_commitment=b'\x56' * 32, + last_poc_change_slot=123, + second_last_poc_change_slot=123, +) +crosslink_record = CrosslinkRecord(slot=12847, shard_block_root=b'\x67' * 32) +crosslink_record_stubs = [crosslink_record for i in range(1024)] + + +def make_state(num_validators): + shard_committee = ShardCommittee( + shard=1, + committee=tuple(range(num_validators // 1024)), + total_validator_count=num_validators, + ) + shard_committee_stubs = tuple(tuple(shard_committee for i in range(16)) for i in range(64)) + state = State( + validator_registry=tuple(validator_record for i in range(num_validators)), + shard_and_committee_for_slots=shard_committee_stubs, + latest_crosslinks=crosslink_record_stubs, + ) + return state + + +def do_test_serialize(state, rounds=100): + for _ in range(rounds): + x = encode(state, cache=True) + return x + + +def do_test_serialize_no_cache(state, rounds=100): + for _ in range(rounds): + x = encode(state, cache=False) + + return x + + +def test_encode_cache(): + state = make_state(2**10) + + start_time = time.time() + without_cache_result = do_test_serialize_no_cache(state) + without_cache_actual_performance = time.time() - start_time + print("Performance of serialization without cache", without_cache_actual_performance) + + state = make_state(2**10) + start_time = time.time() + with_cache_result = do_test_serialize(state) + with_cache_actual_performance = time.time() - start_time + print("Performance of serialization with cache", with_cache_actual_performance) + + assert with_cache_result == without_cache_result + assert with_cache_actual_performance * 10 < without_cache_actual_performance From 89eb0e7ad2d1c745242840f9aa939c30e5d0b806 Mon Sep 17 00:00:00 2001 From: jannikluhn Date: Wed, 6 Feb 2019 20:20:20 +0800 Subject: [PATCH 2/3] Update ssz/codec.py Co-Authored-By: hwwhww --- ssz/codec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ssz/codec.py b/ssz/codec.py index c578df24..05ad050a 100644 --- a/ssz/codec.py +++ b/ssz/codec.py @@ -24,7 +24,7 @@ def encode(value, sedes=None, cache=True): """ if isinstance(value, Serializable): cached_ssz = value._cached_ssz - if sedes is None and cached_ssz: + if sedes is None and cached_ssz is not None: return cached_ssz else: really_cache = ( From d166d6c96deb080c73c2d13f7389bdd34c7bc8f1 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 6 Feb 2019 20:28:07 +0800 Subject: [PATCH 3/3] Add docstring for `encode` --- ssz/codec.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ssz/codec.py b/ssz/codec.py index 05ad050a..cfee8563 100644 --- a/ssz/codec.py +++ b/ssz/codec.py @@ -21,6 +21,14 @@ def encode(value, sedes=None, cache=True): of integers(as of now). `sedes` parameter could be given as a string or as the actual sedes object itself. + + If `value` has an attribute :attr:`_cached_ssz` (as, notably, + :class:`ssz.sedes.Serializable`) and its value is not `None`, this value is + returned bypassing serialization and encoding, unless `sedes` is given (as + the cache is assumed to refer to the standard serialization which can be + replaced by specifying `sedes`). + If `value` is a :class:`ssz.sedes.Serializable` and `cache` is true, the result of + the encoding will be stored in :attr:`_cached_ssz` if it is empty. """ if isinstance(value, Serializable): cached_ssz = value._cached_ssz