From 828dd455ba158aaf187c2805a9561d90bbd0b20e Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 Mar 2019 10:18:57 -0600 Subject: [PATCH] add basic dependencies and build script for phase0 testing --- .gitignore | 5 + Makefile | 15 +++ requirements.txt | 6 + scripts/__init__.py | 0 scripts/phase0/__init__.py | 0 scripts/phase0/bls_stub.py | 12 ++ scripts/phase0/build_spec.py | 43 +++++++ scripts/phase0/function_puller.py | 46 +++++++ scripts/phase0/minimal_ssz.py | 190 +++++++++++++++++++++++++++++ scripts/phase0/monkey_patches.py | 29 +++++ scripts/phase0/state_transition.py | 84 +++++++++++++ tests/phase0/conftest.py | 6 + 12 files changed, 436 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 requirements.txt create mode 100644 scripts/__init__.py create mode 100644 scripts/phase0/__init__.py create mode 100644 scripts/phase0/bls_stub.py create mode 100644 scripts/phase0/build_spec.py create mode 100644 scripts/phase0/function_puller.py create mode 100644 scripts/phase0/minimal_ssz.py create mode 100644 scripts/phase0/monkey_patches.py create mode 100644 scripts/phase0/state_transition.py create mode 100644 tests/phase0/conftest.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..36c14f3434 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +*.pyc +/__pycache__ +/venv + +/build \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000..724a0392ec --- /dev/null +++ b/Makefile @@ -0,0 +1,15 @@ +SPEC_DIR = ./specs +SCRIPT_DIR = ./scripts +BUILD_DIR = ./build + +.PHONY: clean all + + +clean: + rm -rf $(BUILD_DIR) + + +$(BUILD_DIR)/phase0: + mkdir -p $@ + python3 $(SCRIPT_DIR)/phase0/build_spec.py $(SPEC_DIR)/core/0_beacon-chain.md $(SCRIPT_DIR)/phase0/minimal_ssz.py \ + $(SCRIPT_DIR)/phase0/bls_stub.py $(SCRIPT_DIR)/phase0/state_transition.py $(SCRIPT_DIR)/phase0/monkey_patches.py > $@/spec.py diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000..9145e951e5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +eth-utils>=1.3.0,<2 +eth-typing>=2.1.0,<3.0.0 +oyaml==0.7 +pycryptodome==3.7.3 +py_ecc>=1.6.0 +pytest>=3.6,<3.7 diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/phase0/__init__.py b/scripts/phase0/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/phase0/bls_stub.py b/scripts/phase0/bls_stub.py new file mode 100644 index 0000000000..7e3a6a308c --- /dev/null +++ b/scripts/phase0/bls_stub.py @@ -0,0 +1,12 @@ + + +def bls_verify(pubkey, message_hash, signature, domain): + return True + + +def bls_verify_multiple(pubkeys, message_hashes, signature, domain): + return True + + +def bls_aggregate_pubkeys(pubkeys): + return b'\x42'*96 diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py new file mode 100644 index 0000000000..c4f8ab38cb --- /dev/null +++ b/scripts/phase0/build_spec.py @@ -0,0 +1,43 @@ +import sys +import function_puller + +code_lines = [] + +for i in (1, 2, 3, 4, 8, 32, 48, 96): + code_lines.append("def int_to_bytes%d(x): return x.to_bytes(%d, 'little')" % (i, i)) +code_lines.append("SLOTS_PER_EPOCH = 64") # stub, will get overwritten by real var +code_lines.append("def slot_to_epoch(x): return x // SLOTS_PER_EPOCH") + +code_lines.append(""" +from typing import ( + Any, + Callable, + List, + NewType, + Tuple, +) + + +Slot = NewType('Slot', int) # uint64 +Epoch = NewType('Epoch', int) # uint64 +Shard = NewType('Shard', int) # uint64 +ValidatorIndex = NewType('ValidatorIndex', int) # uint64 +Gwei = NewType('Gwei', int) # uint64 +Bytes32 = NewType('Bytes32', bytes) # bytes32 +BLSPubkey = NewType('BLSPubkey', bytes) # bytes48 +BLSSignature = NewType('BLSSignature', bytes) # bytes96 +Any = None +Store = None +""") + + +code_lines += function_puller.get_lines(sys.argv[1]) + +print(open(sys.argv[2]).read()) +print(open(sys.argv[3]).read()) + +for line in code_lines: + print(line) + +print(open(sys.argv[4]).read()) +print(open(sys.argv[5]).read()) diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py new file mode 100644 index 0000000000..8d1c1a0cc0 --- /dev/null +++ b/scripts/phase0/function_puller.py @@ -0,0 +1,46 @@ +import sys + + +def get_lines(file_name): + code_lines = [] + pulling_from = None + current_name = None + processing_typedef = False + for linenum, line in enumerate(open(sys.argv[1]).readlines()): + line = line.rstrip() + if pulling_from is None and len(line) > 0 and line[0] == '#' and line[-1] == '`': + current_name = line[line[:-1].rfind('`')+1: -1] + if line[:9] == '```python': + assert pulling_from is None + pulling_from = linenum + 1 + elif line[:3] == '```': + if pulling_from is None: + pulling_from = linenum + else: + if processing_typedef: + assert code_lines[-1] == '}' + code_lines[-1] = '})' + pulling_from = None + processing_typedef = False + else: + if pulling_from == linenum and line == '{': + code_lines.append('%s = SSZType({' % current_name) + processing_typedef = True + elif pulling_from is not None: + code_lines.append(line) + elif pulling_from is None and len(line) > 0 and line[0] == '|': + row = line[1:].split('|') + if len(row) >= 2: + for i in range(2): + row[i] = row[i].strip().strip('`') + if '`' in row[i]: + row[i] = row[i][:row[i].find('`')] + eligible = True + if row[0][0] not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_': + eligible = False + for c in row[0]: + if c not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_0123456789': + eligible = False + if eligible: + code_lines.append(row[0] + ' = ' + (row[1].replace('**TBD**', '0x1234567890123567890123456789012357890'))) + return code_lines diff --git a/scripts/phase0/minimal_ssz.py b/scripts/phase0/minimal_ssz.py new file mode 100644 index 0000000000..5caaf8f099 --- /dev/null +++ b/scripts/phase0/minimal_ssz.py @@ -0,0 +1,190 @@ +from utils.hash import hash + + +BYTES_PER_CHUNK = 32 +BYTES_PER_LENGTH_PREFIX = 4 +ZERO_CHUNK = b'\x00' * BYTES_PER_CHUNK + +def SSZType(fields): + class SSZObject(): + def __init__(self, **kwargs): + for f in fields: + if f not in kwargs: + raise Exception("Missing constructor argument: %s" % f) + setattr(self, f, kwargs[f]) + + def __eq__(self, other): + return ( + self.fields == other.fields and + self.serialize() == other.serialize() + ) + + def __hash__(self): + return int.from_bytes(self.hash_tree_root(), byteorder="little") + + def __str__(self): + output = [] + for field in self.fields: + output.append(f'{field}: {getattr(self, field)}') + return "\n".join(output) + + def serialize(self): + return serialize_value(self, self.__class__) + + def hash_tree_root(self): + return hash_tree_root(self, self.__class__) + + SSZObject.fields = fields + return SSZObject + +class Vector(list): + def __init__(self, x): + list.__init__(self, x) + self.length = len(x) + + def append(*args): + raise Exception("Cannot change the length of a vector") + + remove = clear = extend = pop = insert = append + +def is_basic(typ): + return isinstance(typ, str) and (typ[:4] in ('uint', 'bool') or typ == 'byte') + +def is_constant_sized(typ): + if is_basic(typ): + return True + elif isinstance(typ, list) and len(typ) == 1: + return is_constant_sized(typ[0]) + elif isinstance(typ, list) and len(typ) == 2: + return False + elif isinstance(typ, str) and typ[:5] == 'bytes': + return len(typ) > 5 + elif hasattr(typ, 'fields'): + for subtype in typ.fields.values(): + if not is_constant_sized(subtype): + return False + return True + else: + raise Exception("Type not recognized") + +def coerce_to_bytes(x): + if isinstance(x, str): + o = x.encode('utf-8') + assert len(o) == len(x) + return o + elif isinstance(x, bytes): + return x + else: + raise Exception("Expecting bytes") + +def serialize_value(value, typ=None): + if typ is None: + typ = infer_type(value) + if isinstance(typ, str) and typ[:4] == 'uint': + length = int(typ[4:]) + assert length in (8, 16, 32, 64, 128, 256) + return value.to_bytes(length // 8, 'little') + elif typ == 'bool': + assert value in (True, False) + return b'\x01' if value is True else b'\x00' + elif (isinstance(typ, list) and len(typ) == 1) or typ == 'bytes': + serialized_bytes = coerce_to_bytes(value) if typ == 'bytes' else b''.join([serialize_value(element, typ[0]) for element in value]) + assert len(serialized_bytes) < 2**(8 * BYTES_PER_LENGTH_PREFIX) + serialized_length = len(serialized_bytes).to_bytes(BYTES_PER_LENGTH_PREFIX, 'little') + return serialized_length + serialized_bytes + elif isinstance(typ, list) and len(typ) == 2: + assert len(value) == typ[1] + return b''.join([serialize_value(element, typ[0]) for element in value]) + elif isinstance(typ, str) and len(typ) > 5 and typ[:5] == 'bytes': + assert len(value) == int(typ[5:]), (value, int(typ[5:])) + return coerce_to_bytes(value) + elif hasattr(typ, 'fields'): + serialized_bytes = b''.join([serialize_value(getattr(value, field), subtype) for field, subtype in typ.fields.items()]) + if is_constant_sized(typ): + return serialized_bytes + else: + assert len(serialized_bytes) < 2**(8 * BYTES_PER_LENGTH_PREFIX) + serialized_length = len(serialized_bytes).to_bytes(BYTES_PER_LENGTH_PREFIX, 'little') + return serialized_length + serialized_bytes + else: + print(value, typ) + raise Exception("Type not recognized") + +def chunkify(bytez): + bytez += b'\x00' * (-len(bytez) % BYTES_PER_CHUNK) + return [bytez[i:i+32] for i in range(0, len(bytez), 32)] + +def pack(values, subtype): + return chunkify(b''.join([serialize_value(value, subtype) for value in values])) + +def is_power_of_two(x): + return x > 0 and x & (x-1) == 0 + +def merkleize(chunks): + tree = chunks[::] + while not is_power_of_two(len(tree)): + tree.append(ZERO_CHUNK) + tree = [ZERO_CHUNK] * len(tree) + tree + for i in range(len(tree)//2-1, 0, -1): + tree[i] = hash(tree[i*2] + tree[i*2+1]) + return tree[1] + +def mix_in_length(root, length): + return hash(root + length.to_bytes(32, 'little')) + +def infer_type(value): + if hasattr(value.__class__, 'fields'): + return value.__class__ + elif isinstance(value, Vector): + return [infer_type(value[0]) if len(value) > 0 else 'uint64', len(value)] + elif isinstance(value, list): + return [infer_type(value[0])] if len(value) > 0 else ['uint64'] + elif isinstance(value, (bytes, str)): + return 'bytes' + elif isinstance(value, int): + return 'uint64' + else: + raise Exception("Failed to infer type") + +def hash_tree_root(value, typ=None): + if typ is None: + typ = infer_type(value) + if is_basic(typ): + return merkleize(pack([value], typ)) + elif isinstance(typ, list) and len(typ) == 1 and is_basic(typ[0]): + return mix_in_length(merkleize(pack(value, typ[0])), len(value)) + elif isinstance(typ, list) and len(typ) == 1 and not is_basic(typ[0]): + return mix_in_length(merkleize([hash_tree_root(element, typ[0]) for element in value]), len(value)) + elif isinstance(typ, list) and len(typ) == 2 and is_basic(typ[0]): + assert len(value) == typ[1] + return merkleize(pack(value, typ[0])) + elif typ == 'bytes': + return mix_in_length(merkleize(chunkify(coerce_to_bytes(value))), len(value)) + elif isinstance(typ, str) and typ[:5] == 'bytes' and len(typ) > 5: + assert len(value) == int(typ[5:]) + return merkleize(chunkify(coerce_to_bytes(value))) + elif isinstance(typ, list) and len(typ) == 2 and not is_basic(typ[0]): + return merkleize([hash_tree_root(element, typ[0]) for element in value]) + elif hasattr(typ, 'fields'): + return merkleize([hash_tree_root(getattr(value, field), subtype) for field, subtype in typ.fields.items()]) + else: + raise Exception("Type not recognized") + +def truncate(container): + field_keys = list(container.fields.keys()) + truncated_fields = { + key: container.fields[key] + for key in field_keys[:-1] + } + truncated_class = SSZType(truncated_fields) + kwargs = { + field: getattr(container, field) + for field in field_keys[:-1] + } + return truncated_class(**kwargs) + +def signed_root(container): + return hash_tree_root(truncate(container)) + +def serialize(ssz_object): + return getattr(ssz_object, 'serialize')() diff --git a/scripts/phase0/monkey_patches.py b/scripts/phase0/monkey_patches.py new file mode 100644 index 0000000000..8a35b8f27c --- /dev/null +++ b/scripts/phase0/monkey_patches.py @@ -0,0 +1,29 @@ +# Monkey patch validator shuffling cache +_get_shuffling = get_shuffling +shuffling_cache = {} +def get_shuffling(seed: Bytes32, + validators: List[Validator], + epoch: Epoch) -> List[List[ValidatorIndex]]: + + param_hash = (seed, hash_tree_root(validators, [Validator]), epoch) + + if param_hash in shuffling_cache: + # print("Cache hit, epoch={0}".format(epoch)) + return shuffling_cache[param_hash] + else: + # print("Cache miss, epoch={0}".format(epoch)) + ret = _get_shuffling(seed, validators, epoch) + shuffling_cache[param_hash] = ret + return ret + + +# Monkey patch hash cache +_hash = hash +hash_cache = {} +def hash(x): + if x in hash_cache: + return hash_cache[x] + else: + ret = _hash(x) + hash_cache[x] = ret + return ret diff --git a/scripts/phase0/state_transition.py b/scripts/phase0/state_transition.py new file mode 100644 index 0000000000..f78119cf25 --- /dev/null +++ b/scripts/phase0/state_transition.py @@ -0,0 +1,84 @@ + + +def process_transaction_type(state: BeaconState, + transactions: List[Any], + max_transactions: int, + tx_fn: Callable[[BeaconState, Any], None]) -> None: + assert len(transactions) <= max_transactions + for transaction in transactions: + tx_fn(state, transaction) + + +def process_transactions(state: BeaconState, block: BeaconBlock) -> None: + process_transaction_type( + state, + block.body.proposer_slashings, + MAX_PROPOSER_SLASHINGS, + process_proposer_slashing, + ) + process_transaction_type( + state, + block.body.attester_slashings, + MAX_ATTESTER_SLASHINGS, + process_attester_slashing, + ) + process_transaction_type( + state, + block.body.attestations, + MAX_ATTESTATIONS, + process_attestation, + ) + process_transaction_type( + state, + block.body.deposits, + MAX_DEPOSITS, + process_deposit, + ) + process_transaction_type( + state, + block.body.voluntary_exits, + MAX_VOLUNTARY_EXITS, + process_voluntary_exit, + ) + assert len(block.body.transfers) == len(set(block.body.transfers)) + process_transaction_type( + state, + block.body.transfers, + MAX_TRANSFERS, + process_transfer, + ) + + +def process_block(state: BeaconState, + block: BeaconBlock, + verify_state_root: bool=False) -> None: + process_block_header(state, block) + process_randao(state, block) + process_eth1_data(state, block) + process_transactions(state, block) + if verify_state_root: + verify_block_state_root(state, block) + + +def process_epoch_transition(state: BeaconState) -> None: + update_justification_and_finalization(state) + process_crosslinks(state) + maybe_reset_eth1_period(state) + apply_rewards(state) + process_ejections(state) + update_registry_and_shuffling_data(state) + process_slashings(state) + process_exit_queue(state) + finish_epoch_update(state) + + +def state_transition(state: BeaconState, + block: BeaconBlock, + verify_state_root: bool=False) -> BeaconState: + while state.slot < block.slot: + cache_state(state) + if (state.slot + 1) % SLOTS_PER_EPOCH == 0: + process_epoch_transition(state) + advance_slot(state) + if block.slot == state.slot: + process_block(state, block) diff --git a/tests/phase0/conftest.py b/tests/phase0/conftest.py new file mode 100644 index 0000000000..d3ebabaa24 --- /dev/null +++ b/tests/phase0/conftest.py @@ -0,0 +1,6 @@ +import pytest +from build.phase0 import spec + + +# @pytest.fixture(autouse=True) +# def build_clean(): \ No newline at end of file