From 4f3f68bc12d6a67ea7f50ec2e92157c6a8c8a7b9 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 11 May 2019 21:31:37 +0200 Subject: [PATCH 01/37] signatures in tests, most block-ops updated, still many to go --- .../test_process_attestation.py | 39 +++- .../test_process_attester_slashing.py | 18 +- .../test_process_block_header.py | 9 +- .../block_processing/test_process_transfer.py | 8 +- test_libs/pyspec/eth2spec/test/context.py | 2 +- .../test_process_crosslinks.py | 14 +- test_libs/pyspec/eth2spec/test/helpers.py | 204 +++++++++++------- .../pyspec/eth2spec/test/test_finality.py | 11 +- test_libs/pyspec/eth2spec/utils/bls_stub.py | 7 +- 9 files changed, 205 insertions(+), 107 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py index 248b04ef4e..98cb62ef7b 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py @@ -3,17 +3,18 @@ import eth2spec.phase0.spec as spec from eth2spec.phase0.state_transition import ( - state_transition, + state_transition_to, ) from eth2spec.phase0.spec import ( get_current_epoch, process_attestation ) from eth2spec.test.helpers import ( - build_empty_block_for_next_slot, get_valid_attestation, + apply_empty_block, next_epoch, next_slot, + make_attestation_signature, ) from eth2spec.test.context import spec_state_test, expect_assertion_error @@ -65,9 +66,8 @@ def test_success(state): @spec_state_test def test_success_previous_epoch(state): attestation = get_valid_attestation(state) - block = build_empty_block_for_next_slot(state) - block.slot = state.slot + spec.SLOTS_PER_EPOCH - state_transition(state, block) + next_epoch(state) + apply_empty_block(state) yield from run_attestation_processing(state, attestation) @@ -83,10 +83,9 @@ def test_before_inclusion_delay(state): @spec_state_test def test_after_epoch_slots(state): attestation = get_valid_attestation(state) - block = build_empty_block_for_next_slot(state) # increment past latest inclusion slot - block.slot = state.slot + spec.SLOTS_PER_EPOCH + 1 - state_transition(state, block) + state_transition_to(state, state.slot + spec.SLOTS_PER_EPOCH + 1) + apply_empty_block(state) yield from run_attestation_processing(state, attestation, False) @@ -105,6 +104,9 @@ def test_old_source_epoch(state): # Now go beyond that, it will be invalid attestation.data.source_epoch -= 1 + # Re do signature + make_attestation_signature(state, attestation) + yield from run_attestation_processing(state, attestation, False) @@ -115,6 +117,9 @@ def test_wrong_shard(state): attestation.data.shard += 1 + # Re do signature + make_attestation_signature(state, attestation) + yield from run_attestation_processing(state, attestation, False) @@ -125,6 +130,9 @@ def test_new_source_epoch(state): attestation.data.source_epoch += 1 + # Re do signature + make_attestation_signature(state, attestation) + yield from run_attestation_processing(state, attestation, False) @@ -135,6 +143,9 @@ def test_source_root_is_target_root(state): attestation.data.source_root = attestation.data.target_root + # Re do signature + make_attestation_signature(state, attestation) + yield from run_attestation_processing(state, attestation, False) @@ -159,6 +170,9 @@ def test_invalid_current_source_root(state): # Make attestation source root invalid: should be previous justified, not current one attestation.data.source_root = state.current_justified_root + # Re do signature + make_attestation_signature(state, attestation) + yield from run_attestation_processing(state, attestation, False) @@ -169,6 +183,9 @@ def test_bad_source_root(state): attestation.data.source_root = b'\x42' * 32 + # Re do signature + make_attestation_signature(state, attestation) + yield from run_attestation_processing(state, attestation, False) @@ -179,15 +196,21 @@ def test_non_zero_crosslink_data_root(state): attestation.data.crosslink_data_root = b'\x42' * 32 + # Re do signature + make_attestation_signature(state, attestation) + yield from run_attestation_processing(state, attestation, False) @spec_state_test def test_bad_previous_crosslink(state): next_epoch(state) + apply_empty_block(state) + attestation = get_valid_attestation(state) for _ in range(spec.MIN_ATTESTATION_INCLUSION_DELAY): next_slot(state) + apply_empty_block(state) state.current_crosslinks[attestation.data.shard].epoch += 10 diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py index 957b9a9f07..a0334c892d 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py @@ -3,14 +3,15 @@ get_beacon_proposer_index, process_attester_slashing, ) +from eth2spec.test.context import spec_state_test, expect_assertion_error from eth2spec.test.helpers import ( get_balance, get_valid_attester_slashing, next_epoch, + apply_empty_block, + make_indexed_attestation_signature ) -from eth2spec.test.context import spec_state_test, expect_assertion_error - def run_attester_slashing_processing(state, attester_slashing, valid=True): """ @@ -47,14 +48,14 @@ def run_attester_slashing_processing(state, attester_slashing, valid=True): # lost whistleblower reward assert ( - get_balance(state, slashed_index) < - pre_slashed_balance + get_balance(state, slashed_index) < + pre_slashed_balance ) # gained whistleblower reward assert ( - get_balance(state, proposer_index) > - pre_proposer_balance + get_balance(state, proposer_index) > + pre_proposer_balance ) yield 'post', state @@ -70,6 +71,8 @@ def test_success_double(state): @spec_state_test def test_success_surround(state): next_epoch(state) + apply_empty_block(state) + state.current_justified_epoch += 1 attester_slashing = get_valid_attester_slashing(state) @@ -77,6 +80,9 @@ def test_success_surround(state): attester_slashing.attestation_1.data.source_epoch = attester_slashing.attestation_2.data.source_epoch - 1 attester_slashing.attestation_1.data.target_epoch = attester_slashing.attestation_2.data.target_epoch + 1 + # correct the signature of attestation 1 + make_indexed_attestation_signature(state, attester_slashing.attestation_1) + yield from run_attester_slashing_processing(state, attester_slashing) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py index a176f79587..f2695ad098 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py @@ -6,13 +6,13 @@ advance_slot, process_block_header, ) +from eth2spec.test.context import spec_state_test, expect_assertion_error from eth2spec.test.helpers import ( build_empty_block_for_next_slot, next_slot, + make_block_signature ) -from eth2spec.test.context import spec_state_test, expect_assertion_error - def prepare_state_for_header_processing(state): cache_state(state) @@ -43,7 +43,7 @@ def run_block_header_processing(state, block, valid=True): @spec_state_test def test_success(state): - block = build_empty_block_for_next_slot(state) + block = build_empty_block_for_next_slot(state, signed=True) yield from run_block_header_processing(state, block) @@ -59,6 +59,7 @@ def test_invalid_slot(state): def test_invalid_previous_block_root(state): block = build_empty_block_for_next_slot(state) block.previous_block_root = b'\12' * 32 # invalid prev root + make_block_signature(state, block) yield from run_block_header_processing(state, block, valid=False) @@ -73,6 +74,6 @@ def test_proposer_slashed(state): # set proposer to slashed state.validator_registry[proposer_index].slashed = True - block = build_empty_block_for_next_slot(state) + block = build_empty_block_for_next_slot(state, signed=True) yield from run_block_header_processing(state, block, valid=False) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py index 10d2ccede0..64539d56e8 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py @@ -1,18 +1,17 @@ import eth2spec.phase0.spec as spec - from eth2spec.phase0.spec import ( get_active_validator_indices, get_beacon_proposer_index, get_current_epoch, process_transfer, ) +from eth2spec.test.context import spec_state_test, expect_assertion_error from eth2spec.test.helpers import ( get_valid_transfer, next_epoch, + apply_empty_block ) -from eth2spec.test.context import spec_state_test, expect_assertion_error - def run_transfer_processing(state, transfer, valid=True): """ @@ -58,6 +57,7 @@ def test_success_non_activated(state): @spec_state_test def test_success_withdrawable(state): next_epoch(state) + apply_empty_block(state) transfer = get_valid_transfer(state) @@ -97,7 +97,7 @@ def test_active_but_transfer_past_effective_balance(state): @spec_state_test def test_incorrect_slot(state): - transfer = get_valid_transfer(state, slot=state.slot+1) + transfer = get_valid_transfer(state, slot=state.slot + 1) # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index afabd4a57e..99c58ce3de 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -5,7 +5,7 @@ from .utils import spectest, with_args # Provides a genesis state as first argument to the function decorated with this -with_state = with_args(lambda: [create_genesis_state(spec.SLOTS_PER_EPOCH * 8, list())]) +with_state = with_args(lambda: [create_genesis_state(spec.SLOTS_PER_EPOCH * 8)]) # shorthand for decorating @with_state @spectest() diff --git a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py index 203978d29b..d4e6dc9381 100644 --- a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py @@ -1,15 +1,15 @@ from copy import deepcopy import eth2spec.phase0.spec as spec - -from eth2spec.phase0.state_transition import ( - state_transition, -) from eth2spec.phase0.spec import ( cache_state, get_crosslink_deltas, process_crosslinks, ) +from eth2spec.phase0.state_transition import ( + state_transition, +) +from eth2spec.test.context import spec_state_test from eth2spec.test.helpers import ( add_attestation_to_state, build_empty_block_for_next_slot, @@ -18,8 +18,8 @@ get_valid_attestation, next_epoch, next_slot, + apply_empty_block ) -from eth2spec.test.context import spec_state_test def run_process_crosslinks(state, valid=True): @@ -115,6 +115,8 @@ def test_double_late_crosslink(state): if attestation_2.data.shard == attestation_1.data.shard: break next_slot(state) + apply_empty_block(state) + fill_aggregate_attestation(state, attestation_2) # add attestation_2 in the next epoch after attestation_1 has @@ -126,7 +128,7 @@ def test_double_late_crosslink(state): assert len(state.current_epoch_attestations) == 0 crosslink_deltas = get_crosslink_deltas(state) - + yield from run_process_crosslinks(state) shard = attestation_2.data.shard diff --git a/test_libs/pyspec/eth2spec/test/helpers.py b/test_libs/pyspec/eth2spec/test/helpers.py index 8eb6d88913..f6cedd4793 100644 --- a/test_libs/pyspec/eth2spec/test/helpers.py +++ b/test_libs/pyspec/eth2spec/test/helpers.py @@ -2,20 +2,20 @@ from py_ecc import bls -from eth2spec.phase0.state_transition import ( - state_transition, -) +from typing import List + import eth2spec.phase0.spec as spec -from eth2spec.utils.minimal_ssz import signing_root from eth2spec.phase0.spec import ( # constants ZERO_HASH, # SSZ Attestation, + IndexedAttestation, AttestationData, AttestationDataAndCustodyBit, AttesterSlashing, BeaconBlock, + BeaconState, BeaconBlockHeader, Deposit, DepositData, @@ -33,7 +33,6 @@ get_current_epoch, get_domain, get_epoch_start_slot, - get_genesis_beacon_state, get_previous_epoch, get_shard_delta, hash_tree_root, @@ -41,12 +40,15 @@ verify_merkle_branch, hash, ) +from eth2spec.phase0.state_transition import ( + state_transition, state_transition_to +) from eth2spec.utils.merkle_minimal import ( calc_merkle_tree_from_leaves, get_merkle_proof, get_merkle_root, ) - +from eth2spec.utils.minimal_ssz import signing_root privkeys = [i + 1 for i in range(1024)] pubkeys = [bls.privtopub(privkey) for privkey in privkeys] @@ -64,72 +66,109 @@ def set_bitfield_bit(bitfield, i): byte_index = i // 8 bit_index = i % 8 return ( - bitfield[:byte_index] + - bytes([bitfield[byte_index] | (1 << bit_index)]) + - bitfield[byte_index+1:] + bitfield[:byte_index] + + bytes([bitfield[byte_index] | (1 << bit_index)]) + + bitfield[byte_index + 1:] ) -def create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves=None): - if not deposit_data_leaves: - deposit_data_leaves = [] - signature = b'\x33' * 96 - - deposit_data_list = [] - for i in range(num_validators): - pubkey = pubkeys[i] - deposit_data = DepositData( - pubkey=pubkey, - # insecurely use pubkey as withdrawal key as well - withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:], - amount=spec.MAX_EFFECTIVE_BALANCE, - signature=signature, - ) - item = deposit_data.hash_tree_root() - deposit_data_leaves.append(item) - tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves)) - root = get_merkle_root((tuple(deposit_data_leaves))) - proof = list(get_merkle_proof(tree, item_index=i)) - assert verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, i, root) - deposit_data_list.append(deposit_data) - - genesis_validator_deposits = [] - for i in range(num_validators): - genesis_validator_deposits.append(Deposit( - proof=list(get_merkle_proof(tree, item_index=i)), - index=i, - data=deposit_data_list[i] - )) - return genesis_validator_deposits, root +def build_mock_validator(i: int, balance: int): + pubkey = pubkeys[i] + # insecurely use pubkey as withdrawal key as well + withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:] + return spec.Validator( + pubkey=pubkeys[i], + withdrawal_credentials=withdrawal_credentials, + activation_eligibility_epoch=spec.FAR_FUTURE_EPOCH, + activation_epoch=spec.FAR_FUTURE_EPOCH, + exit_epoch=spec.FAR_FUTURE_EPOCH, + withdrawable_epoch=spec.FAR_FUTURE_EPOCH, + effective_balance=min(balance - balance % spec.EFFECTIVE_BALANCE_INCREMENT, spec.MAX_EFFECTIVE_BALANCE) + ) -def create_genesis_state(num_validators, deposit_data_leaves=None): - initial_deposits, deposit_root = create_mock_genesis_validator_deposits( - num_validators, - deposit_data_leaves, - ) - return get_genesis_beacon_state( - initial_deposits, +def create_genesis_state(num_validators): + deposit_root = b'\x42' * 32 + + state = spec.BeaconState( genesis_time=0, - genesis_eth1_data=Eth1Data( + deposit_index=num_validators, + latest_eth1_data=Eth1Data( deposit_root=deposit_root, - deposit_count=len(initial_deposits), + deposit_count=num_validators, block_hash=spec.ZERO_HASH, - ), + )) + + # We "hack" in the initial validators, + # as it is much faster than creating and processing genesis deposits for every single test case. + state.balances = [spec.MAX_EFFECTIVE_BALANCE] * num_validators + state.validator_registry = [build_mock_validator(i, state.balances[i]) for i in range(num_validators)] + + # Process genesis activations + for validator in state.validator_registry: + if validator.effective_balance >= spec.MAX_EFFECTIVE_BALANCE: + validator.activation_eligibility_epoch = spec.GENESIS_EPOCH + validator.activation_epoch = spec.GENESIS_EPOCH + + genesis_active_index_root = hash_tree_root(get_active_validator_indices(state, spec.GENESIS_EPOCH)) + for index in range(spec.LATEST_ACTIVE_INDEX_ROOTS_LENGTH): + state.latest_active_index_roots[index] = genesis_active_index_root + + return state + + +def make_block_signature(state, block): + assert block.slot == state.slot or block.slot == state.slot + 1 + if block.slot == state.slot: + proposer_index = spec.get_beacon_proposer_index(state) + else: + # use stub state to get proposer index of next slot + stub_state = deepcopy(state) + next_slot(stub_state) + proposer_index = spec.get_beacon_proposer_index(stub_state) + + privkey = privkeys[proposer_index] + + block.body.randao_reveal = bls.sign( + privkey=privkey, + message_hash=hash_tree_root(slot_to_epoch(block.slot)), + domain=get_domain( + state, + message_epoch=slot_to_epoch(block.slot), + domain_type=spec.DOMAIN_RANDAO, + ) ) + block.signature = bls.sign( + message_hash=signing_root(block), + privkey=privkey, + domain=get_domain( + state, + spec.DOMAIN_BEACON_PROPOSER, + slot_to_epoch(block.slot))) + return block -def build_empty_block_for_next_slot(state): +def build_empty_block(state, slot=None, signed=False): + if slot is None: + slot = state.slot empty_block = BeaconBlock() - empty_block.slot = state.slot + 1 + empty_block.slot = slot empty_block.body.eth1_data.deposit_count = state.deposit_index previous_block_header = deepcopy(state.latest_block_header) if previous_block_header.state_root == spec.ZERO_HASH: previous_block_header.state_root = state.hash_tree_root() empty_block.previous_block_root = signing_root(previous_block_header) + + if signed: + make_block_signature(state, empty_block) + return empty_block +def build_empty_block_for_next_slot(state, signed=False): + return build_empty_block(state, state.slot + 1, signed=signed) + + def build_deposit_data(state, pubkey, privkey, amount): deposit_data = DepositData( pubkey=pubkey, @@ -172,7 +211,8 @@ def build_attestation_data(state, slot, shard): justified_epoch = state.current_justified_epoch justified_block_root = state.current_justified_root - crosslinks = state.current_crosslinks if slot_to_epoch(slot) == get_current_epoch(state) else state.previous_crosslinks + crosslinks = state.current_crosslinks if slot_to_epoch(slot) == get_current_epoch( + state) else state.previous_crosslinks return AttestationData( shard=shard, beacon_block_root=block_root, @@ -270,6 +310,8 @@ def get_valid_attester_slashing(state): attestation_2 = deepcopy(attestation_1) attestation_2.data.target_root = b'\x01' * 32 + make_attestation_signature(state, attestation_2) + return AttesterSlashing( attestation_1=convert_to_indexed(state, attestation_1), attestation_2=convert_to_indexed(state, attestation_2), @@ -299,26 +341,38 @@ def get_valid_attestation(state, slot=None): data=attestation_data, custody_bitfield=custody_bitfield, ) - participants = get_attesting_indices( - state, - attestation.data, - attestation.aggregation_bitfield, - ) - assert len(participants) == 2 + make_attestation_signature(state, attestation) + return attestation + +def make_aggregate_attestation_signature(state: BeaconState, data: AttestationData, participants: List[int]): signatures = [] for validator_index in participants: privkey = privkeys[validator_index] signatures.append( get_attestation_signature( state, - attestation.data, + data, privkey ) ) - attestation.aggregation_signature = bls.aggregate_signatures(signatures) - return attestation + return bls.aggregate_signatures(signatures) + + +def make_indexed_attestation_signature(state, indexed_attestation: IndexedAttestation): + participants = indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices + indexed_attestation.signature = make_aggregate_attestation_signature(state, indexed_attestation.data, participants) + + +def make_attestation_signature(state, attestation: Attestation): + participants = get_attesting_indices( + state, + attestation.data, + attestation.aggregation_bitfield, + ) + + attestation.signature = make_aggregate_attestation_signature(state, attestation.data, participants) def get_valid_transfer(state, slot=None, sender_index=None, amount=None, fee=None): @@ -357,7 +411,7 @@ def get_valid_transfer(state, slot=None, sender_index=None, amount=None, fee=Non # ensure withdrawal_credentials reproducable state.validator_registry[transfer.sender].withdrawal_credentials = ( - spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(transfer.pubkey)[1:] + spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(transfer.pubkey)[1:] ) return transfer @@ -390,26 +444,32 @@ def add_attestation_to_state(state, attestation, slot): block = build_empty_block_for_next_slot(state) block.slot = slot block.body.attestations.append(attestation) + state_transition_to(state, block.slot) + make_block_signature(state, block) state_transition(state, block) def next_slot(state): """ - Transition to the next slot via an empty block. - Return the empty block that triggered the transition. + Transition to the next slot. """ - block = build_empty_block_for_next_slot(state) - state_transition(state, block) - return block + state_transition_to(state, state.slot + 1) def next_epoch(state): """ - Transition to the start slot of the next epoch via an empty block. - Return the empty block that triggered the transition. + Transition to the start slot of the next epoch """ - block = build_empty_block_for_next_slot(state) - block.slot += spec.SLOTS_PER_EPOCH - (state.slot % spec.SLOTS_PER_EPOCH) + slot = state.slot + spec.SLOTS_PER_EPOCH - (state.slot % spec.SLOTS_PER_EPOCH) + state_transition_to(state, slot) + + +def apply_empty_block(state): + """ + Transition via an empty block (on current slot, assuming no block has been applied yet). + :return: the empty block that triggered the transition. + """ + block = build_empty_block(state) state_transition(state, block) return block diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index 16bf24a4e6..75191e59aa 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -1,10 +1,10 @@ from copy import deepcopy import eth2spec.phase0.spec as spec - from eth2spec.phase0.state_transition import ( state_transition, ) +from .context import spec_state_test from .helpers import ( build_empty_block_for_next_slot, fill_aggregate_attestation, @@ -12,10 +12,9 @@ get_epoch_start_slot, get_valid_attestation, next_epoch, + apply_empty_block ) -from .context import spec_state_test - def check_finality(state, prev_state, @@ -101,7 +100,9 @@ def test_finality_rule_4(state): def test_finality_rule_1(state): # get past first two epochs that finality does not run on next_epoch(state) + apply_empty_block(state) next_epoch(state) + apply_empty_block(state) yield 'pre', state @@ -128,7 +129,9 @@ def test_finality_rule_1(state): def test_finality_rule_2(state): # get past first two epochs that finality does not run on next_epoch(state) + apply_empty_block(state) next_epoch(state) + apply_empty_block(state) yield 'pre', state @@ -161,7 +164,9 @@ def test_finality_rule_3(state): """ # get past first two epochs that finality does not run on next_epoch(state) + apply_empty_block(state) next_epoch(state) + apply_empty_block(state) yield 'pre', state diff --git a/test_libs/pyspec/eth2spec/utils/bls_stub.py b/test_libs/pyspec/eth2spec/utils/bls_stub.py index 108c4ef710..0a52d2f659 100644 --- a/test_libs/pyspec/eth2spec/utils/bls_stub.py +++ b/test_libs/pyspec/eth2spec/utils/bls_stub.py @@ -1,12 +1,13 @@ +from py_ecc import bls def bls_verify(pubkey, message_hash, signature, domain): - return True + return bls.verify(message_hash=message_hash, pubkey=pubkey, signature=signature, domain=domain) def bls_verify_multiple(pubkeys, message_hashes, signature, domain): - return True + return bls.verify_multiple(pubkeys, message_hashes, signature, domain) def bls_aggregate_pubkeys(pubkeys): - return b'\x42' * 96 + return bls.aggregate_pubkeys(pubkeys) From 904e2e9c0c348043a5ad028c3b151df382a14956 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 13 May 2019 23:15:02 +0200 Subject: [PATCH 02/37] BLS on/off deco --- scripts/phase0/build_spec.py | 2 +- test_libs/pyspec/eth2spec/test/context.py | 27 ++++++++++++++++++++- test_libs/pyspec/eth2spec/test/utils.py | 4 +++ test_libs/pyspec/eth2spec/utils/bls.py | 25 +++++++++++++++++++ test_libs/pyspec/eth2spec/utils/bls_stub.py | 12 --------- 5 files changed, 56 insertions(+), 14 deletions(-) create mode 100644 test_libs/pyspec/eth2spec/utils/bls.py delete mode 100644 test_libs/pyspec/eth2spec/utils/bls_stub.py diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index da5845951d..26b0e5a8a6 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -13,7 +13,7 @@ def build_phase0_spec(sourcefile, outfile): Tuple, ) from eth2spec.utils.minimal_ssz import * -from eth2spec.utils.bls_stub import * +from eth2spec.utils.bls import * """) for i in (1, 2, 3, 4, 8, 32, 48, 96): diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index afabd4a57e..3ef62eae50 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -1,4 +1,5 @@ from eth2spec.phase0 import spec +from eth2spec.utils import bls from .helpers import create_genesis_state @@ -10,7 +11,7 @@ # shorthand for decorating @with_state @spectest() def spec_state_test(fn): - return with_state(spectest()(fn)) + return with_state(bls_switch(spectest()(fn))) def expect_assertion_error(fn): @@ -25,3 +26,27 @@ def expect_assertion_error(fn): pass if bad: raise AssertionError('expected an assertion error, but got none.') + + +def always_bls(fn): + """ + Decorator to apply on ``bls_switch`` decorator to force BLS activation. Useful to mark tests as BLS-dependent. + """ + def entry(*args, **kw): + # override bls setting + kw['bls_active'] = True + fn(*args, **kw) + return entry + + +def bls_switch(fn): + """ + Decorator to make a function execute with BLS ON, or BLS off. + Based on an optional bool argument ``bls_active``, passed to the function at runtime. + """ + def entry(*args, **kw): + old_state = bls.bls_active + bls.bls_active = kw.pop('bls_active', False) + fn(*args, **kw) + bls.bls_active = old_state + return entry diff --git a/test_libs/pyspec/eth2spec/test/utils.py b/test_libs/pyspec/eth2spec/test/utils.py index b19d4df595..1c157bcee1 100644 --- a/test_libs/pyspec/eth2spec/test/utils.py +++ b/test_libs/pyspec/eth2spec/test/utils.py @@ -1,4 +1,5 @@ from eth2spec.debug.encode import encode +from eth2spec.utils import bls def spectest(description: str = None): @@ -18,6 +19,9 @@ def entry(*args, **kw): else: # description can be explicit out['description'] = description + # If BLS is not active, mark the test as BLS-ignorant. + if not bls.bls_active: + out['stub_bls'] = True # put all generated data into a dict. for data in fn(*args, **kw): # If there is a type argument, encode it as that type. diff --git a/test_libs/pyspec/eth2spec/utils/bls.py b/test_libs/pyspec/eth2spec/utils/bls.py new file mode 100644 index 0000000000..daeb361a9d --- /dev/null +++ b/test_libs/pyspec/eth2spec/utils/bls.py @@ -0,0 +1,25 @@ +from py_ecc import bls + +# Flag to make BLS active or not. Used for testing, do not ignore BLS in production unless you know what you are doing. +bls_active = True + + +def bls_verify(pubkey, message_hash, signature, domain): + if bls_active: + return bls.verify(message_hash=message_hash, pubkey=pubkey, signature=signature, domain=domain) + else: + return True + + +def bls_verify_multiple(pubkeys, message_hashes, signature, domain): + if bls_active: + return bls.verify_multiple(pubkeys, message_hashes, signature, domain) + else: + return True + + +def bls_aggregate_pubkeys(pubkeys): + if bls_active: + return bls.aggregate_pubkeys(pubkeys) + else: + return b'\x42' * 96 diff --git a/test_libs/pyspec/eth2spec/utils/bls_stub.py b/test_libs/pyspec/eth2spec/utils/bls_stub.py deleted file mode 100644 index 108c4ef710..0000000000 --- a/test_libs/pyspec/eth2spec/utils/bls_stub.py +++ /dev/null @@ -1,12 +0,0 @@ - - -def bls_verify(pubkey, message_hash, signature, domain): - return True - - -def bls_verify_multiple(pubkeys, message_hashes, signature, domain): - return True - - -def bls_aggregate_pubkeys(pubkeys): - return b'\x42' * 96 From 51c82c5b81cb3c7e72023e3edada1d01c3f59b6e Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 15 May 2019 18:36:32 +0200 Subject: [PATCH 03/37] clean up helpers, make helpers pkg --- test_libs/pyspec/eth2spec/test/helpers.py | 505 ------------------ .../pyspec/eth2spec/test/helpers/__init__.py | 0 .../eth2spec/test/helpers/attestations.py | 145 +++++ .../test/helpers/attester_slashings.py | 17 + .../pyspec/eth2spec/test/helpers/bitfields.py | 11 + .../pyspec/eth2spec/test/helpers/block.py | 62 +++ .../pyspec/eth2spec/test/helpers/deposits.py | 74 +++ .../pyspec/eth2spec/test/helpers/genesis.py | 51 ++ .../pyspec/eth2spec/test/helpers/keys.py | 5 + .../test/helpers/proposer_slashings.py | 48 ++ .../pyspec/eth2spec/test/helpers/state.py | 42 ++ .../pyspec/eth2spec/test/helpers/transfers.py | 50 ++ .../eth2spec/test/helpers/voluntary_exits.py | 24 + 13 files changed, 529 insertions(+), 505 deletions(-) delete mode 100644 test_libs/pyspec/eth2spec/test/helpers.py create mode 100644 test_libs/pyspec/eth2spec/test/helpers/__init__.py create mode 100644 test_libs/pyspec/eth2spec/test/helpers/attestations.py create mode 100644 test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py create mode 100644 test_libs/pyspec/eth2spec/test/helpers/bitfields.py create mode 100644 test_libs/pyspec/eth2spec/test/helpers/block.py create mode 100644 test_libs/pyspec/eth2spec/test/helpers/deposits.py create mode 100644 test_libs/pyspec/eth2spec/test/helpers/genesis.py create mode 100644 test_libs/pyspec/eth2spec/test/helpers/keys.py create mode 100644 test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py create mode 100644 test_libs/pyspec/eth2spec/test/helpers/state.py create mode 100644 test_libs/pyspec/eth2spec/test/helpers/transfers.py create mode 100644 test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py diff --git a/test_libs/pyspec/eth2spec/test/helpers.py b/test_libs/pyspec/eth2spec/test/helpers.py deleted file mode 100644 index f6cedd4793..0000000000 --- a/test_libs/pyspec/eth2spec/test/helpers.py +++ /dev/null @@ -1,505 +0,0 @@ -from copy import deepcopy - -from py_ecc import bls - -from typing import List - -import eth2spec.phase0.spec as spec -from eth2spec.phase0.spec import ( - # constants - ZERO_HASH, - # SSZ - Attestation, - IndexedAttestation, - AttestationData, - AttestationDataAndCustodyBit, - AttesterSlashing, - BeaconBlock, - BeaconState, - BeaconBlockHeader, - Deposit, - DepositData, - Eth1Data, - ProposerSlashing, - Transfer, - VoluntaryExit, - # functions - convert_to_indexed, - get_active_validator_indices, - get_attesting_indices, - get_block_root, - get_block_root_at_slot, - get_crosslink_committee, - get_current_epoch, - get_domain, - get_epoch_start_slot, - get_previous_epoch, - get_shard_delta, - hash_tree_root, - slot_to_epoch, - verify_merkle_branch, - hash, -) -from eth2spec.phase0.state_transition import ( - state_transition, state_transition_to -) -from eth2spec.utils.merkle_minimal import ( - calc_merkle_tree_from_leaves, - get_merkle_proof, - get_merkle_root, -) -from eth2spec.utils.minimal_ssz import signing_root - -privkeys = [i + 1 for i in range(1024)] -pubkeys = [bls.privtopub(privkey) for privkey in privkeys] -pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys, pubkeys)} - - -def get_balance(state, index): - return state.balances[index] - - -def set_bitfield_bit(bitfield, i): - """ - Set the bit in ``bitfield`` at position ``i`` to ``1``. - """ - byte_index = i // 8 - bit_index = i % 8 - return ( - bitfield[:byte_index] + - bytes([bitfield[byte_index] | (1 << bit_index)]) + - bitfield[byte_index + 1:] - ) - - -def build_mock_validator(i: int, balance: int): - pubkey = pubkeys[i] - # insecurely use pubkey as withdrawal key as well - withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:] - return spec.Validator( - pubkey=pubkeys[i], - withdrawal_credentials=withdrawal_credentials, - activation_eligibility_epoch=spec.FAR_FUTURE_EPOCH, - activation_epoch=spec.FAR_FUTURE_EPOCH, - exit_epoch=spec.FAR_FUTURE_EPOCH, - withdrawable_epoch=spec.FAR_FUTURE_EPOCH, - effective_balance=min(balance - balance % spec.EFFECTIVE_BALANCE_INCREMENT, spec.MAX_EFFECTIVE_BALANCE) - ) - - -def create_genesis_state(num_validators): - deposit_root = b'\x42' * 32 - - state = spec.BeaconState( - genesis_time=0, - deposit_index=num_validators, - latest_eth1_data=Eth1Data( - deposit_root=deposit_root, - deposit_count=num_validators, - block_hash=spec.ZERO_HASH, - )) - - # We "hack" in the initial validators, - # as it is much faster than creating and processing genesis deposits for every single test case. - state.balances = [spec.MAX_EFFECTIVE_BALANCE] * num_validators - state.validator_registry = [build_mock_validator(i, state.balances[i]) for i in range(num_validators)] - - # Process genesis activations - for validator in state.validator_registry: - if validator.effective_balance >= spec.MAX_EFFECTIVE_BALANCE: - validator.activation_eligibility_epoch = spec.GENESIS_EPOCH - validator.activation_epoch = spec.GENESIS_EPOCH - - genesis_active_index_root = hash_tree_root(get_active_validator_indices(state, spec.GENESIS_EPOCH)) - for index in range(spec.LATEST_ACTIVE_INDEX_ROOTS_LENGTH): - state.latest_active_index_roots[index] = genesis_active_index_root - - return state - - -def make_block_signature(state, block): - assert block.slot == state.slot or block.slot == state.slot + 1 - if block.slot == state.slot: - proposer_index = spec.get_beacon_proposer_index(state) - else: - # use stub state to get proposer index of next slot - stub_state = deepcopy(state) - next_slot(stub_state) - proposer_index = spec.get_beacon_proposer_index(stub_state) - - privkey = privkeys[proposer_index] - - block.body.randao_reveal = bls.sign( - privkey=privkey, - message_hash=hash_tree_root(slot_to_epoch(block.slot)), - domain=get_domain( - state, - message_epoch=slot_to_epoch(block.slot), - domain_type=spec.DOMAIN_RANDAO, - ) - ) - block.signature = bls.sign( - message_hash=signing_root(block), - privkey=privkey, - domain=get_domain( - state, - spec.DOMAIN_BEACON_PROPOSER, - slot_to_epoch(block.slot))) - return block - - -def build_empty_block(state, slot=None, signed=False): - if slot is None: - slot = state.slot - empty_block = BeaconBlock() - empty_block.slot = slot - empty_block.body.eth1_data.deposit_count = state.deposit_index - previous_block_header = deepcopy(state.latest_block_header) - if previous_block_header.state_root == spec.ZERO_HASH: - previous_block_header.state_root = state.hash_tree_root() - empty_block.previous_block_root = signing_root(previous_block_header) - - if signed: - make_block_signature(state, empty_block) - - return empty_block - - -def build_empty_block_for_next_slot(state, signed=False): - return build_empty_block(state, state.slot + 1, signed=signed) - - -def build_deposit_data(state, pubkey, privkey, amount): - deposit_data = DepositData( - pubkey=pubkey, - # insecurely use pubkey as withdrawal key as well - withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:], - amount=amount, - ) - signature = bls.sign( - message_hash=signing_root(deposit_data), - privkey=privkey, - domain=get_domain( - state, - spec.DOMAIN_DEPOSIT, - ) - ) - deposit_data.signature = signature - return deposit_data - - -def build_attestation_data(state, slot, shard): - assert state.slot >= slot - - if slot == state.slot: - block_root = build_empty_block_for_next_slot(state).previous_block_root - else: - block_root = get_block_root_at_slot(state, slot) - - current_epoch_start_slot = get_epoch_start_slot(get_current_epoch(state)) - if slot < current_epoch_start_slot: - epoch_boundary_root = get_block_root(state, get_previous_epoch(state)) - elif slot == current_epoch_start_slot: - epoch_boundary_root = block_root - else: - epoch_boundary_root = get_block_root(state, get_current_epoch(state)) - - if slot < current_epoch_start_slot: - justified_epoch = state.previous_justified_epoch - justified_block_root = state.previous_justified_root - else: - justified_epoch = state.current_justified_epoch - justified_block_root = state.current_justified_root - - crosslinks = state.current_crosslinks if slot_to_epoch(slot) == get_current_epoch( - state) else state.previous_crosslinks - return AttestationData( - shard=shard, - beacon_block_root=block_root, - source_epoch=justified_epoch, - source_root=justified_block_root, - target_epoch=slot_to_epoch(slot), - target_root=epoch_boundary_root, - crosslink_data_root=spec.ZERO_HASH, - previous_crosslink_root=hash_tree_root(crosslinks[shard]), - ) - - -def build_voluntary_exit(state, epoch, validator_index, privkey): - voluntary_exit = VoluntaryExit( - epoch=epoch, - validator_index=validator_index, - ) - voluntary_exit.signature = bls.sign( - message_hash=signing_root(voluntary_exit), - privkey=privkey, - domain=get_domain( - state=state, - domain_type=spec.DOMAIN_VOLUNTARY_EXIT, - message_epoch=epoch, - ) - ) - - return voluntary_exit - - -def build_deposit(state, - deposit_data_leaves, - pubkey, - privkey, - amount): - deposit_data = build_deposit_data(state, pubkey, privkey, amount) - - item = deposit_data.hash_tree_root() - index = len(deposit_data_leaves) - deposit_data_leaves.append(item) - tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves)) - root = get_merkle_root((tuple(deposit_data_leaves))) - proof = list(get_merkle_proof(tree, item_index=index)) - assert verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, index, root) - - deposit = Deposit( - proof=list(proof), - index=index, - data=deposit_data, - ) - - return deposit, root, deposit_data_leaves - - -def get_valid_proposer_slashing(state): - current_epoch = get_current_epoch(state) - validator_index = get_active_validator_indices(state, current_epoch)[-1] - privkey = pubkey_to_privkey[state.validator_registry[validator_index].pubkey] - slot = state.slot - - header_1 = BeaconBlockHeader( - slot=slot, - previous_block_root=ZERO_HASH, - state_root=ZERO_HASH, - block_body_root=ZERO_HASH, - ) - header_2 = deepcopy(header_1) - header_2.previous_block_root = b'\x02' * 32 - header_2.slot = slot + 1 - - domain = get_domain( - state=state, - domain_type=spec.DOMAIN_BEACON_PROPOSER, - ) - header_1.signature = bls.sign( - message_hash=signing_root(header_1), - privkey=privkey, - domain=domain, - ) - header_2.signature = bls.sign( - message_hash=signing_root(header_2), - privkey=privkey, - domain=domain, - ) - - return ProposerSlashing( - proposer_index=validator_index, - header_1=header_1, - header_2=header_2, - ) - - -def get_valid_attester_slashing(state): - attestation_1 = get_valid_attestation(state) - attestation_2 = deepcopy(attestation_1) - attestation_2.data.target_root = b'\x01' * 32 - - make_attestation_signature(state, attestation_2) - - return AttesterSlashing( - attestation_1=convert_to_indexed(state, attestation_1), - attestation_2=convert_to_indexed(state, attestation_2), - ) - - -def get_valid_attestation(state, slot=None): - if slot is None: - slot = state.slot - - if slot_to_epoch(slot) == get_current_epoch(state): - shard = (state.latest_start_shard + slot) % spec.SLOTS_PER_EPOCH - else: - previous_shard_delta = get_shard_delta(state, get_previous_epoch(state)) - shard = (state.latest_start_shard - previous_shard_delta + slot) % spec.SHARD_COUNT - - attestation_data = build_attestation_data(state, slot, shard) - - crosslink_committee = get_crosslink_committee(state, attestation_data.target_epoch, attestation_data.shard) - - committee_size = len(crosslink_committee) - bitfield_length = (committee_size + 7) // 8 - aggregation_bitfield = b'\xC0' + b'\x00' * (bitfield_length - 1) - custody_bitfield = b'\x00' * bitfield_length - attestation = Attestation( - aggregation_bitfield=aggregation_bitfield, - data=attestation_data, - custody_bitfield=custody_bitfield, - ) - make_attestation_signature(state, attestation) - return attestation - - -def make_aggregate_attestation_signature(state: BeaconState, data: AttestationData, participants: List[int]): - signatures = [] - for validator_index in participants: - privkey = privkeys[validator_index] - signatures.append( - get_attestation_signature( - state, - data, - privkey - ) - ) - - return bls.aggregate_signatures(signatures) - - -def make_indexed_attestation_signature(state, indexed_attestation: IndexedAttestation): - participants = indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices - indexed_attestation.signature = make_aggregate_attestation_signature(state, indexed_attestation.data, participants) - - -def make_attestation_signature(state, attestation: Attestation): - participants = get_attesting_indices( - state, - attestation.data, - attestation.aggregation_bitfield, - ) - - attestation.signature = make_aggregate_attestation_signature(state, attestation.data, participants) - - -def get_valid_transfer(state, slot=None, sender_index=None, amount=None, fee=None): - if slot is None: - slot = state.slot - current_epoch = get_current_epoch(state) - if sender_index is None: - sender_index = get_active_validator_indices(state, current_epoch)[-1] - recipient_index = get_active_validator_indices(state, current_epoch)[0] - transfer_pubkey = pubkeys[-1] - transfer_privkey = privkeys[-1] - - if fee is None: - fee = get_balance(state, sender_index) // 32 - if amount is None: - amount = get_balance(state, sender_index) - fee - - transfer = Transfer( - sender=sender_index, - recipient=recipient_index, - amount=amount, - fee=fee, - slot=slot, - pubkey=transfer_pubkey, - signature=ZERO_HASH, - ) - transfer.signature = bls.sign( - message_hash=signing_root(transfer), - privkey=transfer_privkey, - domain=get_domain( - state=state, - domain_type=spec.DOMAIN_TRANSFER, - message_epoch=get_current_epoch(state), - ) - ) - - # ensure withdrawal_credentials reproducable - state.validator_registry[transfer.sender].withdrawal_credentials = ( - spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(transfer.pubkey)[1:] - ) - - return transfer - - -def get_attestation_signature(state, attestation_data, privkey, custody_bit=0b0): - message_hash = AttestationDataAndCustodyBit( - data=attestation_data, - custody_bit=custody_bit, - ).hash_tree_root() - - return bls.sign( - message_hash=message_hash, - privkey=privkey, - domain=get_domain( - state=state, - domain_type=spec.DOMAIN_ATTESTATION, - message_epoch=attestation_data.target_epoch, - ) - ) - - -def fill_aggregate_attestation(state, attestation): - crosslink_committee = get_crosslink_committee(state, attestation.data.target_epoch, attestation.data.shard) - for i in range(len(crosslink_committee)): - attestation.aggregation_bitfield = set_bitfield_bit(attestation.aggregation_bitfield, i) - - -def add_attestation_to_state(state, attestation, slot): - block = build_empty_block_for_next_slot(state) - block.slot = slot - block.body.attestations.append(attestation) - state_transition_to(state, block.slot) - make_block_signature(state, block) - state_transition(state, block) - - -def next_slot(state): - """ - Transition to the next slot. - """ - state_transition_to(state, state.slot + 1) - - -def next_epoch(state): - """ - Transition to the start slot of the next epoch - """ - slot = state.slot + spec.SLOTS_PER_EPOCH - (state.slot % spec.SLOTS_PER_EPOCH) - state_transition_to(state, slot) - - -def apply_empty_block(state): - """ - Transition via an empty block (on current slot, assuming no block has been applied yet). - :return: the empty block that triggered the transition. - """ - block = build_empty_block(state) - state_transition(state, block) - return block - - -def get_state_root(state, slot) -> bytes: - """ - Return the state root at a recent ``slot``. - """ - assert slot < state.slot <= slot + spec.SLOTS_PER_HISTORICAL_ROOT - return state.latest_state_roots[slot % spec.SLOTS_PER_HISTORICAL_ROOT] - - -def prepare_state_and_deposit(state, validator_index, amount): - """ - Prepare the state for the deposit, and create a deposit for the given validator, depositing the given amount. - """ - pre_validator_count = len(state.validator_registry) - # fill previous deposits with zero-hash - deposit_data_leaves = [ZERO_HASH] * pre_validator_count - - pubkey = pubkeys[validator_index] - privkey = privkeys[validator_index] - deposit, root, deposit_data_leaves = build_deposit( - state, - deposit_data_leaves, - pubkey, - privkey, - amount, - ) - - state.latest_eth1_data.deposit_root = root - state.latest_eth1_data.deposit_count = len(deposit_data_leaves) - return deposit diff --git a/test_libs/pyspec/eth2spec/test/helpers/__init__.py b/test_libs/pyspec/eth2spec/test/helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py new file mode 100644 index 0000000000..d7fab4c177 --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -0,0 +1,145 @@ +from typing import List + +# Access constants from spec pkg reference. +import eth2spec.phase0.spec as spec + +from eth2spec.phase0.spec import ( + Attestation, + AttestationData, + AttestationDataAndCustodyBit, + get_epoch_start_slot, get_block_root, get_current_epoch, get_previous_epoch, slot_to_epoch, get_shard_delta, + get_crosslink_committee, get_domain, IndexedAttestation, get_attesting_indices, BeaconState, get_block_root_at_slot) + +from eth2spec.phase0.state_transition import ( + state_transition, state_transition_to +) +from eth2spec.test.helpers.bitfields import set_bitfield_bit +from eth2spec.test.helpers.block import build_empty_block_for_next_slot, make_block_signature +from eth2spec.test.helpers.keys import privkeys +from eth2spec.utils.bls import bls_sign, bls_aggregate_signatures + + +def build_attestation_data(state, slot, shard): + assert state.slot >= slot + + if slot == state.slot: + block_root = build_empty_block_for_next_slot(state).previous_block_root + else: + block_root = get_block_root_at_slot(state, slot) + + current_epoch_start_slot = get_epoch_start_slot(get_current_epoch(state)) + if slot < current_epoch_start_slot: + epoch_boundary_root = get_block_root(state, get_previous_epoch(state)) + elif slot == current_epoch_start_slot: + epoch_boundary_root = block_root + else: + epoch_boundary_root = get_block_root(state, get_current_epoch(state)) + + if slot < current_epoch_start_slot: + justified_epoch = state.previous_justified_epoch + justified_block_root = state.previous_justified_root + else: + justified_epoch = state.current_justified_epoch + justified_block_root = state.current_justified_root + + crosslinks = state.current_crosslinks if slot_to_epoch(slot) == get_current_epoch( + state) else state.previous_crosslinks + return AttestationData( + shard=shard, + beacon_block_root=block_root, + source_epoch=justified_epoch, + source_root=justified_block_root, + target_epoch=slot_to_epoch(slot), + target_root=epoch_boundary_root, + crosslink_data_root=spec.ZERO_HASH, + previous_crosslink_root=hash_tree_root(crosslinks[shard]), + ) + + +def get_valid_attestation(state, slot=None): + if slot is None: + slot = state.slot + + if slot_to_epoch(slot) == get_current_epoch(state): + shard = (state.latest_start_shard + slot) % spec.SLOTS_PER_EPOCH + else: + previous_shard_delta = get_shard_delta(state, get_previous_epoch(state)) + shard = (state.latest_start_shard - previous_shard_delta + slot) % spec.SHARD_COUNT + + attestation_data = build_attestation_data(state, slot, shard) + + crosslink_committee = get_crosslink_committee(state, attestation_data.target_epoch, attestation_data.shard) + + committee_size = len(crosslink_committee) + bitfield_length = (committee_size + 7) // 8 + aggregation_bitfield = b'\xC0' + b'\x00' * (bitfield_length - 1) + custody_bitfield = b'\x00' * bitfield_length + attestation = Attestation( + aggregation_bitfield=aggregation_bitfield, + data=attestation_data, + custody_bitfield=custody_bitfield, + ) + make_attestation_signature(state, attestation) + return attestation + + +def make_aggregate_attestation_signature(state: BeaconState, data: AttestationData, participants: List[int]): + signatures = [] + for validator_index in participants: + privkey = privkeys[validator_index] + signatures.append( + get_attestation_signature( + state, + data, + privkey + ) + ) + + return bls_aggregate_signatures(signatures) + + +def make_indexed_attestation_signature(state, indexed_attestation: IndexedAttestation): + participants = indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices + indexed_attestation.signature = make_aggregate_attestation_signature(state, indexed_attestation.data, participants) + + +def make_attestation_signature(state, attestation: Attestation): + participants = get_attesting_indices( + state, + attestation.data, + attestation.aggregation_bitfield, + ) + + attestation.signature = make_aggregate_attestation_signature(state, attestation.data, participants) + + +def get_attestation_signature(state, attestation_data, privkey, custody_bit=0b0): + message_hash = AttestationDataAndCustodyBit( + data=attestation_data, + custody_bit=custody_bit, + ).hash_tree_root() + + return bls_sign( + message_hash=message_hash, + privkey=privkey, + domain=get_domain( + state=state, + domain_type=spec.DOMAIN_ATTESTATION, + message_epoch=attestation_data.target_epoch, + ) + ) + + +def fill_aggregate_attestation(state, attestation): + crosslink_committee = get_crosslink_committee(state, attestation.data.target_epoch, attestation.data.shard) + for i in range(len(crosslink_committee)): + attestation.aggregation_bitfield = set_bitfield_bit(attestation.aggregation_bitfield, i) + + +def add_attestation_to_state(state, attestation, slot): + block = build_empty_block_for_next_slot(state) + block.slot = slot + block.body.attestations.append(attestation) + state_transition_to(state, block.slot) + make_block_signature(state, block) + state_transition(state, block) diff --git a/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py b/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py new file mode 100644 index 0000000000..2895c77718 --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py @@ -0,0 +1,17 @@ +from copy import deepcopy + +from eth2spec.phase0.spec import AttesterSlashing, convert_to_indexed +from eth2spec.test.helpers.attestations import get_valid_attestation, make_attestation_signature + + +def get_valid_attester_slashing(state): + attestation_1 = get_valid_attestation(state) + attestation_2 = deepcopy(attestation_1) + attestation_2.data.target_root = b'\x01' * 32 + + make_attestation_signature(state, attestation_2) + + return AttesterSlashing( + attestation_1=convert_to_indexed(state, attestation_1), + attestation_2=convert_to_indexed(state, attestation_2), + ) diff --git a/test_libs/pyspec/eth2spec/test/helpers/bitfields.py b/test_libs/pyspec/eth2spec/test/helpers/bitfields.py new file mode 100644 index 0000000000..7c25d073ab --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/helpers/bitfields.py @@ -0,0 +1,11 @@ +def set_bitfield_bit(bitfield, i): + """ + Set the bit in ``bitfield`` at position ``i`` to ``1``. + """ + byte_index = i // 8 + bit_index = i % 8 + return ( + bitfield[:byte_index] + + bytes([bitfield[byte_index] | (1 << bit_index)]) + + bitfield[byte_index + 1:] + ) diff --git a/test_libs/pyspec/eth2spec/test/helpers/block.py b/test_libs/pyspec/eth2spec/test/helpers/block.py new file mode 100644 index 0000000000..9069018679 --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/helpers/block.py @@ -0,0 +1,62 @@ +from eth2spec.phase0 import spec +from eth2spec.phase0.spec import get_beacon_proposer_index, slot_to_epoch, get_domain, BeaconBlock + +from copy import deepcopy + +from eth2spec.test.helpers.keys import privkeys +from eth2spec.test.helpers.state import next_slot +from eth2spec.utils.bls import bls_sign +from eth2spec.utils.minimal_ssz import signing_root, hash_tree_root + + +def make_block_signature(state, block): + assert block.slot == state.slot or block.slot == state.slot + 1 + if block.slot == state.slot: + proposer_index = get_beacon_proposer_index(state) + else: + # use stub state to get proposer index of next slot + stub_state = deepcopy(state) + next_slot(stub_state) + proposer_index = get_beacon_proposer_index(stub_state) + + privkey = privkeys[proposer_index] + + block.body.randao_reveal = bls_sign( + privkey=privkey, + message_hash=hash_tree_root(slot_to_epoch(block.slot)), + domain=get_domain( + state, + message_epoch=slot_to_epoch(block.slot), + domain_type=spec.DOMAIN_RANDAO, + ) + ) + block.signature = bls_sign( + message_hash=signing_root(block), + privkey=privkey, + domain=get_domain( + state, + spec.DOMAIN_BEACON_PROPOSER, + slot_to_epoch(block.slot))) + return block + + +def build_empty_block(state, slot=None, signed=False): + if slot is None: + slot = state.slot + empty_block = BeaconBlock() + empty_block.slot = slot + empty_block.body.eth1_data.deposit_count = state.deposit_index + previous_block_header = deepcopy(state.latest_block_header) + if previous_block_header.state_root == spec.ZERO_HASH: + previous_block_header.state_root = state.hash_tree_root() + empty_block.previous_block_root = signing_root(previous_block_header) + + if signed: + make_block_signature(state, empty_block) + + return empty_block + + +def build_empty_block_for_next_slot(state, signed=False): + return build_empty_block(state, state.slot + 1, signed=signed) + diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py new file mode 100644 index 0000000000..cfc6add5ea --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -0,0 +1,74 @@ +# Access constants from spec pkg reference. +import eth2spec.phase0.spec as spec + +from eth2spec.phase0.spec import get_domain, DepositData, verify_merkle_branch, Deposit, ZERO_HASH +from eth2spec.test.helpers.keys import pubkeys, privkeys +from eth2spec.utils.bls import bls_sign +from eth2spec.utils.merkle_minimal import calc_merkle_tree_from_leaves, get_merkle_root, get_merkle_proof +from eth2spec.utils.minimal_ssz import signing_root + + +def build_deposit_data(state, pubkey, privkey, amount): + deposit_data = DepositData( + pubkey=pubkey, + # insecurely use pubkey as withdrawal key as well + withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:], + amount=amount, + ) + signature = bls_sign( + message_hash=signing_root(deposit_data), + privkey=privkey, + domain=get_domain( + state, + spec.DOMAIN_DEPOSIT, + ) + ) + deposit_data.signature = signature + return deposit_data + + +def build_deposit(state, + deposit_data_leaves, + pubkey, + privkey, + amount): + deposit_data = build_deposit_data(state, pubkey, privkey, amount) + + item = deposit_data.hash_tree_root() + index = len(deposit_data_leaves) + deposit_data_leaves.append(item) + tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves)) + root = get_merkle_root((tuple(deposit_data_leaves))) + proof = list(get_merkle_proof(tree, item_index=index)) + assert verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, index, root) + + deposit = Deposit( + proof=list(proof), + index=index, + data=deposit_data, + ) + + return deposit, root, deposit_data_leaves + + +def prepare_state_and_deposit(state, validator_index, amount): + """ + Prepare the state for the deposit, and create a deposit for the given validator, depositing the given amount. + """ + pre_validator_count = len(state.validator_registry) + # fill previous deposits with zero-hash + deposit_data_leaves = [ZERO_HASH] * pre_validator_count + + pubkey = pubkeys[validator_index] + privkey = privkeys[validator_index] + deposit, root, deposit_data_leaves = build_deposit( + state, + deposit_data_leaves, + pubkey, + privkey, + amount, + ) + + state.latest_eth1_data.deposit_root = root + state.latest_eth1_data.deposit_count = len(deposit_data_leaves) + return deposit diff --git a/test_libs/pyspec/eth2spec/test/helpers/genesis.py b/test_libs/pyspec/eth2spec/test/helpers/genesis.py new file mode 100644 index 0000000000..b991f74e71 --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/helpers/genesis.py @@ -0,0 +1,51 @@ +# Access constants from spec pkg reference. +import eth2spec.phase0.spec as spec + +from eth2spec.phase0.spec import Eth1Data, ZERO_HASH, get_active_validator_indices +from eth2spec.test.helpers.keys import pubkeys +from eth2spec.utils.minimal_ssz import hash_tree_root + + +def build_mock_validator(i: int, balance: int): + pubkey = pubkeys[i] + # insecurely use pubkey as withdrawal key as well + withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:] + return spec.Validator( + pubkey=pubkeys[i], + withdrawal_credentials=withdrawal_credentials, + activation_eligibility_epoch=spec.FAR_FUTURE_EPOCH, + activation_epoch=spec.FAR_FUTURE_EPOCH, + exit_epoch=spec.FAR_FUTURE_EPOCH, + withdrawable_epoch=spec.FAR_FUTURE_EPOCH, + effective_balance=min(balance - balance % spec.EFFECTIVE_BALANCE_INCREMENT, spec.MAX_EFFECTIVE_BALANCE) + ) + + +def create_genesis_state(num_validators): + deposit_root = b'\x42' * 32 + + state = spec.BeaconState( + genesis_time=0, + deposit_index=num_validators, + latest_eth1_data=Eth1Data( + deposit_root=deposit_root, + deposit_count=num_validators, + block_hash=ZERO_HASH, + )) + + # We "hack" in the initial validators, + # as it is much faster than creating and processing genesis deposits for every single test case. + state.balances = [spec.MAX_EFFECTIVE_BALANCE] * num_validators + state.validator_registry = [build_mock_validator(i, state.balances[i]) for i in range(num_validators)] + + # Process genesis activations + for validator in state.validator_registry: + if validator.effective_balance >= spec.MAX_EFFECTIVE_BALANCE: + validator.activation_eligibility_epoch = spec.GENESIS_EPOCH + validator.activation_epoch = spec.GENESIS_EPOCH + + genesis_active_index_root = hash_tree_root(get_active_validator_indices(state, spec.GENESIS_EPOCH)) + for index in range(spec.LATEST_ACTIVE_INDEX_ROOTS_LENGTH): + state.latest_active_index_roots[index] = genesis_active_index_root + + return state diff --git a/test_libs/pyspec/eth2spec/test/helpers/keys.py b/test_libs/pyspec/eth2spec/test/helpers/keys.py new file mode 100644 index 0000000000..fe27b78a1e --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/helpers/keys.py @@ -0,0 +1,5 @@ +from py_ecc import bls + +privkeys = [i + 1 for i in range(1024)] +pubkeys = [bls.privtopub(privkey) for privkey in privkeys] +pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys, pubkeys)} diff --git a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py new file mode 100644 index 0000000000..be4f32d1c0 --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py @@ -0,0 +1,48 @@ +from copy import deepcopy + +# Access constants from spec pkg reference. +import eth2spec.phase0.spec as spec + +from eth2spec.phase0.spec import get_current_epoch, get_active_validator_indices, BeaconBlockHeader, ZERO_HASH, \ + get_domain, ProposerSlashing +from eth2spec.test.helpers.keys import pubkey_to_privkey +from eth2spec.utils.bls import bls_sign +from eth2spec.utils.minimal_ssz import signing_root + + +def get_valid_proposer_slashing(state): + current_epoch = get_current_epoch(state) + validator_index = get_active_validator_indices(state, current_epoch)[-1] + privkey = pubkey_to_privkey[state.validator_registry[validator_index].pubkey] + slot = state.slot + + header_1 = BeaconBlockHeader( + slot=slot, + previous_block_root=ZERO_HASH, + state_root=ZERO_HASH, + block_body_root=ZERO_HASH, + ) + header_2 = deepcopy(header_1) + header_2.previous_block_root = b'\x02' * 32 + header_2.slot = slot + 1 + + domain = get_domain( + state=state, + domain_type=spec.DOMAIN_BEACON_PROPOSER, + ) + header_1.signature = bls_sign( + message_hash=signing_root(header_1), + privkey=privkey, + domain=domain, + ) + header_2.signature = bls_sign( + message_hash=signing_root(header_2), + privkey=privkey, + domain=domain, + ) + + return ProposerSlashing( + proposer_index=validator_index, + header_1=header_1, + header_2=header_2, + ) diff --git a/test_libs/pyspec/eth2spec/test/helpers/state.py b/test_libs/pyspec/eth2spec/test/helpers/state.py new file mode 100644 index 0000000000..0261801df7 --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/helpers/state.py @@ -0,0 +1,42 @@ +# Access constants from spec pkg reference. +import eth2spec.phase0.spec as spec + +from eth2spec.phase0.state_transition import state_transition_to, state_transition +from eth2spec.test.helpers.block import build_empty_block + + +def get_balance(state, index): + return state.balances[index] + + +def next_slot(state): + """ + Transition to the next slot. + """ + state_transition_to(state, state.slot + 1) + + +def next_epoch(state): + """ + Transition to the start slot of the next epoch + """ + slot = state.slot + spec.SLOTS_PER_EPOCH - (state.slot % spec.SLOTS_PER_EPOCH) + state_transition_to(state, slot) + + +def apply_empty_block(state): + """ + Transition via an empty block (on current slot, assuming no block has been applied yet). + :return: the empty block that triggered the transition. + """ + block = build_empty_block(state) + state_transition(state, block) + return block + + +def get_state_root(state, slot) -> bytes: + """ + Return the state root at a recent ``slot``. + """ + assert slot < state.slot <= slot + spec.SLOTS_PER_HISTORICAL_ROOT + return state.latest_state_roots[slot % spec.SLOTS_PER_HISTORICAL_ROOT] diff --git a/test_libs/pyspec/eth2spec/test/helpers/transfers.py b/test_libs/pyspec/eth2spec/test/helpers/transfers.py new file mode 100644 index 0000000000..b78359f4ef --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/helpers/transfers.py @@ -0,0 +1,50 @@ +# Access constants from spec pkg reference. +import eth2spec.phase0.spec as spec + +from eth2spec.phase0.spec import get_current_epoch, get_active_validator_indices, Transfer, ZERO_HASH, get_domain +from eth2spec.test.helpers.keys import pubkeys, privkeys +from eth2spec.test.helpers.state import get_balance +from eth2spec.utils.bls import bls_sign +from eth2spec.utils.minimal_ssz import signing_root + + +def get_valid_transfer(state, slot=None, sender_index=None, amount=None, fee=None): + if slot is None: + slot = state.slot + current_epoch = get_current_epoch(state) + if sender_index is None: + sender_index = get_active_validator_indices(state, current_epoch)[-1] + recipient_index = get_active_validator_indices(state, current_epoch)[0] + transfer_pubkey = pubkeys[-1] + transfer_privkey = privkeys[-1] + + if fee is None: + fee = get_balance(state, sender_index) // 32 + if amount is None: + amount = get_balance(state, sender_index) - fee + + transfer = Transfer( + sender=sender_index, + recipient=recipient_index, + amount=amount, + fee=fee, + slot=slot, + pubkey=transfer_pubkey, + signature=ZERO_HASH, + ) + transfer.signature = bls_sign( + message_hash=signing_root(transfer), + privkey=transfer_privkey, + domain=get_domain( + state=state, + domain_type=spec.DOMAIN_TRANSFER, + message_epoch=get_current_epoch(state), + ) + ) + + # ensure withdrawal_credentials reproducable + state.validator_registry[transfer.sender].withdrawal_credentials = ( + spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(transfer.pubkey)[1:] + ) + + return transfer diff --git a/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py b/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py new file mode 100644 index 0000000000..a42830f832 --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py @@ -0,0 +1,24 @@ +# Access constants from spec pkg reference. +import eth2spec.phase0.spec as spec + +from eth2spec.phase0.spec import VoluntaryExit, get_domain +from eth2spec.utils.bls import bls_sign +from eth2spec.utils.minimal_ssz import signing_root + + +def build_voluntary_exit(state, epoch, validator_index, privkey): + voluntary_exit = VoluntaryExit( + epoch=epoch, + validator_index=validator_index, + ) + voluntary_exit.signature = bls_sign( + message_hash=signing_root(voluntary_exit), + privkey=privkey, + domain=get_domain( + state=state, + domain_type=spec.DOMAIN_VOLUNTARY_EXIT, + message_epoch=epoch, + ) + ) + + return voluntary_exit From a4e22639f346e7a0fbc71f0338fc3e9d9709cea7 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 15 May 2019 18:37:11 +0200 Subject: [PATCH 04/37] fix/update bls funcs for testing --- test_libs/pyspec/eth2spec/utils/bls.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/bls.py b/test_libs/pyspec/eth2spec/utils/bls.py index daeb361a9d..23a9a7529f 100644 --- a/test_libs/pyspec/eth2spec/utils/bls.py +++ b/test_libs/pyspec/eth2spec/utils/bls.py @@ -22,4 +22,18 @@ def bls_aggregate_pubkeys(pubkeys): if bls_active: return bls.aggregate_pubkeys(pubkeys) else: - return b'\x42' * 96 + return b'\xaa' * 48 + + +def bls_aggregate_signatures(signatures): + if bls_active: + return bls.aggregate_signatures(signatures) + else: + return b'\x22' * 96 + + +def bls_sign(message_hash, privkey, domain): + if bls_active: + return bls.sign(message_hash=message_hash, privkey=privkey, domain=domain) + else: + return b'\x11' * 96 From e07f1bc98ffe259175efd489b200606dc44fdca1 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 15 May 2019 18:45:25 +0200 Subject: [PATCH 05/37] update imports to new helpers + fix up imports --- .../block_processing/test_process_attestation.py | 16 ++++++++-------- .../test_process_attester_slashing.py | 6 +++--- .../test_process_block_header.py | 4 ++-- .../block_processing/test_process_deposit.py | 16 +++------------- .../test_process_proposer_slashing.py | 7 ++----- .../block_processing/test_process_transfer.py | 7 ++----- .../test_process_voluntary_exit.py | 8 ++------ .../pyspec/eth2spec/test/helpers/attestations.py | 2 -- test_libs/pyspec/eth2spec/test/helpers/block.py | 5 ++--- .../eth2spec/test/helpers/proposer_slashings.py | 1 - 10 files changed, 24 insertions(+), 48 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py index 98cb62ef7b..7f8ac6e412 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py @@ -1,24 +1,24 @@ from copy import deepcopy import eth2spec.phase0.spec as spec - -from eth2spec.phase0.state_transition import ( - state_transition_to, -) from eth2spec.phase0.spec import ( get_current_epoch, process_attestation ) -from eth2spec.test.helpers import ( +from eth2spec.phase0.state_transition import ( + state_transition_to, +) +from eth2spec.test.context import spec_state_test, expect_assertion_error +from eth2spec.test.helpers.attestations import ( get_valid_attestation, + make_attestation_signature, +) +from eth2spec.test.helpers.state import ( apply_empty_block, next_epoch, next_slot, - make_attestation_signature, ) -from eth2spec.test.context import spec_state_test, expect_assertion_error - def run_attestation_processing(state, attestation, valid=True): """ diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py index a0334c892d..0bbdb7d4af 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py @@ -4,12 +4,12 @@ process_attester_slashing, ) from eth2spec.test.context import spec_state_test, expect_assertion_error -from eth2spec.test.helpers import ( +from eth2spec.test.helpers.attestations import make_indexed_attestation_signature +from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing +from eth2spec.test.helpers.state import ( get_balance, - get_valid_attester_slashing, next_epoch, apply_empty_block, - make_indexed_attestation_signature ) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py index f2695ad098..674f659a00 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py @@ -7,11 +7,11 @@ process_block_header, ) from eth2spec.test.context import spec_state_test, expect_assertion_error -from eth2spec.test.helpers import ( +from eth2spec.test.helpers.block import ( build_empty_block_for_next_slot, - next_slot, make_block_signature ) +from eth2spec.test.helpers.state import next_slot def prepare_state_for_header_processing(state): diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py index fe2dae6a84..8f663c5c22 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py @@ -1,18 +1,8 @@ import eth2spec.phase0.spec as spec - -from eth2spec.phase0.spec import ( - ZERO_HASH, - process_deposit, -) -from eth2spec.test.helpers import ( - get_balance, - build_deposit, - prepare_state_and_deposit, - privkeys, - pubkeys, -) - +from eth2spec.phase0.spec import process_deposit from eth2spec.test.context import spec_state_test, expect_assertion_error +from eth2spec.test.helpers.deposits import prepare_state_and_deposit +from eth2spec.test.helpers.state import get_balance def run_deposit_processing(state, deposit, validator_index, valid=True): diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py index 609c97ce6d..6efed86850 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py @@ -3,12 +3,9 @@ get_current_epoch, process_proposer_slashing, ) -from eth2spec.test.helpers import ( - get_balance, - get_valid_proposer_slashing, -) - from eth2spec.test.context import spec_state_test, expect_assertion_error +from eth2spec.test.helpers.proposer_slashings import get_valid_proposer_slashing +from eth2spec.test.helpers.state import get_balance def run_proposer_slashing_processing(state, proposer_slashing, valid=True): diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py index 64539d56e8..fed948e636 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py @@ -6,11 +6,8 @@ process_transfer, ) from eth2spec.test.context import spec_state_test, expect_assertion_error -from eth2spec.test.helpers import ( - get_valid_transfer, - next_epoch, - apply_empty_block -) +from eth2spec.test.helpers.state import next_epoch, apply_empty_block +from eth2spec.test.helpers.transfers import get_valid_transfer def run_transfer_processing(state, transfer, valid=True): diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py index be0ef1e7a4..f842d8b078 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py @@ -1,17 +1,13 @@ import eth2spec.phase0.spec as spec - from eth2spec.phase0.spec import ( get_active_validator_indices, get_churn_limit, get_current_epoch, process_voluntary_exit, ) -from eth2spec.test.helpers import ( - build_voluntary_exit, - pubkey_to_privkey, -) - from eth2spec.test.context import spec_state_test, expect_assertion_error +from eth2spec.test.helpers.keys import pubkey_to_privkey +from eth2spec.test.helpers.voluntary_exits import build_voluntary_exit def run_voluntary_exit_processing(state, voluntary_exit, valid=True): diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index d7fab4c177..ebc76c27f8 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -2,14 +2,12 @@ # Access constants from spec pkg reference. import eth2spec.phase0.spec as spec - from eth2spec.phase0.spec import ( Attestation, AttestationData, AttestationDataAndCustodyBit, get_epoch_start_slot, get_block_root, get_current_epoch, get_previous_epoch, slot_to_epoch, get_shard_delta, get_crosslink_committee, get_domain, IndexedAttestation, get_attesting_indices, BeaconState, get_block_root_at_slot) - from eth2spec.phase0.state_transition import ( state_transition, state_transition_to ) diff --git a/test_libs/pyspec/eth2spec/test/helpers/block.py b/test_libs/pyspec/eth2spec/test/helpers/block.py index 9069018679..25874e75b9 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block.py @@ -1,8 +1,7 @@ -from eth2spec.phase0 import spec -from eth2spec.phase0.spec import get_beacon_proposer_index, slot_to_epoch, get_domain, BeaconBlock - from copy import deepcopy +from eth2spec.phase0 import spec +from eth2spec.phase0.spec import get_beacon_proposer_index, slot_to_epoch, get_domain, BeaconBlock from eth2spec.test.helpers.keys import privkeys from eth2spec.test.helpers.state import next_slot from eth2spec.utils.bls import bls_sign diff --git a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py index be4f32d1c0..d14621b1c5 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py +++ b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py @@ -2,7 +2,6 @@ # Access constants from spec pkg reference. import eth2spec.phase0.spec as spec - from eth2spec.phase0.spec import get_current_epoch, get_active_validator_indices, BeaconBlockHeader, ZERO_HASH, \ get_domain, ProposerSlashing from eth2spec.test.helpers.keys import pubkey_to_privkey From 5c3e760c2810e5ef567fd6f34940c17b20db3e4b Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 15 May 2019 18:58:12 +0200 Subject: [PATCH 06/37] update remaining imports --- .../test_process_crosslinks.py | 10 ++++++---- .../test_process_registry_updates.py | 5 +---- .../pyspec/eth2spec/test/test_finality.py | 10 ++++++---- test_libs/pyspec/eth2spec/test/test_sanity.py | 18 ++++++++++-------- 4 files changed, 23 insertions(+), 20 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py index d4e6dc9381..f342c5e6d7 100644 --- a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py @@ -10,15 +10,17 @@ state_transition, ) from eth2spec.test.context import spec_state_test -from eth2spec.test.helpers import ( +from eth2spec.test.helpers.state import ( + next_epoch, + next_slot, + apply_empty_block +) +from eth2spec.test.helpers.attestations import ( add_attestation_to_state, build_empty_block_for_next_slot, fill_aggregate_attestation, get_crosslink_committee, get_valid_attestation, - next_epoch, - next_slot, - apply_empty_block ) diff --git a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py index 970c309429..2086f4ef2d 100644 --- a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py +++ b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py @@ -4,10 +4,7 @@ get_current_epoch, is_active_validator, ) -from eth2spec.test.helpers import ( - next_epoch, -) - +from eth2spec.test.helpers.state import next_epoch from eth2spec.test.context import spec_state_test diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index 75191e59aa..732dd8f0c4 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -5,14 +5,16 @@ state_transition, ) from .context import spec_state_test -from .helpers import ( - build_empty_block_for_next_slot, +from .helpers.state import ( + next_epoch, + apply_empty_block +) +from .helpers.block import build_empty_block_for_next_slot +from .helpers.attestations import ( fill_aggregate_attestation, get_current_epoch, get_epoch_start_slot, get_valid_attestation, - next_epoch, - apply_empty_block ) diff --git a/test_libs/pyspec/eth2spec/test/test_sanity.py b/test_libs/pyspec/eth2spec/test/test_sanity.py index 1951415acb..1509a9843a 100644 --- a/test_libs/pyspec/eth2spec/test/test_sanity.py +++ b/test_libs/pyspec/eth2spec/test/test_sanity.py @@ -19,18 +19,20 @@ from eth2spec.phase0.state_transition import ( state_transition, ) -from .helpers import ( +from .helpers.state import ( get_balance, - build_empty_block_for_next_slot, - get_state_root, - get_valid_attestation, - get_valid_attester_slashing, - get_valid_proposer_slashing, - get_valid_transfer, - prepare_state_and_deposit, + get_state_root +) +from .helpers.transfers import get_valid_transfer +from .helpers.block import build_empty_block_for_next_slot +from .helpers.keys import ( privkeys, pubkeys, ) +from .helpers.attester_slashings import get_valid_attester_slashing +from .helpers.proposer_slashings import get_valid_proposer_slashing +from .helpers.attestations import get_valid_attestation +from .helpers.deposits import prepare_state_and_deposit from .context import spec_state_test From 8a43ec01326cc5d8b0cdfad6b4d4e7a452c2d1ac Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 15 May 2019 19:31:02 +0200 Subject: [PATCH 07/37] add signing methods --- .../test_process_attestation.py | 16 ++++++------- .../test_process_attester_slashing.py | 4 ++-- .../test_process_block_header.py | 4 ++-- .../eth2spec/test/helpers/attestations.py | 17 ++++++------- .../test/helpers/attester_slashings.py | 4 ++-- .../pyspec/eth2spec/test/helpers/block.py | 4 ++-- .../eth2spec/test/helpers/block_header.py | 18 ++++++++++++++ .../pyspec/eth2spec/test/helpers/deposits.py | 17 +++++++++---- .../test/helpers/proposer_slashings.py | 24 ++++--------------- .../pyspec/eth2spec/test/helpers/transfers.py | 22 ++++++++++------- .../eth2spec/test/helpers/voluntary_exits.py | 12 ++++++---- 11 files changed, 81 insertions(+), 61 deletions(-) create mode 100644 test_libs/pyspec/eth2spec/test/helpers/block_header.py diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py index 7f8ac6e412..bbfab9a13b 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py @@ -11,7 +11,7 @@ from eth2spec.test.context import spec_state_test, expect_assertion_error from eth2spec.test.helpers.attestations import ( get_valid_attestation, - make_attestation_signature, + sign_attestation, ) from eth2spec.test.helpers.state import ( apply_empty_block, @@ -105,7 +105,7 @@ def test_old_source_epoch(state): attestation.data.source_epoch -= 1 # Re do signature - make_attestation_signature(state, attestation) + sign_attestation(state, attestation) yield from run_attestation_processing(state, attestation, False) @@ -118,7 +118,7 @@ def test_wrong_shard(state): attestation.data.shard += 1 # Re do signature - make_attestation_signature(state, attestation) + sign_attestation(state, attestation) yield from run_attestation_processing(state, attestation, False) @@ -131,7 +131,7 @@ def test_new_source_epoch(state): attestation.data.source_epoch += 1 # Re do signature - make_attestation_signature(state, attestation) + sign_attestation(state, attestation) yield from run_attestation_processing(state, attestation, False) @@ -144,7 +144,7 @@ def test_source_root_is_target_root(state): attestation.data.source_root = attestation.data.target_root # Re do signature - make_attestation_signature(state, attestation) + sign_attestation(state, attestation) yield from run_attestation_processing(state, attestation, False) @@ -171,7 +171,7 @@ def test_invalid_current_source_root(state): attestation.data.source_root = state.current_justified_root # Re do signature - make_attestation_signature(state, attestation) + sign_attestation(state, attestation) yield from run_attestation_processing(state, attestation, False) @@ -184,7 +184,7 @@ def test_bad_source_root(state): attestation.data.source_root = b'\x42' * 32 # Re do signature - make_attestation_signature(state, attestation) + sign_attestation(state, attestation) yield from run_attestation_processing(state, attestation, False) @@ -197,7 +197,7 @@ def test_non_zero_crosslink_data_root(state): attestation.data.crosslink_data_root = b'\x42' * 32 # Re do signature - make_attestation_signature(state, attestation) + sign_attestation(state, attestation) yield from run_attestation_processing(state, attestation, False) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py index 0bbdb7d4af..56b3711a41 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py @@ -4,7 +4,7 @@ process_attester_slashing, ) from eth2spec.test.context import spec_state_test, expect_assertion_error -from eth2spec.test.helpers.attestations import make_indexed_attestation_signature +from eth2spec.test.helpers.attestations import sign_indexed_attestation from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing from eth2spec.test.helpers.state import ( get_balance, @@ -81,7 +81,7 @@ def test_success_surround(state): attester_slashing.attestation_1.data.target_epoch = attester_slashing.attestation_2.data.target_epoch + 1 # correct the signature of attestation 1 - make_indexed_attestation_signature(state, attester_slashing.attestation_1) + sign_indexed_attestation(state, attester_slashing.attestation_1) yield from run_attester_slashing_processing(state, attester_slashing) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py index 674f659a00..09c0c53885 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py @@ -9,7 +9,7 @@ from eth2spec.test.context import spec_state_test, expect_assertion_error from eth2spec.test.helpers.block import ( build_empty_block_for_next_slot, - make_block_signature + sign_block ) from eth2spec.test.helpers.state import next_slot @@ -59,7 +59,7 @@ def test_invalid_slot(state): def test_invalid_previous_block_root(state): block = build_empty_block_for_next_slot(state) block.previous_block_root = b'\12' * 32 # invalid prev root - make_block_signature(state, block) + sign_block(state, block) yield from run_block_header_processing(state, block, valid=False) diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index ebc76c27f8..de5f2d521a 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -12,9 +12,10 @@ state_transition, state_transition_to ) from eth2spec.test.helpers.bitfields import set_bitfield_bit -from eth2spec.test.helpers.block import build_empty_block_for_next_slot, make_block_signature +from eth2spec.test.helpers.block import build_empty_block_for_next_slot, sign_block from eth2spec.test.helpers.keys import privkeys from eth2spec.utils.bls import bls_sign, bls_aggregate_signatures +from eth2spec.utils.minimal_ssz import hash_tree_root def build_attestation_data(state, slot, shard): @@ -77,11 +78,11 @@ def get_valid_attestation(state, slot=None): data=attestation_data, custody_bitfield=custody_bitfield, ) - make_attestation_signature(state, attestation) + sign_attestation(state, attestation) return attestation -def make_aggregate_attestation_signature(state: BeaconState, data: AttestationData, participants: List[int]): +def sign_aggregate_attestation(state: BeaconState, data: AttestationData, participants: List[int]): signatures = [] for validator_index in participants: privkey = privkeys[validator_index] @@ -96,19 +97,19 @@ def make_aggregate_attestation_signature(state: BeaconState, data: AttestationDa return bls_aggregate_signatures(signatures) -def make_indexed_attestation_signature(state, indexed_attestation: IndexedAttestation): +def sign_indexed_attestation(state, indexed_attestation: IndexedAttestation): participants = indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices - indexed_attestation.signature = make_aggregate_attestation_signature(state, indexed_attestation.data, participants) + indexed_attestation.signature = sign_aggregate_attestation(state, indexed_attestation.data, participants) -def make_attestation_signature(state, attestation: Attestation): +def sign_attestation(state, attestation: Attestation): participants = get_attesting_indices( state, attestation.data, attestation.aggregation_bitfield, ) - attestation.signature = make_aggregate_attestation_signature(state, attestation.data, participants) + attestation.signature = sign_aggregate_attestation(state, attestation.data, participants) def get_attestation_signature(state, attestation_data, privkey, custody_bit=0b0): @@ -139,5 +140,5 @@ def add_attestation_to_state(state, attestation, slot): block.slot = slot block.body.attestations.append(attestation) state_transition_to(state, block.slot) - make_block_signature(state, block) + sign_block(state, block) state_transition(state, block) diff --git a/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py b/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py index 2895c77718..b34f6ff453 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py @@ -1,7 +1,7 @@ from copy import deepcopy from eth2spec.phase0.spec import AttesterSlashing, convert_to_indexed -from eth2spec.test.helpers.attestations import get_valid_attestation, make_attestation_signature +from eth2spec.test.helpers.attestations import get_valid_attestation, sign_attestation def get_valid_attester_slashing(state): @@ -9,7 +9,7 @@ def get_valid_attester_slashing(state): attestation_2 = deepcopy(attestation_1) attestation_2.data.target_root = b'\x01' * 32 - make_attestation_signature(state, attestation_2) + sign_attestation(state, attestation_2) return AttesterSlashing( attestation_1=convert_to_indexed(state, attestation_1), diff --git a/test_libs/pyspec/eth2spec/test/helpers/block.py b/test_libs/pyspec/eth2spec/test/helpers/block.py index 25874e75b9..36659f8fa4 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block.py @@ -8,7 +8,7 @@ from eth2spec.utils.minimal_ssz import signing_root, hash_tree_root -def make_block_signature(state, block): +def sign_block(state, block): assert block.slot == state.slot or block.slot == state.slot + 1 if block.slot == state.slot: proposer_index = get_beacon_proposer_index(state) @@ -51,7 +51,7 @@ def build_empty_block(state, slot=None, signed=False): empty_block.previous_block_root = signing_root(previous_block_header) if signed: - make_block_signature(state, empty_block) + sign_block(state, empty_block) return empty_block diff --git a/test_libs/pyspec/eth2spec/test/helpers/block_header.py b/test_libs/pyspec/eth2spec/test/helpers/block_header.py new file mode 100644 index 0000000000..9aba62d37d --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/helpers/block_header.py @@ -0,0 +1,18 @@ +# Access constants from spec pkg reference. +import eth2spec.phase0.spec as spec + +from eth2spec.phase0.spec import get_domain +from eth2spec.utils.bls import bls_sign +from eth2spec.utils.minimal_ssz import signing_root + + +def sign_block_header(state, header, privkey): + domain = get_domain( + state=state, + domain_type=spec.DOMAIN_BEACON_PROPOSER, + ) + header.signature = bls_sign( + message_hash=signing_root(header), + privkey=privkey, + domain=domain, + ) diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index cfc6add5ea..1bea2f923b 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -8,13 +8,19 @@ from eth2spec.utils.minimal_ssz import signing_root -def build_deposit_data(state, pubkey, privkey, amount): +def build_deposit_data(state, pubkey, privkey, amount, signed=False): deposit_data = DepositData( pubkey=pubkey, # insecurely use pubkey as withdrawal key as well withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:], amount=amount, ) + if signed: + sign_deposit_data(state, deposit_data, privkey) + return deposit_data + + +def sign_deposit_data(state, deposit_data, privkey): signature = bls_sign( message_hash=signing_root(deposit_data), privkey=privkey, @@ -24,15 +30,15 @@ def build_deposit_data(state, pubkey, privkey, amount): ) ) deposit_data.signature = signature - return deposit_data def build_deposit(state, deposit_data_leaves, pubkey, privkey, - amount): - deposit_data = build_deposit_data(state, pubkey, privkey, amount) + amount, + signed): + deposit_data = build_deposit_data(state, pubkey, privkey, amount, signed) item = deposit_data.hash_tree_root() index = len(deposit_data_leaves) @@ -51,7 +57,7 @@ def build_deposit(state, return deposit, root, deposit_data_leaves -def prepare_state_and_deposit(state, validator_index, amount): +def prepare_state_and_deposit(state, validator_index, amount, signed=False): """ Prepare the state for the deposit, and create a deposit for the given validator, depositing the given amount. """ @@ -67,6 +73,7 @@ def prepare_state_and_deposit(state, validator_index, amount): pubkey, privkey, amount, + signed ) state.latest_eth1_data.deposit_root = root diff --git a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py index d14621b1c5..0dd1ce5c12 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py +++ b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py @@ -1,12 +1,8 @@ from copy import deepcopy -# Access constants from spec pkg reference. -import eth2spec.phase0.spec as spec -from eth2spec.phase0.spec import get_current_epoch, get_active_validator_indices, BeaconBlockHeader, ZERO_HASH, \ - get_domain, ProposerSlashing +from eth2spec.phase0.spec import get_current_epoch, get_active_validator_indices, BeaconBlockHeader, ZERO_HASH, ProposerSlashing +from eth2spec.test.helpers.block_header import sign_block_header from eth2spec.test.helpers.keys import pubkey_to_privkey -from eth2spec.utils.bls import bls_sign -from eth2spec.utils.minimal_ssz import signing_root def get_valid_proposer_slashing(state): @@ -25,20 +21,8 @@ def get_valid_proposer_slashing(state): header_2.previous_block_root = b'\x02' * 32 header_2.slot = slot + 1 - domain = get_domain( - state=state, - domain_type=spec.DOMAIN_BEACON_PROPOSER, - ) - header_1.signature = bls_sign( - message_hash=signing_root(header_1), - privkey=privkey, - domain=domain, - ) - header_2.signature = bls_sign( - message_hash=signing_root(header_2), - privkey=privkey, - domain=domain, - ) + sign_block_header(state, header_1, privkey) + sign_block_header(state, header_2, privkey) return ProposerSlashing( proposer_index=validator_index, diff --git a/test_libs/pyspec/eth2spec/test/helpers/transfers.py b/test_libs/pyspec/eth2spec/test/helpers/transfers.py index b78359f4ef..37547bf97b 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/transfers.py +++ b/test_libs/pyspec/eth2spec/test/helpers/transfers.py @@ -8,7 +8,7 @@ from eth2spec.utils.minimal_ssz import signing_root -def get_valid_transfer(state, slot=None, sender_index=None, amount=None, fee=None): +def get_valid_transfer(state, slot=None, sender_index=None, amount=None, fee=None, signed=False): if slot is None: slot = state.slot current_epoch = get_current_epoch(state) @@ -32,19 +32,25 @@ def get_valid_transfer(state, slot=None, sender_index=None, amount=None, fee=Non pubkey=transfer_pubkey, signature=ZERO_HASH, ) + if signed: + sign_transfer(state, transfer, transfer_privkey) + + # ensure withdrawal_credentials reproducible + state.validator_registry[transfer.sender].withdrawal_credentials = ( + spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(transfer.pubkey)[1:] + ) + + return transfer + + +def sign_transfer(state, transfer, privkey): transfer.signature = bls_sign( message_hash=signing_root(transfer), - privkey=transfer_privkey, + privkey=privkey, domain=get_domain( state=state, domain_type=spec.DOMAIN_TRANSFER, message_epoch=get_current_epoch(state), ) ) - - # ensure withdrawal_credentials reproducable - state.validator_registry[transfer.sender].withdrawal_credentials = ( - spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(transfer.pubkey)[1:] - ) - return transfer diff --git a/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py b/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py index a42830f832..54376d694b 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py @@ -6,19 +6,23 @@ from eth2spec.utils.minimal_ssz import signing_root -def build_voluntary_exit(state, epoch, validator_index, privkey): +def build_voluntary_exit(state, epoch, validator_index, privkey, signed=False): voluntary_exit = VoluntaryExit( epoch=epoch, validator_index=validator_index, ) + if signed: + sign_voluntary_exit(state, voluntary_exit, privkey) + return voluntary_exit + + +def sign_voluntary_exit(state, voluntary_exit, privkey): voluntary_exit.signature = bls_sign( message_hash=signing_root(voluntary_exit), privkey=privkey, domain=get_domain( state=state, domain_type=spec.DOMAIN_VOLUNTARY_EXIT, - message_epoch=epoch, + message_epoch=voluntary_exit.epoch, ) ) - - return voluntary_exit From 4dad74eec3f68b4e682bfba3c102a3196c1ecbe5 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 15 May 2019 19:47:50 +0200 Subject: [PATCH 08/37] attestation signing --- .../test_process_attestation.py | 39 +++++++++---------- .../eth2spec/test/helpers/attestations.py | 5 ++- 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py index bbfab9a13b..c44e8e7b17 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py @@ -57,7 +57,7 @@ def run_attestation_processing(state, attestation, valid=True): @spec_state_test def test_success(state): - attestation = get_valid_attestation(state) + attestation = get_valid_attestation(state, signed=True) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY yield from run_attestation_processing(state, attestation) @@ -65,7 +65,7 @@ def test_success(state): @spec_state_test def test_success_previous_epoch(state): - attestation = get_valid_attestation(state) + attestation = get_valid_attestation(state, signed=True) next_epoch(state) apply_empty_block(state) @@ -74,7 +74,7 @@ def test_success_previous_epoch(state): @spec_state_test def test_before_inclusion_delay(state): - attestation = get_valid_attestation(state) + attestation = get_valid_attestation(state, signed=True) # do not increment slot to allow for inclusion delay yield from run_attestation_processing(state, attestation, False) @@ -82,7 +82,7 @@ def test_before_inclusion_delay(state): @spec_state_test def test_after_epoch_slots(state): - attestation = get_valid_attestation(state) + attestation = get_valid_attestation(state, signed=True) # increment past latest inclusion slot state_transition_to(state, state.slot + spec.SLOTS_PER_EPOCH + 1) apply_empty_block(state) @@ -96,7 +96,7 @@ def test_old_source_epoch(state): state.finalized_epoch = 2 state.previous_justified_epoch = 3 state.current_justified_epoch = 4 - attestation = get_valid_attestation(state, slot=(spec.SLOTS_PER_EPOCH * 3) + 1) + attestation = get_valid_attestation(state, slot=(spec.SLOTS_PER_EPOCH * 3) + 1, signed=False) # test logic sanity check: make sure the attestation is pointing to oldest known source epoch assert attestation.data.source_epoch == state.previous_justified_epoch @@ -104,7 +104,6 @@ def test_old_source_epoch(state): # Now go beyond that, it will be invalid attestation.data.source_epoch -= 1 - # Re do signature sign_attestation(state, attestation) yield from run_attestation_processing(state, attestation, False) @@ -112,12 +111,11 @@ def test_old_source_epoch(state): @spec_state_test def test_wrong_shard(state): - attestation = get_valid_attestation(state) + attestation = get_valid_attestation(state, signed=False) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.data.shard += 1 - # Re do signature sign_attestation(state, attestation) yield from run_attestation_processing(state, attestation, False) @@ -125,12 +123,11 @@ def test_wrong_shard(state): @spec_state_test def test_new_source_epoch(state): - attestation = get_valid_attestation(state) + attestation = get_valid_attestation(state, signed=False) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.data.source_epoch += 1 - # Re do signature sign_attestation(state, attestation) yield from run_attestation_processing(state, attestation, False) @@ -138,12 +135,11 @@ def test_new_source_epoch(state): @spec_state_test def test_source_root_is_target_root(state): - attestation = get_valid_attestation(state) + attestation = get_valid_attestation(state, signed=False) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.data.source_root = attestation.data.target_root - # Re do signature sign_attestation(state, attestation) yield from run_attestation_processing(state, attestation, False) @@ -160,7 +156,7 @@ def test_invalid_current_source_root(state): state.current_justified_epoch = 4 state.current_justified_root = b'\xff' * 32 - attestation = get_valid_attestation(state, slot=(spec.SLOTS_PER_EPOCH * 3) + 1) + attestation = get_valid_attestation(state, slot=(spec.SLOTS_PER_EPOCH * 3) + 1, signed=False) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY # Test logic sanity checks: @@ -170,7 +166,6 @@ def test_invalid_current_source_root(state): # Make attestation source root invalid: should be previous justified, not current one attestation.data.source_root = state.current_justified_root - # Re do signature sign_attestation(state, attestation) yield from run_attestation_processing(state, attestation, False) @@ -178,12 +173,11 @@ def test_invalid_current_source_root(state): @spec_state_test def test_bad_source_root(state): - attestation = get_valid_attestation(state) + attestation = get_valid_attestation(state, signed=False) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.data.source_root = b'\x42' * 32 - # Re do signature sign_attestation(state, attestation) yield from run_attestation_processing(state, attestation, False) @@ -191,12 +185,11 @@ def test_bad_source_root(state): @spec_state_test def test_non_zero_crosslink_data_root(state): - attestation = get_valid_attestation(state) + attestation = get_valid_attestation(state, signed=False) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.data.crosslink_data_root = b'\x42' * 32 - # Re do signature sign_attestation(state, attestation) yield from run_attestation_processing(state, attestation, False) @@ -207,7 +200,7 @@ def test_bad_previous_crosslink(state): next_epoch(state) apply_empty_block(state) - attestation = get_valid_attestation(state) + attestation = get_valid_attestation(state, signed=True) for _ in range(spec.MIN_ATTESTATION_INCLUSION_DELAY): next_slot(state) apply_empty_block(state) @@ -219,19 +212,23 @@ def test_bad_previous_crosslink(state): @spec_state_test def test_non_empty_custody_bitfield(state): - attestation = get_valid_attestation(state) + attestation = get_valid_attestation(state, signed=False) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.custody_bitfield = deepcopy(attestation.aggregation_bitfield) + + sign_attestation(state, attestation) yield from run_attestation_processing(state, attestation, False) @spec_state_test def test_empty_aggregation_bitfield(state): - attestation = get_valid_attestation(state) + attestation = get_valid_attestation(state, signed=False) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.aggregation_bitfield = b'\x00' * len(attestation.aggregation_bitfield) + sign_attestation(state, attestation) + yield from run_attestation_processing(state, attestation, False) diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index de5f2d521a..8b667f27d1 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -55,7 +55,7 @@ def build_attestation_data(state, slot, shard): ) -def get_valid_attestation(state, slot=None): +def get_valid_attestation(state, slot=None, signed=False): if slot is None: slot = state.slot @@ -78,7 +78,8 @@ def get_valid_attestation(state, slot=None): data=attestation_data, custody_bitfield=custody_bitfield, ) - sign_attestation(state, attestation) + if signed: + sign_attestation(state, attestation) return attestation From a9069cbf9e12b6154c90e5bfe524496bb84aacd3 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 15 May 2019 19:53:35 +0200 Subject: [PATCH 09/37] attester slashing signing --- .../test_process_attester_slashing.py | 14 ++++++++------ .../eth2spec/test/helpers/attester_slashings.py | 8 +++++--- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py index 56b3711a41..35e6b0f71f 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py @@ -74,13 +74,12 @@ def test_success_surround(state): apply_empty_block(state) state.current_justified_epoch += 1 - attester_slashing = get_valid_attester_slashing(state) + attester_slashing = get_valid_attester_slashing(state, signed_1=False, signed_2=True) # set attestion1 to surround attestation 2 attester_slashing.attestation_1.data.source_epoch = attester_slashing.attestation_2.data.source_epoch - 1 attester_slashing.attestation_1.data.target_epoch = attester_slashing.attestation_2.data.target_epoch + 1 - # correct the signature of attestation 1 sign_indexed_attestation(state, attester_slashing.attestation_1) yield from run_attester_slashing_processing(state, attester_slashing) @@ -88,25 +87,27 @@ def test_success_surround(state): @spec_state_test def test_same_data(state): - attester_slashing = get_valid_attester_slashing(state) + attester_slashing = get_valid_attester_slashing(state, signed_1=False, signed_2=True) attester_slashing.attestation_1.data = attester_slashing.attestation_2.data + sign_indexed_attestation(state, attester_slashing.attestation_1) yield from run_attester_slashing_processing(state, attester_slashing, False) @spec_state_test def test_no_double_or_surround(state): - attester_slashing = get_valid_attester_slashing(state) + attester_slashing = get_valid_attester_slashing(state, signed_1=False, signed_2=True) attester_slashing.attestation_1.data.target_epoch += 1 + sign_indexed_attestation(state, attester_slashing.attestation_1) yield from run_attester_slashing_processing(state, attester_slashing, False) @spec_state_test def test_participants_already_slashed(state): - attester_slashing = get_valid_attester_slashing(state) + attester_slashing = get_valid_attester_slashing(state, signed_1=True, signed_2=True) # set all indices to slashed attestation_1 = attester_slashing.attestation_1 @@ -119,10 +120,11 @@ def test_participants_already_slashed(state): @spec_state_test def test_custody_bit_0_and_1(state): - attester_slashing = get_valid_attester_slashing(state) + attester_slashing = get_valid_attester_slashing(state, signed_1=False, signed_2=True) attester_slashing.attestation_1.custody_bit_1_indices = ( attester_slashing.attestation_1.custody_bit_0_indices ) + sign_indexed_attestation(state, attester_slashing.attestation_1) yield from run_attester_slashing_processing(state, attester_slashing, False) diff --git a/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py b/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py index b34f6ff453..d19b41dfec 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py @@ -4,12 +4,14 @@ from eth2spec.test.helpers.attestations import get_valid_attestation, sign_attestation -def get_valid_attester_slashing(state): - attestation_1 = get_valid_attestation(state) +def get_valid_attester_slashing(state, signed_1=False, signed_2=False): + attestation_1 = get_valid_attestation(state, signed=signed_1) + attestation_2 = deepcopy(attestation_1) attestation_2.data.target_root = b'\x01' * 32 - sign_attestation(state, attestation_2) + if signed_2: + sign_attestation(state, attestation_2) return AttesterSlashing( attestation_1=convert_to_indexed(state, attestation_1), From b92a9d88573652d0a275651bb89802eb509acab9 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 15 May 2019 20:50:11 +0200 Subject: [PATCH 10/37] sign block headers --- .../test/block_processing/test_process_block_header.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py index 09c0c53885..9d4d6192e1 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py @@ -49,15 +49,16 @@ def test_success(state): @spec_state_test def test_invalid_slot(state): - block = build_empty_block_for_next_slot(state) + block = build_empty_block_for_next_slot(state, signed=False) block.slot = state.slot + 2 # invalid slot + sign_block(state, block) yield from run_block_header_processing(state, block, valid=False) @spec_state_test def test_invalid_previous_block_root(state): - block = build_empty_block_for_next_slot(state) + block = build_empty_block_for_next_slot(state, signed=False) block.previous_block_root = b'\12' * 32 # invalid prev root sign_block(state, block) From 242bb8c912a5acc836cf9f008c762aeb9879ac0d Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 15 May 2019 20:50:20 +0200 Subject: [PATCH 11/37] sign deposits --- .../test/block_processing/test_process_deposit.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py index 8f663c5c22..a796be8178 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py @@ -1,8 +1,9 @@ import eth2spec.phase0.spec as spec from eth2spec.phase0.spec import process_deposit from eth2spec.test.context import spec_state_test, expect_assertion_error -from eth2spec.test.helpers.deposits import prepare_state_and_deposit +from eth2spec.test.helpers.deposits import prepare_state_and_deposit, sign_deposit_data from eth2spec.test.helpers.state import get_balance +from eth2spec.test.helpers.keys import privkeys def run_deposit_processing(state, deposit, validator_index, valid=True): @@ -51,7 +52,7 @@ def test_success(state): # fresh deposit = next validator index = validator appended to registry validator_index = len(state.validator_registry) amount = spec.MAX_EFFECTIVE_BALANCE - deposit = prepare_state_and_deposit(state, validator_index, amount) + deposit = prepare_state_and_deposit(state, validator_index, amount, signed=True) yield from run_deposit_processing(state, deposit, validator_index) @@ -60,7 +61,7 @@ def test_success(state): def test_success_top_up(state): validator_index = 0 amount = spec.MAX_EFFECTIVE_BALANCE // 4 - deposit = prepare_state_and_deposit(state, validator_index, amount) + deposit = prepare_state_and_deposit(state, validator_index, amount, signed=True) yield from run_deposit_processing(state, deposit, validator_index) @@ -69,11 +70,13 @@ def test_success_top_up(state): def test_wrong_index(state): validator_index = len(state.validator_registry) amount = spec.MAX_EFFECTIVE_BALANCE - deposit = prepare_state_and_deposit(state, validator_index, amount) + deposit = prepare_state_and_deposit(state, validator_index, amount, signed=False) # mess up deposit_index deposit.index = state.deposit_index + 1 + sign_deposit_data(state, deposit.data, privkeys[validator_index]) + yield from run_deposit_processing(state, deposit, validator_index, valid=False) @@ -84,9 +87,11 @@ def test_wrong_index(state): def test_bad_merkle_proof(state): validator_index = len(state.validator_registry) amount = spec.MAX_EFFECTIVE_BALANCE - deposit = prepare_state_and_deposit(state, validator_index, amount) + deposit = prepare_state_and_deposit(state, validator_index, amount, signed=False) # mess up merkle branch deposit.proof[-1] = spec.ZERO_HASH + sign_deposit_data(state, deposit.data, privkeys[validator_index]) + yield from run_deposit_processing(state, deposit, validator_index, valid=False) From 32efe49b3358ecfbf234c71764e80ee1b069cb8c Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 15 May 2019 21:07:24 +0200 Subject: [PATCH 12/37] proposer slashing signing --- .../test_process_proposer_slashing.py | 17 +++++++++------- .../test/helpers/proposer_slashings.py | 20 +++++++++++-------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py index 6efed86850..65d748ab5f 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py @@ -4,6 +4,8 @@ process_proposer_slashing, ) from eth2spec.test.context import spec_state_test, expect_assertion_error +from eth2spec.test.helpers.block_header import sign_block_header +from eth2spec.test.helpers.keys import privkeys from eth2spec.test.helpers.proposer_slashings import get_valid_proposer_slashing from eth2spec.test.helpers.state import get_balance @@ -45,14 +47,14 @@ def run_proposer_slashing_processing(state, proposer_slashing, valid=True): @spec_state_test def test_success(state): - proposer_slashing = get_valid_proposer_slashing(state) + proposer_slashing = get_valid_proposer_slashing(state, signed_1=True, signed_2=True) yield from run_proposer_slashing_processing(state, proposer_slashing) @spec_state_test def test_invalid_proposer_index(state): - proposer_slashing = get_valid_proposer_slashing(state) + proposer_slashing = get_valid_proposer_slashing(state, signed_1=True, signed_2=True) # Index just too high (by 1) proposer_slashing.proposer_index = len(state.validator_registry) @@ -61,17 +63,18 @@ def test_invalid_proposer_index(state): @spec_state_test def test_epochs_are_different(state): - proposer_slashing = get_valid_proposer_slashing(state) + proposer_slashing = get_valid_proposer_slashing(state, signed_1=True, signed_2=False) # set slots to be in different epochs proposer_slashing.header_2.slot += spec.SLOTS_PER_EPOCH + sign_block_header(state, proposer_slashing.header_2, privkeys[proposer_slashing.validator_index]) yield from run_proposer_slashing_processing(state, proposer_slashing, False) @spec_state_test def test_headers_are_same(state): - proposer_slashing = get_valid_proposer_slashing(state) + proposer_slashing = get_valid_proposer_slashing(state, signed_1=True, signed_2=False) # set headers to be the same proposer_slashing.header_2 = proposer_slashing.header_1 @@ -81,7 +84,7 @@ def test_headers_are_same(state): @spec_state_test def test_proposer_is_not_activated(state): - proposer_slashing = get_valid_proposer_slashing(state) + proposer_slashing = get_valid_proposer_slashing(state, signed_1=True, signed_2=True) # set proposer to be not active yet state.validator_registry[proposer_slashing.proposer_index].activation_epoch = get_current_epoch(state) + 1 @@ -91,7 +94,7 @@ def test_proposer_is_not_activated(state): @spec_state_test def test_proposer_is_slashed(state): - proposer_slashing = get_valid_proposer_slashing(state) + proposer_slashing = get_valid_proposer_slashing(state, signed_1=True, signed_2=True) # set proposer to slashed state.validator_registry[proposer_slashing.proposer_index].slashed = True @@ -101,7 +104,7 @@ def test_proposer_is_slashed(state): @spec_state_test def test_proposer_is_withdrawn(state): - proposer_slashing = get_valid_proposer_slashing(state) + proposer_slashing = get_valid_proposer_slashing(state, signed_1=True, signed_2=True) # set proposer withdrawable_epoch in past current_epoch = get_current_epoch(state) diff --git a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py index 0dd1ce5c12..dfb8895dc2 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py +++ b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py @@ -1,11 +1,13 @@ from copy import deepcopy -from eth2spec.phase0.spec import get_current_epoch, get_active_validator_indices, BeaconBlockHeader, ZERO_HASH, ProposerSlashing +from eth2spec.phase0.spec import ( + get_current_epoch, get_active_validator_indices, BeaconBlockHeader, ProposerSlashing +) from eth2spec.test.helpers.block_header import sign_block_header from eth2spec.test.helpers.keys import pubkey_to_privkey -def get_valid_proposer_slashing(state): +def get_valid_proposer_slashing(state, signed_1=False, signed_2=False): current_epoch = get_current_epoch(state) validator_index = get_active_validator_indices(state, current_epoch)[-1] privkey = pubkey_to_privkey[state.validator_registry[validator_index].pubkey] @@ -13,16 +15,18 @@ def get_valid_proposer_slashing(state): header_1 = BeaconBlockHeader( slot=slot, - previous_block_root=ZERO_HASH, - state_root=ZERO_HASH, - block_body_root=ZERO_HASH, + previous_block_root=b'\x33' * 32, + state_root=b'\x44' * 32, + block_body_root=b'\x55' * 32, ) header_2 = deepcopy(header_1) - header_2.previous_block_root = b'\x02' * 32 + header_2.previous_block_root = b'\x99' * 32 header_2.slot = slot + 1 - sign_block_header(state, header_1, privkey) - sign_block_header(state, header_2, privkey) + if signed_1: + sign_block_header(state, header_1, privkey) + if signed_2: + sign_block_header(state, header_2, privkey) return ProposerSlashing( proposer_index=validator_index, From 9f00e4f0e464eddd09b06aabb2ade470e0e57e39 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 15 May 2019 21:07:36 +0200 Subject: [PATCH 13/37] sign transfers --- .../block_processing/test_process_transfer.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py index fed948e636..dc63bf491e 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py @@ -44,7 +44,7 @@ def run_transfer_processing(state, transfer, valid=True): @spec_state_test def test_success_non_activated(state): - transfer = get_valid_transfer(state) + transfer = get_valid_transfer(state, signed=True) # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH @@ -56,7 +56,7 @@ def test_success_withdrawable(state): next_epoch(state) apply_empty_block(state) - transfer = get_valid_transfer(state) + transfer = get_valid_transfer(state, signed=True) # withdrawable_epoch in past so can transfer state.validator_registry[transfer.sender].withdrawable_epoch = get_current_epoch(state) - 1 @@ -68,7 +68,7 @@ def test_success_withdrawable(state): def test_success_active_above_max_effective(state): sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + 1 - transfer = get_valid_transfer(state, sender_index=sender_index, amount=1, fee=0) + transfer = get_valid_transfer(state, sender_index=sender_index, amount=1, fee=0, signed=True) yield from run_transfer_processing(state, transfer) @@ -77,7 +77,7 @@ def test_success_active_above_max_effective(state): def test_success_active_above_max_effective_fee(state): sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + 1 - transfer = get_valid_transfer(state, sender_index=sender_index, amount=0, fee=1) + transfer = get_valid_transfer(state, sender_index=sender_index, amount=0, fee=1, signed=True) yield from run_transfer_processing(state, transfer) @@ -87,14 +87,14 @@ def test_active_but_transfer_past_effective_balance(state): sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] amount = spec.MAX_EFFECTIVE_BALANCE // 32 state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE - transfer = get_valid_transfer(state, sender_index=sender_index, amount=amount, fee=0) + transfer = get_valid_transfer(state, sender_index=sender_index, amount=amount, fee=0, signed=True) yield from run_transfer_processing(state, transfer, False) @spec_state_test def test_incorrect_slot(state): - transfer = get_valid_transfer(state, slot=state.slot + 1) + transfer = get_valid_transfer(state, slot=state.slot + 1, signed=True) # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH @@ -105,7 +105,7 @@ def test_incorrect_slot(state): def test_insufficient_balance_for_fee(state): sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE - transfer = get_valid_transfer(state, sender_index=sender_index, amount=0, fee=1) + transfer = get_valid_transfer(state, sender_index=sender_index, amount=0, fee=1, signed=True) # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH @@ -117,7 +117,7 @@ def test_insufficient_balance_for_fee(state): def test_insufficient_balance(state): sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE - transfer = get_valid_transfer(state, sender_index=sender_index, amount=1, fee=0) + transfer = get_valid_transfer(state, sender_index=sender_index, amount=1, fee=0, signed=True) # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH @@ -129,7 +129,7 @@ def test_insufficient_balance(state): def test_no_dust_sender(state): sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] balance = state.balances[sender_index] - transfer = get_valid_transfer(state, sender_index=sender_index, amount=balance - spec.MIN_DEPOSIT_AMOUNT + 1, fee=0) + transfer = get_valid_transfer(state, sender_index=sender_index, amount=balance - spec.MIN_DEPOSIT_AMOUNT + 1, fee=0, signed=True) # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH @@ -141,7 +141,7 @@ def test_no_dust_sender(state): def test_no_dust_recipient(state): sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + 1 - transfer = get_valid_transfer(state, sender_index=sender_index, amount=1, fee=0) + transfer = get_valid_transfer(state, sender_index=sender_index, amount=1, fee=0, signed=True) state.balances[transfer.recipient] = 0 # un-activate so validator can transfer @@ -152,7 +152,7 @@ def test_no_dust_recipient(state): @spec_state_test def test_invalid_pubkey(state): - transfer = get_valid_transfer(state) + transfer = get_valid_transfer(state, signed=True) state.validator_registry[transfer.sender].withdrawal_credentials = spec.ZERO_HASH # un-activate so validator can transfer From eea6c8f6456c9db7079ad4d90041a14ced5fd60b Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 15 May 2019 23:26:05 +0200 Subject: [PATCH 14/37] voluntary exit testing sigs --- .../block_processing/test_process_voluntary_exit.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py index f842d8b078..e5be38722f 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py @@ -7,7 +7,7 @@ ) from eth2spec.test.context import spec_state_test, expect_assertion_error from eth2spec.test.helpers.keys import pubkey_to_privkey -from eth2spec.test.helpers.voluntary_exits import build_voluntary_exit +from eth2spec.test.helpers.voluntary_exits import build_voluntary_exit, sign_voluntary_exit def run_voluntary_exit_processing(state, voluntary_exit, valid=True): @@ -52,6 +52,7 @@ def test_success(state): current_epoch, validator_index, privkey, + signed=True, ) yield from run_voluntary_exit_processing(state, voluntary_exit) @@ -76,6 +77,7 @@ def test_success_exit_queue(state): current_epoch, index, privkey, + signed=True, )) # Now run all the exits @@ -92,6 +94,7 @@ def test_success_exit_queue(state): current_epoch, validator_index, privkey, + signed=True, ) # This is the interesting part of the test: on a pre-state with a full exit queue, @@ -118,8 +121,10 @@ def test_validator_exit_in_future(state): current_epoch, validator_index, privkey, + signed=False, ) voluntary_exit.epoch += 1 + sign_voluntary_exit(state, voluntary_exit, privkey) yield from run_voluntary_exit_processing(state, voluntary_exit, False) @@ -138,8 +143,10 @@ def test_validator_invalid_validator_index(state): current_epoch, validator_index, privkey, + signed=False, ) voluntary_exit.validator_index = len(state.validator_registry) + sign_voluntary_exit(state, voluntary_exit, privkey) yield from run_voluntary_exit_processing(state, voluntary_exit, False) @@ -158,6 +165,7 @@ def test_validator_not_active(state): current_epoch, validator_index, privkey, + signed=True, ) yield from run_voluntary_exit_processing(state, voluntary_exit, False) @@ -180,6 +188,7 @@ def test_validator_already_exited(state): current_epoch, validator_index, privkey, + signed=True, ) yield from run_voluntary_exit_processing(state, voluntary_exit, False) @@ -196,6 +205,7 @@ def test_validator_not_active_long_enough(state): current_epoch, validator_index, privkey, + signed=True, ) assert ( From 62999d8ded3e52bf7c3f60a0e30cec3218faa73f Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 17 May 2019 20:31:21 +0200 Subject: [PATCH 15/37] import fixes + fix import loop + fix minor signing things --- .../block_processing/test_process_attestation.py | 2 +- .../test_process_attester_slashing.py | 4 ++-- .../block_processing/test_process_block_header.py | 3 +-- .../test_process_proposer_slashing.py | 2 +- .../test/block_processing/test_process_transfer.py | 3 ++- test_libs/pyspec/eth2spec/test/context.py | 2 +- test_libs/pyspec/eth2spec/test/helpers/block.py | 11 +++++++++++ test_libs/pyspec/eth2spec/test/helpers/deposits.py | 2 +- test_libs/pyspec/eth2spec/test/helpers/genesis.py | 2 +- test_libs/pyspec/eth2spec/test/helpers/state.py | 13 +------------ 10 files changed, 22 insertions(+), 22 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py index c44e8e7b17..2c88f12b5e 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py @@ -14,10 +14,10 @@ sign_attestation, ) from eth2spec.test.helpers.state import ( - apply_empty_block, next_epoch, next_slot, ) +from eth2spec.test.helpers.block import apply_empty_block def run_attestation_processing(state, attestation, valid=True): diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py index 35e6b0f71f..ea3a31aea3 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py @@ -6,10 +6,10 @@ from eth2spec.test.context import spec_state_test, expect_assertion_error from eth2spec.test.helpers.attestations import sign_indexed_attestation from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing +from eth2spec.test.helpers.block import apply_empty_block from eth2spec.test.helpers.state import ( get_balance, next_epoch, - apply_empty_block, ) @@ -63,7 +63,7 @@ def run_attester_slashing_processing(state, attester_slashing, valid=True): @spec_state_test def test_success_double(state): - attester_slashing = get_valid_attester_slashing(state) + attester_slashing = get_valid_attester_slashing(state, signed_1=True, signed_2=True) yield from run_attester_slashing_processing(state, attester_slashing) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py index 9d4d6192e1..69bacc84f5 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py @@ -49,9 +49,8 @@ def test_success(state): @spec_state_test def test_invalid_slot(state): - block = build_empty_block_for_next_slot(state, signed=False) + block = build_empty_block_for_next_slot(state, signed=True) block.slot = state.slot + 2 # invalid slot - sign_block(state, block) yield from run_block_header_processing(state, block, valid=False) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py index 65d748ab5f..f3d1593a81 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py @@ -67,7 +67,7 @@ def test_epochs_are_different(state): # set slots to be in different epochs proposer_slashing.header_2.slot += spec.SLOTS_PER_EPOCH - sign_block_header(state, proposer_slashing.header_2, privkeys[proposer_slashing.validator_index]) + sign_block_header(state, proposer_slashing.header_2, privkeys[proposer_slashing.proposer_index]) yield from run_proposer_slashing_processing(state, proposer_slashing, False) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py index dc63bf491e..2bbc022d00 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py @@ -6,7 +6,8 @@ process_transfer, ) from eth2spec.test.context import spec_state_test, expect_assertion_error -from eth2spec.test.helpers.state import next_epoch, apply_empty_block +from eth2spec.test.helpers.state import next_epoch +from eth2spec.test.helpers.block import apply_empty_block from eth2spec.test.helpers.transfers import get_valid_transfer diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index ce088f81c8..b2e67d7614 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -1,7 +1,7 @@ from eth2spec.phase0 import spec from eth2spec.utils import bls -from .helpers import create_genesis_state +from .helpers.genesis import create_genesis_state from .utils import spectest, with_args diff --git a/test_libs/pyspec/eth2spec/test/helpers/block.py b/test_libs/pyspec/eth2spec/test/helpers/block.py index 36659f8fa4..b9c4ca712b 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block.py @@ -2,6 +2,7 @@ from eth2spec.phase0 import spec from eth2spec.phase0.spec import get_beacon_proposer_index, slot_to_epoch, get_domain, BeaconBlock +from eth2spec.phase0.state_transition import state_transition from eth2spec.test.helpers.keys import privkeys from eth2spec.test.helpers.state import next_slot from eth2spec.utils.bls import bls_sign @@ -39,6 +40,16 @@ def sign_block(state, block): return block +def apply_empty_block(state): + """ + Transition via an empty block (on current slot, assuming no block has been applied yet). + :return: the empty block that triggered the transition. + """ + block = build_empty_block(state, signed=True) + state_transition(state, block) + return block + + def build_empty_block(state, slot=None, signed=False): if slot is None: slot = state.slot diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index 1bea2f923b..c5deb124e6 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -12,7 +12,7 @@ def build_deposit_data(state, pubkey, privkey, amount, signed=False): deposit_data = DepositData( pubkey=pubkey, # insecurely use pubkey as withdrawal key as well - withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:], + withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(pubkey)[1:], amount=amount, ) if signed: diff --git a/test_libs/pyspec/eth2spec/test/helpers/genesis.py b/test_libs/pyspec/eth2spec/test/helpers/genesis.py index b991f74e71..01011cacd0 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/genesis.py +++ b/test_libs/pyspec/eth2spec/test/helpers/genesis.py @@ -9,7 +9,7 @@ def build_mock_validator(i: int, balance: int): pubkey = pubkeys[i] # insecurely use pubkey as withdrawal key as well - withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:] + withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(pubkey)[1:] return spec.Validator( pubkey=pubkeys[i], withdrawal_credentials=withdrawal_credentials, diff --git a/test_libs/pyspec/eth2spec/test/helpers/state.py b/test_libs/pyspec/eth2spec/test/helpers/state.py index 0261801df7..e720a9709f 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/state.py +++ b/test_libs/pyspec/eth2spec/test/helpers/state.py @@ -1,8 +1,7 @@ # Access constants from spec pkg reference. import eth2spec.phase0.spec as spec -from eth2spec.phase0.state_transition import state_transition_to, state_transition -from eth2spec.test.helpers.block import build_empty_block +from eth2spec.phase0.state_transition import state_transition_to def get_balance(state, index): @@ -24,16 +23,6 @@ def next_epoch(state): state_transition_to(state, slot) -def apply_empty_block(state): - """ - Transition via an empty block (on current slot, assuming no block has been applied yet). - :return: the empty block that triggered the transition. - """ - block = build_empty_block(state) - state_transition(state, block) - return block - - def get_state_root(state, slot) -> bytes: """ Return the state root at a recent ``slot``. From f937dec2a11551c4558b8dd170fb3c786b10a1b1 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 17 May 2019 20:54:34 +0200 Subject: [PATCH 16/37] more keys, more validators, fix import --- test_libs/pyspec/eth2spec/test/context.py | 2 +- test_libs/pyspec/eth2spec/test/helpers/keys.py | 3 ++- test_libs/pyspec/eth2spec/test/test_finality.py | 7 ++----- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index b2e67d7614..d3d9be5885 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -6,7 +6,7 @@ from .utils import spectest, with_args # Provides a genesis state as first argument to the function decorated with this -with_state = with_args(lambda: [create_genesis_state(spec.SLOTS_PER_EPOCH * 8)]) +with_state = with_args(lambda: [create_genesis_state(spec.SHARD_COUNT * 2)]) # shorthand for decorating @with_state @spectest() diff --git a/test_libs/pyspec/eth2spec/test/helpers/keys.py b/test_libs/pyspec/eth2spec/test/helpers/keys.py index fe27b78a1e..f59fbb1590 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/keys.py +++ b/test_libs/pyspec/eth2spec/test/helpers/keys.py @@ -1,5 +1,6 @@ from py_ecc import bls +from eth2spec.phase0 import spec -privkeys = [i + 1 for i in range(1024)] +privkeys = [i + 1 for i in range(spec.SHARD_COUNT * 8)] pubkeys = [bls.privtopub(privkey) for privkey in privkeys] pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys, pubkeys)} diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index 732dd8f0c4..f47ee5f77f 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -5,11 +5,8 @@ state_transition, ) from .context import spec_state_test -from .helpers.state import ( - next_epoch, - apply_empty_block -) -from .helpers.block import build_empty_block_for_next_slot +from .helpers.state import next_epoch +from .helpers.block import build_empty_block_for_next_slot, apply_empty_block from .helpers.attestations import ( fill_aggregate_attestation, get_current_epoch, From 183e3a515330fe7deb52c87993655c08f0bb19cb Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 17 May 2019 21:02:00 +0200 Subject: [PATCH 17/37] minor refactor import fix --- .../eth2spec/test/epoch_processing/test_process_crosslinks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py index f342c5e6d7..5cada0aa51 100644 --- a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py @@ -12,9 +12,9 @@ from eth2spec.test.context import spec_state_test from eth2spec.test.helpers.state import ( next_epoch, - next_slot, - apply_empty_block + next_slot ) +from eth2spec.test.helpers.block import apply_empty_block from eth2spec.test.helpers.attestations import ( add_attestation_to_state, build_empty_block_for_next_slot, From 08d0ff9336df627ec67bafc3bee538079b15be51 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 17 May 2019 21:20:09 +0200 Subject: [PATCH 18/37] fix attestation aggregate bitfields --- test_libs/pyspec/eth2spec/test/helpers/attestations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index 8b667f27d1..9e00489a7b 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -71,7 +71,7 @@ def get_valid_attestation(state, slot=None, signed=False): committee_size = len(crosslink_committee) bitfield_length = (committee_size + 7) // 8 - aggregation_bitfield = b'\xC0' + b'\x00' * (bitfield_length - 1) + aggregation_bitfield = b'\x00' * bitfield_length custody_bitfield = b'\x00' * bitfield_length attestation = Attestation( aggregation_bitfield=aggregation_bitfield, From 0f00b436986dccbd17b19270ab62be58bf47974f Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 17 May 2019 21:26:18 +0200 Subject: [PATCH 19/37] reduce validator key count again, fix valid attestation creation - snippet from Danny --- test_libs/pyspec/eth2spec/test/context.py | 2 +- .../pyspec/eth2spec/test/helpers/attestations.py | 14 +++++++------- test_libs/pyspec/eth2spec/test/helpers/keys.py | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index d3d9be5885..b2e67d7614 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -6,7 +6,7 @@ from .utils import spectest, with_args # Provides a genesis state as first argument to the function decorated with this -with_state = with_args(lambda: [create_genesis_state(spec.SHARD_COUNT * 2)]) +with_state = with_args(lambda: [create_genesis_state(spec.SLOTS_PER_EPOCH * 8)]) # shorthand for decorating @with_state @spectest() diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index 9e00489a7b..7164cf160f 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -6,8 +6,9 @@ Attestation, AttestationData, AttestationDataAndCustodyBit, - get_epoch_start_slot, get_block_root, get_current_epoch, get_previous_epoch, slot_to_epoch, get_shard_delta, - get_crosslink_committee, get_domain, IndexedAttestation, get_attesting_indices, BeaconState, get_block_root_at_slot) + get_epoch_start_slot, get_block_root, get_current_epoch, get_previous_epoch, slot_to_epoch, + get_crosslink_committee, get_domain, IndexedAttestation, get_attesting_indices, BeaconState, get_block_root_at_slot, + get_epoch_start_shard, get_epoch_committee_count) from eth2spec.phase0.state_transition import ( state_transition, state_transition_to ) @@ -59,11 +60,10 @@ def get_valid_attestation(state, slot=None, signed=False): if slot is None: slot = state.slot - if slot_to_epoch(slot) == get_current_epoch(state): - shard = (state.latest_start_shard + slot) % spec.SLOTS_PER_EPOCH - else: - previous_shard_delta = get_shard_delta(state, get_previous_epoch(state)) - shard = (state.latest_start_shard - previous_shard_delta + slot) % spec.SHARD_COUNT + epoch = slot_to_epoch(slot) + epoch_start_shard = get_epoch_start_shard(state, epoch) + committees_per_slot = get_epoch_committee_count(state, epoch) // spec.SLOTS_PER_EPOCH + shard = (epoch_start_shard + committees_per_slot * (slot % spec.SLOTS_PER_EPOCH)) % spec.SHARD_COUNT attestation_data = build_attestation_data(state, slot, shard) diff --git a/test_libs/pyspec/eth2spec/test/helpers/keys.py b/test_libs/pyspec/eth2spec/test/helpers/keys.py index f59fbb1590..f47cd7c10b 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/keys.py +++ b/test_libs/pyspec/eth2spec/test/helpers/keys.py @@ -1,6 +1,6 @@ from py_ecc import bls from eth2spec.phase0 import spec -privkeys = [i + 1 for i in range(spec.SHARD_COUNT * 8)] +privkeys = [i + 1 for i in range(spec.SLOTS_PER_EPOCH * 16)] pubkeys = [bls.privtopub(privkey) for privkey in privkeys] pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys, pubkeys)} From a10aba4bb9009e6c82f59491e0eb057f48c40d25 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 17 May 2019 21:36:17 +0200 Subject: [PATCH 20/37] make valid attest creation fill aggregate bitfield --- test_libs/pyspec/eth2spec/test/helpers/attestations.py | 1 + test_libs/pyspec/eth2spec/test/test_finality.py | 2 -- test_libs/pyspec/eth2spec/test/test_sanity.py | 3 ++- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index 7164cf160f..b541e610f4 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -78,6 +78,7 @@ def get_valid_attestation(state, slot=None, signed=False): data=attestation_data, custody_bitfield=custody_bitfield, ) + fill_aggregate_attestation(state, attestation) if signed: sign_attestation(state, attestation) return attestation diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index f47ee5f77f..26900340da 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -53,13 +53,11 @@ def next_epoch_with_attestations(state, slot_to_attest = post_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 if slot_to_attest >= get_epoch_start_slot(get_current_epoch(post_state)): cur_attestation = get_valid_attestation(post_state, slot_to_attest) - fill_aggregate_attestation(post_state, cur_attestation) block.body.attestations.append(cur_attestation) if fill_prev_epoch: slot_to_attest = post_state.slot - spec.SLOTS_PER_EPOCH + 1 prev_attestation = get_valid_attestation(post_state, slot_to_attest) - fill_aggregate_attestation(post_state, prev_attestation) block.body.attestations.append(prev_attestation) state_transition(post_state, block) diff --git a/test_libs/pyspec/eth2spec/test/test_sanity.py b/test_libs/pyspec/eth2spec/test/test_sanity.py index 1509a9843a..5e3b324166 100644 --- a/test_libs/pyspec/eth2spec/test/test_sanity.py +++ b/test_libs/pyspec/eth2spec/test/test_sanity.py @@ -157,7 +157,8 @@ def test_attester_slashing(state): pre_state = deepcopy(state) attester_slashing = get_valid_attester_slashing(state) - validator_index = attester_slashing.attestation_1.custody_bit_0_indices[0] + validator_index = (attester_slashing.attestation_1.custody_bit_0_indices + + attester_slashing.attestation_1.custody_bit_1_indices)[0] assert not state.validator_registry[validator_index].slashed From 90bcbd6ff49cf1b836105e2e1e5345d054699dbc Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 17 May 2019 21:56:41 +0200 Subject: [PATCH 21/37] fix attester slashing test --- .../test_process_attester_slashing.py | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py index ea3a31aea3..01861f04ca 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py @@ -46,17 +46,22 @@ def run_attester_slashing_processing(state, attester_slashing, valid=True): assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH - # lost whistleblower reward - assert ( - get_balance(state, slashed_index) < - pre_slashed_balance - ) - - # gained whistleblower reward - assert ( - get_balance(state, proposer_index) > - pre_proposer_balance - ) + if slashed_index != proposer_index: + # lost whistleblower reward + assert ( + get_balance(state, slashed_index) < + pre_slashed_balance + ) + + # gained whistleblower reward + assert ( + get_balance(state, proposer_index) > + pre_proposer_balance + ) + else: + # gained rewards for all slashings, which may include others. And only lost that of themselves. + # Netto at least 0, if more people where slashed, a balance increase. + assert get_balance(state, slashed_index) >= pre_slashed_balance yield 'post', state From b0aea2a111d8be31b992863f0ca0261e5a1ce56f Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 20 May 2019 19:38:18 +0200 Subject: [PATCH 22/37] bugfix block proc transfer test --- test_libs/pyspec/eth2spec/test/context.py | 11 +++++++++++ .../pyspec/eth2spec/test/helpers/transfers.py | 3 +-- test_libs/pyspec/eth2spec/test/test_finality.py | 7 +++++-- test_libs/pyspec/eth2spec/test/test_sanity.py | 14 ++++++++------ 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index b2e67d7614..a9eaea349a 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -28,6 +28,17 @@ def expect_assertion_error(fn): raise AssertionError('expected an assertion error, but got none.') +def never_bls(fn): + """ + Decorator to apply on ``bls_switch`` decorator to force BLS de-activation. Useful to mark tests as BLS-ignorant. + """ + def entry(*args, **kw): + # override bls setting + kw['bls_active'] = False + fn(*args, **kw) + return entry + + def always_bls(fn): """ Decorator to apply on ``bls_switch`` decorator to force BLS activation. Useful to mark tests as BLS-dependent. diff --git a/test_libs/pyspec/eth2spec/test/helpers/transfers.py b/test_libs/pyspec/eth2spec/test/helpers/transfers.py index 37547bf97b..2045f48ad6 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/transfers.py +++ b/test_libs/pyspec/eth2spec/test/helpers/transfers.py @@ -1,7 +1,7 @@ # Access constants from spec pkg reference. import eth2spec.phase0.spec as spec -from eth2spec.phase0.spec import get_current_epoch, get_active_validator_indices, Transfer, ZERO_HASH, get_domain +from eth2spec.phase0.spec import get_current_epoch, get_active_validator_indices, Transfer, get_domain from eth2spec.test.helpers.keys import pubkeys, privkeys from eth2spec.test.helpers.state import get_balance from eth2spec.utils.bls import bls_sign @@ -30,7 +30,6 @@ def get_valid_transfer(state, slot=None, sender_index=None, amount=None, fee=Non fee=fee, slot=slot, pubkey=transfer_pubkey, - signature=ZERO_HASH, ) if signed: sign_transfer(state, transfer, transfer_privkey) diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index 26900340da..56f65eca9a 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -4,11 +4,10 @@ from eth2spec.phase0.state_transition import ( state_transition, ) -from .context import spec_state_test +from .context import spec_state_test, never_bls from .helpers.state import next_epoch from .helpers.block import build_empty_block_for_next_slot, apply_empty_block from .helpers.attestations import ( - fill_aggregate_attestation, get_current_epoch, get_epoch_start_slot, get_valid_attestation, @@ -66,6 +65,7 @@ def next_epoch_with_attestations(state, return state, blocks, post_state +@never_bls @spec_state_test def test_finality_rule_4(state): yield 'pre', state @@ -93,6 +93,7 @@ def test_finality_rule_4(state): yield 'post', state +@never_bls @spec_state_test def test_finality_rule_1(state): # get past first two epochs that finality does not run on @@ -122,6 +123,7 @@ def test_finality_rule_1(state): yield 'post', state +@never_bls @spec_state_test def test_finality_rule_2(state): # get past first two epochs that finality does not run on @@ -153,6 +155,7 @@ def test_finality_rule_2(state): yield 'post', state +@never_bls @spec_state_test def test_finality_rule_3(state): """ diff --git a/test_libs/pyspec/eth2spec/test/test_sanity.py b/test_libs/pyspec/eth2spec/test/test_sanity.py index 5e3b324166..27b40f93be 100644 --- a/test_libs/pyspec/eth2spec/test/test_sanity.py +++ b/test_libs/pyspec/eth2spec/test/test_sanity.py @@ -1,7 +1,7 @@ from copy import deepcopy -from py_ecc import bls import eth2spec.phase0.spec as spec +from eth2spec.utils.bls import bls_sign from eth2spec.utils.minimal_ssz import signing_root from eth2spec.phase0.spec import ( @@ -24,7 +24,7 @@ get_state_root ) from .helpers.transfers import get_valid_transfer -from .helpers.block import build_empty_block_for_next_slot +from .helpers.block import build_empty_block_for_next_slot, sign_block from .helpers.keys import ( privkeys, pubkeys, @@ -58,7 +58,7 @@ def test_empty_block_transition(state): yield 'pre', state - block = build_empty_block_for_next_slot(state) + block = build_empty_block_for_next_slot(state, signed=True) yield 'blocks', [block], [spec.BeaconBlock] state_transition(state, block) @@ -73,8 +73,9 @@ def test_skipped_slots(state): pre_slot = state.slot yield 'pre', state - block = build_empty_block_for_next_slot(state) + block = build_empty_block_for_next_slot(state, signed=False) block.slot += 3 + sign_block(state, block) yield 'blocks', [block], [spec.BeaconBlock] state_transition(state, block) @@ -90,8 +91,9 @@ def test_empty_epoch_transition(state): pre_slot = state.slot yield 'pre', state - block = build_empty_block_for_next_slot(state) + block = build_empty_block_for_next_slot(state, signed=False) block.slot += spec.SLOTS_PER_EPOCH + sign_block(state, block) yield 'blocks', [block], [spec.BeaconBlock] state_transition(state, block) @@ -289,7 +291,7 @@ def test_voluntary_exit(state): epoch=get_current_epoch(state), validator_index=validator_index, ) - voluntary_exit.signature = bls.sign( + voluntary_exit.signature = bls_sign( message_hash=signing_root(voluntary_exit), privkey=privkeys[validator_index], domain=get_domain( From ab251d47970ce7fed5ae3d6fe3551708c244ed86 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 20 May 2019 20:44:40 +0200 Subject: [PATCH 23/37] make tests work fast + bls signed optionally --- .../test_process_block_header.py | 4 +- .../test_process_crosslinks.py | 13 +- .../eth2spec/test/helpers/attestations.py | 2 +- .../pyspec/eth2spec/test/helpers/block.py | 26 ++-- test_libs/pyspec/eth2spec/test/test_sanity.py | 132 ++++++++++-------- 5 files changed, 97 insertions(+), 80 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py index 69bacc84f5..3faf51e1bd 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py @@ -42,13 +42,13 @@ def run_block_header_processing(state, block, valid=True): @spec_state_test -def test_success(state): +def test_success_block_header(state): block = build_empty_block_for_next_slot(state, signed=True) yield from run_block_header_processing(state, block) @spec_state_test -def test_invalid_slot(state): +def test_invalid_slot_block_header(state): block = build_empty_block_for_next_slot(state, signed=True) block.slot = state.slot + 2 # invalid slot diff --git a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py index 5cada0aa51..688bb54ac8 100644 --- a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py @@ -14,7 +14,7 @@ next_epoch, next_slot ) -from eth2spec.test.helpers.block import apply_empty_block +from eth2spec.test.helpers.block import apply_empty_block, sign_block from eth2spec.test.helpers.attestations import ( add_attestation_to_state, build_empty_block_for_next_slot, @@ -33,8 +33,9 @@ def run_process_crosslinks(state, valid=True): """ # transition state to slot before state transition slot = state.slot + (spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH) - 1 - block = build_empty_block_for_next_slot(state) + block = build_empty_block_for_next_slot(state, signed=False) block.slot = slot + sign_block(state, block) state_transition(state, block) # cache state before epoch transition @@ -57,7 +58,7 @@ def test_no_attestations(state): def test_single_crosslink_update_from_current_epoch(state): next_epoch(state) - attestation = get_valid_attestation(state) + attestation = get_valid_attestation(state, signed=True) fill_aggregate_attestation(state, attestation) add_attestation_to_state(state, attestation, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) @@ -77,7 +78,7 @@ def test_single_crosslink_update_from_current_epoch(state): def test_single_crosslink_update_from_previous_epoch(state): next_epoch(state) - attestation = get_valid_attestation(state) + attestation = get_valid_attestation(state, signed=True) fill_aggregate_attestation(state, attestation) add_attestation_to_state(state, attestation, state.slot + spec.SLOTS_PER_EPOCH) @@ -105,7 +106,7 @@ def test_double_late_crosslink(state): next_epoch(state) state.slot += 4 - attestation_1 = get_valid_attestation(state) + attestation_1 = get_valid_attestation(state, signed=True) fill_aggregate_attestation(state, attestation_1) # add attestation_1 in the next epoch @@ -113,7 +114,7 @@ def test_double_late_crosslink(state): add_attestation_to_state(state, attestation_1, state.slot + 1) for slot in range(spec.SLOTS_PER_EPOCH): - attestation_2 = get_valid_attestation(state) + attestation_2 = get_valid_attestation(state, signed=True) if attestation_2.data.shard == attestation_1.data.shard: break next_slot(state) diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index b541e610f4..e9b863463c 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -138,7 +138,7 @@ def fill_aggregate_attestation(state, attestation): def add_attestation_to_state(state, attestation, slot): - block = build_empty_block_for_next_slot(state) + block = build_empty_block_for_next_slot(state, signed=False) block.slot = slot block.body.attestations.append(attestation) state_transition_to(state, block.slot) diff --git a/test_libs/pyspec/eth2spec/test/helpers/block.py b/test_libs/pyspec/eth2spec/test/helpers/block.py index b9c4ca712b..6f1a0fe602 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block.py @@ -2,22 +2,26 @@ from eth2spec.phase0 import spec from eth2spec.phase0.spec import get_beacon_proposer_index, slot_to_epoch, get_domain, BeaconBlock -from eth2spec.phase0.state_transition import state_transition +from eth2spec.phase0.state_transition import state_transition, state_transition_to from eth2spec.test.helpers.keys import privkeys -from eth2spec.test.helpers.state import next_slot from eth2spec.utils.bls import bls_sign from eth2spec.utils.minimal_ssz import signing_root, hash_tree_root -def sign_block(state, block): - assert block.slot == state.slot or block.slot == state.slot + 1 - if block.slot == state.slot: - proposer_index = get_beacon_proposer_index(state) - else: - # use stub state to get proposer index of next slot - stub_state = deepcopy(state) - next_slot(stub_state) - proposer_index = get_beacon_proposer_index(stub_state) +def sign_block(state, block, proposer_index=None): + assert state.slot <= block.slot + + if proposer_index is None: + if block.slot == state.slot: + proposer_index = get_beacon_proposer_index(state) + else: + if slot_to_epoch(state.slot) + 1 > slot_to_epoch(block.slot): + print("warning: block slot far away, and no proposer index manually given." + " Signing block is slow due to transition for proposer index calculation.") + # use stub state to get proposer index of future slot + stub_state = deepcopy(state) + state_transition_to(stub_state, block.slot) + proposer_index = get_beacon_proposer_index(stub_state) privkey = privkeys[proposer_index] diff --git a/test_libs/pyspec/eth2spec/test/test_sanity.py b/test_libs/pyspec/eth2spec/test/test_sanity.py index 27b40f93be..6d65cc7f4b 100644 --- a/test_libs/pyspec/eth2spec/test/test_sanity.py +++ b/test_libs/pyspec/eth2spec/test/test_sanity.py @@ -34,7 +34,7 @@ from .helpers.attestations import get_valid_attestation from .helpers.deposits import prepare_state_and_deposit -from .context import spec_state_test +from .context import spec_state_test, never_bls @spec_state_test @@ -51,6 +51,7 @@ def test_slot_transition(state): assert get_state_root(state, pre_slot) == pre_root +@never_bls @spec_state_test def test_empty_block_transition(state): pre_slot = state.slot @@ -68,6 +69,7 @@ def test_empty_block_transition(state): assert get_block_root_at_slot(state, pre_slot) == block.previous_block_root +@never_bls @spec_state_test def test_skipped_slots(state): pre_slot = state.slot @@ -104,30 +106,31 @@ def test_empty_epoch_transition(state): assert get_block_root_at_slot(state, slot) == block.previous_block_root -@spec_state_test -def test_empty_epoch_transition_not_finalizing(state): - # copy for later balance lookups. - pre_state = deepcopy(state) - yield 'pre', state - - block = build_empty_block_for_next_slot(state) - block.slot += spec.SLOTS_PER_EPOCH * 5 - yield 'blocks', [block], [spec.BeaconBlock] - - state_transition(state, block) - yield 'post', state - - assert state.slot == block.slot - assert state.finalized_epoch < get_current_epoch(state) - 4 - for index in range(len(state.validator_registry)): - assert get_balance(state, index) < get_balance(pre_state, index) +# @spec_state_test +# def test_empty_epoch_transition_not_finalizing(state): +# # copy for later balance lookups. +# pre_state = deepcopy(state) +# yield 'pre', state +# +# block = build_empty_block_for_next_slot(state, signed=False) +# block.slot += spec.SLOTS_PER_EPOCH * 5 +# sign_block(state, block, proposer_index=0) +# yield 'blocks', [block], [spec.BeaconBlock] +# +# state_transition(state, block) +# yield 'post', state +# +# assert state.slot == block.slot +# assert state.finalized_epoch < get_current_epoch(state) - 4 +# for index in range(len(state.validator_registry)): +# assert get_balance(state, index) < get_balance(pre_state, index) @spec_state_test def test_proposer_slashing(state): # copy for later balance lookups. pre_state = deepcopy(state) - proposer_slashing = get_valid_proposer_slashing(state) + proposer_slashing = get_valid_proposer_slashing(state, signed_1=True, signed_2=True) validator_index = proposer_slashing.proposer_index assert not state.validator_registry[validator_index].slashed @@ -137,8 +140,9 @@ def test_proposer_slashing(state): # # Add to state via block transition # - block = build_empty_block_for_next_slot(state) + block = build_empty_block_for_next_slot(state, signed=False) block.body.proposer_slashings.append(proposer_slashing) + sign_block(state, block) yield 'blocks', [block], [spec.BeaconBlock] state_transition(state, block) @@ -158,7 +162,7 @@ def test_attester_slashing(state): # copy for later balance lookups. pre_state = deepcopy(state) - attester_slashing = get_valid_attester_slashing(state) + attester_slashing = get_valid_attester_slashing(state, signed_1=True, signed_2=True) validator_index = (attester_slashing.attestation_1.custody_bit_0_indices + attester_slashing.attestation_1.custody_bit_1_indices)[0] @@ -169,8 +173,9 @@ def test_attester_slashing(state): # # Add to state via block transition # - block = build_empty_block_for_next_slot(state) + block = build_empty_block_for_next_slot(state, signed=False) block.body.attester_slashings.append(attester_slashing) + sign_block(state, block) yield 'blocks', [block], [spec.BeaconBlock] state_transition(state, block) @@ -200,12 +205,13 @@ def test_deposit_in_block(state): validator_index = len(state.validator_registry) amount = spec.MAX_EFFECTIVE_BALANCE - deposit = prepare_state_and_deposit(state, validator_index, amount) + deposit = prepare_state_and_deposit(state, validator_index, amount, signed=True) yield 'pre', state - block = build_empty_block_for_next_slot(state) + block = build_empty_block_for_next_slot(state, signed=False) block.body.deposits.append(deposit) + sign_block(state, block) yield 'blocks', [block], [spec.BeaconBlock] @@ -230,8 +236,9 @@ def test_deposit_top_up(state): yield 'pre', state - block = build_empty_block_for_next_slot(state) + block = build_empty_block_for_next_slot(state, signed=False) block.body.deposits.append(deposit) + sign_block(state, block) yield 'blocks', [block], [spec.BeaconBlock] @@ -249,23 +256,24 @@ def test_attestation(state): yield 'pre', state - attestation = get_valid_attestation(state) + attestation = get_valid_attestation(state, signed=True) # Add to state via block transition pre_current_attestations_len = len(state.current_epoch_attestations) - attestation_block = build_empty_block_for_next_slot(state) + attestation_block = build_empty_block_for_next_slot(state, signed=False) attestation_block.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation_block.body.attestations.append(attestation) + sign_block(state, attestation_block) state_transition(state, attestation_block) assert len(state.current_epoch_attestations) == pre_current_attestations_len + 1 - # Epoch transition should move to previous_epoch_attestations pre_current_attestations_root = spec.hash_tree_root(state.current_epoch_attestations) - epoch_block = build_empty_block_for_next_slot(state) + epoch_block = build_empty_block_for_next_slot(state, signed=False) epoch_block.slot += spec.SLOTS_PER_EPOCH + sign_block(state, epoch_block) state_transition(state, epoch_block) yield 'blocks', [attestation_block, epoch_block], [spec.BeaconBlock] @@ -301,15 +309,17 @@ def test_voluntary_exit(state): ) # Add to state via block transition - initiate_exit_block = build_empty_block_for_next_slot(state) + initiate_exit_block = build_empty_block_for_next_slot(state, signed=False) initiate_exit_block.body.voluntary_exits.append(voluntary_exit) + sign_block(state, initiate_exit_block) state_transition(state, initiate_exit_block) assert state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH # Process within epoch transition - exit_block = build_empty_block_for_next_slot(state) + exit_block = build_empty_block_for_next_slot(state, signed=False) exit_block.slot += spec.SLOTS_PER_EPOCH + sign_block(state, exit_block) state_transition(state, exit_block) yield 'blocks', [initiate_exit_block, exit_block], [spec.BeaconBlock] @@ -326,7 +336,7 @@ def test_transfer(state): sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] amount = get_balance(state, sender_index) - transfer = get_valid_transfer(state, state.slot + 1, sender_index, amount) + transfer = get_valid_transfer(state, state.slot + 1, sender_index, amount, signed=True) recipient_index = transfer.recipient pre_transfer_recipient_balance = get_balance(state, recipient_index) @@ -336,8 +346,9 @@ def test_transfer(state): yield 'pre', state # Add to state via block transition - block = build_empty_block_for_next_slot(state) + block = build_empty_block_for_next_slot(state, signed=False) block.body.transfers.append(transfer) + sign_block(state, block) yield 'blocks', [block], [spec.BeaconBlock] @@ -363,8 +374,9 @@ def test_balance_driven_status_transitions(state): yield 'pre', state # trigger epoch transition - block = build_empty_block_for_next_slot(state) + block = build_empty_block_for_next_slot(state, signed=False) block.slot += spec.SLOTS_PER_EPOCH + sign_block(state, block) state_transition(state, block) yield 'blocks', [block], [spec.BeaconBlock] @@ -380,7 +392,7 @@ def test_historical_batch(state): yield 'pre', state - block = build_empty_block_for_next_slot(state) + block = build_empty_block_for_next_slot(state, signed=True) state_transition(state, block) yield 'blocks', [block], [spec.BeaconBlock] @@ -391,28 +403,28 @@ def test_historical_batch(state): assert len(state.historical_roots) == pre_historical_roots_len + 1 -@spec_state_test -def test_eth1_data_votes(state): - yield 'pre', state - - expected_votes = 0 - assert len(state.eth1_data_votes) == expected_votes - - blocks = [] - for _ in range(spec.SLOTS_PER_ETH1_VOTING_PERIOD - 1): - block = build_empty_block_for_next_slot(state) - state_transition(state, block) - expected_votes += 1 - assert len(state.eth1_data_votes) == expected_votes - blocks.append(block) - - block = build_empty_block_for_next_slot(state) - blocks.append(block) - - state_transition(state, block) - - yield 'blocks', [block], [spec.BeaconBlock] - yield 'post', state - - assert state.slot % spec.SLOTS_PER_ETH1_VOTING_PERIOD == 0 - assert len(state.eth1_data_votes) == 1 +# @spec_state_test +# def test_eth1_data_votes(state): +# yield 'pre', state +# +# expected_votes = 0 +# assert len(state.eth1_data_votes) == expected_votes +# +# blocks = [] +# for _ in range(spec.SLOTS_PER_ETH1_VOTING_PERIOD - 1): +# block = build_empty_block_for_next_slot(state, signed=False) +# state_transition(state, block) +# expected_votes += 1 +# assert len(state.eth1_data_votes) == expected_votes +# blocks.append(block) +# +# block = build_empty_block_for_next_slot(state, signed=False) +# blocks.append(block) +# +# state_transition(state, block) +# +# yield 'blocks', [block], [spec.BeaconBlock] +# yield 'post', state +# +# assert state.slot % spec.SLOTS_PER_ETH1_VOTING_PERIOD == 0 +# assert len(state.eth1_data_votes) == 1 From 4d08e9d7d6117b876f09cdc6f49784819c265cc1 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 21 May 2019 21:22:32 +0200 Subject: [PATCH 24/37] signed block header --- .../test/block_processing/test_process_block_header.py | 3 ++- test_libs/pyspec/eth2spec/test/context.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py index 3faf51e1bd..9918e6283b 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py @@ -49,8 +49,9 @@ def test_success_block_header(state): @spec_state_test def test_invalid_slot_block_header(state): - block = build_empty_block_for_next_slot(state, signed=True) + block = build_empty_block_for_next_slot(state, signed=False) block.slot = state.slot + 2 # invalid slot + sign_block(state, block) yield from run_block_header_processing(state, block, valid=False) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index a9eaea349a..4c8ff255b5 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -57,7 +57,7 @@ def bls_switch(fn): """ def entry(*args, **kw): old_state = bls.bls_active - bls.bls_active = kw.pop('bls_active', False) + bls.bls_active = kw.pop('bls_active', True) fn(*args, **kw) bls.bls_active = old_state return entry From ee9c1d911fd8cf1516a24136175115ef8857aa5e Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 21 May 2019 21:51:28 +0200 Subject: [PATCH 25/37] test tagging pattern --- test_libs/pyspec/eth2spec/test/context.py | 26 ++++++++++++++++--- .../pyspec/eth2spec/test/helpers/block.py | 9 ++++--- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index 4c8ff255b5..7a8e959d06 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -3,12 +3,22 @@ from .helpers.genesis import create_genesis_state -from .utils import spectest, with_args +from .utils import spectest, with_args, with_tags # Provides a genesis state as first argument to the function decorated with this with_state = with_args(lambda: [create_genesis_state(spec.SLOTS_PER_EPOCH * 8)]) +# BLS is turned off by default *for performance purposes during TESTING*. +# The runner of the test can indicate the preferred setting (test generators prefer BLS to be ON). +# - Some tests are marked as BLS-requiring, and ignore this setting. +# (tests that express differences caused by BLS, e.g. invalid signatures being rejected) +# - Some other tests are marked as BLS-ignoring, and ignore this setting. +# (tests that are heavily performance impacted / require unsigned state transitions) +# - Most tests respect the BLS setting. +DEFAULT_BLS_ACTIVE = False + + # shorthand for decorating @with_state @spectest() def spec_state_test(fn): return with_state(bls_switch(spectest()(fn))) @@ -28,6 +38,10 @@ def expect_assertion_error(fn): raise AssertionError('expected an assertion error, but got none.') +# Tags a test to be ignoring BLS for it to pass. +bls_ignored = with_tags({'bls_ignored': True}) + + def never_bls(fn): """ Decorator to apply on ``bls_switch`` decorator to force BLS de-activation. Useful to mark tests as BLS-ignorant. @@ -36,7 +50,11 @@ def entry(*args, **kw): # override bls setting kw['bls_active'] = False fn(*args, **kw) - return entry + return bls_ignored(entry) + + +# Tags a test to be requiring BLS for it to pass. +bls_required = with_tags({'bls_required': True}) def always_bls(fn): @@ -47,7 +65,7 @@ def entry(*args, **kw): # override bls setting kw['bls_active'] = True fn(*args, **kw) - return entry + return bls_required(entry) def bls_switch(fn): @@ -57,7 +75,7 @@ def bls_switch(fn): """ def entry(*args, **kw): old_state = bls.bls_active - bls.bls_active = kw.pop('bls_active', True) + bls.bls_active = kw.pop('bls_active', DEFAULT_BLS_ACTIVE) fn(*args, **kw) bls.bls_active = old_state return entry diff --git a/test_libs/pyspec/eth2spec/test/helpers/block.py b/test_libs/pyspec/eth2spec/test/helpers/block.py index 6f1a0fe602..094423ab3c 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block.py @@ -4,13 +4,16 @@ from eth2spec.phase0.spec import get_beacon_proposer_index, slot_to_epoch, get_domain, BeaconBlock from eth2spec.phase0.state_transition import state_transition, state_transition_to from eth2spec.test.helpers.keys import privkeys -from eth2spec.utils.bls import bls_sign +from eth2spec.utils import bls from eth2spec.utils.minimal_ssz import signing_root, hash_tree_root def sign_block(state, block, proposer_index=None): assert state.slot <= block.slot + if not bls.bls_active: + return + if proposer_index is None: if block.slot == state.slot: proposer_index = get_beacon_proposer_index(state) @@ -25,7 +28,7 @@ def sign_block(state, block, proposer_index=None): privkey = privkeys[proposer_index] - block.body.randao_reveal = bls_sign( + block.body.randao_reveal = bls.bls_sign( privkey=privkey, message_hash=hash_tree_root(slot_to_epoch(block.slot)), domain=get_domain( @@ -34,7 +37,7 @@ def sign_block(state, block, proposer_index=None): domain_type=spec.DOMAIN_RANDAO, ) ) - block.signature = bls_sign( + block.signature = bls.bls_sign( message_hash=signing_root(block), privkey=privkey, domain=get_domain( From cfc037fe75eff3d16a963ae0973de55688082544 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 21 May 2019 22:28:47 +0200 Subject: [PATCH 26/37] tags applied after generation of vector (and only if), make BLS decorator --- .../pyspec/eth2spec/test/helpers/block.py | 12 ++--- test_libs/pyspec/eth2spec/test/utils.py | 33 ++++++++++--- test_libs/pyspec/eth2spec/utils/bls.py | 47 +++++++++++-------- 3 files changed, 59 insertions(+), 33 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/block.py b/test_libs/pyspec/eth2spec/test/helpers/block.py index 094423ab3c..81c5e9ef5b 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block.py @@ -4,16 +4,15 @@ from eth2spec.phase0.spec import get_beacon_proposer_index, slot_to_epoch, get_domain, BeaconBlock from eth2spec.phase0.state_transition import state_transition, state_transition_to from eth2spec.test.helpers.keys import privkeys -from eth2spec.utils import bls +from eth2spec.utils.bls import bls_sign, only_with_bls from eth2spec.utils.minimal_ssz import signing_root, hash_tree_root +# Fully ignore the function if BLS is off, beacon-proposer index calculation is slow. +@only_with_bls() def sign_block(state, block, proposer_index=None): assert state.slot <= block.slot - if not bls.bls_active: - return - if proposer_index is None: if block.slot == state.slot: proposer_index = get_beacon_proposer_index(state) @@ -28,7 +27,7 @@ def sign_block(state, block, proposer_index=None): privkey = privkeys[proposer_index] - block.body.randao_reveal = bls.bls_sign( + block.body.randao_reveal = bls_sign( privkey=privkey, message_hash=hash_tree_root(slot_to_epoch(block.slot)), domain=get_domain( @@ -37,14 +36,13 @@ def sign_block(state, block, proposer_index=None): domain_type=spec.DOMAIN_RANDAO, ) ) - block.signature = bls.bls_sign( + block.signature = bls_sign( message_hash=signing_root(block), privkey=privkey, domain=get_domain( state, spec.DOMAIN_BEACON_PROPOSER, slot_to_epoch(block.slot))) - return block def apply_empty_block(state): diff --git a/test_libs/pyspec/eth2spec/test/utils.py b/test_libs/pyspec/eth2spec/test/utils.py index 1c157bcee1..c1d4241099 100644 --- a/test_libs/pyspec/eth2spec/test/utils.py +++ b/test_libs/pyspec/eth2spec/test/utils.py @@ -1,5 +1,5 @@ +from typing import Dict, Any, Callable, Iterable from eth2spec.debug.encode import encode -from eth2spec.utils import bls def spectest(description: str = None): @@ -19,9 +19,6 @@ def entry(*args, **kw): else: # description can be explicit out['description'] = description - # If BLS is not active, mark the test as BLS-ignorant. - if not bls.bls_active: - out['stub_bls'] = True # put all generated data into a dict. for data in fn(*args, **kw): # If there is a type argument, encode it as that type. @@ -44,10 +41,34 @@ def entry(*args, **kw): return runner -def with_args(create_args): +def with_tags(tags: Dict[str, Any]): + """ + Decorator factory, adds tags (key, value) pairs to the output of the function. + Useful to build test-vector annotations with. + This decorator is applied after the ``spectest`` decorator is applied. + :param tags: dict of tags + :return: Decorator. + """ + def runner(fn): + def entry(*args, **kw): + fn_out = fn(*args, **kw) + # do not add tags if the function is not returning a dict at all (i.e. not in generator mode) + if fn_out is None: + return fn_out + return {**tags, **fn_out} + return entry + return runner + + +def with_args(create_args: Callable[[], Iterable[Any]]): + """ + Decorator factory, adds given extra arguments to the decorated function. + :param create_args: function to create arguments with. + :return: Decorator. + """ def runner(fn): # this wraps the function, to hide that the function actually yielding data. def entry(*args, **kw): - return fn(*(create_args() + list(args)), **kw) + return fn(*(list(create_args()) + list(args)), **kw) return entry return runner diff --git a/test_libs/pyspec/eth2spec/utils/bls.py b/test_libs/pyspec/eth2spec/utils/bls.py index 23a9a7529f..52f1fed632 100644 --- a/test_libs/pyspec/eth2spec/utils/bls.py +++ b/test_libs/pyspec/eth2spec/utils/bls.py @@ -3,37 +3,44 @@ # Flag to make BLS active or not. Used for testing, do not ignore BLS in production unless you know what you are doing. bls_active = True +STUB_SIGNATURE = b'\x11' * 96 +STUB_PUBKEY = b'\x22' * 48 + +def only_with_bls(alt_return=None): + """ + Decorator factory to make a function only run when BLS is active. Otherwise return the default. + """ + def runner(fn): + def entry(*args, **kw): + if bls_active: + return fn(*args, **kw) + else: + return alt_return + return entry + return runner + + +@only_with_bls(alt_return=True) def bls_verify(pubkey, message_hash, signature, domain): - if bls_active: - return bls.verify(message_hash=message_hash, pubkey=pubkey, signature=signature, domain=domain) - else: - return True + return bls.verify(message_hash=message_hash, pubkey=pubkey, signature=signature, domain=domain) +@only_with_bls(alt_return=True) def bls_verify_multiple(pubkeys, message_hashes, signature, domain): - if bls_active: - return bls.verify_multiple(pubkeys, message_hashes, signature, domain) - else: - return True + return bls.verify_multiple(pubkeys, message_hashes, signature, domain) +@only_with_bls(alt_return=STUB_PUBKEY) def bls_aggregate_pubkeys(pubkeys): - if bls_active: - return bls.aggregate_pubkeys(pubkeys) - else: - return b'\xaa' * 48 + return bls.aggregate_pubkeys(pubkeys) +@only_with_bls(alt_return=STUB_SIGNATURE) def bls_aggregate_signatures(signatures): - if bls_active: - return bls.aggregate_signatures(signatures) - else: - return b'\x22' * 96 + return bls.aggregate_signatures(signatures) +@only_with_bls(alt_return=STUB_SIGNATURE) def bls_sign(message_hash, privkey, domain): - if bls_active: - return bls.sign(message_hash=message_hash, privkey=privkey, domain=domain) - else: - return b'\x11' * 96 + return bls.sign(message_hash=message_hash, privkey=privkey, domain=domain) From 8303c58aa41248ffef99175e2707c6e63fe63591 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 21 May 2019 22:53:46 +0200 Subject: [PATCH 27/37] bugfix, make BLS wrapper propagate test output properly --- test_libs/pyspec/eth2spec/test/context.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index 7a8e959d06..2136a60538 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -76,6 +76,7 @@ def bls_switch(fn): def entry(*args, **kw): old_state = bls.bls_active bls.bls_active = kw.pop('bls_active', DEFAULT_BLS_ACTIVE) - fn(*args, **kw) + out = fn(*args, **kw) bls.bls_active = old_state + return out return entry From f1ba5aadceb755623ee11e5ada52b2b83a1f2f3e Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 21 May 2019 22:55:02 +0200 Subject: [PATCH 28/37] make test generator use BLS --- test_generators/operations/suite_creator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_generators/operations/suite_creator.py b/test_generators/operations/suite_creator.py index d27e3efc19..c0bc1aa44d 100644 --- a/test_generators/operations/suite_creator.py +++ b/test_generators/operations/suite_creator.py @@ -15,7 +15,7 @@ def generate_from_tests(pkg): for name in fn_names: tfn = getattr(pkg, name) try: - out.append(tfn(generator_mode=True)) + out.append(tfn(generator_mode=True, bls_active=True)) except AssertionError: print("ERROR: failed to generate vector from test: %s (pkg: %s)" % (name, pkg.__name__)) return out From 0e136b1104d6a5eed90dba8d2779768bbc3a8c83 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 21 May 2019 22:55:22 +0200 Subject: [PATCH 29/37] update yaml and eth-utils dependencies --- test_generators/README.md | 2 +- test_generators/bls/requirements.txt | 2 +- test_generators/operations/requirements.txt | 2 +- test_generators/shuffling/requirements.txt | 2 +- test_generators/ssz_generic/requirements.txt | 2 +- test_generators/ssz_static/requirements.txt | 2 +- test_libs/config_helpers/requirements.txt | 2 +- test_libs/config_helpers/setup.py | 2 +- test_libs/gen_helpers/requirements.txt | 4 ++-- test_libs/gen_helpers/setup.py | 4 ++-- 10 files changed, 12 insertions(+), 12 deletions(-) diff --git a/test_generators/README.md b/test_generators/README.md index 43bf7af031..309a64bd92 100644 --- a/test_generators/README.md +++ b/test_generators/README.md @@ -58,7 +58,7 @@ It's recommended to extend the base-generator. Create a `requirements.txt` in the root of your generator directory: ``` -eth-utils==1.4.1 +eth-utils==1.6.0 ../../test_libs/gen_helpers ../../test_libs/config_helpers ../../test_libs/pyspec diff --git a/test_generators/bls/requirements.txt b/test_generators/bls/requirements.txt index 8a933d41ca..329a4ce152 100644 --- a/test_generators/bls/requirements.txt +++ b/test_generators/bls/requirements.txt @@ -1,3 +1,3 @@ py-ecc==1.6.0 -eth-utils==1.4.1 +eth-utils==1.6.0 ../../test_libs/gen_helpers diff --git a/test_generators/operations/requirements.txt b/test_generators/operations/requirements.txt index 8f9bede8f3..595cee69cd 100644 --- a/test_generators/operations/requirements.txt +++ b/test_generators/operations/requirements.txt @@ -1,4 +1,4 @@ -eth-utils==1.4.1 +eth-utils==1.6.0 ../../test_libs/gen_helpers ../../test_libs/config_helpers ../../test_libs/pyspec \ No newline at end of file diff --git a/test_generators/shuffling/requirements.txt b/test_generators/shuffling/requirements.txt index 8f9bede8f3..595cee69cd 100644 --- a/test_generators/shuffling/requirements.txt +++ b/test_generators/shuffling/requirements.txt @@ -1,4 +1,4 @@ -eth-utils==1.4.1 +eth-utils==1.6.0 ../../test_libs/gen_helpers ../../test_libs/config_helpers ../../test_libs/pyspec \ No newline at end of file diff --git a/test_generators/ssz_generic/requirements.txt b/test_generators/ssz_generic/requirements.txt index 94afc9d91b..dcdb0824ff 100644 --- a/test_generators/ssz_generic/requirements.txt +++ b/test_generators/ssz_generic/requirements.txt @@ -1,4 +1,4 @@ -eth-utils==1.4.1 +eth-utils==1.6.0 ../../test_libs/gen_helpers ../../test_libs/config_helpers ssz==0.1.0a2 diff --git a/test_generators/ssz_static/requirements.txt b/test_generators/ssz_static/requirements.txt index 8f9bede8f3..595cee69cd 100644 --- a/test_generators/ssz_static/requirements.txt +++ b/test_generators/ssz_static/requirements.txt @@ -1,4 +1,4 @@ -eth-utils==1.4.1 +eth-utils==1.6.0 ../../test_libs/gen_helpers ../../test_libs/config_helpers ../../test_libs/pyspec \ No newline at end of file diff --git a/test_libs/config_helpers/requirements.txt b/test_libs/config_helpers/requirements.txt index e441a474b8..f2f208c3fb 100644 --- a/test_libs/config_helpers/requirements.txt +++ b/test_libs/config_helpers/requirements.txt @@ -1 +1 @@ -ruamel.yaml==0.15.87 +ruamel.yaml==0.15.96 diff --git a/test_libs/config_helpers/setup.py b/test_libs/config_helpers/setup.py index 90ad94ee44..9f0ea06419 100644 --- a/test_libs/config_helpers/setup.py +++ b/test_libs/config_helpers/setup.py @@ -4,6 +4,6 @@ name='config_helpers', packages=['preset_loader'], install_requires=[ - "ruamel.yaml==0.15.87" + "ruamel.yaml==0.15.96" ] ) diff --git a/test_libs/gen_helpers/requirements.txt b/test_libs/gen_helpers/requirements.txt index 3d6a39458e..557cae6317 100644 --- a/test_libs/gen_helpers/requirements.txt +++ b/test_libs/gen_helpers/requirements.txt @@ -1,2 +1,2 @@ -ruamel.yaml==0.15.87 -eth-utils==1.4.1 +ruamel.yaml==0.15.96 +eth-utils==1.6.0 diff --git a/test_libs/gen_helpers/setup.py b/test_libs/gen_helpers/setup.py index 5de27a6dbe..29cf04fd19 100644 --- a/test_libs/gen_helpers/setup.py +++ b/test_libs/gen_helpers/setup.py @@ -4,7 +4,7 @@ name='gen_helpers', packages=['gen_base'], install_requires=[ - "ruamel.yaml==0.15.87", - "eth-utils==1.4.1" + "ruamel.yaml==0.15.96", + "eth-utils==1.6.0" ] ) From 6b3b5121f29fcb7b0ff2d316e7b50e1f17054629 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 21 May 2019 22:59:47 +0200 Subject: [PATCH 30/37] fix header format of operations tests --- test_generators/operations/suite_creator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_generators/operations/suite_creator.py b/test_generators/operations/suite_creator.py index c0bc1aa44d..caff0c7db9 100644 --- a/test_generators/operations/suite_creator.py +++ b/test_generators/operations/suite_creator.py @@ -34,6 +34,6 @@ def suite_definition(configs_path: str) -> gen_typing.TestSuiteOutput: forks=["phase0"], config=config_name, runner="operations", - handler=config_name, + handler=operation_name, test_cases=get_cases())) return suite_definition From 12af078a6c4765f8486262f7f3a36fc11eafaf0a Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 21 May 2019 23:03:59 +0200 Subject: [PATCH 31/37] make decorators return wrapped fn results properly --- test_libs/pyspec/eth2spec/test/context.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index 2136a60538..a484cc995f 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -49,7 +49,7 @@ def never_bls(fn): def entry(*args, **kw): # override bls setting kw['bls_active'] = False - fn(*args, **kw) + return fn(*args, **kw) return bls_ignored(entry) @@ -64,7 +64,7 @@ def always_bls(fn): def entry(*args, **kw): # override bls setting kw['bls_active'] = True - fn(*args, **kw) + return fn(*args, **kw) return bls_required(entry) From b919d08ab2362915b88ef4eae3b214236eea2570 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 22 May 2019 00:07:52 +0200 Subject: [PATCH 32/37] comment on the deposit signature being soft-rejected --- specs/core/0_beacon-chain.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index e56fd976cc..46c811fedb 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1756,7 +1756,8 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: amount = deposit.data.amount validator_pubkeys = [v.pubkey for v in state.validator_registry] if pubkey not in validator_pubkeys: - # Verify the deposit signature (proof of possession) + # Verify the deposit signature (proof of possession). + # Invalid signatures are allowed by the deposit contract, and hence included on-chain, but must not be processed. if not bls_verify(pubkey, signing_root(deposit.data), deposit.data.signature, get_domain(state, DOMAIN_DEPOSIT)): return From a4363ba0969f5ec6347665d5da087e286c730eec Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 22 May 2019 00:52:57 +0200 Subject: [PATCH 33/37] tests for invalid signatures --- .../test_process_attestation.py | 11 ++++- .../test_process_attester_slashing.py | 23 +++++++++- .../test_process_block_header.py | 9 +++- .../block_processing/test_process_deposit.py | 45 +++++++++++++++---- .../test_process_proposer_slashing.py | 23 +++++++++- .../block_processing/test_process_transfer.py | 12 ++++- .../test_process_voluntary_exit.py | 25 +++++++---- 7 files changed, 126 insertions(+), 22 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py index 2c88f12b5e..9491ee0011 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py @@ -8,7 +8,7 @@ from eth2spec.phase0.state_transition import ( state_transition_to, ) -from eth2spec.test.context import spec_state_test, expect_assertion_error +from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls from eth2spec.test.helpers.attestations import ( get_valid_attestation, sign_attestation, @@ -72,6 +72,15 @@ def test_success_previous_epoch(state): yield from run_attestation_processing(state, attestation) +@always_bls +@spec_state_test +def test_invalid_attestation_signature(state): + attestation = get_valid_attestation(state, signed=False) + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + + yield from run_attestation_processing(state, attestation, False) + + @spec_state_test def test_before_inclusion_delay(state): attestation = get_valid_attestation(state, signed=True) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py index 01861f04ca..1481c2b87a 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py @@ -3,7 +3,7 @@ get_beacon_proposer_index, process_attester_slashing, ) -from eth2spec.test.context import spec_state_test, expect_assertion_error +from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls from eth2spec.test.helpers.attestations import sign_indexed_attestation from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing from eth2spec.test.helpers.block import apply_empty_block @@ -90,6 +90,27 @@ def test_success_surround(state): yield from run_attester_slashing_processing(state, attester_slashing) +@always_bls +@spec_state_test +def test_invalid_sig_1(state): + attester_slashing = get_valid_attester_slashing(state, signed_1=False, signed_2=True) + yield from run_attester_slashing_processing(state, attester_slashing, False) + + +@always_bls +@spec_state_test +def test_invalid_sig_2(state): + attester_slashing = get_valid_attester_slashing(state, signed_1=True, signed_2=False) + yield from run_attester_slashing_processing(state, attester_slashing, False) + + +@always_bls +@spec_state_test +def test_invalid_sig_1_and_2(state): + attester_slashing = get_valid_attester_slashing(state, signed_1=False, signed_2=False) + yield from run_attester_slashing_processing(state, attester_slashing, False) + + @spec_state_test def test_same_data(state): attester_slashing = get_valid_attester_slashing(state, signed_1=False, signed_2=True) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py index 9918e6283b..28e215a3a0 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py @@ -6,7 +6,7 @@ advance_slot, process_block_header, ) -from eth2spec.test.context import spec_state_test, expect_assertion_error +from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls from eth2spec.test.helpers.block import ( build_empty_block_for_next_slot, sign_block @@ -47,6 +47,13 @@ def test_success_block_header(state): yield from run_block_header_processing(state, block) +@always_bls +@spec_state_test +def test_invalid_sig_block_header(state): + block = build_empty_block_for_next_slot(state, signed=False) + yield from run_block_header_processing(state, block, valid=False) + + @spec_state_test def test_invalid_slot_block_header(state): block = build_empty_block_for_next_slot(state, signed=False) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py index a796be8178..f734508aae 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py @@ -1,12 +1,12 @@ import eth2spec.phase0.spec as spec from eth2spec.phase0.spec import process_deposit -from eth2spec.test.context import spec_state_test, expect_assertion_error +from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls from eth2spec.test.helpers.deposits import prepare_state_and_deposit, sign_deposit_data from eth2spec.test.helpers.state import get_balance from eth2spec.test.helpers.keys import privkeys -def run_deposit_processing(state, deposit, validator_index, valid=True): +def run_deposit_processing(state, deposit, validator_index, valid=True, non_effective=False): """ Run ``process_deposit``, yielding: - pre-state ('pre') @@ -34,21 +34,27 @@ def run_deposit_processing(state, deposit, validator_index, valid=True): yield 'post', state - if validator_index < pre_validator_count: - # top-up + if non_effective: assert len(state.validator_registry) == pre_validator_count assert len(state.balances) == pre_validator_count + if validator_index < pre_validator_count: + assert get_balance(state, validator_index) == pre_balance else: - # new validator - assert len(state.validator_registry) == pre_validator_count + 1 - assert len(state.balances) == pre_validator_count + 1 + if validator_index < pre_validator_count: + # top-up + assert len(state.validator_registry) == pre_validator_count + assert len(state.balances) == pre_validator_count + else: + # new validator + assert len(state.validator_registry) == pre_validator_count + 1 + assert len(state.balances) == pre_validator_count + 1 + assert get_balance(state, validator_index) == pre_balance + deposit.data.amount assert state.deposit_index == state.latest_eth1_data.deposit_count - assert get_balance(state, validator_index) == pre_balance + deposit.data.amount @spec_state_test -def test_success(state): +def test_new_deposit(state): # fresh deposit = next validator index = validator appended to registry validator_index = len(state.validator_registry) amount = spec.MAX_EFFECTIVE_BALANCE @@ -57,6 +63,16 @@ def test_success(state): yield from run_deposit_processing(state, deposit, validator_index) +@always_bls +@spec_state_test +def test_invalid_sig_new_deposit(state): + # fresh deposit = next validator index = validator appended to registry + validator_index = len(state.validator_registry) + amount = spec.MAX_EFFECTIVE_BALANCE + deposit = prepare_state_and_deposit(state, validator_index, amount, signed=False) + yield from run_deposit_processing(state, deposit, validator_index, valid=True, non_effective=True) + + @spec_state_test def test_success_top_up(state): validator_index = 0 @@ -66,6 +82,17 @@ def test_success_top_up(state): yield from run_deposit_processing(state, deposit, validator_index) +@always_bls +@spec_state_test +def test_invalid_sig_top_up(state): + validator_index = 0 + amount = spec.MAX_EFFECTIVE_BALANCE // 4 + deposit = prepare_state_and_deposit(state, validator_index, amount, signed=False) + + # invalid signatures, in top-ups, are allowed! + yield from run_deposit_processing(state, deposit, validator_index, valid=True, non_effective=False) + + @spec_state_test def test_wrong_index(state): validator_index = len(state.validator_registry) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py index f3d1593a81..7ec10406d6 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py @@ -3,7 +3,7 @@ get_current_epoch, process_proposer_slashing, ) -from eth2spec.test.context import spec_state_test, expect_assertion_error +from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls from eth2spec.test.helpers.block_header import sign_block_header from eth2spec.test.helpers.keys import privkeys from eth2spec.test.helpers.proposer_slashings import get_valid_proposer_slashing @@ -52,6 +52,27 @@ def test_success(state): yield from run_proposer_slashing_processing(state, proposer_slashing) +@always_bls +@spec_state_test +def test_invalid_sig_1(state): + proposer_slashing = get_valid_proposer_slashing(state, signed_1=False, signed_2=True) + yield from run_proposer_slashing_processing(state, proposer_slashing, False) + + +@always_bls +@spec_state_test +def test_invalid_sig_2(state): + proposer_slashing = get_valid_proposer_slashing(state, signed_1=True, signed_2=False) + yield from run_proposer_slashing_processing(state, proposer_slashing, False) + + +@always_bls +@spec_state_test +def test_invalid_sig_1_and_2(state): + proposer_slashing = get_valid_proposer_slashing(state, signed_1=False, signed_2=False) + yield from run_proposer_slashing_processing(state, proposer_slashing, False) + + @spec_state_test def test_invalid_proposer_index(state): proposer_slashing = get_valid_proposer_slashing(state, signed_1=True, signed_2=True) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py index 2bbc022d00..e5f52e209f 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py @@ -5,7 +5,7 @@ get_current_epoch, process_transfer, ) -from eth2spec.test.context import spec_state_test, expect_assertion_error +from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls from eth2spec.test.helpers.state import next_epoch from eth2spec.test.helpers.block import apply_empty_block from eth2spec.test.helpers.transfers import get_valid_transfer @@ -83,6 +83,16 @@ def test_success_active_above_max_effective_fee(state): yield from run_transfer_processing(state, transfer) +@always_bls +@spec_state_test +def test_invalid_signature(state): + transfer = get_valid_transfer(state, signed=False) + # un-activate so validator can transfer + state.validator_registry[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + + yield from run_transfer_processing(state, transfer, False) + + @spec_state_test def test_active_but_transfer_past_effective_balance(state): sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py index e5be38722f..fe33fb6318 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py @@ -5,7 +5,7 @@ get_current_epoch, process_voluntary_exit, ) -from eth2spec.test.context import spec_state_test, expect_assertion_error +from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls from eth2spec.test.helpers.keys import pubkey_to_privkey from eth2spec.test.helpers.voluntary_exits import build_voluntary_exit, sign_voluntary_exit @@ -47,17 +47,26 @@ def test_success(state): validator_index = get_active_validator_indices(state, current_epoch)[0] privkey = pubkey_to_privkey[state.validator_registry[validator_index].pubkey] - voluntary_exit = build_voluntary_exit( - state, - current_epoch, - validator_index, - privkey, - signed=True, - ) + voluntary_exit = build_voluntary_exit(state, current_epoch, validator_index, privkey, signed=True) yield from run_voluntary_exit_processing(state, voluntary_exit) +@always_bls +@spec_state_test +def test_invalid_signature(state): + # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit + state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + + current_epoch = get_current_epoch(state) + validator_index = get_active_validator_indices(state, current_epoch)[0] + privkey = pubkey_to_privkey[state.validator_registry[validator_index].pubkey] + + voluntary_exit = build_voluntary_exit(state, current_epoch, validator_index, privkey, signed=False) + + yield from run_voluntary_exit_processing(state, voluntary_exit, False) + + @spec_state_test def test_success_exit_queue(state): # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit From 7d3147d418e320c0b433d9de872f8b4dad0d5186 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 22 May 2019 04:19:54 +0200 Subject: [PATCH 34/37] add inconsistent bitfields test --- .../block_processing/test_process_attestation.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py index 9491ee0011..0dae852f0b 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py @@ -219,6 +219,18 @@ def test_bad_previous_crosslink(state): yield from run_attestation_processing(state, attestation, False) +@spec_state_test +def test_inconsistent_bitfields(state): + attestation = get_valid_attestation(state, signed=False) + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + + attestation.custody_bitfield = deepcopy(attestation.aggregation_bitfield) + b'\x00' + + sign_attestation(state, attestation) + + yield from run_attestation_processing(state, attestation, False) + + @spec_state_test def test_non_empty_custody_bitfield(state): attestation = get_valid_attestation(state, signed=False) From dad89ae926988fa4a484c301e24f57646d8980a2 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 21 May 2019 23:20:12 -0600 Subject: [PATCH 35/37] minor lint --- .../test_process_attester_slashing.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py index 1481c2b87a..28e2322772 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py @@ -48,16 +48,9 @@ def run_attester_slashing_processing(state, attester_slashing, valid=True): if slashed_index != proposer_index: # lost whistleblower reward - assert ( - get_balance(state, slashed_index) < - pre_slashed_balance - ) - + assert get_balance(state, slashed_index) < pre_slashed_balance # gained whistleblower reward - assert ( - get_balance(state, proposer_index) > - pre_proposer_balance - ) + assert get_balance(state, proposer_index) > pre_proposer_balance else: # gained rewards for all slashings, which may include others. And only lost that of themselves. # Netto at least 0, if more people where slashed, a balance increase. From 58fd712607ba3fd635feb29e5364dff72add5fdf Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 22 May 2019 20:50:18 +0200 Subject: [PATCH 36/37] fix style issue with deposit processing helper --- .../test/block_processing/test_process_deposit.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py index f734508aae..b520c809f8 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py @@ -6,7 +6,7 @@ from eth2spec.test.helpers.keys import privkeys -def run_deposit_processing(state, deposit, validator_index, valid=True, non_effective=False): +def run_deposit_processing(state, deposit, validator_index, valid=True, effective=True): """ Run ``process_deposit``, yielding: - pre-state ('pre') @@ -34,7 +34,7 @@ def run_deposit_processing(state, deposit, validator_index, valid=True, non_effe yield 'post', state - if non_effective: + if not effective: assert len(state.validator_registry) == pre_validator_count assert len(state.balances) == pre_validator_count if validator_index < pre_validator_count: @@ -70,7 +70,7 @@ def test_invalid_sig_new_deposit(state): validator_index = len(state.validator_registry) amount = spec.MAX_EFFECTIVE_BALANCE deposit = prepare_state_and_deposit(state, validator_index, amount, signed=False) - yield from run_deposit_processing(state, deposit, validator_index, valid=True, non_effective=True) + yield from run_deposit_processing(state, deposit, validator_index, valid=True, effective=False) @spec_state_test @@ -90,7 +90,7 @@ def test_invalid_sig_top_up(state): deposit = prepare_state_and_deposit(state, validator_index, amount, signed=False) # invalid signatures, in top-ups, are allowed! - yield from run_deposit_processing(state, deposit, validator_index, valid=True, non_effective=False) + yield from run_deposit_processing(state, deposit, validator_index, valid=True, effective=True) @spec_state_test From 958f71bb68e6dede30a1e875cadb85edb880a9a4 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 22 May 2019 21:03:46 +0200 Subject: [PATCH 37/37] minor underflow fix for proposer slashing test --- .../test/block_processing/test_process_proposer_slashing.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py index 7ec10406d6..07ccc25f1c 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py @@ -127,6 +127,8 @@ def test_proposer_is_slashed(state): def test_proposer_is_withdrawn(state): proposer_slashing = get_valid_proposer_slashing(state, signed_1=True, signed_2=True) + # move 1 epoch into future, to allow for past withdrawable epoch + state.slot += spec.SLOTS_PER_EPOCH # set proposer withdrawable_epoch in past current_epoch = get_current_epoch(state) proposer_index = proposer_slashing.proposer_index