From c84bb014a855fd10491a968940b1ed11871ab7d5 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Fri, 8 Mar 2019 19:25:19 +0800 Subject: [PATCH 01/23] Split `latest_attestations` --- eth2/beacon/epoch_processing_helpers.py | 40 +------------ eth2/beacon/on_genesis.py | 3 +- .../forks/serenity/epoch_processing.py | 32 ++--------- .../forks/serenity/operation_processing.py | 56 +++++++++++++------ eth2/beacon/types/states.py | 13 +++-- tests/eth2/beacon/conftest.py | 3 +- .../forks/test_serenity_epoch_processing.py | 47 ++++++++-------- .../test_serenity_operation_processing.py | 2 +- .../beacon/test_epoch_processing_helpers.py | 19 +++---- tests/eth2/beacon/test_on_genesis.py | 3 +- 10 files changed, 98 insertions(+), 120 deletions(-) diff --git a/eth2/beacon/epoch_processing_helpers.py b/eth2/beacon/epoch_processing_helpers.py index 58be9ac50f..a0921f2ebe 100644 --- a/eth2/beacon/epoch_processing_helpers.py +++ b/eth2/beacon/epoch_processing_helpers.py @@ -31,7 +31,6 @@ get_epoch_start_slot, get_effective_balance, get_total_balance, - slot_to_epoch, ) from eth2.beacon.typing import ( Epoch, @@ -53,39 +52,13 @@ from eth2.beacon.state_machines.configs import BeaconConfig # noqa: F401 -@to_tuple -def get_current_epoch_attestations( - state: 'BeaconState', - slots_per_epoch: int) -> Iterable[PendingAttestationRecord]: - current_epoch = state.current_epoch(slots_per_epoch) - for attestation in state.latest_attestations: - if current_epoch == slot_to_epoch(attestation.data.slot, slots_per_epoch): - yield attestation - - -@to_tuple -def get_previous_epoch_attestations( - state: 'BeaconState', - slots_per_epoch: int, - genesis_epoch: Epoch) -> Iterable[PendingAttestationRecord]: - previous_epoch = state.previous_epoch(slots_per_epoch, genesis_epoch) - for attestation in state.latest_attestations: - if previous_epoch == slot_to_epoch(attestation.data.slot, slots_per_epoch): - yield attestation - - @to_tuple def get_previous_epoch_head_attestations( state: 'BeaconState', slots_per_epoch: int, genesis_epoch: Epoch, latest_block_roots_length: int) -> Iterable[PendingAttestationRecord]: - previous_epoch_attestations = get_previous_epoch_attestations( - state, - slots_per_epoch, - genesis_epoch, - ) - for attestation in previous_epoch_attestations: + for attestation in state.previous_epoch_attestations: beacon_block_root = get_block_root( state, attestation.data.slot, @@ -161,13 +134,6 @@ def get_epoch_boundary_attesting_balances( state: 'BeaconState', config: 'BeaconConfig') -> Tuple[Gwei, Gwei]: - current_epoch_attestations = get_current_epoch_attestations(state, config.SLOTS_PER_EPOCH) - previous_epoch_attestations = get_previous_epoch_attestations( - state, - config.SLOTS_PER_EPOCH, - config.GENESIS_EPOCH, - ) - previous_epoch_boundary_root = get_block_root( state, get_epoch_start_slot(previous_epoch, config.SLOTS_PER_EPOCH), @@ -176,7 +142,7 @@ def get_epoch_boundary_attesting_balances( previous_epoch_boundary_attester_indices = get_epoch_boundary_attester_indices( state, - current_epoch_attestations + previous_epoch_attestations, + state.current_epoch_attestations + state.previous_epoch_attestations, state.previous_justified_epoch, previous_epoch_boundary_root, CommitteeConfig(config), @@ -196,7 +162,7 @@ def get_epoch_boundary_attesting_balances( current_epoch_boundary_attester_indices = get_epoch_boundary_attester_indices( state, - current_epoch_attestations, + state.current_epoch_attestations, state.justified_epoch, current_epoch_boundary_root, CommitteeConfig(config), diff --git a/eth2/beacon/on_genesis.py b/eth2/beacon/on_genesis.py index 15ba2bb5f8..92594140c5 100644 --- a/eth2/beacon/on_genesis.py +++ b/eth2/beacon/on_genesis.py @@ -101,6 +101,8 @@ def get_genesis_beacon_state(*, current_shuffling_seed=ZERO_HASH32, # Finality + previous_epoch_attestations=(), + current_epoch_attestations=(), previous_justified_epoch=genesis_epoch, justified_epoch=genesis_epoch, justification_bitfield=0, @@ -113,7 +115,6 @@ def get_genesis_beacon_state(*, latest_block_roots=(ZERO_HASH32,) * latest_block_roots_length, latest_active_index_roots=(ZERO_HASH32,) * latest_active_index_roots_length, latest_slashed_balances=(Gwei(0),) * latest_slashed_exit_length, - latest_attestations=(), batched_block_roots=(), # Ethereum 1.0 chain data diff --git a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py index 12f2eeafc8..a22ea7620a 100644 --- a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py +++ b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py @@ -44,9 +44,7 @@ ) from eth2.beacon.epoch_processing_helpers import ( get_base_reward, - get_current_epoch_attestations, get_inclusion_infos, - get_previous_epoch_attestations, get_previous_epoch_head_attestations, get_winning_root, get_total_balance, @@ -59,7 +57,6 @@ get_effective_balance, get_epoch_start_slot, get_randao_mix, - slot_to_epoch, ) from eth2.beacon.validator_status_helpers import ( activate_validator, @@ -285,12 +282,6 @@ def process_crosslinks(state: BeaconState, config: BeaconConfig) -> BeaconState: Return resulting ``state`` """ latest_crosslinks = state.latest_crosslinks - previous_epoch_attestations = get_previous_epoch_attestations( - state, - config.SLOTS_PER_EPOCH, - config.GENESIS_EPOCH, - ) - current_epoch_attestations = get_current_epoch_attestations(state, config.SLOTS_PER_EPOCH) previous_epoch_start_slot = get_epoch_start_slot( state.previous_epoch(config.SLOTS_PER_EPOCH, config.GENESIS_EPOCH), config.SLOTS_PER_EPOCH, @@ -314,7 +305,7 @@ def process_crosslinks(state: BeaconState, config: BeaconConfig) -> BeaconState: # not attesting to this shard so we don't need to going over # irrelevent attestations over and over again. attestations=_filter_attestations_by_shard( - previous_epoch_attestations + current_epoch_attestations, + state.previous_epoch_attestations + state.current_epoch_attestations, shard, ), max_deposit_amount=config.MAX_DEPOSIT_AMOUNT, @@ -691,7 +682,6 @@ def _process_rewards_and_penalties_for_crosslinks( config.SLOTS_PER_EPOCH, ) # Also need current epoch attestations to compute the winning root. - current_epoch_attestations = get_current_epoch_attestations(state, config.SLOTS_PER_EPOCH) for slot in range(previous_epoch_start_slot, current_epoch_start_slot): crosslink_committees_at_slot = get_crosslink_committees_at_slot( state, @@ -700,7 +690,7 @@ def _process_rewards_and_penalties_for_crosslinks( ) for crosslink_committee, shard in crosslink_committees_at_slot: filtered_attestations = _filter_attestations_by_shard( - previous_epoch_attestations + current_epoch_attestations, + previous_epoch_attestations + state.current_epoch_attestations, shard, ) try: @@ -755,11 +745,7 @@ def process_rewards_and_penalties(state: BeaconState, config: BeaconConfig) -> B # Compute previous epoch attester indices and the total balance they account for # for later use. - previous_epoch_attestations = get_previous_epoch_attestations( - state, - config.SLOTS_PER_EPOCH, - config.GENESIS_EPOCH, - ) + previous_epoch_attestations = state.previous_epoch_attestations previous_epoch_attester_indices = get_attester_indices_from_attesttion( state=state, attestations=previous_epoch_attestations, @@ -1318,16 +1304,10 @@ def process_final_updates(state: BeaconState, ), ) - latest_attestations = tuple( - filter( - lambda attestation: ( - slot_to_epoch(attestation.data.slot, config.SLOTS_PER_EPOCH) >= current_epoch - ), - state.latest_attestations - ) - ) + # Rotate current/previous epoch attestations state = state.copy( - latest_attestations=latest_attestations, + previous_epoch_attestations=state.current_epoch_attestations, + current_epoch_attestations=(), ) return state diff --git a/eth2/beacon/state_machines/forks/serenity/operation_processing.py b/eth2/beacon/state_machines/forks/serenity/operation_processing.py index 6e4219653f..4117bf9526 100644 --- a/eth2/beacon/state_machines/forks/serenity/operation_processing.py +++ b/eth2/beacon/state_machines/forks/serenity/operation_processing.py @@ -10,10 +10,6 @@ BeaconConfig, CommitteeConfig, ) -from eth2.beacon.types.attester_slashings import AttesterSlashing -from eth2.beacon.types.blocks import BaseBeaconBlock -from eth2.beacon.types.pending_attestation_records import PendingAttestationRecord -from eth2.beacon.types.states import BeaconState from eth2.beacon.validator_status_helpers import ( initiate_validator_exit, slash_validator, @@ -21,6 +17,13 @@ from eth2.beacon.typing import ( ValidatorIndex, ) +from eth2.beacon.committee_helpers import ( + slot_to_epoch, +) +from eth2.beacon.types.attester_slashings import AttesterSlashing +from eth2.beacon.types.blocks import BaseBeaconBlock +from eth2.beacon.types.pending_attestation_records import PendingAttestationRecord +from eth2.beacon.types.states import BeaconState from .block_validation import ( validate_attestation, @@ -112,7 +115,8 @@ def process_attestations(state: BeaconState, Validate the ``attestations`` contained within the ``block`` in the context of ``state``. If any invalid, throw ``ValidationError``. - Otherwise, append an ``PendingAttestationRecords`` for each to ``latest_attestations``. + Otherwise, append an ``PendingAttestationRecords`` for each to ``previous_epoch_attestations`` + or ``current_epoch_attestations``. Return resulting ``state``. """ if len(block.body.attestations) > config.MAX_ATTESTATIONS: @@ -131,18 +135,38 @@ def process_attestations(state: BeaconState, CommitteeConfig(config), ) - # update_latest_attestations - additional_pending_attestations = tuple( - PendingAttestationRecord( - data=attestation.data, - aggregation_bitfield=attestation.aggregation_bitfield, - custody_bitfield=attestation.custody_bitfield, - slot_included=state.slot, - ) - for attestation in block.body.attestations - ) + # update attestations + previous_epoch = state.previous_epoch(config.SLOTS_PER_EPOCH, config.GENESIS_EPOCH) + current_epoch = state.current_epoch(config.SLOTS_PER_EPOCH) + new_previous_epoch_pending_attestations = [] + new_current_epoch_pending_attestations = [] + for attestation in block.body.attestations: + if slot_to_epoch(attestation.data.slot, config.SLOTS_PER_EPOCH) == current_epoch: + new_current_epoch_pending_attestations.append( + PendingAttestationRecord( + data=attestation.data, + aggregation_bitfield=attestation.aggregation_bitfield, + custody_bitfield=attestation.custody_bitfield, + slot_included=state.slot, + ) + ) + elif slot_to_epoch(attestation.data.slot, config.SLOTS_PER_EPOCH) == previous_epoch: + new_previous_epoch_pending_attestations.append( + PendingAttestationRecord( + data=attestation.data, + aggregation_bitfield=attestation.aggregation_bitfield, + custody_bitfield=attestation.custody_bitfield, + slot_included=state.slot, + ) + ) + state = state.copy( - latest_attestations=state.latest_attestations + additional_pending_attestations, + previous_epoch_attestations=( + state.previous_epoch_attestations + tuple(new_previous_epoch_pending_attestations) + ), + current_epoch_attestations=( + state.current_epoch_attestations + tuple(new_current_epoch_pending_attestations) + ), ) return state diff --git a/eth2/beacon/types/states.py b/eth2/beacon/types/states.py index 7f910337fc..8943d906aa 100644 --- a/eth2/beacon/types/states.py +++ b/eth2/beacon/types/states.py @@ -64,6 +64,9 @@ class BeaconState(ssz.Serializable): ('current_shuffling_seed', bytes32), # Finality + ('previous_epoch_attestations', List(PendingAttestationRecord)), + ('current_epoch_attestations', List(PendingAttestationRecord)), + ('previous_justified_epoch', uint64), ('justified_epoch', uint64), @@ -77,7 +80,6 @@ class BeaconState(ssz.Serializable): ('latest_block_roots', List(bytes32)), # Needed to process attestations, older to newer # noqa: E501 ('latest_active_index_roots', List(bytes32)), ('latest_slashed_balances', List(uint64)), # Balances slashed at every withdrawal period # noqa: E501 - ('latest_attestations', List(PendingAttestationRecord)), ('batched_block_roots', List(bytes32)), # allow for a log-sized Merkle proof from any block to any historical block root" # noqa: E501 # Ethereum 1.0 chain @@ -106,6 +108,8 @@ def __init__( previous_shuffling_seed: Hash32, current_shuffling_seed: Hash32, # Finality + previous_epoch_attestations: Sequence[PendingAttestationRecord], + current_epoch_attestations: Sequence[PendingAttestationRecord], previous_justified_epoch: Epoch, justified_epoch: Epoch, justification_bitfield: int, @@ -116,7 +120,6 @@ def __init__( latest_active_index_roots: Sequence[Hash32], latest_slashed_balances: Sequence[Gwei], batched_block_roots: Sequence[Hash32], - latest_attestations: Sequence[PendingAttestationRecord], # Ethereum 1.0 chain latest_eth1_data: Eth1Data, eth1_data_votes: Sequence[Eth1DataVote], @@ -143,6 +146,8 @@ def __init__( previous_shuffling_seed=previous_shuffling_seed, current_shuffling_seed=current_shuffling_seed, # Finality + previous_epoch_attestations=previous_epoch_attestations, + current_epoch_attestations=current_epoch_attestations, previous_justified_epoch=previous_justified_epoch, justified_epoch=justified_epoch, justification_bitfield=justification_bitfield, @@ -152,7 +157,6 @@ def __init__( latest_block_roots=latest_block_roots, latest_active_index_roots=latest_active_index_roots, latest_slashed_balances=latest_slashed_balances, - latest_attestations=latest_attestations, batched_block_roots=batched_block_roots, # Ethereum 1.0 chain latest_eth1_data=latest_eth1_data, @@ -225,6 +229,8 @@ def create_filled_state(cls, current_shuffling_seed=ZERO_HASH32, # Finality + previous_epoch_attestations=(), + current_epoch_attestations=(), previous_justified_epoch=genesis_epoch, justified_epoch=genesis_epoch, justification_bitfield=genesis_slot, @@ -242,7 +248,6 @@ def create_filled_state(cls, latest_block_roots=(ZERO_HASH32,) * latest_block_roots_length, latest_active_index_roots=(ZERO_HASH32,) * latest_active_index_roots_length, latest_slashed_balances=(Gwei(0),) * latest_slashed_exit_length, - latest_attestations=(), batched_block_roots=(), # Ethereum 1.0 chain data diff --git a/tests/eth2/beacon/conftest.py b/tests/eth2/beacon/conftest.py index 9024f1c3eb..94a5f59546 100644 --- a/tests/eth2/beacon/conftest.py +++ b/tests/eth2/beacon/conftest.py @@ -168,6 +168,8 @@ def sample_beacon_state_params(genesis_slot, 'current_shuffling_epoch': genesis_epoch, 'previous_shuffling_seed': b'\x77' * 32, 'current_shuffling_seed': b'\x88' * 32, + 'previous_epoch_attestations': (), + 'current_epoch_attestations': (), 'previous_justified_epoch': 0, 'justified_epoch': 0, 'justification_bitfield': 0, @@ -176,7 +178,6 @@ def sample_beacon_state_params(genesis_slot, 'latest_block_roots': (), 'latest_active_index_roots': (), 'latest_slashed_balances': (), - 'latest_attestations': (), 'batched_block_roots': (), 'latest_eth1_data': Eth1Data(**sample_eth1_data_params), 'eth1_data_votes': (), diff --git a/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py b/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py index d1a6ca9e9c..0cc433bd10 100644 --- a/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py +++ b/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py @@ -485,7 +485,7 @@ def test_process_crosslinks( ) state = state.copy( - latest_attestations=cur_epoch_attestations, + current_epoch_attestations=cur_epoch_attestations, ) assert (state.latest_crosslinks[shard].epoch == config.GENESIS_EPOCH and state.latest_crosslinks[shard].crosslink_data_root == ZERO_HASH32) @@ -690,7 +690,7 @@ def test_process_rewards_and_penalties_for_finality( ) ) state = state.copy( - latest_attestations=prev_epoch_attestations, + previous_epoch_attestations=prev_epoch_attestations, ) rewards_received = _process_rewards_and_penalties_for_finality( @@ -1621,41 +1621,41 @@ def test_update_latest_active_index_roots(genesis_state, @pytest.mark.parametrize( ( 'num_validators,' - 'state_slot,' - 'attestation_slot,' - 'len_latest_attestations,' - 'expected_result_len_latest_attestations,' 'slots_per_epoch' ), [ - (10, 4, 4, 2, 2, 4), # slot_to_epoch(attestation.data.slot) >= state.current_epoch, -> expected_result_len_latest_attestations = len_latest_attestations # noqa: E501 - (10, 4, 8, 2, 2, 4), # slot_to_epoch(attestation.data.slot) >= state.current_epoch, -> expected_result_len_latest_attestations = len_latest_attestations # noqa: E501 - (10, 16, 8, 2, 0, 4), # slot_to_epoch(attestation.data.slot) < state.current_epoch, -> expected_result_len_latest_attestations = 0 # noqa: E501 + (10, 4), ] ) def test_process_final_updates(genesis_state, - state_slot, - attestation_slot, - len_latest_attestations, - expected_result_len_latest_attestations, config, sample_attestation_params): + current_slot = 10 state = genesis_state.copy( - slot=state_slot, + slot=current_slot, ) current_index = state.next_epoch(config.SLOTS_PER_EPOCH) % config.LATEST_SLASHED_EXIT_LENGTH previous_index = state.current_epoch(config.SLOTS_PER_EPOCH) % config.LATEST_SLASHED_EXIT_LENGTH - # Assume `len_latest_attestations` attestations in state.latest_attestations - # with attestation.data.slot = attestation_slot attestation = Attestation(**sample_attestation_params) - latest_attestations = [ + previous_epoch_attestation_slot = current_slot - config.SLOTS_PER_EPOCH + num_previous_epoch_attestations = 2 + previous_epoch_attestations = [ + attestation.copy( + data=attestation.data.copy( + slot=previous_epoch_attestation_slot + ) + ) + for _ in range(num_previous_epoch_attestations) + ] + num_current_epoch_attestations = 3 + current_epoch_attestations = [ attestation.copy( data=attestation.data.copy( - slot=attestation_slot + slot=current_slot ) ) - for i in range(len_latest_attestations) + for _ in range(num_current_epoch_attestations) ] # Fill latest_slashed_balances @@ -1667,7 +1667,8 @@ def test_process_final_updates(genesis_state, ) state = state.copy( latest_slashed_balances=latest_slashed_balances, - latest_attestations=latest_attestations, + previous_epoch_attestations=previous_epoch_attestations, + current_epoch_attestations=current_epoch_attestations, ) result_state = process_final_updates(state, config) @@ -1686,6 +1687,6 @@ def test_process_final_updates(genesis_state, ) ) - assert len(result_state.latest_attestations) == expected_result_len_latest_attestations - for attestation in result_state.latest_attestations: - assert attestation.data.slot >= state_slot - config.SLOTS_PER_EPOCH + for attestation in result_state.previous_epoch_attestations: + assert attestation.data.slot == current_slot + assert len(result_state.current_epoch_attestations) == 0 diff --git a/tests/eth2/beacon/state_machines/forks/test_serenity_operation_processing.py b/tests/eth2/beacon/state_machines/forks/test_serenity_operation_processing.py index acab85c3ab..aec4874348 100644 --- a/tests/eth2/beacon/state_machines/forks/test_serenity_operation_processing.py +++ b/tests/eth2/beacon/state_machines/forks/test_serenity_operation_processing.py @@ -288,7 +288,7 @@ def test_process_attestations(genesis_state, config, ) - assert len(new_state.latest_attestations) == len(attestations) + assert len(new_state.current_epoch_attestations) == len(attestations) else: with pytest.raises(ValidationError): process_attestations( diff --git a/tests/eth2/beacon/test_epoch_processing_helpers.py b/tests/eth2/beacon/test_epoch_processing_helpers.py index d257eeeb00..fa9c4b45a8 100644 --- a/tests/eth2/beacon/test_epoch_processing_helpers.py +++ b/tests/eth2/beacon/test_epoch_processing_helpers.py @@ -23,11 +23,9 @@ ) from eth2.beacon.configs import CommitteeConfig from eth2.beacon.epoch_processing_helpers import ( - get_current_epoch_attestations, get_epoch_boundary_attester_indices, get_epoch_boundary_attesting_balances, get_inclusion_infos, - get_previous_epoch_attestations, get_previous_epoch_head_attestations, get_total_balance, get_winning_root, @@ -119,12 +117,11 @@ def test_get_current_and_previous_epoch_attestations(random, state = sample_state.copy( slot=(slots_per_epoch * 2 - 1), - latest_attestations=(previous_epoch_attestations + current_epoch_attestations), + previous_epoch_attestations=previous_epoch_attestations, + current_epoch_attestations=current_epoch_attestations, ) - assert set(previous_epoch_attestations) == set( - get_previous_epoch_attestations(state, slots_per_epoch, genesis_epoch)) - assert set(current_epoch_attestations) == set( - get_current_epoch_attestations(state, slots_per_epoch)) + assert set(previous_epoch_attestations) == set(state.previous_epoch_attestations) + assert set(current_epoch_attestations) == set(state.current_epoch_attestations) @settings(max_examples=1) @@ -190,11 +187,12 @@ def test_get_previous_epoch_head_attestations( ) ) - latest_attestations = previous_epoch_head_attestations + previous_epoch_not_head_attestations state = sample_state.copy( slot=current_slot, latest_block_roots=latest_block_roots, - latest_attestations=latest_attestations, + previous_epoch_attestations=( + previous_epoch_head_attestations + previous_epoch_not_head_attestations + ), ) result = get_previous_epoch_head_attestations( @@ -550,7 +548,8 @@ def mock_get_crosslink_committees_at_slot(state, slot=slot, justified_epoch=justified_epoch, previous_justified_epoch=previous_justified_epoch, - latest_attestations=current_epoch_attestations + previous_epoch_attestations, + previous_epoch_attestations=previous_epoch_attestations, + current_epoch_attestations=current_epoch_attestations, latest_block_roots=tuple(latest_block_roots), ) ( diff --git a/tests/eth2/beacon/test_on_genesis.py b/tests/eth2/beacon/test_on_genesis.py index c52e177497..6fcfb916cf 100644 --- a/tests/eth2/beacon/test_on_genesis.py +++ b/tests/eth2/beacon/test_on_genesis.py @@ -146,6 +146,8 @@ def test_get_genesis_beacon_state( assert state.previous_shuffling_seed == ZERO_HASH32 # Finality + assert len(state.previous_epoch_attestations) == 0 + assert len(state.current_epoch_attestations) == 0 assert state.previous_justified_epoch == genesis_epoch assert state.justified_epoch == genesis_epoch assert state.justification_bitfield == 0 @@ -162,7 +164,6 @@ def test_get_genesis_beacon_state( assert len(state.latest_slashed_balances) == latest_slashed_exit_length assert state.latest_slashed_balances[0] == Gwei(0) - assert len(state.latest_attestations) == 0 assert len(state.batched_block_roots) == 0 # Ethereum 1.0 chain data From 63a3b02c83c037c93954456feb7ebd898f071174 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Sat, 9 Mar 2019 01:07:24 +0800 Subject: [PATCH 02/23] Update `get_winning_root` --- eth2/beacon/epoch_processing_helpers.py | 6 ++-- .../forks/serenity/epoch_processing.py | 35 ++++++++++++++----- .../beacon/test_epoch_processing_helpers.py | 16 +++++++-- 3 files changed, 42 insertions(+), 15 deletions(-) diff --git a/eth2/beacon/epoch_processing_helpers.py b/eth2/beacon/epoch_processing_helpers.py index a0921f2ebe..d7d67c12b2 100644 --- a/eth2/beacon/epoch_processing_helpers.py +++ b/eth2/beacon/epoch_processing_helpers.py @@ -71,7 +71,6 @@ def get_previous_epoch_head_attestations( def get_winning_root( *, state: 'BeaconState', - shard: Shard, attestations: Sequence[PendingAttestationRecord], max_deposit_amount: Gwei, committee_config: CommitteeConfig) -> Tuple[Hash32, Gwei]: @@ -80,7 +79,6 @@ def get_winning_root( crosslink_data_roots = set( [ a.data.crosslink_data_root for a in attestations - if a.data.shard == shard ] ) for crosslink_data_root in crosslink_data_roots: @@ -89,7 +87,7 @@ def get_winning_root( attestations=[ a for a in attestations - if a.data.shard == shard and a.data.crosslink_data_root == crosslink_data_root + if a.data.crosslink_data_root == crosslink_data_root ], committee_config=committee_config, ) @@ -102,7 +100,7 @@ def get_winning_root( winning_root = crosslink_data_root winning_root_balance = total_attesting_balance elif total_attesting_balance == winning_root_balance and winning_root_balance > 0: - if crosslink_data_root < winning_root: + if crosslink_data_root > winning_root: winning_root = crosslink_data_root if winning_root is None: diff --git a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py index a22ea7620a..d08407bc64 100644 --- a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py +++ b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py @@ -37,6 +37,7 @@ get_beacon_proposer_index, get_crosslink_committees_at_slot, get_current_epoch_committee_count, + slot_to_epoch, ) from eth2.beacon.configs import ( BeaconConfig, @@ -262,6 +263,20 @@ def process_justification(state: BeaconState, config: BeaconConfig) -> BeaconSta # # Crosslinks # +@to_tuple +def _filter_attestations_by_latest_crosslinks( + attestations: Sequence[Attestation], + latest_crosslink: CrosslinkRecord) -> Iterable[Attestation]: + for attestation in attestations: + # if attestation.data.latest_crosslink == latest_crosslink: + epoch_matched = attestation.data.latest_crosslink.epoch == latest_crosslink.epoch + crosslink_data_root_matched = ( + attestation.data.latest_crosslink.crosslink_data_root == latest_crosslink.crosslink_data_root + ) + if (epoch_matched and crosslink_data_root_matched): + yield attestation + + @to_tuple def _filter_attestations_by_shard( attestations: Sequence[Attestation], @@ -300,13 +315,15 @@ def process_crosslinks(state: BeaconState, config: BeaconConfig) -> BeaconState: try: winning_root, total_attesting_balance = get_winning_root( state=state, - shard=shard, # Use `_filter_attestations_by_shard` to filter out attestations # not attesting to this shard so we don't need to going over # irrelevent attestations over and over again. - attestations=_filter_attestations_by_shard( - state.previous_epoch_attestations + state.current_epoch_attestations, - shard, + attestations=_filter_attestations_by_latest_crosslinks( + _filter_attestations_by_shard( + state.previous_epoch_attestations + state.current_epoch_attestations, + shard, + ), + state.latest_crosslinks[shard], ), max_deposit_amount=config.MAX_DEPOSIT_AMOUNT, committee_config=CommitteeConfig(config), @@ -689,14 +706,16 @@ def _process_rewards_and_penalties_for_crosslinks( CommitteeConfig(config), ) for crosslink_committee, shard in crosslink_committees_at_slot: - filtered_attestations = _filter_attestations_by_shard( - previous_epoch_attestations + state.current_epoch_attestations, - shard, + filtered_attestations = _filter_attestations_by_latest_crosslinks( + _filter_attestations_by_shard( + previous_epoch_attestations + state.current_epoch_attestations, + shard, + ), + state.latest_crosslinks[shard], ) try: winning_root, total_attesting_balance = get_winning_root( state=state, - shard=shard, attestations=filtered_attestations, max_deposit_amount=config.MAX_DEPOSIT_AMOUNT, committee_config=CommitteeConfig(config), diff --git a/tests/eth2/beacon/test_epoch_processing_helpers.py b/tests/eth2/beacon/test_epoch_processing_helpers.py index fa9c4b45a8..4eddf2f7fc 100644 --- a/tests/eth2/beacon/test_epoch_processing_helpers.py +++ b/tests/eth2/beacon/test_epoch_processing_helpers.py @@ -10,6 +10,8 @@ strategies as st, ) +from eth.constants import ZERO_HASH32 + from eth2._utils.bitfield import ( set_voted, get_empty_bitfield, @@ -40,6 +42,7 @@ from eth2.beacon.types.attestation_data import ( AttestationData, ) +from eth2.beacon.types.crosslink_records import CrosslinkRecord from eth2.beacon.types.pending_attestation_records import PendingAttestationRecord @@ -220,7 +223,7 @@ def test_get_previous_epoch_head_attestations( ), ( 16, - # vote tie; lower root value is favored + # vote tie; higher root value is favored (1, 3, 5, 7), (2, 4, 6, 8) ), @@ -284,6 +287,10 @@ def mock_get_crosslink_committees_at_slot(state, data=AttestationData(**sample_attestation_data_params).copy( shard=shard, crosslink_data_root=competing_block_roots[0], + latest_crosslink=CrosslinkRecord( + epoch=config.GENESIS_EPOCH, + crosslink_data_root=ZERO_HASH32, + ), ), ), # Attestation to `crosslink_data_root_2` by `attestation_participants_2` @@ -292,6 +299,10 @@ def mock_get_crosslink_committees_at_slot(state, data=AttestationData(**sample_attestation_data_params).copy( shard=shard, crosslink_data_root=competing_block_roots[1], + latest_crosslink=CrosslinkRecord( + epoch=config.GENESIS_EPOCH, + crosslink_data_root=ZERO_HASH32, + ), ), ), ) @@ -299,7 +310,6 @@ def mock_get_crosslink_committees_at_slot(state, try: winning_root, attesting_balance = get_winning_root( state=n_validators_state, - shard=shard, attestations=attestations, max_deposit_amount=config.MAX_DEPOSIT_AMOUNT, committee_config=committee_config, @@ -323,7 +333,7 @@ def mock_get_crosslink_committees_at_slot(state, assert len(block_root_1_participants) == 0 and len(block_root_2_participants) == 0 else: if len(block_root_1_participants) == len(block_root_2_participants): - if competing_block_roots[0] < competing_block_roots[1]: + if competing_block_roots[0] > competing_block_roots[1]: assert winning_root == competing_block_roots[0] else: assert winning_root == competing_block_roots[1] From a3ac0caf2243e6477b43f05e89a1cee2bfbff819 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Sat, 9 Mar 2019 20:44:27 +0800 Subject: [PATCH 03/23] Update `process_crosslinks` --- .../forks/serenity/epoch_processing.py | 11 +- .../forks/test_serenity_epoch_processing.py | 112 ++++++++++++++---- 2 files changed, 92 insertions(+), 31 deletions(-) diff --git a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py index d08407bc64..d202142b6e 100644 --- a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py +++ b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py @@ -268,12 +268,7 @@ def _filter_attestations_by_latest_crosslinks( attestations: Sequence[Attestation], latest_crosslink: CrosslinkRecord) -> Iterable[Attestation]: for attestation in attestations: - # if attestation.data.latest_crosslink == latest_crosslink: - epoch_matched = attestation.data.latest_crosslink.epoch == latest_crosslink.epoch - crosslink_data_root_matched = ( - attestation.data.latest_crosslink.crosslink_data_root == latest_crosslink.crosslink_data_root - ) - if (epoch_matched and crosslink_data_root_matched): + if attestation.data.latest_crosslink == latest_crosslink: yield attestation @@ -342,7 +337,7 @@ def process_crosslinks(state: BeaconState, config: BeaconConfig) -> BeaconState: latest_crosslinks, shard, CrosslinkRecord( - epoch=state.current_epoch(config.SLOTS_PER_EPOCH), + epoch=slot_to_epoch(slot, config.SLOTS_PER_EPOCH), crosslink_data_root=winning_root, ), ) @@ -730,7 +725,7 @@ def _process_rewards_and_penalties_for_crosslinks( attestations=( a for a in filtered_attestations - if a.data.shard == shard and a.data.crosslink_data_root == winning_root + if a.data.crosslink_data_root == winning_root ), committee_config=CommitteeConfig(config), ) diff --git a/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py b/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py index 0cc433bd10..6f7a55b355 100644 --- a/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py +++ b/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py @@ -404,7 +404,6 @@ def mock_current_previous_epochs_justifiable(current_epoch, previous_epoch, stat 'slots_per_epoch,' 'target_committee_size,' 'shard_count,' - 'success_crosslink_in_cur_epoch,' 'genesis_slot,' ), [ @@ -413,16 +412,27 @@ def mock_current_previous_epochs_justifiable(current_epoch, previous_epoch, stat 10, 9, 10, - False, 0, ), + ] +) +@pytest.mark.parametrize( + ( + 'success_crosslink_in_previous_epoch,' + 'success_crosslink_in_current_epoch,' + ), + [ + ( + False, + False, + ), ( - 90, - 10, - 9, - 10, True, - 0, + False, + ), + ( + False, + True, ), ] ) @@ -433,11 +443,13 @@ def test_process_crosslinks( slots_per_epoch, target_committee_size, shard_count, - success_crosslink_in_cur_epoch, + success_crosslink_in_previous_epoch, + success_crosslink_in_current_epoch, sample_attestation_data_params, sample_attestation_params): shard = 1 - crosslink_data_root = hash_eth2(b'crosslink_data_root') + previous_epoch_crosslink_data_root = hash_eth2(b'previous_epoch_crosslink_data_root') + current_epoch_crosslink_data_root = hash_eth2(b'current_epoch_crosslink_data_root') current_slot = config.SLOTS_PER_EPOCH * 2 - 1 genesis_crosslinks = tuple([ @@ -449,21 +461,65 @@ def test_process_crosslinks( latest_crosslinks=genesis_crosslinks, ) + # Generate previous epoch attestations + current_epoch = state.current_epoch(config.SLOTS_PER_EPOCH) + previous_epoch = current_epoch - 1 + previous_epoch_start_slot = get_epoch_start_slot(previous_epoch, config.SLOTS_PER_EPOCH) + current_epoch_start_slot = get_epoch_start_slot(current_epoch, config.SLOTS_PER_EPOCH) + previous_epoch_attestations = [] + for slot_in_previous_epoch in range(previous_epoch_start_slot, current_epoch_start_slot): + if len(previous_epoch_attestations) > 0: + break + for committee, _shard in get_crosslink_committees_at_slot( + state, + slot_in_previous_epoch, + CommitteeConfig(config), + ): + if _shard == shard: + # Sample validators attesting to this shard. + # Number of attesting validators sampled depends on `success_crosslink_in_previous_epoch` + # if True, have >2/3 committee attest + if success_crosslink_in_previous_epoch: + attesting_validators = random.sample(committee, (2 * len(committee) // 3 + 1)) + else: + attesting_validators = random.sample(committee, (2 * len(committee) // 3 - 1)) + # Generate the bitfield + aggregation_bitfield = get_empty_bitfield(len(committee)) + for v_index in attesting_validators: + aggregation_bitfield = set_voted( + aggregation_bitfield, committee.index(v_index)) + # Generate the attestation + previous_epoch_attestations.append( + Attestation(**sample_attestation_params).copy( + aggregation_bitfield=aggregation_bitfield, + data=AttestationData(**sample_attestation_data_params).copy( + slot=slot_in_previous_epoch, + shard=shard, + crosslink_data_root=previous_epoch_crosslink_data_root, + latest_crosslink=CrosslinkRecord( + epoch=config.GENESIS_EPOCH, + crosslink_data_root=ZERO_HASH32, + ), + ), + ) + ) + # Generate current epoch attestations - cur_epoch_attestations = [] - for slot_in_cur_epoch in range(state.slot - config.SLOTS_PER_EPOCH, state.slot): - if len(cur_epoch_attestations) > 0: + next_epoch_start_slot = current_epoch_start_slot + config.SLOTS_PER_EPOCH + current_epoch_attestations = [] + for slot_in_current_epoch in range(current_epoch_start_slot, next_epoch_start_slot): + if len(current_epoch_attestations) > 0: break for committee, _shard in get_crosslink_committees_at_slot( state, - slot_in_cur_epoch, + slot_in_current_epoch, CommitteeConfig(config), ): if _shard == shard: # Sample validators attesting to this shard. - # Number of attesting validators sampled depends on `success_crosslink_in_cur_epoch` + # Number of attesting validators sampled depends on `success_crosslink_in_current_epoch` # if True, have >2/3 committee attest - if success_crosslink_in_cur_epoch: + if success_crosslink_in_current_epoch: attesting_validators = random.sample(committee, (2 * len(committee) // 3 + 1)) else: attesting_validators = random.sample(committee, (2 * len(committee) // 3 - 1)) @@ -473,30 +529,40 @@ def test_process_crosslinks( aggregation_bitfield = set_voted( aggregation_bitfield, committee.index(v_index)) # Generate the attestation - cur_epoch_attestations.append( + current_epoch_attestations.append( Attestation(**sample_attestation_params).copy( aggregation_bitfield=aggregation_bitfield, data=AttestationData(**sample_attestation_data_params).copy( - slot=slot_in_cur_epoch, + slot=slot_in_current_epoch, shard=shard, - crosslink_data_root=crosslink_data_root, + crosslink_data_root=current_epoch_crosslink_data_root, + latest_crosslink=CrosslinkRecord( + epoch=config.GENESIS_EPOCH, + crosslink_data_root=ZERO_HASH32, + ), ), ) ) state = state.copy( - current_epoch_attestations=cur_epoch_attestations, + previous_epoch_attestations=previous_epoch_attestations, + current_epoch_attestations=current_epoch_attestations, ) assert (state.latest_crosslinks[shard].epoch == config.GENESIS_EPOCH and state.latest_crosslinks[shard].crosslink_data_root == ZERO_HASH32) new_state = process_crosslinks(state, config) crosslink_record = new_state.latest_crosslinks[shard] - if success_crosslink_in_cur_epoch: - attestation = cur_epoch_attestations[0] - assert (crosslink_record.epoch == slot_to_epoch(current_slot, slots_per_epoch) and + if success_crosslink_in_current_epoch: + attestation = current_epoch_attestations[0] + assert (crosslink_record.epoch == current_epoch and + crosslink_record.crosslink_data_root == attestation.data.crosslink_data_root and + attestation.data.crosslink_data_root == current_epoch_crosslink_data_root) + elif success_crosslink_in_previous_epoch: + attestation = previous_epoch_attestations[0] + assert (crosslink_record.epoch == current_epoch and crosslink_record.crosslink_data_root == attestation.data.crosslink_data_root and - attestation.data.crosslink_data_root == crosslink_data_root) + attestation.data.crosslink_data_root == previous_epoch_crosslink_data_root) else: assert (crosslink_record.epoch == config.GENESIS_EPOCH and crosslink_record.crosslink_data_root == ZERO_HASH32) From 96879ae37dfbe73203d99d4332f9e57e909b1594 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Sat, 9 Mar 2019 21:02:51 +0800 Subject: [PATCH 04/23] Expand rewards/penalites map to entire registry and fix latest_crosslinks in test --- .../state_machines/forks/serenity/epoch_processing.py | 8 ++++---- .../forks/test_serenity_epoch_processing.py | 8 ++++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py index d202142b6e..274ef78889 100644 --- a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py +++ b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py @@ -780,7 +780,7 @@ def process_rewards_and_penalties(state: BeaconState, config: BeaconConfig) -> B index, config.MAX_DEPOSIT_AMOUNT, ) - for index in previous_epoch_active_validator_indices + for index in range(len(state.validator_registry)) } # Compute base reward of each previous epoch active validator for later use _base_reward_quotient = ( @@ -793,13 +793,13 @@ def process_rewards_and_penalties(state: BeaconState, config: BeaconConfig) -> B base_reward_quotient=_base_reward_quotient, max_deposit_amount=config.MAX_DEPOSIT_AMOUNT, ) - for index in previous_epoch_active_validator_indices + for index in range(len(state.validator_registry)) } # Initialize the reward (validator) received map rewards_received = { index: SignedGwei(0) - for index in previous_epoch_active_validator_indices + for index in range(len(state.validator_registry)) } # 1. Process rewards and penalties for justification and finalization @@ -833,7 +833,7 @@ def process_rewards_and_penalties(state: BeaconState, config: BeaconConfig) -> B ) # Apply the overall rewards/penalties - for index in previous_epoch_active_validator_indices: + for index in range(len(state.validator_registry)): state = state.update_validator_balance( index, # Prevent validator balance under flow diff --git a/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py b/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py index 6f7a55b355..ea452ae1d9 100644 --- a/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py +++ b/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py @@ -705,12 +705,12 @@ def test_process_rewards_and_penalties_for_finality( effective_balances = { index: effective_balance - for index in previous_epoch_active_validator_indices + for index in range(len(state.validator_registry)) } base_rewards = { index: base_reward - for index in previous_epoch_active_validator_indices + for index in range(len(state.validator_registry)) } rewards_received = { @@ -993,6 +993,10 @@ def test_process_rewards_and_penalties_for_crosslinks( data=AttestationData(**sample_attestation_data_params).copy( slot=data_slot, shard=shard, + latest_crosslink=CrosslinkRecord( + epoch=config.GENESIS_EPOCH, + crosslink_data_root=ZERO_HASH32, + ), ), aggregation_bitfield=participants_bitfield, slot_included=(data_slot + min_attestation_inclusion_delay), From f9bde80a143c43c961b5e3ce18f670201c841342 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Sat, 9 Mar 2019 21:13:25 +0800 Subject: [PATCH 05/23] Move penalties for `attesting_active_validators` section up for better readability --- .../forks/serenity/epoch_processing.py | 42 ++++++++++--------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py index 274ef78889..9410266677 100644 --- a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py +++ b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py @@ -550,6 +550,28 @@ def _process_rewards_and_penalties_for_finality( # epochs_since_finality > 4 else: + # Punish active validators in `previous_epoch_attester_indices` + attesting_active_validators_indices = previous_epoch_active_validator_indices.intersection( + previous_epoch_attester_indices, + ) + penalties = { + index: Gwei( + base_rewards[index] - ( + base_rewards[index] * + config.MIN_ATTESTATION_INCLUSION_DELAY // + inclusion_infos[index].inclusion_distance + ) + ) + for index in attesting_active_validators_indices + } + rewards_received, penalties_received = _apply_rewards_and_penalties( + RewardSettlementContext( + penalties=penalties, + indices_to_penalize=attesting_active_validators_indices, + rewards_received=rewards_received, + penalties_received=penalties_received, + ), + ) # Punish active validators not in `previous_epoch_attester_indices` excluded_active_validators_indices = previous_epoch_active_validator_indices.difference( previous_epoch_attester_indices, @@ -626,26 +648,6 @@ def _process_rewards_and_penalties_for_finality( ), ) - # Punish validators in `previous_epoch_attester_indices` - penalties = { - index: Gwei( - base_rewards[index] - ( - base_rewards[index] * - config.MIN_ATTESTATION_INCLUSION_DELAY // - inclusion_infos[index].inclusion_distance - ) - ) - for index in previous_epoch_attester_indices - } - rewards_received, penalties_received = _apply_rewards_and_penalties( - RewardSettlementContext( - penalties=penalties, - indices_to_penalize=previous_epoch_attester_indices, - rewards_received=rewards_received, - penalties_received=penalties_received, - ), - ) - historical_rewards_received = old_rewards_received.copy() for index in rewards_received: historical_rewards_received = _update_rewards_or_penalies( From 7bf7c31a2df7fccd7d7fff524571cbeec3348797 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Sat, 9 Mar 2019 22:51:23 +0800 Subject: [PATCH 06/23] Update `_process_rewards_and_penalties_for_finality` --- eth2/beacon/epoch_processing_helpers.py | 1 - .../forks/serenity/epoch_processing.py | 32 +++++--- .../forks/test_serenity_epoch_processing.py | 76 +++++++++---------- 3 files changed, 59 insertions(+), 50 deletions(-) diff --git a/eth2/beacon/epoch_processing_helpers.py b/eth2/beacon/epoch_processing_helpers.py index d7d67c12b2..afdd9f1939 100644 --- a/eth2/beacon/epoch_processing_helpers.py +++ b/eth2/beacon/epoch_processing_helpers.py @@ -35,7 +35,6 @@ from eth2.beacon.typing import ( Epoch, Gwei, - Shard, ValidatorIndex, ) diff --git a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py index 9410266677..aefbe1997d 100644 --- a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py +++ b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py @@ -74,11 +74,13 @@ from eth2.beacon.types.eth1_data_vote import Eth1DataVote from eth2.beacon.types.pending_attestation_records import PendingAttestationRecord from eth2.beacon.types.states import BeaconState +from eth2.beacon.types.validator_records import ValidatorRecord from eth2.beacon.typing import ( Epoch, Gwei, Shard, SignedGwei, + Slot, ValidatorIndex, ) @@ -337,7 +339,7 @@ def process_crosslinks(state: BeaconState, config: BeaconConfig) -> BeaconState: latest_crosslinks, shard, CrosslinkRecord( - epoch=slot_to_epoch(slot, config.SLOTS_PER_EPOCH), + epoch=slot_to_epoch(slot, Slot(config.SLOTS_PER_EPOCH)), crosslink_data_root=winning_root, ), ) @@ -381,6 +383,12 @@ def _apply_rewards_and_penalties( return rewards_received, penalties_received +def _is_eligible_for_punishment( + validator: ValidatorRecord, + current_epoch: Epoch) -> bool: + return validator.slashed and validator.withdrawable_epoch > current_epoch + + @curry def _process_rewards_and_penalties_for_finality( state: BeaconState, @@ -630,14 +638,18 @@ def _process_rewards_and_penalties_for_finality( ) # Punish penalized active validators + current_epoch = state.current_epoch(config.SLOTS_PER_EPOCH) penalties = { - index: 3 * base_rewards[index] + 2 * ( - effective_balances[index] * + ValidatorIndex(index): 3 * base_rewards[ValidatorIndex(index)] + 2 * ( + effective_balances[ValidatorIndex(index)] * epochs_since_finality // config.INACTIVITY_PENALTY_QUOTIENT // 2 ) - for index in previous_epoch_active_validator_indices - if state.validator_registry[index].slashed is True + for index in range(len(state.validator_registry)) + if ( + (index not in previous_epoch_active_validator_indices) and + _is_eligible_for_punishment(state.validator_registry[index], current_epoch) + ) } rewards_received, penalties_received = _apply_rewards_and_penalties( RewardSettlementContext( @@ -777,9 +789,9 @@ def process_rewards_and_penalties(state: BeaconState, config: BeaconConfig) -> B # Compute effective balance of each previous epoch active validator for later use effective_balances = { - index: get_effective_balance( + ValidatorIndex(index): get_effective_balance( state.validator_balances, - index, + ValidatorIndex(index), config.MAX_DEPOSIT_AMOUNT, ) for index in range(len(state.validator_registry)) @@ -789,9 +801,9 @@ def process_rewards_and_penalties(state: BeaconState, config: BeaconConfig) -> B integer_squareroot(previous_total_balance) // config.BASE_REWARD_QUOTIENT ) base_rewards = { - index: get_base_reward( + ValidatorIndex(index): get_base_reward( state=state, - index=index, + index=ValidatorIndex(index), base_reward_quotient=_base_reward_quotient, max_deposit_amount=config.MAX_DEPOSIT_AMOUNT, ) @@ -837,7 +849,7 @@ def process_rewards_and_penalties(state: BeaconState, config: BeaconConfig) -> B # Apply the overall rewards/penalties for index in range(len(state.validator_registry)): state = state.update_validator_balance( - index, + ValidatorIndex(index), # Prevent validator balance under flow max(state.validator_balances[index] + rewards_received[index], 0), ) diff --git a/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py b/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py index ea452ae1d9..be944a4fda 100644 --- a/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py +++ b/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py @@ -477,8 +477,7 @@ def test_process_crosslinks( ): if _shard == shard: # Sample validators attesting to this shard. - # Number of attesting validators sampled depends on `success_crosslink_in_previous_epoch` - # if True, have >2/3 committee attest + # if `success_crosslink_in_previous_epoch` is True, have >2/3 committee attest if success_crosslink_in_previous_epoch: attesting_validators = random.sample(committee, (2 * len(committee) // 3 + 1)) else: @@ -517,8 +516,7 @@ def test_process_crosslinks( ): if _shard == shard: # Sample validators attesting to this shard. - # Number of attesting validators sampled depends on `success_crosslink_in_current_epoch` - # if True, have >2/3 committee attest + # if `success_crosslink_in_current_epoch` is True, have >2/3 committee attest if success_crosslink_in_current_epoch: attesting_validators = random.sample(committee, (2 * len(committee) // 3 + 1)) else: @@ -605,36 +603,36 @@ def test_process_crosslinks( 'expected_rewards_received' ), [ - ( - 4, 15, # epochs_since_finality <= 4 - {6, 7}, - {0, 1, 2, 3, 4, 5, 6, 7}, - {2, 3, 4, 5, 6}, - {2, 3, 4}, - { - 2: 4, - 3: 4, - 4: 4, - 5: 5, - 6: 6, - }, - 1000, 100, - { - 0: -300, # -3 * 100 - 1: -300, # -3 * 100 - 2: 236, # 100 * 5 // 8 + 100 * 3 // 8 + 100 * 3 // 8 + 100 * 4 // 4 - 3: 236, # 100 * 5 // 8 + 100 * 3 // 8 + 100 * 3 // 8 + 100 * 4 // 4 - 4: 236, # 100 * 5 // 8 + 100 * 3 // 8 + 100 * 3 // 8 + 100 * 4 // 4 - 5: -58, # 100 * 5 // 8 - 100 - 100 + 100 * 4 // 5 - 6: -72, # 100 * 5 // 5 - 100 - 100 + 100 * 4 // 6 - 7: -300, # -3 * 100 - 8: 0, # not active - 9: 0, # not active - } - ), + # ( + # 4, 15, # epochs_since_finality <= 4 + # {8, 9}, + # {0, 1, 2, 3, 4, 5, 6, 7}, + # {2, 3, 4, 5, 6}, + # {2, 3, 4}, + # { + # 2: 4, + # 3: 4, + # 4: 4, + # 5: 5, + # 6: 6, + # }, + # 1000, 100, + # { + # 0: -300, # -3 * 100 + # 1: -300, # -3 * 100 + # 2: 236, # 100 * 5 // 8 + 100 * 3 // 8 + 100 * 3 // 8 + 100 * 4 // 4 + # 3: 236, # 100 * 5 // 8 + 100 * 3 // 8 + 100 * 3 // 8 + 100 * 4 // 4 + # 4: 236, # 100 * 5 // 8 + 100 * 3 // 8 + 100 * 3 // 8 + 100 * 4 // 4 + # 5: -58, # 100 * 5 // 8 - 100 - 100 + 100 * 4 // 5 + # 6: -72, # 100 * 5 // 5 - 100 - 100 + 100 * 4 // 6 + # 7: -300, # -3 * 100 + # 8: 0, # not active + # 9: 0, # not active + # } + # ), ( 3, 15, # epochs_since_finality > 4 - {6, 7}, + {8, 9}, {0, 1, 2, 3, 4, 5, 6, 7}, {2, 3, 4, 5, 6}, {2, 3, 4}, @@ -647,16 +645,16 @@ def test_process_crosslinks( }, 1000, 100, { - 0: -800, # -2 * (100 + 1000 * 5 // 10 // 2) - 100 - (100 - 100 * 4 // 4) - 1: -800, # -2 * (100 + 1000 * 5 // 10 // 2) - 100 - (100 - 100 * 4 // 4) + 0: -800, # -(100 - 100 * 4 // 4) - 2 * (100 + 1000 * 5 // 10 // 2) - 100 + 1: -800, # -(100 - 100 * 4 // 4) - 2 * (100 + 1000 * 5 // 10 // 2) - 100 2: 0, # -(100 - 100 * 4 // 4) 3: 0, # -(100 - 100 * 4 // 4) 4: 0, # -(100 - 100 * 4 // 4) - 5: -470, # -(100 * 2 + 1000 * 5 // 10 // 2) - (100 - 100 * 4 // 5) - 6: -1284, # -(100 * 2 + 1000 * 5 // 10 // 2) - (2 * (100 + 1000 * 5 // 10 // 2) + 100) - (100 - 100 * 4 // 6) # noqa: E501 - 7: -1600, # -2 * (100 + 1000 * 5 // 10 // 2) - 100 - (2 * (100 + 1000 * 5 // 10 // 2) + 100) - (100 - 100 * 4 // 4) # noqa: E501 - 8: 0, # not active - 9: 0, # not active + 5: -470, # -(100 - 100 * 4 // 5) - (100 * 2 + 1000 * 5 // 10 // 2) + 6: -484, # -(100 - 100 * 4 // 6) - (100 * 2 + 1000 * 5 // 10 // 2) + 7: -800, # -(100 - 100 * 4 // 4) - 2 * (100 + 1000 * 5 // 10 // 2) - 100 + 8: -800, # -(2 * (100 + 1000 * 5 // 10 // 2) + 100) + 9: -800, # -(2 * (100 + 1000 * 5 // 10 // 2) + 100) } ), ] From b96f3b4ad1910e554a4c13e8512c94555abc5483 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Sat, 9 Mar 2019 23:26:41 +0800 Subject: [PATCH 07/23] Handle `get_base_reward` when no previous balance --- eth2/beacon/epoch_processing_helpers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eth2/beacon/epoch_processing_helpers.py b/eth2/beacon/epoch_processing_helpers.py index afdd9f1939..240b72f0a3 100644 --- a/eth2/beacon/epoch_processing_helpers.py +++ b/eth2/beacon/epoch_processing_helpers.py @@ -190,6 +190,8 @@ def get_base_reward( index: ValidatorIndex, base_reward_quotient: int, max_deposit_amount: Gwei) -> Gwei: + if base_reward_quotient == 0: + return 0 return Gwei( get_effective_balance( state.validator_balances, From 3afb6a441ff8d5ef47f1cd4ea92e96423a99fd4d Mon Sep 17 00:00:00 2001 From: NIC619 Date: Sat, 9 Mar 2019 23:39:34 +0800 Subject: [PATCH 08/23] Fix linting error and move `_is_eligible_for_punishment` into `_process_rewards_and_penalties_for_finality` --- eth2/beacon/epoch_processing_helpers.py | 2 +- .../forks/serenity/epoch_processing.py | 14 ++++++-------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/eth2/beacon/epoch_processing_helpers.py b/eth2/beacon/epoch_processing_helpers.py index 240b72f0a3..b52c17e060 100644 --- a/eth2/beacon/epoch_processing_helpers.py +++ b/eth2/beacon/epoch_processing_helpers.py @@ -191,7 +191,7 @@ def get_base_reward( base_reward_quotient: int, max_deposit_amount: Gwei) -> Gwei: if base_reward_quotient == 0: - return 0 + return Gwei(0) return Gwei( get_effective_balance( state.validator_balances, diff --git a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py index aefbe1997d..2333435c59 100644 --- a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py +++ b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py @@ -339,7 +339,7 @@ def process_crosslinks(state: BeaconState, config: BeaconConfig) -> BeaconState: latest_crosslinks, shard, CrosslinkRecord( - epoch=slot_to_epoch(slot, Slot(config.SLOTS_PER_EPOCH)), + epoch=slot_to_epoch(Slot(slot), config.SLOTS_PER_EPOCH), crosslink_data_root=winning_root, ), ) @@ -383,12 +383,6 @@ def _apply_rewards_and_penalties( return rewards_received, penalties_received -def _is_eligible_for_punishment( - validator: ValidatorRecord, - current_epoch: Epoch) -> bool: - return validator.slashed and validator.withdrawable_epoch > current_epoch - - @curry def _process_rewards_and_penalties_for_finality( state: BeaconState, @@ -639,6 +633,10 @@ def _process_rewards_and_penalties_for_finality( # Punish penalized active validators current_epoch = state.current_epoch(config.SLOTS_PER_EPOCH) + + def _is_eligible_for_punishment(validator: ValidatorRecord) -> bool: + return validator.slashed and validator.withdrawable_epoch > current_epoch + penalties = { ValidatorIndex(index): 3 * base_rewards[ValidatorIndex(index)] + 2 * ( effective_balances[ValidatorIndex(index)] * @@ -648,7 +646,7 @@ def _process_rewards_and_penalties_for_finality( for index in range(len(state.validator_registry)) if ( (index not in previous_epoch_active_validator_indices) and - _is_eligible_for_punishment(state.validator_registry[index], current_epoch) + _is_eligible_for_punishment(state.validator_registry[index]) ) } rewards_received, penalties_received = _apply_rewards_and_penalties( From c358dcc8e986572cc2f5513cda7305611fa2b706 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Mon, 11 Mar 2019 11:38:12 +0800 Subject: [PATCH 09/23] Implement `get_inactivity_penalty` --- eth2/beacon/epoch_processing_helpers.py | 12 +++++ .../forks/serenity/epoch_processing.py | 24 ++++----- .../forks/test_serenity_epoch_processing.py | 54 +++++++++---------- 3 files changed, 49 insertions(+), 41 deletions(-) diff --git a/eth2/beacon/epoch_processing_helpers.py b/eth2/beacon/epoch_processing_helpers.py index b52c17e060..1c7bbfd1c1 100644 --- a/eth2/beacon/epoch_processing_helpers.py +++ b/eth2/beacon/epoch_processing_helpers.py @@ -201,6 +201,18 @@ def get_base_reward( ) +def get_inactivity_penalty( + *, + base_reward: Gwei, + effective_balance: Gwei, + epochs_since_finality: int, + inactivity_penalty_quotient: int) -> Gwei: + return Gwei( + base_reward + + effective_balance * epochs_since_finality // inactivity_penalty_quotient // 2 + ) + + def get_inclusion_infos( *, state: 'BeaconState', diff --git a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py index 2333435c59..660b151f72 100644 --- a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py +++ b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py @@ -45,6 +45,7 @@ ) from eth2.beacon.epoch_processing_helpers import ( get_base_reward, + get_inactivity_penalty, get_inclusion_infos, get_previous_epoch_head_attestations, get_winning_root, @@ -552,6 +553,15 @@ def _process_rewards_and_penalties_for_finality( # epochs_since_finality > 4 else: + inactivity_penalties = { + index: get_inactivity_penalty( + base_reward=base_rewards[index], + effective_balance=effective_balances[index], + epochs_since_finality=epochs_since_finality, + inactivity_penalty_quotient=config.INACTIVITY_PENALTY_QUOTIENT, + ) + for index in previous_epoch_active_validator_indices + } # Punish active validators in `previous_epoch_attester_indices` attesting_active_validators_indices = previous_epoch_active_validator_indices.intersection( previous_epoch_attester_indices, @@ -578,13 +588,6 @@ def _process_rewards_and_penalties_for_finality( excluded_active_validators_indices = previous_epoch_active_validator_indices.difference( previous_epoch_attester_indices, ) - inactivity_penalties = { - index: base_rewards[index] + ( - effective_balances[index] * - epochs_since_finality // config.INACTIVITY_PENALTY_QUOTIENT // 2 - ) - for index in excluded_active_validators_indices - } rewards_received, penalties_received = _apply_rewards_and_penalties( RewardSettlementContext( penalties=inactivity_penalties, @@ -598,13 +601,6 @@ def _process_rewards_and_penalties_for_finality( excluded_active_validators_indices = previous_epoch_active_validator_indices.difference( previous_epoch_boundary_attester_indices, ) - inactivity_penalties = { - index: base_rewards[index] + ( - effective_balances[index] * - epochs_since_finality // config.INACTIVITY_PENALTY_QUOTIENT // 2 - ) - for index in excluded_active_validators_indices - } rewards_received, penalties_received = _apply_rewards_and_penalties( RewardSettlementContext( penalties=inactivity_penalties, diff --git a/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py b/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py index be944a4fda..4755244338 100644 --- a/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py +++ b/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py @@ -603,33 +603,33 @@ def test_process_crosslinks( 'expected_rewards_received' ), [ - # ( - # 4, 15, # epochs_since_finality <= 4 - # {8, 9}, - # {0, 1, 2, 3, 4, 5, 6, 7}, - # {2, 3, 4, 5, 6}, - # {2, 3, 4}, - # { - # 2: 4, - # 3: 4, - # 4: 4, - # 5: 5, - # 6: 6, - # }, - # 1000, 100, - # { - # 0: -300, # -3 * 100 - # 1: -300, # -3 * 100 - # 2: 236, # 100 * 5 // 8 + 100 * 3 // 8 + 100 * 3 // 8 + 100 * 4 // 4 - # 3: 236, # 100 * 5 // 8 + 100 * 3 // 8 + 100 * 3 // 8 + 100 * 4 // 4 - # 4: 236, # 100 * 5 // 8 + 100 * 3 // 8 + 100 * 3 // 8 + 100 * 4 // 4 - # 5: -58, # 100 * 5 // 8 - 100 - 100 + 100 * 4 // 5 - # 6: -72, # 100 * 5 // 5 - 100 - 100 + 100 * 4 // 6 - # 7: -300, # -3 * 100 - # 8: 0, # not active - # 9: 0, # not active - # } - # ), + ( + 4, 15, # epochs_since_finality <= 4 + {8, 9}, + {0, 1, 2, 3, 4, 5, 6, 7}, + {2, 3, 4, 5, 6}, + {2, 3, 4}, + { + 2: 4, + 3: 4, + 4: 4, + 5: 5, + 6: 6, + }, + 1000, 100, + { + 0: -300, # -3 * 100 + 1: -300, # -3 * 100 + 2: 236, # 100 * 5 // 8 + 100 * 3 // 8 + 100 * 3 // 8 + 100 * 4 // 4 + 3: 236, # 100 * 5 // 8 + 100 * 3 // 8 + 100 * 3 // 8 + 100 * 4 // 4 + 4: 236, # 100 * 5 // 8 + 100 * 3 // 8 + 100 * 3 // 8 + 100 * 4 // 4 + 5: -58, # 100 * 5 // 8 - 100 - 100 + 100 * 4 // 5 + 6: -72, # 100 * 5 // 5 - 100 - 100 + 100 * 4 // 6 + 7: -300, # -3 * 100 + 8: 0, # not active + 9: 0, # not active + } + ), ( 3, 15, # epochs_since_finality > 4 {8, 9}, From adc512f12c25d1e312cf617d7df0fb8100bd2a63 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Mon, 11 Mar 2019 14:03:20 +0800 Subject: [PATCH 10/23] Change `get_winning_root` to `get_winning_root_and_participants` --- eth2/beacon/epoch_processing_helpers.py | 113 ++++++++++++------ eth2/beacon/exceptions.py | 7 -- .../forks/serenity/epoch_processing.py | 102 +++++----------- eth2/beacon/types/states.py | 1 - .../forks/test_serenity_epoch_processing.py | 4 +- .../beacon/test_epoch_processing_helpers.py | 53 ++++---- 6 files changed, 135 insertions(+), 145 deletions(-) diff --git a/eth2/beacon/epoch_processing_helpers.py b/eth2/beacon/epoch_processing_helpers.py index 1c7bbfd1c1..eae33a1337 100644 --- a/eth2/beacon/epoch_processing_helpers.py +++ b/eth2/beacon/epoch_processing_helpers.py @@ -16,6 +16,7 @@ to_set, ) +from eth.constants import ZERO_HASH32 from eth2.beacon.committee_helpers import ( get_attestation_participants, get_attester_indices_from_attesttion, @@ -23,9 +24,6 @@ from eth2.beacon.configs import ( CommitteeConfig, ) -from eth2.beacon.exceptions import ( - NoWinningRootError, -) from eth2.beacon.helpers import ( get_block_root, get_epoch_start_slot, @@ -35,10 +33,12 @@ from eth2.beacon.typing import ( Epoch, Gwei, + Shard, ValidatorIndex, ) from eth2.beacon.datastructures.inclusion_info import InclusionInfo +from eth2.beacon.types.crosslink_records import CrosslinkRecord from eth2.beacon.types.pending_attestation_records import ( PendingAttestationRecord, ) @@ -67,44 +67,69 @@ def get_previous_epoch_head_attestations( yield attestation -def get_winning_root( +@to_tuple +def _filter_attestations_by_latest_crosslinks( + attestations: Iterable[PendingAttestationRecord], + latest_crosslink: CrosslinkRecord) -> Iterable[PendingAttestationRecord]: + for attestation in attestations: + if attestation.data.latest_crosslink == latest_crosslink: + yield attestation + + +@to_tuple +def _filter_attestations_by_shard( + attestations: Iterable[PendingAttestationRecord], + shard: Shard) -> Iterable[PendingAttestationRecord]: + for attestation in attestations: + if attestation.data.shard == shard: + yield attestation + + +def get_winning_root_and_participants( *, state: 'BeaconState', - attestations: Sequence[PendingAttestationRecord], - max_deposit_amount: Gwei, - committee_config: CommitteeConfig) -> Tuple[Hash32, Gwei]: - winning_root = None - winning_root_balance: Gwei = Gwei(0) - crosslink_data_roots = set( - [ - a.data.crosslink_data_root for a in attestations - ] + shard: Shard, + effective_balances: Dict[ValidatorIndex, Gwei], + committee_config: CommitteeConfig) -> Tuple[Hash32, Sequence[ValidatorIndex]]: + valid_attestations = _filter_attestations_by_latest_crosslinks( + _filter_attestations_by_shard( + state.current_epoch_attestations + state.previous_epoch_attestations, + shard, + ), + state.latest_crosslinks[shard], + ) + all_roots = set([a.data.crosslink_data_root for a in valid_attestations]) + + # handle when no attestations for shard available + if len(all_roots) == 0: + return (Hash32(ZERO_HASH32), tuple()) + + def get_attestations_for(root: Hash32) -> Iterable[PendingAttestationRecord]: + return [a for a in valid_attestations if a.data.crosslink_data_root == root] + + # Winning crosslink root is the root with the most votes for it, ties broken in favor of + # lexicographically higher hash + winning_root = max( + all_roots, + key=lambda r: ( + get_attesting_balance_from_attesttion( + state=state, + effective_balances=effective_balances, + attestations=get_attestations_for(r), + committee_config=committee_config, + ), + r, + ), ) - for crosslink_data_root in crosslink_data_roots: - attesting_validator_indices = get_attester_indices_from_attesttion( + + return ( + winning_root, + get_attester_indices_from_attesttion( state=state, - attestations=[ - a - for a in attestations - if a.data.crosslink_data_root == crosslink_data_root - ], + attestations=get_attestations_for(winning_root), committee_config=committee_config, - ) - total_attesting_balance = get_total_balance( - state.validator_balances, - attesting_validator_indices, - max_deposit_amount, - ) - if total_attesting_balance > winning_root_balance: - winning_root = crosslink_data_root - winning_root_balance = total_attesting_balance - elif total_attesting_balance == winning_root_balance and winning_root_balance > 0: - if crosslink_data_root > winning_root: - winning_root = crosslink_data_root - - if winning_root is None: - raise NoWinningRootError - return (winning_root, winning_root_balance) + ), + ) @to_tuple @@ -184,6 +209,22 @@ def get_total_balance_from_effective_balances( ) +def get_attesting_balance_from_attesttion( + *, + state: 'BeaconState', + effective_balances: Dict[ValidatorIndex, Gwei], + attestations: Iterable[PendingAttestationRecord], + committee_config: CommitteeConfig) -> Gwei: + return get_total_balance_from_effective_balances( + effective_balances, + get_attester_indices_from_attesttion( + state=state, + attestations=attestations, + committee_config=committee_config, + ), + ) + + def get_base_reward( *, state: 'BeaconState', diff --git a/eth2/beacon/exceptions.py b/eth2/beacon/exceptions.py index d5dea3faee..e0d0d7d7c2 100644 --- a/eth2/beacon/exceptions.py +++ b/eth2/beacon/exceptions.py @@ -25,13 +25,6 @@ class ProposerIndexError(PyEVMError): pass -class NoWinningRootError(PyEVMError): - """ - Raised when no shard block root is attested to among the attestations provided. - """ - pass - - class NoCommitteeAssignment(PyEVMError): """ Raised when no potential crosslink committee assignment. diff --git a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py index 660b151f72..eae5cdd4c7 100644 --- a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py +++ b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py @@ -10,7 +10,6 @@ from eth_utils import ( to_dict, - to_tuple, ) from eth_utils.toolz import ( curry, @@ -29,9 +28,6 @@ from eth2.beacon.constants import ( FAR_FUTURE_EPOCH, ) -from eth2.beacon.exceptions import ( - NoWinningRootError, -) from eth2.beacon.committee_helpers import ( get_attester_indices_from_attesttion, get_beacon_proposer_index, @@ -48,7 +44,7 @@ get_inactivity_penalty, get_inclusion_infos, get_previous_epoch_head_attestations, - get_winning_root, + get_winning_root_and_participants, get_total_balance, get_total_balance_from_effective_balances, get_epoch_boundary_attesting_balances, @@ -73,13 +69,11 @@ from eth2.beacon.types.attestations import Attestation from eth2.beacon.types.crosslink_records import CrosslinkRecord from eth2.beacon.types.eth1_data_vote import Eth1DataVote -from eth2.beacon.types.pending_attestation_records import PendingAttestationRecord from eth2.beacon.types.states import BeaconState from eth2.beacon.types.validator_records import ValidatorRecord from eth2.beacon.typing import ( Epoch, Gwei, - Shard, SignedGwei, Slot, ValidatorIndex, @@ -266,24 +260,6 @@ def process_justification(state: BeaconState, config: BeaconConfig) -> BeaconSta # # Crosslinks # -@to_tuple -def _filter_attestations_by_latest_crosslinks( - attestations: Sequence[Attestation], - latest_crosslink: CrosslinkRecord) -> Iterable[Attestation]: - for attestation in attestations: - if attestation.data.latest_crosslink == latest_crosslink: - yield attestation - - -@to_tuple -def _filter_attestations_by_shard( - attestations: Sequence[Attestation], - shard: Shard) -> Iterable[Attestation]: - for attestation in attestations: - if attestation.data.shard == shard: - yield attestation - - def process_crosslinks(state: BeaconState, config: BeaconConfig) -> BeaconState: """ Implement 'per-epoch-processing.crosslinks' portion of Phase 0 spec: @@ -295,6 +271,14 @@ def process_crosslinks(state: BeaconState, config: BeaconConfig) -> BeaconState: Return resulting ``state`` """ latest_crosslinks = state.latest_crosslinks + effective_balances = { + ValidatorIndex(index): get_effective_balance( + state.validator_balances, + ValidatorIndex(index), + config.MAX_DEPOSIT_AMOUNT, + ) + for index in range(len(state.validator_registry)) + } previous_epoch_start_slot = get_epoch_start_slot( state.previous_epoch(config.SLOTS_PER_EPOCH, config.GENESIS_EPOCH), config.SLOTS_PER_EPOCH, @@ -310,26 +294,21 @@ def process_crosslinks(state: BeaconState, config: BeaconConfig) -> BeaconState: CommitteeConfig(config), ) for crosslink_committee, shard in crosslink_committees_at_slot: - try: - winning_root, total_attesting_balance = get_winning_root( - state=state, - # Use `_filter_attestations_by_shard` to filter out attestations - # not attesting to this shard so we don't need to going over - # irrelevent attestations over and over again. - attestations=_filter_attestations_by_latest_crosslinks( - _filter_attestations_by_shard( - state.previous_epoch_attestations + state.current_epoch_attestations, - shard, - ), - state.latest_crosslinks[shard], - ), - max_deposit_amount=config.MAX_DEPOSIT_AMOUNT, - committee_config=CommitteeConfig(config), - ) - except NoWinningRootError: + winning_root, attesting_validator_indices = get_winning_root_and_participants( + state=state, + shard=shard, + effective_balances=effective_balances, + committee_config=CommitteeConfig(config), + ) + if len(attesting_validator_indices) == 0: # No winning shard block root found for this shard. pass else: + total_attesting_balance = get_total_balance( + state.validator_balances, + attesting_validator_indices, + config.MAX_DEPOSIT_AMOUNT, + ) total_balance = get_total_balance( state.validator_balances, crosslink_committee, @@ -688,7 +667,6 @@ def _process_rewards_and_penalties_for_attestation_inclusion( def _process_rewards_and_penalties_for_crosslinks( state: BeaconState, config: BeaconConfig, - previous_epoch_attestations: Iterable[PendingAttestationRecord], effective_balances: Dict[ValidatorIndex, Gwei], base_rewards: Dict[ValidatorIndex, Gwei], old_rewards_received: Dict[ValidatorIndex, SignedGwei]) -> Dict[ValidatorIndex, SignedGwei]: @@ -709,34 +687,17 @@ def _process_rewards_and_penalties_for_crosslinks( CommitteeConfig(config), ) for crosslink_committee, shard in crosslink_committees_at_slot: - filtered_attestations = _filter_attestations_by_latest_crosslinks( - _filter_attestations_by_shard( - previous_epoch_attestations + state.current_epoch_attestations, - shard, - ), - state.latest_crosslinks[shard], + winning_root, attesting_validator_indices = get_winning_root_and_participants( + state=state, + shard=shard, + effective_balances=effective_balances, + committee_config=CommitteeConfig(config), + ) + total_attesting_balance = get_total_balance( + state.validator_balances, + attesting_validator_indices, + config.MAX_DEPOSIT_AMOUNT, ) - try: - winning_root, total_attesting_balance = get_winning_root( - state=state, - attestations=filtered_attestations, - max_deposit_amount=config.MAX_DEPOSIT_AMOUNT, - committee_config=CommitteeConfig(config), - ) - except NoWinningRootError: - # No winning shard block root found for this shard. - # Hence no one is counted as attesting validator. - attesting_validator_indices: Iterable[ValidatorIndex] = set() - else: - attesting_validator_indices = get_attester_indices_from_attesttion( - state=state, - attestations=( - a - for a in filtered_attestations - if a.data.crosslink_data_root == winning_root - ), - committee_config=CommitteeConfig(config), - ) total_balance = get_total_balance_from_effective_balances( effective_balances, crosslink_committee, @@ -834,7 +795,6 @@ def process_rewards_and_penalties(state: BeaconState, config: BeaconConfig) -> B _process_rewards_and_penalties_for_crosslinks( state, config, - previous_epoch_attestations, effective_balances, base_rewards, ) diff --git a/eth2/beacon/types/states.py b/eth2/beacon/types/states.py index 8943d906aa..9b7d46884f 100644 --- a/eth2/beacon/types/states.py +++ b/eth2/beacon/types/states.py @@ -66,7 +66,6 @@ class BeaconState(ssz.Serializable): # Finality ('previous_epoch_attestations', List(PendingAttestationRecord)), ('current_epoch_attestations', List(PendingAttestationRecord)), - ('previous_justified_epoch', uint64), ('justified_epoch', uint64), diff --git a/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py b/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py index 4755244338..e552d209df 100644 --- a/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py +++ b/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py @@ -1000,6 +1000,9 @@ def test_process_rewards_and_penalties_for_crosslinks( slot_included=(data_slot + min_attestation_inclusion_delay), ) ) + state = state.copy( + previous_epoch_attestations=tuple(previous_epoch_attestations), + ) active_validators = set( [ @@ -1040,7 +1043,6 @@ def test_process_rewards_and_penalties_for_crosslinks( rewards_received = _process_rewards_and_penalties_for_crosslinks( state, config, - tuple(previous_epoch_attestations), effective_balances, base_rewards, rewards_received, diff --git a/tests/eth2/beacon/test_epoch_processing_helpers.py b/tests/eth2/beacon/test_epoch_processing_helpers.py index 4eddf2f7fc..f0670e1cc5 100644 --- a/tests/eth2/beacon/test_epoch_processing_helpers.py +++ b/tests/eth2/beacon/test_epoch_processing_helpers.py @@ -19,21 +19,16 @@ from eth2.beacon._utils.hash import ( hash_eth2, ) - -from eth2.beacon.committee_helpers import ( - get_attester_indices_from_attesttion, -) from eth2.beacon.configs import CommitteeConfig from eth2.beacon.epoch_processing_helpers import ( get_epoch_boundary_attester_indices, get_epoch_boundary_attesting_balances, get_inclusion_infos, get_previous_epoch_head_attestations, - get_total_balance, - get_winning_root, + get_winning_root_and_participants, ) -from eth2.beacon.exceptions import NoWinningRootError from eth2.beacon.helpers import ( + get_effective_balance, get_epoch_start_slot, ) from eth2.beacon.types.attestations import ( @@ -235,7 +230,7 @@ def test_get_previous_epoch_head_attestations( ), ] ) -def test_get_winning_root( +def test_get_winning_root_and_participants( random, monkeypatch, target_committee_size, @@ -307,40 +302,40 @@ def mock_get_crosslink_committees_at_slot(state, ), ) - try: - winning_root, attesting_balance = get_winning_root( - state=n_validators_state, - attestations=attestations, - max_deposit_amount=config.MAX_DEPOSIT_AMOUNT, - committee_config=committee_config, - ) - attesting_validators_indices = get_attester_indices_from_attesttion( - state=n_validators_state, - attestations=( - a - for a in attestations - if a.data.shard == shard and a.data.crosslink_data_root == winning_root - ), - committee_config=committee_config, - ) - total_attesting_balance = get_total_balance( - n_validators_state.validator_balances, - attesting_validators_indices, + state = n_validators_state.copy( + previous_epoch_attestations=attestations, + ) + effective_balances = { + index: get_effective_balance( + state.validator_balances, + index, config.MAX_DEPOSIT_AMOUNT, ) - assert attesting_balance == total_attesting_balance - except NoWinningRootError: + for index in range(len(state.validator_registry)) + } + + winning_root, attesting_validator_indices = get_winning_root_and_participants( + state=state, + shard=shard, + effective_balances=effective_balances, + committee_config=committee_config, + ) + if len(attesting_validator_indices) == 0: assert len(block_root_1_participants) == 0 and len(block_root_2_participants) == 0 else: if len(block_root_1_participants) == len(block_root_2_participants): if competing_block_roots[0] > competing_block_roots[1]: assert winning_root == competing_block_roots[0] + assert attesting_validator_indices == set(block_root_1_participants) else: assert winning_root == competing_block_roots[1] + assert attesting_validator_indices == set(block_root_2_participants) elif len(block_root_1_participants) < len(block_root_2_participants): assert winning_root == competing_block_roots[1] + assert attesting_validator_indices == set(block_root_2_participants) else: assert winning_root == competing_block_roots[0] + assert attesting_validator_indices == set(block_root_1_participants) @settings(max_examples=1) From e79333e3e7fafca79628f7e586fe7d924b333c4e Mon Sep 17 00:00:00 2001 From: NIC619 Date: Mon, 11 Mar 2019 16:30:28 +0800 Subject: [PATCH 11/23] Merge attestion inclusion reward into finality reward --- .../forks/serenity/epoch_processing.py | 54 ++--- .../forks/test_serenity_epoch_processing.py | 229 +++++------------- 2 files changed, 89 insertions(+), 194 deletions(-) diff --git a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py index eae5cdd4c7..a1ca31551c 100644 --- a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py +++ b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py @@ -530,6 +530,31 @@ def _process_rewards_and_penalties_for_finality( ), ) + # 1.5 Attestation inclusion + rewards = { + ValidatorIndex(index): Gwei(0) + for index in range(len(state.validator_registry)) + } + for index in previous_epoch_attester_indices: + proposer_index = get_beacon_proposer_index( + state, + inclusion_infos[index].inclusion_slot, + CommitteeConfig(config), + ) + rewards[proposer_index] = Gwei( + rewards[proposer_index] + ( + base_rewards[index] // config.ATTESTATION_INCLUSION_REWARD_QUOTIENT + ) + ) + rewards_received, penalties_received = _apply_rewards_and_penalties( + RewardSettlementContext( + rewards=rewards, + indices_to_reward=set(rewards.keys()), + rewards_received=rewards_received, + penalties_received=penalties_received, + ), + ) + # epochs_since_finality > 4 else: inactivity_penalties = { @@ -606,7 +631,7 @@ def _process_rewards_and_penalties_for_finality( ), ) - # Punish penalized active validators + # Punish penalized inactive validators current_epoch = state.current_epoch(config.SLOTS_PER_EPOCH) def _is_eligible_for_punishment(validator: ValidatorRecord) -> bool: @@ -643,26 +668,6 @@ def _is_eligible_for_punishment(validator: ValidatorRecord) -> bool: return historical_rewards_received -@curry -def _process_rewards_and_penalties_for_attestation_inclusion( - state: BeaconState, - config: BeaconConfig, - previous_epoch_attester_indices: Set[ValidatorIndex], - inclusion_infos: Dict[ValidatorIndex, InclusionInfo], - base_rewards: Dict[ValidatorIndex, Gwei], - old_rewards_received: Dict[ValidatorIndex, SignedGwei]) -> Dict[ValidatorIndex, SignedGwei]: - rewards_received = old_rewards_received.copy() - for index in previous_epoch_attester_indices: - proposer_index = get_beacon_proposer_index( - state, - inclusion_infos[index].inclusion_slot, - CommitteeConfig(config), - ) - reward = base_rewards[index] // config.ATTESTATION_INCLUSION_REWARD_QUOTIENT - rewards_received[proposer_index] = SignedGwei(rewards_received[proposer_index] + reward) - return rewards_received - - @curry def _process_rewards_and_penalties_for_crosslinks( state: BeaconState, @@ -785,13 +790,6 @@ def process_rewards_and_penalties(state: BeaconState, config: BeaconConfig) -> B effective_balances, base_rewards, ), - _process_rewards_and_penalties_for_attestation_inclusion( - state, - config, - previous_epoch_attester_indices, - inclusion_infos, - base_rewards, - ), _process_rewards_and_penalties_for_crosslinks( state, config, diff --git a/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py b/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py index e552d209df..26aa4159d9 100644 --- a/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py +++ b/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py @@ -67,7 +67,6 @@ _get_finalized_epoch, _is_majority_vote, _majority_threshold, - _process_rewards_and_penalties_for_attestation_inclusion, _process_rewards_and_penalties_for_crosslinks, _process_rewards_and_penalties_for_finality, _update_eth1_vote_if_exists, @@ -576,15 +575,17 @@ def test_process_crosslinks( 'target_committee_size,' 'shard_count,' 'min_attestation_inclusion_delay,' + 'attestation_inclusion_reward_quotient,' 'inactivity_penalty_quotient,' 'genesis_slot,' ), [ ( - 10, - 2, + 15, + 3, 5, - 2, + 3, + 1, 4, 10, 0, @@ -610,56 +611,67 @@ def test_process_crosslinks( {2, 3, 4, 5, 6}, {2, 3, 4}, { - 2: 4, - 3: 4, - 4: 4, - 5: 5, - 6: 6, + 2: 1, + 3: 1, + 4: 1, + 5: 2, + 6: 3, }, 1000, 100, { 0: -300, # -3 * 100 - 1: -300, # -3 * 100 - 2: 236, # 100 * 5 // 8 + 100 * 3 // 8 + 100 * 3 // 8 + 100 * 4 // 4 - 3: 236, # 100 * 5 // 8 + 100 * 3 // 8 + 100 * 3 // 8 + 100 * 4 // 4 - 4: 236, # 100 * 5 // 8 + 100 * 3 // 8 + 100 * 3 // 8 + 100 * 4 // 4 - 5: -58, # 100 * 5 // 8 - 100 - 100 + 100 * 4 // 5 - 6: -72, # 100 * 5 // 5 - 100 - 100 + 100 * 4 // 6 + 1: -275, # -3 * 100 + 1 * 100 // 4 + 2: 236, # 100 * 5 // 8 + 100 * 3 // 8 + 100 * 3 // 8 + 100 * 1 // 1 + 3: 236, # 100 * 5 // 8 + 100 * 3 // 8 + 100 * 3 // 8 + 100 * 1 // 1 + 4: 236, # 100 * 5 // 8 + 100 * 3 // 8 + 100 * 3 // 8 + 100 * 1 // 1 + 5: -63, # 100 * 5 // 8 - 100 - 100 + 100 * 1 // 2 + 1 * 100 // 4 + 6: -105, # 100 * 5 // 8 - 100 - 100 + 100 * 1 // 3 7: -300, # -3 * 100 - 8: 0, # not active - 9: 0, # not active + 8: 0, + 9: 0, + 10: 0, + 11: 0, + 12: 75, # 3 * 100 // 4 + 13: 0, + 14: 0, } ), ( - 3, 15, # epochs_since_finality > 4 + 3, 23, # epochs_since_finality > 4 {8, 9}, {0, 1, 2, 3, 4, 5, 6, 7}, {2, 3, 4, 5, 6}, {2, 3, 4}, { - 2: 4, - 3: 4, - 4: 4, - 5: 5, - 6: 6, + 2: 1, + 3: 1, + 4: 1, + 5: 2, + 6: 3, }, 1000, 100, { - 0: -800, # -(100 - 100 * 4 // 4) - 2 * (100 + 1000 * 5 // 10 // 2) - 100 - 1: -800, # -(100 - 100 * 4 // 4) - 2 * (100 + 1000 * 5 // 10 // 2) - 100 - 2: 0, # -(100 - 100 * 4 // 4) - 3: 0, # -(100 - 100 * 4 // 4) - 4: 0, # -(100 - 100 * 4 // 4) - 5: -470, # -(100 - 100 * 4 // 5) - (100 * 2 + 1000 * 5 // 10 // 2) - 6: -484, # -(100 - 100 * 4 // 6) - (100 * 2 + 1000 * 5 // 10 // 2) - 7: -800, # -(100 - 100 * 4 // 4) - 2 * (100 + 1000 * 5 // 10 // 2) - 100 + 0: -800, # 2 * (100 + 1000 * 5 // 10 // 2) - 100 + 1: -800, # 2 * (100 + 1000 * 5 // 10 // 2) - 100 + 2: 0, # -(100 - 100 * 1 // 1) + 3: 0, # -(100 - 100 * 1 // 1) + 4: 0, # -(100 - 100 * 1 // 1) + 5: -500, # -(100 - 100 * 1 // 2) - (100 * 2 + 1000 * 5 // 10 // 2) + 6: -517, # -(100 - 100 * 1 // 3) - (100 * 2 + 1000 * 5 // 10 // 2) + 7: -800, # 2 * (100 + 1000 * 5 // 10 // 2) - 100 8: -800, # -(2 * (100 + 1000 * 5 // 10 // 2) + 100) 9: -800, # -(2 * (100 + 1000 * 5 // 10 // 2) + 100) + 10: 0, + 11: 0, + 12: 0, + 13: 0, + 14: 0, } ), ] ) def test_process_rewards_and_penalties_for_finality( + monkeypatch, n_validators_state, config, slots_per_epoch, @@ -679,6 +691,26 @@ def test_process_rewards_and_penalties_for_finality( expected_rewards_received, sample_pending_attestation_record_params, sample_attestation_data_params): + # Mock `get_beacon_proposer_index + from eth2.beacon.state_machines.forks.serenity import epoch_processing + + def mock_get_beacon_proposer_index(state, + slot, + committee_config, + registry_change=False): + mock_proposer_for_slot = { + 13: 12, + 14: 5, + 15: 1, + } + return mock_proposer_for_slot[slot] + + monkeypatch.setattr( + epoch_processing, + 'get_beacon_proposer_index', + mock_get_beacon_proposer_index + ) + validator_registry = n_validators_state.validator_registry for index in penalized_validator_indices: validator_record = validator_registry[index].copy( @@ -774,141 +806,6 @@ def test_process_rewards_and_penalties_for_finality( assert reward_received == expected_rewards_received[index] -@pytest.mark.parametrize( - ( - 'n,' - 'genesis_slot,' - 'slots_per_epoch,' - 'target_committee_size,' - 'shard_count,' - 'attestation_inclusion_reward_quotient,' - 'current_slot,' - 'previous_epoch_attester_indices,' - 'inclusion_slots,' - 'base_reward,' - 'expected_rewards_received' - ), - [ - ( - 20, - 0, - 10, - 2, - 10, - 4, - 40, - {2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13, 15, 16, 17}, - { - 2: 31, # proposer index for inclusion slot 31: 6 - 3: 31, - 4: 32, # proposer index for inclusion slot 32: 16 - 5: 32, - 6: 32, - 7: 32, - 9: 35, # proposer index for inclusion slot 35: 19 - 10: 35, - 11: 35, - 12: 35, - 13: 35, - 15: 38, # proposer index for inclusion slot 38: 15 - 16: 38, - 17: 38, - }, - 100, - { - 0: 0, - 1: 0, - 2: 0, - 3: 0, - 4: 0, - 5: 0, - 6: 50, # 2 * (100 // 4) - 7: 0, - 8: 0, - 9: 0, - 10: 0, - 11: 0, - 12: 0, - 13: 0, - 14: 0, - 15: 75, # 3 * (100 // 4) - 16: 100, # 4 * (100 // 4) - 17: 0, - 18: 0, - 19: 125, # 5 * (100 // 4) - } - ), - ] -) -def test_process_rewards_and_penalties_for_attestation_inclusion( - monkeypatch, - n_validators_state, - config, - slots_per_epoch, - target_committee_size, - shard_count, - attestation_inclusion_reward_quotient, - current_slot, - previous_epoch_attester_indices, - inclusion_slots, - base_reward, - expected_rewards_received): - # Mock `get_beacon_proposer_index - from eth2.beacon.state_machines.forks.serenity import epoch_processing - - def mock_get_beacon_proposer_index(state, - slot, - committee_config, - registry_change=False): - mock_proposer_for_slot = { - 31: 6, - 32: 16, - 35: 19, - 38: 15, - } - return mock_proposer_for_slot[slot] - - monkeypatch.setattr( - epoch_processing, - 'get_beacon_proposer_index', - mock_get_beacon_proposer_index - ) - - state = n_validators_state.copy( - slot=current_slot, - ) - inclusion_infos = { - index: InclusionInfo( - inclusion_slots[index], - inclusion_slots[index] - config.MIN_ATTESTATION_INCLUSION_DELAY, - ) - for index in previous_epoch_attester_indices - } - - base_rewards = { - index: base_reward - for index in previous_epoch_attester_indices - } - - rewards_received = { - index: 0 - for index in range(len(n_validators_state.validator_registry)) - } - - # Process the rewards and penalties for attestation inclusion - rewards_received = _process_rewards_and_penalties_for_attestation_inclusion( - state, - config, - previous_epoch_attester_indices, - inclusion_infos, - base_rewards, - rewards_received, - ) - - for index, reward_received in rewards_received.items(): - assert reward_received == expected_rewards_received[index] - - @settings(max_examples=1) @given(random=st.randoms()) @pytest.mark.parametrize( From ae91d6ed449d7adfd3fde570f2fa951f770feb1a Mon Sep 17 00:00:00 2001 From: NIC619 Date: Mon, 11 Mar 2019 18:20:29 +0800 Subject: [PATCH 12/23] Make `get_previous_epoch_boundary_attestations` helper funciton --- eth2/beacon/epoch_processing_helpers.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/eth2/beacon/epoch_processing_helpers.py b/eth2/beacon/epoch_processing_helpers.py index eae33a1337..0d1be6b0f4 100644 --- a/eth2/beacon/epoch_processing_helpers.py +++ b/eth2/beacon/epoch_processing_helpers.py @@ -51,6 +51,25 @@ from eth2.beacon.state_machines.configs import BeaconConfig # noqa: F401 +@to_tuple +def get_previous_epoch_boundary_attestations( + state: 'BeaconState', + slots_per_epoch: int, + genesis_epoch: Epoch, + latest_block_roots_length: int) -> Iterable[PendingAttestationRecord]: + beacon_block_root = get_block_root( + state, + get_epoch_start_slot( + state.previous_epoch(slots_per_epoch, genesis_epoch), + slots_per_epoch, + ), + latest_block_roots_length, + ) + for attestation in state.previous_epoch_attestations: + if attestation.data.beacon_block_root == beacon_block_root: + yield attestation + + @to_tuple def get_previous_epoch_head_attestations( state: 'BeaconState', From 17c83456df49afba90b1d96d1a846a391df902c0 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Mon, 11 Mar 2019 18:56:06 +0800 Subject: [PATCH 13/23] Separate `_process_rewards_and_penalties_for_finality` into `compute_normal_justification_and_finalization_deltas` and `compute_inactivity_leak_deltas` --- .../forks/serenity/epoch_processing.py | 447 ++++++++---------- 1 file changed, 186 insertions(+), 261 deletions(-) diff --git a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py index a1ca31551c..84a63ad173 100644 --- a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py +++ b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py @@ -363,6 +363,170 @@ def _apply_rewards_and_penalties( return rewards_received, penalties_received +def _compute_normal_justification_and_finalization_deltas( + state: BeaconState, + config: BeaconConfig, + previous_epoch_active_validator_indices: Set[ValidatorIndex], + previous_total_balance: Gwei, + previous_epoch_attester_indices: Set[ValidatorIndex], + previous_epoch_boundary_attester_indices: Set[ValidatorIndex], + previous_epoch_head_attester_indices: Set[ValidatorIndex], + inclusion_infos: Dict[ValidatorIndex, InclusionInfo], + effective_balances: Dict[ValidatorIndex, Gwei], + base_rewards: Dict[ValidatorIndex, Gwei]) -> Tuple[Dict[ValidatorIndex, SignedGwei], Dict[ValidatorIndex, SignedGwei]]: # noqa: E501 + rewards_received = { + index: Gwei(0) + for index in range(len(state.validator_registry)) + } + penalties_received = rewards_received.copy() + previous_epoch_attesting_balance = get_total_balance_from_effective_balances( + effective_balances, + previous_epoch_attester_indices, + ) + previous_epoch_boundary_attesting_balance = get_total_balance_from_effective_balances( + effective_balances, + previous_epoch_boundary_attester_indices, + ) + previous_epoch_head_attesting_balance = get_total_balance_from_effective_balances( + effective_balances, + previous_epoch_head_attester_indices, + ) + for index in previous_epoch_active_validator_indices: + # Expected FFG source + if index in previous_epoch_attester_indices: + rewards_received = _update_rewards_or_penalies( + index, + base_rewards[index] * previous_epoch_attesting_balance // previous_total_balance, + rewards_received, + ) + # Inclusion speed bonus + rewards_received = _update_rewards_or_penalies( + index, + base_rewards[index] * config.MIN_ATTESTATION_INCLUSION_DELAY // inclusion_infos[index].inclusion_distance, + rewards_received, + ) + else: + penalties_received = _update_rewards_or_penalies( + index, + base_rewards[index], + penalties_received, + ) + # Expected FFG target + if index in previous_epoch_boundary_attester_indices: + rewards_received = _update_rewards_or_penalies( + index, + base_rewards[index] * previous_epoch_boundary_attesting_balance // previous_total_balance, + rewards_received, + ) + else: + penalties_received = _update_rewards_or_penalies( + index, + base_rewards[index], + penalties_received, + ) + # Expected head + if index in previous_epoch_head_attester_indices: + rewards_received = _update_rewards_or_penalies( + index, + base_rewards[index] * previous_epoch_head_attesting_balance // previous_total_balance, + rewards_received, + ) + else: + penalties_received = _update_rewards_or_penalies( + index, + base_rewards[index], + penalties_received, + ) + # Proposer bonus + if index in previous_epoch_attester_indices: + proposer_index = get_beacon_proposer_index( + state, + inclusion_infos[index].inclusion_slot, + CommitteeConfig(config), + ) + rewards_received = _update_rewards_or_penalies( + proposer_index, + base_rewards[index] // config.ATTESTATION_INCLUSION_REWARD_QUOTIENT, + rewards_received, + ) + return (rewards_received, penalties_received) + + +def _compute_inactivity_leak_deltas( + state: BeaconState, + config: BeaconConfig, + previous_epoch_active_validator_indices: Set[ValidatorIndex], + previous_epoch_attester_indices: Set[ValidatorIndex], + previous_epoch_boundary_attester_indices: Set[ValidatorIndex], + previous_epoch_head_attester_indices: Set[ValidatorIndex], + inclusion_infos: Dict[ValidatorIndex, InclusionInfo], + effective_balances: Dict[ValidatorIndex, Gwei], + base_rewards: Dict[ValidatorIndex, Gwei], + epochs_since_finality: int) -> Tuple[Dict[ValidatorIndex, SignedGwei], Dict[ValidatorIndex, SignedGwei]]: # noqa: E501 + inactivity_penalties = { + index: get_inactivity_penalty( + base_reward=base_rewards[index], + effective_balance=effective_balances[index], + epochs_since_finality=epochs_since_finality, + inactivity_penalty_quotient=config.INACTIVITY_PENALTY_QUOTIENT, + ) + for index in range(len(state.validator_registry)) + } + rewards_received = { + index: Gwei(0) + for index in range(len(state.validator_registry)) + } + penalties_received = rewards_received.copy() + for index in previous_epoch_active_validator_indices: + if index not in previous_epoch_attester_indices: + penalties_received = _update_rewards_or_penalies( + index, + inactivity_penalties[index], + penalties_received, + ) + else: + # If a validator did attest, apply a small penalty + # for getting attestations included late + rewards_received = _update_rewards_or_penalies( + index, + base_rewards[index] // config.MIN_ATTESTATION_INCLUSION_DELAY // inclusion_infos[index].inclusion_distance, + rewards_received, + ) + penalties_received = _update_rewards_or_penalies( + index, + base_rewards[index], + penalties_received, + ) + if index not in previous_epoch_boundary_attester_indices: + penalties_received = _update_rewards_or_penalies( + index, + inactivity_penalties[index], + penalties_received, + ) + if index not in previous_epoch_head_attester_indices: + penalties_received = _update_rewards_or_penalies( + index, + base_rewards[index], + penalties_received, + ) + + # Penalize slashed-but-inactive validators as though they were active but offline + current_epoch = state.current_epoch(config.SLOTS_PER_EPOCH) + for index in range(len(state.validator_registry)): + eligible = ( + index not in previous_epoch_active_validator_indices and + state.validator_registry[index].slashed and + current_epoch < state.validator_registry[index].withdrawable_epoch + ) + if eligible: + penalties_received = _update_rewards_or_penalies( + index, + 2 * inactivity_penalties[index] + base_rewards[index], + penalties_received, + ) + return (rewards_received, penalties_received) + + @curry def _process_rewards_and_penalties_for_finality( state: BeaconState, @@ -373,19 +537,12 @@ def _process_rewards_and_penalties_for_finality( previous_epoch_attester_indices: Set[ValidatorIndex], inclusion_infos: Dict[ValidatorIndex, InclusionInfo], effective_balances: Dict[ValidatorIndex, Gwei], - base_rewards: Dict[ValidatorIndex, Gwei], - old_rewards_received: Dict[ValidatorIndex, SignedGwei]) -> Dict[ValidatorIndex, SignedGwei]: - previous_epoch_boundary_attestations = ( - a - for a in previous_epoch_attestations - if a.data.epoch_boundary_root == get_block_root( - state, - get_epoch_start_slot( - state.previous_epoch(config.SLOTS_PER_EPOCH, config.GENESIS_EPOCH), - config.SLOTS_PER_EPOCH, - ), - config.LATEST_BLOCK_ROOTS_LENGTH, - ) + base_rewards: Dict[ValidatorIndex, Gwei]) -> Tuple[Dict[ValidatorIndex, SignedGwei], Dict[ValidatorIndex, SignedGwei]]: # noqa: E501 + previous_epoch_boundary_attestations = get_previous_epoch_boundary_attestations( + state, + config.SLOTS_PER_EPOCH, + config.GENESIS_EPOCH, + config.LATEST_BLOCK_ROOTS_LENGTH, ) previous_epoch_boundary_attester_indices = get_attester_indices_from_attesttion( state=state, @@ -405,267 +562,35 @@ def _process_rewards_and_penalties_for_finality( committee_config=CommitteeConfig(config), ) - rewards_received = { - index: Gwei(0) - for index in old_rewards_received - } - penalties_received = rewards_received.copy() epochs_since_finality = state.next_epoch(config.SLOTS_PER_EPOCH) - state.finalized_epoch if epochs_since_finality <= 4: - # 1.1 Expected FFG source: - previous_epoch_attesting_balance = get_total_balance_from_effective_balances( - effective_balances, - previous_epoch_attester_indices, - ) - # Reward validators in `previous_epoch_attester_indices` - # # Punish active validators not in `previous_epoch_attester_indices` - excluded_active_validators_indices = previous_epoch_active_validator_indices.difference( + return _compute_normal_justification_and_finalization_deltas( + state, + config, + previous_epoch_active_validator_indices, + previous_total_balance, previous_epoch_attester_indices, - ) - rewards = { - index: Gwei( - base_rewards[index] * - previous_epoch_attesting_balance // - previous_total_balance - ) - for index in previous_epoch_attester_indices - } - penalties = { - index: base_rewards[index] - for index in excluded_active_validators_indices - } - rewards_received, penalties_received = _apply_rewards_and_penalties( - RewardSettlementContext( - rewards=rewards, - indices_to_reward=previous_epoch_attester_indices, - penalties=penalties, - indices_to_penalize=excluded_active_validators_indices, - rewards_received=rewards_received, - penalties_received=penalties_received, - ), - ) - - # 1.2 Expected FFG target: - previous_epoch_boundary_attesting_balance = get_total_balance_from_effective_balances( - effective_balances, previous_epoch_boundary_attester_indices, - ) - # Reward validators in `previous_epoch_boundary_attester_indices` - # Punish active validators not in `previous_epoch_boundary_attester_indices` - excluded_active_validators_indices = previous_epoch_active_validator_indices.difference( - previous_epoch_boundary_attester_indices, - ) - rewards = { - index: Gwei( - base_rewards[index] * - previous_epoch_boundary_attesting_balance // - previous_total_balance - ) - for index in previous_epoch_boundary_attester_indices - } - penalties = { - index: base_rewards[index] - for index in excluded_active_validators_indices - } - rewards_received, penalties_received = _apply_rewards_and_penalties( - RewardSettlementContext( - rewards=rewards, - indices_to_reward=previous_epoch_boundary_attester_indices, - penalties=penalties, - indices_to_penalize=excluded_active_validators_indices, - rewards_received=rewards_received, - penalties_received=penalties_received, - ), - ) - - # 1.3 Expected beacon chain head: - previous_epoch_head_attesting_balance = get_total_balance_from_effective_balances( - effective_balances, previous_epoch_head_attester_indices, - ) - # Reward validators in `previous_epoch_head_attester_indices` - # Punish active validators not in `previous_epoch_head_attester_indices` - excluded_active_validators_indices = previous_epoch_active_validator_indices.difference( - previous_epoch_head_attester_indices, - ) - rewards = { - index: Gwei( - base_rewards[index] * - previous_epoch_head_attesting_balance // - previous_total_balance - ) - for index in previous_epoch_head_attester_indices - } - penalties = { - index: base_rewards[index] - for index in excluded_active_validators_indices - } - rewards_received, penalties_received = _apply_rewards_and_penalties( - RewardSettlementContext( - rewards=rewards, - indices_to_reward=previous_epoch_head_attester_indices, - penalties=penalties, - indices_to_penalize=excluded_active_validators_indices, - rewards_received=rewards_received, - penalties_received=penalties_received, - ), - ) - - # 1.4 Inclusion distance: - # Reward validators in `previous_epoch_attester_indices` - rewards = { - index: Gwei( - base_rewards[index] * - config.MIN_ATTESTATION_INCLUSION_DELAY // - inclusion_infos[index].inclusion_distance - ) - for index in previous_epoch_attester_indices - } - rewards_received, penalties_received = _apply_rewards_and_penalties( - RewardSettlementContext( - rewards=rewards, - indices_to_reward=previous_epoch_attester_indices, - rewards_received=rewards_received, - penalties_received=penalties_received, - ), - ) - - # 1.5 Attestation inclusion - rewards = { - ValidatorIndex(index): Gwei(0) - for index in range(len(state.validator_registry)) - } - for index in previous_epoch_attester_indices: - proposer_index = get_beacon_proposer_index( - state, - inclusion_infos[index].inclusion_slot, - CommitteeConfig(config), - ) - rewards[proposer_index] = Gwei( - rewards[proposer_index] + ( - base_rewards[index] // config.ATTESTATION_INCLUSION_REWARD_QUOTIENT - ) - ) - rewards_received, penalties_received = _apply_rewards_and_penalties( - RewardSettlementContext( - rewards=rewards, - indices_to_reward=set(rewards.keys()), - rewards_received=rewards_received, - penalties_received=penalties_received, - ), + inclusion_infos, + effective_balances, + base_rewards, ) # epochs_since_finality > 4 else: - inactivity_penalties = { - index: get_inactivity_penalty( - base_reward=base_rewards[index], - effective_balance=effective_balances[index], - epochs_since_finality=epochs_since_finality, - inactivity_penalty_quotient=config.INACTIVITY_PENALTY_QUOTIENT, - ) - for index in previous_epoch_active_validator_indices - } - # Punish active validators in `previous_epoch_attester_indices` - attesting_active_validators_indices = previous_epoch_active_validator_indices.intersection( - previous_epoch_attester_indices, - ) - penalties = { - index: Gwei( - base_rewards[index] - ( - base_rewards[index] * - config.MIN_ATTESTATION_INCLUSION_DELAY // - inclusion_infos[index].inclusion_distance - ) - ) - for index in attesting_active_validators_indices - } - rewards_received, penalties_received = _apply_rewards_and_penalties( - RewardSettlementContext( - penalties=penalties, - indices_to_penalize=attesting_active_validators_indices, - rewards_received=rewards_received, - penalties_received=penalties_received, - ), - ) - # Punish active validators not in `previous_epoch_attester_indices` - excluded_active_validators_indices = previous_epoch_active_validator_indices.difference( + return _compute_inactivity_leak_deltas( + state, + config, + previous_epoch_active_validator_indices, previous_epoch_attester_indices, - ) - rewards_received, penalties_received = _apply_rewards_and_penalties( - RewardSettlementContext( - penalties=inactivity_penalties, - indices_to_penalize=excluded_active_validators_indices, - rewards_received=rewards_received, - penalties_received=penalties_received, - ), - ) - - # Punish active validators not in `previous_epoch_boundary_attester_indices` - excluded_active_validators_indices = previous_epoch_active_validator_indices.difference( previous_epoch_boundary_attester_indices, - ) - rewards_received, penalties_received = _apply_rewards_and_penalties( - RewardSettlementContext( - penalties=inactivity_penalties, - indices_to_penalize=excluded_active_validators_indices, - rewards_received=rewards_received, - penalties_received=penalties_received, - ), - ) - - # Punish active validators not in `previous_epoch_head_attester_indices` - excluded_active_validators_indices = previous_epoch_active_validator_indices.difference( previous_epoch_head_attester_indices, + inclusion_infos, + effective_balances, + base_rewards, + epochs_since_finality, ) - penalties = { - index: base_rewards[index] - for index in excluded_active_validators_indices - } - rewards_received, penalties_received = _apply_rewards_and_penalties( - RewardSettlementContext( - penalties=penalties, - indices_to_penalize=excluded_active_validators_indices, - rewards_received=rewards_received, - penalties_received=penalties_received, - ), - ) - - # Punish penalized inactive validators - current_epoch = state.current_epoch(config.SLOTS_PER_EPOCH) - - def _is_eligible_for_punishment(validator: ValidatorRecord) -> bool: - return validator.slashed and validator.withdrawable_epoch > current_epoch - - penalties = { - ValidatorIndex(index): 3 * base_rewards[ValidatorIndex(index)] + 2 * ( - effective_balances[ValidatorIndex(index)] * - epochs_since_finality // - config.INACTIVITY_PENALTY_QUOTIENT // 2 - ) - for index in range(len(state.validator_registry)) - if ( - (index not in previous_epoch_active_validator_indices) and - _is_eligible_for_punishment(state.validator_registry[index]) - ) - } - rewards_received, penalties_received = _apply_rewards_and_penalties( - RewardSettlementContext( - penalties=penalties, - indices_to_penalize={index for index in penalties}, - rewards_received=rewards_received, - penalties_received=penalties_received, - ), - ) - - historical_rewards_received = old_rewards_received.copy() - for index in rewards_received: - historical_rewards_received = _update_rewards_or_penalies( - index, - rewards_received[index] - penalties_received[index], - historical_rewards_received, - ) - return historical_rewards_received @curry From e7e9e8e578c3a9ce83a488265bf89caa114f2ede Mon Sep 17 00:00:00 2001 From: NIC619 Date: Mon, 11 Mar 2019 18:57:22 +0800 Subject: [PATCH 14/23] Return both rewards and penalties map instead of just rewards map --- .../forks/serenity/epoch_processing.py | 78 +++++++++++-------- 1 file changed, 44 insertions(+), 34 deletions(-) diff --git a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py index 84a63ad173..b6c942f8c2 100644 --- a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py +++ b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py @@ -598,9 +598,7 @@ def _process_rewards_and_penalties_for_crosslinks( state: BeaconState, config: BeaconConfig, effective_balances: Dict[ValidatorIndex, Gwei], - base_rewards: Dict[ValidatorIndex, Gwei], - old_rewards_received: Dict[ValidatorIndex, SignedGwei]) -> Dict[ValidatorIndex, SignedGwei]: - rewards_received = old_rewards_received.copy() + base_rewards: Dict[ValidatorIndex, Gwei]) -> Tuple[Dict[ValidatorIndex, SignedGwei], Dict[ValidatorIndex, SignedGwei]]: # noqa: E501 previous_epoch_start_slot = get_epoch_start_slot( state.previous_epoch(config.SLOTS_PER_EPOCH, config.GENESIS_EPOCH), config.SLOTS_PER_EPOCH, @@ -609,6 +607,11 @@ def _process_rewards_and_penalties_for_crosslinks( state.current_epoch(config.SLOTS_PER_EPOCH), config.SLOTS_PER_EPOCH, ) + rewards_received = { + index: Gwei(0) + for index in range(len(state.validator_registry)) + } + penalties_received = rewards_received.copy() # Also need current epoch attestations to compute the winning root. for slot in range(previous_epoch_start_slot, current_epoch_start_slot): crosslink_committees_at_slot = get_crosslink_committees_at_slot( @@ -633,12 +636,18 @@ def _process_rewards_and_penalties_for_crosslinks( crosslink_committee, ) for index in attesting_validator_indices: - reward = base_rewards[index] * total_attesting_balance // total_balance - rewards_received[index] = SignedGwei(rewards_received[index] + reward) + rewards_received = _update_rewards_or_penalies( + index, + base_rewards[index] * total_attesting_balance // total_balance, + rewards_received, + ) for index in set(crosslink_committee).difference(attesting_validator_indices): - penalty = base_rewards[index] - rewards_received[index] = SignedGwei(rewards_received[index] - penalty) - return rewards_received + penalties_received = _update_rewards_or_penalies( + index, + base_rewards[index], + penalties_received, + ) + return (rewards_received, penalties_received) def process_rewards_and_penalties(state: BeaconState, config: BeaconConfig) -> BeaconState: @@ -695,32 +704,24 @@ def process_rewards_and_penalties(state: BeaconState, config: BeaconConfig) -> B for index in range(len(state.validator_registry)) } - # Initialize the reward (validator) received map - rewards_received = { - index: SignedGwei(0) - for index in range(len(state.validator_registry)) - } - # 1. Process rewards and penalties for justification and finalization - rewards_received = pipe( - rewards_received, - _process_rewards_and_penalties_for_finality( - state, - config, - previous_epoch_active_validator_indices, - previous_total_balance, - previous_epoch_attestations, - previous_epoch_attester_indices, - inclusion_infos, - effective_balances, - base_rewards, - ), - _process_rewards_and_penalties_for_crosslinks( - state, - config, - effective_balances, - base_rewards, - ) + finality_rewards, finality_penalties = _process_rewards_and_penalties_for_finality( + state, + config, + previous_epoch_active_validator_indices, + previous_total_balance, + previous_epoch_attestations, + previous_epoch_attester_indices, + inclusion_infos, + effective_balances, + base_rewards, + ) + # 2. Process rewards and penalties for crosslinks + crosslinks_rewards, crosslinks_penalties = _process_rewards_and_penalties_for_crosslinks( + state, + config, + effective_balances, + base_rewards, ) # Apply the overall rewards/penalties @@ -728,7 +729,16 @@ def process_rewards_and_penalties(state: BeaconState, config: BeaconConfig) -> B state = state.update_validator_balance( ValidatorIndex(index), # Prevent validator balance under flow - max(state.validator_balances[index] + rewards_received[index], 0), + max( + ( + state.validator_balances[index] + + finality_rewards[index] + + crosslinks_rewards[index] - + finality_penalties[index] - + crosslinks_penalties[index] + ), + 0, + ), ) return state From 6e253c6c09dc6bd00feb25975cf2eb5dffff362b Mon Sep 17 00:00:00 2001 From: NIC619 Date: Mon, 11 Mar 2019 18:59:49 +0800 Subject: [PATCH 15/23] Update rewards and penalties tests --- .../forks/test_serenity_epoch_processing.py | 28 +++++++------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py b/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py index 26aa4159d9..325000cb00 100644 --- a/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py +++ b/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py @@ -743,11 +743,6 @@ def mock_get_beacon_proposer_index(state, for index in range(len(state.validator_registry)) } - rewards_received = { - index: 0 - for index in range(len(state.validator_registry)) - } - prev_epoch_start_slot = get_epoch_start_slot( state.previous_epoch(config.SLOTS_PER_EPOCH, config.GENESIS_EPOCH), slots_per_epoch, ) @@ -789,7 +784,7 @@ def mock_get_beacon_proposer_index(state, previous_epoch_attestations=prev_epoch_attestations, ) - rewards_received = _process_rewards_and_penalties_for_finality( + rewards_received, penalties_received = _process_rewards_and_penalties_for_finality( state, config, previous_epoch_active_validator_indices, @@ -799,11 +794,12 @@ def mock_get_beacon_proposer_index(state, inclusion_infos, effective_balances, base_rewards, - rewards_received, ) - for index, reward_received in rewards_received.items(): - assert reward_received == expected_rewards_received[index] + for index in range(len(state.validator_registry)): + assert ( + rewards_received[index] - penalties_received[index] == expected_rewards_received[index] + ) @settings(max_examples=1) @@ -932,17 +928,11 @@ def test_process_rewards_and_penalties_for_crosslinks( for index in active_validators } - rewards_received = { - index: 0 - for index in range(len(state.validator_registry)) - } - - rewards_received = _process_rewards_and_penalties_for_crosslinks( + rewards_received, penalties_received = _process_rewards_and_penalties_for_crosslinks( state, config, effective_balances, base_rewards, - rewards_received, ) expected_rewards_received = { @@ -975,8 +965,10 @@ def test_process_rewards_and_penalties_for_crosslinks( expected_rewards_received[index] -= penalty # Check the rewards/penalties match - for index, _ in rewards_received.items(): - assert rewards_received[index] == expected_rewards_received[index] + for index in range(len(state.validator_registry)): + assert ( + rewards_received[index] - penalties_received[index] == expected_rewards_received[index] + ) # From 93f39350a35404fc23994da3377759b3e63c4330 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Mon, 11 Mar 2019 19:36:54 +0800 Subject: [PATCH 16/23] Remove `SignedGwei` type and `RewardSettlementContext` --- .../reward_settlement_context.py | 19 ----- .../forks/serenity/epoch_processing.py | 77 ++++++++----------- eth2/beacon/typing.py | 1 - .../forks/test_serenity_epoch_processing.py | 6 +- 4 files changed, 36 insertions(+), 67 deletions(-) delete mode 100644 eth2/beacon/datastructures/reward_settlement_context.py diff --git a/eth2/beacon/datastructures/reward_settlement_context.py b/eth2/beacon/datastructures/reward_settlement_context.py deleted file mode 100644 index e1402923c5..0000000000 --- a/eth2/beacon/datastructures/reward_settlement_context.py +++ /dev/null @@ -1,19 +0,0 @@ -from typing import ( - Dict, - NamedTuple, - Set, -) - -from eth2.beacon.typing import ( - Gwei, - ValidatorIndex, -) - - -class RewardSettlementContext(NamedTuple): - rewards_received: Dict[ValidatorIndex, Gwei] - penalties_received: Dict[ValidatorIndex, Gwei] - rewards: Dict[ValidatorIndex, Gwei] = dict() - indices_to_reward: Set[ValidatorIndex] = set() - penalties: Dict[ValidatorIndex, Gwei] = dict() - indices_to_penalize: Set[ValidatorIndex] = set() diff --git a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py index b6c942f8c2..61b92b3656 100644 --- a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py +++ b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py @@ -13,7 +13,6 @@ ) from eth_utils.toolz import ( curry, - pipe, first, ) @@ -43,6 +42,7 @@ get_base_reward, get_inactivity_penalty, get_inclusion_infos, + get_previous_epoch_boundary_attestations, get_previous_epoch_head_attestations, get_winning_root_and_participants, get_total_balance, @@ -51,7 +51,6 @@ ) from eth2.beacon.helpers import ( get_active_validator_indices, - get_block_root, get_effective_balance, get_epoch_start_slot, get_randao_mix, @@ -65,16 +64,13 @@ hash_eth2, ) from eth2.beacon.datastructures.inclusion_info import InclusionInfo -from eth2.beacon.datastructures.reward_settlement_context import RewardSettlementContext from eth2.beacon.types.attestations import Attestation from eth2.beacon.types.crosslink_records import CrosslinkRecord from eth2.beacon.types.eth1_data_vote import Eth1DataVote from eth2.beacon.types.states import BeaconState -from eth2.beacon.types.validator_records import ValidatorRecord from eth2.beacon.typing import ( Epoch, Gwei, - SignedGwei, Slot, ValidatorIndex, ) @@ -344,25 +340,6 @@ def _update_rewards_or_penalies( yield i, rewards_or_penalties[i] -def _apply_rewards_and_penalties( - reward_settlement_context: RewardSettlementContext) -> Tuple[Dict[ValidatorIndex, Gwei], Dict[ValidatorIndex, Gwei]]: # noqa: E501 - rewards_received = reward_settlement_context.rewards_received - penalties_received = reward_settlement_context.penalties_received - for index in reward_settlement_context.indices_to_reward: - rewards_received = _update_rewards_or_penalies( - index, - reward_settlement_context.rewards[index], - rewards_received, - ) - for index in reward_settlement_context.indices_to_penalize: - penalties_received = _update_rewards_or_penalies( - index, - reward_settlement_context.penalties[index], - penalties_received, - ) - return rewards_received, penalties_received - - def _compute_normal_justification_and_finalization_deltas( state: BeaconState, config: BeaconConfig, @@ -373,9 +350,9 @@ def _compute_normal_justification_and_finalization_deltas( previous_epoch_head_attester_indices: Set[ValidatorIndex], inclusion_infos: Dict[ValidatorIndex, InclusionInfo], effective_balances: Dict[ValidatorIndex, Gwei], - base_rewards: Dict[ValidatorIndex, Gwei]) -> Tuple[Dict[ValidatorIndex, SignedGwei], Dict[ValidatorIndex, SignedGwei]]: # noqa: E501 + base_rewards: Dict[ValidatorIndex, Gwei]) -> Tuple[Dict[ValidatorIndex, Gwei], Dict[ValidatorIndex, Gwei]]: # noqa: E501 rewards_received = { - index: Gwei(0) + ValidatorIndex(index): Gwei(0) for index in range(len(state.validator_registry)) } penalties_received = rewards_received.copy() @@ -402,7 +379,10 @@ def _compute_normal_justification_and_finalization_deltas( # Inclusion speed bonus rewards_received = _update_rewards_or_penalies( index, - base_rewards[index] * config.MIN_ATTESTATION_INCLUSION_DELAY // inclusion_infos[index].inclusion_distance, + ( + base_rewards[index] * config.MIN_ATTESTATION_INCLUSION_DELAY // + inclusion_infos[index].inclusion_distance + ), rewards_received, ) else: @@ -415,7 +395,10 @@ def _compute_normal_justification_and_finalization_deltas( if index in previous_epoch_boundary_attester_indices: rewards_received = _update_rewards_or_penalies( index, - base_rewards[index] * previous_epoch_boundary_attesting_balance // previous_total_balance, + ( + base_rewards[index] * previous_epoch_boundary_attesting_balance // + previous_total_balance + ), rewards_received, ) else: @@ -428,7 +411,10 @@ def _compute_normal_justification_and_finalization_deltas( if index in previous_epoch_head_attester_indices: rewards_received = _update_rewards_or_penalies( index, - base_rewards[index] * previous_epoch_head_attesting_balance // previous_total_balance, + ( + base_rewards[index] * previous_epoch_head_attesting_balance // + previous_total_balance + ), rewards_received, ) else: @@ -462,18 +448,18 @@ def _compute_inactivity_leak_deltas( inclusion_infos: Dict[ValidatorIndex, InclusionInfo], effective_balances: Dict[ValidatorIndex, Gwei], base_rewards: Dict[ValidatorIndex, Gwei], - epochs_since_finality: int) -> Tuple[Dict[ValidatorIndex, SignedGwei], Dict[ValidatorIndex, SignedGwei]]: # noqa: E501 + epochs_since_finality: int) -> Tuple[Dict[ValidatorIndex, Gwei], Dict[ValidatorIndex, Gwei]]: # noqa: E501 inactivity_penalties = { - index: get_inactivity_penalty( - base_reward=base_rewards[index], - effective_balance=effective_balances[index], + ValidatorIndex(index): get_inactivity_penalty( + base_reward=base_rewards[ValidatorIndex(index)], + effective_balance=effective_balances[ValidatorIndex(index)], epochs_since_finality=epochs_since_finality, inactivity_penalty_quotient=config.INACTIVITY_PENALTY_QUOTIENT, ) for index in range(len(state.validator_registry)) } rewards_received = { - index: Gwei(0) + ValidatorIndex(index): Gwei(0) for index in range(len(state.validator_registry)) } penalties_received = rewards_received.copy() @@ -489,7 +475,10 @@ def _compute_inactivity_leak_deltas( # for getting attestations included late rewards_received = _update_rewards_or_penalies( index, - base_rewards[index] // config.MIN_ATTESTATION_INCLUSION_DELAY // inclusion_infos[index].inclusion_distance, + ( + base_rewards[index] // config.MIN_ATTESTATION_INCLUSION_DELAY // + inclusion_infos[index].inclusion_distance + ), rewards_received, ) penalties_received = _update_rewards_or_penalies( @@ -512,16 +501,16 @@ def _compute_inactivity_leak_deltas( # Penalize slashed-but-inactive validators as though they were active but offline current_epoch = state.current_epoch(config.SLOTS_PER_EPOCH) - for index in range(len(state.validator_registry)): + for i in range(len(state.validator_registry)): eligible = ( - index not in previous_epoch_active_validator_indices and - state.validator_registry[index].slashed and - current_epoch < state.validator_registry[index].withdrawable_epoch + i not in previous_epoch_active_validator_indices and + state.validator_registry[ValidatorIndex(i)].slashed and + current_epoch < state.validator_registry[i].withdrawable_epoch ) if eligible: penalties_received = _update_rewards_or_penalies( - index, - 2 * inactivity_penalties[index] + base_rewards[index], + ValidatorIndex(i), + 2 * inactivity_penalties[ValidatorIndex(i)] + base_rewards[ValidatorIndex(i)], penalties_received, ) return (rewards_received, penalties_received) @@ -537,7 +526,7 @@ def _process_rewards_and_penalties_for_finality( previous_epoch_attester_indices: Set[ValidatorIndex], inclusion_infos: Dict[ValidatorIndex, InclusionInfo], effective_balances: Dict[ValidatorIndex, Gwei], - base_rewards: Dict[ValidatorIndex, Gwei]) -> Tuple[Dict[ValidatorIndex, SignedGwei], Dict[ValidatorIndex, SignedGwei]]: # noqa: E501 + base_rewards: Dict[ValidatorIndex, Gwei]) -> Tuple[Dict[ValidatorIndex, Gwei], Dict[ValidatorIndex, Gwei]]: # noqa: E501 previous_epoch_boundary_attestations = get_previous_epoch_boundary_attestations( state, config.SLOTS_PER_EPOCH, @@ -598,7 +587,7 @@ def _process_rewards_and_penalties_for_crosslinks( state: BeaconState, config: BeaconConfig, effective_balances: Dict[ValidatorIndex, Gwei], - base_rewards: Dict[ValidatorIndex, Gwei]) -> Tuple[Dict[ValidatorIndex, SignedGwei], Dict[ValidatorIndex, SignedGwei]]: # noqa: E501 + base_rewards: Dict[ValidatorIndex, Gwei]) -> Tuple[Dict[ValidatorIndex, Gwei], Dict[ValidatorIndex, Gwei]]: # noqa: E501 previous_epoch_start_slot = get_epoch_start_slot( state.previous_epoch(config.SLOTS_PER_EPOCH, config.GENESIS_EPOCH), config.SLOTS_PER_EPOCH, @@ -608,7 +597,7 @@ def _process_rewards_and_penalties_for_crosslinks( config.SLOTS_PER_EPOCH, ) rewards_received = { - index: Gwei(0) + ValidatorIndex(index): Gwei(0) for index in range(len(state.validator_registry)) } penalties_received = rewards_received.copy() diff --git a/eth2/beacon/typing.py b/eth2/beacon/typing.py index 6a511b0a8d..7744e88adc 100644 --- a/eth2/beacon/typing.py +++ b/eth2/beacon/typing.py @@ -15,7 +15,6 @@ CommitteeIndex = NewType('CommitteeIndex', int) Gwei = NewType('Gwei', int) # uint64 -SignedGwei = NewType('SignedGwei', int) # uint64 Timestamp = NewType('Timestamp', int) Second = NewType('Second', int) diff --git a/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py b/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py index 325000cb00..3737bc6720 100644 --- a/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py +++ b/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py @@ -651,14 +651,14 @@ def test_process_crosslinks( }, 1000, 100, { - 0: -800, # 2 * (100 + 1000 * 5 // 10 // 2) - 100 - 1: -800, # 2 * (100 + 1000 * 5 // 10 // 2) - 100 + 0: -800, # -2 * (100 + 1000 * 5 // 10 // 2) - 100 + 1: -800, # -2 * (100 + 1000 * 5 // 10 // 2) - 100 2: 0, # -(100 - 100 * 1 // 1) 3: 0, # -(100 - 100 * 1 // 1) 4: 0, # -(100 - 100 * 1 // 1) 5: -500, # -(100 - 100 * 1 // 2) - (100 * 2 + 1000 * 5 // 10 // 2) 6: -517, # -(100 - 100 * 1 // 3) - (100 * 2 + 1000 * 5 // 10 // 2) - 7: -800, # 2 * (100 + 1000 * 5 // 10 // 2) - 100 + 7: -800, # -2 * (100 + 1000 * 5 // 10 // 2) - 100 8: -800, # -(2 * (100 + 1000 * 5 // 10 // 2) + 100) 9: -800, # -(2 * (100 + 1000 * 5 // 10 // 2) + 100) 10: 0, From 71623470a7c783cdf153f9be1feaf45885ebe148 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Wed, 13 Mar 2019 15:26:01 +0800 Subject: [PATCH 17/23] Apply PR feedback: fix namings and comments --- eth2/beacon/committee_helpers.py | 2 +- eth2/beacon/epoch_processing_helpers.py | 10 +++++----- .../state_machines/forks/serenity/epoch_processing.py | 11 +++++------ .../forks/serenity/operation_processing.py | 2 +- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/eth2/beacon/committee_helpers.py b/eth2/beacon/committee_helpers.py index 66fc12f65d..c8501188da 100644 --- a/eth2/beacon/committee_helpers.py +++ b/eth2/beacon/committee_helpers.py @@ -457,7 +457,7 @@ def get_attestation_participants(state: 'BeaconState', @to_set -def get_attester_indices_from_attesttion( +def get_attester_indices_from_attestations( *, state: 'BeaconState', attestations: Iterable['Attestation'], diff --git a/eth2/beacon/epoch_processing_helpers.py b/eth2/beacon/epoch_processing_helpers.py index 0d1be6b0f4..f3375d9ac1 100644 --- a/eth2/beacon/epoch_processing_helpers.py +++ b/eth2/beacon/epoch_processing_helpers.py @@ -19,7 +19,7 @@ from eth.constants import ZERO_HASH32 from eth2.beacon.committee_helpers import ( get_attestation_participants, - get_attester_indices_from_attesttion, + get_attester_indices_from_attestations, ) from eth2.beacon.configs import ( CommitteeConfig, @@ -131,7 +131,7 @@ def get_attestations_for(root: Hash32) -> Iterable[PendingAttestationRecord]: winning_root = max( all_roots, key=lambda r: ( - get_attesting_balance_from_attesttion( + get_attesting_balance_from_attestations( state=state, effective_balances=effective_balances, attestations=get_attestations_for(r), @@ -143,7 +143,7 @@ def get_attestations_for(root: Hash32) -> Iterable[PendingAttestationRecord]: return ( winning_root, - get_attester_indices_from_attesttion( + get_attester_indices_from_attestations( state=state, attestations=get_attestations_for(winning_root), committee_config=committee_config, @@ -228,7 +228,7 @@ def get_total_balance_from_effective_balances( ) -def get_attesting_balance_from_attesttion( +def get_attesting_balance_from_attestations( *, state: 'BeaconState', effective_balances: Dict[ValidatorIndex, Gwei], @@ -236,7 +236,7 @@ def get_attesting_balance_from_attesttion( committee_config: CommitteeConfig) -> Gwei: return get_total_balance_from_effective_balances( effective_balances, - get_attester_indices_from_attesttion( + get_attester_indices_from_attestations( state=state, attestations=attestations, committee_config=committee_config, diff --git a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py index 61b92b3656..017c1fab62 100644 --- a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py +++ b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py @@ -28,7 +28,7 @@ FAR_FUTURE_EPOCH, ) from eth2.beacon.committee_helpers import ( - get_attester_indices_from_attesttion, + get_attester_indices_from_attestations, get_beacon_proposer_index, get_crosslink_committees_at_slot, get_current_epoch_committee_count, @@ -298,7 +298,7 @@ def process_crosslinks(state: BeaconState, config: BeaconConfig) -> BeaconState: ) if len(attesting_validator_indices) == 0: # No winning shard block root found for this shard. - pass + continue else: total_attesting_balance = get_total_balance( state.validator_balances, @@ -533,7 +533,7 @@ def _process_rewards_and_penalties_for_finality( config.GENESIS_EPOCH, config.LATEST_BLOCK_ROOTS_LENGTH, ) - previous_epoch_boundary_attester_indices = get_attester_indices_from_attesttion( + previous_epoch_boundary_attester_indices = get_attester_indices_from_attestations( state=state, attestations=previous_epoch_boundary_attestations, committee_config=CommitteeConfig(config), @@ -545,7 +545,7 @@ def _process_rewards_and_penalties_for_finality( config.GENESIS_EPOCH, config.LATEST_BLOCK_ROOTS_LENGTH, ) - previous_epoch_head_attester_indices = get_attester_indices_from_attesttion( + previous_epoch_head_attester_indices = get_attester_indices_from_attestations( state=state, attestations=previous_epoch_head_attestations, committee_config=CommitteeConfig(config), @@ -601,7 +601,6 @@ def _process_rewards_and_penalties_for_crosslinks( for index in range(len(state.validator_registry)) } penalties_received = rewards_received.copy() - # Also need current epoch attestations to compute the winning root. for slot in range(previous_epoch_start_slot, current_epoch_start_slot): crosslink_committees_at_slot = get_crosslink_committees_at_slot( state, @@ -657,7 +656,7 @@ def process_rewards_and_penalties(state: BeaconState, config: BeaconConfig) -> B # Compute previous epoch attester indices and the total balance they account for # for later use. previous_epoch_attestations = state.previous_epoch_attestations - previous_epoch_attester_indices = get_attester_indices_from_attesttion( + previous_epoch_attester_indices = get_attester_indices_from_attestations( state=state, attestations=previous_epoch_attestations, committee_config=CommitteeConfig(config), diff --git a/eth2/beacon/state_machines/forks/serenity/operation_processing.py b/eth2/beacon/state_machines/forks/serenity/operation_processing.py index 4117bf9526..25daaf175f 100644 --- a/eth2/beacon/state_machines/forks/serenity/operation_processing.py +++ b/eth2/beacon/state_machines/forks/serenity/operation_processing.py @@ -115,7 +115,7 @@ def process_attestations(state: BeaconState, Validate the ``attestations`` contained within the ``block`` in the context of ``state``. If any invalid, throw ``ValidationError``. - Otherwise, append an ``PendingAttestationRecords`` for each to ``previous_epoch_attestations`` + Otherwise, append a ``PendingAttestationRecords`` for each to ``previous_epoch_attestations`` or ``current_epoch_attestations``. Return resulting ``state``. """ From 4bf74ef885d98f98cec2cc2801b48bd9fa342c35 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Wed, 13 Mar 2019 15:35:41 +0800 Subject: [PATCH 18/23] Apply PR feedback: Compute adjusted base reward quotient in `get_base_reward` --- eth2/beacon/epoch_processing_helpers.py | 9 ++++++--- .../forks/serenity/epoch_processing.py | 7 ++----- .../forks/test_serenity_epoch_processing.py | 19 ++++++------------- 3 files changed, 14 insertions(+), 21 deletions(-) diff --git a/eth2/beacon/epoch_processing_helpers.py b/eth2/beacon/epoch_processing_helpers.py index f3375d9ac1..330cbf5b37 100644 --- a/eth2/beacon/epoch_processing_helpers.py +++ b/eth2/beacon/epoch_processing_helpers.py @@ -17,6 +17,7 @@ ) from eth.constants import ZERO_HASH32 +from eth2._utils.numeric import integer_squareroot from eth2.beacon.committee_helpers import ( get_attestation_participants, get_attester_indices_from_attestations, @@ -249,15 +250,17 @@ def get_base_reward( state: 'BeaconState', index: ValidatorIndex, base_reward_quotient: int, + previous_total_balance: Gwei, max_deposit_amount: Gwei) -> Gwei: - if base_reward_quotient == 0: - return Gwei(0) + adjusted_quotient = ( + integer_squareroot(previous_total_balance) // base_reward_quotient + ) return Gwei( get_effective_balance( state.validator_balances, index, max_deposit_amount, - ) // base_reward_quotient // 5 + ) // adjusted_quotient // 5 ) diff --git a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py index 017c1fab62..648389a389 100644 --- a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py +++ b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py @@ -18,7 +18,6 @@ from eth2.beacon import helpers from eth2._utils.numeric import ( - integer_squareroot, is_power_of_two, ) from eth2._utils.tuple import ( @@ -679,14 +678,12 @@ def process_rewards_and_penalties(state: BeaconState, config: BeaconConfig) -> B for index in range(len(state.validator_registry)) } # Compute base reward of each previous epoch active validator for later use - _base_reward_quotient = ( - integer_squareroot(previous_total_balance) // config.BASE_REWARD_QUOTIENT - ) base_rewards = { ValidatorIndex(index): get_base_reward( state=state, index=ValidatorIndex(index), - base_reward_quotient=_base_reward_quotient, + base_reward_quotient=config.BASE_REWARD_QUOTIENT, + previous_total_balance=previous_total_balance, max_deposit_amount=config.MAX_DEPOSIT_AMOUNT, ) for index in range(len(state.validator_registry)) diff --git a/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py b/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py index 3737bc6720..664335ec4f 100644 --- a/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py +++ b/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py @@ -6,10 +6,6 @@ strategies as st, ) -from eth._utils.numeric import ( - integer_squareroot -) - from eth.constants import ( ZERO_HASH32, ) @@ -915,14 +911,12 @@ def test_process_rewards_and_penalties_for_crosslinks( validator_balance = max_deposit_amount total_active_balance = len(active_validators) * validator_balance - _base_reward_quotient = ( - integer_squareroot(total_active_balance) // config.BASE_REWARD_QUOTIENT - ) base_rewards = { index: get_base_reward( state=state, index=index, - base_reward_quotient=_base_reward_quotient, + base_reward_quotient=config.BASE_REWARD_QUOTIENT, + previous_total_balance=total_active_balance, max_deposit_amount=max_deposit_amount, ) for index in active_validators @@ -944,14 +938,12 @@ def test_process_rewards_and_penalties_for_crosslinks( attesting_validators = each_slot_attestion_validators_list[i] total_attesting_balance = len(attesting_validators) * validator_balance total_committee_balance = len(crosslink_committee) * validator_balance - _base_reward_quotient = ( - integer_squareroot(total_active_balance) // config.BASE_REWARD_QUOTIENT - ) for index in attesting_validators: reward = get_base_reward( state=state, index=index, - base_reward_quotient=_base_reward_quotient, + base_reward_quotient=config.BASE_REWARD_QUOTIENT, + previous_total_balance=total_active_balance, max_deposit_amount=max_deposit_amount, ) * total_attesting_balance // total_committee_balance expected_rewards_received[index] += reward @@ -959,7 +951,8 @@ def test_process_rewards_and_penalties_for_crosslinks( penalty = get_base_reward( state=state, index=index, - base_reward_quotient=_base_reward_quotient, + base_reward_quotient=config.BASE_REWARD_QUOTIENT, + previous_total_balance=total_active_balance, max_deposit_amount=max_deposit_amount, ) expected_rewards_received[index] -= penalty From a8ad3a044f024e603830c1bcca84ae5cc26f74e5 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Sun, 17 Mar 2019 10:16:10 +0800 Subject: [PATCH 19/23] Apply PR feedback: naming and type checking --- eth2/beacon/committee_helpers.py | 3 ++- eth2/beacon/epoch_processing_helpers.py | 12 ++++++------ .../forks/serenity/epoch_processing.py | 4 ++-- tests/eth2/beacon/test_epoch_processing_helpers.py | 6 +++--- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/eth2/beacon/committee_helpers.py b/eth2/beacon/committee_helpers.py index c8501188da..2ed89d1741 100644 --- a/eth2/beacon/committee_helpers.py +++ b/eth2/beacon/committee_helpers.py @@ -456,11 +456,12 @@ def get_attestation_participants(state: 'BeaconState', return get_members_from_bitfield(committee, bitfield) +@to_tuple @to_set def get_attester_indices_from_attestations( *, state: 'BeaconState', - attestations: Iterable['Attestation'], + attestations: Sequence['Attestation'], committee_config: CommitteeConfig) -> Iterable[ValidatorIndex]: for a in attestations: yield from get_attestation_participants( diff --git a/eth2/beacon/epoch_processing_helpers.py b/eth2/beacon/epoch_processing_helpers.py index 330cbf5b37..1f65423acc 100644 --- a/eth2/beacon/epoch_processing_helpers.py +++ b/eth2/beacon/epoch_processing_helpers.py @@ -72,7 +72,7 @@ def get_previous_epoch_boundary_attestations( @to_tuple -def get_previous_epoch_head_attestations( +def get_previous_epoch_matching_head_attestations( state: 'BeaconState', slots_per_epoch: int, genesis_epoch: Epoch, @@ -89,7 +89,7 @@ def get_previous_epoch_head_attestations( @to_tuple def _filter_attestations_by_latest_crosslinks( - attestations: Iterable[PendingAttestationRecord], + attestations: Sequence[PendingAttestationRecord], latest_crosslink: CrosslinkRecord) -> Iterable[PendingAttestationRecord]: for attestation in attestations: if attestation.data.latest_crosslink == latest_crosslink: @@ -98,7 +98,7 @@ def _filter_attestations_by_latest_crosslinks( @to_tuple def _filter_attestations_by_shard( - attestations: Iterable[PendingAttestationRecord], + attestations: Sequence[PendingAttestationRecord], shard: Shard) -> Iterable[PendingAttestationRecord]: for attestation in attestations: if attestation.data.shard == shard: @@ -124,12 +124,12 @@ def get_winning_root_and_participants( if len(all_roots) == 0: return (Hash32(ZERO_HASH32), tuple()) - def get_attestations_for(root: Hash32) -> Iterable[PendingAttestationRecord]: + def get_attestations_for(root: Hash32) -> Sequence[PendingAttestationRecord]: return [a for a in valid_attestations if a.data.crosslink_data_root == root] # Winning crosslink root is the root with the most votes for it, ties broken in favor of # lexicographically higher hash - winning_root = max( + winning_root: Hash32 = max( all_roots, key=lambda r: ( get_attesting_balance_from_attestations( @@ -233,7 +233,7 @@ def get_attesting_balance_from_attestations( *, state: 'BeaconState', effective_balances: Dict[ValidatorIndex, Gwei], - attestations: Iterable[PendingAttestationRecord], + attestations: Sequence[PendingAttestationRecord], committee_config: CommitteeConfig) -> Gwei: return get_total_balance_from_effective_balances( effective_balances, diff --git a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py index 648389a389..0b24c59eb1 100644 --- a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py +++ b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py @@ -42,7 +42,7 @@ get_inactivity_penalty, get_inclusion_infos, get_previous_epoch_boundary_attestations, - get_previous_epoch_head_attestations, + get_previous_epoch_matching_head_attestations, get_winning_root_and_participants, get_total_balance, get_total_balance_from_effective_balances, @@ -538,7 +538,7 @@ def _process_rewards_and_penalties_for_finality( committee_config=CommitteeConfig(config), ) - previous_epoch_head_attestations = get_previous_epoch_head_attestations( + previous_epoch_head_attestations = get_previous_epoch_matching_head_attestations( state, config.SLOTS_PER_EPOCH, config.GENESIS_EPOCH, diff --git a/tests/eth2/beacon/test_epoch_processing_helpers.py b/tests/eth2/beacon/test_epoch_processing_helpers.py index f0670e1cc5..16b3352efd 100644 --- a/tests/eth2/beacon/test_epoch_processing_helpers.py +++ b/tests/eth2/beacon/test_epoch_processing_helpers.py @@ -24,7 +24,7 @@ get_epoch_boundary_attester_indices, get_epoch_boundary_attesting_balances, get_inclusion_infos, - get_previous_epoch_head_attestations, + get_previous_epoch_matching_head_attestations, get_winning_root_and_participants, ) from eth2.beacon.helpers import ( @@ -132,7 +132,7 @@ def test_get_current_and_previous_epoch_attestations(random, (10, 100, 0), ] ) -def test_get_previous_epoch_head_attestations( +def test_get_previous_epoch_matching_head_attestations( random, sample_state, genesis_epoch, @@ -193,7 +193,7 @@ def test_get_previous_epoch_head_attestations( ), ) - result = get_previous_epoch_head_attestations( + result = get_previous_epoch_matching_head_attestations( state, slots_per_epoch, genesis_epoch, From 9dc8c6c81cab8d24b33a30c138456b9f6c8b830c Mon Sep 17 00:00:00 2001 From: NIC619 Date: Sun, 17 Mar 2019 10:27:27 +0800 Subject: [PATCH 20/23] Combine two `_filter_attestations_by` helpers --- eth2/beacon/epoch_processing_helpers.py | 24 +++++++------------ .../beacon/test_epoch_processing_helpers.py | 8 +++---- 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/eth2/beacon/epoch_processing_helpers.py b/eth2/beacon/epoch_processing_helpers.py index 1f65423acc..b87fdbf340 100644 --- a/eth2/beacon/epoch_processing_helpers.py +++ b/eth2/beacon/epoch_processing_helpers.py @@ -88,20 +88,14 @@ def get_previous_epoch_matching_head_attestations( @to_tuple -def _filter_attestations_by_latest_crosslinks( - attestations: Sequence[PendingAttestationRecord], - latest_crosslink: CrosslinkRecord) -> Iterable[PendingAttestationRecord]: - for attestation in attestations: - if attestation.data.latest_crosslink == latest_crosslink: - yield attestation - - -@to_tuple -def _filter_attestations_by_shard( +def _filter_attestations_by_latest_crosslinks_and_shard( attestations: Sequence[PendingAttestationRecord], + latest_crosslink: CrosslinkRecord, shard: Shard) -> Iterable[PendingAttestationRecord]: for attestation in attestations: - if attestation.data.shard == shard: + is_latest_crosslink_matched = attestation.data.latest_crosslink == latest_crosslink + is_shard_matched = attestation.data.shard == shard + if is_latest_crosslink_matched and is_shard_matched: yield attestation @@ -111,12 +105,10 @@ def get_winning_root_and_participants( shard: Shard, effective_balances: Dict[ValidatorIndex, Gwei], committee_config: CommitteeConfig) -> Tuple[Hash32, Sequence[ValidatorIndex]]: - valid_attestations = _filter_attestations_by_latest_crosslinks( - _filter_attestations_by_shard( - state.current_epoch_attestations + state.previous_epoch_attestations, - shard, - ), + valid_attestations = _filter_attestations_by_latest_crosslinks_and_shard( + state.current_epoch_attestations + state.previous_epoch_attestations, state.latest_crosslinks[shard], + shard, ) all_roots = set([a.data.crosslink_data_root for a in valid_attestations]) diff --git a/tests/eth2/beacon/test_epoch_processing_helpers.py b/tests/eth2/beacon/test_epoch_processing_helpers.py index 16b3352efd..eeb605ab7a 100644 --- a/tests/eth2/beacon/test_epoch_processing_helpers.py +++ b/tests/eth2/beacon/test_epoch_processing_helpers.py @@ -326,16 +326,16 @@ def mock_get_crosslink_committees_at_slot(state, if len(block_root_1_participants) == len(block_root_2_participants): if competing_block_roots[0] > competing_block_roots[1]: assert winning_root == competing_block_roots[0] - assert attesting_validator_indices == set(block_root_1_participants) + assert set(attesting_validator_indices) == set(block_root_1_participants) else: assert winning_root == competing_block_roots[1] - assert attesting_validator_indices == set(block_root_2_participants) + assert set(attesting_validator_indices) == set(block_root_2_participants) elif len(block_root_1_participants) < len(block_root_2_participants): assert winning_root == competing_block_roots[1] - assert attesting_validator_indices == set(block_root_2_participants) + assert set(attesting_validator_indices) == set(block_root_2_participants) else: assert winning_root == competing_block_roots[0] - assert attesting_validator_indices == set(block_root_1_participants) + assert set(attesting_validator_indices) == set(block_root_1_participants) @settings(max_examples=1) From ea0d44fb37734a514af1c8479a4de3c1d0bc8220 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Sun, 17 Mar 2019 10:28:47 +0800 Subject: [PATCH 21/23] Add check for 0 total balance in `get_base_reward` --- eth2/beacon/epoch_processing_helpers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eth2/beacon/epoch_processing_helpers.py b/eth2/beacon/epoch_processing_helpers.py index b87fdbf340..c1a3b89aad 100644 --- a/eth2/beacon/epoch_processing_helpers.py +++ b/eth2/beacon/epoch_processing_helpers.py @@ -244,6 +244,8 @@ def get_base_reward( base_reward_quotient: int, previous_total_balance: Gwei, max_deposit_amount: Gwei) -> Gwei: + if previous_total_balance == 0: + return Gwei(0) adjusted_quotient = ( integer_squareroot(previous_total_balance) // base_reward_quotient ) From 9691de31606d31c530fee751b72c4b292fc415d9 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Sun, 17 Mar 2019 13:06:21 +0800 Subject: [PATCH 22/23] Apply PR feedback: update return type syntax in `get_winning_root_and_participants` --- eth2/beacon/epoch_processing_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth2/beacon/epoch_processing_helpers.py b/eth2/beacon/epoch_processing_helpers.py index c1a3b89aad..27a1bab170 100644 --- a/eth2/beacon/epoch_processing_helpers.py +++ b/eth2/beacon/epoch_processing_helpers.py @@ -104,7 +104,7 @@ def get_winning_root_and_participants( state: 'BeaconState', shard: Shard, effective_balances: Dict[ValidatorIndex, Gwei], - committee_config: CommitteeConfig) -> Tuple[Hash32, Sequence[ValidatorIndex]]: + committee_config: CommitteeConfig) -> Tuple[Hash32, Tuple[ValidatorIndex, ...]]: valid_attestations = _filter_attestations_by_latest_crosslinks_and_shard( state.current_epoch_attestations + state.previous_epoch_attestations, state.latest_crosslinks[shard], From 73a4377ee312738982b981e507acad09b3f53cb5 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 19 Mar 2019 21:22:53 +0800 Subject: [PATCH 23/23] Remove no-op if/else branches --- .../state_machines/forks/serenity/epoch_processing.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py index 0b24c59eb1..524aa58db2 100644 --- a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py +++ b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py @@ -295,10 +295,7 @@ def process_crosslinks(state: BeaconState, config: BeaconConfig) -> BeaconState: effective_balances=effective_balances, committee_config=CommitteeConfig(config), ) - if len(attesting_validator_indices) == 0: - # No winning shard block root found for this shard. - continue - else: + if len(attesting_validator_indices) > 0: total_attesting_balance = get_total_balance( state.validator_balances, attesting_validator_indices, @@ -318,9 +315,6 @@ def process_crosslinks(state: BeaconState, config: BeaconConfig) -> BeaconState: crosslink_data_root=winning_root, ), ) - else: - # Don't update the crosslink of this shard - pass state = state.copy( latest_crosslinks=latest_crosslinks, )