From 6650ee89b39fda0b97a35216cd783e019c524255 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Wed, 6 Mar 2019 20:46:15 -0800 Subject: [PATCH 1/9] Add per-epoch processing for eth1 data votes --- .../forks/serenity/epoch_processing.py | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py index ce425e0a09..34170282ec 100644 --- a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py +++ b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py @@ -15,6 +15,7 @@ from eth_utils.toolz import ( curry, pipe, + first, ) from eth2.beacon import helpers @@ -72,6 +73,7 @@ from eth2.beacon.datastructures.reward_settlement_context import RewardSettlementContext from eth2.beacon.types.attestations import Attestation from eth2.beacon.types.crosslink_records import CrosslinkRecord +from eth2.beacon.types.eth1_data_vote import Eth1DataVote from eth2.beacon.types.pending_attestation_records import PendingAttestationRecord from eth2.beacon.types.states import BeaconState from eth2.beacon.typing import ( @@ -83,6 +85,47 @@ ) +# +# Eth1 data votes +# + +@curry +def _is_majority_vote(config: BeaconConfig, vote: Eth1DataVote) -> bool: + return vote.vote_count * 2 > config.EPOCHS_PER_ETH1_VOTING_PERIOD * config.SLOTS_PER_EPOCH + +def _update_eth1_vote_if_exists(state: BeaconState, config: BeaconConfig) -> BeaconState: + """ + This function searches the 'pending' Eth1 data votes in ``state`` to find one Eth1 data vote + containing majority support. + + If such a vote is found, update the ``state`` entry for the latest vote. + Regardless of the existence of such a vote, clear the 'pending' storage. + """ + + latest_eth1_data = state.latest_eth1_data + + try: + majority_vote = first( + filter(_is_majority_vote(config), state.eth1_data_votes) + ) + latest_eth1_data = majority_vote.eth1_data + except StopIteration: + pass + + return state.copy( + latest_eth1_data=latest_eth1_data, + eth1_data_votes=(), + ) + + +def process_eth1_data_votes(state: BeaconState, config: BeaconConfig) -> BeaconState: + next_epoch = state.next_epoch(config.SLOTS_PER_EPOCH) + should_process = next_epoch % config.EPOCHS_PER_ETH1_VOTING_PERIOD == 0 + if should_process: + return _update_eth1_vote_if_exists(state, config) + return state + + # # Justification # From 15db550ff5e44210a116089cb944e099c3051a2d Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Wed, 6 Mar 2019 20:46:38 -0800 Subject: [PATCH 2/9] Enable eth1 data vote processing per-epoch --- eth2/beacon/state_machines/forks/serenity/state_transitions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eth2/beacon/state_machines/forks/serenity/state_transitions.py b/eth2/beacon/state_machines/forks/serenity/state_transitions.py index 021cf40f8d..fc4ed2d73d 100644 --- a/eth2/beacon/state_machines/forks/serenity/state_transitions.py +++ b/eth2/beacon/state_machines/forks/serenity/state_transitions.py @@ -18,6 +18,7 @@ validate_proposer_signature, ) from .epoch_processing import ( + process_eth1_data_votes, process_justification, process_crosslinks, process_ejections, @@ -88,7 +89,7 @@ def per_block_transition(self, return state def per_epoch_transition(self, state: BeaconState, block: BaseBeaconBlock) -> BeaconState: - # TODO: state = process_eth1_data_votes(state, self.config) + state = process_eth1_data_votes(state, self.config) state = process_justification(state, self.config) state = process_crosslinks(state, self.config) state = process_rewards_and_penalties(state, self.config) From 5ef2bf47d7b1878f164f14bcff9ab8453ab399cb Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Wed, 6 Mar 2019 20:52:49 -0800 Subject: [PATCH 3/9] linter fix --- eth2/beacon/state_machines/forks/serenity/epoch_processing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py index 34170282ec..95cfdff573 100644 --- a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py +++ b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py @@ -93,6 +93,7 @@ def _is_majority_vote(config: BeaconConfig, vote: Eth1DataVote) -> bool: return vote.vote_count * 2 > config.EPOCHS_PER_ETH1_VOTING_PERIOD * config.SLOTS_PER_EPOCH + def _update_eth1_vote_if_exists(state: BeaconState, config: BeaconConfig) -> BeaconState: """ This function searches the 'pending' Eth1 data votes in ``state`` to find one Eth1 data vote From 2311571de6e9e01e25e68e274bbe7b93408e1c9f Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 7 Mar 2019 15:22:49 -0800 Subject: [PATCH 4/9] Test periodicity of eth1 data vote processing per-epoch --- .../forks/test_serenity_epoch_processing.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py b/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py index 86003c0d82..87b815aea8 100644 --- a/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py +++ b/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py @@ -65,6 +65,7 @@ process_crosslinks, process_ejections, process_exit_queue, + process_eth1_data_votes, process_final_updates, process_justification, process_slashings, @@ -76,6 +77,42 @@ from eth2.beacon.types.validator_records import ValidatorRecord +# +# Eth1 data votes +# +def test_only_process_eth1_data_votes_per_period(sample_beacon_state_params, + config): + slots_per_epoch = config.SLOTS_PER_EPOCH + epochs_per_voting_period = config.EPOCHS_PER_ETH1_VOTING_PERIOD + number_of_epochs_to_sample = 3 + + # NOTE: we process if the _next_ epoch is on a voting period, so subtract 1 here + # NOTE: we also avoid the epoch 0 so change range bounds + epochs_to_process_votes = [ + (epochs_per_voting_period * epoch) - 1 for epoch in range(1, number_of_epochs_to_sample + 1) + ] + state = BeaconState(**sample_beacon_state_params) + + last_epoch_to_process_votes = epochs_to_process_votes[-1] + # NOTE: we arbitrarily pick two after; if this fails here, think about how to + # change so we avoid including another voting period + some_epochs_after_last_target = last_epoch_to_process_votes + 2 + assert some_epochs_after_last_target % epochs_per_voting_period != 0 + + for epoch in range(some_epochs_after_last_target): + slot = get_epoch_start_slot(epoch, slots_per_epoch) + state = state.copy(slot=slot) + updated_state = process_eth1_data_votes(state, config) + if epoch in epochs_to_process_votes: + # we should get back a different state object + assert id(state) != id(updated_state) + # in particular, with no eth1 data votes + assert not updated_state.eth1_data_votes + else: + # we get back the same state (by value) + assert state == updated_state + + # # Justification # From b1183a877e822132eb427f220e749223e82e7d79 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 7 Mar 2019 16:27:20 -0800 Subject: [PATCH 5/9] break the majority threshold calculation into its own function --- .../state_machines/forks/serenity/epoch_processing.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py index 95cfdff573..dbc870cc86 100644 --- a/eth2/beacon/state_machines/forks/serenity/epoch_processing.py +++ b/eth2/beacon/state_machines/forks/serenity/epoch_processing.py @@ -88,10 +88,16 @@ # # Eth1 data votes # +def _majority_threshold(config: BeaconConfig) -> int: + """ + Return the value constituting the majority threshold for an Eth1 data vote. + """ + return config.EPOCHS_PER_ETH1_VOTING_PERIOD * config.SLOTS_PER_EPOCH + @curry def _is_majority_vote(config: BeaconConfig, vote: Eth1DataVote) -> bool: - return vote.vote_count * 2 > config.EPOCHS_PER_ETH1_VOTING_PERIOD * config.SLOTS_PER_EPOCH + return vote.vote_count * 2 > _majority_threshold(config) def _update_eth1_vote_if_exists(state: BeaconState, config: BeaconConfig) -> BeaconState: From c0441ee5e46e4f10f54205dd46e7b1e2c91b6629 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 7 Mar 2019 16:30:54 -0800 Subject: [PATCH 6/9] add tests for eth1 data update per-epoch, conditional on majority vote --- .../forks/test_serenity_epoch_processing.py | 91 ++++++++++++++++++- 1 file changed, 89 insertions(+), 2 deletions(-) diff --git a/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py b/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py index 87b815aea8..e3ba956fba 100644 --- a/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py +++ b/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py @@ -1,4 +1,5 @@ import pytest +import random from hypothesis import ( given, @@ -14,6 +15,11 @@ ZERO_HASH32, ) +from eth_utils.toolz import ( + assoc, + curry, +) + from eth2._utils.tuple import ( update_tuple_item, ) @@ -50,6 +56,8 @@ from eth2.beacon.datastructures.inclusion_info import InclusionInfo from eth2.beacon.types.attestations import Attestation from eth2.beacon.types.attestation_data import AttestationData +from eth2.beacon.types.eth1_data import Eth1Data +from eth2.beacon.types.eth1_data_vote import Eth1DataVote from eth2.beacon.types.crosslink_records import CrosslinkRecord from eth2.beacon.types.pending_attestation_records import PendingAttestationRecord from eth2.beacon.state_machines.forks.serenity.epoch_processing import ( @@ -58,9 +66,12 @@ _compute_total_penalties, _current_previous_epochs_justifiable, _get_finalized_epoch, + _is_majority_vote, + _majority_threshold, _process_rewards_and_penalties_for_attestation_inclusion, _process_rewards_and_penalties_for_crosslinks, _process_rewards_and_penalties_for_finality, + _update_eth1_vote_if_exists, _update_latest_active_index_roots, process_crosslinks, process_ejections, @@ -80,8 +91,84 @@ # # Eth1 data votes # -def test_only_process_eth1_data_votes_per_period(sample_beacon_state_params, - config): +def test_majority_threshold(config): + threshold = config.EPOCHS_PER_ETH1_VOTING_PERIOD * config.SLOTS_PER_EPOCH + assert _majority_threshold(config) == threshold + + +@curry +def _mk_eth1_data_vote(params, vote_count): + return Eth1DataVote(**assoc(params, "vote_count", vote_count)) + + +def test_ensure_majority_votes(sample_eth1_data_vote_params, config): + threshold = _majority_threshold(config) + votes = map(_mk_eth1_data_vote(sample_eth1_data_vote_params), range(2 * threshold)) + for vote in votes: + if vote.vote_count * 2 > threshold: + assert _is_majority_vote(config, vote) + else: + assert not _is_majority_vote(config, vote) + + +def _random_bytes(size): + return bytes(random.getrandbits(8) for _ in range(size)) + + +@pytest.mark.parametrize( + ( + 'vote_offsets' # a tuple of offsets against the majority threshold + ), + ( + # no eth1_data_votes + (), + # a minority of eth1_data_votes (single) + (-2,), + # a plurality of eth1_data_votes (multiple but not majority) + (-2, -2), + # almost a majority! + (0,), + # a majority of eth1_data_votes + (12,), + # NOTE: we are accepting more than one block per slot if + # there are multiple majorities so no need to test this + ) +) +def test_ensure_update_eth1_vote_if_exists(sample_beacon_state_params, + config, + vote_offsets): + # one less than a majority is the majority divided by 2 + threshold = _majority_threshold(config) / 2 + data_votes = tuple( + Eth1DataVote( + eth1_data=Eth1Data( + deposit_root=_random_bytes(32), + block_hash=_random_bytes(32), + ), + vote_count=threshold + offset, + ) for offset in vote_offsets + ) + params = assoc(sample_beacon_state_params, "eth1_data_votes", data_votes) + state = BeaconState(**params) + + if data_votes: # we should have non-empty votes for non-empty inputs + assert state.eth1_data_votes + + updated_state = _update_eth1_vote_if_exists(state, config) + + # we should *always* clear the pending set + assert not updated_state.eth1_data_votes + + # we should update the 'latest' entry if we have a majority + for offset in vote_offsets: + if offset <= 0: + assert state.latest_eth1_data == updated_state.latest_eth1_data + else: + assert len(data_votes) == 1 # sanity check + assert updated_state.latest_eth1_data == data_votes[0].eth1_data + + +def test_only_process_eth1_data_votes_per_period(sample_beacon_state_params, config): slots_per_epoch = config.SLOTS_PER_EPOCH epochs_per_voting_period = config.EPOCHS_PER_ETH1_VOTING_PERIOD number_of_epochs_to_sample = 3 From 83c8cf55a94996bf0dfd1af3956e918e64cc9ab8 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Mon, 11 Mar 2019 16:58:14 -0700 Subject: [PATCH 7/9] Update mock data generation so it is more reproducible Via feedback from @hwwhww --- .../forks/test_serenity_epoch_processing.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py b/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py index e3ba956fba..a89190448d 100644 --- a/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py +++ b/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py @@ -111,8 +111,8 @@ def test_ensure_majority_votes(sample_eth1_data_vote_params, config): assert not _is_majority_vote(config, vote) -def _random_bytes(size): - return bytes(random.getrandbits(8) for _ in range(size)) +def _some_bytes(seed): + return hash_eth2(b'some_hash' + seed.to_bytes(32, 'little')) @pytest.mark.parametrize( @@ -142,8 +142,8 @@ def test_ensure_update_eth1_vote_if_exists(sample_beacon_state_params, data_votes = tuple( Eth1DataVote( eth1_data=Eth1Data( - deposit_root=_random_bytes(32), - block_hash=_random_bytes(32), + deposit_root=_some_bytes(offset), + block_hash=_some_bytes(offset), ), vote_count=threshold + offset, ) for offset in vote_offsets From 2ed16e77bfed039c73a2a1a7b509a05716f2e694 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Mon, 11 Mar 2019 17:07:33 -0700 Subject: [PATCH 8/9] linter fix --- .../state_machines/forks/test_serenity_epoch_processing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py b/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py index a89190448d..d50a54aff5 100644 --- a/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py +++ b/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py @@ -1,5 +1,4 @@ import pytest -import random from hypothesis import ( given, From f6dcc48f163dd3b18e2eb2a4de82956520f94b29 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Mon, 11 Mar 2019 17:07:39 -0700 Subject: [PATCH 9/9] fix bug where we need to supply a positive value --- .../state_machines/forks/test_serenity_epoch_processing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py b/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py index d50a54aff5..3514238e68 100644 --- a/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py +++ b/tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py @@ -111,7 +111,7 @@ def test_ensure_majority_votes(sample_eth1_data_vote_params, config): def _some_bytes(seed): - return hash_eth2(b'some_hash' + seed.to_bytes(32, 'little')) + return hash_eth2(b'some_hash' + abs(seed).to_bytes(32, 'little')) @pytest.mark.parametrize(