From dd39d25c86d812e7d5ac24e6bc5f043426e3617d Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 19 Mar 2019 09:32:06 -0500 Subject: [PATCH 1/7] Replace committee exponential backoff with max progress Removes the mechanism that only rotates committees if blocks have been finalized and every shard has been crosslinked or at exponentially decreasing intervals, and replaces it with a rule that shard committees can only progress a maximum of 64 epochs at a time to preserve the invariant that maximum possible work required per epoch for a validator is O(1). --- specs/core/0_beacon-chain.md | 95 +++++++++++------------------------- 1 file changed, 28 insertions(+), 67 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index a631bf2fc6..6877b93585 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -185,8 +185,10 @@ Code snippets appearing in `this style` are to be interpreted as Python code. | `MAX_INDICES_PER_SLASHABLE_VOTE` | `2**12` (= 4,096) | | `MAX_EXIT_DEQUEUES_PER_EPOCH` | `2**2` (= 4) | | `SHUFFLE_ROUND_COUNT` | 90 | +| `MAX_CROSSLINK_EPOCHS` | `2**6` (= 64) | * For the safety of crosslinks `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) +* `MAX_CROSSLINK_EPOCHS` should be a small constant times `SHARD_COUNT // EPOCH_LENGTH` ### Deposit contract @@ -598,12 +600,7 @@ The types are defined topologically to aid in facilitating an executable version # Randomness and committees 'latest_randao_mixes': ['bytes32', LATEST_RANDAO_MIXES_LENGTH], - 'previous_shuffling_start_shard': 'uint64', 'current_shuffling_start_shard': 'uint64', - 'previous_shuffling_epoch': 'uint64', - 'current_shuffling_epoch': 'uint64', - 'previous_shuffling_seed': 'bytes32', - 'current_shuffling_seed': 'bytes32', # Finality 'previous_epoch_attestations': [PendingAttestation], @@ -849,7 +846,7 @@ def get_current_epoch_committee_count(state: BeaconState) -> int: """ current_active_validators = get_active_validator_indices( state.validator_registry, - state.current_shuffling_epoch, + get_current_epoch(state), ) return get_epoch_committee_count(len(current_active_validators)) ``` @@ -886,40 +883,30 @@ def get_crosslink_committees_at_slot(state: BeaconState, next_epoch = current_epoch + 1 assert previous_epoch <= epoch <= next_epoch + committees_per_epoch = get_epoch_committee_count(get_active_validator_indices( + state.validator_registry, + epoch, + )) if epoch == current_epoch: - committees_per_epoch = get_current_epoch_committee_count(state) - seed = state.current_shuffling_seed - shuffling_epoch = state.current_shuffling_epoch shuffling_start_shard = state.current_shuffling_start_shard elif epoch == previous_epoch: - committees_per_epoch = get_previous_epoch_committee_count(state) - seed = state.previous_shuffling_seed - shuffling_epoch = state.previous_shuffling_epoch - shuffling_start_shard = state.previous_shuffling_start_shard + shuffling_start_shard = ( + state.current_shuffling_start_shard - EPOCH_LENGTH * committees_per_epoch + ) % SHARD_COUNT elif epoch == next_epoch: - epochs_since_last_registry_update = current_epoch - state.validator_registry_update_epoch - if registry_change: - committees_per_epoch = get_next_epoch_committee_count(state) - seed = generate_seed(state, next_epoch) - shuffling_epoch = next_epoch - current_committees_per_epoch = get_current_epoch_committee_count(state) - shuffling_start_shard = (state.current_shuffling_start_shard + current_committees_per_epoch) % SHARD_COUNT - elif epochs_since_last_registry_update > 1 and is_power_of_two(epochs_since_last_registry_update): - committees_per_epoch = get_next_epoch_committee_count(state) - seed = generate_seed(state, next_epoch) - shuffling_epoch = next_epoch - shuffling_start_shard = state.current_shuffling_start_shard - else: - committees_per_epoch = get_current_epoch_committee_count(state) - seed = state.current_shuffling_seed - shuffling_epoch = state.current_shuffling_epoch - shuffling_start_shard = state.current_shuffling_start_shard + current_epoch_committees = get_epoch_committee_count(get_active_validator_indices( + state.validator_registry, + current_epoch, + )) + shuffling_start_shard = ( + state.current_shuffling_start_shard + EPOCH_LENGTH * current_epoch_committees + ) % SHARD_COUNT shuffling = get_shuffling( - seed, + generate_seed(state, epoch), state.validator_registry, - shuffling_epoch, + epoch, ) offset = slot % SLOTS_PER_EPOCH committees_per_slot = committees_per_epoch // SLOTS_PER_EPOCH @@ -1529,12 +1516,7 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], # Randomness and committees latest_randao_mixes=Vector([ZERO_HASH for _ in range(LATEST_RANDAO_MIXES_LENGTH)]), - previous_shuffling_start_shard=GENESIS_START_SHARD, current_shuffling_start_shard=GENESIS_START_SHARD, - previous_shuffling_epoch=GENESIS_EPOCH, - current_shuffling_epoch=GENESIS_EPOCH, - previous_shuffling_seed=ZERO_HASH, - current_shuffling_seed=ZERO_HASH, # Finality previous_epoch_attestations=[], @@ -1574,7 +1556,6 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], genesis_active_index_root = hash_tree_root(get_active_validator_indices(state.validator_registry, GENESIS_EPOCH)) for index in range(LATEST_ACTIVE_INDEX_ROOTS_LENGTH): state.latest_active_index_roots[index] = genesis_active_index_root - state.current_shuffling_seed = generate_seed(state, GENESIS_EPOCH) return state ``` @@ -1855,7 +1836,7 @@ def process_crosslinks(state: BeaconState) -> None: total_balance = get_total_balance(state, crosslink_committee) if 3 * participating_balance >= 2 * total_balance: state.latest_crosslinks[shard] = Crosslink( - epoch=slot_to_epoch(slot), + epoch=min(slot_to_epoch(slot), state.latest_crosslinks[shard].epoch + MAX_CROSSLINK_EPOCHS), crosslink_data_root=winning_root ) ``` @@ -2060,14 +2041,6 @@ def should_update_validator_registry(state: BeaconState) -> bool: # Must have finalized a new block if state.finalized_epoch <= state.validator_registry_update_epoch: return False - # Must have processed new crosslinks on all shards of the current epoch - shards_to_check = [ - (state.current_shuffling_start_shard + i) % SHARD_COUNT - for i in range(get_current_epoch_committee_count(state)) - ] - for shard in shards_to_check: - if state.latest_crosslinks[shard].epoch <= state.validator_registry_update_epoch: - return False return True ``` @@ -2119,30 +2092,17 @@ def update_validator_registry(state: BeaconState) -> None: Run the following function: ```python -def update_registry_and_shuffling_data(state: BeaconState) -> None: - # First set previous shuffling data to current shuffling data - state.previous_shuffling_epoch = state.current_shuffling_epoch - state.previous_shuffling_start_shard = state.current_shuffling_start_shard - state.previous_shuffling_seed = state.current_shuffling_seed +def update_registry(state: BeaconState) -> None: current_epoch = get_current_epoch(state) next_epoch = current_epoch + 1 # Check if we should update, and if so, update if should_update_validator_registry(state): update_validator_registry(state) - # If we update the registry, update the shuffling data and shards as well - state.current_shuffling_epoch = next_epoch - state.current_shuffling_start_shard = ( - state.current_shuffling_start_shard + - get_current_epoch_committee_count(state) % SHARD_COUNT - ) - state.current_shuffling_seed = generate_seed(state, state.current_shuffling_epoch) - else: - # If processing at least one crosslink keeps failing, then reshuffle every power of two, - # but don't update the current_shuffling_start_shard - epochs_since_last_registry_update = current_epoch - state.validator_registry_update_epoch - if epochs_since_last_registry_update > 1 and is_power_of_two(epochs_since_last_registry_update): - state.current_shuffling_epoch = next_epoch - state.current_shuffling_seed = generate_seed(state, state.current_shuffling_epoch) + # If we update the registry, update the shuffling data 2/3 or and shards as well + state.current_shuffling_start_shard = ( + state.current_shuffling_start_shard + + get_current_epoch_committee_count(state) % SHARD_COUNT + ) ``` **Invariant**: the active index root that is hashed into the shuffling seed actually is the `hash_tree_root` of the validator set that is used for that epoch. @@ -2397,7 +2357,8 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: # the attestation is trying to create Crosslink( crosslink_data_root=attestation.data.crosslink_data_root, - epoch=slot_to_epoch(attestation.data.slot) + epoch=min(slot_to_epoch(attestation.data.slot), + attestation.data.previous_crosslink.epoch + MAX_CROSSLINK_EPOCHS) ) } assert state.latest_crosslinks[attestation.data.shard] in acceptable_crosslink_data From db92235d9ed3eeffa846e50eef82895567cf77c7 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 19 Mar 2019 09:34:37 -0500 Subject: [PATCH 2/7] Removed some no-longer-necessary functions --- specs/core/0_beacon-chain.md | 35 +---------------------------------- 1 file changed, 1 insertion(+), 34 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 6877b93585..9e52148a0f 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -64,9 +64,7 @@ - [`split`](#split) - [`get_epoch_committee_count`](#get_epoch_committee_count) - [`get_shuffling`](#get_shuffling) - - [`get_previous_epoch_committee_count`](#get_previous_epoch_committee_count) - [`get_current_epoch_committee_count`](#get_current_epoch_committee_count) - - [`get_next_epoch_committee_count`](#get_next_epoch_committee_count) - [`get_crosslink_committees_at_slot`](#get_crosslink_committees_at_slot) - [`get_block_root`](#get_block_root) - [`get_state_root`](#get_state_root) @@ -823,20 +821,6 @@ def get_shuffling(seed: Bytes32, **Note**: this definition and the next few definitions make heavy use of repetitive computing. Production implementations are expected to appropriately use caching/memoization to avoid redoing work. -### `get_previous_epoch_committee_count` - -```python -def get_previous_epoch_committee_count(state: BeaconState) -> int: - """ - Return the number of committees in the previous epoch of the given ``state``. - """ - previous_active_validators = get_active_validator_indices( - state.validator_registry, - state.previous_shuffling_epoch, - ) - return get_epoch_committee_count(len(previous_active_validators)) -``` - ### `get_current_epoch_committee_count` ```python @@ -851,20 +835,6 @@ def get_current_epoch_committee_count(state: BeaconState) -> int: return get_epoch_committee_count(len(current_active_validators)) ``` -### `get_next_epoch_committee_count` - -```python -def get_next_epoch_committee_count(state: BeaconState) -> int: - """ - Return the number of committees in the next epoch of the given ``state``. - """ - next_active_validators = get_active_validator_indices( - state.validator_registry, - get_current_epoch(state) + 1, - ) - return get_epoch_committee_count(len(next_active_validators)) -``` - ### `get_crosslink_committees_at_slot` ```python @@ -895,10 +865,7 @@ def get_crosslink_committees_at_slot(state: BeaconState, state.current_shuffling_start_shard - EPOCH_LENGTH * committees_per_epoch ) % SHARD_COUNT elif epoch == next_epoch: - current_epoch_committees = get_epoch_committee_count(get_active_validator_indices( - state.validator_registry, - current_epoch, - )) + current_epoch_committees = get_current_epoch_committee_count(state) shuffling_start_shard = ( state.current_shuffling_start_shard + EPOCH_LENGTH * current_epoch_committees ) % SHARD_COUNT From c5ee74d5e03376ec5c3bef1d294aaa9a3da831f6 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 19 Mar 2019 11:21:17 -0500 Subject: [PATCH 3/7] Justin fixes --- specs/core/0_beacon-chain.md | 37 ++++++++++++++---------------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 9e52148a0f..c14ff9736c 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -183,10 +183,8 @@ Code snippets appearing in `this style` are to be interpreted as Python code. | `MAX_INDICES_PER_SLASHABLE_VOTE` | `2**12` (= 4,096) | | `MAX_EXIT_DEQUEUES_PER_EPOCH` | `2**2` (= 4) | | `SHUFFLE_ROUND_COUNT` | 90 | -| `MAX_CROSSLINK_EPOCHS` | `2**6` (= 64) | * For the safety of crosslinks `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) -* `MAX_CROSSLINK_EPOCHS` should be a small constant times `SHARD_COUNT // EPOCH_LENGTH` ### Deposit contract @@ -232,6 +230,10 @@ Code snippets appearing in `this style` are to be interpreted as Python code. | `SLOTS_PER_HISTORICAL_ROOT` | `2**13` (= 8,192) | slots | ~13 hours | | `MIN_VALIDATOR_WITHDRAWABILITY_DELAY` | `2**8` (= 256) | epochs | ~27 hours | | `PERSISTENT_COMMITTEE_PERIOD` | `2**11` (= 2,048) | epochs | 9 days | +| `MAX_CROSSLINK_EPOCHS` | `2**6` (= 64) | + +* `MAX_CROSSLINK_EPOCHS` should be a small constant times `SHARD_COUNT // SLOTS_PER_EPOCH` + ### State list lengths @@ -598,7 +600,7 @@ The types are defined topologically to aid in facilitating an executable version # Randomness and committees 'latest_randao_mixes': ['bytes32', LATEST_RANDAO_MIXES_LENGTH], - 'current_shuffling_start_shard': 'uint64', + 'latest_start_shard': 'uint64', # Finality 'previous_epoch_attestations': [PendingAttestation], @@ -859,15 +861,15 @@ def get_crosslink_committees_at_slot(state: BeaconState, )) if epoch == current_epoch: - shuffling_start_shard = state.current_shuffling_start_shard + shuffling_start_shard = state.latest_start_shard elif epoch == previous_epoch: shuffling_start_shard = ( - state.current_shuffling_start_shard - EPOCH_LENGTH * committees_per_epoch + state.latest_start_shard - SLOTS_PER_EPOCH * committees_per_epoch ) % SHARD_COUNT elif epoch == next_epoch: current_epoch_committees = get_current_epoch_committee_count(state) shuffling_start_shard = ( - state.current_shuffling_start_shard + EPOCH_LENGTH * current_epoch_committees + state.latest_start_shard + EPOCH_LENGTH * current_epoch_committees ) % SHARD_COUNT shuffling = get_shuffling( @@ -1483,7 +1485,7 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], # Randomness and committees latest_randao_mixes=Vector([ZERO_HASH for _ in range(LATEST_RANDAO_MIXES_LENGTH)]), - current_shuffling_start_shard=GENESIS_START_SHARD, + latest_start_shard=GENESIS_START_SHARD, # Finality previous_epoch_attestations=[], @@ -2003,14 +2005,6 @@ def process_ejections(state: BeaconState) -> None: #### Validator registry and shuffling seed data -```python -def should_update_validator_registry(state: BeaconState) -> bool: - # Must have finalized a new block - if state.finalized_epoch <= state.validator_registry_update_epoch: - return False - return True -``` - ```python def update_validator_registry(state: BeaconState) -> None: """ @@ -2060,16 +2054,13 @@ Run the following function: ```python def update_registry(state: BeaconState) -> None: - current_epoch = get_current_epoch(state) - next_epoch = current_epoch + 1 # Check if we should update, and if so, update - if should_update_validator_registry(state): + if state.finalized_epoch > state.validator_registry_update_epoch: update_validator_registry(state) - # If we update the registry, update the shuffling data 2/3 or and shards as well - state.current_shuffling_start_shard = ( - state.current_shuffling_start_shard + - get_current_epoch_committee_count(state) % SHARD_COUNT - ) + state.latest_start_shard = ( + state.latest_start_shard + + get_current_epoch_committee_count(state) + ) % SHARD_COUNT ``` **Invariant**: the active index root that is hashed into the shuffling seed actually is the `hash_tree_root` of the validator set that is used for that epoch. From b50e148642d4a19d5517ab1ab689708b33ed7b53 Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 19 Mar 2019 17:13:25 +0000 Subject: [PATCH 4/7] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index c14ff9736c..07179aa0ab 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -74,7 +74,6 @@ - [`get_beacon_proposer_index`](#get_beacon_proposer_index) - [`verify_merkle_branch`](#verify_merkle_branch) - [`get_attestation_participants`](#get_attestation_participants) - - [`is_power_of_two`](#is_power_of_two) - [`int_to_bytes1`, `int_to_bytes2`, ...](#int_to_bytes1-int_to_bytes2-) - [`bytes_to_int`](#bytes_to_int) - [`get_effective_balance`](#get_effective_balance) @@ -861,16 +860,12 @@ def get_crosslink_committees_at_slot(state: BeaconState, )) if epoch == current_epoch: - shuffling_start_shard = state.latest_start_shard + start_shard = state.latest_start_shard elif epoch == previous_epoch: - shuffling_start_shard = ( - state.latest_start_shard - SLOTS_PER_EPOCH * committees_per_epoch - ) % SHARD_COUNT + start_shard = (state.latest_start_shard - SLOTS_PER_EPOCH * committees_per_epoch) % SHARD_COUNT elif epoch == next_epoch: current_epoch_committees = get_current_epoch_committee_count(state) - shuffling_start_shard = ( - state.latest_start_shard + EPOCH_LENGTH * current_epoch_committees - ) % SHARD_COUNT + start_shard = (state.latest_start_shard + EPOCH_LENGTH * current_epoch_committees) % SHARD_COUNT shuffling = get_shuffling( generate_seed(state, epoch), @@ -879,7 +874,7 @@ def get_crosslink_committees_at_slot(state: BeaconState, ) offset = slot % SLOTS_PER_EPOCH committees_per_slot = committees_per_epoch // SLOTS_PER_EPOCH - slot_start_shard = (shuffling_start_shard + committees_per_slot * offset) % SHARD_COUNT + slot_start_shard = (start_shard + committees_per_slot * offset) % SHARD_COUNT return [ ( @@ -1017,16 +1012,6 @@ def get_attestation_participants(state: BeaconState, return participants ``` -### `is_power_of_two` - -```python -def is_power_of_two(value: int) -> bool: - """ - Check if ``value`` is a power of two integer. - """ - return (value > 0) and (value & (value - 1) == 0) -``` - ### `int_to_bytes1`, `int_to_bytes2`, ... `int_to_bytes1(x): return x.to_bytes(1, 'little')`, `int_to_bytes2(x): return x.to_bytes(2, 'little')`, and so on for all integers, particularly 1, 2, 3, 4, 8, 32, 48, 96. From a8410b8b843608bbbcec9c4cad76898f7435ae07 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 26 Mar 2019 11:27:07 -0600 Subject: [PATCH 5/7] add some attestation tests. fix genesi crosslink bug --- specs/core/0_beacon-chain.md | 11 ++- .../test_process_attestation.py | 67 +++++++++++++++++++ tests/phase0/helpers.py | 50 ++++++++++++++ tests/phase0/test_sanity.py | 50 +------------- 4 files changed, 124 insertions(+), 54 deletions(-) create mode 100644 tests/phase0/block_processing/test_process_attestation.py diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index e628c70579..4b52bd2fa0 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -905,21 +905,20 @@ def get_crosslink_committees_at_slot(state: BeaconState, next_epoch = current_epoch + 1 assert previous_epoch <= epoch <= next_epoch - active_validator_indices = get_active_validator_indices( + indices = get_active_validator_indices( state.validator_registry, epoch, ) - committees_per_epoch = get_epoch_committee_count(len(active_validator_indices)) + committees_per_epoch = get_epoch_committee_count(len(indices)) if epoch == current_epoch: start_shard = state.latest_start_shard elif epoch == previous_epoch: - start_shard = (state.latest_start_shard - SLOTS_PER_EPOCH * committees_per_epoch) % SHARD_COUNT + start_shard = (state.latest_start_shard - committees_per_epoch) % SHARD_COUNT elif epoch == next_epoch: current_epoch_committees = get_current_epoch_committee_count(state) - start_shard = (state.latest_start_shard + EPOCH_LENGTH * current_epoch_committees) % SHARD_COUNT + start_shard = (state.latest_start_shard + current_epoch_committees) % SHARD_COUNT - indices = get_active_validator_indices(state.validator_registry, epoch) committees_per_slot = committees_per_epoch // SLOTS_PER_EPOCH offset = slot % SLOTS_PER_EPOCH slot_start_shard = (start_shard + committees_per_slot * offset) % SHARD_COUNT @@ -1830,7 +1829,7 @@ Run the following function: ```python def process_crosslinks(state: BeaconState) -> None: current_epoch = get_current_epoch(state) - previous_epoch = current_epoch - 1 + previous_epoch = max(current_epoch - 1, GENESIS_EPOCH) next_epoch = current_epoch + 1 for slot in range(get_epoch_start_slot(previous_epoch), get_epoch_start_slot(next_epoch)): for crosslink_committee, shard in get_crosslink_committees_at_slot(state, slot): diff --git a/tests/phase0/block_processing/test_process_attestation.py b/tests/phase0/block_processing/test_process_attestation.py new file mode 100644 index 0000000000..80770fdf92 --- /dev/null +++ b/tests/phase0/block_processing/test_process_attestation.py @@ -0,0 +1,67 @@ +from copy import deepcopy +import pytest + +import build.phase0.spec as spec + +from build.phase0.state_transition import ( + state_transition, +) +from build.phase0.spec import ( + ZERO_HASH, + get_current_epoch, + process_attestation, + slot_to_epoch, +) +from tests.phase0.helpers import ( + build_empty_block_for_next_slot, + get_valid_attestation, +) + + +# mark entire file as 'attestations' +pytestmark = pytest.mark.attestations + + +def run_attestation_processing(state, attestation, valid=True): + """ + Run ``process_attestation`` returning the pre and post state. + If ``valid == False``, run expecting ``AssertionError`` + """ + post_state = deepcopy(state) + + if not valid: + with pytest.raises(AssertionError): + process_attestation(post_state, attestation) + return state, None + + process_attestation(post_state, attestation) + + current_epoch = get_current_epoch(state) + target_epoch = slot_to_epoch(attestation.data.slot) + if target_epoch == current_epoch: + assert len(post_state.current_epoch_attestations) == len(state.current_epoch_attestations) + 1 + else: + assert len(post_state.previous_epoch_attestations) == len(state.previous_epoch_attestations) + 1 + + + return state, post_state + + +def test_success(state): + attestation = get_valid_attestation(state) + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + + pre_state, post_state = run_attestation_processing(state, attestation) + + return pre_state, attestation, post_state + + +def test_success_prevous_epoch(state): + attestation = get_valid_attestation(state) + block = build_empty_block_for_next_slot(state) + block.slot = state.slot + spec.SLOTS_PER_EPOCH + state_transition(state, block) + + pre_state, post_state = run_attestation_processing(state, attestation) + + return pre_state, attestation, post_state diff --git a/tests/phase0/helpers.py b/tests/phase0/helpers.py index 3c68c2c8c1..d7f4ae6e8f 100644 --- a/tests/phase0/helpers.py +++ b/tests/phase0/helpers.py @@ -9,7 +9,9 @@ EMPTY_SIGNATURE, ZERO_HASH, # SSZ + Attestation, AttestationData, + AttestationDataAndCustodyBit, BeaconBlockHeader, Deposit, DepositData, @@ -18,7 +20,9 @@ VoluntaryExit, # functions get_active_validator_indices, + get_attestation_participants, get_block_root, + get_crosslink_committees_at_slot, get_current_epoch, get_domain, get_empty_block, @@ -236,3 +240,49 @@ def get_valid_proposer_slashing(state): header_1=header_1, header_2=header_2, ) + + +def get_valid_attestation(state, slot=None): + if slot is None: + slot = state.slot + shard = state.latest_start_shard + attestation_data = build_attestation_data(state, slot, shard) + + crosslink_committees = get_crosslink_committees_at_slot(state, slot) + crosslink_committee = [committee for committee, _shard in crosslink_committees if _shard == attestation_data.shard][0] + + committee_size = len(crosslink_committee) + bitfield_length = (committee_size + 7) // 8 + aggregation_bitfield = b'\x01' + b'\x00' * (bitfield_length - 1) + custody_bitfield = b'\x00' * bitfield_length + attestation = Attestation( + aggregation_bitfield=aggregation_bitfield, + data=attestation_data, + custody_bitfield=custody_bitfield, + aggregate_signature=EMPTY_SIGNATURE, + ) + participants = get_attestation_participants( + state, + attestation.data, + attestation.aggregation_bitfield, + ) + assert len(participants) == 1 + + validator_index = participants[0] + privkey = privkeys[validator_index] + + message_hash = AttestationDataAndCustodyBit( + data=attestation.data, + custody_bit=0b0, + ).hash_tree_root() + + attestation.aggregation_signature = bls.sign( + message_hash=message_hash, + privkey=privkey, + domain=get_domain( + fork=state.fork, + epoch=get_current_epoch(state), + domain_type=spec.DOMAIN_ATTESTATION, + ) + ) + return attestation diff --git a/tests/phase0/test_sanity.py b/tests/phase0/test_sanity.py index a2cbadd9a2..b287bde51b 100644 --- a/tests/phase0/test_sanity.py +++ b/tests/phase0/test_sanity.py @@ -11,19 +11,13 @@ EMPTY_SIGNATURE, ZERO_HASH, # SSZ - Attestation, - AttestationDataAndCustodyBit, - BeaconBlockHeader, Deposit, Transfer, - ProposerSlashing, VoluntaryExit, # functions get_active_validator_indices, - get_attestation_participants, get_balance, get_block_root, - get_crosslink_committees_at_slot, get_current_epoch, get_domain, get_state_root, @@ -42,10 +36,10 @@ get_merkle_root, ) from tests.phase0.helpers import ( - build_attestation_data, build_deposit_data, build_empty_block_for_next_slot, force_registry_change_at_next_epoch, + get_valid_attestation, get_valid_proposer_slashing, privkeys, pubkeys, @@ -222,47 +216,7 @@ def test_deposit_top_up(state): def test_attestation(state): test_state = deepcopy(state) - slot = state.slot - shard = state.latest_start_shard - attestation_data = build_attestation_data(state, slot, shard) - - crosslink_committees = get_crosslink_committees_at_slot(state, slot) - crosslink_committee = [committee for committee, _shard in crosslink_committees if _shard == attestation_data.shard][0] - - committee_size = len(crosslink_committee) - bitfield_length = (committee_size + 7) // 8 - aggregation_bitfield = b'\x01' + b'\x00' * (bitfield_length - 1) - custody_bitfield = b'\x00' * bitfield_length - attestation = Attestation( - aggregation_bitfield=aggregation_bitfield, - data=attestation_data, - custody_bitfield=custody_bitfield, - aggregate_signature=EMPTY_SIGNATURE, - ) - participants = get_attestation_participants( - test_state, - attestation.data, - attestation.aggregation_bitfield, - ) - assert len(participants) == 1 - - validator_index = participants[0] - privkey = privkeys[validator_index] - - message_hash = AttestationDataAndCustodyBit( - data=attestation.data, - custody_bit=0b0, - ).hash_tree_root() - - attestation.aggregation_signature = bls.sign( - message_hash=message_hash, - privkey=privkey, - domain=get_domain( - fork=test_state.fork, - epoch=get_current_epoch(test_state), - domain_type=spec.DOMAIN_ATTESTATION, - ) - ) + attestation = get_valid_attestation(state) # # Add to state via block transition From 9fa6055a8a5b2c73774f143d3abdbe23323e93b4 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 26 Mar 2019 11:41:15 -0600 Subject: [PATCH 6/7] add more attestation tests --- .../test_process_attestation.py | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/tests/phase0/block_processing/test_process_attestation.py b/tests/phase0/block_processing/test_process_attestation.py index 80770fdf92..b34c64d956 100644 --- a/tests/phase0/block_processing/test_process_attestation.py +++ b/tests/phase0/block_processing/test_process_attestation.py @@ -65,3 +65,90 @@ def test_success_prevous_epoch(state): pre_state, post_state = run_attestation_processing(state, attestation) return pre_state, attestation, post_state + + +def test_before_inclusion_delay(state): + attestation = get_valid_attestation(state) + # do not increment slot to allow for inclusion delay + + pre_state, post_state = run_attestation_processing(state, attestation, False) + + return pre_state, attestation, post_state + + +def test_after_epoch_slots(state): + attestation = get_valid_attestation(state) + block = build_empty_block_for_next_slot(state) + # increment past latest inclusion slot + block.slot = state.slot + spec.SLOTS_PER_EPOCH + 1 + state_transition(state, block) + + pre_state, post_state = run_attestation_processing(state, attestation, False) + + return pre_state, attestation, post_state + + +def test_bad_source_epoch(state): + attestation = get_valid_attestation(state) + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + + attestation.data.source_epoch += 10 + + pre_state, post_state = run_attestation_processing(state, attestation, False) + + return pre_state, attestation, post_state + + +def test_bad_source_root(state): + attestation = get_valid_attestation(state) + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + + attestation.data.source_root = b'\x42'*32 + + pre_state, post_state = run_attestation_processing(state, attestation, False) + + return pre_state, attestation, post_state + + +def test_non_zero_crosslink_data_root(state): + attestation = get_valid_attestation(state) + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + + attestation.data.crosslink_data_root = b'\x42'*32 + + pre_state, post_state = run_attestation_processing(state, attestation, False) + + return pre_state, attestation, post_state + + +def test_bad_previous_crosslink(state): + attestation = get_valid_attestation(state) + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + + state.latest_crosslinks[attestation.data.shard].epoch += 10 + + pre_state, post_state = run_attestation_processing(state, attestation, False) + + return pre_state, attestation, post_state + + +def test_non_empty_custody_bitfield(state): + attestation = get_valid_attestation(state) + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + + attestation.custody_bitfield = b'\x01' + attestation.custody_bitfield[1:] + + pre_state, post_state = run_attestation_processing(state, attestation, False) + + return pre_state, attestation, post_state + + +def test_empty_aggregation_bitfield(state): + attestation = get_valid_attestation(state) + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + + attestation.aggregation_bitfield = b'\x00' * len(attestation.aggregation_bitfield) + + pre_state, post_state = run_attestation_processing(state, attestation, False) + + return pre_state, attestation, post_state From 2c5a68b5b5d4348ede49f07e50b943eb22c03414 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 26 Mar 2019 12:32:24 -0600 Subject: [PATCH 7/7] remove registry_change options from shuffling functions --- specs/core/0_beacon-chain.md | 8 ++----- specs/validator/0_beacon-chain-validator.md | 21 +++++-------------- .../test_process_attestation.py | 1 - 3 files changed, 7 insertions(+), 23 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 4b52bd2fa0..38f5f56c53 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -891,13 +891,9 @@ def get_current_epoch_committee_count(state: BeaconState) -> int: ```python def get_crosslink_committees_at_slot(state: BeaconState, - slot: Slot, - registry_change: bool=False) -> List[Tuple[List[ValidatorIndex], Shard]]: + slot: Slot) -> List[Tuple[List[ValidatorIndex], Shard]]: """ Return the list of ``(committee, shard)`` tuples for the ``slot``. - - Note: There are two possible shufflings for crosslink committees for a - ``slot`` in the next epoch -- with and without a `registry_change` """ epoch = slot_to_epoch(slot) current_epoch = get_current_epoch(state) @@ -2339,7 +2335,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: # Check target epoch, source epoch, and source root target_epoch = slot_to_epoch(attestation.data.slot) assert (target_epoch, attestation.data.source_epoch, attestation.data.source_root) in { - (get_current_epoch(state), state.current_justified_epoch, state.current_justified_root), + (get_current_epoch(state), state.current_justified_epoch, state.current_justified_root), (get_previous_epoch(state), state.previous_justified_epoch, state.previous_justified_root), } diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index f1a10a048a..4a4c63836d 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -331,16 +331,15 @@ signed_attestation_data = bls_sign( ## Validator assignments -A validator can get the current and previous epoch committee assignments using the following helper via `get_committee_assignment(state, epoch, validator_index)` where `previous_epoch <= epoch <= current_epoch`. +A validator can get the current, previous, and next epoch committee assignments using the following helper via `get_committee_assignment(state, epoch, validator_index)` where `previous_epoch <= epoch <= next_epoch`. ```python def get_committee_assignment( state: BeaconState, epoch: Epoch, - validator_index: ValidatorIndex, - registry_change: bool=False) -> Tuple[List[ValidatorIndex], Shard, Slot]: + validator_index: ValidatorIndex) -> Tuple[List[ValidatorIndex], Shard, Slot]: """ - Return the committee assignment in the ``epoch`` for ``validator_index`` and ``registry_change``. + Return the committee assignment in the ``epoch`` for ``validator_index``. ``assignment`` returned is a tuple of the following form: * ``assignment[0]`` is the list of validators in the committee * ``assignment[1]`` is the shard to which the committee is assigned @@ -355,7 +354,6 @@ def get_committee_assignment( crosslink_committees = get_crosslink_committees_at_slot( state, slot, - registry_change=registry_change, ) selected_committees = [ committee # Tuple[List[ValidatorIndex], Shard] @@ -389,18 +387,9 @@ _Note_: If a validator is assigned to the 0th slot of an epoch, the validator mu The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahead on the validator's upcoming committee assignments for attesting dictated by the shuffling and slot. Note that this lookahead does not apply to proposing which must checked during the epoch in question. -There are three possibilities for the shuffling at the next epoch: -1. The shuffling changes due to a "validator registry change". -2. The shuffling changes due to `epochs_since_last_registry_update` being an exact power of 2 greater than 1. -3. The shuffling remains the same (i.e. the validator is in the same shard committee). +`get_committee_assignment` should be called at the start of each epoch to get the assignment for the next epoch (`current_epoch + 1`). A validator should plan for future assignments which involves noting at which future slot one will have to attest and also which shard one should begin syncing (in phase 1+). -Either (2) or (3) occurs if (1) fails. The choice between (2) and (3) is deterministic based upon `epochs_since_last_registry_update`. - -When querying for assignments in the next epoch there are two options -- with and without a `registry_change` -- which is the optional fourth parameter of the `get_committee_assignment`. - -`get_committee_assignment` should be called at the start of each epoch to get the assignment for the next epoch (`current_epoch + 1`). A validator should always plan for assignments from both values of `registry_change` unless the validator can concretely eliminate one of the options. Planning for future assignments involves noting at which future slot one might have to attest and also which shard one should begin syncing (in phase 1+). - -Specifically, a validator should call both `get_committee_assignment(state, next_epoch, validator_index, registry_change=True)` and `get_committee_assignment(state, next_epoch, validator_index, registry_change=False)` when checking for next epoch assignments. +Specifically, a validator should call `get_committee_assignment(state, next_epoch, validator_index)` when checking for next epoch assignments. ## How to avoid slashing diff --git a/tests/phase0/block_processing/test_process_attestation.py b/tests/phase0/block_processing/test_process_attestation.py index b34c64d956..08cab11ff1 100644 --- a/tests/phase0/block_processing/test_process_attestation.py +++ b/tests/phase0/block_processing/test_process_attestation.py @@ -43,7 +43,6 @@ def run_attestation_processing(state, attestation, valid=True): else: assert len(post_state.previous_epoch_attestations) == len(state.previous_epoch_attestations) + 1 - return state, post_state