Merge pull request #228 from NIC619/Process_crosslinks

Implement process_crosslink

eth2/beacon/exceptions.py

@@ -23,3 +23,10 @@ class ProposerIndexError(PyEVMError):
     of proposer of the given ``slot``
     """
     pass
+
+
+class NoWinningRootError(Exception):
+    """
+    Raised when no shard block root is attested to among the attestations provided.
+    """
+    pass

eth2/beacon/helpers.py

@@ -9,6 +9,7 @@
 
 from eth_utils import (
     to_tuple,
+    to_set,
     ValidationError,
 )
 from eth_typing import (
@@ -30,9 +31,13 @@
     shuffle,
     split,
 )
+from eth2.beacon.exceptions import NoWinningRootError
 from eth2.beacon.enums import (
     SignatureDomain,
 )
+from eth2.beacon.types.pending_attestation_records import (
+    PendingAttestationRecord,
+)
 from eth2.beacon.typing import (
     Bitfield,
     BLSPubkey,
@@ -400,11 +405,8 @@ def get_attestation_participants(state: 'BeaconState',
     committee_size = len(committee)
     if len(aggregation_bitfield) != get_bitfield_length(committee_size):
         raise ValidationError(
-            'Invalid bitfield length,'
-            "\texpected: {}, found: {}".format(
-                get_bitfield_length(committee_size),
-                len(attestation_data),
-            )
+            f"Invalid bitfield length,"
+            f"\texpected: {get_bitfield_length(committee_size)}, found: {len(aggregation_bitfield)}"
         )
 
     # Find the participating attesters in the committee
@@ -413,6 +415,130 @@ def get_attestation_participants(state: 'BeaconState',
             yield validator_index
 
 
+#
+# Per-epoch processing helpers
+#
+@to_tuple
+def get_current_epoch_attestations(
+        state: 'BeaconState',
+        epoch_length: int) -> Iterable[PendingAttestationRecord]:
+    """
+    Note that this function should only be called at epoch boundaries as
+    it's computed based on current slot number.
+    """
+    for attestation in state.latest_attestations:
+        if state.slot - epoch_length <= attestation.data.slot < state.slot:
+            yield attestation
+
+
+@to_tuple
+def get_previous_epoch_attestations(
+        state: 'BeaconState',
+        epoch_length: int) -> Iterable[PendingAttestationRecord]:
+    """
+    Note that this function should only be called at epoch boundaries as
+    it's computed based on current slot number.
+    """
+    for attestation in state.latest_attestations:
+        if state.slot - 2 * epoch_length <= attestation.data.slot < state.slot - epoch_length:
+            yield attestation
+
+
+@to_tuple
+@to_set
+def get_attesting_validator_indices(
+        *,
+        state: 'BeaconState',
+        attestations: Sequence[PendingAttestationRecord],
+        shard: ShardNumber,
+        shard_block_root: Hash32,
+        epoch_length: int,
+        target_committee_size: int,
+        shard_count: int) -> Iterable[ValidatorIndex]:
+    """
+    Loop through ``attestations`` and check if ``shard``/``shard_block_root`` in the attestation
+    matches the given ``shard``/``shard_block_root``.
+    If the attestation matches, get the index of the participating validators.
+    Finally, return the union of the indices.
+    """
+    for a in attestations:
+        if a.data.shard == shard and a.data.shard_block_root == shard_block_root:
+            yield from get_attestation_participants(
+                state,
+                a.data,
+                a.aggregation_bitfield,
+                epoch_length,
+                target_committee_size,
+                shard_count,
+            )
+
+
+def get_total_attesting_balance(
+        *,
+        state: 'BeaconState',
+        shard: ShardNumber,
+        shard_block_root: Hash32,
+        attestations: Sequence[PendingAttestationRecord],
+        epoch_length: int,
+        max_deposit_amount: Gwei,
+        target_committee_size: int,
+        shard_count: int) -> Gwei:
+    return Gwei(
+        sum(
+            get_effective_balance(state.validator_balances, i, max_deposit_amount)
+            for i in get_attesting_validator_indices(
+                state=state,
+                attestations=attestations,
+                shard=shard,
+                shard_block_root=shard_block_root,
+                epoch_length=epoch_length,
+                target_committee_size=target_committee_size,
+                shard_count=shard_count,
+            )
+        )
+    )
+
+
+def get_winning_root(
+        *,
+        state: 'BeaconState',
+        shard: ShardNumber,
+        attestations: Sequence[PendingAttestationRecord],
+        epoch_length: int,
+        max_deposit_amount: Gwei,
+        target_committee_size: int,
+        shard_count: int) -> Tuple[Hash32, Gwei]:
+    winning_root = None
+    winning_root_balance: Gwei = Gwei(0)
+    shard_block_roots = set(
+        [
+            a.data.shard_block_root for a in attestations
+            if a.data.shard == shard
+        ]
+    )
+    for shard_block_root in shard_block_roots:
+        total_attesting_balance = get_total_attesting_balance(
+            state=state,
+            shard=shard,
+            shard_block_root=shard_block_root,
+            attestations=attestations,
+            epoch_length=epoch_length,
+            max_deposit_amount=max_deposit_amount,
+            target_committee_size=target_committee_size,
+            shard_count=shard_count,
+        )
+        if total_attesting_balance > winning_root_balance:
+            winning_root = shard_block_root
+            winning_root_balance = total_attesting_balance
+        elif total_attesting_balance == winning_root_balance and winning_root_balance > 0:
+            if shard_block_root < winning_root:
+                winning_root = shard_block_root
+
+    if winning_root is None:
+        raise NoWinningRootError
+    return (winning_root, winning_root_balance)
+
+
 #
 # Misc
 #

eth2/beacon/state_machines/forks/serenity/epoch_processing.py

@@ -1,25 +1,112 @@
 from typing import (
+    Iterable,
+    Sequence,
     Tuple,
 )
 
+from eth_utils import to_tuple
+
+from eth2.beacon import helpers
 from eth2._utils.numeric import (
     is_power_of_two,
 )
 from eth2._utils.tuple import (
     update_tuple_item,
 )
-from eth2.beacon._utils.hash import (
-    hash_eth2,
-)
-from eth2.beacon import helpers
+from eth2.beacon.exceptions import NoWinningRootError
 from eth2.beacon.helpers import (
     get_active_validator_indices,
+    get_crosslink_committees_at_slot,
     get_current_epoch_committee_count_per_slot,
+    get_current_epoch_attestations,
+    get_effective_balance,
+    get_winning_root,
 )
+from eth2.beacon.typing import ShardNumber
+from eth2.beacon._utils.hash import (
+    hash_eth2,
+)
+from eth2.beacon.types.attestations import Attestation
+from eth2.beacon.types.crosslink_records import CrosslinkRecord
 from eth2.beacon.types.states import BeaconState
 from eth2.beacon.state_machines.configs import BeaconConfig
 
 
+#
+# Crosslinks
+#
+@to_tuple
+def _filter_attestations_by_shard(
+        attestations: Sequence[Attestation],
+        shard: ShardNumber) -> Iterable[Attestation]:
+    for attestation in attestations:
+        if attestation.data.shard == shard:
+            yield attestation
+
+
+def process_crosslinks(state: BeaconState, config: BeaconConfig) -> BeaconState:
+    """
+    Implement 'per-epoch-processing.crosslinks' portion of Phase 0 spec:
+    https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#crosslinks
+
+    For each shard from the past two epochs, find the shard block
+    root that has been attested to by the most stake.
+    If enough(>= 2/3 total stake) attesting stake, update the crosslink record of that shard.
+    Return resulting ``state``
+    """
+    latest_crosslinks = state.latest_crosslinks
+    current_epoch_attestations = get_current_epoch_attestations(state, config.EPOCH_LENGTH)
+    # TODO: STUB, in spec it was
+    # `for slot in range(state.slot - 2 * config.EPOCH_LENGTH, state.slot):``
+    # waiting for ethereum/eth2.0-specs#492 to update the spec
+    for slot in range(state.slot - 1 * config.EPOCH_LENGTH, state.slot):
+        crosslink_committees_at_slot = get_crosslink_committees_at_slot(
+            state,
+            slot,
+            config.EPOCH_LENGTH,
+            config.TARGET_COMMITTEE_SIZE,
+            config.SHARD_COUNT,
+        )
+        for crosslink_committee, shard in crosslink_committees_at_slot:
+            try:
+                winning_root, total_attesting_balance = get_winning_root(
+                    state=state,
+                    shard=shard,
+                    # Use `_filter_attestations_by_shard` to filter out attestations
+                    # not attesting to this shard so we don't need to going over
+                    # irrelevent attestations over and over again.
+                    attestations=_filter_attestations_by_shard(current_epoch_attestations, shard),
+                    epoch_length=config.EPOCH_LENGTH,
+                    max_deposit_amount=config.MAX_DEPOSIT_AMOUNT,
+                    target_committee_size=config.TARGET_COMMITTEE_SIZE,
+                    shard_count=config.SHARD_COUNT,
+                )
+            except NoWinningRootError:
+                # No winning shard block root found for this shard.
+                pass
+            else:
+                total_balance = sum(
+                    get_effective_balance(state.validator_balances, i, config.MAX_DEPOSIT_AMOUNT)
+                    for i in crosslink_committee
+                )
+                if 3 * total_attesting_balance >= 2 * total_balance:
+                    latest_crosslinks = update_tuple_item(
+                        latest_crosslinks,
+                        shard,
+                        CrosslinkRecord(
+                            slot=state.slot,
+                            shard_block_root=winning_root,
+                        ),
+                    )
+                else:
+                    # Don't update the crosslink of this shard
+                    pass
+    state = state.copy(
+        latest_crosslinks=latest_crosslinks,
+    )
+    return state
+
+
 #
 # Validator registry and shuffling seed data
 #

eth2/beacon/state_machines/forks/serenity/state_transitions.py

@@ -10,6 +10,7 @@
 from eth2.beacon.types.states import BeaconState
 
 from .epoch_processing import (
+    process_crosslinks,
     process_final_updates,
     process_validator_registry,
 )
@@ -121,7 +122,7 @@ def per_block_transition(self,
     def per_epoch_transition(self, state: BeaconState, block: BaseBeaconBlock) -> BeaconState:
         # TODO: state = process_et1_data_votes(state, self.config)
         # TODO: state = process_justification(state, self.config)
-        # TODO: state = process_crosslinks(state, self.config)
+        state = process_crosslinks(state, self.config)
         # TODO: state = process_rewards_and_penalties(state, self.config)
         # TODO: state = process_ejections(state, self.config)
         state = process_validator_registry(state, self.config)

eth2/beacon/types/attestations.py

@@ -20,7 +20,7 @@ class Attestation(rlp.Serializable):
     """
     fields = [
         ('data', AttestationData),
-        # Attester participation bitfield
+        # Attester aggregation bitfield
         ('aggregation_bitfield', binary),
         # Proof of custody bitfield
         ('custody_bitfield', binary),

eth2/beacon/types/pending_attestation_records.py

@@ -23,7 +23,7 @@ class PendingAttestationRecord(rlp.Serializable):
     fields = [
         # Signed data
         ('data', AttestationData),
-        # Attester participation bitfield
+        # Attester aggregation bitfield
         ('aggregation_bitfield', binary),
         # Custody bitfield
         ('custody_bitfield', binary),

tests/eth2/beacon/conftest.py

@@ -344,8 +344,13 @@ def filled_beacon_state(genesis_slot,
 
 
 @pytest.fixture()
-def ten_validators_state(filled_beacon_state, max_deposit_amount):
-    validator_count = 10
+def n():
+    return 10
+
+
+@pytest.fixture()
+def n_validators_state(filled_beacon_state, max_deposit_amount, n):
+    validator_count = n
     return filled_beacon_state.copy(
         validator_registry=tuple(
             mock_validator_record(