Merge pull request #2938 from etan-status/lc-testsuite

Add functions for deriving light client data
ethereum · Jul 20, 2022 · 1d2ef9f · 1d2ef9f
2 parents 2b4613e + 0941114
commit 1d2ef9f
Show file tree

Hide file tree

Showing 24 changed files with 716 additions and 37 deletions.
diff --git a/Makefile b/Makefile
@@ -23,13 +23,15 @@ GENERATOR_VENVS = $(patsubst $(GENERATOR_DIR)/%, $(GENERATOR_DIR)/%venv, $(GENER
 # To check generator matching:
 #$(info $$GENERATOR_TARGETS is [${GENERATOR_TARGETS}])
 
-MARKDOWN_FILES = $(wildcard $(SPEC_DIR)/phase0/*.md) $(wildcard $(SPEC_DIR)/altair/*.md) $(wildcard $(SSZ_DIR)/*.md) \
+MARKDOWN_FILES = $(wildcard $(SPEC_DIR)/phase0/*.md) \
+                 $(wildcard $(SPEC_DIR)/altair/*.md) $(wildcard $(SPEC_DIR)/altair/**/*.md) \
                  $(wildcard $(SPEC_DIR)/bellatrix/*.md) \
                  $(wildcard $(SPEC_DIR)/capella/*.md) \
                  $(wildcard $(SPEC_DIR)/custody/*.md) \
                  $(wildcard $(SPEC_DIR)/das/*.md) \
                  $(wildcard $(SPEC_DIR)/sharding/*.md) \
-                 $(wildcard $(SPEC_DIR)/eip4844/*.md)
+                 $(wildcard $(SPEC_DIR)/eip4844/*.md) \
+                 $(wildcard $(SSZ_DIR)/*.md)
 
 COV_HTML_OUT=.htmlcov
 COV_HTML_OUT_DIR=$(PY_SPEC_DIR)/$(COV_HTML_OUT)

diff --git a/README.md b/README.md
@@ -18,11 +18,11 @@ Features are researched and developed in parallel, and then consolidated into se
 | Seq. | Code Name | Fork Epoch | Specs |
 | - | - | - | - |
 | 0 | **Phase0** |`0` | <ul><li>Core</li><ul><li>[The beacon chain](specs/phase0/beacon-chain.md)</li><li>[Deposit contract](specs/phase0/deposit-contract.md)</li><li>[Beacon chain fork choice](specs/phase0/fork-choice.md)</li></ul><li>Additions</li><ul><li>[Honest validator guide](specs/phase0/validator.md)</li><li>[P2P networking](specs/phase0/p2p-interface.md)</li><li>[Weak subjectivity](specs/phase0/weak-subjectivity.md)</li></ul></ul> |
-| 1 |  **Altair** | `74240` | <ul><li>Core</li><ul><li>[Beacon chain changes](specs/altair/beacon-chain.md)</li><li>[Altair fork](specs/altair/fork.md)</li></ul><li>Additions</li><ul><li>[Light client sync protocol](specs/altair/sync-protocol.md)</li><li>[Honest validator guide changes](specs/altair/validator.md)</li><li>[P2P networking](specs/altair/p2p-interface.md)</li></ul></ul> |
+| 1 |  **Altair** | `74240` | <ul><li>Core</li><ul><li>[Beacon chain changes](specs/altair/beacon-chain.md)</li><li>[Altair fork](specs/altair/fork.md)</li></ul><li>Additions</li><ul><li>[Light client sync protocol](specs/altair/light-client/sync-protocol.md) ([full node](specs/altair/light-client/full-node.md))</li><li>[Honest validator guide changes](specs/altair/validator.md)</li><li>[P2P networking](specs/altair/p2p-interface.md)</li></ul></ul> |
 | 2 | **Bellatrix** <br/> (["The Merge"](https://ethereum.org/en/upgrades/merge/)) | TBD | <ul><li>Core</li><ul><li>[Beacon Chain changes](specs/bellatrix/beacon-chain.md)</li><li>[Bellatrix fork](specs/bellatrix/fork.md)</li><li>[Fork choice changes](specs/bellatrix/fork-choice.md)</li></ul><li>Additions</li><ul><li>[Honest validator guide changes](specs/bellatrix/validator.md)</li><li>[P2P networking](specs/bellatrix/p2p-interface.md)</li></ul></ul> |
 
 ### In-development Specifications
-| Code Name or Topic | Specs | Notes | 
+| Code Name or Topic | Specs | Notes |
 | - | - | - |
 | Capella (tentative) | <ul><li>Core</li><ul><li>[Beacon chain changes](specs/capella/beacon-chain.md)</li><li>[Capella fork](specs/capella/fork.md)</li></ul><li>Additions</li><ul><li>[Validator additions](specs/capella/validator.md)</li></ul></ul> |
 | EIP4844 (tentative) | <ul><li>Core</li><ul><li>[Beacon Chain changes](specs/eip4844/beacon-chain.md)</li><li>[EIP-4844 fork](specs/eip4844/fork.md)</li><li>[Polynomial commitments](specs/eip4844/polynomial-commitments.md)</li></ul><li>Additions</li><ul><li>[Honest validator guide changes](specs/eip4844/validator.md)</li><li>[P2P networking](specs/eip4844/p2p-interface.md)</li></ul></ul> |

diff --git a/setup.py b/setup.py
@@ -458,6 +458,7 @@ def imports(cls, preset_name: str) -> str:
 from typing import NewType, Union as PyUnion
 
 from eth2spec.phase0 import {preset_name} as phase0
+from eth2spec.test.helpers.merkle import build_proof
 from eth2spec.utils.ssz.ssz_typing import Path
 '''
 
@@ -475,7 +476,12 @@ def get_generalized_index(ssz_class: Any, *path: Sequence[PyUnion[int, SSZVariab
     ssz_path = Path(ssz_class)
     for item in path:
         ssz_path = ssz_path / item
-    return GeneralizedIndex(ssz_path.gindex())'''
+    return GeneralizedIndex(ssz_path.gindex())
+
+
+def compute_merkle_proof_for_state(state: BeaconState,
+                                   index: GeneralizedIndex) -> Sequence[Bytes32]:
+    return build_proof(state.get_backing(), index)'''
 
 
     @classmethod
@@ -656,7 +662,11 @@ def format_protocol(protocol_name: str, protocol_def: ProtocolDefinition) -> str
 
     protocols_spec = '\n\n\n'.join(format_protocol(k, v) for k, v in spec_object.protocols.items())
     for k in list(spec_object.functions):
-        if "ceillog2" in k or "floorlog2" in k:
+        if k in [
+            "ceillog2",
+            "floorlog2",
+            "compute_merkle_proof_for_state",
+        ]:
             del spec_object.functions[k]
     functions = builder.implement_optimizations(spec_object.functions)
     functions_spec = '\n\n\n'.join(functions.values())
@@ -817,7 +827,7 @@ def parse_config_vars(conf: Dict[str, str]) -> Dict[str, str]:
     for k, v in conf.items():
         if isinstance(v, str) and (v.startswith("0x") or k == 'PRESET_BASE' or k == 'CONFIG_NAME'):
             # Represent byte data with string, to avoid misinterpretation as big-endian int.
-            # Everything is either byte data or an integer, with PRESET_BASE as one exception.
+            # Everything except PRESET_BASE and CONFIG_NAME is either byte data or an integer.
             out[k] = f"'{v}'"
         else:
             out[k] = str(int(v))
@@ -923,12 +933,13 @@ def finalize_options(self):
                 """
             if self.spec_fork in (ALTAIR, BELLATRIX, CAPELLA, EIP4844):
                 self.md_doc_paths += """
+                    specs/altair/light-client/full-node.md
+                    specs/altair/light-client/sync-protocol.md
                     specs/altair/beacon-chain.md
                     specs/altair/bls.md
                     specs/altair/fork.md
                     specs/altair/validator.md
                     specs/altair/p2p-interface.md
-                    specs/altair/sync-protocol.md
                 """
             if self.spec_fork in (BELLATRIX, CAPELLA, EIP4844):
                 self.md_doc_paths += """

diff --git a/specs/altair/light-client/full-node.md b/specs/altair/light-client/full-node.md
@@ -0,0 +1,135 @@
+# Altair Light Client -- Full Node
+
+**Notice**: This document is a work-in-progress for researchers and implementers.
+
+## Table of contents
+
+<!-- TOC -->
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+
+- [Introduction](#introduction)
+- [Helper functions](#helper-functions)
+  - [`compute_merkle_proof_for_state`](#compute_merkle_proof_for_state)
+- [Deriving light client data](#deriving-light-client-data)
+  - [`create_light_client_bootstrap`](#create_light_client_bootstrap)
+  - [`create_light_client_update`](#create_light_client_update)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+<!-- /TOC -->
+
+## Introduction
+
+This document provides helper functions to enable full nodes to serve light client data. Full nodes SHOULD implement the described functionality to enable light clients to sync with the network.
+
+## Helper functions
+
+### `compute_merkle_proof_for_state`
+
+```python
+def compute_merkle_proof_for_state(state: BeaconState,
+                                   index: GeneralizedIndex) -> Sequence[Bytes32]:
+    ...
+```
+
+## Deriving light client data
+
+Full nodes are expected to derive light client data from historic blocks and states and provide it to other clients.
+
+### `create_light_client_bootstrap`
+
+```python
+def create_light_client_bootstrap(state: BeaconState) -> LightClientBootstrap:
+    assert compute_epoch_at_slot(state.slot) >= ALTAIR_FORK_EPOCH
+    assert state.slot == state.latest_block_header.slot
+
+    return LightClientBootstrap(
+        header=BeaconBlockHeader(
+            slot=state.latest_block_header.slot,
+            proposer_index=state.latest_block_header.proposer_index,
+            parent_root=state.latest_block_header.parent_root,
+            state_root=hash_tree_root(state),
+            body_root=state.latest_block_header.body_root,
+        ),
+        current_sync_committee=state.current_sync_committee,
+        current_sync_committee_branch=compute_merkle_proof_for_state(state, CURRENT_SYNC_COMMITTEE_INDEX)
+    )
+```
+
+Full nodes SHOULD provide `LightClientBootstrap` for all finalized epoch boundary blocks in the epoch range `[max(ALTAIR_FORK_EPOCH, current_epoch - MIN_EPOCHS_FOR_BLOCK_REQUESTS), current_epoch]` where `current_epoch` is defined by the current wall-clock time. Full nodes MAY also provide `LightClientBootstrap` for other blocks.
+
+Blocks are considered to be epoch boundary blocks if their block root can occur as part of a valid `Checkpoint`, i.e., if their slot is the initial slot of an epoch, or if all following slots through the initial slot of the next epoch are empty (no block proposed / orphaned).
+
+`LightClientBootstrap` is computed from the block's immediate post state (without applying empty slots).
+
+### `create_light_client_update`
+
+To form a `LightClientUpdate`, the following historical states and blocks are needed:
+- `state`: the post state of any block with a post-Altair parent block
+- `block`: the corresponding block
+- `attested_state`: the post state of the block referred to by `block.parent_root`
+- `finalized_block`: the block referred to by `attested_state.finalized_checkpoint.root`, if locally available (may be unavailable, e.g., when using checkpoint sync, or if it was pruned locally)
+
+```python
+def create_light_client_update(state: BeaconState,
+                               block: SignedBeaconBlock,
+                               attested_state: BeaconState,
+                               finalized_block: Optional[SignedBeaconBlock]) -> LightClientUpdate:
+    assert compute_epoch_at_slot(attested_state.slot) >= ALTAIR_FORK_EPOCH
+    assert sum(block.message.body.sync_aggregate.sync_committee_bits) >= MIN_SYNC_COMMITTEE_PARTICIPANTS
+
+    assert state.slot == state.latest_block_header.slot
+    header = state.latest_block_header.copy()
+    header.state_root = hash_tree_root(state)
+    assert hash_tree_root(header) == hash_tree_root(block.message)
+    update_signature_period = compute_sync_committee_period(compute_epoch_at_slot(block.message.slot))
+
+    assert attested_state.slot == attested_state.latest_block_header.slot
+    attested_header = attested_state.latest_block_header.copy()
+    attested_header.state_root = hash_tree_root(attested_state)
+    assert hash_tree_root(attested_header) == block.message.parent_root
+    update_attested_period = compute_sync_committee_period(compute_epoch_at_slot(attested_header.slot))
+
+    # `next_sync_committee` is only useful if the message is signed by the current sync committee
+    if update_attested_period == update_signature_period:
+        next_sync_committee = attested_state.next_sync_committee
+        next_sync_committee_branch = compute_merkle_proof_for_state(attested_state, NEXT_SYNC_COMMITTEE_INDEX)
+    else:
+        next_sync_committee = SyncCommittee()
+        next_sync_committee_branch = [Bytes32() for _ in range(floorlog2(NEXT_SYNC_COMMITTEE_INDEX))]
+
+    # Indicate finality whenever possible
+    if finalized_block is not None:
+        if finalized_block.message.slot != GENESIS_SLOT:
+            finalized_header = BeaconBlockHeader(
+                slot=finalized_block.message.slot,
+                proposer_index=finalized_block.message.proposer_index,
+                parent_root=finalized_block.message.parent_root,
+                state_root=finalized_block.message.state_root,
+                body_root=hash_tree_root(finalized_block.message.body),
+            )
+            assert hash_tree_root(finalized_header) == attested_state.finalized_checkpoint.root
+        else:
+            assert attested_state.finalized_checkpoint.root == Bytes32()
+            finalized_header = BeaconBlockHeader()
+        finality_branch = compute_merkle_proof_for_state(attested_state, FINALIZED_ROOT_INDEX)
+    else:
+        finalized_header = BeaconBlockHeader()
+        finality_branch = [Bytes32() for _ in range(floorlog2(FINALIZED_ROOT_INDEX))]
+
+    return LightClientUpdate(
+        attested_header=attested_header,
+        next_sync_committee=next_sync_committee,
+        next_sync_committee_branch=next_sync_committee_branch,
+        finalized_header=finalized_header,
+        finality_branch=finality_branch,
+        sync_aggregate=block.message.body.sync_aggregate,
+        signature_slot=block.message.slot,
+    )
+```
+
+Full nodes SHOULD provide the best derivable `LightClientUpdate` (according to `is_better_update`) for each sync committee period covering any epochs in range `[max(ALTAIR_FORK_EPOCH, current_epoch - MIN_EPOCHS_FOR_BLOCK_REQUESTS), current_epoch]` where `current_epoch` is defined by the current wall-clock time. Full nodes MAY also provide `LightClientUpdate` for other sync committee periods.
+
+- `LightClientUpdate` are assigned to sync committee periods based on their `attested_header.slot`
+- `LightClientUpdate` are only considered if `compute_sync_committee_period(compute_epoch_at_slot(update.attested_header.slot)) == compute_sync_committee_period(compute_epoch_at_slot(update.signature_slot))`
+- Only `LightClientUpdate` with `next_sync_committee` as selected by fork choice are provided, regardless of ranking by `is_better_update`. To uniquely identify a non-finalized sync committee fork, all of `period`, `current_sync_committee` and `next_sync_committee` need to be incorporated, as sync committees may reappear over time.
diff --git a/specs/altair/sync-protocol.md → specs/altair/light-client/sync-protocol.md b/specs/altair/sync-protocol.md → specs/altair/light-client/sync-protocol.md
@@ -1,4 +1,4 @@
-# Altair -- Minimal Light Client
+# Altair Light Client -- Sync Protocol
 
 **Notice**: This document is a work-in-progress for researchers and implementers.
 
@@ -45,6 +45,9 @@ and metered VMs (e.g. blockchain VMs for cross-chain bridges).
 This document suggests a minimal light client design for the beacon chain that
 uses sync committees introduced in [this beacon chain extension](./beacon-chain.md).
 
+Additional documents describe how the light client sync protocol can be used:
+- [Full node](./full-node.md)
+
 ## Constants
 
 | Name | Value |
@@ -307,8 +310,8 @@ def validate_light_client_update(store: LightClientStore,
         assert update.finalized_header == BeaconBlockHeader()
     else:
         if update.finalized_header.slot == GENESIS_SLOT:
-            finalized_root = Bytes32()
             assert update.finalized_header == BeaconBlockHeader()
+            finalized_root = Bytes32()
         else:
             finalized_root = hash_tree_root(update.finalized_header)
         assert is_valid_merkle_branch(

diff --git a/specs/altair/validator.md b/specs/altair/validator.md
@@ -54,7 +54,7 @@ It builds on the [previous document for the behavior of an "honest validator" fr
 This previous document is referred to below as the "Phase 0 document".
 
 Altair introduces a new type of committee: the sync committee. Sync committees are responsible for signing each block of the canonical chain and there exists an efficient algorithm for light clients to sync the chain using the output of the sync committees.
-See the [sync protocol](./sync-protocol.md) for further details on the light client sync.
+See the [sync protocol](./light-client/sync-protocol.md) for further details on the light client sync.
 Under this network upgrade, validators track their participation in this new committee type and produce the relevant signatures as required.
 Block proposers incorporate the (aggregated) sync committee signatures into each block they produce.
 
@@ -265,7 +265,7 @@ This process occurs each slot.
 
 ##### Prepare sync committee message
 
-If a validator is in the current sync committee (i.e. `is_assigned_to_sync_committee()` above returns `True`), then for every `slot` in the current sync committee period, the validator should prepare a `SyncCommitteeMessage` for the previous slot (`slot - 1`) according to the logic in `get_sync_committee_message` as soon as they have determined the head block of `slot - 1`. This means that when assigned to `slot` a `SyncCommitteeMessage` is prepared and broadcast in `slot-1 ` instead of `slot`.   
+If a validator is in the current sync committee (i.e. `is_assigned_to_sync_committee()` above returns `True`), then for every `slot` in the current sync committee period, the validator should prepare a `SyncCommitteeMessage` for the previous slot (`slot - 1`) according to the logic in `get_sync_committee_message` as soon as they have determined the head block of `slot - 1`. This means that when assigned to `slot` a `SyncCommitteeMessage` is prepared and broadcast in `slot-1 ` instead of `slot`.
 
 This logic is triggered upon the same conditions as when producing an attestation.
 Meaning, a sync committee member should produce and broadcast a `SyncCommitteeMessage` either when (a) the validator has received a valid block from the expected block proposer for the current `slot` or (b) one-third of the slot has transpired (`SECONDS_PER_SLOT / INTERVALS_PER_SLOT` seconds after the start of the slot) -- whichever comes first.

diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py
@@ -1,3 +1,4 @@
+from eth_utils import encode_hex
 import os
 import time
 import shutil
@@ -97,6 +98,20 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
     yaml = YAML(pure=True)
     yaml.default_flow_style = None
 
+    # Spec config is using a YAML subset
+    cfg_yaml = YAML(pure=True)
+    cfg_yaml.default_flow_style = False  # Emit separate line for each key
+
+    def cfg_represent_bytes(self, data):
+        return self.represent_int(encode_hex(data))
+
+    cfg_yaml.representer.add_representer(bytes, cfg_represent_bytes)
+
+    def cfg_represent_quoted_str(self, data):
+        return self.represent_scalar(u'tag:yaml.org,2002:str', data, style="'")
+
+    cfg_yaml.representer.add_representer(context.quoted_str, cfg_represent_quoted_str)
+
     log_file = Path(output_dir) / 'testgen_error_log.txt'
 
     print(f"Generating tests into {output_dir}")
@@ -169,10 +184,14 @@ def output_part(out_kind: str, name: str, fn: Callable[[Path, ], None]):
                         written_part = True
                         if out_kind == "meta":
                             meta[name] = data
-                        if out_kind == "data":
-                            output_part("data", name, dump_yaml_fn(data, name, file_mode, yaml))
-                        if out_kind == "ssz":
-                            output_part("ssz", name, dump_ssz_fn(data, name, file_mode))
+                        elif out_kind == "cfg":
+                            output_part(out_kind, name, dump_yaml_fn(data, name, file_mode, cfg_yaml))
+                        elif out_kind == "data":
+                            output_part(out_kind, name, dump_yaml_fn(data, name, file_mode, yaml))
+                        elif out_kind == "ssz":
+                            output_part(out_kind, name, dump_ssz_fn(data, name, file_mode))
+                        else:
+                            assert False  # Unknown kind
                 except SkippedTest as e:
                     print(e)
                     skipped_test_count += 1

diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_typing.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_typing.py
@@ -10,9 +10,10 @@
 # Elements: name, out_kind, data
 #
 # out_kind is the type of data:
+#  - "meta" for generic data to collect into a meta data dict
+#  - "cfg" for a spec config dictionary
 #  - "data" for generic
 #  - "ssz" for SSZ encoded bytes
-#  - "meta" for generic data to collect into a meta data dict.
 TestCasePart = NewType("TestCasePart", Tuple[str, str, Any])
 
 

diff --git a/...pec/test/altair/sync_protocol/__init__.py → ...spec/test/altair/light_client/__init__.py b/...pec/test/altair/sync_protocol/__init__.py → ...spec/test/altair/light_client/__init__.py