EIP-6475: Add SSZ Optional[T] type (and drop selector byte in Union None case)

setup.py

@@ -378,7 +378,8 @@ def imports(cls, preset_name: str) -> str:
     field,
 )
 from typing import (
-    Any, Callable, Dict, Set, Sequence, Tuple, Optional, TypeVar, NamedTuple, Final
+    Any, Callable, Dict, Set, Sequence, Tuple, TypeVar, NamedTuple, Final,
+    Optional as PyOptional
 )
 
 from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes
@@ -564,7 +565,7 @@ def sundry_functions(cls) -> str:
 ExecutionState = Any
 
 
-def get_pow_block(hash: Bytes32) -> Optional[PowBlock]:
+def get_pow_block(hash: Bytes32) -> PyOptional[PowBlock]:
     return PowBlock(block_hash=hash, parent_hash=Bytes32(), total_difficulty=uint256(0))
 
 
@@ -585,7 +586,7 @@ def notify_forkchoice_updated(self: ExecutionEngine,
                                   head_block_hash: Hash32,
                                   safe_block_hash: Hash32,
                                   finalized_block_hash: Hash32,
-                                  payload_attributes: Optional[PayloadAttributes]) -> Optional[PayloadId]:
+                                  payload_attributes: PyOptional[PayloadAttributes]) -> PyOptional[PayloadId]:
         pass
 
     def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> ExecutionPayload:
@@ -815,7 +816,7 @@ def combine_dicts(old_dict: Dict[str, T], new_dict: Dict[str, T]) -> Dict[str, T
     'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256',
     'bytes', 'byte', 'ByteList', 'ByteVector',
     'Dict', 'dict', 'field', 'ceillog2', 'floorlog2', 'Set',
-    'Optional', 'Sequence',
+    'PyOptional', 'Sequence',
 ]
 
 

specs/_features/das/das-core.md

@@ -105,7 +105,7 @@ Implementations:
 - [Old approach in Go](https://github.com/protolambda/go-kate/blob/master/recovery.go)
 
 ```python
-def recover_data(data: Sequence[Optional[Sequence[Point]]]) -> Sequence[Point]:
+def recover_data(data: Sequence[PyOptional[Sequence[Point]]]) -> Sequence[Point]:
     """Given an a subset of half or more of subgroup-aligned ranges of values, recover the None values."""
     ...
 ```
@@ -183,7 +183,7 @@ def verify_sample(sample: DASSample, sample_count: uint64, commitment: BLSCommit
 ```
 
 ```python
-def reconstruct_extended_data(samples: Sequence[Optional[DASSample]]) -> Sequence[Point]:
+def reconstruct_extended_data(samples: Sequence[PyOptional[DASSample]]) -> Sequence[Point]:
     # Instead of recovering with a point-by-point approach, recover the samples by recovering missing subgroups.
     subgroups = [None if sample is None else reverse_bit_order_list(sample.data) for sample in samples]
     return recover_data(subgroups)

specs/altair/light-client/full-node.md

@@ -97,7 +97,7 @@ def create_light_client_update(state: BeaconState,
                                block: SignedBeaconBlock,
                                attested_state: BeaconState,
                                attested_block: SignedBeaconBlock,
-                               finalized_block: Optional[SignedBeaconBlock]) -> LightClientUpdate:
+                               finalized_block: PyOptional[SignedBeaconBlock]) -> LightClientUpdate:
     assert compute_epoch_at_slot(attested_state.slot) >= ALTAIR_FORK_EPOCH
     assert sum(block.message.body.sync_aggregate.sync_committee_bits) >= MIN_SYNC_COMMITTEE_PARTICIPANTS
 

specs/altair/light-client/sync-protocol.md

@@ -152,7 +152,7 @@ class LightClientStore(object):
     current_sync_committee: SyncCommittee
     next_sync_committee: SyncCommittee
     # Best available header to switch finalized head to if we see nothing else
-    best_valid_update: Optional[LightClientUpdate]
+    best_valid_update: PyOptional[LightClientUpdate]
     # Most recent available reasonably-safe header
     optimistic_header: LightClientHeader
     # Max number of active participants in a sync committee (used to calculate safety threshold)

specs/bellatrix/fork-choice.md

@@ -60,7 +60,7 @@ def notify_forkchoice_updated(self: ExecutionEngine,
                               head_block_hash: Hash32,
                               safe_block_hash: Hash32,
                               finalized_block_hash: Hash32,
-                              payload_attributes: Optional[PayloadAttributes]) -> Optional[PayloadId]:
+                              payload_attributes: PyOptional[PayloadAttributes]) -> PyOptional[PayloadId]:
     ...
 ```
 
@@ -101,7 +101,7 @@ class PowBlock(Container):
 
 ### `get_pow_block`
 
-Let `get_pow_block(block_hash: Hash32) -> Optional[PowBlock]` be the function that given the hash of the PoW block returns its data.
+Let `get_pow_block(block_hash: Hash32) -> PyOptional[PowBlock]` be the function that given the hash of the PoW block returns its data.
 It may result in `None` if the requested block is not yet available.
 
 *Note*: The `eth_getBlockByHash` JSON-RPC method may be used to pull this information from an execution client.

specs/bellatrix/validator.md

@@ -39,7 +39,7 @@ Please see related Beacon Chain doc before continuing and use them as a referenc
 ### `get_pow_block_at_terminal_total_difficulty`
 
 ```python
-def get_pow_block_at_terminal_total_difficulty(pow_chain: Dict[Hash32, PowBlock]) -> Optional[PowBlock]:
+def get_pow_block_at_terminal_total_difficulty(pow_chain: Dict[Hash32, PowBlock]) -> PyOptional[PowBlock]:
     # `pow_chain` abstractly represents all blocks in the PoW chain
     for block in pow_chain.values():
         block_reached_ttd = block.total_difficulty >= TERMINAL_TOTAL_DIFFICULTY
@@ -58,7 +58,7 @@ def get_pow_block_at_terminal_total_difficulty(pow_chain: Dict[Hash32, PowBlock]
 ### `get_terminal_pow_block`
 
 ```python
-def get_terminal_pow_block(pow_chain: Dict[Hash32, PowBlock]) -> Optional[PowBlock]:
+def get_terminal_pow_block(pow_chain: Dict[Hash32, PowBlock]) -> PyOptional[PowBlock]:
     if TERMINAL_BLOCK_HASH != Hash32():
         # Terminal block hash override takes precedence over terminal total difficulty
         if TERMINAL_BLOCK_HASH in pow_chain:
@@ -122,7 +122,7 @@ def prepare_execution_payload(state: BeaconState,
                               safe_block_hash: Hash32,
                               finalized_block_hash: Hash32,
                               suggested_fee_recipient: ExecutionAddress,
-                              execution_engine: ExecutionEngine) -> Optional[PayloadId]:
+                              execution_engine: ExecutionEngine) -> PyOptional[PayloadId]:
     if not is_merge_transition_complete(state):
         is_terminal_block_hash_set = TERMINAL_BLOCK_HASH != Hash32()
         is_activation_epoch_reached = get_current_epoch(state) >= TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH
@@ -157,7 +157,7 @@ def prepare_execution_payload(state: BeaconState,
 2. Set `block.body.execution_payload = get_execution_payload(payload_id, execution_engine)`, where:
 
 ```python
-def get_execution_payload(payload_id: Optional[PayloadId], execution_engine: ExecutionEngine) -> ExecutionPayload:
+def get_execution_payload(payload_id: PyOptional[PayloadId], execution_engine: ExecutionEngine) -> ExecutionPayload:
     if payload_id is None:
         # Pre-merge, empty payload
         return ExecutionPayload()

specs/capella/fork-choice.md

@@ -42,7 +42,7 @@ def notify_forkchoice_updated(self: ExecutionEngine,
                               head_block_hash: Hash32,
                               safe_block_hash: Hash32,
                               finalized_block_hash: Hash32,
-                              payload_attributes: Optional[PayloadAttributes]) -> Optional[PayloadId]:
+                              payload_attributes: PyOptional[PayloadAttributes]) -> PyOptional[PayloadId]:
     ...
 ```
 

specs/capella/validator.md

@@ -73,7 +73,7 @@ def prepare_execution_payload(state: BeaconState,
                               safe_block_hash: Hash32,
                               finalized_block_hash: Hash32,
                               suggested_fee_recipient: ExecutionAddress,
-                              execution_engine: ExecutionEngine) -> Optional[PayloadId]:
+                              execution_engine: ExecutionEngine) -> PyOptional[PayloadId]:
     if not is_merge_transition_complete(state):
         is_terminal_block_hash_set = TERMINAL_BLOCK_HASH != Hash32()
         is_activation_epoch_reached = get_current_epoch(state) >= TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH

specs/phase0/validator.md

@@ -215,7 +215,7 @@ A validator can get committee assignments for a given epoch using the following
 def get_committee_assignment(state: BeaconState,
                              epoch: Epoch,
                              validator_index: ValidatorIndex
-                             ) -> Optional[Tuple[Sequence[ValidatorIndex], CommitteeIndex, Slot]]:
+                             ) -> PyOptional[Tuple[Sequence[ValidatorIndex], CommitteeIndex, Slot]]:
     """
     Return the committee assignment in the ``epoch`` for ``validator_index``.
     ``assignment`` returned is a tuple of the following form:

ssz/simple-serialize.md

@@ -20,6 +20,7 @@
   - [`Bitvector[N]`](#bitvectorn)
   - [`Bitlist[N]`](#bitlistn)
   - [Vectors, containers, lists](#vectors-containers-lists)
+  - [Optional](#optional)
   - [Union](#union)
 - [Deserialization](#deserialization)
 - [Merkleization](#merkleization)
@@ -60,6 +61,8 @@
     * notation `Bitvector[N]`
 * **bitlist**: ordered variable-length collection of `boolean` values, limited to `N` bits
     * notation `Bitlist[N]`
+* **optional**: either a wrapped value of the given subtype, or `None`
+    * notation `Optional[type]`, e.g. `Optional[uint64]`
 * **union**: union type containing one of the given subtypes
     * notation `Union[type_0, type_1, ...]`, e.g. `union[None, uint64, uint32]`
 
@@ -90,6 +93,7 @@ Assuming a helper function `default(type)` which returns the default value for `
 | `Bitvector[N]` | `[False] * N` |
 | `List[type, N]` | `[]` |
 | `Bitlist[N]` | `[]` |
+| `Optional[type]` | `None` |
 | `Union[type_0, type_1, ...]` | `default(type_0)` |
 
 #### `is_zero`
@@ -163,6 +167,15 @@ fixed_parts = [part if part != None else variable_offsets[i] for i, part in enum
 return b"".join(fixed_parts + variable_parts)
 ```
 
+### Optional
+
+```python
+if value is None:
+    return b""
+else:
+    return b"\x01" + serialize(value)
+```
+
 ### Union
 
 A `value` as `Union[T...]` type has properties `value.value` with the contained value, and `value.selector` which indexes the selected `Union` type option `T`.
@@ -178,7 +191,7 @@ A `Union`:
 ```python
 if value.value is None:
     assert value.selector == 0
-    return b"\x00"
+    return b""
 else:
     serialized_bytes = serialize(value.value)
     serialized_selector_index = value.selector.to_bytes(1, "little")
@@ -196,14 +209,15 @@ Deserialization can be implemented using a recursive algorithm. The deserializat
   * The size of each object in the vector/list can be inferred from the difference of two offsets. To get the size of the last object, the total number of bytes has to be known (it is not generally possible to deserialize an SSZ object of unknown length)
 * Containers follow the same principles as vectors, with the difference that there may be fixed-size objects in a container as well. This means the `fixed_parts` data will contain offsets as well as fixed-size objects.
 * In the case of bitlists, the length in bits cannot be uniquely inferred from the number of bytes in the object. Because of this, they have a bit at the end that is always set. This bit has to be used to infer the size of the bitlist in bits.
-* In the case of unions, the first byte of the deserialization scope is deserialized as type selector, the remainder of the scope is deserialized as the selected type.
+* In the case of optional, if the serialized data has length 0, it represents `None`. Otherwise, the first byte of the deserialization scope must be checked to be `0x01`, the remainder of the scope is deserialized same as `T`.
+* In the case of unions, if the serialized data has length 0, it represents `None`. Otherwise, the first byte of the deserialization scope is deserialized as type selector, the remainder of the scope is deserialized as the selected type (cannot refer to `None`).
 
 Note that deserialization requires hardening against invalid inputs. A non-exhaustive list:
 
 - Offsets: out of order, out of range, mismatching minimum element size.
 - Scope: Extra unused bytes, not aligned with element size.
 - More elements than a list limit allows. Part of enforcing consensus.
-- An out-of-bounds selected index in an `Union`
+- An out-of-bounds selected index in an `Union`, or a `None` value for a type that doesn't support it.
 
 Efficient algorithms for computing this object can be found in [the implementations](#implementations).
 
@@ -244,6 +258,8 @@ We now define Merkleization `hash_tree_root(value)` of an object `value` recursi
 * `mix_in_length(merkleize(pack_bits(value), limit=chunk_count(type)), len(value))` if `value` is a bitlist.
 * `merkleize([hash_tree_root(element) for element in value])` if `value` is a vector of composite objects or a container.
 * `mix_in_length(merkleize([hash_tree_root(element) for element in value], limit=chunk_count(type)), len(value))` if `value` is a list of composite objects.
+* `mix_in_length(hash_tree_root(value), 1)` if `value` is of optional type, and `value` is not `None`
+* `mix_in_length(Bytes32(), 0)` if `value` is of optional type, and `value` is `None`
 * `mix_in_selector(hash_tree_root(value.value), value.selector)` if `value` is of union type, and `value.value` is not `None`
 * `mix_in_selector(Bytes32(), 0)` if `value` is of union type, and `value.value` is `None`