Skip to content

Commit

Permalink
Sidecar inclusion proof (#4900)
Browse files Browse the repository at this point in the history 
* Refactor BlobSidecar to new type

* Fix some compile errors

* Gossip verification compiles

* Fix http api types take 1

* Fix another round of compile errors

* Beacon node crate compiles

* EF tests compile

* Remove all blob signing from VC

* fmt

* Tests compile

* Fix some tests

* Fix more http tests

* get compiling

* Fix gossip conditions and tests

* Add basic proof generation and verification

* remove unnecessary ssz decode

* add back build_sidecar

* remove default at fork for blobs

* fix beacon chain tests

* get relase tests compiling

* fix lints

* fix existing spec tests

* add new ef tests

* fix gossip duplicate rule

* lints

* add back sidecar signature check in gossip

* add finalized descendant check to blob sidecar gossip

* fix error conversion

* fix release tests

* sidecar inclusion self review cleanup

* Add proof verification and computation metrics

* Remove accidentally committed file

* Unify some block and blob errors; add slashing conditions for sidecars

* Address review comment

* Clean up re-org tests (#4957)

* Address more review comments

* Add Comments & Eliminate Unnecessary Clones

* update names

* Update beacon_node/beacon_chain/src/metrics.rs

Co-authored-by: Jimmy Chen <jchen.tc@gmail.com>

* Update beacon_node/network/src/network_beacon_processor/tests.rs

Co-authored-by: Jimmy Chen <jchen.tc@gmail.com>

* pr feedback

* fix test compile

* Sidecar Inclusion proof small refactor and updates (#4967)

* Update some comments, variables and small cosmetic fixes.

* Couple blobs and proofs into a tuple in `PayloadAndBlobs` for simplicity and safety.

* Update function comment.

* Update testing/ef_tests/src/cases/merkle_proof_validity.rs

Co-authored-by: Jimmy Chen <jchen.tc@gmail.com>

* Rename the block and blob wrapper types used in the beacon API interfaces.

* make sure gossip invalid blobs are passed to the slasher (#4970)

* Add blob headers to slasher before adding to DA checker

* Replace Vec with HashSet in BlockQueue

* fmt

* Rename gindex -> index

* Simplify gossip condition

---------

Co-authored-by: realbigsean <seananderson33@gmail.com>
Co-authored-by: realbigsean <sean@sigmaprime.io>
Co-authored-by: Michael Sproul <michael@sigmaprime.io>
Co-authored-by: Mark Mackey <mark@sigmaprime.io>
Co-authored-by: Jimmy Chen <jchen.tc@gmail.com>
  • Loading branch information
@pawanjay176 @realbigsean
@michaelsproul @ethDreamer @jimmygchen
6 people committed Dec 5, 2023
1 parent ec8edfb commit 3104440
Show file tree
Hide file tree
Showing 74 changed files with 1,953 additions and 2,273 deletions.
1 change: 1 addition & 0 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

149 changes: 93 additions & 56 deletions beacon_node/beacon_chain/src/beacon_chain.rs
View file
Expand Up @@ -7,7 +7,7 @@ use crate::attester_cache::{AttesterCache, AttesterCacheKey};
use crate::beacon_block_streamer::{BeaconBlockStreamer, CheckEarlyAttesterCache};
use crate::beacon_proposer_cache::compute_proposer_duties_from_head;
use crate::beacon_proposer_cache::BeaconProposerCache;
use crate::blob_verification::{self, GossipBlobError, GossipVerifiedBlob};
use crate::blob_verification::{GossipBlobError, GossipVerifiedBlob};
use crate::block_times_cache::BlockTimesCache;
use crate::block_verification::POS_PANDA_BANNER;
use crate::block_verification::{
Expand Down Expand Up @@ -121,7 +121,6 @@ use tree_hash::TreeHash;
use types::beacon_state::CloneConfig;
use types::blob_sidecar::{BlobSidecarList, FixedBlobSidecarList};
use types::payload::BlockProductionVersion;
use types::sidecar::BlobItems;
use types::*;

pub type ForkChoiceError = fork_choice::Error<crate::ForkChoiceStoreError>;
Expand Down Expand Up @@ -489,16 +488,49 @@ pub struct BeaconChain<T: BeaconChainTypes> {
pub block_production_state: Arc<Mutex<Option<(Hash256, BlockProductionPreState<T::EthSpec>)>>>,
}

pub enum BeaconBlockResponseType<T: EthSpec> {
pub enum BeaconBlockResponseWrapper<T: EthSpec> {
Full(BeaconBlockResponse<T, FullPayload<T>>),
Blinded(BeaconBlockResponse<T, BlindedPayload<T>>),
}

impl<E: EthSpec> BeaconBlockResponseWrapper<E> {
pub fn fork_name(&self, spec: &ChainSpec) -> Result<ForkName, InconsistentFork> {
Ok(match self {
BeaconBlockResponseWrapper::Full(resp) => resp.block.to_ref().fork_name(spec)?,
BeaconBlockResponseWrapper::Blinded(resp) => resp.block.to_ref().fork_name(spec)?,
})
}

pub fn execution_payload_value(&self) -> Option<Uint256> {
match self {
BeaconBlockResponseWrapper::Full(resp) => resp.execution_payload_value,
BeaconBlockResponseWrapper::Blinded(resp) => resp.execution_payload_value,
}
}

pub fn consensus_block_value(&self) -> Option<u64> {
match self {
BeaconBlockResponseWrapper::Full(resp) => resp.consensus_block_value,
BeaconBlockResponseWrapper::Blinded(resp) => resp.consensus_block_value,
}
}

pub fn is_blinded(&self) -> bool {
matches!(self, BeaconBlockResponseWrapper::Blinded(_))
}
}

/// The components produced when the local beacon node creates a new block to extend the chain
pub struct BeaconBlockResponse<T: EthSpec, Payload: AbstractExecPayload<T>> {
/// The newly produced beacon block
pub block: BeaconBlock<T, Payload>,
/// The post-state after applying the new block
pub state: BeaconState<T>,
pub maybe_side_car: Option<SidecarList<T, <Payload as AbstractExecPayload<T>>::Sidecar>>,
/// The Blobs / Proofs associated with the new block
pub blob_items: Option<(KzgProofs<T>, BlobsList<T>)>,
/// The execution layer reward for the block
pub execution_payload_value: Option<Uint256>,
/// The consensus layer reward to the proposer
pub consensus_block_value: Option<u64>,
}

Expand Down Expand Up @@ -2022,17 +2054,15 @@ impl<T: BeaconChainTypes> BeaconChain<T> {

pub fn verify_blob_sidecar_for_gossip(
self: &Arc<Self>,
blob_sidecar: SignedBlobSidecar<T::EthSpec>,
blob_sidecar: Arc<BlobSidecar<T::EthSpec>>,
subnet_id: u64,
) -> Result<GossipVerifiedBlob<T>, GossipBlobError<T::EthSpec>> {
metrics::inc_counter(&metrics::BLOBS_SIDECAR_PROCESSING_REQUESTS);
let _timer = metrics::start_timer(&metrics::BLOBS_SIDECAR_GOSSIP_VERIFICATION_TIMES);
blob_verification::validate_blob_sidecar_for_gossip(blob_sidecar, subnet_id, self).map(
|v| {
metrics::inc_counter(&metrics::BLOBS_SIDECAR_PROCESSING_SUCCESSES);
v
},
)
GossipVerifiedBlob::new(blob_sidecar, subnet_id, self).map(|v| {
metrics::inc_counter(&metrics::BLOBS_SIDECAR_PROCESSING_SUCCESSES);
v
})
}

/// Accepts some 'LightClientOptimisticUpdate' from the network and attempts to verify it
Expand Down Expand Up @@ -2832,7 +2862,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
}

self.data_availability_checker
.notify_gossip_blob(blob.as_blob().slot, block_root, &blob);
.notify_gossip_blob(blob.slot(), block_root, &blob);
let r = self.check_gossip_blob_availability_and_import(blob).await;
self.remove_notified(&block_root, r)
}
Expand Down Expand Up @@ -2942,6 +2972,20 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
// Increment the Prometheus counter for block processing requests.
metrics::inc_counter(&metrics::BLOCK_PROCESSING_REQUESTS);

// Set observed time if not already set. Usually this should be set by gossip or RPC,
// but just in case we set it again here (useful for tests).
if let (Some(seen_timestamp), Some(current_slot)) =
(self.slot_clock.now_duration(), self.slot_clock.now())
{
self.block_times_cache.write().set_time_observed(
block_root,
current_slot,
seen_timestamp,
None,
None,
);
}

let block_slot = unverified_block.block().slot();

// A small closure to group the verification and import errors.
Expand Down Expand Up @@ -3097,6 +3141,9 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
blob: GossipVerifiedBlob<T>,
) -> Result<AvailabilityProcessingStatus, BlockError<T::EthSpec>> {
let slot = blob.slot();
if let Some(slasher) = self.slasher.as_ref() {
slasher.accept_block_header(blob.signed_block_header());
}
let availability = self.data_availability_checker.put_gossip_blob(blob)?;

self.process_availability(slot, availability).await
Expand All @@ -3110,6 +3157,11 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
block_root: Hash256,
blobs: FixedBlobSidecarList<T::EthSpec>,
) -> Result<AvailabilityProcessingStatus, BlockError<T::EthSpec>> {
if let Some(slasher) = self.slasher.as_ref() {
for blob_sidecar in blobs.iter().filter_map(|blob| blob.clone()) {
slasher.accept_block_header(blob_sidecar.signed_block_header.clone());
}
}
let availability = self
.data_availability_checker
.put_rpc_blobs(block_root, blobs)?;
Expand Down Expand Up @@ -3968,7 +4020,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
validator_graffiti: Option<Graffiti>,
verification: ProduceBlockVerification,
block_production_version: BlockProductionVersion,
) -> Result<BeaconBlockResponseType<T::EthSpec>, BlockProductionError> {
) -> Result<BeaconBlockResponseWrapper<T::EthSpec>, BlockProductionError> {
metrics::inc_counter(&metrics::BLOCK_PRODUCTION_REQUESTS);
let _complete_timer = metrics::start_timer(&metrics::BLOCK_PRODUCTION_TIMES);
// Part 1/2 (blocking)
Expand Down Expand Up @@ -4414,7 +4466,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
/// This function uses heuristics that align quite closely but not exactly with the re-org
/// conditions set out in `get_state_for_re_org` and `get_proposer_head`. The differences are
/// documented below.
fn overridden_forkchoice_update_params(
pub fn overridden_forkchoice_update_params(
&self,
canonical_forkchoice_params: ForkchoiceUpdateParameters,
) -> Result<ForkchoiceUpdateParameters, Error> {
Expand All @@ -4432,7 +4484,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
})
}

fn overridden_forkchoice_update_params_or_failure_reason(
pub fn overridden_forkchoice_update_params_or_failure_reason(
&self,
canonical_forkchoice_params: &ForkchoiceUpdateParameters,
) -> Result<ForkchoiceUpdateParameters, ProposerHeadError<Error>> {
Expand Down Expand Up @@ -4573,7 +4625,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
.unwrap_or_else(|| Duration::from_secs(0)),
);
block_delays.observed.map_or(false, |delay| {
delay > self.slot_clock.unagg_attestation_production_delay()
delay >= self.slot_clock.unagg_attestation_production_delay()
})
}

Expand All @@ -4599,7 +4651,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
validator_graffiti: Option<Graffiti>,
verification: ProduceBlockVerification,
block_production_version: BlockProductionVersion,
) -> Result<BeaconBlockResponseType<T::EthSpec>, BlockProductionError> {
) -> Result<BeaconBlockResponseWrapper<T::EthSpec>, BlockProductionError> {
// Part 1/3 (blocking)
//
// Perform the state advance and block-packing functions.
Expand Down Expand Up @@ -4658,7 +4710,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
.await
.map_err(BlockProductionError::TokioJoin)??;

Ok(BeaconBlockResponseType::Full(beacon_block_response))
Ok(BeaconBlockResponseWrapper::Full(beacon_block_response))
}
BlockProposalContentsType::Blinded(block_contents) => {
let chain = self.clone();
Expand All @@ -4678,7 +4730,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
.await
.map_err(BlockProductionError::TokioJoin)??;

Ok(BeaconBlockResponseType::Blinded(beacon_block_response))
Ok(BeaconBlockResponseWrapper::Blinded(beacon_block_response))
}
}
} else {
Expand All @@ -4699,7 +4751,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
.await
.map_err(BlockProductionError::TokioJoin)??;

Ok(BeaconBlockResponseType::Full(beacon_block_response))
Ok(BeaconBlockResponseWrapper::Full(beacon_block_response))
}
}

Expand Down Expand Up @@ -4977,7 +5029,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
bls_to_execution_changes,
} = partial_beacon_block;

let (inner_block, blobs_opt, proofs_opt, execution_payload_value) = match &state {
let (inner_block, maybe_blobs_and_proofs, execution_payload_value) = match &state {
BeaconState::Base(_) => (
BeaconBlock::Base(BeaconBlockBase {
slot,
Expand All @@ -4997,7 +5049,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
},
}),
None,
None,
Uint256::zero(),
),
BeaconState::Altair(_) => (
Expand All @@ -5021,7 +5072,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
},
}),
None,
None,
Uint256::zero(),
),
BeaconState::Merge(_) => {
Expand Down Expand Up @@ -5052,7 +5102,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
},
}),
None,
None,
execution_payload_value,
)
}
Expand Down Expand Up @@ -5086,12 +5135,11 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
},
}),
None,
None,
execution_payload_value,
)
}
BeaconState::Deneb(_) => {
let (payload, kzg_commitments, blobs, proofs, execution_payload_value) =
let (payload, kzg_commitments, maybe_blobs_and_proofs, execution_payload_value) =
block_contents
.ok_or(BlockProductionError::MissingExecutionPayload)?
.deconstruct();
Expand Down Expand Up @@ -5121,8 +5169,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
.ok_or(BlockProductionError::InvalidPayloadFork)?,
},
}),
blobs,
proofs,
maybe_blobs_and_proofs,
execution_payload_value,
)
}
Expand Down Expand Up @@ -5181,51 +5228,41 @@ impl<T: BeaconChainTypes> BeaconChain<T> {

let blobs_verification_timer =
metrics::start_timer(&metrics::BLOCK_PRODUCTION_BLOBS_VERIFICATION_TIMES);
let maybe_sidecar_list = match (blobs_opt, proofs_opt) {
(Some(blobs_or_blobs_roots), Some(proofs)) => {
let blob_items = match maybe_blobs_and_proofs {
Some((blobs, proofs)) => {
let expected_kzg_commitments =
block.body().blob_kzg_commitments().map_err(|_| {
BlockProductionError::InvalidBlockVariant(
"deneb block does not contain kzg commitments".to_string(),
)
})?;

if expected_kzg_commitments.len() != blobs_or_blobs_roots.len() {
if expected_kzg_commitments.len() != blobs.len() {
return Err(BlockProductionError::MissingKzgCommitment(format!(
"Missing KZG commitment for slot {}. Expected {}, got: {}",
block.slot(),
blobs_or_blobs_roots.len(),
blobs.len(),
expected_kzg_commitments.len()
)));
}

let kzg_proofs = Vec::from(proofs);

if let Some(blobs) = blobs_or_blobs_roots.blobs() {
let kzg = self
.kzg
.as_ref()
.ok_or(BlockProductionError::TrustedSetupNotInitialized)?;
kzg_utils::validate_blobs::<T::EthSpec>(
kzg,
expected_kzg_commitments,
blobs.iter().collect(),
&kzg_proofs,
)
.map_err(BlockProductionError::KzgError)?;
}

Some(
Sidecar::build_sidecar(
blobs_or_blobs_roots,
&block,
expected_kzg_commitments,
kzg_proofs,
)
.map_err(BlockProductionError::FailedToBuildBlobSidecars)?,
let kzg = self
.kzg
.as_ref()
.ok_or(BlockProductionError::TrustedSetupNotInitialized)?;
kzg_utils::validate_blobs::<T::EthSpec>(
kzg,
expected_kzg_commitments,
blobs.iter().collect(),
&kzg_proofs,
)
.map_err(BlockProductionError::KzgError)?;

Some((kzg_proofs.into(), blobs))
}
_ => None,
None => None,
};

drop(blobs_verification_timer);
Expand All @@ -5243,7 +5280,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
Ok(BeaconBlockResponse {
block,
state,
maybe_side_car: maybe_sidecar_list,
blob_items,
execution_payload_value: Some(execution_payload_value),
consensus_block_value: Some(consensus_block_value),
})
Expand Down

0 comments on commit 3104440

Please sign in to comment.