From ffb6336d2000ac166ce42009f0baba17221189ae Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 15 Nov 2022 21:27:30 +0100 Subject: [PATCH] WIP --- packages/api/src/utils/types.ts | 15 --- packages/beacon-node/package.json | 1 + .../src/api/impl/beacon/blocks/index.ts | 39 ++++---- packages/beacon-node/src/chain/chain.ts | 15 +-- .../src/chain/errors/blobsSidecarError.ts | 23 +++++ packages/beacon-node/src/chain/interface.ts | 2 + .../chain/produceBlock/produceBlockBody.ts | 27 ++++-- .../src/chain/validation/blobsSidecar.ts | 68 ++++++++++++++ .../beacon-node/src/execution/engine/http.ts | 2 +- .../src/execution/engine/interface.ts | 6 +- .../src/network/gossip/handlers/index.ts | 48 +++++----- .../src/network/gossip/validation/queue.ts | 2 +- packages/beacon-node/src/network/interface.ts | 4 + packages/beacon-node/src/network/network.ts | 92 ++++++++++++------- packages/beacon-node/src/util/promises.ts | 14 +++ packages/state-transition/src/block/index.ts | 13 ++- .../src/block/processBlobKzgCommitments.ts | 8 ++ .../state-transition/src/cache/stateCache.ts | 9 +- packages/state-transition/src/cache/types.ts | 6 +- packages/state-transition/src/index.ts | 2 +- .../src/slot/upgradeStateTo4844.ts | 4 +- packages/state-transition/src/types.ts | 2 +- packages/state-transition/src/util/blobs.ts | 64 +++++++++++++ packages/state-transition/src/util/index.ts | 2 + 24 files changed, 349 insertions(+), 119 deletions(-) create mode 100644 packages/beacon-node/src/chain/errors/blobsSidecarError.ts create mode 100644 packages/beacon-node/src/chain/validation/blobsSidecar.ts create mode 100644 packages/beacon-node/src/util/promises.ts create mode 100644 packages/state-transition/src/block/processBlobKzgCommitments.ts create mode 100644 packages/state-transition/src/util/blobs.ts diff --git a/packages/api/src/utils/types.ts b/packages/api/src/utils/types.ts index f1f6779b1b4d..e969a3943e09 100644 --- a/packages/api/src/utils/types.ts +++ b/packages/api/src/utils/types.ts @@ -2,8 +2,6 @@ import {isBasicType, ListBasicType, Type, isCompositeType, ListCompositeType, Ar import {ForkName} from "@lodestar/params"; import {IChainForkConfig} from "@lodestar/config"; import {objectToExpectedCase} from "@lodestar/utils"; -import {ssz} from "@lodestar/types"; -import {Blobs} from "@lodestar/types/eip4844"; import {Schema, SchemaDefinition} from "./schema.js"; // See /packages/api/src/routes/index.ts for reasoning @@ -196,19 +194,6 @@ export function WithExecutionOptimistic( }; } -export function WithBlobs(type: TypeJson): TypeJson { - return { - toJson: ({blobs, ...data}) => ({ - ...(type.toJson((data as unknown) as T) as Record), - blobs: ssz.eip4844.Blobs.toJson(blobs), - }), - fromJson: ({blobs, ...data}: T & {blobs: Blobs}) => ({ - ...type.fromJson(data), - blobs: ssz.eip4844.Blobs.fromJson(blobs), - }), - }; -} - type JsonCase = "snake" | "constant" | "camel" | "param" | "header" | "pascal" | "dot" | "notransform"; /** Helper to only translate casing */ diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index d7c1c8425de8..e3b7a408ce58 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -130,6 +130,7 @@ "@multiformats/multiaddr": "^11.0.0", "@types/datastore-level": "^3.0.0", "buffer-xor": "^2.0.2", + "c-kzg": "^1.0.0", "cross-fetch": "^3.1.4", "datastore-core": "^8.0.1", "datastore-level": "^9.0.1", diff --git a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts index a1733d5be0cd..f959b5befcdd 100644 --- a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts @@ -4,6 +4,7 @@ import {computeTimeAtSlot} from "@lodestar/state-transition"; import {SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params"; import {sleep} from "@lodestar/utils"; import {fromHexString, toHexString} from "@chainsafe/ssz"; +import {promiseAllMaybeAsync} from "../../../../util/promises.js"; import {BlockError, BlockErrorCode} from "../../../../chain/errors/index.js"; import {OpSource} from "../../../../metrics/validatorMonitor.js"; import {NetworkEvent} from "../../../../network/index.js"; @@ -23,16 +24,6 @@ export function getBeaconBlockApi({ network, db, }: Pick): routes.beacon.block.Api { - const waitForSlot = async (slot: number): Promise => { - // Simple implementation of a pending block queue. Keeping the block here recycles the API logic, and keeps the - // REST request promise without any extra infrastructure. - const msToBlockSlot = computeTimeAtSlot(config, slot, chain.genesisTime) * 1000 - Date.now(); - if (msToBlockSlot <= MAX_API_CLOCK_DISPARITY_MS && msToBlockSlot > 0) { - // If block is a bit early, hold it in a promise. Equivalent to a pending queue. - await sleep(msToBlockSlot); - } - }; - return { async getBlockHeaders(filters) { // TODO - SLOW CODE: This code seems like it could be improved @@ -183,23 +174,31 @@ export function getBeaconBlockApi({ async publishBlock(signedBlock) { const seenTimestampSec = Date.now() / 1000; - await waitForSlot(signedBlock.message.slot); + + // Simple implementation of a pending block queue. Keeping the block here recycles the API logic, and keeps the + // REST request promise without any extra infrastructure. + const msToBlockSlot = computeTimeAtSlot(config, signedBlock.message.slot, chain.genesisTime) * 1000 - Date.now(); + if (msToBlockSlot <= MAX_API_CLOCK_DISPARITY_MS && msToBlockSlot > 0) { + // If block is a bit early, hold it in a promise. Equivalent to a pending queue. + await sleep(msToBlockSlot); + } // TODO: Validate block metrics?.registerBeaconBlock(OpSource.api, seenTimestampSec, signedBlock.message); - await Promise.all([ + await promiseAllMaybeAsync([ // Send the block, regardless of whether or not it is valid. The API // specification is very clear that this is the desired behaviour. - network.gossip.publishBeaconBlock(signedBlock), - - chain.processBlock(signedBlock).catch((e) => { - if (e instanceof BlockError && e.type.code === BlockErrorCode.PARENT_UNKNOWN) { - network.events.emit(NetworkEvent.unknownBlockParent, signedBlock, network.peerId.toString()); - } - throw e; - }), + () => network.publishBeaconBlockMaybeBlobs(signedBlock), + + () => + chain.processBlock(signedBlock).catch((e) => { + if (e instanceof BlockError && e.type.code === BlockErrorCode.PARENT_UNKNOWN) { + network.events.emit(NetworkEvent.unknownBlockParent, signedBlock, network.peerId.toString()); + } + throw e; + }), ]); }, }; diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 4cfda7dff008..b202cd92c39a 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -119,6 +119,7 @@ export class BeaconChain implements IBeaconChain { private successfulExchangeTransition = false; private readonly exchangeTransitionConfigurationEverySlots: number; + /** Map keyed by executionPayload.blockHash of the block for those blobs */ private readonly producedBlobsCache = new Map(); private readonly faultInspectionWindow: number; @@ -383,7 +384,7 @@ export class BeaconChain implements IBeaconChain { // Cache for latter broadcasting if (blobs.type === BlobsResultType.produced) { - this.producedBlobsCache.set(blobs.blockRoot, blobs.blobs); + this.producedBlobsCache.set(blobs.blockHash, blobs.blobs); } return block; @@ -399,15 +400,17 @@ export class BeaconChain implements IBeaconChain { * kzg_aggregated_proof=compute_proof_from_blobs(blobs), * ) */ - getBlobsSidecar(beaconBlockRoot: RootHex): eip4844.BlobsSidecar { - const blobs = this.producedBlobsCache.get(beaconBlockRoot); + getBlobsSidecar(beaconBlock: eip4844.BeaconBlock): eip4844.BlobsSidecar { + const blockHash = toHex(beaconBlock.body.executionPayload.blockHash); + const blobs = this.producedBlobsCache.get(blockHash); if (!blobs) { - throw Error(`No blobs for beaconBlockRoot ${beaconBlockRoot}`); + throw Error(`No blobs for beaconBlockRoot ${blockHash}`); } return { - beaconBlockRoot: beaconBlockRoot, - beaconBlockSlot: blobs.slot, + // TODO EIP-4844: Optimize, hashing the full block is not free. + beaconBlockRoot: this.config.getForkTypes(beaconBlock.slot).BeaconBlock.hashTreeRoot(beaconBlock), + beaconBlockSlot: beaconBlock.slot, blobs: blobs, kzgAggregatedProof: computeAggregateKzgProof(blobs), }; diff --git a/packages/beacon-node/src/chain/errors/blobsSidecarError.ts b/packages/beacon-node/src/chain/errors/blobsSidecarError.ts new file mode 100644 index 000000000000..d5cc3c1e384c --- /dev/null +++ b/packages/beacon-node/src/chain/errors/blobsSidecarError.ts @@ -0,0 +1,23 @@ +import {Slot} from "@lodestar/types"; +import {GossipActionError} from "./gossipValidation.js"; + +export enum BlobsSidecarErrorCode { + /** !bls.KeyValidate(block.body.blob_kzg_commitments[i]) */ + INVALID_KZG = "BLOBS_SIDECAR_ERROR_INVALID_KZG", + /** !verify_kzg_commitments_against_transactions(block.body.execution_payload.transactions, block.body.blob_kzg_commitments) */ + INVALID_KZG_TXS = "BLOBS_SIDECAR_ERROR_INVALID_KZG_TXS", + /** sidecar.beacon_block_slot != block.slot */ + INCORRECT_SLOT = "BLOBS_SIDECAR_ERROR_INCORRECT_SLOT", + /** BLSFieldElement in valid range (x < BLS_MODULUS) */ + INVALID_BLOB = "BLOBS_SIDECAR_ERROR_INVALID_BLOB", + /** !bls.KeyValidate(blobs_sidecar.kzg_aggregated_proof) */ + INVALID_KZG_PROOF = "BLOBS_SIDECAR_ERROR_INVALID_KZG_PROOF", +} + +export type BlobsSidecarErrorType = + | {code: BlobsSidecarErrorCode.INVALID_KZG; kzgIdx: number} + | {code: BlobsSidecarErrorCode.INVALID_KZG_TXS} + | {code: BlobsSidecarErrorCode.INCORRECT_SLOT; blockSlot: Slot; blobSlot: Slot} + | {code: BlobsSidecarErrorCode.INVALID_KZG_PROOF}; + +export class BlobsSidecarError extends GossipActionError {} diff --git a/packages/beacon-node/src/chain/interface.ts b/packages/beacon-node/src/chain/interface.ts index 021626b5b58f..b2421bb340fc 100644 --- a/packages/beacon-node/src/chain/interface.ts +++ b/packages/beacon-node/src/chain/interface.ts @@ -111,6 +111,8 @@ export interface IBeaconChain { produceBlock(blockAttributes: BlockAttributes): Promise; produceBlindedBlock(blockAttributes: BlockAttributes): Promise; + getBlobsSidecar(beaconBlock: eip4844.BeaconBlock): eip4844.BlobsSidecar; + /** Process a block until complete */ processBlock(block: allForks.SignedBeaconBlock, opts?: ImportBlockOpts): Promise; /** Process a chain of blocks until complete */ diff --git a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts index 86ccdebde6aa..d7828b9b7223 100644 --- a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts +++ b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts @@ -1,3 +1,4 @@ +import {blobToKzgCommitment} from "c-kzg"; import { Bytes32, phase0, @@ -22,6 +23,7 @@ import { getRandaoMix, getCurrentEpoch, isMergeTransitionComplete, + verifyKzgCommitmentsAgainstTransactions, } from "@lodestar/state-transition"; import {IChainForkConfig} from "@lodestar/config"; import {ForkName, ForkSeq} from "@lodestar/params"; @@ -64,7 +66,9 @@ export enum BlobsResultType { produced, } -export type BlobsResult = {type: BlobsResultType.preEIP4844} | {type: BlobsResultType.produced; blobs: eip4844.Blobs}; +export type BlobsResult = + | {type: BlobsResultType.preEIP4844} + | {type: BlobsResultType.produced; blobs: eip4844.Blobs; blockHash: RootHex}; export async function produceBlockBody( this: BeaconChain, @@ -102,7 +106,7 @@ export async function produceBlockBody( const {eth1Data, deposits} = await this.eth1.getEth1DataAndDeposits(currentState); // We assign this in an EIP-4844 branch below and return it - let blobs: eip4844.Blobs | null = null; + let blobs: {blobs: eip4844.Blobs; blockHash: RootHex} | null = null; const blockBody: phase0.BeaconBlockBody = { randaoReveal, @@ -157,11 +161,12 @@ export async function produceBlockBody( // For MeV boost integration, this is where the execution header will be // fetched from the payload id and a blinded block will be produced instead of // fullblock for the validator to sign - (blockBody as allForks.BlindedBeaconBlockBody).executionPayloadHeader = await prepareExecutionPayloadHeader( + const executionPayloadHeader = await prepareExecutionPayloadHeader( this, currentState as CachedBeaconStateBellatrix, proposerPubKey ); + (blockBody as allForks.BlindedBeaconBlockBody).executionPayloadHeader = executionPayloadHeader; // Capella and later forks have withdrawalRoot on their ExecutionPayloadHeader // TODO Capella: Remove this. It will come from the execution client. @@ -174,7 +179,7 @@ export async function produceBlockBody( if (forkName === ForkName.eip4844) { // Empty blobs for now (blockBody as eip4844.BeaconBlockBody).blobKzgCommitments = []; - blobs = []; + blobs = {blobs: [], blockHash: executionPayloadHeader.blockHash}; } } @@ -224,9 +229,15 @@ export async function produceBlockBody( // payload_id to retrieve blobs and blob_kzg_commitments via get_blobs_and_kzg_commitments(payload_id) const blobsBundle = await this.executionEngine.getBlobsBundle(payloadId); + // Sanity check consistency between getPayload() and getBlobsBundle() + const blockHash = toHex(payload.blockHash); + if (blobsBundle.blockHash !== blockHash) { + throw Error(`blobsBundle incorrect blockHash ${blobsBundle.blockHash} != ${blockHash}`); + } + if (this.opts.sanityCheckExecutionEngineBlocks) { // Optionally sanity-check that the KZG commitments match the versioned hashes in the transactions - verify_kzg_commitments_against_transactions(payload.transactions, blobsBundle.kzgs); + verifyKzgCommitmentsAgainstTransactions(payload.transactions, blobsBundle.kzgs); // Optionally sanity-check that the KZG commitments match the blobs (as produced by the execution engine) if (blobsBundle.blobs.length !== blobsBundle.kzgs.length) { @@ -236,7 +247,7 @@ export async function produceBlockBody( } for (let i = 0; i < blobsBundle.blobs.length; i++) { - const kzg = blob_to_kzg_commitment(blobsBundle.blobs[i]) as eip4844.KZGCommitment; + const kzg = blobToKzgCommitment(blobsBundle.blobs[i]) as eip4844.KZGCommitment; if (!byteArrayEquals(kzg, blobsBundle.kzgs[i])) { throw Error(`Wrong KZG[${i}] ${toHex(blobsBundle.kzgs[i])} expected ${toHex(kzg)}`); } @@ -244,7 +255,7 @@ export async function produceBlockBody( } (blockBody as eip4844.BeaconBlockBody).blobKzgCommitments = blobsBundle.kzgs; - blobs = blobsBundle.blobs; + blobs = {blobs: blobsBundle.blobs, blockHash}; } const fetchedTime = Date.now() / 1000 - computeTimeAtSlot(this.config, blockSlot, this.genesisTime); @@ -284,7 +295,7 @@ export async function produceBlockBody( if (!blobs) { throw Error("Blobs are null post eip4844"); } - blobsResult = {type: BlobsResultType.produced, blobs}; + blobsResult = {type: BlobsResultType.produced, ...blobs}; } else { blobsResult = {type: BlobsResultType.preEIP4844}; } diff --git a/packages/beacon-node/src/chain/validation/blobsSidecar.ts b/packages/beacon-node/src/chain/validation/blobsSidecar.ts new file mode 100644 index 000000000000..00ebc4b5d19f --- /dev/null +++ b/packages/beacon-node/src/chain/validation/blobsSidecar.ts @@ -0,0 +1,68 @@ +import {IChainForkConfig} from "@lodestar/config"; +import {eip4844} from "@lodestar/types"; +import {verifyKzgCommitmentsAgainstTransactions} from "@lodestar/state-transition"; +import {IBeaconChain} from "../interface.js"; +import {BlobsSidecarError, BlobsSidecarErrorCode} from "../errors/blobsSidecarError.js"; +import {GossipAction} from "../errors/gossipValidation.js"; + +export async function validateGossipBlobsSidecar( + config: IChainForkConfig, + chain: IBeaconChain, + signedBlock: eip4844.SignedBeaconBlock, + blobsSidecar: eip4844.BlobsSidecar +): Promise { + const block = signedBlock.message; + + // Spec: https://github.com/ethereum/consensus-specs/blob/4cb6fd1c8c8f190d147d15b182c2510d0423ec61/specs/eip4844/p2p-interface.md#beacon_block_and_blobs_sidecar + // [REJECT] The KZG commitments of the blobs are all correctly encoded compressed BLS G1 Points. + // -- i.e. all(bls.KeyValidate(commitment) for commitment in block.body.blob_kzg_commitments) + const {blobKzgCommitments} = block.body; + for (let i = 0; i < blobKzgCommitments.length; i++) { + if (!bls.keyValidate(blobKzgCommitments[i])) { + throw new BlobsSidecarError(GossipAction.REJECT, {code: BlobsSidecarErrorCode.INVALID_KZG, kzgIdx: i}); + } + } + + // [REJECT] The KZG commitments correspond to the versioned hashes in the transactions list. + // -- i.e. verify_kzg_commitments_against_transactions(block.body.execution_payload.transactions, block.body.blob_kzg_commitments) + if ( + !verifyKzgCommitmentsAgainstTransactions(block.body.executionPayload.transactions, block.body.blobKzgCommitments) + ) { + throw new BlobsSidecarError(GossipAction.REJECT, {code: BlobsSidecarErrorCode.INVALID_KZG_TXS}); + } + + // [IGNORE] the sidecar.beacon_block_slot is for the current slot (with a MAXIMUM_GOSSIP_CLOCK_DISPARITY allowance) + // -- i.e. sidecar.beacon_block_slot == block.slot. + if (blobsSidecar.beaconBlockSlot !== block.slot) { + throw new BlobsSidecarError(GossipAction.IGNORE, { + code: BlobsSidecarErrorCode.INCORRECT_SLOT, + blobSlot: blobsSidecar.beaconBlockSlot, + blockSlot: block.slot, + }); + } + + // [REJECT] the sidecar.blobs are all well formatted, i.e. the BLSFieldElement in valid range (x < BLS_MODULUS). + TODO; + + // [REJECT] The KZG proof is a correctly encoded compressed BLS G1 Point + // -- i.e. bls.KeyValidate(blobs_sidecar.kzg_aggregated_proof) + if (!bls.KeyValidate(blobsSidecar.kzgAggregatedProof)) { + throw new BlobsSidecarError(GossipAction.REJECT, {code: BlobsSidecarErrorCode.INVALID_KZG_PROOF}); + } +} + +type Result = {ok: true; result: T} | {ok: false; error: Error}; + +function rustOk(): Result; + +function Ok(result: T): Result { + return {ok: true, result}; +} + +function okUser(): Result { + const res = rustOk(); + if (!res.ok) return res; + const resStr = res.result; + + return Ok(parseInt(resStr)); +} diff --git a/packages/beacon-node/src/execution/engine/http.ts b/packages/beacon-node/src/execution/engine/http.ts index cc1fa752ad86..e482cf9b2dfe 100644 --- a/packages/beacon-node/src/execution/engine/http.ts +++ b/packages/beacon-node/src/execution/engine/http.ts @@ -499,7 +499,7 @@ export function parseExecutionPayload(data: ExecutionPayloadRpc): allForks.Execu export function parseBlobsBundle(data: BlobsBundleRpc): BlobsBundle { return { - blockHash: dataToBytes(data.blockHash), + blockHash: data.blockHash, kzgs: data.kzgs.map((kzg) => dataToBytes(kzg)), blobs: data.blobs.map((blob) => dataToBytes(blob)), }; diff --git a/packages/beacon-node/src/execution/engine/interface.ts b/packages/beacon-node/src/execution/engine/interface.ts index 67e2091149a2..de20d12abf06 100644 --- a/packages/beacon-node/src/execution/engine/interface.ts +++ b/packages/beacon-node/src/execution/engine/interface.ts @@ -56,7 +56,11 @@ export type TransitionConfigurationV1 = { }; export type BlobsBundle = { - blockHash: Uint8Array; + /** + * Execution payload `blockHash` for the caller to sanity-check the consistency with the `engine_getPayload` call + * https://github.com/protolambda/execution-apis/blob/bf44a8d08ab34b861ef97fa9ef5c5e7806194547/src/engine/blob-extension.md?plain=1#L49 + */ + blockHash: RootHex; kzgs: KZGCommitment[]; blobs: Blob[]; }; diff --git a/packages/beacon-node/src/network/gossip/handlers/index.ts b/packages/beacon-node/src/network/gossip/handlers/index.ts index b54547443046..adae10309437 100644 --- a/packages/beacon-node/src/network/gossip/handlers/index.ts +++ b/packages/beacon-node/src/network/gossip/handlers/index.ts @@ -33,6 +33,7 @@ import {NetworkEvent} from "../../events.js"; import {PeerAction} from "../../peers/index.js"; import {validateLightClientFinalityUpdate} from "../../../chain/validation/lightClientFinalityUpdate.js"; import {validateLightClientOptimisticUpdate} from "../../../chain/validation/lightClientOptimisticUpdate.js"; +import {validateGossipBlobsSidecar} from "../../../chain/validation/blobsSidecar.js"; /** * Gossip handler options as part of network options @@ -76,23 +77,10 @@ const MAX_UNKNOWN_BLOCK_ROOT_RETRIES = 1; export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipHandlerOpts): GossipHandlers { const {chain, config, metrics, network, logger} = modules; - async function handleBeaconBlock( - signedBlock: SignedBeaconBlock, - fork: ForkName, - peerIdStr: string, - seenTimestampSec: number - ): Promise { + async function validateBeaconBlock(signedBlock: SignedBeaconBlock, fork: ForkName, peerIdStr: string): Promise { const slot = signedBlock.message.slot; const forkTypes = config.getForkTypes(slot); const blockHex = prettyBytes(forkTypes.BeaconBlock.hashTreeRoot(signedBlock.message)); - const delaySec = chain.clock.secFromSlot(slot, seenTimestampSec); - logger.verbose("Received gossip block", { - slot: slot, - root: blockHex, - curentSlot: chain.clock.currentSlot, - peerId: peerIdStr, - delaySec, - }); try { await validateGossipBlock(config, chain, signedBlock, fork); @@ -110,6 +98,20 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH throw e; } + } + + function handleValidBeaconBlock(signedBlock: SignedBeaconBlock, peerIdStr: string, seenTimestampSec: number): void { + const slot = signedBlock.message.slot; + const forkTypes = config.getForkTypes(slot); + const blockHex = prettyBytes(forkTypes.BeaconBlock.hashTreeRoot(signedBlock.message)); + const delaySec = chain.clock.secFromSlot(slot, seenTimestampSec); + logger.verbose("Received gossip block", { + slot: slot, + root: blockHex, + curentSlot: chain.clock.currentSlot, + peerId: peerIdStr, + delaySec, + }); // Handler - MUST NOT `await`, to allow validation result to be propagated @@ -146,15 +148,19 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH } return { - [GossipType.beacon_block_and_blobs_sidecar]: async (signedBlock, topic, peerIdStr, seenTimestampSec) => { - const {beaconBlock, blobsSidecar: _} = signedBlock; - // TODO EIP-4844: Validate blobs - - return handleBeaconBlock(beaconBlock, topic.fork, peerIdStr, seenTimestampSec); + [GossipType.beacon_block]: async (signedBlock, topic, peerIdStr, seenTimestampSec) => { + await validateBeaconBlock(signedBlock, topic.fork, peerIdStr); + handleValidBeaconBlock(signedBlock, peerIdStr, seenTimestampSec); }, - [GossipType.beacon_block]: async (signedBlock, topic, peerIdStr, seenTimestampSec) => { - return handleBeaconBlock(signedBlock, topic.fork, peerIdStr, seenTimestampSec); + [GossipType.beacon_block_and_blobs_sidecar]: async (signedBlock, topic, peerIdStr, seenTimestampSec) => { + const {beaconBlock, blobsSidecar} = signedBlock; + + // Validate block + blob. Then forward, then handle both + await validateBeaconBlock(beaconBlock, topic.fork, peerIdStr); + await validateGossipBlobsSidecar(config, chain, beaconBlock, blobsSidecar); + handleValidBeaconBlock(beaconBlock, peerIdStr, seenTimestampSec); + // TODO: EIP-4844 handle blobs }, [GossipType.beacon_aggregate_and_proof]: async (signedAggregateAndProof, _topic, _peer, seenTimestampSec) => { diff --git a/packages/beacon-node/src/network/gossip/validation/queue.ts b/packages/beacon-node/src/network/gossip/validation/queue.ts index 3a389da7823c..ccd37d425f04 100644 --- a/packages/beacon-node/src/network/gossip/validation/queue.ts +++ b/packages/beacon-node/src/network/gossip/validation/queue.ts @@ -8,6 +8,7 @@ import {GossipJobQueues, GossipType, GossipValidatorFn, ResolvedType, ValidatorF */ const gossipQueueOpts: {[K in GossipType]: Pick} = { [GossipType.beacon_block]: {maxLength: 1024, type: QueueType.FIFO}, + [GossipType.beacon_block_and_blobs_sidecar]: {maxLength: 1024, type: QueueType.FIFO}, // lighthoue has aggregate_queue 4096 and unknown_block_aggregate_queue 1024, we use single queue [GossipType.beacon_aggregate_and_proof]: {maxLength: 5120, type: QueueType.LIFO, maxConcurrency: 16}, // lighthouse has attestation_queue 16384 and unknown_block_attestation_queue 8192, we use single queue @@ -19,7 +20,6 @@ const gossipQueueOpts: {[K in GossipType]: Pick; getConnectedPeers(): PeerId[]; hasSomeConnectedPeer(): boolean; + + publishBeaconBlockMaybeBlobs(signedBlock: allForks.SignedBeaconBlock): Promise; + /** Subscribe, search peers, join long-lived attnets */ prepareBeaconCommitteeSubnet(subscriptions: CommitteeSubscription[]): void; /** Subscribe, search peers, join long-lived syncnets */ diff --git a/packages/beacon-node/src/network/network.ts b/packages/beacon-node/src/network/network.ts index 2359a5071fc4..3faac24ce4d4 100644 --- a/packages/beacon-node/src/network/network.ts +++ b/packages/beacon-node/src/network/network.ts @@ -5,16 +5,17 @@ import {PeerId} from "@libp2p/interface-peer-id"; import {Multiaddr} from "@multiformats/multiaddr"; import {IBeaconConfig} from "@lodestar/config"; import {ILogger, sleep} from "@lodestar/utils"; -import {ATTESTATION_SUBNET_COUNT, ForkName, SYNC_COMMITTEE_SUBNET_COUNT} from "@lodestar/params"; +import {ATTESTATION_SUBNET_COUNT, ForkName, ForkSeq, SYNC_COMMITTEE_SUBNET_COUNT} from "@lodestar/params"; import {Discv5, ENR} from "@chainsafe/discv5"; import {computeEpochAtSlot, computeTimeAtSlot} from "@lodestar/state-transition"; -import {altair, Epoch} from "@lodestar/types"; +import {allForks, altair, Epoch, RootHex} from "@lodestar/types"; +import {promiseAllMaybeAsync} from "../util/promises.js"; import {IMetrics} from "../metrics/index.js"; import {ChainEvent, IBeaconChain, IBeaconClock} from "../chain/index.js"; import {INetworkOptions} from "./options.js"; import {INetwork} from "./interface.js"; import {IReqResp, IReqRespOptions, ReqResp, ReqRespHandlers} from "./reqresp/index.js"; -import {Eth2Gossipsub, getGossipHandlers, GossipHandlers, GossipType} from "./gossip/index.js"; +import {Eth2Gossipsub, getGossipHandlers, GossipHandlers, GossipTopicTypeMap, GossipType} from "./gossip/index.js"; import {MetadataController} from "./metadata.js"; import {FORK_EPOCH_LOOKAHEAD, getActiveForks} from "./forks.js"; import {PeerManager} from "./peers/peerManager.js"; @@ -195,6 +196,21 @@ export class Network implements INetwork { return this.peerManager.hasSomeConnectedPeer(); } + async publishBeaconBlockMaybeBlobs(beaconBlock: allForks.SignedBeaconBlock): Promise { + const fns: (() => Promise)[] = [ + // Always publish on beacon topic + () => this.gossip.publishBeaconBlock(beaconBlock), + ]; + + // TODO EIP-4844: Open question if broadcast to both block topic + block_and_blobs topic + if (this.config.getForkSeq(beaconBlock.message.slot) >= ForkSeq.eip4844) { + const blobsSidecar = this.chain.getBlobsSidecar(beaconBlock.message as eip4844.BeaconBlock); + fns.push(() => this.gossip.publishSignedBeaconBlockAndBlobsSidecar({beaconBlock, blobsSidecar})); + } + + await promiseAllMaybeAsync(fns); + } + /** * Request att subnets up `toSlot`. Network will ensure to mantain some peers for each */ @@ -313,25 +329,8 @@ export class Network implements INetwork { if (this.subscribedForks.has(fork)) return; this.subscribedForks.add(fork); - this.gossip.subscribeTopic({type: GossipType.beacon_block, fork}); - this.gossip.subscribeTopic({type: GossipType.beacon_aggregate_and_proof, fork}); - this.gossip.subscribeTopic({type: GossipType.voluntary_exit, fork}); - this.gossip.subscribeTopic({type: GossipType.proposer_slashing, fork}); - this.gossip.subscribeTopic({type: GossipType.attester_slashing, fork}); - // Any fork after altair included - if (fork !== ForkName.phase0) { - this.gossip.subscribeTopic({type: GossipType.sync_committee_contribution_and_proof, fork}); - this.gossip.subscribeTopic({type: GossipType.light_client_optimistic_update, fork}); - this.gossip.subscribeTopic({type: GossipType.light_client_finality_update, fork}); - } - - if (this.opts.subscribeAllSubnets) { - for (let subnet = 0; subnet < ATTESTATION_SUBNET_COUNT; subnet++) { - this.gossip.subscribeTopic({type: GossipType.beacon_attestation, fork, subnet}); - } - for (let subnet = 0; subnet < SYNC_COMMITTEE_SUBNET_COUNT; subnet++) { - this.gossip.subscribeTopic({type: GossipType.sync_committee, fork, subnet}); - } + for (const topic of this.coreTopicsAtFork(fork)) { + this.gossip.subscribeTopic({...topic, fork}); } }; @@ -339,27 +338,52 @@ export class Network implements INetwork { if (!this.subscribedForks.has(fork)) return; this.subscribedForks.delete(fork); - this.gossip.unsubscribeTopic({type: GossipType.beacon_block, fork}); - this.gossip.unsubscribeTopic({type: GossipType.beacon_aggregate_and_proof, fork}); - this.gossip.unsubscribeTopic({type: GossipType.voluntary_exit, fork}); - this.gossip.unsubscribeTopic({type: GossipType.proposer_slashing, fork}); - this.gossip.unsubscribeTopic({type: GossipType.attester_slashing, fork}); + for (const topic of this.coreTopicsAtFork(fork)) { + this.gossip.unsubscribeTopic({...topic, fork}); + } + }; + + /** + * De-duplicate logic to pick fork topics between subscribeCoreTopicsAtFork and unsubscribeCoreTopicsAtFork + */ + private coreTopicsAtFork(fork: ForkName): GossipTopicTypeMap[keyof GossipTopicTypeMap][] { + // Common topics for all forks + const topics: GossipTopicTypeMap[keyof GossipTopicTypeMap][] = [ + // {type: GossipType.beacon_block}, // Handled below + {type: GossipType.beacon_aggregate_and_proof}, + {type: GossipType.voluntary_exit}, + {type: GossipType.proposer_slashing}, + {type: GossipType.attester_slashing}, + ]; + + // After EIP4844 only track beacon_block_and_blobs_sidecar topic + const forkSeq = this.config.forks[fork].seq; + if (forkSeq < ForkSeq.eip4844) { + topics.push({type: GossipType.beacon_block}); + } else { + topics.push({type: GossipType.beacon_block_and_blobs_sidecar}); + } + // Any fork after altair included - if (fork !== ForkName.phase0) { - this.gossip.unsubscribeTopic({type: GossipType.sync_committee_contribution_and_proof, fork}); - this.gossip.unsubscribeTopic({type: GossipType.light_client_optimistic_update, fork}); - this.gossip.unsubscribeTopic({type: GossipType.light_client_finality_update, fork}); + if (forkSeq >= ForkSeq.altair) { + topics.push({type: GossipType.sync_committee_contribution_and_proof}); + topics.push({type: GossipType.light_client_optimistic_update}); + topics.push({type: GossipType.light_client_finality_update}); } if (this.opts.subscribeAllSubnets) { for (let subnet = 0; subnet < ATTESTATION_SUBNET_COUNT; subnet++) { this.gossip.unsubscribeTopic({type: GossipType.beacon_attestation, fork, subnet}); } - for (let subnet = 0; subnet < SYNC_COMMITTEE_SUBNET_COUNT; subnet++) { - this.gossip.unsubscribeTopic({type: GossipType.sync_committee, fork, subnet}); + if (forkSeq >= ForkSeq.altair) { + for (let subnet = 0; subnet < SYNC_COMMITTEE_SUBNET_COUNT; subnet++) { + this.gossip.unsubscribeTopic({type: GossipType.sync_committee, fork, subnet}); + } } } - }; + + return topics; + } private onLightClientFinalityUpdate = async (finalityUpdate: altair.LightClientFinalityUpdate): Promise => { if (this.hasAttachedSyncCommitteeMember()) { diff --git a/packages/beacon-node/src/util/promises.ts b/packages/beacon-node/src/util/promises.ts new file mode 100644 index 000000000000..e0bba250f310 --- /dev/null +++ b/packages/beacon-node/src/util/promises.ts @@ -0,0 +1,14 @@ +/** + * Promise.all() but allows all functions to run even if one throws syncronously + */ +export function promiseAllMaybeAsync(fns: Array<() => Promise>): Promise { + return Promise.all( + fns.map((fn) => { + try { + return fn(); + } catch (e) { + return Promise.reject(e); + } + }) + ); +} diff --git a/packages/state-transition/src/block/index.ts b/packages/state-transition/src/block/index.ts index 3d2ae169b77c..ee4e8927e48b 100644 --- a/packages/state-transition/src/block/index.ts +++ b/packages/state-transition/src/block/index.ts @@ -1,14 +1,15 @@ import {ForkSeq} from "@lodestar/params"; -import {allForks, altair} from "@lodestar/types"; +import {allForks, altair, eip4844} from "@lodestar/types"; import {ExecutionEngine} from "../util/executionEngine.js"; import {getFullOrBlindedPayload, isExecutionEnabled} from "../util/execution.js"; -import {CachedBeaconStateAllForks, CachedBeaconStateBellatrix} from "../types.js"; +import {CachedBeaconStateAllForks, CachedBeaconStateBellatrix, CachedBeaconStateEip4844} from "../types.js"; import {processExecutionPayload} from "./processExecutionPayload.js"; import {processSyncAggregate} from "./processSyncCommittee.js"; import {processBlockHeader} from "./processBlockHeader.js"; import {processEth1Data} from "./processEth1Data.js"; import {processOperations} from "./processOperations.js"; import {processRandao} from "./processRandao.js"; +import {processBlobKzgCommitments} from "./processBlobKzgCommitments.js"; // Spec tests export {processBlockHeader, processExecutionPayload, processRandao, processEth1Data, processSyncAggregate}; @@ -42,4 +43,12 @@ export function processBlock( if (fork >= ForkSeq.altair) { processSyncAggregate(state, block as altair.BeaconBlock, verifySignatures); } + + if (fork >= ForkSeq.eip4844) { + processBlobKzgCommitments(block.body as eip4844.BeaconBlockBody); + + // New in EIP-4844, note: Can sync optimistically without this condition, see note on `is_data_available` + // NOTE: Ommitted and should be verified beforehand + // assert is_data_available(block.slot, hash_tree_root(block), block.body.blob_kzg_commitments) + } } diff --git a/packages/state-transition/src/block/processBlobKzgCommitments.ts b/packages/state-transition/src/block/processBlobKzgCommitments.ts new file mode 100644 index 000000000000..3696d327dbef --- /dev/null +++ b/packages/state-transition/src/block/processBlobKzgCommitments.ts @@ -0,0 +1,8 @@ +import {eip4844} from "@lodestar/types"; +import {verifyKzgCommitmentsAgainstTransactions} from "../util/index.js"; + +export function processBlobKzgCommitments(body: eip4844.BeaconBlockBody): void { + if (!verifyKzgCommitmentsAgainstTransactions(body.executionPayload.transactions, body.blobKzgCommitments)) { + throw Error("Invalid KZG commitments against transactions"); + } +} diff --git a/packages/state-transition/src/cache/stateCache.ts b/packages/state-transition/src/cache/stateCache.ts index 8cf3b1be7e90..8cbedd9ae0f4 100644 --- a/packages/state-transition/src/cache/stateCache.ts +++ b/packages/state-transition/src/cache/stateCache.ts @@ -5,7 +5,7 @@ import { BeaconStateAltair, BeaconStateBellatrix, BeaconStateCapella, - BeaconState4844, + BeaconStateEip4844, BeaconStateAllForks, } from "./types.js"; @@ -116,9 +116,12 @@ export type CachedBeaconStatePhase0 = CachedBeaconState; export type CachedBeaconStateAltair = CachedBeaconState; export type CachedBeaconStateBellatrix = CachedBeaconState; export type CachedBeaconStateCapella = CachedBeaconState; -export type CachedBeaconState4844 = CachedBeaconState; +export type CachedBeaconStateEip4844 = CachedBeaconState; export type CachedBeaconStateAllForks = CachedBeaconState; -export type CachedBeaconStateExecutions = CachedBeaconStateBellatrix | CachedBeaconStateCapella | CachedBeaconState4844; +export type CachedBeaconStateExecutions = + | CachedBeaconStateBellatrix + | CachedBeaconStateCapella + | CachedBeaconStateEip4844; /** * Create CachedBeaconState computing a new EpochContext instance diff --git a/packages/state-transition/src/cache/types.ts b/packages/state-transition/src/cache/types.ts index fffc9ac80def..9a51df0c3f92 100644 --- a/packages/state-transition/src/cache/types.ts +++ b/packages/state-transition/src/cache/types.ts @@ -5,7 +5,7 @@ export type BeaconStatePhase0 = CompositeViewDU; export type BeaconStateAltair = CompositeViewDU; export type BeaconStateBellatrix = CompositeViewDU; export type BeaconStateCapella = CompositeViewDU; -export type BeaconState4844 = CompositeViewDU; +export type BeaconStateEip4844 = CompositeViewDU; // Union at the TreeViewDU level // - Works well as function argument and as generic type for allForks functions @@ -17,6 +17,6 @@ export type BeaconStateAllForks = | BeaconStateAltair | BeaconStateBellatrix | BeaconStateCapella - | BeaconState4844; + | BeaconStateEip4844; -export type BeaconStateExecutions = BeaconStateBellatrix | BeaconStateCapella | BeaconState4844; +export type BeaconStateExecutions = BeaconStateBellatrix | BeaconStateCapella | BeaconStateEip4844; diff --git a/packages/state-transition/src/index.ts b/packages/state-transition/src/index.ts index 20e7c1fff2a3..8b9422aacfd9 100644 --- a/packages/state-transition/src/index.ts +++ b/packages/state-transition/src/index.ts @@ -9,7 +9,7 @@ export { CachedBeaconStateAltair, CachedBeaconStateBellatrix, CachedBeaconStateCapella, - CachedBeaconState4844, + CachedBeaconStateEip4844, CachedBeaconStateAllForks, CachedBeaconStateExecutions, // Non-cached states diff --git a/packages/state-transition/src/slot/upgradeStateTo4844.ts b/packages/state-transition/src/slot/upgradeStateTo4844.ts index c6113872144f..7a09e8f71894 100644 --- a/packages/state-transition/src/slot/upgradeStateTo4844.ts +++ b/packages/state-transition/src/slot/upgradeStateTo4844.ts @@ -1,12 +1,12 @@ import {ssz} from "@lodestar/types"; -import {CachedBeaconState4844} from "../types.js"; +import {CachedBeaconStateEip4844} from "../types.js"; import {getCachedBeaconState} from "../cache/stateCache.js"; import {CachedBeaconStateCapella} from "../types.js"; /** * Upgrade a state from Capella to 4844. */ -export function upgradeStateTo4844(stateCapella: CachedBeaconStateCapella): CachedBeaconState4844 { +export function upgradeStateTo4844(stateCapella: CachedBeaconStateCapella): CachedBeaconStateEip4844 { const {config} = stateCapella; const stateCapellaNode = ssz.capella.BeaconState.commitViewDU(stateCapella); diff --git a/packages/state-transition/src/types.ts b/packages/state-transition/src/types.ts index 7710674d125c..59ccc1a5be14 100644 --- a/packages/state-transition/src/types.ts +++ b/packages/state-transition/src/types.ts @@ -8,7 +8,7 @@ export { CachedBeaconStateAllForks, CachedBeaconStateCapella, CachedBeaconStateExecutions, - CachedBeaconState4844, + CachedBeaconStateEip4844, } from "./cache/stateCache.js"; export { diff --git a/packages/state-transition/src/util/blobs.ts b/packages/state-transition/src/util/blobs.ts new file mode 100644 index 000000000000..82b73a9b3603 --- /dev/null +++ b/packages/state-transition/src/util/blobs.ts @@ -0,0 +1,64 @@ +import SHA256 from "@chainsafe/as-sha256"; +import {byteArrayEquals} from "@chainsafe/ssz"; +import {bellatrix, eip4844} from "@lodestar/types"; +import {toHex} from "@lodestar/utils"; + +// TODO EIP-4844: Move to params +const BLOB_TX_TYPE = 0x05; +const VERSIONED_HASH_VERSION_KZG = 0x01; + +type VersionHash = Uint8Array; + +export function verifyKzgCommitmentsAgainstTransactions( + transactions: bellatrix.Transaction[], + blobKzgCommitments: eip4844.KZGCommitment[] +): boolean { + const allVersionedHashes: VersionHash[] = []; + for (const tx of transactions) { + if (tx[0] === BLOB_TX_TYPE) { + // TODO EIP-4844: Optimize array manipulation + allVersionedHashes.push(...txPeekBlobVersionedHashes(tx)); + } + } + + for (let i = 0; i < blobKzgCommitments.length; i++) { + const versionedHash = kzgCommitmentToVersionedHash(blobKzgCommitments[i]); + if (!byteArrayEquals(allVersionedHashes[i], versionedHash)) { + throw Error(`Wrong versionedHash ${i} ${toHex(allVersionedHashes[i])} != ${toHex(versionedHash)}`); + } + } + + // TODO EIP-4844: Use proper API, either throw error or return boolean + return true; +} + +function txPeekBlobVersionedHashes(opaqueTx: bellatrix.Transaction): VersionHash[] { + if (opaqueTx[0] !== BLOB_TX_TYPE) { + throw Error(`tx type ${opaqueTx[0]} != BLOB_TX_TYPE`); + } + + const opaqueTxDv = new DataView(opaqueTx.buffer, opaqueTx.byteOffset, opaqueTx.byteLength); + + // uint32.decode_bytes(opaque_tx[1:5]) + // true = little endian + const messageOffset = 1 + opaqueTxDv.getUint32(1, true); + // field offset: 32 + 8 + 32 + 32 + 8 + 4 + 32 + 4 + 4 + 32 = 188 + // Reference: https://gist.github.com/protolambda/23bd106b66f6d4bb854ce46044aa3ca3 + const blobVersionedHashesOffset = messageOffset + opaqueTxDv.getUint32(188); + + const versionedHashes: VersionHash[] = []; + + // iterate from x to end of data, in steps of 32, to get all hashes + for (let i = blobVersionedHashesOffset; i < opaqueTx.length; i += 32) { + versionedHashes.push(opaqueTx.subarray(i, i + 32)); + } + + return versionedHashes; +} + +function kzgCommitmentToVersionedHash(kzgCommitment: eip4844.KZGCommitment): VersionHash { + const hash = SHA256.digest(kzgCommitment); + // Equivalent to `VERSIONED_HASH_VERSION_KZG + hash(kzg_commitment)[1:]` + hash[0] = VERSIONED_HASH_VERSION_KZG; + return hash; +} diff --git a/packages/state-transition/src/util/index.ts b/packages/state-transition/src/util/index.ts index c01e9dce4d20..5e51d0c8ac42 100644 --- a/packages/state-transition/src/util/index.ts +++ b/packages/state-transition/src/util/index.ts @@ -3,6 +3,8 @@ export * from "./array.js"; export * from "./attestation.js"; export * from "./attesterStatus.js"; export * from "./balance.js"; +export * from "./blindedBlock.js"; +export * from "./blobs.js"; export * from "./execution.js"; export * from "./blockRoot.js"; export * from "./domain.js";