diff --git a/packages/beacon-node/src/chain/blocks/importBlock.ts b/packages/beacon-node/src/chain/blocks/importBlock.ts index cd258e42c146..feaddfbad39d 100644 --- a/packages/beacon-node/src/chain/blocks/importBlock.ts +++ b/packages/beacon-node/src/chain/blocks/importBlock.ts @@ -63,6 +63,7 @@ export async function importBlock( const blockRootHex = toHexString(blockRoot); const currentEpoch = computeEpochAtSlot(this.forkChoice.getTime()); const blockEpoch = computeEpochAtSlot(block.message.slot); + const parentEpoch = computeEpochAtSlot(parentBlockSlot); const prevFinalizedEpoch = this.forkChoice.getFinalizedCheckpoint().epoch; const blockDelaySec = (fullyVerifiedBlock.seenTimestampSec - postState.genesisTime) % this.config.SECONDS_PER_SLOT; @@ -347,6 +348,12 @@ export async function importBlock( this.logger.verbose("After importBlock caching postState without SSZ cache", {slot: postState.slot}); } + if (parentEpoch < blockEpoch) { + // current epoch and previous epoch are likely cached in previous states + this.shufflingCache.processState(postState, postState.epochCtx.nextShuffling.epoch); + this.logger.verbose("Processed shuffling for next epoch", {parentEpoch, blockEpoch, slot: block.message.slot}); + } + if (block.message.slot % SLOTS_PER_EPOCH === 0) { // Cache state to preserve epoch transition work const checkpointState = postState; diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 18c37a3ba437..ba1f7ad309db 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -11,6 +11,7 @@ import { isCachedBeaconState, Index2PubkeyCache, PubkeyIndexMap, + EpochShuffling, } from "@lodestar/state-transition"; import {BeaconConfig} from "@lodestar/config"; import { @@ -39,7 +40,6 @@ import {IExecutionEngine, IExecutionBuilder} from "../execution/index.js"; import {Clock, ClockEvent, IClock} from "../util/clock.js"; import {ensureDir, writeIfNotExist} from "../util/file.js"; import {isOptimisticBlock} from "../util/forkChoice.js"; -import {CheckpointStateCache, StateContextCache} from "./stateCache/index.js"; import {BlockProcessor, ImportBlockOpts} from "./blocks/index.js"; import {ChainEventEmitter, ChainEvent} from "./emitter.js"; import {IBeaconChain, ProposerPreparationData, BlockHash, StateGetOpts} from "./interface.js"; @@ -75,6 +75,9 @@ import {BlockAttributes, produceBlockBody} from "./produceBlock/produceBlockBody import {computeNewStateRoot} from "./produceBlock/computeNewStateRoot.js"; import {BlockInput} from "./blocks/types.js"; import {SeenAttestationDatas} from "./seenCache/seenAttestationData.js"; +import {ShufflingCache} from "./shufflingCache.js"; +import {StateContextCache} from "./stateCache/stateContextCache.js"; +import {CheckpointStateCache} from "./stateCache/stateContextCheckpointsCache.js"; /** * Arbitrary constants, blobs and payloads should be consumed immediately in the same slot @@ -129,6 +132,7 @@ export class BeaconChain implements IBeaconChain { readonly beaconProposerCache: BeaconProposerCache; readonly checkpointBalancesCache: CheckpointBalancesCache; + readonly shufflingCache: ShufflingCache; /** Map keyed by executionPayload.blockHash of the block for those blobs */ readonly producedBlobSidecarsCache = new Map(); readonly producedBlindedBlobSidecarsCache = new Map(); @@ -209,6 +213,7 @@ export class BeaconChain implements IBeaconChain { this.beaconProposerCache = new BeaconProposerCache(opts); this.checkpointBalancesCache = new CheckpointBalancesCache(); + this.shufflingCache = new ShufflingCache(metrics, this.opts); // Restore state caches // anchorState may already by a CachedBeaconState. If so, don't create the cache again, since deserializing all @@ -223,6 +228,9 @@ export class BeaconChain implements IBeaconChain { pubkey2index: new PubkeyIndexMap(), index2pubkey: [], }); + this.shufflingCache.processState(cachedState, cachedState.epochCtx.previousShuffling.epoch); + this.shufflingCache.processState(cachedState, cachedState.epochCtx.currentShuffling.epoch); + this.shufflingCache.processState(cachedState, cachedState.epochCtx.nextShuffling.epoch); // Persist single global instance of state caches this.pubkey2index = cachedState.epochCtx.pubkey2index; @@ -640,6 +648,49 @@ export class BeaconChain implements IBeaconChain { } } + /** + * Regenerate state for attestation verification, this does not happen with default chain option of maxSkipSlots = 32 . + * However, need to handle just in case. Lodestar doesn't support multiple regen state requests for attestation verification + * at the same time, bounded inside "ShufflingCache.insertPromise()" function. + * Leave this function in chain instead of attestatation verification code to make sure we're aware of its performance impact. + */ + async regenStateForAttestationVerification( + attEpoch: Epoch, + shufflingDependentRoot: RootHex, + attHeadBlock: ProtoBlock, + regenCaller: RegenCaller + ): Promise { + // this is to prevent multiple calls to get shuffling for the same epoch and dependent root + // any subsequent calls of the same epoch and dependent root will wait for this promise to resolve + this.shufflingCache.insertPromise(attEpoch, shufflingDependentRoot); + const blockEpoch = computeEpochAtSlot(attHeadBlock.slot); + + let state: CachedBeaconStateAllForks; + if (blockEpoch < attEpoch - 1) { + // thanks to one epoch look ahead, we don't need to dial up to attEpoch + const targetSlot = computeStartSlotAtEpoch(attEpoch - 1); + this.metrics?.gossipAttestation.useHeadBlockStateDialedToTargetEpoch.inc({caller: regenCaller}); + state = await this.regen.getBlockSlotState( + attHeadBlock.blockRoot, + targetSlot, + {dontTransferCache: true}, + regenCaller + ); + } else if (blockEpoch > attEpoch) { + // should not happen, handled inside attestation verification code + throw Error(`Block epoch ${blockEpoch} is after attestation epoch ${attEpoch}`); + } else { + // should use either current or next shuffling of head state + // it's not likely to hit this since these shufflings are cached already + // so handle just in case + this.metrics?.gossipAttestation.useHeadBlockState.inc({caller: regenCaller}); + state = await this.regen.getState(attHeadBlock.stateRoot, regenCaller); + } + + // resolve the promise to unblock other calls of the same epoch and dependent root + return this.shufflingCache.processState(state, attEpoch); + } + /** * `ForkChoice.onBlock` must never throw for a block that is valid with respect to the network * `justifiedBalancesGetter()` must never throw and it should always return a state. diff --git a/packages/beacon-node/src/chain/errors/attestationError.ts b/packages/beacon-node/src/chain/errors/attestationError.ts index a93f5b42e439..8e0dc925f32e 100644 --- a/packages/beacon-node/src/chain/errors/attestationError.ts +++ b/packages/beacon-node/src/chain/errors/attestationError.ts @@ -1,5 +1,5 @@ import {toHexString} from "@chainsafe/ssz"; -import {CommitteeIndex, Epoch, Slot, ValidatorIndex, RootHex} from "@lodestar/types"; +import {Epoch, Slot, ValidatorIndex, RootHex} from "@lodestar/types"; import {GossipActionError} from "./gossipValidation.js"; export enum AttestationErrorCode { @@ -65,11 +65,6 @@ export enum AttestationErrorCode { * A signature on the attestation is invalid. */ INVALID_SIGNATURE = "ATTESTATION_ERROR_INVALID_SIGNATURE", - /** - * There is no committee for the slot and committee index of this attestation - * and the attestation should not have been produced. - */ - NO_COMMITTEE_FOR_SLOT_AND_INDEX = "ATTESTATION_ERROR_NO_COMMITTEE_FOR_SLOT_AND_INDEX", /** * The unaggregated attestation doesn't have only one aggregation bit set. */ @@ -150,7 +145,6 @@ export type AttestationErrorType = | {code: AttestationErrorCode.HEAD_NOT_TARGET_DESCENDANT} | {code: AttestationErrorCode.UNKNOWN_TARGET_ROOT; root: Uint8Array} | {code: AttestationErrorCode.INVALID_SIGNATURE} - | {code: AttestationErrorCode.NO_COMMITTEE_FOR_SLOT_AND_INDEX; slot: Slot; index: CommitteeIndex} | {code: AttestationErrorCode.NOT_EXACTLY_ONE_AGGREGATION_BIT_SET} | {code: AttestationErrorCode.PRIOR_ATTESTATION_KNOWN; validatorIndex: ValidatorIndex; epoch: Epoch} | {code: AttestationErrorCode.FUTURE_EPOCH; attestationEpoch: Epoch; currentEpoch: Epoch} diff --git a/packages/beacon-node/src/chain/interface.ts b/packages/beacon-node/src/chain/interface.ts index 8d6f7f419d7b..7fa60fd76ace 100644 --- a/packages/beacon-node/src/chain/interface.ts +++ b/packages/beacon-node/src/chain/interface.ts @@ -3,6 +3,7 @@ import {allForks, UintNum64, Root, phase0, Slot, RootHex, Epoch, ValidatorIndex, import { BeaconStateAllForks, CachedBeaconStateAllForks, + EpochShuffling, Index2PubkeyCache, PubkeyIndexMap, } from "@lodestar/state-transition"; @@ -36,6 +37,7 @@ import {CheckpointBalancesCache} from "./balancesCache.js"; import {IChainOptions} from "./options.js"; import {AssembledBlockType, BlockAttributes, BlockType} from "./produceBlock/produceBlockBody.js"; import {SeenAttestationDatas} from "./seenCache/seenAttestationData.js"; +import {ShufflingCache} from "./shufflingCache.js"; export {BlockType, type AssembledBlockType}; export {type ProposerPreparationData}; @@ -96,6 +98,7 @@ export interface IBeaconChain { readonly producedBlobSidecarsCache: Map; readonly producedBlockRoot: Map; readonly producedBlindedBlobSidecarsCache: Map; + readonly shufflingCache: ShufflingCache; readonly producedBlindedBlockRoot: Set; readonly opts: IChainOptions; @@ -160,6 +163,12 @@ export interface IBeaconChain { persistInvalidSszBytes(type: string, sszBytes: Uint8Array, suffix?: string): void; /** Persist bad items to persistInvalidSszObjectsDir dir, for example invalid state, attestations etc. */ persistInvalidSszView(view: TreeView, suffix?: string): void; + regenStateForAttestationVerification( + attEpoch: Epoch, + shufflingDependentRoot: RootHex, + attHeadBlock: ProtoBlock, + regenCaller: RegenCaller + ): Promise; updateBuilderStatus(clockSlot: Slot): void; regenCanAcceptWork(): boolean; diff --git a/packages/beacon-node/src/chain/options.ts b/packages/beacon-node/src/chain/options.ts index 9f826d1a2403..ab105a6e3261 100644 --- a/packages/beacon-node/src/chain/options.ts +++ b/packages/beacon-node/src/chain/options.ts @@ -3,12 +3,14 @@ import {defaultOptions as defaultValidatorOptions} from "@lodestar/validator"; import {ArchiverOpts} from "./archiver/index.js"; import {ForkChoiceOpts} from "./forkChoice/index.js"; import {LightClientServerOpts} from "./lightClient/index.js"; +import {ShufflingCacheOpts} from "./shufflingCache.js"; export type IChainOptions = BlockProcessOpts & PoolOpts & SeenCacheOpts & ForkChoiceOpts & ArchiverOpts & + ShufflingCacheOpts & LightClientServerOpts & { blsVerifyAllMainThread?: boolean; blsVerifyAllMultiThread?: boolean; diff --git a/packages/beacon-node/src/chain/shufflingCache.ts b/packages/beacon-node/src/chain/shufflingCache.ts new file mode 100644 index 000000000000..c8468f3b6db5 --- /dev/null +++ b/packages/beacon-node/src/chain/shufflingCache.ts @@ -0,0 +1,193 @@ +import {toHexString} from "@chainsafe/ssz"; +import {CachedBeaconStateAllForks, EpochShuffling, getShufflingDecisionBlock} from "@lodestar/state-transition"; +import {Epoch, RootHex, ssz} from "@lodestar/types"; +import {MapDef, pruneSetToMax} from "@lodestar/utils"; +import {GENESIS_SLOT} from "@lodestar/params"; +import {Metrics} from "../metrics/metrics.js"; +import {computeAnchorCheckpoint} from "./initState.js"; + +/** + * Same value to CheckpointBalancesCache, with the assumption that we don't have to use it for old epochs. In the worse case: + * - when loading state bytes from disk, we need to compute shuffling for all epochs (~1s as of Sep 2023) + * - don't have shuffling to verify attestations, need to do 1 epoch transition to add shuffling to this cache. This never happens + * with default chain option of maxSkipSlots = 32 + **/ +const MAX_EPOCHS = 4; + +/** + * With default chain option of maxSkipSlots = 32, there should be no shuffling promise. If that happens a lot, it could blow up Lodestar, + * with MAX_EPOCHS = 4, only allow 2 promise at a time. Note that regen already bounds number of concurrent requests at 1 already. + */ +const MAX_PROMISES = 2; + +enum CacheItemType { + shuffling, + promise, +} + +type ShufflingCacheItem = { + type: CacheItemType.shuffling; + shuffling: EpochShuffling; +}; + +type PromiseCacheItem = { + type: CacheItemType.promise; + promise: Promise; + resolveFn: (shuffling: EpochShuffling) => void; +}; + +type CacheItem = ShufflingCacheItem | PromiseCacheItem; + +export type ShufflingCacheOpts = { + maxShufflingCacheEpochs?: number; +}; + +/** + * A shuffling cache to help: + * - get committee quickly for attestation verification + * - if a shuffling is not available (which does not happen with default chain option of maxSkipSlots = 32), track a promise to make sure we don't compute the same shuffling twice + * - skip computing shuffling when loading state bytes from disk + */ +export class ShufflingCache { + /** LRU cache implemented as a map, pruned every time we add an item */ + private readonly itemsByDecisionRootByEpoch: MapDef> = new MapDef( + () => new Map() + ); + + private readonly maxEpochs: number; + + constructor( + private readonly metrics: Metrics | null = null, + opts: ShufflingCacheOpts = {} + ) { + if (metrics) { + metrics.shufflingCache.size.addCollect(() => + metrics.shufflingCache.size.set( + Array.from(this.itemsByDecisionRootByEpoch.values()).reduce((total, innerMap) => total + innerMap.size, 0) + ) + ); + } + + this.maxEpochs = opts.maxShufflingCacheEpochs ?? MAX_EPOCHS; + } + + /** + * Extract shuffling from state and add to cache + */ + processState(state: CachedBeaconStateAllForks, shufflingEpoch: Epoch): EpochShuffling { + const decisionBlockHex = getDecisionBlock(state, shufflingEpoch); + let shuffling: EpochShuffling; + switch (shufflingEpoch) { + case state.epochCtx.nextShuffling.epoch: + shuffling = state.epochCtx.nextShuffling; + break; + case state.epochCtx.currentShuffling.epoch: + shuffling = state.epochCtx.currentShuffling; + break; + case state.epochCtx.previousShuffling.epoch: + shuffling = state.epochCtx.previousShuffling; + break; + default: + throw new Error(`Shuffling not found from state ${state.slot} for epoch ${shufflingEpoch}`); + } + + let cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(shufflingEpoch).get(decisionBlockHex); + if (cacheItem !== undefined) { + // update existing promise + if (isPromiseCacheItem(cacheItem)) { + // unblock consumers of this promise + cacheItem.resolveFn(shuffling); + // then update item type to shuffling + cacheItem = { + type: CacheItemType.shuffling, + shuffling, + }; + this.add(shufflingEpoch, decisionBlockHex, cacheItem); + // we updated type to CacheItemType.shuffling so the above fields are not used anyway + this.metrics?.shufflingCache.processStateUpdatePromise.inc(); + } else { + // ShufflingCacheItem, do nothing + this.metrics?.shufflingCache.processStateNoOp.inc(); + } + } else { + // not found, new shuffling + this.add(shufflingEpoch, decisionBlockHex, {type: CacheItemType.shuffling, shuffling}); + this.metrics?.shufflingCache.processStateInsertNew.inc(); + } + + return shuffling; + } + + /** + * Insert a promise to make sure we don't regen state for the same shuffling. + * Bound by MAX_SHUFFLING_PROMISE to make sure our node does not blow up. + */ + insertPromise(shufflingEpoch: Epoch, decisionRootHex: RootHex): void { + const promiseCount = Array.from(this.itemsByDecisionRootByEpoch.values()) + .flatMap((innerMap) => Array.from(innerMap.values())) + .filter((item) => isPromiseCacheItem(item)).length; + if (promiseCount >= MAX_PROMISES) { + throw new Error( + `Too many shuffling promises: ${promiseCount}, shufflingEpoch: ${shufflingEpoch}, decisionRootHex: ${decisionRootHex}` + ); + } + let resolveFn: ((shuffling: EpochShuffling) => void) | null = null; + const promise = new Promise((resolve) => { + resolveFn = resolve; + }); + if (resolveFn === null) { + throw new Error("Promise Constructor was not executed immediately"); + } + + const cacheItem: PromiseCacheItem = { + type: CacheItemType.promise, + promise, + resolveFn, + }; + this.add(shufflingEpoch, decisionRootHex, cacheItem); + this.metrics?.shufflingCache.insertPromiseCount.inc(); + } + + /** + * Most of the time, this should return a shuffling immediately. + * If there's a promise, it means we are computing the same shuffling, so we wait for the promise to resolve. + * Return null if we don't have a shuffling for this epoch and dependentRootHex. + */ + async get(shufflingEpoch: Epoch, decisionRootHex: RootHex): Promise { + const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(shufflingEpoch).get(decisionRootHex); + if (cacheItem === undefined) { + return null; + } + + if (isShufflingCacheItem(cacheItem)) { + return cacheItem.shuffling; + } else { + // promise + return cacheItem.promise; + } + } + + private add(shufflingEpoch: Epoch, decisionBlock: RootHex, cacheItem: CacheItem): void { + this.itemsByDecisionRootByEpoch.getOrDefault(shufflingEpoch).set(decisionBlock, cacheItem); + pruneSetToMax(this.itemsByDecisionRootByEpoch, this.maxEpochs); + } +} + +function isShufflingCacheItem(item: CacheItem): item is ShufflingCacheItem { + return item.type === CacheItemType.shuffling; +} + +function isPromiseCacheItem(item: CacheItem): item is PromiseCacheItem { + return item.type === CacheItemType.promise; +} + +/** + * Get the shuffling decision block root for the given epoch of given state + * - Special case close to genesis block, return the genesis block root + * - This is similar to forkchoice.getDependentRoot() function, otherwise we cannot get cached shuffing in attestation verification when syncing from genesis. + */ +function getDecisionBlock(state: CachedBeaconStateAllForks, epoch: Epoch): RootHex { + return state.slot > GENESIS_SLOT + ? getShufflingDecisionBlock(state, epoch) + : toHexString(ssz.phase0.BeaconBlockHeader.hashTreeRoot(computeAnchorCheckpoint(state.config, state).blockHeader)); +} diff --git a/packages/beacon-node/src/chain/validation/aggregateAndProof.ts b/packages/beacon-node/src/chain/validation/aggregateAndProof.ts index 0cd96a8278ec..5c6a308808ce 100644 --- a/packages/beacon-node/src/chain/validation/aggregateAndProof.ts +++ b/packages/beacon-node/src/chain/validation/aggregateAndProof.ts @@ -4,8 +4,6 @@ import {phase0, RootHex, ssz, ValidatorIndex} from "@lodestar/types"; import { computeEpochAtSlot, isAggregatorFromCommitteeLength, - getIndexedAttestationSignatureSet, - ISignatureSet, createAggregateSignatureSetFromComponents, } from "@lodestar/state-transition"; import {IBeaconChain} from ".."; @@ -14,8 +12,9 @@ import {RegenCaller} from "../regen/index.js"; import {getAttDataBase64FromSignedAggregateAndProofSerialized} from "../../util/sszBytes.js"; import {getSelectionProofSignatureSet, getAggregateAndProofSignatureSet} from "./signatureSets/index.js"; import { + getAttestationDataSigningRoot, getCommitteeIndices, - getStateForAttestationVerification, + getShufflingForAttestationVerification, verifyHeadBlockAndTargetRoot, verifyPropagationSlotRange, } from "./attestation.js"; @@ -142,17 +141,16 @@ async function validateAggregateAndProof( // -- i.e. get_ancestor(store, aggregate.data.beacon_block_root, compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)) == store.finalized_checkpoint.root // > Altready check in `chain.forkChoice.hasBlock(attestation.data.beaconBlockRoot)` - const attHeadState = await getStateForAttestationVerification( + const shuffling = await getShufflingForAttestationVerification( chain, - attSlot, attEpoch, attHeadBlock, - RegenCaller.validateGossipAggregateAndProof + RegenCaller.validateGossipAttestation ); const committeeIndices: number[] = cachedAttData ? cachedAttData.committeeIndices - : getCommitteeIndices(attHeadState, attSlot, attIndex); + : getCommitteeIndices(shuffling, attSlot, attIndex); const attestingIndices = aggregate.aggregationBits.intersectValues(committeeIndices); const indexedAttestation: phase0.IndexedAttestation = { @@ -185,21 +183,16 @@ async function validateAggregateAndProof( // by the validator with index aggregate_and_proof.aggregator_index. // [REJECT] The aggregator signature, signed_aggregate_and_proof.signature, is valid. // [REJECT] The signature of aggregate is valid. - const aggregator = attHeadState.epochCtx.index2pubkey[aggregateAndProof.aggregatorIndex]; - let indexedAttestationSignatureSet: ISignatureSet; - if (cachedAttData) { - const {signingRoot} = cachedAttData; - indexedAttestationSignatureSet = createAggregateSignatureSetFromComponents( - indexedAttestation.attestingIndices.map((i) => chain.index2pubkey[i]), - signingRoot, - indexedAttestation.signature - ); - } else { - indexedAttestationSignatureSet = getIndexedAttestationSignatureSet(attHeadState, indexedAttestation); - } + const aggregator = chain.index2pubkey[aggregateAndProof.aggregatorIndex]; + const signingRoot = cachedAttData ? cachedAttData.signingRoot : getAttestationDataSigningRoot(chain.config, attData); + const indexedAttestationSignatureSet = createAggregateSignatureSetFromComponents( + indexedAttestation.attestingIndices.map((i) => chain.index2pubkey[i]), + signingRoot, + indexedAttestation.signature + ); const signatureSets = [ - getSelectionProofSignatureSet(attHeadState, attSlot, aggregator, signedAggregateAndProof), - getAggregateAndProofSignatureSet(attHeadState, attEpoch, aggregator, signedAggregateAndProof), + getSelectionProofSignatureSet(chain.config, attSlot, aggregator, signedAggregateAndProof), + getAggregateAndProofSignatureSet(chain.config, attEpoch, aggregator, signedAggregateAndProof), indexedAttestationSignatureSet, ]; // no need to write to SeenAttestationDatas diff --git a/packages/beacon-node/src/chain/validation/attestation.ts b/packages/beacon-node/src/chain/validation/attestation.ts index 0b642101f010..31e105911ab4 100644 --- a/packages/beacon-node/src/chain/validation/attestation.ts +++ b/packages/beacon-node/src/chain/validation/attestation.ts @@ -1,16 +1,18 @@ import {toHexString} from "@chainsafe/ssz"; import {phase0, Epoch, Root, Slot, RootHex, ssz} from "@lodestar/types"; import {ProtoBlock} from "@lodestar/fork-choice"; -import {ATTESTATION_SUBNET_COUNT, SLOTS_PER_EPOCH, ForkName, ForkSeq} from "@lodestar/params"; +import {ATTESTATION_SUBNET_COUNT, SLOTS_PER_EPOCH, ForkName, ForkSeq, DOMAIN_BEACON_ATTESTER} from "@lodestar/params"; import { computeEpochAtSlot, - CachedBeaconStateAllForks, - getAttestationDataSigningRoot, createSingleSignatureSetFromComponents, SingleSignatureSet, EpochCacheError, EpochCacheErrorCode, + EpochShuffling, + computeStartSlotAtEpoch, + computeSigningRoot, } from "@lodestar/state-transition"; +import {BeaconConfig} from "@lodestar/config"; import {AttestationError, AttestationErrorCode, GossipAction} from "../errors/index.js"; import {MAXIMUM_GOSSIP_CLOCK_DISPARITY_SEC} from "../../constants/index.js"; import {RegenCaller} from "../regen/index.js"; @@ -24,6 +26,7 @@ import {AttestationDataCacheEntry} from "../seenCache/seenAttestationData.js"; import {sszDeserializeAttestation} from "../../network/gossip/topic.js"; import {Result, wrapError} from "../../util/wrapError.js"; import {IBeaconChain} from "../interface.js"; +import {getShufflingDependentRoot} from "../../util/dependentRoot.js"; export type BatchResult = { results: Result[]; @@ -56,12 +59,6 @@ export type Step0Result = AttestationValidationResult & { validatorIndex: number; }; -/** - * The beacon chain shufflings are designed to provide 1 epoch lookahead - * At each state, we have previous shuffling, current shuffling and next shuffling - */ -const SHUFFLING_LOOK_AHEAD_EPOCHS = 1; - /** * Validate a single gossip attestation, do not prioritize bls signature set */ @@ -359,9 +356,8 @@ async function validateGossipAttestationNoSignatureCheck( // --i.e. get_ancestor(store, attestation.data.beacon_block_root, compute_start_slot_at_epoch(attestation.data.target.epoch)) == attestation.data.target.root // > Altready check in `verifyHeadBlockAndTargetRoot()` - const attHeadState = await getStateForAttestationVerification( + const shuffling = await getShufflingForAttestationVerification( chain, - attSlot, attEpoch, attHeadBlock, RegenCaller.validateGossipAttestation @@ -369,9 +365,9 @@ async function validateGossipAttestationNoSignatureCheck( // [REJECT] The committee index is within the expected range // -- i.e. data.index < get_committee_count_per_slot(state, data.target.epoch) - committeeIndices = getCommitteeIndices(attHeadState, attSlot, attIndex); - getSigningRoot = () => getAttestationDataSigningRoot(attHeadState, attData); - expectedSubnet = attHeadState.epochCtx.computeSubnetForSlot(attSlot, attIndex); + committeeIndices = getCommitteeIndices(shuffling, attSlot, attIndex); + getSigningRoot = () => getAttestationDataSigningRoot(chain.config, attData); + expectedSubnet = computeSubnetForSlot(shuffling, attSlot, attIndex); } const validatorIndex = committeeIndices[bitIndex]; @@ -568,36 +564,46 @@ export function verifyHeadBlockAndTargetRoot( } /** - * Get a state for attestation verification. - * Use head state if: - * - attestation slot is in the same fork as head block - * - head state includes committees of target epoch + * Get a shuffling for attestation verification from the ShufflingCache. + * - if blockEpoch is attEpoch, use current shuffling of head state + * - if blockEpoch is attEpoch - 1, use next shuffling of head state + * - if blockEpoch is less than attEpoch - 1, dial head state to attEpoch - 1, and add to ShufflingCache + * + * This implementation does not require to dial head state to attSlot at fork boundary because we always get domain of attSlot + * in consumer context. * - * Otherwise, regenerate state from head state dialing to target epoch + * This is similar to the old getStateForAttestationVerification + * see https://github.com/ChainSafe/lodestar/blob/v1.11.3/packages/beacon-node/src/chain/validation/attestation.ts#L566 */ -export async function getStateForAttestationVerification( +export async function getShufflingForAttestationVerification( chain: IBeaconChain, - attSlot: Slot, attEpoch: Epoch, attHeadBlock: ProtoBlock, regenCaller: RegenCaller -): Promise { - const isSameFork = chain.config.getForkSeq(attSlot) === chain.config.getForkSeq(attHeadBlock.slot); - // thanks for 1 epoch look ahead of shuffling, a state at epoch n can get committee for epoch n+1 - const headStateHasTargetEpochCommmittee = - attEpoch - computeEpochAtSlot(attHeadBlock.slot) <= SHUFFLING_LOOK_AHEAD_EPOCHS; - try { - if (isSameFork && headStateHasTargetEpochCommmittee) { - // most of the time it should just use head state - chain.metrics?.gossipAttestation.useHeadBlockState.inc({caller: regenCaller}); - return await chain.regen.getState(attHeadBlock.stateRoot, regenCaller); - } +): Promise { + const blockEpoch = computeEpochAtSlot(attHeadBlock.slot); + const shufflingDependentRoot = getShufflingDependentRoot(chain.forkChoice, attEpoch, blockEpoch, attHeadBlock); + + const shuffling = await chain.shufflingCache.get(attEpoch, shufflingDependentRoot); + if (shuffling) { + // most of the time, we should get the shuffling from cache + chain.metrics?.gossipAttestation.shufflingCacheHit.inc({caller: regenCaller}); + return shuffling; + } - // at fork boundary we should dial head state to target epoch - // see https://github.com/ChainSafe/lodestar/pull/4849 - chain.metrics?.gossipAttestation.useHeadBlockStateDialedToTargetEpoch.inc({caller: regenCaller}); - return await chain.regen.getBlockSlotState(attHeadBlock.blockRoot, attSlot, {dontTransferCache: true}, regenCaller); + chain.metrics?.gossipAttestation.shufflingCacheMiss.inc({caller: regenCaller}); + try { + // for the 1st time of the same epoch and dependent root, it awaits for the regen state + // from the 2nd time, it should use the same cached promise and it should reach the above code + chain.metrics?.gossipAttestation.shufflingCacheRegenHit.inc({caller: regenCaller}); + return await chain.regenStateForAttestationVerification( + attEpoch, + shufflingDependentRoot, + attHeadBlock, + regenCaller + ); } catch (e) { + chain.metrics?.gossipAttestation.shufflingCacheRegenMiss.inc({caller: regenCaller}); throw new AttestationError(GossipAction.IGNORE, { code: AttestationErrorCode.MISSING_STATE_TO_VERIFY_ATTESTATION, error: e as Error, @@ -605,6 +611,19 @@ export async function getStateForAttestationVerification( } } +/** + * Different version of getAttestationDataSigningRoot in state-transition which doesn't require a state. + */ +export function getAttestationDataSigningRoot(config: BeaconConfig, data: phase0.AttestationData): Uint8Array { + const slot = computeStartSlotAtEpoch(data.target.epoch); + // previously, we call `domain = config.getDomain(state.slot, DOMAIN_BEACON_ATTESTER, slot)` + // at fork boundary, it's required to dial to target epoch https://github.com/ChainSafe/lodestar/blob/v1.11.3/packages/beacon-node/src/chain/validation/attestation.ts#L573 + // instead of that, just use the fork at slot in the attestation data + const fork = config.getForkName(slot); + const domain = config.getDomainAtFork(fork, DOMAIN_BEACON_ATTESTER); + return computeSigningRoot(ssz.phase0.AttestationData, data, domain); +} + /** * Checks if the `attestation.data.beaconBlockRoot` is known to this chain. * @@ -680,21 +699,10 @@ function verifyAttestationTargetRoot(headBlock: ProtoBlock, targetRoot: Root, at } export function getCommitteeIndices( - attestationTargetState: CachedBeaconStateAllForks, + shuffling: EpochShuffling, attestationSlot: Slot, attestationIndex: number ): number[] { - const shuffling = attestationTargetState.epochCtx.getShufflingAtSlotOrNull(attestationSlot); - if (shuffling === null) { - // this may come from an out-of-synced node, the spec did not define it so should not REJECT - // see https://github.com/ChainSafe/lodestar/issues/4396 - throw new AttestationError(GossipAction.IGNORE, { - code: AttestationErrorCode.NO_COMMITTEE_FOR_SLOT_AND_INDEX, - index: attestationIndex, - slot: attestationSlot, - }); - } - const {committees} = shuffling; const slotCommittees = committees[attestationSlot % SLOTS_PER_EPOCH]; @@ -710,9 +718,8 @@ export function getCommitteeIndices( /** * Compute the correct subnet for a slot/committee index */ -export function computeSubnetForSlot(state: CachedBeaconStateAllForks, slot: number, committeeIndex: number): number { +export function computeSubnetForSlot(shuffling: EpochShuffling, slot: number, committeeIndex: number): number { const slotsSinceEpochStart = slot % SLOTS_PER_EPOCH; - const committeesPerSlot = state.epochCtx.getCommitteeCountPerSlot(computeEpochAtSlot(slot)); - const committeesSinceEpochStart = committeesPerSlot * slotsSinceEpochStart; + const committeesSinceEpochStart = shuffling.committeesPerSlot * slotsSinceEpochStart; return (committeesSinceEpochStart + committeeIndex) % ATTESTATION_SUBNET_COUNT; } diff --git a/packages/beacon-node/src/chain/validation/signatureSets/aggregateAndProof.ts b/packages/beacon-node/src/chain/validation/signatureSets/aggregateAndProof.ts index 099590ee019e..2bc2e62c861f 100644 --- a/packages/beacon-node/src/chain/validation/signatureSets/aggregateAndProof.ts +++ b/packages/beacon-node/src/chain/validation/signatureSets/aggregateAndProof.ts @@ -3,32 +3,36 @@ import {DOMAIN_AGGREGATE_AND_PROOF} from "@lodestar/params"; import {ssz} from "@lodestar/types"; import {Epoch, phase0} from "@lodestar/types"; import { - CachedBeaconStateAllForks, computeSigningRoot, computeStartSlotAtEpoch, createSingleSignatureSetFromComponents, ISignatureSet, } from "@lodestar/state-transition"; +import {BeaconConfig} from "@lodestar/config"; export function getAggregateAndProofSigningRoot( - state: CachedBeaconStateAllForks, + config: BeaconConfig, epoch: Epoch, aggregateAndProof: phase0.SignedAggregateAndProof ): Uint8Array { + // previously, we call `const aggregatorDomain = state.config.getDomain(state.slot, DOMAIN_AGGREGATE_AND_PROOF, slot);` + // at fork boundary, it's required to dial to target epoch https://github.com/ChainSafe/lodestar/blob/v1.11.3/packages/beacon-node/src/chain/validation/attestation.ts#L573 + // instead of that, just use the fork of slot in the attestation data const slot = computeStartSlotAtEpoch(epoch); - const aggregatorDomain = state.config.getDomain(state.slot, DOMAIN_AGGREGATE_AND_PROOF, slot); + const fork = config.getForkName(slot); + const aggregatorDomain = config.getDomainAtFork(fork, DOMAIN_AGGREGATE_AND_PROOF); return computeSigningRoot(ssz.phase0.AggregateAndProof, aggregateAndProof.message, aggregatorDomain); } export function getAggregateAndProofSignatureSet( - state: CachedBeaconStateAllForks, + config: BeaconConfig, epoch: Epoch, aggregator: PublicKey, aggregateAndProof: phase0.SignedAggregateAndProof ): ISignatureSet { return createSingleSignatureSetFromComponents( aggregator, - getAggregateAndProofSigningRoot(state, epoch, aggregateAndProof), + getAggregateAndProofSigningRoot(config, epoch, aggregateAndProof), aggregateAndProof.signature ); } diff --git a/packages/beacon-node/src/chain/validation/signatureSets/selectionProof.ts b/packages/beacon-node/src/chain/validation/signatureSets/selectionProof.ts index dbb8e3380606..09e0a5ef12be 100644 --- a/packages/beacon-node/src/chain/validation/signatureSets/selectionProof.ts +++ b/packages/beacon-node/src/chain/validation/signatureSets/selectionProof.ts @@ -1,27 +1,27 @@ import type {PublicKey} from "@chainsafe/bls/types"; import {DOMAIN_SELECTION_PROOF} from "@lodestar/params"; import {phase0, Slot, ssz} from "@lodestar/types"; -import { - CachedBeaconStateAllForks, - computeSigningRoot, - createSingleSignatureSetFromComponents, - ISignatureSet, -} from "@lodestar/state-transition"; +import {computeSigningRoot, createSingleSignatureSetFromComponents, ISignatureSet} from "@lodestar/state-transition"; +import {BeaconConfig} from "@lodestar/config"; -export function getSelectionProofSigningRoot(state: CachedBeaconStateAllForks, slot: Slot): Uint8Array { - const selectionProofDomain = state.config.getDomain(state.slot, DOMAIN_SELECTION_PROOF, slot); +export function getSelectionProofSigningRoot(config: BeaconConfig, slot: Slot): Uint8Array { + // previously, we call `const selectionProofDomain = config.getDomain(state.slot, DOMAIN_SELECTION_PROOF, slot)` + // at fork boundary, it's required to dial to target epoch https://github.com/ChainSafe/lodestar/blob/v1.11.3/packages/beacon-node/src/chain/validation/attestation.ts#L573 + // instead of that, just use the fork of slot in the attestation data + const fork = config.getForkName(slot); + const selectionProofDomain = config.getDomainAtFork(fork, DOMAIN_SELECTION_PROOF); return computeSigningRoot(ssz.Slot, slot, selectionProofDomain); } export function getSelectionProofSignatureSet( - state: CachedBeaconStateAllForks, + config: BeaconConfig, slot: Slot, aggregator: PublicKey, aggregateAndProof: phase0.SignedAggregateAndProof ): ISignatureSet { return createSingleSignatureSetFromComponents( aggregator, - getSelectionProofSigningRoot(state, slot), + getSelectionProofSigningRoot(config, slot), aggregateAndProof.message.selectionProof ); } diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index b6f0d78f3b06..a68fdae0551f 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -590,6 +590,26 @@ export function createLodestarMetrics( labelNames: ["caller"], buckets: [0, 1, 2, 4, 8, 16, 32, 64], }), + shufflingCacheHit: register.gauge<"caller">({ + name: "lodestar_gossip_attestation_shuffling_cache_hit_count", + help: "Count of gossip attestation verification shuffling cache hit", + labelNames: ["caller"], + }), + shufflingCacheMiss: register.gauge<"caller">({ + name: "lodestar_gossip_attestation_shuffling_cache_miss_count", + help: "Count of gossip attestation verification shuffling cache miss", + labelNames: ["caller"], + }), + shufflingCacheRegenHit: register.gauge<"caller">({ + name: "lodestar_gossip_attestation_shuffling_cache_regen_hit_count", + help: "Count of gossip attestation verification shuffling cache regen hit", + labelNames: ["caller"], + }), + shufflingCacheRegenMiss: register.gauge<"caller">({ + name: "lodestar_gossip_attestation_shuffling_cache_regen_miss_count", + help: "Count of gossip attestation verification shuffling cache regen miss", + labelNames: ["caller"], + }), attestationSlotToClockSlot: register.histogram<"caller">({ name: "lodestar_gossip_attestation_attestation_slot_to_clock_slot", help: "Slot distance between clock slot and attestation slot", @@ -1072,6 +1092,29 @@ export function createLodestarMetrics( }), }, + shufflingCache: { + size: register.gauge({ + name: "lodestar_shuffling_cache_size", + help: "Shuffling cache size", + }), + processStateInsertNew: register.gauge({ + name: "lodestar_shuffling_cache_process_state_insert_new_total", + help: "Total number of times processState is called resulting a new shuffling", + }), + processStateUpdatePromise: register.gauge({ + name: "lodestar_shuffling_cache_process_state_update_promise_total", + help: "Total number of times processState is called resulting a promise being updated with shuffling", + }), + processStateNoOp: register.gauge({ + name: "lodestar_shuffling_cache_process_state_no_op_total", + help: "Total number of times processState is called resulting no changes", + }), + insertPromiseCount: register.gauge({ + name: "lodestar_shuffling_cache_insert_promise_count", + help: "Total number of times insertPromise is called", + }), + }, + seenCache: { aggregatedAttestations: { superSetCheckTotal: register.histogram({ diff --git a/packages/beacon-node/src/util/dependentRoot.ts b/packages/beacon-node/src/util/dependentRoot.ts new file mode 100644 index 000000000000..58b26c30b872 --- /dev/null +++ b/packages/beacon-node/src/util/dependentRoot.ts @@ -0,0 +1,47 @@ +import {EpochDifference, IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; +import {Epoch, RootHex} from "@lodestar/types"; + +/** + * Get dependent root of a shuffling given attestation epoch and head block. + */ +export function getShufflingDependentRoot( + forkChoice: IForkChoice, + attEpoch: Epoch, + blockEpoch: Epoch, + attHeadBlock: ProtoBlock +): RootHex { + let shufflingDependentRoot: RootHex; + if (blockEpoch === attEpoch) { + // current shuffling, this is equivalent to `headState.currentShuffling` + // given blockEpoch = attEpoch = n + // epoch: (n-2) (n-1) n (n+1) + // |-------|-------|-------|-------| + // attHeadBlock ------------------------^ + // shufflingDependentRoot ------^ + shufflingDependentRoot = forkChoice.getDependentRoot(attHeadBlock, EpochDifference.previous); + } else if (blockEpoch === attEpoch - 1) { + // next shuffling, this is equivalent to `headState.nextShuffling` + // given blockEpoch = n-1, attEpoch = n + // epoch: (n-2) (n-1) n (n+1) + // |-------|-------|-------|-------| + // attHeadBlock -------------------^ + // shufflingDependentRoot ------^ + shufflingDependentRoot = forkChoice.getDependentRoot(attHeadBlock, EpochDifference.current); + } else if (blockEpoch < attEpoch - 1) { + // this never happens with default chain option of maxSkipSlots = 32, however we still need to handle it + // check the verifyHeadBlockAndTargetRoot() function above + // given blockEpoch = n-2, attEpoch = n + // epoch: (n-2) (n-1) n (n+1) + // |-------|-------|-------|-------| + // attHeadBlock -----------^ + // shufflingDependentRoot -----^ + shufflingDependentRoot = attHeadBlock.blockRoot; + // use lodestar_gossip_attestation_head_slot_to_attestation_slot metric to track this case + } else { + // blockEpoch > attEpoch + // should not happen, handled in verifyAttestationTargetRoot + throw Error(`attestation epoch ${attEpoch} is before head block epoch ${blockEpoch}`); + } + + return shufflingDependentRoot; +} diff --git a/packages/beacon-node/test/__mocks__/mockedBeaconChain.ts b/packages/beacon-node/test/__mocks__/mockedBeaconChain.ts index 6e8056ea7458..3c5dacc9c971 100644 --- a/packages/beacon-node/test/__mocks__/mockedBeaconChain.ts +++ b/packages/beacon-node/test/__mocks__/mockedBeaconChain.ts @@ -13,6 +13,7 @@ import {BeaconProposerCache} from "../../src/chain/beaconProposerCache.js"; import {QueuedStateRegenerator} from "../../src/chain/regen/index.js"; import {LightClientServer} from "../../src/chain/lightClient/index.js"; import {Clock} from "../../src/util/clock.js"; +import {ShufflingCache} from "../../src/chain/shufflingCache.js"; import {getMockedLogger} from "./loggerMock.js"; export type MockedBeaconChain = MockedObject & { @@ -24,6 +25,7 @@ export type MockedBeaconChain = MockedObject & { opPool: MockedObject; aggregatedAttestationPool: MockedObject; beaconProposerCache: MockedObject; + shufflingCache: MockedObject; regen: MockedObject; bls: { verifySignatureSets: Mock<[boolean]>; @@ -40,6 +42,7 @@ vi.mock("../../src/eth1/index.js"); vi.mock("../../src/chain/opPools/opPool.js"); vi.mock("../../src/chain/opPools/aggregatedAttestationPool.js"); vi.mock("../../src/chain/beaconProposerCache.js"); +vi.mock("../../src/chain/shufflingCache.js"); vi.mock("../../src/chain/regen/index.js"); vi.mock("../../src/chain/lightClient/index.js"); vi.mock("../../src/chain/index.js", async (requireActual) => { @@ -75,6 +78,7 @@ vi.mock("../../src/chain/index.js", async (requireActual) => { // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error beaconProposerCache: new BeaconProposerCache(), + shufflingCache: new ShufflingCache(), produceBlock: vi.fn(), produceBlindedBlock: vi.fn(), getCanonicalBlockAtSlot: vi.fn(), @@ -83,6 +87,7 @@ vi.mock("../../src/chain/index.js", async (requireActual) => { getHeadState: vi.fn(), updateBuilderStatus: vi.fn(), processBlock: vi.fn(), + regenStateForAttestationVerification: vi.fn(), close: vi.fn(), logger: getMockedLogger(), regen: new QueuedStateRegenerator({} as any), diff --git a/packages/beacon-node/test/unit/chain/shufflingCache.test.ts b/packages/beacon-node/test/unit/chain/shufflingCache.test.ts new file mode 100644 index 000000000000..186739ff2475 --- /dev/null +++ b/packages/beacon-node/test/unit/chain/shufflingCache.test.ts @@ -0,0 +1,54 @@ +import {describe, it, expect, beforeEach} from "vitest"; + +import {getShufflingDecisionBlock} from "@lodestar/state-transition"; +// eslint-disable-next-line import/no-relative-packages +import {generateTestCachedBeaconStateOnlyValidators} from "../../../../state-transition/test/perf/util.js"; +import {ShufflingCache} from "../../../src/chain/shufflingCache.js"; + +describe("ShufflingCache", function () { + const vc = 64; + const stateSlot = 100; + const state = generateTestCachedBeaconStateOnlyValidators({vc, slot: stateSlot}); + const currentEpoch = state.epochCtx.currentShuffling.epoch; + let shufflingCache: ShufflingCache; + + beforeEach(() => { + shufflingCache = new ShufflingCache(null, {maxShufflingCacheEpochs: 1}); + shufflingCache.processState(state, currentEpoch); + }); + + it("should get shuffling from cache", async function () { + const decisionRoot = getShufflingDecisionBlock(state, currentEpoch); + expect(await shufflingCache.get(currentEpoch, decisionRoot)).to.deep.equal(state.epochCtx.currentShuffling); + }); + + it("should bound by maxSize(=1)", async function () { + const decisionRoot = getShufflingDecisionBlock(state, currentEpoch); + expect(await shufflingCache.get(currentEpoch, decisionRoot)).to.deep.equal(state.epochCtx.currentShuffling); + // insert promises at the same epoch does not prune the cache + shufflingCache.insertPromise(currentEpoch, "0x00"); + expect(await shufflingCache.get(currentEpoch, decisionRoot)).to.deep.equal(state.epochCtx.currentShuffling); + // insert shufflings at other epochs does prune the cache + shufflingCache.processState(state, currentEpoch + 1); + // the current shuffling is not available anymore + expect(await shufflingCache.get(currentEpoch, decisionRoot)).to.be.null; + }); + + it("should return shuffling from promise", async function () { + const nextDecisionRoot = getShufflingDecisionBlock(state, currentEpoch + 1); + shufflingCache.insertPromise(currentEpoch + 1, nextDecisionRoot); + const shufflingRequest0 = shufflingCache.get(currentEpoch + 1, nextDecisionRoot); + const shufflingRequest1 = shufflingCache.get(currentEpoch + 1, nextDecisionRoot); + shufflingCache.processState(state, currentEpoch + 1); + expect(await shufflingRequest0).to.deep.equal(state.epochCtx.nextShuffling); + expect(await shufflingRequest1).to.deep.equal(state.epochCtx.nextShuffling); + }); + + it("should support up to 2 promises at a time", async function () { + // insert 2 promises at the same epoch + shufflingCache.insertPromise(currentEpoch, "0x00"); + shufflingCache.insertPromise(currentEpoch, "0x01"); + // inserting other promise should throw error + expect(() => shufflingCache.insertPromise(currentEpoch, "0x02")).to.throw(); + }); +}); diff --git a/packages/beacon-node/test/unit/chain/validation/aggregateAndProof.test.ts b/packages/beacon-node/test/unit/chain/validation/aggregateAndProof.test.ts index 99b3783574e9..f614ec10551d 100644 --- a/packages/beacon-node/test/unit/chain/validation/aggregateAndProof.test.ts +++ b/packages/beacon-node/test/unit/chain/validation/aggregateAndProof.test.ts @@ -2,7 +2,6 @@ import {toHexString} from "@chainsafe/ssz"; import {describe, it} from "vitest"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {phase0, ssz} from "@lodestar/types"; -import {processSlots} from "@lodestar/state-transition"; // eslint-disable-next-line import/no-relative-packages import {generateTestCachedBeaconStateOnlyValidators} from "../../../../../state-transition/test/perf/util.js"; import {IBeaconChain} from "../../../../src/chain/index.js"; @@ -14,7 +13,6 @@ import { getAggregateAndProofValidData, AggregateAndProofValidDataOpts, } from "../../../utils/validationData/aggregateAndProof.js"; -import {IStateRegenerator} from "../../../../src/chain/regen/interface.js"; describe("chain / validation / aggregateAndProof", () => { const vc = 64; @@ -112,22 +110,6 @@ describe("chain / validation / aggregateAndProof", () => { await expectError(chain, signedAggregateAndProof, AttestationErrorCode.INVALID_TARGET_ROOT); }); - it("NO_COMMITTEE_FOR_SLOT_AND_INDEX", async () => { - const {chain, signedAggregateAndProof} = getValidData(); - // slot is out of the commitee range - // simulate https://github.com/ChainSafe/lodestar/issues/4396 - // this way we cannot get committeeIndices - const committeeState = processSlots( - getState(), - signedAggregateAndProof.message.aggregate.data.slot + 2 * SLOTS_PER_EPOCH - ); - (chain as {regen: IStateRegenerator}).regen = { - getState: async () => committeeState, - } as Partial as IStateRegenerator; - - await expectError(chain, signedAggregateAndProof, AttestationErrorCode.NO_COMMITTEE_FOR_SLOT_AND_INDEX); - }); - it("EMPTY_AGGREGATION_BITFIELD", async () => { const {chain, signedAggregateAndProof} = getValidData(); // Unset all aggregationBits diff --git a/packages/beacon-node/test/unit/chain/validation/attestation.test.ts b/packages/beacon-node/test/unit/chain/validation/attestation.test.ts index 8d7394ecc4ed..efd7d3c00cbb 100644 --- a/packages/beacon-node/test/unit/chain/validation/attestation.test.ts +++ b/packages/beacon-node/test/unit/chain/validation/attestation.test.ts @@ -3,11 +3,9 @@ import type {PublicKey, SecretKey} from "@chainsafe/bls/types"; import bls from "@chainsafe/bls"; import {describe, it, expect, beforeEach, afterEach, vi} from "vitest"; import {ForkName, SLOTS_PER_EPOCH} from "@lodestar/params"; -import {defaultChainConfig, createChainForkConfig} from "@lodestar/config"; -import {ProtoBlock} from "@lodestar/fork-choice"; -// eslint-disable-next-line import/no-relative-packages -import {SignatureSetType, computeEpochAtSlot, computeStartSlotAtEpoch, processSlots} from "@lodestar/state-transition"; -import {Slot, ssz} from "@lodestar/types"; +import {EpochDifference, ProtoBlock} from "@lodestar/fork-choice"; +import {EpochShuffling, SignatureSetType, computeStartSlotAtEpoch} from "@lodestar/state-transition"; +import {ssz} from "@lodestar/types"; // eslint-disable-next-line import/no-relative-packages import {generateTestCachedBeaconStateOnlyValidators} from "../../../../../state-transition/test/perf/util.js"; import {IBeaconChain} from "../../../../src/chain/index.js"; @@ -20,17 +18,16 @@ import { import { ApiAttestation, GossipAttestation, - getStateForAttestationVerification, validateApiAttestation, Step0Result, validateAttestation, validateGossipAttestationsSameAttData, + getShufflingForAttestationVerification, } from "../../../../src/chain/validation/index.js"; import {expectRejectedWithLodestarError} from "../../../utils/errors.js"; import {memoOnce} from "../../../utils/cache.js"; import {getAttestationValidData, AttestationValidDataOpts} from "../../../utils/validationData/attestation.js"; -import {IStateRegenerator, RegenCaller} from "../../../../src/chain/regen/interface.js"; -import {StateRegenerator} from "../../../../src/chain/regen/regen.js"; +import {RegenCaller} from "../../../../src/chain/regen/interface.js"; import {ZERO_HASH_HEX} from "../../../../src/constants/constants.js"; import {BlsSingleThreadVerifier} from "../../../../src/chain/bls/singleThread.js"; @@ -343,35 +340,6 @@ describe("validateAttestation", () => { ); }); - it("NO_COMMITTEE_FOR_SLOT_AND_INDEX", async () => { - const {chain, attestation, subnet} = getValidData(); - // slot is out of the commitee range - // simulate https://github.com/ChainSafe/lodestar/issues/4396 - // this way we cannot get committeeIndices - const committeeState = processSlots(getState(), attestation.data.slot + 2 * SLOTS_PER_EPOCH); - (chain as {regen: IStateRegenerator}).regen = { - getState: async () => committeeState, - } as Partial as IStateRegenerator; - const serializedData = ssz.phase0.Attestation.serialize(attestation); - - await expectApiError( - chain, - {attestation, serializedData: null}, - AttestationErrorCode.NO_COMMITTEE_FOR_SLOT_AND_INDEX - ); - await expectGossipError( - chain, - { - attestation: null, - serializedData, - attSlot: attestation.data.slot, - attDataBase64: getAttDataBase64FromAttestationSerialized(serializedData), - }, - subnet, - AttestationErrorCode.NO_COMMITTEE_FOR_SLOT_AND_INDEX - ); - }); - it("WRONG_NUMBER_OF_AGGREGATION_BITS", async () => { const {chain, attestation, subnet} = getValidData(); // Increase the length of aggregationBits beyond the committee size @@ -481,15 +449,17 @@ describe("validateAttestation", () => { } }); -describe("getStateForAttestationVerification", () => { - // eslint-disable-next-line @typescript-eslint/naming-convention - const config = createChainForkConfig({...defaultChainConfig, CAPELLA_FORK_EPOCH: 2}); +describe("getShufflingForAttestationVerification", () => { let regenStub: MockedBeaconChain["regen"]; + let forkchoiceStub: MockedBeaconChain["forkChoice"]; + let shufflingCacheStub: MockedBeaconChain["shufflingCache"]; let chain: MockedBeaconChain; beforeEach(() => { chain = getMockedBeaconChain(); regenStub = chain.regen; + forkchoiceStub = chain.forkChoice; + shufflingCacheStub = chain.shufflingCache; vi.spyOn(regenStub, "getBlockSlotState"); vi.spyOn(regenStub, "getState"); }); @@ -498,52 +468,124 @@ describe("getStateForAttestationVerification", () => { vi.clearAllMocks(); }); - const forkSlot = computeStartSlotAtEpoch(config.CAPELLA_FORK_EPOCH); - const getBlockSlotStateTestCases: {id: string; attSlot: Slot; headSlot: Slot; regenCall: keyof StateRegenerator}[] = [ - // TODO: This case is not passing inspect later - // { - // id: "should call regen.getBlockSlotState at fork boundary", - // attSlot: forkSlot + 1, - // headSlot: forkSlot - 1, - // regenCall: "getBlockSlotState", - // }, - { - id: "should call regen.getBlockSlotState if > 1 epoch difference", - attSlot: forkSlot + 2 * SLOTS_PER_EPOCH, - headSlot: forkSlot + 1, - regenCall: "getBlockSlotState", - }, - { - id: "should call getState if 1 epoch difference", - attSlot: forkSlot + 2 * SLOTS_PER_EPOCH, - headSlot: forkSlot + SLOTS_PER_EPOCH, - regenCall: "getState", - }, - { - id: "should call getState if 0 epoch difference", - attSlot: forkSlot + 2 * SLOTS_PER_EPOCH, - headSlot: forkSlot + 2 * SLOTS_PER_EPOCH, - regenCall: "getState", - }, - ]; + const attEpoch = 1000; + const blockRoot = "0xd76aed834b4feef32efb53f9076e407c0d344cfdb70f0a770fa88416f70d304d"; + + it("block epoch is the same to attestation epoch", async () => { + const headSlot = computeStartSlotAtEpoch(attEpoch); + const attHeadBlock = { + slot: headSlot, + stateRoot: ZERO_HASH_HEX, + blockRoot, + } as Partial as ProtoBlock; + const previousDependentRoot = "0xa916b57729dbfb89a082820e0eb2b669d9d511a675d3d8c888b2f300f10b0bdf"; + forkchoiceStub.getDependentRoot.mockImplementationOnce((block, epochDiff) => { + if (block === attHeadBlock && epochDiff === EpochDifference.previous) { + return previousDependentRoot; + } else { + throw new Error("Unexpected input"); + } + }); + const expectedShuffling = {epoch: attEpoch} as EpochShuffling; + shufflingCacheStub.get.mockImplementationOnce((epoch, root) => { + if (epoch === attEpoch && root === previousDependentRoot) { + return Promise.resolve(expectedShuffling); + } else { + return Promise.resolve(null); + } + }); + const resultShuffling = await getShufflingForAttestationVerification( + chain, + attEpoch, + attHeadBlock, + RegenCaller.validateGossipAttestation + ); + expect(resultShuffling).to.be.deep.equal(expectedShuffling); + }); + + it("block epoch is previous attestation epoch", async () => { + const headSlot = computeStartSlotAtEpoch(attEpoch - 1); + const attHeadBlock = { + slot: headSlot, + stateRoot: ZERO_HASH_HEX, + blockRoot, + } as Partial as ProtoBlock; + const currentDependentRoot = "0xa916b57729dbfb89a082820e0eb2b669d9d511a675d3d8c888b2f300f10b0bdf"; + forkchoiceStub.getDependentRoot.mockImplementationOnce((block, epochDiff) => { + if (block === attHeadBlock && epochDiff === EpochDifference.current) { + return currentDependentRoot; + } else { + throw new Error("Unexpected input"); + } + }); + const expectedShuffling = {epoch: attEpoch} as EpochShuffling; + shufflingCacheStub.get.mockImplementationOnce((epoch, root) => { + if (epoch === attEpoch && root === currentDependentRoot) { + return Promise.resolve(expectedShuffling); + } else { + return Promise.resolve(null); + } + }); + const resultShuffling = await getShufflingForAttestationVerification( + chain, + attEpoch, + attHeadBlock, + RegenCaller.validateGossipAttestation + ); + expect(resultShuffling).to.be.deep.equal(expectedShuffling); + }); - for (const {id, attSlot, headSlot, regenCall} of getBlockSlotStateTestCases) { - it(id, async () => { - const attEpoch = computeEpochAtSlot(attSlot); - const attHeadBlock = { - slot: headSlot, - stateRoot: ZERO_HASH_HEX, - blockRoot: ZERO_HASH_HEX, - } as Partial as ProtoBlock; - expect(regenStub[regenCall]).toBeCalledTimes(0); - await getStateForAttestationVerification( + it("block epoch is attestation epoch - 2", async () => { + const headSlot = computeStartSlotAtEpoch(attEpoch - 2); + const attHeadBlock = { + slot: headSlot, + stateRoot: ZERO_HASH_HEX, + blockRoot, + } as Partial as ProtoBlock; + const expectedShuffling = {epoch: attEpoch} as EpochShuffling; + let callCount = 0; + shufflingCacheStub.get.mockImplementationOnce((epoch, root) => { + if (epoch === attEpoch && root === blockRoot) { + if (callCount === 0) { + callCount++; + return Promise.resolve(null); + } else { + return Promise.resolve(expectedShuffling); + } + } else { + return Promise.resolve(null); + } + }); + chain.regenStateForAttestationVerification.mockImplementationOnce(() => Promise.resolve(expectedShuffling)); + + const resultShuffling = await getShufflingForAttestationVerification( + chain, + attEpoch, + attHeadBlock, + RegenCaller.validateGossipAttestation + ); + // sandbox.assert.notCalled(forkchoiceStub.getDependentRoot); + expect(forkchoiceStub.getDependentRoot).not.toHaveBeenCalledTimes(1); + expect(resultShuffling).to.be.deep.equal(expectedShuffling); + }); + + it("block epoch is attestation epoch + 1", async () => { + const headSlot = computeStartSlotAtEpoch(attEpoch + 1); + const attHeadBlock = { + slot: headSlot, + stateRoot: ZERO_HASH_HEX, + blockRoot, + } as Partial as ProtoBlock; + try { + await getShufflingForAttestationVerification( chain, - attSlot, attEpoch, attHeadBlock, RegenCaller.validateGossipAttestation ); - expect(regenStub[regenCall]).toBeCalledTimes(1); - }); - } + expect.fail("Expect error because attestation epoch is greater than block epoch"); + } catch (e) { + expect(e instanceof Error).to.be.true; + } + }); }); diff --git a/packages/beacon-node/test/unit/util/dependentRoot.test.ts b/packages/beacon-node/test/unit/util/dependentRoot.test.ts new file mode 100644 index 000000000000..e2f11acc3eba --- /dev/null +++ b/packages/beacon-node/test/unit/util/dependentRoot.test.ts @@ -0,0 +1,67 @@ +import {describe, it, expect, beforeEach, afterEach, vi} from "vitest"; +import {EpochDifference, ProtoBlock} from "@lodestar/fork-choice"; +import {computeEpochAtSlot} from "@lodestar/state-transition"; +import {getShufflingDependentRoot} from "../../../src/util/dependentRoot.js"; +import {MockedBeaconChain, getMockedBeaconChain} from "../../__mocks__/mockedBeaconChain.js"; + +describe("util / getShufflingDependentRoot", () => { + let forkchoiceStub: MockedBeaconChain["forkChoice"]; + + const headBattHeadBlock = { + slot: 100, + } as ProtoBlock; + const blockEpoch = computeEpochAtSlot(headBattHeadBlock.slot); + + beforeEach(() => { + forkchoiceStub = getMockedBeaconChain().forkChoice; + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + it("should return current dependent root", () => { + const attEpoch = blockEpoch; + forkchoiceStub.getDependentRoot.mockImplementation((block, epochDiff) => { + if (block === headBattHeadBlock && epochDiff === EpochDifference.previous) { + return "current"; + } else { + throw new Error("should not be called"); + } + }); + expect(getShufflingDependentRoot(forkchoiceStub, attEpoch, blockEpoch, headBattHeadBlock)).to.be.equal("current"); + }); + + it("should return next dependent root", () => { + const attEpoch = blockEpoch + 1; + // forkchoiceStub.getDependentRoot.withArgs(headBattHeadBlock, EpochDifference.current).returns("previous"); + forkchoiceStub.getDependentRoot.mockImplementation((block, epochDiff) => { + if (block === headBattHeadBlock && epochDiff === EpochDifference.current) { + return "0x000"; + } else { + throw new Error("should not be called"); + } + }); + expect(getShufflingDependentRoot(forkchoiceStub, attEpoch, blockEpoch, headBattHeadBlock)).to.be.equal("0x000"); + }); + + it("should return head block root as dependent root", () => { + const attEpoch = blockEpoch + 2; + // forkchoiceStub.getDependentRoot.throws("should not be called"); + forkchoiceStub.getDependentRoot.mockImplementation(() => { + throw Error("should not be called"); + }); + expect(getShufflingDependentRoot(forkchoiceStub, attEpoch, blockEpoch, headBattHeadBlock)).to.be.equal( + headBattHeadBlock.blockRoot + ); + }); + + it("should throw error if attestation epoch is before head block epoch", () => { + const attEpoch = blockEpoch - 1; + // forkchoiceStub.getDependentRoot.throws("should not be called"); + forkchoiceStub.getDependentRoot.mockImplementation(() => { + throw Error("should not be called"); + }); + expect(() => getShufflingDependentRoot(forkchoiceStub, attEpoch, blockEpoch, headBattHeadBlock)).to.throw(); + }); +}); diff --git a/packages/beacon-node/test/utils/validationData/attestation.ts b/packages/beacon-node/test/utils/validationData/attestation.ts index 6f768227e5cd..fa3c4d479ade 100644 --- a/packages/beacon-node/test/utils/validationData/attestation.ts +++ b/packages/beacon-node/test/utils/validationData/attestation.ts @@ -1,10 +1,13 @@ import {BitArray, toHexString} from "@chainsafe/ssz"; -import {computeEpochAtSlot, computeSigningRoot, computeStartSlotAtEpoch} from "@lodestar/state-transition"; +import { + computeEpochAtSlot, + computeSigningRoot, + computeStartSlotAtEpoch, + getShufflingDecisionBlock, +} from "@lodestar/state-transition"; import {ProtoBlock, IForkChoice, ExecutionStatus} from "@lodestar/fork-choice"; import {DOMAIN_BEACON_ATTESTER} from "@lodestar/params"; import {phase0, Slot, ssz} from "@lodestar/types"; -import {config} from "@lodestar/config/default"; -import {BeaconConfig} from "@lodestar/config"; import { generateTestCachedBeaconStateOnlyValidators, getSecretKeyFromIndexCached, @@ -21,6 +24,7 @@ import {SeenAggregatedAttestations} from "../../../src/chain/seenCache/seenAggre import {SeenAttestationDatas} from "../../../src/chain/seenCache/seenAttestationData.js"; import {defaultChainOptions} from "../../../src/chain/options.js"; import {testLogger} from "../logger.js"; +import {ShufflingCache} from "../../../src/chain/shufflingCache.js"; export type AttestationValidDataOpts = { currentSlot?: Slot; @@ -73,6 +77,12 @@ export function getAttestationValidData(opts: AttestationValidDataOpts): { ...{executionPayloadBlockHash: null, executionStatus: ExecutionStatus.PreMerge}, }; + + const shufflingCache = new ShufflingCache(); + shufflingCache.processState(state, state.epochCtx.currentShuffling.epoch); + shufflingCache.processState(state, state.epochCtx.nextShuffling.epoch); + const dependentRoot = getShufflingDecisionBlock(state, state.epochCtx.currentShuffling.epoch); + const forkChoice = { getBlock: (root) => { if (!ssz.Root.equals(root, beaconBlockRoot)) return null; @@ -82,6 +92,7 @@ export function getAttestationValidData(opts: AttestationValidDataOpts): { if (rootHex !== toHexString(beaconBlockRoot)) return null; return headBlock; }, + getDependentRoot: () => dependentRoot, } as Partial as IForkChoice; const committeeIndices = state.epochCtx.getBeaconCommittee(attSlot, attIndex); @@ -117,11 +128,13 @@ export function getAttestationValidData(opts: AttestationValidDataOpts): { // Add state to regen const regen = { getState: async () => state, + // TODO: remove this once we have a better way to get state + getStateSync: () => state, } as Partial as IStateRegenerator; const chain = { clock, - config: config as BeaconConfig, + config: state.config, forkChoice, regen, seenAttesters: new SeenAttesters(), @@ -132,6 +145,7 @@ export function getAttestationValidData(opts: AttestationValidDataOpts): { : new BlsMultiThreadWorkerPool({}, {logger: testLogger(), metrics: null}), waitForBlock: () => Promise.resolve(false), index2pubkey: state.epochCtx.index2pubkey, + shufflingCache, opts: defaultChainOptions, } as Partial as IBeaconChain; diff --git a/packages/cli/src/options/beaconNodeOptions/chain.ts b/packages/cli/src/options/beaconNodeOptions/chain.ts index 359b77740b00..ce37135f9689 100644 --- a/packages/cli/src/options/beaconNodeOptions/chain.ts +++ b/packages/cli/src/options/beaconNodeOptions/chain.ts @@ -24,6 +24,7 @@ export type ChainArgs = { emitPayloadAttributes?: boolean; broadcastValidationStrictness?: string; "chain.minSameMessageSignatureSetsToBatch"?: number; + "chain.maxShufflingCacheEpochs"?: number; }; export function parseArgs(args: ChainArgs): IBeaconNodeOptions["chain"] { @@ -49,6 +50,7 @@ export function parseArgs(args: ChainArgs): IBeaconNodeOptions["chain"] { broadcastValidationStrictness: args["broadcastValidationStrictness"], minSameMessageSignatureSetsToBatch: args["chain.minSameMessageSignatureSetsToBatch"] ?? defaultOptions.chain.minSameMessageSignatureSetsToBatch, + maxShufflingCacheEpochs: args["chain.maxShufflingCacheEpochs"] ?? defaultOptions.chain.maxShufflingCacheEpochs, }; } @@ -193,4 +195,12 @@ Will double processing times. Use only for debugging purposes.", default: defaultOptions.chain.minSameMessageSignatureSetsToBatch, group: "chain", }, + + "chain.maxShufflingCacheEpochs": { + hidden: true, + description: "Maximum ShufflingCache epochs to keep in memory", + type: "number", + default: defaultOptions.chain.maxShufflingCacheEpochs, + group: "chain", + }, }; diff --git a/packages/cli/test/unit/options/beaconNodeOptions.test.ts b/packages/cli/test/unit/options/beaconNodeOptions.test.ts index 4f5050d87daf..5e9491ab1436 100644 --- a/packages/cli/test/unit/options/beaconNodeOptions.test.ts +++ b/packages/cli/test/unit/options/beaconNodeOptions.test.ts @@ -34,6 +34,7 @@ describe("options / beaconNodeOptions", () => { "chain.archiveStateEpochFrequency": 1024, "chain.trustedSetup": "", "chain.minSameMessageSignatureSetsToBatch": 32, + "chain.maxShufflingCacheEpochs": 100, emitPayloadAttributes: false, eth1: true, @@ -135,6 +136,7 @@ describe("options / beaconNodeOptions", () => { emitPayloadAttributes: false, trustedSetup: "", minSameMessageSignatureSetsToBatch: 32, + maxShufflingCacheEpochs: 100, }, eth1: { enabled: true,