From e70e4ab9e2ed38dd71d65049947e851466bd6245 Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Thu, 18 Jan 2024 15:31:25 +0700 Subject: [PATCH] feat: metrics for epoch transition by callers and log block replays --- .../src/chain/blocks/importBlock.ts | 2 +- .../beacon-node/src/chain/regen/queued.ts | 1 - packages/beacon-node/src/chain/regen/regen.ts | 19 +++++++++++++------ .../chain/stateCache/fifoBlockStateCache.ts | 2 +- .../stateCache/persistentCheckpointsCache.ts | 10 +++++----- .../src/metrics/metrics/lodestar.ts | 6 +++++- 6 files changed, 25 insertions(+), 15 deletions(-) diff --git a/packages/beacon-node/src/chain/blocks/importBlock.ts b/packages/beacon-node/src/chain/blocks/importBlock.ts index 89ed52b66750..b92446945777 100644 --- a/packages/beacon-node/src/chain/blocks/importBlock.ts +++ b/packages/beacon-node/src/chain/blocks/importBlock.ts @@ -359,7 +359,7 @@ export async function importBlock( const checkpointState = postState; const cp = getCheckpointFromState(checkpointState); this.regen.addCheckpointState(cp, checkpointState); - this.emitter.emit(ChainEvent.checkpoint, cp, checkpointState); + this.emitter.emit(ChainEvent.checkpoint, cp, checkpointState.clone()); // Note: in-lined code from previos handler of ChainEvent.checkpoint this.logger.verbose("Checkpoint processed", toCheckpointHex(cp)); diff --git a/packages/beacon-node/src/chain/regen/queued.ts b/packages/beacon-node/src/chain/regen/queued.ts index 94c9d6e77542..446f77d76458 100644 --- a/packages/beacon-node/src/chain/regen/queued.ts +++ b/packages/beacon-node/src/chain/regen/queued.ts @@ -18,7 +18,6 @@ const REGEN_CAN_ACCEPT_WORK_THRESHOLD = 16; type QueuedStateRegeneratorModules = RegenModules & { signal: AbortSignal; - logger: Logger; }; type RegenRequestKey = keyof IStateRegeneratorInternal; diff --git a/packages/beacon-node/src/chain/regen/regen.ts b/packages/beacon-node/src/chain/regen/regen.ts index b2fe3a792558..0f0c52b93cad 100644 --- a/packages/beacon-node/src/chain/regen/regen.ts +++ b/packages/beacon-node/src/chain/regen/regen.ts @@ -10,7 +10,7 @@ import { stateTransition, } from "@lodestar/state-transition"; import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; -import {sleep} from "@lodestar/utils"; +import {Logger, sleep} from "@lodestar/utils"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {ChainForkConfig} from "@lodestar/config"; import {Metrics} from "../../metrics/index.js"; @@ -28,6 +28,7 @@ export type RegenModules = { checkpointStateCache: CheckpointStateCache; config: ChainForkConfig; emitter: ChainEventEmitter; + logger: Logger; metrics: Metrics | null; }; @@ -127,14 +128,14 @@ export class StateRegenerator implements IStateRegeneratorInternal { // If a checkpoint state exists with the given checkpoint root, it either is in requested epoch // or needs to have empty slots processed until the requested epoch if (latestCheckpointStateCtx) { - return processSlotsByCheckpoint(this.modules, latestCheckpointStateCtx, slot, opts); + return processSlotsByCheckpoint(this.modules, latestCheckpointStateCtx, slot, rCaller, opts); } // Otherwise, use the fork choice to get the stateRoot from block at the checkpoint root // regenerate that state, // then process empty slots until the requested epoch const blockStateCtx = await this.getState(block.stateRoot, rCaller, shouldReload); - return processSlotsByCheckpoint(this.modules, blockStateCtx, slot, opts); + return processSlotsByCheckpoint(this.modules, blockStateCtx, slot, rCaller, opts); } /** @@ -189,6 +190,8 @@ export class StateRegenerator implements IStateRegeneratorInternal { }); } + const replaySlots = blocksToReplay.map((b) => b.slot).join(","); + this.modules.logger.debug("Replaying blocks to get state", {stateRoot, replaySlots}); for (const b of blocksToReplay.reverse()) { const block = await this.modules.db.block.get(fromHexString(b.blockRoot)); if (!block) { @@ -212,7 +215,7 @@ export class StateRegenerator implements IStateRegeneratorInternal { verifyProposer: false, verifySignatures: false, }, - null + this.modules.metrics ); const stateRoot = toHexString(state.hashTreeRoot()); @@ -239,6 +242,7 @@ export class StateRegenerator implements IStateRegeneratorInternal { }); } } + this.modules.logger.debug("Replayed blocks to get state", {stateRoot, replaySlots}); return state; } @@ -266,9 +270,10 @@ async function processSlotsByCheckpoint( modules: {checkpointStateCache: CheckpointStateCache; metrics: Metrics | null; emitter: ChainEventEmitter}, preState: CachedBeaconStateAllForks, slot: Slot, + rCaller: RegenCaller, opts: StateCloneOpts ): Promise { - let postState = await processSlotsToNearestCheckpoint(modules, preState, slot, opts); + let postState = await processSlotsToNearestCheckpoint(modules, preState, slot, rCaller, opts); if (postState.slot < slot) { postState = processSlots(postState, slot, opts, modules.metrics); } @@ -286,6 +291,7 @@ async function processSlotsToNearestCheckpoint( modules: {checkpointStateCache: CheckpointStateCache; metrics: Metrics | null; emitter: ChainEventEmitter}, preState: CachedBeaconStateAllForks, slot: Slot, + rCaller: RegenCaller, opts: StateCloneOpts ): Promise { const preSlot = preState.slot; @@ -301,6 +307,7 @@ async function processSlotsToNearestCheckpoint( ) { // processSlots calls .clone() before mutating postState = processSlots(postState, nextEpochSlot, opts, metrics); + modules.metrics?.epochTransitionByCaller.inc({caller: rCaller}); // this is usually added when we prepare for next slot or validate gossip block // then when we process the 1st block of epoch, we don't have to do state transition again @@ -309,7 +316,7 @@ async function processSlotsToNearestCheckpoint( const checkpointState = postState; const cp = getCheckpointFromState(checkpointState); checkpointStateCache.add(cp, checkpointState); - emitter.emit(ChainEvent.checkpoint, cp, checkpointState); + emitter.emit(ChainEvent.checkpoint, cp, checkpointState.clone()); // this avoids keeping our node busy processing blocks await sleep(0); diff --git a/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts b/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts index 854983101c04..63cd806d1451 100644 --- a/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts +++ b/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts @@ -80,7 +80,7 @@ export class FIFOBlockStateCache implements BlockStateCache { this.metrics?.hits.inc(); this.metrics?.stateClonedCount.observe(item.clonedCount); - return item; + return item.clone(); } /** diff --git a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts index 36bc11175624..57121a61bf5e 100644 --- a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts @@ -187,7 +187,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { async getOrReload(cp: CheckpointHex): Promise { const stateOrStateBytesData = await this.getStateOrLoadDb(cp); if (stateOrStateBytesData === null || isCachedBeaconState(stateOrStateBytesData)) { - return stateOrStateBytesData; + return stateOrStateBytesData?.clone() ?? null; } const {persistedKey, stateBytes} = stateOrStateBytesData; const logMeta = {persistedKey: toHexString(persistedKey)}; @@ -237,7 +237,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { this.cache.set(cpKey, {type: CacheItemType.inMemory, state: newCachedState, persistedKey}); this.epochIndex.getOrDefault(cp.epoch).add(cp.rootHex); // don't prune from memory here, call it at the last 1/3 of slot 0 of an epoch - return newCachedState; + return newCachedState.clone(); } catch (e) { this.logger.debug("Reload: error loading cached state", logMeta, e as Error); return null; @@ -311,7 +311,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { if (isInMemoryCacheItem(cacheItem)) { const {state} = cacheItem; this.metrics?.stateClonedCount.observe(state.clonedCount); - return state; + return state.clone(); } return null; @@ -353,7 +353,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { if (this.epochIndex.get(epoch)?.has(rootHex)) { const inMemoryState = this.get({rootHex, epoch}); if (inMemoryState) { - return inMemoryState; + return inMemoryState.clone(); } } } @@ -377,7 +377,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { try { const state = await this.getOrReload({rootHex, epoch}); if (state) { - return state; + return state.clone(); } } catch (e) { this.logger.debug("Error get or reload state", {epoch, rootHex}, e as Error); diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index 284f4e75c064..db3c2b37515c 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -290,7 +290,11 @@ export function createLodestarMetrics( }, // Beacon state transition metrics - + epochTransitionByCaller: register.gauge<{caller: RegenCaller}>({ + name: "lodestar_epoch_transition_by_caller_total", + help: "Total count of epoch transition by caller", + labelNames: ["caller"], + }), epochTransitionTime: register.histogram({ name: "lodestar_stfn_epoch_transition_seconds", help: "Time to process a single epoch transition in seconds",