From ddf3add7ef386dc192771bcf22a5818d22ee29ee Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Wed, 31 Mar 2021 16:38:58 +0200 Subject: [PATCH 1/2] Refactor metrics --- .../src/options/beaconNodeOptions/metrics.ts | 9 - .../unit/options/beaconNodeOptions.test.ts | 2 - .../lodestar/src/chain/blocks/processor.ts | 15 +- packages/lodestar/src/chain/chain.ts | 6 +- packages/lodestar/src/chain/initState.ts | 4 +- packages/lodestar/src/metrics/beacon.ts | 260 ------------------ packages/lodestar/src/metrics/index.ts | 3 +- packages/lodestar/src/metrics/interface.ts | 149 +--------- packages/lodestar/src/metrics/metrics.ts | 38 +-- .../lodestar/src/metrics/metrics/beacon.ts | 124 +++++++++ .../lodestar/src/metrics/metrics/lodestar.ts | 92 +++++++ packages/lodestar/src/metrics/options.ts | 2 - packages/lodestar/src/metrics/server/http.ts | 44 +-- packages/lodestar/src/metrics/server/index.ts | 1 - packages/lodestar/src/metrics/server/push.ts | 23 -- packages/lodestar/src/metrics/utils/gauge.ts | 70 +++++ .../src/metrics/{ => utils}/gitData.ts | 0 .../lodestar/src/metrics/utils/histogram.ts | 48 ++++ .../metrics/utils/registryMetricCreator.ts | 24 ++ .../lodestar/src/network/gossip/gossipsub.ts | 6 +- .../lodestar/src/network/gossip/validator.ts | 34 +-- packages/lodestar/src/network/network.ts | 4 +- .../lodestar/src/network/peers/peerManager.ts | 6 +- packages/lodestar/src/node/nodejs.ts | 22 +- packages/lodestar/src/sync/interface.ts | 4 +- packages/lodestar/src/util/queue/index.ts | 19 +- packages/lodestar/src/util/queue/metrics.ts | 49 ---- .../e2e/network/peers/peerManager.test.ts | 4 +- .../lodestar/test/unit/chain/chain.test.ts | 6 +- .../lodestar/test/unit/metrics/beacon.test.ts | 22 +- .../test/unit/metrics/metrics.test.ts | 10 +- .../test/unit/metrics/server/http.test.ts | 11 +- .../lodestar/test/unit/metrics/utils.test.ts | 76 +++++ 33 files changed, 570 insertions(+), 617 deletions(-) delete mode 100644 packages/lodestar/src/metrics/beacon.ts create mode 100644 packages/lodestar/src/metrics/metrics/beacon.ts create mode 100644 packages/lodestar/src/metrics/metrics/lodestar.ts delete mode 100644 packages/lodestar/src/metrics/server/push.ts create mode 100644 packages/lodestar/src/metrics/utils/gauge.ts rename packages/lodestar/src/metrics/{ => utils}/gitData.ts (100%) create mode 100644 packages/lodestar/src/metrics/utils/histogram.ts create mode 100644 packages/lodestar/src/metrics/utils/registryMetricCreator.ts delete mode 100644 packages/lodestar/src/util/queue/metrics.ts create mode 100644 packages/lodestar/test/unit/metrics/utils.test.ts diff --git a/packages/cli/src/options/beaconNodeOptions/metrics.ts b/packages/cli/src/options/beaconNodeOptions/metrics.ts index 42c53425b746..173d6de75a58 100644 --- a/packages/cli/src/options/beaconNodeOptions/metrics.ts +++ b/packages/cli/src/options/beaconNodeOptions/metrics.ts @@ -4,7 +4,6 @@ import {ICliCommandOptions} from "../../util"; export interface IMetricsArgs { "metrics.enabled": boolean; "metrics.gatewayUrl": string; - "metrics.pushGateway": boolean; "metrics.serverPort": number; "metrics.timeout": number; "metrics.listenAddr": string; @@ -14,7 +13,6 @@ export function parseArgs(args: IMetricsArgs): IBeaconNodeOptions["metrics"] { return { enabled: args["metrics.enabled"], gatewayUrl: args["metrics.gatewayUrl"], - pushGateway: args["metrics.pushGateway"], serverPort: args["metrics.serverPort"], timeout: args["metrics.timeout"], listenAddr: args["metrics.listenAddr"], @@ -36,13 +34,6 @@ export const options: ICliCommandOptions = { group: "metrics", }, - "metrics.pushGateway": { - type: "boolean", - description: "Enable/disable Prometheus Pushgateway for metrics", - defaultDescription: String(defaultOptions.metrics.pushGateway), - group: "metrics", - }, - "metrics.serverPort": { type: "number", description: "Server port for metrics", diff --git a/packages/cli/test/unit/options/beaconNodeOptions.test.ts b/packages/cli/test/unit/options/beaconNodeOptions.test.ts index 4d96b3f116bd..51aba1843a32 100644 --- a/packages/cli/test/unit/options/beaconNodeOptions.test.ts +++ b/packages/cli/test/unit/options/beaconNodeOptions.test.ts @@ -22,7 +22,6 @@ describe("options / beaconNodeOptions", () => { "metrics.enabled": true, "metrics.gatewayUrl": "http://localhost:8000", - "metrics.pushGateway": true, "metrics.serverPort": 8765, "metrics.timeout": 5000, "metrics.listenAddr": "0.0.0.0", @@ -61,7 +60,6 @@ describe("options / beaconNodeOptions", () => { metrics: { enabled: true, gatewayUrl: "http://localhost:8000", - pushGateway: true, serverPort: 8765, timeout: 5000, listenAddr: "0.0.0.0", diff --git a/packages/lodestar/src/chain/blocks/processor.ts b/packages/lodestar/src/chain/blocks/processor.ts index 2b23fc862a7c..eaf43cc4db81 100644 --- a/packages/lodestar/src/chain/blocks/processor.ts +++ b/packages/lodestar/src/chain/blocks/processor.ts @@ -11,19 +11,17 @@ import {IStateRegenerator} from "../regen"; import {JobQueue} from "../../util/queue"; import {CheckpointStateCache} from "../stateCache"; import {BlockError, BlockErrorCode, ChainSegmentError} from "../errors"; -import {IBeaconMetrics} from "../../metrics"; +import {IMetrics} from "../../metrics"; import {processBlock, processChainSegment} from "./process"; import {validateBlock} from "./validate"; -const metricsPrefix = "lodestar_block_processor_queue"; - type BlockProcessorModules = { config: IBeaconConfig; forkChoice: IForkChoice; regen: IStateRegenerator; emitter: ChainEventEmitter; - metrics?: IBeaconMetrics; + metrics?: IMetrics; clock: IBeaconClock; checkpointStateCache: CheckpointStateCache; }; @@ -44,7 +42,14 @@ export class BlockProcessor { maxLength?: number; }) { this.modules = modules; - this.jobQueue = new JobQueue({maxLength, signal}, {metrics: modules.metrics, prefix: metricsPrefix}); + this.jobQueue = new JobQueue( + {maxLength, signal}, + modules.metrics && { + length: modules.metrics.blockProcessorQueueLength, + droppedJobs: modules.metrics.blockProcessorQueueDroppedJobs, + jobTime: modules.metrics.blockProcessorQueueJobTime, + } + ); } async processBlockJob(job: IBlockJob): Promise { diff --git a/packages/lodestar/src/chain/chain.ts b/packages/lodestar/src/chain/chain.ts index d66a370923ab..eb4996230cd1 100644 --- a/packages/lodestar/src/chain/chain.ts +++ b/packages/lodestar/src/chain/chain.ts @@ -19,7 +19,7 @@ import {AbortController} from "abort-controller"; import {FAR_FUTURE_EPOCH, GENESIS_EPOCH, ZERO_HASH} from "../constants"; import {IBeaconDb} from "../db"; import {CheckpointStateCache, StateContextCache} from "./stateCache"; -import {IBeaconMetrics} from "../metrics"; +import {IMetrics} from "../metrics"; import {AttestationPool, AttestationProcessor} from "./attestation"; import {BlockPool, BlockProcessor} from "./blocks"; import {IBeaconClock, LocalClock} from "./clock"; @@ -36,7 +36,7 @@ export interface IBeaconChainModules { config: IBeaconConfig; db: IBeaconDb; logger: ILogger; - metrics?: IBeaconMetrics; + metrics?: IMetrics; anchorState: TreeBacked; } @@ -58,7 +58,7 @@ export class BeaconChain implements IBeaconChain { protected readonly config: IBeaconConfig; protected readonly db: IBeaconDb; protected readonly logger: ILogger; - protected readonly metrics?: IBeaconMetrics; + protected readonly metrics?: IMetrics; protected readonly opts: IChainOptions; /** * Internal event emitter is used internally to the chain to update chain state diff --git a/packages/lodestar/src/chain/initState.ts b/packages/lodestar/src/chain/initState.ts index 5fd5ff467cce..9e61d135d4e0 100644 --- a/packages/lodestar/src/chain/initState.ts +++ b/packages/lodestar/src/chain/initState.ts @@ -16,7 +16,7 @@ import {toHexString, TreeBacked} from "@chainsafe/ssz"; import {GENESIS_SLOT, ZERO_HASH} from "../constants"; import {IBeaconDb} from "../db"; import {Eth1Provider} from "../eth1"; -import {IBeaconMetrics} from "../metrics"; +import {IMetrics} from "../metrics"; import {GenesisBuilder} from "./genesis/genesis"; import {IGenesisResult} from "./genesis/interface"; import {CheckpointStateCache, StateContextCache} from "./stateCache"; @@ -176,7 +176,7 @@ export function restoreStateCaches( return cachedBeaconState; } -export function initBeaconMetrics(metrics: IBeaconMetrics, state: TreeBacked): void { +export function initBeaconMetrics(metrics: IMetrics, state: TreeBacked): void { metrics.headSlot.set(state.slot); metrics.previousJustifiedEpoch.set(state.previousJustifiedCheckpoint.epoch); metrics.currentJustifiedEpoch.set(state.currentJustifiedCheckpoint.epoch); diff --git a/packages/lodestar/src/metrics/beacon.ts b/packages/lodestar/src/metrics/beacon.ts deleted file mode 100644 index ff983770fff9..000000000000 --- a/packages/lodestar/src/metrics/beacon.ts +++ /dev/null @@ -1,260 +0,0 @@ -/** - * @module metrics - */ -import {Gauge, Counter} from "prom-client"; - -import {IBeaconMetrics} from "./interface"; -import {IMetricsOptions} from "./options"; -import {Metrics} from "./metrics"; -import {readLodestarGitData} from "./gitData"; -import {ILogger} from "@chainsafe/lodestar-utils"; - -export class BeaconMetrics extends Metrics implements IBeaconMetrics { - peers: Gauge; - slot: Gauge; - headSlot: Gauge; - headRoot: Gauge; - finalizedEpoch: Gauge; - finalizedRoot: Gauge; - currentJustifiedEpoch: Gauge; - currentJustifiedRoot: Gauge; - previousJustifiedEpoch: Gauge; - previousJustifiedRoot: Gauge; - currentValidators: Gauge; - previousValidators: Gauge; - currentLiveValidators: Gauge; - previousLiveValidators: Gauge; - pendingDeposits: Gauge; - processedDepositsTotal: Gauge; - pendingExits: Gauge; - previousEpochOrphanedBlocks: Gauge; - reorgEventsTotal: Counter; - currentEpochActiveGwei: Gauge; - currentEpochSourceGwei: Gauge; - currentEpochTargetGwei: Gauge; - previousEpochActiveGwei: Gauge; - previousEpochSourceGwei: Gauge; - previousEpochTargetGwei: Gauge; - observedEpochAttesters: Gauge; - observedEpochAggregators: Gauge; - peersByDirection: Gauge; - peerConnectedEvent: Gauge; - peerDisconnectedEvent: Gauge; - peerGoodbyeReceived: Gauge; - peerGoodbyeSent: Gauge; - peersTotalUniqueConnected: Gauge; - gossipMeshPeersByType: Gauge; - gossipMeshPeersByBeaconAttestationSubnet: Gauge; - - private lodestarVersion: Gauge; - private logger: ILogger; - - constructor(opts: IMetricsOptions, {logger}: {logger: ILogger}) { - super(opts); - const registers = [this.registry]; - this.logger = logger; - this.peers = new Gauge({ - name: "libp2p_peers", - help: "number of connected peers", - registers, - }); - this.slot = new Gauge({ - name: "beacon_slot", - help: "latest slot", - registers, - }); - this.headSlot = new Gauge({ - name: "beacon_head_slot", - help: "slot of the head block of the beacon chain", - registers, - }); - this.headRoot = new Gauge({ - name: "beacon_head_root", - help: "root of the head block of the beacon chain", - registers, - }); - this.finalizedEpoch = new Gauge({ - name: "beacon_finalized_epoch", - help: "current finalized epoch", - registers, - }); - this.finalizedRoot = new Gauge({ - name: "beacon_finalized_root", - help: "current finalized root", - registers, - }); - this.currentJustifiedEpoch = new Gauge({ - name: "beacon_current_justified_epoch", - help: "current justified epoch", - registers, - }); - this.currentJustifiedRoot = new Gauge({ - name: "beacon_current_justified_root", - help: "current justified root", - registers, - }); - this.previousJustifiedEpoch = new Gauge({ - name: "beacon_previous_justified_epoch", - help: "previous justified epoch", - registers, - }); - this.previousJustifiedRoot = new Gauge({ - name: "beacon_previous_justified_root", - help: "previous justified root", - registers, - }); - this.currentValidators = new Gauge({ - name: "beacon_current_validators", - labelNames: ["status"], - help: "number of validators in current epoch", - registers, - }); - this.previousValidators = new Gauge({ - name: "beacon_previous_validators", - labelNames: ["status"], - help: "number of validators in previous epoch", - registers, - }); - this.currentLiveValidators = new Gauge({ - name: "beacon_current_live_validators", - help: "number of active validators that successfully included attestation on chain for current epoch", - registers, - }); - this.previousLiveValidators = new Gauge({ - name: "beacon_previous_live_validators", - help: "number of active validators that successfully included attestation on chain for previous epoch", - registers, - }); - this.pendingDeposits = new Gauge({ - name: "beacon_pending_deposits", - help: "number of pending deposits", - registers, - }); - this.processedDepositsTotal = new Gauge({ - name: "beacon_processed_deposits_total", - help: "number of total deposits included on chain", - registers, - }); - this.pendingExits = new Gauge({ - name: "beacon_pending_exits", - help: "number of pending voluntary exits", - registers, - }); - this.previousEpochOrphanedBlocks = new Gauge({ - name: "beacon_previous_epoch_orphaned_blocks", - help: "number of blocks not included into the chain in previous epoch", - registers, - }); - this.reorgEventsTotal = new Counter({ - name: "beacon_reorg_events_total", - help: "number of chain reorganizations", - registers, - }); - this.currentEpochActiveGwei = new Gauge({ - name: "beacon_current_epoch_active_gwei", - help: "current epoch active balances", - registers, - }); - this.currentEpochSourceGwei = new Gauge({ - name: "beacon_current_epoch_source_gwei", - help: "current epoch source balances", - registers, - }); - this.currentEpochTargetGwei = new Gauge({ - name: "beacon_current_epoch_target_gwei", - help: "current epoch target balances", - registers, - }); - this.previousEpochActiveGwei = new Gauge({ - name: "beacon_previous_epoch_active_gwei", - help: "previous epoch active balances", - registers, - }); - this.previousEpochSourceGwei = new Gauge({ - name: "beacon_previous_epoch_source_gwei", - help: "previous epoch source balances", - registers, - }); - this.previousEpochTargetGwei = new Gauge({ - name: "beacon_previous_epoch_target_gwei", - help: "previous epoch target balances", - registers, - }); - this.observedEpochAttesters = new Gauge({ - name: "beacon_observed_epoch_attesters", - help: "number of attesters for which we have seen an attestation, not necessarily included on chain.", - registers, - }); - this.observedEpochAggregators = new Gauge({ - name: "beacon_observed_epoch_aggregators", - help: "number of aggregators for which we have seen an attestation, not necessarily included on chain.", - registers, - }); - - // Extra Lodestar custom metrics - - this.peersByDirection = new Gauge({ - name: "lodestar_peers_by_direction", - help: "number of peers, labeled by direction", - labelNames: ["direction"], - registers, - }); - - this.peerConnectedEvent = new Gauge({ - name: "lodestar_peer_connected", - help: "Number of peer:connected event, labeled by direction", - labelNames: ["direction"], - registers, - }); - - this.peerDisconnectedEvent = new Gauge({ - name: "lodestar_peer_disconnected", - help: "Number of peer:disconnected event, labeled by direction", - labelNames: ["direction"], - registers, - }); - - this.peerGoodbyeReceived = new Gauge({ - name: "lodestar_peer_goodbye_received", - help: "Number of goodbye received, labeled by reason", - labelNames: ["reason"], - registers, - }); - - this.peerGoodbyeSent = new Gauge({ - name: "lodestar_peer_goodbye_sent", - help: "Number of goodbye sent, labeled by reason", - labelNames: ["reason"], - registers, - }); - - this.peersTotalUniqueConnected = new Gauge({ - name: "lodestar_peers_total_unique_connected", - help: "Total number of unique peers that have had a connection with", - registers, - }); - - this.gossipMeshPeersByType = new Gauge({ - name: "lodestar_gossip_mesh_peers_by_type", - help: "Number of connected mesh peers per gossip type", - labelNames: ["gossipType"], - registers, - }); - - this.gossipMeshPeersByBeaconAttestationSubnet = new Gauge({ - name: "lodestar_gossip_mesh_peers_by_beacon_attestation_subnet", - help: "Number of connected mesh peers per beacon attestation subnet", - labelNames: ["subnet"], - registers, - }); - - // Private - only used once now - this.lodestarVersion = new Gauge({ - name: "lodestar_version", - help: "Lodestar version", - labelNames: ["semver", "branch", "commit", "version"], - registers, - }); - this.lodestarVersion.set(readLodestarGitData(), 1); - } -} diff --git a/packages/lodestar/src/metrics/index.ts b/packages/lodestar/src/metrics/index.ts index 7f8942f5123c..5efb48856c34 100644 --- a/packages/lodestar/src/metrics/index.ts +++ b/packages/lodestar/src/metrics/index.ts @@ -1,7 +1,6 @@ /** * @module metrics */ -export * from "./interface"; export * from "./metrics"; -export * from "./beacon"; export * from "./server"; +export * from "./interface"; diff --git a/packages/lodestar/src/metrics/interface.ts b/packages/lodestar/src/metrics/interface.ts index 2c029e4bcd8f..ba844e7ea5e9 100644 --- a/packages/lodestar/src/metrics/interface.ts +++ b/packages/lodestar/src/metrics/interface.ts @@ -1,146 +1,7 @@ -/** - * @module metrics - */ -import {Registry, Gauge, Counter} from "prom-client"; +import {Gauge, Histogram} from "prom-client"; -export interface IMetrics { - registry: Registry; -} +export type IGauge = Pick, "inc" | "set"> & { + addCollect: (collectFn: () => void) => void; +}; -/** - * Metrics from: - * https://github.com/ethereum/eth2.0-metrics/ and - * https://hackmd.io/D5FmoeFZScim_squBFl8oA - */ -export interface IBeaconMetrics extends IMetrics { - /** - * Tracks the number of libp2p peers - */ - peers: Gauge; - /** - * Latest slot of the beacon chain state - */ - slot: Gauge; - /** - * Slot of the head block of the beacon chain - */ - headSlot: Gauge; - /** - * Root of the head block of the beacon chain - */ - headRoot: Gauge; - /** - * Current finalized epoch - */ - finalizedEpoch: Gauge; - /** - * Current finalized root - */ - finalizedRoot: Gauge; - /** - * Current justified epoch - */ - currentJustifiedEpoch: Gauge; - /** - * Current justified root - */ - currentJustifiedRoot: Gauge; - /** - * Current previously justified epoch - */ - previousJustifiedEpoch: Gauge; - /** - * Current previously justified root - */ - previousJustifiedRoot: Gauge; - /** - * Number of `status="pending|active|exited|withdrawable" validators in current epoch - */ - currentValidators: Gauge; - /** - * Number of `status="pending|active|exited|withdrawable" validators in current epoch - */ - previousValidators: Gauge; - /** - * Number of active validators that successfully included attestation on chain for current epoch - */ - currentLiveValidators: Gauge; - /** - * Number of active validators that successfully included attestation on chain for previous epoch - */ - previousLiveValidators: Gauge; - /** - * Number of pending deposits (`state.eth1Data.depositCount - state.eth1DepositIndex`) - */ - pendingDeposits: Gauge; - /** - * Number of total deposits included on chain - */ - processedDepositsTotal: Gauge; - /** - * Number of pending voluntary exits in local operation pool - */ - pendingExits: Gauge; - /** - * Number of blocks orphaned in the previous epoch - */ - previousEpochOrphanedBlocks: Gauge; - /** - * Total occurances of reorganizations of the chain - */ - reorgEventsTotal: Counter; - /** - * Track current epoch active balances - */ - currentEpochActiveGwei: Gauge; - /** - * Track current epoch active balances - */ - currentEpochSourceGwei: Gauge; - /** - * Track current epoch active balances - */ - currentEpochTargetGwei: Gauge; - /** - * Track previous epoch active balances - */ - previousEpochActiveGwei: Gauge; - /** - * Track previous epoch active balances - */ - previousEpochSourceGwei: Gauge; - /** - * Track previous epoch active balances - */ - previousEpochTargetGwei: Gauge; - /** - * Track number of attesters for which we have seen an attestation. - * That attestation is not necessarily included on chain. - */ - observedEpochAttesters: Gauge; - /** - * Track number of aggregators for which we have seen an attestation. - * That attestation is not necessarily included on chain. - */ - observedEpochAggregators: Gauge; - /** Peers labeled by direction */ - peersByDirection: Gauge; - /** Number of peer:connected event, labeled by direction */ - peerConnectedEvent: Gauge; - /** Number of peer:disconnected event, labeled by direction */ - peerDisconnectedEvent: Gauge; - /** Number of goodbye received, labeled by reason */ - peerGoodbyeReceived: Gauge; - /** Number of goodbye sent, labeled by reason */ - peerGoodbyeSent: Gauge; - /** Total number of unique peers that have had a connection with */ - peersTotalUniqueConnected: Gauge; - - /** Gossip mesh peer count by GossipType */ - gossipMeshPeersByType: Gauge; - /** Gossip mesh peer count by beacon attestation subnet */ - gossipMeshPeersByBeaconAttestationSubnet: Gauge; -} - -// eslint-disable-next-line @typescript-eslint/no-empty-interface -export interface IMetricsServer {} +export type IHistogram = Pick, "observe" | "startTimer">; diff --git a/packages/lodestar/src/metrics/metrics.ts b/packages/lodestar/src/metrics/metrics.ts index 2a3d53fbdeb2..1d493f9c210b 100644 --- a/packages/lodestar/src/metrics/metrics.ts +++ b/packages/lodestar/src/metrics/metrics.ts @@ -3,28 +3,28 @@ */ import {collectDefaultMetrics, Registry} from "prom-client"; import gcStats from "prometheus-gc-stats"; +import {createBeaconMetrics, IBeaconMetrics} from "./metrics/beacon"; +import {createLodestarMetrics, ILodestarMetrics} from "./metrics/lodestar"; +import {RegistryMetricCreator} from "./utils/registryMetricCreator"; -import {IMetrics} from "./interface"; -import {IMetricsOptions} from "./options"; +export type IMetrics = IBeaconMetrics & ILodestarMetrics & {register: Registry}; -export class Metrics implements IMetrics { - registry: Registry; +export function createMetrics(): IMetrics { + const register = new RegistryMetricCreator(); + const beacon = createBeaconMetrics(register); + const lodestar = createLodestarMetrics(register); - constructor(opts: IMetricsOptions) { - this.registry = new Registry(); + collectDefaultMetrics({ + register, + // eventLoopMonitoringPrecision with sampling rate in milliseconds + eventLoopMonitoringPrecision: 10, + }); - if (opts.enabled) { - collectDefaultMetrics({ - register: this.registry, - // eventLoopMonitoringPrecision with sampling rate in milliseconds - eventLoopMonitoringPrecision: 10, - }); + // Collects GC metrics using a native binding module + // - nodejs_gc_runs_total: Counts the number of time GC is invoked + // - nodejs_gc_pause_seconds_total: Time spent in GC in seconds + // - nodejs_gc_reclaimed_bytes_total: The number of bytes GC has freed + gcStats(register)(); - // Collects GC metrics using a native binding module - // - nodejs_gc_runs_total: Counts the number of time GC is invoked - // - nodejs_gc_pause_seconds_total: Time spent in GC in seconds - // - nodejs_gc_reclaimed_bytes_total: The number of bytes GC has freed - gcStats(this.registry)(); - } - } + return {...beacon, ...lodestar, register}; } diff --git a/packages/lodestar/src/metrics/metrics/beacon.ts b/packages/lodestar/src/metrics/metrics/beacon.ts new file mode 100644 index 000000000000..0723d796c445 --- /dev/null +++ b/packages/lodestar/src/metrics/metrics/beacon.ts @@ -0,0 +1,124 @@ +import {RegistryMetricCreator} from "../utils/registryMetricCreator"; + +export type IBeaconMetrics = ReturnType; + +/** + * Metrics from: + * https://github.com/ethereum/eth2.0-metrics/ and + * https://hackmd.io/D5FmoeFZScim_squBFl8oA + */ +// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/explicit-function-return-type +export function createBeaconMetrics(register: RegistryMetricCreator) { + return { + peers: register.gauge({ + name: "libp2p_peers", + help: "number of connected peers", + }), + slot: register.gauge({ + name: "beacon_slot", + help: "latest slot", + }), + headSlot: register.gauge({ + name: "beacon_head_slot", + help: "slot of the head block of the beacon chain", + }), + headRoot: register.gauge({ + name: "beacon_head_root", + help: "root of the head block of the beacon chain", + }), + finalizedEpoch: register.gauge({ + name: "beacon_finalized_epoch", + help: "current finalized epoch", + }), + finalizedRoot: register.gauge({ + name: "beacon_finalized_root", + help: "current finalized root", + }), + currentJustifiedEpoch: register.gauge({ + name: "beacon_current_justified_epoch", + help: "current justified epoch", + }), + currentJustifiedRoot: register.gauge({ + name: "beacon_current_justified_root", + help: "current justified root", + }), + previousJustifiedEpoch: register.gauge({ + name: "beacon_previous_justified_epoch", + help: "previous justified epoch", + }), + previousJustifiedRoot: register.gauge({ + name: "beacon_previous_justified_root", + help: "previous justified root", + }), + currentValidators: register.gauge<"status">({ + name: "beacon_current_validators", + labelNames: ["status"], + help: "number of validators in current epoch", + }), + previousValidators: register.gauge<"status">({ + name: "beacon_previous_validators", + labelNames: ["status"], + help: "number of validators in previous epoch", + }), + currentLiveValidators: register.gauge({ + name: "beacon_current_live_validators", + help: "number of active validators that successfully included attestation on chain for current epoch", + }), + previousLiveValidators: register.gauge({ + name: "beacon_previous_live_validators", + help: "number of active validators that successfully included attestation on chain for previous epoch", + }), + pendingDeposits: register.gauge({ + name: "beacon_pending_deposits", + help: "number of pending deposits", + }), + processedDepositsTotal: register.gauge({ + name: "beacon_processed_deposits_total", + help: "number of total deposits included on chain", + }), + pendingExits: register.gauge({ + name: "beacon_pending_exits", + help: "number of pending voluntary exits", + }), + previousEpochOrphanedBlocks: register.gauge({ + name: "beacon_previous_epoch_orphaned_blocks", + help: "number of blocks not included into the chain in previous epoch", + }), + reorgEventsTotal: register.gauge({ + name: "beacon_reorg_events_total", + help: "number of chain reorganizations", + }), + currentEpochActiveGwei: register.gauge({ + name: "beacon_current_epoch_active_gwei", + help: "current epoch active balances", + }), + currentEpochSourceGwei: register.gauge({ + name: "beacon_current_epoch_source_gwei", + help: "current epoch source balances", + }), + currentEpochTargetGwei: register.gauge({ + name: "beacon_current_epoch_target_gwei", + help: "current epoch target balances", + }), + previousEpochActiveGwei: register.gauge({ + name: "beacon_previous_epoch_active_gwei", + help: "previous epoch active balances", + }), + previousEpochSourceGwei: register.gauge({ + name: "beacon_previous_epoch_source_gwei", + help: "previous epoch source balances", + }), + previousEpochTargetGwei: register.gauge({ + name: "beacon_previous_epoch_target_gwei", + help: "previous epoch target balances", + }), + observedEpochAttesters: register.gauge({ + name: "beacon_observed_epoch_attesters", + help: "number of attesters for which we have seen an attestation, not necessarily included on chain.", + }), + observedEpochAggregators: register.gauge({ + name: "beacon_observed_epoch_aggregators", + help: "number of aggregators for which we have seen an attestation, not necessarily included on chain.", + }), + }; +} diff --git a/packages/lodestar/src/metrics/metrics/lodestar.ts b/packages/lodestar/src/metrics/metrics/lodestar.ts new file mode 100644 index 000000000000..4ec7900010ad --- /dev/null +++ b/packages/lodestar/src/metrics/metrics/lodestar.ts @@ -0,0 +1,92 @@ +import {RegistryMetricCreator} from "../utils/registryMetricCreator"; +import {readLodestarGitData} from "../utils/gitData"; + +export type ILodestarMetrics = ReturnType; + +/** + * Extra Lodestar custom metrics + */ +// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/explicit-function-return-type +export function createLodestarMetrics(register: RegistryMetricCreator) { + // Private - only used once now + register.static<"semver" | "branch" | "commit" | "version">({ + name: "lodestar_version", + help: "Lodestar version", + value: readLodestarGitData(), + }); + + return { + peersByDirection: register.gauge<"direction">({ + name: "lodestar_peers_by_direction", + help: "number of peers, labeled by direction", + labelNames: ["direction"], + }), + peerConnectedEvent: register.gauge<"direction">({ + name: "lodestar_peer_connected", + help: "Number of peer:connected event, labeled by direction", + labelNames: ["direction"], + }), + peerDisconnectedEvent: register.gauge<"direction">({ + name: "lodestar_peer_disconnected", + help: "Number of peer:disconnected event, labeled by direction", + labelNames: ["direction"], + }), + peerGoodbyeReceived: register.gauge<"reason">({ + name: "lodestar_peer_goodbye_received", + help: "Number of goodbye received, labeled by reason", + labelNames: ["reason"], + }), + peerGoodbyeSent: register.gauge<"reason">({ + name: "lodestar_peer_goodbye_sent", + help: "Number of goodbye sent, labeled by reason", + labelNames: ["reason"], + }), + peersTotalUniqueConnected: register.gauge({ + name: "lodestar_peers_total_unique_connected", + help: "Total number of unique peers that have had a connection with", + }), + + gossipMeshPeersByType: register.gauge<"gossipType">({ + name: "lodestar_gossip_mesh_peers_by_type", + help: "Number of connected mesh peers per gossip type", + labelNames: ["gossipType"], + }), + gossipMeshPeersByBeaconAttestationSubnet: register.gauge<"subnet">({ + name: "lodestar_gossip_mesh_peers_by_beacon_attestation_subnet", + help: "Number of connected mesh peers per beacon attestation subnet", + labelNames: ["subnet"], + }), + + gossipValidationQueueLength: register.gauge<"topic">({ + name: "lodestar_gossip_validation_queue_length", + help: "Count of total gossip validation queue length", + labelNames: ["topic"], + }), + gossipValidationQueueDroppedJobs: register.gauge<"topic">({ + name: "lodestar_gossip_validation_queue_dropped_jobs_total", + help: "Count of total gossip validation queue dropped jobs", + labelNames: ["topic"], + }), + gossipValidationQueueJobTime: register.histogram<"topic">({ + name: "lodestar_gossip_validation_queue_job_time_seconds", + help: "Time to process gossip validation queue job in seconds", + labelNames: ["topic"], + }), + + blockProcessorQueueLength: register.gauge<"topic">({ + name: "lodestar_block_processor_queue_length", + help: "Count of total block processor queue length", + labelNames: ["topic"], + }), + blockProcessorQueueDroppedJobs: register.gauge<"topic">({ + name: "lodestar_block_processor_queue_dropped_jobs_total", + help: "Count of total block processor queue dropped jobs", + labelNames: ["topic"], + }), + blockProcessorQueueJobTime: register.histogram<"topic">({ + name: "lodestar_block_processor_queue_job_time_seconds", + help: "Time to process block processor queue job in seconds", + labelNames: ["topic"], + }), + }; +} diff --git a/packages/lodestar/src/metrics/options.ts b/packages/lodestar/src/metrics/options.ts index 1ee038f147e6..84cc23f74532 100644 --- a/packages/lodestar/src/metrics/options.ts +++ b/packages/lodestar/src/metrics/options.ts @@ -5,7 +5,6 @@ export interface IMetricsOptions { enabled: boolean; timeout: number; - pushGateway: boolean; serverPort?: number; gatewayUrl?: string; listenAddr?: string; @@ -14,6 +13,5 @@ export interface IMetricsOptions { export const defaultMetricsOptions: IMetricsOptions = { enabled: false, timeout: 5000, - pushGateway: false, serverPort: 8008, }; diff --git a/packages/lodestar/src/metrics/server/http.ts b/packages/lodestar/src/metrics/server/http.ts index 9e6927a9aa22..b44122d203fa 100644 --- a/packages/lodestar/src/metrics/server/http.ts +++ b/packages/lodestar/src/metrics/server/http.ts @@ -3,53 +3,57 @@ */ import http from "http"; import {createHttpTerminator, HttpTerminator} from "http-terminator"; +import {Registry} from "prom-client"; import {ILogger} from "@chainsafe/lodestar-utils"; -import {IMetrics, IMetricsServer} from "../interface"; import {IMetricsOptions} from "../options"; import {wrapError} from "../../util/wrapError"; +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface IMetricsServer {} + +type RegistryHolder = { + register: Registry; +}; + export class HttpMetricsServer implements IMetricsServer { http: http.Server; private terminator: HttpTerminator; private opts: IMetricsOptions; - private metrics: IMetrics; + private register: Registry; private logger: ILogger; - constructor(opts: IMetricsOptions, {metrics, logger}: {metrics: IMetrics; logger: ILogger}) { + constructor(opts: IMetricsOptions, {metrics, logger}: {metrics: RegistryHolder; logger: ILogger}) { this.opts = opts; this.logger = logger; - this.metrics = metrics; + this.register = metrics.register; this.http = http.createServer(this.onRequest.bind(this)); this.terminator = createHttpTerminator({server: this.http}); } async start(): Promise { - if (this.opts.enabled) { - const {serverPort, listenAddr} = this.opts; - this.logger.info("Starting metrics HTTP server", {port: serverPort || null}); - const listen = this.http.listen.bind(this.http); - return new Promise((resolve, reject) => { - listen(serverPort, listenAddr).once("listening", resolve).once("error", reject); - }); - } + const {serverPort, listenAddr} = this.opts; + this.logger.info("Starting metrics HTTP server", {port: serverPort || null}); + const listen = this.http.listen.bind(this.http); + return new Promise((resolve, reject) => { + listen(serverPort, listenAddr).once("listening", resolve).once("error", reject); + }); } + async stop(): Promise { - if (this.opts.enabled) { - try { - await this.terminator.terminate(); - } catch (e) { - this.logger.warn("Failed to stop metrics server", e); - } + try { + await this.terminator.terminate(); + } catch (e) { + this.logger.warn("Failed to stop metrics server", e); } } private async onRequest(req: http.IncomingMessage, res: http.ServerResponse): Promise { if (req.method === "GET" && req.url && req.url.includes("/metrics")) { - const metricsRes = await wrapError(this.metrics.registry.metrics()); + const metricsRes = await wrapError(this.register.metrics()); if (metricsRes.err) { res.writeHead(500, {"content-type": "text/plain"}).end(metricsRes.err.stack); } else { - res.writeHead(200, {"content-type": this.metrics.registry.contentType}).end(metricsRes.result); + res.writeHead(200, {"content-type": this.register.contentType}).end(metricsRes.result); } } else { res.writeHead(404).end(); diff --git a/packages/lodestar/src/metrics/server/index.ts b/packages/lodestar/src/metrics/server/index.ts index 3410342025d2..0271a016bf91 100644 --- a/packages/lodestar/src/metrics/server/index.ts +++ b/packages/lodestar/src/metrics/server/index.ts @@ -2,4 +2,3 @@ * @module metrics/server */ export * from "./http"; -export * from "./push"; diff --git a/packages/lodestar/src/metrics/server/push.ts b/packages/lodestar/src/metrics/server/push.ts deleted file mode 100644 index 2cf833c55d63..000000000000 --- a/packages/lodestar/src/metrics/server/push.ts +++ /dev/null @@ -1,23 +0,0 @@ -/** - * @module metrics/server - */ -import {Pushgateway} from "prom-client"; - -import {IMetrics, IMetricsServer} from "../interface"; -import {IMetricsOptions} from "../options"; - -export class PushMetricsServer implements IMetricsServer { - private metrics: IMetrics; - private opts: IMetricsOptions; - private gateway: Pushgateway | null = null; - constructor(opts: IMetricsOptions, {metrics}: {metrics: IMetrics}) { - this.opts = opts; - this.metrics = metrics; - } - start(): void { - this.gateway = new Pushgateway(this.opts.gatewayUrl as string, {}, this.metrics.registry); - } - stop(): void { - this.gateway = null; - } -} diff --git a/packages/lodestar/src/metrics/utils/gauge.ts b/packages/lodestar/src/metrics/utils/gauge.ts new file mode 100644 index 000000000000..3af46726bae6 --- /dev/null +++ b/packages/lodestar/src/metrics/utils/gauge.ts @@ -0,0 +1,70 @@ +import {Gauge, GaugeConfiguration} from "prom-client"; +import {IGauge} from "../interface"; + +type CollectFn = (metric: IGauge) => void; +type Labels = Partial>; + +/** + * Extends the prom-client Gauge with extra features: + * - Add multiple collect functions after instantiation + * - Create child gauges with fixed labels + */ +export class GaugeExtra extends Gauge implements IGauge { + private collectFns: CollectFn[] = []; + + constructor(configuration: GaugeConfiguration) { + super(configuration); + } + + addCollect(collectFn: CollectFn): void { + this.collectFns.push(collectFn); + } + + child(labels: Labels): GaugeChild { + return new GaugeChild(labels, this); + } + + /** + * @override Metric.collect + */ + private collect(): void { + for (const collectFn of this.collectFns) { + collectFn(this); + } + } +} + +export class GaugeChild implements IGauge { + gauge: GaugeExtra; + labelsParent: Labels; + constructor(labelsParent: Labels, gauge: GaugeExtra) { + this.gauge = gauge; + this.labelsParent = labelsParent; + } + + // Sorry for this mess, `prom-client` API choices are not great + // If the function signature was `inc(value: number, labels?: Labels)`, this would be simpler + inc(value?: number): void; + inc(labels: Labels, value?: number): void; + inc(arg1?: Labels | number, arg2?: number): void { + if (typeof arg1 === "object") { + this.gauge.inc({...this.labelsParent, ...arg1}, arg2 || 0); + } else { + this.gauge.inc(this.labelsParent, arg1 || 0); + } + } + + set(value: number): void; + set(labels: Labels, value: number): void; + set(arg1?: Labels | number, arg2?: number): void { + if (typeof arg1 === "object") { + this.gauge.set({...this.labelsParent, ...arg1}, arg2 || 0); + } else { + this.gauge.set(this.labelsParent, arg1 || 0); + } + } + + addCollect(collectFn: CollectFn): void { + this.gauge.addCollect(() => collectFn(this)); + } +} diff --git a/packages/lodestar/src/metrics/gitData.ts b/packages/lodestar/src/metrics/utils/gitData.ts similarity index 100% rename from packages/lodestar/src/metrics/gitData.ts rename to packages/lodestar/src/metrics/utils/gitData.ts diff --git a/packages/lodestar/src/metrics/utils/histogram.ts b/packages/lodestar/src/metrics/utils/histogram.ts new file mode 100644 index 000000000000..2612f4934d7d --- /dev/null +++ b/packages/lodestar/src/metrics/utils/histogram.ts @@ -0,0 +1,48 @@ +import {Histogram, HistogramConfiguration} from "prom-client"; +import {IHistogram} from "../interface"; + +type Labels = Partial>; + +/** + * Extends the prom-client Histogram with extra features: + * - Add multiple collect functions after instantiation + * - Create child histograms with fixed labels + */ +export class HistogramExtra extends Histogram implements IHistogram { + constructor(configuration: HistogramConfiguration) { + super(configuration); + } + + child(labels: Labels): HistogramChild { + return new HistogramChild(labels, this); + } +} + +export class HistogramChild implements IHistogram { + histogram: HistogramExtra; + labelsParent: Labels; + constructor(labelsParent: Labels, histogram: HistogramExtra) { + this.histogram = histogram; + this.labelsParent = labelsParent; + } + + // Sorry for this mess, `prom-client` API choices are not great + // If the function signature was `observe(value: number, labels?: Labels)`, this would be simpler + observe(value?: number): void; + observe(labels: Labels, value?: number): void; + observe(arg1?: Labels | number, arg2?: number): void { + if (typeof arg1 === "object") { + this.histogram.observe({...this.labelsParent, ...arg1}, arg2 || 0); + } else { + this.histogram.observe(this.labelsParent, arg1 || 0); + } + } + + startTimer(arg1?: Labels): (labels?: Labels) => number { + if (typeof arg1 === "object") { + return this.histogram.startTimer({...this.labelsParent, ...arg1}); + } else { + return this.histogram.startTimer(this.labelsParent); + } + } +} diff --git a/packages/lodestar/src/metrics/utils/registryMetricCreator.ts b/packages/lodestar/src/metrics/utils/registryMetricCreator.ts new file mode 100644 index 000000000000..591d84e8c529 --- /dev/null +++ b/packages/lodestar/src/metrics/utils/registryMetricCreator.ts @@ -0,0 +1,24 @@ +import {Gauge, GaugeConfiguration, Registry, HistogramConfiguration} from "prom-client"; +import {GaugeExtra} from "./gauge"; +import {HistogramExtra} from "./histogram"; + +type StaticConfiguration = { + name: GaugeConfiguration["name"]; + help: GaugeConfiguration["help"]; + value: Record; +}; + +export class RegistryMetricCreator extends Registry { + gauge(configuration: GaugeConfiguration): GaugeExtra { + return new GaugeExtra({...configuration, registers: [this]}); + } + + histogram(configuration: HistogramConfiguration): HistogramExtra { + return new HistogramExtra({...configuration, registers: [this]}); + } + + /** Static metric to send string-based data such as versions, config params, etc */ + static({name, help, value}: StaticConfiguration): void { + new Gauge({name, help, labelNames: Object.keys(value), registers: [this]}).set(value, 1); + } +} diff --git a/packages/lodestar/src/network/gossip/gossipsub.ts b/packages/lodestar/src/network/gossip/gossipsub.ts index 0bbbf7232613..efb697632684 100644 --- a/packages/lodestar/src/network/gossip/gossipsub.ts +++ b/packages/lodestar/src/network/gossip/gossipsub.ts @@ -7,7 +7,7 @@ import {ATTESTATION_SUBNET_COUNT, phase0, Root} from "@chainsafe/lodestar-types" import {ILogger, toJson} from "@chainsafe/lodestar-utils"; import {computeEpochAtSlot} from "@chainsafe/lodestar-beacon-state-transition"; -import {IBeaconMetrics} from "../../metrics"; +import {IMetrics} from "../../metrics"; import {GossipHandlerFn, GossipObject, GossipTopic, GossipType, IGossipMessage, TopicValidatorFnMap} from "./interface"; import {msgIdToString, getMsgId, messageIsValid} from "./utils"; import {getGossipSSZSerializer, getGossipTopic, getGossipTopicString} from "./topic"; @@ -23,7 +23,7 @@ interface IGossipsubModules { libp2p: Libp2p; validatorFns: TopicValidatorFnMap; logger: ILogger; - metrics?: IBeaconMetrics; + metrics?: IMetrics; } /** @@ -43,7 +43,7 @@ export class Eth2Gossipsub extends Gossipsub { private readonly config: IBeaconConfig; private readonly genesisValidatorsRoot: Root; private readonly logger: ILogger; - private readonly metrics?: IBeaconMetrics; + private readonly metrics?: IMetrics; /** * Cached gossip objects * diff --git a/packages/lodestar/src/network/gossip/validator.ts b/packages/lodestar/src/network/gossip/validator.ts index faeb1f4bfdf1..17e6a4c33306 100644 --- a/packages/lodestar/src/network/gossip/validator.ts +++ b/packages/lodestar/src/network/gossip/validator.ts @@ -1,7 +1,7 @@ import {AbortSignal} from "abort-controller"; import {ATTESTATION_SUBNET_COUNT} from "@chainsafe/lodestar-types"; import {mapValues} from "@chainsafe/lodestar-utils"; -import {IBeaconMetrics} from "../../metrics"; +import {IMetrics} from "../../metrics"; import {JobQueue, JobQueueOpts, QueueType} from "../../util/queue"; import {getGossipTopicString} from "./topic"; import {DEFAULT_ENCODING} from "./constants"; @@ -27,28 +27,13 @@ const gossipQueueOpts: {[K in GossipType]: {maxLength: number; type: QueueType}} [GossipType.attester_slashing]: {maxLength: 4096, type: QueueType.FIFO}, }; -const gossipMetricsPrefix: {[K in GossipType]: string} = { - [GossipType.beacon_block]: "lodestar_gossip_beacon_block_queue", - [GossipType.beacon_aggregate_and_proof]: "lodestar_gossip_beacon_aggregate_and_proof_queue", - [GossipType.beacon_attestation]: "lodestar_gossip_beacon_attestation_queue", - [GossipType.voluntary_exit]: "lodestar_gossip_voluntary_exit_queue", - [GossipType.proposer_slashing]: "lodestar_gossip_proposer_slashing_queue", - [GossipType.attester_slashing]: "lodestar_gossip_attester_slashing_queue", -}; - export function createTopicValidatorFnMap( modules: IObjectValidatorModules, - metrics: IBeaconMetrics | undefined, + metrics: IMetrics | undefined, signal: AbortSignal ): TopicValidatorFnMap { const wrappedValidatorFns = mapValues(validatorFns, (validatorFn, type) => - wrapWithQueue( - validatorFn as ValidatorFn, - modules, - {signal, ...gossipQueueOpts[type]}, - metrics, - gossipMetricsPrefix[type] - ) + wrapWithQueue(validatorFn as ValidatorFn, modules, {signal, ...gossipQueueOpts[type]}, metrics, type) ); return createValidatorFnsByTopic(modules, wrappedValidatorFns); @@ -72,10 +57,17 @@ export function wrapWithQueue( validatorFn: ValidatorFn, modules: IObjectValidatorModules, queueOpts: JobQueueOpts, - metrics: IBeaconMetrics | undefined, - metricsPrefix: string + metrics: IMetrics | undefined, + type: GossipType ): TopicValidatorFn { - const jobQueue = new JobQueue(queueOpts, {metrics, prefix: metricsPrefix}); + const jobQueue = new JobQueue( + queueOpts, + metrics && { + length: metrics.gossipValidationQueueLength.child({topic: type}), + droppedJobs: metrics.gossipValidationQueueDroppedJobs.child({topic: type}), + jobTime: metrics.gossipValidationQueueJobTime.child({topic: type}), + } + ); return async function (_topicStr, gossipMsg) { const {gossipTopic, gossipObject} = parseGossipMsg(gossipMsg); await jobQueue.push(async () => await validatorFn(modules, gossipTopic, gossipObject)); diff --git a/packages/lodestar/src/network/network.ts b/packages/lodestar/src/network/network.ts index 86720db00716..882d04eb49cb 100644 --- a/packages/lodestar/src/network/network.ts +++ b/packages/lodestar/src/network/network.ts @@ -7,7 +7,7 @@ import PeerId from "peer-id"; import Multiaddr from "multiaddr"; import {IBeaconConfig} from "@chainsafe/lodestar-config"; import {ILogger} from "@chainsafe/lodestar-utils"; -import {IBeaconMetrics} from "../metrics"; +import {IMetrics} from "../metrics"; import {ReqResp, IReqResp, IReqRespOptions} from "./reqresp"; import {INetworkOptions} from "./options"; import {INetwork} from "./interface"; @@ -28,7 +28,7 @@ interface INetworkModules { config: IBeaconConfig; libp2p: LibP2p; logger: ILogger; - metrics?: IBeaconMetrics; + metrics?: IMetrics; chain: IBeaconChain; db: IBeaconDb; reqRespHandler: IReqRespHandler; diff --git a/packages/lodestar/src/network/peers/peerManager.ts b/packages/lodestar/src/network/peers/peerManager.ts index a566948f78c6..ff313590581e 100644 --- a/packages/lodestar/src/network/peers/peerManager.ts +++ b/packages/lodestar/src/network/peers/peerManager.ts @@ -5,7 +5,7 @@ import {ILogger, LodestarError} from "@chainsafe/lodestar-utils"; import PeerId from "peer-id"; import {IBeaconChain} from "../../chain"; import {GoodByeReasonCode, GOODBYE_KNOWN_CODES, Libp2pEvent, Method} from "../../constants"; -import {IBeaconMetrics} from "../../metrics"; +import {IMetrics} from "../../metrics"; import {NetworkEvent, INetworkEventBus} from "../events"; import {IReqResp} from "../reqresp"; import {Libp2pPeerMetadataStore} from "./metastore"; @@ -49,7 +49,7 @@ export type PeerManagerOpts = { export type PeerManagerModules = { libp2p: LibP2p; logger: ILogger; - metrics?: IBeaconMetrics; + metrics?: IMetrics; reqResp: IReqResp; chain: IBeaconChain; config: IBeaconConfig; @@ -69,7 +69,7 @@ export type PeerManagerModules = { export class PeerManager { private libp2p: LibP2p; private logger: ILogger; - private metrics?: IBeaconMetrics; + private metrics?: IMetrics; private reqResp: IReqResp; private chain: IBeaconChain; private config: IBeaconConfig; diff --git a/packages/lodestar/src/node/nodejs.ts b/packages/lodestar/src/node/nodejs.ts index 0eddf2c65560..7a0c8bdf8bd8 100644 --- a/packages/lodestar/src/node/nodejs.ts +++ b/packages/lodestar/src/node/nodejs.ts @@ -14,7 +14,7 @@ import {IBeaconDb} from "../db"; import {INetwork, Network, ReqRespHandler} from "../network"; import {BeaconSync, IBeaconSync} from "../sync"; import {BeaconChain, IBeaconChain, initBeaconMetrics} from "../chain"; -import {BeaconMetrics, HttpMetricsServer, IBeaconMetrics} from "../metrics"; +import {createMetrics, IMetrics, HttpMetricsServer} from "../metrics"; import {Api, IApi, RestApi} from "../api"; import {TasksService} from "../tasks"; import {IBeaconNodeOptions} from "./options"; @@ -27,7 +27,7 @@ export interface IBeaconNodeModules { opts: IBeaconNodeOptions; config: IBeaconConfig; db: IBeaconDb; - metrics?: IBeaconMetrics; + metrics?: IMetrics; network: INetwork; chain: IBeaconChain; api: IApi; @@ -61,7 +61,7 @@ export class BeaconNode { opts: IBeaconNodeOptions; config: IBeaconConfig; db: IBeaconDb; - metrics?: IBeaconMetrics; + metrics?: IMetrics; metricsServer?: HttpMetricsServer; network: INetwork; chain: IBeaconChain; @@ -121,9 +121,8 @@ export class BeaconNode { // start db if not already started await db.start(); - let metrics; - if (opts.metrics.enabled) { - metrics = new BeaconMetrics(opts.metrics, {logger: logger.child(opts.logger.metrics)}); + const metrics = opts.metrics.enabled ? createMetrics() : undefined; + if (metrics) { initBeaconMetrics(metrics, anchorState); } @@ -180,14 +179,13 @@ export class BeaconNode { chain, }); - let metricsServer; - if (metrics) { - metricsServer = new HttpMetricsServer(opts.metrics, { - metrics: metrics, - logger: logger.child(opts.logger.metrics), - }); + const metricsServer = metrics + ? new HttpMetricsServer(opts.metrics, {metrics, logger: logger.child(opts.logger.metrics)}) + : undefined; + if (metricsServer) { await metricsServer.start(); } + const restApi = await RestApi.init(opts.api.rest, { config, logger: logger.child(opts.logger.api), diff --git a/packages/lodestar/src/sync/interface.ts b/packages/lodestar/src/sync/interface.ts index a5702134b222..065577b69d79 100644 --- a/packages/lodestar/src/sync/interface.ts +++ b/packages/lodestar/src/sync/interface.ts @@ -3,7 +3,7 @@ import {ILogger} from "@chainsafe/lodestar-utils"; import {CommitteeIndex, Slot, phase0} from "@chainsafe/lodestar-types"; import {IRegularSync} from "./regular"; import {IBeaconChain} from "../chain"; -import {IBeaconMetrics} from "../metrics"; +import {IMetrics} from "../metrics"; import {IBeaconConfig} from "@chainsafe/lodestar-config"; import {IBeaconDb} from "../db/api"; import {AttestationCollector} from "./utils"; @@ -41,7 +41,7 @@ export interface ISyncModules { db: IBeaconDb; logger: ILogger; chain: IBeaconChain; - metrics?: IBeaconMetrics; + metrics?: IMetrics; regularSync?: IRegularSync; gossipHandler?: BeaconGossipHandler; attestationCollector?: AttestationCollector; diff --git a/packages/lodestar/src/util/queue/index.ts b/packages/lodestar/src/util/queue/index.ts index bf6df2117fd3..acbafa8a4142 100644 --- a/packages/lodestar/src/util/queue/index.ts +++ b/packages/lodestar/src/util/queue/index.ts @@ -2,8 +2,8 @@ import {AbortSignal} from "abort-controller"; import {sleep} from "@chainsafe/lodestar-utils"; import {wrapError} from "../wrapError"; import {QueueError, QueueErrorCode} from "./errors"; -import {QueueMetricsOpts, IQueueMetrics, createQueueMetrics} from "./metrics"; -export {QueueError, QueueErrorCode, QueueMetricsOpts}; +import {IGauge, IHistogram} from "../../metrics"; +export {QueueError, QueueErrorCode}; export type JobQueueOpts = { maxLength: number; @@ -23,6 +23,13 @@ enum QueueState { Yielding, } +export interface IQueueMetrics { + length: IGauge; + droppedJobs: IGauge; + /** Compute async utilization rate with `rate(metrics_name[1m])` */ + jobTime: IHistogram; +} + // eslint-disable-next-line @typescript-eslint/no-explicit-any type Job = (...args: any) => Promise; @@ -39,10 +46,14 @@ export class JobQueue { private readonly jobs: JobQueueItem>[] = []; private readonly metrics?: IQueueMetrics; - constructor(opts: JobQueueOpts, metricsOpts?: QueueMetricsOpts) { + constructor(opts: JobQueueOpts, metrics?: IQueueMetrics) { this.opts = opts; this.opts.signal.addEventListener("abort", this.abortAllJobs, {once: true}); - this.metrics = metricsOpts && createQueueMetrics(metricsOpts, {getQueueLength: () => this.jobs.length}); + + if (metrics) { + this.metrics = metrics; + metrics.length.addCollect(() => metrics.length.set(this.jobs.length)); + } } async push = Job>(job: Fn): Promise { diff --git a/packages/lodestar/src/util/queue/metrics.ts b/packages/lodestar/src/util/queue/metrics.ts deleted file mode 100644 index 49eaf802e72e..000000000000 --- a/packages/lodestar/src/util/queue/metrics.ts +++ /dev/null @@ -1,49 +0,0 @@ -import {Gauge, Histogram} from "prom-client"; -import {IBeaconMetrics} from "../../metrics"; - -export type QueueMetricsOpts = { - metrics: IBeaconMetrics | undefined; - prefix: string; -}; - -export interface IQueueMetrics { - length: Gauge; - droppedJobs: Gauge; - /** - * Total number of seconds spent completing queue jobs - * Useful to compute the utilitzation ratio of this queue with: - * `rate(metrics_name[1m])` - */ - jobTime: Histogram; -} - -export function createQueueMetrics( - metricsOpts: QueueMetricsOpts, - hooks: {getQueueLength: () => number} -): IQueueMetrics | undefined { - const {metrics, prefix} = metricsOpts; - if (!metrics) return; - - return { - length: new Gauge({ - name: `${prefix}_length`, - help: `Count of total queue length of ${prefix}`, - registers: [metrics.registry], - collect() { - this.set(hooks.getQueueLength()); - }, - }), - - droppedJobs: new Gauge({ - name: `${prefix}_dropped_jobs_total`, - help: `Count of total dropped jobs of ${prefix}`, - registers: [metrics.registry], - }), - - jobTime: new Histogram({ - name: `${prefix}_job_time_seconds`, - help: `Time to process queue job of ${prefix} in seconds`, - registers: [metrics.registry], - }), - }; -} diff --git a/packages/lodestar/test/e2e/network/peers/peerManager.test.ts b/packages/lodestar/test/e2e/network/peers/peerManager.test.ts index 99a4b3be3697..8d4a0fce765c 100644 --- a/packages/lodestar/test/e2e/network/peers/peerManager.test.ts +++ b/packages/lodestar/test/e2e/network/peers/peerManager.test.ts @@ -7,7 +7,7 @@ import {IReqResp} from "../../../../src/network/reqresp"; import {PeerRpcScoreStore, PeerManager, Libp2pPeerMetadataStore} from "../../../../src/network/peers"; import {NetworkEvent, NetworkEventBus} from "../../../../src/network"; import {Method} from "../../../../src/constants"; -import {BeaconMetrics} from "../../../../src/metrics"; +import {createMetrics} from "../../../../src/metrics"; import {createNode, getAttnets} from "../../../utils/network"; import {MockBeaconChain} from "../../../utils/mocks/chain/chain"; import {generateEmptySignedBlock} from "../../../utils/block"; @@ -22,7 +22,7 @@ const logger = testLogger(); describe("network / peers / PeerManager", function () { const peerId1 = getValidPeerId(); - const metrics = new BeaconMetrics({enabled: true, timeout: 5000, pushGateway: false}, {logger}); + const metrics = createMetrics(); const afterEachCallbacks: (() => Promise | void)[] = []; afterEach(async () => { diff --git a/packages/lodestar/test/unit/chain/chain.test.ts b/packages/lodestar/test/unit/chain/chain.test.ts index 088536423804..ac3c5a64a4c7 100644 --- a/packages/lodestar/test/unit/chain/chain.test.ts +++ b/packages/lodestar/test/unit/chain/chain.test.ts @@ -6,7 +6,6 @@ import {createCachedBeaconState} from "@chainsafe/lodestar-beacon-state-transiti import {BeaconChain, IBeaconChain} from "../../../src/chain"; import {defaultChainOptions} from "../../../src/chain/options"; -import {BeaconMetrics, IBeaconMetrics} from "../../../src/metrics"; import {generateBlockSummary} from "../../utils/block"; import {generateState} from "../../utils/state"; import {StubbedBeaconDb} from "../../utils/stub"; @@ -15,16 +14,15 @@ import {testLogger} from "../../utils/logger"; describe("BeaconChain", function () { const sandbox = sinon.createSandbox(); - let dbStub: StubbedBeaconDb, metrics: IBeaconMetrics | undefined; + let dbStub: StubbedBeaconDb; const logger = testLogger(); let chain: IBeaconChain; beforeEach(() => { dbStub = new StubbedBeaconDb(sandbox); - metrics = new BeaconMetrics({enabled: false} as any, {logger}); const state = generateState({}, config); dbStub.stateArchive.lastValue.resolves(state as any); - chain = new BeaconChain({opts: defaultChainOptions, config, db: dbStub, logger, metrics, anchorState: state}); + chain = new BeaconChain({opts: defaultChainOptions, config, db: dbStub, logger, anchorState: state}); chain.stateCache = (sandbox.createStubInstance(StateContextCache) as unknown) as StateContextCache; (chain.stateCache as SinonStubbedInstance & StateContextCache).get.returns( createCachedBeaconState(config, state) diff --git a/packages/lodestar/test/unit/metrics/beacon.test.ts b/packages/lodestar/test/unit/metrics/beacon.test.ts index 7ee7088f97db..330283249794 100644 --- a/packages/lodestar/test/unit/metrics/beacon.test.ts +++ b/packages/lodestar/test/unit/metrics/beacon.test.ts @@ -1,23 +1,21 @@ import {expect} from "chai"; -import {BeaconMetrics} from "../../../src/metrics"; -import {testLogger} from "../../utils/logger"; +import {createMetrics} from "../../../src/metrics"; describe("BeaconMetrics", () => { - const logger = testLogger(); - it("updated metrics should be reflected in the registry", async () => { - const m = new BeaconMetrics({enabled: true, timeout: 5000, pushGateway: false, serverPort: 0}, {logger}); - const metricsAsArray = await m.registry.getMetricsAsArray(); - const metricsAsText = await m.registry.metrics(); + it("updated metrics should be reflected in the register", async () => { + const metrics = createMetrics(); + const metricsAsArray = await metrics.register.getMetricsAsArray(); + const metricsAsText = await metrics.register.metrics(); // basic assumptions expect(metricsAsArray.length).to.be.gt(0); expect(metricsAsText).to.not.equal(""); // check updating beacon-specific metrics - expect((await m.registry.getSingleMetricAsString("libp2p_peers")).includes("libp2p_peers 0")); - m.peers.set(1); - expect((await m.registry.getSingleMetricAsString("libp2p_peers")).includes("libp2p_peers 1")); - m.peers.set(20); - expect((await m.registry.getSingleMetricAsString("libp2p_peers")).includes("libp2p_peers 20")); + expect((await metrics.register.getSingleMetricAsString("libp2p_peers")).includes("libp2p_peers 0")); + metrics.peers.set(1); + expect((await metrics.register.getSingleMetricAsString("libp2p_peers")).includes("libp2p_peers 1")); + metrics.peers.set(20); + expect((await metrics.register.getSingleMetricAsString("libp2p_peers")).includes("libp2p_peers 20")); }); }); diff --git a/packages/lodestar/test/unit/metrics/metrics.test.ts b/packages/lodestar/test/unit/metrics/metrics.test.ts index 3c83bb8724c5..a0ac859cc1f9 100644 --- a/packages/lodestar/test/unit/metrics/metrics.test.ts +++ b/packages/lodestar/test/unit/metrics/metrics.test.ts @@ -1,11 +1,11 @@ import {expect} from "chai"; -import {Metrics} from "../../../src/metrics"; +import {createMetrics} from "../../../src/metrics"; describe("Metrics", () => { - it("should get default metrics from registry", async () => { - const m = new Metrics({enabled: true, timeout: 5000, serverPort: 0, pushGateway: false}); - const metricsAsArray = await m.registry.getMetricsAsArray(); - const metricsAsText = await m.registry.metrics(); + it("should get default metrics from register", async () => { + const metrics = createMetrics(); + const metricsAsArray = await metrics.register.getMetricsAsArray(); + const metricsAsText = await metrics.register.metrics(); expect(metricsAsArray.length).to.be.gt(0); expect(metricsAsText).to.not.equal(""); }); diff --git a/packages/lodestar/test/unit/metrics/server/http.test.ts b/packages/lodestar/test/unit/metrics/server/http.test.ts index 9de9ebafaa4e..203a6ee0b732 100644 --- a/packages/lodestar/test/unit/metrics/server/http.test.ts +++ b/packages/lodestar/test/unit/metrics/server/http.test.ts @@ -1,22 +1,21 @@ import request from "supertest"; -import {Metrics, HttpMetricsServer} from "../../../../src/metrics"; +import {createMetrics, HttpMetricsServer} from "../../../../src/metrics"; import {testLogger} from "../../../utils/logger"; describe("HttpMetricsServer", () => { const logger = testLogger(); - let server: HttpMetricsServer; + let server: HttpMetricsServer | null = null; it("should serve metrics on /metrics", async () => { - const options = {enabled: true, timeout: 5000, serverPort: 0, pushGateway: false}; - const metrics = new Metrics(options); - server = new HttpMetricsServer(options, {metrics, logger}); + const metrics = createMetrics(); + server = new HttpMetricsServer({enabled: true, timeout: 5000, serverPort: 0}, {metrics, logger}); await server.start(); await request(server.http).get("/metrics").expect(200); }); after(async () => { - await server.stop(); + if (server) await server.stop(); }); }); diff --git a/packages/lodestar/test/unit/metrics/utils.test.ts b/packages/lodestar/test/unit/metrics/utils.test.ts new file mode 100644 index 000000000000..0a50fe5718de --- /dev/null +++ b/packages/lodestar/test/unit/metrics/utils.test.ts @@ -0,0 +1,76 @@ +import {expect} from "chai"; +import {Gauge, Registry} from "prom-client"; +import {GaugeExtra} from "../../../src/metrics/utils/gauge"; + +type MetricValue = { + value: number; + labels: Record; +}; + +describe("Metrics Gauge collect fn", () => { + const name = "test_metric"; + const help = name; + + async function getMetric(register: Registry): Promise { + const metrics = await register.getMetricsAsJSON(); + const metric = metrics.find((m) => m.name === name); + if (!metric) throw Error(`Metric ${name} not found`); + return ((metric as unknown) as {values: MetricValue[]}).values; + } + + it("Use no collect function", async () => { + const register = new Registry(); + new Gauge({ + name, + help, + registers: [register], + }); + + expect(await getMetric(register)).to.deep.equal([{value: 0, labels: {}}]); + }); + + it("Use collect function in constructor", async () => { + const num = 5; + const register = new Registry(); + new Gauge({ + name, + help, + registers: [register], + collect() { + this.set(num); + }, + }); + + expect(await getMetric(register)).to.deep.equal([{value: num, labels: {}}]); + }); + + it("Override collect function", async () => { + const num = 10; + const register = new Registry(); + const gauge = new Gauge({ + name, + help, + registers: [register], + }); + + (gauge as Gauge & {collect: () => void}).collect = function () { + this.set(num); + }; + + expect(await getMetric(register)).to.deep.equal([{value: num, labels: {}}]); + }); + + it("Override collect function with GaugeCollectable", async () => { + const num = 15; + const register = new Registry(); + const gauge = new GaugeExtra({ + name, + help, + registers: [register], + }); + + gauge.addCollect((g) => g.set(num)); + + expect(await getMetric(register)).to.deep.equal([{value: num, labels: {}}]); + }); +}); From 0c133c5b08f026db457db19c7c612f9e2c013d87 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Wed, 31 Mar 2021 16:44:43 +0200 Subject: [PATCH 2/2] Fix metric name in grafana dashboard --- docker-compose.yml | 2 +- docker/grafana/provisioning/dashboards/lodestar.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index df282e1d03f5..19528c7655b1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ services: ports: - "9000:9000" # P2P port - "9596:9596" # REST API port - command: beacon --rootDir /data --api.rest.host 0.0.0.0 --metrics.enabled true + command: beacon --rootDir /data --api.rest.host 0.0.0.0 --metrics.enabled prometheus: build: docker/prometheus diff --git a/docker/grafana/provisioning/dashboards/lodestar.json b/docker/grafana/provisioning/dashboards/lodestar.json index 05a0894dad07..43afe68b7f08 100644 --- a/docker/grafana/provisioning/dashboards/lodestar.json +++ b/docker/grafana/provisioning/dashboards/lodestar.json @@ -2631,7 +2631,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(lodestar_block_processor_queue_job_time_seconds[$__rate_interval])", + "expr": "rate(lodestar_block_processor_queue_job_time_seconds_sum[$__rate_interval])", "instant": false, "interval": "", "legendFormat": "",