From 63036363e4aeba1bcfbb8a10443e8e2d843a7777 Mon Sep 17 00:00:00 2001 From: Tomasz Kopacki Date: Tue, 10 Feb 2026 19:31:48 +0100 Subject: [PATCH] Integrating "local" Ponder Client with Indexing Status Builder into ENSIndexer API --- .../ponder/src/api/handlers/ensnode-api.ts | 30 ++- .../src/api/lib/chains-config-blockrange.ts | 171 ++++++++++++++++++ .../ponder/src/api/lib/local-ponder-client.ts | 70 +++++++ .../{src/ponder => ponder/src}/config.ts | 0 .../chain-block-refs.ts | 24 +-- apps/ensindexer/tsconfig.json | 3 +- 6 files changed, 277 insertions(+), 21 deletions(-) create mode 100644 apps/ensindexer/ponder/src/api/lib/chains-config-blockrange.ts create mode 100644 apps/ensindexer/ponder/src/api/lib/local-ponder-client.ts rename apps/ensindexer/{src/ponder => ponder/src}/config.ts (100%) diff --git a/apps/ensindexer/ponder/src/api/handlers/ensnode-api.ts b/apps/ensindexer/ponder/src/api/handlers/ensnode-api.ts index e45f9844f..bb1188f2f 100644 --- a/apps/ensindexer/ponder/src/api/handlers/ensnode-api.ts +++ b/apps/ensindexer/ponder/src/api/handlers/ensnode-api.ts @@ -1,6 +1,5 @@ import config from "@/config"; -import { publicClients } from "ponder:api"; import { getUnixTime } from "date-fns"; import { Hono } from "hono"; @@ -15,10 +14,13 @@ import { } from "@ensnode/ensnode-sdk"; import { buildENSIndexerPublicConfig } from "@/config/public"; +import { createCrossChainIndexingStatusSnapshotOmnichain } from "@/lib/indexing-status/build-index-status"; +import { buildOmnichainIndexingStatusSnapshot } from "@/lib/indexing-status-builder/omnichain-indexing-status-snapshot"; import { - buildOmnichainIndexingStatusSnapshot, - createCrossChainIndexingStatusSnapshotOmnichain, -} from "@/lib/indexing-status/build-index-status"; + cachedChainsBlockRefs, + indexedChainIds, + ponderClient, +} from "@/ponder/api/lib/local-ponder-client"; const app = new Hono(); @@ -38,14 +40,26 @@ app.get("/indexing-status", async (c) => { let omnichainSnapshot: OmnichainIndexingStatusSnapshot | undefined; try { - omnichainSnapshot = await buildOmnichainIndexingStatusSnapshot(publicClients); + const [ponderIndexingMetrics, ponderIndexingStatus] = await Promise.all([ + ponderClient.metrics(), + ponderClient.status(), + ]); + + omnichainSnapshot = buildOmnichainIndexingStatusSnapshot( + indexedChainIds, + cachedChainsBlockRefs, + ponderIndexingMetrics, + ponderIndexingStatus, + ); } catch (error) { const errorMessage = error instanceof Error ? error.message : "Unknown error"; - console.error(`Omnichain snapshot is currently not available: ${errorMessage}`); + console.error( + `Indexing Status is currently not available. Failed to fetch Omnichain Indexing Status snapshot: ${errorMessage}`, + ); } // return IndexingStatusResponseError - if (typeof omnichainSnapshot === "undefined") { + if (!omnichainSnapshot) { return c.json( serializeIndexingStatusResponse({ responseCode: IndexingStatusResponseCodes.Error, @@ -53,8 +67,6 @@ app.get("/indexing-status", async (c) => { 500, ); } - - // otherwise, proceed with creating IndexingStatusResponseOk const crossChainSnapshot = createCrossChainIndexingStatusSnapshotOmnichain( omnichainSnapshot, snapshotTime, diff --git a/apps/ensindexer/ponder/src/api/lib/chains-config-blockrange.ts b/apps/ensindexer/ponder/src/api/lib/chains-config-blockrange.ts new file mode 100644 index 000000000..d545fa854 --- /dev/null +++ b/apps/ensindexer/ponder/src/api/lib/chains-config-blockrange.ts @@ -0,0 +1,171 @@ +/** + * This file is about parsing the object that is exported by `ponder.config.ts`. + * + * Each Ponder datasource defined in the aforementioned Ponder Config object + * can include information about startBlock and endBlock. This is to let + * Ponder know which blockrange to index for a particular Ponder Datasource. + * + * ENSIndexer, however, needs a blockrange for each indexed chain. This is why + * we examine Ponder Config object, looking for the "lowest" startBlock, and + * the "highest" endBlock defined for each of the indexed chains. + */ + +import type { AddressConfig, ChainConfig, CreateConfigReturnType } from "ponder"; + +import { + type BlockNumber, + type Blockrange, + type ChainId, + type ChainIdString, + deserializeBlockNumber, + deserializeBlockrange, + deserializeChainId, +} from "@ensnode/ensnode-sdk"; + +/** + * Ponder config datasource with a flat `chain` value. + */ +export type PonderConfigDatasourceFlat = { + chain: ChainIdString; +} & AddressConfig & + Blockrange; + +/** + * Ponder config datasource with a nested `chain` value. + */ +export type PonderConfigDatasourceNested = { + chain: Record; +}; + +/** + * Ponder config datasource + */ +export type PonderConfigDatasource = PonderConfigDatasourceFlat | PonderConfigDatasourceNested; + +/** + * Ponder config datasource + */ +type PonderConfigDatasources = { + [datasourceId: string]: PonderConfigDatasource; +}; + +/** + * Ponder chains config + * + * Chain config for each indexed chain. + */ +type PonderConfigChains = { + [chainId: ChainIdString]: ChainConfig; +}; + +/** + * Ponder Config + * + * A utility type describing Ponder Config. + */ +export type PonderConfigType = CreateConfigReturnType< + PonderConfigChains, + PonderConfigDatasources, + PonderConfigDatasources, + PonderConfigDatasources +>; + +/** + * Ensure the `ponderDatasource` is {@link PonderConfigDatasourceFlat}. + */ +function isPonderDatasourceFlat( + ponderDatasource: PonderConfigDatasource, +): ponderDatasource is PonderConfigDatasourceFlat { + return typeof ponderDatasource.chain === "string"; +} + +/** + * Ensure the `ponderDatasource` is {@link PonderConfigDatasourceNested}. + */ +function isPonderDatasourceNested( + ponderDatasource: PonderConfigDatasource, +): ponderDatasource is PonderConfigDatasourceNested { + return typeof ponderDatasource.chain === "object"; +} + +/** + * Build {@link Blockrange} for each indexed chain. + * + * Invariants: + * - every chain include a startBlock, + * - some chains may include an endBlock, + * - all present startBlock and endBlock values are valid {@link BlockNumber} values. + */ +export function buildChainsBlockrange(ponderConfig: PonderConfigType): Map { + const chainsBlockrange = new Map(); + + // 0. Get all ponder sources (includes chain + startBlock & endBlock) + const ponderSources = [ + ...Object.values(ponderConfig.accounts ?? {}), + ...Object.values(ponderConfig.blocks ?? {}), + ...Object.values(ponderConfig.contracts ?? {}), + ] as PonderConfigDatasource[]; + + // 1. For every indexed chain + for (const serializedChainId of Object.keys(ponderConfig.chains)) { + const chainStartBlocks: BlockNumber[] = []; + const chainEndBlocks: BlockNumber[] = []; + + // 1.1. For every Ponder source (accounts, blocks, contracts), + // extract startBlock number (required) and endBlock number (optional). + for (const ponderSource of ponderSources) { + let startBlock: Blockrange["startBlock"]; + let endBlock: Blockrange["endBlock"]; + + if (isPonderDatasourceFlat(ponderSource) && ponderSource.chain === serializedChainId) { + startBlock = ponderSource.startBlock; + endBlock = ponderSource.endBlock; + } else if (isPonderDatasourceNested(ponderSource) && ponderSource.chain[serializedChainId]) { + startBlock = ponderSource.chain[serializedChainId].startBlock; + endBlock = ponderSource.chain[serializedChainId].endBlock; + } + + if (typeof startBlock === "number") { + chainStartBlocks.push(deserializeBlockNumber(startBlock)); + } + + if (typeof endBlock === "number") { + chainEndBlocks.push(deserializeBlockNumber(endBlock)); + } + } + + // 2. Get the lowest startBlock for the chain. + const chainLowestStartBlock = + chainStartBlocks.length > 0 ? Math.min(...chainStartBlocks) : undefined; + + // 3.a) The endBlock can only be set for a chain if and only if every + // ponderSource for that chain has its respective `endBlock` defined. + const isEndBlockForChainAllowed = chainEndBlocks.length === chainStartBlocks.length; + + // 3.b) Get the highest endBLock for the chain. + const chainHighestEndBlock = + isEndBlockForChainAllowed && chainEndBlocks.length > 0 + ? Math.max(...chainEndBlocks) + : undefined; + + // 4. Enforce invariants + + // Invariant: the indexed chain must have its startBlock defined as number. + if (typeof chainLowestStartBlock === "undefined") { + throw new Error( + `No minimum start block found for chain '${serializedChainId}'. Either all contracts, accounts, and block intervals use "latest" (unsupported) or the chain is misconfigured.`, + ); + } + + // 5. Assign a valid blockrange to the chain + chainsBlockrange.set( + deserializeChainId(serializedChainId), + deserializeBlockrange({ + startBlock: chainLowestStartBlock, + endBlock: chainHighestEndBlock, + }), + ); + } + + return chainsBlockrange; +} diff --git a/apps/ensindexer/ponder/src/api/lib/local-ponder-client.ts b/apps/ensindexer/ponder/src/api/lib/local-ponder-client.ts new file mode 100644 index 000000000..8be28540d --- /dev/null +++ b/apps/ensindexer/ponder/src/api/lib/local-ponder-client.ts @@ -0,0 +1,70 @@ +import config from "@/config"; + +import { publicClients as ponderPublicClients } from "ponder:api"; +import type { PublicClient } from "viem"; + +import { deserializeChainId } from "@ensnode/ensnode-sdk"; +import { type ChainId, PonderClient } from "@ensnode/ponder-sdk"; + +import type { ChainBlockRefs } from "@/lib/indexing-status/ponder-metadata"; +import { fetchChainsBlockRefs } from "@/lib/indexing-status-builder/chain-block-refs"; +import { buildChainsBlockrange } from "@/ponder/api/lib/chains-config-blockrange"; +import ponderConfig from "@/ponder/config"; + +/** + * Cached PublicClient instances for indexed chains, keyed by chain ID. + */ +const publicClients = new Map( + Object.entries(ponderPublicClients).map(([chainId, publicClient]) => [ + deserializeChainId(chainId), + publicClient, + ]), +); + +/** + * Configured block range for indexed chains, based on Ponder Config. + */ +const chainsConfigBlockrange = buildChainsBlockrange(ponderConfig); + +/** + * Indexed chain IDs, based on Ponder Config. + */ +export const indexedChainIds = Array.from(publicClients.keys()) satisfies ChainId[]; + +/** + * PonderClient instance for fetching indexing status and metrics from + * Ponder API. + */ +export const ponderClient = new PonderClient(config.ensIndexerUrl); + +/** + * Cached block references based on Ponder Config and Ponder API metrics, + * keyed by chain ID. + * + * Note: this value is initialized on application startup, and is + * not expected to change during runtime, as it is based on + * the configured block range and block references fetched + * from RPCs at startup. + */ +// TODO: this operation may fail, so it should be wrapped in auto-retry logic. +// pRetry could be used, with a retry strategy that includes +// exponential backoff and jitter, to avoid overwhelming the RPC endpoints +// in case of transient errors or rate limits. +export const cachedChainsBlockRefs = await initializeCachedChainsBlockRefs(); + +async function initializeCachedChainsBlockRefs(): Promise>> { + const ponderIndexingMetrics = await ponderClient.metrics(); + + const chainsBlockRefs = await fetchChainsBlockRefs( + indexedChainIds, + chainsConfigBlockrange, + ponderIndexingMetrics.chains, + publicClients, + ); + + if (chainsBlockRefs.size === 0) { + throw new Error("Failed to fetch chainsBlockRefs: no block refs found"); + } + + return Object.freeze(chainsBlockRefs); +} diff --git a/apps/ensindexer/src/ponder/config.ts b/apps/ensindexer/ponder/src/config.ts similarity index 100% rename from apps/ensindexer/src/ponder/config.ts rename to apps/ensindexer/ponder/src/config.ts diff --git a/apps/ensindexer/src/lib/indexing-status-builder/chain-block-refs.ts b/apps/ensindexer/src/lib/indexing-status-builder/chain-block-refs.ts index b4d54e595..45a705340 100644 --- a/apps/ensindexer/src/lib/indexing-status-builder/chain-block-refs.ts +++ b/apps/ensindexer/src/lib/indexing-status-builder/chain-block-refs.ts @@ -56,41 +56,43 @@ export interface ChainBlockRefs { } /** - * Get {@link IndexedChainBlockRefs} for indexed chains. + * Fetch {@link ChainBlockRefs} for indexed chains. * * Guaranteed to include {@link ChainBlockRefs} for each indexed chain. */ -export async function getChainsBlockRefs( +export async function fetchChainsBlockRefs( chainIds: ChainId[], - chainsConfigBlockrange: Record, + chainsConfigBlockrange: Map, chainsIndexingMetrics: Map, - publicClients: Record, + publicClients: Map, ): Promise> { const chainsBlockRefs = new Map(); for (const chainId of chainIds) { - const blockrange = chainsConfigBlockrange[chainId]; + const blockrange = chainsConfigBlockrange.get(chainId); const startBlock = blockrange?.startBlock; const endBlock = blockrange?.endBlock; - const publicClient = publicClients[chainId]; - const indexingMetrics = chainsIndexingMetrics.get(chainId); if (typeof startBlock !== "number") { - throw new Error(`startBlock not found for chain ${chainId}`); + throw new Error(`startBlock must be defined for chain ${chainId}`); } + const publicClient = publicClients.get(chainId); + if (typeof publicClient === "undefined") { - throw new Error(`publicClient not found for chain ${chainId}`); + throw new Error(`publicClient must be defined for chain ${chainId}`); } + const indexingMetrics = chainsIndexingMetrics.get(chainId); + if (typeof indexingMetrics === "undefined") { - throw new Error(`indexingMetrics not found for chain ${chainId}`); + throw new Error(`indexingMetrics must be defined for chain ${chainId}`); } const historicalTotalBlocks = indexingMetrics.backfillSyncBlocksTotal; if (typeof historicalTotalBlocks !== "number") { - throw new Error(`No historical total blocks metric found for chain ${chainId}`); + throw new Error(`historicalTotalBlocks must be defined for chain ${chainId}`); } const backfillEndBlock = startBlock + historicalTotalBlocks - 1; diff --git a/apps/ensindexer/tsconfig.json b/apps/ensindexer/tsconfig.json index c9a80eb67..44dbb8ae9 100644 --- a/apps/ensindexer/tsconfig.json +++ b/apps/ensindexer/tsconfig.json @@ -2,7 +2,8 @@ "extends": "@ensnode/shared-configs/tsconfig.ponder.json", "compilerOptions": { "paths": { - "@/*": ["./src/*"] + "@/*": ["./src/*"], + "@/ponder/*": ["./ponder/src/*"] }, "typeRoots": ["./types"] },