[WIP] Integration of Indexing Status Builder into ENSIndexer API

apps/ensindexer/ponder/src/api/handlers/ensnode-api.ts

@@ -1,6 +1,5 @@
 import config from "@/config";
 
-import { publicClients } from "ponder:api";
 import { getUnixTime } from "date-fns";
 import { Hono } from "hono";
 
@@ -15,10 +14,13 @@ import {
 } from "@ensnode/ensnode-sdk";
 
 import { buildENSIndexerPublicConfig } from "@/config/public";
+import { createCrossChainIndexingStatusSnapshotOmnichain } from "@/lib/indexing-status/build-index-status";
+import { buildOmnichainIndexingStatusSnapshot } from "@/lib/indexing-status-builder/omnichain-indexing-status-snapshot";
 import {
-  buildOmnichainIndexingStatusSnapshot,
-  createCrossChainIndexingStatusSnapshotOmnichain,
-} from "@/lib/indexing-status/build-index-status";
+  cachedChainsBlockRefs,
+  indexedChainIds,
+  ponderClient,
+} from "@/ponder/api/lib/local-ponder-client";
 
 const app = new Hono();
 
@@ -38,23 +40,33 @@ app.get("/indexing-status", async (c) => {
   let omnichainSnapshot: OmnichainIndexingStatusSnapshot | undefined;
 
   try {
-    omnichainSnapshot = await buildOmnichainIndexingStatusSnapshot(publicClients);
+    const [ponderIndexingMetrics, ponderIndexingStatus] = await Promise.all([
+      ponderClient.metrics(),
+      ponderClient.status(),
+    ]);
+
+    omnichainSnapshot = buildOmnichainIndexingStatusSnapshot(
+      indexedChainIds,
+      cachedChainsBlockRefs,
+      ponderIndexingMetrics,
+      ponderIndexingStatus,
+    );
   } catch (error) {
     const errorMessage = error instanceof Error ? error.message : "Unknown error";
-    console.error(`Omnichain snapshot is currently not available: ${errorMessage}`);
+    console.error(
+      `Indexing Status is currently not available. Failed to fetch Omnichain Indexing Status snapshot: ${errorMessage}`,
+    );
   }
 
   // return IndexingStatusResponseError
-  if (typeof omnichainSnapshot === "undefined") {
+  if (!omnichainSnapshot) {
     return c.json(
       serializeIndexingStatusResponse({
         responseCode: IndexingStatusResponseCodes.Error,
       } satisfies IndexingStatusResponseError),
       500,
     );
   }
-
-  // otherwise, proceed with creating IndexingStatusResponseOk
   const crossChainSnapshot = createCrossChainIndexingStatusSnapshotOmnichain(
     omnichainSnapshot,
     snapshotTime,

apps/ensindexer/ponder/src/api/lib/chains-config-blockrange.ts

@@ -0,0 +1,171 @@
+/**
+ * This file is about parsing the object that is exported by `ponder.config.ts`.
+ *
+ * Each Ponder datasource defined in the aforementioned Ponder Config object
+ * can include information about startBlock and endBlock. This is to let
+ * Ponder know which blockrange to index for a particular Ponder Datasource.
+ *
+ * ENSIndexer, however, needs a blockrange for each indexed chain. This is why
+ * we examine Ponder Config object, looking for the "lowest" startBlock, and
+ * the "highest" endBlock defined for each of the indexed chains.
+ */
+
+import type { AddressConfig, ChainConfig, CreateConfigReturnType } from "ponder";
+
+import {
+  type BlockNumber,
+  type Blockrange,
+  type ChainId,
+  type ChainIdString,
+  deserializeBlockNumber,
+  deserializeBlockrange,
+  deserializeChainId,
+} from "@ensnode/ensnode-sdk";
+
+/**
+ * Ponder config datasource with a flat `chain` value.
+ */
+export type PonderConfigDatasourceFlat = {
+  chain: ChainIdString;
+} & AddressConfig &
+  Blockrange;
+
+/**
+ * Ponder config datasource with a nested `chain` value.
+ */
+export type PonderConfigDatasourceNested = {
+  chain: Record<ChainIdString, AddressConfig & Blockrange>;
+};
+
+/**
+ * Ponder config datasource
+ */
+export type PonderConfigDatasource = PonderConfigDatasourceFlat | PonderConfigDatasourceNested;
+
+/**
+ * Ponder config datasource
+ */
+type PonderConfigDatasources = {
+  [datasourceId: string]: PonderConfigDatasource;
+};
+
+/**
+ * Ponder chains config
+ *
+ * Chain config for each indexed chain.
+ */
+type PonderConfigChains = {
+  [chainId: ChainIdString]: ChainConfig;
+};
+
+/**
+ * Ponder Config
+ *
+ * A utility type describing Ponder Config.
+ */
+export type PonderConfigType = CreateConfigReturnType<
+  PonderConfigChains,
+  PonderConfigDatasources,
+  PonderConfigDatasources,
+  PonderConfigDatasources
+>;
+
+/**
+ * Ensure the `ponderDatasource` is {@link PonderConfigDatasourceFlat}.
+ */
+function isPonderDatasourceFlat(
+  ponderDatasource: PonderConfigDatasource,
+): ponderDatasource is PonderConfigDatasourceFlat {
+  return typeof ponderDatasource.chain === "string";
+}
+
+/**
+ * Ensure the `ponderDatasource` is {@link PonderConfigDatasourceNested}.
+ */
+function isPonderDatasourceNested(
+  ponderDatasource: PonderConfigDatasource,
+): ponderDatasource is PonderConfigDatasourceNested {
+  return typeof ponderDatasource.chain === "object";
+}
+
+/**
+ * Build {@link Blockrange} for each indexed chain.
+ *
+ * Invariants:
+ * - every chain include a startBlock,
+ * - some chains may include an endBlock,
+ * - all present startBlock and endBlock values are valid {@link BlockNumber} values.
+ */
+export function buildChainsBlockrange(ponderConfig: PonderConfigType): Map<ChainId, Blockrange> {
+  const chainsBlockrange = new Map<ChainId, Blockrange>();
+
+  // 0. Get all ponder sources (includes chain + startBlock & endBlock)
+  const ponderSources = [
+    ...Object.values(ponderConfig.accounts ?? {}),
+    ...Object.values(ponderConfig.blocks ?? {}),
+    ...Object.values(ponderConfig.contracts ?? {}),
+  ] as PonderConfigDatasource[];
+
+  // 1. For every indexed chain
+  for (const serializedChainId of Object.keys(ponderConfig.chains)) {
+    const chainStartBlocks: BlockNumber[] = [];
+    const chainEndBlocks: BlockNumber[] = [];
+
+    // 1.1. For every Ponder source (accounts, blocks, contracts),
+    //      extract startBlock number (required) and endBlock number (optional).
+    for (const ponderSource of ponderSources) {
+      let startBlock: Blockrange["startBlock"];
+      let endBlock: Blockrange["endBlock"];
+
+      if (isPonderDatasourceFlat(ponderSource) && ponderSource.chain === serializedChainId) {
+        startBlock = ponderSource.startBlock;
+        endBlock = ponderSource.endBlock;
+      } else if (isPonderDatasourceNested(ponderSource) && ponderSource.chain[serializedChainId]) {
+        startBlock = ponderSource.chain[serializedChainId].startBlock;
+        endBlock = ponderSource.chain[serializedChainId].endBlock;
+      }
+
+      if (typeof startBlock === "number") {
+        chainStartBlocks.push(deserializeBlockNumber(startBlock));
+      }
+
+      if (typeof endBlock === "number") {
+        chainEndBlocks.push(deserializeBlockNumber(endBlock));
+      }
+    }
+
+    // 2. Get the lowest startBlock for the chain.
+    const chainLowestStartBlock =
+      chainStartBlocks.length > 0 ? Math.min(...chainStartBlocks) : undefined;
+
+    // 3.a) The endBlock can only be set for a chain if and only if every
+    //      ponderSource for that chain has its respective `endBlock` defined.
+    const isEndBlockForChainAllowed = chainEndBlocks.length === chainStartBlocks.length;
+
+    // 3.b) Get the highest endBLock for the chain.
+    const chainHighestEndBlock =
+      isEndBlockForChainAllowed && chainEndBlocks.length > 0
+        ? Math.max(...chainEndBlocks)
+        : undefined;
+
+    // 4. Enforce invariants
+
+    // Invariant: the indexed chain must have its startBlock defined as number.
+    if (typeof chainLowestStartBlock === "undefined") {
+      throw new Error(
+        `No minimum start block found for chain '${serializedChainId}'. Either all contracts, accounts, and block intervals use "latest" (unsupported) or the chain is misconfigured.`,
+      );
+    }
+
+    // 5. Assign a valid blockrange to the chain
+    chainsBlockrange.set(
+      deserializeChainId(serializedChainId),
+      deserializeBlockrange({
+        startBlock: chainLowestStartBlock,
+        endBlock: chainHighestEndBlock,
+      }),
+    );
+  }
+
+  return chainsBlockrange;
+}

apps/ensindexer/ponder/src/api/lib/local-ponder-client.ts

@@ -0,0 +1,70 @@
+import config from "@/config";
+
+import { publicClients as ponderPublicClients } from "ponder:api";
+import type { PublicClient } from "viem";
+
+import { deserializeChainId } from "@ensnode/ensnode-sdk";
+import { type ChainId, PonderClient } from "@ensnode/ponder-sdk";
+
+import type { ChainBlockRefs } from "@/lib/indexing-status/ponder-metadata";
+import { fetchChainsBlockRefs } from "@/lib/indexing-status-builder/chain-block-refs";
+import { buildChainsBlockrange } from "@/ponder/api/lib/chains-config-blockrange";
+import ponderConfig from "@/ponder/config";
+
+/**
+ * Cached PublicClient instances for indexed chains, keyed by chain ID.
+ */
+const publicClients = new Map<ChainId, PublicClient>(
+  Object.entries(ponderPublicClients).map(([chainId, publicClient]) => [
+    deserializeChainId(chainId),
+    publicClient,
+  ]),
+);
+
+/**
+ * Configured block range for indexed chains, based on Ponder Config.
+ */
+const chainsConfigBlockrange = buildChainsBlockrange(ponderConfig);
+
+/**
+ * Indexed chain IDs, based on Ponder Config.
+ */
+export const indexedChainIds = Array.from(publicClients.keys()) satisfies ChainId[];
+
+/**
+ * PonderClient instance for fetching indexing status and metrics from
+ * Ponder API.
+ */
+export const ponderClient = new PonderClient(config.ensIndexerUrl);
+
+/**
+ * Cached block references based on Ponder Config and Ponder API metrics,
+ * keyed by chain ID.
+ *
+ * Note: this value is initialized on application startup, and is
+ * not expected to change during runtime, as it is based on
+ * the configured block range and block references fetched
+ * from RPCs at startup.
+ */
+// TODO: this operation may fail, so it should be wrapped in auto-retry logic.
+// pRetry could be used, with a retry strategy that includes
+// exponential backoff and jitter, to avoid overwhelming the RPC endpoints
+// in case of transient errors or rate limits.
+export const cachedChainsBlockRefs = await initializeCachedChainsBlockRefs();
+
+async function initializeCachedChainsBlockRefs(): Promise<Readonly<Map<ChainId, ChainBlockRefs>>> {
+  const ponderIndexingMetrics = await ponderClient.metrics();
+
+  const chainsBlockRefs = await fetchChainsBlockRefs(
+    indexedChainIds,
+    chainsConfigBlockrange,
+    ponderIndexingMetrics.chains,
+    publicClients,
+  );
+
+  if (chainsBlockRefs.size === 0) {
+    throw new Error("Failed to fetch chainsBlockRefs: no block refs found");
+  }
+
+  return Object.freeze(chainsBlockRefs);
+}

apps/ensindexer/src/lib/indexing-status-builder/chain-block-refs.ts

@@ -56,41 +56,43 @@ export interface ChainBlockRefs {
 }
 
 /**
- * Get {@link IndexedChainBlockRefs} for indexed chains.
+ * Fetch {@link ChainBlockRefs} for indexed chains.
  *
  * Guaranteed to include {@link ChainBlockRefs} for each indexed chain.
  */
-export async function getChainsBlockRefs(
+export async function fetchChainsBlockRefs(
   chainIds: ChainId[],
-  chainsConfigBlockrange: Record<ChainId, Blockrange>,
+  chainsConfigBlockrange: Map<ChainId, Blockrange>,
   chainsIndexingMetrics: Map<ChainId, ChainIndexingMetrics>,
-  publicClients: Record<ChainId, PublicClient>,
+  publicClients: Map<ChainId, PublicClient>,
 ): Promise<Map<ChainId, ChainBlockRefs>> {
   const chainsBlockRefs = new Map<ChainId, ChainBlockRefs>();
 
   for (const chainId of chainIds) {
-    const blockrange = chainsConfigBlockrange[chainId];
+    const blockrange = chainsConfigBlockrange.get(chainId);
     const startBlock = blockrange?.startBlock;
     const endBlock = blockrange?.endBlock;
-    const publicClient = publicClients[chainId];
-    const indexingMetrics = chainsIndexingMetrics.get(chainId);
 
     if (typeof startBlock !== "number") {
-      throw new Error(`startBlock not found for chain ${chainId}`);
+      throw new Error(`startBlock must be defined for chain ${chainId}`);
     }
 
+    const publicClient = publicClients.get(chainId);
+
     if (typeof publicClient === "undefined") {
-      throw new Error(`publicClient not found for chain ${chainId}`);
+      throw new Error(`publicClient must be defined for chain ${chainId}`);
     }
 
+    const indexingMetrics = chainsIndexingMetrics.get(chainId);
+
     if (typeof indexingMetrics === "undefined") {
-      throw new Error(`indexingMetrics not found for chain ${chainId}`);
+      throw new Error(`indexingMetrics must be defined for chain ${chainId}`);
     }
 
     const historicalTotalBlocks = indexingMetrics.backfillSyncBlocksTotal;
 
     if (typeof historicalTotalBlocks !== "number") {
-      throw new Error(`No historical total blocks metric found for chain ${chainId}`);
+      throw new Error(`historicalTotalBlocks must be defined for chain ${chainId}`);
     }
 
     const backfillEndBlock = startBlock + historicalTotalBlocks - 1;

apps/ensindexer/tsconfig.json

@@ -2,7 +2,8 @@
   "extends": "@ensnode/shared-configs/tsconfig.ponder.json",
   "compilerOptions": {
     "paths": {
-      "@/*": ["./src/*"]
+      "@/*": ["./src/*"],
+      "@/ponder/*": ["./ponder/src/*"]
     },
     "typeRoots": ["./types"]
   },