feat(store-sync,store-indexer): add followBlockTag option (#2315)

latticexyz · Feb 27, 2024 · 3622e39 · 3622e39
1 parent 3be4dee
commit 3622e39
Show file tree

Hide file tree

Showing 8 changed files with 20 additions and 2 deletions.
diff --git a/.changeset/brave-ghosts-walk.md b/.changeset/brave-ghosts-walk.md
@@ -0,0 +1,8 @@
+---
+"@latticexyz/store-indexer": minor
+"@latticexyz/store-sync": minor
+---
+
+Added a `followBlockTag` option to configure which block number to follow when running `createStoreSync`. It defaults to `latest` (current behavior), which is recommended for individual clients so that you always have the latest chain state.
+
+Indexers now default to `safe` to avoid issues with reorgs and load-balanced RPCs being out of sync. This means indexers will be slightly behind the latest block number, but clients can quickly catch up. Indexers can override this setting using `FOLLOW_BLOCK_TAG` environment variable.
diff --git a/e2e/packages/sync-test/setup/startIndexer.ts b/e2e/packages/sync-test/setup/startIndexer.ts
@@ -37,6 +37,7 @@ export async function startIndexer(opts: StartIndexerOptions) {
     RPC_HTTP_URL: opts.rpcHttpUrl,
     SQLITE_FILENAME: opts.indexer === "sqlite" ? opts.sqliteFilename : undefined,
     DATABASE_URL: opts.indexer === "postgres" ? opts.databaseUrl : undefined,
+    FOLLOW_BLOCK_TAG: "latest",
   };
   console.log(chalk.magenta("[indexer]:"), "starting indexer", env);
 

diff --git a/packages/store-indexer/bin/parseEnv.ts b/packages/store-indexer/bin/parseEnv.ts
@@ -8,6 +8,7 @@ export const frontendEnvSchema = z.object({
 
 export const indexerEnvSchema = z.intersection(
   z.object({
+    FOLLOW_BLOCK_TAG: z.enum(["latest", "safe", "finalized"]).default("safe"),
     START_BLOCK: z.coerce.bigint().nonnegative().default(0n),
     MAX_BLOCK_RANGE: z.coerce.bigint().positive().default(1000n),
     POLLING_INTERVAL: z.coerce.number().positive().default(1000),

diff --git a/packages/store-indexer/bin/postgres-decoded-indexer.ts b/packages/store-indexer/bin/postgres-decoded-indexer.ts
@@ -69,6 +69,7 @@ try {
 const { latestBlockNumber$, storedBlockLogs$ } = await createStoreSync({
   storageAdapter,
   publicClient,
+  followBlockTag: env.FOLLOW_BLOCK_TAG,
   startBlock,
   maxBlockRange: env.MAX_BLOCK_RANGE,
   address: env.STORE_ADDRESS,

diff --git a/packages/store-indexer/bin/postgres-indexer.ts b/packages/store-indexer/bin/postgres-indexer.ts
@@ -70,6 +70,7 @@ try {
 const { latestBlockNumber$, storedBlockLogs$ } = await createStoreSync({
   storageAdapter,
   publicClient,
+  followBlockTag: env.FOLLOW_BLOCK_TAG,
   startBlock,
   maxBlockRange: env.MAX_BLOCK_RANGE,
   address: env.STORE_ADDRESS,

diff --git a/packages/store-indexer/bin/sqlite-indexer.ts b/packages/store-indexer/bin/sqlite-indexer.ts
@@ -75,6 +75,7 @@ try {
 const { latestBlockNumber$, storedBlockLogs$ } = await syncToSqlite({
   database,
   publicClient,
+  followBlockTag: env.FOLLOW_BLOCK_TAG,
   startBlock,
   maxBlockRange: env.MAX_BLOCK_RANGE,
   address: env.STORE_ADDRESS,

diff --git a/packages/store-sync/src/common.ts b/packages/store-sync/src/common.ts
@@ -77,6 +77,10 @@ export type SyncOptions<TConfig extends StoreConfig = StoreConfig> = {
    * @deprecated Use `filters` option instead.
    * */
   tableIds?: Hex[];
+  /**
+   * Optional block tag to follow for the latest block number. Defaults to `latest`. It's recommended to use `safe` for indexers.
+   */
+  followBlockTag?: "latest" | "safe" | "finalized";
   /**
    * Optional block number to start indexing from. Useful for resuming the indexer from a particular point in time or starting after a particular contract deployment.
    */

diff --git a/packages/store-sync/src/createStoreSync.ts b/packages/store-sync/src/createStoreSync.ts
@@ -55,6 +55,7 @@ export async function createStoreSync<TConfig extends StoreConfig = StoreConfig>
   address,
   filters: initialFilters = [],
   tableIds = [],
+  followBlockTag = "latest",
   startBlock: initialStartBlock = 0n,
   maxBlockRange,
   initialState,
@@ -176,11 +177,11 @@ export async function createStoreSync<TConfig extends StoreConfig = StoreConfig>
     tap((startBlock) => debug("starting sync from block", startBlock))
   );
 
-  const latestBlock$ = createBlockStream({ publicClient, blockTag: "latest" }).pipe(shareReplay(1));
+  const latestBlock$ = createBlockStream({ publicClient, blockTag: followBlockTag }).pipe(shareReplay(1));
   const latestBlockNumber$ = latestBlock$.pipe(
     map((block) => block.number),
     tap((blockNumber) => {
-      debug("latest block number", blockNumber);
+      debug("on block number", blockNumber, "for", followBlockTag, "block tag");
     }),
     shareReplay(1)
   );