feat(store-sync): add flag to define chunking behavior during initial hydration (#3745)

Co-authored-by: Kevin Ingersoll <kingersoll@gmail.com>

Author: alvrs

.changeset/rotten-windows-dance.md

@@ -0,0 +1,9 @@
+---
+"@latticexyz/store-sync": patch
+---
+
+The sync stack now supports defining the chunking behavior during initial hydration. Chunking remains enabled by default.
+
+Chunking is useful to avoid blocking the main thread for too long, but it can lead to updates that happened in the same block being split across multiple chunks.
+If chunking is enabled, clients should account for this by waiting for full hydration before using the update stream.
+If atomicity of updates is important and blocking the main thread is not an issue, set this to `false`.

packages/store-sync/src/common.ts

@@ -110,6 +110,13 @@ export type SyncOptions = GetRpcClientOptions & {
     blockNumber: bigint;
     logs: readonly StorageAdapterLog[];
   };
+  /**
+   * Optional flag to define the chunking behavior during the initial hydration. Defaults to `true`.
+   * Chunking is useful to avoid blocking the main thread for too long, but it can lead to updates that happened in the same block being split across multiple chunks.
+   * If chunking is enabled, clients should account for this by waiting for full hydration before using the update stream.
+   * If atomicity of updates is important and blocking the main thread is not an issue, set this to `false`.
+   */
+  enableHydrationChunking?: boolean;
 };
 
 export type WaitForTransactionResult = Pick<TransactionReceipt, "blockNumber" | "status" | "transactionHash">;

packages/store-sync/src/createStoreSync.ts

@@ -72,6 +72,7 @@ export async function createStoreSync({
   maxBlockRange,
   initialState,
   initialBlockLogs,
+  enableHydrationChunking = true,
   ...opts
 }: CreateStoreSyncOptions): Promise<SyncResult> {
   const filters: SyncFilter[] =
@@ -155,26 +156,31 @@ export async function createStoreSync({
         message: "Hydrating from snapshot",
       });
 
-      // Split snapshot operations into chunks so we can update the progress callback (and ultimately render visual progress for the user).
-      // This isn't ideal if we want to e.g. batch load these into a DB in a single DB tx, but we'll take it.
-      //
-      // Split into 50 equal chunks (for better `onProgress` updates) but only if we have 100+ items per chunk
-      const chunkSize = Math.max(100, Math.floor(logs.length / 50));
-      const chunks = Array.from(chunk(logs, chunkSize));
-      for (const [i, chunk] of chunks.entries()) {
-        await storageAdapter({ blockNumber, logs: chunk });
-        onProgress?.({
-          step: SyncStep.SNAPSHOT,
-          percentage: ((i + 1) / chunks.length) * 100,
-          latestBlockNumber: 0n,
-          lastBlockNumberProcessed: blockNumber,
-          message: "Hydrating from snapshot",
-        });
-
-        // RECS is a synchronous API so hydrating in a loop like this blocks downstream render cycles
-        // that would display the percentage climbing up to 100.
-        // We wait for idle callback here to give rendering a chance to complete.
-        await waitForIdle();
+      if (enableHydrationChunking) {
+        // Split snapshot operations into chunks so we can update the progress callback (and ultimately render visual progress for the user).
+        // This isn't ideal if we want to e.g. batch load these into a DB in a single DB tx, but we'll take it.
+        //
+        // Split into 50 equal chunks (for better `onProgress` updates) but only if we have 100+ items per chunk
+        const chunkSize = Math.max(100, Math.floor(logs.length / 50));
+        const chunks = Array.from(chunk(logs, chunkSize));
+        for (const [i, chunk] of chunks.entries()) {
+          debug(`hydrating chunk ${i}/${chunks.length}`);
+          await storageAdapter({ blockNumber, logs: chunk });
+          onProgress?.({
+            step: SyncStep.SNAPSHOT,
+            percentage: ((i + 1) / chunks.length) * 100,
+            latestBlockNumber: 0n,
+            lastBlockNumberProcessed: blockNumber,
+            message: "Hydrating from snapshot",
+          });
+          // RECS is a synchronous API so hydrating in a loop like this blocks downstream render cycles
+          // that would display the percentage climbing up to 100.
+          // We wait for idle callback here to give rendering a chance to complete.
+          await waitForIdle();
+        }
+      } else {
+        debug("hydrating all logs without chunking");
+        await storageAdapter({ blockNumber, logs });
       }
 
       onProgress?.({