feat: allow blob archival for bigger time periods (#6393)

* feat: allow blob archival for bigger time periods * fix spellcheck and spellings * change cli flag and desc
ChainSafe · Feb 16, 2024 · 6a68b1e · 6a68b1e
1 parent 0e01efd
commit 6a68b1e
Show file tree

Hide file tree

Showing 5 changed files with 35 additions and 13 deletions.
diff --git a/packages/beacon-node/src/chain/archiver/archiveBlocks.ts b/packages/beacon-node/src/chain/archiver/archiveBlocks.ts
@@ -35,7 +35,8 @@ export async function archiveBlocks(
   lightclientServer: LightClientServer,
   logger: Logger,
   finalizedCheckpoint: CheckpointHex,
-  currentEpoch: Epoch
+  currentEpoch: Epoch,
+  archiveBlobEpochs?: number
 ): Promise<void> {
   // Use fork choice to determine the blocks to archive and delete
   // getAllAncestorBlocks response includes the finalized block, so it's also moved to the cold db
@@ -81,19 +82,25 @@ export async function archiveBlocks(
   }
 
   // Delete expired blobs
-  // Keep only `[max(GENESIS_EPOCH, current_epoch - MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS), current_epoch]`
+  // Keep only `[current_epoch - max(MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS, archiveBlobEpochs)]
+  // if archiveBlobEpochs set to Infinity do not prune`
   if (finalizedPostDeneb) {
-    const blobSidecarsMinEpoch = currentEpoch - config.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS;
-    if (blobSidecarsMinEpoch >= config.DENEB_FORK_EPOCH) {
-      const slotsToDelete = await db.blobSidecarsArchive.keys({lt: computeStartSlotAtEpoch(blobSidecarsMinEpoch)});
-      if (slotsToDelete.length > 0) {
-        await db.blobSidecarsArchive.batchDelete(slotsToDelete);
-        logger.verbose(
-          `blobSidecars prune: batchDelete range ${slotsToDelete[0]}..${slotsToDelete[slotsToDelete.length - 1]}`
-        );
-      } else {
-        logger.verbose(`blobSidecars prune: no entries before epoch ${blobSidecarsMinEpoch}`);
+    if (archiveBlobEpochs !== Infinity) {
+      const blobsArchiveWindow = Math.max(config.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS, archiveBlobEpochs ?? 0);
+      const blobSidecarsMinEpoch = currentEpoch - blobsArchiveWindow;
+      if (blobSidecarsMinEpoch >= config.DENEB_FORK_EPOCH) {
+        const slotsToDelete = await db.blobSidecarsArchive.keys({lt: computeStartSlotAtEpoch(blobSidecarsMinEpoch)});
+        if (slotsToDelete.length > 0) {
+          await db.blobSidecarsArchive.batchDelete(slotsToDelete);
+          logger.verbose(
+            `blobSidecars prune: batchDelete range ${slotsToDelete[0]}..${slotsToDelete[slotsToDelete.length - 1]}`
+          );
+        } else {
+          logger.verbose(`blobSidecars prune: no entries before epoch ${blobSidecarsMinEpoch}`);
+        }
       }
+    } else {
+      logger.verbose("blobSidecars pruning skipped: archiveBlobEpochs set to Infinity");
     }
   }
 

diff --git a/packages/beacon-node/src/chain/archiver/index.ts b/packages/beacon-node/src/chain/archiver/index.ts
@@ -11,6 +11,7 @@ const PROCESS_FINALIZED_CHECKPOINT_QUEUE_LEN = 256;
 
 export type ArchiverOpts = StatesArchiverOpts & {
   disableArchiveOnCheckpoint?: boolean;
+  archiveBlobEpochs?: number;
 };
 
 type ProposalStats = {
@@ -37,6 +38,7 @@ export class Archiver {
 
   private prevFinalized: CheckpointWithHex;
   private readonly statesArchiver: StatesArchiver;
+  private archiveBlobEpochs?: number;
 
   constructor(
     private readonly db: IBeaconDb,
@@ -45,6 +47,7 @@ export class Archiver {
     signal: AbortSignal,
     opts: ArchiverOpts
   ) {
+    this.archiveBlobEpochs = opts.archiveBlobEpochs;
     this.statesArchiver = new StatesArchiver(chain.regen, db, logger, opts);
     this.prevFinalized = chain.forkChoice.getFinalizedCheckpoint();
     this.jobQueue = new JobItemQueue<[CheckpointWithHex], void>(this.processFinalizedCheckpoint, {
@@ -96,7 +99,8 @@ export class Archiver {
         this.chain.lightClientServer,
         this.logger,
         finalized,
-        this.chain.clock.currentEpoch
+        this.chain.clock.currentEpoch,
+        this.archiveBlobEpochs
       );
       this.prevFinalized = finalized;
 

diff --git a/packages/beacon-node/src/chain/options.ts b/packages/beacon-node/src/chain/options.ts
@@ -30,6 +30,7 @@ export type IChainOptions = BlockProcessOpts &
     trustedSetup?: string;
     broadcastValidationStrictness?: string;
     minSameMessageSignatureSetsToBatch: number;
+    archiveBlobEpochs?: number;
   };
 
 export type BlockProcessOpts = {

diff --git a/packages/cli/src/options/beaconNodeOptions/chain.ts b/packages/cli/src/options/beaconNodeOptions/chain.ts
@@ -26,6 +26,7 @@ export type ChainArgs = {
   broadcastValidationStrictness?: string;
   "chain.minSameMessageSignatureSetsToBatch"?: number;
   "chain.maxShufflingCacheEpochs"?: number;
+  "chain.archiveBlobEpochs"?: number;
 };
 
 export function parseArgs(args: ChainArgs): IBeaconNodeOptions["chain"] {
@@ -53,6 +54,7 @@ export function parseArgs(args: ChainArgs): IBeaconNodeOptions["chain"] {
     minSameMessageSignatureSetsToBatch:
       args["chain.minSameMessageSignatureSetsToBatch"] ?? defaultOptions.chain.minSameMessageSignatureSetsToBatch,
     maxShufflingCacheEpochs: args["chain.maxShufflingCacheEpochs"] ?? defaultOptions.chain.maxShufflingCacheEpochs,
+    archiveBlobEpochs: args["chain.archiveBlobEpochs"],
   };
 }
 
@@ -212,4 +214,10 @@ Will double processing times. Use only for debugging purposes.",
     default: defaultOptions.chain.maxShufflingCacheEpochs,
     group: "chain",
   },
+
+  "chain.archiveBlobEpochs": {
+    description: "Number of epochs to retain finalized blobs (minimum of MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS)",
+    type: "number",
+    group: "chain",
+  },
 };
diff --git a/packages/cli/test/unit/options/beaconNodeOptions.test.ts b/packages/cli/test/unit/options/beaconNodeOptions.test.ts
@@ -36,6 +36,7 @@ describe("options / beaconNodeOptions", () => {
       "chain.trustedSetup": "",
       "chain.minSameMessageSignatureSetsToBatch": 32,
       "chain.maxShufflingCacheEpochs": 100,
+      "chain.archiveBlobEpochs": 10000,
       emitPayloadAttributes: false,
 
       eth1: true,
@@ -139,6 +140,7 @@ describe("options / beaconNodeOptions", () => {
         trustedSetup: "",
         minSameMessageSignatureSetsToBatch: 32,
         maxShufflingCacheEpochs: 100,
+        archiveBlobEpochs: 10000,
       },
       eth1: {
         enabled: true,