From 3f2f4ef5e6ce57f2e3bd9cba938bc0bb598273f7 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Wed, 22 Oct 2025 19:33:53 +1100 Subject: [PATCH 1/5] New design for blob/column pruning --- beacon_node/store/src/hot_cold_store.rs | 134 +++++++++++++----------- 1 file changed, 71 insertions(+), 63 deletions(-) diff --git a/beacon_node/store/src/hot_cold_store.rs b/beacon_node/store/src/hot_cold_store.rs index 895afa4f336..c2a1423e280 100644 --- a/beacon_node/store/src/hot_cold_store.rs +++ b/beacon_node/store/src/hot_cold_store.rs @@ -3212,9 +3212,9 @@ impl, Cold: ItemStore> HotColdDB return Ok(()); } + // FIXME(sproul): reimplement support for epochs-per-blob-prune? maybe it doesn't matter? let pruning_enabled = self.get_config().prune_blobs; let margin_epochs = self.get_config().blob_prune_margin_epochs; - let epochs_per_blob_prune = self.get_config().epochs_per_blob_prune; if !force && !pruning_enabled { debug!(prune_blobs = pruning_enabled, "Blob pruning is disabled"); @@ -3223,18 +3223,10 @@ impl, Cold: ItemStore> HotColdDB let blob_info = self.get_blob_info(); let data_column_info = self.get_data_column_info(); - let Some(oldest_blob_slot) = blob_info.oldest_blob_slot else { - error!("Slot of oldest blob is not known"); - return Err(HotColdDBError::BlobPruneLogicError.into()); - }; - - // Start pruning from the epoch of the oldest blob stored. - // The start epoch is inclusive (blobs in this epoch will be pruned). - let start_epoch = oldest_blob_slot.epoch(E::slots_per_epoch()); // Prune blobs up until the `data_availability_boundary - margin` or the split // slot's epoch, whichever is older. We can't prune blobs newer than the split. - // The end epoch is also inclusive (blobs in this epoch will be pruned). + // The end epoch is inclusive (blobs in this epoch will be pruned). let split = self.get_split_info(); let end_epoch = std::cmp::min( data_availability_boundary - margin_epochs - 1, @@ -3242,35 +3234,30 @@ impl, Cold: ItemStore> HotColdDB ); let end_slot = end_epoch.end_slot(E::slots_per_epoch()); - let can_prune = end_epoch != 0 && start_epoch <= end_epoch; - let should_prune = start_epoch + epochs_per_blob_prune <= end_epoch + 1; - - if !force && !should_prune || !can_prune { + // Iterate blocks backwards from the `end_epoch` (usually the data availability boundary). + let Some((end_block_root, _)) = self + .forwards_block_roots_iterator_until(end_slot, end_slot, || { + self.get_hot_state(&split.state_root, true)? + .ok_or(HotColdDBError::MissingSplitState( + split.state_root, + split.slot, + )) + .map(|state| (state, split.state_root)) + .map_err(Into::into) + })? + .next() + .transpose()? + else { + // Can't prune blobs if we don't know the block at `end_slot`. This is expected if we + // have checkpoint synced and haven't backfilled to the DA boundary yet. debug!( - %oldest_blob_slot, - %data_availability_boundary, - %split.slot, %end_epoch, - %start_epoch, - "Blobs are pruned" + %data_availability_boundary, + "No blobs to prune" ); return Ok(()); - } - - // Sanity checks. - let anchor = self.get_anchor_info(); - if oldest_blob_slot < anchor.oldest_block_slot { - error!( - %oldest_blob_slot, - oldest_block_slot = %anchor.oldest_block_slot, - "Oldest blob is older than oldest block" - ); - return Err(HotColdDBError::BlobPruneLogicError.into()); - } - - // Iterate block roots forwards from the oldest blob slot. + }; debug!( - %start_epoch, %end_epoch, %data_availability_boundary, "Pruning blobs" @@ -3279,39 +3266,42 @@ impl, Cold: ItemStore> HotColdDB // We collect block roots of deleted blobs in memory. Even for 10y of blob history this // vec won't go beyond 1GB. We can probably optimise this out eventually. let mut removed_block_roots = vec![]; + let mut blobs_db_ops = vec![]; - let remove_blob_if = |blobs_bytes: &[u8]| { - let blobs = Vec::from_ssz_bytes(blobs_bytes)?; - let Some(blob): Option<&Arc>> = blobs.first() else { - return Ok(false); - }; - - if blob.slot() <= end_slot { - // Store the block root so we can delete from the blob cache - removed_block_roots.push(blob.block_root()); - // Delete from the on-disk db - return Ok(true); - }; - Ok(false) - }; - - self.blobs_db - .delete_if(DBColumn::BeaconBlob, remove_blob_if)?; + // Iterate blocks backwards until we reach a block for which we've already pruned + // blobs/columns. + for tuple in ParentRootBlockIterator::new(self, end_block_root) { + let (block_root, blinded_block) = tuple?; - if self.spec.is_peer_das_enabled_for_epoch(start_epoch) { - let remove_data_column_if = |blobs_bytes: &[u8]| { - let data_column: DataColumnSidecar = - DataColumnSidecar::from_ssz_bytes(blobs_bytes)?; - - if data_column.slot() <= end_slot { - return Ok(true); - }; + // If the block has no blobs we can't tell if they've been pruned, and there is nothing + // to prune, so we just skip. + if !blinded_block.message().body().has_blobs() { + continue; + } - Ok(false) + // Check if we have blobs or columns stored. If not, we assume pruning has already + // reached this point. + let column = if blinded_block.fork_name_unchecked().fulu_enabled() { + DBColumn::BeaconDataColumn + } else { + DBColumn::BeaconBlob }; - - self.blobs_db - .delete_if(DBColumn::BeaconDataColumn, remove_data_column_if)?; + if self.blobs_db.key_exists(column, block_root.as_slice())? { + blobs_db_ops.push(KeyValueStoreOp::DeleteKey( + column, + block_root.as_slice().to_vec(), + )); + removed_block_roots.push(block_root); + } else { + // FIXME(sproul): consider continuing if `--force` is set (to account for gaps due + // to bugs). + debug!( + slot = %blinded_block.slot(), + ?block_root, + "Reached slot with blobs/columns already pruned" + ); + break; + } } // Remove deleted blobs from the cache. @@ -3321,6 +3311,24 @@ impl, Cold: ItemStore> HotColdDB } } + // Remove from disk. + if !blobs_db_ops.is_empty() { + debug!( + num_deleted = blobs_db_ops.len(), + "Deleting blobs/data columns from disk" + ); + self.blobs_db.do_atomically(blobs_db_ops)?; + } + + // FIXME(sproul): this is a bit scuffed, we're only doing this for metadata purposes now + let Some(oldest_blob_slot) = blob_info.oldest_blob_slot else { + error!("Slot of oldest blob is not known"); + return Err(HotColdDBError::BlobPruneLogicError.into()); + }; + + // Start pruning from the epoch of the oldest blob stored. + // The start epoch is inclusive (blobs in this epoch will be pruned). + let start_epoch = oldest_blob_slot.epoch(E::slots_per_epoch()); self.update_blob_or_data_column_info(start_epoch, end_slot, blob_info, data_column_info)?; debug!("Blob pruning complete"); From 5bb96ed3fc86b1065492eb811e20b90c2158646c Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 23 Oct 2025 11:47:10 +1100 Subject: [PATCH 2/5] This works --- beacon_node/store/src/hot_cold_store.rs | 81 ++++++++++++++++++------- 1 file changed, 60 insertions(+), 21 deletions(-) diff --git a/beacon_node/store/src/hot_cold_store.rs b/beacon_node/store/src/hot_cold_store.rs index c2a1423e280..3b33123d565 100644 --- a/beacon_node/store/src/hot_cold_store.rs +++ b/beacon_node/store/src/hot_cold_store.rs @@ -2553,6 +2553,16 @@ impl, Cold: ItemStore> HotColdDB .collect() } + /// Fetch all possible data column keys for a given `block_root`. + /// + /// Unlike `get_data_column_keys`, these keys are not necessarily all present in the database, + /// due to the node's custody requirements many just store a subset. + pub fn get_all_data_column_keys(&self, block_root: Hash256) -> Vec> { + (0..E::number_of_columns() as u64) + .map(|column_index| get_data_column_key(&block_root, &column_index)) + .collect() + } + /// Fetch a single data_column for a given block from the store. pub fn get_data_column( &self, @@ -3212,9 +3222,9 @@ impl, Cold: ItemStore> HotColdDB return Ok(()); } - // FIXME(sproul): reimplement support for epochs-per-blob-prune? maybe it doesn't matter? let pruning_enabled = self.get_config().prune_blobs; let margin_epochs = self.get_config().blob_prune_margin_epochs; + let epochs_per_blob_prune = self.get_config().epochs_per_blob_prune; if !force && !pruning_enabled { debug!(prune_blobs = pruning_enabled, "Blob pruning is disabled"); @@ -3223,6 +3233,15 @@ impl, Cold: ItemStore> HotColdDB let blob_info = self.get_blob_info(); let data_column_info = self.get_data_column_info(); + let Some(oldest_blob_slot) = blob_info.oldest_blob_slot else { + error!("Slot of oldest blob is not known"); + return Err(HotColdDBError::BlobPruneLogicError.into()); + }; + + // The start epoch is not necessarily iterated back to, but is used for deciding whether we + // should attempt pruning. We could probably refactor it out eventually (while reducing our + // dependence on BlobInfo). + let start_epoch = oldest_blob_slot.epoch(E::slots_per_epoch()); // Prune blobs up until the `data_availability_boundary - margin` or the split // slot's epoch, whichever is older. We can't prune blobs newer than the split. @@ -3234,6 +3253,21 @@ impl, Cold: ItemStore> HotColdDB ); let end_slot = end_epoch.end_slot(E::slots_per_epoch()); + let can_prune = end_epoch != 0 && start_epoch <= end_epoch; + let should_prune = start_epoch + epochs_per_blob_prune <= end_epoch + 1; + + if !force && !should_prune || !can_prune { + debug!( + %oldest_blob_slot, + %data_availability_boundary, + %split.slot, + %end_epoch, + %start_epoch, + "Blobs are pruned" + ); + return Ok(()); + } + // Iterate blocks backwards from the `end_epoch` (usually the data availability boundary). let Some((end_block_root, _)) = self .forwards_block_roots_iterator_until(end_slot, end_slot, || { @@ -3272,6 +3306,7 @@ impl, Cold: ItemStore> HotColdDB // blobs/columns. for tuple in ParentRootBlockIterator::new(self, end_block_root) { let (block_root, blinded_block) = tuple?; + let slot = blinded_block.slot(); // If the block has no blobs we can't tell if they've been pruned, and there is nothing // to prune, so we just skip. @@ -3281,22 +3316,35 @@ impl, Cold: ItemStore> HotColdDB // Check if we have blobs or columns stored. If not, we assume pruning has already // reached this point. - let column = if blinded_block.fork_name_unchecked().fulu_enabled() { - DBColumn::BeaconDataColumn + let (db_column, db_keys) = if blinded_block.fork_name_unchecked().fulu_enabled() { + ( + DBColumn::BeaconDataColumn, + self.get_all_data_column_keys(block_root), + ) } else { - DBColumn::BeaconBlob + (DBColumn::BeaconBlob, vec![block_root.as_slice().to_vec()]) }; - if self.blobs_db.key_exists(column, block_root.as_slice())? { - blobs_db_ops.push(KeyValueStoreOp::DeleteKey( - column, - block_root.as_slice().to_vec(), - )); + + // For data columns, consider a block pruned if ALL column indices are absent. + let mut data_stored_for_block = false; + for db_key in db_keys { + if self.blobs_db.key_exists(db_column, &db_key)? { + data_stored_for_block = true; + debug!( + ?db_column, + ?block_root, + %slot, + "Pruning blob/column" + ); + blobs_db_ops.push(KeyValueStoreOp::DeleteKey(db_column, db_key)); + } + } + + if data_stored_for_block { removed_block_roots.push(block_root); } else { - // FIXME(sproul): consider continuing if `--force` is set (to account for gaps due - // to bugs). debug!( - slot = %blinded_block.slot(), + %slot, ?block_root, "Reached slot with blobs/columns already pruned" ); @@ -3320,15 +3368,6 @@ impl, Cold: ItemStore> HotColdDB self.blobs_db.do_atomically(blobs_db_ops)?; } - // FIXME(sproul): this is a bit scuffed, we're only doing this for metadata purposes now - let Some(oldest_blob_slot) = blob_info.oldest_blob_slot else { - error!("Slot of oldest blob is not known"); - return Err(HotColdDBError::BlobPruneLogicError.into()); - }; - - // Start pruning from the epoch of the oldest blob stored. - // The start epoch is inclusive (blobs in this epoch will be pruned). - let start_epoch = oldest_blob_slot.epoch(E::slots_per_epoch()); self.update_blob_or_data_column_info(start_epoch, end_slot, blob_info, data_column_info)?; debug!("Blob pruning complete"); From 8d5318af52a5e50196177d343b383519d29d5605 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 23 Oct 2025 11:52:12 +1100 Subject: [PATCH 3/5] Add note about potential optimisation --- beacon_node/store/src/hot_cold_store.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/beacon_node/store/src/hot_cold_store.rs b/beacon_node/store/src/hot_cold_store.rs index 3b33123d565..7d33a4d2117 100644 --- a/beacon_node/store/src/hot_cold_store.rs +++ b/beacon_node/store/src/hot_cold_store.rs @@ -3326,6 +3326,9 @@ impl, Cold: ItemStore> HotColdDB }; // For data columns, consider a block pruned if ALL column indices are absent. + // In future we might want to refactor this to read the data column indices that *exist* + // from the DB, which could be slightly more efficient than checking existence for every + // possible column. let mut data_stored_for_block = false; for db_key in db_keys { if self.blobs_db.key_exists(db_column, &db_key)? { From 7bb4c646bcd67206263e164ca1a57f8c849de401 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 23 Oct 2025 13:01:35 +1100 Subject: [PATCH 4/5] Adjust log messages --- beacon_node/store/src/hot_cold_store.rs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/beacon_node/store/src/hot_cold_store.rs b/beacon_node/store/src/hot_cold_store.rs index 7d33a4d2117..5fc3699bae9 100644 --- a/beacon_node/store/src/hot_cold_store.rs +++ b/beacon_node/store/src/hot_cold_store.rs @@ -3333,23 +3333,22 @@ impl, Cold: ItemStore> HotColdDB for db_key in db_keys { if self.blobs_db.key_exists(db_column, &db_key)? { data_stored_for_block = true; - debug!( - ?db_column, - ?block_root, - %slot, - "Pruning blob/column" - ); blobs_db_ops.push(KeyValueStoreOp::DeleteKey(db_column, db_key)); } } if data_stored_for_block { + debug!( + ?block_root, + %slot, + "Pruning blobs or columns for block" + ); removed_block_roots.push(block_root); } else { debug!( %slot, ?block_root, - "Reached slot with blobs/columns already pruned" + "Reached slot with blobs or columns already pruned" ); break; } @@ -3366,7 +3365,7 @@ impl, Cold: ItemStore> HotColdDB if !blobs_db_ops.is_empty() { debug!( num_deleted = blobs_db_ops.len(), - "Deleting blobs/data columns from disk" + "Deleting blobs and data columns from disk" ); self.blobs_db.do_atomically(blobs_db_ops)?; } From 738c9ce0bf92f023575501644839add628364be0 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 23 Oct 2025 14:07:20 +1100 Subject: [PATCH 5/5] Fiddle with the block cache --- beacon_node/store/src/hot_cold_store.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/beacon_node/store/src/hot_cold_store.rs b/beacon_node/store/src/hot_cold_store.rs index 5fc3699bae9..a0a75dbb0d4 100644 --- a/beacon_node/store/src/hot_cold_store.rs +++ b/beacon_node/store/src/hot_cold_store.rs @@ -146,9 +146,13 @@ impl BlockCache { pub fn delete_blobs(&mut self, block_root: &Hash256) { let _ = self.blob_cache.pop(block_root); } + pub fn delete_data_columns(&mut self, block_root: &Hash256) { + let _ = self.data_column_cache.pop(block_root); + } pub fn delete(&mut self, block_root: &Hash256) { - let _ = self.block_cache.pop(block_root); - let _ = self.blob_cache.pop(block_root); + self.delete_block(block_root); + self.delete_blobs(block_root); + self.delete_data_columns(block_root); } } @@ -3358,6 +3362,7 @@ impl, Cold: ItemStore> HotColdDB if let Some(mut block_cache) = self.block_cache.as_ref().map(|cache| cache.lock()) { for block_root in removed_block_roots { block_cache.delete_blobs(&block_root); + block_cache.delete_data_columns(&block_root); } }