DLPX-82781 [Backport of DLPX-82727 to 6.0.16.0] zcache removal: free …

…slabs exhausted due to rebalancing sleeping for a minute

cmd/zfs_object_agent/zettacache/src/slab_allocator.rs

@@ -9,6 +9,7 @@
 //! Allocator which slabs are allocated, by calling SlabAllocatorBuilder::claim().
 
 use std::cmp::max;
+use std::cmp::min;
 use std::collections::BTreeMap;
 use std::collections::HashSet;
 use std::mem;
@@ -70,6 +71,12 @@ tunable! {
     // With the above in mind we set the minimum to 32 which should be ~1GB assuming 32 MB slabs.
     static ref RESERVED_SLABS_MIN: u64 = 32;
 
+    // Device removal can quickly push the cache to run low on space. In order to avoid running
+    // out of space we want to make sure that a portion from our reserved slabs is available
+    // before finalizing a removal in the slab allocator. This tunable controls the percentage of
+    // our reserved slabs that we want to be allocatable before finalizing a removal.
+    pub static ref RESERVED_SLABS_REMOVAL_LIMIT_PCT: Percent = Percent::new(100.0);
+
     // We never allow these slabs to be allocated.  If we get down to this few free slabs, we'll
     // panic.  By doing this before we get to zero, we preserve the possibility of changing
     // code/tunables to recover from running nearly out of space.
@@ -528,6 +535,10 @@ impl SlabAllocator {
         trace!("slab reservation set to {:} slabs", inner.reserved_slabs);
     }
 
+    pub fn reserved_slabs(&self) -> u64 {
+        self.inner.lock().unwrap().reserved_slabs
+    }
+
     pub fn allocate_reserved(&self) -> SlabId {
         let mut inner = self.inner.lock().unwrap();
         let super_reserved_slabs = SUPER_RESERVED_SLABS_PCT
@@ -602,6 +613,12 @@ impl SlabAllocator {
         inner.allocatable.len() as u64 + noalloc_slabs
     }
 
+    /// Return the number of reserved slabs that are available for allocation.
+    pub fn available_reserved_slabs(&self) -> u64 {
+        let inner = self.inner.lock().unwrap();
+        min(inner.allocatable.len() as u64, inner.reserved_slabs)
+    }
+
     /// Returns the number of slabs that we would like the block allocator to evacuate and free.
     pub fn num_slabs_to_evacuate(&self) -> u64 {
         let inner = self.inner.lock().unwrap();

cmd/zfs_object_agent/zettacache/src/zettacache/mod.rs

@@ -100,6 +100,7 @@ use crate::slab_allocator::SlabAllocatorBuilder;
 use crate::slab_allocator::SlabAllocatorPhys;
 use crate::slab_allocator::SlabId;
 use crate::slab_allocator::RESERVED_SLABS_PCT;
+use crate::slab_allocator::RESERVED_SLABS_REMOVAL_LIMIT_PCT;
 use crate::superblock::DiskPhys;
 use crate::superblock::PrimaryPhys;
 use crate::superblock::SuperblockPhys;
@@ -2047,17 +2048,30 @@ impl Locked {
                             );
                         }
                         None => {
-                            self.slab_allocator.finalize_removal(disk);
-                            let slabs_moved = self
-                                .block_allocator
-                                .transfer_metadata_for_removal(disk)
-                                .await;
-                            progress.merge_cycles_left =
-                                Some(REMOVAL_MAX_MERGES_FOR_METADATA_EVACUATION);
-                            info!(
+                            // Before we forbid allocations from the removing device and finalize
+                            // its removal we need to make sure that there are enough reserved
+                            // slabs for metadata allocations to continue pushing checkpoints and
+                            // not run out of space. If we don't have enough reserved slabs skip
+                            // finalizing the removal until rebalance and eviction free up some
+                            // slabs.
+                            let reserved = self.slab_allocator.available_reserved_slabs();
+                            let threshold = RESERVED_SLABS_REMOVAL_LIMIT_PCT
+                                .apply(self.slab_allocator.reserved_slabs());
+                            if reserved >= threshold {
+                                self.slab_allocator.finalize_removal(disk);
+                                let slabs_moved = self
+                                    .block_allocator
+                                    .transfer_metadata_for_removal(disk)
+                                    .await;
+                                progress.merge_cycles_left =
+                                    Some(REMOVAL_MAX_MERGES_FOR_METADATA_EVACUATION);
+                                info!(
                             "removal: {disk:?}: {slabs_moved} metadata slabs moved; awaiting at most {} merge cycles",
                             REMOVAL_MAX_MERGES_FOR_METADATA_EVACUATION
                         );
+                            } else {
+                                debug!("removal: can't finalize - reserved slabs: {reserved} less than threshold: {threshold}");
+                            }
                         }
                     }
                 } else {