feat: sleep 10ms while trying to acquire exclusive lock (#65)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

foyer-storage/src/admission/rated_random.rs

@@ -59,7 +59,10 @@ where
     V: Value,
 {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("DynamicRandom").finish()
+        f.debug_struct("DynamicRandom")
+            .field("rate", &self.rate)
+            .field("probability", &self.probability.load(Ordering::Relaxed))
+            .finish()
     }
 }
 

foyer-storage/src/region.rs

@@ -555,13 +555,16 @@ impl<A: BufferAllocator> ErwLock<A> {
         can_physical_read: bool,
     ) -> OwnedRwLockWriteGuard<RegionInner<A>> {
         loop {
-            let guard = self.inner.clone().write_owned().await;
-            let is_ready = (can_write || guard.writers == 0)
-                && (can_buffered_read || guard.buffered_readers == 0)
-                && (can_physical_read || guard.physical_readers == 0);
-            if is_ready {
-                return guard;
+            {
+                let guard = self.inner.clone().write_owned().await;
+                let is_ready = (can_write || guard.writers == 0)
+                    && (can_buffered_read || guard.buffered_readers == 0)
+                    && (can_physical_read || guard.physical_readers == 0);
+                if is_ready {
+                    return guard;
+                }
             }
+            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
         }
     }
 }

foyer-storage/src/region_manager.rs

@@ -199,7 +199,6 @@ where
         }
     }
 
-    #[tracing::instrument(skip(self))]
     pub fn clean_regions(&self) -> &AsyncQueue<RegionId> {
         &self.clean_regions
     }

foyer-storage/src/store.rs

@@ -208,6 +208,7 @@ where
         Ok(())
     }
 
+    #[tracing::instrument(skip(self))]
     pub async fn insert(&self, key: K, value: V) -> Result<bool> {
         let _timer = self.metrics.latency_insert.start_timer();
 
@@ -257,6 +258,7 @@ where
         Ok(true)
     }
 
+    #[tracing::instrument(skip(self))]
     pub async fn lookup(&self, key: &K) -> Result<Option<V>> {
         let now = Instant::now();
 
@@ -300,6 +302,7 @@ where
         res
     }
 
+    #[tracing::instrument(skip(self))]
     pub fn remove(&self, key: &K) {
         let _timer = self.metrics.latency_remove.start_timer();
 
@@ -335,6 +338,7 @@ where
         Ok(())
     }
 
+    #[tracing::instrument(skip(self))]
     async fn recover(&self, concurrency: usize) -> Result<()> {
         tracing::info!("start store recovery");