fix(disk usage based eviction): do not contend for layermap rwlock

pageserver/src/disk_usage_eviction_task.rs

@@ -807,14 +807,21 @@ async fn collect_eviction_candidates(
             if !tl.is_active() {
                 continue;
             }
-            let info = tl.get_local_layers_for_disk_usage_eviction().await;
+            let Some(info) = tl.get_local_layers_for_disk_usage_eviction().await else {
+                // if we cannot access the layers right now, we'll try again later.
+                continue;
+            };
             debug!(tenant_id=%tl.tenant_shard_id.tenant_id, shard_id=%tl.tenant_shard_id.shard_slug(), timeline_id=%tl.timeline_id, "timeline resident layers count: {}", info.resident_layers.len());
             tenant_candidates.extend(info.resident_layers.into_iter());
             max_layer_size = max_layer_size.max(info.max_layer_size.unwrap_or(0));
 
             if cancel.is_cancelled() {
                 return Ok(EvictionCandidates::Cancelled);
             }
+
+            // because we no longer do waiting on `RwLock::read` and so make tokio::coop progress,
+            // yield after each successful timeline
+            tokio::task::yield_now().await;
         }
 
         // `min_resident_size` defaults to maximum layer file size of the tenant.

pageserver/src/tenant/timeline.rs

@@ -4656,9 +4656,13 @@ impl Timeline {
 }
 
 impl Timeline {
-    /// Returns non-remote layers for eviction.
-    pub(crate) async fn get_local_layers_for_disk_usage_eviction(&self) -> DiskUsageEvictionInfo {
-        let guard = self.layers.read().await;
+    /// Returns non-remote layers for eviction if possible right now.
+    pub(crate) async fn get_local_layers_for_disk_usage_eviction(
+        &self,
+    ) -> Option<DiskUsageEvictionInfo> {
+        let Ok(guard) = self.layers.try_read() else {
+            return None;
+        };
         let layers = guard.layer_map();
 
         let mut max_layer_size: Option<u64> = None;
@@ -4695,10 +4699,10 @@ impl Timeline {
             });
         }
 
-        DiskUsageEvictionInfo {
+        Some(DiskUsageEvictionInfo {
             max_layer_size,
             resident_layers,
-        }
+        })
     }
 
     pub(crate) fn get_shard_index(&self) -> ShardIndex {