Move cleaned pages into read cache instead of discarding them.

Just after dirty pages have been written out to disk, they are currently discard
and need to be read in again when these pages are accessed in the future.

This changes moves them into the read cache instead thereby keeping the clean
data available and cache hot in memory.

src/tree_store/page_store/cached_file.rs

@@ -237,10 +237,9 @@ impl PagedCachedFile {
         max_read_cache_bytes: usize,
         max_write_buffer_bytes: usize,
     ) -> Result<Self, DatabaseError> {
-        let mut read_cache = Vec::with_capacity(Self::lock_stripes());
-        for _ in 0..Self::lock_stripes() {
-            read_cache.push(RwLock::new(PrioritizedCache::new()));
-        }
+        let read_cache = (0..Self::lock_stripes())
+            .map(|_| RwLock::new(PrioritizedCache::new()))
+            .collect();
 
         Ok(Self {
             file,
@@ -263,7 +262,7 @@ impl PagedCachedFile {
         self.file.len().map_err(StorageError::from)
     }
 
-    const fn lock_stripes() -> usize {
+    const fn lock_stripes() -> u64 {
         131
     }
 
@@ -285,12 +284,43 @@ impl PagedCachedFile {
         self.check_fsync_failure()?;
         let mut write_buffer = self.write_buffer.lock().unwrap();
 
-        for (offset, buffer) in write_buffer.cache.iter() {
-            self.file.write(*offset, buffer.as_ref().unwrap())?;
+        for (&offset, buffer) in write_buffer.cache.iter_mut() {
+            let buffer = buffer.take().unwrap();
+
+            self.file.write(offset, &buffer)?;
+
+            let cache_size = self
+                .read_cache_bytes
+                .fetch_add(buffer.len(), Ordering::AcqRel);
+
+            if cache_size + buffer.len() <= self.max_read_cache_bytes {
+                let cache_slot: usize = (offset % Self::lock_stripes()).try_into().unwrap();
+                let mut lock = self.read_cache[cache_slot].write().unwrap();
+                lock.insert(offset, buffer, CachePriority::High);
+            } else {
+                self.read_cache_bytes
+                    .fetch_sub(buffer.len(), Ordering::AcqRel);
+            }
         }
-        for (offset, buffer) in write_buffer.low_pri_cache.iter() {
-            self.file.write(*offset, buffer.as_ref().unwrap())?;
+        for (&offset, buffer) in write_buffer.low_pri_cache.iter_mut() {
+            let buffer = buffer.take().unwrap();
+
+            self.file.write(offset, &buffer)?;
+
+            let cache_size = self
+                .read_cache_bytes
+                .fetch_add(buffer.len(), Ordering::AcqRel);
+
+            if cache_size + buffer.len() <= self.max_read_cache_bytes {
+                let cache_slot: usize = (offset % Self::lock_stripes()).try_into().unwrap();
+                let mut lock = self.read_cache[cache_slot].write().unwrap();
+                lock.insert(offset, buffer, CachePriority::Low);
+            } else {
+                self.read_cache_bytes
+                    .fetch_sub(buffer.len(), Ordering::AcqRel);
+            }
         }
+
         self.write_buffer_bytes.store(0, Ordering::Release);
         write_buffer.clear();
 
@@ -353,7 +383,7 @@ impl PagedCachedFile {
             }
         }
 
-        let cache_slot: usize = (offset % Self::lock_stripes() as u64).try_into().unwrap();
+        let cache_slot: usize = (offset % Self::lock_stripes()).try_into().unwrap();
         {
             let read_lock = self.read_cache[cache_slot].read().unwrap();
             if let Some(cached) = read_lock.get(&offset) {
@@ -398,7 +428,7 @@ impl PagedCachedFile {
     //
     // NOTE: Invalidating a cached region in subsections is permitted, as long as all subsections are invalidated
     pub(super) fn invalidate_cache(&self, offset: u64, len: usize) {
-        let cache_slot: usize = (offset % self.read_cache.len() as u64).try_into().unwrap();
+        let cache_slot: usize = (offset % Self::lock_stripes()).try_into().unwrap();
         let mut lock = self.read_cache[cache_slot].write().unwrap();
         if let Some(removed) = lock.remove(&offset) {
             assert_eq!(len, removed.len());
@@ -431,7 +461,7 @@ impl PagedCachedFile {
         let mut lock = self.write_buffer.lock().unwrap();
 
         // TODO: allow hint that page is known to be dirty and will not be in the read cache
-        let cache_slot: usize = (offset % self.read_cache.len() as u64).try_into().unwrap();
+        let cache_slot: usize = (offset % Self::lock_stripes()).try_into().unwrap();
         let existing = {
             let mut lock = self.read_cache[cache_slot].write().unwrap();
             if let Some(removed) = lock.remove(&offset) {