feat: add new option allow skip disk cache check when cache in memory (…

…#3253)
openobserve · Apr 17, 2024 · c2c2d50 · c2c2d50
1 parent 7e457c3
commit c2c2d50
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 3 deletions.
diff --git a/src/config/src/config.rs b/src/config/src/config.rs
@@ -793,10 +793,12 @@ pub struct MemoryCache {
     // MB, when cache is full will release how many data once time, default is 1% of max_size
     #[env_config(name = "ZO_MEMORY_CACHE_RELEASE_SIZE", default = 0)]
     pub release_size: usize,
-    #[env_config(name = "ZO_MEMORY_CACHE_GC_SIZE", default = 10)] // MB
+    #[env_config(name = "ZO_MEMORY_CACHE_GC_SIZE", default = 50)] // MB
     pub gc_size: usize,
     #[env_config(name = "ZO_MEMORY_CACHE_GC_INTERVAL", default = 0)] // seconds
     pub gc_interval: u64,
+    #[env_config(name = "ZO_MEMORY_CACHE_SKIP_DISK_CHECK", default = false)]
+    pub skip_disk_check: bool,
     // MB, default is 50% of system memory
     #[env_config(name = "ZO_MEMORY_CACHE_DATAFUSION_MAX_SIZE", default = 0)]
     pub datafusion_max_size: usize,
@@ -821,7 +823,7 @@ pub struct DiskCache {
     // MB, when cache is full will release how many data once time, default is 1% of max_size
     #[env_config(name = "ZO_DISK_CACHE_RELEASE_SIZE", default = 0)]
     pub release_size: usize,
-    #[env_config(name = "ZO_DISK_CACHE_GC_SIZE", default = 10)] // MB
+    #[env_config(name = "ZO_DISK_CACHE_GC_SIZE", default = 100)] // MB
     pub gc_size: usize,
     #[env_config(name = "ZO_DISK_CACHE_GC_INTERVAL", default = 0)] // seconds
     pub gc_interval: u64,

diff --git a/src/service/search/grpc/storage.rs b/src/service/search/grpc/storage.rs
@@ -401,7 +401,8 @@ async fn cache_parquet_files(
             let ret = match cache_type {
                 file_data::CacheType::Memory => {
                     if !file_data::memory::exist(&file_name).await
-                        && !file_data::disk::exist(&file_name).await
+                        && (CONFIG.memory_cache.skip_disk_check
+                            || !file_data::disk::exist(&file_name).await)
                     {
                         file_data::memory::download(&trace_id, &file_name)
                             .await