Less contention during dynamic resize of filesystem cache

src/Interpreters/Cache/FileCache.cpp

@@ -783,6 +783,17 @@ bool FileCache::tryReserve(
     ProfileEventTimeIncrement<Microseconds> watch(ProfileEvents::FilesystemCacheReserveMicroseconds);
 
     assertInitialized();
+
+    /// A logical race on cache_is_being_resized is still possible,
+    /// in this case we will try to lock cache with timeout, this is ok, timeout is small
+    /// and as resizing of cache can take a long time then this small chance of a race is
+    /// ok compared to the number of cases this check will help.
+    if (cache_is_being_resized.load(std::memory_order_relaxed))
+    {
+        ProfileEvents::increment(ProfileEvents::FilesystemCacheFailToReserveSpaceBecauseOfLockContention);
+        return false;
+    }
+
     auto cache_lock = tryLockCache(std::chrono::milliseconds(lock_wait_timeout_milliseconds));
     if (!cache_lock)
     {
@@ -1264,12 +1275,14 @@ std::vector<String> FileCache::tryGetCachePaths(const Key & key)
 
 size_t FileCache::getUsedCacheSize() const
 {
-    return main_priority->getSize(lockCache());
+    /// We use this method for metrics, so it is ok to get approximate result.
+    return main_priority->getSizeApprox();
 }
 
 size_t FileCache::getFileSegmentsNum() const
 {
-    return main_priority->getElementsCount(lockCache());
+    /// We use this method for metrics, so it is ok to get approximate result.
+    return main_priority->getElementsCountApprox();
 }
 
 void FileCache::assertCacheCorrectness()
@@ -1327,8 +1340,12 @@ void FileCache::applySettingsIfPossible(const FileCacheSettings & new_settings,
     if (new_settings.max_size != actual_settings.max_size
         || new_settings.max_elements != actual_settings.max_elements)
     {
-        auto cache_lock = lockCache();
+        cache_is_being_resized.store(true, std::memory_order_relaxed);
+        SCOPE_EXIT({
+            cache_is_being_resized.store(false, std::memory_order_relaxed);
+        });
 
+        auto cache_lock = lockCache();
         bool updated = false;
         try
         {

src/Interpreters/Cache/FileCache.h

@@ -202,6 +202,7 @@ class FileCache : private boost::noncopyable
     mutable std::mutex init_mutex;
     std::unique_ptr<StatusFile> status_file;
     std::atomic<bool> shutdown = false;
+    std::atomic<bool> cache_is_being_resized = false;
 
     std::mutex apply_settings_mutex;
 

src/Interpreters/Cache/IFileCachePriority.h

@@ -63,8 +63,12 @@ class IFileCachePriority : private boost::noncopyable
 
     virtual size_t getSize(const CachePriorityGuard::Lock &) const = 0;
 
+    virtual size_t getSizeApprox() const = 0;
+
     virtual size_t getElementsCount(const CachePriorityGuard::Lock &) const = 0;
 
+    virtual size_t getElementsCountApprox() const = 0;
+
     /// Throws exception if there is not enough size to fit it.
     virtual IteratorPtr add( /// NOLINT
         KeyMetadataPtr key_metadata,

src/Interpreters/Cache/LRUFileCachePriority.h

@@ -28,6 +28,10 @@ class LRUFileCachePriority final : public IFileCachePriority
 
     size_t getElementsCount(const CachePriorityGuard::Lock &) const override { return state->current_elements_num; }
 
+    size_t getSizeApprox() const override { return state->current_size; }
+
+    size_t getElementsCountApprox() const override { return state->current_elements_num; }
+
     bool canFit( /// NOLINT
         size_t size,
         const CachePriorityGuard::Lock &,

src/Interpreters/Cache/SLRUFileCachePriority.cpp

@@ -44,6 +44,16 @@ size_t SLRUFileCachePriority::getElementsCount(const CachePriorityGuard::Lock &
     return protected_queue.getElementsCount(lock) + probationary_queue.getElementsCount(lock);
 }
 
+size_t SLRUFileCachePriority::getSizeApprox() const
+{
+    return protected_queue.getSizeApprox() + probationary_queue.getSizeApprox();
+}
+
+size_t SLRUFileCachePriority::getElementsCountApprox() const
+{
+    return protected_queue.getElementsCountApprox() + probationary_queue.getElementsCountApprox();
+}
+
 bool SLRUFileCachePriority::canFit( /// NOLINT
     size_t size,
     const CachePriorityGuard::Lock & lock,

src/Interpreters/Cache/SLRUFileCachePriority.h

@@ -25,6 +25,10 @@ class SLRUFileCachePriority : public IFileCachePriority
 
     size_t getElementsCount(const CachePriorityGuard::Lock &) const override;
 
+    size_t getSizeApprox() const override;
+
+    size_t getElementsCountApprox() const override;
+
     bool canFit( /// NOLINT
         size_t size,
         const CachePriorityGuard::Lock &,