fs cache improvement for big reads

docker/test/upgrade/run.sh

@@ -78,6 +78,7 @@ remove_keeper_config "create_if_not_exists" "[01]"
 rm /etc/clickhouse-server/config.d/merge_tree.xml
 rm /etc/clickhouse-server/config.d/enable_wait_for_shutdown_replicated_tables.xml
 rm /etc/clickhouse-server/users.d/nonconst_timezone.xml
+rm /etc/clickhouse-server/users.d/s3_cache_new.xml
 
 start
 stop
@@ -114,6 +115,7 @@ sudo chgrp clickhouse /etc/clickhouse-server/config.d/s3_storage_policy_by_defau
 rm /etc/clickhouse-server/config.d/merge_tree.xml
 rm /etc/clickhouse-server/config.d/enable_wait_for_shutdown_replicated_tables.xml
 rm /etc/clickhouse-server/users.d/nonconst_timezone.xml
+rm /etc/clickhouse-server/users.d/s3_cache_new.xml
 
 start
 

src/Core/Settings.h

@@ -723,6 +723,7 @@ class IColumn;
     M(Bool, skip_download_if_exceeds_query_cache, true, "Skip download from remote filesystem if exceeds query cache size", 0) \
     M(UInt64, filesystem_cache_max_download_size, (128UL * 1024 * 1024 * 1024), "Max remote filesystem cache size that can be downloaded by a single query", 0) \
     M(Bool, throw_on_error_from_cache_on_write_operations, false, "Ignore error from cache when caching on write operations (INSERT, merges)", 0) \
+    M(UInt64, filesystem_cache_getorset_batch_size, 20, "A batch size for holding file segments for a single read range", 0) \
     \
     M(Bool, load_marks_asynchronously, false, "Load MergeTree marks asynchronously", 0) \
     M(Bool, enable_filesystem_read_prefetches_log, false, "Log to system.filesystem prefetch_log during query. Should be used only for testing or debugging, not recommended to be turned on by default", 0) \

src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp

@@ -114,30 +114,40 @@ void CachedOnDiskReadBufferFromFile::appendFilesystemCacheLog(
     cache_log->add(std::move(elem));
 }
 
-void CachedOnDiskReadBufferFromFile::initialize(size_t offset, size_t size)
+bool CachedOnDiskReadBufferFromFile::nextFileSegmentsBatch()
 {
-    if (initialized)
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Caching buffer already initialized");
-
-    implementation_buffer.reset();
+    size_t size = getRemainingSizeToRead();
+    if (!size)
+        return false;
 
     if (settings.read_from_filesystem_cache_if_exists_otherwise_bypass_cache)
     {
-        file_segments = cache->get(cache_key, offset, size);
+        file_segments = cache->get(cache_key, file_offset_of_buffer_end, size, settings.filesystem_cache_getorset_batch_size);
     }
     else
     {
         CreateFileSegmentSettings create_settings(FileSegmentKind::Regular);
-        file_segments = cache->getOrSet(cache_key, offset, size, file_size.value(), create_settings);
+        file_segments = cache->getOrSet(cache_key, file_offset_of_buffer_end, size, file_size.value(), create_settings, settings.filesystem_cache_getorset_batch_size);
     }
+    return !file_segments->empty();
+}
+
+void CachedOnDiskReadBufferFromFile::initialize()
+{
+    if (initialized)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Caching buffer already initialized");
+
+    implementation_buffer.reset();
 
     /**
      * Segments in returned list are ordered in ascending order and represent a full contiguous
      * interval (no holes). Each segment in returned list has state: DOWNLOADED, DOWNLOADING or EMPTY.
      */
-    if (file_segments->empty())
+    if (!nextFileSegmentsBatch())
         throw Exception(ErrorCodes::LOGICAL_ERROR, "List of file segments cannot be empty");
 
+    chassert(!file_segments->empty());
+
     LOG_TEST(
         log,
         "Having {} file segments to read: {}, current offset: {}",
@@ -512,7 +522,7 @@ bool CachedOnDiskReadBufferFromFile::completeFileSegmentAndGetNext()
     cache_file_reader.reset();
 
     file_segments->popFront();
-    if (file_segments->empty())
+    if (file_segments->empty() && !nextFileSegmentsBatch())
         return false;
 
     current_file_segment = &file_segments->front();
@@ -788,9 +798,9 @@ bool CachedOnDiskReadBufferFromFile::nextImplStep()
         return false;
 
     if (!initialized)
-        initialize(file_offset_of_buffer_end, getTotalSizeToRead());
+        initialize();
 
-    if (file_segments->empty())
+    if (file_segments->empty() && !nextFileSegmentsBatch())
         return false;
 
     const size_t original_buffer_size = internal_buffer.size();
@@ -1159,7 +1169,7 @@ off_t CachedOnDiskReadBufferFromFile::seek(off_t offset, int whence)
     return new_pos;
 }
 
-size_t CachedOnDiskReadBufferFromFile::getTotalSizeToRead()
+size_t CachedOnDiskReadBufferFromFile::getRemainingSizeToRead()
 {
     /// Last position should be guaranteed to be set, as at least we always know file size.
     if (!read_until_position)

src/Disks/IO/CachedOnDiskReadBufferFromFile.h

@@ -63,7 +63,7 @@ class CachedOnDiskReadBufferFromFile : public ReadBufferFromFileBase
 private:
     using ImplementationBufferPtr = std::shared_ptr<ReadBufferFromFileBase>;
 
-    void initialize(size_t offset, size_t size);
+    void initialize();
 
     /**
      * Return a list of file segments ordered in ascending order. This list represents
@@ -85,7 +85,7 @@ class CachedOnDiskReadBufferFromFile : public ReadBufferFromFileBase
 
     bool nextImplStep();
 
-    size_t getTotalSizeToRead();
+    size_t getRemainingSizeToRead();
 
     bool completeFileSegmentAndGetNext();
 
@@ -95,6 +95,8 @@ class CachedOnDiskReadBufferFromFile : public ReadBufferFromFileBase
 
     static bool canStartFromCache(size_t current_offset, const FileSegment & file_segment);
 
+    bool nextFileSegmentsBatch();
+
     Poco::Logger * log;
     FileCache::Key cache_key;
     String source_file_path;

src/IO/ReadSettings.h

@@ -100,6 +100,7 @@ struct ReadSettings
     bool enable_filesystem_cache_log = false;
     /// Don't populate cache when the read is not part of query execution (e.g. background thread).
     bool avoid_readthrough_cache_outside_query_context = true;
+    size_t filesystem_cache_getorset_batch_size = 100;
 
     size_t filesystem_cache_max_download_size = (128UL * 1024 * 1024 * 1024);
     bool skip_download_if_exceeds_query_cache = true;