diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 560cbf2d6ee0ad..76df3225bec8f2 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1329,6 +1329,8 @@ DEFINE_mBool(enable_delete_when_cumu_compaction, "false"); // max_write_buffer_number for rocksdb DEFINE_Int32(rocksdb_max_write_buffer_number, "5"); +// bytes_per_sec for rocksdb rate limiter +DEFINE_Int64(rocksdb_rate_limiter_bytes_per_sec, "1048576"); // 1MB/s DEFINE_mBool(allow_zero_date, "false"); DEFINE_Bool(allow_invalid_decimalv2_literal, "false"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 95bf94667d7529..f67d23594fdd94 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1372,6 +1372,8 @@ DECLARE_mBool(enable_delete_when_cumu_compaction); // max_write_buffer_number for rocksdb DECLARE_Int32(rocksdb_max_write_buffer_number); +// bytes_per_sec for rocksdb rate limiter +DECLARE_Int64(rocksdb_rate_limiter_bytes_per_sec); // Convert date 0000-00-00 to 0000-01-01. It's recommended to set to false. DECLARE_mBool(allow_zero_date); diff --git a/be/src/io/cache/cache_block_meta_store.cpp b/be/src/io/cache/cache_block_meta_store.cpp index 369d81537969e0..36002c8d8cd5c4 100644 --- a/be/src/io/cache/cache_block_meta_store.cpp +++ b/be/src/io/cache/cache_block_meta_store.cpp @@ -33,6 +33,7 @@ #include "common/status.h" #include "exec/common/hex.h" +#include "rocksdb/rate_limiter.h" #include "storage/field.h" #include "storage/field.h" // For OLAP_FIELD_TYPE_BIGINT #include "storage/key_coder.h" @@ -88,6 +89,10 @@ Status CacheBlockMetaStore::init() { _options.write_buffer_size = 64 * 1024 * 1024; // 64MB _options.target_file_size_base = 64 * 1024 * 1024; + auto bytes_per_sec = config::rocksdb_rate_limiter_bytes_per_sec; + _options.rate_limiter.reset(rocksdb::NewGenericRateLimiter( + bytes_per_sec, 100 * 1000, 10, rocksdb::RateLimiter::Mode::kWritesOnly, true)); + rocksdb::BlockBasedTableOptions table_options; table_options.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, false)); table_options.block_size = 16 * 1024; diff --git a/be/src/storage/olap_meta.cpp b/be/src/storage/olap_meta.cpp index 262dd3bcc6d02f..3d8e1e36842ca0 100644 --- a/be/src/storage/olap_meta.cpp +++ b/be/src/storage/olap_meta.cpp @@ -37,6 +37,7 @@ #include "rocksdb/convenience.h" #include "rocksdb/db.h" #include "rocksdb/options.h" +#include "rocksdb/rate_limiter.h" #include "rocksdb/slice.h" #include "rocksdb/slice_transform.h" #include "runtime/runtime_profile.h" @@ -83,6 +84,10 @@ Status OlapMeta::init() { options.info_log = std::make_shared(); options.info_log_level = rocksdb::WARN_LEVEL; + auto bytes_per_sec = config::rocksdb_rate_limiter_bytes_per_sec; + options.rate_limiter.reset(rocksdb::NewGenericRateLimiter( + bytes_per_sec, 100 * 1000, 10, rocksdb::RateLimiter::Mode::kWritesOnly, true)); + std::string db_path = _root_path + META_POSTFIX; std::vector column_families; // default column family is required