Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Rocksdb new options configuration #11053

Open
wants to merge 1 commit into
base: release-7.1
Choose a base branch
from
Open

Rocksdb new options configuration #11053

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
9 changes: 9 additions & 0 deletions fdbclient/ServerKnobs.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -420,6 +420,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
init( ROCKSDB_READ_RANGE_BOUNDED_ITERATORS_MAX_LIMIT, 200 );
// Set to 0 to disable rocksdb write rate limiting. Rate limiter unit: bytes per second.
init( ROCKSDB_WRITE_RATE_LIMITER_BYTES_PER_SEC, 0 );
init( ROCKSDB_WRITE_RATE_LIMITER_FAIRNESS, 10 ); // RocksDB default 10
// If true, enables dynamic adjustment of ROCKSDB_WRITE_RATE_LIMITER_BYTES according to the recent demand of background IO.
init( ROCKSDB_WRITE_RATE_LIMITER_AUTO_TUNE, true );
init( DEFAULT_FDB_ROCKSDB_COLUMN_FAMILY, "fdb");
Expand Down Expand Up @@ -461,6 +462,14 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
init( ROCKSDB_CAN_COMMIT_DELAY_TIMES_ON_OVERLOAD, 20 );
init( ROCKSDB_COMPACTION_READAHEAD_SIZE, 32768 ); // 32 KB, performs bigger reads when doing compaction.
init( ROCKSDB_BLOCK_SIZE, 32768 ); // 32 KB, size of the block in rocksdb cache.
init( ROCKSDB_WRITE_BUFFER_SIZE, isSimulated ? 256 << 10 : 64 << 20 ); // 64 MB
init( ROCKSDB_MAX_WRITE_BUFFER_NUMBER, 6 ); // RocksDB default.
init( ROCKSDB_MIN_WRITE_BUFFER_NUMBER_TO_MERGE, 2 ); // RocksDB default.
init( ROCKSDB_LEVEL0_FILENUM_COMPACTION_TRIGGER, 2 ); // RocksDB default.
init( ROCKSDB_LEVEL0_SLOWDOWN_WRITES_TRIGGER, 20 ); // RocksDB default.
init( ROCKSDB_LEVEL0_STOP_WRITES_TRIGGER, 36 ); // RocksDB default.
init( ROCKSDB_TARGET_FILE_SIZE_BASE, 0 ); // If 0, pick RocksDB default.
init( ROCKSDB_TARGET_FILE_SIZE_MULTIPLIER, 1 ); // RocksDB default.
init( ROCKSDB_MAX_LOG_FILE_SIZE, 10485760 ); // 10MB.
init( ROCKSDB_KEEP_LOG_FILE_NUM, 200 ); // Keeps 2GB log per storage server.
// Temporary knob to enable trace events which prints details about all the backup keys update(write/clear) operations.
Expand Down
9 changes: 9 additions & 0 deletions fdbclient/ServerKnobs.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,7 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
bool ROCKSDB_READ_RANGE_REUSE_BOUNDED_ITERATORS;
int ROCKSDB_READ_RANGE_BOUNDED_ITERATORS_MAX_LIMIT;
int64_t ROCKSDB_WRITE_RATE_LIMITER_BYTES_PER_SEC;
int ROCKSDB_WRITE_RATE_LIMITER_FAIRNESS;
bool ROCKSDB_WRITE_RATE_LIMITER_AUTO_TUNE;
std::string DEFAULT_FDB_ROCKSDB_COLUMN_FAMILY;
bool ROCKSDB_DISABLE_AUTO_COMPACTIONS;
Expand Down Expand Up @@ -377,6 +378,14 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
int ROCKSDB_STATS_LEVEL;
int64_t ROCKSDB_COMPACTION_READAHEAD_SIZE;
int64_t ROCKSDB_BLOCK_SIZE;
int64_t ROCKSDB_WRITE_BUFFER_SIZE;
int ROCKSDB_MAX_WRITE_BUFFER_NUMBER;
int ROCKSDB_MIN_WRITE_BUFFER_NUMBER_TO_MERGE;
int ROCKSDB_LEVEL0_FILENUM_COMPACTION_TRIGGER;
int ROCKSDB_LEVEL0_SLOWDOWN_WRITES_TRIGGER;
int ROCKSDB_LEVEL0_STOP_WRITES_TRIGGER;
int ROCKSDB_TARGET_FILE_SIZE_BASE;
int ROCKSDB_TARGET_FILE_SIZE_MULTIPLIER;
int ROCKSDB_MAX_LOG_FILE_SIZE;
int ROCKSDB_KEEP_LOG_FILE_NUM;
bool SS_BACKUP_KEYS_OP_LOGS;
Expand Down
12 changes: 11 additions & 1 deletion fdbserver/KeyValueStoreRocksDB.actor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -324,6 +324,16 @@ rocksdb::ColumnFamilyOptions getCFOptions() {
options.hard_pending_compaction_bytes_limit = SERVER_KNOBS->ROCKSDB_HARD_PENDING_COMPACT_BYTES_LIMIT;
}
options.paranoid_file_checks = SERVER_KNOBS->ROCKSDB_PARANOID_FILE_CHECKS;
if (SERVER_KNOBS->ROCKSDB_TARGET_FILE_SIZE_BASE > 0) {
options.target_file_size_base = SERVER_KNOBS->ROCKSDB_TARGET_FILE_SIZE_BASE;
}
options.target_file_size_multiplier = SERVER_KNOBS->ROCKSDB_TARGET_FILE_SIZE_MULTIPLIER;
options.write_buffer_size = SERVER_KNOBS->ROCKSDB_WRITE_BUFFER_SIZE;
options.max_write_buffer_number = SERVER_KNOBS->ROCKSDB_MAX_WRITE_BUFFER_NUMBER;
options.min_write_buffer_number_to_merge = SERVER_KNOBS->ROCKSDB_MIN_WRITE_BUFFER_NUMBER_TO_MERGE;
options.level0_file_num_compaction_trigger = SERVER_KNOBS->ROCKSDB_LEVEL0_FILENUM_COMPACTION_TRIGGER;
options.level0_slowdown_writes_trigger = SERVER_KNOBS->ROCKSDB_LEVEL0_SLOWDOWN_WRITES_TRIGGER;
options.level0_stop_writes_trigger = SERVER_KNOBS->ROCKSDB_LEVEL0_STOP_WRITES_TRIGGER;

// Compact sstables when there's too much deleted stuff.
if (SERVER_KNOBS->ROCKSDB_ENABLE_COMPACT_ON_DELETION) {
Expand Down Expand Up @@ -1143,7 +1153,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
? rocksdb::NewGenericRateLimiter(
SERVER_KNOBS->ROCKSDB_WRITE_RATE_LIMITER_BYTES_PER_SEC, // rate_bytes_per_sec
100 * 1000, // refill_period_us
10, // fairness
SERVER_KNOBS->ROCKSDB_WRITE_RATE_LIMITER_FAIRNESS, // fairness
rocksdb::RateLimiter::Mode::kAllIo,
SERVER_KNOBS->ROCKSDB_WRITE_RATE_LIMITER_AUTO_TUNE)
: nullptr) {
Expand Down