From a31fea7c7f5664d3fec1f54868336c350992bfd4 Mon Sep 17 00:00:00 2001 From: Wallace Date: Mon, 9 Sep 2019 20:13:39 +0800 Subject: [PATCH] [release-3.0]: config support rocksdb doubly skiplist to optimize reverse-scan (#5368) * config support rocksdb doubly skiplist to optimize reverse-scan Signed-off-by: Little-Wallace --- Cargo.lock | 8 ++++---- src/config.rs | 14 +++++++++++++- tests/integrations/config/mod.rs | 5 +++++ tests/integrations/config/test-custom.toml | 1 + 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5f29623fa05..1262456ed5b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -155,7 +155,7 @@ dependencies = [ [[package]] name = "bzip2-sys" version = "0.1.7" -source = "git+https://github.com/alexcrichton/bzip2-rs.git#f47e7400da86931cf1e9300e7e46452238bf08b9" +source = "git+https://github.com/alexcrichton/bzip2-rs.git#02096d6f16e6b78cde379ce2305e08d2933e23b7" dependencies = [ "cc 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.45 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1068,7 +1068,7 @@ dependencies = [ [[package]] name = "librocksdb_sys" version = "0.1.0" -source = "git+https://github.com/pingcap/rust-rocksdb.git?branch=tikv-3.0#494eb5743b21b47ebf838413066e125a1db6388f" +source = "git+https://github.com/pingcap/rust-rocksdb.git?branch=tikv-3.0#a0eb4933c0fb3cc2211355303c6039817ce7704a" dependencies = [ "bzip2-sys 0.1.7 (git+https://github.com/alexcrichton/bzip2-rs.git)", "cc 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1085,7 +1085,7 @@ dependencies = [ [[package]] name = "libtitan_sys" version = "0.0.1" -source = "git+https://github.com/pingcap/rust-rocksdb.git?branch=tikv-3.0#494eb5743b21b47ebf838413066e125a1db6388f" +source = "git+https://github.com/pingcap/rust-rocksdb.git?branch=tikv-3.0#a0eb4933c0fb3cc2211355303c6039817ce7704a" dependencies = [ "bzip2-sys 0.1.7 (git+https://github.com/alexcrichton/bzip2-rs.git)", "cc 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1845,7 +1845,7 @@ dependencies = [ [[package]] name = "rocksdb" version = "0.3.0" -source = "git+https://github.com/pingcap/rust-rocksdb.git?branch=tikv-3.0#494eb5743b21b47ebf838413066e125a1db6388f" +source = "git+https://github.com/pingcap/rust-rocksdb.git?branch=tikv-3.0#a0eb4933c0fb3cc2211355303c6039817ce7704a" dependencies = [ "crc 1.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.45 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/src/config.rs b/src/config.rs index 8de5bd58539..1d8e51ccadc 100644 --- a/src/config.rs +++ b/src/config.rs @@ -160,6 +160,7 @@ macro_rules! cf_config { pub hard_pending_compaction_bytes_limit: ReadableSize, pub prop_size_index_distance: u64, pub prop_keys_index_distance: u64, + pub enable_doubly_skiplist: bool, pub titan: TitanCfConfig, } @@ -264,6 +265,10 @@ macro_rules! write_into_metrics { $metrics .with_label_values(&[$tag, "hard_pending_compaction_bytes_limit"]) .set($cf.hard_pending_compaction_bytes_limit.0 as f64); + $metrics + .with_label_values(&[$tag, "enable_doubly_skiplist"]) + .set(($cf.enable_doubly_skiplist as i32).into()); + $metrics .with_label_values(&[$tag, "titan_min_blob_size"]) .set($cf.titan.min_blob_size.0 as f64); @@ -334,7 +339,9 @@ macro_rules! build_cf_opt { cf_opts.set_soft_pending_compaction_bytes_limit($opt.soft_pending_compaction_bytes_limit.0); cf_opts.set_hard_pending_compaction_bytes_limit($opt.hard_pending_compaction_bytes_limit.0); cf_opts.set_optimize_filters_for_hits($opt.optimize_filters_for_hits); - + if $opt.enable_doubly_skiplist { + cf_opts.set_doubly_skiplist(); + } cf_opts }}; } @@ -383,6 +390,7 @@ impl Default for DefaultCfConfig { hard_pending_compaction_bytes_limit: ReadableSize::gb(256), prop_size_index_distance: DEFAULT_PROP_SIZE_INDEX_DISTANCE, prop_keys_index_distance: DEFAULT_PROP_KEYS_INDEX_DISTANCE, + enable_doubly_skiplist: false, titan: TitanCfConfig::default(), } } @@ -448,6 +456,7 @@ impl Default for WriteCfConfig { hard_pending_compaction_bytes_limit: ReadableSize::gb(256), prop_size_index_distance: DEFAULT_PROP_SIZE_INDEX_DISTANCE, prop_keys_index_distance: DEFAULT_PROP_KEYS_INDEX_DISTANCE, + enable_doubly_skiplist: false, titan, } } @@ -515,6 +524,7 @@ impl Default for LockCfConfig { hard_pending_compaction_bytes_limit: ReadableSize::gb(256), prop_size_index_distance: DEFAULT_PROP_SIZE_INDEX_DISTANCE, prop_keys_index_distance: DEFAULT_PROP_KEYS_INDEX_DISTANCE, + enable_doubly_skiplist: false, titan, } } @@ -572,6 +582,7 @@ impl Default for RaftCfConfig { hard_pending_compaction_bytes_limit: ReadableSize::gb(256), prop_size_index_distance: DEFAULT_PROP_SIZE_INDEX_DISTANCE, prop_keys_index_distance: DEFAULT_PROP_KEYS_INDEX_DISTANCE, + enable_doubly_skiplist: false, titan, } } @@ -839,6 +850,7 @@ impl Default for RaftDefaultCfConfig { hard_pending_compaction_bytes_limit: ReadableSize::gb(256), prop_size_index_distance: DEFAULT_PROP_SIZE_INDEX_DISTANCE, prop_keys_index_distance: DEFAULT_PROP_KEYS_INDEX_DISTANCE, + enable_doubly_skiplist: false, titan: TitanCfConfig::default(), } } diff --git a/tests/integrations/config/mod.rs b/tests/integrations/config/mod.rs index d4058dca100..855cab509f6 100644 --- a/tests/integrations/config/mod.rs +++ b/tests/integrations/config/mod.rs @@ -239,6 +239,7 @@ fn test_serde_custom_tikv_config() { }, prop_size_index_distance: 4000000, prop_keys_index_distance: 40000, + enable_doubly_skiplist: true, }, writecf: WriteCfConfig { block_size: ReadableSize::kb(12), @@ -291,6 +292,7 @@ fn test_serde_custom_tikv_config() { }, prop_size_index_distance: 4000000, prop_keys_index_distance: 40000, + enable_doubly_skiplist: false, }, lockcf: LockCfConfig { block_size: ReadableSize::kb(12), @@ -343,6 +345,7 @@ fn test_serde_custom_tikv_config() { }, prop_size_index_distance: 4000000, prop_keys_index_distance: 40000, + enable_doubly_skiplist: false, }, raftcf: RaftCfConfig { block_size: ReadableSize::kb(12), @@ -395,6 +398,7 @@ fn test_serde_custom_tikv_config() { }, prop_size_index_distance: 4000000, prop_keys_index_distance: 40000, + enable_doubly_skiplist: false, }, titan: TitanDBConfig { enabled: true, @@ -469,6 +473,7 @@ fn test_serde_custom_tikv_config() { titan: TitanCfConfig::default(), prop_size_index_distance: 4000000, prop_keys_index_distance: 40000, + enable_doubly_skiplist: false, }, }; value.storage = StorageConfig { diff --git a/tests/integrations/config/test-custom.toml b/tests/integrations/config/test-custom.toml index 9435faecca1..1f5ac4de160 100644 --- a/tests/integrations/config/test-custom.toml +++ b/tests/integrations/config/test-custom.toml @@ -216,6 +216,7 @@ soft-pending-compaction-bytes-limit = "12GB" hard-pending-compaction-bytes-limit = "12GB" prop-size-index-distance = 4000000 prop-keys-index-distance = 40000 +enable-doubly-skiplist = true [rocksdb.defaultcf.titan] min-blob-size = "2018B"