Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[release-3.0]: config support rocksdb doubly skiplist to optimize reverse-scan #5368

Merged
merged 6 commits into from Sep 9, 2019
Merged

[release-3.0]: config support rocksdb doubly skiplist to optimize reverse-scan #5368

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
70ec092
config support rocksdb doubly skiplist to optimize reverse-scan
Little-Wallace Aug 30, 2019
846db18
fix unit test
Little-Wallace Aug 30, 2019
cfce0e5
Merge branch 'release-3.0' into tikv/release-3.0
zhouqiang-cl Sep 9, 2019
8facfa0
empty commit, to restart CI
Little-Wallace Sep 9, 2019
e99f8d3
fix fmt
Little-Wallace Sep 9, 2019
637602b
Merge branch 'release-3.0' into tikv/release-3.0
zhangjinpeng87 Sep 9, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
8 changes: 4 additions & 4 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

14 changes: 13 additions & 1 deletion src/config.rs
View file
Open in desktop
Expand Up @@ -160,6 +160,7 @@ macro_rules! cf_config {
pub hard_pending_compaction_bytes_limit: ReadableSize,
pub prop_size_index_distance: u64,
pub prop_keys_index_distance: u64,
pub enable_doubly_skiplist: bool,
pub titan: TitanCfConfig,
}

Expand Down Expand Up @@ -264,6 +265,10 @@ macro_rules! write_into_metrics {
$metrics
.with_label_values(&[$tag, "hard_pending_compaction_bytes_limit"])
.set($cf.hard_pending_compaction_bytes_limit.0 as f64);
$metrics
.with_label_values(&[$tag, "enable_doubly_skiplist"])
.set(($cf.enable_doubly_skiplist as i32).into());

$metrics
.with_label_values(&[$tag, "titan_min_blob_size"])
.set($cf.titan.min_blob_size.0 as f64);
Expand Down Expand Up @@ -334,7 +339,9 @@ macro_rules! build_cf_opt {
cf_opts.set_soft_pending_compaction_bytes_limit($opt.soft_pending_compaction_bytes_limit.0);
cf_opts.set_hard_pending_compaction_bytes_limit($opt.hard_pending_compaction_bytes_limit.0);
cf_opts.set_optimize_filters_for_hits($opt.optimize_filters_for_hits);

if $opt.enable_doubly_skiplist {
cf_opts.set_doubly_skiplist();
}
cf_opts
}};
}
Expand Down Expand Up @@ -383,6 +390,7 @@ impl Default for DefaultCfConfig {
hard_pending_compaction_bytes_limit: ReadableSize::gb(256),
prop_size_index_distance: DEFAULT_PROP_SIZE_INDEX_DISTANCE,
prop_keys_index_distance: DEFAULT_PROP_KEYS_INDEX_DISTANCE,
enable_doubly_skiplist: false,
titan: TitanCfConfig::default(),
}
}
Expand Down Expand Up @@ -448,6 +456,7 @@ impl Default for WriteCfConfig {
hard_pending_compaction_bytes_limit: ReadableSize::gb(256),
prop_size_index_distance: DEFAULT_PROP_SIZE_INDEX_DISTANCE,
prop_keys_index_distance: DEFAULT_PROP_KEYS_INDEX_DISTANCE,
enable_doubly_skiplist: false,
titan,
}
}
Expand Down Expand Up @@ -515,6 +524,7 @@ impl Default for LockCfConfig {
hard_pending_compaction_bytes_limit: ReadableSize::gb(256),
prop_size_index_distance: DEFAULT_PROP_SIZE_INDEX_DISTANCE,
prop_keys_index_distance: DEFAULT_PROP_KEYS_INDEX_DISTANCE,
enable_doubly_skiplist: false,
titan,
}
}
Expand Down Expand Up @@ -572,6 +582,7 @@ impl Default for RaftCfConfig {
hard_pending_compaction_bytes_limit: ReadableSize::gb(256),
prop_size_index_distance: DEFAULT_PROP_SIZE_INDEX_DISTANCE,
prop_keys_index_distance: DEFAULT_PROP_KEYS_INDEX_DISTANCE,
enable_doubly_skiplist: false,
titan,
}
}
Expand Down Expand Up @@ -839,6 +850,7 @@ impl Default for RaftDefaultCfConfig {
hard_pending_compaction_bytes_limit: ReadableSize::gb(256),
prop_size_index_distance: DEFAULT_PROP_SIZE_INDEX_DISTANCE,
prop_keys_index_distance: DEFAULT_PROP_KEYS_INDEX_DISTANCE,
enable_doubly_skiplist: false,
titan: TitanCfConfig::default(),
}
}
Expand Down
5 changes: 5 additions & 0 deletions tests/integrations/config/mod.rs
View file
Open in desktop
Expand Up @@ -239,6 +239,7 @@ fn test_serde_custom_tikv_config() {
},
prop_size_index_distance: 4000000,
prop_keys_index_distance: 40000,
enable_doubly_skiplist: true,
},
writecf: WriteCfConfig {
block_size: ReadableSize::kb(12),
Expand Down Expand Up @@ -291,6 +292,7 @@ fn test_serde_custom_tikv_config() {
},
prop_size_index_distance: 4000000,
prop_keys_index_distance: 40000,
enable_doubly_skiplist: false,
},
lockcf: LockCfConfig {
block_size: ReadableSize::kb(12),
Expand Down Expand Up @@ -343,6 +345,7 @@ fn test_serde_custom_tikv_config() {
},
prop_size_index_distance: 4000000,
prop_keys_index_distance: 40000,
enable_doubly_skiplist: false,
},
raftcf: RaftCfConfig {
block_size: ReadableSize::kb(12),
Expand Down Expand Up @@ -395,6 +398,7 @@ fn test_serde_custom_tikv_config() {
},
prop_size_index_distance: 4000000,
prop_keys_index_distance: 40000,
enable_doubly_skiplist: false,
},
titan: TitanDBConfig {
enabled: true,
Expand Down Expand Up @@ -469,6 +473,7 @@ fn test_serde_custom_tikv_config() {
titan: TitanCfConfig::default(),
prop_size_index_distance: 4000000,
prop_keys_index_distance: 40000,
enable_doubly_skiplist: false,
},
};
value.storage = StorageConfig {
Expand Down
1 change: 1 addition & 0 deletions tests/integrations/config/test-custom.toml
View file
Open in desktop
Expand Up @@ -216,6 +216,7 @@ soft-pending-compaction-bytes-limit = "12GB"
hard-pending-compaction-bytes-limit = "12GB"
prop-size-index-distance = 4000000
prop-keys-index-distance = 40000
enable-doubly-skiplist = true

[rocksdb.defaultcf.titan]
min-blob-size = "2018B"
Expand Down