Skip to content

Commit

Permalink
import: fix #4257, restrict memory usage by RocksDB (#4350)
Browse files Browse the repository at this point in the history
* import: put index and filter in block cache to restrict memory usage

Signed-off-by: Lonng <chris@lonng.org>
  • Loading branch information
kennytm authored and Connor1996 committed Mar 12, 2019
1 parent cc70ef3 commit 5cb9ac9
Showing 1 changed file with 3 additions and 2 deletions.
5 changes: 3 additions & 2 deletions src/import/engine.rs
Original file line number Diff line number Diff line change
Expand Up @@ -292,9 +292,10 @@ fn tune_dboptions_for_bulk_load(opts: &DbConfig) -> (DBOptions, CFOptions) {
// RocksDB preserves `max_background_jobs/4` for flush.
db_opts.set_max_background_jobs(opts.max_background_jobs);

// Put index and filter in block cache to restrict memory usage.
let mut block_base_opts = BlockBasedOptions::new();
// Use a large block size for sequential access.
block_base_opts.set_block_size(MB as usize);
block_base_opts.set_lru_cache(128 * MB as usize, -1, 0, 0.0);
block_base_opts.set_cache_index_and_filter_blocks(true);
let mut cf_opts = ColumnFamilyOptions::new();
cf_opts.set_block_based_table_factory(&block_base_opts);
cf_opts.compression_per_level(&opts.defaultcf.compression_per_level);
Expand Down

0 comments on commit 5cb9ac9

Please sign in to comment.