This repository has been archived by the owner on Feb 26, 2020. It is now read-only.
forked from rust-rocksdb/rust-rocksdb
-
Notifications
You must be signed in to change notification settings - Fork 23
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
A utility function to help users migrate DB after options change
Summary: Add a utility function that trigger necessary full compaction and put output to the correct level by looking at new options and old options. Test Plan: Add unit tests for it. Reviewers: andrewkr, igor, IslamAbdelRahman Reviewed By: IslamAbdelRahman Subscribers: muthu, sumeet, leveldb, andrewkr, dhruba Differential Revision: https://reviews.facebook.net/D60783
- Loading branch information
Showing
7 changed files
with
388 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. | ||
| // This source code is licensed under the BSD-style license found in the | ||
| // LICENSE file in the root directory of this source tree. An additional grant | ||
| // of patent rights can be found in the PATENTS file in the same directory. | ||
|
|
||
| #pragma once | ||
|
|
||
| #include <string> | ||
| #include "rocksdb/options.h" | ||
| #include "rocksdb/status.h" | ||
|
|
||
| namespace rocksdb { | ||
| // Try to migrate DB created with old_opts to be use new_opts. | ||
| // Multiple column families is not supported. | ||
| // It is best-effort. No guarantee to succeed. | ||
| // A full compaction may be executed. | ||
| Status OptionChangeMigration(std::string dbname, const Options& old_opts, | ||
| const Options& new_opts); | ||
| } // namespace rocksdb |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
153 changes: 153 additions & 0 deletions
153
utilities/option_change_migration/option_change_migration.cc
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,153 @@ | ||
| // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. | ||
| // This source code is licensed under the BSD-style license found in the | ||
| // LICENSE file in the root directory of this source tree. An additional grant | ||
| // of patent rights can be found in the PATENTS file in the same directory. | ||
|
|
||
| #include "rocksdb/utilities/option_change_migration.h" | ||
|
|
||
| #ifndef ROCKSDB_LITE | ||
| #include "rocksdb/db.h" | ||
|
|
||
| namespace rocksdb { | ||
| namespace { | ||
| // Return a version of Options `opts` that allow us to open/write into a DB | ||
| // without triggering an automatic compaction or stalling. This is guaranteed | ||
| // by disabling automatic compactions and using huge values for stalling | ||
| // triggers. | ||
| Options GetNoCompactionOptions(const Options& opts) { | ||
| Options ret_opts = opts; | ||
| ret_opts.disable_auto_compactions = true; | ||
| ret_opts.level0_slowdown_writes_trigger = 999999; | ||
| ret_opts.level0_stop_writes_trigger = 999999; | ||
| ret_opts.soft_pending_compaction_bytes_limit = 0; | ||
| ret_opts.hard_pending_compaction_bytes_limit = 0; | ||
| return ret_opts; | ||
| } | ||
|
|
||
| Status OpenDb(const Options& options, const std::string& dbname, | ||
| std::unique_ptr<DB>* db) { | ||
| db->reset(); | ||
| DB* tmpdb; | ||
| Status s = DB::Open(options, dbname, &tmpdb); | ||
| if (s.ok()) { | ||
| db->reset(tmpdb); | ||
| } | ||
| return s; | ||
| } | ||
|
|
||
| Status CompactToLevel(const Options& options, const std::string& dbname, | ||
| int dest_level, bool need_reopen) { | ||
| std::unique_ptr<DB> db; | ||
| Options no_compact_opts = GetNoCompactionOptions(options); | ||
| if (dest_level == 0) { | ||
| // L0 has strict sequenceID requirements to files to it. It's safer | ||
| // to only put one compacted file to there. | ||
| // This is only used for converting to universal compaction with | ||
| // only one level. In this case, compacting to one file is also | ||
| // optimal. | ||
| no_compact_opts.target_file_size_base = 999999999999999; | ||
| } | ||
| Status s = OpenDb(no_compact_opts, dbname, &db); | ||
| if (!s.ok()) { | ||
| return s; | ||
| } | ||
| CompactRangeOptions cro; | ||
| cro.change_level = true; | ||
| cro.target_level = dest_level; | ||
| db->CompactRange(cro, nullptr, nullptr); | ||
|
|
||
| if (need_reopen) { | ||
| // Need to restart DB to rewrite the manifest file. | ||
| // In order to open a DB with specific num_levels, the manifest file should | ||
| // contain no record that mentiones any level beyond num_levels. Issuing a | ||
| // full compaction will move all the data to a level not exceeding | ||
| // num_levels, but the manifest may still contain previous record mentioning | ||
| // a higher level. Reopening the DB will force the manifest to be rewritten | ||
| // so that those records will be cleared. | ||
| db.reset(); | ||
| s = OpenDb(no_compact_opts, dbname, &db); | ||
| } | ||
| return s; | ||
| } | ||
|
|
||
| Status MigrateToUniversal(std::string dbname, const Options& old_opts, | ||
| const Options& new_opts) { | ||
| if (old_opts.num_levels <= new_opts.num_levels) { | ||
| return Status::OK(); | ||
| } else { | ||
| bool need_compact = false; | ||
| { | ||
| std::unique_ptr<DB> db; | ||
| Options opts = GetNoCompactionOptions(old_opts); | ||
| Status s = OpenDb(opts, dbname, &db); | ||
| if (!s.ok()) { | ||
| return s; | ||
| } | ||
| ColumnFamilyMetaData metadata; | ||
| db->GetColumnFamilyMetaData(&metadata); | ||
| if (!metadata.levels.empty() && | ||
| metadata.levels.back().level >= new_opts.num_levels) { | ||
| need_compact = true; | ||
| } | ||
| } | ||
| if (need_compact) { | ||
| return CompactToLevel(old_opts, dbname, new_opts.num_levels - 1, true); | ||
| } | ||
| return Status::OK(); | ||
| } | ||
| } | ||
|
|
||
| Status MigrateToLevelBase(std::string dbname, const Options& old_opts, | ||
| const Options& new_opts) { | ||
| if (!new_opts.level_compaction_dynamic_level_bytes) { | ||
| if (old_opts.num_levels == 1) { | ||
| return Status::OK(); | ||
| } | ||
| // Compact everything to level 1 to guarantee it can be safely opened. | ||
| Options opts = old_opts; | ||
| opts.target_file_size_base = new_opts.target_file_size_base; | ||
| // Although sometimes we can open the DB with the new option without error, | ||
| // We still want to compact the files to avoid the LSM tree to stuck | ||
| // in bad shape. For example, if the user changed the level size | ||
| // multiplier from 4 to 8, with the same data, we will have fewer | ||
| // levels. Unless we issue a full comaction, the LSM tree may stuck | ||
| // with more levels than needed and it won't recover automatically. | ||
| return CompactToLevel(opts, dbname, 1, true); | ||
| } else { | ||
| // Compact everything to the last level to guarantee it can be safely | ||
| // opened. | ||
| if (old_opts.num_levels == 1) { | ||
| return Status::OK(); | ||
| } else if (new_opts.num_levels > old_opts.num_levels) { | ||
| // Dynamic level mode requires data to be put in the last level first. | ||
| return CompactToLevel(new_opts, dbname, new_opts.num_levels - 1, false); | ||
| } else { | ||
| Options opts = old_opts; | ||
| opts.target_file_size_base = new_opts.target_file_size_base; | ||
| return CompactToLevel(opts, dbname, new_opts.num_levels - 1, true); | ||
| } | ||
| } | ||
| } | ||
| } // namespace | ||
|
|
||
| Status OptionChangeMigration(std::string dbname, const Options& old_opts, | ||
| const Options& new_opts) { | ||
| if (new_opts.compaction_style == CompactionStyle::kCompactionStyleUniversal) { | ||
| return MigrateToUniversal(dbname, old_opts, new_opts); | ||
| } else if (new_opts.compaction_style == | ||
| CompactionStyle::kCompactionStyleLevel) { | ||
| return MigrateToLevelBase(dbname, old_opts, new_opts); | ||
| } else { | ||
| return Status::NotSupported( | ||
| "Do not how to migrate to this compaction style"); | ||
| } | ||
| } | ||
| } // namespace rocksdb | ||
| #else | ||
| namespace rocksdb { | ||
| Status OptionChangeMigration(std::string dbname, const Options& old_opts, | ||
| const Options& new_opts) { | ||
| return Status::NotSupported(); | ||
| } | ||
| } // namespace rocksdb | ||
| #endif // ROCKSDB_LITE |
Oops, something went wrong.