From 437942e48119a4108ca34724cf1fde5fc440f96b Mon Sep 17 00:00:00 2001 From: Yi Wu Date: Wed, 2 Nov 2016 15:22:13 -0700 Subject: [PATCH] Add avoid_flush_during_shutdown DB option Summary: Add avoid_flush_during_shutdown DB option. Closes https://github.com/facebook/rocksdb/pull/1451 Differential Revision: D4108643 Pulled By: yiwu-arbug fbshipit-source-id: abdaf4d --- HISTORY.md | 3 +++ db/db_impl.cc | 3 ++- db/db_options_test.cc | 22 ++++++++++++++++++++++ include/rocksdb/options.h | 9 +++++++++ util/db_options.cc | 9 +++++++-- util/db_options.h | 1 + util/options.cc | 6 ++++-- util/options_helper.cc | 2 ++ util/options_helper.h | 6 +++++- util/options_settable_test.cc | 3 ++- util/testutil.cc | 2 ++ 11 files changed, 59 insertions(+), 7 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 5473989f7f2..34074189b46 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -3,6 +3,9 @@ ### Public API Change * Options::max_bytes_for_level_multiplier is now a double along with all getters and setters. +### New Features +* Add avoid_flush_during_shutdown option, which speeds up DB shutdown by not flushing unpersisted data (i.e. with disableWAL = true). Unpersisted data will be lost. The options is dynamically changeable. + ## 4.13.0 (10/18/2016) ### Public API Change * DB::GetOptions() reflect dynamic changed options (i.e. through DB::SetOptions()) and return copy of options instead of reference. diff --git a/db/db_impl.cc b/db/db_impl.cc index e7bbf4f4fbc..ad04bc396c4 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -375,7 +375,8 @@ void DBImpl::CancelAllBackgroundWork(bool wait) { InstrumentedMutexLock l(&mutex_); if (!shutting_down_.load(std::memory_order_acquire) && - has_unpersisted_data_) { + has_unpersisted_data_ && + !mutable_db_options_.avoid_flush_during_shutdown) { for (auto cfd : *versions_->GetColumnFamilySet()) { if (!cfd->IsDropped() && !cfd->mem()->IsEmpty()) { cfd->Ref(); diff --git a/db/db_options_test.cc b/db/db_options_test.cc index ad315c31636..0b8b8739d36 100644 --- a/db/db_options_test.cc +++ b/db/db_options_test.cc @@ -256,6 +256,28 @@ TEST_F(DBOptionsTest, SetBackgroundCompactionThreads) { ASSERT_EQ(3, dbfull()->TEST_BGCompactionsAllowed()); } +TEST_F(DBOptionsTest, AvoidFlushDuringShutdown) { + Options options; + options.create_if_missing = true; + options.disable_auto_compactions = true; + WriteOptions write_without_wal; + write_without_wal.disableWAL = true; + + ASSERT_FALSE(options.avoid_flush_during_shutdown); + DestroyAndReopen(options); + ASSERT_OK(Put("foo", "v1", write_without_wal)); + Reopen(options); + ASSERT_EQ("v1", Get("foo")); + ASSERT_EQ("1", FilesPerLevel()); + + DestroyAndReopen(options); + ASSERT_OK(Put("foo", "v2", write_without_wal)); + ASSERT_OK(dbfull()->SetDBOptions({{"avoid_flush_during_shutdown", "true"}})); + Reopen(options); + ASSERT_EQ("NOT_FOUND", Get("foo")); + ASSERT_EQ("", FilesPerLevel()); +} + #endif // ROCKSDB_LITE } // namespace rocksdb diff --git a/include/rocksdb/options.h b/include/rocksdb/options.h index 6da0b39443f..2d7272d4875 100644 --- a/include/rocksdb/options.h +++ b/include/rocksdb/options.h @@ -1354,6 +1354,15 @@ struct DBOptions { // // DEFAULT: false bool avoid_flush_during_recovery; + + // By default RocksDB will flush all memtables on DB close if there are + // unpersisted data (i.e. with WAL disabled) The flush can be skip to speedup + // DB close. Unpersisted data WILL BE LOST. + // + // DEFAULT: false + // + // Dynamically changeable through SetDBOptions() API. + bool avoid_flush_during_shutdown; }; // Options to control the behavior of a database (passed to DB::Open) diff --git a/util/db_options.cc b/util/db_options.cc index e40840b682a..4f20d568668 100644 --- a/util/db_options.cc +++ b/util/db_options.cc @@ -224,17 +224,22 @@ void ImmutableDBOptions::Dump(Logger* log) const { } MutableDBOptions::MutableDBOptions() - : base_background_compactions(1), max_background_compactions(1) {} + : base_background_compactions(1), + max_background_compactions(1), + avoid_flush_during_shutdown(false) {} MutableDBOptions::MutableDBOptions(const DBOptions& options) : base_background_compactions(options.base_background_compactions), - max_background_compactions(options.max_background_compactions) {} + max_background_compactions(options.max_background_compactions), + avoid_flush_during_shutdown(options.avoid_flush_during_shutdown) {} void MutableDBOptions::Dump(Logger* log) const { Header(log, " Options.base_background_compactions: %d", base_background_compactions); Header(log, " Options.max_background_compactions: %d", max_background_compactions); + Header(log, " Options.avoid_flush_during_shutdown: %d", + avoid_flush_during_shutdown); } } // namespace rocksdb diff --git a/util/db_options.h b/util/db_options.h index 813e4f2cf57..1841742aff2 100644 --- a/util/db_options.h +++ b/util/db_options.h @@ -94,6 +94,7 @@ struct MutableDBOptions { int base_background_compactions; int max_background_compactions; + bool avoid_flush_during_shutdown; }; } // namespace rocksdb diff --git a/util/options.cc b/util/options.cc index b8f3f51cb6c..bdacb7bf2c2 100644 --- a/util/options.cc +++ b/util/options.cc @@ -228,7 +228,8 @@ DBOptions::DBOptions() #endif // ROCKSDB_LITE fail_if_options_file_error(false), dump_malloc_stats(false), - avoid_flush_during_recovery(false) { + avoid_flush_during_recovery(false), + avoid_flush_during_shutdown(false) { } DBOptions::DBOptions(const Options& options) @@ -301,7 +302,8 @@ DBOptions::DBOptions(const Options& options) #endif // ROCKSDB_LITE fail_if_options_file_error(options.fail_if_options_file_error), dump_malloc_stats(options.dump_malloc_stats), - avoid_flush_during_recovery(options.avoid_flush_during_recovery) { + avoid_flush_during_recovery(options.avoid_flush_during_recovery), + avoid_flush_during_shutdown(options.avoid_flush_during_shutdown) { } static const char* const access_hints[] = { diff --git a/util/options_helper.cc b/util/options_helper.cc index 2818b64e496..32a0e4a8c59 100644 --- a/util/options_helper.cc +++ b/util/options_helper.cc @@ -118,6 +118,8 @@ DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options, options.dump_malloc_stats = immutable_db_options.dump_malloc_stats; options.avoid_flush_during_recovery = immutable_db_options.avoid_flush_during_recovery; + options.avoid_flush_during_shutdown = + mutable_db_options.avoid_flush_during_shutdown; return options; } diff --git a/util/options_helper.h b/util/options_helper.h index 272b5a88210..373c4bcf0f3 100644 --- a/util/options_helper.h +++ b/util/options_helper.h @@ -343,7 +343,11 @@ static std::unordered_map db_options_type_info = { OptionVerificationType::kNormal, false, 0}}, {"avoid_flush_during_recovery", {offsetof(struct DBOptions, avoid_flush_during_recovery), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}}; + OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, + {"avoid_flush_during_shutdown", + {offsetof(struct DBOptions, avoid_flush_during_shutdown), + OptionType::kBoolean, OptionVerificationType::kNormal, true, + offsetof(struct MutableDBOptions, avoid_flush_during_shutdown)}}}; static std::unordered_map cf_options_type_info = { /* not yet supported diff --git a/util/options_settable_test.cc b/util/options_settable_test.cc index e562769536c..a4d2c437fa5 100644 --- a/util/options_settable_test.cc +++ b/util/options_settable_test.cc @@ -288,7 +288,8 @@ TEST_F(OptionsSettableTest, DBOptionsAllFieldsSettable) { "info_log_level=DEBUG_LEVEL;" "dump_malloc_stats=false;" "allow_2pc=false;" - "avoid_flush_during_recovery=false;", + "avoid_flush_during_recovery=false;" + "avoid_flush_during_shutdown=false;", new_options)); ASSERT_EQ(unset_bytes_base, NumUnsetBytes(new_options_ptr, sizeof(DBOptions), diff --git a/util/testutil.cc b/util/testutil.cc index 3e8c4f42cfe..a04db034928 100644 --- a/util/testutil.cc +++ b/util/testutil.cc @@ -253,6 +253,8 @@ void RandomInitDBOptions(DBOptions* db_opt, Random* rnd) { db_opt->use_adaptive_mutex = rnd->Uniform(2); db_opt->use_fsync = rnd->Uniform(2); db_opt->recycle_log_file_num = rnd->Uniform(2); + db_opt->avoid_flush_during_recovery = rnd->Uniform(2); + db_opt->avoid_flush_during_shutdown = rnd->Uniform(2); // int options db_opt->max_background_compactions = rnd->Uniform(100);