Skip to content

Commit

Permalink
Serialize only options that have changed from the some default (#648)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mrambacher @udi-speedb
mrambacher authored and udi-speedb committed Nov 23, 2023
1 parent e95aa35 commit d09fe38
Show file tree
Hide file tree
Showing 6 changed files with 339 additions and 50 deletions.
2 changes: 1 addition & 1 deletion HISTORY.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
### New Features
* Non-Blocking Manual Compaction (CompactRange()) - Support non-blocking manual compactions by setting a new CompactRangeOptions option (async_completion_cb). When set, the CompactRange() call will return control to the caller immediately. The manual compaction iteslf will be performed in an internally created thread. The manual compaction will ALWAYS call the specified callback upon completion and provide the completion status (#597).
* Change the internal Configurable API SerializeOptions to return UserProperties (instead of the final string representation). Added ToString methods to the ConfigurableOptions class to complete the serialization of Options properties.

* Added ConfigOptions::compare_to. When set, this value causes only values that have been changed to be part of the serialized output.

### Enhancements
* Unit Testing: Expose the disallow_trivial_move flag in the MoveFilesToLevel testing utility (#677).
Expand Down
3 changes: 3 additions & 0 deletions include/rocksdb/convenience.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,9 @@ struct ConfigOptions {
// The object registry to use for this options
std::shared_ptr<ObjectRegistry> registry;

// If set, only changes from this reference version will be serialized.
Configurable* compare_to = nullptr;

bool IsShallow() const { return depth == Depth::kDepthShallow; }
bool IsDetailed() const { return depth == Depth::kDepthDetailed; }

Expand Down
107 changes: 100 additions & 7 deletions options/configurable.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -541,16 +541,90 @@ Status ConfigurableHelper::SerializeOptions(const ConfigOptions& config_options,
const std::string& prefix,
OptionProperties* props) {
assert(props);
for (auto const& opt_iter : configurable.options_) {
if (opt_iter.type_map != nullptr) {
Status s = OptionTypeInfo::SerializeType(config_options, prefix,
*(opt_iter.type_map),
opt_iter.opt_ptr, props);
if (!s.ok()) {
return s;
ConfigOptions copy = config_options;
auto compare_to = config_options.compare_to;
if (compare_to != nullptr && !MayBeEquivalent(configurable, *compare_to)) {
// If we are comparing this type to another, first see if the types
// are the same. If not, forget it
compare_to = nullptr;
}

Status s;
for (size_t i = 0; i < configurable.options_.size(); i++) {
const auto& opt = configurable.options_[i];
if (opt.type_map != nullptr) {
const auto opt_addr = opt.opt_ptr;
for (const auto& opt_iter : *(opt.type_map)) {
std::string single;
const auto& opt_name = opt_iter.first;
const auto& opt_info = opt_iter.second;
bool should_serialize = opt_info.ShouldSerialize();
if (should_serialize && compare_to != nullptr) {
// This option should be serialized but there is a possiblity that it
// matches the default. Check to see if we really should serialize it
std::string mismatch;
if (opt_info.IsConfigurable() &&
opt_info.IsEnabled(OptionTypeFlags::kStringNameOnly) &&
!config_options.IsDetailed()) {
// If it is a Configurable name-only and we are not printing the
// details, then compare loosely
copy.sanity_level = ConfigOptions::kSanityLevelLooselyCompatible;
if (opt_info.AreEqual(copy, opt_name, opt_addr,
compare_to->options_[i].opt_ptr, &mismatch)) {
should_serialize = false;
}
copy.sanity_level = config_options.sanity_level;
} else if (opt_info.AreEqual(config_options, opt_name, opt_addr,
compare_to->options_[i].opt_ptr,
&mismatch)) {
should_serialize = false;
}
}
if (should_serialize) {
if (compare_to != nullptr && opt_info.IsCustomizable()) {
copy.compare_to = opt_info.AsRawPointer<Customizable>(
compare_to->options_[i].opt_ptr);
} else {
copy.compare_to = compare_to;
}
s = SerializeOption(copy,
OptionTypeInfo::MakePrefix(prefix, opt_name),
opt_info, opt_addr, &single);
if (!s.ok()) {
return s;
} else if (!single.empty()) {
props->insert_or_assign(opt_name, single);
}
}
}
}
}
return s;
}

Status ConfigurableHelper::SerializeOption(const ConfigOptions& config_options,
const std::string& opt_name,
const OptionTypeInfo& opt_info,
const void* opt_addr,
std::string* value) {
if (opt_info.ShouldSerialize()) {
if (!config_options.mutable_options_only) {
return opt_info.Serialize(config_options, opt_name, opt_addr, value);
} else if (opt_info.IsMutable()) {
ConfigOptions copy = config_options;
copy.mutable_options_only = false;
return opt_info.Serialize(copy, opt_name, opt_addr, value);
} else if (opt_info.IsConfigurable()) {
// If it is a Configurable and we are either printing all of the
// details or not printing only the name, this option should be
// included in the list
if (config_options.IsDetailed() ||
!opt_info.IsEnabled(OptionTypeFlags::kStringNameOnly)) {
return opt_info.Serialize(config_options, opt_name, opt_addr, value);
}
}
}
value->clear();
return Status::OK();
}

Expand Down Expand Up @@ -596,6 +670,25 @@ Status ConfigurableHelper::ListOptions(
//
//*******************************************************************************

bool ConfigurableHelper::MayBeEquivalent(const Configurable& this_one,
const Configurable& that_one) {
if (this_one.options_.size() != that_one.options_.size()) {
// The two types do not have the same number of registered options,
// therefore they cannot be the same.
return false;
}

for (size_t i = 0; i < this_one.options_.size(); i++) {
const auto& this_opt = this_one.options_[i];
const auto& that_opt = that_one.options_[i];
if (this_opt.name != that_opt.name ||
this_opt.type_map != that_opt.type_map) {
return false;
}
}
return true;
}

bool Configurable::AreEquivalent(const ConfigOptions& config_options,
const Configurable* other,
std::string* name) const {
Expand Down
17 changes: 17 additions & 0 deletions options/configurable_helper.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,19 @@ class ConfigurableHelper {
const std::string& prefix,
OptionProperties* props);

// Serializes a single option to its string representation
// @param opt_name The name of the option
// @param opt_info The type and related information of the option
// @param opt_addr The address of the option
// @param value The string representation of the option.
// @return OK If the options for this object wer successfully serialized.
// @return InvalidArgument If one or more of the options could not be
// serialized.
static Status SerializeOption(const ConfigOptions& config_options,
const std::string& opt_name,
const OptionTypeInfo& opt_info,
const void* opt_addr, std::string* value);

// Internal method to list the option names for this object.
// Classes may override this value to change its behavior.
// @see ListOptions for more details
Expand All @@ -173,6 +186,10 @@ class ConfigurableHelper {
const Configurable& that_one,
std::string* mismatch);

// Checks to see if the two Configurable classes may be equivalent
static bool MayBeEquivalent(const Configurable& this_one,
const Configurable& that_one);

private:
// Looks for the option specified by name in the RegisteredOptions.
// This method traverses the types in the input options vector. If an entry
Expand Down
50 changes: 8 additions & 42 deletions options/options_helper.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1144,38 +1144,22 @@ Status OptionTypeInfo::SerializeType(
const ConfigOptions& config_options, const std::string& prefix,
const std::unordered_map<std::string, OptionTypeInfo>& type_map,
const void* opt_addr, OptionProperties* props) {
Status status;
for (const auto& iter : type_map) {
std::string single;
const auto& opt_name = iter.first;
const auto& opt_info = iter.second;
if (opt_info.ShouldSerialize()) {
if (!config_options.mutable_options_only) {
status = opt_info.Serialize(
config_options, MakePrefix(prefix, opt_name), opt_addr, &single);
} else if (opt_info.IsMutable()) {
ConfigOptions copy = config_options;
copy.mutable_options_only = false;
status = opt_info.Serialize(copy, MakePrefix(prefix, opt_name),
opt_addr, &single);
} else if (opt_info.IsConfigurable()) {
// If it is a Configurable and we are either printing all of the
// details or not printing only the name, this option should be
// included in the list
if (config_options.IsDetailed() ||
!opt_info.IsEnabled(OptionTypeFlags::kStringNameOnly)) {
status = opt_info.Serialize(
config_options, MakePrefix(prefix, opt_name), opt_addr, &single);
}
}
Status status = ConfigurableHelper::SerializeOption(
config_options, MakePrefix(prefix, opt_name), opt_info, opt_addr,
&single);
if (!status.ok()) {
return status;
} else if (!single.empty()) {
props->insert_or_assign(iter.first, single);
}
}
}
return status;
return Status::OK();
}

Status OptionTypeInfo::SerializeStruct(
Expand Down Expand Up @@ -1305,7 +1289,8 @@ bool OptionTypeInfo::AreEqual(const ConfigOptions& config_options,
const void* const that_ptr,
std::string* mismatch) const {
auto level = GetSanityLevel();
if (!config_options.IsCheckEnabled(level)) {
if (config_options.compare_to == nullptr &&
!config_options.IsCheckEnabled(level)) {
return true; // If the sanity level is not being checked, skip it
}
if (this_ptr == nullptr || that_ptr == nullptr) {
Expand All @@ -1332,7 +1317,8 @@ bool OptionTypeInfo::AreEqual(const ConfigOptions& config_options,
} else if (this_config != nullptr && that_config != nullptr) {
std::string bad_name;
bool matches;
if (level < config_options.sanity_level) {
if (config_options.compare_to == nullptr &&
level < config_options.sanity_level) {
ConfigOptions copy = config_options;
copy.sanity_level = level;
matches = this_config->AreEquivalent(copy, that_config, &bad_name);
Expand Down Expand Up @@ -1414,26 +1400,6 @@ bool OptionTypeInfo::StructsAreEqual(
return matches;
}

bool MatchesOptionsTypeFromMap(
const ConfigOptions& config_options,
const std::unordered_map<std::string, OptionTypeInfo>& type_map,
const void* const this_ptr, const void* const that_ptr,
std::string* mismatch) {
for (auto& pair : type_map) {
// We skip checking deprecated variables as they might
// contain random values since they might not be initialized
if (config_options.IsCheckEnabled(pair.second.GetSanityLevel())) {
if (!pair.second.AreEqual(config_options, pair.first, this_ptr, that_ptr,
mismatch) &&
!pair.second.AreEqualByName(config_options, pair.first, this_ptr,
that_ptr)) {
return false;
}
}
}
return true;
}

bool OptionTypeInfo::AreEqualByName(const ConfigOptions& config_options,
const std::string& opt_name,
const void* const this_ptr,
Expand Down

0 comments on commit d09fe38

Please sign in to comment.