From c9c777045fc7594ffbc9a014643e4a6385e3144f Mon Sep 17 00:00:00 2001 From: zhaojintaozhao Date: Sat, 2 Dec 2023 00:17:53 +0800 Subject: [PATCH] issue_913 add SimpleMergeSelector settings configuration --- src/CloudServices/selectPartsToMerge.cpp | 1 + .../MergeTree/MergeTreeDataMergerMutator.cpp | 3 + .../MergeTree/SimpleMergeSelector.cpp | 30 +++- src/Storages/MergeTree/SimpleMergeSelector.h | 154 +++++++++--------- 4 files changed, 111 insertions(+), 77 deletions(-) diff --git a/src/CloudServices/selectPartsToMerge.cpp b/src/CloudServices/selectPartsToMerge.cpp index f3d292a4e29..ab2364a7abc 100644 --- a/src/CloudServices/selectPartsToMerge.cpp +++ b/src/CloudServices/selectPartsToMerge.cpp @@ -215,6 +215,7 @@ ServerSelectPartsDecision selectPartsToMerge( else { SimpleMergeSelector::Settings merge_settings; + merge_settings.loadFromConfig(config); /// Override value from table settings merge_settings.max_parts_to_merge_at_once = data_settings->max_parts_to_merge_at_once; merge_settings.max_total_rows_to_merge = data_settings->cnch_merge_max_total_rows_to_merge; diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index d3883feee74..688dd37ffa8 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -399,6 +399,8 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectPartsToMerge( { SimpleMergeSelector::Settings merge_settings; /// Override value from table settings + auto & config = data.getContext()->getConfigRef(); + merge_settings.loadFromConfig(config); merge_settings.max_parts_to_merge_at_once = data_settings->max_parts_to_merge_at_once; if (aggressive) @@ -617,6 +619,7 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectPartsToMergeMulti( else { SimpleMergeSelector::Settings merge_settings; + merge_settings.loadFromConfig(config); /// Override value from table settings merge_settings.max_parts_to_merge_at_once = std::min(data_settings->cnch_merge_max_parts_to_merge, data_settings->max_parts_to_merge_at_once); merge_settings.enable_batch_select = enable_batch_select; diff --git a/src/Storages/MergeTree/SimpleMergeSelector.cpp b/src/Storages/MergeTree/SimpleMergeSelector.cpp index a60c4642865..a531ad26352 100644 --- a/src/Storages/MergeTree/SimpleMergeSelector.cpp +++ b/src/Storages/MergeTree/SimpleMergeSelector.cpp @@ -32,6 +32,34 @@ namespace DB { +namespace ErrorCodes +{ + extern const int UNKNOWN_SETTING; +} + +IMPLEMENT_SETTINGS_TRAITS(SimpleMergeSelectorSettingsTraits, LIST_OF_SIMPLE_MERGE_SELECTOR_SETTINGS) + +void SimpleMergeSelectorSettings::loadFromConfig(const Poco::Util::AbstractConfiguration & config) +{ + static std::string config_elem = "simple_merge_selector"; + if (!config.has(config_elem)) + return; + + Poco::Util::AbstractConfiguration::Keys config_keys; + config.keys(config_elem, config_keys); + + try + { + for (auto & key : config_keys) + set(key, config.getString(config_elem + "." + key)); + } + catch (Exception & e) + { + if (e.code() == ErrorCodes::UNKNOWN_SETTING) + e.addMessage("in DanceMergeSelector config"); + throw; + } +} namespace { @@ -157,7 +185,7 @@ bool allow( // std::cerr << "combined_ratio: " << combined_ratio << "\n"; - double lowered_base = interpolateLinear(settings.base, 2.0, combined_ratio); + double lowered_base = interpolateLinear(settings.base, settings.standard_baseline, combined_ratio); // std::cerr << "------- lowered_base: " << lowered_base << "\n"; diff --git a/src/Storages/MergeTree/SimpleMergeSelector.h b/src/Storages/MergeTree/SimpleMergeSelector.h index 7c339155e33..e5a882b0673 100644 --- a/src/Storages/MergeTree/SimpleMergeSelector.h +++ b/src/Storages/MergeTree/SimpleMergeSelector.h @@ -106,85 +106,87 @@ namespace DB class MergeScheduler; +#define LIST_OF_SIMPLE_MERGE_SELECTOR_SETTINGS(M) \ + /** Zero means unlimited. Can be overridden by the same merge tree setting. */ \ + M(UInt64, max_parts_to_merge_at_once, 100, "", 0) \ + M(UInt64, min_parts_to_merge_at_once, 3, "", 0) \ +\ + /** Zero means unlimited.*/ \ + /** Unique table will set it to a value < 2^32 in order to prevent rowid(UInt32) overflow */ \ + /** Too large part has no advantage since we cannot utilize parallelism. We set max_total_rows_to_merge as 2147483647.*/ \ + M(UInt64, max_total_rows_to_merge, 0xFFFFFFFF, "", 0) \ +\ + /** Minimum ratio of size of one part to all parts in set of parts to merge (for usual cases).*/ \ + /** For example, if all parts have equal size, it means, that at least 'base' number of parts should be merged.*/ \ + /** If parts has non-uniform sizes, then minimum number of parts to merge is effectively increased.*/ \ + /** This behaviour balances merge-tree workload.*/ \ + /** It called 'base', because merge-tree depth could be estimated as logarithm with that base.*/ \ + /** If base is higher - then tree gets more wide and narrow, lowering write amplification.*/ \ + /** If base is lower - then merges occurs more frequently, lowering number of parts in average.*/ \ + /** */ \ + /** We need some balance between write amplification and number of parts.*/ \ + M(Float, base, 5, "", 0) \ + /** Simple standard baseline for merge to lower the base. */ \ + M(Float, standard_baseline, 2.0, "", 0) \ +\ + /** Base is lowered until 1 (effectively means "merge any two parts") depending on several variables:*/ \ + /** */ \ + /** 1. Total number of parts in partition. If too many - then base is lowered.*/ \ + /** It means: when too many parts - do merges more urgently.*/ \ + /** */ \ + /** 2. Minimum age of parts participating in merge. If higher age - then base is lowered.*/ \ + /** It means: do less wide merges only rarely.*/ \ + /** */ \ + /** 3. Sum size of parts participating in merge. If higher - then more age is required to lower base. So, base is lowered slower.*/ \ + /** It means: for small parts, it's worth to merge faster, even not so wide or balanced.*/ \ + /** */ \ + /** We have multivariative dependency. Let it be logarithmic of size and somewhat multi-linear by other variables,*/ \ + /** between some boundary points, and constant outside.*/ \ + M(UInt64, min_size_to_lower_base, 1024 * 1024, "", 0) \ + M(UInt64, max_size_to_lower_base, 100ULL * 1024 * 1024 * 1024, "", 0) \ +\ + M(UInt64, min_age_to_lower_base_at_min_size, 10, "", 0) \ + M(UInt64, min_age_to_lower_base_at_max_size, 10, "", 0) \ + M(UInt64, max_age_to_lower_base_at_min_size, 3600, "", 0) \ + M(UInt64, max_age_to_lower_base_at_max_size, 30 * 86400, "", 0) \ +\ + M(UInt64, min_parts_to_lower_base, 10, "", 0) \ + M(UInt64, max_parts_to_lower_base, 50, "", 0) \ +\ + /** Add this to size before all calculations. It means: merging even very small parts has it's fixed cost.*/ \ + M(UInt64, size_fixed_cost_to_add, 5 * 1024 * 1024, "", 0) \ +\ + /** Heuristic:*/ \ + /** Make some preference for ranges, that sum_size is like (in terms of ratio) to part previous at left.*/ \ + M(Bool, enable_heuristic_to_align_parts, true, "", 0) \ + M(Float, heuristic_to_align_parts_min_ratio_of_sum_size_to_prev_part, 0.9, "", 0) \ + M(Float, heuristic_to_align_parts_max_absolute_difference_in_powers_of_two, 0.5, "", 0) \ + M(Float, heuristic_to_align_parts_max_score_adjustment, 0.75, "", 0) \ +\ + /** Heuristic:*/ \ + /** From right side of range, remove all parts, that size is less than specified ratio of sum_size.*/ \ + M(Bool, enable_heuristic_to_remove_small_parts_at_right, true, "", 0) \ + M(Float, heuristic_to_remove_small_parts_at_right_max_ratio, 0.01, "", 0) \ +\ + /** For batch select mode.*/ \ + /** Currently, only part-merger tool use thesis options.*/ \ + M(Bool, enable_batch_select, false, "", 0) \ + M(UInt64, max_rows_to_merge_at_once, 30000000, "", 0) + +DECLARE_SETTINGS_TRAITS(SimpleMergeSelectorSettingsTraits, LIST_OF_SIMPLE_MERGE_SELECTOR_SETTINGS) + + +class SimpleMergeSelectorSettings : public BaseSettings +{ +public: + void loadFromConfig(const Poco::Util::AbstractConfiguration & config); +}; + + class SimpleMergeSelector final : public IMergeSelector { public: - struct Settings - { - /// Zero means unlimited. Can be overridden by the same merge tree setting. - size_t max_parts_to_merge_at_once = 100; - - /** Zero means unlimited. - * Unique table will set it to a value < 2^32 in order to prevent rowid(UInt32) overflow - * Too large part has no advantage since we cannot utilize parallelism. We set max_total_rows_to_merge as 2147483647. - */ - size_t max_total_rows_to_merge = 0xFFFFFFFF; - - /** Minimum ratio of size of one part to all parts in set of parts to merge (for usual cases). - * For example, if all parts have equal size, it means, that at least 'base' number of parts should be merged. - * If parts has non-uniform sizes, then minimum number of parts to merge is effectively increased. - * This behaviour balances merge-tree workload. - * It called 'base', because merge-tree depth could be estimated as logarithm with that base. - * - * If base is higher - then tree gets more wide and narrow, lowering write amplification. - * If base is lower - then merges occurs more frequently, lowering number of parts in average. - * - * We need some balance between write amplification and number of parts. - */ - double base = 5; - - /** Base is lowered until 1 (effectively means "merge any two parts") depending on several variables: - * - * 1. Total number of parts in partition. If too many - then base is lowered. - * It means: when too many parts - do merges more urgently. - * - * 2. Minimum age of parts participating in merge. If higher age - then base is lowered. - * It means: do less wide merges only rarely. - * - * 3. Sum size of parts participating in merge. If higher - then more age is required to lower base. So, base is lowered slower. - * It means: for small parts, it's worth to merge faster, even not so wide or balanced. - * - * We have multivariative dependency. Let it be logarithmic of size and somewhat multi-linear by other variables, - * between some boundary points, and constant outside. - */ - - size_t min_size_to_lower_base = 1024 * 1024; - size_t max_size_to_lower_base = 100ULL * 1024 * 1024 * 1024; - - time_t min_age_to_lower_base_at_min_size = 10; - time_t min_age_to_lower_base_at_max_size = 10; - time_t max_age_to_lower_base_at_min_size = 3600; - time_t max_age_to_lower_base_at_max_size = 30 * 86400; - - size_t min_parts_to_lower_base = 10; - size_t max_parts_to_lower_base = 50; - - /// Add this to size before all calculations. It means: merging even very small parts has it's fixed cost. - size_t size_fixed_cost_to_add = 5 * 1024 * 1024; - - /** Heuristic: - * Make some preference for ranges, that sum_size is like (in terms of ratio) to part previous at left. - */ - bool enable_heuristic_to_align_parts = true; - double heuristic_to_align_parts_min_ratio_of_sum_size_to_prev_part = 0.9; - double heuristic_to_align_parts_max_absolute_difference_in_powers_of_two = 0.5; - double heuristic_to_align_parts_max_score_adjustment = 0.75; - - /** Heuristic: - * From right side of range, remove all parts, that size is less than specified ratio of sum_size. - */ - bool enable_heuristic_to_remove_small_parts_at_right = true; - double heuristic_to_remove_small_parts_at_right_max_ratio = 0.01; - - /** - * For batch select mode. - * - * Currently, only part-merger tool use thesis options. - */ - bool enable_batch_select = false; - size_t min_parts_to_merge_at_once = 3; - size_t max_rows_to_merge_at_once = 30000000; - }; + using Settings = SimpleMergeSelectorSettings; explicit SimpleMergeSelector(const Settings & settings_) : settings(settings_) {