From d2929c2ade5bd0125a73d53280c82ae5da86218e Mon Sep 17 00:00:00 2001 From: Amnon Heiman Date: Sun, 14 Jan 2024 12:18:21 +0200 Subject: [PATCH 1/4] metrics_api.hh: Remove duplication of metric family metadata Metric family metadata consists of information relevant to all metrics with the same name (but with different labels). For example, name, description, and aggregate labels. The metric family metadata is found in two data structures: the _value_map that holds all registered metrics and _metadata that contains only reported metrics. This patch makes the _metadata hold a reference to the data in the _value_map instead of copying it, which serves two purposes: 1. Minor performance gain (memory and computation). 2. It will now be possible for a change in _value_map to be reflected in _metadata. Signed-off-by: Amnon Heiman --- include/seastar/core/metrics_api.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/seastar/core/metrics_api.hh b/include/seastar/core/metrics_api.hh index d84870b6f5b..42d7c25b5bb 100644 --- a/include/seastar/core/metrics_api.hh +++ b/include/seastar/core/metrics_api.hh @@ -333,7 +333,7 @@ using metric_metadata_fifo = std::deque; * each of the metric. */ struct metric_family_metadata { - metric_family_info mf; + metric_family_info& mf; metric_metadata_fifo metrics; }; From 352c8ce00f328f3fe95b3c053c2b81a7692d30d4 Mon Sep 17 00:00:00 2001 From: Amnon Heiman Date: Mon, 26 Feb 2024 16:46:50 +0200 Subject: [PATCH 2/4] core/relabel_config.hh: support metric family config This patch adds support for metric family config. A metric family config is a configuration that relates to all the metrics with the same name (but with different labels). The configuration would allow changing the labels aggregation for metrics. Besides the configuration class, which is similar to metrics_relabel_config, two enhancements were added to the relabel_config_regex: 1. empty() which check if it's empty or not. 2. match() that checks if the regex is nonempty and matches a string without returning the matched group. Signed-off-by: Amnon Heiman --- include/seastar/core/relabel_config.hh | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/include/seastar/core/relabel_config.hh b/include/seastar/core/relabel_config.hh index 1e0e17ee659..d8fa1e89c05 100644 --- a/include/seastar/core/relabel_config.hh +++ b/include/seastar/core/relabel_config.hh @@ -58,6 +58,13 @@ public: _regex = std::regex(_regex_str); return *this; } + bool empty() const noexcept { + return _regex_str.empty(); + } + + bool match(const std::string& str) const noexcept { + return !empty() && std::regex_match(str, _regex); + } }; /*! @@ -103,6 +110,25 @@ struct relabel_config { */ relabel_config::relabel_action relabel_config_action(const std::string& action); +/*! + * \brief metric_family_config allow changing metrics family configuration + * + * Allow changing metrics family configuration + * Supports changing the aggregation labels. + * The metrics family can be identified by a name or by regex; name-matching is + * more efficient and should be used for a single metrics family. + * + * name - optional exact metric name + * regex_name - if set, all the metrics name that match the regular expression + * aggregate_labels - The labels to aggregate the metrics by. + * + */ +struct metric_family_config { + std::string name; + relabel_config_regex regex_name = ""; + std::vector aggregate_labels; +}; + SEASTAR_MODULE_EXPORT_END } From 9c148f395d2f9079396f5b1eab9d3f38d9606e30 Mon Sep 17 00:00:00 2001 From: Amnon Heiman Date: Mon, 26 Feb 2024 17:07:54 +0200 Subject: [PATCH 3/4] metrics: Adds metric_family_config support Family config is a configuration that relates to all metrics with the same name but with different labels values. set_metric_family_configs allows changing that configuration during run time. Specifically, change the label aggregation based on a metric name. The following is an example for setting the aggregate labels for the metric test_gauge_1 and all metrics matching the regex test_gauge1.*: std::vector fc(2); fc[0].name = "test_gauge_1"; fc[0].aggregate_labels = { "lb" }; fc[1].regex_name = "test_gauge1.*"; fc[1].aggregate_labels = { "ll", "aa" }; sm::set_metric_family_configs(fc); Signed-off-by: Amnon Heiman include/seastar/core/metrics_api.hh --- include/seastar/core/metrics_api.hh | 45 +++++++++++++++++++++++++++-- src/core/metrics.cc | 34 ++++++++++++++++++++++ 2 files changed, 77 insertions(+), 2 deletions(-) diff --git a/include/seastar/core/metrics_api.hh b/include/seastar/core/metrics_api.hh index 42d7c25b5bb..b77276d876d 100644 --- a/include/seastar/core/metrics_api.hh +++ b/include/seastar/core/metrics_api.hh @@ -69,6 +69,8 @@ namespace metrics { SEASTAR_MODULE_EXPORT struct relabel_config; +SEASTAR_MODULE_EXPORT +struct metric_family_config; /*! * \brief result of metric relabeling * @@ -330,10 +332,17 @@ using metric_metadata_fifo = std::deque; * * The meta data of a metric family compose of the * metadata of the family, and a vector of the metadata for - * each of the metric. + * each of the metrics. + * + * The struct is used for two purposes. First, it allows iterating over all metric_families + * and all metrics related to them. Second, it only contains enabled metrics, + * making disabled metrics more efficient. + * The struct is recreated when impl._value_map changes + * Using a pointer to the family_info metadata is an efficient way to get + * from a metric_family to its metadata based on its name. */ struct metric_family_metadata { - metric_family_info& mf; + metric_family_info& mf; //This points to impl._value_map metric_metadata_fifo metrics; }; @@ -358,6 +367,7 @@ class impl { std::set _labels; std::vector> _current_metrics; std::vector _relabel_configs; + std::vector _metric_family_configs; public: value_map& get_value_map() { return _value_map; @@ -398,6 +408,13 @@ public: const std::vector& get_relabel_configs() const noexcept { return _relabel_configs; } + const std::vector& get_metric_family_configs() const noexcept { + return _metric_family_configs; + } + + void set_metric_family_configs(const std::vector& metrics_config); + + void update_aggregate(metric_family_info& mf) const noexcept; }; const value_map& get_value_map(); @@ -486,5 +503,29 @@ future set_relabel_configs(const std::vector& get_relabel_configs(); +/* + * \brief change the metrics family config + * + * Family config is a configuration that relates to all metrics with the same name but with different labels. + * set_metric_family_configs allows changing that configuration during run time. + * Specifically, change the label aggregation based on a metric name. + * + * The following is an example for setting the aggregate labels for the metric test_gauge_1 + * and all metrics matching the regex test_gauge1.*: + * + * std::vector fc(2); + * fc[0].name = "test_gauge_1"; + * fc[0].aggregate_labels = { "lb" }; + * fc[1].regex_name = "test_gauge1.*"; + * fc[1].aggregate_labels = { "ll", "aa" }; + * sm::set_metric_family_configs(fc); + */ +void set_metric_family_configs(const std::vector& metrics_config); + +/* + * \brief return the current metric_family_config + * This function returns a vector of the current metrics family config + */ +const std::vector& get_metric_family_configs(); } } diff --git a/src/core/metrics.cc b/src/core/metrics.cc index af2e8937e0f..3c703df826f 100644 --- a/src/core/metrics.cc +++ b/src/core/metrics.cc @@ -130,7 +130,20 @@ future set_relabel_configs(const std::vector& get_relabel_configs() { return impl::get_local_impl()->get_relabel_configs(); } +void impl::impl::update_aggregate(metric_family_info& mf) const noexcept { + for (const auto& fc : _metric_family_configs) { + if (fc.name == mf.name || fc.regex_name.match(mf.name)) { + mf.aggregate_labels = fc.aggregate_labels; + } + } +} +void set_metric_family_configs(const std::vector& family_config) { + impl::get_local_impl()->set_metric_family_configs(family_config); +} +const std::vector& get_metric_family_configs() { + return impl::get_local_impl()->get_metric_family_configs(); +} static bool apply_relabeling(const relabel_config& rc, impl::metric_info& info) { std::stringstream s; @@ -454,6 +467,7 @@ void impl::add_registration(const metric_id& id, const metric_type& type, metric _value_map[name].info().inherit_type = type.type_name; _value_map[name].info().name = id.full_name(); _value_map[name].info().aggregate_labels = aggregate_labels; + impl::update_aggregate(_value_map[name].info()); _value_map[name][rm->info().id.labels()] = rm; } dirty(); @@ -503,6 +517,26 @@ future impl::set_relabel_configs(const std::vector(conflicts); } + +void impl::set_metric_family_configs(const std::vector& family_config) { + _metric_family_configs = family_config; + bool has_regex = false; + for (const auto& fc : family_config) { + has_regex |= !fc.regex_name.empty(); + if (fc.name != "" && _value_map.find(fc.name) != _value_map.end()) { + _value_map[fc.name].info().aggregate_labels = fc.aggregate_labels; + } + } + if (has_regex) { + for (auto& [name, family] : _value_map) { + for (const auto& fc : family_config) { + if (fc.regex_name.match(name)) { + family.info().aggregate_labels = fc.aggregate_labels; + } + } + } + } +} } const bool metric_disabled = false; From 9a80dce9e90629ce32f057e01e1a4e1331996e81 Mon Sep 17 00:00:00 2001 From: Amnon Heiman Date: Mon, 26 Feb 2024 17:09:24 +0200 Subject: [PATCH 4/4] tests/unit/metrics_test.cc: Test metrics family config This patch adds a test for the family config API. it register metrics and change their aggregate labels. It check that both metrics that create before and after the call to set_metric_family_configs works as expected. Signed-off-by: Amnon Heiman --- tests/unit/metrics_test.cc | 63 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/tests/unit/metrics_test.cc b/tests/unit/metrics_test.cc index 8d96d547851..0012e56b44a 100644 --- a/tests/unit/metrics_test.cc +++ b/tests/unit/metrics_test.cc @@ -224,6 +224,69 @@ SEASTAR_THREAD_TEST_CASE(test_relabel_add_labels) { sm::set_relabel_configs({}).get(); } +SEASTAR_THREAD_TEST_CASE(test_metrics_family_aggregate) { + using namespace seastar::metrics; + namespace sm = seastar::metrics; + sm::metric_groups app_metrics; + sm::label lb("lb"); + app_metrics.add_group("test", { + sm::make_gauge("gauge_1", sm::description("gague 1"), [] { return 1; })(lb("1")), + sm::make_gauge("gauge_1", sm::description("gague 1"), [] { return 2; })(lb("2")), + sm::make_counter("counter_1", sm::description("counter 1"), [] { return 3; })(lb("1")), + sm::make_counter("counter_1", sm::description("counter 1"), [] { return 4; })(lb("2")) + }); + std::vector rl(2); + rl[0].source_labels = {"__name__"}; + rl[0].action = sm::relabel_config::relabel_action::drop; + + rl[1].source_labels = {"lb"}; + rl[1].action = sm::relabel_config::relabel_action::keep; + // Dropping the lev label would cause a conflict, but not crash the system + sm::set_relabel_configs(rl).get(); + + std::vector fc(2); + fc[0].name = "test_gauge_1"; + fc[0].aggregate_labels = { "lb" }; + fc[1].regex_name = "test_gauge1.*"; + fc[1].aggregate_labels = { "ll", "aa" }; + sm::set_metric_family_configs(fc); + seastar::foreign_ptr values = seastar::metrics::impl::get_values(); + int count = 0; + for (auto&& md : (*values->metadata)) { + if (md.mf.name == "test_gauge_1") { + BOOST_CHECK_EQUAL(md.mf.aggregate_labels.size(), 1); + BOOST_CHECK_EQUAL(md.mf.aggregate_labels[0], "lb"); + } else { + BOOST_CHECK_EQUAL(md.mf.aggregate_labels.size(), 0); + } + count++; + } + BOOST_CHECK_EQUAL(count, 2); + app_metrics.add_group("test", { + sm::make_gauge("gauge1_1", sm::description("gague 1"), [] { return 1; })(lb("1")), + sm::make_gauge("gauge1_1", sm::description("gague 1"), [] { return 2; })(lb("2")), + sm::make_counter("counter1_1", sm::description("counter 1"), [] { return 3; })(lb("1")), + sm::make_counter("counter1_1", sm::description("counter 1"), [] { return 4; })(lb("2")) + }); + values = seastar::metrics::impl::get_values(); + count = 0; + for (auto&& md : (*values->metadata)) { + if (md.mf.name == "test_gauge_1") { + BOOST_CHECK_EQUAL(md.mf.aggregate_labels.size(), 1); + BOOST_CHECK_EQUAL(md.mf.aggregate_labels[0], "lb"); + } else if (md.mf.name == "test_gauge1_1") { + BOOST_CHECK_EQUAL(md.mf.aggregate_labels.size(), 2); + BOOST_CHECK_EQUAL(md.mf.aggregate_labels[0], "ll"); + } else { + BOOST_CHECK_EQUAL(md.mf.aggregate_labels.size(), 0); + } + count++; + } + BOOST_CHECK_EQUAL(count, 4); + std::vector rl1; + sm::set_relabel_configs(rl1).get(); +} + SEASTAR_THREAD_TEST_CASE(test_relabel_drop_label_prevent_runtime_conflicts) { using namespace seastar::metrics; namespace sm = seastar::metrics;