Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add metrics aggregation capabilities #26

Merged
merged 4 commits into from
Jun 27, 2022
Merged

Add metrics aggregation capabilities #26

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ccdd278
metrics.hh: Support SUMMARY type
amnonh Jun 23, 2022
893e2de
metrics.cc: missing count and sum when aggregating histograms
amnonh May 27, 2022
9c84d52
metrics: support aggregate by labels and skip_when_empty
amnonh Jun 23, 2022
3f35820
prometheus.cc: Optimize reporting
amnonh Nov 4, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 29 additions & 2 deletions include/seastar/core/metrics.hh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,7 @@ enum class data_type : uint8_t {
REAL_COUNTER,
GAUGE,
HISTOGRAM,
SUMMARY,
};

template <bool callable, typename T>
Expand Down Expand Up @@ -340,7 +341,16 @@ public:
const histogram& get_histogram() const {
return std::get<histogram>(u);
}

/*!
* \brief return true if this is metrics was never used
*
* Histograms, Summaries and counters are ever growing by nature, so
* it possible to check if they have been used or not.
*/
bool is_empty() const {
return ((_type == data_type::HISTOGRAM || _type == data_type::SUMMARY) && get_histogram().sample_count == 0) ||
((_type == data_type::COUNTER || _type == data_type::REAL_COUNTER) && d() == 0);
}
private:
static void ulong_conversion_error(double d);
};
Expand All @@ -358,16 +368,21 @@ struct metric_definition_impl {
metric_function f;
description d;
bool enabled = true;
bool _skip_when_empty = false;
std::vector<std::string> aggregate_labels;
VladLazar marked this conversation as resolved.
Show resolved Hide resolved
std::map<sstring, sstring> labels;
metric_definition_impl& operator ()(bool enabled);
metric_definition_impl& operator ()(const label_instance& label);
metric_definition_impl& aggregate(const std::vector<label>& labels);
metric_definition_impl& skip_when_empty(bool skip=true);
metric_definition_impl& set_type(const sstring& type_name);
metric_definition_impl(
metric_name_type name,
metric_type type,
metric_function f,
description d,
std::vector<label_instance> labels);
std::vector<label_instance> labels,
std::vector<label> aggregate_labels = {});
VladLazar marked this conversation as resolved.
Show resolved Hide resolved
};

class metric_groups_def {
Expand Down Expand Up @@ -595,6 +610,18 @@ impl::metric_definition_impl make_histogram(metric_name_type name,
return {name, {impl::data_type::HISTOGRAM, "histogram"}, make_function(std::forward<T>(val), impl::data_type::HISTOGRAM), d, {}};
}

/*!
* \brief create a summary metric.
*
* Summaries are a different kind of histograms. It reports in quantiles.
* For example, the p99 and p95 latencies.
*/
template<typename T>
impl::metric_definition_impl make_summary(metric_name_type name,
description d, T&& val) {
return {name, {impl::data_type::SUMMARY, "summary"}, make_function(std::forward<T>(val), impl::data_type::SUMMARY), d, {}};
}


/*!
* \brief create a total_bytes metric.
Expand Down
10 changes: 7 additions & 3 deletions include/seastar/core/metrics_api.hh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,7 @@ struct metric_family_info {
metric_type_def inherit_type;
description d;
sstring name;
std::vector<std::string> aggregate_labels;
};


Expand All @@ -161,6 +162,7 @@ struct metric_family_info {
struct metric_info {
metric_id id;
bool enabled;
bool skip_when_empty;
};


Expand All @@ -185,7 +187,7 @@ class registered_metric {
metric_function _f;
shared_ptr<impl> _impl;
public:
registered_metric(metric_id id, metric_function f, bool enabled=true);
registered_metric(metric_id id, metric_function f, bool enabled=true, bool skip_when_empty=false);
virtual ~registered_metric() {}
virtual metric_value operator()() const {
return _f();
Expand All @@ -198,7 +200,9 @@ public:
void set_enabled(bool b) {
_info.enabled = b;
}

void set_skip_when_empty(bool skip) {
_info.skip_when_empty = skip;
}
const metric_id& get_id() const {
return _info.id;
}
Expand Down Expand Up @@ -330,7 +334,7 @@ public:
return _value_map;
}

void add_registration(const metric_id& id, const metric_type& type, metric_function f, const description& d, bool enabled);
void add_registration(const metric_id& id, const metric_type& type, metric_function f, const description& d, bool enabled, bool skip_when_empty, const std::vector<std::string>& aggregate_labels);
void remove_registration(const metric_id& id);
future<> stop() {
return make_ready_future<>();
Expand Down
28 changes: 23 additions & 5 deletions src/core/metrics.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,9 +118,10 @@ bool label_instance::operator!=(const label_instance& id2) const {
label shard_label("shard");
namespace impl {

registered_metric::registered_metric(metric_id id, metric_function f, bool enabled) :
registered_metric::registered_metric(metric_id id, metric_function f, bool enabled, bool skip_when_empty) :
_f(f), _impl(get_local_impl()) {
_info.enabled = enabled;
set_skip_when_empty(skip_when_empty);
_info.id = id;
}

Expand All @@ -146,7 +147,8 @@ metric_definition_impl::metric_definition_impl(
metric_type type,
metric_function f,
description d,
std::vector<label_instance> _labels)
std::vector<label_instance> _labels,
std::vector<label> _aggregate_labels)
: name(name), type(type), f(f)
, d(d), enabled(true) {
for (auto i: _labels) {
Expand All @@ -155,6 +157,7 @@ metric_definition_impl::metric_definition_impl(
if (labels.find(shard_label.name()) == labels.end()) {
labels[shard_label.name()] = shard();
}
aggregate(_aggregate_labels);
}

metric_definition_impl& metric_definition_impl::operator ()(bool _enabled) {
Expand All @@ -172,6 +175,18 @@ metric_definition_impl& metric_definition_impl::set_type(const sstring& type_nam
return *this;
}

metric_definition_impl& metric_definition_impl::aggregate(const std::vector<label>& _labels) {
aggregate_labels.reserve(_labels.size());
std::transform(_labels.begin(), _labels.end(),std::back_inserter(aggregate_labels),
[](const label& l) { return l.name(); });
return *this;
}

metric_definition_impl& metric_definition_impl::skip_when_empty(bool skip) {
_skip_when_empty = skip;
return *this;
}

std::unique_ptr<metric_groups_def> create_metric_groups() {
return std::make_unique<metric_groups_impl>();
}
Expand All @@ -186,7 +201,7 @@ metric_groups_impl& metric_groups_impl::add_metric(group_name_type name, const m

metric_id id(name, md._impl->name, md._impl->labels);

get_local_impl()->add_registration(id, md._impl->type, md._impl->f, md._impl->d, md._impl->enabled);
get_local_impl()->add_registration(id, md._impl->type, md._impl->f, md._impl->d, md._impl->enabled, md._impl->_skip_when_empty, md._impl->aggregate_labels);

_registration.push_back(id);
return *this;
Expand Down Expand Up @@ -327,8 +342,8 @@ std::vector<std::vector<metric_function>>& impl::functions() {
return _current_metrics;
}

void impl::add_registration(const metric_id& id, const metric_type& type, metric_function f, const description& d, bool enabled) {
auto rm = ::seastar::make_shared<registered_metric>(id, f, enabled);
void impl::add_registration(const metric_id& id, const metric_type& type, metric_function f, const description& d, bool enabled, bool skip_when_empty, const std::vector<std::string>& aggregate_labels) {
auto rm = ::seastar::make_shared<registered_metric>(id, f, enabled, skip_when_empty);
sstring name = id.full_name();
if (_value_map.find(name) != _value_map.end()) {
auto& metric = _value_map[name];
Expand All @@ -344,6 +359,7 @@ void impl::add_registration(const metric_id& id, const metric_type& type, metric
_value_map[name].info().d = d;
_value_map[name].info().inherit_type = type.type_name;
_value_map[name].info().name = id.full_name();
_value_map[name].info().aggregate_labels = aggregate_labels;
_value_map[name][id.labels()] = rm;
}
dirty();
Expand All @@ -365,6 +381,8 @@ histogram& histogram::operator+=(const histogram& c) {
buckets[i].count += c.buckets[i].count;
}
}
sample_count += c.sample_count;
sample_sum += c.sample_sum;
return *this;
}

Expand Down
Loading