Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix(userspace/falco): properly format numeric values in metrics #2569

Merged
merged 4 commits into from May 23, 2023
Merged

fix(userspace/falco): properly format numeric values in metrics #2569

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
cc1e299
fix(userspace/falco): properly format numeric values in metrics
jasondellaluce May 23, 2023
80e062c
fix(userspace/faclco): output drop perc metric only if drops are present
jasondellaluce May 23, 2023
be47d4b
fix(userspace/falco): solve escape issues in grpc output
jasondellaluce May 23, 2023
632eba2
fix(userspace/falco): right boundary checks for strncat
jasondellaluce May 23, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 2 additions & 2 deletions userspace/falco/app/actions/create_signal_handlers.cpp
View file
Open in desktop
Expand Up @@ -131,9 +131,9 @@ falco::app::run_result falco::app::actions::create_signal_handlers(falco::app::s
{
std::string rule = "Falco internal: hot restart failure";
std::string msg = rule + ": " + err;
std::map<std::string, std::string> o = {};
auto fields = nlohmann::json::object();
auto now = std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
s.outputs->handle_msg(now, falco_common::PRIORITY_CRITICAL, msg, rule, o);
s.outputs->handle_msg(now, falco_common::PRIORITY_CRITICAL, msg, rule, fields);
}

return success;
Expand Down
7 changes: 3 additions & 4 deletions userspace/falco/app/actions/process_events.cpp
View file
Open in desktop
Expand Up @@ -228,11 +228,10 @@ static falco::app::run_result do_inspect(
{
sinsp_utils::ts_to_string(duration_start, &last_event_time_str, false, true);
}
std::map<std::string, std::string> o = {
{"last_event_time", last_event_time_str},
};
nlohmann::json fields;
fields["last_event_time"] = last_event_time_str;
auto now = std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
s.outputs->handle_msg(now, falco_common::PRIORITY_DEBUG, msg, rule, o);
s.outputs->handle_msg(now, falco_common::PRIORITY_DEBUG, msg, rule, fields);
// Reset the timeouts counter, Falco alerted
timeouts_since_last_success_or_msg = 0;
}
Expand Down
2 changes: 1 addition & 1 deletion userspace/falco/event_drops.cpp
View file
Open in desktop
Expand Up @@ -156,7 +156,7 @@ bool syscall_evt_drop_mgr::perform_actions(uint64_t now, scap_stats &delta, bool

case syscall_evt_drop_action::ALERT:
{
std::map<std::string, std::string> output_fields;
nlohmann::json output_fields;
output_fields["n_evts"] = std::to_string(delta.n_evts); /* Total number of kernel side events actively traced (not including events discarded due to simple consumer mode in eBPF case). */
output_fields["n_drops"] = std::to_string(delta.n_drops); /* Number of all kernel side event drops out of n_evts. */
output_fields["n_drops_buffer_total"] = std::to_string(delta.n_drops_buffer); /* Total number of kernel side drops due to full buffer, includes all categories below, likely higher than sum of syscall categories. */
Expand Down
15 changes: 12 additions & 3 deletions userspace/falco/falco_outputs.cpp
View file
Open in desktop
Expand Up @@ -161,8 +161,13 @@ void falco_outputs::handle_msg(uint64_t ts,
falco_common::priority_type priority,
std::string &msg,
std::string &rule,
std::map<std::string, std::string> &output_fields)
nlohmann::json &output_fields)
{
if (!output_fields.is_object())
{
throw falco_exception("falco_outputs: output fields must be key-value maps");
}

falco_outputs::ctrl_msg cmsg = {};
cmsg.ts = ts;
cmsg.priority = priority;
Expand Down Expand Up @@ -202,7 +207,7 @@ void falco_outputs::handle_msg(uint64_t ts,

sinsp_utils::ts_to_string(ts, &timestr, false, true);
cmsg.msg = timestr + ": " + falco_common::format_priority(priority) + " " + msg + " (";
for(auto &pair : output_fields)
for(auto &pair : output_fields.items())
{
if(first)
{
Expand All @@ -212,7 +217,11 @@ void falco_outputs::handle_msg(uint64_t ts,
{
cmsg.msg += " ";
}
cmsg.msg += pair.first + "=" + pair.second;
if (!pair.value().is_primitive())
{
throw falco_exception("falco_outputs: output fields must be key-value maps");
}
cmsg.msg += pair.key() + "=" + pair.value().dump();
}
cmsg.msg += ")";
}
Expand Down
2 changes: 1 addition & 1 deletion userspace/falco/falco_outputs.h
View file
Open in desktop
Expand Up @@ -66,7 +66,7 @@ class falco_outputs
falco_common::priority_type priority,
std::string &msg,
std::string &rule,
std::map<std::string, std::string> &output_fields);
nlohmann::json &output_fields);

/*!
\brief Sends a cleanup message to all outputs.
Expand Down
3 changes: 2 additions & 1 deletion userspace/falco/outputs.h
View file
Open in desktop
Expand Up @@ -21,6 +21,7 @@ limitations under the License.

#include "falco_common.h"
#include "gen_filter.h"
#include <nlohmann/json.hpp>

namespace falco
{
Expand Down Expand Up @@ -49,7 +50,7 @@ struct message
std::string msg;
std::string rule;
std::string source;
std::map<std::string, std::string> fields;
nlohmann::json fields;
std::set<std::string> tags;
};

Expand Down
10 changes: 8 additions & 2 deletions userspace/falco/outputs_grpc.cpp
View file
Open in desktop
Expand Up @@ -79,9 +79,15 @@ void falco::outputs::output_grpc::output(const message *msg)

// output fields
auto &fields = *grpc_res.mutable_output_fields();
for(const auto &kv : msg->fields)
for(const auto &kv : msg->fields.items())
{
fields[kv.first] = kv.second;
if (!kv.value().is_primitive())
{
throw falco_exception("output_grpc: output fields must be key-value maps");
}
fields[kv.key()] = (kv.value().is_string())
? kv.value().get<std::string>()
: kv.value().dump();
}

// hostname
Expand Down
69 changes: 34 additions & 35 deletions userspace/falco/stats_writer.cpp
View file
Open in desktop
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@jasondellaluce
please consider fixing #2333 (comment) here :)

Expand Up @@ -162,8 +162,7 @@ void stats_writer::worker() noexcept
{
std::string rule = "Falco internal: metrics snapshot";
std::string msg = "Falco metrics snapshot";
std::map<std::string,std::string> fields = {m.output_fields.begin(), m.output_fields.end()};
m_outputs->handle_msg(m.ts, falco_common::PRIORITY_INFORMATIONAL, msg, rule, fields);
m_outputs->handle_msg(m.ts, falco_common::PRIORITY_INFORMATIONAL, msg, rule, m.output_fields);
}

if (use_file)
Expand All @@ -188,7 +187,7 @@ stats_writer::collector::collector(const std::shared_ptr<stats_writer>& writer)
}

void stats_writer::collector::get_metrics_output_fields_wrapper(
std::unordered_map<std::string, std::string>& output_fields,
nlohmann::json& output_fields,
const std::shared_ptr<sinsp>& inspector, uint64_t now,
const std::string& src, uint64_t num_evts, double stats_snapshot_time_delta_sec)
{
Expand All @@ -199,14 +198,14 @@ void stats_writer::collector::get_metrics_output_fields_wrapper(
const scap_machine_info* machine_info = inspector->get_machine_info();

/* Wrapper fields useful for statistical analyses and attributions. Always enabled. */
output_fields["evt.time"] = std::to_string(now); /* Some ETLs may prefer a consistent timestamp within output_fields. */
output_fields["evt.time"] = now; /* Some ETLs may prefer a consistent timestamp within output_fields. */
output_fields["falco.version"] = FALCO_VERSION;
output_fields["falco.start_ts"] = std::to_string(agent_info->start_ts_epoch);
output_fields["falco.duration_sec"] = std::to_string((now - agent_info->start_ts_epoch) / ONE_SECOND_IN_NS);
output_fields["falco.start_ts"] = agent_info->start_ts_epoch;
output_fields["falco.duration_sec"] = (uint64_t)((now - agent_info->start_ts_epoch) / ONE_SECOND_IN_NS);
output_fields["falco.kernel_release"] = agent_info->uname_r;
output_fields["falco.host_boot_ts"] = std::to_string(machine_info->boot_ts_epoch);
output_fields["falco.host_boot_ts"] = machine_info->boot_ts_epoch;
output_fields["falco.hostname"] = machine_info->hostname; /* Explicitly add hostname to log msg in case hostname rule output field is disabled. */
output_fields["falco.host_num_cpus"] = std::to_string(machine_info->num_cpus);
output_fields["falco.host_num_cpus"] = machine_info->num_cpus;

output_fields["evt.source"] = src;
for (size_t i = 0; i < sizeof(all_driver_engines) / sizeof(const char*); i++)
Expand All @@ -222,15 +221,15 @@ void stats_writer::collector::get_metrics_output_fields_wrapper(
if (m_last_num_evts != 0 && stats_snapshot_time_delta_sec > 0)
{
/* Successfully processed userspace event rate. */
output_fields["falco.evts_rate_sec"] = std::to_string((num_evts - m_last_num_evts) / (double)stats_snapshot_time_delta_sec);
output_fields["falco.evts_rate_sec"] = (double)((num_evts - m_last_num_evts) / (double)stats_snapshot_time_delta_sec);
}
output_fields["falco.num_evts"] = std::to_string(num_evts);
output_fields["falco.num_evts_prev"] = std::to_string(m_last_num_evts);
output_fields["falco.num_evts"] = num_evts;
output_fields["falco.num_evts_prev"] = m_last_num_evts;
m_last_num_evts = num_evts;
}

void stats_writer::collector::get_metrics_output_fields_additional(
std::unordered_map<std::string, std::string>& output_fields,
nlohmann::json& output_fields,
const std::shared_ptr<sinsp>& inspector,
double stats_snapshot_time_delta_sec, const std::string& src)
{
Expand All @@ -251,32 +250,32 @@ void stats_writer::collector::get_metrics_output_fields_additional(
for(uint32_t stat = 0; stat < nstats; stat++)
{
char metric_name[STATS_NAME_MAX] = "falco.";
strncat(metric_name, utilization[stat].name, sizeof(metric_name) - strlen(metric_name));
strncat(metric_name, utilization[stat].name, sizeof(metric_name) - strlen(metric_name) - 1);
switch(utilization[stat].type)
{
case STATS_VALUE_TYPE_U64:

if (m_writer->m_config->m_metrics_convert_memory_to_mb && strncmp(utilization[stat].name, "container_memory_used", 21) == 0)
{
output_fields[metric_name] = std::to_string(utilization[stat].value.u64 / (double)1024 / (double)1024);
output_fields[metric_name] = (uint64_t)(utilization[stat].value.u64 / (double)1024 / (double)1024);
}
else
{
output_fields[metric_name] = std::to_string(utilization[stat].value.u64);
output_fields[metric_name] = utilization[stat].value.u64;
}
break;
case STATS_VALUE_TYPE_U32:
if (m_writer->m_config->m_metrics_convert_memory_to_mb && strncmp(utilization[stat].name, "memory_", 7) == 0)
{
output_fields[metric_name] = std::to_string(utilization[stat].value.u32 / (double)1024);
output_fields[metric_name] = (uint32_t)(utilization[stat].value.u32 / (double)1024);
}
else
{
output_fields[metric_name] = std::to_string(utilization[stat].value.u32);
output_fields[metric_name] = utilization[stat].value.u32;
}
break;
case STATS_VALUE_TYPE_D:
output_fields[metric_name] = std::to_string(utilization[stat].value.d);
output_fields[metric_name] = utilization[stat].value.d;
break;
default:
break;
Expand Down Expand Up @@ -314,7 +313,7 @@ void stats_writer::collector::get_metrics_output_fields_additional(
// todo: as we expand scap_stats_v2 prefix may be pushed to scap or we may need to expand
// functionality here for example if we add userspace syscall counters that should be prefixed w/ `falco.`
char metric_name[STATS_NAME_MAX] = "scap.";
strncat(metric_name, stats_v2[stat].name, sizeof(metric_name) - strlen(metric_name));
strncat(metric_name, stats_v2[stat].name, sizeof(metric_name) - strlen(metric_name) - 1);
switch(stats_v2[stat].type)
{
case STATS_VALUE_TYPE_U64:
Expand All @@ -324,42 +323,42 @@ void stats_writer::collector::get_metrics_output_fields_additional(
if (m_last_n_evts != 0 && stats_snapshot_time_delta_sec > 0)
{
/* n_evts is total number of kernel side events. */
output_fields["scap.evts_rate_sec"] = std::to_string((n_evts - m_last_n_evts) / stats_snapshot_time_delta_sec);
output_fields["scap.evts_rate_sec"] = (double)((n_evts - m_last_n_evts) / stats_snapshot_time_delta_sec);
}
else
{
output_fields["scap.evts_rate_sec"] = std::to_string(0);
output_fields["scap.evts_rate_sec"] = (double)(0);
}
output_fields["scap.n_evts_prev"] = std::to_string(m_last_n_evts);
output_fields["scap.n_evts_prev"] = m_last_n_evts;
}
else if (strncmp(stats_v2[stat].name, "n_drops", 7) == 0)
{
n_drops = stats_v2[stat].value.u64;
if (m_last_n_drops != 0 && stats_snapshot_time_delta_sec > 0)
{
/* n_drops is total number of kernel side event drops. */
output_fields["scap.evts_drop_rate_sec"] = std::to_string((n_drops - m_last_n_drops) / stats_snapshot_time_delta_sec);
output_fields["scap.evts_drop_rate_sec"] = (double)((n_drops - m_last_n_drops) / stats_snapshot_time_delta_sec);
}
else
{
output_fields["scap.evts_drop_rate_sec"] = std::to_string(0);
output_fields["scap.evts_drop_rate_sec"] = (double)(0);
}
output_fields["scap.n_drops_prev"] = std::to_string(m_last_n_drops);
if((n_evts - m_last_n_evts) > 0)
{
output_fields["scap.n_drops_perc"] = (double)((100.0 * (n_drops - m_last_n_drops)) / (n_evts - m_last_n_evts));
}
else
{
output_fields["scap.n_drops_perc"] = (double)(0);
}
output_fields["scap.n_drops_prev"] = m_last_n_drops;
}
output_fields[metric_name] = std::to_string(stats_v2[stat].value.u64);
output_fields[metric_name] = stats_v2[stat].value.u64;
break;
default:
break;
}
}
if((n_evts - m_last_n_evts) > 0)
{
output_fields["scap.n_drops_perc"] = std::to_string((100.0 * (n_drops - m_last_n_drops)) / (n_evts - m_last_n_evts));
}
else
{
output_fields["scap.n_drops_perc"] = std::to_string(0);
}
m_last_n_evts = n_evts;
m_last_n_drops = n_drops;
}
Expand All @@ -386,7 +385,7 @@ void stats_writer::collector::collect(const std::shared_ptr<sinsp>& inspector, c
double stats_snapshot_time_delta_sec = (stats_snapshot_time_delta / (double)ONE_SECOND_IN_NS);

/* Get respective metrics output_fields. */
std::unordered_map<std::string, std::string> output_fields;
nlohmann::json output_fields;
get_metrics_output_fields_wrapper(output_fields, inspector, now, src, num_evts, stats_snapshot_time_delta_sec);
get_metrics_output_fields_additional(output_fields, inspector, stats_snapshot_time_delta_sec, src);

Expand Down
6 changes: 3 additions & 3 deletions userspace/falco/stats_writer.h
View file
Open in desktop
Expand Up @@ -64,12 +64,12 @@ class stats_writer
/*!
\brief Collect snapshot metrics wrapper fields as internal rule formatted output fields.
*/
void get_metrics_output_fields_wrapper(std::unordered_map<std::string, std::string>& output_fields, const std::shared_ptr<sinsp>& inspector, uint64_t now, const std::string& src, uint64_t num_evts, double stats_snapshot_time_delta_sec);
void get_metrics_output_fields_wrapper(nlohmann::json& output_fields, const std::shared_ptr<sinsp>& inspector, uint64_t now, const std::string& src, uint64_t num_evts, double stats_snapshot_time_delta_sec);

/*!
\brief Collect snapshot metrics syscalls related metrics as internal rule formatted output fields.
*/
void get_metrics_output_fields_additional(std::unordered_map<std::string, std::string>& output_fields, const std::shared_ptr<sinsp>& inspector, double stats_snapshot_time_delta_sec, const std::string& src);
void get_metrics_output_fields_additional(nlohmann::json& output_fields, const std::shared_ptr<sinsp>& inspector, double stats_snapshot_time_delta_sec, const std::string& src);


std::shared_ptr<stats_writer> m_writer;
Expand Down Expand Up @@ -132,7 +132,7 @@ class stats_writer
bool stop;
uint64_t ts;
std::string source;
std::unordered_map<std::string, std::string> output_fields;
nlohmann::json output_fields;
};

void worker() noexcept;
Expand Down