From 261d47c6d7aa77dc62c4fb1ec06d6b41bc64326f Mon Sep 17 00:00:00 2001 From: Jason Dellaluce Date: Thu, 31 Aug 2023 13:52:29 +0000 Subject: [PATCH 1/9] update(userspace/engine): refactor rule describe methods to accept plugins Signed-off-by: Jason Dellaluce --- userspace/engine/falco_engine.cpp | 98 +++++++++++-------- userspace/engine/falco_engine.h | 38 ++++--- .../falco/app/actions/load_rules_files.cpp | 6 +- 3 files changed, 86 insertions(+), 56 deletions(-) diff --git a/userspace/engine/falco_engine.cpp b/userspace/engine/falco_engine.cpp index 505237aa63d..e218d1c3cbb 100644 --- a/userspace/engine/falco_engine.cpp +++ b/userspace/engine/falco_engine.cpp @@ -469,7 +469,7 @@ std::size_t falco_engine::add_source(const std::string &source, return m_sources.insert(src, source); } -void falco_engine::describe_rule(std::string *rule, bool json) const +void falco_engine::describe_rule(std::string *rule, const std::vector>& plugins, bool json) const { if(!json) { @@ -498,7 +498,6 @@ void falco_engine::describe_rule(std::string *rule, bool json) const return; } - std::unique_ptr insp(new sinsp()); Json::FastWriter writer; std::string json_str; @@ -541,7 +540,7 @@ void falco_engine::describe_rule(std::string *rule, bool json) const { auto ri = m_rule_collector.rules().at(r.name); Json::Value rule; - get_json_details(r, *ri, insp.get(), rule); + get_json_details(rule, r, *ri, plugins); // Append to rule array rules_array.append(rule); @@ -553,7 +552,7 @@ void falco_engine::describe_rule(std::string *rule, bool json) const for(const auto &m : m_rule_collector.macros()) { Json::Value macro; - get_json_details(m, macro); + get_json_details(macro, m, plugins); macros_array.append(macro); } output["macros"] = macros_array; @@ -563,7 +562,7 @@ void falco_engine::describe_rule(std::string *rule, bool json) const for(const auto &l : m_rule_collector.lists()) { Json::Value list; - get_json_details(l, list); + get_json_details(list, l); lists_array.append(list); } output["lists"] = lists_array; @@ -580,17 +579,18 @@ void falco_engine::describe_rule(std::string *rule, bool json) const } auto r = m_rules.at(ri->name); Json::Value rule; - get_json_details(*r, *ri, insp.get(), rule); + get_json_details(rule, *r, *ri, plugins); json_str = writer.write(rule); } fprintf(stdout, "%s", json_str.c_str()); } -void falco_engine::get_json_details(const falco_rule &r, +void falco_engine::get_json_details( + Json::Value &out, + const falco_rule &r, const rule_loader::rule_info &ri, - sinsp *insp, - Json::Value &rule) const + const std::vector>& plugins) const { Json::Value rule_info; @@ -608,25 +608,26 @@ void falco_engine::get_json_details(const falco_rule &r, tags.append(t); } rule_info["tags"] = tags; - rule["info"] = rule_info; + out["info"] = rule_info; // Parse rule condition and build the AST // Assumption: no exception because rules have already been loaded. auto ast = libsinsp::filter::parser(ri.cond).parse(); Json::Value json_details; - get_json_details(ast.get(), json_details); - rule["details"] = json_details; + get_json_details(json_details, ast.get()); + out["details"] = json_details; // Get fields from output string auto fmt = create_formatter(r.source, r.output); std::vector out_fields; fmt->get_field_names(out_fields); + // TODO: check these too for plugins Json::Value outputFields = Json::arrayValue; for(const auto &of : out_fields) { outputFields.append(of); } - rule["details"]["output_fields"] = outputFields; + out["details"]["output_fields"] = outputFields; // Get fields from exceptions Json::Value exception_fields = Json::arrayValue; @@ -634,7 +635,7 @@ void falco_engine::get_json_details(const falco_rule &r, { exception_fields.append(f); } - rule["details"]["exception_fields"] = exception_fields; + out["details"]["exception_fields"] = exception_fields; // Get names and operators from exceptions Json::Value exception_names = Json::arrayValue; @@ -665,42 +666,45 @@ void falco_engine::get_json_details(const falco_rule &r, exception_operators.append(e.comps.item); } } - rule["details"]["exceptions"] = exception_names; - rule["details"]["exception_operators"] = exception_operators; + out["details"]["exceptions"] = exception_names; + out["details"]["exception_operators"] = exception_operators; if(ri.source == falco_common::syscall_source) { // Store event types Json::Value events; - get_json_evt_types(ast.get(), events); - rule["details"]["events"] = events; + get_json_evt_types(events, ast.get()); + out["details"]["events"] = events; } } -void falco_engine::get_json_details(const rule_loader::macro_info& m, - Json::Value& macro) const +void falco_engine::get_json_details( + Json::Value& out, + const rule_loader::macro_info& m, + const std::vector>& plugins) const { Json::Value macro_info; macro_info["name"] = m.name; macro_info["condition"] = m.cond; - macro["info"] = macro_info; + out["info"] = macro_info; // Assumption: no exception because rules have already been loaded. auto ast = libsinsp::filter::parser(m.cond).parse(); Json::Value json_details; - get_json_details(ast.get(), json_details); - macro["details"] = json_details; + get_json_details(json_details, ast.get()); + out["details"] = json_details; // Store event types Json::Value events; - get_json_evt_types(ast.get(), events); - macro["details"]["events"] = events; + get_json_evt_types(events, ast.get()); + out["details"]["events"] = events; } -void falco_engine::get_json_details(const rule_loader::list_info& l, - Json::Value& list) const +void falco_engine::get_json_details( + Json::Value& out, + const rule_loader::list_info& l) const { Json::Value list_info; list_info["name"] = l.name; @@ -718,12 +722,14 @@ void falco_engine::get_json_details(const rule_loader::list_info& l, } list_info["items"] = items; - list["info"] = list_info; - list["details"]["lists"] = lists; + out["info"] = list_info; + out["details"]["lists"] = lists; + out["details"]["plugins"] = Json::arrayValue; } -void falco_engine::get_json_details(libsinsp::filter::ast::expr* ast, - Json::Value& output) const +void falco_engine::get_json_details( + Json::Value& out, + libsinsp::filter::ast::expr* ast) const { filter_details details; for(const auto &m : m_rule_collector.macros()) @@ -745,46 +751,56 @@ void falco_engine::get_json_details(libsinsp::filter::ast::expr* ast, { macros.append(m); } - output["macros"] = macros; + out["macros"] = macros; Json::Value operators = Json::arrayValue; for(const auto &o : details.operators) { operators.append(o); } - output["operators"] = operators; + out["operators"] = operators; Json::Value condition_fields = Json::arrayValue; for(const auto &f : details.fields) { condition_fields.append(f); } - output["condition_fields"] = condition_fields; + out["condition_fields"] = condition_fields; Json::Value lists = Json::arrayValue; for(const auto &l : details.lists) { lists.append(l); } - output["lists"] = lists; - - details.reset(); + out["lists"] = lists; } -void falco_engine::get_json_evt_types(libsinsp::filter::ast::expr* ast, - Json::Value& output) const +void falco_engine::get_json_evt_types( + Json::Value& out, + libsinsp::filter::ast::expr* ast) const { - output = Json::arrayValue; + out = Json::arrayValue; auto evtcodes = libsinsp::filter::ast::ppm_event_codes(ast); auto syscodes = libsinsp::filter::ast::ppm_sc_codes(ast); auto syscodes_to_evt_names = libsinsp::events::sc_set_to_event_names(syscodes); auto evtcodes_to_evt_names = libsinsp::events::event_set_to_names(evtcodes, false); for (const auto& n : unordered_set_union(syscodes_to_evt_names, evtcodes_to_evt_names)) { - output.append(n); + out.append(n); } } +void falco_engine::get_json_used_plugins( + Json::Value& out, + const std::string& source, + const std::unordered_set& evttypes, + const std::unordered_set& fields, + const std::vector>& plugins) const +{ + // TODO + out = Json::arrayValue; +} + void falco_engine::print_stats() const { diff --git a/userspace/engine/falco_engine.h b/userspace/engine/falco_engine.h index cb24e3c6c12..e1210410bdf 100644 --- a/userspace/engine/falco_engine.h +++ b/userspace/engine/falco_engine.h @@ -125,7 +125,7 @@ class falco_engine // Print details on the given rule. If rule is NULL, print // details on all rules. // - void describe_rule(std::string *rule, bool json) const; + void describe_rule(std::string *rule, const std::vector>& plugins, bool json) const; // // Print statistics on how many events matched each rule. @@ -303,18 +303,30 @@ class falco_engine inline bool should_drop_evt() const; // Retrieve json details from rules, macros, lists - void get_json_details(const falco_rule& r, - const rule_loader::rule_info& ri, - sinsp* insp, - Json::Value& rule) const; - void get_json_details(const rule_loader::macro_info& m, - Json::Value& macro) const; - void get_json_details(const rule_loader::list_info& l, - Json::Value& list) const; - void get_json_details(libsinsp::filter::ast::expr* ast, - Json::Value& output) const; - void get_json_evt_types(libsinsp::filter::ast::expr* ast, - Json::Value& output) const; + void get_json_details( + Json::Value& out, + const falco_rule& r, + const rule_loader::rule_info& ri, + const std::vector>& plugins) const; + void get_json_details( + Json::Value& out, + const rule_loader::macro_info& m, + const std::vector>& plugins) const; + void get_json_details( + Json::Value& out, + const rule_loader::list_info& l) const; + void get_json_details( + Json::Value& out, + libsinsp::filter::ast::expr* ast) const; + void get_json_evt_types( + Json::Value& out, + libsinsp::filter::ast::expr* ast) const; + void get_json_used_plugins( + Json::Value& out, + const std::string& source, + const std::unordered_set& evttypes, + const std::unordered_set& fields, + const std::vector>& plugins) const; rule_loader::collector m_rule_collector; indexed_vector m_rules; diff --git a/userspace/falco/app/actions/load_rules_files.cpp b/userspace/falco/app/actions/load_rules_files.cpp index 592b018471b..5b4d8de1b47 100644 --- a/userspace/falco/app/actions/load_rules_files.cpp +++ b/userspace/falco/app/actions/load_rules_files.cpp @@ -157,13 +157,15 @@ falco::app::run_result falco::app::actions::load_rules_files(falco::app::state& if (s.options.describe_all_rules) { - s.engine->describe_rule(NULL, s.config->m_json_output); + const auto& plugins = s.offline_inspector->get_plugin_manager()->plugins(); + s.engine->describe_rule(NULL, plugins, s.config->m_json_output); return run_result::exit(); } if (!s.options.describe_rule.empty()) { - s.engine->describe_rule(&(s.options.describe_rule), s.config->m_json_output); + const auto& plugins = s.offline_inspector->get_plugin_manager()->plugins(); + s.engine->describe_rule(&(s.options.describe_rule), plugins, s.config->m_json_output); return run_result::exit(); } From 9ae00d983c777a3579351d671d7d1c92d4607a29 Mon Sep 17 00:00:00 2001 From: Jason Dellaluce Date: Thu, 31 Aug 2023 13:58:31 +0000 Subject: [PATCH 2/9] update(userspace/engine): find evt names in filter resolver Signed-off-by: Jason Dellaluce --- userspace/engine/filter_details_resolver.cpp | 30 ++++++++++++++++++-- userspace/engine/filter_details_resolver.h | 2 ++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/userspace/engine/filter_details_resolver.cpp b/userspace/engine/filter_details_resolver.cpp index 7130ab11d53..7ab1d8e3971 100644 --- a/userspace/engine/filter_details_resolver.cpp +++ b/userspace/engine/filter_details_resolver.cpp @@ -25,6 +25,7 @@ void filter_details::reset() macros.clear(); operators.clear(); lists.clear(); + evtnames.clear(); } void filter_details_resolver::run(ast::expr* filter, filter_details& details) @@ -70,6 +71,16 @@ void filter_details_resolver::visitor::visit(ast::list_expr* e) } } } + if (m_expect_evtname) + { + for(const auto& item : e->values) + { + if(m_details.known_lists.find(item) == m_details.known_lists.end()) + { + m_details.evtnames.insert(item); + } + } + } } void filter_details_resolver::visitor::visit(ast::binary_check_expr* e) @@ -77,9 +88,18 @@ void filter_details_resolver::visitor::visit(ast::binary_check_expr* e) m_expect_macro = false; m_details.fields.insert(e->field); m_details.operators.insert(e->op); - m_expect_list = true; - e->value->accept(this); - m_expect_list = false; + if (e->field == "evt.type" || e->field == "evt.asynctype") + { + m_expect_evtname = true; + e->value->accept(this); + m_expect_evtname = false; + } + else + { + m_expect_list = true; + e->value->accept(this); + m_expect_list = false; + } } void filter_details_resolver::visitor::visit(ast::unary_check_expr* e) @@ -101,4 +121,8 @@ void filter_details_resolver::visitor::visit(ast::value_expr* e) m_details.macros.insert(e->value); } + if(m_expect_evtname) + { + m_details.evtnames.insert(e->value); + } } diff --git a/userspace/engine/filter_details_resolver.h b/userspace/engine/filter_details_resolver.h index bbf2534a789..df0e66ec70a 100644 --- a/userspace/engine/filter_details_resolver.h +++ b/userspace/engine/filter_details_resolver.h @@ -33,6 +33,7 @@ struct filter_details std::unordered_set macros; std::unordered_set operators; std::unordered_set lists; + std::unordered_set evtnames; void reset(); }; @@ -76,5 +77,6 @@ class filter_details_resolver filter_details& m_details; bool m_expect_list; bool m_expect_macro; + bool m_expect_evtname; }; }; From 3c2e5b9acca26196e55fd8aa251ff62b73c206b8 Mon Sep 17 00:00:00 2001 From: Jason Dellaluce Date: Tue, 5 Sep 2023 11:11:52 +0000 Subject: [PATCH 3/9] update(userspace/engine): support printing plugins used by rules Signed-off-by: Jason Dellaluce --- userspace/engine/falco_engine.cpp | 163 ++++++++++++++++++++++-------- userspace/engine/falco_engine.h | 10 +- 2 files changed, 124 insertions(+), 49 deletions(-) diff --git a/userspace/engine/falco_engine.cpp b/userspace/engine/falco_engine.cpp index e218d1c3cbb..a59bb20152c 100644 --- a/userspace/engine/falco_engine.cpp +++ b/userspace/engine/falco_engine.cpp @@ -46,7 +46,6 @@ limitations under the License. #include "utils.h" #include "banned.h" // This raises a compilation error when certain functions are used #include "evttype_index_ruleset.h" -#include "filter_details_resolver.h" const std::string falco_engine::s_default_ruleset = "falco-default-ruleset"; @@ -548,11 +547,11 @@ void falco_engine::describe_rule(std::string *rule, const std::vector out_fields; fmt->get_field_names(out_fields); - // TODO: check these too for plugins Json::Value outputFields = Json::arrayValue; for(const auto &of : out_fields) { @@ -669,19 +679,27 @@ void falco_engine::get_json_details( out["details"]["exceptions"] = exception_names; out["details"]["exception_operators"] = exception_operators; - if(ri.source == falco_common::syscall_source) - { - // Store event types - Json::Value events; - get_json_evt_types(events, ast.get()); - out["details"]["events"] = events; - } + // Store event types + Json::Value events; + get_json_evt_types(events, ri.source, details); + out["details"]["events"] = events; + + // Compute the plugins that are actually used by this rule. This is involves: + // - The rule's event source, that can be implemented by a plugin + // - The fields used in the rule's condition, output, and exceptions + // - The evt types used in the rule's condition checks, that can potentially + // match plugin-provided async events + Json::Value used_plugins; + // note: making a union of conditions's and output's fields + // note: the condition's AST should include all resolved refs and exceptions + details.fields.insert(out_fields.begin(), out_fields.end()); + get_json_used_plugins(used_plugins, ri.source, details.evtnames, details.fields, plugins); + out["details"]["plugins"] = used_plugins; } void falco_engine::get_json_details( Json::Value& out, - const rule_loader::macro_info& m, - const std::vector>& plugins) const + const rule_loader::macro_info& m) const { Json::Value macro_info; @@ -689,16 +707,28 @@ void falco_engine::get_json_details( macro_info["condition"] = m.cond; out["info"] = macro_info; + // Parse rule condition and build the AST // Assumption: no exception because rules have already been loaded. auto ast = libsinsp::filter::parser(m.cond).parse(); + // get details related to the condition's filter + filter_details details; Json::Value json_details; - get_json_details(json_details, ast.get()); + for(const auto &m : m_rule_collector.macros()) + { + details.known_macros.insert(m.name); + } + for(const auto &l : m_rule_collector.lists()) + { + details.known_lists.insert(l.name); + } + filter_details_resolver().run(ast.get(), details); + get_json_filter_details(json_details, details); out["details"] = json_details; // Store event types Json::Value events; - get_json_evt_types(events, ast.get()); + get_json_evt_types(events, "", details); out["details"]["events"] = events; } @@ -724,28 +754,12 @@ void falco_engine::get_json_details( list_info["items"] = items; out["info"] = list_info; out["details"]["lists"] = lists; - out["details"]["plugins"] = Json::arrayValue; } -void falco_engine::get_json_details( +void falco_engine::get_json_filter_details( Json::Value& out, - libsinsp::filter::ast::expr* ast) const + const filter_details& details) const { - filter_details details; - for(const auto &m : m_rule_collector.macros()) - { - details.known_macros.insert(m.name); - } - - for(const auto &l : m_rule_collector.lists()) - { - details.known_lists.insert(l.name); - } - - // Resolve the AST details - filter_details_resolver resolver; - resolver.run(ast, details); - Json::Value macros = Json::arrayValue; for(const auto &m : details.macros) { @@ -777,31 +791,92 @@ void falco_engine::get_json_details( void falco_engine::get_json_evt_types( Json::Value& out, - libsinsp::filter::ast::expr* ast) const + const std::string& source, + const filter_details& details) const { out = Json::arrayValue; - auto evtcodes = libsinsp::filter::ast::ppm_event_codes(ast); - auto syscodes = libsinsp::filter::ast::ppm_sc_codes(ast); - auto syscodes_to_evt_names = libsinsp::events::sc_set_to_event_names(syscodes); - auto evtcodes_to_evt_names = libsinsp::events::event_set_to_names(evtcodes, false); - for (const auto& n : unordered_set_union(syscodes_to_evt_names, evtcodes_to_evt_names)) + for (const auto& n : details.evtnames) { out.append(n); } + + // plugin-implemented sources always match pluginevent. + // note: macros have no source and can potentially be referenced in a rule + // used for a plugin-implemented source. + if(source.empty() || source != falco_common::syscall_source) + { + for (const auto& n : libsinsp::events::event_set_to_names({PPME_PLUGINEVENT_E}, false)) + { + out.append(n); + } + } } void falco_engine::get_json_used_plugins( Json::Value& out, const std::string& source, - const std::unordered_set& evttypes, + const std::unordered_set& evtnames, const std::unordered_set& fields, const std::vector>& plugins) const { - // TODO - out = Json::arrayValue; + out = Json::arrayValue; + for (const auto& p : plugins) + { + bool used = false; + if (p->caps() & CAP_SOURCING) + { + // The rule's source is implemented by a plugin with event + // sourcing capability. + // Note: if Falco loads two plugins implementing the same source, + // they will both be included in the list. + if (!used && p->event_source() == source) + { + out.append(p->name()); + used = true; + } + } + if (!used && p->caps() & CAP_EXTRACTION) + { + // The rule uses a field implemented by a plugin with field + // extraction capability that is compatible with the rule's source. + // Note: here we're assuming that Falco will prevent loading + // plugins implementing fields with the same name for the same + // event source (implemented in init_inspectors app action). + if (sinsp_plugin::is_source_compatible(p->extract_event_sources(), source)) + { + for (const auto &f : p->fields()) + { + if (!used && fields.find(f.m_name) != fields.end()) + { + out.append(p->name()); + used = true; + break; + } + } + } + } + if (!used && p->caps() & CAP_ASYNC) + { + // The rule matches an event type implemented by a plugin with + // async events capability that is compatible with the rule's source. + // Note: if Falco loads two plugins implementing async events with + // the same name, they will both be included in the list. + if (sinsp_plugin::is_source_compatible(p->async_event_sources(), source)) + { + for (const auto &n : p->async_event_names()) + { + if (!used && evtnames.find(n) != evtnames.end()) + { + out.append(p->name()); + used = true; + break; + } + } + } + } + } } - void falco_engine::print_stats() const { std::string out; diff --git a/userspace/engine/falco_engine.h b/userspace/engine/falco_engine.h index e1210410bdf..696a6921036 100644 --- a/userspace/engine/falco_engine.h +++ b/userspace/engine/falco_engine.h @@ -310,17 +310,17 @@ class falco_engine const std::vector>& plugins) const; void get_json_details( Json::Value& out, - const rule_loader::macro_info& m, - const std::vector>& plugins) const; + const rule_loader::macro_info& m) const; void get_json_details( Json::Value& out, const rule_loader::list_info& l) const; - void get_json_details( + void get_json_filter_details( Json::Value& out, - libsinsp::filter::ast::expr* ast) const; + const filter_details& details) const; void get_json_evt_types( Json::Value& out, - libsinsp::filter::ast::expr* ast) const; + const std::string& source, + const filter_details& details) const; void get_json_used_plugins( Json::Value& out, const std::string& source, From 9d36c357fdc912ea667e45d49bd35943721bbeaa Mon Sep 17 00:00:00 2001 From: Jason Dellaluce Date: Tue, 5 Sep 2023 11:26:58 +0000 Subject: [PATCH 4/9] fix(userspace/engine): print rules fields with arguments Signed-off-by: Jason Dellaluce --- userspace/engine/falco_engine.cpp | 15 ++++++++++++++- userspace/engine/filter_details_resolver.cpp | 14 ++++++++++++-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/userspace/engine/falco_engine.cpp b/userspace/engine/falco_engine.cpp index a59bb20152c..98d2ca04dda 100644 --- a/userspace/engine/falco_engine.cpp +++ b/userspace/engine/falco_engine.cpp @@ -819,6 +819,19 @@ void falco_engine::get_json_used_plugins( const std::unordered_set& fields, const std::vector>& plugins) const { + // note: condition and output fields may have an argument, so + // we need to isolate the field names + std::unordered_set fieldnames; + for (auto f: fields) + { + auto argpos = f.find('['); + if (argpos != std::string::npos) + { + f = f.substr(0, argpos); + } + fieldnames.insert(f); + } + out = Json::arrayValue; for (const auto& p : plugins) { @@ -846,7 +859,7 @@ void falco_engine::get_json_used_plugins( { for (const auto &f : p->fields()) { - if (!used && fields.find(f.m_name) != fields.end()) + if (!used && fieldnames.find(f.m_name) != fieldnames.end()) { out.append(p->name()); used = true; diff --git a/userspace/engine/filter_details_resolver.cpp b/userspace/engine/filter_details_resolver.cpp index 7ab1d8e3971..81a9a407935 100644 --- a/userspace/engine/filter_details_resolver.cpp +++ b/userspace/engine/filter_details_resolver.cpp @@ -19,6 +19,16 @@ limitations under the License. using namespace libsinsp::filter; +std::string get_field_name(const std::string& name, const std::string& arg) +{ + std::string fld = name; + if (!arg.empty()) + { + fld += "[" + arg + "]"; + } + return fld; +} + void filter_details::reset() { fields.clear(); @@ -86,7 +96,7 @@ void filter_details_resolver::visitor::visit(ast::list_expr* e) void filter_details_resolver::visitor::visit(ast::binary_check_expr* e) { m_expect_macro = false; - m_details.fields.insert(e->field); + m_details.fields.insert(get_field_name(e->field, e->arg)); m_details.operators.insert(e->op); if (e->field == "evt.type" || e->field == "evt.asynctype") { @@ -105,7 +115,7 @@ void filter_details_resolver::visitor::visit(ast::binary_check_expr* e) void filter_details_resolver::visitor::visit(ast::unary_check_expr* e) { m_expect_macro = false; - m_details.fields.insert(e->field); + m_details.fields.insert(get_field_name(e->field, e->arg)); m_details.operators.insert(e->op); } From 64223f93b30b72bb2f4146ab717193c861f6ea76 Mon Sep 17 00:00:00 2001 From: Jason Dellaluce Date: Tue, 5 Sep 2023 16:01:46 +0000 Subject: [PATCH 5/9] refactor(userspace/engine): modularize rules files compilation Signed-off-by: Jason Dellaluce --- userspace/engine/falco_engine.cpp | 50 +++++++- userspace/engine/falco_rule.h | 42 +++++++ userspace/engine/rule_loader.cpp | 4 +- userspace/engine/rule_loader.h | 13 +- userspace/engine/rule_loader_compiler.cpp | 147 ++++++++++++---------- userspace/engine/rule_loader_compiler.h | 29 ++++- 6 files changed, 194 insertions(+), 91 deletions(-) diff --git a/userspace/engine/falco_engine.cpp b/userspace/engine/falco_engine.cpp index 98d2ca04dda..99c088642c1 100644 --- a/userspace/engine/falco_engine.cpp +++ b/userspace/engine/falco_engine.cpp @@ -190,22 +190,60 @@ void falco_engine::load_rules(const std::string &rules_content, bool verbose, bo std::unique_ptr falco_engine::load_rules(const std::string &rules_content, const std::string &name) { rule_loader::configuration cfg(rules_content, m_sources, name); - cfg.min_priority = m_min_priority; cfg.output_extra = m_extra; cfg.replace_output_container_info = m_replace_container_info; - cfg.default_ruleset_id = m_default_ruleset_id; + // read rules YAML file and collect its definitions rule_loader::reader reader; if (reader.read(cfg, m_rule_collector)) - { + { + // compile the definitions (resolve macro/list refs, exceptions, ...) + rule_loader::compiler::compile_output out; + rule_loader::compiler().compile(cfg, m_rule_collector, out); + + // clear the rules known by the engine and each ruleset + m_rules.clear(); for (auto &src : m_sources) { src.ruleset = src.ruleset_factory->new_ruleset(); } - rule_loader::compiler compiler; - m_rules.clear(); - compiler.compile(cfg, m_rule_collector, m_rules); + // add rules to the engine and the rulesets + for (const auto& rule : out.rules) + { + // skip the rule if below the minimum priority + if (rule.priority > m_min_priority) + { + continue; + } + + auto info = m_rule_collector.rules().at(rule.name); + if (!info) + { + // this is just defensive, it should never happen + throw falco_exception("can't find internal rule info at name: " + name); + } + + // the rule is ok, we can add it to the engine and the rulesets + // note: the compiler should guarantee that the rule's condition + // is a valid sinsp filter + auto source = find_source(rule.source); + std::shared_ptr filter( + sinsp_filter_compiler(source->filter_factory, rule.condition.get()).compile()); + auto rule_id = m_rules.insert(rule, rule.name); + m_rules.at(rule_id)->id = rule_id; + source->ruleset->add(rule, filter, rule.condition); + + // By default rules are enabled/disabled for the default ruleset + if(info->enabled) + { + source->ruleset->enable(rule.name, true, m_default_ruleset_id); + } + else + { + source->ruleset->disable(rule.name, true, m_default_ruleset_id); + } + } } if (cfg.res->successful()) diff --git a/userspace/engine/falco_rule.h b/userspace/engine/falco_rule.h index 482f1357159..d2028bfe9d3 100644 --- a/userspace/engine/falco_rule.h +++ b/userspace/engine/falco_rule.h @@ -21,6 +21,46 @@ limitations under the License. #include #include "falco_common.h" +#include + +/*! + \brief Represents a list in the Falco Engine. + The rule ID must be unique across all the lists loaded in the engine. +*/ +struct falco_list +{ + falco_list(): used(false), id(0) { } + falco_list(falco_list&&) = default; + falco_list& operator = (falco_list&&) = default; + falco_list(const falco_list&) = default; + falco_list& operator = (const falco_list&) = default; + ~falco_list() = default; + + bool used; + std::size_t id; + std::string name; + std::vector items; +}; + +/*! + \brief Represents a macro in the Falco Engine. + The rule ID must be unique across all the macros loaded in the engine. +*/ +struct falco_macro +{ + falco_macro(): used(false), id(0) { } + falco_macro(falco_macro&&) = default; + falco_macro& operator = (falco_macro&&) = default; + falco_macro(const falco_macro&) = default; + falco_macro& operator = (const falco_macro&) = default; + ~falco_macro() = default; + + bool used; + std::size_t id; + std::string name; + std::shared_ptr condition; +}; + /*! \brief Represents a rule in the Falco Engine. The rule ID must be unique across all the rules loaded in the engine. @@ -32,6 +72,7 @@ struct falco_rule falco_rule& operator = (falco_rule&&) = default; falco_rule(const falco_rule&) = default; falco_rule& operator = (const falco_rule&) = default; + ~falco_rule() = default; std::size_t id; std::string source; @@ -41,4 +82,5 @@ struct falco_rule std::set tags; std::set exception_fields; falco_common::priority_type priority; + std::shared_ptr condition; }; diff --git a/userspace/engine/rule_loader.cpp b/userspace/engine/rule_loader.cpp index d4076412910..9036a67e340 100644 --- a/userspace/engine/rule_loader.cpp +++ b/userspace/engine/rule_loader.cpp @@ -532,12 +532,12 @@ rule_loader::plugin_version_info::plugin_version_info(context &ctx) } rule_loader::list_info::list_info(context &ctx) - : ctx(ctx), used(false), index(0), visibility(0) + : ctx(ctx), index(0), visibility(0) { } rule_loader::macro_info::macro_info(context &ctx) - : ctx(ctx), cond_ctx(ctx), used(false), index(0), visibility(0) + : ctx(ctx), cond_ctx(ctx), index(0), visibility(0) { } diff --git a/userspace/engine/rule_loader.h b/userspace/engine/rule_loader.h index 44481fc50df..c866625e0e5 100644 --- a/userspace/engine/rule_loader.h +++ b/userspace/engine/rule_loader.h @@ -273,8 +273,7 @@ namespace rule_loader const indexed_vector& srcs, const std::string& name) : content(cont), sources(srcs), name(name), - default_ruleset_id(0), replace_output_container_info(false), - min_priority(falco_common::PRIORITY_DEBUG) + output_extra(), replace_output_container_info(false) { res.reset(new result(name)); } @@ -283,14 +282,15 @@ namespace rule_loader configuration(const configuration&) = delete; configuration& operator = (const configuration&) = delete; + // inputs const std::string& content; const indexed_vector& sources; std::string name; - std::unique_ptr res; std::string output_extra; - uint16_t default_ruleset_id; bool replace_output_container_info; - falco_common::priority_type min_priority; + + // outputs + std::unique_ptr res; }; /*! @@ -359,7 +359,6 @@ namespace rule_loader list_info& operator = (const list_info&) = default; context ctx; - bool used; size_t index; size_t visibility; std::string name; @@ -380,12 +379,10 @@ namespace rule_loader context ctx; context cond_ctx; - bool used; size_t index; size_t visibility; std::string name; std::string cond; - std::shared_ptr cond_ast; }; /*! diff --git a/userspace/engine/rule_loader_compiler.cpp b/userspace/engine/rule_loader_compiler.cpp index 074c9da7bdd..393aae4ff59 100644 --- a/userspace/engine/rule_loader_compiler.cpp +++ b/userspace/engine/rule_loader_compiler.cpp @@ -160,8 +160,30 @@ static void build_rule_exception_infos( } } +static inline rule_loader::list_info* list_info_from_name( + const rule_loader::collector& c, const std::string& name) +{ + auto ret = c.lists().at(name); + if (!ret) + { + throw falco_exception("can't find internal list info at name: " + name); + } + return ret; +} + +static inline rule_loader::macro_info* macro_info_from_name( + const rule_loader::collector& c, const std::string& name) +{ + auto ret = c.macros().at(name); + if (!ret) + { + throw falco_exception("can't find internal macro info at name: " + name); + } + return ret; +} + // todo(jasondellaluce): this breaks string escaping in lists -static bool resolve_list(std::string& cnd, const rule_loader::list_info& list) +static bool resolve_list(std::string& cnd, const falco_list& list) { static std::string blanks = " \t\n\r"; static std::string delims = blanks + "(),="; @@ -232,18 +254,20 @@ static bool resolve_list(std::string& cnd, const rule_loader::list_info& list) } static void resolve_macros( - indexed_vector& macros, + const indexed_vector& infos, + indexed_vector& macros, std::shared_ptr& ast, const std::string& condition, uint32_t visibility, const rule_loader::context &ctx) { filter_macro_resolver macro_resolver; - for (auto &m : macros) + for (auto &m : infos) { if (m.index < visibility) { - macro_resolver.set_macro(m.name, m.cond_ast); + auto macro = macros.at(m.name); + macro_resolver.set_macro(m.name, macro->condition); } } macro_resolver.run(ast); @@ -272,7 +296,7 @@ static void resolve_macros( // note: there is no visibility order between filter conditions and lists static std::shared_ptr parse_condition( std::string condition, - indexed_vector& lists, + indexed_vector& lists, const rule_loader::context &ctx) { for (auto &l : lists) @@ -319,13 +343,14 @@ static void apply_output_substitutions( void rule_loader::compiler::compile_list_infos( configuration& cfg, const collector& col, - indexed_vector& out) const + indexed_vector& out) const { std::string tmp; std::vector used; for (auto &list : col.lists()) { - list_info v = list; + falco_list v; + v.name = list.name; v.items.clear(); for (auto &item : list.items) { @@ -347,7 +372,8 @@ void rule_loader::compiler::compile_list_infos( } } v.used = false; - out.insert(v, v.name); + auto list_id = out.insert(v, v.name); + out.at(list_id)->id = list_id; } for (auto &v : used) { @@ -359,20 +385,23 @@ void rule_loader::compiler::compile_list_infos( void rule_loader::compiler::compile_macros_infos( configuration& cfg, const collector& col, - indexed_vector& lists, - indexed_vector& out) const + indexed_vector& lists, + indexed_vector& out) const { for (auto &m : col.macros()) { - macro_info entry = m; - entry.cond_ast = parse_condition(m.cond, lists, m.cond_ctx); + falco_macro entry; + entry.name = m.name; + entry.condition = parse_condition(m.cond, lists, m.cond_ctx); entry.used = false; - out.insert(entry, m.name); + auto macro_id = out.insert(entry, m.name); + out.at(macro_id)->id = macro_id; } for (auto &m : out) { - resolve_macros(out, m.cond_ast, m.cond, m.visibility, m.ctx); + auto info = macro_info_from_name(col, m.name); + resolve_macros(col.macros(), out, m.condition, info->cond, info->visibility, info->ctx); } } @@ -386,8 +415,8 @@ static bool err_is_unknown_type_or_field(const std::string& err) void rule_loader::compiler::compile_rule_infos( configuration& cfg, const collector& col, - indexed_vector& lists, - indexed_vector& macros, + indexed_vector& lists, + indexed_vector& macros, indexed_vector& out) const { std::string err, condition; @@ -401,12 +430,6 @@ void rule_loader::compiler::compile_rule_infos( continue; } - // skip the rule if below the minimum priority - if (r.priority > cfg.min_priority) - { - continue; - } - // note: this should not be nullptr if the source is not unknown auto source = cfg.sources.at(r.source); THROW(!source, @@ -423,12 +446,12 @@ void rule_loader::compiler::compile_rule_infos( build_rule_exception_infos( r.exceptions, rule.exception_fields, condition); } - auto ast = parse_condition(condition, lists, r.cond_ctx); - resolve_macros(macros, ast, condition, MAX_VISIBILITY, r.ctx); + rule.condition = parse_condition(condition, lists, r.cond_ctx); + resolve_macros(col.macros(), macros, rule.condition, condition, MAX_VISIBILITY, r.ctx); // check for warnings in the filtering condition warn_codes.clear(); - if (warn_resolver.run(ast.get(), warn_codes)) + if (warn_resolver.run(rule.condition.get(), warn_codes)) { for (auto &w : warn_codes) { @@ -443,8 +466,11 @@ void rule_loader::compiler::compile_rule_infos( apply_output_substitutions(cfg, rule.output); } + // validate the rule's output if(!is_format_valid(*cfg.sources.at(r.source), rule.output, err)) { + // skip the rule silently if skip_if_unknown_filter is true and + // we encountered some specific kind of errors if (err_is_unknown_type_or_field(err) && r.skip_if_unknown_filter) { cfg.res->add_warning( @@ -459,30 +485,18 @@ void rule_loader::compiler::compile_rule_infos( r.output_ctx); } - // construct rule definition and compile it to a filter - rule.name = r.name; - rule.source = r.source; - rule.description = r.desc; - rule.priority = r.priority; - rule.tags = r.tags; - - auto rule_id = out.insert(rule, rule.name); - out.at(rule_id)->id = rule_id; - - // This also compiles the filter, and might throw a - // falco_exception with details on the compilation - // failure. - sinsp_filter_compiler compiler(cfg.sources.at(r.source)->filter_factory, ast.get()); - try { - std::shared_ptr filter(compiler.compile()); - source->ruleset->add(*out.at(rule_id), filter, ast); + // validate the rule's condiiton: we compile it into a sinsp filter + // on-the-fly and we throw an exception with details on failure + sinsp_filter_compiler compiler(cfg.sources.at(r.source)->filter_factory, rule.condition.get()); + try + { + compiler.compile(); } catch (const sinsp_exception& e) { - // Allow errors containing "nonexistent field" if - // skip_if_unknown_filter is true + // skip the rule silently if skip_if_unknown_filter is true and + // we encountered some specific kind of errors std::string err = e.what(); - if (err_is_unknown_type_or_field(err) && r.skip_if_unknown_filter) { cfg.res->add_warning( @@ -491,7 +505,6 @@ void rule_loader::compiler::compile_rule_infos( r.cond_ctx); continue; } - rule_loader::context ctx(compiler.get_pos(), condition, r.cond_ctx); throw rule_loader::rule_load_exception( falco::load_result::load_result::LOAD_ERR_COMPILE_CONDITION, @@ -499,20 +512,10 @@ void rule_loader::compiler::compile_rule_infos( ctx); } - // By default rules are enabled/disabled for the default ruleset - if(r.enabled) - { - source->ruleset->enable(rule.name, true, cfg.default_ruleset_id); - } - else - { - source->ruleset->disable(rule.name, true, cfg.default_ruleset_id); - } - // populate set of event types and emit an special warning - if(rule.source == falco_common::syscall_source) + if(r.source == falco_common::syscall_source) { - auto evttypes = libsinsp::filter::ast::ppm_event_codes(ast.get()); + auto evttypes = libsinsp::filter::ast::ppm_event_codes(rule.condition.get()); if ((evttypes.empty() || evttypes.size() > 100) && r.warn_evttypes) { cfg.res->add_warning( @@ -521,23 +524,29 @@ void rule_loader::compiler::compile_rule_infos( r.ctx); } } + + // finalize the rule definition and add it to output + rule.name = r.name; + rule.source = r.source; + rule.description = r.desc; + rule.priority = r.priority; + rule.tags = r.tags; + auto rule_id = out.insert(rule, rule.name); + out.at(rule_id)->id = rule_id; } } void rule_loader::compiler::compile( configuration& cfg, const collector& col, - indexed_vector& out) const + compile_output& out) const { - indexed_vector lists; - indexed_vector macros; - // expand all lists, macros, and rules try { - compile_list_infos(cfg, col, lists); - compile_macros_infos(cfg, col, lists, macros); - compile_rule_infos(cfg, col, lists, macros, out); + compile_list_infos(cfg, col, out.lists); + compile_macros_infos(cfg, col, out.lists, out.macros); + compile_rule_infos(cfg, col, out.lists, out.macros, out.rules); } catch(rule_load_exception &e) { @@ -546,24 +555,24 @@ void rule_loader::compiler::compile( } // print info on any dangling lists or macros that were not used anywhere - for (auto &m : macros) + for (auto &m : out.macros) { if (!m.used) { cfg.res->add_warning( falco::load_result::load_result::LOAD_UNUSED_MACRO, "Macro not referred to by any other rule/macro", - m.ctx); + macro_info_from_name(col, m.name)->ctx); } } - for (auto &l : lists) + for (auto &l : out.lists) { if (!l.used) { cfg.res->add_warning( falco::load_result::LOAD_UNUSED_LIST, "List not referred to by any other rule/macro", - l.ctx); + list_info_from_name(col, l.name)->ctx); } } } diff --git a/userspace/engine/rule_loader_compiler.h b/userspace/engine/rule_loader_compiler.h index 699d74b6159..bc7b53d9e70 100644 --- a/userspace/engine/rule_loader_compiler.h +++ b/userspace/engine/rule_loader_compiler.h @@ -31,6 +31,23 @@ namespace rule_loader class compiler { public: + /*! + \brief The output of a compilation. + */ + struct compile_output + { + compile_output() = default; + virtual ~compile_output() = default; + compile_output(compile_output&&) = default; + compile_output& operator = (compile_output&&) = default; + compile_output(const compile_output&) = default; + compile_output& operator = (const compile_output&) = default; + + indexed_vector lists; + indexed_vector macros; + indexed_vector rules; + }; + compiler() = default; virtual ~compiler() = default; compiler(compiler&&) = default; @@ -44,25 +61,25 @@ class compiler virtual void compile( configuration& cfg, const collector& col, - indexed_vector& out) const; + compile_output& out) const; private: void compile_list_infos( configuration& cfg, const collector& col, - indexed_vector& out) const; + indexed_vector& out) const; void compile_macros_infos( configuration& cfg, const collector& col, - indexed_vector& lists, - indexed_vector& out) const; + indexed_vector& lists, + indexed_vector& out) const; void compile_rule_infos( configuration& cfg, const collector& col, - indexed_vector& lists, - indexed_vector& macros, + indexed_vector& lists, + indexed_vector& macros, indexed_vector& out) const; }; From 95e3af4137fa9c80e07c74540803e9dc66b5b1e9 Mon Sep 17 00:00:00 2001 From: Jason Dellaluce Date: Wed, 6 Sep 2023 13:12:31 +0000 Subject: [PATCH 6/9] fix(userspace/engine): solve issues in describing rules/macros/lists Signed-off-by: Jason Dellaluce --- userspace/engine/falco_engine.cpp | 210 ++++++++++++++++-------------- userspace/engine/falco_engine.h | 15 ++- 2 files changed, 120 insertions(+), 105 deletions(-) diff --git a/userspace/engine/falco_engine.cpp b/userspace/engine/falco_engine.cpp index 99c088642c1..0177051982d 100644 --- a/userspace/engine/falco_engine.cpp +++ b/userspace/engine/falco_engine.cpp @@ -506,6 +506,16 @@ std::size_t falco_engine::add_source(const std::string &source, return m_sources.insert(src, source); } +template inline Json::Value sequence_to_json_array(const T& seq) +{ + Json::Value ret = Json::arrayValue; + for (auto it = seq.begin(); it != seq.end(); it++) + { + ret.append(*it); + } + return ret; +} + void falco_engine::describe_rule(std::string *rule, const std::vector>& plugins, bool json) const { if(!json) @@ -535,9 +545,20 @@ void falco_engine::describe_rule(std::string *rule, const std::vector>& plugins) const { Json::Value rule_info; // Fill general rule information rule_info["name"] = r.name; - rule_info["condition"] = ri.cond; + rule_info["condition"] = info.cond; rule_info["priority"] = format_priority(r.priority, false); - rule_info["output"] = r.output; + rule_info["output"] = info.output; rule_info["description"] = r.description; - rule_info["enabled"] = ri.enabled; + rule_info["enabled"] = info.enabled; rule_info["source"] = r.source; - Json::Value tags = Json::arrayValue; - for(const auto &t : ri.tags) - { - tags.append(t); - } - rule_info["tags"] = tags; + rule_info["tags"] = sequence_to_json_array(info.tags); out["info"] = rule_info; - // Parse rule condition and build the AST - // Assumption: no exception because rules have already been loaded. - auto ast = libsinsp::filter::parser(ri.cond).parse(); + // Parse rule condition and build the non-compiled AST + // Assumption: no error because rules have already been loaded. + auto ast = libsinsp::filter::parser(info.cond).parse(); // get details related to the condition's filter filter_details details; + filter_details compiled_details; Json::Value json_details; for(const auto &m : m_rule_collector.macros()) { details.known_macros.insert(m.name); + compiled_details.known_macros.insert(m.name); } for(const auto &l : m_rule_collector.lists()) { details.known_lists.insert(l.name); + compiled_details.known_lists.insert(l.name); } filter_details_resolver().run(ast.get(), details); - get_json_filter_details(json_details, details); - out["details"] = json_details; + filter_details_resolver().run(r.condition.get(), compiled_details); + + out["details"]["macros"] = sequence_to_json_array(details.macros); + out["details"]["lists"] = sequence_to_json_array(details.lists); + out["details"]["operators"] = sequence_to_json_array(compiled_details.operators); + out["details"]["condition_fields"] = sequence_to_json_array(compiled_details.fields); // Get fields from output string auto fmt = create_formatter(r.source, r.output); std::vector out_fields; fmt->get_field_names(out_fields); - Json::Value outputFields = Json::arrayValue; - for(const auto &of : out_fields) - { - outputFields.append(of); - } - out["details"]["output_fields"] = outputFields; + out["details"]["output_fields"] = sequence_to_json_array(out_fields); // Get fields from exceptions - Json::Value exception_fields = Json::arrayValue; - for(const auto &f : r.exception_fields) - { - exception_fields.append(f); - } - out["details"]["exception_fields"] = exception_fields; + out["details"]["exception_fields"] = sequence_to_json_array(r.exception_fields); // Get names and operators from exceptions Json::Value exception_names = Json::arrayValue; Json::Value exception_operators = Json::arrayValue; - for(const auto &e : ri.exceptions) + for(const auto &e : info.exceptions) { exception_names.append(e.name); if(e.comps.is_list) @@ -719,9 +732,13 @@ void falco_engine::get_json_details( // Store event types Json::Value events; - get_json_evt_types(events, ri.source, details); + get_json_evt_types(events, info.source, r.condition.get()); out["details"]["events"] = events; + // Store compiled condition and output + out["details"]["condition_compiled"] = libsinsp::filter::ast::as_string(r.condition.get()); + out["details"]["output_compiled"] = r.output; + // Compute the plugins that are actually used by this rule. This is involves: // - The rule's event source, that can be implemented by a plugin // - The fields used in the rule's condition, output, and exceptions @@ -729,59 +746,86 @@ void falco_engine::get_json_details( // match plugin-provided async events Json::Value used_plugins; // note: making a union of conditions's and output's fields - // note: the condition's AST should include all resolved refs and exceptions - details.fields.insert(out_fields.begin(), out_fields.end()); - get_json_used_plugins(used_plugins, ri.source, details.evtnames, details.fields, plugins); + // note: the condition's AST accounts for all the resolved refs and exceptions + compiled_details.fields.insert(out_fields.begin(), out_fields.end()); + get_json_used_plugins(used_plugins, info.source, compiled_details.evtnames, compiled_details.fields, plugins); out["details"]["plugins"] = used_plugins; } void falco_engine::get_json_details( Json::Value& out, - const rule_loader::macro_info& m) const + const falco_macro& m, + const rule_loader::macro_info& info, + const std::vector>& plugins) const { Json::Value macro_info; macro_info["name"] = m.name; - macro_info["condition"] = m.cond; + macro_info["condition"] = info.cond; out["info"] = macro_info; - // Parse rule condition and build the AST + // Parse the macro condition and build the non-compiled AST // Assumption: no exception because rules have already been loaded. - auto ast = libsinsp::filter::parser(m.cond).parse(); + auto ast = libsinsp::filter::parser(info.cond).parse(); // get details related to the condition's filter filter_details details; + filter_details compiled_details; Json::Value json_details; for(const auto &m : m_rule_collector.macros()) { details.known_macros.insert(m.name); + compiled_details.known_macros.insert(m.name); } for(const auto &l : m_rule_collector.lists()) { details.known_lists.insert(l.name); + compiled_details.known_lists.insert(l.name); } filter_details_resolver().run(ast.get(), details); - get_json_filter_details(json_details, details); - out["details"] = json_details; + filter_details_resolver().run(m.condition.get(), compiled_details); + + out["details"]["used"] = m.used; + out["details"]["macros"] = sequence_to_json_array(details.macros); + out["details"]["lists"] = sequence_to_json_array(details.lists); + out["details"]["operators"] = sequence_to_json_array(compiled_details.operators); + out["details"]["condition_fields"] = sequence_to_json_array(compiled_details.fields); // Store event types Json::Value events; - get_json_evt_types(events, "", details); + get_json_evt_types(events, "", m.condition.get()); out["details"]["events"] = events; + + // Store compiled condition + out["details"]["condition_compiled"] = libsinsp::filter::ast::as_string(m.condition.get()); + + // Compute the plugins that are actually used by this macro. + // Note: macros have no specidic source, we need to set an empty list of used + // plugins because we can't be certain about their actual usage. For example, + // if a macro uses a plugin's field, we can't be sure which plugin actually + // is used until we resolve the macro ref in a rule providing a source for + // disambiguation. + out["details"]["plugins"] = Json::arrayValue; } void falco_engine::get_json_details( Json::Value& out, - const rule_loader::list_info& l) const + const falco_list& l, + const rule_loader::list_info& info, + const std::vector>& plugins) const { Json::Value list_info; list_info["name"] = l.name; + // note: the syntactic definitions still has the list refs unresolved Json::Value items = Json::arrayValue; Json::Value lists = Json::arrayValue; - for(const auto &i : l.items) + for(const auto &i : info.items) { - if(m_rule_collector.lists().at(i) != nullptr) + // if an item is present in the syntactic def of a list, but not + // on the compiled_items of the same list, then we can assume it + // being a resolved list ref + if(std::find(l.items.begin(), l.items.end(), i) == l.items.end()) { lists.append(i); continue; @@ -791,62 +835,32 @@ void falco_engine::get_json_details( list_info["items"] = items; out["info"] = list_info; + out["details"]["used"] = l.used; out["details"]["lists"] = lists; -} - -void falco_engine::get_json_filter_details( - Json::Value& out, - const filter_details& details) const -{ - Json::Value macros = Json::arrayValue; - for(const auto &m : details.macros) - { - macros.append(m); - } - out["macros"] = macros; - - Json::Value operators = Json::arrayValue; - for(const auto &o : details.operators) - { - operators.append(o); - } - out["operators"] = operators; - - Json::Value condition_fields = Json::arrayValue; - for(const auto &f : details.fields) - { - condition_fields.append(f); - } - out["condition_fields"] = condition_fields; - - Json::Value lists = Json::arrayValue; - for(const auto &l : details.lists) - { - lists.append(l); - } - out["lists"] = lists; + out["details"]["items_compiled"] = sequence_to_json_array(l.items); + out["details"]["plugins"] = Json::arrayValue; // always empty } void falco_engine::get_json_evt_types( Json::Value& out, const std::string& source, - const filter_details& details) const + libsinsp::filter::ast::expr* ast) const { - out = Json::arrayValue; - for (const auto& n : details.evtnames) + // note: this duplicates part of the logic of evttype_index_ruleset, + // not good but it's our best option for now + if (source.empty() || source == falco_common::syscall_source) { - out.append(n); + auto evtcodes = libsinsp::filter::ast::ppm_event_codes(ast); + evtcodes.insert(ppm_event_code::PPME_ASYNCEVENT_E); + auto syscodes = libsinsp::filter::ast::ppm_sc_codes(ast); + auto syscodes_to_evt_names = libsinsp::events::sc_set_to_event_names(syscodes); + auto evtcodes_to_evt_names = libsinsp::events::event_set_to_names(evtcodes, false); + out = sequence_to_json_array(unordered_set_union(syscodes_to_evt_names, evtcodes_to_evt_names)); } - - // plugin-implemented sources always match pluginevent. - // note: macros have no source and can potentially be referenced in a rule - // used for a plugin-implemented source. - if(source.empty() || source != falco_common::syscall_source) + else { - for (const auto& n : libsinsp::events::event_set_to_names({PPME_PLUGINEVENT_E}, false)) - { - out.append(n); - } + out = sequence_to_json_array(libsinsp::events::event_set_to_names( + {ppm_event_code::PPME_PLUGINEVENT_E, ppm_event_code::PPME_ASYNCEVENT_E})); } } diff --git a/userspace/engine/falco_engine.h b/userspace/engine/falco_engine.h index 696a6921036..38a66801ed6 100644 --- a/userspace/engine/falco_engine.h +++ b/userspace/engine/falco_engine.h @@ -306,21 +306,22 @@ class falco_engine void get_json_details( Json::Value& out, const falco_rule& r, - const rule_loader::rule_info& ri, + const rule_loader::rule_info& info, const std::vector>& plugins) const; void get_json_details( Json::Value& out, - const rule_loader::macro_info& m) const; + const falco_macro& m, + const rule_loader::macro_info& info, + const std::vector>& plugins) const; void get_json_details( Json::Value& out, - const rule_loader::list_info& l) const; - void get_json_filter_details( - Json::Value& out, - const filter_details& details) const; + const falco_list& l, + const rule_loader::list_info& info, + const std::vector>& plugins) const; void get_json_evt_types( Json::Value& out, const std::string& source, - const filter_details& details) const; + libsinsp::filter::ast::expr* ast) const; void get_json_used_plugins( Json::Value& out, const std::string& source, From b4752b966f90c428d0269bf42fda0f1aac4fd0d6 Mon Sep 17 00:00:00 2001 From: Jason Dellaluce Date: Wed, 6 Sep 2023 13:23:19 +0000 Subject: [PATCH 7/9] fix(userspace/engine): solve issues with filter details resolver Signed-off-by: Jason Dellaluce --- userspace/engine/filter_details_resolver.cpp | 32 ++++++++------------ userspace/engine/filter_details_resolver.h | 3 +- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/userspace/engine/filter_details_resolver.cpp b/userspace/engine/filter_details_resolver.cpp index 81a9a407935..d4b88414792 100644 --- a/userspace/engine/filter_details_resolver.cpp +++ b/userspace/engine/filter_details_resolver.cpp @@ -46,6 +46,7 @@ void filter_details_resolver::run(ast::expr* filter, filter_details& details) void filter_details_resolver::visitor::visit(ast::and_expr* e) { + m_expect_macro = false; for(size_t i = 0; i < e->children.size(); i++) { m_expect_macro = true; @@ -56,6 +57,7 @@ void filter_details_resolver::visitor::visit(ast::and_expr* e) void filter_details_resolver::visitor::visit(ast::or_expr* e) { + m_expect_macro = false; for(size_t i = 0; i < e->children.size(); i++) { m_expect_macro = true; @@ -98,18 +100,11 @@ void filter_details_resolver::visitor::visit(ast::binary_check_expr* e) m_expect_macro = false; m_details.fields.insert(get_field_name(e->field, e->arg)); m_details.operators.insert(e->op); - if (e->field == "evt.type" || e->field == "evt.asynctype") - { - m_expect_evtname = true; - e->value->accept(this); - m_expect_evtname = false; - } - else - { - m_expect_list = true; - e->value->accept(this); - m_expect_list = false; - } + m_expect_list = true; + m_expect_evtname = e->field == "evt.type" || e->field == "evt.asynctype"; + e->value->accept(this); + m_expect_evtname = false; + m_expect_list = false; } void filter_details_resolver::visitor::visit(ast::unary_check_expr* e) @@ -121,17 +116,16 @@ void filter_details_resolver::visitor::visit(ast::unary_check_expr* e) void filter_details_resolver::visitor::visit(ast::value_expr* e) { - if(m_expect_macro) + if (m_expect_macro) { - auto it = m_details.known_macros.find(e->value); - if(it == m_details.known_macros.end()) + if(m_details.known_macros.find(e->value) != m_details.known_macros.end()) { - return; + m_details.macros.insert(e->value); } - - m_details.macros.insert(e->value); + // todo(jasondellaluce): should we throw an error if we + // encounter an unknown macro? } - if(m_expect_evtname) + else if (m_expect_evtname) { m_details.evtnames.insert(e->value); } diff --git a/userspace/engine/filter_details_resolver.h b/userspace/engine/filter_details_resolver.h index df0e66ec70a..4030537b712 100644 --- a/userspace/engine/filter_details_resolver.h +++ b/userspace/engine/filter_details_resolver.h @@ -60,7 +60,8 @@ class filter_details_resolver visitor(filter_details& details) : m_details(details), m_expect_list(false), - m_expect_macro(false) {} + m_expect_macro(false), + m_expect_evtname(false) {} visitor(visitor&&) = default; visitor& operator = (visitor&&) = default; visitor(const visitor&) = delete; From 8baa8829748b5fdd215c21b44ae3f6733f2eb463 Mon Sep 17 00:00:00 2001 From: Jason Dellaluce Date: Wed, 6 Sep 2023 13:25:24 +0000 Subject: [PATCH 8/9] refactor(userspace/engine)!: rename some description details outputs Signed-off-by: Jason Dellaluce --- userspace/engine/falco_engine.cpp | 36 ++++++++++++++++--------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/userspace/engine/falco_engine.cpp b/userspace/engine/falco_engine.cpp index 0177051982d..466c012d08f 100644 --- a/userspace/engine/falco_engine.cpp +++ b/userspace/engine/falco_engine.cpp @@ -686,7 +686,7 @@ void falco_engine::get_json_details( out["details"]["macros"] = sequence_to_json_array(details.macros); out["details"]["lists"] = sequence_to_json_array(details.lists); - out["details"]["operators"] = sequence_to_json_array(compiled_details.operators); + out["details"]["condition_operators"] = sequence_to_json_array(compiled_details.operators); out["details"]["condition_fields"] = sequence_to_json_array(compiled_details.fields); // Get fields from output string @@ -699,11 +699,11 @@ void falco_engine::get_json_details( out["details"]["exception_fields"] = sequence_to_json_array(r.exception_fields); // Get names and operators from exceptions - Json::Value exception_names = Json::arrayValue; - Json::Value exception_operators = Json::arrayValue; + std::unordered_set exception_names; + std::unordered_set exception_operators; for(const auto &e : info.exceptions) { - exception_names.append(e.name); + exception_names.insert(e.name); if(e.comps.is_list) { for(const auto& c : e.comps.items) @@ -713,22 +713,22 @@ void falco_engine::get_json_details( // considering max two levels of lists for(const auto& i : c.items) { - exception_operators.append(i.item); + exception_operators.insert(i.item); } } else { - exception_operators.append(c.item); + exception_operators.insert(c.item); } } } else { - exception_operators.append(e.comps.item); + exception_operators.insert(e.comps.item); } } - out["details"]["exceptions"] = exception_names; - out["details"]["exception_operators"] = exception_operators; + out["details"]["exception_names"] = sequence_to_json_array(exception_names); + out["details"]["exception_operators"] = sequence_to_json_array(exception_operators); // Store event types Json::Value events; @@ -788,7 +788,7 @@ void falco_engine::get_json_details( out["details"]["used"] = m.used; out["details"]["macros"] = sequence_to_json_array(details.macros); out["details"]["lists"] = sequence_to_json_array(details.lists); - out["details"]["operators"] = sequence_to_json_array(compiled_details.operators); + out["details"]["condition_operators"] = sequence_to_json_array(compiled_details.operators); out["details"]["condition_fields"] = sequence_to_json_array(compiled_details.fields); // Store event types @@ -819,7 +819,7 @@ void falco_engine::get_json_details( // note: the syntactic definitions still has the list refs unresolved Json::Value items = Json::arrayValue; - Json::Value lists = Json::arrayValue; + std::unordered_set lists; for(const auto &i : info.items) { // if an item is present in the syntactic def of a list, but not @@ -827,7 +827,7 @@ void falco_engine::get_json_details( // being a resolved list ref if(std::find(l.items.begin(), l.items.end(), i) == l.items.end()) { - lists.append(i); + lists.insert(i); continue; } items.append(i); @@ -836,7 +836,7 @@ void falco_engine::get_json_details( list_info["items"] = items; out["info"] = list_info; out["details"]["used"] = l.used; - out["details"]["lists"] = lists; + out["details"]["lists"] = sequence_to_json_array(lists); out["details"]["items_compiled"] = sequence_to_json_array(l.items); out["details"]["plugins"] = Json::arrayValue; // always empty } @@ -884,7 +884,7 @@ void falco_engine::get_json_used_plugins( fieldnames.insert(f); } - out = Json::arrayValue; + std::unordered_set used_plugins; for (const auto& p : plugins) { bool used = false; @@ -896,7 +896,7 @@ void falco_engine::get_json_used_plugins( // they will both be included in the list. if (!used && p->event_source() == source) { - out.append(p->name()); + used_plugins.insert(p->name()); used = true; } } @@ -913,7 +913,7 @@ void falco_engine::get_json_used_plugins( { if (!used && fieldnames.find(f.m_name) != fieldnames.end()) { - out.append(p->name()); + used_plugins.insert(p->name()); used = true; break; } @@ -932,7 +932,7 @@ void falco_engine::get_json_used_plugins( { if (!used && evtnames.find(n) != evtnames.end()) { - out.append(p->name()); + used_plugins.insert(p->name()); used = true; break; } @@ -940,6 +940,8 @@ void falco_engine::get_json_used_plugins( } } } + + out = sequence_to_json_array(used_plugins); } void falco_engine::print_stats() const From 3df2c3b5a0cc55d57284420a7bac2e7f2461a39e Mon Sep 17 00:00:00 2001 From: Jason Dellaluce Date: Tue, 19 Sep 2023 15:06:05 +0000 Subject: [PATCH 9/9] chore(userspace/engine): apply codespell suggestions Signed-off-by: Jason Dellaluce --- userspace/engine/falco_engine.cpp | 2 +- userspace/engine/rule_loader_compiler.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/userspace/engine/falco_engine.cpp b/userspace/engine/falco_engine.cpp index 466c012d08f..1b0f3d73e59 100644 --- a/userspace/engine/falco_engine.cpp +++ b/userspace/engine/falco_engine.cpp @@ -800,7 +800,7 @@ void falco_engine::get_json_details( out["details"]["condition_compiled"] = libsinsp::filter::ast::as_string(m.condition.get()); // Compute the plugins that are actually used by this macro. - // Note: macros have no specidic source, we need to set an empty list of used + // Note: macros have no specific source, we need to set an empty list of used // plugins because we can't be certain about their actual usage. For example, // if a macro uses a plugin's field, we can't be sure which plugin actually // is used until we resolve the macro ref in a rule providing a source for diff --git a/userspace/engine/rule_loader_compiler.cpp b/userspace/engine/rule_loader_compiler.cpp index 393aae4ff59..431e95384cc 100644 --- a/userspace/engine/rule_loader_compiler.cpp +++ b/userspace/engine/rule_loader_compiler.cpp @@ -485,7 +485,7 @@ void rule_loader::compiler::compile_rule_infos( r.output_ctx); } - // validate the rule's condiiton: we compile it into a sinsp filter + // validate the rule's condition: we compile it into a sinsp filter // on-the-fly and we throw an exception with details on failure sinsp_filter_compiler compiler(cfg.sources.at(r.source)->filter_factory, rule.condition.get()); try