update(userspace/engine): modularize rule compiler, fix and enrich rule descriptions

userspace/engine/falco_engine.h

@@ -125,7 +125,7 @@ class falco_engine
 	// Print details on the given rule. If rule is NULL, print
 	// details on all rules.
 	//
-	void describe_rule(std::string *rule, bool json) const;
+	void describe_rule(std::string *rule, const std::vector<std::shared_ptr<sinsp_plugin>>& plugins, bool json) const;
 
 	//
 	// Print statistics on how many events matched each rule.
@@ -303,18 +303,31 @@ class falco_engine
 	inline bool should_drop_evt() const;
 
 	// Retrieve json details from rules, macros, lists
-	void get_json_details(const falco_rule& r,
-					const rule_loader::rule_info& ri,
-					sinsp* insp,
-					Json::Value& rule) const;
-	void get_json_details(const rule_loader::macro_info& m,
-					Json::Value& macro) const;
-	void get_json_details(const rule_loader::list_info& l,
-					Json::Value& list) const;
-	void get_json_details(libsinsp::filter::ast::expr* ast,
-					Json::Value& output) const;
-	void get_json_evt_types(libsinsp::filter::ast::expr* ast,
-					Json::Value& output) const;
+	void get_json_details(
+		Json::Value& out,
+		const falco_rule& r,
+		const rule_loader::rule_info& info,
+		const std::vector<std::shared_ptr<sinsp_plugin>>& plugins) const;
+	void get_json_details(
+		Json::Value& out,
+		const falco_macro& m,
+		const rule_loader::macro_info& info,
+		const std::vector<std::shared_ptr<sinsp_plugin>>& plugins) const;
+	void get_json_details(
+		Json::Value& out,
+		const falco_list& l,
+		const rule_loader::list_info& info,
+		const std::vector<std::shared_ptr<sinsp_plugin>>& plugins) const;
+	void get_json_evt_types(
+		Json::Value& out,
+		const std::string& source,
+		libsinsp::filter::ast::expr* ast) const;
+	void get_json_used_plugins(
+		Json::Value& out,
+		const std::string& source,
+		const std::unordered_set<std::string>& evttypes,
+		const std::unordered_set<std::string>& fields,
+		const std::vector<std::shared_ptr<sinsp_plugin>>& plugins) const;
 
 	rule_loader::collector m_rule_collector;
 	indexed_vector<falco_rule> m_rules;

userspace/engine/falco_rule.h

@@ -21,6 +21,46 @@ limitations under the License.
 #include <string>
 #include "falco_common.h"
 
+#include <filter/ast.h>
+
+/*!
+	\brief Represents a list in the Falco Engine.
+	The rule ID must be unique across all the lists loaded in the engine.
+*/
+struct falco_list
+{
+	falco_list(): used(false), id(0) { }
+	falco_list(falco_list&&) = default;
+	falco_list& operator = (falco_list&&) = default;
+	falco_list(const falco_list&) = default;
+	falco_list& operator = (const falco_list&) = default;
+	~falco_list() = default;
+
+	bool used;
+	std::size_t id;
+	std::string name;
+	std::vector<std::string> items;
+};
+
+/*!
+	\brief Represents a macro in the Falco Engine.
+	The rule ID must be unique across all the macros loaded in the engine.
+*/
+struct falco_macro
+{
+	falco_macro(): used(false), id(0) { }
+	falco_macro(falco_macro&&) = default;
+	falco_macro& operator = (falco_macro&&) = default;
+	falco_macro(const falco_macro&) = default;
+	falco_macro& operator = (const falco_macro&) = default;
+	~falco_macro() = default;
+
+	bool used;
+	std::size_t id;
+	std::string name;
+	std::shared_ptr<libsinsp::filter::ast::expr> condition;
+};
+
 /*!
 	\brief Represents a rule in the Falco Engine.
 	The rule ID must be unique across all the rules loaded in the engine.
@@ -32,6 +72,7 @@ struct falco_rule
 	falco_rule& operator = (falco_rule&&) = default;
 	falco_rule(const falco_rule&) = default;
 	falco_rule& operator = (const falco_rule&) = default;
+	~falco_rule() = default;
 
 	std::size_t id;
 	std::string source;
@@ -41,4 +82,5 @@ struct falco_rule
 	std::set<std::string> tags;
 	std::set<std::string> exception_fields;
 	falco_common::priority_type priority;
+	std::shared_ptr<libsinsp::filter::ast::expr> condition;
 };

userspace/engine/filter_details_resolver.cpp

@@ -19,12 +19,23 @@ limitations under the License.
 
 using namespace libsinsp::filter;
 
+std::string get_field_name(const std::string& name, const std::string& arg)
+{
+	std::string fld = name;
+	if (!arg.empty())
+	{
+		fld += "[" + arg + "]";
+	}
+	return fld;
+}
+
 void filter_details::reset()
 {
 	fields.clear();
 	macros.clear();
 	operators.clear();
 	lists.clear();
+	evtnames.clear();
 }
 
 void filter_details_resolver::run(ast::expr* filter, filter_details& details)
@@ -35,6 +46,7 @@ void filter_details_resolver::run(ast::expr* filter, filter_details& details)
 
 void filter_details_resolver::visitor::visit(ast::and_expr* e)
 {
+	m_expect_macro = false;
 	for(size_t i = 0; i < e->children.size(); i++)
 	{
 		m_expect_macro = true;
@@ -45,6 +57,7 @@ void filter_details_resolver::visitor::visit(ast::and_expr* e)
 
 void filter_details_resolver::visitor::visit(ast::or_expr* e)
 {
+	m_expect_macro = false;
 	for(size_t i = 0; i < e->children.size(); i++)
 	{
 		m_expect_macro = true;
@@ -70,35 +83,50 @@ void filter_details_resolver::visitor::visit(ast::list_expr* e)
 			}
 		}
 	}
+	if (m_expect_evtname)
+	{
+		for(const auto& item : e->values)
+		{
+			if(m_details.known_lists.find(item) == m_details.known_lists.end())
+			{
+				m_details.evtnames.insert(item);
+			}
+		}
+	}
 }
 
 void filter_details_resolver::visitor::visit(ast::binary_check_expr* e)
 {
 	m_expect_macro = false;
-	m_details.fields.insert(e->field);
+	m_details.fields.insert(get_field_name(e->field, e->arg));
 	m_details.operators.insert(e->op);
 	m_expect_list = true;
+	m_expect_evtname = e->field == "evt.type" || e->field == "evt.asynctype";
 	e->value->accept(this);
+	m_expect_evtname = false;
 	m_expect_list = false;
 }
 
 void filter_details_resolver::visitor::visit(ast::unary_check_expr* e)
 {
 	m_expect_macro = false;
-	m_details.fields.insert(e->field);
+	m_details.fields.insert(get_field_name(e->field, e->arg));
 	m_details.operators.insert(e->op);
 }
 
 void filter_details_resolver::visitor::visit(ast::value_expr* e)
 {
-	if(m_expect_macro)
+	if (m_expect_macro)
 	{
-		auto it = m_details.known_macros.find(e->value);
-		if(it == m_details.known_macros.end())
+		if(m_details.known_macros.find(e->value) != m_details.known_macros.end())
 		{
-			return;
+			m_details.macros.insert(e->value);
 		}
-
-		m_details.macros.insert(e->value);
+		// todo(jasondellaluce): should we throw an error if we
+		// encounter an unknown macro?
+	}
+	else if (m_expect_evtname)
+	{
+		m_details.evtnames.insert(e->value);
 	}
 }

userspace/engine/filter_details_resolver.h

@@ -33,6 +33,7 @@ struct filter_details
 	std::unordered_set<std::string> macros;
 	std::unordered_set<std::string> operators;
 	std::unordered_set<std::string> lists;
+	std::unordered_set<std::string> evtnames;
 
 	void reset();
 };
@@ -59,7 +60,8 @@ class filter_details_resolver
 		visitor(filter_details& details) : 
 			m_details(details),
 			m_expect_list(false),
-			m_expect_macro(false) {}
+			m_expect_macro(false),
+			m_expect_evtname(false) {}
 		visitor(visitor&&) = default;
 		visitor& operator = (visitor&&) = default;
 		visitor(const visitor&) = delete;
@@ -76,5 +78,6 @@ class filter_details_resolver
 		filter_details& m_details;
 		bool m_expect_list;
 		bool m_expect_macro;
+		bool m_expect_evtname;
 	};
 };

userspace/engine/rule_loader.cpp

@@ -532,12 +532,12 @@ rule_loader::plugin_version_info::plugin_version_info(context &ctx)
 }
 
 rule_loader::list_info::list_info(context &ctx)
-	: ctx(ctx), used(false), index(0), visibility(0)
+	: ctx(ctx), index(0), visibility(0)
 {
 }
 
 rule_loader::macro_info::macro_info(context &ctx)
-	: ctx(ctx), cond_ctx(ctx), used(false), index(0), visibility(0)
+	: ctx(ctx), cond_ctx(ctx), index(0), visibility(0)
 {
 }
 

userspace/engine/rule_loader.h

@@ -273,8 +273,7 @@ namespace rule_loader
 			const indexed_vector<falco_source>& srcs,
 			const std::string& name)
 				: content(cont), sources(srcs), name(name),
-				  default_ruleset_id(0), replace_output_container_info(false),
-				  min_priority(falco_common::PRIORITY_DEBUG)
+				  output_extra(), replace_output_container_info(false)
 			{
 				res.reset(new result(name));
 			}
@@ -283,14 +282,15 @@ namespace rule_loader
 		configuration(const configuration&) = delete;
 		configuration& operator = (const configuration&) = delete;
 
+		// inputs
 		const std::string& content;
 		const indexed_vector<falco_source>& sources;
 		std::string name;
-		std::unique_ptr<result> res;
 		std::string output_extra;
-		uint16_t default_ruleset_id;
 		bool replace_output_container_info;
-		falco_common::priority_type min_priority;
+
+		// outputs
+		std::unique_ptr<result> res;
 	};
 
 	/*!
@@ -359,7 +359,6 @@ namespace rule_loader
 		list_info& operator = (const list_info&) = default;
 
 		context ctx;
-		bool used;
 		size_t index;
 		size_t visibility;
 		std::string name;
@@ -380,12 +379,10 @@ namespace rule_loader
 
 		context ctx;
 		context cond_ctx;
-		bool used;
 		size_t index;
 		size_t visibility;
 		std::string name;
 		std::string cond;
-		std::shared_ptr<libsinsp::filter::ast::expr> cond_ast;
 	};
 
 	/*!