Add support for event-specific filters.

Instead of combining all rules into one huge filter expression and
giving it to the inspector, keep each filter expression separate and
annotate it with the events for which the rule applies.

This uses the capabilties in draios/sysdig#627
to have multiple sets of event-specific filters.

Within the compiler, a new pass over the ast get_evttypes looks for
evt.type clauses, converts the evt.type as a string to any event type
ids for which it may apply, and passes that back with the compiled
rule.

As rule conditions may refer to evt.types in negative
contexts (i.e. evt.type != XXX, or not evt.type = XXX), this pass
prefers rules that list event type checks at the beginning of
conditions, and allows other rules with a warning.

When traversing the ast looking for evt.type checks, once any "!=" or
"not ..." is seen, no other evt.type checks are "allowed". If one
is found, the rule is considered ambiguous wrt event types. In this
case, a warning is printed and the rule is associated with a catchall
set that runs for all event types.

Also, instead of rejecting rules with no event type check, print a
warning and associate it with the catchall set.

In the rule loader, create a new global events that maps each event as a
string to the list of event ids for which it may apply. Also there's a
callback add_filter from lua which is called for each rule. In turn, it
passes the rule and list of event ids to the inspector using
add_evttype_filter().

Also, with -v (verbose) also print the exact set of events found for
each event type. This is used by a upcoming change to the set of unit
tests.

userspace/falco/falco.cpp

@@ -399,11 +399,11 @@ int falco_init(int argc, char **argv)
 		falco_fields::init(inspector, ls);
 
 		falco_logger::init(ls);
+		falco_rules::init(ls);
 
 
 		inspector->set_drop_event_flags(EF_DROP_FALCO);
 		rules->load_rules(config.m_rules_filename, verbose);
-		inspector->set_filter(rules->get_filter());
 		falco_logger::log(LOG_INFO, "Parsed rules from file " + config.m_rules_filename + "\n");
 
 		if (describe_all_rules)

userspace/falco/lua/compiler.lua

@@ -160,7 +160,103 @@ function check_for_ignored_syscalls_events(ast, filter_type, source)
       end
    end
 
-   parser.traverse_ast(ast, "BinaryRelOp", cb)
+   parser.traverse_ast(ast, {BinaryRelOp=1}, cb)
+end
+
+-- Examine the ast and find the event types for which the rule should
+-- run. All evt.type references are added as event types up until the
+-- first "!=" binary operator or unary not operator. If no event type
+-- checks are found afterward in the rule, the rule is considered
+-- optimized and is associated with the event type(s).
+--
+-- Otherwise, the rule is associated with a 'catchall' category and is
+-- run for all event types. (Also, a warning is printed).
+--
+
+function get_evttypes(name, ast, source)
+
+   local evttypes = {}
+   local evtnames = {}
+   local found_event = false
+   local found_not = false
+   local found_event_after_not = false
+
+   function cb(node)
+     if node.type == "UnaryBoolOp" then
+	if node.operator == "not" then
+	   found_not = true
+	end
+     else
+	 if node.operator == "!=" then
+	    found_not = true
+	 end
+	 if node.left.type == "FieldName" and node.left.value == "evt.type" then
+	    found_event = true
+	    if found_not then
+	       found_event_after_not = true
+	    end
+	    if node.operator == "in" then
+	       for i, v in ipairs(node.right.elements) do
+		  if v.type == "BareString" then
+		     evtnames[v.value] = 1
+		     for id in string.gmatch(events[v.value], "%S+") do
+			evttypes[id] = 1
+		     end
+		  end
+	       end
+	    else
+	       if node.right.type == "BareString" then
+		  evtnames[node.right.value] = 1
+		  for id in string.gmatch(events[node.right.value], "%S+") do
+		     evttypes[id] = 1
+		  end
+	       end
+	    end
+	 end
+      end
+   end
+
+   parser.traverse_ast(ast.filter.value, {BinaryRelOp=1, UnaryBoolOp=1} , cb)
+
+   if not found_event then
+      io.stderr:write("Rule "..name..": warning (no-evttype):\n")
+      io.stderr:write(source.."\n")
+      io.stderr:write("         did not contain any evt.type restriction, meaning it will run for all event types.\n")
+      io.stderr:write("         This has a significant performance penalty. Consider adding an evt.type restriction if possible.\n")
+      evttypes = {}
+      evtnames = {}
+   end
+
+   if found_event_after_not then
+      io.stderr:write("Rule "..name..": warning (trailing-evttype):\n")
+      io.stderr:write(source.."\n")
+      io.stderr:write("         does not have all evt.type restrictions at the beginning of the condition,\n")
+      io.stderr:write("         or uses a negative match (i.e. \"not\"/\"!=\") for some evt.type restriction.\n")
+      io.stderr:write("         This has a performance penalty, as the rule can not be limited to specific event types.\n")
+      io.stderr:write("         Consider moving all evt.type restrictions to the beginning of the rule and/or\n")
+      io.stderr:write("         replacing negative matches with positive matches if possible.\n")
+      evttypes = {}
+      evtnames = {}
+   end
+
+   evtnames_only = {}
+   local num_evtnames = 0
+   for name, dummy in pairs(evtnames) do
+      table.insert(evtnames_only, name)
+      num_evtnames = num_evtnames + 1
+   end
+
+   if num_evtnames == 0 then
+      table.insert(evtnames_only, "all")
+   end
+
+   table.sort(evtnames_only)
+
+   if compiler.verbose then
+      io.stderr:write("Event types for rule "..name..": "..table.concat(evtnames_only, ",").."\n")
+   end
+
+   return evttypes
 end
 
 function compiler.compile_macro(line, list_defs)
@@ -186,7 +282,7 @@ end
 --[[
    Parses a single filter, then expands macros using passed-in table of definitions. Returns resulting AST.
 --]]
-function compiler.compile_filter(source, macro_defs, list_defs)
+function compiler.compile_filter(name, source, macro_defs, list_defs)
 
    for name, items in pairs(list_defs) do
       source = string.gsub(source, name, table.concat(items, ", "))
@@ -213,7 +309,9 @@ function compiler.compile_filter(source, macro_defs, list_defs)
       error("Unexpected top-level AST type: "..ast.type)
    end
 
-   return ast
+   evttypes = get_evttypes(name, ast, source)
+
+   return ast, evttypes
 end
 
 

userspace/falco/lua/parser.lua

@@ -303,33 +303,33 @@ parser.print_ast = print_ast
 -- have the signature:
 --     cb(ast_node, ctx)
 -- ctx is optional.
-function traverse_ast(ast, node_type, cb, ctx)
+function traverse_ast(ast, node_types, cb, ctx)
    local t = ast.type
 
-   if t == node_type then
+   if node_types[t] ~= nil then
       cb(ast, ctx)
    end
 
    if t == "Rule" then
-      traverse_ast(ast.filter, node_type, cb, ctx)
+      traverse_ast(ast.filter, node_types, cb, ctx)
 
    elseif t == "Filter" then
-      traverse_ast(ast.value, node_type, cb, ctx)
+      traverse_ast(ast.value, node_types, cb, ctx)
 
    elseif t == "BinaryBoolOp" or t == "BinaryRelOp" then
-      traverse_ast(ast.left, node_type, cb, ctx)
-      traverse_ast(ast.right, node_type, cb, ctx)
+      traverse_ast(ast.left, node_types, cb, ctx)
+      traverse_ast(ast.right, node_types, cb, ctx)
 
    elseif t == "UnaryRelOp" or t == "UnaryBoolOp" then
-      traverse_ast(ast.argument, node_type, cb, ctx)
+      traverse_ast(ast.argument, node_types, cb, ctx)
 
    elseif t == "List" then
       for i, v in ipairs(ast.elements) do
-         traverse_ast(v, node_type, cb, ctx)
+         traverse_ast(v, node_types, cb, ctx)
       end
 
    elseif t == "MacroDef" then
-      traverse_ast(ast.value, node_type, cb, ctx)
+      traverse_ast(ast.value, node_types, cb, ctx)
 
    elseif t == "FieldName" or t == "Number" or t == "String" or t == "BareString" or t == "Macro" then
       -- do nothing, no traversal needed

userspace/falco/lua/rule_loader.lua

@@ -117,7 +117,7 @@ end
 -- to a rule.
 local state = {macros={}, lists={}, filter_ast=nil, rules_by_name={}, n_rules=0, rules_by_idx={}}
 
-function load_rules(filename, verbose)
+function load_rules(filename, rules_mgr, verbose)
 
    compiler.set_verbose(verbose)
 
@@ -171,7 +171,8 @@ function load_rules(filename, verbose)
 	 v['level'] = priority(v['priority'])
 	 state.rules_by_name[v['rule']] = v
 
-	 local filter_ast = compiler.compile_filter(v['condition'], state.macros, state.lists)
+	 local filter_ast, evttypes = compiler.compile_filter(v['rule'], v['condition'],
+							      state.macros, state.lists)
 
 	 if (filter_ast.type == "Rule") then
 	    state.n_rules = state.n_rules + 1
@@ -185,6 +186,11 @@ function load_rules(filename, verbose)
 	    -- event.
 	    mark_relational_nodes(filter_ast.filter.value, state.n_rules)
 
+	    install_filter(filter_ast.filter.value)
+
+	    -- Pass the filter and event types back up
+	    falco_rules.add_filter(rules_mgr, evttypes)
+
 	    -- Rule ASTs are merged together into one big AST, with "OR" between each
 	    -- rule.
 	    if (state.filter_ast == nil) then
@@ -198,7 +204,6 @@ function load_rules(filename, verbose)
       end
    end
 
-   install_filter(state.filter_ast)
    io.flush()
 end
 

userspace/falco/rules.cpp

@@ -1,11 +1,17 @@
 #include "rules.h"
+#include "logger.h"
 
 extern "C" {
 #include "lua.h"
 #include "lualib.h"
 #include "lauxlib.h"
 }
 
+const static struct luaL_reg ll_falco_rules [] =
+{
+	{"add_filter", &falco_rules::add_filter},
+	{NULL,NULL}
+};
 
 falco_rules::falco_rules(sinsp* inspector, lua_State *ls, string lua_main_filename)
 {
@@ -17,6 +23,48 @@ falco_rules::falco_rules(sinsp* inspector, lua_State *ls, string lua_main_filena
 	load_compiler(lua_main_filename);
 }
 
+void falco_rules::init(lua_State *ls)
+{
+	luaL_openlib(ls, "falco_rules", ll_falco_rules, 0);
+}
+
+int falco_rules::add_filter(lua_State *ls)
+{
+	if (! lua_islightuserdata(ls, -2) ||
+	    ! lua_istable(ls, -1))
+	{
+		falco_logger::log(LOG_ERR, "Invalid arguments passed to add_filter()\n");
+		throw sinsp_exception("add_filter error");
+	}
+
+	falco_rules *rules = (falco_rules *) lua_topointer(ls, -2);
+
+	list<uint16_t> evttypes;
+
+	lua_pushnil(ls);  /* first key */
+	while (lua_next(ls, -2) != 0) {
+                // key is at index -2, value is at index
+                // -1. We want the keys.
+		evttypes.push_back(luaL_checknumber(ls, -2));
+
+		// Remove value, keep key for next iteration
+		lua_pop(ls, 1);
+	}
+
+	rules->add_filter(evttypes);
+
+	return 0;
+}
+
+void falco_rules::add_filter(list<uint16_t> &evttypes)
+{
+	// While the current rule was being parsed, a sinsp_filter
+	// object was being populated by lua_parser. Grab that filter
+	// and pass it to the inspector.
+	sinsp_filter *filter = m_lua_parser->get_filter(true);
+
+	m_inspector->add_evttype_filter(evttypes, filter);
+}
 
 void falco_rules::load_compiler(string lua_main_filename)
 {
@@ -45,13 +93,42 @@ void falco_rules::load_rules(string rules_filename, bool verbose)
 	lua_getglobal(m_ls, m_lua_load_rules.c_str());
 	if(lua_isfunction(m_ls, -1))
 	{
+		// Create a table containing all events, so they can
+		// be mapped to event ids.
+		sinsp_evttables* einfo = m_inspector->get_event_info_tables();
+		const struct ppm_event_info* etable = einfo->m_event_info;
+		const struct ppm_syscall_desc* stable = einfo->m_syscall_info_table;
+
+		map<string,string> events_by_name;
+		for(uint32_t j = 0; j < PPM_EVENT_MAX; j++)
+		{
+			auto it = events_by_name.find(etable[j].name);
+
+			if (it == events_by_name.end()) {
+				events_by_name[etable[j].name] = to_string(j);
+			} else {
+				string cur = it->second;
+				cur += " ";
+				cur += to_string(j);
+				events_by_name[etable[j].name] = cur;
+			}
+		}
+
+		lua_newtable(m_ls);
+
+		for( auto kv : events_by_name)
+		{
+			lua_pushstring(m_ls, kv.first.c_str());
+			lua_pushstring(m_ls, kv.second.c_str());
+			lua_settable(m_ls, -3);
+		}
+
+		lua_setglobal(m_ls, m_lua_events.c_str());
+
 		// Create a table containing the syscalls/events that
 		// are ignored by the kernel module. load_rules will
 		// return an error if any rule references one of these
 		// syscalls/events.
-		sinsp_evttables* einfo = m_inspector->get_event_info_tables();
-		const struct ppm_event_info* etable = einfo->m_event_info;
-		const struct ppm_syscall_desc* stable = einfo->m_syscall_info_table;
 
 		lua_newtable(m_ls);
 
@@ -82,8 +159,9 @@ void falco_rules::load_rules(string rules_filename, bool verbose)
 		lua_setglobal(m_ls, m_lua_ignored_syscalls.c_str());
 
 		lua_pushstring(m_ls, rules_filename.c_str());
+		lua_pushlightuserdata(m_ls, this);
 		lua_pushboolean(m_ls, (verbose ? 1 : 0));
-		if(lua_pcall(m_ls, 2, 0, 0) != 0)
+		if(lua_pcall(m_ls, 3, 0, 0) != 0)
 		{
 			const char* lerr = lua_tostring(m_ls, -1);
 			string err = "Error loading rules:" + string(lerr);

userspace/falco/rules.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <list>
+
 #include "sinsp.h"
 #include "lua_parser.h"
 
@@ -12,16 +14,22 @@ class falco_rules
 	void describe_rule(string *rule);
 	sinsp_filter* get_filter();
 
+	static void init(lua_State *ls);
+	static int add_filter(lua_State *ls);
+
  private:
 	void load_compiler(string lua_main_filename);
 
+	void add_filter(list<uint16_t> &evttypes);
+
 	lua_parser* m_lua_parser;
 	sinsp* m_inspector;
 	lua_State* m_ls;
 
 	string m_lua_load_rules = "load_rules";
 	string m_lua_ignored_syscalls = "ignored_syscalls";
 	string m_lua_ignored_events = "ignored_events";
+	string m_lua_events = "events";
 	string m_lua_on_event = "on_event";
 	string m_lua_describe_rule = "describe_rule";
 };