Merge pull request #103 from draios/falco-engine

Falco engine

.gitignore

@@ -6,6 +6,8 @@ test/traces-negative
 test/traces-positive
 test/traces-info
 test/job-results
+test/.phoronix-test-suite
+test/results*.json.*
 
 userspace/falco/lua/re.lua
 userspace/falco/lua/lpeg.so

CMakeLists.txt

@@ -6,8 +6,8 @@ if(NOT DEFINED FALCO_VERSION)
 	set(FALCO_VERSION "0.1.1dev")
 endif()
 
-if(NOT DEFINED DIR_ETC)
-	set(DIR_ETC "/etc")
+if(NOT DEFINED FALCO_ETC_DIR)
+	set(FALCO_ETC_DIR "/etc")
 endif()
 
 if(NOT CMAKE_BUILD_TYPE)
@@ -39,6 +39,7 @@ set(PACKAGE_NAME "falco")
 set(PROBE_VERSION "${FALCO_VERSION}")
 set(PROBE_NAME "sysdig-probe")
 set(PROBE_DEVICE_NAME "sysdig")
+set(CMAKE_INSTALL_PREFIX /usr)
 
 set(CMD_MAKE make)
 
@@ -160,11 +161,12 @@ ExternalProject_Add(luajit
 		INSTALL_COMMAND "")
 
 set (LPEG_SRC "${PROJECT_BINARY_DIR}/lpeg-prefix/src/lpeg")
+set (LPEG_LIB "${PROJECT_BINARY_DIR}/lpeg-prefix/src/lpeg/build/lpeg.a")
 ExternalProject_Add(lpeg
                     DEPENDS luajit
 		    URL "http://s3.amazonaws.com/download.draios.com/dependencies/lpeg-1.0.0.tar.gz"
                     URL_MD5 "0aec64ccd13996202ad0c099e2877ece"
-                    BUILD_COMMAND LUA_INCLUDE=${LUAJIT_INCLUDE} "${PROJECT_SOURCE_DIR}/scripts/build-lpeg.sh"
+                    BUILD_COMMAND LUA_INCLUDE=${LUAJIT_INCLUDE} "${PROJECT_SOURCE_DIR}/scripts/build-lpeg.sh" "${LPEG_SRC}/build"
                     BUILD_IN_SOURCE 1
 		    CONFIGURE_COMMAND ""
                     INSTALL_COMMAND "")
@@ -188,17 +190,19 @@ ExternalProject_Add(lyaml
                     BUILD_COMMAND ${CMD_MAKE}
                     BUILD_IN_SOURCE 1
 		    CONFIGURE_COMMAND ./configure --enable-static LIBS=-L../../../libyaml-prefix/src/libyaml/src/.libs CFLAGS=-I../../../libyaml-prefix/src/libyaml/include CPPFLAGS=-I../../../libyaml-prefix/src/libyaml/include LUA_INCLUDE=-I../../../luajit-prefix/src/luajit/src LUA=../../../luajit-prefix/src/luajit/src/luajit
-                    INSTALL_COMMAND sh -c "cp -R ${PROJECT_BINARY_DIR}/lyaml-prefix/src/lyaml/lib/* ${PROJECT_SOURCE_DIR}/userspace/falco/lua")
+                    INSTALL_COMMAND sh -c "cp -R ${PROJECT_BINARY_DIR}/lyaml-prefix/src/lyaml/lib/* ${PROJECT_SOURCE_DIR}/userspace/engine/lua")
 
 install(FILES falco.yaml
-	DESTINATION "${DIR_ETC}")
+	DESTINATION "${FALCO_ETC_DIR}")
 
 add_subdirectory("${SYSDIG_DIR}/driver" "${PROJECT_BINARY_DIR}/driver")
 add_subdirectory("${SYSDIG_DIR}/userspace/libscap" "${PROJECT_BINARY_DIR}/userspace/libscap")
 add_subdirectory("${SYSDIG_DIR}/userspace/libsinsp" "${PROJECT_BINARY_DIR}/userspace/libsinsp")
 
-add_subdirectory(rules)
 add_subdirectory(scripts)
+set(FALCO_SINSP_LIBRARY sinsp)
+set(FALCO_SHARE_DIR share/falco)
+add_subdirectory(userspace/engine)
 add_subdirectory(userspace/falco)
 
 

rules/CMakeLists.txt

@@ -1,3 +1,13 @@
+if(NOT DEFINED FALCO_ETC_DIR)
+  set(FALCO_ETC_DIR "/etc")
+endif()
+
+if(DEFINED FALCO_COMPONENT)
+install(FILES falco_rules.yaml
+        COMPONENT "${FALCO_COMPONENT}"
+	DESTINATION "${FALCO_ETC_DIR}")
+else()
 install(FILES falco_rules.yaml
-	DESTINATION "${DIR_ETC}")
+	DESTINATION "${FALCO_ETC_DIR}")
+endif()
 

scripts/build-lpeg.sh

@@ -1,17 +1,29 @@
-#!/bin/sh
+#!/bin/bash
 
-gcc -O2 -fPIC -I$LUA_INCLUDE -c lpcap.c -o lpcap.o
-gcc -O2 -fPIC -I$LUA_INCLUDE -c lpcode.c -o lpcode.o
-gcc -O2 -fPIC -I$LUA_INCLUDE -c lpprint.c -o lpprint.o
-gcc -O2 -fPIC -I$LUA_INCLUDE -c lptree.c -o lptree.o
-gcc -O2 -fPIC -I$LUA_INCLUDE -c lpvm.c -o lpvm.o
+set -ex
+
+PREFIX=$1
+
+if [ -z $PREFIX ]; then
+    PREFIX=.
+fi
+
+mkdir -p $PREFIX
+
+gcc -O2 -fPIC -I$LUA_INCLUDE -c lpcap.c -o $PREFIX/lpcap.o
+gcc -O2 -fPIC -I$LUA_INCLUDE -c lpcode.c -o $PREFIX/lpcode.o
+gcc -O2 -fPIC -I$LUA_INCLUDE -c lpprint.c -o $PREFIX/lpprint.o
+gcc -O2 -fPIC -I$LUA_INCLUDE -c lptree.c -o $PREFIX/lptree.o
+gcc -O2 -fPIC -I$LUA_INCLUDE -c lpvm.c -o $PREFIX/lpvm.o
 
 
 # For building lpeg.so, which we don't need now that we're statically linking lpeg.a into falco
 #gcc -shared -o lpeg.so -L/usr/local/lib lpcap.o lpcode.o lpprint.o lptree.o lpvm.o
 #gcc -shared -o lpeg.so -L/usr/local/lib lpcap.o lpcode.o lpprint.o lptree.o lpvm.o
 
+pushd $PREFIX
 /usr/bin/ar cr lpeg.a lpcap.o lpcode.o lpprint.o lptree.o lpvm.o
 /usr/bin/ranlib lpeg.a
+popd
 
 chmod ug+w re.lua

test/falco_test.py

@@ -26,8 +26,28 @@ def setUp(self):
         self.json_output = self.params.get('json_output', '*', default=False)
         self.rules_file = self.params.get('rules_file', '*', default=os.path.join(self.basedir, '../rules/falco_rules.yaml'))
 
-        if not os.path.isabs(self.rules_file):
-            self.rules_file = os.path.join(self.basedir, self.rules_file)
+        if not isinstance(self.rules_file, list):
+            self.rules_file = [self.rules_file]
+
+        self.rules_args = ""
+
+        for file in self.rules_file:
+            if not os.path.isabs(file):
+                file = os.path.join(self.basedir, file)
+            self.rules_args = self.rules_args + "-r " + file + " "
+
+        self.disabled_rules = self.params.get('disabled_rules', '*', default='')
+
+        if self.disabled_rules == '':
+            self.disabled_rules = []
+
+        if not isinstance(self.disabled_rules, list):
+            self.disabled_rules = [self.disabled_rules]
+
+        self.disabled_args = ""
+
+        for rule in self.disabled_rules:
+            self.disabled_args = self.disabled_args + "-D " + rule + " "
 
         self.rules_warning = self.params.get('rules_warning', '*', default=False)
         if self.rules_warning == False:
@@ -49,6 +69,9 @@ def setUp(self):
         if self.should_detect:
             self.detect_level = self.params.get('detect_level', '*')
 
+            if not isinstance(self.detect_level, list):
+                self.detect_level = [self.detect_level]
+
         # Doing this in 2 steps instead of simply using
         # module_is_loaded to avoid logging lsmod output to the log.
         lsmod_output = process.system_output("lsmod", verbose=False)
@@ -105,16 +128,17 @@ def check_detections(self, res):
             if events_detected == 0:
                 self.fail("Detected {} events when should have detected > 0".format(events_detected))
 
-            level_line = '{}: (\d+)'.format(self.detect_level)
-            match = re.search(level_line, res.stdout)
+            for level in self.detect_level:
+                level_line = '(?i){}: (\d+)'.format(level)
+                match = re.search(level_line, res.stdout)
 
-            if match is None:
-                self.fail("Could not find a line '{}: <count>' in falco output".format(self.detect_level))
+                if match is None:
+                    self.fail("Could not find a line '{}: <count>' in falco output".format(level))
 
-            events_detected = int(match.group(1))
+                    events_detected = int(match.group(1))
 
-            if not events_detected > 0:
-                self.fail("Detected {} events at level {} when should have detected > 0".format(events_detected, self.detect_level))
+                    if not events_detected > 0:
+                        self.fail("Detected {} events at level {} when should have detected > 0".format(events_detected, level))
 
     def check_json_output(self, res):
         if self.json_output:
@@ -131,8 +155,8 @@ def test(self):
         self.log.info("Trace file %s", self.trace_file)
 
         # Run the provided trace file though falco
-        cmd = '{}/userspace/falco/falco -r {} -c {}/../falco.yaml -e {} -o json_output={} -v'.format(
-            self.falcodir, self.rules_file, self.falcodir, self.trace_file, self.json_output)
+        cmd = '{}/userspace/falco/falco {} {} -c {}/../falco.yaml -e {} -o json_output={} -v'.format(
+            self.falcodir, self.rules_args, self.disabled_args, self.falcodir, self.trace_file, self.json_output)
 
         self.falco_proc = process.SubProcess(cmd)
 

test/falco_tests.yaml.in

@@ -1,13 +1,13 @@
 trace_files: !mux
   builtin_rules_no_warnings:
     detect: False
-    trace_file: empty.scap
+    trace_file: trace_files/empty.scap
     rules_warning: False
 
   test_warnings:
     detect: False
-    trace_file: empty.scap
-    rules_file: falco_rules_warnings.yaml
+    trace_file: trace_files/empty.scap
+    rules_file: rules/falco_rules_warnings.yaml
     rules_warning:
       - no_evttype
       - evttype_not_equals
@@ -60,3 +60,48 @@ trace_files: !mux
       - repeated_evttypes_with_in: [open]
       - repeated_evttypes_with_separate_in: [open]
       - repeated_evttypes_with_mix: [open]
+
+  multiple_rules_first_empty:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/empty_rules.yaml
+      - rules/single_rule.yaml
+    trace_file: trace_files/cat_write.scap
+
+  multiple_rules_last_empty:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/single_rule.yaml
+      - rules/empty_rules.yaml
+    trace_file: trace_files/cat_write.scap
+
+  multiple_rules:
+    detect: True
+    detect_level:
+      - WARNING
+      - INFO
+      - ERROR
+    rules_file:
+      - rules/single_rule.yaml
+      - rules/double_rule.yaml
+    trace_file: trace_files/cat_write.scap
+
+  disabled_rules:
+    detect: False
+    rules_file:
+      - rules/empty_rules.yaml
+      - rules/single_rule.yaml
+    disabled_rules:
+      - open_from_cat
+    trace_file: trace_files/cat_write.scap
+
+  disabled_rules_using_regex:
+    detect: False
+    rules_file:
+      - rules/empty_rules.yaml
+      - rules/single_rule.yaml
+    disabled_rules:
+      - "open.*"
+    trace_file: trace_files/cat_write.scap

test/rules/double_rule.yaml

@@ -0,0 +1,13 @@
+# This ruleset depends on the is_cat macro defined in single_rule.yaml
+
+- rule: exec_from_cat
+  desc: A process named cat does execve
+  condition: evt.type=execve and is_cat
+  output: "An exec was seen (command=%proc.cmdline)"
+  priority: ERROR
+
+- rule: access_from_cat
+  desc: A process named cat does an access
+  condition: evt.type=access and is_cat
+  output: "An access was seen (command=%proc.cmdline)"
+  priority: INFO

test/rules/single_rule.yaml

@@ -0,0 +1,8 @@
+- macro: is_cat
+  condition: proc.name=cat
+
+- rule: open_from_cat
+  desc: A process named cat does an open
+  condition: evt.type=open and is_cat
+  output: "An open was seen (command=%proc.cmdline)"
+  priority: WARNING

test/run_regression_tests.sh

@@ -38,12 +38,12 @@ EOF
 function prepare_multiplex_file() {
     cp $SCRIPTDIR/falco_tests.yaml.in $MULT_FILE
 
-    prepare_multiplex_fileset traces-positive True Warning False
-    prepare_multiplex_fileset traces-negative False Warning True
-    prepare_multiplex_fileset traces-info True Informational False
+    prepare_multiplex_fileset traces-positive True WARNING False
+    prepare_multiplex_fileset traces-negative False WARNING True
+    prepare_multiplex_fileset traces-info True INFO False
 
-    prepare_multiplex_fileset traces-positive True Warning True
-    prepare_multiplex_fileset traces-info True Informational True
+    prepare_multiplex_fileset traces-positive True WARNING True
+    prepare_multiplex_fileset traces-info True INFO True
 
     echo "Contents of $MULT_FILE:"
     cat $MULT_FILE

userspace/engine/CMakeLists.txt

@@ -0,0 +1,31 @@
+include_directories("${PROJECT_SOURCE_DIR}/../sysdig/userspace/libsinsp/third-party/jsoncpp")
+include_directories("${PROJECT_SOURCE_DIR}/../sysdig/userspace/libscap")
+include_directories("${PROJECT_SOURCE_DIR}/../sysdig/userspace/libsinsp")
+include_directories("${PROJECT_BINARY_DIR}/userspace/engine")
+include_directories("${LUAJIT_INCLUDE}")
+
+add_library(falco_engine STATIC rules.cpp falco_common.cpp falco_engine.cpp)
+
+target_include_directories(falco_engine PUBLIC
+  "${LUAJIT_INCLUDE}")
+
+target_link_libraries(falco_engine
+  "${FALCO_SINSP_LIBRARY}"
+  "${LPEG_LIB}"
+  "${LYAML_LIB}"
+  "${LIBYAML_LIB}")
+
+configure_file(config_falco_engine.h.in config_falco_engine.h)
+
+if(DEFINED FALCO_COMPONENT)
+install(DIRECTORY lua
+	DESTINATION "${FALCO_SHARE_DIR}"
+	COMPONENT "${FALCO_COMPONENT}"
+	FILES_MATCHING PATTERN *.lua)
+else()
+install(DIRECTORY lua
+	DESTINATION "${FALCO_SHARE_DIR}"
+	FILES_MATCHING PATTERN *.lua)
+endif()
+
+add_subdirectory("${PROJECT_SOURCE_DIR}/../falco/rules" "${PROJECT_BINARY_DIR}/rules")

userspace/engine/config_falco_engine.h.in

@@ -0,0 +1,4 @@
+#pragma once
+
+#define FALCO_ENGINE_LUA_DIR "${CMAKE_INSTALL_PREFIX}/${FALCO_SHARE_DIR}/lua/"
+#define FALCO_ENGINE_SOURCE_LUA_DIR "${PROJECT_SOURCE_DIR}/../falco/userspace/engine/lua/"