Merge pull request #83 from draios/add-correctness-tests

Add correctness tests

.gitignore

@@ -1,4 +1,9 @@
 /build*
+*~
+test/falco_test.pyc
+test/falco_tests.yaml
+test/traces-negative
+test/traces-positive
 
 userspace/falco/lua/re.lua
 userspace/falco/lua/lpeg.so

.travis.yml

@@ -9,6 +9,14 @@ install:
     - sudo apt-get --force-yes install g++-4.8
     - sudo apt-get install rpm linux-headers-$(uname -r)
     - git clone https://github.com/draios/sysdig.git ../sysdig
+    - sudo apt-get install -y python-pip libvirt-dev jq
+    - cd ..
+    - curl -Lo avocado-36.0-tar.gz https://github.com/avocado-framework/avocado/archive/36.0lts.tar.gz
+    - tar -zxvf avocado-36.0-tar.gz
+    - cd avocado-36.0lts
+    - sudo pip install -r requirements-travis.txt
+    - sudo python setup.py install
+    - cd ../falco
 before_script:
     - export KERNELDIR=/lib/modules/$(ls /lib/modules | sort | head -1)/build
 script:
@@ -28,7 +36,7 @@ script:
     - make VERBOSE=1
     - make package
     - cd ..
-    - sudo test/falco_trace_regression.sh build/userspace/falco/falco
+    - sudo test/run_regression_tests.sh
 notifications:
   webhooks:
     urls:

test/falco_test.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+
+import os
+import re
+
+from avocado import Test
+from avocado.utils import process
+from avocado.utils import linux_modules
+
+class FalcoTest(Test):
+
+    def setUp(self):
+        """
+        Load the sysdig kernel module if not already loaded.
+        """
+        self.falcodir = self.params.get('falcodir', '/', default=os.path.join(self.basedir, '../build'))
+
+        self.should_detect = self.params.get('detect', '*')
+        self.trace_file = self.params.get('trace_file', '*')
+
+        # Doing this in 2 steps instead of simply using
+        # module_is_loaded to avoid logging lsmod output to the log.
+        lsmod_output = process.system_output("lsmod", verbose=False)
+
+        if linux_modules.parse_lsmod_for_module(lsmod_output, 'sysdig_probe') == {}:
+            self.log.debug("Loading sysdig kernel module")
+            process.run('sudo insmod {}/driver/sysdig-probe.ko'.format(self.falcodir))
+
+        self.str_variant = self.trace_file
+
+    def test(self):
+        self.log.info("Trace file %s", self.trace_file)
+
+        # Run the provided trace file though falco
+        cmd = '{}/userspace/falco/falco -r {}/../rules/falco_rules.yaml -c {}/../falco.yaml -e {}'.format(
+            self.falcodir, self.falcodir, self.falcodir, self.trace_file)
+
+        self.falco_proc = process.SubProcess(cmd)
+
+        res = self.falco_proc.run(timeout=60, sig=9)
+
+        if res.exit_status != 0:
+            self.error("Falco command \"{}\" exited with non-zero return value {}".format(
+                cmd, res.exit_status))
+
+        # Get the number of events detected.
+        res = re.search('Events detected: (\d+)', res.stdout)
+        if res is None:
+            self.fail("Could not find a line 'Events detected: <count>' in falco output")
+
+        events_detected = int(res.group(1))
+
+        if not self.should_detect and events_detected > 0:
+            self.fail("Detected {} events when should have detected none".format(events_detected))
+
+        if self.should_detect and events_detected == 0:
+            self.fail("Detected {} events when should have detected > 0".format(events_detected))
+
+        pass
+
+
+if __name__ == "__main__":
+    main()

test/run_regression_tests.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+SCRIPT=$(readlink -f $0)
+SCRIPTDIR=$(dirname $SCRIPT)
+MULT_FILE=$SCRIPTDIR/falco_tests.yaml
+
+function download_trace_files() {
+    for TRACE in traces-positive traces-negative ; do
+	curl -so $SCRIPTDIR/$TRACE.zip https://s3.amazonaws.com/download.draios.com/falco-tests/$TRACE.zip &&
+	unzip -d $SCRIPTDIR $SCRIPTDIR/$TRACE.zip &&
+	rm -rf $SCRIPTDIR/$TRACE.zip
+    done
+}
+
+function prepare_multiplex_file() {
+    echo "trace_files: !mux" > $MULT_FILE
+
+    for trace in $SCRIPTDIR/traces-positive/*.scap ; do
+	[ -e "$trace" ] || continue
+	NAME=`basename $trace .scap`
+	cat << EOF >> $MULT_FILE
+  $NAME:
+    detect: True
+    trace_file: $trace
+EOF
+    done
+
+    for trace in $SCRIPTDIR/traces-negative/*.scap ; do
+	[ -e "$trace" ] || continue
+	NAME=`basename $trace .scap`
+	cat << EOF >> $MULT_FILE
+  $NAME:
+    detect: False
+    trace_file: $trace
+EOF
+    done
+
+    echo "Contents of $MULT_FILE:"
+    cat $MULT_FILE
+}
+
+function run_tests() {
+    CMD="avocado run --multiplex $MULT_FILE --job-results-dir $SCRIPTDIR/job-results -- $SCRIPTDIR/falco_test.py"
+    echo "Running: $CMD"
+    $CMD
+    TEST_RC=$?
+}
+
+
+function print_test_failure_details() {
+    echo "Showing full job logs for any tests that failed:"
+    jq '.tests[] | select(.status != "PASS") | .logfile' $SCRIPTDIR/job-results/latest/results.json  | xargs cat
+}
+
+download_trace_files
+prepare_multiplex_file
+run_tests
+if [ $TEST_RC -ne 0 ]; then
+   print_test_failure_details
+fi
+
+exit $TEST_RC

test/utils/run_sysdig.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# Run sysdig excluding all events that aren't used by falco and also
+# excluding other high-volume events that aren't essential. This
+# results in smaller trace files.
+
+# The remaining arguments are taken from the command line.
+
+exec sudo sysdig not evt.type in '(mprotect,brk,mq_timedreceive,mq_receive,mq_timedsend,mq_send,getrusage,procinfo,rt_sigprocmask,rt_sigaction,ioctl,clock_getres,clock_gettime,clock_nanosleep,clock_settime,close,epoll_create,epoll_create1,epoll_ctl,epoll_pwait,epoll_wait,eventfd,fcntl,fcntl64,fstat,fstat64,fstatat64,fstatfs,fstatfs64,futex,getitimer,gettimeofday,ioprio_get,ioprio_set,llseek,lseek,lstat,lstat64,mmap,mmap2,munmap,nanosleep,poll,ppoll,pread,pread64,preadv,procinfo,pselect6,pwrite,pwrite64,pwritev,read,readv,recv,recvfrom,recvmmsg,recvmsg,sched_yield,select,send,sendfile,sendfile64,sendmmsg,sendmsg,sendto,setitimer,settimeofday,shutdown,splice,stat,stat64,statfs,statfs64,switch,tee,timer_create,timer_delete,timerfd_create,timerfd_gettime,timerfd_settime,timer_getoverrun,timer_gettime,timer_settime,wait4,write,writev) and user.name!=ec2-user' $@

userspace/falco/falco.cpp

@@ -28,6 +28,14 @@ extern "C" {
 #include "utils.h"
 #include <yaml-cpp/yaml.h>
 
+bool g_terminate = false;
+//
+// Helper functions
+//
+static void signal_callback(int signal)
+{
+	g_terminate = true;
+}
 
 //
 // Program help
@@ -67,6 +75,7 @@ static void display_fatal_err(const string &msg, bool daemon)
 
 string lua_on_event = "on_event";
 string lua_add_output = "add_output";
+string lua_print_stats = "print_stats";
 
 // Splitting into key=value or key.subkey=value will be handled by configuration class.
 std::list<string> cmdline_options;
@@ -90,7 +99,11 @@ void do_inspect(sinsp* inspector,
 
 		res = inspector->next(&ev);
 
-		if(res == SCAP_TIMEOUT)
+		if (g_terminate)
+		{
+			break;
+		}
+		else if(res == SCAP_TIMEOUT)
 		{
 			continue;
 		}
@@ -199,6 +212,26 @@ void add_output(lua_State *ls, output_config oc)
 
 }
 
+// Print statistics on the the rules that triggered
+void print_stats(lua_State *ls)
+{
+	lua_getglobal(ls, lua_print_stats.c_str());
+
+	if(lua_isfunction(ls, -1))
+	{
+		if(lua_pcall(ls, 0, 0, 0) != 0)
+		{
+			const char* lerr = lua_tostring(ls, -1);
+			string err = "Error invoking function print_stats: " + string(lerr);
+			throw sinsp_exception(err);
+		}
+	}
+	else
+	{
+		throw sinsp_exception("No function " + lua_print_stats + " found in lua rule loader module");
+	}
+
+}
 
 //
 // ARGUMENT PARSING AND PROGRAM SETUP
@@ -398,6 +431,20 @@ int falco_init(int argc, char **argv)
 			add_output(ls, *it);
 		}
 
+		if(signal(SIGINT, signal_callback) == SIG_ERR)
+		{
+			fprintf(stderr, "An error occurred while setting SIGINT signal handler.\n");
+			result = EXIT_FAILURE;
+			goto exit;
+		}
+
+		if(signal(SIGTERM, signal_callback) == SIG_ERR)
+		{
+			fprintf(stderr, "An error occurred while setting SIGTERM signal handler.\n");
+			result = EXIT_FAILURE;
+			goto exit;
+		}
+
 		if (scap_filename.size())
 		{
 			inspector->open(scap_filename);
@@ -406,7 +453,7 @@ int falco_init(int argc, char **argv)
 		{
 			try
 			{
-				inspector->open();
+				inspector->open(200);
 			}
 			catch(sinsp_exception e)
 			{
@@ -478,6 +525,8 @@ int falco_init(int argc, char **argv)
 			   ls);
 
 		inspector->close();
+
+		print_stats(ls);
 	}
 	catch(sinsp_exception& e)
 	{

userspace/falco/lua/output.lua

@@ -2,6 +2,8 @@ local mod = {}
 
 levels = {"Emergency", "Alert", "Critical", "Error", "Warning", "Notice", "Informational", "Debug"}
 
+mod.levels = levels
+
 local outputs = {}
 
 function mod.stdout(evt, level, format)

userspace/falco/lua/rule_loader.lua

@@ -230,12 +230,51 @@ function describe_rule(name)
    end
 end
 
+local rule_output_counts = {total=0, by_level={}, by_name={}}
+
+for idx, level in ipairs(output.levels) do
+   rule_output_counts[level] = 0
+end
+
 function on_event(evt_, rule_id)
 
    if state.rules_by_idx[rule_id] == nil then
       error ("rule_loader.on_event(): event with invalid rule_id: ", rule_id)
    end
 
-   output.event(evt_, state.rules_by_idx[rule_id].level, state.rules_by_idx[rule_id].output)
+   rule_output_counts.total = rule_output_counts.total + 1
+   local rule = state.rules_by_idx[rule_id]
+
+   if rule_output_counts.by_level[rule.level] == nil then
+      rule_output_counts.by_level[rule.level] = 1
+   else
+      rule_output_counts.by_level[rule.level] = rule_output_counts.by_level[rule.level] + 1
+   end
+
+   if rule_output_counts.by_name[rule.rule] == nil then
+      rule_output_counts.by_name[rule.rule] = 1
+   else
+      rule_output_counts.by_name[rule.rule] = rule_output_counts.by_name[rule.rule] + 1
+   end
+
+   output.event(evt_, rule.level, rule.output)
 end
 
+function print_stats()
+   print("Events detected: "..rule_output_counts.total)
+   print("Rule counts by severity:")
+   for idx, level in ipairs(output.levels) do
+      -- To keep the output concise, we only print 0 counts for error, warning, and info levels
+      if rule_output_counts[level] > 0 or level == "Error" or level == "Warning" or level == "Informational" then
+	 print ("   "..level..": "..rule_output_counts[level])
+      end
+   end
+
+   print("Triggered rules by rule name:")
+   for name, count in pairs(rule_output_counts.by_name) do
+      print ("   "..name..": "..count)
+   end
+end
+
+
+