Add regression tests for configurable outputs.

 - In the regression tests, make the config file configurable in the
   multiplex file via 'conf_file'.
 - A new multiplex file item 'outputs' containing a list of <filename>:
   <regex> tuples. For each item, the test reads the file and matches
   each line against the regex. A match must be found for the test to
   pass.
 - Add 2 new tests that test file output and program output. They write
   to files below /tmp/falco_outputs/ and the contents are checked to
   ensure that alerts are written.

test/falco_test.py

@@ -36,6 +36,10 @@ def setUp(self):
                 file = os.path.join(self.basedir, file)
             self.rules_args = self.rules_args + "-r " + file + " "
 
+        self.conf_file = self.params.get('conf_file', '*', default=os.path.join(self.basedir, '../falco.yaml'))
+        if not os.path.isabs(self.conf_file):
+            self.conf_file = os.path.join(self.basedir, self.conf_file)
+
         self.disabled_rules = self.params.get('disabled_rules', '*', default='')
 
         if self.disabled_rules == '':
@@ -82,6 +86,20 @@ def setUp(self):
 
         self.str_variant = self.trace_file
 
+        self.outputs = self.params.get('outputs', '*', default='')
+
+        if self.outputs == '':
+            self.outputs = {}
+        else:
+            outputs = []
+            for item in self.outputs:
+                for item2 in item:
+                    output = {}
+                    output['file'] = item2[0]
+                    output['line'] = item2[1]
+                    outputs.append(output)
+            self.outputs = outputs
+
     def check_rules_warnings(self, res):
 
         found_warning = sets.Set()
@@ -140,6 +158,23 @@ def check_detections(self, res):
                     if not events_detected > 0:
                         self.fail("Detected {} events at level {} when should have detected > 0".format(events_detected, level))
 
+    def check_outputs(self):
+        for output in self.outputs:
+            # Open the provided file and match each line against the
+            # regex in line.
+            file = open(output['file'], 'r')
+            found = False
+            for line in file:
+                match = re.search(output['line'], line)
+
+                if match is not None:
+                    found = True
+
+            if found == False:
+                self.fail("Could not find a line '{}' in file '{}'".format(output['line'], output['file']))
+
+        return True
+
     def check_json_output(self, res):
         if self.json_output:
             # Just verify that any lines starting with '{' are valid json objects.
@@ -155,8 +190,8 @@ def test(self):
         self.log.info("Trace file %s", self.trace_file)
 
         # Run the provided trace file though falco
-        cmd = '{}/userspace/falco/falco {} {} -c {}/../falco.yaml -e {} -o json_output={} -v'.format(
-            self.falcodir, self.rules_args, self.disabled_args, self.falcodir, self.trace_file, self.json_output)
+        cmd = '{}/userspace/falco/falco {} {} -c {} -e {} -o json_output={} -v'.format(
+            self.falcodir, self.rules_args, self.disabled_args, self.conf_file, self.trace_file, self.json_output)
 
         self.falco_proc = process.SubProcess(cmd)
 
@@ -171,6 +206,7 @@ def test(self):
             self.check_rules_events(res)
         self.check_detections(res)
         self.check_json_output(res)
+        self.check_outputs()
         pass
 
 

test/falco_tests.yaml.in

@@ -105,3 +105,23 @@ trace_files: !mux
     disabled_rules:
       - "open.*"
     trace_file: trace_files/cat_write.scap
+
+  file_output:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/single_rule.yaml
+    conf_file: confs/file_output.yaml
+    trace_file: trace_files/cat_write.scap
+    outputs:
+      - /tmp/falco_outputs/file_output.txt: Warning An open was seen
+
+  program_output:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/single_rule.yaml
+    conf_file: confs/program_output.yaml
+    trace_file: trace_files/cat_write.scap
+    outputs:
+      - /tmp/falco_outputs/program_output.txt: Warning An open was seen

test/run_regression_tests.sh

@@ -50,6 +50,8 @@ function prepare_multiplex_file() {
 }
 
 function run_tests() {
+    rm -rf /tmp/falco_outputs
+    mkdir /tmp/falco_outputs
     CMD="avocado run --multiplex $MULT_FILE --job-results-dir $SCRIPTDIR/job-results -- $SCRIPTDIR/falco_test.py"
     echo "Running: $CMD"
     $CMD