Improvements to the integration tests build

docker/tester/root/usr/bin/entrypoint

@@ -69,7 +69,7 @@ case "$CMD" in
     # run tests
     echo "Running regression tests ..."
     cd "$SOURCE_DIR/falco/test"
-    ./run_regression_tests.sh "$BUILD_DIR/$BUILD_TYPE"
+    ./run_regression_tests.sh -d "$BUILD_DIR/$BUILD_TYPE"
 
     # clean docker images
     clean_image "deb"

test/CMakeLists.txt

@@ -1 +1,4 @@
 add_subdirectory(trace_files)
+
+add_custom_target(test-trace-files ALL)
+add_dependencies(test-trace-files trace-files-base-scap trace-files-psp trace-files-k8s-audit)

test/README.md

@@ -7,13 +7,25 @@ You can find instructions on how to run this test suite on the Falco website [he
 ## Test suites
 
 - [falco_tests](./falco_tests.yaml)
-- [falco_traces](./falco_traces.yaml)
+- [falco_traces](./falco_traces.yaml.in)
 - [falco_tests_package](./falco_tests_package.yaml)
 - [falco_k8s_audit_tests](./falco_k8s_audit_tests.yaml)
 - [falco_tests_psp](./falco_tests_psp.yaml)
 
 ## Running locally
 
+This step assumes you already built Falco.
+
+Note that the tests are intended to be run against a [release build](https://falco.org/docs/source/#specify-the-build-type) of Falco, at the moment.
+
+Also, it assumes you prepared [falco_traces](#falco_traces) (see the section below) and you already run the following command from the build directory:
+
+```console
+make test-trace-files
+```
+
+It prepares the fixtures (`json` and `scap` files) needed by the integration tests.
+
 Using `virtualenv` the steps to locally run a specific test suite are the following ones (from this directory):
 
 ```console
@@ -32,8 +44,72 @@ In case you want to only execute a specific test case, use the `--mux-filter-onl
 BUILD_DIR="../build" avocado run --mux-yaml falco_tests.yaml --job-results-dir /tmp/job-results --mux-filter-only /run/trace_files/program_output -- falco_test.py
 ```
 
-To obtain the path of all the available variants, execute:
+To obtain the path of all the available variants for a given test suite, execute:
+
+```console
+avocado variants --mux-yaml falco_tests.yaml
+```
+
+### falco_traces
+
+The `falco_traces.yaml` test suite gets generated through the `falco_traces.yaml.in` file and some fixtures (`scap` files) downloaded from the web at execution time.
+
+1. Ensure you have `unzip` and `xargs` utilities
+2. Prepare the test suite with the following command:
+
+    ```console
+    bash run_regression_tests.sh -p -v
+    ```
+
+### falco_tests_package
+
+The `falco_tests_package.yaml` test suite requires some additional setup steps to be succesfully run on your local machine.
+
+In particular, it requires some runners (ie., docker images) to be already built and present into your local machine.
+
+1. Ensure you have `docker` up and running
+2. Ensure you build Falco (with bundled deps)
+
+    The recommended way of doing it by running the `falcosecurity/falco-builder` docker image from the project root:
+
+    ```console
+    docker run -v $PWD/..:/source -v $PWD/mybuild:/build falcosecurity/falco-builder cmake
+    docker run -v $PWD/..:/source -v $PWD/mybuild:/build falcosecurity/falco-builder falco
+    ```
+
+3. Ensure you build the Falco packages from the Falco above:
+
+    ```console
+    docker run -v $PWD/..:/source -v $PWD/mybuild:/build falcosecurity/falco-builder package
+    ```
+
+4. Ensure you build the runners:
+
+    ```console
+    FALCO_VERSION=$(./mybuild/release/userspace/falco/falco --version  | head -n 1 | cut -d' ' -f3 | tr -d '\r')
+    mkdir -p /tmp/runners-rootfs
+    cp -R ./test/rules /tmp/runners-rootfs
+    cp -R ./test/trace_files /tmp/runners-rootfs
+    cp ./mybuild/release/falco-${FALCO_VERSION}-x86_64.{deb,rpm,tar.gz} /tmp/runners-rootfs
+    docker build -f docker/tester/root/runners/deb.Dockerfile --build-arg FALCO_VERSION=${FALCO_VERSION} -t falcosecurity/falco:test-deb /tmp/runners-rootfs
+    docker build -f docker/tester/root/runners/rpm.Dockerfile --build-arg FALCO_VERSION=${FALCO_VERSION} -t falcosecurity/falco:test-rpm /tmp/runners-rootfs
+    docker build -f docker/tester/root/runners/tar.gz.Dockerfile --build-arg FALCO_VERSION=${FALCO_VERSION} -t falcosecurity/falco:test-tar.gz /tmp/runners-rootfs
+    ```
+
+5. Run the `falco_tests_package.yaml` test suite from the `test` directory
+
+    ```console
+    cd test
+    BUILD_DIR="../mybuild" avocado run --mux-yaml falco_tests_package.yaml --job-results-dir /tmp/job-results -- falco_test.py
+    ```
+
+### Execute all the test suites
+
+In case you want to run all the test suites at once, you can directly use the `run_regression_tests.sh` runner script.
 
 ```console
-avocado variants --mux-yaml falco_test.yaml
-```
+cd test
+./run_regression_tests.sh -v
+```
+
+Just make sure you followed all the previous setup steps.

test/driver-loader/run_test.sh

@@ -20,17 +20,17 @@ set -euo pipefail
 BUILD_DIR=$1
 
 SCRIPT=$(readlink -f $0)
-SCRIPTDIR=$(dirname $SCRIPT)
+SCRIPTDIR=$(dirname "$SCRIPT")
 RUNNERDIR="${SCRIPTDIR}/runner"
 FALCO_VERSION=$(cat ${BUILD_DIR}/userspace/falco/config_falco.h | grep 'FALCO_VERSION ' | cut -d' ' -f3 | sed -e 's/^"//' -e 's/"$//')
 DRIVER_VERSION=$(cat ${BUILD_DIR}/userspace/falco/config_falco.h | grep 'DRIVER_VERSION ' | cut -d' ' -f3 | sed -e 's/^"//' -e 's/"$//')
 FALCO_PACKAGE="falco-${FALCO_VERSION}-x86_64.tar.gz"
 
 cp "${BUILD_DIR}/${FALCO_PACKAGE}" "${RUNNERDIR}"
-pushd ${RUNNERDIR}
+pushd "${RUNNERDIR}"
 docker build --build-arg FALCO_VERSION="$FALCO_VERSION" \
     -t falcosecurity/falco:test-driver-loader \
-    -f "${RUNNERDIR}/Dockerfile" ${RUNNERDIR}
+    -f "${RUNNERDIR}/Dockerfile" "${RUNNERDIR}"
 popd
 rm -f "${RUNNERDIR}/${FALCO_PACKAGE}"
 

test/falco_traces.yaml.in

@@ -1,5 +1,5 @@
 #
-# Copyright (C) 2019 The Falco Authors.
+# Copyright (C) 2020 The Falco Authors.
 #
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

test/run_regression_tests.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 #
-# Copyright (C) 2019 The Falco Authors.
+# Copyright (C) 2020 The Falco Authors.
 #
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,45 +18,46 @@
 set -euo pipefail
 
 SCRIPT=$(readlink -f $0)
-SCRIPTDIR=$(dirname $SCRIPT)
-BUILD_DIR=$1
-BRANCH=${2:-none}
-
-TRACE_DIR=$BUILD_DIR/test
-
-mkdir -p $TRACE_DIR
+SCRIPTDIR=$(dirname "$SCRIPT")
 
 function download_trace_files() {
-    echo "branch=$BRANCH"
     for TRACE in traces-positive traces-negative traces-info ; do
-	if [ ! -e $TRACE_DIR/$TRACE ]; then
-	    if [ $BRANCH != "none" ]; then
-		curl -fso $TRACE_DIR/$TRACE.zip https://s3.amazonaws.com/download.draios.com/falco-tests/$TRACE-$BRANCH.zip
-	    else
-		curl -fso $TRACE_DIR/$TRACE.zip https://s3.amazonaws.com/download.draios.com/falco-tests/$TRACE.zip
-	    fi
-	    unzip -d $TRACE_DIR $TRACE_DIR/$TRACE.zip
-	    rm -rf $TRACE_DIR/$TRACE.zip
-	fi
+    if [ ! -e "$TRACE_DIR/$TRACE" ]; then
+        if [ "$OPT_BRANCH" != "none" ]; then
+        curl -fso "$TRACE_DIR/$TRACE.zip" https://s3.amazonaws.com/download.draios.com/falco-tests/$TRACE-$OPT_BRANCH.zip
+        else
+        curl -fso "$TRACE_DIR/$TRACE.zip" https://s3.amazonaws.com/download.draios.com/falco-tests/$TRACE.zip
+        fi
+        unzip -d "$TRACE_DIR" "$TRACE_DIR/$TRACE.zip"
+        rm -rf "$TRACE_DIR/$TRACE.zip"
+    else
+        if ${OPT_VERBOSE}; then
+            echo "Trace directory $TRACE_DIR/$TRACE already exist: skipping"
+        fi
+    fi
     done
 }
 
 function prepare_multiplex_fileset() {
-
     dir=$1
     detect=$2
 
-    for trace in $TRACE_DIR/$dir/*.scap ; do
-	[ -e "$trace" ] || continue
-	NAME=`basename $trace .scap`
+    for trace in "$TRACE_DIR/$dir"/*.scap ; do
+        [ -e "$trace" ] || continue
+        NAME=$(basename "$trace" .scap)
 
-	# falco_traces.yaml might already have an entry for this trace
-	# file, with specific detection levels and counts. If so, skip
-	# it. Otherwise, add a generic entry showing whether or not to
-	# detect anything.
-	grep -q "$NAME:" $SCRIPTDIR/falco_traces.yaml && continue
+        # falco_traces.yaml might already have an entry for this trace file, with specific detection levels and counts.
+        # If so, skip it.
+        # Otherwise, add a generic entry showing whether or not to detect anything.
+        if grep -q "$NAME:" "$SCRIPTDIR/falco_traces.yaml"; then
+            if ${OPT_VERBOSE}; then
+                echo "Entry $NAME already exist: skipping"
+            fi
+            continue
+        fi
+
+        cat << EOF >> "$SCRIPTDIR/falco_traces.yaml"
 
-	cat << EOF >> $SCRIPTDIR/falco_traces.yaml
   $NAME:
     detect: $detect
     detect_level: WARNING
@@ -66,41 +67,96 @@ EOF
 }
 
 function prepare_multiplex_file() {
-    cp $SCRIPTDIR/falco_traces.yaml.in $SCRIPTDIR/falco_traces.yaml
+    /bin/cp -f "$SCRIPTDIR/falco_traces.yaml.in" "$SCRIPTDIR/falco_traces.yaml"
 
     prepare_multiplex_fileset traces-positive True
     prepare_multiplex_fileset traces-negative False
     prepare_multiplex_fileset traces-info True
 
-    echo "Contents of $SCRIPTDIR/falco_traces.yaml:"
-    cat $SCRIPTDIR/falco_traces.yaml
+    if ${OPT_VERBOSE}; then
+        echo "Contents of $SCRIPTDIR/falco_traces.yaml"
+        cat "$SCRIPTDIR/falco_traces.yaml"
+    fi
 }
 
 function print_test_failure_details() {
     echo "Showing full job logs for any tests that failed:"
-    jq '.tests[] | select(.status != "PASS") | .logfile' $SCRIPTDIR/job-results/latest/results.json  | xargs cat
+    jq '.tests[] | select(.status != "PASS") | .logfile' "$SCRIPTDIR/job-results/latest/results.json" | xargs cat
 }
 
 function run_tests() {
     rm -rf /tmp/falco_outputs
     mkdir /tmp/falco_outputs
-    # If we got this far, we can undo set -e, as we're watching the
-    # return status when running avocado.
+    # If we got this far, we can undo set -e,
+    # as we're watching the return status when running avocado.
     set +e
     TEST_RC=0
     for mult in $SCRIPTDIR/falco_traces.yaml $SCRIPTDIR/falco_tests.yaml $SCRIPTDIR/falco_tests_package.yaml $SCRIPTDIR/falco_k8s_audit_tests.yaml $SCRIPTDIR/falco_tests_psp.yaml; do
-	CMD="avocado run --mux-yaml $mult --job-results-dir $SCRIPTDIR/job-results -- $SCRIPTDIR/falco_test.py"
-	echo "Running: $CMD"
-	BUILD_DIR=${BUILD_DIR} $CMD
-	RC=$?
-	TEST_RC=$((TEST_RC+$RC))
-	if [ $RC -ne 0 ]; then
-	    print_test_failure_details
-	fi
+        CMD="avocado run --mux-yaml $mult --job-results-dir $SCRIPTDIR/job-results -- $SCRIPTDIR/falco_test.py"
+        echo "Running $CMD"
+        BUILD_DIR=${OPT_BUILD_DIR} $CMD
+        RC=$?
+        TEST_RC=$((TEST_RC+RC))
+        if [ $RC -ne 0 ]; then
+            print_test_failure_details
+        fi
     done
 }
 
+OPT_ONLY_PREPARE="false"
+OPT_VERBOSE="false"
+OPT_BUILD_DIR="$(dirname "$SCRIPTDIR")/build"
+OPT_BRANCH="none"
+while getopts ':p :h :v :b: :d:' 'OPTKEY'; do
+    case ${OPTKEY} in
+        'p')
+            OPT_ONLY_PREPARE="true"
+            ;;
+        'h')
+            /bin/bash usage
+            exit 0
+            ;;
+        'v')
+            OPT_VERBOSE="true"
+            ;;
+        'd')
+            OPT_BUILD_DIR=${OPTARG}
+            ;;
+        'b')
+            OPT_BRANCH=${OPTARG}
+            ;;
+        '?')
+            echo "Invalid option: ${OPTARG}." >&2
+            /bin/bash usage
+            exit 1
+            ;;
+        ':')
+            echo "Missing argument for option: ${OPTARG}." >&2
+            /bin/bash usage
+            exit 1
+            ;;
+        *)
+            echo "Unimplemented option: ${OPTKEY}." >&2
+            /bin/bash usage
+            exit 1
+            ;;
+        esac
+done
+
+TRACE_DIR=$OPT_BUILD_DIR/test
+
+if ${OPT_VERBOSE}; then
+    echo "Build directory = $OPT_BUILD_DIR"
+    echo "Trace directory = $TRACE_DIR"
+    echo "Custom branch   = $OPT_BRANCH"
+fi
+
+mkdir -p "$TRACE_DIR"
+
 download_trace_files
 prepare_multiplex_file
-run_tests
-exit $TEST_RC
+
+if ! ${OPT_ONLY_PREPARE}; then
+    run_tests
+    exit $TEST_RC
+fi

test/trace_files/CMakeLists.txt

@@ -1,5 +1,6 @@
 add_subdirectory(k8s_audit)
 add_subdirectory(psp)
+
 # Note: list of traces is created at cmake time, not build time
 file(GLOB test_trace_files
 	"${CMAKE_CURRENT_SOURCE_DIR}/*.scap")
@@ -11,4 +12,8 @@ foreach(trace_file_path ${test_trace_files})
 	add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${trace_file}
 		COMMAND ${CMAKE_COMMAND} -E copy ${trace_file_path} ${CMAKE_CURRENT_BINARY_DIR}/${trace_file}
 		DEPENDS ${trace_file_path})
+	list(APPEND BASE_SCAP_TRACE_FILES_TARGETS test-trace-${trace_file})
 endforeach()
+
+add_custom_target(trace-files-base-scap ALL)
+add_dependencies(trace-files-base-scap ${BASE_SCAP_TRACE_FILES_TARGETS})

test/trace_files/k8s_audit/CMakeLists.txt

@@ -9,4 +9,8 @@ foreach(trace_file_path ${test_trace_files})
 	add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${trace_file}
 		COMMAND ${CMAKE_COMMAND} -E copy ${trace_file_path} ${CMAKE_CURRENT_BINARY_DIR}/${trace_file}
 		DEPENDS ${trace_file_path})
+	list(APPEND K8S_AUDIT_TRACE_FILES_TARGETS test-trace-${trace_file})
 endforeach()
+
+add_custom_target(trace-files-k8s-audit ALL)
+add_dependencies(trace-files-k8s-audit ${K8S_AUDIT_TRACE_FILES_TARGETS})

test/trace_files/psp/CMakeLists.txt

@@ -10,4 +10,8 @@ foreach(trace_file_path ${test_trace_files})
 	add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${trace_file}
 		COMMAND ${CMAKE_COMMAND} -E copy ${trace_file_path} ${CMAKE_CURRENT_BINARY_DIR}/${trace_file}
 		DEPENDS ${trace_file_path})
+	list(APPEND PSP_TRACE_FILES_TARGETS test-trace-${trace_file})
 endforeach()
+
+add_custom_target(trace-files-psp ALL)
+add_dependencies(trace-files-psp ${PSP_TRACE_FILES_TARGETS})