tests: improve automated testing framework

This patch modifies number of shell and Python scripts under scripts/tests/ directory
to improve and enhance automated testing framework that is described in this
[Wiki page](https://github.com/cloudius-systems/osv/wiki/Automated-Testing-Framework)
in following ways:

- refine compose_and_test_selected_apps.sh to properly build and independently run
  ZFS and ROFS images

- refine compose_and_test_selected_apps.sh to log success/fail messages
  to summary files under /tmp

- refine compose_and_test_selected_apps.sh to support running both ZFS and ROFS
  unit test images as well as httpserver API tests

- refine python test scripts to support running test images with arbitrary
  OSv kernel (kernel.elf)

- revamp python script intended to test http apps (new name test_http_app.py) to support
  both ab (Apache Bench) and wrk stress test tools

Signed-off-by: Waldemar Kozaczuk <jwkozaczuk@gmail.com>

Author: wkozaczuk

scripts/tests/compose_and_test_selected_apps.sh

@@ -22,6 +22,18 @@ usage() {
 	  -R              Compose test app image with RoFS (ZFS is the default)
 	  -l              Use latest OSv kernel from build/last to build test image
 	  -f              Run OSv on firecracker
+
+	Test groups:
+	  simple - simple apps like golang-example
+	  http - httpserver apps
+	  http-java - java httpserver apps
+	  java <name> - java app
+	  http-node - node http apps
+	  node <name> - node app
+	  with_tester - apps tested with extra tester script like redis
+	  unit_tests - unit tests
+	  httpserver_api_tests - httpserver API unit tests
+	  all - all apps
 	EOF
 	exit ${1:-0}
 }
@@ -68,7 +80,7 @@ compose_test_app()
   local DEPENDANT_PKG1=$2
   local DEPENDANT_PKG2=$3
 
-  local IMAGE_PATH="$CAPSTAN_REPO/repository/test-$APP_NAME/test-$APP_NAME.qemu"
+  local IMAGE_PATH="$CAPSTAN_REPO/repository/test-$APP_NAME-$FS/test-$APP_NAME-$FS.qemu"
 
   if [ $RUN_ONLY == false ]; then
     local DEPENDENCIES="--require osv.$APP_NAME"
@@ -100,15 +112,22 @@ EOF
       echo "Using latest OSv kernel from $OSV_DIR/build/last/loader.img !"
     fi
 
+    if [ "$FS" == "rofs" ]; then
+      FSTAB=static/etc/fstab_rofs
+    else
+      FSTAB=static/etc/fstab
+    fi
+
     TEMPDIR=$(mktemp -d) && pushd $TEMPDIR > /dev/null && \
-      capstan package compose $DEPENDENCIES --fs $FS $LOADER_OPTION "test-$APP_NAME" && \
-      rmdir $TEMPDIR && popd > /dev/null
+      mkdir -p etc && cp $OSV_DIR/$FSTAB etc/fstab && \
+      $HOME/projects/go/src/github.com/cloudius-systems/capstan/capstan package compose $DEPENDENCIES --fs $FS $LOADER_OPTION "test-$APP_NAME-$FS" && \
+      rm -rf $TEMPDIR && popd > /dev/null
   else
     echo "Reusing the test image: $IMAGE_PATH that must have been composed before!"
   fi
 
   if [ -f "$IMAGE_PATH" ]; then
-    cp $CAPSTAN_REPO/repository/"test-$APP_NAME"/"test-$APP_NAME".qemu $OSV_DIR/build/last/usr.img
+    cp $CAPSTAN_REPO/repository/"test-$APP_NAME-$FS"/"test-$APP_NAME-$FS".qemu $OSV_DIR/build/last/usr.img
   else
     echo "Could not find test image: $IMAGE_PATH!"
     exit 1
@@ -121,15 +140,16 @@ run_test_app()
   local TEST_PARAMETER=$2
 
   if [ $COMPOSE_ONLY == false ]; then
-    echo "-------------------------------------------------------"
-    echo " Testing $OSV_APP_NAME ... "
-    echo "-------------------------------------------------------"
+    echo "-------------------------------------------------------"  | tee -a $STATUS_FILE
+    echo " Testing $OSV_APP_NAME ... "  | tee -a $STATUS_FILE
 
     if [ -f $OSV_DIR/apps/$OSV_APP_NAME/test.sh ]; then
       $OSV_DIR/apps/$OSV_APP_NAME/test.sh $TEST_PARAMETER
     elif [ -f $OSV_DIR/modules/$OSV_APP_NAME/test.sh ]; then
       $OSV_DIR/modules/$OSV_APP_NAME/test.sh $TEST_PARAMETER
     fi
+
+    echo "-------------------------------------------------------"  | tee -a $STATUS_FILE
   fi
   echo ''
 }
@@ -141,86 +161,184 @@ compose_and_run_test_app()
   run_test_app $APP_NAME
 }
 
-test_simple_apps()
+# Simple stateless apps that should work with both ZFS and ROFS
+test_simple_apps() #stateless
 {
   compose_and_run_test_app "golang-example"
   compose_and_run_test_app "golang-pie-example"
   compose_and_run_test_app "graalvm-example"
-  compose_and_run_test_app "graalvm-example"
   compose_and_run_test_app "lua-hello-from-host"
   compose_and_run_test_app "rust-example"
   compose_and_run_test_app "stream"
   compose_test_app "python2-from-host" && run_test_app "python-from-host"
   compose_test_app "python3-from-host" && run_test_app "python-from-host"
 }
 
-test_http_apps()
+test_java_app()
+{
+  compose_test_app "$1" "run-java" "openjdk8-zulu-compact3-with-java-beans" && run_test_app "$1"
+}
+
+# Stateless http java apps that should work with both ZFS and ROFS
+test_http_java_apps()
+{
+  #TODO: Test with multiple versions of java
+  test_java_app "jetty"
+  test_java_app "tomcat"
+  test_java_app "vertx"
+  test_java_app "spring-boot-example" #Really slow
+}
+
+test_node_app()
+{
+  compose_test_app "$1" "node-from-host" && run_test_app "$1"
+}
+
+# Stateless node apps that should work with both ZFS and ROFS
+test_http_node_apps()
+{
+  #TODO: Test with multiple versions of node
+  test_node_app "node-express-example"
+  test_node_app "node-socketio-example"
+}
+
+# Stateless http apps that should work with both ZFS and ROFS except for nginx
+test_http_apps() #stateless
 {
   compose_and_run_test_app "golang-httpserver"
   compose_and_run_test_app "golang-pie-httpserver"
   compose_and_run_test_app "graalvm-httpserver"
   compose_and_run_test_app "lighttpd"
-  compose_and_run_test_app "nginx"
+  if [ "$FS" == "zfs" ]; then #TODO: Fix configuration to make it work with ROFS
+    compose_and_run_test_app "nginx" #Not ROFS
+  fi
   compose_and_run_test_app "rust-httpserver"
-  #TODO: Test with multiple versions of java
-  compose_test_app "jetty" "run-java" "openjdk8-zulu-compact3-with-java-beans" && run_test_app "jetty"
-  compose_test_app "tomcat" "run-java" "openjdk8-zulu-compact3-with-java-beans" && run_test_app "tomcat"
-  compose_test_app "vertx" "run-java" "openjdk8-zulu-compact3-with-java-beans" && run_test_app "vertx"
-  compose_test_app "spring-boot-example" "run-java" "openjdk8-zulu-compact3-with-java-beans" && run_test_app "spring-boot-example" #Really slow
-  #TODO: Test with multiple versions of node
-  compose_test_app "node-express-example" "node-from-host" && run_test_app "node-express-example"
-  compose_test_app "node-socketio-example" "node-from-host" && run_test_app "node-socketio-example"
+  test_http_java_apps
+  test_http_node_apps
+}
+
+test_ffmpeg()
+{
+  if [ "$OSV_HYPERVISOR" == "firecracker" ]; then
+    echo "Skipping ffmpeg as at this time it cannot run on Firecracker. Needs to chenge this script to setup bridge parameter"
+  else
+    compose_test_app "ffmpeg" && run_test_app "ffmpeg" "video_subclip" && run_test_app "ffmpeg" "video_transcode"
+  fi
+}
+
+test_redis()
+{
+  compose_and_run_test_app "redis-memonly" && run_test_app "redis-memonly" "ycsb"
+}
+
+test_keydb()
+{
+  compose_and_run_test_app "keydb" && run_test_app "keydb" "ycsb"
 }
 
-test_apps_with_tester()
+test_apps_with_tester() #most stateless
 {
   compose_and_run_test_app "iperf3"
   compose_and_run_test_app "graalvm-netty-plot"
-  compose_test_app "ffmpeg" && run_test_app "ffmpeg" "video_subclip" && run_test_app "ffmpeg" "video_transcode"
-  compose_and_run_test_app "redis-memonly"
+  test_ffmpeg
+  test_redis
+  test_keydb
   compose_and_run_test_app "cli"
-  compose_and_run_test_app "mysql"
-  compose_test_app "apache-derby" "run-java" "openjdk8-zulu-compact3-with-java-beans" && run_test_app "apache-derby"
-  compose_test_app "apache-kafka" "run-java" "openjdk8-zulu-compact3-with-java-beans" && run_test_app "apache-kafka"
-  compose_and_run_test_app "elasticsearch"
+  if [ "$FS" == "zfs" ]; then #These are stateful apps
+    compose_and_run_test_app "mysql"
+    test_java_app "apache-derby"
+    test_java_app "apache-kafka"
+    compose_and_run_test_app "elasticsearch"
+  fi
 }
 
-run_unit_tests()
+run_unit_tests() #regular unit tests are stateful
 {
   # Unit tests are special as the unit tests runner depends on usr.manifest which
   # needs to be placed in the tests module. So let us gegnerate it on the fly from the unit tests mpm
-  capstan package describe osv.unit-tests -c | grep "/tests/tst-" | grep -o "/tests/tst-.*" | sed 's/$/: dummy/' > $OSV_DIR/modules/tests/usr.manifest
-  compose_test_app "unit-tests" && run_test_app "tests"
-  compose_test_app "httpserver-api-tests" && run_test_app "httpserver-api" "http"
-  #compose_test_app "httpserver-api-https-tests" "httpserver-api-tests" && run_test_app "httpserver-api" "https"
+  capstan package describe osv.common-tests -c | grep "/tests/tst-" | grep -o "/tests/tst-.*" | sed 's/$/: dummy/' > $OSV_DIR/modules/tests/usr.manifest
+  capstan package describe "osv.$FS-tests" -c | grep "/tests/tst-" | grep -o "/tests/tst-.*" | sed 's/$/: dummy/' >> $OSV_DIR/modules/tests/usr.manifest
+  compose_test_app "$FS-tests" "openjdk8-from-host" "common-tests" && run_test_app "tests"
+}
+
+run_httpserver_api_tests()
+{
+  if [ "$FS" == "zfs" ]; then #These are stateful apps
+    compose_test_app "httpserver-api-tests" && run_test_app "httpserver-api" "http"
+    rm -rf $OSV_DIR/modules/certs/build && mkdir -p $OSV_DIR/modules/certs/build && pushd $OSV_DIR/modules/certs/build
+    tar xf $CAPSTAN_REPO/packages/osv.httpserver-api-tests.mpm /client/client.key && \
+    tar xf $CAPSTAN_REPO/packages/osv.httpserver-api-tests.mpm /etc/pki/CA/cacert.pem && \
+    tar xf $CAPSTAN_REPO/packages/osv.httpserver-api-tests.mpm /client/client.pem && \
+    mv client/* . && mv etc/pki/CA/cacert.pem .
+    popd
+    compose_test_app "httpserver-api-https-tests" "httpserver-api-tests" && run_test_app "httpserver-api" "https"
+  fi
 }
 
+export STATUS_FILE="/tmp/$TEST_APP_PACKAGE_NAME"
+rm -f $STATUS_FILE
 
 case "$TEST_APP_PACKAGE_NAME" in
   simple)
-    echo "Testing simple apps ..."
-    echo "-----------------------------------"
+    echo "Testing simple apps ..." | tee -a $STATUS_FILE
+    echo "-----------------------------------" | tee -a $STATUS_FILE
     test_simple_apps;;
   http)
-    echo "Testing HTTP apps ..."
-    echo "-----------------------------------"
+    echo "Testing HTTP apps ..." | tee -a $STATUS_FILE
+    echo "-----------------------------------" | tee -a $STATUS_FILE
     test_http_apps;;
+  http-java)
+    echo "Testing HTTP Java apps ..." | tee -a $STATUS_FILE
+    echo "-----------------------------------" | tee -a $STATUS_FILE
+    test_http_java_apps;;
+  java)
+    echo "Testing Java app ..." | tee -a $STATUS_FILE
+    echo "-----------------------------------" | tee -a $STATUS_FILE
+    test_java_app $TEST_OSV_APP_NAME;;
+  http-node)
+    echo "Testing HTTP Node apps ..." | tee -a $STATUS_FILE
+    echo "-----------------------------------" | tee -a $STATUS_FILE
+    test_http_node_apps;;
+  node)
+    echo "Testing Node app ..." | tee -a $STATUS_FILE
+    echo "-----------------------------------" | tee -a $STATUS_FILE
+    test_node_app $TEST_OSV_APP_NAME;;
   with_tester)
-    echo "Testing apps with custom tester ..."
-    echo "-----------------------------------"
+    echo "Testing apps with custom tester ..." | tee -a $STATUS_FILE
+    echo "-----------------------------------" | tee -a $STATUS_FILE
     test_apps_with_tester;;
   unit_tests)
-    echo "Running unit tests ..."
-    echo "-----------------------------------"
+    echo "Running unit tests ..." | tee -a $STATUS_FILE
+    echo "-----------------------------------" | tee -a $STATUS_FILE
     run_unit_tests;;
+  redis)
+    echo "Running redis test..." | tee -a $STATUS_FILE
+    echo "-----------------------------------" | tee -a $STATUS_FILE
+    test_redis;;
+  keydb)
+    echo "Running keydb test..." | tee -a $STATUS_FILE
+    echo "-----------------------------------" | tee -a $STATUS_FILE
+    test_keydb;;
+  ffmpeg)
+    echo "Running ffmpeg test..." | tee -a $STATUS_FILE
+    echo "-----------------------------------" | tee -a $STATUS_FILE
+    test_ffmpeg;;
+  httpserver_api_tests)
+    echo "Running httpserver api tests ..." | tee -a $STATUS_FILE
+    echo "-----------------------------------" | tee -a $STATUS_FILE
+    run_httpserver_api_tests;;
   all)
-    echo "Running all tests ..."
-    echo "-----------------------------------"
+    echo "Running all tests ..." | tee -a $STATUS_FILE
+    echo "-----------------------------------" | tee -a $STATUS_FILE
     run_unit_tests
+    run_httpserver_api_tests
     test_simple_apps
     test_http_apps
+    compose_and_run_test_app specjvm
     test_apps_with_tester;;
   *)
+    echo "Running $TEST_APP_PACKAGE_NAME ..." | tee -a $STATUS_FILE
+    echo "-----------------------------------" | tee -a $STATUS_FILE
     if [ "$TEST_OSV_APP_NAME" == "" ]; then
       compose_and_run_test_app "$TEST_APP_PACKAGE_NAME"
     else

scripts/tests/test_app.py

@@ -1,18 +1,30 @@
 #!/usr/bin/python3
 from testing import *
 import argparse
+import os
 from time import sleep
 
-def run(command, hypervisor_name, image_path=None, line=None, guest_port=None, host_port=None, input_lines=[], kill_app=False):
+def run(command, hypervisor_name, image_path=None, line=None, guest_port=None, host_port=None,
+        input_lines=[], kill_app=False, kernel_path=None):
+
     py_args = []
     if image_path != None:
         py_args = ['--image', image_path]
 
+    if kernel_path != None:
+       print('Using kernel at %s' % kernel_path)
+       if hypervisor_name == 'firecracker':
+          py_args += ['-k', kernel_path]
+       else:
+          py_args += ['-k', '--kernel-path', kernel_path]
+
     pipe_guest_stdin= len(input_lines) > 0
     if guest_port != None and host_port != None:
-        app = run_command_in_guest(command, hypervisor=hypervisor_name, run_py_args=py_args, forward=[(host_port, guest_port)], pipe_stdin=pipe_guest_stdin)
+        app = run_command_in_guest(command, hypervisor=hypervisor_name, run_py_args=py_args,
+                                   forward=[(host_port, guest_port)], pipe_stdin=pipe_guest_stdin)
     else:
-        app = run_command_in_guest(command, hypervisor=hypervisor_name, run_py_args=py_args, pipe_stdin=pipe_guest_stdin)
+        app = run_command_in_guest(command, hypervisor=hypervisor_name, run_py_args=py_args,
+                                   pipe_stdin=pipe_guest_stdin)
 
     if line != None:
         if len(input_lines) > 0:
@@ -28,7 +40,12 @@ def run(command, hypervisor_name, image_path=None, line=None, guest_port=None, h
     app.join()
 
     print('----------')
-    print('SUCCESS')
+    print('  SUCCESS')
+
+    status_file_name = os.getenv('STATUS_FILE')
+    if status_file_name:
+       with open(status_file_name, "a+") as status_file:
+         status_file.write('  SUCCESS\n')
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(prog='test_app')
@@ -44,6 +61,7 @@ def run(command, hypervisor_name, image_path=None, line=None, guest_port=None, h
     parser.add_argument("--host_port", action="store", default=None, help="host port")
     parser.add_argument("--input_line", action="append", default=[], help="input line")
     parser.add_argument("--kill", action="store_true", help="kill the app instead of waiting until terminates itself")
+    parser.add_argument("--kernel_path", action="store", help="path to kernel.elf.")
 
     cmdargs = parser.parse_args()
 
@@ -55,6 +73,14 @@ def run(command, hypervisor_name, image_path=None, line=None, guest_port=None, h
         if hypervisor_from_env != None:
             hypervisor_name = hypervisor_from_env
 
+    kernel_path = cmdargs.kernel_path
+    if not kernel_path and os.getenv('OSV_KERNEL'):
+        kernel_path = os.getenv('OSV_KERNEL')
+
+    if kernel_path and not os.path.exists(kernel_path):
+        print("The file %s does not exist!" % kernel_path)
+        sys.exit(-1)
+
     set_verbose_output(True)
     run(cmdargs.execute, hypervisor_name, cmdargs.image, cmdargs.line,
-        cmdargs.guest_port, cmdargs.host_port, cmdargs.input_line, cmdargs.kill)
+        cmdargs.guest_port, cmdargs.host_port, cmdargs.input_line, cmdargs.kill, kernel_path)