Merge pull request #462 from m-1-k-3/refactoring

Refactoring

README.md

@@ -73,4 +73,5 @@ We welcome [pull requests](https://github.com/e-m-b-a/emba/pulls) and [issues](h
 ## Team
 
 [The core EMBA Team](https://github.com/orgs/e-m-b-a/people)
+
 [Contributors](https://github.com/e-m-b-a/emba/blob/master/CONTRIBUTORS.md)

check_project.sh

@@ -13,7 +13,7 @@
 #
 # Author(s): Michael Messner, Pascal Eckmann
 
-# Description:  Check all shell scripts inside ./helpers, ./modules, emba.sh and itself with shellchecker
+# Description:  Check all shell scripts inside ./helpers, ./modules, emba and itself with shellchecker
 
 STRICT_MODE=1
 
@@ -106,7 +106,7 @@ import_installer() {
 
 import_emba_main() {
   MODULES=()
-  mapfile -t MODULES < <(find ./ -iname "emba.sh" -o -iname "installer.sh" -o -iname "check_project.sh" 2>/dev/null)
+  mapfile -t MODULES < <(find ./ -iname "emba" -o -iname "installer.sh" -o -iname "check_project.sh" 2>/dev/null)
   for LINE in "${MODULES[@]}"; do
     if (file "$LINE" | grep -q "shell script"); then
       echo "$LINE"
@@ -172,7 +172,7 @@ check() {
   echo -e "\\n""$GREEN""Run shellcheck and semgrep:""$NC""\\n"
   for SOURCE in "${SOURCES[@]}"; do
     echo -e "\\n""$GREEN""Run ${ORANGE}shellcheck$GREEN on $ORANGE$SOURCE""$NC""\\n"
-    if shellcheck -P "$HELP_DIR":"$MOD_DIR":"$MOD_DIR_LOCAL" -a ./emba.sh "$SOURCE" || [[ $? -ne 1 && $? -ne 2 ]]; then
+    if shellcheck -P "$HELP_DIR":"$MOD_DIR":"$MOD_DIR_LOCAL" -a ./emba "$SOURCE" || [[ $? -ne 1 && $? -ne 2 ]]; then
       echo -e "$GREEN""$BOLD""==> SUCCESS""$NC""\\n"
     else
       echo -e "\\n""$ORANGE""$BOLD""==> FIX ERRORS""$NC""\\n"
@@ -257,8 +257,6 @@ summary() {
     done
     echo -e "$ORANGE""WARNING: Fix the errors before pushing to the EMBA repository!"
   fi
-
-
 }
 
 # check that all tools are installed

config/bin_version_strings.cfg

@@ -336,7 +336,7 @@ libpcap;;bsd;"^libpcap\ version\ [0-9](\.[0-9]+)+?$";"sed -r 's/libpcap\ version
 libpcre;;bsd;"libpcre\.so\.[0-9]\.[0-9](\.[0-9]+)+?$";"sed -r 's/libpcre\.so\.([0-9](\.[0-9]+)+?)$/pcre:\1/'";
 libpng;;libpng;"libpng\ version\ [0-9](\.[0-9]+)+?\ ";"sed -r 's/libpng\ version\ ([0-9](\.[0-9]+)+?)\ .*/libpng:\1/'";
 libreswan;;gplv2;"^Libreswan\ [\.0-9]+";"sed -r 's/Libreswan\ ([0-9](\.[0-9]+)+?).*/libreswan:\1/'";
-libsensors;;unknown;"libsensors\ version\ [.\0-9]+$";"sed -r 's/libsensors\ version\ ([0-9](\.[0-9]+)+?)$/libsensors:\1/'";
+libsensors;;unknown;"libsensors\ version\ [\.0-9]+$";"sed -r 's/libsensors\ version\ ([0-9](\.[0-9]+)+?)$/libsensors:\1/'";
 libtiff;;unknown;"^LIBTIFF,\ Version\ [0-9](\.[0-9]+)+?$";"sed -r 's/LIBTIFF,\ Version\ ([0-9](\.[0-9]+)+?)$/libtiff:libtiff:\1/'";
 lighttpd;;bsd;"^lighttpd\/[0-9](\.[0-9]+)+?\ .*\ -\ a\ light\ and\ fast\ webserver$";"sed -r 's/lighttpd\/([0-9](\.[0-9]+)+?)\ .*/lighttpd:\1/'";
 lighttpd;live;bsd;"^lighttpd\/[0-9](\.[0-9]+)+?(-devel-[0-9]+[A-Z])?$";"sed -r 's/lighttpd\/([0-9](\.[0-9]+)+?).*/lighttpd:\1/'";

config/distri_id.cfg

@@ -18,4 +18,4 @@ D-Link;/image_sign;grep -a -o -E ".*_d.*_.*";sort -u | cut -d_ -f3 | sed -r 's/(
 VERSION.LTM;/VERSION.LTM;grep -a -o -E -e "^Product:.*" -a -o -E -e "^Version:.*";sort -u | tr -d '\n' | sed 's/Product: BIG-IP/BIG-IP LTM/g' | sed 's/Version://g' | sed 's/^\ //'
 # F5 BigIP - application security manager
 VERSION.ASM;/VERSION.ASM;grep -a -o -E -e "^Product:.*" -a -o -E -e "^Version:.*";sort -u | tr -d '\n' | sed 's/Product: BIG-IP/BIG-IP ASM/g' | sed 's/Version://g' | sed 's/^\ //'
-Mikrotik-router;/nova/lib/console/logo.txt;grep -a -o -E -e "MikroTik\ routerOS\ V[0-9]\.[0-9]+\ \(c\) [0-9]+-[0-9].*";sed -r 's/.*MikroTik\ routerOS\ V([0-9]\.[0-9]+)\ .*/MikroTik\ routerOS\ V\1/'
+Mikrotik-router;/nova/lib/console/logo.txt;grep -a -o -E -e "MikroTik\ routerOS\ V[0-9]\.[0-9]+\ \(c\) [0-9]+-[0-9].*";sed -r 's/.*MikroTik\ routerOS\ V([0-9]\.[0-9]+).*/MikroTik\ routerOS\ V\1/'

config/trickest_blacklist.txt

@@ -39,3 +39,4 @@ andir/nixos-issue-db-example
 andrewwebber/kate
 jenkinsci-cert/nvd-cwe
 xaviermerino/ECE1552
+evdenis/cvehound

emba.sh → emba

@@ -135,6 +135,7 @@ run_modules()
       mapfile -t MODULES_LOCAL < <(find "${MOD_DIR_LOCAL}" -name "${MODULE_GROUP^^}""*.sh" 2>/dev/null | sort -V 2> /dev/null)
     fi
     MODULES=( "${MODULES_EMBA[@]}" "${MODULES_LOCAL[@]}" )
+    MODULES_EXPORTED+=("${MODULES[@]}")
     if [[ $THREADING_SET -eq 1 && "${MODULE_GROUP^^}" != "P" ]] ; then
       sort_modules
     fi
@@ -203,6 +204,8 @@ run_modules()
       if [[ "$SELECT_NUM" =~ ^["${MODULE_GROUP,,}","${MODULE_GROUP^^}"]{1}[0-9]+ ]]; then
         local MODULE=""
         MODULE=$(find "$MOD_DIR" -name "${MODULE_GROUP^^}""${SELECT_NUM:1}""_*.sh" | sort -V 2> /dev/null)
+        # we need the whole module name including path in our array for later checks on it
+        export MODULES_EXPORTED+=("${MODULE}")
         if ( file "$MODULE" | grep -q "shell script" ) && ! [[ "$MODULE" =~ \ |\' ]] ; then
           MODULE_BN=$(basename "$MODULE")
           MODULE_MAIN=${MODULE_BN%.*}
@@ -246,9 +249,9 @@ run_modules()
           mapfile -t MODULES_LOCAL < <(find "${MOD_DIR_LOCAL}" -name "${MODULE_GROUP^^}""*.sh" 2>/dev/null | sort -V 2> /dev/null)
         fi
         MODULES=( "${MODULES_EMBA[@]}" "${MODULES_LOCAL[@]}" )
-        if [[ $THREADING_SET -eq 1 ]] ; then
-          sort_modules
-        fi
+
+        [[ $THREADING_SET -eq 1 ]] && sort_modules
+
         for MODULE_FILE in "${MODULES[@]}" ; do
           # check if "$MODULE_NAME" is in blacklist from config directory and skip it
           MODULE_NAME=$(basename -s .sh "$MODULE_FILE")
@@ -346,6 +349,7 @@ main()
   export ARCH=""
   export EXLUDE=()
   export SELECT_MODULES=()
+  export MODULES_EXPORTED=()
   export ROOT_PATH=()
   export FILE_ARR=()
   export LOG_GREP=0
@@ -433,7 +437,7 @@ main()
   fi
 
   export EMBA_COMMAND
-  EMBA_COMMAND="$(dirname "$0")""/emba.sh ""$*"
+  EMBA_COMMAND="$(dirname "$0")""/emba ""$*"
 
   while getopts a:bBA:cC:dDe:Ef:Fghijk:l:m:N:p:P:QrsStT:UxX:yY:WzZ: OPT ; do
     case $OPT in
@@ -594,9 +598,7 @@ main()
   fi
 
   # print it only once per EMBA run - not again from started container
-  if [[ $IN_DOCKER -eq 0 ]]; then
-    banner_printer
-  fi
+  [[ $IN_DOCKER -eq 0 ]] && banner_printer
 
   if [[ $IN_DOCKER -eq 1 ]] ; then
     # set external path new for docker
@@ -621,7 +623,7 @@ main()
       print_bar "no_log"
     fi
 
-    enable_strict_mode "$STRICT_MODE"
+    enable_strict_mode "$STRICT_MODE" 1
 
     # profile handling
     if [[ -n "${PROFILE:-}" ]]; then
@@ -661,16 +663,12 @@ main()
       fi
     fi
 
-    if [[ $IN_DOCKER -eq 0 ]]; then
-      # check if LOG_DIR exists and prompt to terminal to delete its content (Y/n)
-      log_folder
-    fi
+    # check if LOG_DIR exists and prompt to terminal to delete its content (Y/n)
+    [[ $IN_DOCKER -eq 0 ]] && log_folder
 
+    # create log directory, if not exists and needed subdirectories
     # do not create a log dir for dep check
-    if [[ "$ONLY_DEP" -eq 0 ]]; then
-      # create log directory, if not exists and needed subdirectories
-      create_log_dir
-    fi
+    [[ "$ONLY_DEP" -eq 0 ]] && create_log_dir
 
     # kernel downloader runs on the host and waits for an identified kernel version. Afterwards
     # it tries to download the kernel sources for further analysis
@@ -887,11 +885,10 @@ main()
 
     write_notification "EMBA starting docker container"
 
-    if [[ "$STRICT_MODE" -eq 1 ]]; then
-      set +e
-    fi
+    [[ "$STRICT_MODE" -eq 1 ]] && set +e
+
     disable_strict_mode "$STRICT_MODE" 0
-    EMBA="$INVOCATION_PATH" FIRMWARE="$FIRMWARE_PATH" LOG="$LOG_DIR" docker-compose run --rm emba -c './emba.sh -l /logs -f /firmware -i "$@"' _ "${ARGUMENTS[@]}"
+    EMBA="$INVOCATION_PATH" FIRMWARE="$FIRMWARE_PATH" LOG="$LOG_DIR" docker-compose run --rm emba -c './emba -l /logs -f /firmware -i "$@"' _ "${ARGUMENTS[@]}"
     D_RETURN=$?
     enable_strict_mode "$STRICT_MODE" 0
 
@@ -907,9 +904,7 @@ main()
         cleaner 0
       else
         # we do not need the log dir from dependency checker
-        if [[ -d "$LOG_DIR" ]]; then
-          rm -r "$LOG_DIR"
-        fi
+        [[ -d "$LOG_DIR" ]] && rm -r "$LOG_DIR"
       fi
       exit 0
     else
@@ -947,9 +942,7 @@ main()
     run_modules "P" "$THREADED" "0"
 
     # if we running threaded we ware going to wait for the slow guys here
-    if [[ $THREADED -eq 1 ]]; then
-      wait_for_pid "${WAIT_PIDS[@]}"
-    fi
+    [[ $THREADED -eq 1 ]] && wait_for_pid "${WAIT_PIDS[@]}"
 
     print_ln "no_log"
 
@@ -985,9 +978,7 @@ main()
 
     run_modules "S" "$THREADED" "$HTML"
 
-    if [[ $THREADED -eq 1 ]]; then
-      wait_for_pid "${WAIT_PIDS[@]}"
-    fi
+    [[ $THREADED -eq 1 ]] && wait_for_pid "${WAIT_PIDS[@]}"
 
     print_ln "no_log"
 
@@ -1065,17 +1056,16 @@ main()
     print_output "$(indent "Try using binwalk or something else to extract the firmware")"
     exit 1
   fi
-  if [[ "$HTML" -eq 1 ]]; then
-    update_index
-  fi
+
+  [[ "$HTML" -eq 1 ]] && update_index
+
   if [[ -f "$HTML_PATH"/index.html ]] && [[ "$IN_DOCKER" -eq 0 ]]; then
     print_output "[*] Web report created HTML report in $ORANGE$LOG_DIR/html-report$NC\\n" "main"
     print_output "[*] Open the web-report with$ORANGE firefox $(abs_path "$HTML_PATH/index.html")$NC\\n" "main"
   fi
-  if [[ "$IN_DOCKER" -eq 1 ]]; then
-    # we need to change the permissions of the LOG_DIR to the orig. user from the host
-    restore_permissions
-  fi
+
+  # we need to change the permissions of the LOG_DIR to the orig. user from the host
+  [[ "$IN_DOCKER" -eq 1 ]] && restore_permissions
   cleaner 0
   exit 0
 }

helpers/helpers_emba_dependency_check.sh

@@ -231,15 +231,11 @@ setup_unblob() {
   fi
   print_output "    ""sasquatch"" - \\c" "no_log"
   if [[ -f /usr/local/bin/sasquatch_binwalk ]]; then
-    if [[ -L "$UNBLOB_PATH"/sasquatch ]]; then
-      rm  "$UNBLOB_PATH"/sasquatch
-    fi
+    [[ -L "$UNBLOB_PATH"/sasquatch ]] && rm  "$UNBLOB_PATH"/sasquatch
     ln -s /usr/local/bin/sasquatch_binwalk "$UNBLOB_PATH"/sasquatch
     echo -e "$GREEN""ok""$NC"
   elif [[ -f /usr/local/bin/sasquatch_unblob ]]; then
-    if [[ -L "$UNBLOB_PATH"/sasquatch ]]; then
-      rm  "$UNBLOB_PATH"/sasquatch
-    fi
+    [[ -L "$UNBLOB_PATH"/sasquatch ]] && rm  "$UNBLOB_PATH"/sasquatch
     ln -s /usr/local/bin/sasquatch_unblob "$UNBLOB_PATH"/sasquatch
     echo -e "$ORANGE""warning""$NC"
     DEP_EXIT=1
@@ -328,7 +324,6 @@ dependency_check()
     fi
   fi
 
-
   print_ln "no_log"
   print_output "[*] Necessary utils on system:" "no_log"
 
@@ -470,9 +465,7 @@ dependency_check()
       # TODO change to portcheck and write one for external hosts
       check_dep_file "cve-search script" "$EXT_DIR""/cve-search/bin/search.py"
       # we have already checked it outside the docker - do not need it again
-      if [[ "$IN_DOCKER" -eq 0 ]]; then
-        check_cve_search
-      fi
+      [[ "$IN_DOCKER" -eq 0 ]] && check_cve_search
       if [[ "$IN_DOCKER" -eq 0 ]]; then
         # really basic check, if cve-search database is running - no check, if populated and also no check, if EMBA in docker
         check_dep_tool "mongo database" "mongod"

helpers/helpers_emba_helpers.sh

@@ -62,9 +62,7 @@ max_pids_protection() {
     # check for really running PIDs and re-create the array
     for PID in "${WAIT_PIDS[@]}"; do
       # print_output "[*] max pid protection: ${#WAIT_PIDS[@]}"
-      if [[ -e /proc/"$PID" ]]; then
-        TEMP_PIDS+=( "$PID" )
-      fi
+      [[ -e /proc/"$PID" ]] && TEMP_PIDS+=( "$PID" )
     done
     # if S115 is running we have to kill old qemu processes
     if [[ -f "$LOG_DIR"/"$MAIN_LOG_FILE" ]] && [[ $(grep -i -c S115_ "$LOG_DIR"/"$MAIN_LOG_FILE" || true) -eq 1 && -n "$QRUNTIME" ]]; then
@@ -96,9 +94,7 @@ cleaner() {
   fi
 
   # Remove status bar and reset screen
-  if [[ "$DISABLE_STATUS_BAR" -eq 0 ]]; then
-    remove_status_bar
-  fi
+  [[ "$DISABLE_STATUS_BAR" -eq 0 ]] && remove_status_bar
 
   # if S115 is found only once in main.log the module was started and we have to clean it up
   # additionally we need to check some variable from a running EMBA instance
@@ -132,9 +128,7 @@ cleaner() {
       reset_network_emulation 2
     fi
   fi
-  if [[ "$IN_DOCKER" -eq 1 ]]; then
-    restore_permissions
-  fi
+  [[ "$IN_DOCKER" -eq 1 ]] && restore_permissions
 
   if [[ "$IN_DOCKER" -eq 0 ]] && [[ -v K_DOWN_PID ]]; then
     if ps -p "$K_DOWN_PID" > /dev/null; then
@@ -293,6 +287,11 @@ backup_var() {
 
 module_wait() {
   local MODULE_TO_WAIT="${1:-}"
+  # if the module we should wait is not in our module array we return without waiting
+  if ! [[ " ${MODULES_EXPORTED[*]} " == *"${MODULE_TO_WAIT}"* ]]; then
+    print_output "[-] Module $ORANGE$MODULE_TO_WAIT$NC not in module array - this will result in unexpected behavior" "main"
+    return
+  fi
 
   while ! [[ -f "$MAIN_LOG" ]]; do
     sleep 1

helpers/helpers_emba_print.sh

@@ -143,12 +143,12 @@ print_output()
       fi
     fi
   fi
-  if [[ "$LOG_SETTING" != "no_log" ]] ; then
+  if [[ "$LOG_SETTING" != "no_log" ]]; then
     write_grep_log "$OUTPUT"
   fi
 }
 
-# echo untrusted data in a secure way:
+# echo unknown data in a consistent way:
 safe_echo() {
   STRING_TO_ECHO="${1:-}"
 
@@ -182,7 +182,7 @@ print_ln()
 
 print_dot()
 {
-  echo "." | tr -d "\n" 2>/dev/null ||true
+  echo -n "." 2>/dev/null ||true
 }
 
 write_log()
@@ -564,8 +564,7 @@ print_etc()
   fi
 }
 
-print_excluded()
-{
+print_excluded() {
   readarray -t EXCLUDE_PATHS_ARR < <(printf '%s' "$EXCLUDE_PATHS")
   if [[ ${#EXCLUDE_PATHS_ARR[@]} -gt 0 ]] ; then
     print_ln "no_log"
@@ -599,7 +598,7 @@ module_start_log() {
     print_output "[*] Found old module log path for $ORANGE$MODULE_MAIN_NAME$NC ... creating a backup" "no_log"
     mv "$LOG_PATH_MODULE" "$LOG_PATH_MODULE".bak."$RANDOM" || true
   fi
-  if ! [[ -d "$LOG_PATH_MODULE" ]] ; then
+  if ! [[ -d "$LOG_PATH_MODULE" ]]; then
     mkdir "$LOG_PATH_MODULE" || true
   fi
 }
@@ -644,9 +643,7 @@ module_end_log() {
       print_bar ""
     fi
   fi
-  if [[ "$HTML" -eq 1 ]]; then
-    run_web_reporter_mod_name "$MODULE_MAIN_NAME"
-  fi
+  [[ "$HTML" -eq 1 ]] && run_web_reporter_mod_name "$MODULE_MAIN_NAME"
   if [[ -v LOG_PATH_MODULE ]]; then
     if [[ -d "$LOG_PATH_MODULE" ]]; then
       if [[ "$(find "$LOG_PATH_MODULE" -type f | wc -l)" -eq 0 ]]; then
@@ -690,12 +687,10 @@ banner_printer() {
 # write notfication is the central notification area
 # if you want to print a notification via the notification system
 # call this function with the message as parameter
-write_notification(){
-  if [[ "$DISABLE_NOTIFICATIONS" -eq 1 ]]; then
-    return
-  fi
+write_notification() {
+  [[ "$DISABLE_NOTIFICATIONS" -eq 1 ]] && return
+  # in case DISPLAY is not set we are not able to show notifications
   if ! [[ -v DISPLAY ]]; then
-    # in case DISPLAY is not set we are not able to show notifications
     return
   fi
 
@@ -716,10 +711,8 @@ write_notification(){
 # print_notification handles the monitoring of the notification tmp file
 # from the docker container. If someone prints something into this file
 # this function will handle it and generate a desktop notification
-print_notification(){
-  if [[ "$DISABLE_NOTIFICATIONS" -eq 1 ]]; then
-    return
-  fi
+print_notification() {
+  [[ "$DISABLE_NOTIFICATIONS" -eq 1 ]] && return
   if ! [[ -v DISPLAY ]]; then
     # in case DISPLAY is not set we are not able to show notifications
     return

helpers/helpers_emba_status_bar.sh

@@ -176,7 +176,7 @@ update_box_status() {
     local RUNTIME=0
     RUNTIME="$(date -d@"$(( "$(date +%s)" - "$DATE_STR" ))" -u +%H:%M:%S)"
     LOG_DIR_SIZE="$(du -sh "$LOG_DIR" 2> /dev/null | cut -d$'\t' -f1 2> /dev/null || true)"
-    RUN_EMBA_PROCESSES="$(ps -C emba.sh | wc -l || true)"
+    RUN_EMBA_PROCESSES="$(ps -C emba | wc -l || true)"
     printf '\e[s\e[%s;29f%s\e[%s;29f%s\e[%s;29f%s\e[u' "$(( LINES - 3 ))" "$(status_util_str 0 "$RUNTIME")" "$(( LINES - 2 ))" "$(status_util_str 1 "$LOG_DIR_SIZE")" "$(( LINES - 1 ))" "$(status_util_str 2 "$RUN_EMBA_PROCESSES")" || true
     sleep .5
     if [[ -f "$STATUS_TMP_PATH" ]] ; then