System emulation improvements, workflow

.github/workflows/default_install.yml

@@ -0,0 +1,44 @@
+name: EMBA default installation on Ubuntu
+
+on:
+  schedule:
+    - cron: '0 0 * * *' # do it every day
+#  push:
+#    branches:
+#      - '**'        # matches every branch
+#  pull_request:
+#    branches:
+#      - '**'
+#  # Allows you to run this workflow manually from the Actions tab
+#  workflow_dispatch:
+
+jobs:
+  default-install:
+    if: github.repository_owner == 'e-m-b-a'
+    runs-on: ubuntu-22.04
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v3
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: true
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          swap-storage: true
+      - name: EMBA default install
+        uses: Wandalen/wretry.action@master
+        with:
+          command: sudo ./installer.sh -g && sudo ./emba -d 1 -y -j
+          attempt_limit: 3
+      - name: EMBA firmware download
+        run: |
+            sudo apt-get update
+            sudo apt-get install wget
+            wget https://ftp.dlink.de/dir/dir-300/archive/driver_software/DIR-300_fw_revb_214b01_ALL_de_20130206.zip
+      - name: EMBA default analysis
+        run: |
+            sudo ./emba -f ./DIR-300_fw_revb_214b01_ALL_de_20130206.zip -l ./logs_emba -S -p ./scan-profiles/default-scan-emulation.emba -y -j

.github/workflows/shellcheck.yml

@@ -29,4 +29,6 @@ jobs:
         # You may pin to the exact commit or the version.
         # uses: ludeeus/action-shellcheck@d586102c117f97e63d7e3b56629d269efc9a7c60
         uses: ludeeus/action-shellcheck@1.1.0
+        env:
+          SHELLCHECK_OPTS: -x
 

check_project.sh

@@ -18,10 +18,10 @@
 STRICT_MODE=1
 
 if [[ "$STRICT_MODE" -eq 1 ]]; then
-  # shellcheck disable=SC1091
+  # shellcheck source=./installer/wickStrictModeFail.sh
   source ./installer/wickStrictModeFail.sh
   export DEBUG_SCRIPT=0
-  # shellcheck disable=SC1091
+  # shellcheck source=./helpers/helpers_emba_load_strict_settings.sh
   source ./helpers/helpers_emba_load_strict_settings.sh
   load_strict_mode_settings
   trap 'wickStrictModeFail $?' ERR  # The ERR trap is triggered when a script catches an error
@@ -174,7 +174,7 @@ check() {
   echo -e "\\n""$GREEN""Run shellcheck and semgrep:""$NC""\\n"
   for SOURCE in "${SOURCES[@]}"; do
     echo -e "\\n""$GREEN""Run ${ORANGE}shellcheck$GREEN on $ORANGE$SOURCE""$NC""\\n"
-    if shellcheck -P "$HELP_DIR":"$MOD_DIR":"$MOD_DIR_LOCAL" "$SOURCE" || [[ $? -ne 1 && $? -ne 2 ]]; then
+    if shellcheck -x -P "./installer":"$HELP_DIR":"$MOD_DIR":"$MOD_DIR_LOCAL" "$SOURCE" || [[ $? -ne 1 && $? -ne 2 ]]; then
       echo -e "$GREEN""$BOLD""==> SUCCESS""$NC""\\n"
     else
       echo -e "\\n""$ORANGE""$BOLD""==> FIX ERRORS""$NC""\\n"
@@ -272,6 +272,7 @@ check_tools() {
   done
   if ! [[ -d ./external/semgrep-rules/bash ]]; then
     echo -e "\\n""$RED""$BOLD""Please install semgrep-rules to directory ./external to perform all checks""$NC""\\n"
+    echo -e "${ORANGE}git clone https://github.com/returntocorp/semgrep-rules.git external/semgrep-rule${NC}"
     exit 1
   fi
 }

emba

@@ -135,7 +135,7 @@ run_modules()
       mapfile -t MODULES_LOCAL < <(find "${MOD_DIR_LOCAL}" -name "${MODULE_GROUP^^}""*.sh" 2>/dev/null | sort -V 2> /dev/null)
     fi
     MODULES=( "${MODULES_EMBA[@]}" "${MODULES_LOCAL[@]}" )
-    MODULES_EXPORTED+=("${MODULES[@]}")
+    export MODULES_EXPORTED+=("${MODULES[@]}")
     if [[ $THREADING_SET -eq 1 && "${MODULE_GROUP^^}" != "P" ]] ; then
       sort_modules
     fi
@@ -827,7 +827,7 @@ main() {
 
   run_modules "F" "0" "$HTML"
 
-  remove_status_bar
+  [[ $DISABLE_STATUS_BAR -eq 0 ]] && remove_status_bar
 
   write_notification "Reporting phase ended"
 

helpers/helpers_emba_helpers.sh

@@ -234,7 +234,7 @@ enable_strict_mode() {
   if [[ "$STRICT_MODE_" -eq 1 ]]; then
     # http://redsymbol.net/articles/unofficial-bash-strict-mode/
     # https://github.com/tests-always-included/wick/blob/master/doc/bash-strict-mode.md
-    # shellcheck disable=SC1091
+    # shellcheck source=./installer/wickStrictModeFail.sh
     source ./installer/wickStrictModeFail.sh
     load_strict_mode_settings
     trap 'wickStrictModeFail $? | tee -a "$LOG_DIR"/emba_error.log' ERR  # The ERR trap is triggered when a script catches an error

helpers/helpers_emba_print.sh

@@ -141,7 +141,7 @@ print_output()
       safe_echo "$(format_log "$OUTPUT")" "$MAIN_LOG"
     elif [[ "$LOG_SETTING" != "no_log" ]] ; then
       if [[ -z "$REF_LINK" ]] ; then
-        safe_echo "$(format_log "$OUTPUT")" "$LOG_FILE"
+        safe_echo "$(format_log "$OUTPUT")" "${LOG_FILE:-}"
         if [[ -n "${LOG_FILE_MOD:-}" ]]; then
           safe_echo "$(format_log "$OUTPUT")" "$LOG_FILE_MOD"
         fi
@@ -587,7 +587,7 @@ print_help()
   echo -e "\\nWeb reporter"
   echo -e "$CYAN""-W""$NC""                Activates web report creation in log path (overwrites -z)"
   echo -e "\\nSystem check"
-  echo -e "$CYAN""-d [0/1]""$NC""          Only checks dependencies (1 - on host and in container, 2 - only container)"
+  echo -e "$CYAN""-d [1/2]""$NC""          Only checks dependencies (1 - on host and in container, 2 - only container)"
   echo -e "$CYAN""-F""$NC""                Checks dependencies but ignore errors"
   echo -e "$CYAN""-U""$NC""                Check and apply available updates and exit"
   echo -e "$CYAN""-V""$NC""                Show EMBA version"

helpers/helpers_emba_system_emulation.sh

@@ -17,7 +17,7 @@
 restart_emulation() {
   local IP_ADDRESS_="${1:-}"
   local IMAGE_NAME_="${2:-}"
-  # restart_scan is used to indicate a restarted scan. There we do not need to restart the network
+  # restart_scan is used to indicate a restarted scan. For this we do not need to restart the network
   local RESTART_SCAN="${3:-0}"
 
   if ping -c 1 "$IP_ADDRESS_" &> /dev/null; then
@@ -47,11 +47,20 @@ restart_emulation() {
   while ! ping -c 1 "$IP_ADDRESS_" &> /dev/null; do
     print_output "[*] Waiting for restarted system ..."
     ((COUNTER+=1))
-    [[ "$COUNTER" -gt 50 ]] && (print_output "[-] System not recovered" && return)
+    if [[ "$COUNTER" -gt 50 ]]; then
+      print_output "[-] System not recovered"
+      break
+    fi
     sleep 6
   done
-  print_output "[*] System automatically maintained and should be available again in a few moments ... check ip address $ORANGE$IP_ADDRESS_$NC"
-  sleep 60
-  export SYS_ONLINE=1
-  export TCP="ok"
+
+  if ping -c 1 "$IP_ADDRESS_" &> /dev/null; then
+    print_output "[*] System automatically maintained and should be available again in a few moments ... check ip address $ORANGE$IP_ADDRESS_$NC"
+    sleep 60
+    export SYS_ONLINE=1
+    export TCP="ok"
+  else
+    export SYS_ONLINE=0
+    export TCP="not ok"
+  fi
 }

helpers/routersploit_db_update.sh

@@ -44,7 +44,7 @@ fi
 
 echo "[*] Building the Routersploit exploit database"
 # search all ruby files in the routersploit directory and create a temporary file with the module path and CVE:
-find "$ROUTERSPLOIT_MOD_PATH" -type f -iname "*.py" -exec grep -i -o -H -E "CVE-[0-9]{4}-[0-9]+" {} \; | sed 's/.*external\/routersploit//' | sort -u > "$ROUTERSPLOIT_CVE_PATH"
+find "$ROUTERSPLOIT_MOD_PATH" -type f -iname "*.py" -exec grep -i -o -H -E "CVE-[0-9]{4}-[0-9]+" {} \; | sed 's/.*external\/routersploit//' | sed 's/cve-/CVE-/' | sort -u > "$ROUTERSPLOIT_CVE_PATH"
 find "$ROUTERSPLOIT_MOD_PATH" -type f -iname "*.py" -exec grep -i -o -H -E "exploit-db.com/exploits/[0-9]+" {} \; | sed 's/exploit-db\.com\/exploits\///' | sed 's/.*external\/routersploit//' | sort -u > "$ROUTERSPLOIT_EDB_PATH"
 
 if [[ -f "$ROUTERSPLOIT_CVE_PATH" ]]; then

installer.sh

@@ -53,6 +53,7 @@ export REMOVE=0
 export OTHER_OS=0
 export UBUNTU_OS=0
 export WSL=0
+export GH_ACTION=0
 
 ## Color definition
 export RED="\033[0;31m"
@@ -93,7 +94,7 @@ if [ "$#" -ne 1 ]; then
   exit 1
 fi
 
-while getopts cCdDFhlr OPT ; do
+while getopts cCdDFghlr OPT ; do
   case $OPT in
     d)
       export DOCKER_SETUP=1
@@ -112,6 +113,13 @@ while getopts cCdDFhlr OPT ; do
       export CVE_SEARCH=1
       echo -e "$GREEN""$BOLD""Install all dependecies for developer mode""$NC"
       ;;
+    g)
+      export DOCKER_SETUP=1
+      export GH_ACTION=1
+      export CVE_SEARCH=0
+      echo -e "$GREEN""$BOLD""Install all dependecies for EMBA test via Github actions""$NC"
+      echo -e "$GREEN""$BOLD""This mode is a default installation without populating the CVE-search database""$NC"
+      ;;
     h)
       print_help
       exit 0

installer/IF20_cve_search.sh

@@ -22,9 +22,9 @@ IF20_cve_search() {
   if [[ "$LIST_DEP" -eq 1 ]] || [[ $IN_DOCKER -eq 1 ]] || [[ $DOCKER_SETUP -eq 1 ]] || [[ $CVE_SEARCH -eq 1 ]] || [[ $FULL -eq 1 ]]; then
 
     print_git_info "trickest cve database" "EMBA-support-repos/trickest-cve" "Trickest CVE to github exploit database"
+    echo -e "$ORANGE""trickest poc database will be downloaded.""$NC"
     print_git_info "cve-search" "EMBA-support-repos/cve-search" "CVE-Search is a tool to import CVE and CPE into a database to facilitate search and processing of CVEs."
     echo -e "$ORANGE""cve-search will be downloaded.""$NC"
-    echo -e "$ORANGE""trickest poc database will be downloaded.""$NC"
 
     if [[ "$LIST_DEP" -eq 1 ]] || [[ $IN_DOCKER -eq 1 ]] ; then
       ANSWER=("n")
@@ -103,15 +103,7 @@ IF20_cve_search() {
     case ${ANSWER:0:1} in
       y|Y )
 
-        cd "$HOME_PATH" || ( echo "Could not install EMBA component Trickest" && exit 1 )
-        # get trickest repository
-        if ! [[ -d external/trickest-cve ]]; then
-          git clone https://github.com/EMBA-support-repos/trickest-cve.git external/trickest-cve
-        else
-          cd external/trickest-cve || ( echo "Could not install EMBA component Trickest" && exit 1 )
-          git pull
-          cd "$HOME_PATH" || ( echo "Could not install EMBA component Trickest" && exit 1 )
-        fi
+        cd "$HOME_PATH" || ( echo "Could not install EMBA component cve-search" && exit 1 )
 
         CVE_INST=1
         echo -e "\\n""$MAGENTA""Check if the cve-search database is already installed and populated.""$NC"
@@ -153,8 +145,9 @@ IF20_cve_search() {
             echo -e "\\n""$MAGENTA""cve-search database not ready.""$NC"
             echo -e "\\n""$MAGENTA""The installer is going to populate the database.""$NC"
           fi
-          # only update and install the database if we have no working database:
-          if [[ "$CVE_INST" -eq 1 ]]; then
+          # only update and install the database if we have no working database
+          # also do not update if we are running as github action (GH_ACTION set to 1)
+          if [[ "$GH_ACTION" -eq 0 ]] && [[ "$CVE_INST" -eq 1 ]]; then
             /etc/init.d/redis-server restart
             ./sbin/db_mgmt_cpe_dictionary.py -p || true
             ./sbin/db_mgmt_json.py -p || true
@@ -171,6 +164,15 @@ IF20_cve_search() {
           echo -e "$MAGENTA""$BOLD""For automatic updates it should be checked and copied to /etc/cron.daily/""$NC"
         fi
         cd "$HOME_PATH" || ( echo "Could not install EMBA component cve-search" && exit 1 )
+
+        # get trickest repository
+        if ! [[ -d external/trickest-cve ]]; then
+          git clone https://github.com/EMBA-support-repos/trickest-cve.git external/trickest-cve
+        else
+          cd external/trickest-cve || ( echo "Could not install EMBA component Trickest" && exit 1 )
+          git pull
+          cd "$HOME_PATH" || ( echo "Could not install EMBA component Trickest" && exit 1 )
+        fi
       ;;
     esac
   fi

installer/helpers.sh

@@ -20,8 +20,9 @@ print_help()
 {
   echo -e "\\n""$CYAN""USAGE""$NC"
   echo -e "$CYAN""-d""$NC""         Default installation of all dependencies needed for EMBA in default/docker mode (typical initial installation)"
-  echo -e "$CYAN""-F""$NC""         Developer installation (for running on your host in developer mode)"
   echo -e "$CYAN""-D""$NC""         Only used via docker-compose for building EMBA docker container"
+  echo -e "$CYAN""-F""$NC""         Developer installation (for running on your host in developer mode)"
+  echo -e "$CYAN""-g""$NC""         Install all dependecies for EMBA tests via Github actions (CVE-search database not populated)""$NC"
   echo -e "$CYAN""-h""$NC""         Print this help message"
   echo -e "$CYAN""-l""$NC""         List all dependencies of EMBA (deprecated)"
   echo -e "$CYAN""-r""$NC""         Remove a default installation of EMBA"

modules/F20_vul_aggregator.sh

@@ -500,10 +500,8 @@ cve_db_lookup_cve () {
   if [[ "$STRICT_MODE" -eq 1 ]]; then
     set +e
   fi
-  "$PATH_CVE_SEARCH" -c "$CVE_ENTRY" -o json | jq -rc '"\(.id):\(.cvss):\(.cvss3)"' | sort -t ':' -k3 -r > "$LOG_PATH_MODULE"/"$CVE_ENTRY".txt || true
 
-  # shellcheck disable=SC2181
-  if [[ "$?" -ne 0 ]]; then
+  if ! "$PATH_CVE_SEARCH" -c "$CVE_ENTRY" -o json | jq -rc '"\(.id):\(.cvss):\(.cvss3)"' | sort -t ':' -k3 -r > "$LOG_PATH_MODULE"/"$CVE_ENTRY".txt; then
     "$PATH_CVE_SEARCH" -c "$CVE_ENTRY" -o json | jq -rc '"\(.id):\(.cvss):\(.cvss3)"' | sort -t ':' -k3 -r > "$LOG_PATH_MODULE"/"$CVE_ENTRY".txt || true
   fi
   if [[ "$STRICT_MODE" -eq 1 ]]; then
@@ -528,15 +526,9 @@ cve_db_lookup_version() {
   local VERSION_PATH="${BIN_VERSION_//:/_}"
   print_output "[*] CVE database lookup with version information: ${ORANGE}$BIN_VERSION_${NC}" "no_log"
 
-  # CVE search:
-  set +e
-  "$PATH_CVE_SEARCH" -p "$BIN_VERSION_" -o json | jq -rc '"\(.id):\(.cvss):\(.cvss3)"' | sort -t ':' -k3 -r > "$LOG_PATH_MODULE"/"$VERSION_PATH".txt || true
-
-  # shellcheck disable=SC2181
-  if [[ "$?" -ne 0 ]]; then
+  if ! "$PATH_CVE_SEARCH" -p "$BIN_VERSION_" -o json | jq -rc '"\(.id):\(.cvss):\(.cvss3)"' | sort -t ':' -k3 -r > "$LOG_PATH_MODULE"/"$VERSION_PATH".txt; then
     "$PATH_CVE_SEARCH" -p "$BIN_VERSION_" -o json | jq -rc '"\(.id):\(.cvss):\(.cvss3)"' | sort -t ':' -k3 -r > "$LOG_PATH_MODULE"/"$VERSION_PATH".txt || true
   fi
-  set -e
 
   if [[ "$BIN_VERSION_" == *"dlink"* ]]; then
     # dlink extrawurst: dlink vs d-link