Lua script analysis support, UPnP live module, improvements

installer/I20_sourcecode_check.sh

@@ -29,6 +29,7 @@ I20_sourcecode_check() {
 
     print_tool_info "shellcheck" 1
     print_tool_info "php" 1
+    print_tool_info "luarocks" 1
     print_pip_info "semgrep"
     print_git_info "semgrep-rules" "returntocorp/semgrep-rules" "Standard library for Semgrep rules"
 
@@ -37,14 +38,16 @@ I20_sourcecode_check() {
     if [[ "$LIST_DEP" -eq 1 ]] || [[ $DOCKER_SETUP -eq 1 ]] ; then
       ANSWER=("n")
     else
-      echo -e "\\n""$MAGENTA""$BOLD""Composer, iniscan and semgrep (if not already on the system) will be downloaded!""$NC"
+      echo -e "\\n""$MAGENTA""$BOLD""Composer, iniscan, luacheck  and semgrep (if not already on the system) will be downloaded!""$NC"
       ANSWER=("y")
     fi
 
     case ${ANSWER:0:1} in
       y|Y )
         apt-get install "${INSTALL_APP_LIST[@]}" -y --no-install-recommends
 
+        luarocks install luacheck
+
         pip_install "semgrep"
         if ! [[ -d external/semgrep-rules ]]; then
           git clone https://github.com/returntocorp/semgrep-rules.git external/semgrep-rules

installer/IL15_emulated_checks_init.sh

@@ -37,6 +37,8 @@ IL15_emulated_checks_init() {
       echo -e "$RED""$BOLD""Not installing snmpcheck. Your EMBA installation will be incomplete""$NC"
     fi
     print_tool_info "python3-pip" 1
+    # mini UPnP client
+    print_tool_info "miniupnpc" 1
     print_tool_info "cutycapt" 1
 
     # needed for cutycapt

modules/L10_system_emulation.sh

@@ -1180,6 +1180,10 @@ get_networking_details_emulation() {
     mapfile -t PORTS < <(grep -a "inet_bind" "$LOG_PATH_MODULE"/qemu.initial.serial.log | sed -E 's/.*inet_bind\[PID:\ [0-9]+\ //' | sort -u || true)
     mapfile -t VLAN_HW_INFO_DEV < <(grep -a -E "adding VLAN [0-9] to HW filter on device eth[0-9]" "$LOG_PATH_MODULE"/qemu.initial.serial.log | awk -F\  '{print $NF}' | sort -u || true)
 
+    # we handle missing files in setup_network_config -> there we already remount the filesystem and we can perform the changes
+    mapfile -t MISSING_FILES_TMP < <(grep -a -E "No such file or directory" "$LOG_PATH_MODULE"/qemu.initial.serial.log | tr ' ' '\n' | grep "/" | grep -v proc | tr -d ':' | sort -u || true)
+    MISSING_FILES+=( "${MISSING_FILES_TMP[@]}" )
+
     NVRAM_TMP=( "${NVRAM[@]}" )
 
     if [[ "${#INTERFACE_CANDIDATES[@]}" -gt 0 || "${#BRIDGE_INTERFACES[@]}" -gt 0 || "${#VLAN_INFOS[@]}" -gt 0 || "${#PORTS[@]}" -gt 0 || "${#NVRAM_TMP[@]}" -gt 0 ]]; then
@@ -1582,6 +1586,28 @@ write_network_config_to_filesystem() {
 
     set_network_config "$IP_ADDRESS_" "$NETWORK_MODE" "$NETWORK_DEVICE" "$ETH_INT"
 
+    # if there were missing files found -> we try to fix this now
+    if [[ -v MISSING_FILES[@] ]]; then
+      for FILE_PATH_MISSING in "${MISSING_FILES[@]}"; do
+        print_output "[!] MISSING_FILE: ${FILE_PATH_MISSING}"
+        [[ "${FILE_PATH_MISSING}" == *"/proc/"* ]] && continue
+        [[ "${FILE_PATH_MISSING}" == *"/sys/"* ]] && continue
+
+        FILENAME_MISSING=$(basename "${FILE_PATH_MISSING}")
+        print_output "[*] Found missing area ${ORANGE}${FILENAME_MISSING}${NC} in filesystem ... trying to fix this now"
+        DIR_NAME_MISSING=$(dirname "${FILE_PATH_MISSING}")
+        if ! [[ -d "${MNT_POINT}""${DIR_NAME_MISSING}" ]]; then
+          print_output "[*] Create missing directory ${ORANGE}${DIR_NAME_MISSING}${NC} in filesystem ... trying to fix this now"
+          mkdir -p "${MNT_POINT}""${DIR_NAME_MISSING}"
+        fi
+        FOUND_MISSING=$(find "${MNT_POINT}" -name "${FILENAME_MISSING}" | head -1)
+        if [[ -f ${FOUND_MISSING} ]]; then
+          print_output "[*] Recover missing file ${ORANGE}${FILENAME_MISSING}${NC} in filesystem ... trying to fix this now"
+          cp "${FOUND_MISSING}" "${MNT_POINT}""${DIR_NAME_MISSING}"/
+        fi
+      done
+    fi
+
     # umount filesystem:
     umount_qemu_image "$DEVICE"
   fi

modules/L10_system_emulation/inferService.sh

@@ -30,6 +30,14 @@ if [ -e /etc/init.d/lighttpd ]; then
   fi
 fi
 
+if [ -e /etc/init.d/lighttpd.sh ]; then
+  if ! "${BUSYBOX}" grep -q "/etc/init.d/lighttpd.sh" /firmadyne/service 2>/dev/null; then
+    "${BUSYBOX}" echo -e "[*] Writing EMBA service for ${ORANGE}lighttpd service${NC}"
+    "${BUSYBOX}" echo -e -n "/etc/init.d/lighttpd.sh start\n" >> /firmadyne/service
+  fi
+fi
+
+
 if [ -e /etc/init.d/ftpd ]; then
   if ! "${BUSYBOX}" grep -q ftpd /firmadyne/service 2>/dev/null; then
     "${BUSYBOX}" echo -e "[*] Writing EMBA service for ${ORANGE}ftpd service${NC}"

modules/L22_upnp_checks.sh

@@ -0,0 +1,80 @@
+#!/bin/bash -p
+
+# EMBA - EMBEDDED LINUX ANALYZER
+#
+# Copyright 2020-2023 Siemens Energy AG
+#
+# EMBA comes with ABSOLUTELY NO WARRANTY. This is free software, and you are
+# welcome to redistribute it under the terms of the GNU General Public License.
+# See LICENSE file for usage of this software.
+#
+# EMBA is licensed under GPLv3
+#
+# Author(s): Michael Messner
+
+# Description:  Tests the emulated live system which is build and started in L10
+#               Currently this is an experimental module and needs to be activated separately via the -Q switch. 
+#               It is also recommended to only use this technique in a dockerized or virtualized environment.
+
+L22_upnp_checks() {
+
+  export UPNP_UP=0
+
+  if [[ "$SYS_ONLINE" -eq 1 ]] && [[ "$TCP" == "ok" ]]; then
+    module_log_init "${FUNCNAME[0]}"
+    module_title "Live UPnP tests of emulated device."
+    pre_module_reporter "${FUNCNAME[0]}"
+
+    if [[ $IN_DOCKER -eq 0 ]] ; then
+      print_output "[!] This module should not be used in developer mode and could harm your host environment."
+    fi
+
+    if [[ -v IP_ADDRESS_ ]]; then
+      if ! ping -c 2 "$IP_ADDRESS_" &> /dev/null; then
+        restart_emulation "$IP_ADDRESS_" "$IMAGE_NAME"
+        if ! ping -c 2 "$IP_ADDRESS_" &> /dev/null; then
+          print_output "[-] System not responding - Not performing UPnP checks"
+          module_end_log "${FUNCNAME[0]}" "$UPNP_UP"
+          return
+        fi
+      fi
+      if [[ -v HOSTNETDEV_0 ]]; then
+        check_basic_upnp "$HOSTNETDEV_0"
+      else
+        print_output "[!] No network interface found"
+      fi
+    else
+      print_output "[!] No IP address found"
+    fi
+
+    write_log ""
+    write_log "Statistics:$UPNP_UP"
+    module_end_log "${FUNCNAME[0]}" "$UPNP_UP"
+  fi
+}
+
+check_basic_upnp() {
+  local INTERFACE="${1:-}"
+
+  sub_module_title "UPnP enumeration for emulated system with IP $ORANGE$IP_ADDRESS_$NC"
+
+  if command -v upnpc > /dev/null; then
+    print_output "[*] UPnP scan with upnpc"
+    upnpc -m "$INTERFACE" -P >> "$LOG_PATH_MODULE"/upnp-discovery-check.txt || true
+    if [[ -f "$LOG_PATH_MODULE"/upnp-discovery-check.txt ]]; then
+      print_ln
+      tee -a "$LOG_FILE" < "$LOG_PATH_MODULE"/upnp-discovery-check.txt
+    fi
+    print_ln
+
+    UPNP_UP=$(grep -c "desc\|IGD" "$LOG_PATH_MODULE"/upnp-discovery-check.txt)
+  fi
+
+  if [[ "$UPNP_UP" -gt 0 ]]; then
+    UPNP_UP=1
+  fi
+
+  print_ln
+  print_output "[*] UPnP basic enumeration finished"
+}
+

modules/L25_web_checks.sh

@@ -226,33 +226,41 @@ web_access_crawler() {
   sub_module_title "Starting web server crawling for $ORANGE$IP_:$PORT$NC"
   print_ln
 
-  # we need files and links (for cgi files)
-  mapfile -t FILE_ARR_EXT < <(find "$FIRMWARE_PATH" -type f -o -type l || true)
-
-  for WEB_PATH in "${FILE_ARR_EXT[@]}"; do
-    if ! ping -c 1 "$IP_" &> /dev/null; then
-      print_output "[-] System not responding - Stopping crawling"
-      break
-    fi
-    print_dot
-    WEB_FILE="$(basename "$WEB_PATH")"
-    echo -e "\\n[*] Testing $ORANGE$PROTO://$IP_:$PORT_/$WEB_FILE$NC" >> "$LOG_PATH_MODULE/crawling_$IP_-$PORT_.log"
-    timeout --preserve-status --signal SIGINT 2 curl "${CURL_OPTS[@]}" - "$PROTO""://""$IP_":"$PORT_""/""$WEB_FILE" -o /dev/null >> "$LOG_PATH_MODULE/crawling_$IP_-$PORT_.log" 2>/dev/null || true
-    WEB_DIR_L1="$(dirname "$WEB_PATH" | rev | cut -d'/' -f1 | rev)"
-    if [[ -n "${WEB_DIR_L1}" ]]; then
-      echo -e "\\n[*] Testing $ORANGE$PROTO://$IP_:$PORT_/${WEB_DIR_L1}/${WEB_FILE}$NC" >> "$LOG_PATH_MODULE/crawling_$IP_-$PORT_.log"
-      timeout --preserve-status --signal SIGINT 2 curl "${CURL_OPTS[@]}" - "$PROTO""://""$IP_":"$PORT_""/""${WEB_DIR_L1}""/""$WEB_FILE" -o /dev/null >> "$LOG_PATH_MODULE/crawling_$IP_-$PORT_.log" 2>/dev/null || true
-    fi
-    WEB_DIR_L2="$(dirname "$WEB_PATH" | rev | cut -d'/' -f1-2 | rev)"
-    if [[ -n "${WEB_DIR_L2}" ]]; then
-      echo -e "\\n[*] Testing $ORANGE$PROTO://$IP_:$PORT_/${WEB_DIR_L2}/${WEB_FILE}$NC" >> "$LOG_PATH_MODULE/crawling_$IP_-$PORT_.log"
-      timeout --preserve-status --signal SIGINT 2 curl "${CURL_OPTS[@]}" - "$PROTO""://""$IP_":"$PORT_""/""${WEB_DIR_L2}""/""$WEB_FILE" -o /dev/null >> "$LOG_PATH_MODULE/crawling_$IP_-$PORT_.log" 2>/dev/null || true
-    fi
-    WEB_DIR_L3="$(dirname "$WEB_PATH" | rev | cut -d'/' -f1-3 | rev)"
-    if [[ -n "${WEB_DIR_L3}" ]]; then
-      echo -e "\\n[*] Testing $ORANGE$PROTO://$IP_:$PORT_/${WEB_DIR_L3}/${WEB_FILE}$NC" >> "$LOG_PATH_MODULE/crawling_$IP_-$PORT_.log"
-      timeout --preserve-status --signal SIGINT 2 curl "${CURL_OPTS[@]}" - "$PROTO""://""$IP_":"$PORT_""/""${WEB_DIR_L3}""/""$WEB_FILE" -o /dev/null >> "$LOG_PATH_MODULE/crawling_$IP_-$PORT_.log" 2>/dev/null || true
-    fi
+  for R_PATH in "${ROOT_PATH[@]}" ; do
+    # we need files and links (for cgi files)
+    mapfile -t FILE_ARR_EXT < <(find "$R_PATH" -type f -o -type l || true)
+
+    for WEB_PATH in "${FILE_ARR_EXT[@]}"; do
+      if ! ping -c 1 "$IP_" &> /dev/null; then
+        print_output "[-] System not responding - Stopping crawling"
+        break
+      fi
+      print_dot
+      WEB_FILE="$(basename "$WEB_PATH")"
+      echo -e "\\n[*] Testing $ORANGE$PROTO://$IP_:$PORT_/$WEB_FILE$NC" >> "$LOG_PATH_MODULE/crawling_$IP_-$PORT_.log"
+      timeout --preserve-status --signal SIGINT 2 curl "${CURL_OPTS[@]}" - "$PROTO""://""$IP_":"$PORT_""/""$WEB_FILE" -o /dev/null >> "$LOG_PATH_MODULE/crawling_$IP_-$PORT_.log" 2>/dev/null || true
+      WEB_DIR_L1="$(dirname "$WEB_PATH" | rev | cut -d'/' -f1 | rev)"
+      if [[ -n "${WEB_DIR_L1}" ]]; then
+        WEB_DIR_L1="${WEB_DIR_L1#\.}"
+        WEB_DIR_L1="${WEB_DIR_L1#\/}"
+        echo -e "\\n[*] Testing $ORANGE$PROTO://$IP_:$PORT_/${WEB_DIR_L1}/${WEB_FILE}$NC" >> "$LOG_PATH_MODULE/crawling_$IP_-$PORT_.log"
+        timeout --preserve-status --signal SIGINT 2 curl "${CURL_OPTS[@]}" - "$PROTO""://""$IP_":"$PORT_""/""${WEB_DIR_L1}""/""$WEB_FILE" -o /dev/null >> "$LOG_PATH_MODULE/crawling_$IP_-$PORT_.log" 2>/dev/null || true
+      fi
+      WEB_DIR_L2="$(dirname "$WEB_PATH" | rev | cut -d'/' -f1-2 | rev)"
+      if [[ -n "${WEB_DIR_L2}" ]] && [[ "${WEB_DIR_L2}" != "${WEB_DIR_L1}" ]]; then
+        WEB_DIR_L2="${WEB_DIR_L2#\.}"
+        WEB_DIR_L2="${WEB_DIR_L2#\/}"
+        echo -e "\\n[*] Testing $ORANGE$PROTO://$IP_:$PORT_/${WEB_DIR_L2}/${WEB_FILE}$NC" >> "$LOG_PATH_MODULE/crawling_$IP_-$PORT_.log"
+        timeout --preserve-status --signal SIGINT 2 curl "${CURL_OPTS[@]}" - "$PROTO""://""$IP_":"$PORT_""/""${WEB_DIR_L2}""/""$WEB_FILE" -o /dev/null >> "$LOG_PATH_MODULE/crawling_$IP_-$PORT_.log" 2>/dev/null || true
+      fi
+      WEB_DIR_L3="$(dirname "$WEB_PATH" | rev | cut -d'/' -f1-3 | rev)"
+      if [[ -n "${WEB_DIR_L3}" ]] && [[ "${WEB_DIR_L3}" != "${WEB_DIR_L2}" ]] && [[ "${WEB_DIR_L3}" != "${WEB_DIR_L1}" ]]; then
+        WEB_DIR_L3="${WEB_DIR_L3#\.}"
+        WEB_DIR_L3="${WEB_DIR_L3#\/}"
+        echo -e "\\n[*] Testing $ORANGE$PROTO://$IP_:$PORT_/${WEB_DIR_L3}/${WEB_FILE}$NC" >> "$LOG_PATH_MODULE/crawling_$IP_-$PORT_.log"
+        timeout --preserve-status --signal SIGINT 2 curl "${CURL_OPTS[@]}" - "$PROTO""://""$IP_":"$PORT_""/""${WEB_DIR_L3}""/""$WEB_FILE" -o /dev/null >> "$LOG_PATH_MODULE/crawling_$IP_-$PORT_.log" 2>/dev/null || true
+      fi
+    done
   done
 
   if [[ -f "$LOG_PATH_MODULE/crawling_$IP_-$PORT_.log" ]]; then

modules/S23_lua_check.sh

@@ -0,0 +1,138 @@
+#!/bin/bash -p
+
+# EMBA - EMBEDDED LINUX ANALYZER
+#
+# Copyright 2020-2023 Siemens Energy AG
+#
+# EMBA comes with ABSOLUTELY NO WARRANTY. This is free software, and you are
+# welcome to redistribute it under the terms of the GNU General Public License.
+# See LICENSE file for usage of this software.
+#
+# EMBA is licensed under GPLv3
+#
+# Author(s): Michael Messner
+
+# Description:  Checks for bugs, stylistic errors, etc. in lua scripts
+
+S23_lua_check()
+{
+  module_log_init "${FUNCNAME[0]}"
+  module_title "Check lua scripts for security issues"
+  pre_module_reporter "${FUNCNAME[0]}"
+
+  local S23_LUA_VULNS=0
+  local LUA_SCRIPT=""
+  local S23_LUA_SCRIPTS=()
+
+  write_csv_log "Script path" "LUA issues detected" "LUA vulnarabilities detected" "common linux file"
+  mapfile -t S23_LUA_SCRIPTS < <(find "$FIRMWARE_PATH" -xdev -type f -iname "*.lua" -exec md5sum {} \; 2>/dev/null | sort -u -k1,1 | cut -d\  -f3 )
+
+  sub_module_title "LUA linter checks module"
+
+  for LUA_SCRIPT in "${S23_LUA_SCRIPTS[@]}" ; do
+    if [[ "$THREADED" -eq 1 ]]; then
+      # linting check:
+      s23_luacheck "$LUA_SCRIPT" &
+      local TMP_PID="$!"
+      store_kill_pids "$TMP_PID"
+      WAIT_PIDS_S23+=( "$TMP_PID" )
+      max_pids_protection "$MAX_MOD_THREADS" "${WAIT_PIDS_S23[@]}"
+      continue
+    else
+      s23_luacheck "$LUA_SCRIPT"
+    fi
+  done
+
+  [[ "$THREADED" -eq 1 ]] && wait_for_pid "${WAIT_PIDS_S23[@]}"
+
+  # simple lua checks to identify files which should be analysed in more detail
+  print_ln
+  s23_luaseccheck
+
+  if [[ "$S23_LUA_VULNS" -gt 0 ]]; then
+    print_ln
+    print_output "[+] Found ""$ORANGE""$S23_LUA_VULNS"" security issues""$GREEN"" in ""$ORANGE""${#LUA_CGI_FILES[@]}""$GREEN"" lua files""$NC""\\n"
+  fi
+
+  write_log ""
+  write_log "[*] Statistics:$S23_LUA_VULNS:${#LUA_CGI_FILES[@]}"
+  module_end_log "${FUNCNAME[0]}" "$S23_LUA_VULNS"
+}
+
+# this is a very basic checker for LUA issues
+s23_luaseccheck() {
+  local NAME=""
+  local LUA_LOG=""
+
+  sub_module_title "LUA Security checks module"
+
+  mapfile -t LUA_CGI_FILES < <(find "${FIRMWARE_PATH}" -type f -exec grep -H cgilua\. {} \; 2>/dev/null | cut -d ':' -f1 | sort -u)
+
+  for QUERY_FILE in "${LUA_CGI_FILES[@]}"; do
+    local ISSUES_FILE=0
+
+    mapfile -t QUERY_ENTRIES < <(grep -E "=.*cgilua\.QUERY" "${QUERY_FILE}" | tr ' ' '\n' | sed 's/.*cgilua.QUERY.//' \
+       | sed 's/.*cgilua.QUERY.//' | grep -o -E "^[[:alnum:]]+" | grep -v "^local$" | sort -u || true)
+
+    for ENTRY in "${QUERY_ENTRIES[@]}"; do
+      ENTRY="$(echo "$ENTRY" | tr -dc '[:print:]')"
+      [[ -z "$ENTRY" ]] && continue
+      ! [[ "$ENTRY" =~ ^[a-zA-Z0-9_-]+$ ]] && continue
+
+      if grep "$ENTRY" "${QUERY_FILE}" | grep -E -q "io\.(p)?open"; then
+        # possible file access
+        S23_LUA_VULNS=$((S23_LUA_VULNS+1))
+        ISSUES_FILE=$((ISSUES_FILE+1))
+        print_output "[+] Found lua QUERY (GET/POST) entry: ${ORANGE}${ENTRY}${GREEN} in file ${ORANGE}${QUERY_FILE}${GREEN} with file access capabilities."
+      fi
+      if grep "$ENTRY" "${QUERY_FILE}" | grep -q "os.execute"; then
+        # command exec - critical
+        S23_LUA_VULNS=$((S23_LUA_VULNS+1))
+        ISSUES_FILE=$((ISSUES_FILE+1))
+        print_output "[+] Found lua QUERY (GET/POST) entry: ${ORANGE}${ENTRY}${GREEN} in file ${ORANGE}${QUERY_FILE}${GREEN} with command execution capabilities."
+      fi
+    done
+    if [[ "${ISSUES_FILE}" -eq 0 ]] && grep -q "os.execute" "${QUERY_FILE}"; then
+      # command exec - not our parameter but we check it
+      print_output "[*] Found lua file ${ORANGE}${QUERY_FILE}${NC} with possible command execution for review."
+    fi
+    if [[ "${ISSUES_FILE}" -eq 0 ]] && grep -E -q "io\.(p)?open" "${QUERY_FILE}"; then
+      # command exec - not our parameter but we check it
+      print_output "[*] Found lua file ${ORANGE}${QUERY_FILE}${NC} with possible file access for review."
+    fi
+
+    if [[ "${ISSUES_FILE}" -gt 0 ]]; then
+      write_csv_log "$(print_path "$QUERY_FILE")" "0" "$ISSUES_FILE" "NA"
+    fi
+  done
+}
+
+s23_luacheck() {
+  local LUA_SCRIPT_="${1:-}"
+  local NAME=""
+  local LUA_LOG=""
+
+  NAME=$(basename "$LUA_SCRIPT_" 2> /dev/null | sed -e 's/:/_/g')
+  LUA_LOG="$LOG_PATH_MODULE""/luacheck_""$NAME"".txt"
+  luacheck "$LUA_SCRIPT_" > "$LUA_LOG" 2> /dev/null || true
+
+  ISSUES=$(strip_color_codes "$(grep Total "$LUA_LOG" | awk '{print $2}' 2> /dev/null || true)")
+  if [[ "$ISSUES" -gt 0 ]] ; then
+    # check if this is common linux file:
+    local COMMON_FILES_FOUND
+    local CFF
+    if [[ -f "$BASE_LINUX_FILES" ]]; then
+      COMMON_FILES_FOUND="(""${RED}""common linux file: no""${GREEN}"")"
+      CFF="no"
+      if grep -q "^$NAME\$" "$BASE_LINUX_FILES" 2>/dev/null; then
+        COMMON_FILES_FOUND="(""${CYAN}""common linux file: yes""${GREEN}"")"
+        CFF="yes"
+      fi
+    else
+      COMMON_FILES_FOUND=""
+      CFF="NA"
+    fi
+    print_output "[+] Found ""$ORANGE""$ISSUES"" coding issues""$GREEN"" in lua script ""$COMMON_FILES_FOUND"":""$NC"" ""$(print_path "$LUA_SCRIPT_")" "" "$LUA_LOG"
+    write_csv_log "$(print_path "$LUA_SCRIPT_")" "$ISSUES" "0" "$CFF"
+  fi
+}