Merge pull request #815 from m-1-k-3/sys_emul_fs_mount

System emulation fs mount improvements

emba

@@ -516,11 +516,12 @@ main() {
       disown "${NOTIFICATION_PID}" 2> /dev/null || true
       print_output "[*] Original user: ${ORANGE}${SUDO_USER:-${USER}}${NC}" "no_log"
       print_output "[*] Notification process started with PID ${ORANGE}${NOTIFICATION_PID}${NC}" "no_log"
+      PROXY="$(sudo -E -u "${SUDO_USER:-${USER}}" env | grep -E "http(s)_proxy" | cut -d = -f2 || true)"
       echo "${SUDO_USER:-${USER}}" > "${LOG_DIR}"/orig_user.log
       {
         echo "UID: $(id -u "${SUDO_USER:-${USER}}")"
         echo "GID: $(id -g "${SUDO_USER:-${USER}}")"
-        echo "PROXY: $(sudo -E -u "${SUDO_USER:-${USER}}" env | grep -E "http(s)_proxy" | cut -d = -f2)"
+        echo "PROXY: ${PROXY}"
       } >> "${LOG_DIR}"/orig_user.log
     fi
 

helpers/fix_bins_lnk_emulation.sh

@@ -28,11 +28,13 @@ chmod +x "$ROOT_DIR"/busybox
 
 echo "[*] Identifying possible executable files"
 mapfile -t POSSIBLE_ELFS < <(find "$ROOT_DIR" -type f -exec file {} \; | grep "ELF\|executable" | cut -d: -f1)
+mapfile -t POSSIBLE_SH < <(find "$ROOT_DIR" -type f -name "*.sh")
+POSSIBLE_EXES_ARR=( "${POSSIBLE_ELFS[@]}" "${POSSIBLE_SH[@]}" )
 
-for POSSIBLE_ELF in "${POSSIBLE_ELFS[@]}"; do
-  [[ -x "${POSSIBLE_ELF}" ]] && continue
-  echo "[*] Processing executable $(basename "$POSSIBLE_ELF") - chmod privileges"
-  chmod +x "$POSSIBLE_ELF"
+for POSSIBLE_EXE in "${POSSIBLE_EXES_ARR[@]}"; do
+  [[ -x "${POSSIBLE_EXE}" ]] && continue
+  echo "[*] Processing executable $(basename "$POSSIBLE_EXE") - chmod privileges"
+  chmod +x "$POSSIBLE_EXE"
 done
 
 HOME_DIR="$(pwd)"

installer/IP00_extractors.sh

@@ -25,6 +25,7 @@ IP00_extractors(){
     print_pip_info "protobuf"
     print_pip_info "bsdiff4"
     print_git_info "payload_dumper" "EMBA-support-repos/payload_dumper" "Android OTA payload.bin extractor"
+    print_git_info "smcbmc" "EMBA-support-repos/smcbmc" "Supermicro BMC firmware image decryptor"
     # ubireader:
     # print_tool_info "python3-lzo" 1
     print_tool_info "liblzo2-dev" 1
@@ -64,6 +65,15 @@ IP00_extractors(){
           cd "$HOME_PATH" || ( echo "Could not install EMBA component payload dumper" && exit 1 )
         fi
 
+        if ! [[ -d external/smcbmc ]]; then
+          git clone https://github.com/EMBA-support-repos/smcbmc.git external/smcbmc
+        else
+          cd external/smcbmc || ( echo "Could not install EMBA component smcbmc" && exit 1 )
+          git pull
+          cd "$HOME_PATH" || ( echo "Could not install EMBA component smcbmc" && exit 1 )
+        fi
+
+
         if ! [[ -f "./external/buffalo-enc.elf" ]] ; then
           # Buffalo decryptor:
           download_file "buffalo-enc.c" "https://git-us.netdef.org/projects/OSR/repos/openwrt-buildroot/raw/tools/firmware-utils/src/buffalo-enc.c" "external/buffalo-enc.c"

modules/L10_system_emulation.sh

@@ -28,6 +28,11 @@ L10_system_emulation() {
   module_log_init "${FUNCNAME[0]}"
   module_title "System emulation of Linux based embedded devices."
 
+  # enable DEBUG_MODE for further debugging capabilities:
+  # * create_emulation_archive for all attempts
+  # * do not stop after 2 deteted network services
+  export DEBUG_MODE=0
+
   export SYS_ONLINE=0
   export TCP=""
   local MODULE_END=0
@@ -137,7 +142,11 @@ L10_system_emulation() {
 
             if [[ "$SYS_ONLINE" -eq 1 ]] && [[ "$TCP" == "ok" ]]; then
               # do not test other root paths if we are already online (some ports are available)
-              break
+              if [[ "${DEBUG_MODE}" -eq 1 ]]; then
+                print_output "[!] Debug mode: We do not stop here ..."
+              else
+                break
+              fi
             fi
           else
             print_output "[!] No supported architecture detected"
@@ -496,6 +505,7 @@ main_emulation() {
     fi
 
     # we deal with a startup script
+    local FS_MOUNTS_INIT=()
     if file "$MNT_POINT""$INIT_FILE" | grep -q "text executable\|ASCII text"; then
       INIT_OUT="$MNT_POINT""$INIT_FILE"
       find "$INIT_OUT" -xdev -maxdepth 1 -ls || true
@@ -504,7 +514,7 @@ main_emulation() {
       BAK_INIT_BACKUP="$LOG_PATH_MODULE"/"$(basename "$INIT_OUT".init)"
       cp -pr "$INIT_OUT" "$BAK_INIT_BACKUP"
 
-      mapfile -t FS_MOUNTS < <(grep -E "^mount\ -t\ .*\ .*mtd.* /.*" "$INIT_OUT" || true)
+      mapfile -t FS_MOUNTS_INIT < <(grep -E "^mount\ -t\ .*\ .*mtd.* /.*" "$INIT_OUT" | sort -u || true)
 
       # just in case we have issues with permissions
       chmod +x "$INIT_OUT"
@@ -513,6 +523,15 @@ main_emulation() {
       sed -i -r 's/(.*exit\ [0-9])$/\#\ \1/' "$INIT_OUT"
     fi
 
+    # Beside the check of init we also try to find other mounts for further filesystems
+    # probably we need to tweak this further to also find mounts in binaries - strings?!?
+    local FS_MOUNTS_FS=()
+    if [[ -d "${FIRMWARE_PATH}" ]]; then
+      mapfile -t FS_MOUNTS_FS < <(find "${FIRMWARE_PATH}"  -xdev -type f -exec grep -a -h -E "^mount\ -t\ .*\ .*mtd.* /.*" {} \; 2>/dev/null | sort -u || true)
+    fi
+
+    FS_MOUNTS=( "${FS_MOUNTS_INIT[@]}" "${FS_MOUNTS_FS[@]}" )
+    eval "FS_MOUNTS=($(for i in "${FS_MOUNTS[@]}" ; do echo "\"$i\"" ; done | sort -u))"
     handle_fs_mounts "${FS_MOUNTS[@]}"
 
     print_output "[*] Add network.sh entry to $ORANGE$INIT_OUT$NC"
@@ -802,14 +821,20 @@ main_emulation() {
               #  print_output "[-] No startup script ${ORANGE}$ARCHIVE_PATH/run.sh${NC} found - this should not be possible!"
               #  reset_network_emulation 2
               # fi
-              break 2
+              if [[ "${DEBUG_MODE}" -ne 1 ]]; then
+                break 2
+              fi
             fi
           fi
         else
           print_output "[-] No working emulation - removing emulation archive."
-          # print_output "[-] Emulation archive: $ARCHIVE_PATH."
-          # create_emulation_archive "$ARCHIVE_PATH"
-          rm -r "$ARCHIVE_PATH" || true
+          if [[ "${DEBUG_MODE}" -ne 1 ]]; then
+            create_emulation_archive "$ARCHIVE_PATH"
+          else
+            # print_output "[-] Emulation archive: $ARCHIVE_PATH."
+            # create_emulation_archive "$ARCHIVE_PATH"
+            rm -r "$ARCHIVE_PATH" || true
+          fi
         fi
 
         stopping_emulation_process "$IMAGE_NAME"
@@ -875,8 +900,19 @@ handle_fs_mounts() {
     # as the original mount will not work, we need to remove it from the startup file:
     sed -i 's|'"$FS_MOUNT"'|\#'"$FS_MOUNT"'|g' "$MNT_POINT""$INIT_FILE"
 
-    MOUNT_PT=$(echo "$FS_MOUNT" | awk '{print $NF}')
+    MOUNT_PT=$(echo "$FS_MOUNT" | awk '{print $5}')
     MOUNT_FS=$(echo "$FS_MOUNT" | grep " \-t " | sed 's/.*-t //g' | awk '{print $1}')
+    if [[ "${MOUNT_FS}" != *"jffs"* ]] && [[ "${MOUNT_FS}" != *"cramfs"* ]]; then
+      print_output "[-] Warning: ${ORANGE}${MOUNT_FS}${NC} filesystem currently not supported"
+      print_output "[-] Warning: If further results are wrong please open a ticket"
+    fi
+    if [[ "${MOUNT_PT}" != *"/"* ]]; then
+      MOUNT_PT=$(echo "$FS_MOUNT" | awk '{print $NF}')
+      if [[ "${MOUNT_PT}" != *"/"* ]]; then
+        print_output "[-] Warning: Mount point ${ORANGE}${MOUNT_PT}${NC} currently not supported"
+        print_output "[-] Warning: If further results are wrong please open a ticket"
+      fi
+    fi
     # we test for paths including the MOUNT_FS part like "jffs2" in the path
     FS_FIND=$(find "$LOG_DIR"/firmware -path "*/*$MOUNT_FS*_extract" | head -1 || true)
 
@@ -930,6 +966,11 @@ handle_fs_mounts() {
       cp -prn "$N_PATH"* "$MNT_POINT""$MOUNT_PT"
       find "$MNT_POINT""$MOUNT_PT" -xdev -ls || true
     done
+
+    print_output "[*] Final copy of ${ORANGE}${FS_FIND}${NC} to ${ORANGE}${MNT_POINT}${MOUNT_PT}${NC} ..."
+    cp -prn "${FS_FIND}"/* "${MNT_POINT}""${MOUNT_PT}"
+    # find "$MNT_POINT""$MOUNT_PT" -xdev -ls || true
+    ls -lh "${MNT_POINT}""${MOUNT_PT}"
   done
 
   # Todo: move this to somewhere, where we only need to do this once

modules/L10_system_emulation/inferService.sh

@@ -131,4 +131,6 @@ for BINARY in $("${BUSYBOX}" find / -name "lighttpd" -type f -o -name "upnp" -ty
   fi
 done
 
+"${BUSYBOX}" sort -u -o /firmadyne/service /firmadyne/service
+
 "${BUSYBOX}" echo "[*] EMBA inferService script finished ..."

modules/L22_upnp_hnap_checks.sh

@@ -126,8 +126,8 @@ check_basic_hnap_jnap() {
         curl -v -L --noproxy '*' --max-redir 0 -f -m 5 -s -X POST -H "${JNAP_ACTION}" -d "{}" http://"${IP_ADDRESS_}":"${PORT}"/JNAP/ >> "$LOG_PATH_MODULE"/jnap-discovery-check.txt || true
       else
         # HNAP - SSL
-        curl -v -L --noproxy '*' --max-redir 0 -f -m 5 -s -X GET https://"${IP_ADDRESS_}":"${PORT}"/HNAP/ >> "$LOG_PATH_MODULE"/hnap-discovery-check.txt || true
-        curl -v -L --noproxy '*' --max-redir 0 -f -m 5 -s -X GET https://"${IP_ADDRESS_}":"${PORT}"/HNAP1/ >> "$LOG_PATH_MODULE"/hnap-discovery-check.txt || true
+        curl -v -L -k --noproxy '*' --max-redir 0 -f -m 5 -s -X GET https://"${IP_ADDRESS_}":"${PORT}"/HNAP/ >> "$LOG_PATH_MODULE"/hnap-discovery-check.txt || true
+        curl -v -L -k --noproxy '*' --max-redir 0 -f -m 5 -s -X GET https://"${IP_ADDRESS_}":"${PORT}"/HNAP1/ >> "$LOG_PATH_MODULE"/hnap-discovery-check.txt || true
         # JNAP - SSL
         curl -v -L --noproxy '*' --max-redir 0 -f -m 5 -s -X POST -H "${JNAP_ACTION}" -d "{}" https://"${IP_ADDRESS_}":"${PORT}"/JNAP/ >> "$LOG_PATH_MODULE"/jnap-discovery-check.txt || true
       fi

modules/S108_stacs_password_search.sh

@@ -37,7 +37,7 @@ S108_stacs_password_search()
     if [[ -f "$TMP_DIR"/stacs.err ]]; then
       print_ln
       print_output "[*] STACS log:"
-      tee -a "$LOG_FILE" < "$TMP_DIR"/stacs.err
+      tee -a "$LOG_FILE" < "$TMP_DIR"/stacs.err || true
     fi
 
     if [[ -f "$STACS_LOG_FILE" && $(jq ".runs[0] .results[] | .message[]" "$STACS_LOG_FILE" | wc -l) -gt 0 ]]; then