Grepit and semgrep improvements

helpers/helpers_emba_print.sh

@@ -221,10 +221,8 @@ write_csv_log() {
   CSV_LOG="${LOG_FILE_NAME/\.txt/\.csv}"
   CSV_LOG="$CSV_DIR""/""$CSV_LOG"
 
-  (
-  IFS=\;
-    echo -e "${CSV_ITEMS[*]}" | tee -a "$CSV_LOG" >/dev/null
-  )
+  printf '%s;' "${CSV_ITEMS[@]}" | tee -a "$CSV_LOG" >/dev/null
+  printf '\n' | tee -a "$CSV_LOG" >/dev/null
 }
 
 write_grep_log()

modules/F50_base_aggregator.sh

@@ -847,8 +847,7 @@ get_data() {
   fi
   if [[ -f "$F20_EXPLOITS_LOG" ]]; then
     #EXPLOIT_COUNTER="$(grep -c -E "Exploit\ .*" "$F20_EXPLOITS_LOG" || true)"
-    # shellcheck disable=SC2126
-    EXPLOIT_COUNTER="$(grep -E "Exploit\ .*" "$F20_EXPLOITS_LOG" | grep -v "Exploit summary" | wc -l || true)"
+    EXPLOIT_COUNTER="$(grep -E "Exploit\ .*" "$F20_EXPLOITS_LOG" | grep -cv "Exploit summary" || true)"
     MSF_MODULE_CNT="$(grep -c -E "Exploit\ .*MSF" "$F20_EXPLOITS_LOG" || true)"
     REMOTE_EXPLOIT_CNT="$(grep -c -E "Exploit\ .*\ \(R\)" "$F20_EXPLOITS_LOG" || true)"
     LOCAL_EXPLOIT_CNT="$(grep -c -E "Exploit\ .*\ \(L\)" "$F20_EXPLOITS_LOG" || true)"

modules/L10_system_emulation.sh

@@ -270,8 +270,7 @@ create_emulation_filesystem() {
         if ! [[ -d "$LOG_PATH_MODULE"/nvram ]]; then
           mkdir "$LOG_PATH_MODULE"/nvram
         fi
-        # shellcheck disable=SC2001
-        NVRAM_FILE=$(echo "$NVRAM_FILE" | sed -e 's/^\.//')
+        NVRAM_FILE="${NVRAM_FILE/\.}"
         print_output "[*] Found possible NVRAM default file $ORANGE$NVRAM_FILE$NC -> setup /firmadyne directory"
         echo "$NVRAM_FILE" >> "$LOG_PATH_MODULE"/nvram/nvram_files
         cp ."$NVRAM_FILE" "$LOG_PATH_MODULE"/nvram/
@@ -350,7 +349,7 @@ main_emulation() {
     # we deal with a startup script
     if file "$MNT_POINT""$INIT_FILE" | grep -q "text executable\|ASCII text"; then
       INIT_OUT="$MNT_POINT""$INIT_FILE"
-      find "$INIT_OUT" -xdev -ls || true
+      find "$INIT_OUT" -xdev -maxdepth 1 -ls || true
       print_output "[*] Backup original init file $ORANGE$INIT_OUT$NC"
       BAK_INIT_ORIG="$INIT_OUT"
       BAK_INIT_BACKUP="$LOG_PATH_MODULE"/"$(basename "$INIT_OUT".init)"
@@ -384,12 +383,10 @@ main_emulation() {
 
     print_ln
     print_output "[*] FirmAE filesytem:"
-    find "$MNT_POINT" -xdev -ls | tee -a "$LOG_FILE" || true
+    find "$MNT_POINT" -xdev -maxdepth 1 -ls | tee -a "$LOG_FILE" || true
 
     print_ln
     print_output "[*] FirmAE firmadyne directory:"
-    # shellcheck disable=SC2012
-    #ls -l "$MNT_POINT/firmadyne" | tee -a "$LOG_FILE"
     find "$MNT_POINT"/firmadyne -xdev -ls | tee -a "$LOG_FILE" || true
     print_ln
 
@@ -940,8 +937,7 @@ get_networking_details_emulation() {
     if [[ -v NVRAM_TMP[@] ]]; then
       for NVRAM_ENTRY in "${NVRAM_TMP[@]}"; do
         if [[ "$NVRAM_ENTRY" =~ [[:print:]] ]]; then
-          # shellcheck disable=SC2076
-          if [[ ! " ${NVRAMS[*]} " =~ " $NVRAM_ENTRY " ]]; then
+          if [[ ! " ${NVRAMS[*]} " =~  $NVRAM_ENTRY  ]]; then
             NVRAMS+=( "$NVRAM_ENTRY" )
           fi
         fi
@@ -1007,8 +1003,8 @@ get_networking_details_emulation() {
         fi
       done
 
-      #shellcheck disable=SC2001
-      IP_="$(echo "$IP_" | sed 's/^\.//')"
+      #IP_="$(echo "$IP_" | sed 's/^\.//')"
+      IP_="${IP_/\.}"
 
       IP_ADDRESS_=""
       if [[ "$D_END" == "eb" ]]; then
@@ -1353,8 +1349,8 @@ nvram_searcher_emulation() {
       fi
     done
     if [[ "$COUNT" -gt 0 ]]; then
-      #shellcheck disable=SC2001
-      NVRAM_FILE=$(echo "$NVRAM_FILE" | sed 's/^\.//')
+      #NVRAM_FILE=$(echo "$NVRAM_FILE" | sed 's/^\.//')
+      NVRAM_FILE="${NVRAM_FILE/\.}"
       #print_output "[*] $NVRAM_FILE $COUNT ASCII_text"
       echo "$NVRAM_FILE $COUNT ASCII_text" >> "$LOG_PATH_MODULE"/nvram/nvram_files_final
     fi
@@ -1529,23 +1525,27 @@ check_online_stat() {
       # write all services into a one liner for output:
       print_ln
       if [[ -v TCP_SERVICES_STARTUP[@] ]]; then
-        TCP_SERV=$(IFS=$' '; echo "${TCP_SERVICES_STARTUP[*]}")
+        #TCP_SERV=$(IFS=$' '; echo "${TCP_SERVICES_STARTUP[*]}")
+        printf -v TCP_SERV "%s " "${TCP_SERVICES_STARTUP[@]}"
         TCP_SERV_STARTUP=${TCP_SERV//\ /,}
         print_output "[*] TCP Services detected via startup: $ORANGE$TCP_SERV_STARTUP$NC"
       fi
       if [[ -v UDP_SERVICES_STARTUP[@] ]]; then
-        UDP_SERV=$(IFS=$' '; echo "${UDP_SERVICES_STARTUP[*]}")
+        #UDP_SERV=$(IFS=$' '; echo "${UDP_SERVICES_STARTUP[*]}")
+        printf -v UDP_SERV "%s " "${UDP_SERVICES_STARTUP[@]}"
         UDP_SERV_STARTUP=${UDP_SERV//\ /,}
         print_output "[*] UDP Services detected via startup: $ORANGE$UDP_SERV_STARTUP$NC"
       fi
 
       if [[ "${#TCP_SERV_NETSTAT_ARR[@]}" -gt 0 ]]; then
-        TCP_SERV=$(IFS=$' '; echo "${TCP_SERV_NETSTAT_ARR[*]}")
+        #TCP_SERV=$(IFS=$' '; echo "${TCP_SERV_NETSTAT_ARR[*]}")
+        printf -v TCP_SERV "%s " "${TCP_SERV_NETSTAT_ARR[@]}"
         TCP_SERV_NETSTAT=${TCP_SERV//\ /,}
         print_output "[*] TCP Services detected via netstat: $ORANGE$TCP_SERV_NETSTAT$NC"
       fi
       if [[ "${#UDP_SERV_NETSTAT_ARR[@]}" -gt 0 ]]; then
-        UDP_SERV=$(IFS=$' '; echo "${UDP_SERV_NETSTAT_ARR[*]}")
+        #UDP_SERV=$(IFS=$' '; echo "${UDP_SERV_NETSTAT_ARR[*]}")
+        printf -v UDP_SERV "%s " "${UDP_SERV_NETSTAT_ARR[@]}"
         UDP_SERV_NETSTAT=${UDP_SERV//\ /,}
         print_output "[*] UDP Services detected via netstat: $ORANGE$UDP_SERV_NETSTAT$NC"
       fi
@@ -1557,22 +1557,22 @@ check_online_stat() {
       eval "TCP_SERV_ARR=($(for i in "${TCP_SERV_ARR[@]}" ; do echo "\"$i\"" ; done | sort -u))"
       eval "UDP_SERV_ARR=($(for i in "${UDP_SERV_ARR[@]}" ; do echo "\"$i\"" ; done | sort -u))"
       if [[ -v TCP_SERV_ARR[@] ]]; then
-        TCP_SERV=$(IFS=$' '; echo "${TCP_SERV_ARR[*]}")
+        #TCP_SERV=$(IFS=$' '; echo "${TCP_SERV_ARR[*]}")
+        printf -v TCP_SERV "%s " "${TCP_SERV_ARR[@]}"
         TCP_SERV=${TCP_SERV//\ /,}
         # print_output "[*] TCP Services detected: $ORANGE$TCP_SERV$NC"
       fi
       if [[ -v UDP_SERV_ARR[@] ]]; then
-        UDP_SERV=$(IFS=$' '; echo "${UDP_SERV_ARR[*]}")
+        #UDP_SERV=$(IFS=$' '; echo "${UDP_SERV_ARR[*]}")
+        printf -v UDP_SERV "%s " "${UDP_SERV_ARR[@]}"
         UDP_SERV=${UDP_SERV//\ /,}
         # print_output "[*] UDP Services detected: $ORANGE$UDP_SERV$NC"
       fi
 
       UDP_SERV="U:""$UDP_SERV"
       TCP_SERV="T:""$TCP_SERV"
-      # shellcheck disable=SC2001
-      TCP_SERV=$(echo "$TCP_SERV" | sed 's/,$//g')
-      # shellcheck disable=SC2001
-      UDP_SERV=$(echo "$UDP_SERV" | sed 's/,$//g')
+      TCP_SERV="${TCP_SERV%,}"
+      UDP_SERV="${UDP_SERV%,}"
 
       local PORTS_TO_SCAN=""
       if [[ "$TCP_SERV" =~ ^T:[0-9].* ]]; then

modules/P05_patools_init.sh

@@ -86,8 +86,7 @@ patools_extractor() {
 
   print_ln
   print_output "[*] Using the following firmware directory ($ORANGE$EXTRACTION_DIR_$NC) as base directory:"
-  #shellcheck disable=SC2012
-  ls -lh "$EXTRACTION_DIR_" | tee -a "$LOG_FILE"
+  find "$EXTRACTION_DIR_" -xdev -maxdepth 1 -ls | tee -a "$LOG_FILE"
   print_ln
 
   FILES_PATOOLS=$(find "$EXTRACTION_DIR_" -type f | wc -l)

modules/P14_ext_mounter.sh

@@ -62,8 +62,7 @@ ext_extractor() {
     cp -pri "$TMP_EXT_MOUNT"/* "$EXTRACTION_DIR_"
     print_ln
     print_output "[*] Using the following firmware directory ($ORANGE$EXTRACTION_DIR_$NC) as base directory:"
-    #shellcheck disable=SC2012
-    ls -lh "$EXTRACTION_DIR_" | tee -a "$LOG_FILE"
+    find "$EXTRACTION_DIR_" -xdev -maxdepth 1 -ls | tee -a "$LOG_FILE"
     print_ln
     print_output "[*] Unmounting $ORANGE$TMP_EXT_MOUNT$NC directory"
 

modules/P18_qnap_decryptor.sh

@@ -223,8 +223,6 @@ qnap_extractor() {
   if [ -e "$INITRAMFS" ]; then
     print_ln
     print_output "[*] Extracting $ORANGE$INITRAMFS$NC."
-    #  # shellcheck disable=SC2002
-    #cat "$INITRAMFS" | (cd "$SYSROOT" && (cpio -i --make-directories||true) )
     (cd "$SYSROOT" && (cpio -i --make-directories||true) ) < "$INITRAMFS"
     print_ln
     print_output "[*] Extracted firmware structure ($ORANGE$SYSROOT$NC):"

modules/P19_bsd_ufs_mounter.sh

@@ -64,8 +64,7 @@ ufs_extractor() {
     cp -pri "$TMP_UFS_MOUNT"/* "$EXTRACTION_DIR_" 2>/dev/null || true
     print_ln
     print_output "[*] Using the following firmware directory ($ORANGE$EXTRACTION_DIR_$NC) as base directory:"
-    #shellcheck disable=SC2012
-    ls -lh "$EXTRACTION_DIR_" | tee -a "$LOG_FILE"
+    find "$EXTRACTION_DIR_" -xdev -maxdepth 1 -ls | tee -a "$LOG_FILE"
     print_ln
     print_output "[*] Unmounting $ORANGE$TMP_UFS_MOUNT$NC directory"
 

modules/P35_UEFI_extractor.sh

@@ -65,8 +65,7 @@ ami_extractor() {
 
   print_ln
   print_output "[*] Using the following firmware directory ($ORANGE$EXTRACTION_DIR_$NC) as base directory:"
-  #shellcheck disable=SC2012
-  ls -lh "$EXTRACTION_DIR_" | tee -a "$LOG_FILE"
+  find "$EXTRACTION_DIR_" -xdev -maxdepth 1 -ls | tee -a "$LOG_FILE"
   print_ln
 
   FILES_UEFI=$(find "$EXTRACTION_DIR_" -type f | wc -l)

modules/S06_distribution_identification.sh

@@ -44,7 +44,6 @@ S06_distribution_identification()
         if [[ -f "$FILE" ]]; then
           PATTERN="$(echo "$CONFIG" | cut -d\; -f3)"
           SED_COMMAND="$(echo "$CONFIG" | cut -d\; -f4)"
-          # shellcheck disable=SC2086
           FILE_QUOTED=$(escape_echo "$FILE")
           OUT1="$(eval "$PATTERN" "$FILE_QUOTED" || true)"
           # echo "SED command: $SED_COMMAND"

modules/S115_usermode_emulator.sh

@@ -198,7 +198,6 @@ copy_firmware() {
   # we just create a backup if the original firmware path was a root directory
   # if it was a binary file we already have extracted it and it is already messed up
   # so we can mess it up a bit more ;)
-  # shellcheck disable=SC2154
   if [[ -d "$FIRMWARE_PATH_BAK" ]]; then
     print_output "[*] Create a firmware backup for emulation ..."
     cp -pri "$FIRMWARE_PATH" "$LOG_PATH_MODULE" 2> /dev/null
@@ -522,8 +521,7 @@ emulate_binary() {
   write_log "[*] Using root directory: $ORANGE$R_PATH$NC ($ORANGE$ROOT_CNT/${#ROOT_PATH[@]}$NC)" "$LOG_FILE_BIN"
   write_log "[*] Using CPU config: $ORANGE$CPU_CONFIG_$NC" "$LOG_FILE_BIN"
   #write_log "[*] Root path used: $ORANGE$R_PATH$NC" "$LOG_FILE_BIN"
-  #shellcheck disable=SC2001
-  write_log "[*] Emulating binary: $ORANGE$(echo "$BIN_" | sed 's/^\.//')$NC" "$LOG_FILE_BIN"
+  write_log "[*] Emulating binary: $ORANGE${BIN_/\.}$NC" "$LOG_FILE_BIN"
   write_log "" "$LOG_FILE_BIN"
 
   # lets assume we now have only ELF files. Sometimes the permissions of firmware updates are completely weird

modules/S99_grepit.sh

@@ -39,10 +39,6 @@ S99_grepit() {
   module_title "Intelligent grepit module"
   print_output "Running intelligent grepit module for identification of interesting spots within the firmware ..." "no_log"
 
-  if [[ "$STRICT_MODE" -eq 1 ]]; then
-    disable_strict_mode "$STRICT_MODE"
-  fi
-
   pre_module_reporter "${FUNCNAME[0]}"
 
   local WAIT_PIDS_S99=()
@@ -60,11 +56,13 @@ S99_grepit() {
   # Weird grep behaviour with clearing to the end of line -.-
   # This variable prevents this behaviour
   export GREP_COLORS=ne
+  # sometimes we have so many results. We need to limit it a bit
+  local LIMIT_GREP=(-m 20000)
   # Do not remove -rP if you don't know what you are doing, otherwise you probably break this script
-  local GREP_ARGUMENTS="-a -n -A 1 -B 3 -rP"
+  local GREP_ARGUMENTS=(-a -n -A 1 -B 3 -rP)
   # Open the colored outputs with "less -R" or cat, otherwise remove --color=always (not recommended, colors help to find the matches in huge text files)
-  local COLOR_ARGUMENTS="--color=always"
-  export _STANDARD_GREP_ARGUMENTS="$GREP_ARGUMENTS $COLOR_ARGUMENTS"
+  local COLOR_ARGUMENTS=(--color=always)
+  export STANDARD_GREP_ARGUMENTS=("${GREP_ARGUMENTS[@]}" "${COLOR_ARGUMENTS[@]}" "${LIMIT_GREP[@]}")
   export ENABLE_LEAST_LIKELY=0
 
   mapfile -t GREPIT_MODULES < <(grep -E "^grepit_module.*\(\) " "$MOD_DIR"/"${FUNCNAME[0]}".sh | sed -e 's/()\ .*//g' | sort -u)
@@ -88,10 +86,6 @@ S99_grepit() {
     wait_for_pid "${WAIT_PIDS_S99[@]}"
   fi
 
-  if [[ "$STRICT_MODE" -eq 1 ]]; then
-    enable_strict_mode "$STRICT_MODE"
-  fi
-
   grepit_reporter
 
   GREPIT_RESULTS=$(grep -v -c -E "\ Searching\ \(" "$LOG_PATH_MODULE"/[0-9]_* | cut -d: -f2 | paste -sd+ | bc)
@@ -127,36 +121,35 @@ grepit_reporter() {
 
 grepit_search() {
   local LINES_OF_OUTPUT=0
-  local _GREP_COMMAND="grep"
+  local GREP_COMMAND="grep"
   local LOG_DETAILS=1
   local COMMENT="$1"
   local EXAMPLE="$2"
   local FALSE_POSITIVES_EXAMPLE="$3"
   local SEARCH_REGEX="$4"
   local OUTFILE="$5"
   if [[ -v 6 ]]; then
-    local _ARGS_FOR_GREP="${6}" # usually just -i for case insensitive or empty, very rare we use -o for match-only part with no context info
+    local ARGS_FOR_GREP=("${6}") # usually just -i for case insensitive or empty, very rare we use -o for match-only part with no context info
   else
-    local _ARGS_FOR_GREP=""
+    local ARGS_FOR_GREP=()
   fi
 
   if [[ "$ENABLE_LEAST_LIKELY" -eq 0 ]] && [[ "$OUTFILE" == 9_* ]]; then
     print_output "[-] Skipping searching for $OUTFILE with regex $SEARCH_REGEX. Set ENABLE_LEAST_LIKELY in the module options to 1 if you would like to." "no_log"
   else
-    write_log "[*] Searching (args for grep: $ORANGE$_ARGS_FOR_GREP$NC) for $ORANGE$SEARCH_REGEX$NC." "$LOG_PATH_MODULE/$OUTFILE"
+    write_log "[*] Searching (args for grep: $ORANGE${ARGS_FOR_GREP[*]}$NC) for $ORANGE$SEARCH_REGEX$NC." "$LOG_PATH_MODULE/$OUTFILE"
 
     if [[ "$LOG_DETAILS" -eq 1 ]]; then
       write_log "[*] Grepit state info - comment: $ORANGE$COMMENT$NC" "$LOG_PATH_MODULE/$OUTFILE"
       write_log "[*] Grepit state info - Filename $ORANGE$OUTFILE$NC" "$LOG_PATH_MODULE/$OUTFILE"
       write_log "[*] Grepit state info - Example: $ORANGE$EXAMPLE$NC" "$LOG_PATH_MODULE/$OUTFILE"
       write_log "[*] Grepit state info - False positive example: $ORANGE$FALSE_POSITIVES_EXAMPLE$NC" "$LOG_PATH_MODULE/$OUTFILE"
-      write_log "[*] Grepit state info - Grep args: $ORANGE$_ARGS_FOR_GREP$NC" "$LOG_PATH_MODULE/$OUTFILE"
+      write_log "[*] Grepit state info - Grep args: $ORANGE${ARGS_FOR_GREP[*]}$NC" "$LOG_PATH_MODULE/$OUTFILE"
       write_log "[*] Grepit state info - Search regex: $ORANGE$SEARCH_REGEX$NC" "$LOG_PATH_MODULE/$OUTFILE"
       write_log "" "$LOG_PATH_MODULE/$OUTFILE"
     fi
 
-    # shellcheck disable=SC2086
-    $_GREP_COMMAND $_ARGS_FOR_GREP $_STANDARD_GREP_ARGUMENTS -- "$SEARCH_REGEX" "$FIRMWARE_PATH" >> "$LOG_PATH_MODULE/$OUTFILE" 2>&1 || true
+    "$GREP_COMMAND" "${ARGS_FOR_GREP[@]}" "${STANDARD_GREP_ARGUMENTS[@]}" -- "$SEARCH_REGEX" "$FIRMWARE_PATH" >> "$LOG_PATH_MODULE/$OUTFILE" 2>&1 || true
 
     if [[ "$LOG_DETAILS" -eq 1 ]]; then
       if [[ -f "$LOG_PATH_MODULE/$OUTFILE" ]] && ! [[ $(grep -v -c -E "\ Searching\ \(" "$LOG_PATH_MODULE/$OUTFILE" 2>/dev/null) -gt 7 ]]; then
@@ -178,7 +171,7 @@ grepit_search() {
       # parse the csv output file and sort it according the test priority - 1-9, where 1 is more interesting
       # (low false positive rate, certainty of "vulnerability") and 9 is only "you might want to have a look when you are desperately looking for vulns")
       print_output "[*] $ORANGE$LINES_OF_OUTPUT$NC results of grepit module $ORANGE$CURRENT_TEST$NC." "no_log"
-      write_csv_log "$CURRENT_TEST" "$LINES_OF_OUTPUT" "$_ARGS_FOR_GREP" "$SEARCH_REGEX" "$COMMENT"
+      write_csv_log "$CURRENT_TEST" "$LINES_OF_OUTPUT" "${ARGS_FOR_GREP[*]}" "$SEARCH_REGEX" "$COMMENT"
     fi
   fi
 }