Merge pull request #479 from m-1-k-3/respect_blaclist

Respect module blacklist in waiting state / Installer fix

config/bin_version_strings.cfg

@@ -306,6 +306,7 @@ kmod;;unknown;"^lsmod\ (version\ )?[0-9](\.[0-9]+)+?$";"sed -r 's/lsmod\ (versio
 kmod;;unknown;"^modinfo\ (version\ )?[0-9](\.[0-9]+)+?$";"sed -r 's/modinfo\ (version\ )?([[0-9](\.[0-9]+)+?)$/kmod:\2/'";
 roaring-l2tpd;;gplv2;"l2tpd\ Version\ [0-9]\.[0-9]+\ Copyright\ [0-9]+\ Roaring\ Penguin\ Software\ Inc\.";"sed -r 's/l2tpd\ Version\ ([0-9](\.[0-9]+)+?).*Roaring\ Penguin\ Software\ Inc\./roaring_penguin:l2tpd:\1/'";
 l2tpd;;gplv2;"l2tpd\ version\ [0-9](\.[0-9]+)+?\ started\ on\ ";"sed -r 's/l2tpd\ version\ ([0-9](\.[0-9]+)+?)\ .*/roaring_penguin:l2tpd:\1/'";
+l2tpd;;gplv2;"l2tpd\ Version\ [0-9](\.[0-9]+)+?(-rtn)?\ Copyright\ [0-9]+\ Roaring\ Penguin\ Software\ Inc\.";"sed -r 's/l2tpd\ Version\ ([0-9](\.[0-9]+)+?).*/roaring_penguin:l2tpd:\1/'";
 openldap;;openldap-lic;"OpenLDAP:\ ldap[a-z]+\ [0-9](\.[0-9]+)+?\ ";"sed -r 's/OpenLDAP:\ ldap[a-z]+\ ([0-9](\.[0-9]+)+?)\ .*/openldap:\1/'";
 glibc;;lgpl;"ldconfig\ \(GNU\ libc\)\ [0-9](\.[0-9]+)+?$";"sed -r 's/ldconfig\ \(GNU\ libc\)\ ([0-9](\.[0-9]+)+?)$/gnu:glibc:\1/'";
 ld-musl-armhf.so.1;strict;unknwon;"^Version\ [0-9](\.[0-9]+)+?$";"sed -r 's/Version\ ([0-9](\.[0-9]+)+?)$/musl-libc:musl:\1/'";
@@ -402,6 +403,7 @@ mini_httpd;;unknown;"mini_httpd\/[0-9](\.[0-9]+)+?(\ [0-9]+)?";"sed -r 's/mini_h
 mini_httpd;live;unknown;"mini_httpd\ [0-9](\.[0-9]+)+?(\ [0-9]+)?";"sed -r 's/mini_httpd\ ([0-9](\.[0-9]+)+?).*/acme:mini_httpd:\1/'";
 miniupnpd;;unknown;"SERVER:.*UPnP\/[0-9](\.[0-9]+)+?\ MiniUPnPd\/[0-9](\.[0-9]+)+?$";"sed -r 's/SERVER:.*UPnP\/[0-9](\.[0-9]+)+?\ MiniUPnPd\/([0-9](\.[0-9]+)+?)$/miniupnpd:\2/'";
 miniupnpd;live;unknown;"MiniUPnP\ [0-9](\.[0-9]+)+?\ ;"sed -r 's/MiniUPnP\ ([0-9](\.[0-9]+)+?)\ .*/miniupnpd:\1/'";
+miniupnpd;;unknown;"^miniupnpd\ [0-9](\.[0-9]+)+?\ ;"sed -r 's/^miniupnpd\ ([0-9](\.[0-9]+)+?)\ .*/miniupnpd:\1/'";
 mkconfig;;unknown;"^mkconfig\ version\ [0-9](\.[0-9]+)+?$";"sed -r 's/mkconfig\ version\ ([0-9](\.[0-9]+)+?)$/mkconfig:\1/'";
 mke2fs;;unknown;"^mke2fs\ [0-9](\.[0-9]+)+?";"sed -r 's/mke2fs\ ([0-9](\.[0-9]+)+?)/e2fsprogs:\1/'";
 mkfs.jffs;;unknown;"mkfs\.jffs[0-9]:\ error!:\ revision\ [0-9]\.[0-9]+$";"sed -r 's/mkfs\.jffs[0-9]:\ error!:\ revision\ ([0-9](\.[0-9]+)+?)$/mtd-utils:\1/'";
@@ -610,7 +612,7 @@ snmpd;;unknown;"^NET-SNMP\ version:\ [0-9](\.[0-9]+)+?$";"sed -r 's/NET-SNMP\ ve
 snmpd;strict;unknown;"^Version:\ \ [0-9]\.[0-9]+\.[0-9]$";"sed -r 's/Version:\ \ ([0-9](\.[0-9]+)+?)$/net-snmp:\1/'";
 snort;strict;gplv2;"^Version\ [0-9](\.[0-9])+?";"sed -r 's/Version:\ ([0-9](\.[0-9]+)+?)/snort:\1/'";
 #sntp;;unknown;"^sntp\ [0-9][\.0-9p]+";"NA";
-socat;;gplv2;"socat\ version\ [0-9](\.[0-9]+)?(-[a-z][0-9]+)?\ ";"sed -r 's/socat\ version\ ([0-9](\.[0-9]+)+?)((-[a-z][0-9]+)?)\ .*/socat:\1:\2/'";
+socat;;gplv2;"socat\ version\ [0-9](\.[0-9]+)+?(-[a-z][0-9]+)?\ ";"sed -r 's/socat\ version\ ([0-9](\.[0-9]+)+?)((-[a-z][0-9]+)?)\ .*/socat:\1:\2/'";
 sqlite3;;public-domain;"SQLite\ version\ 3(\.[0-9]+)+?";"sed -r 's/SQLite\ version\ (3(\.[0-9]+)+?).*/sqlite:\1/'";
 sqlite3;strict;public-domain;"^3\.[0-9]+\.[0-9]+\ ";"sed -r 's/(3(\.[0-9]+)+?)\ /sqlite:\1/'";
 sqlite3;strict;public-domain;"^3\.[0-9]+\.[0-9]+$";"sed -r 's/(3(\.[0-9]+)+?)$/sqlite:\1/'";
@@ -624,6 +626,7 @@ start-stop-deamon;;unknown;"^start-stop-daemon\ [0-9][\.0-9]+\ ";"sed -r 's/star
 strace;;unknown;"strace\ --\ version\ [0-9](\.[0-9]+)+?$";"sed -r 's/strace\ --\ version\ ([0-9](\.[0-9]+)+?)$/strace:\1/'";
 ltrace;;unknown;"^ltrace\ version\ [0-9](\.[0-9]+)+?\.$";"sed -r 's/ltrace\ version\ ([0-9](\.[0-9]+)+?)\.$/ltrace:\1/'";
 strongswan;;gplv2;"Linux\ strongSwan\ [0-9](\.[0-9]+)+?$";"sed -r 's/Linux\ strongSwan\ ([0-9](\.[0-9]+)+?)$/strongswan:\1/'";
+strongswan;;gplv2;"\ strongSwan\ [0-9](\.[0-9]+)+?$";"sed -r 's/.*\ strongSwan\ ([0-9](\.[0-9]+)+?)$/strongswan:\1/'";
 strongswan;;gplv2;"^ipsec _copyright strongSwan [0-9](\.[0-9]+)+?$";"sed -r 's/ipsec _copyright strongSwan ([0-9](\.[0-9]+)+?)/strongswan:\1/'";
 stunnel;;gplv2;"stunnel\ [0-9](\.[0-9]+)+?\ on\ ";"sed -r 's/stunnel\ ([0-9](\.[0-9]+)+?)\ .*/stunnel:\1/'";
 sudoreplay;;unknown;"sudoreplay\ version\ [0-9](\.[0-9]+)+?(p[0-9]+)?$";"sed -r 's/sudoreplay\ version\ ([0-9](\.[0-9]+)+?(p[0-9]+)?)$/sudo:\1/'";

helpers/helpers_emba_helpers.sh

@@ -101,12 +101,16 @@ cleaner() {
   # otherwise the unmounter runs crazy in some corner cases
   if [[ -f "$LOG_DIR"/"$MAIN_LOG_FILE" && "${#FILE_ARR[@]}" -gt 0 ]]; then
     if [[ $(grep -i -c S115 "$LOG_DIR"/"$MAIN_LOG_FILE") -eq 1 ]]; then
+
       print_output "[*] Terminating qemu processes - check it with ps" "no_log"
       killall -9 --quiet -r .*qemu.*sta.* || true
+
       print_output "[*] Cleaning the emulation environment\\n" "no_log"
-      find "$FIRMWARE_PATH_CP" -xdev -iname "qemu*static" -exec rm {} \; 2>/dev/null
+      find "$FIRMWARE_PATH_CP" -xdev -iname "qemu*static" -exec rm {} \; 2>/dev/null || true
+      find "$LOG_DIR/s115_usermode_emulator" -xdev -iname "qemu*static" -exec rm {} \; 2>/dev/null || true
+
       print_output "[*] Umounting proc, sys and run" "no_log"
-      mapfile -t CHECK_MOUNTS < <(mount | grep "$FIRMWARE_PATH_CP" 2>/dev/null || true)
+      mapfile -t CHECK_MOUNTS < <(mount | grep "s115_usermode_emulator" 2>/dev/null || true)
       # now we can unmount the stuff from emulator and delete temporary stuff
       for MOUNT in "${CHECK_MOUNTS[@]}"; do
         print_output "[*] Unmounting $MOUNT" "no_log"
@@ -294,7 +298,7 @@ module_wait() {
   local MODULE_TO_WAIT="${1:-}"
   # if the module we should wait is not in our module array we return without waiting
   if ! [[ " ${MODULES_EXPORTED[*]} " == *"${MODULE_TO_WAIT}"* ]]; then
-    print_output "[-] Module $ORANGE$MODULE_TO_WAIT$NC not in module array - this will result in unexpected behavior" "main"
+    print_output "[-] $(date) - $MODULE_TO_WAIT not in module array - this will result in unexpected behavior" "main"
     return
   fi
 
@@ -303,6 +307,11 @@ module_wait() {
   done
 
   while [[ $(grep -i -c "$MODULE_TO_WAIT finished" "$MAIN_LOG" || true) -ne 1 ]]; do
+    if grep -q "$MODULE_TO_WAIT not executed - blacklist triggered" "$MAIN_LOG"; then
+      print_output "[-] $(date) - $MODULE_TO_WAIT blacklisted - not waiting" "main"
+      # if our module which we are waiting is on the blacklist we can just return
+      return
+    fi
     sleep 1
   done
 }

installer/ID1_ubuntu_os.sh

@@ -39,12 +39,12 @@ ID1_ubuntu_os() {
         # libssl1.1 missing
         echo -e "\\n""$BOLD""Installing libssl1.1 for mongodb!""$NC"
         # echo "deb http://security.ubuntu.com/ubuntu impish-security main" | tee /etc/apt/sources.list.d/impish-security.list
-        wget http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl-dev_1.1.1-1ubuntu2.1~18.04.20_amd64.deb -O external/libssl-dev_1.1.1-1ubuntu2.1~18.04.20_amd64.deb
-        wget http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1-1ubuntu2.1~18.04.20_amd64.deb -O external/libssl1.1_1.1.1-1ubuntu2.1~18.04.20_amd64.deb
-        dpkg -i external/libssl1.1_1.1.1-1ubuntu2.1~18.04.20_amd64.deb
-        dpkg -i external/libssl-dev_1.1.1-1ubuntu2.1~18.04.20_amd64.deb
-        rm external/libssl1.1_1.1.1-1ubuntu2.1~18.04.20_amd64.deb
-        rm external/libssl-dev_1.1.1-1ubuntu2.1~18.04.20_amd64.deb
+        wget http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl-dev_1.1.1-1ubuntu2.1~18.04.21_amd64.deb -O external/libssl-dev_1.1.1-1ubuntu2.1~18.04.21_amd64.deb
+        wget http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1-1ubuntu2.1~18.04.21_amd64.deb -O external/libssl1.1_1.1.1-1ubuntu2.1~18.04.21_amd64.deb
+        dpkg -i external/libssl1.1_1.1.1-1ubuntu2.1~18.04.21_amd64.deb
+        dpkg -i external/libssl-dev_1.1.1-1ubuntu2.1~18.04.21_amd64.deb
+        rm external/libssl1.1_1.1.1-1ubuntu2.1~18.04.21_amd64.deb
+        rm external/libssl-dev_1.1.1-1ubuntu2.1~18.04.21_amd64.deb
     fi
 
     if [[ "$WSL" -eq 1 ]]; then

modules/L10_system_emulation.sh

@@ -631,7 +631,7 @@ main_emulation() {
         fi
 
         RESULT_SOURCE="EMBA"
-        write_results
+        write_results "$ARCHIVE_PATH"
 
         # if we are going to execute L15 then we do not reset the network environment now
         # we just write the commands to run.sh
@@ -2045,7 +2045,9 @@ write_results() {
     local FIRMWARE_PATH_orig
     FIRMWARE_PATH_orig="$(cat "$TMP_DIR"/fw_name.log)"
   fi
-  echo "$FIRMWARE_PATH_orig;$RESULT_SOURCE;Booted $BOOTED; ICMP $ICMP; TCP-0 $TCP_0;TCP $TCP; IP address: $IP_ADDRESS_; Network mode: $NETWORK_MODE ($NETWORK_DEVICE/$ETH_INT/$INIT_FILE)" >> "$LOG_DIR"/emulator_online_results.log
+  local ARCHIVE_PATH_="${1:-}"
+  ARCHIVE_PATH_="$(echo "$ARCHIVE_PATH_" | rev | cut -d '/' -f1 | rev)"
+  echo "$FIRMWARE_PATH_orig;$RESULT_SOURCE;Booted $BOOTED; ICMP $ICMP; TCP-0 $TCP_0;TCP $TCP; IP address: $IP_ADDRESS_; Network mode: $NETWORK_MODE ($NETWORK_DEVICE/$ETH_INT/$INIT_FILE);$ARCHIVE_PATH_" >> "$LOG_DIR"/emulator_online_results.log
   print_bar ""
 }
 

modules/S115_usermode_emulator.sh

@@ -68,11 +68,9 @@ S115_usermode_emulator() {
     readarray -t BIN_BLACKLIST < "$CONFIG_DIR"/emulation_blacklist.cfg
 
     # as we modify the firmware area, we copy it to the log directory and do the modifications in this area
-    # Note: only for firmware directories - if we have already extracted the firmware we do not copy it again
     copy_firmware
 
-    # we only need to detect the root directory again if we have copied it before
-    [[ -d "$FIRMWARE_PATH_BAK" ]] && detect_root_dir_helper "$EMULATION_PATH_BASE"
+    detect_root_dir_helper "$EMULATION_PATH_BASE"
     kill_qemu_threader &
     PID_killer+="$!"
 

modules/S99_grepit.sh

@@ -46,7 +46,7 @@ S99_grepit() {
   local GREPIT_RESULTS=0
 
   local MAX_MOD_THREADS=1
-  local MEM_LIMIT=$(( "$TOTAL_MEMORY"/2 ))
+  local MEM_LIMIT=$(( "$TOTAL_MEMORY"/3 ))
 
 
   # grepit options: