Merge pull request #248 from m-1-k-3/master

#247 and removed FACT

config/bin_version_strings.cfg

@@ -34,6 +34,7 @@ aircrack-ng;;gplv2;"^\ \ Packetforge-ng\ [0-9](\.[0-9]+)+?\ -\ \(C\)\ ";"sed -r
 alsa;;unknown;"alsactl\ version\ [0-9](\.[0-9]+)+?$";"sed -r 's/alsactl\ version\ ([0-9](\.[0-9]+)+?)$/alsa:\1/'";
 amixer;;unknown;"amixer\ version\ [0-9](\.[0-9]+)+?$";"sed -r 's/amixer\ version\ ([0-9](\.[0-9]+)+?)$/alsa:\1/'";
 amule;;gplv2;"aMuleD\ [0-9](\.[0-9]+)+?\ compiled\ with\ ";"sed -r 's/aMuleD\ ([0-9](\.[0-9]+)+?)\ .*/amuled:\1/'";
+apache;;apachev2;"^Server version: Apache/[0-9](\.[0-9]+)+?";"sed -r 's/^Server version: Apache\/([0-9](\.[0-9]+)+?)/apache:\1/g'";
 apstats;;unknown;"apstats\ v[0-9](\.[0-9]+)+?:\ Display\ Access\ Point\ Statistics\.";"sed -r 's/apstats\ v([0-9](\.[0-9]+)+?):\ .*/apstats:\1/'";
 apt;;gplv3;"apt\ [0-9]+(\.[0-9]+)+?\ ";"sed -r 's/apt\ ([0-9]+(\.[0-9]+)+?)\ .*/apt:\1/'";
 apt;;gplv3;"apt-Version\ [0-9](\.[0-9]+)+?\ ";"sed -r 's/apt-Version\ ([0-9](\.[0-9]+)+?)\ .*/apt:\1/'";
@@ -162,6 +163,7 @@ libext2fs;;unknown;"^EXT2FS\ Library\ version\ [0-9](\.[0-9]+)+?$";"sed -r 's/EX
 ez-ipupdate;;gplv2;"[0-9](\.[0-9]+)+?\ -\ $Id:\ ez-ipupdate.c,v\ [0-9]\.[0-9]+";"sed -r 's/([0-9](\.[0-9]+)+?)\ -\ $Id:\ ez-ipupdate.c,v\ [0-9]\.[0-9]+/ez-ipupdate:\1/'";
 ez-ipupdate;;gplv2;"ez-ipupdate:\ -\ [0-9](\.[0-9]+)+?";"sed -r 's/ez-ipupdate:\ -\ ([0-9](\.[0-9]+)+?)/ez-ipupdate:\1/'";
 fdisk;;gplv2;"^fdisk\ v[0-9](\.[0-9]+)+?([a-z])?$";"sed -r 's/fdisk\ v([0-9](\.[0-9]+)+?([a-z])?)$/fdisk:\1/'";
+ffmpeg;;lgpl;"^ffmpeg version [0-9](\.[0-9]+)+? Copyright \(c\) 20[0-9]+-20[0-9]+ the FFmpeg developers";"sed -r 's/^ffmpeg version ([0-9](\.[0-9]+)+?) Copyright \(c\) 20[0-9]+-20[0-9]+ the FFmpeg developers$/ffmpeg:ffmpeg:\1/'";
 file;;unknown;"file-[0-9](\.[0-9]+)+?$";"sed -r 's/file-([0-9](\.[0-9]+)+?)$/file:\1/'";
 flash_eraseall;;unknown;"flash_eraseall\ \$Revision:\ [0-9](\.[0-9]+)+?\ \$";"sed -r 's/flash_eraseall\ \$Revision:\ ([0-9](\.[0-9]+)+?).*/mtd-utils:\1/'";
 flash_erase;;unknown;"flash_erase\ version\ [0-9](\.[0-9]+)+?";"sed -r 's/flash_erase\ version\ ([0-9](\.[0-9]+)+?).*/mtd-utils:\1/'";
@@ -195,6 +197,7 @@ glibc;;lgpl;"GNU\ C\ Library\ stable\ release\ version\ [0-9](\.[0-9]+)+?,\ ";"s
 gnu_cpio;;gplv3;"\(GNU\ cpio\)\ [0-9](\.[0-9]+)+?$";"sed -r 's/\(GNU\ cpio\)\ ([0-9](\.[0-9]+)+?)$/gnu:cpio:\1/'";
 gnu_findutils;;gplv3;"\(GNU\ findutils\)\ [0-9](\.[0-9]+)+?$";"sed -r 's/\(GNU\ findutils\)\ ([0-9](\.[0-9]+)+?)$/gnu:findutils:\1/'";
 gnu_findutils;;gplv3;"\(GNU\ findutils\)\ [0-9](\.[0-9]+)+?-[a-z]+$";"sed -r 's/\(GNU\ findutils\)\ ([0-9](\.[0-9]+)+?-[a-z]+)$/gnu:findutils:\1/'";
+gnu_fribidi;;lgpl;"^\(GNU FriBidi\) [0-9](\.[0-9]+)+?$";"sed -r 's/^\(GNU FriBidi\) ([0-9](\.[0-9]+)+?)$/gnu:fribidi:\1/'";
 gnu_gettext-runtime;;unknown;"\(GNU\ gettext-runtime\)\ [0-9](\.[0-9]+)+?$";"sed -r 's/\(GNU\ gettext-runtime\)\ ([0-9](\.[0-9]+)+?)$/gnu:gettext-runtime:\1/'";
 gnu_groff;;gplv3;"\(GNU\ groff\)\ version\ [0-9](\.[0-9]+)+?$";"sed -r 's/\(GNU\ groff\)\ version\ ([0-9](\.[0-9]+)+?)$/gnu:groff:\1/'";
 gnu_groff;;gplv3;"^GNU\ refer\ \(groff\)\ version\ [0-9]+(\.[0-9]+)+?$";"sed -r 's/GNU\ refer\ \(groff\)\ version\ ([0-9]+(\.[0-9]+)+?)$/gnu:groff:\1/'";
@@ -206,9 +209,6 @@ goahead;;commercial;"^CONFIG_SET\ \(\/runtime\/VerInfo\/Web,\ [0-9](\.[0-9])+?(b
 goahead;;commercial;"^goahead\ \ [0-9](\.[0-9]+)+?\ start$";"sed -r 's/goahead\ \ ([0-9](\.[0-9]+)+?)\ start$/goahead:\1/'";
 gpg;;gplv3;"gpg.*\(GnuPG\)\ [0-9](\.[0-9]+)+?$";"sed -r 's/gpg.*\(GnuPG\)\ ([0-9](\.[0-9]+)+?)$/gnupg:\1/'";
 gpgsplit;;gplv3;"gpgsplit\ \(GnuPG\)\ [0-9](\.[0-9]+)+?";"sed -r 's/gpg.*\(GnuPG\)\ ([0-9](\.[0-9]+)+?)/gnupg:\1/'";
-libgpiod;;lgpl;"^gpiofind\ \(libgpiod\)\ [0-9](\.[0-9]+)+?$";"sed -r 's/gpiofind\ \(libgpiod\)\ [0-9](\.[0-9]+)+?)$/libgpiod:\1/'";
-libgpiod;;lgpl;"^gpioinfo\ \(libgpiod\)\ [0-9](\.[0-9]+)+?$";"sed -r 's/gpioinfo\ \(libgpiod\)\ [0-9](\.[0-9]+)+?)$/libgpiod:\1/'";
-libgpiod;;lgpl;"^gpiomon\ \(libgpiod\)\ [0-9](\.[0-9]+)+?$";"sed -r 's/gpiomon\ \(libgpiod\)\ [0-9](\.[0-9]+)+?)$/libgpiod:\1/'";
 grep;;gplv3;"\(GNU\ grep\)\ [0-9](\.[0-9]+)+?$";"sed -r 's/\(GNU\ grep\)\ ([0-9](\.[0-9]+)+?)$/gnu:grep:\1/'";
 grep;;gplv3;"GNU\ grep\ [0-9](\.[0-9]+)+?";"sed -r 's/GNU\ grep\ ([0-9](\.[0-9]+)+?).*/gnu:grep:\1/'";
 grub2;;gplv3;"\ \(GRUB\)\ [0-9](\.[0-9]+)+?";"sed -r 's/\ \(GRUB\)\ 2((\.[0-9]+)+?).*/grub2:2\1/'";
@@ -319,6 +319,12 @@ libgcrypt;;lgplv2.1;"^This\ is\ Libgcrypt\ [0-9]\.([0-9]+)+?\ -\ The\ GNU\ Crypt
 libjpeg;;bsd-style;"Independent\ JPEG\ Group.s\ CJPEG,\ version\ [0-9][a-z]\ \ [0-9]+-[A-Za-z]+-[0-9]+$";"sed -r 's/Independent\ JPEG\ Group.s\ CJPEG,\ version\ ([0-9][a-z])\ .*/libjpeg:\1/'";
 libjpeg-turbo;;bsd-style;"^libjpeg-turbo\ version\ [0-9](\.[0-9]+)+?\ \(build\ [0-9]+\)$";"sed -r 's/libjpeg-turbo\ version\ ([0-9](\.[0-9]+)+?)\ .*/libgjpeg-turbo:\1/'";
 liblzma;;public-domain;"liblzma\ [0-9](\.[0-9]+)+?$";"sed -r 's/liblzma\ ([0-9](\.[0-9]+)+?)$/liblzma:\1/'";
+libgpiod;;lgpl;"^gpiofind\ \(libgpiod\)\ [0-9](\.[0-9]+)+?$";"sed -r 's/gpiofind\ \(libgpiod\)\ [0-9](\.[0-9]+)+?)$/libgpiod:\1/'";
+libgpiod;;lgpl;"^gpioinfo\ \(libgpiod\)\ [0-9](\.[0-9]+)+?$";"sed -r 's/gpioinfo\ \(libgpiod\)\ [0-9](\.[0-9]+)+?)$/libgpiod:\1/'";
+libgpiod;;lgpl;"^gpiomon\ \(libgpiod\)\ [0-9](\.[0-9]+)+?$";"sed -r 's/gpiomon\ \(libgpiod\)\ [0-9](\.[0-9]+)+?)$/libgpiod:\1/'";
+libsoup;;lgpl;"^libsoup\/[0-9](\.[0-9]+)+?$";"sed -r 's/^libsoup\/([0-9](\.[0-9]+)+?)$/libsoup:\1/'";
+libmicrohttpd.so.12;strict;lgplv2.1;"^[0-9]\.[0-9](\.[0-9]+)+?$";"sed -r 's/^([0-9]\.[0-9](\.[0-9]+)+?)/gnu:libmicrohttpd:\1/'";
+libharfbuzz.so.0;strict;mit;"^[0-9]\.[0-9](\.[0-9]+)+?$";"sed -r 's/^([0-9]\.[0-9](\.[0-9]+)+?)/harfbuzz_project:harfbuzz:\1/'";
 libnss3.so;strict;unknown;"^[0-9](\.[0-9]+)+?$";"sed -r 's/([0-9](\.[0-9]+)+?)$/libnss3:\1/'";
 libpcap;;bsd;"^libpcap\ version\ [0-9](\.[0-9]+)+?$";"sed -r 's/libpcap\ version\ ([0-9](\.[0-9]+)+?)$/libpcap:\1/'";
 libpcre;;bsd;"libpcre\.so\.[0-9]\.[0-9](\.[0-9]+)+?$";"sed -r 's/libpcre\.so\.([0-9](\.[0-9]+)+?)$/pcre:\1/'";
@@ -407,6 +413,7 @@ mount;;unknown;"^mount:\ mount-[0-9](\.[0-9]+)+?([a-z])?$";"sed -r 's/mount:\ mo
 mongodb;;unknown;"^MongoDB\ shell\ version\ v[0-9](\.[0-9]+)+?$";"sed -r 's/MongoDB\ shell\ version v([0-9](\.[0-9]+)+?)$/mongodb:mongodb:\1/'";
 mongodb;;unknown;"^db\ version\ v[0-9](\.[0-9]+)+?$";"sed -r 's/db\ version\ v([0-9](\.[0-9]+)+?)$/mongodb:mongodb:\1/'";
 mpicalc;;unknown;"mpicalc\ [0-9]\.[0-9]+";"sed -r 's/mpicalc\ ([0-9](\.[0-9]+)+?)/mpicalc:\1/'";
+msmtp;;unknown;"^msmtp version [0-9](\.[0-9]+)+?$";"sed -r 's/^msmtp version ([0-9](\.[0-9]+)+?)$/martin_lambers:msmtp:\1/'";
 mtd-utils;;gplv2;"mtdinfo\ [0-9]\.[0-9]\.[0-9]+$";"sed -r 's/mtdinfo\ ([0-9](\.[0-9]+)+?)$/mtd-utils:\1/'";
 mtd-utils;;gplv2;"\(mtd-utils\)\ [0-9]\.[0-9]\.[0-9]+$";"sed -r 's/\(mtd-utils\)\ ([0-9](\.[0-9]+)+?)$/mtd-utils:\1/'";
 mtd-utils;;gplv2;"nanddump\ [0-9](\.[0-9]+)+?$";"sed -r 's/nanddump\ ([0-9](\.[0-9]+)+?)$/mtd-utils:\1/'";
@@ -665,6 +672,7 @@ uvm;;unknown;"UVM\ [0-9](\.[0-9]+)+?";"sed -r 's/UVM\ ([0-9](\.[0-9]+)+?)$/uvm:\
 veritysetup;;unknown;"veritysetup\ [0-9](\.[0-9]+)+?$";"sed -r 's/veritysetup\ ([0-9](\.[0-9]+)+?)$/veritysetup:\1/'";
 vim;;unknown;"VIM\ -\ Vi\ IMproved\ [0-9](\.[0-9]+)+?\ ";"sed -r 's/VIM\ -\ Vi\ IMproved\ ([0-9]+(\.[0-9]+)+?)\ .*/vim:\1/'";
 visudo;;unknown;"visudo\ version\ [0-9](\.[0-9]+)+?(p[0-9]+)?$";"sed -r 's/visudo\ version\ ([0-9](\.[0-9]+)+?(p[0-9]+)?)$/sudo:\1/'";
+vsftpd;;unknown;"^vsftpd: version [0-9](\.[0-9]+)+?$"; "sed -r 's/vsftpd:\ version\ ([0-9](\.[0-9]+)+?)/vsftpd_project:vsftpd:\1/'
 vxworks;;unknown;"^VxWorks\ [0-9](\.[0-9]+)+?$";"sed -r 's/VxWorks\ ([0-9](\.[0-9]+)+?)$/vxworks:\1/'";
 vxworks;;unknown;"VxWorks[0-9](\.[0-9]+)+?$";"sed -r 's/VxWorks([0-9](\.[0-9]+)+?)$/vxworks:\1/'";
 vxworks;;unknown;"VxWorks\ [0-9](\.[0-9]+)+?\ SMP";"sed -r 's/VxWorks\ ([0-9](\.[0-9]+)+\ SMP?)$/vxworks:\1/'";
@@ -684,6 +692,7 @@ wimaxd;;unknown;"Beceem\ CM\ Server\ [0-9](\.[0-9]+)+?";"sed -r 's/Beceem\ CM\ S
 wireless-tools;;unknown;"Wireless-Tools\ version\ [0-9]+";"sed -r 's/Wireless-Tools\ version\ ([0-9]+).*$/wireless_tools:\1/'";
 wlxmlpatch;;unknown;"wlxmlpatch\ v[0-9](\.[0-9]+)+?$";"sed -r 's/wlxmlpatch\ v([0-9](\.[0-9]+)+?)$/wlxmlpatch:\1/'";
 wol;;unknown;"wol\ [0-9](\.[0-9]+)+?";"sed -r 's/wol\ ([0-9](\.[0-9]+)+?)/wol:\1/'";
+libwolfssl.so;strict;gplv3;"^[0-9]\.[0-9](\.[0-9])+?$";"sed -r 's/([0-9](\.[0-9]+)+?)$/wolfssl:\1/'";
 wpa_cli;;bsd;"wpa_cli\ v[0-9](\.[0-9]+)+?$";"sed -r 's/wpa_cli\ v([0-9](\.[0-9]+)+?).*/wpa_cli:\1/'";
 wpa_cli;;bsd;"wpa_cli\ v[0-9](\.[0-9]+)+?\.x$";"sed -r 's/wpa_cli\ v([0-9](\.[0-9]+)+?).*/wpa_cli:\1/'";
 wpa_supplicant;;bsd;"^wpa_supplicant\ v[0-9](\.[0-9]+)+?(-devel)?$";"sed -r 's/wpa_supplicant\ v([0-9](\.[0-9]+)+?).*/wpa_supplicant:\1/'";

helpers/helpers_emba_dependency_check.sh

@@ -288,19 +288,6 @@ dependency_check()
     # progpilot for php code checks
     check_dep_file "progpilot php ini checker" "$EXT_DIR""/progpilot"
 
-    # Check if fact extractor is on the system - disable, if not
-    export FACT_EXTRACTOR=1 
-
-    print_output "    fact-extractor start script - \\c" "no_log"
-    if [[ -f "$EXT_DIR""/fact_extractor/fact_extractor/fact_extract.py" ]] ; then
-      echo -e "$GREEN""ok""$NC"
-    else
-      echo -e "$RED""not ok""$NC"
-      echo -e "$RED""    Missing fact-extractor start script - check your installation""$NC"
-      FACT_EXTRACTOR=0
-      DEP_ERROR=1
-    fi
-
     # patool extractor - https://wummel.github.io/patool/
     check_dep_tool "patool"
 

installer.sh

@@ -144,9 +144,11 @@ fi
 I01_default_apps_host
 
 DOCKER_COMP_VER=$(docker-compose -v | grep version | awk '{print $3}' | tr -d ',')
-if [[ $(version "$DOCKER_COMP_VER") -lt $(version "1.29.0") ]]; then
+if [[ $(version "$DOCKER_COMP_VER") -lt $(version "1.28.5") ]]; then
   echo -e "\n${ORANGE}WARNING: compatibility of the used docker-compose version is unknown!$NC"
-  read -p "Press any key to continue..." -n1 -s -r
+  echo -e "\n${ORANGE}Please consider updating your docker-compose installation to version 1.28.5 or later.$NC"
+  echo -e "\n${ORANGE}Please check the EMBA wiki for further details: https://github.com/e-m-b-a/emba/wiki/Installation#prerequisites$NC"
+  read -p "If you know what you are doing you can press any key to continue ..." -n1 -s -r
 fi
 
 INSTALL_APP_LIST=()
@@ -165,8 +167,6 @@ if [[ "$CVE_SEARCH" -ne 1 ]] || [[ "$DOCKER_SETUP" -ne 1 ]] || [[ "$IN_DOCKER" -
 
   IP99_binwalk_default
 
-  IP60_fact_extractor
-
   I13_objdump
 
   I20_php_check

installer/IL10_system_emulator.sh

@@ -37,6 +37,7 @@ IL10_system_emulator() {
     print_tool_info "qemu-system-x86" 1
     print_tool_info "qemu-utils" 1
     print_tool_info "hping3" 1
+    print_tool_info "traceroute" 1
 
     # future use:
     print_file_info "vmlinux.mipsel.2" "FirmAE - Linux kernel 2.6 - MIPSel" "https://github.com/pr0v3rbs/FirmAE_kernel-v2.6/releases/download/v1.0/vmlinux.mipsel.2" "external/firmae/binaries/vmlinux.mipsel.2"

modules/P60_firmware_bin_extractor.sh

@@ -14,7 +14,7 @@
 # Author(s): Michael Messner, Pascal Eckmann
 
 # Description:  Analyzes firmware with binwalk, checks entropy and extracts firmware to the log directory.
-#               If binwalk fails to extract the firmware, it will be extracted with FACT-extractor.
+
 # Pre-checker threading mode - if set to 1, these modules will run in threaded mode
 # This module extracts the firmware and is blocking modules that needs executed before the following modules can run
 export PRE_THREAD_ENA=0
@@ -25,8 +25,6 @@ P60_firmware_bin_extractor() {
   pre_module_reporter "${FUNCNAME[0]}"
 
   DISK_SPACE_CRIT=0
-  FILES_FACT=0
-  FILES_BINWALK=0
   LINUX_PATH_COUNTER=0
 
   # typically FIRMWARE_PATH is only a file if none of the EMBA extractors were able to extract something
@@ -38,28 +36,6 @@ P60_firmware_bin_extractor() {
 
   linux_basic_identification_helper
 
-  # Typically FIRMWARE_PATH is only a file if none of the EMBA extractors (including binwalk) were able
-  # to extract something - we try FACT extractor
-  if [[ -f "$FIRMWARE_PATH" ]]; then
-    # if we have not found a linux filesystem we try to extract the firmware again with FACT-extractor
-    # shellcheck disable=SC2153
-    if [[ $FACT_EXTRACTOR -eq 1 && $LINUX_PATH_COUNTER -lt 2 ]]; then
-      fact_extractor
-      linux_basic_identification_helper
-    fi
-
-    FILES_BINWALK=$(find "$OUTPUT_DIR_binwalk" -xdev -type f | wc -l )
-    if [[ -n "${OUTPUT_DIR_fact:-}" && -d "$OUTPUT_DIR_fact" ]]; then
-      FILES_FACT=$(find "$OUTPUT_DIR_fact" -xdev -type f | wc -l )
-    fi
-    print_output ""
-    print_output "[*] Default binwalk extractor extracted $ORANGE$FILES_BINWALK$NC files."
-  fi
-
-  if [[ ${FILES_FACT-0} -gt 0 ]]; then
-    print_output "[*] Default FACT-extractor extracted $ORANGE$FILES_FACT$NC files."
-  fi
-
   # If we have not found a linux filesystem we try to do a binwalk -e -M on every file for two times
   # Manual activation via -x switch:
   if [[ $LINUX_PATH_COUNTER -lt 2 || $DEEP_EXTRACTOR -eq 1 ]] ; then
@@ -242,30 +218,6 @@ deeper_extractor_helper() {
   fi
 }
 
-fact_extractor() {
-  sub_module_title "Extracting binary firmware blob with FACT-extractor"
-
-  export OUTPUT_DIR_fact
-  OUTPUT_DIR_fact=$(basename "$FIRMWARE_PATH")
-  OUTPUT_DIR_fact="$FIRMWARE_PATH_CP""/""$OUTPUT_DIR_fact"_fact_emba
-
-  print_output "[*] Extracting firmware to directory $OUTPUT_DIR_fact"
-
-  # this is not working in background. I have created a new function that gets executed in the background
-  # probably there is a more elegant way
-  #mapfile -t FACT_EXTRACT < <(./external/extract.py -o "$OUTPUT_DIR_fact" "$FIRMWARE_PATH" 2>/dev/null &)
-  extract_fact_helper &
-  WAIT_PIDS+=( "$!" )
-  wait_for_extractor
-  WAIT_PIDS=( )
-
-  # as we probably kill FACT and to not loose the results we need to execute FACT in a function 
-  # and read the results from the caller
-  if [[ -f "$TMP_DIR"/FACTer.txt ]] ; then
-    tee -a "$LOG_FILE" < "$TMP_DIR"/FACTer.txt 
-  fi
-}
-
 binwalking() {
   sub_module_title "Analyze binary firmware blob with binwalk"
 
@@ -322,21 +274,6 @@ extract_binwalk_helper() {
   fi
 }
 
-extract_fact_helper() {
-  if [[ -d /tmp/extractor ]]; then
-    # This directory is currently hard coded in FACT-extractor
-    rm -rf /tmp/extractor
-  fi
-
-  "$EXT_DIR"/fact_extractor/fact_extractor/fact_extract.py -d "$FIRMWARE_PATH" >> "$TMP_DIR"/FACTer.txt
-
-  if [[ -d /tmp/extractor/files ]]; then
-    cat /tmp/extractor/reports/meta.json >> "$TMP_DIR"/FACTer.txt
-    cp -r /tmp/extractor/files "$OUTPUT_DIR_fact"
-    rm -rf /tmp/extractor
-  fi
-}
-
 binwalk_deep_extract_helper() {
   if [[ "$BINWALK_VER_CHECK" == 1 ]]; then
     binwalk --run-as=root --preserve-symlinks -e -M -C "$FIRMWARE_PATH_CP" "$FILE_TMP" | tee -a "$LOG_FILE" || true