Check project updates

.github/workflows/check_project.yml

@@ -0,0 +1,31 @@
+# This workflow tests the project for proper linting
+
+name: Check the project with check_project.sh
+
+on:
+  push:
+    branches:    
+      - '**'        # matches every branch
+  pull_request:
+    branches:
+      - '**'
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout Branch
+      uses: actions/checkout@v2
+    - name: Install dependencies for check script
+      run: |
+        sudo apt-get install -y shellcheck python3-pip
+        pip3 install -U requests
+        pip3 install semgrep
+        mkdir ./external
+        git clone https://github.com/returntocorp/semgrep-rules.git external/semgrep-rules
+    - name: Run check_project.sh
+      run: |
+        ./check_project.sh

check_project.sh

@@ -45,6 +45,7 @@ MODULES_TO_CHECK_ARR=()
 MODULES_TO_CHECK_ARR_TAB=()
 MODULES_TO_CHECK_ARR_SEMGREP=()
 MODULES_TO_CHECK_ARR_DOCKER=()
+MODULES_TO_CHECK_ARR_PERM=()
 
 import_config_scripts() {
   mapfile -t HELPERS < <(find "$CONF_DIR" -iname "*.sh" 2>/dev/null)
@@ -102,6 +103,18 @@ import_installer() {
   done
 }
 
+import_emba_main() {
+  MODULES=()
+  mapfile -t MODULES < <(find ./ -iname "emba.sh" -o -iname "installer.sh" -o -iname "check_project.sh" 2>/dev/null)
+  for LINE in "${MODULES[@]}"; do
+    if (file "$LINE" | grep -q "shell script"); then
+      echo "$LINE"
+      SOURCES+=("$LINE")
+    fi
+  done
+}
+
+
 dockerchecker() {
   echo -e "\\n""$ORANGE""$BOLD""EMBA docker-files check""$NC""\\n""$BOLD""=================================================================""$NC"
   mapfile -t DOCKER_COMPS < <(find . -maxdepth 1 -iname "docker-compose*.yml")
@@ -120,24 +133,9 @@ dockerchecker() {
 check() {
   echo -e "\\n""$ORANGE""$BOLD""Embedded Linux Analyzer Shellcheck""$NC""\\n""$BOLD""=================================================================""$NC"
 
-  echo -e "\\n""$GREEN""Run shellcheck on this script:""$NC""\\n"
-  if shellcheck ./check_project.sh || [[ $? -ne 1 && $? -ne 2 ]]; then
-    echo -e "$GREEN""$BOLD""==> SUCCESS""$NC""\\n"
-  else
-    echo -e "\\n""$ORANGE$BOLD==> FIX ERRORS""$NC""\\n"
-    MODULES_TO_CHECK_ARR+=("check_project.sh")
-  fi
-
-  echo -e "\\n""$GREEN""Run shellcheck on installer:""$NC""\\n"
-  if shellcheck ./installer.sh || [[ $? -ne 1 && $? -ne 2 ]]; then
-    echo -e "$GREEN""$BOLD""==> SUCCESS""$NC""\\n"
-  else
-    echo -e "\\n""$ORANGE$BOLD==> FIX ERRORS""$NC""\\n"
-    MODULES_TO_CHECK_ARR+=("installer.sh")
-  fi
-
   echo -e "\\n""$GREEN""Load all files for check:""$NC""\\n"
-  echo "./emba.sh"
+
+  import_emba_main
   import_installer
   import_helper
   import_config_scripts
@@ -166,14 +164,25 @@ check() {
     fi
 
     echo -e "\\n""$GREEN""Run ${ORANGE}semgrep$GREEN on $ORANGE$SOURCE""$NC""\\n"
-    semgrep --disable-version-check --config "$EXT_DIR"/semgrep-rules/bash "$SOURCE" | tee /tmp/emba_semgrep.log
+    semgrep --disable-version-check --metrics=off --config "$EXT_DIR"/semgrep-rules/bash "$SOURCE" | tee /tmp/emba_semgrep.log
     if grep -q "Findings:" /tmp/emba_semgrep.log; then
       echo -e "\\n""$ORANGE""$BOLD""==> FIX ERRORS""$NC""\\n"
       MODULES_TO_CHECK_ARR_SEMGREP+=("$SOURCE")
     else
       echo -e "$GREEN""$BOLD""==> SUCCESS""$NC""\\n"
     fi
   done
+
+  echo -e "\\n""$GREEN""Check all scripts for correct permissions:""$NC""\\n"
+  for SOURCE in "${SOURCES[@]}"; do
+    echo -e "\\n""$GREEN""Check ${ORANGE}permission$GREEN on $ORANGE$SOURCE""$NC""\\n"
+    if stat -L -c "%a" "$SOURCE" | grep -q "755"; then
+      echo -e "$GREEN""$BOLD""==> SUCCESS""$NC""\\n"
+    else
+      echo -e "\\n""$ORANGE""$BOLD""==> FIX ERRORS""$NC""\\n"
+      MODULES_TO_CHECK_ARR_PERM+=("$SOURCE")
+    fi
+  done
 }
 
 summary() {
@@ -215,6 +224,15 @@ summary() {
     done
     echo -e "$ORANGE""WARNING: Fix the errors before pushing to the EMBA repository!"
   fi
+  if [[ "${#MODULES_TO_CHECK_ARR_PERM[@]}" -gt 0 ]]; then
+    echo -e "\\n\\n""$GREEN$BOLD""SUMMARY:$NC\\n"
+    echo -e "Modules to check (permissions): ${#MODULES_TO_CHECK_ARR_PERM[@]}\\n"
+    for MODULE in "${MODULES_TO_CHECK_ARR_PERM[@]}"; do
+      echo -e "$ORANGE$BOLD==> FIX MODULE: ""$MODULE""$NC"
+    done
+    echo -e "$ORANGE""WARNING: Fix the errors before pushing to the EMBA repository!"
+  fi
+
 
 }
 
@@ -239,3 +257,7 @@ check
 dockerchecker
 summary
 
+if [[ "${#MODULES_TO_CHECK_ARR_TAB[@]}" -gt 0 ]] || [[ "${#MODULES_TO_CHECK_ARR[@]}" -gt 0 ]] || [[ "${#MODULES_TO_CHECK_ARR[@]}" -gt 0 ]] || \
+  [[ "${#MODULES_TO_CHECK_ARR_SEMGREP[@]}" -gt 0 ]] || [[ "${#MODULES_TO_CHECK_ARR_DOCKER[@]}" -gt 0 ]] || [[ "${#MODULES_TO_CHECK_ARR_PERM[@]}" -gt 0 ]]; then
+  exit 1
+fi

helpers/fixImage_user_mode_emulation.sh

@@ -89,7 +89,7 @@ if ("$BUSYBOX" grep -sq "/dev/gpio/in" /bin/gpio) ||
   ("$BUSYBOX" grep -sq "/dev/gpio/in" /usr/lib/libshared.so); then
   echo "[*] Creating /dev/gpio/in (required for some linksys devices)"
   "$BUSYBOX" mkdir -p /dev/gpio
-  # shellcheck disable=SC3037
+  # shellcheck disable=SC3037,2039
   echo -ne "\xff\xff\xff\xff" > /dev/gpio/in
 fi
 

helpers/helpers_emba_prepare.sh

@@ -413,6 +413,23 @@ prepare_binary_arr()
   #rm_proc_binary "${BINARIES[@]}"
 }
 
+prepare_file_arr_limited() {
+  local FIRMWARE_PATH="${1:-}"
+  export FILE_ARR_LIMITED=()
+
+  if ! [[ -d "$FIRMWARE_PATH" ]]; then
+    return
+  fi
+
+  echo ""
+  print_output "[*] Unique and limited file array generation for $ORANGE$FIRMWARE_PATH$NC (could take some time)\\n"
+
+  readarray -t FILE_ARR_LIMITED < <(find "$FIRMWARE_PATH" -xdev "${EXCL_FIND[@]}" -type f ! \( -iname "*.udeb" -o -iname "*.deb" \
+    -o -iname "*.ipk" -o -iname "*.pdf" -o -iname "*.php" -o -iname "*.txt" -o -iname "*.doc" -o -iname "*.rtf" -o -iname "*.docx" \
+    -o -iname "*.htm" -o -iname "*.html" -o -iname "*.md5" -o -iname "*.sha1" -o -iname "*.torrent" -o -iname "*.png" -o -iname "*.svg" \
+    -o -iname "*.js" -o -iname "*.info" \) -exec md5sum {} \; 2>/dev/null | sort -u -k1,1 | cut -d\  -f3-)
+}
+
 set_etc_paths()
 {
   # For the case if ./etc isn't in root of provided firmware or is renamed like e.g. ./etc-ro:

installer/IP35_uefi_extraction.sh

@@ -18,33 +18,34 @@
 IP35_uefi_extraction() {
   module_title "${FUNCNAME[0]}"
 
-
-  INSTALL_APP_LIST=()
+  if [[ "$LIST_DEP" -eq 1 ]] || [[ $IN_DOCKER -eq 1 ]] || [[ $DOCKER_SETUP -eq 0 ]] || [[ $FULL -eq 1 ]]; then
+    INSTALL_APP_LIST=()
 
-  print_file_info "UEFIExtract_NE_A62_linux_x86_64.zip" "Release-version A62" "https://github.com/LongSoft/UEFITool/releases/download/A62/UEFIExtract_NE_A62_linux_x86_64.zip" "external/UEFITool/UEFIExtract_NE_A62_linux_x86_64.zip"
-  print_tool_info "unzip" 1
+    print_file_info "UEFIExtract_NE_A62_linux_x86_64.zip" "Release-version A62" "https://github.com/LongSoft/UEFITool/releases/download/A62/UEFIExtract_NE_A62_linux_x86_64.zip" "external/UEFITool/UEFIExtract_NE_A62_linux_x86_64.zip"
+    print_tool_info "unzip" 1
 
-  if [[ "$LIST_DEP" -eq 1 ]] || [[ $DOCKER_SETUP -eq 1 ]]; then
+    if [[ "$LIST_DEP" -eq 1 ]] || [[ $DOCKER_SETUP -eq 1 ]]; then
       ANSWER=("n")
-  else
+    else
       echo -e "\\n""$MAGENTA""$BOLD""UEFI Extraction Tool"" will be downloaded (if not already on the system) installed!""$NC"
       ANSWER=("y")
-  fi
+    fi
 
-  case ${ANSWER:0:1} in
-    y|Y )
-      apt-get install "${INSTALL_APP_LIST[@]}" -y --no-install-recommends
-      if ! [[ -d external/UEFITool ]]; then
-        mkdir external/UEFITool
-      fi
-      download_file "UEFIExtract_NE_A62_linux_x86_64.zip" "https://github.com/LongSoft/UEFITool/releases/download/A62/UEFIExtract_NE_A62_linux_x86_64.zip" "external/UEFITool/UEFIExtract_NE_A62_linux_x86_64.zip"
-      if [[ -f "external/UEFITool/UEFIExtract_NE_A62_linux_x86_64.zip" ]]; then
-        if ! [[ -f external/UEFITool/UEFIExtract ]]; then
-          unzip external/UEFITool/UEFIExtract_NE_A62_linux_x86_64.zip -d external/UEFITool
+    case ${ANSWER:0:1} in
+      y|Y )
+        apt-get install "${INSTALL_APP_LIST[@]}" -y --no-install-recommends
+        if ! [[ -d external/UEFITool ]]; then
+          mkdir external/UEFITool
         fi
-      else
-        echo -e "$ORANGE""UEFITool installation failed - check it manually""$NC"
-      fi
-      ;;
-  esac
-}
+        download_file "UEFIExtract_NE_A62_linux_x86_64.zip" "https://github.com/LongSoft/UEFITool/releases/download/A62/UEFIExtract_NE_A62_linux_x86_64.zip" "external/UEFITool/UEFIExtract_NE_A62_linux_x86_64.zip"
+        if [[ -f "external/UEFITool/UEFIExtract_NE_A62_linux_x86_64.zip" ]]; then
+          if ! [[ -f external/UEFITool/UEFIExtract ]]; then
+            unzip external/UEFITool/UEFIExtract_NE_A62_linux_x86_64.zip -d external/UEFITool
+          fi
+        else
+          echo -e "$ORANGE""UEFITool installation failed - check it manually""$NC"
+        fi
+        ;;
+    esac
+  fi
+}

modules/L10_system_emulation.sh

@@ -216,8 +216,8 @@ create_emulation_filesystem() {
     mkdir -p "$MNT_POINT/firmadyne/libnvram.override/" || true
 
     print_output "[*] Patching filesystem (chroot)"
-    cp "$(which busybox)" "$MNT_POINT" || true
-    cp "$(which bash-static)" "$MNT_POINT" || true
+    cp "$(command -v busybox)" "$MNT_POINT" || true
+    cp "$(command -v bash-static)" "$MNT_POINT" || true
 
     if [[ -f "$CSV_DIR"/s24_kernel_bin_identifier.csv ]]; then
       # kernelInit is getting the output of the init command line we get from s24
@@ -781,8 +781,8 @@ handle_fs_mounts() {
   done
 
   # now we need to startup the inferFile/inferService script again
-  cp "$(which bash-static)" "$MNT_POINT" || true
-  cp "$(which busybox)" "$MNT_POINT" || true
+  cp "$(command -v bash-static)" "$MNT_POINT" || true
+  cp "$(command -v busybox)" "$MNT_POINT" || true
   cp "$MODULE_SUB_PATH/inferService.sh" "${MNT_POINT}" || true
   print_output "[*] inferService.sh (chroot)"
   FIRMAE_BOOT=${FIRMAE_BOOT} FIRMAE_ETC=${FIRMAE_ETC} timeout --preserve-status --signal SIGINT 120 chroot "${MNT_POINT}" /bash-static /inferService.sh | tee -a "$LOG_FILE"

modules/P22_Zyxel_zip_decrypt.sh

@@ -100,12 +100,12 @@ zyxel_zip_extractor() {
 
       print_output "[*] Running Zyxel emulation for key extraction ..."
 
-      if ! [[ -e "$(which "$EMULATOR")" ]]; then
+      if ! [[ -e "$(command -v "$EMULATOR")" ]]; then
         print_output "[-] No valid emulator ($ORANGE$EMULATOR$NC) found in your environment"
         return
       fi
 
-      cp "$(which "$EMULATOR")" "$ZLD_DIR" || ( print_output "[-] Something went wrong" && return)
+      cp "$(command -v "$EMULATOR")" "$ZLD_DIR" || ( print_output "[-] Something went wrong" && return)
       cp "$RI_FILE_BIN_PATH" "$ZLD_DIR" || ( print_output "[-] Something went wrong" && return)
       ZLD_BIN=$(basename "$ZLD_BIN")
 

modules/P35_UEFI_extractor.sh

@@ -105,7 +105,7 @@ uefi_extractor(){
     mkdir -p "$EXTRACTION_DIR_"
   fi
   cp "$FIRMWARE_PATH_" "$EXTRACTION_DIR_"
-  $UEFI_EXTRACT_BIN "$EXTRACTION_DIR_"firmware all &> "$LOG_PATH_MODULE"/uefi_extractor_"$FIRMWARE_NAME_".log
+  "$UEFI_EXTRACT_BIN" "$EXTRACTION_DIR_"firmware all &> "$LOG_PATH_MODULE"/uefi_extractor_"$FIRMWARE_NAME_".log
   UEFI_EXTRACT_REPORT_FILE="$EXTRACTION_DIR_"firmware.report.txt
   mv "$UEFI_EXTRACT_REPORT_FILE" "$LOG_PATH_MODULE"
   UEFI_EXTRACT_REPORT_FILE="$LOG_PATH_MODULE"/firmware.report.txt

modules/P60_firmware_bin_extractor.sh

@@ -89,7 +89,6 @@ disk_space_protection() {
 deep_extractor() {
   sub_module_title "Deep extraction mode"
 
-  FILE_ARR_TMP=()
   FILE_MD5=""
 
   FILES_BEFORE_DEEP=$(find "$FIRMWARE_PATH_CP" -xdev -type f | wc -l )
@@ -141,17 +140,12 @@ deeper_extractor_helper() {
   else
     local MATRYOSHKA=0
   fi
-  local FILE_ARR_TMP=()
   local FILE_TMP=""
   local FILE_MD5=""
 
-  readarray -t FILE_ARR_TMP < <(find "$FIRMWARE_PATH_CP" -xdev "${EXCL_FIND[@]}" -type f ! \( -iname "*.udeb" -o -iname "*.deb" \
-    -o -iname "*.ipk" -o -iname "*.pdf" -o -iname "*.php" -o -iname "*.txt" -o -iname "*.doc" -o -iname "*.rtf" -o -iname "*.docx" \
-    -o -iname "*.htm" -o -iname "*.html" -o -iname "*.md5" -o -iname "*.sha1" -o -iname "*.torrent" -o -iname "*.png" -o -iname "*.svg" \
-    -o -iname "*.js" \) \
-    -exec md5sum {} \; 2>/dev/null | sort -u -k1,1 | cut -d\  -f3- )
+  prepare_file_arr_limited "$FIRMWARE_PATH_CP"
 
-  for FILE_TMP in "${FILE_ARR_TMP[@]}"; do
+  for FILE_TMP in "${FILE_ARR_LIMITED[@]}"; do
 
     FILE_MD5="$(md5sum "$FILE_TMP" | awk '{print $1}')"
     # let's check the current md5sum against our array of unique md5sums - if we have a match this is already extracted

modules/S02_UEFI_FwHunt.sh

@@ -31,9 +31,9 @@ S02_UEFI_FwHunt() {
   local MAX_MOD_THREADS=$((MAX_MOD_THREADS/2))
   local EXTRACTED_FILE=""
 
-  if [[ "$RTOS" -eq 1 ]] && [[ "$UEFI_DETECTED" -eq 1 ]]; then
+  if [[ "$RTOS" -eq 1 ]] && [[ "$UEFI_DETECTED" -eq 1 ]] && [[ -v FILE_ARR_LIMITED ]]; then
     print_output "[*] Starting FwHunter UEFI firmware vulnerability detection"
-    for EXTRACTED_FILE in "${FILE_ARR[@]}"; do
+    for EXTRACTED_FILE in "${FILE_ARR_LIMITED[@]}"; do
       if [[ $THREADED -eq 1 ]]; then
         fwhunter "$EXTRACTED_FILE" &
         WAIT_PIDS_S02+=( "$!" )

modules/S115_usermode_emulator.sh

@@ -243,7 +243,7 @@ prepare_emulator() {
       print_output "$(indent "$(red "Terminating EMBA now.\\n")")"
       exit 1
     else
-      cp "$(which "$EMULATOR")" "$R_PATH"
+      cp "$(command -v "$EMULATOR")" "$R_PATH"
     fi
 
     if ! [[ -d "$R_PATH""/proc" ]] ; then
@@ -281,7 +281,7 @@ prepare_emulator() {
     print_output "[*] Final fixes of the root filesytem in a chroot environment"
     cp "$HELP_DIR"/fixImage_user_mode_emulation.sh "$R_PATH"
     chmod +x "$R_PATH"/fixImage_user_mode_emulation.sh
-    cp "$(which busybox)" "$R_PATH"
+    cp "$(command -v busybox)" "$R_PATH"
     chmod +x "$R_PATH"/busybox
     if [[ "$CHROOT" == "jchroot" ]]; then
       "$CHROOT" "${OPTS[@]}" "$R_PATH" -- /busybox ash /fixImage_user_mode_emulation.sh | tee -a "$LOG_PATH_MODULE"/chroot_fixes.txt || print_output "[-] Something weird going wrong in chroot filesystem fixing"

modules/S24_kernel_bin_identifier.sh

@@ -22,21 +22,19 @@ S24_kernel_bin_identifier()
   pre_module_reporter "${FUNCNAME[0]}"
 
   local NEG_LOG=0
-  local FILE_ARR_TMP=()
   local FILE=""
   local K_VER=""
   local K_INIT=""
   local CFG_MD5=""
   export KCFG_MD5=()
 
-  readarray -t FILE_ARR_TMP < <(find "$FIRMWARE_PATH_CP" -xdev "${EXCL_FIND[@]}" -type f ! \( -iname "*.udeb" -o -iname "*.deb" \
-    -o -iname "*.ipk" -o -iname "*.pdf" -o -iname "*.php" -o -iname "*.txt" -o -iname "*.doc" -o -iname "*.rtf" -o -iname "*.docx" \
-    -o -iname "*.htm" -o -iname "*.html" -o -iname "*.md5" -o -iname "*.sha1" -o -iname "*.torrent" \) \
-    -exec md5sum {} \; 2>/dev/null | sort -u -k1,1 | cut -d\  -f3 )
+  if ! [[ -v FILE_ARR_LIMITED ]] || [[ "${#FILE_ARR_LIMITED[@]}" -eq 0 ]]; then
+    prepare_file_arr_limited "$FIRMWARE_PATH_CP"
+  fi
 
   write_csv_log "Kernel version" "file" "identified init"
 
-  for FILE in "${FILE_ARR_TMP[@]}" ; do
+  for FILE in "${FILE_ARR_LIMITED[@]}" ; do
     if file "$FILE" | grep -q "ASCII text"; then
       # reduce false positive rate
       continue

modules/S99_grepit.sh

@@ -63,7 +63,7 @@ S99_grepit() {
   # Do not remove -rP if you don't know what you are doing, otherwise you probably break this script
   local GREP_ARGUMENTS=(-a -n -A 1 -B 3 -rP)
   # Open the colored outputs with "less -R" or cat, otherwise remove --color=always (not recommended, colors help to find the matches in huge text files)
-  local COLOR_ARGUMENTS=(--color=always)
+  local COLOR_ARGUMENTS=("--color=always")
   export STANDARD_GREP_ARGUMENTS=("${GREP_ARGUMENTS[@]}" "${COLOR_ARGUMENTS[@]}" "${LIMIT_GREP[@]}")
   export ENABLE_LEAST_LIKELY=0