Merge pull request #946 from m-1-k-3/master

Include 0xdea semgrep rules and haruspex ghidra script, improve cwe-search integration

SECURITY.md

@@ -1,10 +1,11 @@
 # Security Policy
 
-EMBA is a platform for optimizing our research and testing tasks in the field of IoT, OT and general embedded analysis. Because of this, we include code quite early and sometimes in a very raw state. We do not recommend setting up EMBA as a productive environment or in an unprotected environment! If you are using EMBA you should know what you are doing.
+_EMBA_ is a platform for optimizing our research and testing tasks in the field of IoT, OT, ICS and general embedded analysis. Because of this, we include code quite early and sometimes in a very raw state. We do not recommend setting up _EMBA_ as a productive environment or in an unprotected environment! If you are using _EMBA_ you should know what you are doing.
 
 ## Reporting a Vulnerability
 
-If there is a security problem within EMBA please open an issue or contact us via one of the following ways:
-* [Issue](https://github.com/e-m-b-a/emba/issues)
-* [Discussions](https://github.com/e-m-b-a/emba/discussions)
-* [PM via Twitter](https://twitter.com/securefirmware)
+If there is a security problem within _EMBA_ please open an issue or contact us via one of the following ways:
+* [Open an Issue](https://github.com/e-m-b-a/emba/issues)
+* [Start a Discussions](https://github.com/e-m-b-a/emba/discussions)
+* [PM us via Twitter](https://twitter.com/securefirmware)
+* [PM us via Mastodon](https://infosec.exchange/@securefirmware)

config/VERSION.txt

@@ -1 +1 @@
-1.3.1-1433017b
+1.3.1-d871699f

helpers/helpers_emba_defaults.sh

@@ -66,6 +66,9 @@ set_defaults() {
   export PYTHON_CHECK=1
   export QEMULATION=0
   export FULL_EMULATION=0
+  export FULL_TEST=0            # with this variable we can control the behavior of s16 and s120 -> 0 is default an tests only
+                                # non Linux binaries (binaries not listed in config/linux_common_files.txt. 1 means we test every
+                                # binary which results in long runtimes
   # to get rid of all the running stuff we are going to kill it after RUNTIME
   export QRUNTIME="20s"
 
@@ -132,4 +135,5 @@ set_defaults() {
   TOTAL_MEMORY="$(grep MemTotal /proc/meminfo | awk '{print $2}' || true)"
   export Q_MOD_PID=""
   export F20_DEEP=1      # F20 module - set to cve-discovery caller for further processing
+  export GHIDRA_PATH="${EXT_DIR}""/ghidra/ghidra_10.3.1_PUBLIC"
 }

helpers/helpers_emba_dependency_check.sh

@@ -648,6 +648,8 @@ dependency_check()
         check_emulation_port "Running Qemu telnet service" "4321"
       fi
 
+      check_dep_file "GHIDRA" "${GHIDRA_PATH}""/ghidraRun"
+
       if [[ "${CWE_CHECKER}" -eq 1 ]]; then
         if [[ -d "${HOME}"/.cargo/bin ]]; then
           export PATH=${PATH}:"${HOME}"/.cargo/bin/:"${EXT_DIR}"/jdk/bin/

installer/I120_cwe_checker.sh

@@ -34,6 +34,7 @@ I120_cwe_checker() {
     echo -e "${ORANGE}""cwe-checker will be downloaded.""${NC}"
     print_file_info "OpenJDK" "OpenJDK for cwe-checker" "https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.12%2B7/OpenJDK11U-jdk_x64_linux_hotspot_11.0.12_7.tar.gz" "external/jdk.tar.gz"
     print_file_info "GHIDRA" "Ghidra for cwe-checker" "https://github.com/NationalSecurityAgency/ghidra/releases/download/Ghidra_10.3.1_build/ghidra_10.3.1_PUBLIC_20230614.zip" "external/ghidra.zip"
+    print_file_info "Ghidra Haruspex script" "Decompiled code exporter" "https://raw.githubusercontent.com/EMBA-support-repos/ghidra-scripts-0xdea/main/Haruspex.java" "external/ghidra_scripts"
 
     if [[ "${LIST_DEP}" -eq 1 ]] || [[ "${DOCKER_SETUP}" -eq 1 ]] ; then
       ANSWER=("n")
@@ -47,8 +48,32 @@ I120_cwe_checker() {
         echo
         apt-get install "${INSTALL_APP_LIST[@]}" -y --no-install-recommends
 
-        if ! [[ -d ./external/cwe_checker ]]; then
+        ## GHIDRA
+
+        # Java SDK for ghidra
+        if [[ -d ./external/jdk ]] ; then rm -R ./external/jdk ; fi
+        curl -L https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.12%2B7/OpenJDK11U-jdk_x64_linux_hotspot_11.0.12_7.tar.gz -Sf -o external/jdk.tar.gz
+        mkdir external/jdk 2>/dev/null
+        tar -xzf external/jdk.tar.gz -C external/jdk --strip-components 1
+        rm external/jdk.tar.gz
+
+        # Ghidra
+        if [[ -d ./external/ghidra ]] ; then rm -R ./external/ghidra ; fi
+        curl -L https://github.com/NationalSecurityAgency/ghidra/releases/download/Ghidra_10.3.1_build/ghidra_10.3.1_PUBLIC_20230614.zip -Sf -o external/ghidra.zip
+        mkdir external/ghidra 2>/dev/null
+        unzip -qo external/ghidra.zip -d external/ghidra
+        if [[ "${IN_DOCKER}" -eq 1 ]]; then
+          sed -i s@JAVA_HOME_OVERRIDE=@JAVA_HOME_OVERRIDE=/external/jdk@g external/ghidra/ghidra_10.3.1_PUBLIC/support/launch.properties
+        else
+          sed -i s@JAVA_HOME_OVERRIDE=@JAVA_HOME_OVERRIDE=external/jdk@g external/ghidra/ghidra_10.3.1_PUBLIC/support/launch.properties
+        fi
+        rm external/ghidra.zip
+
+        # further Ghidra installation stuff:
+        mkdir external/ghidra_scripts
+        download_file "Ghidra Haruspex script" "https://raw.githubusercontent.com/EMBA-support-repos/ghidra-scripts-0xdea/main/Haruspex.java" "external/ghidra_scripts/Haruspex.java"
 
+        if ! [[ -d ./external/cwe_checker ]]; then
           # cleanup first
           rm "${HOME}"/.cargo -r -f
           rm "${HOME}"/.config -r -f
@@ -58,25 +83,6 @@ I120_cwe_checker() {
 
           export PATH="${PATH}":"${HOME}"/.cargo/bin
 
-          # Java SDK for ghidra
-          if [[ -d ./external/jdk ]] ; then rm -R ./external/jdk ; fi
-          curl -L https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.12%2B7/OpenJDK11U-jdk_x64_linux_hotspot_11.0.12_7.tar.gz -Sf -o external/jdk.tar.gz
-          mkdir external/jdk 2>/dev/null
-          tar -xzf external/jdk.tar.gz -C external/jdk --strip-components 1
-          rm external/jdk.tar.gz
-
-          # Ghidra
-          if [[ -d ./external/ghidra ]] ; then rm -R ./external/ghidra ; fi
-          curl -L https://github.com/NationalSecurityAgency/ghidra/releases/download/Ghidra_10.3.1_build/ghidra_10.3.1_PUBLIC_20230614.zip -Sf -o external/ghidra.zip
-          mkdir external/ghidra 2>/dev/null
-          unzip -qo external/ghidra.zip -d external/ghidra
-          if [[ "${IN_DOCKER}" -eq 1 ]]; then
-            sed -i s@JAVA_HOME_OVERRIDE=@JAVA_HOME_OVERRIDE=/external/jdk@g external/ghidra/ghidra_10.3.1_PUBLIC/support/launch.properties
-          else
-            sed -i s@JAVA_HOME_OVERRIDE=@JAVA_HOME_OVERRIDE=external/jdk@g external/ghidra/ghidra_10.3.1_PUBLIC/support/launch.properties
-          fi
-          rm external/ghidra.zip
-
           if [[ -d ./external/cwe_checker ]] ; then rm -R ./external/cwe_checker ; fi
           mkdir ./external/cwe_checker 2>/dev/null
           git clone https://github.com/EMBA-support-repos/cwe_checker.git external/cwe_checker

installer/I20_sourcecode_check.sh

@@ -32,6 +32,7 @@ I20_sourcecode_check() {
     print_tool_info "luarocks" 1
     print_pip_info "semgrep"
     print_git_info "semgrep-rules" "returntocorp/semgrep-rules" "Standard library for Semgrep rules"
+    print_git_info "0xdea C/C++ semgrep-rules" "EMBA-support-repos/semgrep-rules-0xdea" "C/C++ Semgrep rules by 0xdea"
 
     print_file_info "iniscan/composer.phar" "A Dependency Manager for PHP" "https://getcomposer.org/installer" "external/iniscan/composer.phar"
 
@@ -52,6 +53,9 @@ I20_sourcecode_check() {
         if ! [[ -d external/semgrep-rules ]]; then
           git clone https://github.com/returntocorp/semgrep-rules.git external/semgrep-rules
         fi
+        if ! [[ -d external/semgrep-rules-0xdea ]]; then
+          git clone https://github.com/EMBA-support-repos/semgrep-rules-0xdea.git external/semgrep-rules-0xdea
+        fi
 
         if ! [[ -d "external/iniscan" ]] ; then
           mkdir external/iniscan

installer/IF20_nvd_feed.sh

@@ -20,7 +20,6 @@ IF20_nvd_feed() {
   if [[ "${LIST_DEP}" -eq 1 ]] || [[ "${IN_DOCKER}" -eq 1 ]] || [[ "${DOCKER_SETUP}" -eq 1 ]] || [[ "${CVE_SEARCH}" -eq 1 ]] || [[ "${FULL}" -eq 1 ]]; then
 
     print_git_info "NVD JSON data feed" "EMBA-support-repos/nvd-json-data-feeds" "The NVD data feed is JSON database to facilitate search and processing of CVEs."
-    echo -e "${ORANGE}""NVD JSON data feed will be downloaded.""${NC}"
 
     if [[ "${LIST_DEP}" -eq 1 ]] || [[ "${IN_DOCKER}" -eq 1 ]] ; then
       ANSWER=("n")

modules/F20_vul_aggregator.sh

@@ -802,7 +802,7 @@ check_kernel_major_v() {
   local lKERNEL_CVE_VER="${2:-}"
   local lCVE_ID="${3:-}"
   if [[ "${lBIN_VERSION_ONLY:0:1}" != "${lKERNEL_CVE_VER:0:1}" ]]; then
-    print_output "[-] Info for CVE ${ORANGE}${lCVE_ID}${NC} - Major kernel version not matching ${ORANGE}${lKERNEL_CVE_VER}${NC} vs ${ORANGE}${lBIN_VERSION_ONLY}${NC} - Higher false positive risk"
+    print_output "[-] Info for CVE ${ORANGE}${lCVE_ID}${NC} - Major kernel version not matching ${ORANGE}${lKERNEL_CVE_VER}${NC} vs ${ORANGE}${lBIN_VERSION_ONLY}${NC} - Higher false positive risk" "no_log"
   fi
 }
 

modules/F50_base_aggregator.sh

@@ -32,6 +32,7 @@ F50_base_aggregator() {
   S12_LOG="s12_binary_protection.txt"
   S13_LOG="s13_weak_func_check.txt"
   S14_LOG="s14_weak_func_radare_check.txt"
+  S16_LOG="s16_ghidra_decompile_checks.txt"
   S17_CSV_LOG="${CSV_DIR}""/s17_apk_check.csv"
   S24_CSV_LOG="${CSV_DIR}""/s24_kernel_bin_identifier.csv"
   S02_LOG="s02_uefi_fwhunt.txt"
@@ -503,6 +504,11 @@ output_binaries() {
 
   cwe_logging
 
+  if [[ "${S16_GHIDRA_SEMGREP:-0}" -gt 0 ]]; then
+    print_output "[+] Found ""${ORANGE}""${S16_GHIDRA_SEMGREP}""${GREEN}"" possible vulnerabilities (via semgrep in Ghidra decompiled code) in ""${ORANGE}""${S16_BINS_CHECKED}""${GREEN}"" tested binaries.""${NC}"
+    write_link "s16"
+  fi
+
   if [[ "${STRCPY_CNT:-0}" -gt 0 ]]; then
     print_output "[+] Found ""${ORANGE}""${STRCPY_CNT}""${GREEN}"" usages of strcpy in ""${ORANGE}""${BINS_CHECKED}""${GREEN}"" binaries.""${NC}"
     if [[ $(find "${LOG_DIR}""/s13_weak_func_check/" -type f 2>/dev/null | wc -l) -gt $(find "${LOG_DIR}""/s14_weak_func_radare_check/" -type f 2>/dev/null | wc -l) ]]; then
@@ -790,6 +796,8 @@ get_data() {
   export S22_PHP_INI_ISSUES=0
   export S22_PHP_INI_CONFIGS=0
   export S24_FAILED_KSETTINGS=0
+  export S16_GHIDRA_SEMGREP=0
+  export S16_BIN_CHECKED=0
   export MOD_DATA_COUNTER=0
   export KMOD_BAD=0
   export S40_WEAK_PERM_COUNTER=0
@@ -874,6 +882,10 @@ get_data() {
   else
     STRCPY_CNT="${STRCPY_CNT_13}"
   fi
+  if [[ -f "${LOG_DIR}"/"${S16_LOG}" ]]; then
+    S16_GHIDRA_SEMGREP=$(grep -a "\[\*\]\ Statistics:" "${LOG_DIR}"/"${S16_LOG}" | cut -d: -f2 || true)
+    S16_BINS_CHECKED=$(grep -a "\[\*\]\ Statistics:" "${LOG_DIR}"/"${S16_LOG}" | cut -d: -f3 || true)
+  fi
   if [[ -f "${LOG_DIR}"/"${S20_LOG}" ]]; then
     S20_SHELL_VULNS=$(grep -a "\[\*\]\ Statistics:" "${LOG_DIR}"/"${S20_LOG}" | cut -d: -f2 || true)
     S20_SCRIPTS=$(grep -a "\[\*\]\ Statistics:" "${LOG_DIR}"/"${S20_LOG}" | cut -d: -f3 || true)
@@ -1141,7 +1153,7 @@ cwe_logging() {
     mapfile -t CWE_OUT < <( jq -r '.[] | "\(.name) \(.description)"' "${LOG_DIR}"/"${LOG_DIR_MOD}"/cwe_*.log | cut -d\) -f1 | tr -d '('  | sort -u|| true)
 
     if [[ ${#CWE_OUT[@]} -gt 0 ]] ; then
-      print_output "[+] cwe-checker found a total of ""${ORANGE}""${TOTAL_CWE_CNT}""${GREEN}"" of the following security issues:"
+      print_output "[+] cwe-checker found a total of ""${ORANGE}""${TOTAL_CWE_CNT}""${GREEN}"" security issues in firmware binaries:"
       write_link "s120"
       for CWE_ENTRY in "${CWE_OUT[@]}"; do
         CWE="$(echo "${CWE_ENTRY}" | awk '{print $1}')"

modules/S120_cwe_checker.sh

@@ -26,23 +26,25 @@ S120_cwe_checker()
 {
   if [[ ${CWE_CHECKER} -eq 1 ]] ; then
     module_log_init "${FUNCNAME[0]}"
-    module_title "Check binaries with cwe-checker"
+    module_title "Check binaries for vulnerabilities with cwe-checker"
     pre_module_reporter "${FUNCNAME[0]}"
-    local CWE_CNT_=0
+    local lCWE_CNT_=0
+    local lTESTED_BINS=0
 
     [[ "${IN_DOCKER}" -eq 1 ]] && cwe_container_prepare
 
     cwe_check
 
     if [[ -f "${TMP_DIR}"/CWE_CNT.tmp ]]; then
-      CWE_CNT_=$(awk '{sum += $1 } END { print sum }' "${TMP_DIR}"/CWE_CNT.tmp || true)
+      lCWE_CNT_=$(awk '{sum += $1 } END { print sum }' "${TMP_DIR}"/CWE_CNT.tmp || true)
+      lTESTED_BINS=$(grep -c " Tested " "${LOG_FILE}" || true)
     fi
 
-    final_cwe_log "${CWE_CNT_}"
+    final_cwe_log "${lCWE_CNT_}"
 
     write_log ""
-    write_log "[*] Statistics:${CWE_CNT_}"
-    module_end_log "${FUNCNAME[0]}" "${CWE_CNT_}"
+    write_log "[*] Statistics:${lCWE_CNT_}:${lTESTED_BINS}"
+    module_end_log "${FUNCNAME[0]}" "${lCWE_CNT_}"
   else
     print_output "[!] Check with cwe-checker is disabled!"
     print_output "[!] Enable it with the -c switch."
@@ -111,6 +113,15 @@ cwe_checker_threaded () {
 
   local NAME=""
   NAME=$(basename "${BINARY_}")
+
+  if [[ -f "${BASE_LINUX_FILES}" && "${FULL_TEST}" -eq 0 ]]; then
+    # if we have the base linux config file we only test non known Linux binaries
+    # with this we do not waste too much time on open source Linux stuff
+    if grep -E -q "^${NAME}$" "${BASE_LINUX_FILES}" 2>/dev/null; then
+      return
+    fi
+  fi
+
   local OLD_LOG_FILE="${LOG_FILE}"
   local LOG_FILE="${LOG_PATH_MODULE}""/cwe_check_""${NAME}"".txt"
   BINARY_=$(readlink -f "${BINARY_}")

modules/S20_shell_check.sh

@@ -118,12 +118,14 @@ s20_eval_script_check() {
   local SH_SCRIPT=""
   local GPT_PRIO_=3
   local GPT_ANCHOR_=""
+  local EVAL_RESULTS=0
 
   sub_module_title "Summary of shell eval usages"
 
   for SH_SCRIPT in "${SH_SCRIPTS_[@]}" ; do
-    print_output "[*] Testing ${ORANGE}${SH_SCRIPT}${NC} for eval usage" "no_log"
+    # print_output "[*] Testing ${ORANGE}${SH_SCRIPT}${NC} for eval usage" "no_log"
     if grep "eval " "${SH_SCRIPT}" | grep -q -v "^#.*"; then
+      EVAL_RESULTS=1
       SH_SCRIPT_NAME="$(basename "${SH_SCRIPT}")"
       local SHELL_LOG="${LOG_PATH_MODULE}"/sh_eval_sources/"${SH_SCRIPT_NAME}".log
       ! [[ -d "${LOG_PATH_MODULE}"/sh_eval_sources/ ]] && mkdir "${LOG_PATH_MODULE}"/sh_eval_sources/
@@ -141,6 +143,9 @@ s20_eval_script_check() {
       fi
     fi
   done
+  if [[ "${EVAL_RESULTS}" -eq 0 ]]; then
+    print_output "[-] No eval usage found in shell scripts"
+  fi
 }
 
 s20_script_check() {

modules/S22_php_check.sh

@@ -241,6 +241,10 @@ s22_check_php_ini(){
   PHP_INI_WARNINGS=0
 
   mapfile -t PHP_INI_FILE < <( find "${FIRMWARE_PATH}" -xdev "${EXCL_FIND[@]}" -iname 'php.ini' -exec md5sum {} \; 2>/dev/null | sort -u -k1,1 | cut -d\  -f3 )
+  if [[ "${#PHP_INI_FILE[@]}" -eq 0 ]]; then
+    print_output "[-] No PHP.ini issues found"
+    return
+  fi
 
   disable_strict_mode "${STRICT_MODE}"
   for PHP_FILE in "${PHP_INI_FILE[@]}" ;  do

scan-profiles/default-scan.emba

@@ -19,6 +19,9 @@ export THREADED=1
 export SHORT_PATH=1
 export HTML=1
 export QEMULATION=1
+# cwe-checker is now only testing non-linux binaries. With this mechanism we can enable
+# it in the default profile.
+export CWE_CHECKER=1
 # the following modules are long running modules which are disabled in the default profile
 export MODULE_BLACKLIST=( "S15_radare_decompile_checks" "S99_grepit" "S110_yara_check" )
 

scan-profiles/full-scan.emba

@@ -21,6 +21,7 @@ export HTML=1
 export CWE_CHECKER=1
 export QEMULATION=1
 export FULL_EMULATION=1
+export FULL_TEST=1
 
 # we output the profile only at the beginning - outside the docker environment
 if [[ $IN_DOCKER -ne 1 ]] ; then