Merge pull request #876 from m-1-k-3/uefi_improvements

UEFI analysis improvements

CONTRIBUTING.md

@@ -66,8 +66,12 @@ It also sketches the typical integration process of patches.
 
 - Don't use backticks anymore, use $(..) instead
 
+- Don't use `grep -R` for recursive grep search. Instead use `find -type f -exec grep something {} \;` or use `grep -r`
+
 - Use double square \[[]] brackets (conditional expressions) instead of single square [] brackets
 
+- We require variable braces. Instead of using `$VARIABLE` please use `${VARIABLE}`
+
 - Whenever possible try to avoid `tr` `sed` `awk` and use bash internal functions instead, see e.g. [bash shell parameter substitution](http://www.cyberciti.biz/tips/bash-shell-parameter-substitution-2.html). Using bash internals is faster as it does not fork, fopen and pipes the results back.
 
 - At least ["weak quoting"](https://wiki.bash-hackers.org/syntax/quoting#weak_quoting) is required - unquoted variable processing is not permitted

README.md

@@ -72,6 +72,12 @@ For further details on EMBA's AI analysis engine check the [wiki](https://github
 ```console
 sudo ./emba -l ~/log -f ~/firmware -p ./scan-profiles/default-scan-gpt.emba
 
+```
+## Quick start in firmware diffing mode
+For further details on EMBA's firmware diffing mechanism check the [wiki](https://github.com/e-m-b-a/emba/wiki/Firmware-diffing).
+```console
+sudo ./emba -l ~/log -f ~/1st-firmware -o ~/2nd-newer-firmware -t
+
 ```
 
 ---

helpers/helpers_emba_defaults.sh

@@ -47,6 +47,7 @@ set_defaults() {
   export FW_DEVICE=""
   export FW_NOTES=""
   export ARCH=""
+  export EFI_ARCH=""
   export EXLUDE=()
   export SELECT_MODULES=()
   export MODULES_EXPORTED=()

helpers/helpers_emba_dependency_check.sh

@@ -214,7 +214,7 @@ dependency_check()
       print_output "[*] Info: Proxy settings detected: ${ORANGE}${PROXY_SETTINGS}${NC}" "no_log"
     fi
 
-    if ! curl -Is https://www.google.com &>/dev/null ; then
+    if ! curl --connect-timeout 5 -Is https://www.google.com &>/dev/null ; then
       echo -e "${RED}""not ok""${NC}"
       print_output "[-] Warning: Quest container has no internet connection!" "no_log"
     else
@@ -730,6 +730,8 @@ architecture_dep_check() {
   elif [[ "${ARCH}" == "x64" ]] ; then
     # ARCH_STR="i386:x86-64"
     ARCH_STR="x86-64"
+  elif [[ "${ARCH}" == "x86-64" ]] ; then
+    ARCH_STR="x86-64"
   elif [[ "${ARCH}" == "PPC" ]] ; then
     # ARCH_STR="powerpc:common"
     ARCH_STR="powerpc"

helpers/helpers_emba_extractors.sh

@@ -39,3 +39,19 @@ docker_container_extractor() {
   fi
 }
 
+binwalker_matryoshka() {
+  local FIRMWARE_PATH_="${1:-}"
+  local OUTPUT_DIR_BINWALK="${2:-}"
+  local BINWALK_BIN="binwalk"
+
+  sub_module_title "Analyze binary firmware blob with binwalk"
+
+  print_output "[*] Extracting firmware to directory ${ORANGE}${OUTPUT_DIR_BINWALK}${NC}"
+
+  if ! [[ -d "${OUTPUT_DIR_BINWALK}" ]]; then
+    mkdir -p "${OUTPUT_DIR_BINWALK}"
+  fi
+
+  timeout --preserve-status --signal SIGINT 300 "${BINWALK_BIN}" --run-as=root --preserve-symlinks -e -M --dd='.*' -C "${OUTPUT_DIR_BINWALK}" "${FIRMWARE_PATH_}" | tee -a "${LOG_FILE}" || true
+  print_ln
+}

helpers/helpers_emba_prepare.sh

@@ -340,6 +340,10 @@ architecture_check()
         ARCH="${D_ARCH}"
         export ARCH
       fi
+    elif [[ -n "${EFI_ARCH}" ]]; then
+      print_output "$(indent "Detected architecture of the UEFI firmware: ""${ORANGE}""${EFI_ARCH}""${NC}")""\\n"
+      ARCH="${EFI_ARCH}"
+      export ARCH
     else
       print_output "$(indent "$(red "Based on binary identification no architecture was detected.")")"
       if [[ -n "${ARCH}" ]] ; then
@@ -468,8 +472,7 @@ check_firmware()
   fi
 
   if [[ ${DIR_COUNT} -lt 5 ]] && [[ "${RTOS}" -eq 1 ]]; then
-    print_ln "no_log"
-    print_output "[!] Your firmware looks not like a regular Linux system, sure that you have entered the correct path?"
+    print_output "[-] Your firmware does not look like a regular Linux system."
   fi
   if [[ "${RTOS}" -eq 0 ]] || [[ ${DIR_COUNT} -gt 4 ]]; then
     print_output "[+] Your firmware looks like a regular Linux system."

installer/I02_UEFI_fwhunt.sh

@@ -32,6 +32,7 @@ I02_UEFI_fwhunt() {
     print_git_info "fwhunt-scan" "EMBA-support-repos/fwhunt-scan" "Tools for analyzing UEFI firmware and checking UEFI modules with FwHunt rules."
     print_git_info "fwhunt-rules" "EMBA-support-repos/FwHunt" "The Binarly Firmware Hunt (FwHunt) rule format was designed to scan for known vulnerabilities in UEFI firmware."
     print_git_info "BIOSUtilities" "EMBA-support-repos/BIOSUtilities" "Various BIOS Utilities for Modding/Research"
+    print_git_info "BGScriptTool" "platomav/BGScriptTool" "The tool allows you to assemble and disassemble BIOS Guard script."
 
     if [[ "${LIST_DEP}" -eq 1 ]] || [[ "${DOCKER_SETUP}" -eq 1 ]] ; then
       ANSWER=("n")
@@ -66,7 +67,14 @@ I02_UEFI_fwhunt() {
         if [[ -d external/BIOSUtilities ]]; then
           rm -r external/BIOSUtilities
         fi
+        if [[ -d external/BGScriptTool ]]; then
+          rm -r external/BGScriptTool
+        fi
         git clone https://github.com/EMBA-support-repos/BIOSUtilities.git external/BIOSUtilities
+        git clone https://github.com/platomav/BGScriptTool.git external/BGScriptTool
+        if [[ -f external/BGScriptTool/big_script_tool.py ]]; then
+          cp external/BGScriptTool/big_script_tool.py external/BIOSUtilities/
+        fi
 
         echo -e "${ORANGE}""${BOLD}""Installing FwHunt""${NC}"
         if [[ -d external/fwhunt-scan ]]; then

installer/IF20_cve_search.sh

@@ -78,6 +78,7 @@ IF20_cve_search() {
       pip_install "dnspython==2.2.1"
       pip_install "Werkzeug"
       pip_install "python-dateutil"
+      pip_install "CveXplore==0.3.14"
 
       REDIS_PW="$(tr -dc A-Za-z0-9 </dev/urandom | head -c 13 || true)"
 

installer/IP35_uefi_extraction.sh

@@ -23,6 +23,7 @@ IP35_uefi_extraction() {
 
     print_file_info "UEFIExtract_NE_A62_linux_x86_64.zip" "Release-version A62" "https://github.com/LongSoft/UEFITool/releases/download/A62/UEFIExtract_NE_A62_linux_x86_64.zip" "external/UEFITool/UEFIExtract_NE_A62_linux_x86_64.zip"
     print_tool_info "unzip" 1
+    print_pip_info "uefi_firmware"
 
     if [[ "${LIST_DEP}" -eq 1 ]] || [[ "${DOCKER_SETUP}" -eq 1 ]]; then
       ANSWER=("n")
@@ -34,6 +35,8 @@ IP35_uefi_extraction() {
     case ${ANSWER:0:1} in
       y|Y )
         apt-get install "${INSTALL_APP_LIST[@]}" -y --no-install-recommends
+        pip_install "uefi_firmware"
+
         if ! [[ -d external/UEFITool ]]; then
           mkdir external/UEFITool
         fi

installer/IP99_binwalk_default.sh

@@ -69,6 +69,11 @@ IP99_binwalk_default() {
     print_tool_info "unrar" 1
     print_tool_info "binwalk" 1
     print_tool_info "python3-binwalk" 1
+    print_tool_info "capstone-tool" 1
+    print_tool_info "libcapstone4:amd64" 1
+    print_tool_info "python3-capstone" 1
+    print_git_info "sasquatch" "devttys0/sasquatch" "The sasquatch project is a set of patches to the standard unsquashfs utility (part of squashfs-tools) that attempts to add support for as many hacked-up vendor-specific SquashFS implementations as possible."
+    echo -e "${ORANGE}""devttys0 - sasquatch will be downloaded.""${NC}"
 
     if [[ "${LIST_DEP}" -eq 1 ]] || [[ "${DOCKER_SETUP}" -eq 1 ]] ; then
       ANSWER=("n")
@@ -91,6 +96,24 @@ IP99_binwalk_default() {
           cp -pr external/cpu_rec/cpu_rec_corpus "${HOME}"/.config/binwalk/modules/
         fi
 
+        if ! [[ -d external/binwalk/sasquatch ]]; then
+          mkdir -p external/binwalk
+          git clone https://github.com/devttys0/sasquatch external/binwalk/sasquatch
+        fi
+        cd external/binwalk/sasquatch || ( echo "Could not install EMBA component sasquatch" && exit 1 )
+        wget https://github.com/devttys0/sasquatch/pull/47.patch
+        patch -p1 < 47.patch
+        CFLAGS="-fcommon -Wno-misleading-indentation" ./build.sh -y
+        cd "${HOME_PATH}" || ( echo "Could not install EMBA component sasquatch" && exit 1 )
+
+        # we have seen issues with the unblob sasquatch version - lets move the binwalk version to another name and link to it
+        # during the testing phase. With this in place we are able to install both versions in ||
+        if [[ -e /usr/local/bin/sasquatch ]]; then
+          echo -e "${GREEN}Backup binwalk sasquatch version to ${ORANGE}/usr/local/bin/sasquatch_binwalk${NC}"
+          mv /usr/local/bin/sasquatch /usr/local/bin/sasquatch_binwalk
+          sed -i 's/squashfs:sasquatch /squashfs:sasquatch_binwalk /' /usr/lib/python3/dist-packages/binwalk/config/extract.conf
+        fi
+
         if command -v binwalk > /dev/null ; then
           echo -e "${GREEN}""binwalk installed successfully""${NC}"
         elif [[ ! -f "/usr/local/bin/binwalk" ]] ; then

modules/F20_vul_aggregator.sh

@@ -455,6 +455,7 @@ generate_cve_details_cves() {
 
   local CVES_AGGREGATED=("$@")
   local CVE_ENTRY=""
+  CVE_COUNTER=0
 
   for CVE_ENTRY in "${CVES_AGGREGATED[@]}"; do
     if [[ "${THREADED}" -eq 1 ]]; then

modules/F50_base_aggregator.sh

@@ -817,6 +817,7 @@ get_data() {
   export CVE_COUNTER=0
   export CVE_SEARCH=1
   export FWHUNTER_CNT=0
+  # export FWHUNTER_CNT_CVE=0
   export MSF_VERIFIED=0
   export K_CVE_VERIFIED_SYMBOLS=0
   export K_CVE_VERIFIED_COMPILED=0
@@ -841,7 +842,8 @@ get_data() {
   fi
 
   if [[ -f "${LOG_DIR}"/"${S02_LOG}" ]]; then
-    FWHUNTER_CNT=$(grep -a "\[\*\]\ Statistics:" "${LOG_DIR}"/"${S02_LOG}" | cut -d: -f2 || true)
+    # FWHUNTER_CNT_CVE=$(grep -a "\[\*\]\ Statistics:" "${LOG_DIR}"/"${S02_LOG}" | cut -d: -f2 || true)
+    FWHUNTER_CNT=$(grep -a "\[\*\]\ Statistics:" "${LOG_DIR}"/"${S02_LOG}" | cut -d: -f3 || true)
   fi
   if [[ -f "${LOG_DIR}"/"${S03_LOG}" ]]; then
     PRE_ARCH="$(strip_color_codes "$(grep -a "Possible architecture details found" "${LOG_DIR}"/"${S03_LOG}" | cut -d: -f2 | sed 's/\ //g' | tr '\r\n' ' ' | sed 's/\ /\ \//' || true)")"

modules/L25_web_checks.sh

@@ -186,7 +186,7 @@ check_for_basic_auth_init() {
   BASIC_AUTH=$(find "${LOG_DIR}"/l15_emulated_checks_nmap/ -name "nmap*" -exec grep -i "401 Unauthorized" {} \; | wc -l)
 
   if [[ "${BASIC_AUTH}" -gt 0 ]]; then
-    disable_strict_mode 1
+    disable_strict_mode "${STRICT_MODE}" 1
     print_output "[*] Web server with basic auth protected ... performing login attempt"
     # basic auth from nmap found
     curl -v -L --noproxy '*' --max-redir 0 -f -m 5 -s -X GET http://"${IP_}"/ 2> >(tee -a "${LOG_FILE}")
@@ -208,7 +208,7 @@ check_for_basic_auth_init() {
       curl -v -L --noproxy '*' --max-redir 0 -f -m 5 -s -X GET -u "${CREDS}" http://"${IP_}"/ 2> >(tee -a "${LOG_FILE}")
       local CURL_RET="$?"
     fi
-    enable_strict_mode 1
+    enable_strict_mode "${STRICT_MODE}" 1
     if [[ "${CURL_RET}" != 22 ]] && [[ "${CREDS}" != "NA" ]]; then
       print_output "[+] Basic auth credentials for web server found: ${ORANGE}${CREDS}${NC}"
       export CURL_CREDS=(-u "${CREDS}")

modules/P02_firmware_bin_file_check.sh

@@ -124,6 +124,7 @@ set_p02_default_exports() {
   export UEFI_AMI_CAPSULE=0
   export ZYXEL_ZIP=0
   export QCOW_DETECTED=0
+  export UEFI_VERIFIED=0
 }
 
 generate_entropy_graph() {
@@ -167,8 +168,8 @@ fw_bin_detector() {
   else
     QNAP_ENC_CHECK=$("${BINWALK_BIN[@]}" -y "qnap encrypted" "${CHECK_FILE}")
   fi
-  UEFI_CHECK=$(grep -c "UEFI" "${TMP_DIR}"/s02_binwalk_output.txt || true)
-  UEFI_CHECK=$(( "${UEFI_CHECK}" + "$(grep -c "UEFI" "${CHECK_FILE}" || true)" ))
+  UEFI_CHECK=$(grep -c "UEFI\|BIOS" "${TMP_DIR}"/s02_binwalk_output.txt || true)
+  UEFI_CHECK=$(( "${UEFI_CHECK}" + "$(grep -c "UEFI\|BIOS" "${CHECK_FILE}" || true)" ))
 
   if [[ -f "${KERNEL_CONFIG}" ]] && [[ "${KERNEL}" -eq 1 ]]; then
     # we set the FIRMWARE_PATH to the kernel config path if we have only -k parameter
@@ -186,7 +187,7 @@ fw_bin_detector() {
     write_csv_log "BMC encrypted" "yes" "NA"
   fi
   if [[ "${UEFI_CHECK}" -gt 0 ]]; then
-    print_output "[+] Identified possible UEFI firmware - using fwhunt-scan vulnerability scanning module"
+    print_output "[+] Identified possible UEFI/BIOS firmware - using UEFI extraction module"
     export UEFI_DETECTED=1
     UEFI_AMI_CAPSULE=$(grep -c "AMI.*EFI.*capsule" "${TMP_DIR}"/s02_binwalk_output.txt || true)
     if [[ "${UEFI_AMI_CAPSULE}" -gt 0 ]]; then