Replacement of current cve query mechanism

.github/workflows/trivy.yml

@@ -0,0 +1,72 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: trivy
+on:
+  schedule:
+    - cron: '0 0 * * *' # do it every day
+#  push:
+#    branches:
+#      - '**'        # matches every branch
+#  pull_request:
+#    branches:
+#      - '**'
+#  # Allows you to run this workflow manually from the Actions tab
+#  workflow_dispatch:
+jobs:
+  build:
+    if: github.repository_owner == 'e-m-b-a'
+    permissions:
+      contents: read # for actions/checkout to fetch code
+      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
+      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
+    name: Trivy EMBA image check
+    runs-on: "ubuntu-20.04"
+    steps:
+      - name: Checkout EMBA
+        uses: actions/checkout@v3
+
+      - name: Free Disk Space
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: true
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: false
+          swap-storage: true
+
+      - name: Build trivy
+        run: |
+          git clone https://github.com/aquasecurity/trivy-action.git /tmp/trivy-action
+          sudo docker build -t d6f297:4714451b1eca4e41a4cc10f0a8a8c25a -f "/tmp/trivy-action/Dockerfile" "/tmp/trivy-action/"
+
+      - name: Run Trivy vulnerability scanner on EMBA image
+        run: |
+          sudo docker run --workdir /github/workspace --rm -e "INPUT_IMAGE-REF" -e "INPUT_FORMAT" -e "INPUT_TEMPLATE" \
+            -e "INPUT_OUTPUT" -e "INPUT_SEVERITY" -e "INPUT_TIMEOUT" -e "INPUT_SKIP-DIRS" -e "INPUT_SCAN-TYPE" -e "INPUT_INPUT" \
+            -e "INPUT_SCAN-REF" -e "INPUT_EXIT-CODE" -e "INPUT_IGNORE-UNFIXED" -e "INPUT_VULN-TYPE" -e "INPUT_SKIP-FILES" \
+            -e "INPUT_CACHE-DIR" -e "INPUT_IGNORE-POLICY" -e "INPUT_HIDE-PROGRESS" -e "INPUT_LIST-ALL-PKGS" -e "INPUT_SECURITY-CHECKS" \
+            -e "INPUT_TRIVYIGNORES" -e "INPUT_ARTIFACT-TYPE" -e "INPUT_GITHUB-PAT" -e "HOME" -e "GITHUB_JOB" -e "GITHUB_REF" \
+            -e "GITHUB_SHA" -e "GITHUB_REPOSITORY" -e "GITHUB_REPOSITORY_OWNER" -e "GITHUB_REPOSITORY_OWNER_ID" -e "GITHUB_RUN_ID" \
+            -e "GITHUB_RUN_NUMBER" -e "GITHUB_RETENTION_DAYS" -e "GITHUB_RUN_ATTEMPT" -e "GITHUB_REPOSITORY_ID" -e "GITHUB_ACTOR_ID" \
+            -e "GITHUB_ACTOR" -e "GITHUB_TRIGGERING_ACTOR" -e "GITHUB_WORKFLOW" -e "GITHUB_HEAD_REF" -e "GITHUB_BASE_REF" -e "GITHUB_EVENT_NAME" \
+            -e "GITHUB_SERVER_URL" -e "GITHUB_API_URL" -e "GITHUB_GRAPHQL_URL" -e "GITHUB_REF_NAME" -e "GITHUB_REF_PROTECTED" -e "GITHUB_REF_TYPE" \
+            -e "GITHUB_WORKFLOW_REF" -e "GITHUB_WORKFLOW_SHA" -e "GITHUB_WORKSPACE" -e "GITHUB_ACTION" -e "GITHUB_EVENT_PATH" -e "GITHUB_ACTION_REPOSITORY" \
+            -e "GITHUB_ACTION_REF" -e "GITHUB_PATH" -e "GITHUB_ENV" -e "GITHUB_STEP_SUMMARY" -e "GITHUB_STATE" -e "GITHUB_OUTPUT" -e "RUNNER_OS" -e "RUNNER_ARCH" \
+            -e "RUNNER_NAME" -e "RUNNER_ENVIRONMENT" -e "RUNNER_TOOL_CACHE" -e "RUNNER_TEMP" -e "RUNNER_WORKSPACE" -e "ACTIONS_RUNTIME_URL" -e "ACTIONS_RUNTIME_TOKEN" \
+            -e "ACTIONS_CACHE_URL" -e "ACTIONS_RESULTS_URL" -e GITHUB_ACTIONS=true -e CI=true -v "/var/run/docker.sock":"/var/run/docker.sock" \
+            -v "/home/runner/work/_temp/_github_home":"/github/home" -v "/home/runner/work/_temp/_github_workflow":"/github/workflow" \
+            -v "/home/runner/work/_temp/_runner_file_commands":"/github/file_commands" -v "/home/runner/work/emba/emba":"/github/workspace" d6f297:4714451b1eca4e41a4cc10f0a8a8c25a  \
+            "-a image" "-b template" "-c @/contrib/sarif.tpl" "-d " "-e false" "-f os,library" "-h trivy-results.sarif" \
+            "-i embeddedanalyzer/emba" \
+            "-j ." "-k /external/semgrep-rules,/external/routersploit,/external/arachni,/etc/ssl/private/,/external/jdk" \
+            "-l " "-m " "-n 60m" "-o " "-p " "-q " "-r false" "-s " "-t " "-u "
+
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: 'trivy-results.sarif'
+

config/bin_version_strings.cfg

@@ -355,11 +355,11 @@ lighttpd;;bsd;"lighttpd-[0-9](\.[0-9]+)+?$";"sed -r 's/lighttpd-([0-9](\.[0-9]+)
 lighttpd;;bsd;"^lighttpd [0-9](\.[0-9]+)+?";"sed -r 's/lighttpd ([0-9](\.[0-9]+)+?)/lighttpd:\1/'";
 lighttpd;;bsd;"^lighttpd\/[0-9](\.[0-9]+)+?(-devel)?(-[0-9]+[A-Za-z]+)?.*\ -\ a\ light\ and\ fast\ webserver$";"sed -r 's/lighttpd\/([0-9](\.[0-9]+)+?).*/lighttpd:\1/'";
 lighttpd;;bsd;"^server\ started\ \(lighttpd\/[0-9](\.[0-9]+)+?\)$";"sed -r 's/server\ started\ \(lighttpd\/([0-9](\.[0-9]+)+?)\)$/lighttpd:\1/'";
-linux_kernel;;gplv2;"^Linux-[1-6]\.[0-9]+\.[0-9]+";"sed -r 's/Linux-([1-6](\.[0-9]+)+?).*/kernel:\1/'";
-linux_kernel;;gplv2;"Linux\ kernel\ version\ [1-6]\.[0-9]+\.[0-9]+\ ";"sed -r 's/Linux\ kernel\ version\ ([1-6](\.[0-9]+)+?)\ .*/kernel:\1/'";
-linux_kernel;;gplv2;"Linux\ kernel\ version\ [1-6]\.[0-9]+\.[0-9]+$";"sed -r 's/Linux\ kernel\ version\ ([1-6](\.[0-9]+)+?)$/kernel:\1/'";
-linux_kernel;;gplv2;"Linux\ version\ [1-6]\.[0-9]+\.[0-9]+\ ";"sed -r 's/Linux\ version\ ([1-6](\.[0-9]+)+?)\ .*/kernel:\1/'";
-linux_kernel;;gplv2;"Linux\ version\ [1-6]\.[0-9]+\.[0-9]+$";"sed -r 's/Linux\ version\ ([1-6](\.[0-9]+)+?)$/kernel:\1/'";
+linux_kernel;;gplv2;"^Linux-[1-6]\.[0-9]+\.[0-9]+";"sed -r 's/Linux-([1-6](\.[0-9]+)+?).*/linux_kernel:\1/'";
+linux_kernel;;gplv2;"Linux\ kernel\ version\ [1-6]\.[0-9]+\.[0-9]+\ ";"sed -r 's/Linux\ kernel\ version\ ([1-6](\.[0-9]+)+?)\ .*/linux_kernel:\1/'";
+linux_kernel;;gplv2;"Linux\ kernel\ version\ [1-6]\.[0-9]+\.[0-9]+$";"sed -r 's/Linux\ kernel\ version\ ([1-6](\.[0-9]+)+?)$/linux_kernel:\1/'";
+linux_kernel;;gplv2;"Linux\ version\ [1-6]\.[0-9]+\.[0-9]+\ ";"sed -r 's/Linux\ version\ ([1-6](\.[0-9]+)+?)\ .*/linux_kernel:\1/'";
+linux_kernel;;gplv2;"Linux\ version\ [1-6]\.[0-9]+\.[0-9]+$";"sed -r 's/Linux\ version\ ([1-6](\.[0-9]+)+?)$/linux_kernel:\1/'";
 lldpd;;unknown;"^Version:\ lldpd\ [0-9](\.[0-9]+)+$";"sed -r 's/Version:\ lldpd\ ([0-9](\.[0-9]+)+?)$/lldpd:\1/'";
 llmnresp;;unknown;"^llmnresp\ versoin\ [0-9](\.[0-9]+)+?$";"sed -r 's/llmnresp\ versoin\ ([0-9](\.[0-9]+)+?)$/llmnresp:\1/'";
 lnstat;;unknown;"lnstat\ Version\ [0-9]\.[0-9]+(\ [0-9]+)?$";"sed -r 's/lnstat\ Version\ ([0-9](\.[0-9]+)+?).*/lnstat:\1/'";
@@ -494,11 +494,11 @@ openssh;;bsd;"OpenSSH_[0-9](\.[0-9]+)+?([a-z][0-9]+)?,\ ";"sed -r 's/OpenSSH_([0
 openssh;;bsd;"OpenSSH_[0-9](\.[0-9]+)+?([a-z][0-9]+)?\ ";"sed -r 's/OpenSSH_([0-9](\.[0-9]+)+?([a-z][0-9]+)?).*$/openssh:\1/'";
 # we run into false positives in static analysis:
 openssh;no_static;bsd;"^OpenSSH_[0-9](\.[0-9]+)+?([a-z][0-9]+)?$";"sed -r 's/OpenSSH_([0-9](\.[0-9]+)+?([a-z][0-9]+)?)$/openssh:\1/'";
-openssl;;bsd;"OpenSSL\ [0-9](\.[0-9]+)+?\ ";"sed -r 's/OpenSSL\ ([0-9](\.[0-9]+)+?).*$/openssl:\1/'";
-openssl;;bsd;"OpenSSL\ [0-9](\.[0-9]+)+?(-[a-z]+)\ ";"sed -r 's/OpenSSL\ ([0-9](\.[0-9]+)+?)((-[a-z]+)?)\ .*$/openssl:\1\2/'";
-openssl;;bsd;"OpenSSL\ [0-9](\.[0-9]+)+?([a-z]+)?-";"sed -r 's/OpenSSL\ ([0-9](\.[0-9]+)+?([a-z]+)?)-.*$/openssl:\1/'";
-openssl;;bsd;"OpenSSL\ [0-9](\.[0-9]+)+?([a-z]+)?\ ";"sed -r 's/OpenSSL\ ([0-9](\.[0-9]+)+?([a-z]+)?)\ .*$/openssl:\1/'";
-openssl;;bsd;"OpenSSL\ [0-9](\.[0-9]+)+?(-[a-z]+)$";"sed -r 's/OpenSSL\ ([0-9](\.[0-9]+)+?)((-[a-z]+)?)$/openssl:\1\2/'";
+openssl;;bsd;"OpenSSL\ [0-9](\.[0-9]+)+?\ ";"sed -r 's/OpenSSL\ ([0-9](\.[0-9]+)+?).*$/openssl:openssl:\1/'";
+openssl;;bsd;"OpenSSL\ [0-9](\.[0-9]+)+?(-[a-z]+)\ ";"sed -r 's/OpenSSL\ ([0-9](\.[0-9]+)+?)((-[a-z]+)?)\ .*$/openssl:openssl:\1\2/'";
+openssl;;bsd;"OpenSSL\ [0-9](\.[0-9]+)+?([a-z]+)?-";"sed -r 's/OpenSSL\ ([0-9](\.[0-9]+)+?([a-z]+)?)-.*$/openssl:openssl:\1/'";
+openssl;;bsd;"OpenSSL\ [0-9](\.[0-9]+)+?([a-z]+)?\ ";"sed -r 's/OpenSSL\ ([0-9](\.[0-9]+)+?([a-z]+)?)\ .*$/openssl:openssl:\1/'";
+openssl;;bsd;"OpenSSL\ [0-9](\.[0-9]+)+?(-[a-z]+)$";"sed -r 's/OpenSSL\ ([0-9](\.[0-9]+)+?)((-[a-z]+)?)$/openssl:openssl:\1\2/'";
 openswan;;gplv2;"^Openswan\ [0-9](\.[0-9]+)+?$";"sed -r 's/Openswan\ ([0-9](\.[0-9]+)+?)$/openswan:\1/'";
 openswan;;gplv2;"^Linux\ Openswan\ [0-9](\.[0-9]+)+?$";"sed -r 's/Linux\ Openswan\ ([0-9](\.[0-9]+)+?)$/openswan:\1/'";
 openvpn;;gpl;"^OpenVPN\ [0-9](\.[0-9]+)+?\ ";"sed -r 's/OpenVPN\ ([0-9](\.[0-9]+)+?)\ .*/openvpn:\1/'";

config/emba_updater.init

@@ -6,7 +6,6 @@ BASE_PATH="$(pwd)"
 LOG_DIR="/var/log"
 
 [ -d EMBA_INSTALL_PATH ] || exit 0
-[ -x /etc/init.d/redis-server ] || exit 0
 [ -d "${LOG_DIR}" ] || exit 0
 
 if [ -f EMBA_INSTALL_PATH/external/emba_venv/bin/activate ]; then
@@ -23,16 +22,10 @@ cd EMBA_INSTALL_PATH || exit
 git pull origin master | tee -a "${LOG_DIR}"/emba_update.log
 cd "${BASE_PATH}" || exit
 
-echo "[*] EMBA update - cve-search update" | tee -a "${LOG_DIR}"/emba_update.log
-service mongod start | tee -a "${LOG_DIR}"/emba_update.log
-/etc/init.d/redis-server start | tee -a "${LOG_DIR}"/emba_update.log
-
-# Find and set Proxy-settings for cvexplore
-if [[ -n "${https_proxy}" ]]; then
-  export HTTP_PROXY_STRING="${https_proxy}"
-fi
-
-MONGODB_HOST="172.36.0.1" cvexplore database update | tee -a "${LOG_DIR}"/emba_update.log
+echo "[*] EMBA update - CVE database update" | tee -a "${LOG_DIR}"/emba_update.log
+cd EMBA_INSTALL_PATH/external/nvd-json-data-feeds || exit
+git pull origin master | tee -a "${LOG_DIR}"/emba_update.log
+cd "${BASE_PATH}" || exit
 
 echo "[*] EMBA update - update local docker image" | tee -a "${LOG_DIR}"/emba_update.log
 docker pull embeddedanalyzer/emba | tee -a "${LOG_DIR}"/emba_update.log

docker-compose.yml

@@ -37,13 +37,12 @@ services:
         - ${LOG}/:/logs
         - ${EMBA}/:/emba:ro
         - ${EMBA}/external/linux_kernel_sources/:/external/linux_kernel_sources:ro
+        - ${EMBA}/external/nvd-json-data-feeds/:/external/nvd-json-data-feeds:ro
         - /etc/localtime:/etc/localtime:ro
         - /dev:/dev
     environment:
         - USER
         - CONTAINER_NUMBER=1
-    networks:
-      - emba_runs
     devices:
       - /dev/fuse:/dev/fuse:rwm
     cap_add:
@@ -81,13 +80,3 @@ services:
     security_opt:
       - no-new-privileges:true
 
-networks:
-  emba_runs:
-    name: emba_runs
-    driver: bridge
-    internal: true
-    driver_opts:
-      com.docker.network.bridge.name: emba_runs
-    ipam:
-      config:
-        - subnet: "172.36.0.0/16"

emba

@@ -84,31 +84,6 @@ sort_modules()
   MODULES=( "${SORTED_MODULES[@]}" )
 }
 
-# lets check cve-search in a background job
-check_cve_search_job() {
-  local EMBA_PID="${1:-}"
-
-  if ! [[ "${EMBA_PID}" =~ [0-9]+ ]]; then
-    print_output "[-] WARNING: No EMBA PID detected ... are we really running?!?"
-    return
-  fi
-
-  while true; do
-    if [[ -f "${LOG_DIR}"/emba.log ]]; then
-      if grep -q "Test ended\|EMBA failed" "${LOG_DIR}"/emba.log 2>/dev/null; then
-        break
-      fi
-    fi
-    # shellcheck disable=SC2009
-    if ! ps aux | grep -v grep | grep -q "${EMBA_PID}"; then
-      break
-    fi
-    check_nw_interface
-    check_cve_search
-    sleep 90
-  done
-}
-
 check_quest_container() {
   print_ln "no_log"
   print_output "[*] Checking Quest container ${QUEST_CONTAINER} dependencies \\n" "no_log"
@@ -414,7 +389,6 @@ main() {
   if [[ ${IN_DOCKER} -eq 1 ]] ; then
     # set external path new for docker
     export EXT_DIR="/external"
-    export PATH_CVE_SEARCH="${EXT_DIR}""/cve-search/bin/search.py"
   fi
 
   # activate the virtual environment - we should have it in external which fits also the docker environment
@@ -661,13 +635,6 @@ main() {
       fi
     fi
 
-    if [[ ${IN_DOCKER} -eq 0 ]] ; then
-      check_cve_search_job "${EMBA_PID}" &
-      local TMP_PID="$!"
-      store_kill_pids "${TMP_PID}"
-      disown "${TMP_PID}" 2> /dev/null || true
-    fi
-
     # disk space monitor not fully working -> removed for now
     # Todo: check it and fix it
     # disk_space_monitor "${EMBA_PID}" &

helpers/helpers_emba_defaults.sh

@@ -81,8 +81,6 @@ set_defaults() {
                                 # 1 -> multi threaded
   export YARA=1
   export OVERWRITE_LOG=0        # automaticially overwrite log directory, if necessary
-  export JUMP_OVER_CVESEARCH_CHECK=0 # ignore long CVEsearch check in dep check
-
   export MAX_EXT_SPACE=11000     # a useful value, could be adjusted if you deal with very big firmware images
   export LOG_DIR="${INVOCATION_PATH}""/logs"
   export TMP_DIR="${LOG_DIR}""/tmp"
@@ -97,7 +95,6 @@ set_defaults() {
   # this will be in TMP_DIR/pid_notes.log
   export PID_LOG_FILE="pid_notes.log"
   export BASE_LINUX_FILES="${CONFIG_DIR}""/linux_common_files.txt"
-  export PATH_CVE_SEARCH="${EXT_DIR}""/cve-search/bin/search.py"
   if [[ -f "${CONFIG_DIR}"/known_exploited_vulnerabilities.csv ]]; then
     export KNOWN_EXP_CSV="${CONFIG_DIR}"/known_exploited_vulnerabilities.csv
   fi
@@ -124,8 +121,7 @@ set_defaults() {
 
   export CVE_BLACKLIST="${CONFIG_DIR}"/cve-blacklist.txt  # include the blacklisted CVE values to this file
   export CVE_WHITELIST="${CONFIG_DIR}"/cve-whitelist.txt  # include the whitelisted CVE values to this file
-  export MONGODB_HOST="172.36.0.1"  # cveXplore mondodb host
-  # export MONGODB_PORT=27017  # cveXplore mondodb port
+  export NVD_DIR="${EXT_DIR}"/nvd-json-data-feeds
 
   export MODULE_BLACKLIST=()
   if [[ -f "${CONFIG_DIR}"/module_blacklist.txt ]]; then