Malcolm v3.1.0 development

Dockerfiles/elasticsearch.Dockerfile

@@ -1,4 +1,4 @@
-FROM amazon/opendistro-for-elasticsearch:1.13.0
+FROM amazon/opendistro-for-elasticsearch:1.13.2
 
 # Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="malcolm.netsec@gmail.com"
@@ -14,6 +14,7 @@ ARG DEFAULT_UID=1000
 ARG DEFAULT_GID=1000
 ENV DEFAULT_UID $DEFAULT_UID
 ENV DEFAULT_GID $DEFAULT_GID
+ENV PUID $DEFAULT_UID
 ENV PUSER "elasticsearch"
 ENV PGROUP "elasticsearch"
 ENV PUSER_PRIV_DROP true
@@ -33,13 +34,19 @@ RUN yum install -y openssl && \
   /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro_security && \
   echo -e 'cluster.name: "docker-cluster"\nnetwork.host: 0.0.0.0' > /usr/share/elasticsearch/config/elasticsearch.yml && \
   chown -R $PUSER:$PGROUP /usr/share/elasticsearch/config/elasticsearch.yml && \
-  sed -i "s/\b1000\b/\${PUID:-${DEFAULT_UID}}/g" /usr/local/bin/docker-entrypoint.sh && \
+  sed -i "s/user=1000\b/user=%(ENV_PUID)s/g" /usr/share/elasticsearch/plugins/opendistro-performance-analyzer/pa_config/supervisord.conf && \
+  sed -i "s/user=1000\b/user=%(ENV_PUID)s/g" /usr/share/elasticsearch/performance-analyzer-rca/pa_config/supervisord.conf && \
   sed -i '/[^#].*\/usr\/share\/elasticsearch\/bin\/elasticsearch.*/i /usr/local/bin/jdk-cacerts-auto-import.sh || true' /usr/local/bin/docker-entrypoint.sh
-
 # just used for initial keystore creation
 ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
 ADD shared/bin/jdk-cacerts-auto-import.sh /usr/local/bin/
 
+USER root
+
+ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh"]
+
+CMD ["/usr/local/bin/docker-entrypoint.sh"]
+
 # to be populated at build-time:
 ARG BUILD_DATE
 ARG MALCOLM_VERSION

Dockerfiles/file-monitor.Dockerfile

@@ -75,14 +75,15 @@ ENV EXTRACTED_FILE_ENABLE_CAPA $EXTRACTED_FILE_ENABLE_CAPA
 ENV EXTRACTED_FILE_CAPA_VERBOSE $EXTRACTED_FILE_CAPA_VERBOSE
 ENV SRC_BASE_DIR "/usr/local/src"
 ENV CLAMAV_RULES_DIR "/var/lib/clamav"
-ENV YARA_VERSION "4.0.5"
+ENV YARA_VERSION "4.1.0"
 ENV YARA_URL "https://github.com/VirusTotal/yara/archive/v${YARA_VERSION}.tar.gz"
 ENV YARA_RULES_URL "https://github.com/Neo23x0/signature-base"
 ENV YARA_RULES_DIR "/yara-rules"
 ENV YARA_RULES_SRC_DIR "$SRC_BASE_DIR/signature-base"
-ENV CAPA_URL "https://github.com/fireeye/capa"
-ENV CAPA_RULES_URL "https://github.com/fireeye/capa-rules"
-ENV CAPA_RULES_DIR "/capa-rules"
+ENV CAPA_VERSION "1.6.3"
+ENV CAPA_URL "https://github.com/fireeye/capa/releases/download/v${CAPA_VERSION}/capa-v${CAPA_VERSION}-linux.zip"
+ENV CAPA_DIR "/opt/capa"
+ENV CAPA_BIN "${CAPA_DIR}/capa"
 ENV EXTRACTED_FILE_HTTP_SERVER_DEBUG $EXTRACTED_FILE_HTTP_SERVER_DEBUG
 ENV EXTRACTED_FILE_HTTP_SERVER_ENABLE $EXTRACTED_FILE_HTTP_SERVER_ENABLE
 ENV EXTRACTED_FILE_HTTP_SERVER_ENCRYPT $EXTRACTED_FILE_HTTP_SERVER_ENCRYPT
@@ -106,6 +107,7 @@ RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list
       curl \
       gcc \
       git \
+      jq \
       libclamunrar9 \
       libjansson-dev \
       libjansson4 \
@@ -121,11 +123,6 @@ RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list
       inotify-tools \
       libzmq5 \
       psmisc \
-      python \
-      python-dev \
-      python-pip \
-      python-backports-shutil-get-terminal-size \
-      python-backports.functools-lru-cache \
       python3 \
       python3-bs4 \
       python3-dev \
@@ -134,7 +131,6 @@ RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list
       python3-requests \
       python3-zmq && \
     pip3 install clamd supervisor yara-python python-magic psutil pycryptodome && \
-    pip2 install flare-capa && \
     curl -fsSLO "$SUPERCRONIC_URL" && \
       echo "${SUPERCRONIC_SHA1SUM}  ${SUPERCRONIC}" | sha1sum -c - && \
       chmod +x "$SUPERCRONIC" && \
@@ -157,17 +153,13 @@ RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list
       git clone --depth 1 --single-branch "${YARA_RULES_URL}" "${YARA_RULES_SRC_DIR}" && \
       mkdir -p "${YARA_RULES_DIR}" && \
       ln -f -s -r "${YARA_RULES_SRC_DIR}"/yara/* "${YARA_RULES_SRC_DIR}"/vendor/yara/* "${YARA_RULES_DIR}"/ && \
-    git clone --depth 1 --single-branch --branch "v$(/usr/local/bin/capa --version 2>&1 | awk '{print $2}')" "${CAPA_URL}" /tmp/capa && \
-      cd /tmp/capa && \
-      git submodule init rules && \
-      (git rev-parse @:./rules > /tmp/capa_rules_sha.txt) && \
-    mkdir -p "${CAPA_RULES_DIR}" && \
-    cd "${CAPA_RULES_DIR}" && \
-      git init && \
-      git remote add origin "${CAPA_RULES_URL}" && \
-      git fetch --depth 1 origin "$(cat /tmp/capa_rules_sha.txt)" && \
-      git reset --hard FETCH_HEAD && \
-      rm -rf /tmp/capa* && \
+    cd /tmp && \
+      curl -fsSL -o ./capa.zip "${CAPA_URL}" && \
+      unzip ./capa.zip && \
+      chmod 755 ./capa && \
+      mkdir -p "${CAPA_DIR}" && \
+      mv ./capa "${CAPA_BIN}" && \
+      rm -f ./capa.zip && \
     apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages --purge remove \
         automake \
         build-essential \
@@ -180,21 +172,19 @@ RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list
         libssl-dev \
         libtool \
         make \
-        python-dev \
-        python3-dev \
-        unzip && \
+        python3-dev && \
       apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages autoremove && \
       apt-get clean && \
-      rm -rf /var/lib/apt/lists/* && \
+      rm -rf /var/lib/apt/lists/* /tmp/* && \
     mkdir -p /var/log/clamav "${CLAMAV_RULES_DIR}" && \
     curl -s -S -L -o "${CLAMAV_RULES_DIR}"/main.cvd http://database.clamav.net/main.cvd && \
       curl -s -S -L -o "${CLAMAV_RULES_DIR}"/daily.cvd http://database.clamav.net/daily.cvd && \
       curl -s -S -L -o "${CLAMAV_RULES_DIR}"/bytecode.cvd http://database.clamav.net/bytecode.cvd && \
     groupadd --gid ${DEFAULT_GID} ${PGROUP} && \
       useradd -m --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} ${PUSER} && \
       usermod -a -G tty ${PUSER} && \
-    chown -R ${PUSER}:${PGROUP} /var/log/clamav "${CLAMAV_RULES_DIR}" "${CAPA_RULES_DIR}" "${YARA_RULES_DIR}" "${YARA_RULES_SRC_DIR}" && \
-    find /var/log/clamav "${CLAMAV_RULES_DIR}" "${CAPA_RULES_DIR}" "${YARA_RULES_DIR}" "${YARA_RULES_SRC_DIR}" -type d -exec chmod 750 "{}" \; && \
+    chown -R ${PUSER}:${PGROUP} /var/log/clamav "${CLAMAV_RULES_DIR}" "${CAPA_DIR}" "${YARA_RULES_DIR}" "${YARA_RULES_SRC_DIR}" && \
+    find /var/log/clamav "${CLAMAV_RULES_DIR}" "${CAPA_DIR}" "${YARA_RULES_DIR}" "${YARA_RULES_SRC_DIR}" -type d -exec chmod 750 "{}" \; && \
     sed -i 's/^Foreground .*$/Foreground true/g' /etc/clamav/clamd.conf && \
       sed -i "s/^User .*$/User ${PUSER}/g" /etc/clamav/clamd.conf && \
       sed -i "s|^LocalSocket .*$|LocalSocket $CLAMD_SOCKET_FILE|g" /etc/clamav/clamd.conf && \
@@ -211,7 +201,7 @@ RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list
       ln -r -s /usr/local/bin/zeek_carve_scanner.py /usr/local/bin/yara_scan.py && \
       ln -r -s /usr/local/bin/zeek_carve_scanner.py /usr/local/bin/capa_scan.py && \
       ln -r -s /usr/local/bin/zeek_carve_scanner.py /usr/local/bin/malass_scan.py && \
-      echo "0 */6 * * * /bin/bash /usr/local/bin/capa-rules-update.sh\n0 */6 * * * /bin/bash /usr/local/bin/yara-rules-update.sh" > ${SUPERCRONIC_CRONTAB}
+      echo "0 */6 * * * /bin/bash /usr/local/bin/capa-update.sh\n0 */6 * * * /bin/bash /usr/local/bin/yara-rules-update.sh" > ${SUPERCRONIC_CRONTAB}
 
 ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
 ADD shared/bin/zeek_carve*.py /usr/local/bin/
@@ -228,7 +218,9 @@ USER root
 
 WORKDIR /data/zeek/extract_files
 
-VOLUME ["$CAPA_RULES_DIR"]
+ENV PATH "${CAPA_DIR}:${PATH}"
+
+VOLUME ["$CAPA_DIR"]
 VOLUME ["$CLAMAV_RULES_DIR"]
 VOLUME ["$YARA_RULES_DIR"]
 VOLUME ["$YARA_RULES_SRC_DIR"]
@@ -240,7 +232,6 @@ ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh", "/docker-entrypoint.sh"]
 
 CMD ["/usr/local/bin/supervisord", "-c", "/etc/supervisord.conf", "-n"]
 
-
 # to be populated at build-time:
 ARG BUILD_DATE
 ARG MALCOLM_VERSION

Dockerfiles/filebeat.Dockerfile

@@ -48,11 +48,10 @@ USER root
 
 RUN yum install -y epel-release && \
         yum update -y && \
-        yum install -y curl inotify-tools file psmisc tar gzip unzip cpio bzip2 lzma xz p7zip p7zip-plugins unar python-setuptools python-pip && \
+        yum install -y curl inotify-tools file psmisc tar gzip unzip cpio bzip2 lzma xz p7zip p7zip-plugins unar python3-setuptools python3-pip && \
         yum clean all && \
     ln -sr /usr/sbin/fuser /bin/fuser && \
-    easy_install supervisor && \
-    pip install patool entrypoint2 pyunpack python-magic ordered-set==3.1.1 && \
+    python3 -m pip install patool entrypoint2 pyunpack python-magic ordered-set supervisor && \
     curl -fsSLO "$SUPERCRONIC_URL" && \
       echo "${SUPERCRONIC_SHA1SUM}  ${SUPERCRONIC}" | sha1sum -c - && \
       chmod +x "$SUPERCRONIC" && \
@@ -96,7 +95,7 @@ VOLUME ["/usr/share/filebeat/data", "/usr/share/filebeat-nginx/data"]
 
 ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh"]
 
-CMD ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf", "-u", "root", "-n"]
+CMD ["/usr/local/bin/supervisord", "-c", "/etc/supervisord.conf", "-u", "root", "-n"]
 
 
 # to be populated at build-time:

Dockerfiles/htadmin.Dockerfile

@@ -77,6 +77,7 @@ RUN apt-get update && \
 ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
 ADD docs/images/favicon/favicon.ico /var/www/htadmin/
 ADD htadmin/supervisord.conf /supervisord.conf
+ADD htadmin/htadmin.sh /usr/local/bin/
 ADD htadmin/src /var/www/htadmin/
 ADD htadmin/php/php.ini /etc/php/$PHP_VERSION/fpm/php.ini
 ADD htadmin/nginx/sites-available/default /etc/nginx/sites-available/default

Dockerfiles/kibana.Dockerfile

@@ -1,4 +1,4 @@
-FROM centos:7 AS build
+FROM amazonlinux:2 AS build
 
 # Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
 
@@ -20,12 +20,8 @@ ENV ELASTIC_VERSION $ELASTIC_VERSION
 
 USER root
 
-RUN yum install -y epel-release && \
-    yum update -y && \
-    yum install -y curl patch psmisc zip unzip gcc-c++ make && \
-    yum install -y https://packages.endpoint.com/rhel/7/os/x86_64/endpoint-repo-1.7-1.x86_64.rpm && \
-    yum install -y git && \
-    yum clean all && \
+RUN amazon-linux-extras install -y epel && \
+    yum install -y curl patch procps psmisc tar zip unzip gcc-c++ make moreutils jq git && \
     groupadd -g ${DEFAULT_GID} ${PGROUP} && \
     adduser -u ${DEFAULT_UID} -d /home/kibana -s /bin/bash -G ${PGROUP} -g ${PUSER} ${PUSER} && \
     mkdir -p /usr/share && \
@@ -65,7 +61,7 @@ RUN eval "$(nodenv init -)" && \
     yarn build --kibana-version "${ELASTIC_VERSION}" && \
     mv ./build/kbnSankeyVis-7.10.2.zip ./build/kbnSankeyVis.zip
 
-FROM amazon/opendistro-for-elasticsearch-kibana:1.13.0
+FROM amazon/opendistro-for-elasticsearch-kibana:1.13.2
 
 LABEL maintainer="malcolm.netsec@gmail.com"
 LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'

Dockerfiles/logstash.Dockerfile

@@ -1,26 +1,45 @@
-FROM centos:7 AS build
+FROM amazonlinux:2 AS build
 
 # Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
 
-RUN yum install -y epel-release && \
-    yum update -y && \
-    yum install -y java-1.8.0-openjdk-devel curl wget tar which \
-                patch libyaml-devel libffi-devel glibc-headers autoconf gcc-c++ glibc-devel \
-                readline-devel zlib-devel openssl-d evel bzip2 automake libtool bison make
-
-ENV OUIFILTER_URL "https://codeload.github.com/mmguero-dev/logstash-filter-ieee_oui/tar.gz/master"
+RUN amazon-linux-extras install -y epel && \
+    yum install -y \
+      autoconf \
+      automake \
+      bison \
+      bzip2 \
+      curl \
+      gcc-c++ \
+      glibc-devel \
+      glibc-headers \
+      java-latest-openjdk-devel \
+      libffi-devel \
+      libtool \
+      libyaml-devel \
+      make \
+      openssl-devel \
+      patch \
+      procps \
+      readline-devel \
+      tar \
+      wget \
+      which \
+      zlib-devel
 
 RUN /bin/bash -lc "command curl -sSL https://rvm.io/mpapis.asc | gpg2 --import -" && \
     /bin/bash -lc "command curl -sSL https://rvm.io/pkuczynski.asc | gpg2 --import -" && \
     /bin/bash -lc "curl -L get.rvm.io | bash -s stable" && \
     /bin/bash -lc "rvm autolibs fail" && \
-    /bin/bash -lc "rvm install jruby-9.2.5.0" && \
-    /bin/bash -lc "rvm use jruby-9.2.5.0 --default" && \
-    /bin/bash -lc "gem install bundler --no-ri --no-rdoc" && \
-    cd /opt && \
-      mkdir -p ./logstash-filter-ieee_oui && \
-      curl -sSL "$OUIFILTER_URL" | tar xzvf - -C ./logstash-filter-ieee_oui --strip-components 1 && \
-      /bin/bash -lc "cd /opt/logstash-filter-ieee_oui && bundle install && gem build logstash-filter-ieee_oui.gemspec && bundle info logstash-filter-ieee_oui"
+    /bin/bash -lc "rvm install jruby-9.2.17.0" && \
+    /bin/bash -lc "rvm use jruby-9.2.17.0 --default" && \
+    /bin/bash -lc "gem install bundler --no-document"
+
+ENV OUIFILTER_URL "https://codeload.github.com/mmguero-dev/logstash-filter-ieee_oui/tar.gz/master"
+
+RUN cd /opt && \
+    mkdir -p ./logstash-filter-ieee_oui && \
+    curl -sSL "$OUIFILTER_URL" | tar xzvf - -C ./logstash-filter-ieee_oui --strip-components 1 && \
+    /bin/bash -lc "export JAVA_HOME=$(realpath $(dirname $(find /usr/lib/jvm -name javac -type f))/../) && cd /opt/logstash-filter-ieee_oui && ( bundle install || bundle install ) && gem build logstash-filter-ieee_oui.gemspec && bundle info logstash-filter-ieee_oui"
 
 FROM docker.elastic.co/logstash/logstash-oss:7.10.2
 

Dockerfiles/nginx.Dockerfile

@@ -7,8 +7,6 @@
 #             jwilder/nginx-proxy         -  https://github.com/jwilder/nginx-proxy/blob/master/Dockerfile.alpine
 
 ####################################################################################
-# build a patched APK of stunnel supporting ldap StartTLS (patched protocols.c)
-# (based on https://www.stunnel.org/pipermail/stunnel-users/2013-November/004437.html)
 
 FROM alpine:3.13 as stunnel_build
 
@@ -20,30 +18,22 @@ ENV PUSER "builder"
 ENV PGROUP "abuild"
 
 ADD https://codeload.github.com/alpinelinux/aports/tar.gz/master /aports-master.tar.gz
-ADD nginx/src/*.patch /usr/src/patches/
 
 USER root
 
 RUN set -x ; \
-    apk add --no-cache alpine-sdk patchutils sudo openssl-dev linux-headers; \
+    apk add --no-cache alpine-sdk sudo openssl-dev linux-headers; \
     sed -i 's/^#\s*\(%wheel\s\+ALL=(ALL)\s\+NOPASSWD:\s\+ALL\)/\1/' /etc/sudoers ; \
     adduser -D -u ${DEFAULT_UID} -h /apkbuild -G ${PGROUP} ${PUSER} ; \
     addgroup ${PUSER} wheel ; \
     chmod 644 /aports-master.tar.gz
 
 USER ${PUSER}
 
-# todo: when aports updates stunnel to 5.58, this will need to be updated
-
 RUN set -x ; \
     cd /apkbuild ; \
     tar xvf /aports-master.tar.gz aports-master/community/stunnel ; \
-    cp /usr/src/patches/stunnel-5.56-open-ldap.patch /apkbuild/aports-master/community/stunnel/ ; \
     cd /apkbuild/aports-master/community/stunnel ; \
-    sed -i 's@https://www.stunnel.org/downloads/stunnel.*gz@ftp://ftp.stunnel.org/stunnel/archive/5.x/stunnel-5.57.tar.gz@' APKBUILD ; \
-    sed -i 's/\(^makedepends="\)/\1patchutils /' APKBUILD ; \
-    sed -i '/^source=/a \ \ \ \ \ \ \ \ stunnel-5.56-open-ldap.patch' APKBUILD ; \
-    sed -i '/^sha512sums/,$d' APKBUILD ; \
     abuild-keygen -a -i -n ; \
     abuild checksum ; \
     abuild -R
@@ -83,9 +73,6 @@ ARG NGINX_BASIC_AUTH=true
 #   and use stunnel to tunnel the connection.
 ARG NGINX_LDAP_TLS_STUNNEL=false
 
-# when initiating the "extendedReq(1) LDAP_START_TLS_OID" command, which protocol to use: winldap or openldap
-ARG NGINX_LDAP_TLS_STUNNEL_PROTOCOL=winldap
-
 # stunnel will require and verify certificates for StartTLS when one or more
 # trusted CA certificate files are placed in the ./nginx/ca-trust directory.
 # For additional security, hostname or IP address checking of the associated
@@ -98,13 +85,12 @@ ARG NGINX_LDAP_TLS_STUNNEL_VERIFY_LEVEL=2
 
 ENV NGINX_BASIC_AUTH $NGINX_BASIC_AUTH
 ENV NGINX_LDAP_TLS_STUNNEL $NGINX_LDAP_TLS_STUNNEL
-ENV NGINX_LDAP_TLS_STUNNEL_PROTOCOL $NGINX_LDAP_TLS_STUNNEL_PROTOCOL
 ENV NGINX_LDAP_TLS_STUNNEL_CHECK_HOST $NGINX_LDAP_TLS_STUNNEL_CHECK_HOST
 ENV NGINX_LDAP_TLS_STUNNEL_CHECK_IP $NGINX_LDAP_TLS_STUNNEL_CHECK_IP
 ENV NGINX_LDAP_TLS_STUNNEL_VERIFY_LEVEL $NGINX_LDAP_TLS_STUNNEL_VERIFY_LEVEL
 
 # build latest nginx with nginx-auth-ldap
-ENV NGINX_VERSION=1.19.7
+ENV NGINX_VERSION=1.20.0
 ENV NGINX_AUTH_LDAP_BRANCH=master
 
 ADD https://codeload.github.com/mmguero-dev/nginx-auth-ldap/tar.gz/$NGINX_AUTH_LDAP_BRANCH /nginx-auth-ldap.tar.gz