Merge pull request #279 from cisagov/v23.09.0_merge_cisagov

Malcolm v23.09.0

.github/workflows/sensor-iso-build-docker-wrap-push-ghcr.yml

@@ -102,6 +102,7 @@ jobs:
           echo "${{ secrets.MAXMIND_GEOIP_DB_LICENSE_KEY }}" > ./shared/maxmind_license.txt
           echo "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" > ./shared/environment.chroot
           echo "VCS_REVSION=${{ steps.extract_commit_sha.outputs.sha }}" > ./shared/environment.chroot
+          echo "BUILD_JOBS=2" > ./shared/environment.chroot
           sudo /usr/bin/env bash ./build.sh
           rm -rf ./shared/ ./docs/ ./_config.yml ./_includes ./_layouts /Gemfile ./README.md
           sudo chmod 644 ./hedgehog-*.*

.github/workflows/zeek-build-and-push-ghcr.yml

@@ -67,6 +67,8 @@ jobs:
           file: ./Dockerfiles/zeek.Dockerfile
           push: true
           tags: ghcr.io/${{ github.repository_owner }}/malcolm/zeek:${{ steps.extract_branch.outputs.branch }}
+          build-args: |
+            BUILD_JOBS=2
       -
         name: Run Trivy vulnerability scanner
         id: trivy-scan

.trigger_iso_workflow_build

@@ -1,2 +1,2 @@
 # this file exists solely for the purpose of being updated and seen by github to trigger a commit build action
-0
+1

.trigger_workflow_build

@@ -1,2 +1,2 @@
 # this file exists solely for the purpose of being updated and seen by github to trigger a commit build action
-0
+1

Dockerfiles/arkime.Dockerfile

@@ -3,8 +3,11 @@ FROM debian:12-slim AS build
 # Copyright (c) 2023 Battelle Energy Alliance, LLC.  All rights reserved.
 
 ENV DEBIAN_FRONTEND noninteractive
+ENV TERM xterm
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
 
-ENV ARKIME_VERSION "v4.4.0"
+ENV ARKIME_VERSION "v4.5.0"
 ENV ARKIME_DIR "/opt/arkime"
 ENV ARKIME_URL "https://github.com/arkime/arkime.git"
 ENV ARKIME_LOCALELASTICSEARCH no
@@ -50,7 +53,7 @@ RUN apt-get -q update && \
         swig \
         wget \
         zlib1g-dev && \
-  python3 -m pip install --break-system-packages --no-cache-dir beautifulsoup4 meson && \
+  python3 -m pip install --break-system-packages --no-compile --no-cache-dir beautifulsoup4 meson && \
   cd /opt && \
     git clone --recurse-submodules --branch="$ARKIME_VERSION" "$ARKIME_URL" "./arkime-"$ARKIME_VERSION && \
     cd "./arkime-"$ARKIME_VERSION && \
@@ -94,6 +97,8 @@ ENV PUSER_RLIMIT_UNLOCK true
 
 ENV DEBIAN_FRONTEND noninteractive
 ENV TERM xterm
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
 
 ARG OPENSEARCH_URL="http://opensearch:9200"
 ARG OPENSEARCH_LOCAL=true
@@ -169,7 +174,7 @@ RUN sed -i "s/main$/main contrib non-free/g" /etc/apt/sources.list.d/debian.sour
       vim-tiny \
       wget \
       tar gzip unzip cpio bzip2 lzma xz-utils p7zip-full unrar zlib1g && \
-    python3 -m pip install --break-system-packages --no-cache-dir beautifulsoup4 pyzmq watchdog && \
+    python3 -m pip install --break-system-packages --no-compile --no-cache-dir beautifulsoup4 pyzmq watchdog && \
     ln -sfr $ARKIME_DIR/bin/npm /usr/local/bin/npm && \
       ln -sfr $ARKIME_DIR/bin/node /usr/local/bin/node && \
       ln -sfr $ARKIME_DIR/bin/npx /usr/local/bin/npx && \

Dockerfiles/file-monitor.Dockerfile

@@ -20,6 +20,8 @@ ENV PUSER_PRIV_DROP true
 
 ENV DEBIAN_FRONTEND noninteractive
 ENV TERM xterm
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
 
 ARG ZEEK_EXTRACTOR_PATH=/zeek/extract_files
 ARG ZEEK_LOG_DIRECTORY=/zeek/logs
@@ -81,7 +83,7 @@ ENV YARA_VERSION "4.3.2"
 ENV YARA_URL "https://github.com/VirusTotal/yara/archive/v${YARA_VERSION}.tar.gz"
 ENV YARA_RULES_SRC_DIR "/yara-rules-src"
 ENV YARA_RULES_DIR "/yara-rules"
-ENV CAPA_VERSION "6.0.0"
+ENV CAPA_VERSION "6.1.0"
 ENV CAPA_URL "https://github.com/fireeye/capa/releases/download/v${CAPA_VERSION}/capa-v${CAPA_VERSION}-linux.zip"
 ENV CAPA_DIR "/opt/capa"
 ENV CAPA_BIN "${CAPA_DIR}/capa"
@@ -135,7 +137,7 @@ RUN sed -i "s/main$/main contrib non-free/g" /etc/apt/sources.list.d/debian.sour
       python3-requests \
       python3-zmq \
       rsync && \
-    python3 -m pip install --break-system-packages --no-cache-dir clamd supervisor yara-python python-magic psutil pycryptodome watchdog && \
+    python3 -m pip install --break-system-packages --no-compile --no-cache-dir clamd supervisor yara-python python-magic psutil pycryptodome watchdog && \
     curl -fsSLO "$SUPERCRONIC_URL" && \
       echo "${SUPERCRONIC_SHA1SUM}  ${SUPERCRONIC}" | sha1sum -c - && \
       chmod +x "$SUPERCRONIC" && \

Dockerfiles/filebeat.Dockerfile

@@ -1,4 +1,4 @@
-FROM docker.elastic.co/beats/filebeat-oss:8.9.0
+FROM docker.elastic.co/beats/filebeat-oss:8.10.0
 
 # Copyright (c) 2023 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="malcolm@inl.gov"
@@ -23,6 +23,8 @@ ENV PUSER_PRIV_DROP false
 
 ENV DEBIAN_FRONTEND noninteractive
 ENV TERM xterm
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
 
 ARG AUTO_TAG=true
 ARG FILEBEAT_SCAN_FREQUENCY=10s
@@ -93,7 +95,7 @@ RUN apt-get -q update && \
         unar \
         unzip \
         xz-utils && \
-    python3 -m pip install --no-cache-dir patool entrypoint2 pyunpack python-magic ordered-set supervisor watchdog && \
+    python3 -m pip install --no-compile --no-cache-dir patool entrypoint2 pyunpack python-magic ordered-set supervisor watchdog && \
     curl -fsSLO "$SUPERCRONIC_URL" && \
       echo "${SUPERCRONIC_SHA1SUM}  ${SUPERCRONIC}" | sha1sum -c - && \
       chmod +x "$SUPERCRONIC" && \

Dockerfiles/freq.Dockerfile

@@ -40,7 +40,7 @@ RUN apt-get -q update && \
       python3-pip \
       rsync \
       tini && \
-    pip3 install --break-system-packages supervisor six && \
+    pip3 install --break-system-packages --no-compile --no-cache-dir supervisor six && \
     cd /opt && \
     mkdir -p ./freq_server && \
       curl -sSL "$FREQ_URL" | tar xzvf - -C ./freq_server --strip-components 1 && \

Dockerfiles/logstash.Dockerfile

@@ -1,4 +1,4 @@
-FROM docker.elastic.co/logstash/logstash-oss:8.9.0
+FROM docker.elastic.co/logstash/logstash-oss:8.10.0
 
 LABEL maintainer="malcolm@inl.gov"
 LABEL org.opencontainers.image.authors='malcolm@inl.gov'
@@ -51,6 +51,7 @@ RUN set -x && \
     apt-get -q update && \
     apt-get -y -q --no-install-recommends upgrade && \
     apt-get -y --no-install-recommends install \
+        curl \
         gettext \
         patch \
         python3-setuptools \
@@ -81,6 +82,7 @@ RUN set -x && \
 
 COPY --chmod=755 shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
 COPY --chmod=755 shared/bin/service_check_passthrough.sh /usr/local/bin/
+COPY --chmod=755 shared/bin/opensearch_status.sh /usr/local/bin/
 COPY --from=ghcr.io/mmguero-dev/gostatic --chmod=755 /goStatic /usr/bin/goStatic
 COPY --chmod=755 shared/bin/manuf-oui-parse.py /usr/local/bin/
 COPY --chmod=755 shared/bin/jdk-cacerts-auto-import.sh /usr/local/bin/

Dockerfiles/netbox.Dockerfile

@@ -1,4 +1,4 @@
-FROM netboxcommunity/netbox:v3.5.7
+FROM netboxcommunity/netbox:v3.6.1
 
 # Copyright (c) 2023 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="malcolm@inl.gov"
@@ -13,13 +13,15 @@ LABEL org.opencontainers.image.description='Malcolm container providing the NetB
 ENV DEBIAN_FRONTEND noninteractive
 ENV TERM xterm
 ENV LANG C.UTF-8
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
 
 ARG DEFAULT_UID=1000
 ARG DEFAULT_GID=1000
 ENV DEFAULT_UID $DEFAULT_UID
 ENV DEFAULT_GID $DEFAULT_GID
-ENV PUSER "boxer"
-ENV PGROUP "boxer"
+ENV PUSER "ubuntu"
+ENV PGROUP "ubuntu"
 ENV PUSER_PRIV_DROP true
 
 ENV SUPERCRONIC_VERSION "0.2.26"
@@ -50,15 +52,24 @@ ADD netbox/patch/* /tmp/netbox-patches/
 RUN apt-get -q update && \
     apt-get -y -q --no-install-recommends upgrade && \
     apt-get install -q -y --no-install-recommends \
+      gcc \
       git \
       jq \
+      libpq-dev \
+      libpq5 \
       patch \
       procps \
       psmisc \
+      python3-dev \
       rsync \
       supervisor \
       tini && \
-    "${NETBOX_PATH}/venv/bin/python" -m pip install --break-system-packages --no-cache-dir 'git+https://github.com/mmguero-dev/netbox-initializers' psycopg2 pynetbox python-slugify randomcolor && \
+    "${NETBOX_PATH}/venv/bin/python" -m pip install --break-system-packages --no-compile --no-cache-dir \
+      'git+https://github.com/tobiasge/netbox-initializers' \
+      psycopg2 \
+      pynetbox \
+      python-slugify \
+      randomcolor && \
     cd "${NETBOX_PATH}" && \
       bash -c 'for i in /tmp/netbox-patches/*; do patch -p 1 -r - --no-backup-if-mismatch < $i || true; done' && \
     curl -fsSLO "${SUPERCRONIC_URL}" && \
@@ -69,13 +80,11 @@ RUN apt-get -q update && \
       touch "${SUPERCRONIC_CRONTAB}" && \
     curl -fsSL -o /usr/bin/yq "${YQ_URL}" && \
         chmod 755 /usr/bin/yq && \
-    apt-get -q -y --purge remove patch git && \
+    apt-get -q -y --purge remove patch gcc git libpq-dev python3-dev && \
       apt-get -q -y --purge autoremove && \
       apt-get clean && \
       rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
-    groupadd --gid ${DEFAULT_GID} ${PUSER} && \
-      useradd -m --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} ${PUSER} && \
-      usermod -a -G tty ${PUSER} && \
+    usermod -a -G tty ${PUSER} && \
     mkdir -p /opt/unit "${NETBOX_DEVICETYPE_LIBRARY_PATH}" && \
     chown -R $PUSER:root /etc/netbox /opt/unit "${NETBOX_PATH}" && \
     cd "$(dirname "${NETBOX_DEVICETYPE_LIBRARY_PATH}")" && \

Dockerfiles/pcap-monitor.Dockerfile

@@ -23,6 +23,8 @@ ENV PUSER_PRIV_DROP false
 
 ENV DEBIAN_FRONTEND noninteractive
 ENV TERM xterm
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
 
 ARG OPENSEARCH_URL="http://opensearch:9200"
 ARG OPENSEARCH_LOCAL=true
@@ -61,7 +63,7 @@ RUN apt-get -q update && \
       vim-tiny && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* && \
-    python3 -m pip install --break-system-packages --no-cache-dir opensearch-py pyzmq python-magic requests watchdog && \
+    python3 -m pip install --break-system-packages --no-compile --no-cache-dir opensearch-py pyzmq python-magic requests watchdog && \
     groupadd --gid ${DEFAULT_GID} ${PGROUP} && \
       useradd -M --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} ${PUSER}
 

Dockerfiles/suricata.Dockerfile

@@ -11,6 +11,8 @@ LABEL org.opencontainers.image.description='Malcolm container providing Suricata
 
 ENV DEBIAN_FRONTEND noninteractive
 ENV TERM xterm
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
 
 # configure unprivileged user and runtime parameters
 ARG DEFAULT_UID=1000
@@ -99,7 +101,7 @@ RUN sed -i "s/main$/main contrib non-free/g" /etc/apt/sources.list.d/debian.sour
         tini \
         vim-tiny \
         zlib1g && \
-    python3 -m pip install --break-system-packages --no-cache-dir watchdog && \
+    python3 -m pip install --break-system-packages --no-compile --no-cache-dir watchdog && \
     curl -fsSLO "$SUPERCRONIC_URL" && \
         echo "${SUPERCRONIC_SHA1SUM}  ${SUPERCRONIC}" | sha1sum -c - && \
         chmod +x "$SUPERCRONIC" && \