Work in progress of idaholab#281, arkime-live container; done in arki…

…me-capture branch

Dockerfiles/arkime.Dockerfile

@@ -118,10 +118,13 @@ ARG ARKIME_PACKET_THREADS=1
 ARG OPENSEARCH_MAX_SHARDS_PER_NODE=2500
 ARG WISE=on
 ARG VIEWER=on
+ARG ARKIME_VIEWER_PORT=8005
 #Whether or not Arkime is in charge of deleting old PCAP files to reclaim space
 ARG MANAGE_PCAP_FILES=false
 ARG ARKIME_PCAP_PROCESSOR=true
 ARG ARKIME_LIVE_CAPTURE=false
+ARG ARKIME_LIVE_NODE_NAME=malcolm
+ARG ARKIME_LIVE_NODE_HOST=
 ARG ARKIME_ROTATED_PCAP=true
 ARG ARKIME_COMPRESSION_TYPE=none
 ARG ARKIME_COMPRESSION_LEVEL=0
@@ -149,12 +152,15 @@ ENV ARKIME_AUTO_ANALYZE_PCAP_THREADS $ARKIME_AUTO_ANALYZE_PCAP_THREADS
 ENV ARKIME_PACKET_THREADS $ARKIME_PACKET_THREADS
 ENV ARKIME_PCAP_PROCESSOR $ARKIME_PCAP_PROCESSOR
 ENV ARKIME_LIVE_CAPTURE $ARKIME_LIVE_CAPTURE
+ENV ARKIME_LIVE_NODE_NAME $ARKIME_LIVE_NODE_NAME
+ENV ARKIME_LIVE_NODE_HOST $ARKIME_LIVE_NODE_HOST
 ENV ARKIME_COMPRESSION_TYPE $ARKIME_COMPRESSION_TYPE
 ENV ARKIME_COMPRESSION_LEVEL $ARKIME_COMPRESSION_LEVEL
 ENV ARKIME_ROTATED_PCAP $ARKIME_ROTATED_PCAP
 ENV OPENSEARCH_MAX_SHARDS_PER_NODE $OPENSEARCH_MAX_SHARDS_PER_NODE
 ENV WISE $WISE
 ENV VIEWER $VIEWER
+ENV ARKIME_VIEWER_PORT $ARKIME_VIEWER_PORT
 ENV MANAGE_PCAP_FILES $MANAGE_PCAP_FILES
 ENV AUTO_TAG $AUTO_TAG
 ENV PCAP_PIPELINE_VERBOSITY $PCAP_PIPELINE_VERBOSITY
@@ -258,7 +264,7 @@ RUN groupadd --gid $DEFAULT_GID $PGROUP && \
 #Update Path
 ENV PATH="/opt:$ARKIME_DIR/bin:${PATH}"
 
-EXPOSE 8000 8005 8081
+EXPOSE 8000 8005 8006 8081
 WORKDIR $ARKIME_DIR
 
 ENTRYPOINT ["/usr/bin/tini", \

arkime/arkime_regression_test_harness/Dockerfiles/arkime.Dockerfile

@@ -66,7 +66,7 @@ RUN git clone --recursive --depth=1 --single-branch -b "$GITHUB_BRANCH" "$GITHUB
 
 ENV PATH="/opt:$ARKIME_DIR/bin:$ARKIME_DIR/node-v10.21.0-linux-x64/bin:${PATH}"
 
-EXPOSE 8000 8005 8081
+EXPOSE 8000 8005 8006 8081
 
 WORKDIR $ARKIME_DIR/tests
 

arkime/etc/config.ini

@@ -17,6 +17,7 @@ geoLite2Country=/opt/arkime/etc/GeoLite2-Country.mmdb
 httpRealm=Arkime
 icmpTimeout=10
 interface=eth0
+bpf=
 keyFile=/opt/arkime/etc/viewer.key
 logESRequests=false
 logEveryXPackets=500000

arkime/scripts/docker_entrypoint.sh

@@ -13,6 +13,7 @@ ARKIME_PASSWORD_SECRET=${ARKIME_PASSWORD_SECRET:-"Malcolm"}
 ARKIME_FREESPACEG=${ARKIME_FREESPACEG:-"10%"}
 CAPTURE_INTERFACE=${PCAP_IFACE:-}
 LIVE_CAPTURE=${ARKIME_LIVE_CAPTURE:-false}
+VIEWER_PORT=${ARKIME_VIEWER_PORT:-8005}
 
 MALCOLM_PROFILE=${MALCOLM_PROFILE:-"malcolm"}
 OPENSEARCH_URL_FINAL=${OPENSEARCH_URL:-"http://opensearch:9200"}
@@ -53,6 +54,7 @@ if [[ -r "${ARKIME_DIR}"/etc/config.orig.ini ]]; then
     sed -i "s|^\(elasticsearch=\).*|\1"${OPENSEARCH_URL_FINAL}"|" "${ARKIME_DIR}"/etc/config.ini
     sed -i "s/^\(passwordSecret=\).*/\1"${ARKIME_PASSWORD_SECRET}"/" "${ARKIME_DIR}"/etc/config.ini
     sed -i "s/^\(freeSpaceG=\).*/\1"${ARKIME_FREESPACEG}"/" "${ARKIME_DIR}"/etc/config.ini
+    sed -i "s/^\(viewPort=\).*/\1"${VIEWER_PORT}"/" "${ARKIME_DIR}"/etc/config.ini
 
     # capture interface(s)
     if [[ -n "$CAPTURE_INTERFACE" ]] && [[ "$LIVE_CAPTURE" == "true" ]] ; then
@@ -63,6 +65,7 @@ if [[ -r "${ARKIME_DIR}"/etc/config.orig.ini ]]; then
       # place capture interfaces in the config file
       sed -r -i "s|(interface)\s*=\s*.*|\1=$ARKIME_CAPTURE_INTERFACE|" "${ARKIME_DIR}"/etc/config.ini
       sed -i "s/^\(readTruncatedPackets=\).*/\1"false"/" "${ARKIME_DIR}"/etc/config.ini
+      sed -r -i "s/(bpf)\s*=\s*.*/\1=${PCAP_FILTER:-}/" "${ARKIME_DIR}"/etc/config.ini
 
       # convert pcap rotation size units (MB to GB) and stick in config file
       if [[ -n $PCAP_ROTATE_MEGABYTES ]]; then
@@ -91,8 +94,8 @@ if [[ -r "${ARKIME_DIR}"/etc/config.orig.ini ]]; then
       setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' "${ZEEK_DIR}"/bin/capture || true
     fi
 
-    # comment-out features that are only unused in hedgehog run profile mode
-    if [[ "$MALCOLM_PROFILE" == "hedgehog" ]]; then
+    # comment-out features that are unused in hedgehog run profile mode and in live-capture mode
+    if [[ "$MALCOLM_PROFILE" == "hedgehog" ]] || [[ "$LIVE_CAPTURE" == "true" ]]; then
         sed -i "s/^\(userNameHeader=\)/# \1/" "${ARKIME_DIR}"/etc/config.ini
         sed -i "s/^\(userAuthIps=\)/# \1/" "${ARKIME_DIR}"/etc/config.ini
         sed -i "s/^\(userAutoCreateTmpl=\)/# \1/" "${ARKIME_DIR}"/etc/config.ini
@@ -103,7 +106,9 @@ if [[ -r "${ARKIME_DIR}"/etc/config.orig.ini ]]; then
         sed -i '/^\[custom-fields\]/,$d' "${ARKIME_DIR}"/etc/config.ini
     fi
 
-    chmod 600 "${ARKIME_DIR}"/etc/config.ini
+    chmod 600 "${ARKIME_DIR}"/etc/config.ini || true
+    [[ -n ${PUID} ]] && chown -f ${PUID} "${ARKIME_DIR}"/etc/config.ini || true
+    [[ -n ${PGID} ]] && chown -f :${PGID} "${ARKIME_DIR}"/etc/config.ini || true
 fi
 
 unset OPENSEARCH_URL_FINAL

arkime/scripts/initarkime.sh

@@ -3,6 +3,7 @@
 # Copyright (c) 2023 Battelle Energy Alliance, LLC.  All rights reserved.
 
 MALCOLM_PROFILE=${MALCOLM_PROFILE:-"malcolm"}
+ARKIME_LIVE_CAPTURE=${ARKIME_LIVE_CAPTURE:-"false"}
 OPENSEARCH_URL=${OPENSEARCH_URL:-"http://opensearch:9200"}
 OPENSEARCH_PRIMARY=${OPENSEARCH_PRIMARY:-"opensearch-local"}
 OPENSEARCH_SSL_CERTIFICATE_VERIFICATION=${OPENSEARCH_SSL_CERTIFICATE_VERIFICATION:-"false"}
@@ -41,70 +42,75 @@ if [[ "$MALCOLM_PROFILE" == "malcolm" ]]; then
   # download and/or update geo updates
   $ARKIME_DIR/bin/arkime_update_geo.sh
 
-  # start and wait patiently for WISE
-  if [[ "$WISE" = "on" ]] ; then
-    touch /var/run/arkime/runwise
-    echo "Giving WISE time to start..."
-    sleep 5
-    until curl -fsS --output /dev/null "http://127.0.0.1:8081/fields?ver=1"
-    do
-        echo "Waiting for WISE to start"
-        sleep 1
-    done
-    echo "WISE is running!"
-    echo
-  fi
+  # don't do database initialization or run wise in arkime-live mode
+  if [[ "$ARKIME_LIVE_CAPTURE" == "false" ]]; then
 
-  # initialize the contents of the OpenSearch database if it has never been initialized (ie., the users_v# table hasn't been created)
-  if [[ $(curl "${CURL_CONFIG_PARAMS[@]}" -fs -XGET -H'Content-Type: application/json' "${OPENSEARCH_URL}/_cat/indices/arkime_users_v*" | wc -l) < 1 ]]; then
+    # start and wait patiently for WISE
+    if [[ "$WISE" = "on" ]] ; then
+      touch /var/run/arkime/runwise
+      echo "Giving WISE time to start..."
+      sleep 5
+      until curl -fsS --output /dev/null "http://127.0.0.1:8081/fields?ver=1"
+      do
+          echo "Waiting for WISE to start"
+          sleep 1
+      done
+      echo "WISE is running!"
+      echo
+    fi
 
-    echo "Initializing $OPENSEARCH_PRIMARY database..."
+    # initialize the contents of the OpenSearch database if it has never been initialized (ie., the users_v# table hasn't been created)
+    if [[ $(curl "${CURL_CONFIG_PARAMS[@]}" -fs -XGET -H'Content-Type: application/json' "${OPENSEARCH_URL}/_cat/indices/arkime_users_v*" | wc -l) < 1 ]]; then
 
-  	$ARKIME_DIR/db/db.pl $DB_SSL_FLAG "${OPENSEARCH_URL_FULL}" initnoprompt
+      echo "Initializing $OPENSEARCH_PRIMARY database..."
 
-    echo "Creating default user..."
+    	$ARKIME_DIR/db/db.pl $DB_SSL_FLAG "${OPENSEARCH_URL_FULL}" initnoprompt
 
-  	# this password isn't going to be used by Arkime, nginx will do the auth instead
-  	$ARKIME_DIR/bin/arkime_add_user.sh "${MALCOLM_USERNAME}" "${MALCOLM_USERNAME}" "ignored" --admin --webauthonly --webauth $DB_SSL_FLAG
+      echo "Creating default user..."
 
-    echo "Initializing fields..."
+    	# this password isn't going to be used by Arkime, nginx will do the auth instead
+    	$ARKIME_DIR/bin/arkime_add_user.sh "${MALCOLM_USERNAME}" "${MALCOLM_USERNAME}" "ignored" --admin --webauthonly --webauth $DB_SSL_FLAG
 
-    # this is a hacky way to get all of the Arkime-parseable field definitions put into E.S.
-    touch /tmp/not_a_packet.pcap
-    $ARKIME_DIR/bin/capture $DB_SSL_FLAG --packetcnt 0 -r /tmp/not_a_packet.pcap >/dev/null 2>&1
-    rm -f /tmp/not_a_packet.pcap
+      echo "Initializing fields..."
 
-    echo "Initializing views..."
+      # this is a hacky way to get all of the Arkime-parseable field definitions put into E.S.
+      touch /tmp/not_a_packet.pcap
+      $ARKIME_DIR/bin/capture $DB_SSL_FLAG --packetcnt 0 -r /tmp/not_a_packet.pcap >/dev/null 2>&1
+      rm -f /tmp/not_a_packet.pcap
 
-    for VIEW_FILE in "$ARKIME_DIR"/etc/views/*.json; do
-      TEMP_JSON=$(mktemp --suffix=.json)
-      RANDOM_ID="$(openssl rand -base64 14 | sed -E 's/[^[:alnum:][:space:]]+/_/g')"
-      echo "Creating view $(jq '.name' < "${VIEW_FILE}")"
-      jq ". += {\"user\": \"${MALCOLM_USERNAME}\"}" < "${VIEW_FILE}" >"${TEMP_JSON}"
-      curl "${CURL_CONFIG_PARAMS[@]}" -sS --output /dev/null -H'Content-Type: application/json' -XPOST "${OPENSEARCH_URL}/arkime_views/_doc/${RANDOM_ID}" -d "@${TEMP_JSON}"
-      rm -f "${TEMP_JSON}"
-    done
+      echo "Initializing views..."
 
-    echo "Setting defaults..."
+      for VIEW_FILE in "$ARKIME_DIR"/etc/views/*.json; do
+        TEMP_JSON=$(mktemp --suffix=.json)
+        RANDOM_ID="$(openssl rand -base64 14 | sed -E 's/[^[:alnum:][:space:]]+/_/g')"
+        echo "Creating view $(jq '.name' < "${VIEW_FILE}")"
+        jq ". += {\"user\": \"${MALCOLM_USERNAME}\"}" < "${VIEW_FILE}" >"${TEMP_JSON}"
+        curl "${CURL_CONFIG_PARAMS[@]}" -sS --output /dev/null -H'Content-Type: application/json' -XPOST "${OPENSEARCH_URL}/arkime_views/_doc/${RANDOM_ID}" -d "@${TEMP_JSON}"
+        rm -f "${TEMP_JSON}"
+      done
 
-    curl "${CURL_CONFIG_PARAMS[@]}" -sS --output /dev/null -H'Content-Type: application/json' -XPOST "${OPENSEARCH_URL}/arkime_users/_update/$MALCOLM_USERNAME" -d "@$ARKIME_DIR/etc/user_settings.json"
+      echo "Setting defaults..."
 
-    echo -e "\n$OPENSEARCH_PRIMARY database initialized!\n"
+      curl "${CURL_CONFIG_PARAMS[@]}" -sS --output /dev/null -H'Content-Type: application/json' -XPOST "${OPENSEARCH_URL}/arkime_users/_update/$MALCOLM_USERNAME" -d "@$ARKIME_DIR/etc/user_settings.json"
 
-  else
-    echo "$OPENSEARCH_PRIMARY database previously initialized!"
-    echo
+      echo -e "\n$OPENSEARCH_PRIMARY database initialized!\n"
 
-    $ARKIME_DIR/db/db.pl $DB_SSL_FLAG "${OPENSEARCH_URL_FULL}" upgradenoprompt --ifneeded
-    echo "$OPENSEARCH_PRIMARY database is up-to-date for Arkime version $ARKIME_VERSION!"
+    else
+      echo "$OPENSEARCH_PRIMARY database previously initialized!"
+      echo
 
-  fi # if/else OpenSearch database initialized
+      $ARKIME_DIR/db/db.pl $DB_SSL_FLAG "${OPENSEARCH_URL_FULL}" upgradenoprompt --ifneeded
+      echo "$OPENSEARCH_PRIMARY database is up-to-date for Arkime version $ARKIME_VERSION!"
 
-  # increase OpenSearch max shards per node from default if desired
-  if [[ -n $OPENSEARCH_MAX_SHARDS_PER_NODE ]]; then
-    # see https://github.com/elastic/elasticsearch/issues/40803
-    curl "${CURL_CONFIG_PARAMS[@]}" -sS --output /dev/null -H'Content-Type: application/json' -XPUT "${OPENSEARCH_URL}/_cluster/settings" -d "{ \"persistent\": { \"cluster.max_shards_per_node\": \"$OPENSEARCH_MAX_SHARDS_PER_NODE\" } }"
-  fi
+    fi # if/else OpenSearch database initialized
+
+    # increase OpenSearch max shards per node from default if desired
+    if [[ -n $OPENSEARCH_MAX_SHARDS_PER_NODE ]]; then
+      # see https://github.com/elastic/elasticsearch/issues/40803
+      curl "${CURL_CONFIG_PARAMS[@]}" -sS --output /dev/null -H'Content-Type: application/json' -XPUT "${OPENSEARCH_URL}/_cluster/settings" -d "{ \"persistent\": { \"cluster.max_shards_per_node\": \"$OPENSEARCH_MAX_SHARDS_PER_NODE\" } }"
+    fi
+
+  fi # "$ARKIME_LIVE_CAPTURE" == "false"
 
   # before running viewer, call _refresh to make sure everything is available for search first
   curl "${CURL_CONFIG_PARAMS[@]}" -sS -XPOST "${OPENSEARCH_URL}/_refresh"

arkime/supervisord.conf

@@ -87,14 +87,13 @@ user=%(ENV_PUSER)s
 command="%(ENV_ARKIME_DIR)s"/bin/capture --insecure
   -c "%(ENV_ARKIME_DIR)s"/etc/config.ini
   -o pcapDir=/data/pcap/processed
-  -o bpf="%(ENV_CAPTURE_FILTER)s"
   -o packetThreads=%(ENV_ARKIME_PACKET_THREADS)s
   -o dropUser=%(ENV_PUSER)s
   -o dropGroup=%(ENV_PGROUP)s
   -o ecsEventProvider=arkime
   -o ecsEventDataset=session
-  --node "%(ENV_PCAP_NODE_NAME)s"
-  --host "%(ENV_PCAP_NODE_HOST)s"
+  --node "%(ENV_ARKIME_LIVE_NODE_NAME)s"
+  --host "%(ENV_ARKIME_LIVE_NODE_HOST)s"
 autostart=%(ENV_ARKIME_LIVE_CAPTURE)s
 autorestart=%(ENV_ARKIME_LIVE_CAPTURE)s
 startsecs=30
@@ -105,6 +104,6 @@ killasgroup=true
 stdout_logfile=/dev/fd/1
 stdout_logfile_maxbytes=0
 redirect_stderr=true
-user=%(ENV_PUSER)s
+user=root
 directory=/data/pcap/processed
 

config/arkime-live.env.example

@@ -1,8 +1,16 @@
+ARKIME_VIEWER_PORT=8006
 # Whether or Arkime should monitor live traffic on a local
 #   interface (PCAP_IFACE in pcap-capture.env specifies interface)
 ARKIME_LIVE_CAPTURE=false
 ARKIME_COMPRESSION_TYPE=none
 ARKIME_COMPRESSION_LEVEL=0
 ARKIME_PACKET_THREADS=1
 
+# The node name (e.g., the hostname of this machine running Malcolm) to associate with
+#   network traffic metadata
+ARKIME_LIVE_NODE_NAME=malcolm
+# The node host (e.g., the IP address of the machine running Malcolm) to associate with
+#   network traffic metadata (optional, defaults to PCAP_NODE_NAME if unspecified)
+ARKIME_LIVE_NODE_HOST=
+
 ARKIME_PCAP_PROCESSOR=false

config/arkime-offline.env.example

@@ -1,3 +1,4 @@
+ARKIME_VIEWER_PORT=8005
 # Whether or not Arkime should analyze uploaded PCAP files
 ARKIME_AUTO_ANALYZE_PCAP_FILES=true
 # The number of Arkime processes for analyzing uploaded PCAP files allowed

sensor-iso/interface/sensor_ctl/arkime/config.ini

@@ -12,6 +12,7 @@ geoLite2ASN=/dummy/GeoLite2-ASN.mmdb
 geoLite2Country=/dummy/GeoLite2-Country.mmdb
 httpRealm=Arkime
 icmpTimeout=10
+bpf=
 interface=enp0s1
 logESRequests=false
 logEveryXPackets=500000

sensor-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh

@@ -38,6 +38,10 @@ if [[ -n $SUPERVISOR_PATH ]] && [[ -r "$SUPERVISOR_PATH"/arkime/config.ini ]]; t
     sed -r -i "s|(elasticsearch)\s*=\s*.*|\1=$ARKIME_ELASTICSEARCH|" "$ARKIME_CONFIG_FILE"
   fi
 
+  if [[ -n $ARKIME_VIEWER_PORT ]]; then
+    sed -r -i "s/(viewPort)\s*=\s*.*/\1=$ARKIME_VIEWER_PORT/" "$ARKIME_CONFIG_FILE"
+  f
+
   if [[ -n $ARKIME_PASSWORD_SECRET ]]; then
     # place the Arkime viewer cluster password hash in the config file
     sed -r -i "s|(passwordSecret)\s*=\s*.*|\1=$ARKIME_PASSWORD_SECRET|" "$ARKIME_CONFIG_FILE"
@@ -64,7 +68,6 @@ if [[ -n $SUPERVISOR_PATH ]] && [[ -r "$SUPERVISOR_PATH"/arkime/config.ini ]]; t
   if [[ -n $ARKIME_FREESPACEG ]]; then
     sed -r -i "s/(freeSpaceG)\s*=\s*.*/\1=$ARKIME_FREESPACEG/" "$ARKIME_CONFIG_FILE"
   fi
-
   # pcap compression
   COMPRESSION_TYPE="${ARKIME_COMPRESSION_TYPE:-none}"
   COMPRESSION_LEVEL="${ARKIME_COMPRESSION_LEVEL:-0}"