idaholab#463, initial work for allowing custom tags to be specified o…

…n hedgehog

hedgehog-iso/config/package-lists/system.list.chroot

@@ -69,6 +69,7 @@ hfsutils
 htop
 hwdata
 javascript-common
+jo
 jq
 less
 libatomic1

hedgehog-iso/interface/sensor_ctl/arkime/live_capture.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Copyright (c) 2024 Battelle Energy Alliance, LLC.  All rights reserved.
+
+ARKIME_DIR=${ARKIME_DIR:-"/opt/arkime"}
+
+[[ "$(uname -s)" = 'Darwin' ]] && REALPATH=grealpath || REALPATH=realpath
+[[ "$(uname -s)" = 'Darwin' ]] && DIRNAME=gdirname || DIRNAME=dirname
+if ! (type "$REALPATH" && type "$DIRNAME") > /dev/null; then
+  echo "$(basename "${BASH_SOURCE[0]}") requires $REALPATH and $DIRNAME"
+  exit 1
+fi
+SCRIPT_PATH="$($DIRNAME $($REALPATH -e "${BASH_SOURCE[0]}"))"
+
+TAGS_ARGS=()
+if [[ -n "${MALCOLM_EXTRA_TAGS}" ]]; then
+  while read EXTRA_TAG; do
+    TAGS_ARGS+=( -t )
+    TAGS_ARGS+=( "${EXTRA_TAG}" )
+  done < <(echo "${MALCOLM_EXTRA_TAGS}" | tr ',' '\n') # loop over ',' separated MALCOLM_EXTRA_TAGS values
+fi
+
+"${ARKIME_DIR}"/bin/capture ${ARKIME_HTTPS_FLAG:-} "${TAGS_ARGS[@]}" \
+  -c "${SCRIPT_PATH}"/config.ini \
+  -o pcapDir="${PCAP_PATH}" \
+  -o bpf="${CAPTURE_FILTER}" \
+  -o dropUser=sensor \
+  -o dropGroup=netdev \
+  -o geoLite2Country="${SCRIPT_PATH}"/GeoLite2-Country.mmdb \
+  -o geoLite2ASN="${SCRIPT_PATH}"/GeoLite2-ASN.mmdb \
+  -o rirFile="${SCRIPT_PATH}"/ipv4-address-space.csv \
+  -o ouiFile="${SCRIPT_PATH}"/oui.txt \
+  -o rulesFiles="${SCRIPT_PATH}"/rules.yml \
+  -o parsersDir="${ARKIME_DIR}"/parsers \
+  -o pluginsDir="${ARKIME_DIR}"/plugins \
+  -o ecsEventProvider="${ARKIME_ECS_PROVIDER:-arkime}" \
+  -o ecsEventDataset="${ARKIME_ECS_DATASET:-session}" \
+  --node "${ARKIME_NODE_NAME}" \
+  --host "${ARKIME_NODE_HOST}"

hedgehog-iso/interface/sensor_ctl/control_vars.conf

@@ -1,5 +1,5 @@
 export CAPTURE_INTERFACE=lo
-export CAPTURE_FILTER=""
+export CAPTURE_FILTER=
 
 export PCAP_PATH=/home/sensor/net_cap
 export PCAP_TCPDUMP_FILENAME_PATTERN=%Y%m%d_%H%M%S.pcap
@@ -55,6 +55,8 @@ export BEAT_LS_SSL_CA_CRT=/opt/sensor/sensor_ctl/logstash-client-certificates/ca
 
 export MALCOLM_REQUEST_ACL=
 export MALCOLM_REQUEST_PORTS=$ARKIME_VIEWER_PORT,$EXTRACTED_FILE_HTTP_SERVER_PORT
+# Comma-separated list of tags for data forwarded to Malcolm via filebeat, A-Za-z0-9._- allowed
+export MALCOLM_EXTRA_TAGS=
 export DOCUMENTATION_PORT=8420
 export MISCBEAT_PORT=9516
 export FLUENTBIT_METRICS_INTERVAL=30
@@ -174,7 +176,7 @@ export OS_PASSWORD=%70%61%73%73%77%6F%72%64
 export OS_SSL_VERIFY=none
 
 export VTOT_REQUESTS_PER_MINUTE=4
-export VTOT_API2_KEY=""
+export VTOT_API2_KEY=
 export CLAMD_MAX_REQUESTS=8
 export EXTRACTED_FILE_YARA_CUSTOM_ONLY=false
 export YARA_MAX_REQUESTS=8

hedgehog-iso/interface/sensor_ctl/filebeat/sensor_filebeat_local.sh

@@ -39,10 +39,28 @@ fi
 SCRIPT_PATH="$($DIRNAME $($REALPATH -e "${BASH_SOURCE[0]}"))"
 pushd "$SCRIPT_PATH" >/dev/null 2>&1
 
+INSTANCE_UID="$(tr -dc A-Za-z0-9 </dev/urandom 2>/dev/null | head -c 16; echo)"
+(( ${#INSTANCE_UID} == 16 )) || INSTANCE_UID=$RANDOM
+
+TMP_CONFIG_FILE="$SCRIPT_PATH/filebeat.$INSTANCE_UID.yml"
+
+function cleanup {
+    rm -f "$TMP_CONFIG_FILE"
+}
+
+trap cleanup EXIT
+
 mkdir -p "$SCRIPT_PATH/data"
 
+if [[ -n "${MALCOLM_EXTRA_TAGS}" ]]; then
+  readarray -td '' EXTRA_TAGS_ARRAY < <(awk '{ gsub(/,/,"\0"); print; }' <<<"$MALCOLM_EXTRA_TAGS,"); unset 'EXTRA_TAGS_ARRAY[-1]';
+  yq -P eval "(.\"filebeat.inputs\"[] | select(.type == \"log\").tags) += $(jo -a "${EXTRA_TAGS_ARRAY[@]}")" "$SCRIPT_PATH/filebeat.yml" > "$TMP_CONFIG_FILE"
+else
+  cp "$SCRIPT_PATH/filebeat.yml" "$TMP_CONFIG_FILE"
+fi
+
 sleep $SLEEP_SEC
 
-filebeat --path.home "$SCRIPT_PATH" --path.config "$SCRIPT_PATH" --path.data "$SCRIPT_PATH/data" -c "$SCRIPT_PATH/filebeat.yml" -e
+filebeat --path.home "$SCRIPT_PATH" --path.config "$SCRIPT_PATH" --path.data "$SCRIPT_PATH/data" -c "$TMP_CONFIG_FILE" -e
 
 popd  >/dev/null 2>&1

hedgehog-iso/interface/sensor_ctl/miscbeat/sensor_miscbeat_local.sh

@@ -12,8 +12,26 @@ fi
 SCRIPT_PATH="$($DIRNAME $($REALPATH -e "${BASH_SOURCE[0]}"))"
 pushd "$SCRIPT_PATH" >/dev/null 2>&1
 
+INSTANCE_UID="$(tr -dc A-Za-z0-9 </dev/urandom 2>/dev/null | head -c 16; echo)"
+(( ${#INSTANCE_UID} == 16 )) || INSTANCE_UID=$RANDOM
+
+TMP_CONFIG_FILE="$SCRIPT_PATH/filebeat.$INSTANCE_UID.yml"
+
+function cleanup {
+    rm -f "$TMP_CONFIG_FILE"
+}
+
+trap cleanup EXIT
+
 mkdir -p "$SCRIPT_PATH/data"
 
-filebeat --path.home "$SCRIPT_PATH" --path.config "$SCRIPT_PATH" --path.data "$SCRIPT_PATH/data" -c "$SCRIPT_PATH/filebeat.yml" -e
+if [[ -n "${MALCOLM_EXTRA_TAGS}" ]]; then
+  readarray -td '' EXTRA_TAGS_ARRAY < <(awk '{ gsub(/,/,"\0"); print; }' <<<"$MALCOLM_EXTRA_TAGS,"); unset 'EXTRA_TAGS_ARRAY[-1]';
+  yq -P eval "(.\"filebeat.inputs\"[] | select(.type == \"log\").tags) += $(jo -a "${EXTRA_TAGS_ARRAY[@]}")" "$SCRIPT_PATH/filebeat.yml" > "$TMP_CONFIG_FILE"
+else
+  cp "$SCRIPT_PATH/filebeat.yml" "$TMP_CONFIG_FILE"
+fi
+
+filebeat --path.home "$SCRIPT_PATH" --path.config "$SCRIPT_PATH" --path.data "$SCRIPT_PATH/data" -c "$TMP_CONFIG_FILE" -e
 
 popd  >/dev/null 2>&1

hedgehog-iso/interface/sensor_ctl/supervisor.d/arkime.conf

@@ -17,23 +17,7 @@ autorestart=%(ENV_AUTOSTART_ARKIME)s
 directory=/opt/arkime/viewer
 
 [program:arkime-capture]
-command=/opt/arkime/bin/capture %(ENV_ARKIME_HTTPS_FLAG)s
-  -c "%(ENV_SUPERVISOR_PATH)s"/arkime/config.ini
-  -o pcapDir="%(ENV_PCAP_PATH)s"
-  -o bpf="%(ENV_CAPTURE_FILTER)s"
-  -o dropUser=sensor
-  -o dropGroup=netdev
-  -o geoLite2Country="%(ENV_SUPERVISOR_PATH)s"/arkime/GeoLite2-Country.mmdb
-  -o geoLite2ASN="%(ENV_SUPERVISOR_PATH)s"/arkime/GeoLite2-ASN.mmdb
-  -o rirFile="%(ENV_SUPERVISOR_PATH)s"/arkime/ipv4-address-space.csv
-  -o ouiFile="%(ENV_SUPERVISOR_PATH)s"/arkime/oui.txt
-  -o rulesFiles="%(ENV_SUPERVISOR_PATH)s"/arkime/rules.yml
-  -o parsersDir=/opt/arkime/parsers
-  -o pluginsDir=/opt/arkime/plugins
-  -o ecsEventProvider="%(ENV_ARKIME_ECS_PROVIDER)s"
-  -o ecsEventDataset="%(ENV_ARKIME_ECS_DATASET)s"
-  --node "%(ENV_ARKIME_NODE_NAME)s"
-  --host "%(ENV_ARKIME_NODE_HOST)s"
+command="%(ENV_SUPERVISOR_PATH)s"/arkime/live_capture.sh
 startsecs=30
 startretries=2000000000
 stopasgroup=true

malcolm-iso/config/package-lists/system.list.chroot

@@ -72,6 +72,7 @@ hfsutils
 htop
 hwdata
 javascript-common
+jo
 jq
 less
 libffi-dev

shared/bin/configure-capture.py

@@ -63,6 +63,7 @@ class Constants:
     ARKIMECAP = 'arkime-capture'
     TX_RX_SECURE = 'ssl-client-receive'
     ACL_CONFIGURE = 'acl-configure'
+    TAGS_CONFIGURE = 'tags-configure'
 
     BEAT_DIR = {
         FILEBEAT: f'/opt/sensor/sensor_ctl/{FILEBEAT}',
@@ -122,6 +123,9 @@ class Constants:
     # ACL for Arkime PCAP reachback and extracted files server
     MALCOLM_REQUEST_ACL = "MALCOLM_REQUEST_ACL"
 
+    # Comma-separated list of tags for data forwarded to Malcolm via filebeat
+    MALCOLM_EXTRA_TAGS = "MALCOLM_EXTRA_TAGS"
+
     MSG_CONFIG_MODE = 'Configuration Mode'
     MSG_CONFIG_MODE_CAPTURE = 'Configure Capture'
     MSG_CONFIG_MODE_FORWARD = 'Configure Forwarding'
@@ -134,6 +138,7 @@ class Constants:
     MSG_CONFIG_MISCBEAT = (f'{MISCBEAT}', f"Configure miscellaneous sensor metrics forwarding via {FILEBEAT}")
     MSG_CONFIG_TXRX = (f'{TX_RX_SECURE}', f'Receive client SSL files for {FILEBEAT} from Malcolm')
     MSG_CONFIG_ACL = (f'{ACL_CONFIGURE}', f'Configure ACL for artifact reachback from Malcolm')
+    MSG_CONFIG_TAGS = (f'{TAGS_CONFIGURE}', f'Define extra tags for logs forwarded to Malcolm')
 
     MSG_OVERWRITE_CONFIG = '{} is already configured, overwrite current settings?'
     MSG_IDENTIFY_NICS = 'Do you need help identifying network interfaces?'
@@ -165,6 +170,7 @@ class Constants:
     )
     MSG_CONFIG_ARKIME_VIEWER_PASSWORD = 'Specify password hash secret for Arkime viewer cluster'
     MSG_CONFIG_REQUEST_ACL = 'Specify IP addresses for ACL for artifact reachback from Malcolm (one per line)'
+    MSG_CONFIG_EXTRA_TAGS = 'Specify extra tags for logs forwarded to Malcolm (one per line)'
     MSG_ERR_PLEBE_REQUIRED = 'this utility should be be run as non-privileged user'
     MSG_ERROR_DIR_NOT_FOUND = 'One or more of the paths specified does not exist'
     MSG_ERROR_FILE_NOT_FOUND = 'One or more of the files specified does not exist'
@@ -397,6 +403,7 @@ def main():
                 Constants.BEAT_OS_PORT: "OS_PORT",
                 Constants.BEAT_HTTP_USERNAME: "OS_USERNAME",
                 Constants.MALCOLM_REQUEST_ACL: Constants.MALCOLM_REQUEST_ACL,
+                Constants.MALCOLM_EXTRA_TAGS: Constants.MALCOLM_EXTRA_TAGS,
                 Constants.ARKIME_PASSWORD_SECRET: Constants.ARKIME_PASSWORD_SECRET,
                 Constants.BEAT_LS_SSL_CA_CRT: Constants.BEAT_LS_SSL_CA_CRT,
                 Constants.BEAT_LS_SSL_CLIENT_CRT: Constants.BEAT_LS_SSL_CLIENT_CRT,
@@ -849,8 +856,9 @@ def main():
                         Constants.MSG_CONFIG_FILEBEAT,
                         Constants.MSG_CONFIG_MISCBEAT,
                         Constants.MSG_CONFIG_ACL,
+                        Constants.MSG_CONFIG_TAGS,
                         Constants.MSG_CONFIG_TXRX,
-                    ][: 5 if txRxScript else -1],
+                    ][: 6 if txRxScript else -1],
                 )
                 if code != Dialog.OK:
                     raise CancelledError
@@ -1247,7 +1255,6 @@ def main():
                 elif fwd_mode == Constants.ACL_CONFIGURE:
 
                     # get list of IP addresses allowed for packet payload retrieval
-                    acl_config_dict = defaultdict(str)
                     lines = previous_config_values[Constants.MALCOLM_REQUEST_ACL].split(",")
                     if Constants.BEAT_OS_HOST in previous_config_values and (
                         previous_config_values[Constants.BEAT_OS_HOST]
@@ -1279,6 +1286,41 @@ def main():
                         )
                     )
 
+                elif fwd_mode == Constants.TAGS_CONFIGURE:
+
+                    # get list of tags for logs forwarded to Malcolm
+                    lines = previous_config_values[Constants.MALCOLM_EXTRA_TAGS].split(",")
+                    code, lines = d.editbox_str(
+                        "\n".join(list(filter(None, list(set(lines))))), title=Constants.MSG_CONFIG_EXTRA_TAGS
+                    )
+                    if code != Dialog.OK:
+                        raise CancelledError
+
+                    # modify specified tags array value in-place in SENSOR_CAPTURE_CONFIG file
+                    newTagsValsDict = {
+                        Constants.MALCOLM_EXTRA_TAGS: ','.join(
+                            [
+                                tag
+                                for tag in list(
+                                    set(
+                                        filter(
+                                            None,
+                                            [re.sub(r'[^A-Za-z0-9 ._-]', '', x.strip()) for x in lines.split('\n')],
+                                        )
+                                    )
+                                )
+                            ]
+                        )
+                    }
+                    rewrite_dict_to_file(newTagsValsDict, Constants.SENSOR_CAPTURE_CONFIG)
+
+                    # hooray
+                    code = d.msgbox(
+                        text=Constants.MSG_CONFIG_FORWARDING_SUCCESS.format(
+                            fwd_mode, "\n".join(newTagsValsDict[Constants.MALCOLM_EXTRA_TAGS].split(','))
+                        )
+                    )
+
                 elif (fwd_mode == Constants.TX_RX_SECURE) and txRxScript:
                     # use tx-rx-secure.sh (via croc) to get certs from Malcolm
                     code = d.msgbox(text='Run auth_setup on Malcolm "Transfer self-signed client certificates..."')