work for idaholab#445, supporting suricata eve.json rotation

Dockerfiles/filebeat.Dockerfile

@@ -124,7 +124,7 @@ RUN for INPUT in nginx tcp; do \
       chmod 770 /usr/share/filebeat-$INPUT/data; \
     done; \
     chmod 755 /usr/local/bin/*.sh /usr/local/bin/*.py && \
-    (echo "* * * * * /usr/local/bin/filebeat-process-zeek-folder.sh\n*/5 * * * * /usr/local/bin/filebeat-clean-zeeklogs-processed-folder.py" > ${SUPERCRONIC_CRONTAB})
+    (echo "* * * * * /usr/local/bin/filebeat-process-zeek-folder.sh\n*/5 * * * * /usr/local/bin/clean-processed-folder.py" > ${SUPERCRONIC_CRONTAB})
 
 ENV AUTO_TAG $AUTO_TAG
 ENV LOG_CLEANUP_MINUTES $LOG_CLEANUP_MINUTES

Dockerfiles/suricata.Dockerfile

@@ -114,7 +114,7 @@ RUN sed -i "s/main$/main contrib non-free/g" /etc/apt/sources.list.d/debian.sour
       useradd -M --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} --home /nonexistant ${PUSER} && \
       usermod -a -G tty ${PUSER} && \
     ln -sfr /usr/local/bin/pcap_processor.py /usr/local/bin/pcap_suricata_processor.py && \
-        (echo "*/5 * * * * /usr/local/bin/eve-clean-logs.sh\n0 */6 * * * /bin/bash /usr/local/bin/suricata-update-rules.sh\n" > ${SUPERCRONIC_CRONTAB}) && \
+        (echo "0 */6 * * * /bin/bash /usr/local/bin/suricata-update-rules.sh\n" > ${SUPERCRONIC_CRONTAB}) && \
     mkdir -p "$SURICATA_CUSTOM_RULES_DIR" "$SURICATA_DEFAULT_RULES_DIR" "$SURICATA_CUSTOM_CONFIG_DIR" && \
         chown -R ${PUSER}:${PGROUP} "$SURICATA_CUSTOM_RULES_DIR" "$SURICATA_DEFAULT_RULES_DIR" "$SURICATA_CUSTOM_CONFIG_DIR" && \
     cp "$(dpkg -L suricata-update | grep 'update\.yaml$' | head -n 1)" \
@@ -136,7 +136,6 @@ COPY --chmod=755 shared/bin/pcap_processor.py /usr/local/bin/
 COPY --chmod=644 scripts/malcolm_utils.py /usr/local/bin/
 COPY --chmod=755 shared/bin/suricata_config_populate.py /usr/local/bin/
 COPY --chmod=755 suricata/scripts/docker_entrypoint.sh /usr/local/bin/
-COPY --chmod=755 suricata/scripts/eve-clean-logs.sh /usr/local/bin/
 COPY --chmod=755 suricata/scripts/suricata-update-rules.sh /usr/local/bin/
 COPY --chmod=u=rwX,go=rX suricata/rules-default/ "$SURICATA_DEFAULT_RULES_DIR"/
 
@@ -148,7 +147,6 @@ ARG SURICATA_CRON=true
 ARG SURICATA_AUTO_ANALYZE_PCAP_FILES=false
 ARG SURICATA_CUSTOM_RULES_ONLY=false
 ARG SURICATA_AUTO_ANALYZE_PCAP_THREADS=1
-ARG LOG_CLEANUP_MINUTES=30
 ARG SURICATA_UPDATE_RULES=false
 ARG SURICATA_UPDATE_DEBUG=false
 ARG SURICATA_UPDATE_ETOPEN=true
@@ -168,7 +166,6 @@ ENV SURICATA_CRON $SURICATA_CRON
 ENV SURICATA_AUTO_ANALYZE_PCAP_FILES $SURICATA_AUTO_ANALYZE_PCAP_FILES
 ENV SURICATA_AUTO_ANALYZE_PCAP_THREADS $SURICATA_AUTO_ANALYZE_PCAP_THREADS
 ENV SURICATA_CUSTOM_RULES_ONLY $SURICATA_CUSTOM_RULES_ONLY
-ENV LOG_CLEANUP_MINUTES $LOG_CLEANUP_MINUTES
 ENV SURICATA_UPDATE_RULES $SURICATA_UPDATE_RULES
 ENV SURICATA_UPDATE_DEBUG $SURICATA_UPDATE_DEBUG
 ENV SURICATA_UPDATE_ETOPEN $SURICATA_UPDATE_ETOPEN

config/filebeat.env.example

@@ -9,6 +9,12 @@ FILEBEAT_CLOSE_RENAMED=true
 FILEBEAT_CLOSE_REMOVED=true
 FILEBEAT_CLOSE_EOF=true
 FILEBEAT_CLEAN_REMOVED=true
+# The age (in minutes) at which already-processed log files containing network traffic metadata should
+#   be pruned from the filesystem
+LOG_CLEANUP_MINUTES=360
+# The age (in minutes) at which the compressed archives containing already-processed log files should
+#   be pruned from the filesystem
+ZIP_CLEANUP_MINUTES=720
 # Whether or not to use polling vs. native inotify API to watch for files
 FILEBEAT_WATCHER_POLLING=false
 # When polling, seconds of inactivity to assume a file is closed and ready for processing

config/upload-common.env.example

@@ -14,10 +14,4 @@ PCAP_PIPELINE_POLLING=false
 PCAP_PIPELINE_POLLING_ASSUME_CLOSED_SEC=10
 # 'pcap-monitor' to match the name of the container providing the uploaded/captured PCAP file
 #   monitoring service
-PCAP_MONITOR_HOST=pcap-monitor
-# The age (in minutes) at which already-processed log files containing network traffic metadata should
-#   be pruned from the filesystem
-LOG_CLEANUP_MINUTES=360
-# The age (in minutes) at which the compressed archives containing already-processed log files should
-#   be pruned from the filesystem
-ZIP_CLEANUP_MINUTES=720
+PCAP_MONITOR_HOST=pcap-monitor

filebeat/filebeat.yml

@@ -72,7 +72,7 @@ filebeat.inputs:
 #-------------------------- Suricata EVE JSON logs -----------------------------
 - type: log
   paths:
-    - ${FILEBEAT_SURICATA_LOG_PATH:/suricata}/eve-*.json
+    - ${FILEBEAT_SURICATA_LOG_PATH:/suricata}/eve*.json
   symlinks: true
   fields_under_root: true
   tags: ["_filebeat_suricata_malcolm_upload"]
@@ -88,7 +88,7 @@ filebeat.inputs:
 
 - type: log
   paths:
-    - ${FILEBEAT_SURICATA_LOG_PATH:/suricata}/live/eve.json
+    - ${FILEBEAT_SURICATA_LOG_PATH:/suricata}/live/eve*.json
   symlinks: true
   fields_under_root: true
   tags: ["_filebeat_suricata_malcolm_live"]

filebeat/scripts/filebeat-clean-zeeklogs-processed-folder.py → filebeat/scripts/clean-processed-folder.py

@@ -16,16 +16,20 @@
 from subprocess import Popen, PIPE
 
 lockFilename = os.path.join(gettempdir(), '{}.lock'.format(os.path.basename(__file__)))
-zeekDir = os.path.join(os.getenv('FILEBEAT_ZEEK_DIR', "/zeek/"), '')
 cleanLogSeconds = int(os.getenv('LOG_CLEANUP_MINUTES', "30")) * 60
 cleanZipSeconds = int(os.getenv('ZIP_CLEANUP_MINUTES', "120")) * 60
 fbRegFilename = os.getenv('FILEBEAT_REGISTRY_FILE', "/usr/share/filebeat/data/registry/filebeat/data.json")
-currentDir = zeekDir + "current/"
-processedDir = zeekDir + "processed/"
-liveDir = zeekDir + "live/logs/"
+
+zeekDir = os.path.join(os.getenv('FILEBEAT_ZEEK_DIR', "/zeek/"), '')
+zeekLiveDir = zeekDir + "live/logs/"
+zeekCurrentDir = zeekDir + "current/"
+zeekProcessedDir = zeekDir + "processed/"
+
+suricataDir = os.path.join(os.getenv('FILEBEAT_SURICATA_LOG_PATH', "/suricata/"), '')
+suricataLiveDir = suricataDir + "live/"
 
 nowTime = time.time()
-logMimeType = "text/plain"
+logMimeTypeRegex = re.compile(r"(text/plain|application/(x-nd)?json)")
 archiveMimeTypeRegex = re.compile(
     r"(application/gzip|application/x-gzip|application/x-7z-compressed|application/x-bzip2|application/x-cpio|application/x-lzip|application/x-lzma|application/x-rar-compressed|application/x-tar|application/x-xz|application/zip)"
 )
@@ -74,7 +78,7 @@ def checkFile(filename, filebeatReg=None, checkLogs=True, checkArchives=True):
 
             # get the file type
             fileType = magic.from_file(filename, mime=True)
-            if (checkLogs is True) and (cleanLogSeconds > 0) and (fileType == logMimeType):
+            if (checkLogs is True) and (cleanLogSeconds > 0) and logMimeTypeRegex.match(fileType) is not None:
                 cleanSeconds = cleanLogSeconds
             elif (checkArchives is True) and (cleanZipSeconds > 0) and archiveMimeTypeRegex.match(fileType) is not None:
                 cleanSeconds = cleanZipSeconds
@@ -100,14 +104,16 @@ def pruneFiles():
         # disabled, don't do anything
         return
 
-    # look for regular files in the processed/ directory
-    foundFiles = [
-        (os.path.join(root, filename)) for root, dirnames, filenames in os.walk(processedDir) for filename in filenames
+    # look for regular Zeek files in the processed/ directory
+    zeekFoundFiles = [
+        (os.path.join(root, filename))
+        for root, dirnames, filenames in os.walk(zeekProcessedDir)
+        for filename in filenames
     ]
 
     # look for rotated files from live zeek instance
-    rotatedFiles = [
-        (os.path.join(root, filename)) for root, dirnames, filenames in os.walk(liveDir) for filename in filenames
+    zeekRotatedFiles = [
+        (os.path.join(root, filename)) for root, dirnames, filenames in os.walk(zeekLiveDir) for filename in filenames
     ]
 
     # look up the filebeat registry file and try to read it
@@ -117,22 +123,22 @@ def pruneFiles():
             fbReg = json.load(f)
 
     # see if the files we found are in use and old enough to be pruned
-    for file in foundFiles:
+    for file in zeekFoundFiles:
         checkFile(file, filebeatReg=fbReg, checkLogs=True, checkArchives=True)
-    for file in rotatedFiles:
+    for file in zeekRotatedFiles:
         checkFile(file, filebeatReg=None, checkLogs=False, checkArchives=True)
 
-    # clean up any broken symlinks in the current/ directory
-    for current in os.listdir(currentDir):
-        currentFileSpec = os.path.join(currentDir, current)
+    # clean up any broken symlinks in the Zeek current/ directory
+    for current in os.listdir(zeekCurrentDir):
+        currentFileSpec = os.path.join(zeekCurrentDir, current)
         if os.path.islink(currentFileSpec) and not os.path.exists(currentFileSpec):
             print('removing dead symlink "{}"'.format(currentFileSpec))
             silentRemove(currentFileSpec)
 
-    # clean up any old and empty directories in processed/ directory
+    # clean up any old and empty directories in Zeek processed/ directory
     cleanDirSeconds = min(i for i in (cleanLogSeconds, cleanZipSeconds) if i > 0)
     candidateDirs = []
-    for root, dirs, files in os.walk(processedDir, topdown=False):
+    for root, dirs, files in os.walk(zeekProcessedDir, topdown=False):
         if root and dirs:
             candidateDirs += [os.path.join(root, tmpDir) for tmpDir in dirs]
     candidateDirs = list(set(candidateDirs))
@@ -148,6 +154,13 @@ def pruneFiles():
             except OSError:
                 pass
 
+    # check the suricata logs (live and otherwise) as well
+    for surDir in [suricataDir, suricataLiveDir]:
+        for eve in os.listdir(surDir):
+            eveFile = os.path.join(surDir, eve)
+            if os.path.isfile(eveFile):
+                checkFile(eveFile, filebeatReg=fbReg, checkLogs=True, checkArchives=False)
+
 
 def main():
     with open(lockFilename, 'w') as lock_file:

hedgehog-iso/interface/sensor_ctl/control_vars.conf

@@ -96,6 +96,7 @@ export ZEEK_DISABLE_BEST_GUESS_ICS=true
 export SURICATA_CUSTOM_RULES_ONLY=false
 export SURICATA_DISABLE_ICS_ALL=false
 export SURICATA_RUNMODE=workers
+export SURICATA_LIVE_CAPTURE=true
 export SURICATA_AF_PACKET_BLOCK_SIZE=32768
 export SURICATA_AF_PACKET_BLOCK_TIMEOUT=10
 export SURICATA_AF_PACKET_BUFFER_SIZE=32768

hedgehog-iso/interface/sensor_ctl/supervisor.init/suricata_config_populate.sh

@@ -2,11 +2,6 @@
 
 if [[ -n $SUPERVISOR_PATH ]] && [[ -r /usr/local/bin/suricata_config_populate.py ]]; then
 
-    # if there's a previous eve.json file, rename it prior to starting up
-    [[ -n $ZEEK_LOG_PATH ]] && [[ -f "$ZEEK_LOG_PATH"/suricata/eve.json ]] && \
-        mv -f "$ZEEK_LOG_PATH/suricata/eve.json" \
-              "$ZEEK_LOG_PATH/suricata/eve.json.$(date -d @$(stat -c%Y "$ZEEK_LOG_PATH/suricata/eve.json") +'%Y%m%d%H%M%S')"
-
     # if there's no configuration files to modify, start with the defaults
     [[ ! -f "$SUPERVISOR_PATH"/suricata/suricata.yaml ]] && cp /etc/suricata/suricata.yaml "$SUPERVISOR_PATH"/suricata/suricata.yaml
     [[ ! -f "$SUPERVISOR_PATH"/suricata/update.yaml ]] && cp "$(dpkg -L suricata-update | grep 'update\.yaml' | head -n 1)" "$SUPERVISOR_PATH"/suricata/update.yaml

shared/bin/suricata_config_populate.py

@@ -100,6 +100,9 @@ def __call__(self, repr, data):
         'ENIP_ENABLED': True,
         'ENIP_EVE_ENABLED': False,
         'ENIP_PORTS': 44818,
+        'EVE_FILENAME_PATTERN': 'eve-%Y%m%d_%H%M%S.json',
+        'EVE_ROTATE_INTERVAL': '300s',
+        'EVE_THREADED': False,
         'EXTERNAL_NET': '!$HOME_NET',
         'FILE_DATA_PORTS': "[$HTTP_PORTS,110,143]",
         'FILES_ENABLED': True,
@@ -134,6 +137,7 @@ def __call__(self, repr, data):
         'IMAP_EVE_ENABLED': False,
         'KRB5_ENABLED': True,
         'KRB5_EVE_ENABLED': False,
+        'LIVE_CAPTURE': False,
         'MANAGED_RULES_DIR': '/var/lib/suricata/rules',
         'MAX_PENDING_PACKETS': 1024,
         'MODBUS_ENABLED': True,
@@ -719,6 +723,7 @@ def main():
         deep_set(cfg, ['vars', 'port-groups', portKey], DEFAULT_VARS[portKey])
 
     # capture parameters
+    liveCapture = val2bool(DEFAULT_VARS['LIVE_CAPTURE'])
     for cfgKey in (
         ['capture', 'disable-offloading', 'CAPTURE_DISABLE_OFFLOADING'],
         ['capture', 'checksum-validation', 'CAPTURE_CHECKSUM_VALIDATION'],
@@ -776,7 +781,13 @@ def main():
                 # enable community-id for easier cross-referencing and pcap-file for
                 # tying back to the original PCAP filename
                 cfg['outputs'][outputIdx][name]['community-id'] = True
-                cfg['outputs'][outputIdx][name]['pcap-file'] = True
+
+                # some options make sense for live capture but not PCAP processing
+                cfg['outputs'][outputIdx][name]['pcap-file'] = not liveCapture
+                if liveCapture:
+                    cfg['outputs'][outputIdx][name]['filename'] = DEFAULT_VARS['EVE_FILENAME_PATTERN']
+                    cfg['outputs'][outputIdx][name]['threaded'] = DEFAULT_VARS['EVE_THREADED']
+                    cfg['outputs'][outputIdx][name]['rotate-interval'] = DEFAULT_VARS['EVE_ROTATE_INTERVAL']
 
                 # configure the various different output types belonging to eve-log
                 if 'types' in cfg['outputs'][outputIdx][name]: