pcap-log: seed ring buffer on start up

On start, look for existing pcap log files and add them to
the ring buffer. This makes pcap-log self maintaining over
restarts removing the need for external tools to clear
orphaned files.

configure.ac

@@ -132,6 +132,7 @@
     AC_CHECK_HEADERS([sys/ioctl.h linux/if_ether.h linux/if_packet.h linux/filter.h])
     AC_CHECK_HEADERS([linux/ethtool.h linux/sockios.h])
     AC_CHECK_HEADER(glob.h,,[AC_ERROR(glob.h not found ...)])
+    AC_CHECK_HEADERS([dirent.h fnmatch.h])
 
     AC_CHECK_HEADERS([sys/socket.h net/if.h sys/mman.h linux/if_arp.h], [], [],
     [[#ifdef HAVE_SYS_SOCKET_H

src/log-pcap.c

@@ -26,6 +26,13 @@
  */
 
 #include "suricata-common.h"
+
+#if defined(HAVE_DIRENT_H) && defined(HAVE_FNMATCH_H)
+#define INIT_RING_BUFFER
+#include <dirent.h>
+#include <fnmatch.h>
+#endif
+
 #include "debug.h"
 #include "detect.h"
 #include "flow.h"
@@ -80,6 +87,14 @@ SC_ATOMIC_DECLARE(uint32_t, thread_cnt);
 typedef struct PcapFileName_ {
     char *filename;
     char *dirname;
+
+    /* Like a struct timeval, but with fixed size. This is only used when
+     * seeding the ring buffer on start. */
+    struct {
+        uint64_t secs;
+        uint32_t usecs;
+    };
+
     TAILQ_ENTRY(PcapFileName_) next; /**< Pointer to next Pcap File for tailq. */
 } PcapFileName;
 
@@ -138,6 +153,11 @@ typedef struct PcapLogThreadData_ {
     PcapLogData *pcap_log;
 } PcapLogThreadData;
 
+/* Pattern for extracting timestamp from pcap log files. */
+static const char timestamp_pattern[] = ".*?(\\d+)(\\.(\\d+))?";
+static pcre *pcre_timestamp_code = NULL;
+static pcre_extra *pcre_timestamp_extra = NULL;
+
 /* global pcap data for when we're using multi mode. At exit we'll
  * merge counters into this one and then report counters. */
 static PcapLogData *g_pcap_data = NULL;
@@ -475,6 +495,194 @@ static PcapLogData *PcapLogDataCopy(const PcapLogData *pl)
     return copy;
 }
 
+#ifdef INIT_RING_BUFFER
+static int PcapLogGetTimeOfFile(const char *filename, uint64_t *secs,
+    uint32_t *usecs)
+{
+    int pcre_ovecsize = 4 * 3;
+    int pcre_ovec[pcre_ovecsize];
+    char buf[PATH_MAX];
+
+    int n = pcre_exec(pcre_timestamp_code, pcre_timestamp_extra,
+        filename, strlen(filename), 0, 0, pcre_ovec,
+        pcre_ovecsize);
+    if (n != 2 && n != 4) {
+        /* No match. */
+        return 0;
+    }
+
+    if (n >= 2) {
+        /* Extract seconds. */
+        if (pcre_copy_substring(filename, pcre_ovec, pcre_ovecsize,
+                1, buf, sizeof(buf)) < 0) {
+            return 0;
+        }
+        if (ByteExtractStringUint64(secs, 10, 0, buf) < 0) {
+            return 0;
+        }
+    }
+    if (n == 4) {
+        /* Extract microseconds. */
+        if (pcre_copy_substring(filename, pcre_ovec, pcre_ovecsize,
+                3, buf, sizeof(buf)) < 0) {
+            return 0;
+        }
+        if (ByteExtractStringUint32(usecs, 10, 0, buf) < 0) {
+            return 0;
+        }
+    }
+
+    return 1;
+}
+
+static TmEcode PcapLogInitRingBuffer(PcapLogData *pl)
+{
+    char pattern[PATH_MAX];
+
+    SCLogInfo("Initializing PCAP ring buffer for %s/%s.",
+        pl->dir, pl->prefix);
+
+    strlcpy(pattern, pl->dir, PATH_MAX);
+    if (pattern[strlen(pattern) - 1] != '/') {
+        strlcat(pattern, "/", PATH_MAX);
+    }
+    if (pl->mode == LOGMODE_MULTI) {
+        for (int i = 0; i < pl->filename_part_cnt; i++) {
+            char *part = pl->filename_parts[i];
+            if (part == NULL || strlen(part) == 0) {
+                continue;
+            }
+            if (part[0] != '%' || strlen(part) < 2) {
+                strlcat(pattern, part, PATH_MAX);
+                continue;
+            }
+            switch (part[1]) {
+                case 'i':
+                    SCLogError(SC_ERR_INVALID_ARGUMENT,
+                        "Thread ID not allowed inring buffer mode.");
+                    return TM_ECODE_FAILED;
+                case 'n': {
+                    char tmp[PATH_MAX];
+                    snprintf(tmp, PATH_MAX, "%"PRIu32, pl->thread_number);
+                    strlcat(pattern, tmp, PATH_MAX);
+                    break;
+                }
+                case 't':
+                    strlcat(pattern, "*", PATH_MAX);
+                    break;
+                default:
+                    SCLogError(SC_ERR_INVALID_ARGUMENT,
+                        "Unsupported format character: %%%s", part);
+                    return TM_ECODE_FAILED;
+            }
+        }
+    } else {
+        strlcat(pattern, pl->prefix, PATH_MAX);
+        strlcat(pattern, ".*", PATH_MAX);
+    }
+
+    char *basename = strrchr(pattern, '/');
+    *basename++ = '\0';
+
+    /* Pattern is now just the directory name. */
+    DIR *dir = opendir(pattern);
+    if (dir == NULL) {
+        SCLogWarning(SC_ERR_DIR_OPEN, "Failed to open directory %s: %s",
+            pattern, strerror(errno));
+        return TM_ECODE_FAILED;
+    }
+
+    for (;;) {
+        struct dirent *entry = readdir(dir);
+        if (entry == NULL) {
+            break;
+        }
+        if (fnmatch(basename, entry->d_name, 0) != 0) {
+            continue;
+        }
+
+        uint64_t secs = 0;
+        uint32_t usecs = 0;
+
+        if (!PcapLogGetTimeOfFile(entry->d_name, &secs, &usecs)) {
+            /* Failed to get time stamp out of file name. Not necessarily a
+             * failure as the file might just not be a pcap log file. */
+            continue;
+        }
+
+        PcapFileName *pf = SCCalloc(sizeof(*pf), 1);
+        if (unlikely(pf == NULL)) {
+            return TM_ECODE_FAILED;
+        }
+        char path[PATH_MAX];
+        snprintf(path, PATH_MAX - 1, "%s/%s", pattern, entry->d_name);
+        if ((pf->filename = SCStrdup(path)) == NULL) {
+            goto fail;
+        }
+        if ((pf->dirname = SCStrdup(pattern)) == NULL) {
+            goto fail;
+        }
+        pf->secs = secs;
+        pf->usecs = usecs;
+
+        if (TAILQ_EMPTY(&pl->pcap_file_list)) {
+            TAILQ_INSERT_TAIL(&pl->pcap_file_list, pf, next);
+        } else {
+            /* Ordered insert. */
+            PcapFileName *it = TAILQ_FIRST(&pl->pcap_file_list);
+            TAILQ_FOREACH(it, &pl->pcap_file_list, next) {
+                if (pf->secs < it->secs) {
+                    break;
+                } else if (pf->secs == it->secs && pf->usecs < it->usecs) {
+                    break;
+                }
+            }
+            if (it == NULL) {
+                TAILQ_INSERT_TAIL(&pl->pcap_file_list, pf, next);
+            } else {
+                TAILQ_INSERT_BEFORE(it, pf, next);
+            }
+        }
+        pl->file_cnt++;
+        continue;
+
+    fail:
+        if (pf != NULL) {
+            if (pf->filename != NULL) {
+                SCFree(pf->filename);
+            }
+            if (pf->dirname != NULL) {
+                SCFree(pf->dirname);
+            }
+            SCFree(pf);
+        }
+        break;
+    }
+
+    if (pl->file_cnt > pl->max_files) {
+        PcapFileName *pf = TAILQ_FIRST(&pl->pcap_file_list);
+        while (pf != NULL && pl->file_cnt > pl->max_files) {
+            SCLogDebug("Removing PCAP file %s", pf->filename);
+            if (remove(pf->filename) != 0) {
+                SCLogWarning(SC_WARN_REMOVE_FILE,
+                    "Failed to remove PCAP file %s: %s", pf->filename,
+                    strerror(errno));
+            }
+            TAILQ_REMOVE(&pl->pcap_file_list, pf, next);
+            pf = TAILQ_FIRST(&pl->pcap_file_list);
+            pl->file_cnt--;
+        }
+    }
+
+    closedir(dir);
+
+    /* For some reason file count is initialized at one, instead of 0. */
+    SCLogNotice("Ring buffer initialized with %d files.", pl->file_cnt - 1);
+
+    return TM_ECODE_OK;
+}
+#endif /* INIT_RING_BUFFER */
+
 static TmEcode PcapLogDataInit(ThreadVars *t, void *initdata, void **data)
 {
     if (initdata == NULL) {
@@ -500,7 +708,9 @@ static TmEcode PcapLogDataInit(ThreadVars *t, void *initdata, void **data)
     td->pcap_log->pkt_cnt = 0;
     td->pcap_log->pcap_dead_handle = NULL;
     td->pcap_log->pcap_dumper = NULL;
-    td->pcap_log->file_cnt = 1;
+    if (td->pcap_log->file_cnt < 1) {
+        td->pcap_log->file_cnt = 1;
+    }
 
     struct timeval ts;
     memset(&ts, 0x00, sizeof(struct timeval));
@@ -518,6 +728,16 @@ static TmEcode PcapLogDataInit(ThreadVars *t, void *initdata, void **data)
 
     *data = (void *)td;
 
+    if (pl->max_files && (pl->mode == LOGMODE_MULTI || pl->threads == 1)) {
+#ifdef INIT_RING_BUFFER
+        if (PcapLogInitRingBuffer(td->pcap_log) == TM_ECODE_FAILED) {
+            return TM_ECODE_FAILED;
+        }
+#else
+        SCLogInfo("Unable to initialize ring buffer on this platform.");
+#endif /* INIT_RING_BUFFER */
+    }
+
     return TM_ECODE_OK;
 }
 
@@ -701,6 +921,9 @@ static int ParseFilename(PcapLogData *pl, const char *filename)
  * */
 static OutputCtx *PcapLogInitCtx(ConfNode *conf)
 {
+    const char *pcre_errbuf;
+    int pcre_erroffset;
+
     PcapLogData *pl = SCMalloc(sizeof(PcapLogData));
     if (unlikely(pl == NULL)) {
         SCLogError(SC_ERR_MEM_ALLOC, "Failed to allocate Memory for PcapLogData");
@@ -727,6 +950,19 @@ static OutputCtx *PcapLogInitCtx(ConfNode *conf)
 
     SCMutexInit(&pl->plog_lock, NULL);
 
+    /* Initialize PCREs. */
+    pcre_timestamp_code = pcre_compile(timestamp_pattern, 0, &pcre_errbuf,
+        &pcre_erroffset, NULL);
+    if (pcre_timestamp_code == NULL) {
+        FatalError(SC_ERR_PCRE_COMPILE,
+            "Failed to compile \"%s\" at offset %"PRIu32": %s",
+            timestamp_pattern, pcre_erroffset, pcre_errbuf);
+    }
+    pcre_timestamp_extra = pcre_study(pcre_timestamp_code, 0, &pcre_errbuf);
+    if (pcre_timestamp_extra == NULL) {
+        FatalError(SC_ERR_PCRE_STUDY, "Fail to study pcre: %s", pcre_errbuf);
+    }
+
     /* conf params */
 
     const char *filename = NULL;

src/util-error.c

@@ -331,6 +331,8 @@ const char * SCErrorToString(SCError err)
         CASE_CODE (SC_ERR_NO_SHA1_SUPPORT);
         CASE_CODE (SC_ERR_NO_SHA256_SUPPORT);
         CASE_CODE (SC_ERR_DNP3_CONFIG);
+        CASE_CODE (SC_ERR_DIR_OPEN);
+        CASE_CODE(SC_WARN_REMOVE_FILE);
     }
 
     return "UNKNOWN_ERROR";

src/util-error.h

@@ -321,6 +321,8 @@ typedef enum {
     SC_ERR_NO_SHA256_SUPPORT,
     SC_ERR_ENIP_CONFIG,
     SC_ERR_DNP3_CONFIG,
+    SC_ERR_DIR_OPEN,
+    SC_WARN_REMOVE_FILE,
 } SCError;
 
 const char *SCErrorToString(SCError);