Next/20190124/v4

src/Makefile.am

@@ -435,6 +435,7 @@ util-fmemopen.c util-fmemopen.h \
 util-hash.c util-hash.h \
 util-hashlist.c util-hashlist.h \
 util-hash-lookup3.c util-hash-lookup3.h \
+util-hash-string.c util-hash-string.h \
 util-host-os-info.c util-host-os-info.h \
 util-host-info.c util-host-info.h \
 util-hyperscan.c util-hyperscan.h \

src/counters.c

@@ -100,6 +100,7 @@ static char stats_enabled = TRUE;
 
 /**< add decoder events as stats? enabled by default */
 bool stats_decoder_events = true;
+const char *stats_decoder_events_prefix = "decoder";
 /**< add stream events as stats? disabled by default */
 bool stats_stream_events = false;
 
@@ -224,7 +225,7 @@ static ConfNode *GetConfig(void) {
 /**
  * \brief Initializes stats context
  */
-static void StatsInitCtx(void)
+static void StatsInitCtxPreOutput(void)
 {
     SCEnter();
 #ifdef AFLFUZZ_DISABLE_MGTTHREADS
@@ -252,6 +253,31 @@ static void StatsInitCtx(void)
         if (ret) {
             stats_stream_events = (b == 1);
         }
+
+        const char *prefix = NULL;
+        if (ConfGet("stats.decoder-events-prefix", &prefix) != 1) {
+            prefix = "decoder";
+            SCLogWarning(SC_WARN_DEFAULT_WILL_CHANGE, "in 5.0 the default "
+                    "for decoder event stats will go from "
+                    "'decoder.<proto>.<event>' to 'decoder.event.<proto>.<event>'. "
+                    "See ticket #2225. To suppress this message, "
+                    "set stats.decoder-events-prefix in the yaml.");
+        }
+        stats_decoder_events_prefix = prefix;
+    }
+    SCReturn;
+}
+
+static void StatsInitCtxPostOutput(void)
+{
+    SCEnter();
+    /* Store the engine start time */
+    time(&stats_start_time);
+
+    /* init the lock used by StatsThreadStore */
+    if (SCMutexInit(&stats_ctx->sts_lock, NULL) != 0) {
+        SCLogError(SC_ERR_INITIALIZATION, "error initializing sts mutex");
+        exit(EXIT_FAILURE);
     }
 
     if (!OutputStatsLoggersRegistered()) {
@@ -266,15 +292,6 @@ static void StatsInitCtx(void)
         }
     }
 
-    /* Store the engine start time */
-    time(&stats_start_time);
-
-    /* init the lock used by StatsThreadStore */
-    if (SCMutexInit(&stats_ctx->sts_lock, NULL) != 0) {
-        SCLogError(SC_ERR_INITIALIZATION, "error initializing sts mutex");
-        exit(EXIT_FAILURE);
-    }
-
     SCReturn;
 }
 
@@ -848,11 +865,17 @@ void StatsInit(void)
     StatsPublicThreadContextInit(&stats_ctx->global_counter_ctx);
 }
 
-void StatsSetupPostConfig(void)
+void StatsSetupPostConfigPreOutput(void)
 {
-    StatsInitCtx();
+    StatsInitCtxPreOutput();
 }
 
+void StatsSetupPostConfigPostOutput(void)
+{
+    StatsInitCtxPostOutput();
+}
+
+
 /**
  * \brief Spawns the wakeup, and the management thread used by the stats api
  *

src/counters.h

@@ -105,7 +105,8 @@ typedef struct StatsPrivateThreadContext_ {
 
 /* the initialization functions */
 void StatsInit(void);
-void StatsSetupPostConfig(void);
+void StatsSetupPostConfigPreOutput(void);
+void StatsSetupPostConfigPostOutput(void);
 void StatsSpawnThreads(void);
 void StatsRegisterTests(void);
 

src/decode-ipv6.c

@@ -631,6 +631,9 @@ int DecodeIPV6(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p, uint8_t *pkt, u
         case IPPROTO_IPV6:
             DecodeIP6inIP6(tv, dtv, p, pkt + IPV6_HEADER_LEN, IPV6_GET_PLEN(p), pq);
             return TM_ECODE_OK;
+        case IPPROTO_GRE:
+            DecodeGRE(tv, dtv, p, pkt + IPV6_HEADER_LEN, IPV6_GET_PLEN(p), pq);
+            break;
         case IPPROTO_FRAGMENT:
         case IPPROTO_HOPOPTS:
         case IPPROTO_ROUTING:

src/decode.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2014 Open Information Security Foundation
+/* Copyright (C) 2007-2019 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -63,11 +63,12 @@
 #include "util-profiling.h"
 #include "pkt-var.h"
 #include "util-mpm-ac.h"
-
+#include "util-hash-string.h"
 #include "output.h"
 #include "output-flow.h"
 
 extern bool stats_decoder_events;
+const char *stats_decoder_events_prefix;
 extern bool stats_stream_events;
 
 int DecodeTunnel(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p,
@@ -125,7 +126,7 @@ void PacketUpdateEngineEventCounters(ThreadVars *tv,
 
         if (e <= DECODE_EVENT_PACKET_MAX && !stats_decoder_events)
             continue;
-        if (e > DECODE_EVENT_PACKET_MAX && !stats_stream_events)
+        else if (e > DECODE_EVENT_PACKET_MAX && !stats_stream_events)
             continue;
         StatsIncr(tv, dtv->counter_engine_events[e]);
     }
@@ -413,6 +414,20 @@ void PacketBypassCallback(Packet *p)
     }
 }
 
+/* counter name store */
+static HashTable *g_counter_table = NULL;
+static SCMutex g_counter_table_mutex = SCMUTEX_INITIALIZER;
+
+void DecodeUnregisterCounters(void)
+{
+    SCMutexLock(&g_counter_table_mutex);
+    if (g_counter_table) {
+        HashTableFree(g_counter_table);
+        g_counter_table = NULL;
+    }
+    SCMutexUnlock(&g_counter_table_mutex);
+}
+
 void DecodeRegisterPerfCounters(DecodeThreadVars *dtv, ThreadVars *tv)
 {
     /* register counters */
@@ -470,11 +485,41 @@ void DecodeRegisterPerfCounters(DecodeThreadVars *dtv, ThreadVars *tv)
 
         if (i <= DECODE_EVENT_PACKET_MAX && !stats_decoder_events)
             continue;
-        if (i > DECODE_EVENT_PACKET_MAX && !stats_stream_events)
+        else if (i > DECODE_EVENT_PACKET_MAX && !stats_stream_events)
             continue;
 
-        dtv->counter_engine_events[i] = StatsRegisterCounter(
-                DEvents[i].event_name, tv);
+        if (i < DECODE_EVENT_PACKET_MAX &&
+                strncmp(DEvents[i].event_name, "decoder.", 8) == 0)
+        {
+            SCMutexLock(&g_counter_table_mutex);
+            if (g_counter_table == NULL) {
+                g_counter_table = HashTableInit(256, StringHashFunc,
+                        StringHashCompareFunc,
+                        StringHashFreeFunc);
+                BUG_ON(g_counter_table == NULL);
+            }
+
+            char name[256];
+            char *dot = index(DEvents[i].event_name, '.');
+            BUG_ON(!dot);
+            snprintf(name, sizeof(name), "%s.%s",
+                    stats_decoder_events_prefix, dot+1);
+
+            const char *found = HashTableLookup(g_counter_table, name, 0);
+            if (!found) {
+                char *add = SCStrdup(name);
+                BUG_ON(!add);
+                HashTableAdd(g_counter_table, add, 0);
+                found = add;
+            }
+            dtv->counter_engine_events[i] = StatsRegisterCounter(
+                    found, tv);
+
+            SCMutexUnlock(&g_counter_table_mutex);
+        } else {
+            dtv->counter_engine_events[i] = StatsRegisterCounter(
+                    DEvents[i].event_name, tv);
+        }
     }
 
     return;

src/decode.h

@@ -971,6 +971,7 @@ int DecoderParseDataFromFile(char *filename, DecoderFunc Decoder);
 int DecoderParseDataFromFileSerie(char *fileprefix, DecoderFunc Decoder);
 #endif
 void DecodeGlobalConfig(void);
+void DecodeUnregisterCounters(void);
 
 /** \brief Set the No payload inspection Flag for the packet.
  *

src/detect-metadata.c

@@ -31,7 +31,7 @@
 #include "detect-parse.h"
 #include "detect-engine.h"
 #include "detect-metadata.h"
-
+#include "util-hash-string.h"
 #include "util-unittest.h"
 
 static int DetectMetadataSetup (DetectEngineCtx *, Signature *, const char *);
@@ -60,38 +60,6 @@ void DetectMetadataFree(DetectMetadata *mdata)
     SCReturn;
 }
 
-/* djb2 string hashing */
-static uint32_t StringHashFunc(HashTable *ht, void *data, uint16_t datalen)
-{
-    uint32_t hash = 5381;
-    int c;
-
-    while ((c = *(char *)data++))
-        hash = ((hash << 5) + hash) + c; /* hash * 33 + c */
-
-    hash = hash % ht->array_size;
-
-    return hash;
-}
-
-static char StringHashCompareFunc(void *data1, uint16_t datalen1,
-                               void *data2, uint16_t datalen2)
-{
-    int len1 = strlen((char *)data1);
-    int len2 = strlen((char *)data2);
-
-    if (len1 == len2 && memcmp(data1, data2, len1) == 0) {
-        return 1;
-    }
-
-    return 0;
-}
-
-static void StringHashFreeFunc(void *data)
-{
-    SCFree(data);
-}
-
 int DetectMetadataHashInit(DetectEngineCtx *de_ctx)
 {
     if (! DetectEngineMustParseMetadata())

src/detect.c

@@ -1039,7 +1039,9 @@ static void DetectRunCleanup(DetectEngineThreadCtx *det_ctx,
 
     if (pflow != NULL) {
         /* update inspected tracker for raw reassembly */
-        if (p->proto == IPPROTO_TCP && pflow->protoctx != NULL) {
+        if (p->proto == IPPROTO_TCP && pflow->protoctx != NULL &&
+            (p->flags & PKT_STREAM_EST))
+        {
             StreamReassembleRawUpdateProgress(pflow->protoctx, p,
                     det_ctx->raw_stream_progress);
 

src/output-json-stats.c

@@ -52,6 +52,9 @@
 
 #ifdef HAVE_LIBJANSSON
 
+extern bool stats_decoder_events;
+const char *stats_decoder_events_prefix;
+
 /**
  * specify which engine info will be printed in stats log.
  * ALL means both last reload and ruleset stats.
@@ -387,6 +390,14 @@ static OutputInitResult OutputStatsLogInit(ConfNode *conf)
         return result;
     }
 
+    if (stats_decoder_events &&
+            strcmp(stats_decoder_events_prefix, "decoder") == 0) {
+        SCLogWarning(SC_WARN_EVE_MISSING_EVENTS, "json stats will not display "
+                "all decoder events correctly. See #2225. Set a prefix in "
+                "stats.decoder-events-prefix. In 5.0 the prefix will default "
+                "to 'decoder.event'.");
+    }
+
     if (SCConfLogOpenGeneric(conf, file_ctx, DEFAULT_LOG_FILENAME, 1) < 0) {
         LogFileFreeCtx(file_ctx);
         return result;
@@ -449,6 +460,14 @@ static OutputInitResult OutputStatsLogInitSub(ConfNode *conf, OutputCtx *parent_
     if (unlikely(stats_ctx == NULL))
         return result;
 
+    if (stats_decoder_events &&
+            strcmp(stats_decoder_events_prefix, "decoder") == 0) {
+        SCLogWarning(SC_WARN_EVE_MISSING_EVENTS, "eve.stats will not display "
+                "all decoder events correctly. See #2225. Set a prefix in "
+                "stats.decoder-events-prefix. In 5.0 the prefix will default "
+                "to 'decoder.event'.");
+    }
+
     stats_ctx->flags = JSON_STATS_TOTALS;
 
     if (conf != NULL) {

src/source-pcap.c

@@ -94,6 +94,7 @@ TmEcode ReceivePcapThreadInit(ThreadVars *, const void *, void **);
 void ReceivePcapThreadExitStats(ThreadVars *, void *);
 TmEcode ReceivePcapThreadDeinit(ThreadVars *, void *);
 TmEcode ReceivePcapLoop(ThreadVars *tv, void *data, void *slot);
+TmEcode ReceivePcapBreakLoop(ThreadVars *tv, void *data);
 
 TmEcode DecodePcapThreadInit(ThreadVars *, const void *, void **);
 TmEcode DecodePcapThreadDeinit(ThreadVars *tv, void *data);
@@ -113,7 +114,7 @@ void TmModuleReceivePcapRegister (void)
     tmm_modules[TMM_RECEIVEPCAP].ThreadInit = ReceivePcapThreadInit;
     tmm_modules[TMM_RECEIVEPCAP].Func = NULL;
     tmm_modules[TMM_RECEIVEPCAP].PktAcqLoop = ReceivePcapLoop;
-    tmm_modules[TMM_RECEIVEPCAP].PktAcqBreakLoop = NULL;
+    tmm_modules[TMM_RECEIVEPCAP].PktAcqBreakLoop = ReceivePcapBreakLoop;
     tmm_modules[TMM_RECEIVEPCAP].ThreadExitPrintStats = ReceivePcapThreadExitStats;
     tmm_modules[TMM_RECEIVEPCAP].ThreadDeinit = NULL;
     tmm_modules[TMM_RECEIVEPCAP].RegisterTests = NULL;
@@ -298,6 +299,20 @@ TmEcode ReceivePcapLoop(ThreadVars *tv, void *data, void *slot)
     SCReturnInt(TM_ECODE_OK);
 }
 
+/**
+ * \brief PCAP Break Loop function.
+ */
+TmEcode ReceivePcapBreakLoop(ThreadVars *tv, void *data)
+{
+    SCEnter();
+    PcapThreadVars *ptv = (PcapThreadVars *)data;
+    if (ptv->pcap_handle == NULL) {
+        SCReturnInt(TM_ECODE_FAILED);
+    }
+    pcap_breakloop(ptv->pcap_handle);
+    SCReturnInt(TM_ECODE_OK);
+}
+
 /**
  * \brief Init function for ReceivePcap.
  *

src/suricata.c

@@ -2289,8 +2289,9 @@ void PreRunPostPrivsDropInit(const int runmode)
     if (runmode == RUNMODE_UNIX_SOCKET)
         return;
 
+    StatsSetupPostConfigPreOutput();
     RunModeInitializeOutputs();
-    StatsSetupPostConfig();
+    StatsSetupPostConfigPostOutput();
 }
 
 /* clean up / shutdown code for both the main modes and for
@@ -2327,6 +2328,7 @@ void PostRunDeinit(const int runmode, struct timeval *start_time)
     /* mgt and ppt threads killed, we can run non thread-safe
      * shutdown functions */
     StatsReleaseResources();
+    DecodeUnregisterCounters();
     RunModeShutDown();
     FlowShutdown();
     IPPairShutdown();

src/util-error.c

@@ -357,6 +357,8 @@ const char * SCErrorToString(SCError err)
         CASE_CODE (SC_ERR_WINDIVERT_INVALID_FILTER);
         CASE_CODE (SC_ERR_WINDIVERT_TOOLONG_FILTER);
         CASE_CODE (SC_WARN_RUST_NOT_AVAILABLE);
+        CASE_CODE (SC_WARN_DEFAULT_WILL_CHANGE);
+        CASE_CODE (SC_WARN_EVE_MISSING_EVENTS);
 
         CASE_CODE (SC_ERR_MAX);
     }

src/util-error.h

@@ -347,6 +347,8 @@ typedef enum {
     SC_ERR_WINDIVERT_INVALID_FILTER,
     SC_ERR_WINDIVERT_TOOLONG_FILTER,
     SC_WARN_RUST_NOT_AVAILABLE,
+    SC_WARN_DEFAULT_WILL_CHANGE,
+    SC_WARN_EVE_MISSING_EVENTS,
 
     SC_ERR_MAX,
 } SCError;