Backports 5.0.x V1 Batched backports of 6 issues, 1 undisclosed.

doc/userguide/performance/tuning-considerations.rst

@@ -20,10 +20,10 @@ The memory is set up at start and the usage is as follows:
 mpm-algo: <ac|hs|ac-bs|ac-ks>
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Controls the pattern matcher algorithm. AC (``Aho–Corasick ``) is the default. 
+Controls the pattern matcher algorithm. AC (``Aho–Corasick``) is the default.
 On supported platforms, :doc:`hyperscan` is the best option. On commodity 
 hardware if Hyperscan is not available the suggested setting is 
-``mpm-algo: ac-ks`` (``Aho–Corasick `` Ken Steele variant) as it performs better than 
+``mpm-algo: ac-ks`` (``Aho–Corasick`` Ken Steele variant) as it performs better than
 ``mpm-algo: ac``
 
 detect.profile: <low|medium|high|custom>

rules/smtp-events.rules

@@ -29,4 +29,5 @@ alert smtp any any -> any any (msg:"SURICATA SMTP Mime boundary length exceeded"
 
 alert smtp any any -> any any (msg:"SURICATA SMTP duplicate fields"; flow:established,to_server; app-layer-event:smtp.duplicate_fields; flowint:smtp.anomaly.count,+,1; classtype:protocol-command-decode; sid:2220018; rev:1;)
 alert smtp any any -> any any (msg:"SURICATA SMTP unparsable content"; flow:established,to_server; app-layer-event:smtp.unparsable_content; flowint:smtp.anomaly.count,+,1; classtype:protocol-command-decode; sid:2220019; rev:1;)
-# next sid 2220020
+alert smtp any any -> any any (msg:"SURICATA SMTP filename truncated"; flow:established,to_server; app-layer-event:smtp.mime_long_filename; flowint:smtp.anomaly.count,+,1; classtype:protocol-command-decode; sid:2220020; rev:1;)
+# next sid 2220021

src/app-layer-smtp.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2012 Open Information Security Foundation
+/* Copyright (C) 2007-2020 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -147,6 +147,8 @@ SCEnumCharMap smtp_decoder_event_table[ ] = {
       SMTP_DECODER_EVENT_MIME_LONG_HEADER_VALUE },
     { "MIME_LONG_BOUNDARY",
       SMTP_DECODER_EVENT_MIME_BOUNDARY_TOO_LONG },
+    { "MIME_LONG_FILENAME",
+      SMTP_DECODER_EVENT_MIME_LONG_FILENAME },
 
     /* Invalid behavior or content */
     { "DUPLICATE_FIELDS",
@@ -558,7 +560,7 @@ int SMTPProcessDataChunk(const uint8_t *chunk, uint32_t len,
  *
  * \param state The smtp state.
  *
- * \retval  0 On suceess.
+ * \retval  0 On success.
  * \retval -1 Either when we don't have any new lines to supply anymore or
  *            on failure.
  */
@@ -873,6 +875,9 @@ static void SetMimeEvents(SMTPState *state)
     if (msg->anomaly_flags & ANOM_LONG_BOUNDARY) {
         SMTPSetEvent(state, SMTP_DECODER_EVENT_MIME_BOUNDARY_TOO_LONG);
     }
+    if (msg->anomaly_flags & ANOM_LONG_FILENAME) {
+        SMTPSetEvent(state, SMTP_DECODER_EVENT_MIME_LONG_FILENAME);
+    }
 }
 
 /**
@@ -885,7 +890,7 @@ static int SMTPProcessCommandDATA(SMTPState *state, Flow *f,
     SCEnter();
 
     if (!(state->parser_state & SMTP_PARSER_STATE_COMMAND_DATA_MODE)) {
-        /* looks like are still waiting for a confirmination from the server */
+        /* looks like are still waiting for a confirmation from the server */
         return 0;
     }
 
@@ -3049,7 +3054,7 @@ static int SMTPParserTest03(void)
 }
 
 /*
- * \test Test smtp with just <LF> delimter instead of <CR><LF>.
+ * \test Test smtp with just <LF> delimiter instead of <CR><LF>.
  */
 static int SMTPParserTest04(void)
 {

src/app-layer-smtp.h

@@ -50,6 +50,7 @@ enum {
     SMTP_DECODER_EVENT_MIME_LONG_HEADER_NAME,
     SMTP_DECODER_EVENT_MIME_LONG_HEADER_VALUE,
     SMTP_DECODER_EVENT_MIME_BOUNDARY_TOO_LONG,
+    SMTP_DECODER_EVENT_MIME_LONG_FILENAME,
 
     /* Invalid behavior or content */
     SMTP_DECODER_EVENT_DUPLICATE_FIELDS,

src/decode-erspan.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2015 Open Information Security Foundation
+/* Copyright (C) 2020 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -27,7 +27,7 @@
  *
  * \author Victor Julien <victor@inliniac.net>
  *
- * Decodes ERSPAN
+ * Decodes ERSPAN Types I and II
  */
 
 #include "suricata-common.h"
@@ -40,10 +40,38 @@
 #include "util-debug.h"
 
 /**
- * \brief Function to decode ERSPAN packets
+ * \brief Functions to decode ERSPAN Type I and II packets
  */
 
-int DecodeERSPAN(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p, const uint8_t *pkt, uint32_t len, PacketQueue *pq)
+bool g_erspan_typeI_enabled = false;
+
+void DecodeERSPANConfig(void)
+{
+    int enabled = 0;
+    if (ConfGetBool("decoder.erspan.typeI.enabled", &enabled) == 1) {
+        g_erspan_typeI_enabled = (enabled == 1);
+    }
+    SCLogDebug("ERSPAN Type I decode support %s", g_erspan_typeI_enabled ? "enabled" : "disabled");
+}
+
+/**
+ * \brief ERSPAN Type I
+ */
+int DecodeERSPANTypeI(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p,
+                      const uint8_t *pkt, uint32_t len, PacketQueue *pq)
+{
+    if (unlikely(!g_erspan_typeI_enabled))
+        return TM_ECODE_FAILED;
+
+    StatsIncr(tv, dtv->counter_erspan);
+
+    return DecodeEthernet(tv, dtv, p, pkt, len, pq);
+}
+
+/**
+ * \brief ERSPAN Type II
+ */
+int DecodeERSPANTypeII(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p, const uint8_t *pkt, uint32_t len, PacketQueue *pq)
 {
     StatsIncr(tv, dtv->counter_erspan);
 

src/decode-erspan.h

@@ -34,4 +34,5 @@ typedef struct ErspanHdr_ {
     uint32_t padding;
 } __attribute__((__packed__)) ErspanHdr;
 
+void DecodeERSPANConfig(void);
 #endif /* __DECODE_ERSPAN_H__ */

src/decode-gre.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2013 Open Information Security Foundation
+/* Copyright (C) 2007-2020 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -66,7 +66,7 @@ int DecodeGRE(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p, const uint8_t *p
     {
         case GRE_VERSION_0:
 
-            /* GRE version 0 doenst support the fields below RFC 1701 */
+            /* GRE version 0 doesn't support the fields below RFC 1701 */
 
             /**
              * \todo We need to make sure this does not allow bypassing
@@ -130,7 +130,7 @@ int DecodeGRE(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p, const uint8_t *p
 
         case GRE_VERSION_1:
 
-            /* GRE version 1 doenst support the fields below RFC 1701 */
+            /* GRE version 1 doesn't support the fields below RFC 1701 */
 
             /**
              * \todo We need to make sure this does not allow bypassing
@@ -252,8 +252,16 @@ int DecodeGRE(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p, const uint8_t *p
         case ETHERNET_TYPE_ERSPAN:
         {
             if (pq != NULL) {
+                // Determine if it's Type I or Type II based on the flags in the GRE header.
+                // Type I:  0|0|0|0|0|00000|000000000|00000
+                // Type II: 0|0|0|1|0|00000|000000000|00000
+                //                Seq
                 Packet *tp = PacketTunnelPktSetup(tv, dtv, p, pkt + header_len,
-                        len - header_len, DECODE_TUNNEL_ERSPAN, pq);
+                        len - header_len,
+                        GRE_FLAG_ISSET_SQ(p->greh) == 0 ?
+                                DECODE_TUNNEL_ERSPANI :
+                                DECODE_TUNNEL_ERSPANII,
+                        pq);
                 if (tp != NULL) {
                     PKT_SET_SRC(tp, PKT_SRC_DECODER_GRE);
                     PacketEnqueue(pq,tp);

src/decode.c

@@ -28,7 +28,7 @@
  * example we have DecodeIPV4() for IPv4 and DecodePPP() for
  * PPP.
  *
- * These functions have all a pkt and and a len argument which
+ * These functions have all a pkt and a len argument which
  * are respectively a pointer to the protocol data and the length
  * of this protocol data.
  *
@@ -88,8 +88,10 @@ int DecodeTunnel(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p,
             return DecodeVLAN(tv, dtv, p, pkt, len, pq);
         case DECODE_TUNNEL_ETHERNET:
             return DecodeEthernet(tv, dtv, p, pkt, len, pq);
-        case DECODE_TUNNEL_ERSPAN:
-            return DecodeERSPAN(tv, dtv, p, pkt, len, pq);
+        case DECODE_TUNNEL_ERSPANII:
+            return DecodeERSPANTypeII(tv, dtv, p, pkt, len, pq);
+        case DECODE_TUNNEL_ERSPANI:
+            return DecodeERSPANTypeI(tv, dtv, p, pkt, len, pq);
         default:
             SCLogDebug("FIXME: DecodeTunnel: protocol %" PRIu32 " not supported.", proto);
             break;
@@ -110,7 +112,7 @@ void PacketFree(Packet *p)
  * \brief Finalize decoding of a packet
  *
  * This function needs to be call at the end of decode
- * functions when decoding has been succesful.
+ * functions when decoding has been successful.
  *
  */
 void PacketDecodeFinalize(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p)
@@ -287,7 +289,7 @@ Packet *PacketTunnelPktSetup(ThreadVars *tv, DecodeThreadVars *dtv, Packet *pare
         SCReturnPtr(NULL, "Packet");
     }
 
-    /* copy packet and set lenght, proto */
+    /* copy packet and set length, proto */
     PacketCopyData(p, pkt, len);
     p->recursion_level = parent->recursion_level + 1;
     p->ts.tv_sec = parent->ts.tv_sec;
@@ -384,7 +386,7 @@ Packet *PacketDefragPktSetup(Packet *parent, const uint8_t *pkt, uint32_t len, u
 
 /**
  *  \brief inform defrag "parent" that a pseudo packet is
- *         now assosiated to it.
+ *         now associated to it.
  */
 void PacketDefragPktSetupParent(Packet *parent)
 {
@@ -646,7 +648,7 @@ void DecodeThreadVarsFree(ThreadVars *tv, DecodeThreadVars *dtv)
 }
 
 /**
- * \brief Set data for Packet and set length when zeo copy is used
+ * \brief Set data for Packet and set length when zero copy is used
  *
  *  \param Pointer to the Packet to modify
  *  \param Pointer to the data
@@ -730,6 +732,7 @@ void DecodeGlobalConfig(void)
 {
     DecodeTeredoConfig();
     DecodeVXLANConfig();
+    DecodeERSPANConfig();
 }
 
 /**

src/decode.h

@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2013 Open Information Security Foundation
+/* Copyright (C) 2007-2020 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -161,7 +161,7 @@ typedef struct Address_ {
         (a)->addr_data32[3] = 0; \
     } while (0)
 
-/* Set the IPv6 addressesinto the Addrs of the Packet.
+/* Set the IPv6 addresses into the Addrs of the Packet.
  * Make sure p->ip6h is initialized and validated. */
 #define SET_IPV6_SRC_ADDR(p, a) do {                    \
         (a)->family = AF_INET6;                         \
@@ -386,7 +386,7 @@ typedef struct PktProfiling_ {
 
 #endif /* PROFILING */
 
-/* forward declartion since Packet struct definition requires this */
+/* forward declaration since Packet struct definition requires this */
 struct PacketQueue_;
 
 /* sizes of the members:
@@ -889,7 +889,8 @@ void CaptureStatsSetup(ThreadVars *tv, CaptureStats *s);
 
 enum DecodeTunnelProto {
     DECODE_TUNNEL_ETHERNET,
-    DECODE_TUNNEL_ERSPAN,
+    DECODE_TUNNEL_ERSPANII,
+    DECODE_TUNNEL_ERSPANI,
     DECODE_TUNNEL_VLAN,
     DECODE_TUNNEL_IPV4,
     DECODE_TUNNEL_IPV6,
@@ -943,6 +944,8 @@ int DecodeVLAN(ThreadVars *, DecodeThreadVars *, Packet *, const uint8_t *, uint
 int DecodeVXLAN(ThreadVars *, DecodeThreadVars *, Packet *, const uint8_t *, uint32_t, PacketQueue *);
 int DecodeMPLS(ThreadVars *, DecodeThreadVars *, Packet *, const uint8_t *, uint32_t, PacketQueue *);
 int DecodeERSPAN(ThreadVars *, DecodeThreadVars *, Packet *, const uint8_t *, uint32_t, PacketQueue *);
+int DecodeERSPANTypeII(ThreadVars *, DecodeThreadVars *, Packet *, const uint8_t *, uint32_t, PacketQueue *);
+int DecodeERSPANTypeI(ThreadVars *, DecodeThreadVars *, Packet *, const uint8_t *, uint32_t, PacketQueue *);
 int DecodeTEMPLATE(ThreadVars *, DecodeThreadVars *, Packet *, const uint8_t *, uint32_t, PacketQueue *);
 
 #ifdef UNITTESTS
@@ -1089,7 +1092,7 @@ void DecodeUnregisterCounters(void);
 #define PKT_ALLOC                       (1<<3)      /**< Packet was alloc'd this run, needs to be freed */
 #define PKT_HAS_TAG                     (1<<4)      /**< Packet has matched a tag */
 #define PKT_STREAM_ADD                  (1<<5)      /**< Packet payload was added to reassembled stream */
-#define PKT_STREAM_EST                  (1<<6)      /**< Packet is part of establised stream */
+#define PKT_STREAM_EST                  (1<<6)      /**< Packet is part of established stream */
 #define PKT_STREAM_EOF                  (1<<7)      /**< Stream is in eof state */
 #define PKT_HAS_FLOW                    (1<<8)
 #define PKT_PSEUDO_STREAM_END           (1<<9)      /**< Pseudo packet to end the stream */

src/detect-engine-address.c

@@ -529,9 +529,6 @@ int DetectAddressParseString(DetectAddress *dd, const char *str)
     int r = 0;
     char ipstr[256];
 
-    while (*str != '\0' && *str == ' ')
-        str++;
-
     /* shouldn't see 'any' here */
     BUG_ON(strcasecmp(str, "any") == 0);
 
@@ -738,6 +735,9 @@ static int DetectAddressSetup(DetectAddressHead *gh, const char *s)
 {
     SCLogDebug("gh %p, s %s", gh, s);
 
+    while (*s != '\0' && isspace(*s))
+        s++;
+
     if (strcasecmp(s, "any") == 0) {
         SCLogDebug("adding 0.0.0.0/0 and ::/0 as we\'re handling \'any\'");
 

src/detect-filestore.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2012 Open Information Security Foundation
+/* Copyright (C) 2007-2020 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -344,6 +344,9 @@ static int DetectFilestoreSetup (DetectEngineCtx *de_ctx, Signature *s, const ch
     sm->type = DETECT_FILESTORE;
 
     if (str != NULL && strlen(str) > 0) {
+        char str_0[32];
+        char str_1[32];
+        char str_2[32];
         SCLogDebug("str %s", str);
 
         ret = pcre_exec(parse_regex, parse_regex_study, str, strlen(str), 0, 0, ov, MAX_SUBSTRINGS);
@@ -353,29 +356,28 @@ static int DetectFilestoreSetup (DetectEngineCtx *de_ctx, Signature *s, const ch
         }
 
         if (ret > 1) {
-            const char *str_ptr;
-            res = pcre_get_substring((char *)str, ov, MAX_SUBSTRINGS, 1, &str_ptr);
+            res = pcre_copy_substring((char *)str, ov, MAX_SUBSTRINGS, 1, str_0, sizeof(str_0));
             if (res < 0) {
-                SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed");
+                SCLogError(SC_ERR_PCRE_COPY_SUBSTRING, "pcre_copy_substring failed");
                 goto error;
             }
-            args[0] = (char *)str_ptr;
+            args[0] = (char *)str_0;
 
             if (ret > 2) {
-                res = pcre_get_substring((char *)str, ov, MAX_SUBSTRINGS, 2, &str_ptr);
+                res = pcre_copy_substring((char *)str, ov, MAX_SUBSTRINGS, 2, str_1, sizeof(str_1));
                 if (res < 0) {
-                    SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed");
+                    SCLogError(SC_ERR_PCRE_COPY_SUBSTRING, "pcre_copy_substring failed");
                     goto error;
                 }
-                args[1] = (char *)str_ptr;
+                args[1] = (char *)str_1;
             }
             if (ret > 3) {
-                res = pcre_get_substring((char *)str, ov, MAX_SUBSTRINGS, 3, &str_ptr);
+                res = pcre_copy_substring((char *)str, ov, MAX_SUBSTRINGS, 3, str_2, sizeof(str_2));
                 if (res < 0) {
-                    SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed");
+                    SCLogError(SC_ERR_PCRE_COPY_SUBSTRING, "pcre_copy_substring failed");
                     goto error;
                 }
-                args[2] = (char *)str_ptr;
+                args[2] = (char *)str_2;
             }
         }