Bug #904: store tx_id with alerts, update XFF -- v1.1

src/alert-debuglog.c

@@ -301,6 +301,13 @@ TmEcode AlertDebugLogger(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq,
                              i,
                              pa->flags & PACKET_ALERT_FLAG_STREAM_MATCH  ? "STREAM" :
                              (pa->flags & PACKET_ALERT_FLAG_STATE_MATCH ? "STATE" : "PACKET"));
+        if (pa->flags & PACKET_ALERT_FLAG_TX) {
+            MemBufferWriteString(aft->buffer,
+                    "ALERT IN TX [%02d]:    %"PRIu64"\n", i, pa->tx_id);
+        } else {
+            MemBufferWriteString(aft->buffer,
+                    "ALERT IN TX [%02d]:    N/A\n", i);
+        }
         if (p->payload_len > 0) {
             MemBufferWriteString(aft->buffer,
                                  "PAYLOAD LEN:         %" PRIu32 "\n"

src/alert-unified2-alert.c

@@ -315,61 +315,79 @@ static int Unified2Write(Unified2AlertThread *aun)
     return 1;
 }
 
-/**
- *  \brief Function to return XFF IP if any...
- *  \retval 1 if the IP has been found and returned in dstbuf
- *  \retval 0 if the IP has not being found or error
- */
-static int GetXFFIP (Packet *p, char *xff_header, char *dstbuf, int dstbuflen)
+static int GetXFFIPFromTx (Packet *p, uint64_t tx_id, char *xff_header, char *dstbuf, int dstbuflen)
 {
     uint8_t xff_chain[UNIFIED2_ALERT_XFF_CHAIN_MAXLEN];
     HtpState *htp_state = NULL;
     htp_tx_t *tx = NULL;
-    uint64_t tx_id = 0;
     uint64_t total_txs = 0;
 
     htp_state = (HtpState *)AppLayerGetProtoStateFromPacket(p);
 
     if (htp_state == NULL) {
         SCLogDebug("no http state, XFF IP cannot be retrieved");
-        goto end;
+        return 0;
     }
 
     total_txs = AppLayerGetTxCnt(ALPROTO_HTTP, htp_state);
+    if (tx_id >= total_txs)
+        return 0;
 
-    for (; tx_id < total_txs; tx_id++) {
-        tx = AppLayerGetTx(ALPROTO_HTTP, htp_state, tx_id);
+    tx = AppLayerGetTx(ALPROTO_HTTP, htp_state, tx_id);
+    if (tx == NULL) {
+        SCLogDebug("tx is NULL, XFF cannot be retrieved");
+        return 0;
+    }
 
-        if (tx == NULL) {
-            SCLogDebug("tx is NULL, XFF cannot be retrieved");
-            continue;
-        }
+    htp_header_t *h_xff = NULL;
+    if (tx->request_headers != NULL) {
+        h_xff = htp_table_get_c(tx->request_headers, xff_header);
+    }
+
+    if (h_xff != NULL && bstr_len(h_xff->value) >= UNIFIED2_ALERT_XFF_CHAIN_MINLEN &&
+            bstr_len(h_xff->value) < UNIFIED2_ALERT_XFF_CHAIN_MAXLEN) {
 
-        htp_header_t *h_xff = NULL;
-        if (tx->request_headers != NULL) {
-            h_xff = htp_table_get_c(tx->request_headers, xff_header);
+        memcpy(xff_chain, bstr_ptr(h_xff->value), bstr_len(h_xff->value));
+        xff_chain[bstr_len(h_xff->value)]=0;
+        /** Check for chained IP's separated by ", ", we will get the last one */
+        uint8_t *p_xff = memrchr(xff_chain, ' ', bstr_len(h_xff->value));
+        if (p_xff == NULL) {
+            p_xff = xff_chain;
+        } else {
+            p_xff++;
+        }
+        /** Sanity check on extracted IP for IPv4 and IPv6 */
+        uint32_t ip[4];
+        if ( inet_pton(AF_INET, (char *)p_xff, ip ) == 1 ||
+                inet_pton(AF_INET6, (char *)p_xff, ip ) == 1 ) {
+            strlcpy(dstbuf, (char *)p_xff, dstbuflen);
+            return 1; // OK
         }
+    }
+    return 0;
+}
 
-        if (h_xff != NULL && bstr_len(h_xff->value) >= UNIFIED2_ALERT_XFF_CHAIN_MINLEN &&
-                bstr_len(h_xff->value) < UNIFIED2_ALERT_XFF_CHAIN_MAXLEN) {
+/**
+ *  \brief Function to return XFF IP if any...
+ *  \retval 1 if the IP has been found and returned in dstbuf
+ *  \retval 0 if the IP has not being found or error
+ */
+static int GetXFFIP (Packet *p, char *xff_header, char *dstbuf, int dstbuflen)
+{
+    HtpState *htp_state = NULL;
+    uint64_t tx_id = 0;
+    uint64_t total_txs = 0;
 
-            memcpy(xff_chain, bstr_ptr(h_xff->value), bstr_len(h_xff->value));
-            xff_chain[bstr_len(h_xff->value)]=0;
-            /** Check for chained IP's separated by ", ", we will get the last one */
-            uint8_t *p_xff = memrchr(xff_chain, ' ', bstr_len(h_xff->value));
-            if (p_xff == NULL) {
-                p_xff = xff_chain;
-            } else {
-                p_xff++;
-            }
-            /** Sanity check on extracted IP for IPv4 and IPv6 */
-            uint32_t ip[4];
-            if ( inet_pton(AF_INET, (char *)p_xff, ip ) == 1 ||
-                    inet_pton(AF_INET6, (char *)p_xff, ip ) == 1 ) {
-                strlcpy(dstbuf, (char *)p_xff, dstbuflen);
-                return 1; // OK
-            }
-        }
+    htp_state = (HtpState *)AppLayerGetProtoStateFromPacket(p);
+    if (htp_state == NULL) {
+        SCLogDebug("no http state, XFF IP cannot be retrieved");
+        goto end;
+    }
+
+    total_txs = AppLayerGetTxCnt(ALPROTO_HTTP, htp_state);
+    for (; tx_id < total_txs; tx_id++) {
+        if (GetXFFIPFromTx(p, tx_id, xff_header, dstbuf, dstbuflen) == 1)
+            return 1;
     }
 
 end:
@@ -384,14 +402,16 @@ static int GetXFFIP (Packet *p, char *xff_header, char *dstbuf, int dstbuflen)
  */
 TmEcode Unified2Alert (ThreadVars *t, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq)
 {
+    int ret = 0;
     Unified2AlertThread *aun = (Unified2AlertThread *)data;
     aun->xff_flags = UNIFIED2_ALERT_XFF_DISABLED;
 
     if (p->alerts.cnt == 0 && !(p->flags & PKT_HAS_TAG))
         return TM_ECODE_OK;
 
-    if (!(aun->unified2alert_ctx->xff_mode & UNIFIED2_ALERT_XFF_DISABLED) &&
-            p->flow != NULL) {
+    /* overwrite mode can only work per u2 block, not per individual
+     * alert. So we'll look for an XFF record once */
+    if ((aun->unified2alert_ctx->xff_mode & UNIFIED2_ALERT_XFF_OVERWRITE) && p->flow != NULL) {
         FLOWLOCK_RDLOCK(p->flow);
 
         if (AppLayerGetProtoFromPacket(p) == ALPROTO_HTTP) {
@@ -401,43 +421,27 @@ TmEcode Unified2Alert (ThreadVars *t, Packet *p, void *data, PacketQueue *pq, Pa
                 /** Be sure that we have a nice zeroed buffer */
                 memset(aun->xff_ip, 0, 4 * sizeof(uint32_t));
 
-                if (aun->unified2alert_ctx->xff_mode & UNIFIED2_ALERT_XFF_OVERWRITE) {
-                    /** We can only have override mode if packet IP version matches
-                     * the XFF IP version, otherwise fall-back to extra data */
-                    if (inet_pton(AF_INET, buffer, aun->xff_ip) == 1) {
-                        aun->xff_flags = UNIFIED2_ALERT_XFF_IPV4;
-
-                        if (PKT_IS_IPV4(p)) {
-                            aun->xff_flags |= UNIFIED2_ALERT_XFF_OVERWRITE;
-                        } else {
-                            aun->xff_flags |= UNIFIED2_ALERT_XFF_EXTRADATA;
-                        }
-                    } else if (inet_pton(AF_INET6, buffer, aun->xff_ip) == 1) {
-                        aun->xff_flags = UNIFIED2_ALERT_XFF_IPV6;
-
-                        if (PKT_IS_IPV6(p)) {
-                            aun->xff_flags |= UNIFIED2_ALERT_XFF_OVERWRITE;
-                        } else {
-                            aun->xff_flags |= UNIFIED2_ALERT_XFF_EXTRADATA;
-                        }
+                /** We can only have override mode if packet IP version matches
+                 * the XFF IP version, otherwise fall-back to extra data */
+                if (inet_pton(AF_INET, buffer, aun->xff_ip) == 1) {
+                    SCLogDebug("valid ipv4 xff, setting flags %s", buffer);
+                    if (PKT_IS_IPV4(p)) {
+                        aun->xff_flags = (UNIFIED2_ALERT_XFF_IPV4|UNIFIED2_ALERT_XFF_OVERWRITE);
+                    } else {
+                        aun->xff_flags = (UNIFIED2_ALERT_XFF_IPV4|UNIFIED2_ALERT_XFF_EXTRADATA);
                     }
-                }
-                else if (aun->unified2alert_ctx->xff_mode & UNIFIED2_ALERT_XFF_EXTRADATA) {
-                    aun->xff_flags = UNIFIED2_ALERT_XFF_EXTRADATA;
-
-                    if (inet_pton(AF_INET, buffer, aun->xff_ip) == 1) {
-                        aun->xff_flags |= UNIFIED2_ALERT_XFF_IPV4;
-                    } else if (inet_pton(AF_INET6, buffer, aun->xff_ip) == 1) {
-                        aun->xff_flags |= UNIFIED2_ALERT_XFF_IPV6;
+                } else if (inet_pton(AF_INET6, buffer, aun->xff_ip) == 1) {
+                    if (PKT_IS_IPV6(p)) {
+                        aun->xff_flags = (UNIFIED2_ALERT_XFF_IPV6|UNIFIED2_ALERT_XFF_OVERWRITE);
+                    } else {
+                        aun->xff_flags = (UNIFIED2_ALERT_XFF_IPV6|UNIFIED2_ALERT_XFF_EXTRADATA);
                     }
                 }
             }
         }
         FLOWLOCK_UNLOCK(p->flow);
     }
 
-    int ret = 0;
-
     if (PKT_IS_IPV4(p)) {
         ret = Unified2IPv4TypeAlert (t, p, data, pq);
     } else if(PKT_IS_IPV6(p)) {
@@ -962,6 +966,31 @@ int Unified2IPv6TypeAlert (ThreadVars *t, Packet *p, void *data, PacketQueue *pq
         if (unlikely(pa->s == NULL))
             continue;
 
+        if ((aun->unified2alert_ctx->xff_mode & UNIFIED2_ALERT_XFF_EXTRADATA) && p->flow != NULL) {
+            FLOWLOCK_RDLOCK(p->flow);
+            if (AppLayerGetProtoFromPacket(p) == ALPROTO_HTTP) {
+                char buffer[UNIFIED2_ALERT_XFF_MAXLEN];
+                int have_xff_ip = 0;
+
+                if (pa->flags & PACKET_ALERT_FLAG_TX) {
+                    have_xff_ip = GetXFFIPFromTx(p, pa->tx_id, aun->unified2alert_ctx->xff_header, buffer, UNIFIED2_ALERT_XFF_MAXLEN);
+                } else {
+                    have_xff_ip = GetXFFIP(p, aun->unified2alert_ctx->xff_header, buffer, UNIFIED2_ALERT_XFF_MAXLEN);
+                }
+                if (have_xff_ip) {
+                    memset(aun->xff_ip, 0, 4 * sizeof(uint32_t));
+
+                    if (inet_pton(AF_INET, buffer, aun->xff_ip) == 1) {
+                        SCLogDebug("valid ipv4 xff, setting flag");
+                        aun->xff_flags = (UNIFIED2_ALERT_XFF_IPV4|UNIFIED2_ALERT_XFF_EXTRADATA);
+                    } else if (inet_pton(AF_INET6, buffer, aun->xff_ip) == 1) {
+                        aun->xff_flags = (UNIFIED2_ALERT_XFF_IPV6|UNIFIED2_ALERT_XFF_EXTRADATA);
+                    }
+                }
+            }
+            FLOWLOCK_UNLOCK(p->flow);
+        }
+
         /* reset length and offset */
         aun->offset = offset;
         aun->length = length;
@@ -1111,6 +1140,32 @@ int Unified2IPv4TypeAlert (ThreadVars *tv, Packet *p, void *data, PacketQueue *p
         if (unlikely(pa->s == NULL))
             continue;
 
+        if ((aun->unified2alert_ctx->xff_mode & UNIFIED2_ALERT_XFF_EXTRADATA) && p->flow != NULL) {
+            FLOWLOCK_RDLOCK(p->flow);
+            if (AppLayerGetProtoFromPacket(p) == ALPROTO_HTTP) {
+                char buffer[UNIFIED2_ALERT_XFF_MAXLEN];
+                int have_xff_ip = 0;
+
+                if (pa->flags & PACKET_ALERT_FLAG_TX) {
+                    have_xff_ip = GetXFFIPFromTx(p, pa->tx_id, aun->unified2alert_ctx->xff_header, buffer, UNIFIED2_ALERT_XFF_MAXLEN);
+                } else {
+                    have_xff_ip = GetXFFIP(p, aun->unified2alert_ctx->xff_header, buffer, UNIFIED2_ALERT_XFF_MAXLEN);
+                }
+                if (have_xff_ip) {
+                    SCLogDebug("buffer %s", buffer);
+                    memset(aun->xff_ip, 0, 4 * sizeof(uint32_t));
+
+                    if (inet_pton(AF_INET, buffer, aun->xff_ip) == 1) {
+                        SCLogDebug("valid ipv4 xff, setting flags %s", buffer);
+                        aun->xff_flags = (UNIFIED2_ALERT_XFF_IPV4|UNIFIED2_ALERT_XFF_EXTRADATA);
+                    } else if (inet_pton(AF_INET6, buffer, aun->xff_ip) == 1) {
+                        aun->xff_flags = (UNIFIED2_ALERT_XFF_IPV6|UNIFIED2_ALERT_XFF_EXTRADATA);
+                    }
+                }
+            }
+            FLOWLOCK_UNLOCK(p->flow);
+        }
+
         /* reset length and offset */
         aun->offset = offset;
         aun->length = length;

src/decode.h

@@ -248,6 +248,7 @@ typedef struct PacketAlert_ {
     uint8_t action; /* Internal num, used for sorting */
     uint8_t flags;
     struct Signature_ *s;
+    uint64_t tx_id;
 } PacketAlert;
 
 /** After processing an alert by the thresholding module, if at
@@ -258,6 +259,8 @@ typedef struct PacketAlert_ {
 #define PACKET_ALERT_FLAG_STATE_MATCH   0x02
 /** alert was generated based on stream */
 #define PACKET_ALERT_FLAG_STREAM_MATCH  0x04
+/** alert is in a tx, tx_id set */
+#define PACKET_ALERT_FLAG_TX            0x08
 
 #define PACKET_ALERT_MAX 15
 

src/detect-engine-alert.c

@@ -175,7 +175,7 @@ int PacketAlertRemove(Packet *p, uint16_t pos)
  *  \param flags alert flags
  *  \param alert_msg ptr to StreamMsg object that the signature matched on
  */
-int PacketAlertAppend(DetectEngineThreadCtx *det_ctx, Signature *s, Packet *p, uint8_t flags)
+int PacketAlertAppend(DetectEngineThreadCtx *det_ctx, Signature *s, Packet *p, uint64_t tx_id, uint8_t flags)
 {
     int i = 0;
 
@@ -192,6 +192,7 @@ int PacketAlertAppend(DetectEngineThreadCtx *det_ctx, Signature *s, Packet *p, u
         p->alerts.alerts[p->alerts.cnt].action = s->action;
         p->alerts.alerts[p->alerts.cnt].flags = flags;
         p->alerts.alerts[p->alerts.cnt].s = s;
+        p->alerts.alerts[p->alerts.cnt].tx_id = tx_id;
     } else {
         /* We need to make room for this s->num
          (a bit ugly with memcpy but we are planning changes here)*/
@@ -205,6 +206,7 @@ int PacketAlertAppend(DetectEngineThreadCtx *det_ctx, Signature *s, Packet *p, u
         p->alerts.alerts[i].action = s->action;
         p->alerts.alerts[i].flags = flags;
         p->alerts.alerts[i].s = s;
+        p->alerts.alerts[i].tx_id = tx_id;
     }
 
     /* Update the count */

src/detect-engine-alert.h

@@ -29,7 +29,7 @@
 #include "detect.h"
 
 void PacketAlertFinalize(DetectEngineCtx *, DetectEngineThreadCtx *, Packet *);
-int PacketAlertAppend(DetectEngineThreadCtx *, Signature *, Packet *, uint8_t);
+int PacketAlertAppend(DetectEngineThreadCtx *, Signature *, Packet *, uint64_t tx_id, uint8_t);
 int PacketAlertCheck(Packet *, uint32_t);
 int PacketAlertRemove(Packet *, uint16_t);
 void PacketAlertTagInit(void);

src/detect-engine-iponly.c

@@ -1084,9 +1084,9 @@ void IPOnlyMatchPacket(ThreadVars *tv,
                     }
                     if (!(s->flags & SIG_FLAG_NOALERT)) {
                         if (s->action & ACTION_DROP)
-                            PacketAlertAppend(det_ctx, s, p, PACKET_ALERT_FLAG_DROP_FLOW);
+                            PacketAlertAppend(det_ctx, s, p, 0, PACKET_ALERT_FLAG_DROP_FLOW);
                         else
-                            PacketAlertAppend(det_ctx, s, p, 0);
+                            PacketAlertAppend(det_ctx, s, p, 0, 0);
                     } else {
                         /* apply actions for noalert/rule suppressed as well */
                         PACKET_UPDATE_ACTION(p, s->action);