Normalization Effort: Part 1: Remove string ip fields **WIP**

examples/demo/sampleData2MozDef.py

@@ -145,7 +145,6 @@ def makeEvents():
                     if 'details' not in event.keys():
                         event['details'] = dict()
                     event['details']['sourceipaddress'] = randomIP
-                    event['details']['sourceipv4address'] = randomIP
 
                 #print(event['timestamp'], event['tags'], event['summary'])
 
@@ -209,7 +208,6 @@ def makeAlerts():
                     if 'details' not in event.keys():
                         event['details'] = dict()
                     event['details']['sourceipaddress'] = randomIP
-                    event['details']['sourceipv4address'] = randomIP
 
                 if 'duplicate' in event.keys():
                     # send this event multiple times to trigger an alert
@@ -278,7 +276,6 @@ def makeAttackers():
                     if 'details' not in event.keys():
                         event['details'] = dict()
                     event['details']['sourceipaddress'] = randomIP
-                    event['details']['sourceipv4address'] = randomIP
 
                 if 'duplicate' in event.keys():
                     # send this event multiple times to trigger an alert

examples/demo/sampleevents/alertcreating-bro-intel.json

@@ -18,7 +18,6 @@
       "seenindicatortype": "HTTP::IN_X_FORWARDED_FOR_HEADER",
       "destinationport": 443,
       "sourceport": 58969,
-      "destinationipv4address": "0.0.82.27",
       "severity": "NOTICE"
     }
   }

examples/demo/sampleevents/events-logins-failure.json

@@ -9,7 +9,6 @@
     "details": {
       "dn": "john@example.com,o=com,dc=example",
       "success": false,
-      "sourceipv4address": "10.20.70.200",
       "result": "LDAP_INVALID_CREDENTIALS",
       "srcip": "10.20.0.200",
       "sourceipaddress": "10.20.0.200"
@@ -25,7 +24,6 @@
     "details": {
       "dn": "bob@example.com,o=com,dc=example",    
       "success": false,
-      "sourceipv4address": "10.20.70.200",
       "result": "LDAP_INVALID_CREDENTIALS",
       "srcip": "10.20.0.200",
       "sourceipaddress": "10.20.0.200"
@@ -41,7 +39,6 @@
     "details": {
       "dn": "mary@example.com,o=com,dc=example",
       "success": false,
-      "sourceipv4address": "10.20.70.200",
       "result": "LDAP_INVALID_CREDENTIALS",
       "srcip": "10.20.0.200",
       "sourceipaddress": "10.20.0.200"
@@ -57,7 +54,6 @@
     "details": {
       "dn": "sue@example.com,o=com,dc=example",
       "success": false,
-      "sourceipv4address": "10.20.70.200",
       "result": "LDAP_INVALID_CREDENTIALS",
       "srcip": "10.20.0.200",
       "sourceipaddress": "10.20.0.200"

examples/demo/sampleevents/events-network.json

@@ -7,7 +7,6 @@
     "file": "networklogs",
     "details": {
       "protocol": "6",
-      "sourceipv4address": "10.2.2.59",
       "payload": "",
       "sourceipaddress": "10.2.2.59",
       "service": "junos-https",
@@ -17,8 +16,7 @@
       "policy": "any--any",
       "destinationnatrule": "None",
       "destinationipaddress": "63.245.215.25",
-      "destinationzone": "external",
-      "destinationipv4address": "63.245.215.25"
+      "destinationzone": "external"
     }
 },
 {

examples/es-docs/bro_intel.json

@@ -18,12 +18,10 @@
       "ts": "1405546326.853474",
       "seenindicator": "0.0.139.213",
       "sources": "CIF - need-to-know",
-      "sourceipv4address": "0.0.82.208",
       "seenindicatortype": "HTTP::IN_X_FORWARDED_FOR_HEADER",
       "destinationport": 443,
       "sourceport": 58969,
       "sourceipaddress": "0.0.82.208",
-      "destinationipv4address": "0.0.82.27",
       "severity": "NOTICE"
     }
   },
@@ -46,12 +44,10 @@
       "seenindicator": "0.0.139.213",
       "ts": "1405546326.853474",
       "sources": "CIF - need-to-know",
-      "sourceipv4address": "0.0.82.208",
       "seenindicatortype": "HTTP::IN_X_FORWARDED_FOR_HEADER",
       "destinationport": 443,
       "sourceport": 13711,
       "sourceipaddress": "0.0.82.208",
-      "destinationipv4address": "0.0.82.28",
       "severity": "NOTICE"
     }
   },
@@ -74,12 +70,10 @@
       "seenindicator": "0.0.139.213",
       "ts": "1405546326.853474",
       "sources": "CIF - need-to-know",
-      "sourceipv4address": "0.0.82.208",
       "seenindicatortype": "HTTP::IN_X_FORWARDED_FOR_HEADER",
       "destinationport": 443,
       "sourceport": 13711,
       "sourceipaddress": "0.0.82.208",
-      "destinationipv4address": "0.0.82.28",
       "severity": "NOTICE"
     }
   },
@@ -102,12 +96,10 @@
       "seenindicator": "0.0.139.213",
       "ts": "1405546326.853474",
       "sources": "CIF - need-to-know",
-      "sourceipv4address": "0.0.82.208",
       "seenindicatortype": "HTTP::IN_X_FORWARDED_FOR_HEADER",
       "destinationport": 443,
       "sourceport": 13711,
       "sourceipaddress": "0.0.82.208",
-      "destinationipv4address": "0.0.82.28",
       "severity": "NOTICE"
     }
   },
@@ -130,12 +122,10 @@
       "seenindicator": "0.0.139.213",
       "ts": "1405546326.853474",
       "sources": "CIF - need-to-know",
-      "sourceipv4address": "0.0.82.208",
       "seenindicatortype": "HTTP::IN_X_FORWARDED_FOR_HEADER",
       "destinationport": 443,
       "sourceport": 13711,
       "sourceipaddress": "0.0.82.208",
-      "destinationipv4address": "0.0.82.28",
       "severity": "NOTICE"
     }
   },
@@ -158,12 +148,10 @@
       "seenindicator": "0.0.139.213",
       "ts": "1405546326.853474",
       "sources": "CIF - need-to-know",
-      "sourceipv4address": "0.0.82.208",
       "seenindicatortype": "HTTP::IN_X_FORWARDED_FOR_HEADER",
       "destinationport": 443,
       "sourceport": 13711,
       "sourceipaddress": "0.0.82.208",
-      "destinationipv4address": "0.0.82.28",
       "severity": "NOTICE"
     }
   }

examples/es-docs/bruteforce_ssh.json

@@ -11,7 +11,6 @@
     "eventsource": "systemslogs",
     "details": {
       "processid": "",
-      "sourceipv4address": "0.0.72.113",
       "timestamp": "Jul 17 14:00:13",
       "hostname": "example.com",
       "program": "sshd",
@@ -46,7 +45,6 @@
     "eventsource": "systemslogs",
     "details": {
       "processid": "",
-      "sourceipv4address": "0.0.72.113",
       "timestamp": "Jul 17 14:00:10",
       "hostname": "example.com",
       "program": "sshd",
@@ -81,7 +79,6 @@
     "eventsource": "systemslogs",
     "details": {
       "processid": "",
-      "sourceipv4address": "0.0.72.113",
       "timestamp": "Jul 17 14:00:07",
       "hostname": "example.com",
       "program": "sshd",

examples/es-docs/fail2ban.json

@@ -11,7 +11,6 @@
     "eventsource": "systemslogs",
     "details": {
       "processid": "",
-      "sourceipv4address": "0.0.141.210",
       "timestamp": "Jul 17 15:57:16",
       "hostname": "pbx1",
       "program": "fail2ban",

mq/plugins/broFixup.py

@@ -386,31 +386,26 @@ def onMessage(self, message, metadata):
                             newmessage[u'details'][u'indicators'].append(newmessage[u'details'][u'src'])
                             # If details.src is present overwrite the source IP address with it
                             newmessage[u'details'][u'sourceipaddress'] = newmessage[u'details'][u'src']
-                            newmessage[u'details'][u'sourceipv4address'] = newmessage[u'details'][u'src']
                         if isIPv6(newmessage[u'details'][u'src']):
                             newmessage[u'details'][u'indicators'].append(newmessage[u'details'][u'src'])
                             # If details.src is present overwrite the source IP address with it
-                            newmessage[u'details'][u'sourceipv6address'] = newmessage[u'details'][u'src']
+                            newmessage[u'details'][u'sourceipaddress'] = newmessage[u'details'][u'src']
                         del newmessage[u'details'][u'src']
                     sumstruct = {}
                     sumstruct['note'] = newmessage['details'][u'note']
-                    if 'sourceipv6address' in newmessage['details']:
-                        sumstruct['src'] = newmessage['details']['sourceipv6address']
+                    if 'sourceipaddress' in newmessage['details']:
+                        sumstruct['src'] = newmessage['details']['sourceipaddress']
                     else:
-                        if 'sourceipv4address' in newmessage['details']:
-                            sumstruct['src'] = newmessage['details']['sourceipv4address']
-                        else:
-                            sumstruct['src'] = u'unknown'
+                        sumstruct['src'] = u'0.0.0.0'
                     if 'dst' in newmessage['details']:
                         sumstruct['dst'] = newmessage['details']['dst']
                         del(newmessage[u'details'][u'dst'])
                         if isIPv4(sumstruct[u'dst']):
                             newmessage['details'][u'destinationipaddress'] = sumstruct['dst']
-                            newmessage['details'][u'destinationipv4address'] = sumstruct['dst']
                         if isIPv6(sumstruct[u'dst']):
-                            newmessage['details'][u'destinationipv6address'] = sumstruct['dst']
+                            newmessage['details'][u'destinationipaddress'] = sumstruct['dst']
                     else:
-                        sumstruct['dst'] = u'unknown'
+                        sumstruct['dst'] = u'0.0.0.0'
                     if 'p' in newmessage['details']:
                         sumstruct['p'] = newmessage['details']['p']
                     else:

mq/plugins/fluentdSqsFixup.py

@@ -66,21 +66,18 @@ def onMessage(self, message, metadata):
 
         # host is used to store dns-style-ip entries in AWS, for ex
         # ip-10-162-8-26 is 10.162.8.26. obviously there is no strong guarantee
-        # that this is always trusted. It's better than nothing though. At the
-        # time of writing, there is  no ipv6 support AWS-side for this kind of
-        # field. It may be overridden later by a better field, if any exists
+        # that this is always trusted. It's better than nothing though. As of
+        # 2018, AWS does not provide ipv6 DNS style hostnames for ipv6 instances.
+        # It may be overridden later by a better field, if this changes.
         if 'host' in message.keys():
             tmp = message['host']
             if tmp.startswith('ip-'):
                 ipText = tmp.split('ip-')[1].replace('-', '.')
                 if isIPv4(ipText):
                     if 'destinationipaddress' not in message.keys():
                         message['details']['destinationipaddress'] = ipText
-                    if 'destinationipv4address' not in message.keys():
-                        message['details']['destinationipv4address'] = ipText
                 else:
                     message['details']['destinationipaddress'] = '0.0.0.0'
-                    message['details']['destinationipv4address'] = '0.0.0.0'
                     addError(message,
                              'plugin: {0} error: {1}:{2}'.format(
                                  'fluentSqsFixUp.py',

mq/plugins/ipFixup.py

@@ -45,15 +45,9 @@ def onMessage(self, message, metadata):
           ipv6 in an ipv4 field
           ipv4 in another field
           '-' or other invalid ip in the ip field
-        Also sets ipv4 in two fields:
-            ipaddress (decimal mapping IP)
-            ipv4address (string mapping)
-            Elastic search is inconsistent about returning IPs as
-            decimal or IPs.
-            In a query an IP field is returned as string.
-            In a facets an IP field is returned as decimal.
-            No ES field type exists for ipv6, so always having
-            a string version is the most flexible option.
+        Sets two ip fields since ES now supports ipv6:
+            sourceipaddress (decimal mapping IP)
+            destinationipaddress (decimal mapping IP)
         '''
 
         if 'details' in message.keys():
@@ -64,67 +58,49 @@ def onMessage(self, message, metadata):
                 ipText = message['details']['http_x_forwarded_for'].split(',')[0]
                 if isIPv4(ipText) and 'sourceipaddress' not in message['details'].keys():
                     message['details']['sourceipaddress'] = ipText
-                if isIPv4(ipText) and 'sourceipv4address' not in message['details'].keys():
-                    message['details']['sourceipv4address'] = ipText
-                if isIPv6(ipText) and 'sourceipv6address' not in message['details'].keys():
-                    message['details']['sourceipv6address'] = ipText
+                if isIPv6(ipText) and 'sourceipaddress' not in message['details'].keys():
+                    message['details']['sourceipaddress'] = ipText
 
             if 'sourceipaddress' in message['details'].keys():
                 ipText = message['details']['sourceipaddress']
-                if isIPv6(ipText):
-                    message['details']['sourceipv6address'] = ipText
-                    message['details']['sourceipaddress'] = '0.0.0.0'
-                    addError(message, 'plugin: {0} error: {1}'.format('ipFixUp.py', 'sourceipaddress is ipv6, moved'))
-                elif isIPv4(ipText):
-                    message['details']['sourceipv4address'] = ipText
-                else:
+                if not (isIPv6(ipText) or isIPv4(ipText)):
                     # Smells like a hostname, let's save it as source field
                     message['details']['source'] = message['details']['sourceipaddress']
-                    message['details']['sourceipaddress'] = None
+                    message['details']['sourceipaddress'] = '0.0.0.0'
 
             if 'destinationipaddress' in message['details'].keys():
                 ipText = message['details']['destinationipaddress']
-                if isIPv6(ipText):
-                    message['details']['destinationipv6address'] = ipText
-                    message['details']['destinationipaddress'] = '0.0.0.0'
-                    addError(message, 'plugin: {0} error: {1}'.format('ipFixUp.py', 'destinationipaddress is ipv6, moved'))
-                elif isIPv4(ipText):
-                    message['details']['destinationipv4address'] = ipText
-                else:
+                if not (isIPv6(ipText) or isIPv4(ipText)):
                     # Smells like a hostname, let's save it as destination field
                     message['details']['destination'] = message['details']['destinationipaddress']
-                    message['details']['destinationipaddress'] = None
+                    message['details']['destinationipaddress'] = '0.0.0.0'
 
             if 'src' in message['details'].keys():
                 ipText = message['details']['src']
                 if isIPv4(ipText):
                     message['details']['sourceipaddress'] = ipText
-                    message['details']['sourceipv4address'] = ipText
                 if isIPv6(ipText):
-                    message['details']['sourceipv6address'] = ipText
+                    message['details']['sourceipaddress'] = ipText
 
             if 'srcip' in message['details'].keys():
                 ipText = message['details']['srcip']
                 if isIPv4(ipText):
                     message['details']['sourceipaddress'] = ipText
-                    message['details']['sourceipv4address'] = ipText
                 if isIPv6(ipText):
-                    message['details']['sourceipv6address'] = ipText
+                    message['details']['sourceipaddress'] = ipText
             if 'dst' in message['details'].keys():
                 ipText = message['details']['dst']
                 if isIPv4(ipText):
                     message['details']['destinationipaddress'] = ipText
-                    message['details']['destinationipv4address'] = ipText
                 if isIPv6(ipText):
-                    message['details']['destinationipv6address'] = ipText
+                    message['details']['destinationipaddress'] = ipText
 
             if 'dstip' in message['details'].keys():
                 ipText = message['details']['dstip']
                 if isIPv4(ipText):
                     message['details']['destinationipaddress'] = ipText
-                    message['details']['destinationipv4address'] = ipText
                 if isIPv6(ipText):
-                    message['details']['destinationipv6address'] = ipText
+                    message['details']['destinationipaddress'] = ipText
 
             if 'cluster_client_ip' in message['details'].keys():
                 ipText = message['details']['cluster_client_ip']