Merge branch 'maintenance' into develop

certtools · Oct 10, 2018 · 2915e9f · 2915e9f
2 parents 5114bee + c95ff56
commit 2915e9f
Show file tree

Hide file tree

Showing 6 changed files with 85 additions and 2 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -75,7 +75,10 @@ CHANGELOG
 - `intelmq.bots.parsers.cymru.parser_cap_program`:
   - Add support for new format (extra data about botnet of 'bots').
   - Handle AS number 0.
-- `intelmq.bots.parsers.shadowserver.config`: Spam URL reports: remove `src_naics`, `src_sic` columns.
+- `intelmq.bots.parsers.shadowserver`:
+  - Spam URL reports: remove `src_naics`, `src_sic` columns.
+  - fix parsing of 'spam' events in ShadowServer's 'Botnet Drone Hadoop' Report (#1271).
+  - Add support in parser to ignore some columns in config file by using `False` as intelmq key.
 
 #### Experts
 

diff --git a/NEWS.md b/NEWS.md
@@ -30,6 +30,13 @@ The bot `intelmq.bots.experts.ripencc_abuse_contact.expert` has been renamed to
 ### Libraries
 
 ### Postgres databases
+The following statements optionally update existing data.
+Please check if you did use these feed names and eventually adapt them for your setup!
+```SQL
+UPDATE events
+   SET "classification.taxonomy" = 'abusive content', "classification.type" = 'spam', "classification.identifier" = 'spam', "malware.name" = NULL, "source.fqdn" = "source.reverse_dns", "source.reverse_dns" = NULL, "source.url" = "destination.url", "destination.url" = NULL
+   WHERE "malware.name" = 'spam' AND "feed.name" = 'Drone';
+```
 
 ### MongoDB databases
 In previous version the MongoDB Output Bot saved the fields `time.observation` and `time.source` as strings in ISO format. But MongoDB does support saving datetime objects directly which are converted to its native date format, enabling certain optimizations and features. The MongoDB Output Bot now saves these values as datetime objects.

diff --git a/intelmq/bots/parsers/shadowserver/config.py b/intelmq/bots/parsers/shadowserver/config.py
@@ -1095,6 +1095,51 @@ def convert_date(value):
         # classification.identifier will be set to (harmonized) malware name by modify expert
     },
 }
+drone_spam = {
+    'required_fields': [
+        ('time.source', 'timestamp', add_UTC_to_timestamp),
+        ('source.ip', 'ip'),
+        ('source.port', 'port'),
+    ],
+    'optional_fields': [
+        ('source.asn', 'asn'),
+        ('source.geolocation.cc', 'geo'),
+        ('source.geolocation.region', 'region'),
+        ('source.geolocation.city', 'city'),
+        ('source.fqdn', 'hostname'),
+        ('protocol.transport', 'type'),
+        (False, 'infection'),  # is just 'spam'
+        ('source.url', 'url', convert_http_host_and_url, True),
+        ('user_agent', 'agent'),
+        ('destination.ip', 'cc_ip', validate_ip),
+        ('destination.port', 'cc_port'),
+        ('destination.asn', 'cc_asn'),
+        ('destination.geolocation.cc', 'cc_geo'),
+        ('destination.fqdn', 'cc_dns', validate_fqdn),
+        ('connection_count', 'count', convert_int),
+        ('extra.', 'proxy', convert_bool),
+        ('protocol.application', 'application'),
+        ('os.name', 'p0f_genre'),
+        ('os.version', 'p0f_detail'),
+        ('extra.', 'machine_name', validate_to_none),
+        ('extra.', 'id', validate_to_none),
+        ('extra.', 'naics', invalidate_zero),
+        ('extra.', 'sic', invalidate_zero),
+        ('extra.destination.naics', 'cc_naics', invalidate_zero),
+        ('extra.destination.sic', 'cc_sic', invalidate_zero),
+        ('extra.destination.sector', 'cc_sector', validate_to_none),
+        ('extra.', 'sector', validate_to_none),
+        ('extra.', 'ssl_cipher', validate_to_none),
+        ('extra.', 'family', validate_to_none),
+        ('extra.', 'tag', validate_to_none),
+        ('extra.', 'public_source', validate_to_none),
+    ],
+    'constant_fields': {
+        'classification.taxonomy': 'abusive content',
+        'classification.type': 'spam',
+        'classification.identifier': 'spam',
+    },
+}
 
 # https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-XDMCP
 open_xdmcp = {

diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py
@@ -43,6 +43,10 @@ def parse_line(self, row, report):
 
         conf = self.sparser_config
 
+        # https://github.com/certtools/intelmq/issues/1271
+        if conf is config.drone and row.get('infection') == 'spam':
+            conf = config.drone_spam
+
         # we need to copy here...
         fields = copy.copy(self.csv_fieldnames)
         # We will use this variable later.
@@ -133,10 +137,14 @@ def parse_line(self, row, report):
                     extra[shadowkey] = value
                     fields.remove(shadowkey)
                     continue
-                elif intelmqkey.startswith('extra.'):
+                elif intelmqkey and intelmqkey.startswith('extra.'):
                     extra[intelmqkey.replace('extra.', '', 1)] = value
                     fields.remove(shadowkey)
                     continue
+                elif intelmqkey is False:
+                    # ignore it explicitly
+                    fields.remove(shadowkey)
+                    continue
                 try:
                     event.add(intelmqkey, value)
                     fields.remove(shadowkey)

diff --git a/intelmq/tests/bots/parsers/shadowserver/drone-hadoop.csv b/intelmq/tests/bots/parsers/shadowserver/drone-hadoop.csv
@@ -7,3 +7,4 @@
 "2011-04-23 00:00:28","124.190.16.11",4095,1221,"AU","VICTORIA","MELBOURNE",,"tcp","sinkhole",,,"87.106.24.200",443,8560,"DE",,,,,"Windows","2000 SP4, XP SP1+",,,0,0,0,0,"Communications","Communications",,,,
 "2011-04-23 00:00:29","124.182.36.33",60837,1221,"AU","WESTERN AUSTRALIA","PERTH",,"tcp","sinkhole",,,"87.106.24.200",443,8560,"DE",,1,,,"Windows","XP/2000 (RFC1323+, w+, tstamp+)",,,0,0,517510,737415,,"Communications",,,,
 "2011-04-23 00:00:33","116.212.205.74",23321,9822,"AU","WESTERN AUSTRALIA","PERTH",,"tcp","sinkhole",,,"74.208.164.166",80,8560,"US",,1,,,"Windows","XP SP1+, 2000 SP3 (2)",,,541690,874899,517510,737415,,"Communications",,,,
+"2018-08-14 02:13:36","192.0.2.15",,65548,"AT","BURGENLAND","EISENSTADT","www.example.com","tcp","spam","https://www.example.com/foobar",,,,,,,,,,,,,,0,0,,,,,,,"spam",
diff --git a/intelmq/tests/bots/parsers/shadowserver/test_drone_hadoop.py b/intelmq/tests/bots/parsers/shadowserver/test_drone_hadoop.py
@@ -224,6 +224,25 @@
            'source.port': 23321,
            'time.observation': '2015-01-01T00:00:00+00:00',
            'time.source': '2011-04-23T00:00:33+00:00'},
+          {'__type': 'Event',
+           'feed.name': 'ShadowServer Drone',
+           'classification.taxonomy': 'abusive content',
+           'classification.type': 'spam',
+           'classification.identifier': 'spam',
+           'protocol.transport': 'tcp',
+           'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0],
+                                                 EXAMPLE_LINES[9]])),
+           'source.asn': 65548,
+           'source.fqdn': 'www.example.com',
+           'source.geolocation.cc': 'AT',
+           'source.geolocation.city': 'EISENSTADT',
+           'source.geolocation.region': 'BURGENLAND',
+           'source.ip': '192.0.2.15',
+           'source.url': 'https://www.example.com/foobar',
+           'time.observation': '2015-01-01T00:00:00+00:00',
+           'time.source': '2018-08-14T02:13:36+00:00',
+           'extra.tag': 'spam',
+           },
           ]