Permalink
Browse files

Merge branch 'maintenance' into develop

  • Loading branch information...
wagner-certat committed Oct 10, 2018
2 parents 5114bee + c95ff56 commit 2915e9fbdb07010e8b3b91d3529e2558daa050c2
View
@@ -75,7 +75,10 @@ CHANGELOG
- `intelmq.bots.parsers.cymru.parser_cap_program`:
- Add support for new format (extra data about botnet of 'bots').
- Handle AS number 0.
- `intelmq.bots.parsers.shadowserver.config`: Spam URL reports: remove `src_naics`, `src_sic` columns.
- `intelmq.bots.parsers.shadowserver`:
- Spam URL reports: remove `src_naics`, `src_sic` columns.
- fix parsing of 'spam' events in ShadowServer's 'Botnet Drone Hadoop' Report (#1271).
- Add support in parser to ignore some columns in config file by using `False` as intelmq key.
#### Experts
View
@@ -30,6 +30,13 @@ The bot `intelmq.bots.experts.ripencc_abuse_contact.expert` has been renamed to
### Libraries
### Postgres databases
The following statements optionally update existing data.
Please check if you did use these feed names and eventually adapt them for your setup!
```SQL
UPDATE events
SET "classification.taxonomy" = 'abusive content', "classification.type" = 'spam', "classification.identifier" = 'spam', "malware.name" = NULL, "source.fqdn" = "source.reverse_dns", "source.reverse_dns" = NULL, "source.url" = "destination.url", "destination.url" = NULL
WHERE "malware.name" = 'spam' AND "feed.name" = 'Drone';
```
### MongoDB databases
In previous version the MongoDB Output Bot saved the fields `time.observation` and `time.source` as strings in ISO format. But MongoDB does support saving datetime objects directly which are converted to its native date format, enabling certain optimizations and features. The MongoDB Output Bot now saves these values as datetime objects.
@@ -1095,6 +1095,51 @@ def convert_date(value):
# classification.identifier will be set to (harmonized) malware name by modify expert
},
}
drone_spam = {
'required_fields': [
('time.source', 'timestamp', add_UTC_to_timestamp),
('source.ip', 'ip'),
('source.port', 'port'),
],
'optional_fields': [
('source.asn', 'asn'),
('source.geolocation.cc', 'geo'),
('source.geolocation.region', 'region'),
('source.geolocation.city', 'city'),
('source.fqdn', 'hostname'),
('protocol.transport', 'type'),
(False, 'infection'), # is just 'spam'
('source.url', 'url', convert_http_host_and_url, True),
('user_agent', 'agent'),
('destination.ip', 'cc_ip', validate_ip),
('destination.port', 'cc_port'),
('destination.asn', 'cc_asn'),
('destination.geolocation.cc', 'cc_geo'),
('destination.fqdn', 'cc_dns', validate_fqdn),
('connection_count', 'count', convert_int),
('extra.', 'proxy', convert_bool),
('protocol.application', 'application'),
('os.name', 'p0f_genre'),
('os.version', 'p0f_detail'),
('extra.', 'machine_name', validate_to_none),
('extra.', 'id', validate_to_none),
('extra.', 'naics', invalidate_zero),
('extra.', 'sic', invalidate_zero),
('extra.destination.naics', 'cc_naics', invalidate_zero),
('extra.destination.sic', 'cc_sic', invalidate_zero),
('extra.destination.sector', 'cc_sector', validate_to_none),
('extra.', 'sector', validate_to_none),
('extra.', 'ssl_cipher', validate_to_none),
('extra.', 'family', validate_to_none),
('extra.', 'tag', validate_to_none),
('extra.', 'public_source', validate_to_none),
],
'constant_fields': {
'classification.taxonomy': 'abusive content',
'classification.type': 'spam',
'classification.identifier': 'spam',
},
}
# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-XDMCP
open_xdmcp = {
@@ -43,6 +43,10 @@ def parse_line(self, row, report):
conf = self.sparser_config
# https://github.com/certtools/intelmq/issues/1271
if conf is config.drone and row.get('infection') == 'spam':
conf = config.drone_spam
# we need to copy here...
fields = copy.copy(self.csv_fieldnames)
# We will use this variable later.
@@ -133,10 +137,14 @@ def parse_line(self, row, report):
extra[shadowkey] = value
fields.remove(shadowkey)
continue
elif intelmqkey.startswith('extra.'):
elif intelmqkey and intelmqkey.startswith('extra.'):
extra[intelmqkey.replace('extra.', '', 1)] = value
fields.remove(shadowkey)
continue
elif intelmqkey is False:
# ignore it explicitly
fields.remove(shadowkey)
continue
try:
event.add(intelmqkey, value)
fields.remove(shadowkey)
@@ -7,3 +7,4 @@
"2011-04-23 00:00:28","124.190.16.11",4095,1221,"AU","VICTORIA","MELBOURNE",,"tcp","sinkhole",,,"87.106.24.200",443,8560,"DE",,,,,"Windows","2000 SP4, XP SP1+",,,0,0,0,0,"Communications","Communications",,,,
"2011-04-23 00:00:29","124.182.36.33",60837,1221,"AU","WESTERN AUSTRALIA","PERTH",,"tcp","sinkhole",,,"87.106.24.200",443,8560,"DE",,1,,,"Windows","XP/2000 (RFC1323+, w+, tstamp+)",,,0,0,517510,737415,,"Communications",,,,
"2011-04-23 00:00:33","116.212.205.74",23321,9822,"AU","WESTERN AUSTRALIA","PERTH",,"tcp","sinkhole",,,"74.208.164.166",80,8560,"US",,1,,,"Windows","XP SP1+, 2000 SP3 (2)",,,541690,874899,517510,737415,,"Communications",,,,
"2018-08-14 02:13:36","192.0.2.15",,65548,"AT","BURGENLAND","EISENSTADT","www.example.com","tcp","spam","https://www.example.com/foobar",,,,,,,,,,,,,,0,0,,,,,,,"spam",
@@ -224,6 +224,25 @@
'source.port': 23321,
'time.observation': '2015-01-01T00:00:00+00:00',
'time.source': '2011-04-23T00:00:33+00:00'},
{'__type': 'Event',
'feed.name': 'ShadowServer Drone',
'classification.taxonomy': 'abusive content',
'classification.type': 'spam',
'classification.identifier': 'spam',
'protocol.transport': 'tcp',
'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0],
EXAMPLE_LINES[9]])),
'source.asn': 65548,
'source.fqdn': 'www.example.com',
'source.geolocation.cc': 'AT',
'source.geolocation.city': 'EISENSTADT',
'source.geolocation.region': 'BURGENLAND',
'source.ip': '192.0.2.15',
'source.url': 'https://www.example.com/foobar',
'time.observation': '2015-01-01T00:00:00+00:00',
'time.source': '2018-08-14T02:13:36+00:00',
'extra.tag': 'spam',
},
]

0 comments on commit 2915e9f

Please sign in to comment.