Skip to content
Permalink
Browse files

Merge branch 'maintenance' into develop

  • Loading branch information...
wagner-certat committed Sep 24, 2019
2 parents eea86f9 + e24f1da commit 27e1b3d48dd8ce25a9a8b14569239d8f7c17f92e
@@ -138,6 +138,7 @@ CHANGELOG
- Support for all proxy types without ports.
- Use Country Code of AS as `source.geolocation.cc`.
- Support for 'scanner' category.
- Handle bogus lines with missing separator.

#### Experts

@@ -168,6 +169,8 @@ CHANGELOG
- Use `statistics_*` parameters.
- Make file executable
- Handle None values in `*.temporary.*` keys and treat them as 0.
- systemd:
- Add `PIDFile` parameter to service file.

### Known issues

@@ -37,6 +37,10 @@ The deprecated parameter `feed` for collectors is again supported as the documen
#### RIPE expert
In the upgrade function for version 1.1.0 (in effect in version 2.0.1) the addition of the parameter `query_ripe_stat_ip` was not correctly done and is maybe missing. A new upgrade function re-adds it with the value of `query_ripe_stat_ip`.

#### Cymru CAP Feed Migration

The Cymru CAP Feed is (being) migrated to a new URL with a different format and more data. Look at the feed's documentation for more information.

### Libraries

### Postgres databases
@@ -12,6 +12,7 @@
[Service]
Type=$type
PIDFile=/run/intelmq/$bot.pid
ExecStart=$bot_run_cmd
User=$INTELMQ_USER
Group=$INTELMQ_GROUP
@@ -1,4 +1,6 @@
# -*- coding: utf-8 -*-
import re

from intelmq.lib import utils
from intelmq.lib.bot import ParserBot

@@ -27,6 +29,7 @@
PROTOCOL_MAPPING = {'6': 'tcp', # TODO: use getent in harmonization
'17': 'udp',
'1': 'icmp'}
BOGUS_HOSTNAME_PORT = re.compile('hostname: ([^:]+)port: ([0-9]+)')


class CymruCAPProgramParserBot(ParserBot):
@@ -214,6 +217,17 @@ def parse_line(self, line, report):

def parse_line_new(self, line, report):
category, ip, asn, timestamp, notes, asninfo = line.split('|')

# to detect bogous lines like 'hostname: sub.example.comport: 80'
bogus = BOGUS_HOSTNAME_PORT.search(notes)
if bogus:
span = bogus.span()
groups = bogus.groups()
notes = '%shostname: %s; port: %s%s' % (notes[:span[0]],
groups[0],
groups[1],
notes[span[1]:])

comment_split = list(filter(lambda x: x, notes.split(';')))
asninfo_split = asninfo.split(', ')
event = self.new_event(report)
@@ -1,11 +1,11 @@
#generated on 20190326
#category|address|asn|timestamp|notes|asninfo
#category|address|asn|timestamp|optional_information|asninfo

bot|172.16.0.21|64496|2019-03-22 11:18:52|family: Conficker;|Example AS Name, AT
bot|172.16.0.21|64496|2019-03-25 03:44:22|family: nivdort;dest_addr: 172.16.0.22; dest_port: 80;protocol: 6;|Example AS Name, AT
bruteforce|172.16.0.21|64496|2019-01-10 22:25:58|ssh;|Example AS Name, AT
controller|172.16.0.21|64496|2019-03-25 17:47:40|family: stealrat;protocol: 6;|Example AS Name, AT
controller|172.16.0.21|64496|2019-03-25 05:01:47|family: http_post;hostname: www.pfeffer.at;|Example AS Name, AT
controller|172.16.0.21|64496|2019-03-25 05:01:47|family: http_post;hostname: www.example.com;|Example AS Name, AT
darknet|172.16.0.21|64496|2019-03-25 17:24:06|darknet_port: 23, 2323;protocol: 6;|Example AS Name, AT
darknet|172.16.0.21|64496|2019-03-25 04:27:11|ports_scanned: 55756;protocol: 17;|Example AS Name, AT
honeypot|172.16.0.21|64496|2019-03-25 14:08:53|honeypot_port: 22;protocol: 6;|Example AS Name, AT
@@ -21,3 +21,4 @@ darknet|172.16.0.21|64496|2019-09-11 11:57:37|destination_port_numbers: 23;port:
darknet|172.16.0.21|64496|2019-09-11 00:49:45|destination_port_numbers: 3;port: 3;protocol: 1;|Example AS Name, AT
proxy|172.16.0.21|64496|2019-09-12 07:01:00|proxy_type: socks4;|Example AS Name, AT
scanner|172.16.0.21|64496|2019-09-17 02:58:48|port: 53912;protocol: 6;|Example AS Name, AT
controller|172.16.0.21|64496|2019-09-22 05:39:38|family: http_post;hostname: sub.example.comport: 80;|Example AS Name, AT
@@ -77,7 +77,7 @@
'source.as_name': 'Example AS Name',
'source.asn': 64496,
'source.ip': '172.16.0.21',
'source.fqdn': 'www.pfeffer.at',
'source.fqdn': 'www.example.com',
'time.source': '2019-03-25T05:01:47+00:00',
'raw': utils.base64_encode('\n'.join(RAW_LINES[:2] + [RAW_LINES[7]])),
'source.geolocation.cc': 'AT',
@@ -291,6 +291,20 @@
'protocol.transport': 'tcp',
'source.geolocation.cc': 'AT',
}
EVENT20 = {'__type': 'Event',
'time.observation': '2015-11-01T00:01:45+00:05',
'source.as_name': 'Example AS Name',
'source.asn': 64496,
'source.ip': '172.16.0.21',
'source.fqdn': 'sub.example.com',
'source.port': 80,
'time.source': '2019-09-22T05:39:38+00:00',
'raw': utils.base64_encode('\n'.join(RAW_LINES[:2] + [RAW_LINES[23]])),
'classification.identifier': 'http_post',
'malware.name': 'http_post',
'classification.type': 'c2server',
'source.geolocation.cc': 'AT',
}


class TestCymruCAPProgramParserBot(test.BotTestCase, unittest.TestCase):
@@ -328,6 +342,7 @@ def test_events(self):
self.assertMessageEqual(18, EVENT17)
self.assertMessageEqual(19, EVENT18)
self.assertMessageEqual(20, EVENT19)
self.assertMessageEqual(21, EVENT20)


if __name__ == '__main__': # pragma: no cover

0 comments on commit 27e1b3d

Please sign in to comment.
You can’t perform that action at this time.