Permalink
Browse files

Merge branch 'maintenance' into develop

  • Loading branch information...
wagner-certat committed Oct 23, 2018
2 parents f3526ff + b10af8f commit 2ff1fe7b3d7e56a6608956f65f39222cb55c920d
@@ -67,11 +67,15 @@ CHANGELOG
- `lib.bot.py`:
- `ParserBot`'s method `recover_line_csv` now also handles given `tempdata`.
- `Bot.acknowledge_message()` deletes `__current_message` to free the memory, saves memory in idling parsers with big reports.
- `process()`: Warn once per run if `error_dump_message` is set to false.
- `lib/message.py`:
- Fix add('extra', ..., overwrite=True): old extra fields have not been deleted previously (#1335).
- Do not ignore empty or ignored (as defined in `_IGNORED_VALUES`) values of `extra.*` fields for backwards compatibility (#1335).
- `lib/pipeline.py` (`Redis.receive`): Wait in 1s steps if redis is busy loading its snapshot from disk (#1334).
### Default configuration
- Set `error_dump_message` to true by default.
### Development
- `bin/rewrite_config_files.py`: Fix ordering of BOTS file (#1327).
@@ -106,6 +110,7 @@ CHANGELOG
- fix parsing of 'spam' events in ShadowServer's 'Botnet Drone Hadoop' Report (#1271).
- Add support in parser to ignore some columns in config file by using `False` as intelmq key.
- Add support for the `Outdated-DNSSEC-Key` and `Outdated-DNSSEC-Key-IPv6` feeds.
- Add support for the `Accessible-Rsync` feed.
- `intelmq.bots.parsers.generic.parser_csv`: If the `skip_header` parameter was set to `True`, the header was not part of the `raw` field as returned by the `recover_line` method. The header is now saved and handled correctly by the fixed recovery method.
#### Experts
@@ -26,6 +26,7 @@ The bot `intelmq.bots.experts.ripencc_abuse_contact.expert` has been renamed to
### Harmonization
### Configuration
In 1.1.0 the default value for the parameter `error_dump_message` was set to `false`. The recommended value, used in previous and future release` is `true` to not loose any data in case of errors. Users are advised to check the values configured in their `defaults.conf` file.
### Libraries
@@ -26,6 +26,7 @@ Possible feednames:
* `Accessible-CWMP`
* `Accessible-Hadoop`
* `Accessible-RDP`
* `Accessible-Rsync`
* `Accessible-SMB`
* `Accessible-Telnet`
* `Accessible-VNC`
@@ -5,7 +5,7 @@
"destination_pipeline_host": "127.0.0.1",
"destination_pipeline_password": null,
"destination_pipeline_port": 6379,
"error_dump_message": false,
"error_dump_message": true,
"error_log_exception": true,
"error_log_message": false,
"error_max_retries": 3,
@@ -225,17 +225,16 @@ def start(self, starting: bool = True, error_on_pipeline: bool = True,
if error_on_message:
delete_message = False
if self.parameters.error_dump_message:
error_traceback = traceback.format_exception(*error_on_message)
self._dump_message(error_traceback,
message=self.__current_message)
delete_message = True
else:
warnings.warn("Message will be removed from the pipeline and not dumped to the disk. "
"Set `error_dump_message` to true to save the message on disk. "
"This warning is only shown once in the runtime of a bot.")
if '_on_error' in self.__destination_queues:
self.send_message(self.__current_message, path='_on_error')
delete_message = True
if delete_message:
self.__current_message = None
# remove message from pipeline
self.acknowledge_message()
@@ -0,0 +1,3 @@
"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","module","motd","password"
"2018-10-22 11:11:11","10.10.10.1","tcp",873,"foo.example.com","rsync",65536,"AT","OBEROSTERREICH","STEYR",0,0,"Public;foo;bar;",,"N"
"2018-10-22 11:11:12","10.10.10.2","tcp",873,"bar.example.com","rsync",65537,"AT","SALZBURG","SALZBURG",0,0,"Shared folder;",,"N"
@@ -0,0 +1,86 @@
# -*- coding: utf-8 -*-
import os
import unittest
import intelmq.lib.test as test
import intelmq.lib.utils as utils
from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot
with open(os.path.join(os.path.dirname(__file__), 'accessible-rsync.csv')) as handle:
EXAMPLE_FILE = handle.read()
EXAMPLE_LINES = EXAMPLE_FILE.splitlines()
EXAMPLE_REPORT = {"feed.name": "ShadowServer Accessible Rsync",
"raw": utils.base64_encode(EXAMPLE_FILE),
"__type": "Report",
"time.observation": "2015-01-01T00:00:00+00:00",
}
EVENTS = [{'__type': 'Event',
'classification.identifier': 'accessible-rsync',
'classification.taxonomy': 'vulnerable',
'classification.type': 'vulnerable service',
'extra.module': 'Public;foo;bar;',
'extra.password': False,
'extra.tag': 'rsync',
'feed.name': 'ShadowServer Accessible Rsync',
'protocol.application': 'rsync',
'protocol.transport': 'tcp',
'source.asn': 65536,
'source.geolocation.cc': 'AT',
'source.geolocation.city': 'STEYR',
'source.geolocation.region': 'OBEROSTERREICH',
'source.ip': '10.10.10.1',
'source.port': 873,
'source.reverse_dns': 'foo.example.com',
'time.source': '2018-10-22T11:11:11+00:00',
'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0],
EXAMPLE_LINES[1]])),
'time.observation': '2015-01-01T00:00:00+00:00',
},
{'__type': 'Event',
'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0],
EXAMPLE_LINES[2]])),
'classification.identifier': 'accessible-rsync',
'classification.taxonomy': 'vulnerable',
'classification.type': 'vulnerable service',
'extra.module': 'Shared folder;',
'extra.password': False,
'extra.tag': 'rsync',
'feed.name': 'ShadowServer Accessible Rsync',
'protocol.application': 'rsync',
'protocol.transport': 'tcp',
'source.asn': 65537,
'source.geolocation.cc': 'AT',
'source.geolocation.city': 'SALZBURG',
'source.geolocation.region': 'SALZBURG',
'source.ip': '10.10.10.2',
'source.port': 873,
'source.reverse_dns': 'bar.example.com',
'time.source': '2018-10-22T11:11:12+00:00',
'time.observation': '2015-01-01T00:00:00+00:00',
},
]
class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase):
"""
A TestCase for a ShadowserverParserBot.
"""
@classmethod
def set_bot(cls):
cls.bot_reference = ShadowserverParserBot
cls.default_input_message = EXAMPLE_REPORT
cls.sysconfig = {'feedname': 'Accessible-Rsync'}
def test_event(self):
""" Test if correct Event has been produced. """
self.run_bot()
for i, EVENT in enumerate(EVENTS):
self.assertMessageEqual(i, EVENT)
if __name__ == '__main__': # pragma: no cover
unittest.main()
Oops, something went wrong.

0 comments on commit 2ff1fe7

Please sign in to comment.