Permalink
Browse files

Merge branch 'maintenance' into develop

  • Loading branch information...
wagner-certat committed Oct 22, 2018
2 parents 4156bc0 + ff022ca commit ba5a0b2121e54d4f2ba5b0c34e9dc5d761972033
@@ -17,4 +17,3 @@
/.* export-ignore
/CONTRIBUTING.md export-ignore
/debian export-ignore
/MANIFEST.in export-ignore
@@ -64,10 +64,13 @@ CHANGELOG
### Core
- `lib/harmonization.py`: Change `parse_utc_isoformat` of `DateTime` class from private to public (related to #1322).
- `lib/utils.py`: Add new function `object_pair_hook_bots`.
- `lib.bot.py`: `ParserBot`'s method `recover_line_csv` now also handles given `tempdata`.
- `lib.bot.py`:
- `ParserBot`'s method `recover_line_csv` now also handles given `tempdata`.
- `Bot.acknowledge_message()` deletes `__current_message` to free the memory, saves memory in idling parsers with big reports.
- `lib/message.py`:
- Fix add('extra', ..., overwrite=True): old extra fields have not been deleted previously (#1335).
- Do not ignore empty or ignored (as defined in `_IGNORED_VALUES`) values of `extra.*` fields for backwards compatibility (#1335).
- `lib/pipeline.py` (`Redis.receive`): Wait in 1s steps if redis is busy loading its snapshot from disk (#1334).
### Development
- `bin/rewrite_config_files.py`: Fix ordering of BOTS file (#1327).
@@ -4,7 +4,7 @@ intelmq (1.2.0~alpha1) UNRELEASED; urgency=medium
-- Sebastian Wagner <wagner@cert.at> Sat, 07 Jul 2018 11:43:23 +0200
intelmq (1.1.1~alpha1-1) UNRELEASED; urgency=medium
intelmq (1.1.1~alpha1-1) unstable; urgency=medium
* Update to version 1.1.1 alpha 1
@@ -1776,7 +1776,6 @@ def convert_date(value):
'required_fields': [
('time.source', 'timestamp', add_UTC_to_timestamp),
('source.ip', 'ip'),
# ('source.port', 'port'), # not available
],
'optional_fields': [
('source.asn', 'asn'),
@@ -385,6 +385,10 @@ def send_message(self, *messages, path="_default", auto_add=None):
'but needed')
self.logger.debug("Sending message.")
raw_message = libmessage.MessageFactory.serialize(message)
self.__destination_pipeline.send(raw_message, path=path)
self.__message_counter += 1
if not self.__message_counter_start:
self.__message_counter_start = datetime.datetime.now()
@@ -394,9 +398,6 @@ def send_message(self, *messages, path="_default", auto_add=None):
self.__message_counter = 0
self.__message_counter_start = datetime.datetime.now()
raw_message = libmessage.MessageFactory.serialize(message)
self.__destination_pipeline.send(raw_message, path=path)
def receive_message(self):
self.logger.debug('Waiting for incoming message.')
message = None
@@ -435,6 +436,9 @@ def acknowledge_message(self):
if self.__source_pipeline:
self.__source_pipeline.acknowledge()
# free memory of last message
self.__current_message = None
def _dump_message(self, error_traceback, message: dict):
if message is None or getattr(self.parameters, 'testing', False):
return
@@ -146,7 +146,13 @@ def receive(self):
if self.source_queue is None:
raise exceptions.ConfigurationError('pipeline', 'No source queue given.')
try:
retval = self.pipe.lindex(self.internal_queue, -1) # returns None if no value
while True:
try:
retval = self.pipe.lindex(self.internal_queue, -1) # returns None if no value
except redis.exceptions.BusyLoadingError: # Just wait at redis' startup #1334
time.sleep(1)
else:
break
if not retval:
retval = self.pipe.brpoplpush(self.source_queue,
self.internal_queue, 0)
@@ -1,4 +1,3 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Aug 9 15:18:24 2018
@@ -92,6 +92,7 @@
'Programming Language :: Python :: 3.4',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7',
'Programming Language :: Python :: 3 :: Only',
'Programming Language :: Python :: Implementation :: CPython',
'Topic :: Security',

0 comments on commit ba5a0b2

Please sign in to comment.