diff --git a/.gitattributes b/.gitattributes index a0417134a..3db097fda 100644 --- a/.gitattributes +++ b/.gitattributes @@ -17,4 +17,3 @@ /.* export-ignore /CONTRIBUTING.md export-ignore /debian export-ignore -/MANIFEST.in export-ignore diff --git a/CHANGELOG.md b/CHANGELOG.md index deba82160..6e721e5a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,10 +64,13 @@ CHANGELOG ### Core - `lib/harmonization.py`: Change `parse_utc_isoformat` of `DateTime` class from private to public (related to #1322). - `lib/utils.py`: Add new function `object_pair_hook_bots`. -- `lib.bot.py`: `ParserBot`'s method `recover_line_csv` now also handles given `tempdata`. +- `lib.bot.py`: + - `ParserBot`'s method `recover_line_csv` now also handles given `tempdata`. + - `Bot.acknowledge_message()` deletes `__current_message` to free the memory, saves memory in idling parsers with big reports. - `lib/message.py`: - Fix add('extra', ..., overwrite=True): old extra fields have not been deleted previously (#1335). - Do not ignore empty or ignored (as defined in `_IGNORED_VALUES`) values of `extra.*` fields for backwards compatibility (#1335). +- `lib/pipeline.py` (`Redis.receive`): Wait in 1s steps if redis is busy loading its snapshot from disk (#1334). ### Development - `bin/rewrite_config_files.py`: Fix ordering of BOTS file (#1327). diff --git a/debian/changelog b/debian/changelog index 4e9d3c453..6815783c9 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,7 +4,7 @@ intelmq (1.2.0~alpha1) UNRELEASED; urgency=medium -- Sebastian Wagner Sat, 07 Jul 2018 11:43:23 +0200 -intelmq (1.1.1~alpha1-1) UNRELEASED; urgency=medium +intelmq (1.1.1~alpha1-1) unstable; urgency=medium * Update to version 1.1.1 alpha 1 diff --git a/intelmq/bots/parsers/shadowserver/config.py b/intelmq/bots/parsers/shadowserver/config.py index 9cbec58aa..cf34367e3 100644 --- a/intelmq/bots/parsers/shadowserver/config.py +++ b/intelmq/bots/parsers/shadowserver/config.py @@ -1776,7 +1776,6 @@ def convert_date(value): 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), ('source.ip', 'ip'), -# ('source.port', 'port'), # not available ], 'optional_fields': [ ('source.asn', 'asn'), diff --git a/intelmq/lib/bot.py b/intelmq/lib/bot.py index 2b4c59c20..fff56ba88 100644 --- a/intelmq/lib/bot.py +++ b/intelmq/lib/bot.py @@ -385,6 +385,10 @@ def send_message(self, *messages, path="_default", auto_add=None): 'but needed') self.logger.debug("Sending message.") + + raw_message = libmessage.MessageFactory.serialize(message) + self.__destination_pipeline.send(raw_message, path=path) + self.__message_counter += 1 if not self.__message_counter_start: self.__message_counter_start = datetime.datetime.now() @@ -394,9 +398,6 @@ def send_message(self, *messages, path="_default", auto_add=None): self.__message_counter = 0 self.__message_counter_start = datetime.datetime.now() - raw_message = libmessage.MessageFactory.serialize(message) - self.__destination_pipeline.send(raw_message, path=path) - def receive_message(self): self.logger.debug('Waiting for incoming message.') message = None @@ -435,6 +436,9 @@ def acknowledge_message(self): if self.__source_pipeline: self.__source_pipeline.acknowledge() + # free memory of last message + self.__current_message = None + def _dump_message(self, error_traceback, message: dict): if message is None or getattr(self.parameters, 'testing', False): return diff --git a/intelmq/lib/pipeline.py b/intelmq/lib/pipeline.py index bd3758a3e..b72bbbd66 100644 --- a/intelmq/lib/pipeline.py +++ b/intelmq/lib/pipeline.py @@ -146,7 +146,13 @@ def receive(self): if self.source_queue is None: raise exceptions.ConfigurationError('pipeline', 'No source queue given.') try: - retval = self.pipe.lindex(self.internal_queue, -1) # returns None if no value + while True: + try: + retval = self.pipe.lindex(self.internal_queue, -1) # returns None if no value + except redis.exceptions.BusyLoadingError: # Just wait at redis' startup #1334 + time.sleep(1) + else: + break if not retval: retval = self.pipe.brpoplpush(self.source_queue, self.internal_queue, 0) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_helpers.py b/intelmq/tests/bots/parsers/shadowserver/test_helpers.py index b7f4fa2ac..3d1f52b28 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_helpers.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_helpers.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Created on Thu Aug 9 15:18:24 2018 diff --git a/setup.py b/setup.py index 6ca2be186..86cc94c9a 100644 --- a/setup.py +++ b/setup.py @@ -92,6 +92,7 @@ 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3 :: Only', 'Programming Language :: Python :: Implementation :: CPython', 'Topic :: Security',