Merge branch 'maintenance' into develop

.gitattributes

@@ -17,4 +17,3 @@
 /.*	export-ignore
 /CONTRIBUTING.md	export-ignore
 /debian	export-ignore
-/MANIFEST.in	export-ignore

CHANGELOG.md

@@ -64,10 +64,13 @@ CHANGELOG
 ### Core
 - `lib/harmonization.py`: Change `parse_utc_isoformat` of `DateTime` class from private to public (related to #1322).
 - `lib/utils.py`: Add new function `object_pair_hook_bots`.
-- `lib.bot.py`: `ParserBot`'s method `recover_line_csv` now also handles given `tempdata`.
+- `lib.bot.py`:
+  - `ParserBot`'s method `recover_line_csv` now also handles given `tempdata`.
+  - `Bot.acknowledge_message()` deletes `__current_message` to free the memory, saves memory in idling parsers with big reports.
 - `lib/message.py`:
   - Fix add('extra', ..., overwrite=True): old extra fields have not been deleted previously (#1335).
   - Do not ignore empty or ignored (as defined in `_IGNORED_VALUES`) values of `extra.*` fields for backwards compatibility (#1335).
+- `lib/pipeline.py` (`Redis.receive`): Wait in 1s steps if redis is busy loading its snapshot from disk (#1334).
 
 ### Development
 - `bin/rewrite_config_files.py`: Fix ordering of BOTS file (#1327).

debian/changelog

@@ -4,7 +4,7 @@ intelmq (1.2.0~alpha1) UNRELEASED; urgency=medium
 
  -- Sebastian Wagner <wagner@cert.at>  Sat, 07 Jul 2018 11:43:23 +0200
 
-intelmq (1.1.1~alpha1-1) UNRELEASED; urgency=medium
+intelmq (1.1.1~alpha1-1) unstable; urgency=medium
 
   * Update to version 1.1.1 alpha 1
 

intelmq/bots/parsers/shadowserver/config.py

@@ -1776,7 +1776,6 @@ def convert_date(value):
     'required_fields': [
         ('time.source', 'timestamp', add_UTC_to_timestamp),
         ('source.ip', 'ip'),
-#        ('source.port', 'port'),  # not available
     ],
     'optional_fields': [
         ('source.asn', 'asn'),

intelmq/lib/bot.py

@@ -385,6 +385,10 @@ def send_message(self, *messages, path="_default", auto_add=None):
                                                     'but needed')
 
             self.logger.debug("Sending message.")
+
+            raw_message = libmessage.MessageFactory.serialize(message)
+            self.__destination_pipeline.send(raw_message, path=path)
+
             self.__message_counter += 1
             if not self.__message_counter_start:
                 self.__message_counter_start = datetime.datetime.now()
@@ -394,9 +398,6 @@ def send_message(self, *messages, path="_default", auto_add=None):
                 self.__message_counter = 0
                 self.__message_counter_start = datetime.datetime.now()
 
-            raw_message = libmessage.MessageFactory.serialize(message)
-            self.__destination_pipeline.send(raw_message, path=path)
-
     def receive_message(self):
         self.logger.debug('Waiting for incoming message.')
         message = None
@@ -435,6 +436,9 @@ def acknowledge_message(self):
         if self.__source_pipeline:
             self.__source_pipeline.acknowledge()
 
+        # free memory of last message
+        self.__current_message = None
+
     def _dump_message(self, error_traceback, message: dict):
         if message is None or getattr(self.parameters, 'testing', False):
             return

intelmq/lib/pipeline.py

@@ -146,7 +146,13 @@ def receive(self):
         if self.source_queue is None:
             raise exceptions.ConfigurationError('pipeline', 'No source queue given.')
         try:
-            retval = self.pipe.lindex(self.internal_queue, -1)  # returns None if no value
+            while True:
+                try:
+                    retval = self.pipe.lindex(self.internal_queue, -1)  # returns None if no value
+                except redis.exceptions.BusyLoadingError:  # Just wait at redis' startup #1334
+                    time.sleep(1)
+                else:
+                    break
             if not retval:
                 retval = self.pipe.brpoplpush(self.source_queue,
                                               self.internal_queue, 0)

intelmq/tests/bots/parsers/shadowserver/test_helpers.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 """
 Created on Thu Aug  9 15:18:24 2018

setup.py

@@ -92,6 +92,7 @@
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3 :: Only',
         'Programming Language :: Python :: Implementation :: CPython',
         'Topic :: Security',