Merge pull request #448 from sebix/bugfixes2

RT Collector, pip installation, logcheck rules

.gitignore

@@ -17,3 +17,4 @@ dist
 htmlcov/
 *.pem
 *.key
+src/

.travis.yml

@@ -7,14 +7,12 @@ install:
   - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then pip install -r REQUIREMENTS2; fi
   - if [[ ${TRAVIS_PYTHON_VERSION%.?} == 3 ]]; then pip install -r REQUIREMENTS; fi
   - for file in intelmq/bots/*/*/REQUIREMENTS.txt; do pip install -r $file; done
-  - "python setup_travis.py install"
   - pip install coveralls
   - pip install codecov
-# command to run tests
+  - pip install .
 script: nosetests --with-coverage --cover-package=intelmq
 services:
   - redis-server
-
 after_success:
   - codecov
   - coveralls

MANIFEST.in

@@ -0,0 +1,8 @@
+exclude .gitignore
+exclude .pep8
+exclude .travis.yml
+graft contrib
+graft docs
+include COPYRIGHT
+include LICENSE
+include README.md

contrib/logcheck/README.md

@@ -0,0 +1,6 @@
+# logcheck ruleset
+
+logcheck is a simple and effective log monitoring tool checking for known and
+unknown patterns in the logfiles. The given rules do ignore all messages with
+loglevels INFO and DEBUG, and parts of tracebacks. ERROR and CRITICAL are
+treated as violations, WARNINGS are thus normal irregular log messages.

contrib/logcheck/ignore

@@ -0,0 +1,8 @@
+^[0-9-]{10} [0-9:,]{12} - [a-zA-Z0-9_-]* - (INFO|DEBUG) - 
+^ 
+^Traceback
+^[a-zA-Z]*Error
+^InvalidValue
+^During handling of the above exception, another exception occurred:$
+^intelmq\.
+^redis\.

contrib/logcheck/logcheck.logfiles

@@ -0,0 +1 @@
+/opt/intelmq/var/log/*.log

contrib/logcheck/violations

@@ -0,0 +1 @@
+^[0-9-]{10} [0-9:,]{12} - [a-zA-Z0-9_-]* - (ERROR|CRITICAL) - 

docs/Developers-Guide.md

@@ -29,6 +29,12 @@ How do you test if things are easy? Let them new programmers test-drive your fea
 
 Similarly, if code does not get accepted upstream by the main developers, it is usually only because of the ease-of-use argument. Do not give up , go back to the drawing board, and re-submit again.
 
+## Installation
+
+Install intelmq with `pip -e`, which gives you a so called *editable* installation. All changed files in the local copy are directly changed in the local installation too!
+
+    pip install -e .
+
 ## Testing
 
 All changes have to be tested and new contributions must must be accompanied by according tests. You can run the tests by changing to the directory with intelmq repository and running either `unittest` or `nosetests`:

docs/User-Guide.md

@@ -120,7 +120,7 @@ cd /tmp/intelmq
 sudo -s
 
 pip3 install -r REQUIREMENTS
-python3 setup.py install
+pip3 install .
 
 useradd -d /opt/intelmq -U -s /bin/bash intelmq
 echo 'export PATH="$PATH:$HOME/bin"' > /opt/intelmq/.profile
@@ -138,7 +138,7 @@ git clone https://github.com/certtools/intelmq.git /tmp/intelmq
 cd /tmp/intelmq
 
 pip2 install -r REQUIREMENTS2
-python2.7 setup.py install
+pip2 install .
 
 useradd -d /opt/intelmq -U -s /bin/bash intelmq
 echo 'export PATH="$PATH:$HOME/bin"' > /opt/intelmq/.profile

intelmq/bots/BOTS

@@ -420,6 +420,28 @@
                 "rate_limit": 28800
             }
         },
+        "Request Tracker": {
+            "description": "Request Tracker Collector fetches attachments from an RTIR instance and optionally decrypts them with gnupg.",
+            "module": "intelmq.bots.collectors.rt.collector_rt",
+            "parameters": {
+                "attachment_regex": "\\.csv\\.zip$",
+                "feed": "Request Tracker",
+                "gnupg_decrypt": false,
+                "gnupg_homedir": "/opt/intelmq/",
+                "gnupg_passphrase": null,
+                "gnupg_trust": true,
+                "password": "password",
+                "rate_limit": 3600,
+                "search_owner": "nobody",
+                "search_queue": "Incident Reports",
+                "search_status": "new",
+                "search_subject_like": "Report",
+                "take_ticket": true,
+                "unzip_attachment": true,
+                "uri": "http://localhost/rt/REST/1.0",
+                "user": "root"
+            }
+        },
         "ShadowServer Chargen": {
             "description": "ShadowServer Chargen Collector is the bot responsible to get the report from source of information. https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-Chargen",
             "module": "intelmq.bots.collectors.mail.mail-url",
@@ -495,11 +517,11 @@
             }
         },
         "Spamhaus CERT": {
-            "description": "Spamhaus CERT Collector is the bot responsible to get the report from source of information.",
+            "description": "Spamhaus CERT Insight Portal. Access limited to CERTs and CSIRTs with national or regional responsibility. https://www.spamhaus.org/news/article/705/spamhaus-launches-cert-insight-portal",
             "module": "intelmq.bots.collectors.http.collector_http",
             "parameters": {
                 "feed": "Spamhaus CERT",
-                "http_url": "https://portal.spamhaus.org/cert/api.php?cert=<CERTNAME>&key=<APIKEY>",
+                "http_url": "<your CERT portal URL>",
                 "rate_limit": 3600
             }
         },
@@ -508,7 +530,7 @@
             "module": "intelmq.bots.collectors.http.collector_http",
             "parameters": {
                 "feed": "Spamhaus Drop",
-                "http_url": "https://www.spamhaus.org/drop/drop.lasso",
+                "http_url": "https://www.spamhaus.org/drop/drop.txt",
                 "rate_limit": 3600
             }
         },

intelmq/bots/collectors/rt/REQUIREMENTS.txt

@@ -0,0 +1 @@
+-e git://git.nic.cz/python-rt.git@52e74e590b27605f33923c54b552ec00ece85a6c#egg=rt

intelmq/bots/collectors/rt/collector_rt.py

@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+"""
+TODO: Arbitrary order of decrypt and unzip
+"""
+from __future__ import unicode_literals
+
+import io
+import re
+import sys
+import zipfile
+
+from intelmq.lib.bot import Bot
+from intelmq.lib.harmonization import DateTime
+from intelmq.lib.message import Report
+
+import rt
+
+try:
+    import gnupg
+except ImportError:
+    gnupg = None
+
+
+class RTCollectorBot(Bot):
+
+    def init(self):
+        if self.parameters.gnupg_decrypt:
+            if gnupg is None:
+                raise ValueError('gnupg module is not available')
+            self.gpg = gnupg.GPG(gnupghome=self.parameters.gnupg_homedir)
+
+    def process(self):
+        RT = rt.Rt(self.parameters.uri, self.parameters.user,
+                   self.parameters.password)
+        if not RT.login():
+            raise ValueError('Login failed.')
+
+        query = RT.search(Queue=self.parameters.search_queue,
+                          Subject__like=self.parameters.search_subject_like,
+                          Owner=self.parameters.search_owner,
+                          Status=self.parameters.search_status)
+        self.logger.info('{} results on search query.'.format(len(query)))
+
+        for ticket in query:
+            ticket_id = int(ticket['id'].split('/')[1])
+            self.logger.debug('Process ticket {}.'.format(ticket_id))
+            for (att_id, att_name, _, _) in RT.get_attachments(ticket_id):
+                if re.search(self.parameters.attachment_regex, att_name):
+                    self.logger.debug('Found attachment {}: {!r}.'
+                                      ''.format(att_id, att_name))
+                    break
+            else:
+                self.logger.debug('No matching attachement name found.')
+                continue
+            attachment = RT.get_attachment_content(ticket_id, att_id)
+
+            if self.parameters.unzip_attachment:
+                file_obj = io.BytesIO(attachment)
+                zipped = zipfile.ZipFile(file_obj)
+                raw = zipped.read(zipped.namelist()[0])
+            else:
+                raw = attachment
+
+            if self.parameters.gnupg_decrypt:
+                raw = str(self.gpg.decrypt(raw,
+                                           always_trust=self.parameters.gnupg_trust,
+                                           passphrase=self.parameters.gnupg_passphrase))
+                self.logger.info('Successfully decrypted attachment.')
+
+            self.logger.debug(raw)
+            report = Report()
+            report.add("raw", raw, sanitize=True)
+            report.add("rtir_id", ticket_id, sanitize=True)
+            report.add("feed.name", self.parameters.feed, sanitize=True)
+            report.add("feed.accuracy", self.parameters.accuracy,
+                       sanitize=True)
+            time_observation = DateTime().generate_datetime_now()
+            report.add('time.observation', time_observation, sanitize=True)
+            self.send_message(report)
+
+            if self.parameters.take_ticket:
+                RT.edit_ticket(ticket_id, Owner=self.parameters.user)
+
+
+if __name__ == "__main__":
+    bot = RTCollectorBot(sys.argv[1])
+    bot.start()

intelmq/conf/harmonization.conf

@@ -334,6 +334,10 @@
             "length": 500000000,
             "type": "Base64"
         },
+        "rtir_id": {
+            "description": "Request Tracker Incident Response incident id.",
+            "type": "Integer"
+        },
         "time.observation": {
             "description": "The time a source bot saw the event. This timestamp becomes especially important should you perform your own attribution on a host DNS name for example. The mechanism to denote the attributed elements with reference to the source provided is detailed below in Reported Identity IOC.(ISO8660).",
             "type": "DateTime"

intelmq/lib/bot.py

@@ -19,6 +19,9 @@
 from intelmq.lib.pipeline import PipelineFactory
 
 
+__all__ = ['Bot']
+
+
 class Bot(object):
 
     def __init__(self, bot_id):
@@ -48,9 +51,8 @@ def __init__(self, bot_id):
                 syslog = self.parameters.logging_syslog
             else:
                 syslog = False
-            self.logger = utils.log(self.bot_id,
+            self.logger = utils.log(self.bot_id, syslog=syslog,
                                     log_level=self.parameters.logging_level)
-#                                    syslog=syslog)
         except:
             self.log_buffer.append(('critical', traceback.format_exc()))
             self.stop()
@@ -301,7 +303,6 @@ def dump_message(self, error_traceback):
         self.logger.info('Message dumped.')
         self.current_message = None
 
-
     def load_defaults_configuration(self):
         self.log_buffer.append(('debug', "Loading defaults configuration."))
         config = utils.load_configuration(DEFAULTS_CONF_FILE)

intelmq/lib/cache.py

@@ -14,6 +14,9 @@
 import intelmq.lib.utils as utils
 
 
+__all__ = ['Cache']
+
+
 class Cache():
 
     def __init__(self, host, port, db, ttl):

intelmq/lib/exceptions.py

@@ -6,6 +6,12 @@
 import traceback
 
 
+__all__ = ['InvalidArgument', 'ConfigurationError', 'IntelMQException',
+           'IntelMQHarmonizationException', 'InvalidKey', 'InvalidValue',
+           'KeyExists', 'KeyNotExists', 'PipelineError',
+           ]
+
+
 class IntelMQException(Exception):
 
     def __init__(self, message):

intelmq/lib/message.py

@@ -16,6 +16,9 @@
 from intelmq.lib import utils
 
 
+__all__ = ['Event', 'Message', 'MessageFactory', 'Report']
+
+
 harm_config = utils.load_configuration(HARMONIZATION_CONF_FILE)
 
 

intelmq/lib/pipeline.py

@@ -10,6 +10,9 @@
 import intelmq.lib.utils as utils
 
 
+__all__ = ['Pipeline', 'PipelineFactory', 'Redis', 'Pythonlist']
+
+
 class PipelineFactory(object):
 
     @staticmethod

intelmq/lib/test.py

@@ -20,6 +20,9 @@
 from intelmq import PIPELINE_CONF_FILE, RUNTIME_CONF_FILE, SYSTEM_CONF_FILE
 
 
+__all__ = ['BotTestCase']
+
+
 BOT_CONFIG = {
     "logging_level": "DEBUG",
     "http_proxy":  None,
@@ -65,7 +68,7 @@ def mock(conf_file):
 
 
 def mocked_logger(logger):
-    def log(name, log_path=None, log_level=None):
+    def log(name, log_path=None, log_level=None, stream=None, syslog=None):
         return logger
     return log
 
@@ -352,9 +355,9 @@ def assertRegexpMatchesLog(self, pattern):
 
         self.assertIsNotNone(self.loglines_buffer)
         try:
-            self.assertRegexpMatches(self.loglines_buffer, pattern)
-        except AttributeError:
             self.assertRegex(self.loglines_buffer, pattern)
+        except AttributeError:  # Py2
+            self.assertRegexpMatches(self.loglines_buffer, pattern)
 
     def assertNotRegexpMatchesLog(self, pattern):
         """Asserts that pattern doesn't match against log."""

intelmq/lib/utils.py

@@ -31,9 +31,11 @@
 except ImportError:
     unicodecsv = None
 
-__all__ = ['decode', 'encode', 'base64_encode', 'base64_decode',
-           'load_configuration', 'load_parameters', 'log', 'reverse_readline',
-           'parse_logline']
+
+__all__ = ['base64_decode', 'base64_encode', 'decode', 'encode',
+           'load_configuration', 'load_parameters', 'log', 'parse_logline',
+           'reverse_readline',
+           ]
 
 # Used loglines format
 LOG_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
@@ -199,7 +201,7 @@ class instance with items of configs as attributes
 
 
 def log(name, log_path=DEFAULT_LOGGING_PATH, log_level="DEBUG", stream=None,
-        syslog=False):
+        syslog=None):
     """
     Returns a logger instance logging to file and sys.stderr or other stream.
 

intelmq/tests/bots/collectors/rt/test_collector.py

@@ -0,0 +1,5 @@
+# -*- coding: utf-8 -*-
+"""
+"""
+
+import intelmq.bots.collectors.rt.collector_rt

intelmq/tests/lib/test_utils.py

@@ -88,7 +88,10 @@ def test_file_logger(self):
 
             line_format = [line.format(name) for line in LINES['long']]
             for ind, line in enumerate(file_lines):
-                self.assertRegexpMatches(line, line_format[ind])
+                try:
+                    self.assertRegex(line, line_format[ind])
+                except AttributeError:  # Py2
+                    self.assertRegexpMatches(line, line_format[ind])
 
     def test_stream_logger(self):
         """Tests if a logger for a stream can be generated with log()."""