diff --git a/core/feed.py b/core/feed.py index 26380e05f..4c02cbbe3 100644 --- a/core/feed.py +++ b/core/feed.py @@ -1,14 +1,14 @@ from __future__ import unicode_literals import csv -import requests -from datetime import datetime import logging from StringIO import StringIO +from datetime import datetime +import requests from lxml import etree -from mongoengine import StringField from mongoengine import DoesNotExist +from mongoengine import StringField from core.config.celeryctl import celery_app from core.config.config import yeti_config @@ -221,26 +221,29 @@ def update_csv(self, delimiter=';', quotechar="'", headers={}, auth=None): for line in reader: yield line - def update_json(self, headers={}, auth=None): + def update_json(self, headers={}, auth=None, params={}): """Helper function. Performs an HTTP request on ``source`` and parses the response JSON, returning a Python ``dict`` object. Args: headers: Optional headers to be added to the HTTP request. auth: Username / password tuple to be sent along with the HTTP request. + params: Optional param to be added to the HTTP request. Returns: Python ``dict`` object representing the response JSON. """ + if auth: r = requests.get( self.source, headers=headers, auth=auth, - proxies=yeti_config.proxy) + proxies=yeti_config.proxy, params=params) else: r = requests.get( - self.source, headers=headers, proxies=yeti_config.proxy) + self.source, headers=headers, proxies=yeti_config.proxy, + params=params) return r.json() diff --git a/plugins/feeds/public/alienvault_ip_reputation.py b/plugins/feeds/public/alienvault_ip_reputation.py new file mode 100644 index 000000000..1209102b7 --- /dev/null +++ b/plugins/feeds/public/alienvault_ip_reputation.py @@ -0,0 +1,53 @@ +import logging +from datetime import timedelta + +from core import Feed +from core.errors import ObservableValidationError +from core.observables import Ip + + +class AlienVaultIPReputation(Feed): + default_values = { + "frequency": + timedelta(hours=4), + "name": + "AlienVaultIPReputation", + "source": + "http://reputation.alienvault.com/reputation.data", + "description": + "Reputation IP generated by Alienvault", + } + + def update(self): + for line in self.update_csv(delimiter='#', quotechar=None): + self.analyze(line) + + def analyze(self, item): + + if not item: + return + try: + context = dict(source=self.name) + + ip_str = item[0] + category = item[3] + country = item[4] + ip = None + try: + ip = Ip.get_or_create(value=ip_str) + except ObservableValidationError as e: + logging.error(e) + return False + + ip.add_source('feed') + + context['country'] = country + context['threat'] = category + + ip.tag(category) + ip.add_context(context) + + except Exception as e: + logging.error('Error to process the item %s %s' % (item, e)) + return False + return True