Merge branch 'master' into reviewfeeds_20191105

plugins/feeds/public/abuseipdb.py

@@ -0,0 +1,43 @@
+import logging
+from datetime import timedelta
+
+from core import Feed
+from core.errors import ObservableValidationError
+from core.observables import Ip
+from core.config.config import yeti_config
+
+
+class AbuseIPDB(Feed):
+    default_values = {
+        "frequency": timedelta(hours=5),
+        "name": "AbuseIPDB",
+        "source": "https://api.abuseipdb.com/api/v2/blacklist",
+        "description":
+            "Black List IP generated by AbuseIPDB",
+    }
+
+    def update(self):
+        api_key = yeti_config.get('abuseIPDB', 'key')
+
+        if api_key:
+            self.source = "https://api.abuseipdb.com/api/v2/blacklist?&key=%s&plaintext&limit=10000" % (api_key)
+            # change the limit rate if you subscribe to a paid plan
+            for line in self.update_lines():
+                self.analyze(line)
+        else:
+            logging.error("Your abuseIPDB API key is not set in the yeti.conf file")
+
+    def analyze(self, line):
+        line = line.strip()
+
+        ip = line
+
+        context = {'source': self.name}
+
+        try:
+            ip = Ip.get_or_create(value=ip)
+            ip.add_context(context)
+            ip.add_source(self.name)
+            ip.tag('abuseIPDB')
+        except ObservableValidationError as e:
+            raise logging.error(e)

plugins/feeds/public/malwaremustdiecncs.py

@@ -22,20 +22,11 @@ def update(self):
             if self.last_run is not None:
 
                 try:
-                    date_description = datetime.strptime(item["description"],
-                                                         "%d/%b/%Y")
-                except ValueError as e:
-                    pass
-
-                try:
-                    date_description = datetime.strptime(item["description"],
-                                                        "%d/%B/%Y")
-                except ValueError as e:
-                    continue
-                if date_description < since_last_run:
+                    if datetime.strptime(item["description"], "%d/%b/%Y") < since_last_run:
+                        continue
+                except ValueError:
+                    if datetime.strptime(item["description"], "%d/%B/%Y") < since_last_run:
                         continue
-
-
             self.analyze(item["title"])
 
     def analyze(self, cnc):

plugins/feeds/public/misp.py

@@ -68,16 +68,29 @@ def get_organisations(self, instance):
             'Accept': 'application/json'
         }
 
-        orgs = requests.get(
-            url, headers=headers, proxies=yeti_config.proxy).json()
+        r = requests.get(
+            url, headers=headers, proxies=yeti_config.proxy)
 
-        for org in orgs:
-            org_id = org['Organisation']['id']
-            org_name = org['Organisation']['name']
-            self.instances[instance]['organisations'][org_id] = org_name
+        if r.status_code == 200:
+
+            orgs = r.json()
+
+            for org in orgs:
+                org_id = org['Organisation']['id']
+                org_name = org['Organisation']['name']
+                self.instances[instance]['organisations'][org_id] = org_name
+        else:
+            logging.error('error http %s to get instances' % r.status_code)
 
     def week_events(self, instance):
         one_week = timedelta(days=7)
+        if not self.instances:
+            logging.error('not instances in MISP')
+            return
+        elif instance not in self.instances:
+            logging.error('error in instances of Misp')
+            return
+
         url = urljoin(self.instances[instance]['url'], '/events/restSearch')
         headers = {'Authorization': self.instances[instance]['key']}
         to = date.today()

plugins/feeds/public/phishing_database.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+"""This class will incorporate the PhishingDatabase feed into yeti."""
+
+from datetime import timedelta
+import logging
+
+from core.observables import Url
+from core.feed import Feed
+from core.errors import ObservableValidationError
+
+class PhishingDatabase(Feed):
+    """This class will pull the PhishingDatabase feed from github on a 12 hour interval."""
+
+    default_values = {
+        'frequency': timedelta(hours=12),
+        'name': 'PhishingDatabase',
+        'source': 'https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-links-NEW-today.txt',
+        'description':
+            'Phishing Domains, urls websites and threats database.'
+    }
+
+    def update(self):
+        for url in self.update_lines():
+            self.analyze(url)
+
+    def analyze(self, url):
+        context = {'source': self.name}
+
+        try:
+            url = Url.get_or_create(value=url)
+            url.add_context(context)
+            url.add_source(self.name)
+            url.tag(['phishing'])
+        except ObservableValidationError as e:
+            logging.error(e)

yeti.conf.sample

@@ -81,5 +81,13 @@
 # w/ token - limit 5k r/h
 # token =
 
+[otx]
+# otx_key = YourOTXKey
+# number_page = 1
+
+[abuseIPDB]
+# key = YourKey
+
+
 [phishtank]
-key=
+# key=