Merge branch 'cudeso-master'

docs/tutorial/PyMISP_tutorial.ipynb

@@ -358,9 +358,10 @@
     "# And the to_ids flag is set\n",
     "attributes = misp.search(controller='attributes', type_attribute='ip-src', to_ids=0, pythonify=True)\n",
     "\n",
+    "# Collect all event_id matching the searched attribute\n",
     "event_ids = set()\n",
     "for attr in attributes:\n",
-    "    event_ids.add(event_id)\n",
+    "    event_ids.add(attr.event_id)\n",
     "\n",
     "# Fetch all related events\n",
     "for event_id in event_ids:\n",

examples/vmray_automation.py

@@ -0,0 +1,213 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+'''
+Koen Van Impe
+
+VMRay automatic import
+Put this script in crontab to run every /15 or /60
+    */5 *    * * *   mispuser   /usr/bin/python3 /home/mispuser/PyMISP/examples/vmray_automation.py
+
+Calls "vmray_import" for all events that have an 'incomplete' VMray analysis
+
+Do inline config in "main"
+
+'''
+
+from pymisp import PyMISP
+from keys import misp_url, misp_key, misp_verifycert
+import argparse
+import os
+import json
+import datetime
+import time
+
+import requests
+import sys
+
+# Suppress those "Unverified HTTPS request is being made"
+import urllib3
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+
+def init(url, key):
+    '''
+        Template to get MISP module started
+    '''
+    return PyMISP(url, key, misp_verifycert, 'json')
+
+
+def get_vmray_config(url, key, misp_verifycert, default_wait_period):
+    '''
+        Fetch configuration settings from MISP
+        Includes VMRay API and modules URL
+    '''
+
+    try:
+        misp_headers = {'Content-Type': 'application/json', 'Accept': 'application/json', 'Authorization': key}
+        req = requests.get(url + 'servers/serverSettings.json', verify=misp_verifycert, headers=misp_headers)
+
+        if req.status_code == 200:
+            req_json = req.json()
+            if 'finalSettings' in req_json:
+                finalSettings = req_json['finalSettings']
+                vmray_api = ''
+                vmray_url = ''
+                vmray_wait_period = 0
+
+                for el in finalSettings:
+                    # Is the vmray import module enabled?
+                    if el['setting'] == 'Plugin.Import_vmray_import_enabled':
+                        vmray_import_enabled = el['value']
+                        if vmray_import_enabled is False:
+                            break
+                    # Get the VMRay API key from the MISP settings
+                    elif el['setting'] == 'Plugin.Import_vmray_import_apikey':
+                        vmray_api = el['value']
+                    # The VMRay URL to query
+                    elif el['setting'] == 'Plugin.Import_vmray_import_url':
+                        vmray_url = el['value'].replace('/', '\\/')
+                    # MISP modules - Port?
+                    elif el['setting'] == 'Plugin.Import_services_port':
+                        module_import_port = el['value']
+                    # MISP modules - URL
+                    elif el['setting'] == 'Plugin.Import_services_url':
+                        module_import_url = el['value'].replace('\/\/', '//')
+                    # Wait period
+                    elif el['setting'] == 'Plugin.Import_vmray_import_wait_period':
+                        vmray_wait_period = abs(int(el['value']))
+
+                if vmray_wait_period < 1:
+                    vmray_wait_period = default_wait_period
+        else:
+            sys.exit('Did not receive a 200 code from MISP')
+
+        if vmray_import_enabled and vmray_api and vmray_url and module_import_port and module_import_url:
+            return {'vmray_wait_period': vmray_wait_period, 'vmray_api': vmray_api, 'vmray_url': vmray_url, 'module_import_port': module_import_port, 'module_import_url': module_import_url}
+        else:
+            sys.exit('Did not receive all the necessary configuration information from MISP')
+
+    except Exception as e:
+        sys.exit('Unable to get VMRay config from MISP')
+
+
+def search_vmray_incomplete(m, url, wait_period, module_import_url, module_import_port, vmray_url, vmray_api, vmray_attribute_category, vmray_include_analysisid, vmray_include_imphash_ssdeep, vmray_include_extracted_files, vmray_include_analysisdetails, vmray_include_vtidetails, custom_tags_incomplete, custom_tags_complete):
+    '''
+       Search for the events with VMRay samples that are marked incomplete
+       and then update these events
+    '''
+
+    controller = 'attributes'
+    vmray_value = 'VMRay Sample ID:'  # How sample IDs are stored in MISP
+    req = None
+
+    # Search for the events
+    try:
+        result = m.search(controller, tags=custom_tags_incomplete)
+        response = result['response']
+
+        if len(response) == 0:
+            sys.exit("No VMRay attributes found that match %s" % custom_tags_incomplete)
+
+        attribute = response['Attribute']
+
+        if len(attribute) == 0:
+            sys.exit("No VMRay attributes found that match %s" % custom_tags_incomplete)
+
+        timestamp = int(attribute[0]["timestamp"])
+        # Not enough time has gone by to lookup the analysis jobs
+        if int((time.time() - timestamp) / 60) < int(wait_period):
+            if module_DEBUG:
+                r_timestamp = datetime.datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S')
+                print("Attribute to recent for wait_period (%s minutes) - timestamp attribute: %s (%s minutes old)" % (wait_period, r_timestamp, round((int(time.time() - timestamp) / 60), 2)))
+            return False
+
+        if module_DEBUG:
+            print("All attributes older than %s" % int(wait_period))
+
+        for att in attribute:
+            value = att['value']
+
+            if vmray_value in value:        # We found a sample ID
+                att_id = att['id']
+                att_uuid = att['uuid']
+
+                # VMRay Sample IDs are stored as VMRay Sample ID: 2796577
+                vmray_sample_id = value.split(vmray_value)[1].strip()
+                if vmray_sample_id.isdigit():
+                    event_id = att['event_id']
+                    if module_DEBUG:
+                        print("Found event %s with matching tags %s for sample id %s " % (event_id, custom_tags_incomplete, vmray_sample_id))
+
+                    # Prepare request to send to vmray_import via misp modules
+                    misp_modules_url = module_import_url + ':' + module_import_port + '/query'
+                    misp_modules_headers = {'Content-Type': 'application/json'}
+                    misp_modules_body = '{ "sample_id":"' + vmray_sample_id + '","module":"vmray_import","event_id":"' + event_id + '","config":{"apikey":"' + vmray_api + '","url":"' + vmray_url + '","include_analysisid":"' + vmray_include_analysisid + '","include_analysisdetails":"' + vmray_include_analysisdetails + '","include_extracted_files":"' + vmray_include_extracted_files + '","include_imphash_ssdeep":"' + vmray_include_imphash_ssdeep + '","include_vtidetails":"' + vmray_include_vtidetails + '","sample_id":"' + vmray_sample_id + '"},"data":""}'
+                    req = requests.post(misp_modules_url, data=misp_modules_body, headers=misp_modules_headers)
+                    if module_DEBUG and req is not None:
+                        print("Response code from submitting to MISP modules %s" % (req.status_code))
+
+                    # Succesful response from the misp modules?
+                    if req.status_code == 200:
+                        req_json = req.json()
+                        if "error" in req_json:
+                            print("Error code in reply %s " % req_json["error"])
+                            continue
+                        else:
+                            results = req_json["results"]
+
+                            # Walk through all results in the misp-module reply
+                            for el in results:
+                                to_ids = True
+                                values = el['values']
+                                types = el['types']
+                                if "to_ids" in el:
+                                    to_ids = el['to_ids']
+                                if "text" in types:
+                                    to_ids = False
+                                comment = el['comment']
+                                if len(comment) < 1:
+                                    comment = "Enriched via the vmray_import module"
+
+                                # Attribute can belong in different types
+                                for type in types:
+                                    try:
+                                        r = m.add_named_attribute(event_id, type, values, vmray_attribute_category, to_ids, comment)
+                                        if module_DEBUG:
+                                            print("Add event %s: %s as %s (%s) (toids: %s)" % (event_id, values, type, comment, to_ids))
+                                    except Exception as e:
+                                        continue
+                                        if module_DEBUG:
+                                            print("Unable to add attribute %s as type %s for event %s" % (values, type, event_id))
+
+                            # Remove 'incomplete' state tags
+                            m.untag(att_uuid, custom_tags_incomplete)
+                            # Update tags to 'complete' state
+                            m.tag(att_uuid, custom_tags_complete)
+                            if module_DEBUG:
+                                print("Updated event %s" % event_id)
+
+                    else:
+                        sys.exit('MISP modules did not return HTTP 200 code (event %s ; sampleid %s)' % (event_id, vmray_sample_id))
+
+    except Exception as e:
+        sys.exit("Invalid response received from MISP : %s", e)
+
+
+if __name__ == '__main__':
+
+    module_DEBUG = True
+
+    # Set some defaults to be used in this module
+    vmray_attribute_category = 'External analysis'
+    vmray_include_analysisid = '0'
+    vmray_include_imphash_ssdeep = '0'
+    vmray_include_extracted_files = '0'
+    vmray_include_analysisdetails = '0'
+    vmray_include_vtidetails = '0'
+    custom_tags_incomplete = 'workflow:state="incomplete"'
+    custom_tags_complete = 'workflow:state="complete"'
+    default_wait_period = 30
+
+    misp = init(misp_url, misp_key)
+    vmray_config = get_vmray_config(misp_url, misp_key, misp_verifycert, default_wait_period)
+    search_vmray_incomplete(misp, misp_url, vmray_config['vmray_wait_period'], vmray_config['module_import_url'], vmray_config['module_import_port'], vmray_config['vmray_url'], vmray_config['vmray_api'], vmray_attribute_category, vmray_include_analysisid, vmray_include_imphash_ssdeep, vmray_include_extracted_files, vmray_include_analysisdetails, vmray_include_vtidetails, custom_tags_incomplete, custom_tags_complete)

pymisp/abstract.py

@@ -11,11 +11,18 @@
 
 from .exceptions import PyMISPInvalidFormat
 
+# Try to import MutableMapping the python 3.3+ way
+try:
+    from collections.abc import MutableMapping
+except:
+    pass
+
 
 logger = logging.getLogger('pymisp')
 
 if sys.version_info < (3, 0):
     logger.warning("You're using python 2, it is strongly recommended to use python >=3.6")
+    from collections import MutableMapping
 
     # This is required because Python 2 is a pain.
     from datetime import tzinfo, timedelta

pymisp/api.py

@@ -70,9 +70,10 @@ class PyMISP(object):
     :param cert: Client certificate, as described there: http://docs.python-requests.org/en/master/user/advanced/#client-side-certificates
     :param asynch: Use asynchronous processing where possible
     :param auth: The auth parameter is passed directly to requests, as described here: http://docs.python-requests.org/en/master/user/authentication/
+    :param tool: The software using PyMISP (string), used to set a unique user-agent
     """
 
-    def __init__(self, url, key, ssl=True, out_type='json', debug=None, proxies=None, cert=None, asynch=False, auth=None):
+    def __init__(self, url, key, ssl=True, out_type='json', debug=None, proxies=None, cert=None, asynch=False, auth=None, tool=None):
         if not url:
             raise NoURL('Please provide the URL of your MISP instance.')
         if not key:
@@ -85,6 +86,7 @@ def __init__(self, url, key, ssl=True, out_type='json', debug=None, proxies=None
         self.cert = cert
         self.asynch = asynch
         self.auth = auth
+        self.tool = tool
         if asynch and not ASYNC_OK:
             logger.critical("You turned on Async, but don't have requests_futures installed")
             self.asynch = False
@@ -171,13 +173,16 @@ def _prepare_request(self, request_type, url, data=None,
         else:
             local_session = requests.Session
         with local_session() as s:
+            ua_suffix = ''
+            if self.tool:
+                ua_suffix = ' - {}'.format(self.tool)
             req.auth = self.auth
             prepped = s.prepare_request(req)
             prepped.headers.update(
                 {'Authorization': self.key,
                  'Accept': 'application/{}'.format(output_type),
                  'content-type': 'application/{}'.format(output_type),
-                 'User-Agent': 'PyMISP {} - Python {}.{}.{}'.format(__version__, *sys.version_info)})
+                 'User-Agent': 'PyMISP {} - Python {}.{}.{}{}'.format(__version__, sys.version_info[0], sys.version_info[1], sys.version_info[2], ua_suffix)})
             if logger.isEnabledFor(logging.DEBUG):
                 logger.debug(prepped.headers)
             settings = s.merge_environment_settings(req.url, proxies=self.proxies or {}, stream=None, verify=self.ssl, cert=self.cert)

tests/test_reportlab.py

@@ -10,6 +10,9 @@
 
 manual_testing = False
 
+import logging
+logging.disable(logging.CRITICAL)
+
 try:
     from pymisp.tools import reportlab_generator
 except Exception: