Merge pull request #87 from knightsc/tasks/modularize-sandbox-support

Modularize sandbox support

README.md

@@ -22,4 +22,9 @@ Pastehunter supports several output modules:
  - Dump to CSV file.
  - Send to syslog.
 
+ ## Supported Sandboxes
+ Pastehunter supports several sandboxes that decoded data can be sent to:
+ - Cuckoo
+ - Viper
+
 For examples of data discovered using pastehunter check out my posts https://techanarchy.net/blog/hunting-pastebin-with-pastehunter and https://techanarchy.net/blog/pastehunter-the-results

docs/postprocess.rst

@@ -29,20 +29,7 @@ when the full paste is a base64 blob, i.e. it will not extract base64 code that
 
 - **rule_list**: List of rules that will trigger the postprocess module. 
 
-
-Cuckoo
-^^^^^^
-If the samples match a binary file format you can optionaly send the file for analysis by a Cuckoo Sandbox.
-
-- **api_host**: IP or hostname for a Cuckoo API endpoint. 
-- **api_port**: Port number for a Cuckoo API endpoint.
-
-Viper
-^^^^^
-If the samples match a binary file format you can optionaly send the file to a Viper instance for further analysis.
-
-- **api_host**: IP or hostname for a Cuckoo API endpoint. 
-- **api_port**: Port number for a Cuckoo API endpoint.
+See the `Sandboxes documentation <sandboxes.rst>`_ for information on how to configure the sandboxes used for scanning decoded base64 data.
 
 
 Entropy

docs/sandboxes.rst

@@ -0,0 +1,25 @@
+Sandboxes
+=========
+
+There are a few sandboxes that can be configured and used in various post process steps.
+
+There are a few generic options for each input.
+
+- **enabled**: This turns the sandbox on and off. 
+- **module**: This is used internally by pastehunter.
+
+Cuckoo
+------
+
+If the samples match a binary file format you can optionaly send the file for analysis by a Cuckoo Sandbox.
+
+- **api_host**: IP or hostname for a Cuckoo API endpoint. 
+- **api_port**: Port number for a Cuckoo API endpoint.
+
+Viper
+-----
+
+If the samples match a binary file format you can optionaly send the file to a Viper instance for further analysis.
+
+- **api_host**: IP or hostname for a Viper API endpoint. 
+- **api_port**: Port number for a Viper API endpoint.

pastehunter.py

@@ -157,7 +157,7 @@ def paste_scanner():
             sleep(0.5)
         else:
             paste_data = q.get()
-            with timeout(seconds=10):
+            with timeout(seconds=conf['general']['process_timeout']):
                 # Start a timer
                 start_time = time.time()
                 logger.debug("Found New {0} paste {1}".format(paste_data['pastesite'], paste_data['pasteid']))
@@ -237,6 +237,17 @@ def paste_scanner():
                     # Else use the rule name
                     else:
                         results.append(match.rule)
+
+                # Store additional fields for passing on to post processing
+                encoded_paste_data = raw_paste_data.encode('utf-8')
+                md5 = hashlib.md5(encoded_paste_data).hexdigest()
+                sha256 = hashlib.sha256(encoded_paste_data).hexdigest()
+                paste_data['MD5'] = md5
+                paste_data['SHA256'] = sha256
+                paste_data['raw_paste'] = raw_paste_data
+                paste_data['YaraRule'] = results
+                # Set the size for all pastes - This will override any size set by the source
+                paste_data['size'] = len(raw_paste_data)
 
                 # Store all OverRides other options. 
                 paste_site = paste_data['confname']
@@ -282,21 +293,6 @@ def paste_scanner():
                         results.append('no_match')
 
                 if len(results) > 0:
-
-                    encoded_paste_data = raw_paste_data.encode('utf-8')
-                    md5 = hashlib.md5(encoded_paste_data).hexdigest()
-                    sha256 = hashlib.sha256(encoded_paste_data).hexdigest()
-                    paste_data['MD5'] = md5
-                    paste_data['SHA256'] = sha256
-                    # It is possible a post module modified or set this field.
-                    if not paste_data.get('raw_paste'):
-                        paste_data['raw_paste'] = raw_paste_data
-                        paste_data['size'] = len(raw_paste_data)
-                    else:
-                        # Set size based on modified value
-                        paste_data['size'] = len(paste_data['raw_paste'])
-
-                    paste_data['YaraRule'] = results
                     for output in outputs:
                         try:
                             output.store_paste(paste_data)

postprocess/post_b64.py

@@ -1,9 +1,8 @@
-import io
 import re
 import hashlib
+import importlib
 import gzip
 import logging
-import requests
 from base64 import b64decode
 # This gets the raw paste and the paste_data json object
 from common import parse_config
@@ -45,6 +44,7 @@ def run(results, raw_paste_data, paste_object):
                     paste_object["decompressed_stream"] = encoded
                 except Exception as e:
                     logger.error("Unable to decompress gzip stream")
+
             if rule == 'b64_exe':
                 try:
                     raw_exe = b64decode(raw_paste_data)
@@ -55,47 +55,18 @@ def run(results, raw_paste_data, paste_object):
                     # We are guessing that the sample has been submitted, and crafting a URL
                     paste_object["VT"] = 'https://www.virustotal.com/#/file/{0}'.format(paste_object["exe_md5"])
 
-                    # Cuckoo
-                    if conf["post_process"]["post_b64"]["cuckoo"]["enabled"]:
-                        logger.info("Submitting to Cuckoo")
-                        try:
-                            task_id = send_to_cuckoo(raw_exe, paste_object["pasteid"])
-                            paste_object["Cuckoo Task ID"] = task_id
-                            logger.info("exe submitted to Cuckoo with task id {0}".format(task_id))
-                        except Exception as e:
-                            logger.error("Unabled to submit sample to cuckoo")
-
-                    # Viper
-                    if conf["post_process"]["post_b64"]["viper"]["enabled"]:
-                        send_to_cuckoo(raw_exe, paste_object["pasteid"])
-
-                    # VirusTotal
+                    # If sandbox modules are enabled then submit the file
+                    for sandbox, sandbox_values in conf["sandboxes"].items():
+                        if sandbox_values["enabled"]:
+                            logger.info("Uploading file {0} using {1}".format(paste_object["pasteid"], sandbox_values["module"]))
+                            sandbox_module = importlib.import_module(sandbox_values["module"])
+                            paste_object = sandbox_module.upload_file(raw_exe, paste_object)
 
                 except Exception as e:
                     logger.error("Unable to decode exe file")
 
-
     # Get unique domain count
     # Update the json
 
     # Send the updated json back
     return paste_object
-
-
-def send_to_cuckoo(raw_exe, pasteid):
-    cuckoo_ip = conf["post_process"]["post_b64"]["cuckoo"]["api_host"]
-    cuckoo_port = conf["post_process"]["post_b64"]["cuckoo"]["api_port"]
-    cuckoo_host = 'http://{0}:{1}'.format(cuckoo_ip, cuckoo_port)
-    submit_file_url = '{0}/tasks/create/file'.format(cuckoo_host)
-    files = {'file': ('{0}.exe'.format(pasteid), io.BytesIO(raw_exe))}
-    submit_file = requests.post(submit_file_url, files=files).json()
-    task_id = None
-    try:
-        task_id = submit_file['task_id']
-    except KeyError:
-        try:
-            task_id = submit_file['task_ids'][0]
-        except KeyError:
-            logger.error(submit_file)
-
-    return task_id

sandboxes/cuckoo.py

@@ -0,0 +1,36 @@
+import io
+import logging
+import requests
+from common import parse_config
+conf = parse_config()
+
+logger = logging.getLogger('pastehunter')
+
+def upload_file(raw_file, paste_object):
+    try:
+        task_id = send_to_cuckoo(raw_file, paste_object["pasteid"])
+        paste_object["Cuckoo Task ID"] = task_id
+        logger.info("exe submitted to Cuckoo with task id {0}".format(task_id))
+    except Exception as e:
+        logger.error("Unabled to submit sample to cuckoo")
+
+    # Send any updated json back
+    return paste_object
+
+def send_to_cuckoo(raw_exe, pasteid):
+    cuckoo_ip = conf["sandboxes"]["cuckoo"]["api_host"]
+    cuckoo_port = conf["sandboxes"]["cuckoo"]["api_port"]
+    cuckoo_host = 'http://{0}:{1}'.format(cuckoo_ip, cuckoo_port)
+    submit_file_url = '{0}/tasks/create/file'.format(cuckoo_host)
+    files = {'file': ('{0}.exe'.format(pasteid), io.BytesIO(raw_exe))}
+    submit_file = requests.post(submit_file_url, files=files).json()
+    task_id = None
+    try:
+        task_id = submit_file['task_id']
+    except KeyError:
+        try:
+            task_id = submit_file['task_ids'][0]
+        except KeyError:
+            logger.error(submit_file)
+
+    return task_id

sandboxes/viper.py

@@ -0,0 +1,19 @@
+import io
+import logging
+import requests
+from common import parse_config
+conf = parse_config()
+
+logger = logging.getLogger('pastehunter')
+
+def upload_file(raw_file, paste_object):
+    viper_ip = conf["sandboxes"]["viper"]["api_host"]
+    viper_port = conf["sandboxes"]["viper"]["api_port"]
+    viper_host = 'http://{0}:{1}'.format(viper_ip, viper_port)
+
+    submit_file_url = '{0}/tasks/create/file'.format(viper_host)
+    files = {'file': ('{0}.exe'.format(paste_object["pasteid"]), io.BytesIO(raw_file))}
+    submit_file = requests.post(submit_file_url, files=files).json()
+
+    # Send any updated json back
+    return paste_object

settings.json.sample

@@ -153,7 +153,22 @@
     "format": "%(asctime)s [%(threadName)-12.12s] %(levelname)s:%(message)s"
   },
   "general": {
-    "run_frequency": 300
+    "run_frequency": 300,
+    "process_timeout": 5
+  },
+  "sandboxes": {
+    "cuckoo": {
+      "enabled": false,
+      "module": "sandboxes.cuckoo",
+      "api_host": "127.0.0.1",
+      "api_port": 8080
+    },
+    "viper": {
+      "enabled": false,
+      "module": "sandboxes.viper",
+      "api_host": "127.0.0.1",
+      "api_port": 8080
+    }
   },
   "post_process": {
     "post_email": {
@@ -164,17 +179,7 @@
     "post_b64": {
       "enabled": true,
       "module": "postprocess.post_b64",
-      "rule_list": ["b64_exe", "b64_rar", "b64_zip", "b64_gzip"],
-      "cuckoo": {
-        "enabled": false,
-        "api_host": "127.0.0.1",
-        "api_port": 8080
-      },
-      "viper": {
-        "enabled": false,
-        "api_host": "127.0.0.1",
-        "api_port": 8080
-      }
+      "rule_list": ["b64_exe", "b64_rar", "b64_zip", "b64_gzip"]
     },
     "post_entropy": {
       "enabled": false,