more cli work.

docs/Haystack reverse CLI.ipynb

@@ -748,7 +748,7 @@
     }
    ],
    "source": [
-    "!haystack --dumpname ../test/dumps/vol/zeus.vmem.856.dump --string refresh test.structures.zeus.records.array_of_wcharp 0xc31e90"
+    "!haystack --dumpname ../test/dumps/vol/zeus.vmem.856.dump --string refresh test.structures.zeus.records.array_of_wcharp 0xc31e90  "
    ]
   },
   {

haystack/dbg.py

@@ -3,6 +3,8 @@
 
 import logging
 import platform
+import time
+
 
 try:
     import ptrace
@@ -243,5 +245,20 @@ def get_debugger(pid):
         return MyWinAppDebugger(pid)
 
 
-class ProcessError(Exception):
-    pass
+def make_local_process_memory_handler(pid, use_mmap=True):
+    if not isinstance(pid, (int, long)):
+        raise TypeError('PID should be a number')
+    from haystack.mappings import process
+    my_debugger = get_debugger(pid)
+    _memory_handler = process.readProcessMappings(my_debugger.get_process())
+    t0 = time.time()
+    for m in _memory_handler:
+        if use_mmap:
+            # force to mmap the memory in local space
+            m.mmap()
+            log.debug('mmap() size:%d', len(m.mmap()))
+    if use_mmap:
+        # mmap done, we can release process...
+        my_debugger.get_process().resume()
+        log.info('MemoryHandler mmaped, process released after %02.02f secs', time.time() - t0)
+    return _memory_handler

haystack/mappings/rek.py

@@ -9,6 +9,7 @@
 """
 
 
+import os
 import sys
 import logging
 import struct
@@ -165,3 +166,51 @@ def make_memory_handler(self):
 
 # RekallProcessMapper('/home/other/outputs/vol/zeus.vmem', 856)
 # RekallProcessMapper('~/outputs/vol/victoria-v8.kcore.img', 1)
+
+def rekall_dump_to_haystack(filename, pid, output_folder_name):
+    # rek.py -f vol/zeus.vmem vaddump -p 856 --dump-dir vol/zeus.vmem.856.dump/ > vol/zeus.vmem.856.dump/mappings.vol
+    # rek2map.py vol/zeus.vmem.856.dump/mappings.vol > vol/zeus.vmem.856.dump/mappings
+    # vaddummp
+    log.debug("rekall_dump_to_haystack %s %p", filename, pid)
+    if not os.access(output_folder_name, os.F_OK):
+        os.mkdir(output_folder_name)
+    from rekall import session
+    from rekall import plugins
+    from rekall.ui import json_renderer
+    s = session.Session(
+            filename = filename,
+            autodetect=["rsds"],
+            logger=logging.getLogger(),
+            profile_path=[
+                "http://profiles.rekall-forensic.com"
+            ])
+
+    task_plugin = s.plugins.vaddump(pid=pid, dump_dir=output_folder_name)
+    # get a renderer.
+    renderer = json_renderer.JsonRenderer()
+    task_plugin.render(renderer)
+    print renderer
+    maps = []
+    # FIXME get stdout in here.
+    with open(filename,'r') as fin:
+        entries = fin.readlines()
+        i_start = entries[0].index('Start')
+        i_end = entries[0].index('End')
+        i_path = entries[0].index('Result')
+        fmt = b'0x%08x'
+        if i_end - i_start > 12:
+            fmt = b'0x%016x'
+        for i, line in enumerate(entries[2:]):
+            start = int(line[i_start:i_end].strip(), 16)
+            end = int(line[i_end:i_path].strip(), 16) + 1
+            path = line[i_path:].strip()
+            o_path = "%s-%s" % (fmt % start, fmt % end)
+            # rename file
+            try:
+                os.rename(path, o_path)
+            except OSError, e:
+                sys.stderr.write('File rename error\n')
+            # offset is unknown.
+            print '%s %s r-xp %s 00:00 %d [vol_mapping_%03d]' % (fmt % start, fmt % end, fmt % 0, 0, i)
+
+    pass

haystack/mappings/vol.py

@@ -11,20 +11,14 @@
 """
 
 import logging
+import sys
 import struct
 from functools import partial
 
 from haystack.mappings.base import MemoryHandler, AMemoryMapping
 from haystack.abc import interfaces
 from haystack import target
 
-__author__ = "Loic Jaquemet"
-__copyright__ = "Copyright (C) 2012 Loic Jaquemet"
-__email__ = "loic.jaquemet+python@gmail.com"
-__license__ = "GPL"
-__maintainer__ = "Loic Jaquemet"
-__status__ = "Production"
-__credits__ = ["Victor Skinner"]
 
 log = logging.getLogger('volmapping')
 
@@ -78,7 +72,6 @@ def read_array(self, addr, basetype, count):
     def reset(self):
         pass
 
-import sys
 
 class VolatilityProcessMapper(interfaces.IMemoryLoader):
 

haystack/memory_dumper.py

@@ -145,11 +145,8 @@ def argparser():
     return dump_parser
 
 
-def main(argv):
+def main():
     logging.basicConfig(level=logging.DEBUG)
     parser = argparser()
-    opts = parser.parse_args(argv)
+    opts = parser.parse_args(sys.argv[1:])
     opts.func(opts)
-
-if __name__ == '__main__':
-    main(sys.argv[1:])

haystack/search/api.py

@@ -17,29 +17,55 @@ class HaystackError(Exception):
     pass
 
 
-def search_record(memory_handler, struct_type, search_constraints=None, extended_search=False):
+def search_record(memory_handler, record_type, search_constraints=None, extended_search=False):
     """
     Search a record in the memory dump of a process represented
     by memory_handler.
 
-    the record type must have been imported using haystack functions.
+    The record type must have been imported using haystack functions.
 
-    if constraints exists, they will be considered during the search.
+    If constraints exists, they will be considered during the search.
 
     :param memory_handler: IMemoryHandler
-    :param struct_type: a ctypes.Structure or ctypes.Union from a module imported by haystack
+    :param record_type: a ctypes.Structure or ctypes.Union from a module imported by haystack
     :param search_constraints: IModuleConstraints to be considered during the search
     :param extended_search: boolean, use allocated chunks only per default (False)
     :rtype a list of (ctypes records, memory offset)
     """
     if extended_search:
         my_searcher = searcher.AnyOffsetRecordSearcher(memory_handler, search_constraints)
-        return my_searcher.search(struct_type)
+        return my_searcher.search(record_type)
     my_searcher = searcher.RecordSearcher(memory_handler, search_constraints)
-    return my_searcher.search(struct_type)
+    return my_searcher.search(record_type)
 
 
-#FIXME TODO change for results == ctypes
+def search_record_hint(memory_handler, record_type, hint, search_constraints=None, extended_search=False):
+    """
+    Search a record in the memory dump of a process, but only on the memory page containing the hinted address.
+
+    The record type must have been imported using haystack functions.
+
+    If constraints exists, they will be considered during the search.
+
+    :param memory_handler: IMemoryHandler
+    :param record_type: a ctypes.Structure or ctypes.Union from a module imported by haystack
+    :param search_constraints: IModuleConstraints to be considered during the search
+    :param extended_search: boolean, use allocated chunks only per default (False)
+    :rtype a list of (ctypes records, memory offset)
+    """
+    hint_mapping = memory_handler.get_mapping_for_address(hint)
+    if extended_search:
+        my_searcher = searcher.AnyOffsetRecordSearcher(memory_handler,
+                                                       my_constraints=search_constraints,
+                                                       target_mappings=[hint_mapping])
+        return my_searcher.search(record_type)
+    my_searcher = searcher.RecordSearcher(memory_handler,
+                                          my_constraints=search_constraints,
+                                          target_mappings=[hint_mapping])
+    return my_searcher.search(record_type)
+
+
+# FIXME TODO change for results == ctypes
 def output_to_string(memory_handler, results):
     """
     Transform ctypes results in a string format
@@ -57,6 +83,7 @@ def output_to_string(memory_handler, results):
     ret += ']'
     return ret
 
+
 def output_to_python(memory_handler, results):
     """
     Transform ctypes results in a non-ctypes python object format
@@ -81,6 +108,7 @@ def output_to_python(memory_handler, results):
             'Bug in framework, some Ctypes are still in the return results. Please Report test unit.')
     return ret
 
+
 def output_to_json(memory_handler, results):
     """
     Transform ctypes results in a json format
@@ -94,6 +122,7 @@ def output_to_json(memory_handler, results):
     # cirular refs kills it check_circular=False,
     return json.dumps(ret, default=python.json_encode_pyobj)
 
+
 def output_to_pickle(memory_handler, results):
     """
     Transform ctypes results in a pickled format
@@ -108,6 +137,7 @@ def output_to_pickle(memory_handler, results):
     #code.interact(local=locals())
     return pickle.dumps(ret)
 
+
 def load_record(memory_handler, struct_type, memory_address, load_constraints=None):
     """
     Load a record from a specific address in memory.
@@ -125,6 +155,7 @@ def load_record(memory_handler, struct_type, memory_address, load_constraints=No
     my_loader = searcher.RecordLoader(memory_handler, load_constraints, target_mappings=memory_handler.get_mappings())
     return my_loader.load(struct_type, memory_address)
 
+
 def validate_record(memory_handler, instance, record_constraints=None, max_depth=10):
     """
     Validate a loaded record against constraints.