Some work on #12 - but the reverse script is definitively not ready f…

…or master

CHANGES.txt

@@ -1,9 +1,6 @@
 Ongoing:
- - Volatility plugin
- - Volatility memorymap
- - merge in master
-
- - move interfaces into an interfaces module, maybe use abc.
+ - stabilisation of search
+ - refactorisation of reverse
 
 Changes to architecture.txt:
  - IMemoryHandler: there is a process memory concept (dump, live, vol, rekall) that need a handler.
@@ -19,14 +16,13 @@ Changes to architecture.txt:
  - the memory l
 
 
- - model should be an instance.
- - mapping should represent the memory dump entity
- - ctypes for internal haystack loading should be a Proxy ctypes, saved the mapping instance
-
 2015-09:
  - Working on Volatility dump reader
  - working a WinXP chunk iterator
  - adding PEB to winXP list of know structures
+ - Working volatility plugins in repo https://github.com/trolldbois/volatility_plugins
+ - winheap to work with heap at w.e. offset, not with heap aligned on mapping.start.
+ - fixed winXP x86 heapwalker
 
 2015-08:
  - Fixes for travis in setup.py
@@ -41,8 +37,7 @@ OK (skipped=7, expected failures=4)
  - start standardizing test dumps in a test file inventory
 Ran 240 tests in 119.036s
 FAILED (failures=2, errors=9, skipped=11, expected failures=3)
-
-
+ - model is an instance.
 
 
 2015-07:

TODO

@@ -1,4 +1,4 @@
-
+- keep the original address in the Ouputters, python output.
 
 - re-correlate the is_valid methods to the record type, so that advanced code-based
 validation is allowed on a record, on top of constraits
@@ -23,16 +23,11 @@ OK: python /home/other/Compil/python-haystack/scripts/haystack --debug --string
 
 
 - use PEB search to double check that we find all HEAPs in standard scenarios.
-- orient winheap to work with heap, not with heap aligned on mapping.start.
-
-- FIX winXP x86 heapwalker, then work on vol.
 
 - use pycallgraph to cProfile a HEAP validation.
 - make a callback profiler that profiles the graph path validation of a structure in graphical format
 - using a decorator would be fun
 
-***** heap walker validator is not a validator all by itself.
-
 - add a depth parameter to constraints loading on list fields.
 - separate listmodel in a constraints like config file ?
 
@@ -43,13 +38,9 @@ OK: python /home/other/Compil/python-haystack/scripts/haystack --debug --string
 
 - use ASN.1 for constraints. ??
 
-- FIX get_heap on windows XP/zeus demo. Extract a process from a vol dump to make a test case.
 - add a hintoffset search thingy, so that one can search for a structure at a particular offset of a mmap ? maybe to complex for API
 
-- make a listmodel method for arrays of structures
-- make the winxp freelists method to work.
-
-- use heapwalker plugins from rekall/volatility to create a proxy heapwalker ?
+- make a listmodel method for arrays of structures, or not.
 
 
 Attribute packed is not correct in ctypeslib
@@ -73,10 +64,11 @@ Todo:
  - documented example
  - Rekall plugin
  - Rekall memorymap
- - make basicmodel:loadable members work with vtypes ?
+ - make basicmodel:loadable members work with vtypes (vol/rekall) ?
  - Check why pdfbparse reports some gaps in structs gap_in_pdb_ofs_3C (HEAP)
  - pylint ignore W0212 in profiles.
  - add PyQt4 as dependency for optional functions
  - Add to ipython
+ - Make reverse work again
 
 

haystack/reverse/context.py

@@ -49,14 +49,14 @@ def _init2(self):
         #ptr_values, ptr_offsets, aligned_ptr, not_aligned_ptr = utils.getHeapPointers(self.dumpname, self._memory_handler)
 
         # FIXME: no use I think
-        ##heap_offsets, heap_values = utils.getHeapPointers(self.dumpname, self._memory_handler)
-        ##self._pointers_values_heap = heap_values
-        ##self._pointers_offsets_heap = heap_offsets
+        heap_offsets, heap_values = utils.getHeapPointers(self.dumpname, self.memory_handler)
+        self._pointers_values_heap = heap_values
+        self._pointers_offsets_heap = heap_offsets
 
         # test with all mmap in target
-        ##all_offsets, all_values = utils.getAllPointers(self.dumpname, self._memory_handler)
-        ##self._pointers_values = all_values
-        ##self._pointers_offsets = all_offsets
+        all_offsets, all_values = utils.getAllPointers(self.dumpname, self.memory_handler)
+        self._pointers_values = all_values
+        self._pointers_offsets = all_offsets
 
         if self.memory_handler.get_target_platform().get_os_name() not in ['winxp', 'win7']:
             log.info('[+] Reversing function pointers names')

haystack/reverse/reversers.py

@@ -501,7 +501,7 @@ def iterateList(self, context, head_addr):
 
     def findHead(self, ctx, members):
         sizes = sorted([(ctx.getStructureSizeForAddr(m), m) for m in members])
-        if sizes[0] < 3 * self.my_target.config.get_word_size():
+        if sizes[0] < 3 * self.my_target.get_word_size():
             log.error('a double linked list element must be 3 WORD at least')
             raise ValueError(
                 'a double linked list element must be 3 WORD at least')
@@ -654,6 +654,8 @@ def reverseInstances(dumpname):
         if not os.access(config.get_record_cache_folder_name(ctx.dumpname), os.F_OK):
             os.mkdir(config.get_record_cache_folder_name(ctx.dumpname))
 
+        log.info("[+] Cache created in %s", config.get_record_cache_folder_name(ctx.dumpname))
+
         # we use common allocators to find structures.
         #log.debug('Reversing malloc')
         #mallocRev = MallocReverser()

haystack/reverse/signature.py

@@ -11,8 +11,9 @@
 import re
 import Levenshtein  # seqmatcher ?
 import networkx
+import numpy
 
-from haystack.reverse.config import Config
+from haystack.reverse import config
 from haystack.utils import xrange
 from haystack.reverse import pointerfinder
 
@@ -43,8 +44,10 @@ def _init_signatures(self):
         # need to force resolve of structures
         self._signatures = []
         for addr in map(long, self._structures_addresses):
+            # decode the fields
             self._context.getStructureForAddr(
                 addr).decodeFields()  # can be long
+            # get the signature for the record
             self._signatures.append(
                 (addr, self._context.getStructureForAddr(addr).getSignature(True)))
         return
@@ -69,8 +72,8 @@ def make(self):
         return
 
     def persist(self):
-        outdir = Config.getCacheFilename(
-            Config.CACHE_SIGNATURE_GROUPS_DIR,
+        outdir = config.get_cache_filename(
+            config.CACHE_SIGNATURE_GROUPS_DIR,
             self._context.dumpname)
         if not os.path.isdir(outdir):
             os.mkdir(outdir)
@@ -82,14 +85,14 @@ def persist(self):
         return
 
     def isPersisted(self):
-        outdir = Config.getCacheFilename(
-            Config.CACHE_SIGNATURE_GROUPS_DIR,
+        outdir = config.get_cache_filename(
+            config.CACHE_SIGNATURE_GROUPS_DIR,
             self._context.dumpname)
         return os.access(os.path.sep.join([outdir, self._name]), os.F_OK)
 
     def load(self):
-        outdir = Config.getCacheFilename(
-            Config.CACHE_SIGNATURE_GROUPS_DIR,
+        outdir = config.get_cache_filename(
+            config.CACHE_SIGNATURE_GROUPS_DIR,
             self._context.dumpname)
         inname = os.path.sep.join([outdir, self._name])
         self._similarities = utils.int_array_cache(inname)
@@ -109,11 +112,11 @@ def __init__(self, ctx):
         self._sizes = None
 
     def _loadCache(self):
-        outdir = Config.getCacheFilename(
-            Config.CACHE_SIGNATURE_SIZES_DIR,
+        outdir = config.get_cache_filename(
+            config.CACHE_SIGNATURE_SIZES_DIR,
             self._context.dumpname)
         fdone = os.path.sep.join(
-            [outdir, Config.CACHE_SIGNATURE_SIZES_DIR_TAG])
+            [outdir, config.CACHE_SIGNATURE_SIZES_DIR_TAG])
         if not os.access(fdone, os.R_OK):
             return False
         for myfile in os.listdir(outdir):
@@ -127,8 +130,8 @@ def _loadCache(self):
     def cacheSizes(self):
         """Find the number of different sizes, and creates that much numpyarray"""
         # if not os.access
-        outdir = Config.getCacheFilename(
-            Config.CACHE_SIGNATURE_SIZES_DIR,
+        outdir = config.get_cache_filename(
+            config.CACHE_SIGNATURE_SIZES_DIR,
             self._context.dumpname)
         if not os.path.isdir(outdir):
             os.mkdir(outdir)
@@ -147,7 +150,7 @@ def cacheSizes(self):
         # saved all sizes dictionaries.
         # tag it as done
         file(
-            os.path.sep.join([outdir, Config.CACHE_SIGNATURE_SIZES_DIR_TAG]), 'w')
+            os.path.sep.join([outdir, config.CACHE_SIGNATURE_SIZES_DIR_TAG]), 'w')
         self._sizes = arrays
         return
 
@@ -409,8 +412,8 @@ def graphStructureGroups(context, chains, originAddr=None):
         print '#', '-' * 78
     networkx.readwrite.gexf.write_gexf(
         graph,
-        Config.getCacheFilename(
-            Config.CACHE_GRAPH,
+        config.get_cache_filename(
+            config.CACHE_GRAPH,
             context.dumpname))
 
 
@@ -494,12 +497,12 @@ def makeGroupSignature(context, sizeCache):
 # FIXME: 100 maybe is a bit short
 try:
     import pkgutil
-    _words = pkgutil.get_data(__name__, Config.WORDS_FOR_REVERSE_TYPES_FILE)
+    _words = pkgutil.get_data(__name__, config.WORDS_FOR_REVERSE_TYPES_FILE)
 except ImportError:
     import pkg_resources
     _words = pkg_resources.resource_string(
         __name__,
-        Config.WORDS_FOR_REVERSE_TYPES_FILE)
+        config.WORDS_FOR_REVERSE_TYPES_FILE)
 
 # global
 _NAMES = [s.strip() for s in _words.split('\n')[:-1]]