fake fix TU

TODO

@@ -1,3 +1,10 @@
+Prio 1: use test ctypes for TU instead of pretty zeus and stuff.
+ensure reversers are actually doing their jobs
+
+
+Fix the Double pointer linked list reversers on a ctypes6 TU.
+    It does not capture head and tails with null pointers as same type.
+
 
 
 List of heuristics:
@@ -11,9 +18,9 @@ List of heuristics:
 - PointerGraphReverser: Create graph of records using pointer as edges
 - ArrayFieldsReverser: TODO
 - InlineRecordReverser: TODO (sub record type)
-- TypeReverser: TODO compare records signatures to match types.
+- TypeReverser: Done
 - CommonTypeReverser: Used by x2LinkedReverser to determine best types from a list of similar record
-- WriteRecordToFile: TODO save to file
+- WriteRecordToFile: done
 
 FIX the predecessors search. Most strings pointer in example are starting non-aligned <address> + 1.
 Because the size is on <address>
@@ -26,7 +33,7 @@ And maybe we need to reverse the aligned check on DSA FieldReverser, or create a
 1) integrate pattern and signature modules in the reverse API
 2) add a Reverser that takes and search for known records types. - Done
 
-# apply librairies structures/know structures search against heap before running anything else.
+# apply libraries structures/know structures search against heap before running anything else.
 # that should cut our reverse time way down
 # prioritize search per reverse struct size. limit to big struct and their graph.
 # then make a Named/Anonymous Struct out of them
@@ -131,13 +138,10 @@ Todo:
  - Add a struct beautifier for the string formatting. (hex vs int)
  - easy API
  - documented example
- - Rekall plugin
- - Rekall memorymap
  - make basicmodel:loadable members work with vtypes (vol/rekall) ?
  - Check why pdfbparse reports some gaps in structs gap_in_pdb_ofs_3C (HEAP)
  - pylint ignore W0212 in profiles.
  - add PyQt4 as dependency for optional functions
  - Add to ipython
- - Make reverse work againe
 
 

haystack/reverse/api.py

@@ -1,19 +1,20 @@
 # -*- coding: utf-8 -*-
 
 from __future__ import print_function
+
 import logging
 
 from haystack.abc import interfaces
 
-import reverse.heuristics.graph
 from haystack.reverse import config
 from haystack.reverse import context
-from haystack.reverse.heuristics import reversers
+from haystack.reverse.heuristics import constraints
 from haystack.reverse.heuristics import dsa
+from haystack.reverse.heuristics import graph
+from haystack.reverse.heuristics import model
 from haystack.reverse.heuristics import pointertypes
+from haystack.reverse.heuristics import reversers
 from haystack.reverse.heuristics import signature
-from haystack.reverse.heuristics import constraints
-from haystack.reverse.heuristics import model
 
 log = logging.getLogger('reverse.api')
 
@@ -78,29 +79,26 @@ def reverse_instances(memory_handler):
     """
     assert isinstance(memory_handler, interfaces.IMemoryHandler)
     process_context = memory_handler.get_reverse_context()
-    #for heap in heaps:
-    #    # reverse all fields in all records from that heap
-    #    ## reverse_heap(memory_handler, heap_addr)
 
     log.info('Reversing Fields')
-    fr = dsa.FieldReverser(memory_handler)
+    fr = dsa.FieldReverser(memory_handler)  # 10
     fr.reverse()
 
     log.info('Fixing Text Fields')
-    tfc = dsa.TextFieldCorrection(memory_handler)
+    tfc = dsa.TextFieldCorrection(memory_handler)  # 11
     tfc.reverse()
 
     # try to find some logical constructs.
     log.info('Reversing DoubleLinkedListReverser')
     # why is this a reverse_context ?
-    doublelink = reversers.DoubleLinkedListReverser(memory_handler)
+    doublelink = reversers.DoubleLinkedListReverser(memory_handler)  # 30
     doublelink.reverse()
     doublelink.rename_all_lists()
 
     # then and only then can we look at the PointerFields
     # identify pointer relation between allocators
     log.info('Reversing PointerFields')
-    pfr = pointertypes.PointerFieldReverser(memory_handler)
+    pfr = pointertypes.PointerFieldReverser(memory_handler)  # 50
     pfr.reverse()
 
     # save that
@@ -110,7 +108,7 @@ def reverse_instances(memory_handler):
 
     # then we try to match similar record type together
     log.info('Grouping records with a similar signature')
-    sig_type_rev = signature.TypeReverser(memory_handler)
+    sig_type_rev = signature.TypeReverser(memory_handler)  # 300
     sig_type_rev.reverse()
 
     # save that
@@ -120,17 +118,17 @@ def reverse_instances(memory_handler):
 
     # then we gather the value space of fields for each record, grouped by similar signature
     log.info('Saving reversed records types')
-    fvrr = constraints.FieldValueRangeReverser(memory_handler, output_to_file=True)
+    fvrr = constraints.FieldValueRangeReverser(memory_handler, output_to_file=True)  # 350
     fvrr.reverse()
 
     # graph pointer relations between allocators
     log.info('Reversing PointerGraph')
-    ptrgraph = reverse.heuristics.graph.PointerGraphReverser(memory_handler)
+    ptrgraph = graph.PointerGraphReverser(memory_handler)  # 150
     ptrgraph.reverse()
 
     # extract all strings
     log.info('Reversing strings')
-    strout = reversers.StringsReverser(memory_handler)
+    strout = reversers.StringsReverser(memory_handler)  # 500
     strout.reverse()
 
     log.info('Analysis results are in %s', config.get_cache_folder_name(memory_handler.get_name()))

haystack/reverse/context.py

@@ -14,9 +14,9 @@
 from haystack.abc import interfaces
 from past.builtins import long
 
-import reverse.heuristics.graph
 from haystack.reverse import config
 from haystack.reverse import enumerators
+from haystack.reverse.heuristics import graph
 from haystack.reverse import matchers
 from haystack.reverse import searchers
 from haystack.reverse import structure
@@ -42,37 +42,7 @@ def __init__(self, memory_handler):
         # no need for that
         # create the cache folder then
         self.create_cache_folders()
-
-    # def get_functions_pointers(self):
-    #     try:
-    #         return self.get_cache_radare()
-    #     except IOError as e:
-    #         return self.save_cache_radare()
-    #
-    # def get_cache_radare(self):
-    #     dumpname = self.memory_handler.get_name()
-    #     fname = config.get_cache_filename(config.CACHE_FUNCTION_NAMES, dumpname)
-    #     functions = None
-    #     try:
-    #         with file(fname, 'r') as fin:
-    #             functions = pickle.load(fin)
-    #     except EOFError as e:
-    #         os.remove(fname)
-    #         log.error('Error in the radare cache file. File cleaned. Please restart.')
-    #         raise RuntimeError('Error in the radare cache file. File cleaned. Please restart.')
-    #     return functions
-    #
-    # def save_cache_radare(self):
-    #     from haystack.reverse.heuristics import radare
-    #     func = radare.RadareAnalysis(self.memory_handler)
-    #     func.init_all_functions()
-    #     import code
-    #     code.interact(local=locals())
-    #     dumpname = self.memory_handler.get_name()
-    #     fname = config.get_cache_filename(config.CACHE_FUNCTION_NAMES, dumpname)
-    #     with file(fname, 'w') as fout:
-    #         pickle.dump(func.functions, fout)
-    #     return func.functions
+        self.reverse_level = 0
 
     def create_cache_folders(self):
         """Removes the cache folder"""
@@ -97,11 +67,27 @@ def get_context_for_heap_walker(self, walker):
             raise TypeError('heap should be a IHeapWalker')
         heap_address = walker.get_heap_address()
         if heap_address not in self.__contextes:
-            heap_context = self.make_context_for_heap_walker(walker)
+            heap_context = self.__make_context_for_heap_walker(walker)
             self._set_context_for_heap_walker(walker, heap_context)
             return heap_context
         return self.__contextes[heap_address]
 
+    def __make_context_for_heap_walker(self, walker):
+        """
+        Make the HeapContext for this heap walker.
+        This will reverse all user allocations from this HEAP into records.
+        """
+        heap_addr = walker.get_heap_address()
+        try:
+            ctx = HeapContext.cacheLoad(self.memory_handler, heap_addr)
+            log.debug("Cache avoided HeapContext initialisation")
+        except IOError as e:
+            # heaps are already generated at initialisation of self
+            mapping = self.memory_handler.get_mapping_for_address(heap_addr)
+            walker = self.memory_handler.get_heap_finder().get_heap_walker(mapping)
+            ctx = HeapContext(self.memory_handler, walker)
+        return ctx
+
     def get_context_for_address(self, address):
         """
         Returns the haystack.reverse.context.HeapContext of the process
@@ -118,29 +104,15 @@ def get_context_for_address(self, address):
         heap_context = self.get_context_for_heap_walker(walker)
         return heap_context
 
-    def make_context_for_heap_walker(self, walker):
-        """
-        Make the HeapContext for this heap walker.
-        This will reverse all user allocations from this HEAP into records.
-        """
-        heap_addr = walker.get_heap_address()
-        try:
-            ctx = HeapContext.cacheLoad(self.memory_handler, heap_addr)
-            log.debug("Cache avoided HeapContext initialisation")
-        except IOError as e:
-            # heaps are already generated at initialisation of self
-            mapping = self.memory_handler.get_mapping_for_address(heap_addr)
-            walker = self.memory_handler.get_heap_finder().get_heap_walker(mapping)
-            ctx = HeapContext(self.memory_handler, walker)
-        return ctx
-
     def list_contextes(self):
         """Returns all known HeapContext"""
         return self.__contextes.values()
 
     def _load_graph_cache(self):
-        from haystack.reverse.heuristics import reversers
-        graph_rev = reverse.heuristics.graph.PointerGraphReverser(self.memory_handler)
+        graph_rev = graph.PointerGraphReverser(self.memory_handler)
+        # if self.reverse_level < graph_rev.REVERSE_LEVEL:
+        #     # we need to reverse the process FIXME
+        #     raise RuntimeError('Please reverse the process first')
         self.__record_graph = graph_rev.load_process_graph()
 
     def get_predecessors(self, record):
@@ -155,7 +127,7 @@ def get_predecessors(self, record):
         predecessors_label = self.__record_graph.predecessors(hex(record.address))
         records = []
         for label in predecessors_label:
-            # FIXME, eradicate all L for PY3 migration
+            # eradicate all L for PY3 migration
             if label[-1] == 'L':
                 label = label[:-1]
             record_addr = int(label, 16)
@@ -167,6 +139,14 @@ def get_filename_cache_headers(self):
         dumpname = self.memory_handler.get_name()
         return config.get_cache_filename(config.CACHE_GENERATED_PY_HEADERS_VALUES, dumpname)
 
+    def _get_reverse_level(self):
+        return self.__reverse_level
+
+    def _set_reverse_level(self, level):
+        self.__reverse_level = level
+
+    reverse_level = property(_get_reverse_level, _set_reverse_level, None)
+
 
 class HeapContext(object):
     """
@@ -222,13 +202,6 @@ def _init2(self):
         heap_mapping = self.memory_handler.get_mapping_for_address(self._heap_start)
         finder = self.memory_handler.get_heap_finder()
         self.walker = finder.get_heap_walker(heap_mapping)
-
-
-        #if self.memory_handler.get_target_platform().get_os_name() not in ['winxp', 'win7']:
-        #    log.info('[+] Reversing function pointers names')
-        #    # TODO in reversers
-        #    # dict(libdl.reverseLocalFonctionPointerNames(self) )
-        #    self._function_names = dict()
         return
 
     def _is_record_cache_dirty(self):
@@ -501,14 +474,3 @@ def get_context_for_address(memory_handler, address):
     for the HEAP that hosts this address
     """
     return memory_handler.get_reverse_context().get_context_for_address(address)
-    #assert isinstance(address, long) or isinstance(address, int)
-    #heap_mapping = memory_handler.get_mapping_for_address(address)
-    #if not heap_mapping:
-    #    raise ValueError("Invalid address: 0x%x", address)
-    #finder = memory_handler.get_heap_finder()
-    # walker = finder.get_heap_walker(heap_mapping)
-    # if not walker:
-    #    raise ValueError("Address is not in heap: 0x%x", address)
-    # _context = memory_handler.get_reverse_context()
-    # heap_context = _context.get_context_for_heap_walker(walker)
-    # return heap_context

haystack/reverse/fieldtypes.py

@@ -300,7 +300,7 @@ def to_string(self):
             raise TypeError('This is not reachable')
         else:
             # unknown
-            comment = '# %s else' % (self.comment )
+            comment = '# %s else' % self.comment
         # prep the string
         fstr = "( '%s' , %s ), %s\n" % (self.name, self.get_typename(), comment)
         return fstr
@@ -431,5 +431,8 @@ def __init__(self, field_name, offset, field_type_name, fields):
 
     def to_string(self):
         comment = '# field struct %s' % self.type_name
-        fstr = "( '%s' , %s ), %s\n" % (self.name, self.type_name, comment)
+        fstr = "( '%s' , %s ), %s\n" % (self.name, self.get_typename(), comment)
         return fstr
+
+    def get_typename(self):
+        return self.__type_name

haystack/reverse/heuristics/graph.py

@@ -1,17 +1,19 @@
+import collections
+import logging
 import os
 import time
-import collections
-
-from reverse import config
-from reverse import context
-from reverse.heuristics import model
-from reverse.heuristics.reversers import log
-from haystack.reverse import utils
 
+import matplotlib.pyplot as plt
 import networkx
 from networkx.drawing.nx_agraph import graphviz_layout
-import matplotlib.pyplot as plt
 
+from haystack.reverse import config
+from haystack.reverse import context
+from haystack.reverse import utils
+from haystack.reverse.heuristics import model
+from haystack.reverse.heuristics.reversers import log
+
+log = logging.getLogger("graph")
 
 class PointerGraphReverser(model.AbstractReverser):
     """
@@ -61,7 +63,7 @@ def reverse_context(self, _context):
         # save the whole graph
         networkx.readwrite.gexf.write_gexf(self._graph, _context.get_filename_cache_graph())
         # save the graph pruned of less connected nodes
-        self.clean_graph(self._graph)
+        self.clean_graph(_context, self._graph)
         networkx.readwrite.gexf.write_gexf(self._graph, _context.get_filename_cache_graph_connected())
         # save a PNG of that
         self.print_graph(self._graph, _context.get_filename_cache_graph_png())

haystack/reverse/heuristics/model.py

@@ -172,7 +172,7 @@ def reverse_record(self, _context, _record):
         _record.saveme(_context)
 
     def _write(self):
-        self._fout.write(b'\n'.join(self._towrite))
+        self._fout.write('\n'.join(self._towrite))
         self._fout.flush()
         self._towrite = []
         pass