working on better detection of heaps and winxp heap.

TODO

@@ -1,8 +1,9 @@
-
+- use PEB search to double check that we find all HEAPs in standard scenarios.
 - FIX winXP x86 heapwalker, then work on vol.
 
 - use pycallgraph to cProfile a HEAP validation.
 - make a callback profiler that profiles the graph path validation of a structure in graphical format
+- using a decorator would be fun
 
 ***** heap walker validator is not a validator all by itself.
 

haystack/basicmodel.py

@@ -36,7 +36,7 @@ def get_field_type(record, fieldname):
 
 def get_fields(record):
     if not isinstance(record, ctypes.Structure) and not isinstance(record, ctypes.Union):
-        raise TypeError('Feed me a ctypes record instance')
+        raise TypeError('Feed me a ctypes record instance. Not: %s'% record)
     return get_record_type_fields(type(record))
 
 def get_record_type_fields(record_type):

haystack/listmodel.py

@@ -342,17 +342,39 @@ def _iterate_list_from_field_with_link_info(self, record, link_info):
         # stop at the first sign of a previously found list entry
         done = [s for s in sentinels] + [head_address]
         # log.info('Ignore headAddress self.%s at 0x%0.8x'%(fieldname, headAddr))
+        log.debug("_iterate_list_from_field_with_link_info from %s at offset %d->%d", fieldname, offset, self._ctypes.sizeof(pointee_record_type))
+        return self._iterate_list_from_field_inner(iterator_fn, head, pointee_record_type, offset, done)
+
+    def _iterate_list_from_field_inner(self, iterator_fn, head, pointee_record_type, offset, sentinels):
+        """
+        Use iterator_fn to iterate on the list structures. (as in struct__LIST_ENTRY)
+        For each list entry address:
+            + verify if the entry address is a sentinel value or already parsed.
+            + calculate <list_entry_address> - offset to find the real list entry address
+            + check if its in cache and yield it
+            + otherwise, if its a valid address, read the record from the new address
+            + save it to cache
+            + yield it.
+
+        :param iterator_fn: an iterator on a registered list entry management structure
+        :param head: the first list entry
+        :param pointee_record_type: the list entry record type we want to load
+        :param offset: the offset between the list entry pointers and the interesting record
+        :param sentinels: values that indicates we should stop iterating.
+        :return: pointee_record_type()
+        """
         # we get all addresses for the instances of the double linked list record_type
         # not, the list_member itself. Only the address of the double linked list field.
-        log.debug("_iterate_list_from_field_with_link_info from %s at offset %d->%d", fieldname, offset, self._ctypes.sizeof(pointee_record_type))
-        for entry in iterator_fn(head):
+        for entry in iterator_fn(head, sentinels):
+            # @ of the list member record
+            list_member_address = entry - offset
             # entry is not null. We also ignore record (head_address).
-            if entry in done:
+            if entry in sentinels:
+                continue
+            elif list_member_address in sentinels:
                 continue
             # save it
-            done.append(entry)
-            # @ of the list member record
-            list_member_address = entry - offset
+            sentinels.append(list_member_address)
             # log.info('Read %s at 0x%0.8x instead of 0x%0.8x'%(fieldname, link, entry))
             # use cache if possible, avoid loops.
             st = self._memory_handler.getRef(pointee_record_type, list_member_address)
@@ -373,7 +395,7 @@ def _iterate_list_from_field_with_link_info(self, record, link_info):
         raise StopIteration
 
     # FIXME add sentinels.
-    def _iterate_double_linked_list(self, record):
+    def _iterate_double_linked_list(self, record, sentinels=None):
         """
         iterate forward, then backward, until null or duplicate
 
@@ -384,12 +406,13 @@ def _iterate_double_linked_list(self, record):
         done = [0]
         obj = record
         record_type = type(record)
-        forward, backward, sentinels = self.get_double_linked_list_type(record_type)
-        # log.debug("sentinels %s", str([hex(s) for s in sentinels]))
+        # we ignore the sentinels here, as this is an internal iterator
+        forward, backward, _ = self.get_double_linked_list_type(record_type)
+        log.debug("sentinels %s", str([hex(s) for s in sentinels]))
         for fieldname in [forward, backward]:
             link = getattr(obj, fieldname)
             addr = self._utils.get_pointee_address(link)
-            log.debug('_iterate_double_linked_list got a <%s>/0x%x', link.__class__.__name__, addr)
+            log.debug('_iterate_double_linked_list %s <%s>/0x%x', fieldname, link.__class__.__name__, addr)
             nb = 0
             while addr not in done and addr not in sentinels:
                 done.append(addr)
@@ -405,10 +428,11 @@ def _iterate_double_linked_list(self, record):
                 # next
                 link = getattr(st, fieldname)
                 addr = self._utils.get_pointee_address(link)
+                log.debug('_iterate_double_linked_list %s <%s>/0x%x', fieldname, link.__class__.__name__, addr)
             # log.debug('going backward after %x', addr)
         raise StopIteration
 
-    def _iterate_single_linked_list(self, record):
+    def _iterate_single_linked_list(self, record, sentinels=None):
         """
         iterate forward, until null or duplicate
 
@@ -419,11 +443,12 @@ def _iterate_single_linked_list(self, record):
         done = [0]
         obj = record
         record_type = type(record)
-        fieldname, sentinels = self.get_single_linked_list_type(record_type)
+        # we ignore the sentinels here as this is an internal iterator
+        fieldname, _ = self.get_single_linked_list_type(record_type)
         # log.debug("sentinels %s", str([hex(s) for s in sentinels]))
         link = getattr(obj, fieldname)
         addr = self._utils.get_pointee_address(link)
-        log.debug('_iterate_single_linked_list got a <%s>/0x%x', link.__class__.__name__, addr)
+        log.debug('_iterate_single_linked_list %s <%s>/0x%x', link.__class__.__name__, addr)
         nb = 0
         while addr not in done and addr not in sentinels:
             done.append(addr)
@@ -439,7 +464,7 @@ def _iterate_single_linked_list(self, record):
             # next
             link = getattr(st, fieldname)
             addr = self._utils.get_pointee_address(link)
-            # log.debug('going backward after %x', addr)
+            log.debug('_iterate_single_linked_list %s <%s>/0x%x', link.__class__.__name__, addr)
         raise StopIteration
 
     def is_valid(self, record):

haystack/outputters/python.py

@@ -6,6 +6,7 @@
 
 import logging
 import numbers
+import sys
 
 from haystack.outputters import Outputter
 from haystack import types
@@ -29,7 +30,12 @@ def parse(self, obj, prefix='', depth=50):
         try:
             obj_module_name = obj.__class__.__module__
             obj_class_name = obj.__class__.__name__
-            obj_module = self._model.get_pythoned_module(obj_module_name)
+            try:
+                obj_module = self._model.get_pythoned_module(obj_module_name)
+            except KeyError:
+                # FIXME - ctypes modules should not be in sys.modules. what about reloading?
+                self._model.build_python_class_clones(sys.modules[obj_module_name])
+                obj_module = self._model.get_pythoned_module(obj_module_name)
             my_class = getattr(obj_module, "%s_py" % obj_class_name)
         except AttributeError as e:
             log.warning('did you forget to register your python structures ?')
@@ -145,7 +151,7 @@ def toString(self, prefix='', maxDepth=10):
             return '#(- not printed by Excessive recursion - )'
         s = '{\n'
         if hasattr(self, '_ctype_'):
-            items = [n for n, t in basicmodel.get_fields(self._ctype_)]
+            items = [n for n, t in basicmodel.get_record_type_fields(self._ctype_)]
         else:
             log.warning('no _ctype_')
             items = [n for n in self.__dict__.keys() if n != '_ctype_']

haystack/search/api.py

@@ -31,6 +31,7 @@ def search_record(memory_handler, struct_type, search_constraints=None):
     my_searcher = searcher.RecordSearcher(memory_handler, search_constraints)
     return my_searcher.search(struct_type)
 
+#FIXME TODO change for results == ctypes
 def output_to_string(memory_handler, results):
     """
     Transform ctypes results in a string format

haystack/search/searcher.py

@@ -141,7 +141,7 @@ def _load_at(self, mem_map, address, struct_type, depth=99):
         # check if data matches
         if validator.load_members(instance, depth):
             # FIXME: should be if validator.is_valid(instance):
-            log.info("found instance %s @ 0x%lx", struct_type, address)
+            log.debug("found instance %s @ 0x%lx", struct_type, address)
             # do stuff with it.
             validated = True
         else:
@@ -167,7 +167,7 @@ class AnyOffsetRecordSearcher(RecordSearcher):
     allocated chunks of memory.
     """
 
-    def _search_in(self, mem_map, struct_type, nb=10, depth=99):
+    def _search_in(self, mem_map, struct_type, nb=10, depth=99, align=None):
         """
             Looks for structType instances in memory, using :
                 hints from structType (default values, and such)
@@ -181,8 +181,13 @@ def _search_in(self, mem_map, struct_type, nb=10, depth=99):
         # where do we look
         start = mem_map.start
         end = mem_map.end
-        # check the word size to use aligned words only
+        # pointer len for alignment
         plen = self._memory_handler.get_target_platform().get_word_size()
+        # # check the word size to use aligned words only
+        if align is None:
+            align = plen
+        else:
+            align = align - align % plen
         # the struct cannot fit after that point.
         my_ctypes = self._memory_handler.get_target_platform().get_target_ctypes()
         end = end - my_ctypes.sizeof(struct_type) + 1
@@ -195,7 +200,7 @@ def _search_in(self, mem_map, struct_type, nb=10, depth=99):
         t0 = time.time()
         p = 0
         # python 2.7 xrange doesn't handle long int. replace with ours.
-        for offset in utils.xrange(start, end, plen):
+        for offset in utils.xrange(start, end, align):
             # print a debug message every now and then
             if offset % (1024 << 6) == 0:
                 p2 = offset - start

haystack/structures/heapwalker.py

@@ -103,6 +103,8 @@ def _is_heap(self, mapping):
         # TODO: optimization. store heap status in object.
         if not isinstance(mapping, interfaces.IMemoryMapping):
             raise TypeError('Feed me a IMemoryMapping object')
+        # FIXME: the Heap is not necessary at @start of mapping.
+        # we find some backend heap at other addresses
         heap = self._read_heap(mapping)
         load = self.get_heap_validator().load_members(heap, self._heap_validation_depth)
         log.debug('HeapFinder._is_heap %s %s' % (mapping, load))

haystack/structures/win32/win7heap.constraints

@@ -28,6 +28,23 @@ SubSegmentCode: IgnoreMember
 # Starting with Windows Vista, this field was encoded using a random value that
 # was also stored as a field in the HEAP handle data structure.
 
+## Undertanding the LFH
+#EncodeFlagMask – A value that is used to determine if a heap chunk header is encoded.
+# This value is initially set to 0x100000 by RtlpCreateHeapEncoding() in RtlCreateHeap().
+#Encoding – Used in an XOR operation to encode the chunk headers, preventing
+# predictable meta-data corruption.
+#BlocksIndex – This is a _HEAP_LIST_LOOKUP structure that is used for a variety of
+# purposes. Due to its importance, it will be discussed in greater detail later in this
+# document.
+#FreeLists – A special linked-list that contains pointers to ALL of the free chunks for this
+# heap. It can almost be thought of as a heap cache, but for chunks of every size (and no
+# single associated bitmap).
+#FrontEndHeapType – An integer is initially set to 0x0, and is subsequently assigned a
+# value of 0x2, indicating the use of a LFH. Note: Windows 7 does not actually have
+# support for using Lookaside Lists.
+#FrontEndHeap – A pointer to the associated front-end heap. This will either be NULL or
+# a pointer to a _LFH_HEAP structure when running under Windows 7.
+
 #[HEAP]
 [struct__HEAP]
 Signature: [0xeeffeeff]

haystack/structures/win32/winheap.py

@@ -85,13 +85,16 @@ def HEAP_get_free_UCR_segment_list(self, record):
     def HEAP_get_chunks(self, record):
         """Returns a list of tuple(address,size) for all chunks in
          the backend allocator."""
+        # FIXME look at segment.LastEntryInSegment
         allocated = list()
         free = list()
         for segment in self.HEAP_get_segment_list(record):
             first_addr = self._utils.get_pointee_address(segment.FirstEntry)
             last_addr = self._utils.get_pointee_address(segment.LastValidEntry)
             # create the skip list for each segment.
             skiplist = dict()
+            # FIXME, in XP, ucrsegments is in HEAP
+            # in win7 ucrsegmentlist is in heap_segment
             for ucr in self.HEAP_SEGMENT_get_UCR_segment_list(segment):
                 ucr_addr = self._utils.get_pointee_address(ucr.Address)
                 # UCR.Size are not chunks sizes. NOT *8

haystack/structures/win32/winheapwalker.py

@@ -84,9 +84,10 @@ def get_heap_children_mmaps(self):
             for x, s in self._get_freelists():
                 m = self._memory_handler.get_mapping_for_address(x)
                 if (m != self._heap_mapping) and (m not in child_heaps):
+                    # FIXME, its actually a segment isn't it ?
                     log.debug(
-                        'mmap 0x%0.8x is extended heap space from 0x%0.8x' %
-                        (m.start, self._heap_mapping.start))
+                        'mmap 0x%0.8x is extended heap space from 0x%0.8x',
+                        m.start, self._heap_mapping.start)
                     child_heaps.add(m)
                     pass
             self._child_heaps = child_heaps
@@ -154,6 +155,6 @@ def _get_freelists(self):
                 self._heap)]
         freesize = sum([c[1] for c in free_lists])
         log.debug(
-            '\t+ freeLists: free: %0.4d [%0.5d B]' %
-            (len(free_lists), freesize))
+            '+ freeLists: nb_free_chunk:0x%0.4x total_size:0x%0.5x',
+            len(free_lists), freesize)
         return free_lists

haystack/structures/win32/winxpheap.constraints

@@ -12,6 +12,7 @@ SubSegmentCode: IgnoreMember
 
 [struct__HEAP]
 Signature: [0xeeffeeff]
+# 1 = LAL, 2 = LFH, 0 = Backend (chunks>= 16k)
 FrontEndHeapType: [0, 1, 2]
 # HEAP CommitRoutine encoded by a global key
 # The HEAP handle data structure includes a function pointer field called
@@ -24,27 +25,21 @@ Segments: IgnoreMember
 # found in wild DEBUG:memorybase:is_valid_address_value = bc63e310 False
 UnusedUnCommittedRanges: IgnoreMember
 VirtualAllocdBlocks: IgnoreMember
-# and a lot of freelists FreeLists[0], <class 'haystack.structures.win32.winxp_32.struct__LIST_ENTRY'>
+# FreeLists[n] could be in UCR ?
+FreeLists: IgnoreMember
 
-
-#    ('Entry', HEAP_ENTRY),
-#    ('HeaderValidateCopy', POINTER_T(None)),
-#    ('TagEntries', POINTER_T(struct__HEAP_TAG_ENTRY)),
-#    ('UCRSegments', POINTER_T(struct__HEAP_UCR_SEGMENT)),
-#    ('UnusedUnCommittedRanges', POINTER_T(struct__HEAP_UNCOMMMTTED_RANGE)),
-#    ('VirtualAllocdBlocks', LIST_ENTRY),
-#    ('Segments', POINTER_T(struct__HEAP_SEGMENT) * 64),
-#    ('u', union__HEAP_0),
-#    ('u2', union__HEAP_1),
-#    ('LargeBlocksIndex', POINTER_T(None)),
-#    ('PseudoTagEntries', POINTER_T(struct__HEAP_PSEUDO_TAG_ENTRY)),
-#    ('FreeLists', struct__LIST_ENTRY * 128),
-#    ('LockVariable', POINTER_T(struct__HEAP_LOCK)),
-#    ('CommitRoutine', POINTER_T(ctypes.CFUNCTYPE(ctypes.c_int32, POINTER_T(None), POINTER_T(POINTER_T(None)), POINTER_T(ctypes.c_uint32)))),
-#    ('FrontEndHeap', POINTER_T(None)),
-#    ('FrontHeapLockCount', ctypes.c_uint16),
-#    ('FrontEndHeapType', ctypes.c_ubyte),
-#    ('LastSegmentIndex', ctypes.c_ubyte),
+## Undertanding the LFH
+#LargeBlocksIndex/BlocksIndex – This is a _HEAP_LIST_LOOKUP structure that is used for a variety of
+# purposes. Due to its importance, it will be discussed in greater detail later in this
+# document.
+#FreeLists – A special linked-list that contains pointers to ALL of the free chunks for this
+# heap. It can almost be thought of as a heap cache, but for chunks of every size (and no
+# single associated bitmap).
+#FrontEndHeapType – An integer is initially set to 0x0, and is subsequently assigned a
+# value of 0x2, indicating the use of a LFH. Note: Windows 7 does not actually have
+# support for using Lookaside Lists.
+#FrontEndHeap – A pointer to the associated front-end heap. This will either be NULL or
+# a pointer to a _LFH_HEAP structure when running under Windows 7.
 
 
 [struct__HEAP_SEGMENT]
@@ -56,3 +51,7 @@ LastValidEntry: IgnoreMember
 #    ('LastValidEntry', POINTER_T(struct__HEAP_ENTRY)),
 #    ('UnCommittedRanges', POINTER_T(struct__HEAP_UNCOMMMTTED_RANGE)),
 #    ('LastEntryInSegment', POINTER_T(struct__HEAP_ENTRY)),
+
+# FIXME, why is 0xffffffff not used as a sentinels in
+[struct__ERESOURCE]
+SystemResourcesList: IgnoreMember