add reasonnable support for winxp allocator

TODO

@@ -1,44 +1,24 @@
 
-+ fix performance issues on test units
+- FIX winXP x86 heapwalker, then work on vol.
 
 ***** heap walker validator is not a validator all by itself.
 
 - add a depth parameter to constraints loading on list fields.
-- find out why its getting super slow in overall tests
 - separate listmodel in a constraints like config file ?
 
-- separate basicmodel from ctypes structures record. That will fix all the serialization errors.
-
-- Model should not depends on ctypesProxy or LoadablesMembers. It could do it on Ctypes.Structure/Union.
-- basicmodel should not be monkey patching ctypes.Structure for the sole purpose
-to add methods. Only cross arch helper is authorised. We should have a worker 
-class assessing a "normal" ctypes.Structure instance (modulo target arch)
 - a working str() on Ctypes.loadableMembers requires no instanciated utils
 
-- separated expectedValues from the record type. It should be in a separate dict, 
-that is passed to the basicmodel searcher at search time.
-constraints should not be applied on module/structure.
-
 - add an searcher configuration for padding value == 0 ( to be forwarded to basicmodel)
 - padding value should be 0 most of the time. make it configurable/monkeypatchable
 
-
-- use ASN.1 for constraints.
+- use ASN.1 for constraints. ??
 
 - FIX get_heap on windows XP/zeus demo. Extract a process from a vol dump to make a test case.
 - add a hintoffset search thingy, so that one can search for a structure at a particular offset of a mmap ? maybe to complex for API
 
 - make a listmodel method for arrays of structures
 - make the winxp freelists method to work.
 
-- FIX the model reloading of ctypes across all modules.
-    - unloading all reverse module is crude.
-    - resetting some haystack sub modules (reverse) seems to work
-
-- too many open files
-    - close mappings file in memory mappings.???
-    - clean all TU to clean memory mappings in bringDown
-
 - use heapwalker plugins from rekall/volatility to create a proxy heapwalker ?
 
 

haystack/mappings/vol.py

@@ -76,7 +76,7 @@ def read_array(self, addr, basetype, count):
 
 import sys
 
-
+@unittest.skipTest('not ready')
 class VolatilityProcessMapper(interfaces.IMemoryLoader):
 
     def __init__(self, imgname, pid):
@@ -183,7 +183,7 @@ def my_render_text(mapper, cmd, outfd, data):
 
             maps.append(pmap)
 
-    mappings = MemoryHandler(maps)
+    mappings = MemoryHandler(maps, )
     # print _memory_handler
     mappings.init_config()
     mapper.mappings = mappings

haystack/search/api.py

@@ -2,10 +2,10 @@
 
 import logging
 import pickle
+
 import json
 
 from haystack.search import searcher
-from haystack import constraints
 from haystack.outputters import text
 from haystack.outputters import python
 

haystack/search/searcher.py

@@ -38,7 +38,7 @@ def __init__(self, memory_handler, my_constraints=None, target_mappings=None, up
         self._memory_handler = memory_handler
         self._my_constraints = my_constraints
         self.__target_mappings = target_mappings
-        self.__update_cb = update_cb
+        self._update_cb = update_cb
         log.debug(
             'StructFinder created for %s. Search Perimeter on %d mappings.',
             self._memory_handler.get_name(),
@@ -118,8 +118,8 @@ def _search_in(self, mem_map, struct_type, nb=10, depth=99):
                 if validated:
                     log.debug("found instance @ 0x%lx", offset)
                     # do stuff with it.
-                    if self.__update_cb is not None:
-                        self.__update_cb(instance, offset)
+                    if self._update_cb is not None:
+                        self._update_cb(instance, offset)
                     outputs.append((instance, offset))
                     # stop when time to stop
                     if len(outputs) >= nb:
@@ -207,8 +207,8 @@ def _search_in(self, mem_map, struct_type, nb=10, depth=99):
             if validated:
                 log.debug("found instance @ 0x%lx", offset)
                 # do stuff with it.
-                if self.__update_cb is not None:
-                    self.__update_cb(instance, offset)
+                if self._update_cb is not None:
+                    self._update_cb(instance, offset)
                 outputs.append((instance, offset))
                 # stop when time to stop
                 if len(outputs) >= nb:

haystack/structures/heapwalker.py

@@ -147,8 +147,8 @@ def make_heap_finder(memory_handler):
         from haystack.structures.libc import libcheapwalker
         return libcheapwalker.LibcHeapFinder(memory_handler)
     elif os_name == 'winxp':
-        from haystack.structures.win32 import winheapwalker
-        return winheapwalker.WinHeapFinder(memory_handler)
+        from haystack.structures.win32 import winxpheapwalker
+        return winxpheapwalker.WinXPHeapFinder(memory_handler)
     elif os_name == 'win7':
         from haystack.structures.win32 import win7heapwalker
         return win7heapwalker.Win7HeapFinder(memory_handler)

haystack/structures/win32/win7heap.py

@@ -46,8 +46,9 @@
 import logging
 
 from haystack import model
-from haystack import listmodel
 from haystack.abc import interfaces
+from haystack.structures.win32 import winheap
+
 
 
 # pylint: disable=pointeless-string-statement
@@ -85,7 +86,7 @@
 ############# Start methods overrides #################
 # constraints are in constraints files
 
-class Win7HeapValidator(listmodel.ListModel):
+class Win7HeapValidator(winheap.WinHeapValidator):
     """
     this listmodel Validator will register know important list fields
     in the win7 HEAP,
@@ -100,18 +101,18 @@ def __init__(self, memory_handler, my_constraints, win7heap_module):
         if not isinstance(my_constraints, interfaces.IModuleConstraints):
             raise TypeError("Feed me a IModuleConstraints")
         super(Win7HeapValidator, self).__init__(memory_handler, my_constraints)
-        self.win7heap = win7heap_module
+        self.win_heap = win7heap_module
         # LIST_ENTRY
         # the lists usually use end of mapping as a sentinel.
         # we have to use all mappings instead of heaps, because of a circular dependency
         sentinels = [mapping.end-0x10 for mapping in self._memory_handler.get_mappings()]
-        self.register_double_linked_list_record_type(self.win7heap.LIST_ENTRY, 'Flink', 'Blink', sentinels)
+        self.register_double_linked_list_record_type(self.win_heap.LIST_ENTRY, 'Flink', 'Blink', sentinels)
 
         # HEAP_SEGMENT
         # HEAP_SEGMENT.UCRSegmentList. points to HEAP_UCR_DESCRIPTOR.SegmentEntry.
         # HEAP_UCR_DESCRIPTOR.SegmentEntry. points to HEAP_SEGMENT.UCRSegmentList.
         # FIXME, use offset size base on self._target.get_word_size()
-        self.register_double_linked_list_field_and_type(self.win7heap.HEAP_SEGMENT, 'UCRSegmentList', self.win7heap.HEAP_UCR_DESCRIPTOR, 'ListEntry') # offset = -8
+        self.register_double_linked_list_field_and_type(self.win_heap.HEAP_SEGMENT, 'UCRSegmentList', self.win_heap.HEAP_UCR_DESCRIPTOR, 'ListEntry') # offset = -8
         #HEAP_SEGMENT._listHead_ = [
         #        ('UCRSegmentList', HEAP_UCR_DESCRIPTOR, 'ListEntry', -8)]
         #HEAP_UCR_DESCRIPTOR._listHead_ = [ ('SegmentEntry', HEAP_SEGMENT, 'Entry')]
@@ -127,15 +128,15 @@ def __init__(self, memory_handler, my_constraints, win7heap_module):
         #                   # for get_freelists. offset is sizeof(HEAP_ENTRY)
         #                   ('FreeLists', HEAP_FREE_ENTRY, 'FreeList', -8),
         #                   ('VirtualAllocdBlocks', HEAP_VIRTUAL_ALLOC_ENTRY, 'Entry', -8)]
-        self.register_double_linked_list_field_and_type(self.win7heap.HEAP, 'SegmentList', self.win7heap.HEAP_SEGMENT, 'SegmentListEntry') # offset = -16
-        self.register_double_linked_list_field_and_type(self.win7heap.HEAP, 'UCRList', self.win7heap.HEAP_UCR_DESCRIPTOR, 'ListEntry') # offset = 0
+        self.register_double_linked_list_field_and_type(self.win_heap.HEAP, 'SegmentList', self.win_heap.HEAP_SEGMENT, 'SegmentListEntry') # offset = -16
+        self.register_double_linked_list_field_and_type(self.win_heap.HEAP, 'UCRList', self.win_heap.HEAP_UCR_DESCRIPTOR, 'ListEntry') # offset = 0
         # for get_freelists. offset is sizeof(HEAP_ENTRY)
-        ## self.register_double_linked_list_field_and_type(self.win7heap.HEAP, 'FreeLists', self.win7heap.HEAP_FREE_ENTRY, 'FreeList') # offset =  -8
+        ## self.register_double_linked_list_field_and_type(self.win_heap.HEAP, 'FreeLists', self.win_heap.HEAP_FREE_ENTRY, 'FreeList') # offset =  -8
         if self._target.get_word_size() == 4:
-            self.register_double_linked_list_field_and_type(self.win7heap.HEAP, 'FreeLists', self.win7heap.struct__HEAP_FREE_ENTRY_0_5, 'FreeList') # offset =  -8
+            self.register_double_linked_list_field_and_type(self.win_heap.HEAP, 'FreeLists', self.win_heap.struct__HEAP_FREE_ENTRY_0_5, 'FreeList') # offset =  -8
         else:
-            self.register_double_linked_list_field_and_type(self.win7heap.HEAP, 'FreeLists', self.win7heap.struct__HEAP_FREE_ENTRY_0_2, 'FreeList') # offset =  -8
-        self.register_double_linked_list_field_and_type(self.win7heap.HEAP, 'VirtualAllocdBlocks', self.win7heap.HEAP_VIRTUAL_ALLOC_ENTRY, 'Entry') # offset = -8
+            self.register_double_linked_list_field_and_type(self.win_heap.HEAP, 'FreeLists', self.win_heap.struct__HEAP_FREE_ENTRY_0_2, 'FreeList') # offset =  -8
+        self.register_double_linked_list_field_and_type(self.win_heap.HEAP, 'VirtualAllocdBlocks', self.win_heap.HEAP_VIRTUAL_ALLOC_ENTRY, 'Entry') # offset = -8
 
         # HEAP.SegmentList. points to SEGMENT.SegmentListEntry.
         # SEGMENT.SegmentListEntry. points to HEAP.SegmentList.
@@ -234,7 +235,7 @@ def HEAP_get_chunks(self, record):
                         chunk_addr, size, chunk_addr + size)
                     chunk_addr += size
                     continue
-                chunk_header = self._memory_handler.getRef(self.win7heap.HEAP_ENTRY, chunk_addr)
+                chunk_header = self._memory_handler.getRef(self.win_heap.HEAP_ENTRY, chunk_addr)
                 if chunk_header is None:  # force read it
                     log.debug('reading chunk from %x', chunk_addr)
                     m = self._memory_handler.get_mapping_for_address(chunk_addr)
@@ -246,8 +247,8 @@ def HEAP_get_chunks(self, record):
                     if not m:
                         log.debug("found a non valid chunk pointer at %x", chunk_addr)
                         break
-                    chunk_header = m.read_struct(chunk_addr, self.win7heap.HEAP_ENTRY)
-                    self._memory_handler.keepRef(chunk_header, self.win7heap.HEAP_ENTRY, chunk_addr)
+                    chunk_header = m.read_struct(chunk_addr, self.win_heap.HEAP_ENTRY)
+                    self._memory_handler.keepRef(chunk_header, self.win_heap.HEAP_ENTRY, chunk_addr)
                     # FIXME what is this hack
                     chunk_header._orig_address_ = chunk_addr
                 if record.EncodeFlagMask:  # heap.EncodeFlagMask
@@ -292,19 +293,19 @@ def HEAP_get_frontend_chunks(self, record):
             for x in range(128):
                 log.debug('finding lookaside %d at @%x' % (x, addr))
                 m = self._memory_handler.get_mapping_for_address(addr)
-                st = m.read_struct(addr, self.win7heap.HEAP_LOOKASIDE)
+                st = m.read_struct(addr, self.win_heap.HEAP_LOOKASIDE)
                 # load members on self.FrontEndHeap car c'est un void *
                 #for free in st.iterateList('ListHead'):  # single link list.
                 for free in self.iterateList(st):
                     # TODO delete this free from the heap-segment entries chunks
                     log.debug('free')
                     res.append(free)  # ???
                     pass
-                addr += ctypes.sizeof(self.win7heap.HEAP_LOOKASIDE)
+                addr += ctypes.sizeof(self.win_heap.HEAP_LOOKASIDE)
         elif record.FrontEndHeapType == 2:  # win7 per default
             log.debug('finding frontend at @%x' % (addr))
             m = self._memory_handler.get_mapping_for_address(addr)
-            st = m.read_struct(addr, self.win7heap.LFH_HEAP)
+            st = m.read_struct(addr, self.win_heap.LFH_HEAP)
             # LFH is a big chunk allocated by the backend allocator, called subsegment
             # but rechopped as small chunks of a heapbin.
             # Active subsegment hold that big chunk.
@@ -326,7 +327,7 @@ def HEAP_get_frontend_chunks(self, record):
                         #log.debug('NULL pointer items')
                         continue
                     m = self._memory_handler.get_mapping_for_address(items_addr)
-                    subsegment = m.read_struct(items_addr, self.win7heap.HEAP_SUBSEGMENT)
+                    subsegment = m.read_struct(items_addr, self.win_heap.HEAP_SUBSEGMENT)
                     # log.debug(subsegment)
                     # TODO current subsegment.SFreeListEntry is on error at some depth.
                     # bad pointer value on the second subsegment
@@ -442,7 +443,7 @@ def HEAP_getFreeLists_by_blocksindex(self, record):
         while bi_addr != 0:
             log.debug('BLocksIndex is at %x' % (bi_addr))
             m = self._memory_handler.get_mapping_for_address(bi_addr)
-            bi = m.read_struct(bi_addr, self.win7heap.HEAP_LIST_LOOKUP)
+            bi = m.read_struct(bi_addr, self.win_heap.HEAP_LIST_LOOKUP)
             """
                 ('ExtendedLookup', POINTER(HEAP_LIST_LOOKUP)),
                 ('ArraySize', __uint32_t),
@@ -480,9 +481,9 @@ def HEAP_ENTRY_decode(self, chunk_header, heap):
         # FIXME BUG, we need to use _0_0_0_0 for 64 bits, otherwise
         # we are reading bad data
         # 64 bits: struct__HEAP_ENTRY_0_0_0_0
-        chunk_len = ctypes.sizeof(self.win7heap.struct__HEAP_ENTRY_0_0)
+        chunk_len = ctypes.sizeof(self.win_heap.struct__HEAP_ENTRY_0_0)
         chunk_header_decoded = (
-            self.win7heap.struct__HEAP_ENTRY_0_0).from_buffer_copy(chunk_header)
+            self.win_heap.struct__HEAP_ENTRY_0_0).from_buffer_copy(chunk_header)
         # decode the heap entry chunk header with the heap.Encoding
         working_array = (
             ctypes.c_ubyte *
@@ -504,8 +505,8 @@ def HEAP_ENTRY_decode(self, chunk_header, heap):
 
     def _get_chunk(self, entry_addr):
         m = self._memory_handler.get_mapping_for_address(entry_addr)
-        chunk_header = m.read_struct(entry_addr, self.win7heap.HEAP_ENTRY)
-        self._memory_handler.keepRef(chunk_header, self.win7heap.HEAP_ENTRY, entry_addr)
+        chunk_header = m.read_struct(entry_addr, self.win_heap.HEAP_ENTRY)
+        self._memory_handler.keepRef(chunk_header, self.win_heap.HEAP_ENTRY, entry_addr)
         # FIXME what is this hack
         chunk_header._orig_address_ = entry_addr
         return chunk_header