Add A Minidump Backend Supporting x86/AMD64 (#209)

* Add the initial support for AMD64 minidump files

* Update to the minidump backend for AMD64

* Change the style to fit in more

* Add x86 support to the minidump backend

* Add the minidump requirement to the setup file

* Add the x86 update_state method for the context

* Fix lint issues for the minidump backend

* Use struct and a dictionary for thread registers

* Add tests for the minidump backend

* Fix some lint issues that were identified

.gitignore

@@ -18,3 +18,6 @@ ccle/x86_64
 .idea/
 
 *.egg-info
+
+Pipfile
+Pipfile.lock

cle/backends/__init__.py

@@ -345,6 +345,7 @@ def register_backend(name, cls):
 from .blob import Blob
 from .cgc import CGC, BackedCGC
 from .ihex import Hex
+from .minidump import Minidump
 from .macho import MachO
 from .named_region import NamedRegion
 from .java.jar import Jar

cle/backends/minidump/__init__.py

@@ -0,0 +1,141 @@
+import archinfo
+import ntpath
+import struct
+
+from minidump import minidumpfile
+from minidump.streams import SystemInfoStream
+
+from .. import register_backend, Backend
+from ..region import Section
+from ... memory import Clemory
+
+class MinidumpMissingStreamError(Exception):
+    def __init__(self, stream, message=None):
+        super(MinidumpMissingStreamError, self).__init__()
+        self.message = message
+        self.stream = stream
+
+class Minidump(Backend):
+    is_default = True
+    def __init__(self, *args, **kwargs):
+        super(Minidump, self).__init__(*args, **kwargs)
+        self.os = 'windows'
+        self.supports_nx = True
+        if self.binary is None:
+            self._mdf = minidumpfile.MinidumpFile.parse_bytes(self.binary_stream.read())
+        else:
+            self._mdf = minidumpfile.MinidumpFile.parse(self.binary)
+
+        if self.arch is None:
+            if getattr(self._mdf, 'sysinfo', None) is None:
+                raise MinidumpMissingStreamError('SystemInfo', 'The architecture was not specified')
+            arch = self._mdf.sysinfo.ProcessorArchitecture
+            if arch == SystemInfoStream.PROCESSOR_ARCHITECTURE.AMD64:
+                self.set_arch(archinfo.ArchAMD64())
+            elif arch == SystemInfoStream.PROCESSOR_ARCHITECTURE.INTEL:
+                self.set_arch(archinfo.ArchX86())
+            else:
+                # has not been tested with other architectures
+                raise ValueError('The minidump architecture is not AMD64 or x86')
+
+        if self._mdf.memory_segments_64 is not None:
+            segments = self._mdf.memory_segments_64.memory_segments
+        elif self._mdf.memory_segments is not None:
+            segments = self._mdf.memory_segments.memory_segments
+        else:
+            raise MinidumpMissingStreamError('MemoryList', 'The memory segments were not defined')
+
+        for segment in segments:
+            clemory = Clemory(self.arch)
+            data = segment.read(segment.start_virtual_address, segment.size, self._mdf.file_handle)
+            clemory.add_backer(0, data)
+            self.memory.add_backer(segment.start_virtual_address, clemory)
+
+        for module in self.modules:
+            for segment in segments:
+                if segment.start_virtual_address == module.baseaddress:
+                    break
+            else:
+                raise RuntimeError('Missing segment for loaded module: ' + module.name)
+            section = Section(module.name, segment.start_file_address, module.baseaddress, module.size)
+            self.sections.append(section)
+            self.sections_map[ntpath.basename(section.name)] = section
+        self.segments = self.sections
+
+    def __getstate__(self):
+        if self.binary is None:
+            raise ValueError("Can't pickle an object loaded from a stream")
+
+        state = dict(self.__dict__)
+
+        state['_mdf'] = None
+        state['binary_stream'] = None
+        return state
+
+    def __setstate__(self, state):
+        self.__dict__.update(state)
+        self._mdf = minidumpfile.MinidumpFile.parse(self.binary)
+
+    @property
+    def file_handle(self):
+        return self._mdf.file_handle
+
+    @staticmethod
+    def is_compatible(stream):
+        identstring = stream.read(4)
+        stream.seek(0)
+        return identstring == b'MDMP'
+
+    @property
+    def modules(self):
+        return self._mdf.modules.modules
+
+    @property
+    def threads(self):
+        return self._mdf.threads.threads
+
+    def get_thread_registers_by_id(self, thread_id):
+        """Get the registers for a specific thread by its id."""
+        for thread in self.threads:
+            if thread.ThreadId == thread_id:
+                break
+        else:
+            raise ValueError('The specified thread id was not found')
+        self.file_handle.seek(thread.ThreadContext.Rva)  # pylint: disable=undefined-loop-variable
+        data = self.file_handle.read(thread.ThreadContext.DataSize)  # pylint: disable=undefined-loop-variable
+        self.file_handle.seek(0)
+
+        if self.arch == archinfo.ArchAMD64():
+            fmt = 'QQQQQQIIHHHHHHIQQQQQQQQQQQQQQQQQQQQQQQ'
+            fmt_registers = {
+                'fs':     11, 'gs':  12,
+                'eflags': 14, 'rax': 21,
+                'rcx':    22, 'rdx': 23,
+                'rbx':    24, 'rsp': 25,
+                'rbp':    26, 'rsi': 27,
+                'rdi':    28, 'r8':  29,
+                'r9':     30, 'r10': 31,
+                'r11':    32, 'r12': 33,
+                'r13':    34, 'r14': 35,
+                'r15':    36, 'rip': 37
+            }
+        elif self.arch == archinfo.ArchX86():
+            fmt = 'IIIIIII112xIIIIIIIIIIIIIIII512x'
+            fmt_registers = {
+                'gs':     7,  'fs':  8,
+                'edi':    11, 'esi': 12,
+                'ebx':    13, 'edx': 14,
+                'ecx':    15, 'eax': 16,
+                'ebp':    17, 'eip': 18,
+                'eflags': 20, 'esp': 21
+            }
+        else:
+            raise ValueError('The architecture is unsupported')
+        data = data[:struct.calcsize(fmt)]
+        members = struct.unpack(fmt, data)
+        thread_registers = {}
+        for register, position in fmt_registers.items():
+            thread_registers[register] = members[position]
+        return thread_registers
+
+register_backend('minidump', Minidump)

setup.py

@@ -23,5 +23,6 @@
         'pyvex==8.19.10.30',
         'pefile',
         'sortedcontainers>=2.0',
+        'minidump==0.0.10',
     ]
 )

tests/test_minidump.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+
+import archinfo
+import logging
+import nose
+import os
+
+import cle
+
+TEST_BASE = os.path.join(os.path.dirname(os.path.realpath(__file__)),
+    os.path.join('..', '..', 'binaries'))
+
+def test_minidump():
+    exe = os.path.join(TEST_BASE, 'tests', 'x86', 'windows', 'jusched_x86.dmp')
+    ld = cle.Loader(exe, auto_load_libs=False)
+    nose.tools.assert_is_instance(ld.main_object, cle.Minidump)
+    nose.tools.assert_is_instance(ld.main_object.arch, archinfo.ArchX86)
+    nose.tools.assert_equal(ld.main_object.os, 'windows')
+    nose.tools.assert_equal(len(ld.main_object.segments), 30)
+
+    sections_map = ld.main_object.sections_map
+    nose.tools.assert_in('jusched.exe', sections_map)
+    nose.tools.assert_in('kernel32.dll', sections_map)
+
+    nose.tools.assert_equal(len(ld.main_object.threads), 2)
+    registers = ld.main_object.get_thread_registers_by_id(0x0548)
+    nose.tools.assert_is_instance(registers, dict)
+    nose.tools.assert_equal(registers, {
+        'gs': 43,
+        'fs': 83,
+        'edi': 2001343136,
+        'esi': 2001343136,
+        'ebx': 0,
+        'edx': 2001343136,
+        'ecx': 2001343136,
+        'eax': 2121117696,
+        'ebp': 33357196,
+        'eip': 2000776736,
+        'eflags': 580,
+        'esp': 33357152
+    })
+
+if __name__ == '__main__':
+    logging.basicConfig(level=logging.INFO)
+    test_minidump()