Refactory clemory to be way faster and simpler

README.md

@@ -25,7 +25,7 @@ the OS's loader.
 >>> hex(libc_main_reloc.addr)       # Address of GOT entry for libc_start_main
 '0x61c1c0'
 >>> import pyvex
->>> some_text_data = ''.join(ld.memory.read_bytes(ld.main_object.entry, 0x100))
+>>> some_text_data = ld.memory.load(ld.main_object.entry, 0x100)
 >>> irsb = pyvex.lift(some_text_data, ld.main_object.entry, ld.main_object.arch)
 >>> irsb.pp()
 IRSB {

cle/backends/binja.py

@@ -236,7 +236,7 @@ def set_got_entry(self, name, newaddr):
             return
 
         addr = self.imports[name]
-        self.memory.write_addr_at(addr, newaddr)
+        self.memory.pack_word(addr, newaddr)
 
     def close(self):
         """

cle/backends/cgc/cgc.py

@@ -25,7 +25,7 @@ def __init__(self, binary, *args, **kwargs):
             stream = PatchedStream(open(binary, 'rb'), [(0, ELF_HEADER)])
         kwargs['filename'] = filename
         super(CGC, self).__init__(stream, *args, **kwargs)
-        self.memory.write_bytes(0, CGC_HEADER) # repair CGC header
+        self.memory.store(0, CGC_HEADER) # repair CGC header
         self.os = 'cgc'
         self.execstack = True  # the stack is always executable in CGC
 

cle/backends/elf/elf.py

@@ -123,7 +123,7 @@ def __init__(self, binary, addend=None, **kwargs):
         try:
             self.mapped_base = self.linked_base = min(seg_addrs)
         except ValueError:
-            l.warn('no segments identified in PT_LOAD')
+            l.warning('no segments identified in PT_LOAD')
 
         self.__register_segments()
         self.__register_sections()
@@ -135,7 +135,7 @@ def __init__(self, binary, addend=None, **kwargs):
         self._populate_demangled_names()
 
         for offset, patch in patch_undo:
-            self.memory.write_bytes(AT.from_lva(self.min_addr + offset, self).to_rva(), patch)
+            self.memory.store(AT.from_lva(self.min_addr + offset, self).to_rva(), patch)
 
 
     #
@@ -220,7 +220,7 @@ def get_symbol(self, symid, symbol_table=None): # pylint: disable=arguments-diff
             return symbol
         elif isinstance(symid, (str, unicode)):
             if not symid:
-                l.warn("Trying to resolve a symbol by its empty name")
+                l.warning("Trying to resolve a symbol by its empty name")
                 return None
             cached = self._symbols_by_name.get(symid, None)
             if cached:
@@ -275,21 +275,21 @@ def _extract_init_fini(self):
             arr_start = AT.from_lva(self._dynamic['DT_PREINIT_ARRAY'], self).to_rva()
             arr_end = arr_start + self._dynamic['DT_PREINIT_ARRAYSZ']
             arr_entsize = self.arch.bytes
-            self._preinit_arr = list(map(self.memory.read_addr_at, range(arr_start, arr_end, arr_entsize)))
+            self._preinit_arr = list(map(self.memory.unpack_word, range(arr_start, arr_end, arr_entsize)))
         if 'DT_INIT' in self._dynamic:
             self._init_func = AT.from_lva(self._dynamic['DT_INIT'], self).to_rva()
         if 'DT_INIT_ARRAY' in self._dynamic and 'DT_INIT_ARRAYSZ' in self._dynamic:
             arr_start = AT.from_lva(self._dynamic['DT_INIT_ARRAY'], self).to_rva()
             arr_end = arr_start + self._dynamic['DT_INIT_ARRAYSZ']
             arr_entsize = self.arch.bytes
-            self._init_arr = list(map(self.memory.read_addr_at, range(arr_start, arr_end, arr_entsize)))
+            self._init_arr = list(map(self.memory.unpack_word, range(arr_start, arr_end, arr_entsize)))
         if 'DT_FINI' in self._dynamic:
             self._fini_func = AT.from_lva(self._dynamic['DT_FINI'], self).to_rva()
         if 'DT_FINI_ARRAY' in self._dynamic and 'DT_FINI_ARRAYSZ' in self._dynamic:
             arr_start = AT.from_lva(self._dynamic['DT_FINI_ARRAY'], self).to_rva()
             arr_end = arr_start + self._dynamic['DT_FINI_ARRAYSZ']
             arr_entsize = self.arch.bytes
-            self._fini_arr = list(map(self.memory.read_addr_at, range(arr_start, arr_end, arr_entsize)))
+            self._fini_arr = list(map(self.memory.unpack_word, range(arr_start, arr_end, arr_entsize)))
         self._inits_extracted = True
 
     def _load_segment(self, seg):
@@ -624,7 +624,7 @@ def __register_relocs(self, section, force_jmprel=False):
             try:
                 dest_sec = self.sections[dest_sec_idx]
             except IndexError:
-                l.warn('the relocation section %s refers to unknown section index: %d', section.name, dest_sec_idx)
+                l.warning('the relocation section %s refers to unknown section index: %d', section.name, dest_sec_idx)
             else:
                 if not dest_sec.occupies_memory:
                     # The target section is not loaded into memory, so just continue
@@ -682,7 +682,7 @@ def __register_tls(self, seg_readelf):
         self.tls_used = True
         self.tls_block_size = seg_readelf.header.p_memsz
         self.tls_tdata_size = seg_readelf.header.p_filesz
-        self.tls_tdata_start = seg_readelf.header.p_vaddr
+        self.tls_tdata_start = AT.from_lva(seg_readelf.header.p_vaddr, self).to_rva()
 
     def __register_sections(self):
         new_addr = 0

cle/backends/elf/metaelf.py

@@ -33,8 +33,7 @@ def _block(self, addr):
         thumb = self.arch.name.startswith("ARM") and addr % 2 == 1
         realaddr = addr
         if thumb: realaddr -= 1
-        self.memory.seek(AT.from_lva(realaddr, self).to_rva())
-        dat = self.memory.read(40)
+        dat = self.memory.load(AT.from_lva(realaddr, self).to_rva(), 40)
         return pyvex.IRSB(dat, addr, self.arch, bytes_offset=1 if thumb else 0, opt_level=1)
 
     def _add_plt_stub(self, name, addr, sanity_check=True):
@@ -82,7 +81,7 @@ def _load_plt(self):
                 try:
                     self._add_plt_stub(
                         name,
-                        self.memory.read_addr_at(reloc.relative_addr) - 6,
+                        self.memory.unpack_word(reloc.relative_addr) - 6,
                         sanity_check=not self.pic)
                 except KeyError:
                     pass
@@ -95,7 +94,7 @@ def _load_plt(self):
         # right before the resolution stubs.
         if self.arch.name in ('PPC32',):
             resolver_stubs = sorted(
-                (self.memory.read_addr_at(reloc.relative_addr), name)
+                (self.memory.unpack_word(reloc.relative_addr), name)
                 for name, reloc in func_jmprel.items()
             )
             if resolver_stubs:
@@ -300,8 +299,8 @@ def _ppc64_abiv1_entry_fix(self):
 
         if self.is_ppc64_abiv1:
             ep_offset = self._entry
-            self._entry = self.memory.read_addr_at(AT.from_lva(ep_offset, self).to_rva())
-            self.ppc64_initial_rtoc = self.memory.read_addr_at(AT.from_lva(ep_offset+8, self).to_rva())
+            self._entry = self.memory.unpack_word(AT.from_lva(ep_offset, self).to_rva())
+            self.ppc64_initial_rtoc = self.memory.unpack_word(AT.from_lva(ep_offset+8, self).to_rva())
 
     @staticmethod
     def extract_soname(path):

cle/backends/elf/relocation/arm.py

@@ -238,6 +238,9 @@ class R_ARM_THM_CALL(ELFReloc):
       - However, the J1/J2 bits are XORed with !S bit in this case (see vex implementation: https://github.com/angr/vex/blob/6d1252c7ce8fe8376318b8f8bb8034058454c841/priv/guest_arm_toIR.c#L19219 )
       - Implementation appears correct with the bits placed into offset[23:22]
     """
+    def __init__(self, *args, **kwargs):
+        super(R_ARM_THM_CALL, self).__init__(*args, **kwargs)
+        self._insn_bytes = None
 
     def resolve_symbol(self, solist, bypass_compatibility=False, thumb=False):
         return super(R_ARM_THM_CALL, self).resolve_symbol(solist,
@@ -255,9 +258,11 @@ def value(self):
         #  Because this 4-byte instruction is treated as two 2-byte instructions,
         #  the bytes are in the order `b3 b4 b1 b2`, where b4 is the most significant.
 
-        raw  = list(map(ord, self.owner_obj.memory.read_bytes(self.relative_addr, 4, orig=True)))
-        hi   = (raw[1] << 8) | raw[0]
-        lo   = (raw[3] << 8) | raw[2]
+        if self._insn_bytes is None:
+            self._insn_bytes = self.owner_obj.memory.load(self.relative_addr, 4)
+
+        hi   = (self._insn_bytes[1] << 8) | self._insn_bytes[0]
+        lo   = (self._insn_bytes[3] << 8) | self._insn_bytes[2]
         inst = (hi << 16) | lo
 
         def gen_mask(n_bits, first_bit):

cle/backends/elf/relocation/elfreloc.py

@@ -8,18 +8,18 @@ class ELFReloc(Relocation):
     def __init__(self, owner, symbol, relative_addr, addend=None):
         super(ELFReloc, self).__init__(owner, symbol, relative_addr)
 
-        self._addend = addend
-
-    @property
-    def is_rela(self):
-        return self._addend is not None
+        if addend is not None:
+            self.is_rela = True
+            self._addend = addend
+        else:
+            self.is_rela = False
+            self._addend = self.owner_obj.memory.unpack_word(self.relative_addr)
 
     @property
     def addend(self):
-        if self.is_rela:
-            return self._addend
-        else:
-            return self.owner_obj.memory.read_addr_at(self.relative_addr, orig=True)
+        if self._addend is None:
+            self._addend = self.owner_obj.memory.unpack_word(self.relative_addr)
+        return self._addend
 
     @property
     def value(self):    # pylint: disable=no-self-use

cle/backends/elf/relocation/generic.py

@@ -21,27 +21,27 @@ class GenericTLSOffsetReloc(ELFReloc):
     def relocate(self, solist, bypass_compatibility=False):
         hell_offset = self.owner_obj.arch.elf_tls.tp_offset
         if self.symbol.type == Symbol.TYPE_NONE:
-            self.owner_obj.memory.write_addr_at(
+            self.owner_obj.memory.pack_word(
                 self.relative_addr,
                 self.owner_obj.tls_block_offset + self.addend + self.symbol.relative_addr - hell_offset)
             self.resolve(None)
         else:
             if not self.resolve_symbol(solist, bypass_compatibility):
                 return False
-            self.owner_obj.memory.write_addr_at(
+            self.owner_obj.memory.pack_word(
                 self.relative_addr,
                 self.resolvedby.owner_obj.tls_block_offset + self.addend + self.symbol.relative_addr - hell_offset)
         return True
 
 class GenericTLSModIdReloc(ELFReloc):
     def relocate(self, solist, bypass_compatibility=False):
         if self.symbol.type == Symbol.TYPE_NONE:
-            self.owner_obj.memory.write_addr_at(self.relative_addr, self.owner_obj.tls_module_id)
+            self.owner_obj.memory.pack_word(self.relative_addr, self.owner_obj.tls_module_id)
             self.resolve(None)
         else:
             if not self.resolve_symbol(solist):
                 return False
-            self.owner_obj.memory.write_addr_at(self.relative_addr, self.resolvedby.owner_obj.tls_module_id)
+            self.owner_obj.memory.pack_word(self.relative_addr, self.resolvedby.owner_obj.tls_module_id)
         return True
 
 class GenericIRelativeReloc(ELFReloc):
@@ -95,7 +95,7 @@ def value(self):
 class GenericCopyReloc(ELFReloc):
     @property
     def value(self):
-        return self.resolvedby.owner_obj.memory.read_addr_at(self.resolvedby.relative_addr)
+        return self.resolvedby.owner_obj.memory.unpack_word(self.resolvedby.relative_addr)
 
 class MipsGlobalReloc(GenericAbsoluteReloc):
     pass
@@ -109,9 +109,9 @@ def relocate(self, solist, bypass_compatibility=False): # pylint: disable=unused
         if delta == 0:
             self.resolve(None)
             return True
-        val = self.owner_obj.memory.read_addr_at(self.relative_addr)
+        val = self.owner_obj.memory.unpack_word(self.relative_addr)
         newval = val + delta
-        self.owner_obj.memory.write_addr_at(self.relative_addr, newval)
+        self.owner_obj.memory.pack_word(self.relative_addr, newval)
         self.resolve(None)
         return True
 
@@ -142,4 +142,4 @@ def relocate(self, solist, bypass_compatibility=False): # pylint: disable=unused
                                     " relevant addresses fit in the 32-bit address space." % self.__class__.__name__)
 
         by = struct.pack(self.owner_obj.arch.struct_fmt(32), val % (2**32))
-        self.owner_obj.memory.write_bytes(self.dest_addr, by)
+        self.owner_obj.memory.store(self.dest_addr, by)

cle/backends/elf/relocation/pcc64.py

@@ -16,14 +16,14 @@ def relocate(self, solist, bypass_compatibility=False):
             # R_PPC64_JMP_SLOT
             # http://osxr.org/glibc/source/sysdeps/powerpc/powerpc64/dl-machine.h?v=glibc-2.15#0405
             # copy an entire function descriptor struct
-            addr = self.resolvedby.owner_obj.memory.read_addr_at(self.resolvedby.relative_addr)
-            toc = self.resolvedby.owner_obj.memory.read_addr_at(self.resolvedby.relative_addr + 8)
-            aux = self.resolvedby.owner_obj.memory.read_addr_at(self.resolvedby.relative_addr + 16)
-            self.owner_obj.memory.write_addr_at(self.relative_addr, addr)
-            self.owner_obj.memory.write_addr_at(self.relative_addr + 8, toc)
-            self.owner_obj.memory.write_addr_at(self.relative_addr + 16, aux)
+            addr = self.resolvedby.owner_obj.memory.unpack_word(self.resolvedby.relative_addr)
+            toc = self.resolvedby.owner_obj.memory.unpack_word(self.resolvedby.relative_addr + 8)
+            aux = self.resolvedby.owner_obj.memory.unpack_word(self.resolvedby.relative_addr + 16)
+            self.owner_obj.memory.pack_word(self.relative_addr, addr)
+            self.owner_obj.memory.pack_word(self.relative_addr + 8, toc)
+            self.owner_obj.memory.pack_word(self.relative_addr + 16, aux)
         else:
-            self.owner_obj.memory.write_addr_at(self.relative_addr, self.resolvedby.rebased_addr)
+            self.owner_obj.memory.pack_word(self.relative_addr, self.resolvedby.rebased_addr)
         return True
 
 class R_PPC64_RELATIVE(generic.GenericRelativeReloc):

cle/backends/externs.py

@@ -36,7 +36,7 @@ def make_extern(self, name, size=1, alignment=8, thumb=False):
             self._symbol_cache[name + '#func'] = func_symbol
 
             toc = self.allocate(0x18, alignment=8)
-            self.memory.write_addr_at(AT.from_mva(toc, self).to_rva(), addr)
+            self.memory.pack_word(AT.from_mva(toc, self).to_rva(), addr)
             addr = toc
 
         new_symbol = Symbol(self, name, AT.from_mva(addr, self).to_rva(), 1, Symbol.TYPE_FUNCTION)

cle/backends/idabin.py

@@ -288,14 +288,14 @@ def resolve_import_dirty(self, sym, new_val):
         # Try IDA's _ptr
         plt_addr = self.get_symbol_addr(sym + "_ptr")
         if plt_addr:
-            self.memory.write_addr_at(plt_addr, new_val)
+            self.memory.pack_word(plt_addr, new_val)
             return
 
         # Try the __imp_name
         plt_addr = self.get_symbol_addr("__imp_" + sym)
         if plt_addr:
             for addr in self.ida.idautils.DataRefsTo(plt_addr):
-                self.memory.write_addr_at(addr, new_val)
+                self.memory.pack_word(addr, new_val)
             return
 
         # Try the normal name
@@ -306,7 +306,7 @@ def resolve_import_dirty(self, sym, new_val):
             if len(addrlist) == 0:
                 addrlist = list(self.ida.idautils.CodeRefsTo(plt_addr))
             for addr in addrlist:
-                self.memory.write_addr_at(addr, new_val)
+                self.memory.pack_word(addr, new_val)
             return
 
         # If none of them has an address, that's a problem
@@ -321,7 +321,7 @@ def set_got_entry(self, name, newaddr):
             return
 
         addr = self.imports[name]
-        self.memory.write_addr_at(addr, newaddr)
+        self.memory.pack_word(addr, newaddr)
 
     def is_thumb(self, addr):
         """

cle/backends/macho/binding.py

@@ -370,23 +370,23 @@ def default_binding_handler(state, binary):
     value = symbol.linked_addr + state.addend
     if state.binding_type == 1:  # POINTER
         l.info("Updating address %#x with symobl %r @ %#x", location, state.sym_name, value)
-        binary.memory.write_bytes(
+        binary.memory.store(
             AT.from_lva(location, binary).to_rva(),
             struct.pack(binary.struct_byteorder + ("Q" if binary.arch.bits == 64 else "I"), value))
         symbol.bind_xrefs.append(location)
     elif state.binding_type == 2:  # ABSOLUTE32
         location_32 = location % (2 ** 32)
         value_32 = value % (2 ** 32)
         l.info("Updating address %#x with symobl %r @ %#x", state.sym_name, location_32, value_32)
-        binary.memory.write_bytes(
+        binary.memory.store(
             AT.from_lva(location_32, binary).to_rva(),
             struct.pack(binary.struct_byteorder + "I", value_32))
         symbol.bind_xrefs.append(location_32)
     elif state.binding_type == 3:  # PCREL32
         location_32 = location % (2 ** 32)
         value_32 = (value - (location + 4)) % (2 ** 32)
         l.info("Updating address %#x with symobl %r @ %#x", state.sym_name, location_32, value_32)
-        binary.memory.write_bytes(
+        binary.memory.store(
             AT.from_lva(location_32, binary).to_rva(),
             struct.pack(binary.struct_byteorder + "I", value_32))
         symbol.bind_xrefs.append(location_32)

cle/backends/macho/macho.py

@@ -186,7 +186,7 @@ def _parse_mod_funcs(self):
         # factoring out common code
         def parse_mod_funcs_internal(s, target):
             for i in range(s.vaddr, s.vaddr + s.memsize, size):
-                addr = self._unpack_with_byteorder(fmt, "".join(self.memory.read_bytes(i, size)))[0]
+                addr = self._unpack_with_byteorder(fmt, "".join(self.memory.load(i, size)))[0]
                 l.debug("Addr: %#x", addr)
                 target.append(addr)