Merge branch 'rip/memory'

Author: rhelmot

angr/__init__.py

@@ -61,6 +61,7 @@
 from .storage.file import SimFileBase, SimFile, SimPackets, SimFileStream, SimPacketsStream, SimFileDescriptor, SimFileDescriptorDuplex
 from .state_plugins.filesystem import SimMount, SimHostFilesystem
 from .state_plugins.heap import SimHeapBrk, SimHeapPTMalloc, PTChunk
+from . import concretization_strategies
 from .distributed import Server
 
 # for compatibility reasons

angr/analyses/cfg/indirect_jump_resolvers/jumptable.py

@@ -385,8 +385,12 @@ def _do_load(self, addr, size):
 class StoreHook:
     @staticmethod
     def hook(state):
-        state.inspect.mem_write_expr = state.solver.BVS('instrumented_store',
-                                                        state.solver.eval(state.inspect.mem_write_length) * 8)
+        write_length = state.inspect.mem_write_length
+        if write_length is None:
+            write_length = len(state.inspect.mem_write_expr)
+        else:
+            write_length = write_length * state.arch.byte_width
+        state.inspect.mem_write_expr = state.solver.BVS('instrumented_store', write_length)
 
 
 class LoadHook:
@@ -421,6 +425,72 @@ def __init__(self, reg_offset, reg_bits, value):
     def hook(self, state):
         state.registers.store(self.reg_offset, state.solver.BVV(self.value, self.reg_bits))
 
+
+class BSSHook:
+    def __init__(self, project, bss_regions):
+        self.project = project
+        self._bss_regions = bss_regions
+        self._written_addrs = set()
+
+    def bss_memory_read_hook(self, state):
+
+        if not self._bss_regions:
+            return
+
+        read_addr = state.inspect.mem_read_address
+        read_length = state.inspect.mem_read_length
+
+        if not isinstance(read_addr, int) and read_addr.symbolic:
+            # don't touch it
+            return
+
+        concrete_read_addr = state.solver.eval(read_addr)
+        concrete_read_length = state.solver.eval(read_length)
+
+        for start, size in self._bss_regions:
+            if start <= concrete_read_addr < start + size:
+                # this is a read from the .bss section
+                break
+        else:
+            return
+
+        if concrete_read_addr not in self._written_addrs:
+            # it was never written to before. we overwrite it with unconstrained bytes
+            for i in range(0, concrete_read_length, self.project.arch.bytes):
+                state.memory.store(concrete_read_addr + i, state.solver.Unconstrained('unconstrained',
+                                                                                      self.project.arch.bits))
+
+                # job done :-)
+
+    def bss_memory_write_hook(self, state):
+
+        if not self._bss_regions:
+            return
+
+        write_addr = state.inspect.mem_write_address
+
+        if not isinstance(write_addr, int) and write_addr.symbolic:
+            return
+
+        concrete_write_addr = state.solver.eval(write_addr)
+        concrete_write_length = state.solver.eval(state.inspect.mem_write_length) \
+            if state.inspect.mem_write_length is not None \
+            else len(state.inspect.mem_write_expr) // state.arch.byte_width
+
+        for start, size in self._bss_regions:
+            if start <= concrete_write_addr < start + size:
+                # hit a BSS section
+                break
+        else:
+            return
+
+        if concrete_write_length > 1024:
+            l.warning("Writing more 1024 bytes to the BSS region, only considering the first 1024 bytes.")
+            concrete_write_length = 1024
+
+        for i in range(concrete_write_addr, concrete_write_length):
+            self._written_addrs.add(i)
+
 #
 # Main class
 #
@@ -578,7 +648,10 @@ def _resolve(self, cfg, addr, func_addr, b):
 
             # any read from an uninitialized segment should be unconstrained
             if self._bss_regions:
-                bss_memory_read_bp = BP(when=BP_BEFORE, enabled=True, action=self._bss_memory_read_hook)
+                bss_hook = BSSHook(self.project, self._bss_regions)
+                bss_memory_write_bp = BP(when=BP_AFTER, enabled=True, action=bss_hook.bss_memory_write_hook)
+                start_state.inspect.add_breakpoint('mem_write', bss_memory_write_bp)
+                bss_memory_read_bp = BP(when=BP_BEFORE, enabled=True, action=bss_hook.bss_memory_read_hook)
                 start_state.inspect.add_breakpoint('mem_read', bss_memory_read_bp)
 
             # instrument specified store/put/load statements
@@ -1338,36 +1411,6 @@ def _find_bss_region(self):
                 self._bss_regions.append((section.vaddr, section.memsize))
                 break
 
-    def _bss_memory_read_hook(self, state):
-
-        if not self._bss_regions:
-            return
-
-        read_addr = state.inspect.mem_read_address
-        read_length = state.inspect.mem_read_length
-
-        if not isinstance(read_addr, int) and read_addr.symbolic:
-            # don't touch it
-            return
-
-        concrete_read_addr = state.solver.eval(read_addr)
-        concrete_read_length = state.solver.eval(read_length)
-
-        for start, size in self._bss_regions:
-            if start <= concrete_read_addr < start + size:
-                # this is a read from the .bss section
-                break
-        else:
-            return
-
-        if not state.memory.was_written_to(concrete_read_addr):
-            # it was never written to before. we overwrite it with unconstrained bytes
-            for i in range(0, concrete_read_length, self.project.arch.bytes):
-                state.memory.store(concrete_read_addr + i, state.solver.Unconstrained('unconstrained',
-                                                                                      self.project.arch.bits))
-
-                # job done :-)
-
     def _init_registers_on_demand(self, state):
         # for uninitialized read using a register as the source address, we replace them in memory on demand
         read_addr = state.inspect.mem_read_address

angr/analyses/congruency_check.py

@@ -83,15 +83,15 @@ def _sync_steps(simgr, max_steps=None):
         elif simgr.left[0].history.block_count < simgr.right[0].history.block_count:
             l.debug("... right is ahead; stepping left %s times",
                     simgr.right[0].history.block_count - simgr.left[0].history.block_count)
-            npg = simgr.step(
+            npg = simgr.run(
                 stash='left',
                 until=lambda lpg: lpg.left[0].history.block_count >= simgr.right[0].history.block_count,
                 n=max_steps
             )
         elif simgr.right[0].history.block_count < simgr.left[0].history.block_count:
             l.debug("... left is ahead; stepping right %s times",
                     simgr.left[0].history.block_count - simgr.right[0].history.block_count)
-            npg = simgr.step(
+            npg = simgr.run(
                 stash='right',
                 until=lambda lpg: lpg.right[0].history.block_count >= simgr.left[0].history.block_count,
                 n=max_steps

angr/analyses/ddg.py

@@ -248,7 +248,7 @@ def kill_def(self, variable, location, size_threshold=32):
 
     def lookup_defs(self, variable, size_threshold=32):
         """
-        Find all definitions of the varaible
+        Find all definitions of the variable.
 
         :param SimVariable variable: The variable to lookup for.
         :param int size_threshold: The maximum bytes to consider for the variable. For example, if the variable is 100

angr/analyses/identifier/runner.py

@@ -59,7 +59,7 @@ def _get_recv_state(self):
             fake_flag_data = entry_state.solver.BVV(FLAG_DATA)
             entry_state.memory.store(0x4347c000, fake_flag_data)
             # map the place where I put arguments
-            entry_state.memory.mem.map_region(0x2000, 0x10000, 7)
+            entry_state.memory.map_region(0x2000, 0x10000, 7)
 
             entry_state.unicorn._register_check_count = 100
             entry_state.unicorn._runs_since_symbolic_data = 100

angr/analyses/vfg.py

@@ -1,7 +1,6 @@
 import logging
 from collections import defaultdict
 
-import angr
 import archinfo
 from archinfo.arch_arm import is_arm_arch
 import claripy
@@ -12,7 +11,7 @@
 from .cfg.cfg_utils import CFGUtils
 from .forward_analysis import ForwardAnalysis
 from .. import sim_options
-from ..engines.procedure import ProcedureMixin
+from ..engines.procedure import ProcedureEngine
 from ..engines import SimSuccessors
 from ..errors import AngrDelayJobNotice, AngrSkipJobNotice, AngrVFGError, AngrError, AngrVFGRestartAnalysisNotice, \
     AngrJobMergingFailureNotice, SimValueError, SimIRSBError, SimError
@@ -1344,22 +1343,22 @@ def _get_simsuccessors(self, state, addr):
             error_occured = True
             inst = SIM_PROCEDURES["stubs"]["PathTerminator"](
                 state, self.project.arch)
-            sim_successors = SimEngineProcedure().process(state, inst)
+            sim_successors = ProcedureEngine().process(state, procedure=inst)
         except claripy.ClaripyError:
             l.error("ClaripyError: ", exc_info=True)
             error_occured = True
             # Generate a PathTerminator to terminate the current path
             inst = SIM_PROCEDURES["stubs"]["PathTerminator"](
                 state, self.project.arch)
-            sim_successors = SimEngineProcedure().process(state, inst)
+            sim_successors = ProcedureEngine().process(state, procedure=inst)
         except SimError:
             l.error("SimError: ", exc_info=True)
 
             error_occured = True
             # Generate a PathTerminator to terminate the current path
             inst = SIM_PROCEDURES["stubs"]["PathTerminator"](
                     state, self.project.arch)
-            sim_successors = SimEngineProcedure().process(state, inst)
+            sim_successors = ProcedureEngine().process(state, procedure=inst)
         except AngrError as ex:
             #segment = self.project.loader.main_object.in_which_segment(addr)
             l.error("AngrError %s when generating SimSuccessors at %#x",
@@ -1412,13 +1411,15 @@ def _create_new_jobs(self, job, successor, new_block_id, new_call_stack):
                     sp_difference = current_function.sp_delta
                 else:
                     sp_difference = 0
-                reg_sp_offset = successor_state.arch.sp_offset
-                reg_sp_expr = successor_state.registers.load(reg_sp_offset) + sp_difference
-                successor_state.registers.store(successor_state.arch.sp_offset, reg_sp_expr)
+                arch = successor_state.arch
+                reg_sp_offset = arch.sp_offset
+                reg_sp_expr = successor_state.registers.load(reg_sp_offset, size=arch.bytes,
+                                                             endness=arch.register_endness) + sp_difference
+                successor_state.registers.store(arch.sp_offset, reg_sp_expr)
 
                 # Clear the return value with a TOP
-                top_si = successor_state.solver.TSI(successor_state.arch.bits)
-                successor_state.registers.store(successor_state.arch.ret_offset, top_si)
+                top_si = successor_state.solver.TSI(arch.bits)
+                successor_state.registers.store(arch.ret_offset, top_si)
 
             if job.call_skipped:
 
@@ -1601,8 +1602,9 @@ def _is_return_jumpkind(jumpkind):
 
     @staticmethod
     def _create_stack_region(successor_state, successor_ip):
-        reg_sp_offset = successor_state.arch.sp_offset
-        reg_sp_expr = successor_state.registers.load(reg_sp_offset)
+        arch = successor_state.arch
+        reg_sp_offset = arch.sp_offset
+        reg_sp_expr = successor_state.registers.load(reg_sp_offset, size=arch.bytes, endness=arch.register_endness)
 
         if type(reg_sp_expr._model_vsa) is claripy.BVV:  # pylint:disable=unidiomatic-typecheck
             reg_sp_val = successor_state.solver.eval(reg_sp_expr)
@@ -1619,7 +1621,7 @@ def _create_stack_region(successor_state, successor_ip):
             reg_sp_si = next(iter(reg_sp_expr._model_vsa.items()))[1]
             reg_sp_val = reg_sp_si.min
 
-        reg_sp_val = reg_sp_val - successor_state.arch.bytes  # TODO: Is it OK?
+        reg_sp_val = reg_sp_val - arch.bytes  # TODO: Is it OK?
         new_stack_region_id = successor_state.memory.stack_id(successor_ip)
         successor_state.memory.set_stack_address_mapping(reg_sp_val,
                                                         new_stack_region_id,

angr/analyses/vsa_ddg.py

@@ -253,7 +253,7 @@ def _dump_edge_from_dict(dict_, key, del_key=True):
             if a.type == "mem":
                 if a.actual_addrs is None:
                     # For now, mem reads don't necessarily have actual_addrs set properly
-                    addr_list = set(state.memory.normalize_address(a.addr.ast, convert_to_valueset=True))
+                    addr_list = set(aw.to_valueset(state) for aw in state.memory._concretize_address_descriptor(a.addr.ast))
                 else:
                     addr_list = set(a.actual_addrs)
 

angr/block.py

@@ -90,7 +90,9 @@ def __init__(self, addr, project=None, arch=None, size=None, byte_string=None, v
         if byte_string is None:
             if backup_state is not None:
                 self._bytes = self._vex_engine._load_bytes(addr - thumb, size, state=backup_state)[0]
-                if type(self._bytes) is not bytes:
+                if type(self._bytes) is memoryview:
+                    self._bytes = bytes(self._bytes)
+                elif type(self._bytes) is not bytes:
                     self._bytes = bytes(pyvex.ffi.buffer(self._bytes, size))
             else:
                 self._bytes = None

angr/concretization_strategies/controlled_data.py

@@ -15,7 +15,7 @@ def __init__(self, limit, fixed_addrs, **kwargs):
 
     def _concretize(self, memory, addr):
         # Get all symbolic variables in memory
-        symbolic_vars = filter(lambda key: not key.startswith("reg_") and not key.startswith("mem_"), memory.mem._name_mapping.keys())
+        symbolic_vars = filter(lambda key: not key.startswith("reg_") and not key.startswith("mem_"), memory._name_mapping.keys())
         controlled_addrs = sorted([_addr for s_var in symbolic_vars for _addr in memory.addrs_for_name(s_var)])
         controlled_addrs.extend(self._fixed_addrs)
 

angr/concretization_strategies/max.py

@@ -1,9 +1,23 @@
+from typing import Optional
+
+from ..errors import SimSolverError
 from . import SimConcretizationStrategy
 
+
 class SimConcretizationStrategyMax(SimConcretizationStrategy):
     """
     Concretization strategy that returns the maximum address.
     """
 
+    def __init__(self, max_addr: Optional[int]=None):
+        super().__init__()
+        self._max_addr = max_addr
+
     def _concretize(self, memory, addr):
-        return [ self._max(memory, addr) ]
+        if self._max_addr is None:
+            return [ self._max(memory, addr) ]
+        else:
+            try:
+                return [ self._max(memory, addr, extra_constraints=(addr <= self._max_addr,)) ]
+            except SimSolverError:
+                return [ self._max(memory, addr) ]