Batch pharos update for Jan 7, 2020.

Primary changes:

* Update for Rose API changes.

* Many changes to OOAnalyzer's Prolog code to improve performance on
  large executables.  Remove several unintentional choicepoints and a
  few other bugs.

* Add the mkir utility, which exports an intermediate representation
  of the program's semantics.  The IR is currently used internally by
  pathanalyzer.

.gitignore

@@ -29,6 +29,11 @@ TAGS
 /tools/fn2yara/fn2yara
 /tools/objdigger/objdigger
 
+# site.cmake is site-specific.  Do not check in.
+/site.cmake
+# Don't complain about the user file either.
+/local.cmake
+
 # test cruft
 /apitests.dot
 /tools/ooanalyzer/tests/**/*.ground.errors

CMakeLists.txt

@@ -28,6 +28,7 @@ if(PHAROS_FAIL_ON_WARNING)
 endif()
 
 set(CMAKE_INSTALL_RPATH_USE_LINK_PATH true)
+set(CMAKE_INSTALL_RPATH "$ORIGIN/../lib")
 
 # Load the user's customizations, then the site customization, then
 # the local customization.  This lets the local customization take

INSTALL.md

@@ -215,12 +215,12 @@ $ cd rose
 ```
 
 This version has a reasonable chance of working or only having minor
-issues.  If you want to be conservative, and use the version of ROSE
+issues.  If you want to be conservative and use the version of ROSE
 that was known to compile with the latest major commit to the Pharos
-repository, you can checkout this commit (ROSE version 0.9.11.114):
+repository, you can checkout this commit (ROSE version 0.9.12.13):
 
 ```
-$ git checkout 6d5586a9c9e9034d42240d104c6e5baf1a41aaf1
+$ git checkout 68630678c53d8a95c6ed5f37caf837acfba5ae92
 ```
 
 ROSE can be configured in a multitude of ways, and some attention to

cmake/BuildPharosPod.cmake

@@ -24,7 +24,7 @@ function(build_pharos_pod target podfile section)
     DEPENDS "${POD_BUILD}" ${POD_DEPENDS})
   add_custom_command(
     OUTPUT "${POD_MAN}"
-    COMMAND pod2man --section=${section} "${BUILT_POD}" "${POD_MAN}"
+    COMMAND pod2man --section=${section} --release=Pharos --center="Pharos Documentation" "${BUILT_POD}" "${POD_MAN}"
     MAIN_DEPENDENCY "${BUILT_POD}")
   add_custom_target(${target} ALL DEPENDS "${POD_MAN}")
   install(

libpharos/apigraph.cpp

@@ -1891,33 +1891,28 @@ bool ApiCfgComponent::ContainsCalls() const {
 void ApiCfgComponent::DisconnectVertex(ApiCfgVertex &v) {
 
   rose_addr_t vaddr = (*cfg_)[v].block->get_address();
+
   GDEBUG << "Disconnecting vertex: " << addr_str(vaddr) << LEND;
 
-  // removing only vertex - the entry and exit are now invalid
   if (boost::num_vertices(*cfg_) == 1) {
     entry_ = exit_ = INVALID_ADDRESS;
   }
   // Number of vertices > 1
   else {
-    // for each in-edge to the vertex to remove
 
-    BGL_FORALL_INEDGES(v, in_edge,*cfg_,ApiCfg) {
+    // for each in-edge to the vertex to remove
+    BGL_FORALL_INEDGES(v, in_edge, *cfg_, ApiCfg) {
       // get the previous vertex (source of the in edge)
       ApiCfgVertex prev = boost::source(in_edge, *cfg_);
       // connect the prev vertex to the next vertex with an edge
 
-      BGL_FORALL_OUTEDGES(v, out_edge,*cfg_,ApiCfg) {
+      BGL_FORALL_OUTEDGES(v, out_edge, *cfg_, ApiCfg) {
         ApiCfgVertex next = boost::target(out_edge, *cfg_);
 
         // if the edge doesn't already exist between these vertices
         if (false == boost::edge(prev, next, *cfg_).second) {
-
           // take the in edge and make it refer to next out edge vertex
           boost::add_edge(prev, next, *cfg_);
-
-          GDEBUG << "Adding edge from "
-                 << addr_str((*cfg_)[prev].block->get_address()) << " to "
-                 << addr_str((*cfg_)[next].block->get_address()) << LEND;
         }
       }
     }
@@ -1926,37 +1921,74 @@ void ApiCfgComponent::DisconnectVertex(ApiCfgVertex &v) {
   GDEBUG << "Deleting edges for " << addr_str(vaddr) << LEND;
   // remove in/out edges from the vertex to delete
   boost::clear_vertex(v, *cfg_);
-
-  // delete the vertex
-  boost::remove_vertex(v, *cfg_);
 }
 
 // remove every vertex that is not the entry, exit, of contains a function call
 void ApiCfgComponent::Simplify() {
 
-  std::vector < rose_addr_t > kill_list;
+  std::set<ApiCfgVertex> kill_list;
 
-  BGL_FORALL_VERTICES(vtx,*cfg_,ApiCfg) {
+  BGL_FORALL_VERTICES(vtx, *cfg_, ApiCfg) {
     ApiVertexInfo &vtxi = (*cfg_)[vtx];
     if (vtxi.block->get_address()!=entry_ && vtxi.block->get_address()!=exit_ && !vtxi.EndsInCall()) {
-      kill_list.push_back(vtxi.block->get_address());
+      kill_list.insert(vtx);
     }
   }
   KillVertices(kill_list);
 }
 
+void ApiCfgComponent::RemoveVertices(const std::set<ApiCfgVertex> &kill_list) {
+
+  struct IgnoreVertices {
+    const std::set<ApiCfgVertex> *kset;
+    IgnoreVertices() {}  // has to have a default constructor!
+    IgnoreVertices(const std::set<ApiCfgVertex> &k) : kset(&k) { }
+    bool operator()(ApiCfgVertex v) const {
+      return kset->find(v) == kset->end();
+    }
+  };
+
+  // ApiCfgPtr new_cfg_ = std::make_shared<ApiCfg>();
+  ApiCfg* new_cfg_ = new ApiCfg();
+  boost::copy_graph(boost::make_filtered_graph(*cfg_, boost::keep_all{},
+                                               IgnoreVertices(kill_list)),
+                    *new_cfg_);
+
+  cfg_.reset(new_cfg_);
+}
+
+void ApiCfgComponent::RemoveVertex(ApiCfgVertex target) {
+
+  if (boost::in_degree(target, *cfg_)>0 || boost::out_degree(target, *cfg_)>0) {
+    boost::clear_vertex(target, *cfg_);
+  }
+
+  struct IgnoreVertex {
+    IgnoreVertex() {}
+    IgnoreVertex(ApiCfgVertex v_) : v(v_) {}
+    bool operator()(ApiCfgVertex x) const { return x != v; }
+    ApiCfgVertex v;
+  };
+
+  ApiCfg* new_cfg_ = new ApiCfg();
+  boost::copy_graph(boost::make_filtered_graph(*cfg_, boost::keep_all{},
+                                               IgnoreVertex(target)),
+                    *new_cfg_);
+  cfg_.reset(new_cfg_);
+}
+
 // disconnect and delete a list of vertices
-void ApiCfgComponent::KillVertices(std::vector< rose_addr_t > &kill_list) {
+void ApiCfgComponent::KillVertices(std::set<ApiCfgVertex> &kill_list) {
 
   if (kill_list.empty() == true) {
     return;
   }
-  for (rose_addr_t kill_addr : kill_list)  {
-    ApiCfgVertex vertex2kill = GetVertexByAddr(kill_addr);
+  for (ApiCfgVertex vertex2kill : kill_list)  {
     if (vertex2kill != NULL_VERTEX) {
       DisconnectVertex(vertex2kill);
     }
   }
+  RemoveVertices(kill_list);
 }
 
 // Fetch the vertex associated with an address (by basic block address)
@@ -2289,10 +2321,8 @@ void ApiCfgComponent::Replace(ApiCfgVertex &out_vertex, ApiCfgVertex &in_entry_v
              << addr_str((*cfg_)[next].block->get_address()) << LEND;
     }
   }
-
   // disconnect and remove the out vertex
-  boost::clear_vertex(out_vertex, *cfg_);
-  boost::remove_vertex(out_vertex, *cfg_);
+  RemoveVertex(out_vertex);
 }
 
 void ApiCfgComponent::Print() {
@@ -2428,7 +2458,7 @@ void ApiGraph::BuildXrefs() {
 
 rose_addr_t ApiCfgComponent::ConsolidateReturns(BlockSet & retns) {
 
-  std::vector < rose_addr_t > kill_list; // the list of vertices (by address) to remove
+  std::set < ApiCfgVertex > kill_list; // the list of vertices (by address) to remove
 
   ApiCfgVertex exit_vertex = GetExitVertex();
   if (exit_vertex == NULL_VERTEX) {
@@ -2438,19 +2468,19 @@ rose_addr_t ApiCfgComponent::ConsolidateReturns(BlockSet & retns) {
   ApiVertexInfo &exit_info = (*cfg_)[exit_vertex]; // the vertex that corresponds to exit_
 
   for (const BlockSet::value_type & block : retns) {
-    if (block->get_address() != exit_info.block->get_address()) { // Not the exit vertex
-      ApiCfgVertex vtx = GetVertexByAddr(block->get_address());
-      if (vtx == NULL_VERTEX) {
+    ApiCfgVertex vtx = GetVertexByAddr(block->get_address());
+    if (vtx == NULL_VERTEX) {
         continue;
-      }
-
+    }
+    if (block->get_address() != exit_info.block->get_address()) { // Not the exit vertex
       // make the predecessors of the vertex to remove point to the one true return
       BGL_FORALL_INEDGES(vtx,in_edge,*cfg_,ApiCfg) {
         ApiCfgVertex src = boost::source(in_edge, *cfg_);
         boost::add_edge(src, exit_vertex, *cfg_);
       }
+
       // queue this vertex to be removed
-      kill_list.push_back(block->get_address());
+      kill_list.insert(vtx);
     }
   }
 
@@ -2504,6 +2534,7 @@ void ApiGraph::ConsolidateEmptyFunctions() {
             else {
 
               calling_cfg_comp->DisconnectVertex(call_vertex);
+              calling_cfg_comp->RemoveVertex(call_vertex);
 
               if (kill_list.find(empty_cfg_comp->GetEntryAddr()) == kill_list.end()) {
                 kill_list.insert(empty_cfg_comp->GetEntryAddr());

libpharos/apigraph.hpp

@@ -108,7 +108,8 @@ using ApiCfgComponentMapIter = ApiCfgComponentMap::iterator;
 // search types
 
 using IndexMap = boost::property_map < ApiCfg, boost::vertex_index_t >::type;
-using PredecessorMap = boost::iterator_property_map <ApiCfgVertex*,IndexMap,ApiCfgVertex,ApiCfgVertex&>;
+using PredecessorMap = boost::iterator_property_map <ApiCfgVertex*,IndexMap, ApiCfgVertex, ApiCfgVertex&>;
+
 using ApiSearchResultVector = boost::ptr_vector<ApiSearchResult>;
 using ApiWaypointVector = std::vector<ApiWaypointDescriptor>;
 
@@ -666,7 +667,6 @@ struct ApiBackEdgeVisitor : public boost::base_visitor<ApiBackEdgeVisitor> {
   ApiSearchExecutor *search_executor_;
 };
 
-
 using ApiCfgPtr = std::shared_ptr<ApiCfg>;
 
 // Represents information about each CFG, notably the CFG itself and entry/exit blocks
@@ -692,6 +692,8 @@ class ApiCfgComponent {
 
   ApiCfgPtr CloneApiCfg(ApiCfgPtr src_cfg);
 
+  ApiCfgPtr CloneApiCfg(ApiCfgPtr src_cfg, ApiCfgVertex exclude_vtx);
+
   ~ApiCfgComponent();
 
   void Initialize(const FunctionDescriptor &fd, AddrSet &api_calls, XrefMap &xrefs);
@@ -707,6 +709,10 @@ class ApiCfgComponent {
   // remove a vertex from a CFG
   void DisconnectVertex(ApiCfgVertex &v);
 
+  void RemoveVertices(const std::set<ApiCfgVertex>& kill_list); 
+
+  void RemoveVertex(ApiCfgVertex ); 
+
   void Replace(ApiCfgVertex &out_vertex, ApiCfgVertex &in_entry, ApiCfgVertex &in_exit_vertex);
 
   void InsertBefore(ApiCfgVertex &insert_vertex, ApiCfgVertex &in_entry_vertex, ApiCfgVertex &in_exit_vertex);
@@ -719,7 +725,7 @@ class ApiCfgComponent {
 
   void Simplify();
 
-  void KillVertices(std::vector< rose_addr_t > &kill_list);
+  void KillVertices(std::set< ApiCfgVertex > &kill_list);
 
   ApiCfgVertex GetVertexByAddr(const rose_addr_t addr) const;
 

libpharos/calls.cpp

@@ -44,7 +44,8 @@ LeafNodePtr CallDescriptor::get_stack_delta_variable() const {
   }
   if (stack_delta_variable == nullptr) {
     size_t arch_bits = ds.get_arch_bits();
-    stack_delta_variable = LeafNode::createVariable(arch_bits, "", UNKNOWN_STACK_DELTA);
+    stack_delta_variable = SymbolicExpr::makeIntegerVariable(
+      arch_bits, "", UNKNOWN_STACK_DELTA);
   }
   return stack_delta_variable;
 }

libpharos/defuse.cpp

@@ -67,8 +67,8 @@ BlockAnalysis::BlockAnalysis(DUAnalysis & _du, const ControlFlowGraph& cfg,
   conditions.reserve(num_conds);
 
   // assume every block can be entered
-  entry_condition = SymbolicValue::treenode_instance(Rose::BinaryAnalysis::SymbolicExpr::makeBoolean(true));
-  exit_condition =  SymbolicValue::treenode_instance(Rose::BinaryAnalysis::SymbolicExpr::makeBoolean(true));
+  entry_condition = SymbolicValue::treenode_instance(SymbolicExpr::makeBooleanConstant(true));
+  exit_condition =  SymbolicValue::treenode_instance(SymbolicExpr::makeBooleanConstant(true));
 
   size_t i = 0;
   for (const SgAsmBlock *pblock : cfg_in_bblocks(cfg, vertex)) {
@@ -493,7 +493,6 @@ BlockAnalysis::handle_stack_delta(SgAsmBlock *bb, SgAsmX86Instruction* insn,
   // By default, we have wrong knowledge of the stack pointer.
   StackDelta newdelta(0, ConfidenceWrong);
   // Get the stack register descriptor.
-  size_t arch_bits = du.ds.get_arch_bits();
   size_t arch_bytes = du.ds.get_arch_bytes();
   RegisterDescriptor esprd = du.ds.get_stack_reg();
   // Are confident in our current knowledge of the stack pointer based on emulation?  With
@@ -587,8 +586,8 @@ BlockAnalysis::handle_stack_delta(SgAsmBlock *bb, SgAsmX86Instruction* insn,
             auto sdv = cd->get_stack_delta_variable();
             if (sdv) {
               using Rose::BinaryAnalysis::SymbolicExpr::OP_ADD;
-              sum = InternalNode::create(arch_bits, OP_ADD, sum, sdv,
-                                         Rose::BinaryAnalysis::SmtSolverPtr());
+              sum = InternalNode::instance(OP_ADD, sum, sdv,
+                                           Rose::BinaryAnalysis::SmtSolverPtr());
             }
           }
           SymbolicValuePtr newesp = SymbolicValue::promote(oldesp->copy());
@@ -1805,7 +1804,7 @@ DUAnalysis::get_address_condition(const BlockAnalysis& pred_analysis,
   static auto nullnode = TreeNodePtr();
 
   if (insn_is_call(last_x86insn_in_block(pblock))) {
-    return SymbolicValue::treenode_instance(SymbolicExpr::makeBoolean(true));
+    return SymbolicValue::treenode_instance(SymbolicExpr::makeBooleanConstant(true));
   }
 
   if (!pred_analysis.output_state) {
@@ -1818,7 +1817,7 @@ DUAnalysis::get_address_condition(const BlockAnalysis& pred_analysis,
   }
 
   size_t arch_bits = ds.get_arch_bits();
-  TreeNodePtr leaf_addr = LeafNode::createInteger(arch_bits, bb_addr);
+  TreeNodePtr leaf_addr = SymbolicExpr::makeIntegerConstant(arch_bits, bb_addr);
   RegisterDescriptor iprd = ds.get_ip_reg();
   SymbolicValuePtr ip_sv = ip_reg_state->read_register(iprd);
 
@@ -2357,7 +2356,7 @@ get_leaf_condition(SymbolicValuePtr sv, TreeNodePtr target_leaf, TreeNodePtr par
         return SymbolicValue::treenode_instance(parent_condition);
       }
       // Can always get here
-      return SymbolicValue::treenode_instance(SymbolicExpr::makeBoolean(true));
+      return SymbolicValue::treenode_instance(SymbolicExpr::makeBooleanConstant(true));
     }
     // This leaf is not the address we are looking for
     return SymbolicValue::incomplete(1);

libpharos/funcs.cpp

@@ -597,7 +597,8 @@ void FunctionDescriptor::update_stack_delta(StackDelta sd) {
   stack_delta.confidence = sd.confidence;
   if (sd.confidence == ConfidenceMissing && !stack_delta_variable) {
     size_t arch_bits = ds.get_arch_bits();
-    stack_delta_variable = LeafNode::createVariable(arch_bits, "", UNKNOWN_STACK_DELTA);
+    stack_delta_variable = SymbolicExpr::makeIntegerVariable(
+      arch_bits, "", UNKNOWN_STACK_DELTA);
   } else {
     stack_delta_variable = LeafNodePtr();
   }