Continue to fix non-deterministic behaviors of parallel parsing

1. Force two rules to simplify the interactions between threads
1.A. Each basic block should be parsed and created only by one thread.
1.B. Each edge should be parsed and created only by one thread. To do
     this, we add edges source address into the ParseWorkElem. We also
     need to adjust non-returning function analysis because only one
     of shared functions will continue parsing and have the return
     status. All other functions should be delayed and wait for the
     winning function to finish parsing

These two rules ensure that we do not need a large critical section
 when we look for the correct source block of an edge during parsing
(See 2 for more detail)

2. When parsing an edge, the source block of the edge could have
   been split by the same thread. We need to follow the fallthrough
   edge to find the correct Block object that contains the edge.

   The two rules above ensure that the source block would not be
   split by a different thread, thus we do not need to build a
   large critical for when following the fall-through edges.

3. At finalizing time, fix the block spliting implementiation
   caused by functions sharing code. During parallel parsing,
   there is no range lookup because building a range lookup
   would cause mutual exclusion for threads. So, during parallel
   parsing, we only split blocks within a thread. For functions
   sharing code, this means overlapping blocks in separate functions.

parseAPI/src/CodeObject.C

@@ -296,13 +296,15 @@ CodeObject::parseNewEdges( vector<NewEdgeToParse> & worklist )
                 ( bundle, 
                   ParseWorkElem::checked_call_ft,
                   parser->link_tempsink(worklist[idx].source, worklist[idx].edge_type),
+                  worklist[idx].source->last(),
                   worklist[idx].target,
                   true,
                   false ));
             } else {
                 elem = bundle->add(new ParseWorkElem
                 ( bundle, 
                   parser->link_tempsink(worklist[idx].source, worklist[idx].edge_type),
+                  worklist[idx].source->last(),
                   worklist[idx].target,
                   true,
                   false ));

parseAPI/src/ParseData.C

@@ -50,6 +50,7 @@ void ParseFrame::set_status(Status s)
 ParseWorkElem * ParseFrame::mkWork(
     ParseWorkBundle * b,
     Edge * e,
+    Address source,
     Address target,
     bool resolvable,
     bool tailcall)
@@ -58,7 +59,7 @@ ParseWorkElem * ParseFrame::mkWork(
         b = new ParseWorkBundle();
         work_bundles.push_back(b); 
     }
-    ParseWorkElem * ret = new ParseWorkElem(b,e,target,resolvable,tailcall);
+    ParseWorkElem * ret = new ParseWorkElem(b,e,source, target,resolvable,tailcall);
     b->add( ret );
     return ret;
 }
@@ -76,6 +77,19 @@ ParseWorkElem * ParseFrame::mkWork(
     return ret;
 }
 
+ParseWorkElem * ParseFrame::mkWork(
+    ParseWorkBundle *b,
+    Function * shared_func)
+{
+    if(!b) {
+        b = new ParseWorkBundle();
+        work_bundles.push_back(b); 
+    }
+    ParseWorkElem * ret = new ParseWorkElem(b,shared_func);
+    b->add( ret );
+    return ret;
+
+}
 /**** Standard [no overlapping regions] ParseData ****/
 
 StandardParseData::StandardParseData(Parser *p) :
@@ -485,3 +499,22 @@ OverlappingParseData::reglookup(CodeRegion *cr, Address /* addr */)
 {
     return cr;
 }
+
+Function*
+OverlappingParseData::setEdgeParsingStatus(CodeRegion *cr, Address addr, Function *f)
+{
+    boost::lock_guard<ParseData> g(*this);
+    if(!HASHDEF(rmap,cr)) {
+        fprintf(stderr,"Error, invalid code region [%lx,%lx) in remove_frame\n",
+            cr->offset(),cr->offset()+cr->length());
+        return NULL;
+    }
+    region_data * rd = rmap[cr];
+    return rd->set_edge_parsed(addr,f);
+}
+
+void
+OverlappingParseData::getAllRegionData(vector<region_data*> &rds) {
+    for (auto rit = rmap.begin(); rit != rmap.end(); ++rit)
+        rds.push_back(rit->second);
+}

parseAPI/src/ParseData.h

@@ -87,14 +87,17 @@ class ParseFrame : public boost::lockable_adapter<boost::recursive_mutex> {
     ParseWorkElem * mkWork(
         ParseWorkBundle * b,
         Edge * e,
+        Address source,
         Address target,
         bool resolvable,
         bool tailcall);
     ParseWorkElem * mkWork(
         ParseWorkBundle * b,
 	Block* block,
         const InsnAdapter::IA_IAPI *ah);
-
+    ParseWorkElem * mkWork(
+        ParseWorkBundle *b,
+        Function* shared_func);
     void pushWork(ParseWorkElem * elem) {
         boost::lock_guard<ParseFrame> g(*this);
         parsing_printf("\t pushing work element for block %p, edge %p, target %p\n", elem->cur(), elem->edge(), elem->target());
@@ -183,6 +186,10 @@ class region_data {
     tbb::concurrent_hash_map<Address, ParseFrame *> frame_map;
     tbb::concurrent_hash_map<Address, ParseFrame::Status> frame_status;
 
+    // Edge parsing records
+    // We only want one thread to create edges for a location
+    tbb::concurrent_hash_map<Address, Function*> edge_parsing_status;
+
     Function * findFunc(Address entry);
     Block * findBlock(Address entry);
     int findFuncs(Address addr, set<Function *> & funcs);
@@ -286,6 +293,36 @@ class region_data {
         blocksByAddr.insert(make_pair(sink->start(),sink));
         blocksByRange.insert(sink);
     }
+
+    Function* set_edge_parsed(Address addr, Function *f) {
+        Function* ret=NULL;
+        race_detector_fake_lock_acquire(race_detector_fake_lock(edge_parsing_status));
+	{
+	  tbb::concurrent_hash_map<Address, Function*>::accessor a;
+          // A successful insertion means the thread should 
+          // continue to create edges. We return the passed in Function*
+          // as indication of successful insertion.
+          //
+          // Otherwise, another thread has started creating edges.
+          // The current thread should give up. We return
+          // the function who succeeded.
+          if (edge_parsing_status.insert(a, make_pair(addr, f)))
+              ret = f;
+          else
+              ret = a->second;
+	}
+        race_detector_fake_lock_release(race_detector_fake_lock(edge_parsing_status));
+        return ret;
+    }
+
+    void getAllBlocks(std::map<Address, Block*> &allBlocks) {
+        // This function should be only called in single-threaded mode,
+        // such as when we are finalizing parsing
+
+        // We convert hash map to an ordered tree
+        for (auto it = blocksByAddr.begin(); it != blocksByAddr.end(); ++it)
+            allBlocks[it->first] = it->second;
+    }
 };
 
 /** region_data inlines **/
@@ -397,6 +434,8 @@ class ParseData : public boost::lockable_adapter<boost::recursive_mutex>  {
     // does the Right Thing(TM) for standard- and overlapping-region 
     // object types
     virtual CodeRegion * reglookup(CodeRegion *cr, Address addr) =0;
+    virtual Function* setEdgeParsingStatus(CodeRegion *cr, Address addr, Function *f) = 0;
+    virtual void getAllRegionData(std::vector<region_data*>&) = 0;
 };
 
 /* StandardParseData represents parse data for Parsers that disallow
@@ -434,6 +473,9 @@ class StandardParseData : public ParseData {
     void remove_extents(const std::vector<FuncExtent*> &extents);
 
     CodeRegion * reglookup(CodeRegion *cr, Address addr);
+    Function* setEdgeParsingStatus(CodeRegion *cr, Address addr, Function *f); 
+    void getAllRegionData(std::vector<region_data*>& rds);
+
 };
 
 inline region_data * StandardParseData::findRegion(CodeRegion * /* cr */)
@@ -449,6 +491,15 @@ inline Block* StandardParseData::record_block(CodeRegion * /* cr */, Block *b)
     return _rdata.record_block(b);
 }
 
+inline Function* StandardParseData::setEdgeParsingStatus(CodeRegion *, Address addr, Function *f)
+{
+    return _rdata.set_edge_parsed(addr, f);
+}
+
+inline void StandardParseData::getAllRegionData(std::vector<region_data*> & rds) {
+    rds.push_back(&_rdata);
+} 
+
 /* OverlappingParseData handles binary code objects like .o files
    where CodeRegions may overlap on the same linear address space */
 class OverlappingParseData : public ParseData {
@@ -485,6 +536,8 @@ class OverlappingParseData : public ParseData {
     void remove_extents(const std::vector<FuncExtent*> &extents);
 
     CodeRegion * reglookup(CodeRegion *cr, Address addr);
+    Function* setEdgeParsingStatus(CodeRegion *cr, Address addr, Function* f); 
+    void getAllRegionData(std::vector<region_data*>&);
 };
 
 }