8253524: C2: Refactor code that clones predicates during loop unswitching

Reviewed-by: chagedorn, kvn, thartmann

Author: rwestrel

src/hotspot/share/opto/loopPredicate.cpp

@@ -210,8 +210,7 @@ ProjNode* PhaseIdealLoop::create_new_if_for_predicate(ProjNode* cont_proj, Node*
 }
 
 //--------------------------clone_predicate-----------------------
-ProjNode* PhaseIdealLoop::clone_predicate_to_unswitched_loop(ProjNode* predicate_proj, Node* new_entry, Deoptimization::DeoptReason reason,
-                                               bool is_slow_loop, uint idx_before_clone, Node_List &old_new) {
+ProjNode* PhaseIdealLoop::clone_predicate_to_unswitched_loop(ProjNode* predicate_proj, Node* new_entry, Deoptimization::DeoptReason reason) {
   ProjNode* new_predicate_proj = create_new_if_for_predicate(predicate_proj, new_entry, reason, Op_If);
   IfNode* iff = new_predicate_proj->in(0)->as_If();
   Node* ctrl  = iff->in(0);
@@ -225,17 +224,14 @@ ProjNode* PhaseIdealLoop::clone_predicate_to_unswitched_loop(ProjNode* predicate
   register_new_node(bol, ctrl);
   _igvn.hash_delete(iff);
   iff->set_req(1, bol);
-  clone_skeleton_predicates_to_unswitched_loop(reason, predicate_proj, new_predicate_proj, is_slow_loop, idx_before_clone, old_new);
   return new_predicate_proj;
 }
 
 // Clones skeleton predicates starting at 'old_predicate_proj' to
 // 'new_predicate_proj' and rewires the control edges of data nodes in
 // the loop from the old predicates to the new cloned predicates.
-void PhaseIdealLoop::clone_skeleton_predicates_to_unswitched_loop(Deoptimization::DeoptReason reason, ProjNode* old_predicate_proj,
-                                                    ProjNode* new_predicate_proj, bool is_slow_loop, uint idx_before_clone,
-                                                    Node_List &old_new) {
-  assert(old_predicate_proj->is_Proj(), "must be projection");
+void PhaseIdealLoop::clone_skeleton_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, Deoptimization::DeoptReason reason,
+                                                                  ProjNode* old_predicate_proj, ProjNode* iffast, ProjNode* ifslow) {
   IfNode* iff = old_predicate_proj->in(0)->as_If();
   ProjNode* uncommon_proj = iff->proj_out(1 - old_predicate_proj->as_Proj()->_con);
   Node* rgn = uncommon_proj->unique_ctrl_out();
@@ -255,21 +251,11 @@ void PhaseIdealLoop::clone_skeleton_predicates_to_unswitched_loop(Deoptimization
       // and doing loop unrolling. Push the original predicates on a list to later process them in reverse order to keep the
       // original predicate order.
       list.push(predicate);
-#ifdef ASSERT
-    } else {
-      // All other If predicates should not have a control input to nodes belonging to the original loop
-      for (DUIterator i = predicate->outs(); predicate->has_out(i); i++) {
-        Node* old_node = predicate->out(i);
-        Node* new_node = old_new[old_node->_idx];
-        if (!old_node->is_CFG() && new_node != NULL && old_node->_idx >= idx_before_clone) {
-          assert(false, "should not be part of the original loop");
-        }
-      }
-#endif
     }
     predicate = predicate->in(0)->in(0);
   }
 
+  Node_List to_process;
   // Process in reverse order such that 'create_new_if_for_predicate' can be used and the original order is maintained
   for (int i = list.size()-1; i >= 0; i--) {
     predicate = list.at(i);
@@ -279,58 +265,41 @@ void PhaseIdealLoop::clone_skeleton_predicates_to_unswitched_loop(Deoptimization
     IfProjNode* predicate_proj = predicate->as_IfProj();
 
     // cloned_proj is the same type of projection as the original predicate projection (IfTrue or IfFalse)
-    ProjNode* cloned_proj = create_new_if_for_predicate(new_predicate_proj, NULL, reason, iff->Opcode(), predicate_proj->is_IfTrue());
+    ProjNode* fast_proj = create_new_if_for_predicate(iffast, NULL, reason, iff->Opcode(), predicate_proj->is_IfTrue());
+    ProjNode* slow_proj = create_new_if_for_predicate(ifslow, NULL, reason, iff->Opcode(), predicate_proj->is_IfTrue());
 
     // Replace bool input by input from original predicate
-    _igvn.replace_input_of(cloned_proj->in(0), 1, iff->in(1));
-
-    if (is_slow_loop) {
-      for (DUIterator i = predicate->outs(); predicate->has_out(i); i++) {
-        Node* slow_node = predicate->out(i);
-        Node* fast_node = old_new[slow_node->_idx];
-        if (!slow_node->is_CFG() && fast_node != NULL && slow_node->_idx > idx_before_clone) {
-          // 'slow_node' is a data node and part of the slow loop. This is a clone of the fast loop node
-          // which was temporarily added below in order to verify that 'slow_node' is a clone of 'fast_node'.
-          // Update the control input and reset the mapping for 'slow_node' back to NULL.
-          _igvn.replace_input_of(slow_node, 0, cloned_proj);
-          old_new.map(slow_node->_idx, NULL);
-          --i;
-        }
-        assert(slow_node->_idx <= idx_before_clone || old_new[slow_node->_idx] == NULL, "mapping of cloned nodes must be null");
-      }
-    } else {
-      // Fast loop
-      for (DUIterator i = predicate->outs(); predicate->has_out(i); i++) {
-        Node* fast_node = predicate->out(i);
+    _igvn.replace_input_of(fast_proj->in(0), 1, iff->in(1));
+    _igvn.replace_input_of(slow_proj->in(0), 1, iff->in(1));
+
+    for (DUIterator i = predicate->outs(); predicate->has_out(i); i++) {
+      Node* fast_node = predicate->out(i);
+      if (loop->is_member(get_loop(ctrl_or_self(fast_node)))) {
+        assert(fast_node->in(0) == predicate, "only control edge");
         Node* slow_node = old_new[fast_node->_idx];
-        if (!fast_node->is_CFG() && slow_node != NULL && slow_node->_idx > idx_before_clone) {
-          // 'fast_node' is a data node and part of the fast loop. Add the clone of the fast loop node
-          // to the 'old_new' mapping in order to verify later when cloning the predicates for the slow loop
-          // that 'slow_node' is a clone of 'fast_node'. Update the control input for 'fast_node'.
-          _igvn.replace_input_of(fast_node, 0, cloned_proj);
-          assert(old_new[slow_node->_idx] == NULL, "mapping must be null for cloned nodes");
-          old_new.map(slow_node->_idx, fast_node);
-          --i;
-        }
+        assert(slow_node->in(0) == predicate, "only control edge");
+        _igvn.replace_input_of(fast_node, 0, fast_proj);
+        to_process.push(slow_node);
+        --i;
       }
     }
+    // Have to delay updates to the slow loop so uses of predicate are
+    // not modified while we iterate on them.
+    while (to_process.size() > 0) {
+      Node* slow_node = to_process.pop();
+      _igvn.replace_input_of(slow_node, 0, slow_proj);
+    }
   }
 }
 
 //--------------------------clone_loop_predicates-----------------------
 // Clone loop predicates to cloned loops when unswitching a loop.
-Node* PhaseIdealLoop::clone_predicates_to_unswitched_loop(Node* old_entry, Node* new_entry, bool clone_limit_check,
-                                            bool is_slow_loop, uint idx_before_clone, Node_List &old_new) {
-#ifdef ASSERT
-  assert(LoopUnswitching, "sanity - only called when unswitching a loop");
-  if (new_entry == NULL || !(new_entry->is_Proj() || new_entry->is_Region() || new_entry->is_SafePoint())) {
-    if (new_entry != NULL)
-      new_entry->dump();
-    assert(false, "not IfTrue, IfFalse, Region or SafePoint");
-  }
-#endif
+void PhaseIdealLoop::clone_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, ProjNode*& iffast, ProjNode*& ifslow) {
+  LoopNode* head = loop->_head->as_Loop();
+  bool clone_limit_check = !head->is_CountedLoop();
+  Node* entry = head->skip_strip_mined()->in(LoopNode::EntryControl);
+
   // Search original predicates
-  Node* entry = old_entry;
   ProjNode* limit_check_proj = NULL;
   limit_check_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check);
   if (limit_check_proj != NULL) {
@@ -349,39 +318,45 @@ Node* PhaseIdealLoop::clone_predicates_to_unswitched_loop(Node* old_entry, Node*
   }
   if (predicate_proj != NULL) { // right pattern that can be used by loop predication
     // clone predicate
-    new_entry = clone_predicate_to_unswitched_loop(predicate_proj, new_entry, Deoptimization::Reason_predicate, is_slow_loop,
-                                     idx_before_clone, old_new);
-    assert(new_entry != NULL && new_entry->is_Proj(), "IfTrue or IfFalse after clone predicate");
-    if (TraceLoopPredicate) {
-      tty->print("Loop Predicate cloned: ");
-      debug_only( new_entry->in(0)->dump(); );
-    }
+    iffast = clone_predicate_to_unswitched_loop(predicate_proj, iffast, Deoptimization::Reason_predicate);
+    ifslow = clone_predicate_to_unswitched_loop(predicate_proj, ifslow, Deoptimization::Reason_predicate);
+    clone_skeleton_predicates_to_unswitched_loop(loop, old_new, Deoptimization::Reason_predicate, predicate_proj, iffast, ifslow);
+
+    check_created_predicate_for_unswitching(iffast);
+    check_created_predicate_for_unswitching(ifslow);
   }
   if (profile_predicate_proj != NULL) { // right pattern that can be used by loop predication
     // clone predicate
-    new_entry = clone_predicate_to_unswitched_loop(profile_predicate_proj, new_entry,Deoptimization::Reason_profile_predicate,
-                                     is_slow_loop, idx_before_clone, old_new);
-    assert(new_entry != NULL && new_entry->is_Proj(), "IfTrue or IfFalse after clone predicate");
-    if (TraceLoopPredicate) {
-      tty->print("Loop Predicate cloned: ");
-      debug_only( new_entry->in(0)->dump(); );
-    }
+    iffast = clone_predicate_to_unswitched_loop(profile_predicate_proj, iffast, Deoptimization::Reason_profile_predicate);
+    ifslow = clone_predicate_to_unswitched_loop(profile_predicate_proj, ifslow, Deoptimization::Reason_profile_predicate);
+    clone_skeleton_predicates_to_unswitched_loop(loop, old_new, Deoptimization::Reason_profile_predicate, profile_predicate_proj, iffast, ifslow);
+
+    check_created_predicate_for_unswitching(iffast);
+    check_created_predicate_for_unswitching(ifslow);
   }
   if (limit_check_proj != NULL && clone_limit_check) {
     // Clone loop limit check last to insert it before loop.
     // Don't clone a limit check which was already finalized
     // for this counted loop (only one limit check is needed).
-    new_entry = clone_predicate_to_unswitched_loop(limit_check_proj, new_entry, Deoptimization::Reason_loop_limit_check,
-                                     is_slow_loop, idx_before_clone, old_new);
-    assert(new_entry != NULL && new_entry->is_Proj(), "IfTrue or IfFalse after clone limit check");
-    if (TraceLoopLimitCheck) {
-      tty->print("Loop Limit Check cloned: ");
-      debug_only( new_entry->in(0)->dump(); )
-    }
+    iffast = clone_predicate_to_unswitched_loop(limit_check_proj, iffast, Deoptimization::Reason_loop_limit_check);
+    ifslow = clone_predicate_to_unswitched_loop(limit_check_proj, ifslow, Deoptimization::Reason_loop_limit_check);
+
+    check_created_predicate_for_unswitching(iffast);
+    check_created_predicate_for_unswitching(ifslow);
   }
-  return new_entry;
 }
 
+#ifndef PRODUCT
+void PhaseIdealLoop::check_created_predicate_for_unswitching(const Node* new_entry) const {
+  assert(new_entry != NULL, "IfTrue or IfFalse after clone predicate");
+  if (TraceLoopPredicate) {
+    tty->print("Loop Predicate cloned: ");
+    debug_only(new_entry->in(0)->dump(););
+  }
+}
+#endif
+
+
 //--------------------------skip_loop_predicates------------------------------
 // Skip related predicates.
 Node* PhaseIdealLoop::skip_loop_predicates(Node* entry) {

src/hotspot/share/opto/loopUnswitch.cpp

@@ -275,19 +275,17 @@ ProjNode* PhaseIdealLoop::create_slow_version_of_loop(IdealLoopTree *loop,
   register_node(iffast, outer_loop, iff, dom_depth(iff));
   ProjNode* ifslow = new IfFalseNode(iff);
   register_node(ifslow, outer_loop, iff, dom_depth(iff));
-  uint idx_before_clone = Compile::current()->unique();
 
   // Clone the loop body.  The clone becomes the slow loop.  The
   // original pre-header will (illegally) have 3 control users
   // (old & new loops & new if).
   clone_loop(loop, old_new, dom_depth(head->skip_strip_mined()), mode, iff);
   assert(old_new[head->_idx]->is_Loop(), "" );
 
-  // Fast (true) control
-  Node* iffast_pred = clone_predicates_to_unswitched_loop(entry, iffast, !counted_loop, false, idx_before_clone, old_new);
-
-  // Slow (false) control
-  Node* ifslow_pred = clone_predicates_to_unswitched_loop(entry, ifslow, !counted_loop, true, idx_before_clone, old_new);
+  // Fast (true) and Slow (false) control
+  ProjNode* iffast_pred = iffast;
+  ProjNode* ifslow_pred = ifslow;
+  clone_predicates_to_unswitched_loop(loop, old_new, iffast_pred, ifslow_pred);
 
   Node* l = head->skip_strip_mined();
   _igvn.replace_input_of(l, LoopNode::EntryControl, iffast_pred);

src/hotspot/share/opto/loopnode.hpp

@@ -1431,13 +1431,11 @@ class PhaseIdealLoop : public PhaseTransform {
   }
 
   // Clone loop predicates to slow and fast loop when unswitching a loop
-  Node* clone_predicates_to_unswitched_loop(Node* old_entry, Node* new_entry, bool clone_limit_check, bool is_slow_loop,
-                              uint idx_before_clone, Node_List &old_new);
-  ProjNode* clone_predicate_to_unswitched_loop(ProjNode* predicate_proj, Node* new_entry, Deoptimization::DeoptReason reason,
-                                 bool is_slow_loop, uint idx_before_clone, Node_List &old_new);
-  void clone_skeleton_predicates_to_unswitched_loop(Deoptimization::DeoptReason reason, ProjNode* old_predicate_proj,
-                                      ProjNode* new_predicate_proj, bool is_slow_loop,
-                                      uint idx_before_clone, Node_List &old_new);
+  void clone_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, ProjNode*& iffast, ProjNode*& ifslow);
+  ProjNode* clone_predicate_to_unswitched_loop(ProjNode* predicate_proj, Node* new_entry, Deoptimization::DeoptReason reason);
+  void clone_skeleton_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, Deoptimization::DeoptReason reason,
+                                      ProjNode* old_predicate_proj, ProjNode* iffast, ProjNode* ifslow);
+  void check_created_predicate_for_unswitching(const Node* new_entry) const PRODUCT_RETURN;
 
   bool _created_loop_node;
 #ifdef ASSERT