8350579: Remove Template Assertion Predicates belonging to a loop once it is folded away

Reviewed-by: epeter, roland

Author: chhagedorn

src/hotspot/share/opto/loopTransform.cpp

@@ -1699,17 +1699,16 @@ bool IdealLoopTree::is_invariant(Node* n) const {
 
 // Search the Assertion Predicates added by loop predication and/or range check elimination and update them according
 // to the new stride.
-void PhaseIdealLoop::update_main_loop_assertion_predicates(CountedLoopNode* main_loop_head) {
-  Node* init = main_loop_head->init_trip();
-
+void PhaseIdealLoop::update_main_loop_assertion_predicates(CountedLoopNode* new_main_loop_head,
+                                                           const int stride_con_before_unroll) {
   // Compute the value of the loop induction variable at the end of the
   // first iteration of the unrolled loop: init + new_stride_con - init_inc
-  int unrolled_stride_con = main_loop_head->stride_con() * 2;
+  int unrolled_stride_con = stride_con_before_unroll * 2;
   Node* unrolled_stride = intcon(unrolled_stride_con);
 
-  Node* loop_entry = main_loop_head->skip_strip_mined()->in(LoopNode::EntryControl);
+  Node* loop_entry = new_main_loop_head->skip_strip_mined()->in(LoopNode::EntryControl);
   PredicateIterator predicate_iterator(loop_entry);
-  UpdateStrideForAssertionPredicates update_stride_for_assertion_predicates(unrolled_stride, this);
+  UpdateStrideForAssertionPredicates update_stride_for_assertion_predicates(unrolled_stride, new_main_loop_head, this);
   predicate_iterator.for_each(update_stride_for_assertion_predicates);
 }
 
@@ -1748,9 +1747,9 @@ void PhaseIdealLoop::initialize_assertion_predicates_for_post_loop(CountedLoopNo
 }
 
 void PhaseIdealLoop::create_assertion_predicates_at_loop(CountedLoopNode* source_loop_head,
-                                                          CountedLoopNode* target_loop_head,
-                                                          const NodeInLoopBody& _node_in_loop_body,
-                                                          const bool clone_template) {
+                                                         CountedLoopNode* target_loop_head,
+                                                         const NodeInLoopBody& _node_in_loop_body,
+                                                         const bool clone_template) {
   CreateAssertionPredicatesVisitor create_assertion_predicates_visitor(target_loop_head, this, _node_in_loop_body,
                                                                        clone_template);
   Node* source_loop_entry = source_loop_head->skip_strip_mined()->in(LoopNode::EntryControl);
@@ -1854,15 +1853,13 @@ void PhaseIdealLoop::do_unroll(IdealLoopTree *loop, Node_List &old_new, bool adj
   C->set_major_progress();
 
   Node* new_limit = nullptr;
-  int stride_con = stride->get_int();
+  const int stride_con = stride->get_int();
   int stride_p = (stride_con > 0) ? stride_con : -stride_con;
   uint old_trip_count = loop_head->trip_count();
   // Verify that unroll policy result is still valid.
   assert(old_trip_count > 1 && (!adjust_min_trip || stride_p <=
     MIN2<int>(max_jint / 2 - 2, MAX2(1<<3, Matcher::max_vector_size(T_BYTE)) * loop_head->unrolled_count())), "sanity");
 
-  update_main_loop_assertion_predicates(loop_head);
-
   // Adjust loop limit to keep valid iterations number after unroll.
   // Use (limit - stride) instead of (((limit - init)/stride) & (-2))*stride
   // which may overflow.
@@ -1996,7 +1993,7 @@ void PhaseIdealLoop::do_unroll(IdealLoopTree *loop, Node_List &old_new, bool adj
       phi   ->set_req(LoopNode::LoopBackControl, C->top());
     }
   }
-  Node *clone_head = old_new[loop_head->_idx];
+  CountedLoopNode* clone_head = old_new[loop_head->_idx]->as_CountedLoop();
   _igvn.hash_delete(clone_head);
   loop_head ->set_req(LoopNode::   EntryControl, clone_head->in(LoopNode::LoopBackControl));
   clone_head->set_req(LoopNode::LoopBackControl, loop_head ->in(LoopNode::LoopBackControl));
@@ -2025,6 +2022,8 @@ void PhaseIdealLoop::do_unroll(IdealLoopTree *loop, Node_List &old_new, bool adj
   loop->record_for_igvn();
   loop_head->clear_strip_mined();
 
+  update_main_loop_assertion_predicates(clone_head, stride_con);
+
 #ifndef PRODUCT
   if (C->do_vector_loop() && (PrintOpto && (VerifyLoopOptimizations || TraceLoopOpts))) {
     tty->print("\nnew loop after unroll\n");       loop->dump_head();

src/hotspot/share/opto/loopUnswitch.cpp

@@ -279,14 +279,16 @@ class UnswitchedLoopSelector : public StackObj {
 // Class to unswitch the original loop and create Predicates at the new unswitched loop versions. The newly cloned loop
 // becomes the false-path-loop while original loop becomes the true-path-loop.
 class OriginalLoop : public StackObj {
-  LoopNode* const _loop_head; // OuterStripMinedLoopNode if loop strip mined, else just the loop head.
+  LoopNode* const _loop_head;
+  LoopNode* const _outer_loop_head; // OuterStripMinedLoopNode if loop strip mined, else just the loop head.
   IdealLoopTree* const _loop;
   Node_List& _old_new;
   PhaseIdealLoop* const _phase;
 
  public:
   OriginalLoop(IdealLoopTree* loop, Node_List& old_new)
-      : _loop_head(loop->_head->as_Loop()->skip_strip_mined()),
+      : _loop_head(loop->_head->as_Loop()),
+        _outer_loop_head(loop->_head->as_Loop()->skip_strip_mined()),
         _loop(loop),
         _old_new(old_new),
         _phase(loop->_phase) {}
@@ -314,14 +316,14 @@ class OriginalLoop : public StackObj {
 
  private:
   void clone_loop(const LoopSelector& loop_selector) {
-    _phase->clone_loop(_loop, _old_new, _phase->dom_depth(_loop_head),
+    _phase->clone_loop(_loop, _old_new, _phase->dom_depth(_outer_loop_head),
                        PhaseIdealLoop::CloneIncludesStripMined, loop_selector.selector());
     fix_loop_entries(loop_selector);
   }
 
-  void fix_loop_entries(const LoopSelector& loop_selector) {
-    _phase->replace_loop_entry(_loop_head, loop_selector.true_path_loop_proj());
-    LoopNode* false_path_loop_strip_mined_head = old_to_new(_loop_head)->as_Loop();
+  void fix_loop_entries(const LoopSelector& loop_selector) const {
+    _phase->replace_loop_entry(_outer_loop_head, loop_selector.true_path_loop_proj());
+    LoopNode* false_path_loop_strip_mined_head = old_to_new(_outer_loop_head)->as_Loop();
     _phase->replace_loop_entry(false_path_loop_strip_mined_head,
                                loop_selector.false_path_loop_proj());
   }

src/hotspot/share/opto/loopnode.hpp

@@ -979,7 +979,7 @@ class PhaseIdealLoop : public PhaseTransform {
  private:
   static void get_opaque_template_assertion_predicate_nodes(ParsePredicateSuccessProj* parse_predicate_proj,
                                                             Unique_Node_List& list);
-  void update_main_loop_assertion_predicates(CountedLoopNode* main_loop_head);
+  void update_main_loop_assertion_predicates(CountedLoopNode* new_main_loop_head, int stride_con_before_unroll);
   void initialize_assertion_predicates_for_peeled_loop(CountedLoopNode* peeled_loop_head,
                                                        CountedLoopNode* remaining_loop_head,
                                                        uint first_node_index_in_cloned_loop_body,

src/hotspot/share/opto/opaquenode.cpp

@@ -112,8 +112,10 @@ const Type* OpaqueNotNullNode::Value(PhaseGVN* phase) const {
   return phase->type(in(1));
 }
 
-OpaqueTemplateAssertionPredicateNode::OpaqueTemplateAssertionPredicateNode(BoolNode* bol): Node(nullptr, bol),
-  _predicate_state(PredicateState::Useful) {
+OpaqueTemplateAssertionPredicateNode::OpaqueTemplateAssertionPredicateNode(BoolNode* bol,  CountedLoopNode* loop_node)
+    : Node(nullptr, bol),
+      _loop_node(loop_node),
+      _predicate_state(PredicateState::Useful) {
   init_class_id(Class_OpaqueTemplateAssertionPredicate);
 }
 
@@ -148,6 +150,7 @@ void OpaqueTemplateAssertionPredicateNode::mark_useless(PhaseIterGVN& igvn) {
 
 #ifndef PRODUCT
 void OpaqueTemplateAssertionPredicateNode::dump_spec(outputStream* st) const {
+  st->print("loop_idx=%d ", _loop_node->_idx);
   if (is_useless()) {
     st->print("#useless ");
   }

src/hotspot/share/opto/opaquenode.hpp

@@ -152,6 +152,9 @@ class OpaqueNotNullNode : public Node {
 // OpaqueTemplateAssertionPredicateNode is replaced by true in order to fold the Template Assertion Predicate away.
 class OpaqueTemplateAssertionPredicateNode : public Node {
 
+  // The counted loop this Template Assertion Predicate is associated with.
+  CountedLoopNode* _loop_node;
+
   // When splitting a loop or when the associated loop dies, the Template Assertion Predicate with this
   // OpaqueTemplateAssertionPredicateNode also needs to be removed. We set this flag and then clean this node up in the
   // next IGVN phase by checking this flag in Value().
@@ -164,14 +167,23 @@ class OpaqueTemplateAssertionPredicateNode : public Node {
   }
 
  public:
-  OpaqueTemplateAssertionPredicateNode(BoolNode* bol);
+  OpaqueTemplateAssertionPredicateNode(BoolNode* bol, CountedLoopNode* loop_node);
 
   virtual int Opcode() const;
   virtual uint size_of() const { return sizeof(*this); }
   virtual Node* Identity(PhaseGVN* phase);
   virtual const Type* Value(PhaseGVN* phase) const;
   virtual const Type* bottom_type() const { return TypeInt::BOOL; }
 
+  CountedLoopNode* loop_node() const {
+    return _loop_node;
+  }
+
+  // Should only be called during Loop Unrolling when we only update the OpaqueLoopStride input but don't require a full
+  // clone of the Template Assertion Expression.
+  void update_loop_node(CountedLoopNode* loop_node) {
+    _loop_node = loop_node;
+  }
 
   bool is_useless() const {
     return _predicate_state == PredicateState::Useless;