8251925: C2: RenaissanceStressTest fails with assert(!had_error): bad dominance

Reviewed-by: kvn, thartmann

Author: chhagedorn

src/hotspot/share/opto/superword.cpp

@@ -69,7 +69,8 @@ SuperWord::SuperWord(PhaseIdealLoop* phase) :
   _nlist(arena(), 8, 0, NULL),            // scratch list of nodes
   _stk(arena(), 8, 0, NULL),              // scratch stack of nodes
   _lpt(NULL),                             // loop tree node
-  _lp(NULL),                              // LoopNode
+  _lp(NULL),                              // CountedLoopNode
+  _pre_loop_end(NULL),                    // Pre loop CountedLoopEndNode
   _bb(NULL),                              // basic block
   _iv(NULL),                              // induction var
   _race_possible(false),                  // cases where SDMU is true
@@ -155,10 +156,15 @@ void SuperWord::transform_loop(IdealLoopTree* lpt, bool do_optimization) {
 
   if (cl->is_main_loop()) {
     // Check for pre-loop ending with CountedLoopEnd(Bool(Cmp(x,Opaque1(limit))))
-    CountedLoopEndNode* pre_end = get_pre_loop_end(cl);
-    if (pre_end == NULL) return;
-    Node *pre_opaq1 = pre_end->limit();
-    if (pre_opaq1->Opcode() != Op_Opaque1) return;
+    CountedLoopEndNode* pre_end = find_pre_loop_end(cl);
+    if (pre_end == NULL) {
+      return;
+    }
+    Node* pre_opaq1 = pre_end->limit();
+    if (pre_opaq1->Opcode() != Op_Opaque1) {
+      return;
+    }
+    set_pre_loop_end(pre_end);
   }
 
   init(); // initialize data structures
@@ -911,8 +917,7 @@ bool SuperWord::ref_is_alignable(SWPointer& p) {
   if (!p.has_iv()) {
     return true;   // no induction variable
   }
-  CountedLoopEndNode* pre_end = get_pre_loop_end(lp()->as_CountedLoop());
-  assert(pre_end != NULL, "we must have a correct pre-loop");
+  CountedLoopEndNode* pre_end = pre_loop_end();
   assert(pre_end->stride_is_con(), "pre loop stride is constant");
   int preloop_stride = pre_end->stride_con();
 
@@ -3431,21 +3436,19 @@ LoadNode::ControlDependency SuperWord::control_dependency(Node_List* p) {
 // to align_to_ref will be a position zero in the vector.
 //   (iv + k) mod vector_align == 0
 void SuperWord::align_initial_loop_index(MemNode* align_to_ref) {
-  CountedLoopNode *main_head = lp()->as_CountedLoop();
-  assert(main_head->is_main_loop(), "");
-  CountedLoopEndNode* pre_end = get_pre_loop_end(main_head);
-  assert(pre_end != NULL, "we must have a correct pre-loop");
-  Node *pre_opaq1 = pre_end->limit();
+  assert(lp()->is_main_loop(), "");
+  CountedLoopEndNode* pre_end = pre_loop_end();
+  Node* pre_opaq1 = pre_end->limit();
   assert(pre_opaq1->Opcode() == Op_Opaque1, "");
-  Opaque1Node *pre_opaq = (Opaque1Node*)pre_opaq1;
-  Node *lim0 = pre_opaq->in(1);
+  Opaque1Node* pre_opaq = (Opaque1Node*)pre_opaq1;
+  Node* lim0 = pre_opaq->in(1);
 
   // Where we put new limit calculations
-  Node *pre_ctrl = pre_end->loopnode()->in(LoopNode::EntryControl);
+  Node* pre_ctrl = pre_loop_head()->in(LoopNode::EntryControl);
 
   // Ensure the original loop limit is available from the
   // pre-loop Opaque1 node.
-  Node *orig_limit = pre_opaq->original_loop_limit();
+  Node* orig_limit = pre_opaq->original_loop_limit();
   assert(orig_limit != NULL && _igvn.type(orig_limit) != Type::TOP, "");
 
   SWPointer align_to_ref_p(align_to_ref, this, NULL, false);
@@ -3596,7 +3599,7 @@ void SuperWord::align_initial_loop_index(MemNode* align_to_ref) {
 
 //----------------------------get_pre_loop_end---------------------------
 // Find pre loop end from main loop.  Returns null if none.
-CountedLoopEndNode* SuperWord::get_pre_loop_end(CountedLoopNode* cl) {
+CountedLoopEndNode* SuperWord::find_pre_loop_end(CountedLoopNode* cl) const {
   // The loop cannot be optimized if the graph shape at
   // the loop entry is inappropriate.
   if (!PhaseIdealLoop::is_canonical_loop_entry(cl)) {
@@ -3724,7 +3727,7 @@ SWPointer::SWPointer(MemNode* mem, SuperWord* slp, Node_Stack *nstack, bool anal
   // Match AddP(base, AddP(ptr, k*iv [+ invariant]), constant)
   Node* base = adr->in(AddPNode::Base);
   // The base address should be loop invariant
-  if (!invariant(base)) {
+  if (is_main_loop_member(base)) {
     assert(!valid(), "base address is loop variant");
     return;
   }
@@ -3753,7 +3756,7 @@ SWPointer::SWPointer(MemNode* mem, SuperWord* slp, Node_Stack *nstack, bool anal
       break; // stop looking at addp's
     }
   }
-  if (!invariant(adr)) {
+  if (is_main_loop_member(adr)) {
     assert(!valid(), "adr is loop variant");
     return;
   }
@@ -3783,12 +3786,23 @@ SWPointer::SWPointer(SWPointer* p) :
   #endif
 {}
 
+bool SWPointer::is_main_loop_member(Node* n) const {
+  Node* n_c = phase()->get_ctrl(n);
+  return lpt()->is_member(phase()->get_loop(n_c));
+}
 
-bool SWPointer::invariant(Node* n) {
+bool SWPointer::invariant(Node* n) const {
   NOT_PRODUCT(Tracer::Depth dd;)
-  Node *n_c = phase()->get_ctrl(n);
+  Node* n_c = phase()->get_ctrl(n);
   NOT_PRODUCT(_tracer.invariant_1(n, n_c);)
-  return !lpt()->is_member(phase()->get_loop(n_c));
+  bool is_not_member = !is_main_loop_member(n);
+  if (is_not_member && _slp->lp()->is_main_loop()) {
+    // Check that n_c dominates the pre loop head node. If it does not, then we cannot use n as invariant for the pre loop
+    // CountedLoopEndNode check because n_c is either part of the pre loop or between the pre and the main loop (illegal
+    // invariant: Happens, for example, when n_c is a CastII node that prevents data nodes to flow above the main loop).
+    return phase()->is_dominator(n_c, _slp->pre_loop_head());
+  }
+  return is_not_member;
 }
 
 //------------------------scaled_iv_plus_offset--------------------
@@ -3851,7 +3865,7 @@ bool SWPointer::scaled_iv(Node* n) {
     NOT_PRODUCT(_tracer.scaled_iv_3(n, _scale);)
     return true;
   }
-  if (_analyze_only && (invariant(n) == false)) {
+  if (_analyze_only && (is_main_loop_member(n))) {
     _nstack->push(n, _stack_idx++);
   }
 
@@ -3937,7 +3951,7 @@ bool SWPointer::offset_plus_k(Node* n, bool negate) {
     return false;
   }
 
-  if (_analyze_only && (invariant(n) == false)) {
+  if (_analyze_only && is_main_loop_member(n)) {
     _nstack->push(n, _stack_idx++);
   }
   if (opc == Op_AddI) {
@@ -3970,20 +3984,27 @@ bool SWPointer::offset_plus_k(Node* n, bool negate) {
       return true;
     }
   }
-  if (invariant(n)) {
+
+  if (!is_main_loop_member(n)) {
+    // 'n' is loop invariant. Skip range check dependent CastII nodes before checking if 'n' is dominating the pre loop.
     if (opc == Op_ConvI2L) {
       n = n->in(1);
       if (n->Opcode() == Op_CastII &&
           n->as_CastII()->has_range_check()) {
         // Skip range check dependent CastII nodes
-        assert(invariant(n), "sanity");
+        assert(!is_main_loop_member(n), "sanity");
         n = n->in(1);
       }
+
+      // Check if 'n' can really be used as invariant (not in main loop and dominating the pre loop).
+      if (invariant(n)) {
+        _negate_invar = negate;
+        _invar = n;
+        NOT_PRODUCT(_tracer.offset_plus_k_10(n, _invar, _negate_invar, _offset);)
+        return true;
+      }
     }
-    _negate_invar = negate;
-    _invar = n;
-    NOT_PRODUCT(_tracer.offset_plus_k_10(n, _invar, _negate_invar, _offset);)
-    return true;
+    return false;
   }
 
   NOT_PRODUCT(_tracer.offset_plus_k_11(n);)
@@ -4004,8 +4025,10 @@ void SWPointer::print() {
 
 //----------------------------tracing------------------------
 #ifndef PRODUCT
-void SWPointer::Tracer::print_depth() {
-  for (int ii = 0; ii<_depth; ++ii) tty->print("  ");
+void SWPointer::Tracer::print_depth() const {
+  for (int ii = 0; ii < _depth; ++ii) {
+    tty->print("  ");
+  }
 }
 
 void SWPointer::Tracer::ctor_1 (Node* mem) {
@@ -4057,7 +4080,7 @@ void SWPointer::Tracer::ctor_6(Node* mem) {
   }
 }
 
-void SWPointer::Tracer::invariant_1(Node *n, Node *n_c) {
+void SWPointer::Tracer::invariant_1(Node *n, Node *n_c) const {
   if (_slp->do_vector_loop() && _slp->is_debug() && _slp->_lpt->is_member(_slp->_phase->get_loop(n_c)) != (int)_slp->in_bb(n)) {
     int is_member =  _slp->_lpt->is_member(_slp->_phase->get_loop(n_c));
     int in_bb     =  _slp->in_bb(n);

src/hotspot/share/opto/superword.hpp

@@ -291,11 +291,11 @@ class SuperWord : public ResourceObj {
   void unrolling_analysis(int &local_loop_unroll_factor);
 
   // Accessors for SWPointer
-  PhaseIdealLoop* phase()          { return _phase; }
-  IdealLoopTree* lpt()             { return _lpt; }
-  PhiNode* iv()                    { return _iv; }
+  PhaseIdealLoop* phase() const    { return _phase; }
+  IdealLoopTree* lpt() const       { return _lpt; }
+  PhiNode* iv() const              { return _iv; }
 
-  bool early_return()              { return _early_return; }
+  bool early_return() const        { return _early_return; }
 
 #ifndef PRODUCT
   bool     is_debug()              { return _vector_loop_debug > 0; }
@@ -310,7 +310,8 @@ class SuperWord : public ResourceObj {
   bool     do_reserve_copy()       { return _do_reserve_copy; }
  private:
   IdealLoopTree* _lpt;             // Current loop tree node
-  LoopNode*      _lp;              // Current LoopNode
+  CountedLoopNode* _lp;            // Current CountedLoopNode
+  CountedLoopEndNode* _pre_loop_end; // Current CountedLoopEndNode of pre loop
   Node*          _bb;              // Current basic block
   PhiNode*       _iv;              // Induction var
   bool           _race_possible;   // In cases where SDMU is true
@@ -330,14 +331,33 @@ class SuperWord : public ResourceObj {
   Arena* arena()                   { return _arena; }
 
   Node* bb()                       { return _bb; }
-  void  set_bb(Node* bb)           { _bb = bb; }
-
+  void set_bb(Node* bb)            { _bb = bb; }
   void set_lpt(IdealLoopTree* lpt) { _lpt = lpt; }
+  CountedLoopNode* lp() const      { return _lp; }
+  void set_lp(CountedLoopNode* lp) {
+    _lp = lp;
+    _iv = lp->as_CountedLoop()->phi()->as_Phi();
+  }
+  int iv_stride() const            { return lp()->stride_con(); }
 
-  LoopNode* lp()                   { return _lp; }
-  void      set_lp(LoopNode* lp)   { _lp = lp;
-                                     _iv = lp->as_CountedLoop()->phi()->as_Phi(); }
-  int      iv_stride()             { return lp()->as_CountedLoop()->stride_con(); }
+  CountedLoopNode* pre_loop_head() const {
+    assert(_pre_loop_end != NULL && _pre_loop_end->loopnode() != NULL, "should find head from pre loop end");
+    return _pre_loop_end->loopnode();
+  }
+  void set_pre_loop_end(CountedLoopEndNode* pre_loop_end) {
+    assert(pre_loop_end, "must be valid");
+    _pre_loop_end = pre_loop_end;
+  }
+  CountedLoopEndNode* pre_loop_end() const {
+#ifdef ASSERT
+    assert(_lp != NULL, "sanity");
+    assert(_pre_loop_end != NULL, "should be set when fetched");
+    Node* found_pre_end = find_pre_loop_end(_lp);
+    assert(_pre_loop_end == found_pre_end && _pre_loop_end == pre_loop_head()->loopexit(),
+           "should find the pre loop end and must be the same result");
+#endif
+    return _pre_loop_end;
+  }
 
   int vector_width(Node* n) {
     BasicType bt = velt_basic_type(n);
@@ -531,7 +551,7 @@ class SuperWord : public ResourceObj {
   // to align_to_ref will be a position zero in the vector.
   void align_initial_loop_index(MemNode* align_to_ref);
   // Find pre loop end from main loop.  Returns null if none.
-  CountedLoopEndNode* get_pre_loop_end(CountedLoopNode *cl);
+  CountedLoopEndNode* find_pre_loop_end(CountedLoopNode *cl) const;
   // Is the use of d1 in u1 at the same operand position as d2 in u2?
   bool opnd_positions_match(Node* d1, Node* u1, Node* d2, Node* u2);
   void init();
@@ -567,11 +587,12 @@ class SWPointer {
   bool        _analyze_only; // Used in loop unrolling only for swpointer trace
   uint        _stack_idx;    // Used in loop unrolling only for swpointer trace
 
-  PhaseIdealLoop* phase() { return _slp->phase(); }
-  IdealLoopTree*  lpt()   { return _slp->lpt(); }
-  PhiNode*        iv()    { return _slp->iv();  } // Induction var
+  PhaseIdealLoop* phase() const { return _slp->phase(); }
+  IdealLoopTree*  lpt() const   { return _slp->lpt(); }
+  PhiNode*        iv() const    { return _slp->iv();  } // Induction var
 
-  bool invariant(Node* n);
+  bool is_main_loop_member(Node* n) const;
+  bool invariant(Node* n) const;
 
   // Match: k*iv + offset
   bool scaled_iv_plus_offset(Node* n);
@@ -638,7 +659,7 @@ class SWPointer {
     SuperWord*   _slp;
     static int   _depth;
     int _depth_save;
-    void print_depth();
+    void print_depth() const;
     int  depth() const    { return _depth; }
     void set_depth(int d) { _depth = d; }
     void inc_depth()      { _depth++;}
@@ -664,7 +685,7 @@ class SWPointer {
     void ctor_5(Node* adr, Node* base,  int i);
     void ctor_6(Node* mem);
 
-    void invariant_1(Node *n, Node *n_c);
+    void invariant_1(Node *n, Node *n_c) const;
 
     void scaled_iv_plus_offset_1(Node* n);
     void scaled_iv_plus_offset_2(Node* n);