8296389: C2: PhaseCFG::convert_NeverBranch_to_Goto must handle both orders of successors

8298568: Fastdebug build fails after JDK-8296389

Reviewed-by: rrich
Backport-of: fabda246960cfdfff13c5a87de53d97169248172

Author: GoeLin

src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp

@@ -1009,7 +1009,7 @@ void G1BarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) co
               if (if_ctrl != load_ctrl) {
                 // Skip possible CProj->NeverBranch in infinite loops
                 if ((if_ctrl->is_Proj() && if_ctrl->Opcode() == Op_CProj)
-                    && (if_ctrl->in(0)->is_MultiBranch() && if_ctrl->in(0)->Opcode() == Op_NeverBranch)) {
+                    && if_ctrl->in(0)->is_NeverBranch()) {
                   if_ctrl = if_ctrl->in(0)->in(0);
                 }
               }

src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp

@@ -1028,7 +1028,7 @@ void ShenandoahBarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase p
                 if (if_ctrl != load_ctrl) {
                   // Skip possible CProj->NeverBranch in infinite loops
                   if ((if_ctrl->is_Proj() && if_ctrl->Opcode() == Op_CProj)
-                      && (if_ctrl->in(0)->is_MultiBranch() && if_ctrl->in(0)->Opcode() == Op_NeverBranch)) {
+                      && if_ctrl->in(0)->is_NeverBranch()) {
                     if_ctrl = if_ctrl->in(0)->in(0);
                   }
                 }

src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp

@@ -757,7 +757,7 @@ Node* ShenandoahBarrierC2Support::no_branches(Node* c, Node* dom, bool allow_one
         return NodeSentinel; // unsupported
       } else if (c->Opcode() == Op_CatchProj) {
         return NodeSentinel; // unsupported
-      } else if (c->Opcode() == Op_CProj && next->Opcode() == Op_NeverBranch) {
+      } else if (c->Opcode() == Op_CProj && next->is_NeverBranch()) {
         return NodeSentinel; // unsupported
       } else {
         assert(next->unique_ctrl_out() == c, "unsupported branch pattern");
@@ -2125,7 +2125,7 @@ Node* ShenandoahIUBarrierNode::Identity(PhaseGVN* phase) {
 static bool has_never_branch(Node* root) {
   for (uint i = 1; i < root->req(); i++) {
     Node* in = root->in(i);
-    if (in != NULL && in->Opcode() == Op_Halt && in->in(0)->is_Proj() && in->in(0)->in(0)->Opcode() == Op_NeverBranch) {
+    if (in != NULL && in->Opcode() == Op_Halt && in->in(0)->is_Proj() && in->in(0)->in(0)->is_NeverBranch()) {
       return true;
     }
   }
@@ -2156,20 +2156,20 @@ void MemoryGraphFixer::collect_memory_nodes() {
           if (in->in(0)->is_Region()) {
             Node* r = in->in(0);
             for (uint j = 1; j < r->req(); j++) {
-              assert(r->in(j)->Opcode() != Op_NeverBranch, "");
+              assert(!r->in(j)->is_NeverBranch(), "");
             }
           } else {
             Node* proj = in->in(0);
             assert(proj->is_Proj(), "");
             Node* in = proj->in(0);
-            assert(in->is_CallStaticJava() || in->Opcode() == Op_NeverBranch || in->Opcode() == Op_Catch || proj->is_IfProj(), "");
+            assert(in->is_CallStaticJava() || in->is_NeverBranch() || in->Opcode() == Op_Catch || proj->is_IfProj(), "");
             if (in->is_CallStaticJava()) {
               mem = in->in(TypeFunc::Memory);
             } else if (in->Opcode() == Op_Catch) {
               Node* call = in->in(0)->in(0);
               assert(call->is_Call(), "");
               mem = call->in(TypeFunc::Memory);
-            } else if (in->Opcode() == Op_NeverBranch) {
+            } else if (in->is_NeverBranch()) {
               mem = collect_memory_for_infinite_loop(in);
             }
           }
@@ -2641,7 +2641,7 @@ void MemoryGraphFixer::fix_mem(Node* ctrl, Node* new_ctrl, Node* mem, Node* mem_
               }
             }
           } else if (!mem_is_valid(m, u) &&
-                     !(u->Opcode() == Op_CProj && u->in(0)->Opcode() == Op_NeverBranch && u->as_Proj()->_con == 1)) {
+                     !(u->Opcode() == Op_CProj && u->in(0)->is_NeverBranch() && u->as_Proj()->_con == 1)) {
             uses.push(u);
           }
         }

src/hotspot/share/opto/block.cpp

@@ -620,10 +620,13 @@ static bool no_flip_branch(Block *b) {
 // fake exit path to infinite loops.  At this late stage they need to turn
 // into Goto's so that when you enter the infinite loop you indeed hang.
 void PhaseCFG::convert_NeverBranch_to_Goto(Block *b) {
-  // Find true target
   int end_idx = b->end_idx();
-  int idx = b->get_node(end_idx+1)->as_Proj()->_con;
-  Block *succ = b->_succs[idx];
+  NeverBranchNode* never_branch = b->get_node(end_idx)->as_NeverBranch();
+  Block* succ = get_block_for_node(never_branch->proj_out(0)->unique_ctrl_out_or_null());
+  Block* dead = get_block_for_node(never_branch->proj_out(1)->unique_ctrl_out_or_null());
+  assert(succ == b->_succs[0] || succ == b->_succs[1], "succ is a successor");
+  assert(dead == b->_succs[0] || dead == b->_succs[1], "dead is a successor");
+
   Node* gto = _goto->clone(); // get a new goto node
   gto->set_req(0, b->head());
   Node *bp = b->get_node(end_idx);
@@ -642,7 +645,6 @@ void PhaseCFG::convert_NeverBranch_to_Goto(Block *b) {
     }
   }
   // Kill alternate exit path
-  Block* dead = b->_succs[1 - idx];
   for (j = 1; j < dead->num_preds(); j++) {
     if (dead->pred(j)->in(0) == bp) {
       break;
@@ -740,7 +742,7 @@ void PhaseCFG::remove_empty_blocks() {
     // to give a fake exit path to infinite loops.  At this late stage they
     // need to turn into Goto's so that when you enter the infinite loop you
     // indeed hang.
-    if (block->get_node(block->end_idx())->Opcode() == Op_NeverBranch) {
+    if (block->get_node(block->end_idx())->is_NeverBranch()) {
       convert_NeverBranch_to_Goto(block);
     }
 

src/hotspot/share/opto/cfgnode.hpp

@@ -602,7 +602,10 @@ class CreateExNode : public TypeNode {
 // empty.
 class NeverBranchNode : public MultiBranchNode {
 public:
-  NeverBranchNode( Node *ctrl ) : MultiBranchNode(1) { init_req(0,ctrl); }
+  NeverBranchNode(Node* ctrl) : MultiBranchNode(1) {
+    init_req(0, ctrl);
+    init_class_id(Class_NeverBranch);
+  }
   virtual int Opcode() const;
   virtual bool pinned() const { return true; };
   virtual const Type *bottom_type() const { return TypeTuple::IFBOTH; }

src/hotspot/share/opto/loopPredicate.cpp

@@ -1433,7 +1433,7 @@ bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) {
   }
   LoopNode* head = loop->_head->as_Loop();
 
-  if (head->unique_ctrl_out()->Opcode() == Op_NeverBranch) {
+  if (head->unique_ctrl_out()->is_NeverBranch()) {
     // do nothing for infinite loops
     return false;
   }

src/hotspot/share/opto/loopnode.cpp

@@ -3711,9 +3711,9 @@ bool PhaseIdealLoop::only_has_infinite_loops() {
     if (n->is_Root()) {
       // Found root -> there was an exit!
       return false;
-    } else if (n->Opcode() == Op_NeverBranch) {
+    } else if (n->is_NeverBranch()) {
       // Only follow the loop-internal projection, not the NeverBranch exit
-      ProjNode* proj = n->as_Multi()->proj_out_or_null(0);
+      ProjNode* proj = n->as_NeverBranch()->proj_out_or_null(0);
       assert(proj != nullptr, "must find loop-internal projection of NeverBranch");
       worklist.push(proj);
     } else {

src/hotspot/share/opto/loopopts.cpp

@@ -774,8 +774,8 @@ static void enqueue_cfg_uses(Node* m, Unique_Node_List& wq) {
   for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
     Node* u = m->fast_out(i);
     if (u->is_CFG()) {
-      if (u->Opcode() == Op_NeverBranch) {
-        u = ((NeverBranchNode*)u)->proj_out(0);
+      if (u->is_NeverBranch()) {
+        u = u->as_NeverBranch()->proj_out(0);
         enqueue_cfg_uses(u, wq);
       } else {
         wq.push(u);
@@ -958,7 +958,7 @@ void PhaseIdealLoop::try_move_store_after_loop(Node* n) {
 #endif
             lca = place_outside_loop(lca, n_loop);
             assert(!n_loop->is_member(get_loop(lca)), "control must not be back in the loop");
-            assert(get_loop(lca)->_nest < n_loop->_nest || lca->in(0)->Opcode() == Op_NeverBranch, "must not be moved into inner loop");
+            assert(get_loop(lca)->_nest < n_loop->_nest || lca->in(0)->is_NeverBranch(), "must not be moved into inner loop");
 
             // Move store out of the loop
             _igvn.replace_node(hook, n->in(MemNode::Memory));
@@ -1159,7 +1159,7 @@ Node* PhaseIdealLoop::place_outside_loop(Node* useblock, IdealLoopTree* loop) co
     Node* dom = idom(useblock);
     if (loop->is_member(get_loop(dom)) ||
         // NeverBranch nodes are not assigned to the loop when constructed
-        (dom->Opcode() == Op_NeverBranch && loop->is_member(get_loop(dom->in(0))))) {
+        (dom->is_NeverBranch() && loop->is_member(get_loop(dom->in(0))))) {
       break;
     }
     useblock = dom;

src/hotspot/share/opto/node.hpp

@@ -914,6 +914,7 @@ class Node {
   DEFINE_CLASS_QUERY(Mul)
   DEFINE_CLASS_QUERY(Multi)
   DEFINE_CLASS_QUERY(MultiBranch)
+  DEFINE_CLASS_QUERY(NeverBranch)
   DEFINE_CLASS_QUERY(Opaque1)
   DEFINE_CLASS_QUERY(OuterStripMinedLoop)
   DEFINE_CLASS_QUERY(OuterStripMinedLoopEnd)
@@ -1070,6 +1071,8 @@ class Node {
   Node* find_similar(int opc);
 
   // Return the unique control out if only one. Null if none or more than one.
+  // Placeholder until 8281732 is backported.
+  Node* unique_ctrl_out_or_null() const { return unique_ctrl_out(); }
   Node* unique_ctrl_out() const;
 
   // Set control or add control as precedence edge

test/hotspot/jtreg/compiler/loopopts/TestPhaseCFGNeverBranchToGoto.jasm

@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+
+super public class TestPhaseCFGNeverBranchToGoto
+{
+    public Method "<init>":"()V"
+    stack 2 locals 1
+    {
+        aload_0;
+        invokespecial  Method java/lang/Object."<init>":"()V";
+        return;
+    }
+    static Method test:"(III)V"
+    stack 200 locals 200
+    {
+        iload      2;
+        ifeq LEND; // at runtime avoid the infinite-loop
+
+        iload      0;
+        ifeq L20;
+        goto L10;
+    L10:
+        goto L11;
+    L11:
+        iinc 0, 1;
+        iload      1;
+        ifge L10;
+        goto L11;
+    L20:
+        iload      1;
+        ifle L21;
+        goto L10;
+    L21:
+        iconst_m1; // eventually false
+        ifge L11;
+        goto L20;
+
+    LEND:
+        return;
+    }
+    public static Method main:"([Ljava/lang/String;)V"
+    stack 100 locals 100
+    {
+        iconst_0;
+        iconst_m1;
+        iconst_0;
+        invokestatic    Method test:"(III)V";
+        return;
+    }
+}

test/hotspot/jtreg/compiler/loopopts/TestPhaseCFGNeverBranchToGotoMain.java

@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8296389
+ * @summary Peeling of Irreducible loop can lead to NeverBranch being visited from either side
+ * @run main/othervm -Xcomp -XX:-TieredCompilation -XX:PerMethodTrapLimit=0
+ *      -XX:CompileCommand=compileonly,TestPhaseCFGNeverBranchToGotoMain::test
+ *      TestPhaseCFGNeverBranchToGotoMain
+ */
+
+/*
+ * @test
+ * @bug 8296389
+ * @compile TestPhaseCFGNeverBranchToGoto.jasm
+ * @summary Peeling of Irreducible loop can lead to NeverBranch being visited from either side
+ * @run main/othervm -Xcomp -XX:-TieredCompilation -XX:PerMethodTrapLimit=0
+ *      -XX:CompileCommand=compileonly,TestPhaseCFGNeverBranchToGoto::test
+ *      TestPhaseCFGNeverBranchToGoto
+ */
+
+
+public class TestPhaseCFGNeverBranchToGotoMain {
+    public static void main (String[] args) {
+        test(false, false);
+    }
+
+    public static void test(boolean flag1, boolean flag2) {
+        if (flag1) { // runtime check, avoid infinite loop
+            int a = 77;
+            int b = 0;
+            do { // empty loop
+                a--;
+                b++;
+            } while (a > 0);
+            // a == 0, b == 77 - after first loop-opts phase
+            int p = 0;
+            for (int i = 0; i < 4; i++) {
+                if ((i % 2) == 0) {
+                    p = 1;
+                }
+            }
+            // p == 1 - after second loop-opts phase (via unrolling)
+            // in first loop-opts phase we have 2x unrolling, 4x after second
+            int x = 1;
+            if (flag2) {
+                x = 3;
+            } // have region here, so that NeverBranch does not get removed
+            do { // infinite loop
+                do {
+                    // NeverBranch inserted here, during loop-opts 1
+                    x *= 2;
+                    if (p == 0) { // reason for peeling in loop-opts 1
+                        // after we know that p == 1, we lose this exit
+                        break;
+                    }
+                    // once we know that b == 77, we lose exit
+            } while (b == 77);
+            // after we lost both exits above, this loop gets cut off
+            int y = 5;
+                do {
+                    y *= 3;
+                } while (b == 77);
+            } while (true);
+        }
+    }
+}