8327110: Refactor create_bool_from_template_assertion_predicate() to …

…separate class and fix identical cloning cases used for Loop Unswitching and Split If

Reviewed-by: epeter, kvn

src/hotspot/share/opto/loopPredicate.cpp

@@ -365,12 +365,15 @@ void PhaseIdealLoop::get_assertion_predicates(Node* predicate, Unique_Node_List&
 // Clone an Assertion Predicate for an unswitched loop. OpaqueLoopInit and OpaqueLoopStride nodes are cloned and uncommon
 // traps are kept for the predicate (a Halt node is used later when creating pre/main/post loops and copying this cloned
 // predicate again).
-IfProjNode* PhaseIdealLoop::clone_assertion_predicate_for_unswitched_loops(Node* iff, IfProjNode* predicate,
+IfProjNode* PhaseIdealLoop::clone_assertion_predicate_for_unswitched_loops(IfNode* template_assertion_predicate,
+                                                                           IfProjNode* predicate,
                                                                            Deoptimization::DeoptReason reason,
                                                                            ParsePredicateSuccessProj* parse_predicate_proj) {
-  Node* bol = create_bool_from_template_assertion_predicate(iff, nullptr, nullptr, parse_predicate_proj);
-  IfProjNode* if_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, iff->Opcode(), false);
-  _igvn.replace_input_of(if_proj->in(0), 1, bol);
+  TemplateAssertionPredicateExpression template_assertion_predicate_expression(
+      template_assertion_predicate->in(1)->as_Opaque4());
+  Opaque4Node* cloned_opaque4_node = template_assertion_predicate_expression.clone(parse_predicate_proj, this);
+  IfProjNode* if_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, template_assertion_predicate->Opcode(), false);
+  _igvn.replace_input_of(if_proj->in(0), 1, cloned_opaque4_node);
   _igvn.replace_input_of(parse_predicate_proj->in(0), 0, if_proj);
   set_idom(parse_predicate_proj->in(0), if_proj, dom_depth(if_proj));
   return if_proj;

src/hotspot/share/opto/loopnode.hpp

@@ -28,6 +28,7 @@
 #include "opto/cfgnode.hpp"
 #include "opto/multnode.hpp"
 #include "opto/phaseX.hpp"
+#include "opto/predicates.hpp"
 #include "opto/subnode.hpp"
 #include "opto/type.hpp"
 #include "utilities/checkedCast.hpp"
@@ -1659,7 +1660,7 @@ class PhaseIdealLoop : public PhaseTransform {
                                                      Deoptimization::DeoptReason reason, IfProjNode* old_predicate_proj,
                                                      ParsePredicateSuccessProj* fast_loop_parse_predicate_proj,
                                                      ParsePredicateSuccessProj* slow_loop_parse_predicate_proj);
-  IfProjNode* clone_assertion_predicate_for_unswitched_loops(Node* iff, IfProjNode* predicate,
+  IfProjNode* clone_assertion_predicate_for_unswitched_loops(IfNode* template_assertion_predicate, IfProjNode* predicate,
                                                              Deoptimization::DeoptReason reason,
                                                              ParsePredicateSuccessProj* parse_predicate_proj);
   static void check_cloned_parse_predicate_for_unswitching(const Node* new_entry, bool is_fast_loop) PRODUCT_RETURN;
@@ -1673,6 +1674,12 @@ class PhaseIdealLoop : public PhaseTransform {
   bool created_loop_node()     { return _created_loop_node; }
   void register_new_node(Node* n, Node* blk);
 
+  Node* clone_and_register(Node* n, Node* ctrl) {
+    n = n->clone();
+    register_new_node(n, ctrl);
+    return n;
+  }
+
 #ifdef ASSERT
   void dump_bad_graph(const char* msg, Node* n, Node* early, Node* LCA);
 #endif
@@ -1906,7 +1913,10 @@ class DataNodeGraph : public StackObj {
  private:
   void clone(Node* node, Node* new_ctrl);
   void clone_data_nodes(Node* new_ctrl);
+  void clone_data_nodes_and_transform_opaque_loop_nodes(const TransformStrategyForOpaqueLoopNodes& transform_strategy,
+                                                        Node* new_ctrl);
   void rewire_clones_to_cloned_inputs();
+  void transform_opaque_node(const TransformStrategyForOpaqueLoopNodes& transform_strategy, Node* node);
 
  public:
   // Clone the provided data node collection and rewire the clones in such a way to create an identical graph copy.
@@ -1917,5 +1927,18 @@ class DataNodeGraph : public StackObj {
     rewire_clones_to_cloned_inputs();
     return _orig_to_new;
   }
+
+  // Create a copy of the data nodes provided to the constructor by doing the following:
+  // Clone all non-OpaqueLoop* nodes and rewire them to create an identical subgraph copy. For the OpaqueLoop* nodes,
+  // apply the provided transformation strategy and include the transformed node into the subgraph copy to get a complete
+  // "cloned-and-transformed" graph copy. For all newly cloned nodes (which could also be new OpaqueLoop* nodes), set
+  // `new_ctrl` as ctrl.
+  const OrigToNewHashtable& clone_with_opaque_loop_transform_strategy(
+      const TransformStrategyForOpaqueLoopNodes& transform_strategy,
+      Node* new_ctrl) {
+    clone_data_nodes_and_transform_opaque_loop_nodes(transform_strategy, new_ctrl);
+    rewire_clones_to_cloned_inputs();
+    return _orig_to_new;
+  }
 };
 #endif // SHARE_OPTO_LOOPNODE_HPP

src/hotspot/share/opto/loopopts.cpp

@@ -4502,7 +4502,7 @@ void DataNodeGraph::clone_data_nodes(Node* new_ctrl) {
   }
 }
 
-// Clone the given node and set it up properly. Set `new_ctrl` as ctrl.
+// Clone the given node and set it up properly. Set 'new_ctrl' as ctrl.
 void DataNodeGraph::clone(Node* node, Node* new_ctrl) {
   Node* clone = node->clone();
   _phase->igvn().register_new_node_with_optimizer(clone);
@@ -4523,3 +4523,30 @@ void DataNodeGraph::rewire_clones_to_cloned_inputs() {
     }
   });
 }
+
+// Clone all non-OpaqueLoop* nodes and apply the provided transformation strategy for OpaqueLoop* nodes.
+// Set 'new_ctrl' as ctrl for all cloned non-OpaqueLoop* nodes.
+void DataNodeGraph::clone_data_nodes_and_transform_opaque_loop_nodes(
+    const TransformStrategyForOpaqueLoopNodes& transform_strategy,
+    Node* new_ctrl) {
+  for (uint i = 0; i < _data_nodes.size(); i++) {
+    Node* data_node = _data_nodes[i];
+    if (data_node->is_Opaque1()) {
+      transform_opaque_node(transform_strategy, data_node);
+    } else {
+      clone(data_node, new_ctrl);
+    }
+  }
+}
+
+void DataNodeGraph::transform_opaque_node(const TransformStrategyForOpaqueLoopNodes& transform_strategy, Node* node) {
+  Node* transformed_node;
+  if (node->is_OpaqueLoopInit()) {
+    transformed_node = transform_strategy.transform_opaque_init(node->as_OpaqueLoopInit());
+  } else {
+    assert(node->is_OpaqueLoopStride(), "must be OpaqueLoopStrideNode");
+    transformed_node = transform_strategy.transform_opaque_stride(node->as_OpaqueLoopStride());
+  }
+  // Add an orig->new mapping to correctly update the inputs of the copied graph in rewire_clones_to_cloned_inputs().
+  _orig_to_new.put(node, transformed_node);
+}

src/hotspot/share/opto/node.hpp

@@ -134,6 +134,9 @@ class NegNode;
 class NegVNode;
 class NeverBranchNode;
 class Opaque1Node;
+class OpaqueLoopInitNode;
+class OpaqueLoopStrideNode;
+class Opaque4Node;
 class OuterStripMinedLoopNode;
 class OuterStripMinedLoopEndNode;
 class Node;
@@ -786,9 +789,12 @@ class Node {
     DEFINE_CLASS_ID(ClearArray, Node, 14)
     DEFINE_CLASS_ID(Halt,     Node, 15)
     DEFINE_CLASS_ID(Opaque1,  Node, 16)
-    DEFINE_CLASS_ID(Move,     Node, 17)
-    DEFINE_CLASS_ID(LShift,   Node, 18)
-    DEFINE_CLASS_ID(Neg,      Node, 19)
+      DEFINE_CLASS_ID(OpaqueLoopInit, Opaque1, 0)
+      DEFINE_CLASS_ID(OpaqueLoopStride, Opaque1, 1)
+    DEFINE_CLASS_ID(Opaque4,  Node, 17)
+    DEFINE_CLASS_ID(Move,     Node, 18)
+    DEFINE_CLASS_ID(LShift,   Node, 19)
+    DEFINE_CLASS_ID(Neg,      Node, 20)
 
     _max_classes  = ClassMask_Neg
   };
@@ -955,6 +961,9 @@ class Node {
   DEFINE_CLASS_QUERY(NegV)
   DEFINE_CLASS_QUERY(NeverBranch)
   DEFINE_CLASS_QUERY(Opaque1)
+  DEFINE_CLASS_QUERY(Opaque4)
+  DEFINE_CLASS_QUERY(OpaqueLoopInit)
+  DEFINE_CLASS_QUERY(OpaqueLoopStride)
   DEFINE_CLASS_QUERY(OuterStripMinedLoop)
   DEFINE_CLASS_QUERY(OuterStripMinedLoopEnd)
   DEFINE_CLASS_QUERY(Parm)

src/hotspot/share/opto/opaquenode.hpp

@@ -60,13 +60,15 @@ class Opaque1Node : public Node {
 class OpaqueLoopInitNode : public Opaque1Node {
   public:
   OpaqueLoopInitNode(Compile* C, Node *n) : Opaque1Node(C, n) {
+    init_class_id(Class_OpaqueLoopInit);
   }
   virtual int Opcode() const;
 };
 
 class OpaqueLoopStrideNode : public Opaque1Node {
   public:
   OpaqueLoopStrideNode(Compile* C, Node *n) : Opaque1Node(C, n) {
+    init_class_id(Class_OpaqueLoopStride);
   }
   virtual int Opcode() const;
 };
@@ -120,6 +122,7 @@ class Opaque3Node : public Node {
 class Opaque4Node : public Node {
   public:
   Opaque4Node(Compile* C, Node *tst, Node* final_tst) : Node(nullptr, tst, final_tst) {
+    init_class_id(Class_Opaque4);
     init_flags(Flag_is_macro);
     C->add_macro_node(this);
   }

src/hotspot/share/opto/predicates.cpp

@@ -24,6 +24,8 @@
 
 #include "precompiled.hpp"
 #include "opto/callnode.hpp"
+#include "opto/loopnode.hpp"
+#include "opto/node.hpp"
 #include "opto/predicates.hpp"
 
 // Walk over all Initialized Assertion Predicates and return the entry into the first Initialized Assertion Predicate
@@ -147,3 +149,127 @@ Node* PredicateBlock::skip_regular_predicates(Node* regular_predicate_proj, Deop
   }
   return entry;
 }
+
+// This strategy clones the OpaqueLoopInit and OpaqueLoopStride nodes.
+class CloneStrategy : public TransformStrategyForOpaqueLoopNodes {
+  PhaseIdealLoop* const _phase;
+  Node* const _new_ctrl;
+
+ public:
+  CloneStrategy(PhaseIdealLoop* phase, Node* new_ctrl)
+      : _phase(phase),
+        _new_ctrl(new_ctrl) {}
+  NONCOPYABLE(CloneStrategy);
+
+  Node* transform_opaque_init(OpaqueLoopInitNode* opaque_init) const override {
+    return _phase->clone_and_register(opaque_init, _new_ctrl)->as_OpaqueLoopInit();
+  }
+
+  Node* transform_opaque_stride(OpaqueLoopStrideNode* opaque_stride) const override {
+    return _phase->clone_and_register(opaque_stride, _new_ctrl)->as_OpaqueLoopStride();
+  }
+};
+
+// Creates an identical clone of this Template Assertion Predicate Expression (i.e.cloning all nodes from the Opaque4Node
+// to and including the OpaqueLoop* nodes). The cloned nodes are rewired to reflect the same graph structure as found for
+// this Template Assertion Predicate Expression. The cloned nodes get 'new_ctrl' as ctrl. There is no other update done
+// for the cloned nodes. Return the newly cloned Opaque4Node.
+Opaque4Node* TemplateAssertionPredicateExpression::clone(Node* new_ctrl, PhaseIdealLoop* phase) {
+  CloneStrategy clone_init_and_stride_strategy(phase, new_ctrl);
+  return clone(clone_init_and_stride_strategy, new_ctrl, phase);
+}
+
+// Class to collect data nodes from a source to target nodes by following the inputs of the source node recursively.
+// The class takes a node filter to decide which input nodes to follow and a target node predicate to start backtracking
+// from. All nodes found on all paths from source->target(s) are returned in a Unique_Node_List (without duplicates).
+class DataNodesOnPathsToTargets : public StackObj {
+  typedef bool (*NodeCheck)(const Node*);
+
+  // Node filter function to decide if we should process a node or not while searching for targets.
+  NodeCheck _node_filter;
+  // Function to decide if a node is a target node (i.e. where we should start backtracking). This check should also
+  // trivially pass the _node_filter.
+  NodeCheck _is_target_node;
+  // The resulting node collection of all nodes on paths from source->target(s).
+  Unique_Node_List _collected_nodes;
+  // List to track all nodes visited on the search for target nodes starting at a start node. These nodes are then used
+  // in backtracking to find the nodes actually being on a start->target(s) path. This list also serves as visited set
+  // to avoid double visits of a node which could happen with diamonds shapes.
+  Unique_Node_List _nodes_to_visit;
+
+ public:
+  DataNodesOnPathsToTargets(NodeCheck node_filter, NodeCheck is_target_node)
+      : _node_filter(node_filter),
+        _is_target_node(is_target_node) {}
+  NONCOPYABLE(DataNodesOnPathsToTargets);
+
+  // Collect all input nodes from 'start_node'->target(s) by applying the node filter to discover new input nodes and
+  // the target node predicate to stop discovering more inputs and start backtracking. The implementation is done
+  // with two BFS traversal: One to collect the target nodes (if any) and one to backtrack from the target nodes to
+  // find all other nodes on the start->target(s) paths.
+  const Unique_Node_List& collect(Node* start_node) {
+    assert(_collected_nodes.size() == 0 && _nodes_to_visit.size() == 0, "should not call this method twice in a row");
+    assert(!_is_target_node(start_node), "no trivial paths where start node is also a target node");
+
+    collect_target_nodes(start_node);
+    backtrack_from_target_nodes();
+    assert(_collected_nodes.size() == 0 || _collected_nodes.member(start_node),
+           "either target node predicate was never true or must find start node again when doing backtracking work");
+    return _collected_nodes;
+  }
+
+ private:
+  // Do a BFS from the start_node to collect all target nodes. We can then do another BFS from the target nodes to
+  // find all nodes on the paths from start->target(s).
+  // Note: We could do a single DFS pass to search targets and backtrack in one walk. But this is much more complex.
+  //       Given that the typical Template Assertion Predicate Expression only consists of a few nodes, we aim for
+  //       simplicity here.
+  void collect_target_nodes(Node* start_node) {
+    _nodes_to_visit.push(start_node);
+    for (uint i = 0; i < _nodes_to_visit.size(); i++) {
+      Node* next = _nodes_to_visit[i];
+      for (uint j = 1; j < next->req(); j++) {
+        Node* input = next->in(j);
+        if (_is_target_node(input)) {
+          assert(_node_filter(input), "must also pass node filter");
+          _collected_nodes.push(input);
+        } else if (_node_filter(input)) {
+          _nodes_to_visit.push(input);
+        }
+      }
+    }
+  }
+
+  // Backtrack from all previously collected target nodes by using the visited set of the start->target(s) search. If no
+  // node was collected in the first place (i.e. target node predicate was never true), then nothing needs to be done.
+  void backtrack_from_target_nodes() {
+    for (uint i = 0; i < _collected_nodes.size(); i++) {
+      Node* node_on_path = _collected_nodes[i];
+      for (DUIterator_Fast jmax, j = node_on_path->fast_outs(jmax); j < jmax; j++) {
+        Node* use = node_on_path->fast_out(j);
+        if (_nodes_to_visit.member(use)) {
+          // use must be on a path from start->target(s) because it was also visited in the first BFS starting from
+          // the start node.
+          _collected_nodes.push(use);
+        }
+      }
+    }
+  }
+};
+
+// Clones this Template Assertion Predicate Expression and applies the given strategy to transform the OpaqueLoop* nodes.
+Opaque4Node* TemplateAssertionPredicateExpression::clone(const TransformStrategyForOpaqueLoopNodes& transform_strategy,
+                                                         Node* new_ctrl, PhaseIdealLoop* phase) {
+  ResourceMark rm;
+  auto is_opaque_loop_node = [](const Node* node) {
+    return node->is_Opaque1();
+  };
+  DataNodesOnPathsToTargets data_nodes_on_path_to_targets(TemplateAssertionPredicateExpression::maybe_contains,
+                                                          is_opaque_loop_node);
+  const Unique_Node_List& collected_nodes = data_nodes_on_path_to_targets.collect(_opaque4_node);
+  DataNodeGraph data_node_graph(collected_nodes, phase);
+  const OrigToNewHashtable& orig_to_new = data_node_graph.clone_with_opaque_loop_transform_strategy(transform_strategy, new_ctrl);
+  assert(orig_to_new.contains(_opaque4_node), "must exist");
+  Node* opaque4_clone = *orig_to_new.get(_opaque4_node);
+  return opaque4_clone->as_Opaque4();
+}

src/hotspot/share/opto/predicates.hpp

@@ -26,6 +26,7 @@
 #define SHARE_OPTO_PREDICATES_HPP
 
 #include "opto/cfgnode.hpp"
+#include "opto/opaquenode.hpp"
 
 /*
  * There are different kinds of predicates throughout the code. We differentiate between the following predicates:
@@ -263,6 +264,51 @@ class RuntimePredicate : public StackObj {
   static bool is_success_proj(Node* node, Deoptimization::DeoptReason deopt_reason);
 };
 
+// Interface to transform OpaqueLoopInit and OpaqueLoopStride nodes of a Template Assertion Predicate Expression.
+class TransformStrategyForOpaqueLoopNodes : public StackObj {
+ public:
+  virtual Node* transform_opaque_init(OpaqueLoopInitNode* opaque_init) const = 0;
+  virtual Node* transform_opaque_stride(OpaqueLoopStrideNode* opaque_stride) const = 0;
+};
+
+// A Template Assertion Predicate Expression represents the Opaque4Node for the initial value or the last value of a
+// Template Assertion Predicate and all the nodes up to and including the OpaqueLoop* nodes.
+class TemplateAssertionPredicateExpression : public StackObj {
+  Opaque4Node* _opaque4_node;
+
+ public:
+  explicit TemplateAssertionPredicateExpression(Opaque4Node* opaque4_node) : _opaque4_node(opaque4_node) {}
+
+ private:
+  Opaque4Node* clone(const TransformStrategyForOpaqueLoopNodes& transform_strategy, Node* new_ctrl, PhaseIdealLoop* phase);
+
+ public:
+  // Is 'n' a node that could be part of a Template Assertion Predicate Expression (i.e. could be found on the input
+  // chain of a Template Assertion Predicate Opaque4Node up to and including the OpaqueLoop* nodes)?
+  static bool maybe_contains(const Node* n) {
+    const int opcode = n->Opcode();
+    return (opcode == Op_OpaqueLoopInit ||
+            opcode == Op_OpaqueLoopStride ||
+            n->is_Bool() ||
+            n->is_Cmp() ||
+            opcode == Op_AndL ||
+            opcode == Op_OrL ||
+            opcode == Op_RShiftL ||
+            opcode == Op_LShiftL ||
+            opcode == Op_LShiftI ||
+            opcode == Op_AddL ||
+            opcode == Op_AddI ||
+            opcode == Op_MulL ||
+            opcode == Op_MulI ||
+            opcode == Op_SubL ||
+            opcode == Op_SubI ||
+            opcode == Op_ConvI2L ||
+            opcode == Op_CastII);
+  }
+
+  Opaque4Node* clone(Node* new_ctrl, PhaseIdealLoop* phase);
+};
+
 // This class represents a Predicate Block (i.e. either a Loop Predicate Block, a Profiled Loop Predicate Block,
 // or a Loop Limit Check Predicate Block). It contains zero or more Regular Predicates followed by a Parse Predicate
 // which, however, does not need to exist (we could already have decided to remove Parse Predicates for this loop).

src/hotspot/share/opto/split_if.cpp

@@ -30,6 +30,7 @@
 #include "opto/movenode.hpp"
 #include "opto/node.hpp"
 #include "opto/opaquenode.hpp"
+#include "opto/predicates.hpp"
 
 //------------------------------split_thru_region------------------------------
 // Split Node 'n' through merge point.
@@ -101,8 +102,9 @@ bool PhaseIdealLoop::split_up( Node *n, Node *blk1, Node *blk2 ) {
       Node* m = wq.at(i);
       if (m->is_If()) {
         assert(assertion_predicate_has_loop_opaque_node(m->as_If()), "opaque node not reachable from if?");
-        Node* bol = create_bool_from_template_assertion_predicate(m, nullptr, nullptr, m->in(0));
-        _igvn.replace_input_of(m, 1, bol);
+        TemplateAssertionPredicateExpression template_assertion_predicate_expression(m->in(1)->as_Opaque4());
+        Opaque4Node* cloned_opaque4_node = template_assertion_predicate_expression.clone(m->in(0), this);
+        _igvn.replace_input_of(m, 1, cloned_opaque4_node);
       } else {
         assert(!m->is_CFG(), "not CFG expected");
         for (DUIterator_Fast jmax, j = m->fast_outs(jmax); j < jmax; j++) {