diff --git a/src/hotspot/share/opto/cfgnode.hpp b/src/hotspot/share/opto/cfgnode.hpp index c2cc0161df1ff..e238259cac518 100644 --- a/src/hotspot/share/opto/cfgnode.hpp +++ b/src/hotspot/share/opto/cfgnode.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -28,6 +28,7 @@ #include "opto/multnode.hpp" #include "opto/node.hpp" #include "opto/opcodes.hpp" +#include "opto/predicates_enums.hpp" #include "opto/type.hpp" // Portions of code courtesy of Clifford Click @@ -58,10 +59,7 @@ class JumpProjNode; class SCMemProjNode; class PhaseIdealLoop; enum class AssertionPredicateType; - -// The success projection of a Parse Predicate is always an IfTrueNode and the uncommon projection an IfFalseNode -typedef IfTrueNode ParsePredicateSuccessProj; -typedef IfFalseNode ParsePredicateUncommonProj; +enum class PredicateState; //------------------------------RegionNode------------------------------------- // The class of RegionNodes, which can be mapped to basic blocks in the @@ -486,7 +484,11 @@ class RangeCheckNode : public IfNode { // More information about predicates can be found in loopPredicate.cpp. class ParsePredicateNode : public IfNode { Deoptimization::DeoptReason _deopt_reason; - bool _useless; // If the associated loop dies, this parse predicate becomes useless and can be cleaned up by Value(). + + // When a Parse Predicate loses its connection to a loop head, it will be marked useless by + // EliminateUselessPredicates and cleaned up by Value(). It can also become useless when cloning it to both loops + // during Loop Multiversioning - we no longer use the old version. + PredicateState _predicate_state; public: ParsePredicateNode(Node* control, Deoptimization::DeoptReason deopt_reason, PhaseGVN* gvn); virtual int Opcode() const; @@ -497,15 +499,21 @@ class ParsePredicateNode : public IfNode { } bool is_useless() const { - return _useless; + return _predicate_state == PredicateState::Useless; + } + + void mark_useless(PhaseIterGVN& igvn); + + void mark_maybe_useful() { + _predicate_state = PredicateState::MaybeUseful; } - void mark_useless() { - _useless = true; + bool is_useful() const { + return _predicate_state == PredicateState::Useful; } void mark_useful() { - _useless = false; + _predicate_state = PredicateState::Useful; } // Return the uncommon trap If projection of this Parse Predicate. diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index 014463ccc21c3..2afdc007a07d5 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -67,6 +67,7 @@ #include "opto/mulnode.hpp" #include "opto/narrowptrnode.hpp" #include "opto/node.hpp" +#include "opto/opaquenode.hpp" #include "opto/opcodes.hpp" #include "opto/output.hpp" #include "opto/parse.hpp" @@ -394,7 +395,7 @@ void Compile::remove_useless_node(Node* dead) { remove_expensive_node(dead); } if (dead->is_OpaqueTemplateAssertionPredicate()) { - remove_template_assertion_predicate_opaq(dead); + remove_template_assertion_predicate_opaque(dead->as_OpaqueTemplateAssertionPredicate()); } if (dead->is_ParsePredicate()) { remove_parse_predicate(dead->as_ParsePredicate()); @@ -450,7 +451,8 @@ void Compile::disconnect_useless_nodes(Unique_Node_List& useful, Unique_Node_Lis remove_useless_nodes(_macro_nodes, useful); // remove useless macro nodes remove_useless_nodes(_parse_predicates, useful); // remove useless Parse Predicate nodes - remove_useless_nodes(_template_assertion_predicate_opaqs, useful); // remove useless Assertion Predicate opaque nodes + // Remove useless Template Assertion Predicate opaque nodes + remove_useless_nodes(_template_assertion_predicate_opaques, useful); remove_useless_nodes(_expensive_nodes, useful); // remove useless expensive nodes remove_useless_nodes(_for_post_loop_igvn, useful); // remove useless node recorded for post loop opts IGVN pass remove_useless_unstable_if_traps(useful); // remove useless unstable_if traps @@ -648,7 +650,7 @@ Compile::Compile(ciEnv* ci_env, ciMethod* target, int osr_bci, _intrinsics(comp_arena(), 0, 0, nullptr), _macro_nodes(comp_arena(), 8, 0, nullptr), _parse_predicates(comp_arena(), 8, 0, nullptr), - _template_assertion_predicate_opaqs(comp_arena(), 8, 0, nullptr), + _template_assertion_predicate_opaques(comp_arena(), 8, 0, nullptr), _expensive_nodes(comp_arena(), 8, 0, nullptr), _for_post_loop_igvn(comp_arena(), 8, 0, nullptr), _unstable_if_traps(comp_arena(), 8, 0, nullptr), @@ -1824,8 +1826,7 @@ void Compile::mark_parse_predicate_nodes_useless(PhaseIterGVN& igvn) { } for (int i = 0; i < parse_predicate_count(); i++) { ParsePredicateNode* parse_predicate = _parse_predicates.at(i); - parse_predicate->mark_useless(); - igvn._worklist.push(parse_predicate); + parse_predicate->mark_useless(igvn); } _parse_predicates.clear(); } diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp index 1fe6180c54fab..cfd2567594d38 100644 --- a/src/hotspot/share/opto/compile.hpp +++ b/src/hotspot/share/opto/compile.hpp @@ -71,6 +71,7 @@ class Node_List; class Node_Notes; class NodeHash; class NodeCloneInfo; +class OpaqueTemplateAssertionPredicateNode; class OptoReg; class ParsePredicateNode; class PhaseCFG; @@ -370,8 +371,9 @@ class Compile : public Phase { GrowableArray _intrinsics; // List of intrinsics. GrowableArray _macro_nodes; // List of nodes which need to be expanded before matching. GrowableArray _parse_predicates; // List of Parse Predicates. - // List of OpaqueTemplateAssertionPredicateNode nodes for Template Assertion Predicates. - GrowableArray _template_assertion_predicate_opaqs; + // List of OpaqueTemplateAssertionPredicateNode nodes for Template Assertion Predicates which can be seen as list + // of Template Assertion Predicates themselves. + GrowableArray _template_assertion_predicate_opaques; GrowableArray _expensive_nodes; // List of nodes that are expensive to compute and that we'd better not let the GVN freely common GrowableArray _for_post_loop_igvn; // List of nodes for IGVN after loop opts are over GrowableArray _unstable_if_traps; // List of ifnodes after IGVN @@ -681,18 +683,21 @@ class Compile : public Phase { static IdealGraphPrinter* debug_network_printer() { return _debug_network_printer; } #endif + const GrowableArray& parse_predicates() const { + return _parse_predicates; + } + + const GrowableArray& template_assertion_predicate_opaques() const { + return _template_assertion_predicate_opaques; + } + int macro_count() const { return _macro_nodes.length(); } int parse_predicate_count() const { return _parse_predicates.length(); } - int template_assertion_predicate_count() const { return _template_assertion_predicate_opaqs.length(); } + int template_assertion_predicate_count() const { return _template_assertion_predicate_opaques.length(); } int expensive_count() const { return _expensive_nodes.length(); } int coarsened_count() const { return _coarsened_locks.length(); } Node* macro_node(int idx) const { return _macro_nodes.at(idx); } - ParsePredicateNode* parse_predicate(int idx) const { return _parse_predicates.at(idx); } - - Node* template_assertion_predicate_opaq_node(int idx) const { - return _template_assertion_predicate_opaqs.at(idx); - } Node* expensive_node(int idx) const { return _expensive_nodes.at(idx); } @@ -728,15 +733,15 @@ class Compile : public Phase { } } - void add_template_assertion_predicate_opaq(Node* n) { - assert(!_template_assertion_predicate_opaqs.contains(n), + void add_template_assertion_predicate_opaque(OpaqueTemplateAssertionPredicateNode* n) { + assert(!_template_assertion_predicate_opaques.contains(n), "Duplicate entry in Template Assertion Predicate OpaqueTemplateAssertionPredicate list"); - _template_assertion_predicate_opaqs.append(n); + _template_assertion_predicate_opaques.append(n); } - void remove_template_assertion_predicate_opaq(Node* n) { + void remove_template_assertion_predicate_opaque(OpaqueTemplateAssertionPredicateNode* n) { if (template_assertion_predicate_count() > 0) { - _template_assertion_predicate_opaqs.remove_if_existing(n); + _template_assertion_predicate_opaques.remove_if_existing(n); } } void add_coarsened_locks(GrowableArray& locks); diff --git a/src/hotspot/share/opto/ifnode.cpp b/src/hotspot/share/opto/ifnode.cpp index 5da8993306f2c..5d349caf103d2 100644 --- a/src/hotspot/share/opto/ifnode.cpp +++ b/src/hotspot/share/opto/ifnode.cpp @@ -31,7 +31,7 @@ #include "opto/connode.hpp" #include "opto/loopnode.hpp" #include "opto/phaseX.hpp" -#include "opto/predicates.hpp" +#include "opto/predicates_enums.hpp" #include "opto/runtime.hpp" #include "opto/rootnode.hpp" #include "opto/subnode.hpp" @@ -2169,7 +2169,7 @@ Node* RangeCheckNode::Ideal(PhaseGVN *phase, bool can_reshape) { ParsePredicateNode::ParsePredicateNode(Node* control, Deoptimization::DeoptReason deopt_reason, PhaseGVN* gvn) : IfNode(control, gvn->intcon(1), PROB_MAX, COUNT_UNKNOWN), _deopt_reason(deopt_reason), - _useless(false) { + _predicate_state(PredicateState::Useful) { init_class_id(Class_ParsePredicate); gvn->C->add_parse_predicate(this); gvn->C->record_for_post_loop_opts_igvn(this); @@ -2186,6 +2186,11 @@ ParsePredicateNode::ParsePredicateNode(Node* control, Deoptimization::DeoptReaso #endif // ASSERT } +void ParsePredicateNode::mark_useless(PhaseIterGVN& igvn) { + _predicate_state = PredicateState::Useless; + igvn._worklist.push(this); +} + Node* ParsePredicateNode::uncommon_trap() const { ParsePredicateUncommonProj* uncommon_proj = proj_out(0)->as_IfFalse(); Node* uct_region_or_call = uncommon_proj->unique_ctrl_out(); @@ -2195,14 +2200,15 @@ Node* ParsePredicateNode::uncommon_trap() const { // Fold this node away once it becomes useless or at latest in post loop opts IGVN. const Type* ParsePredicateNode::Value(PhaseGVN* phase) const { + assert(_predicate_state != PredicateState::MaybeUseful, "should only be MaybeUseful when eliminating useless " + "predicates during loop opts"); if (phase->type(in(0)) == Type::TOP) { return Type::TOP; } - if (_useless || phase->C->post_loop_opts_phase()) { + if (_predicate_state == PredicateState::Useless || phase->C->post_loop_opts_phase()) { return TypeTuple::IFTRUE; - } else { - return bottom_type(); } + return bottom_type(); } #ifndef PRODUCT @@ -2224,7 +2230,7 @@ void ParsePredicateNode::dump_spec(outputStream* st) const { default: fatal("unknown kind"); } - if (_useless) { + if (_predicate_state == PredicateState::Useless) { st->print("#useless "); } } diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 7d83d602ac9d8..cb3bcee7194eb 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -4399,116 +4399,13 @@ void PhaseIdealLoop::log_loop_tree() { // Eliminate all Parse and Template Assertion Predicates that are not associated with a loop anymore. The eliminated // predicates will be removed during the next round of IGVN. -void PhaseIdealLoop::eliminate_useless_predicates() { +void PhaseIdealLoop::eliminate_useless_predicates() const { if (C->parse_predicate_count() == 0 && C->template_assertion_predicate_count() == 0) { return; // No predicates left. } - eliminate_useless_parse_predicates(); - eliminate_useless_template_assertion_predicates(); -} - -// Eliminate all Parse Predicates that do not belong to a loop anymore by marking them useless. These will be removed -// during the next round of IGVN. -void PhaseIdealLoop::eliminate_useless_parse_predicates() { - mark_all_parse_predicates_useless(); - if (C->has_loops()) { - mark_loop_associated_parse_predicates_useful(); - } - add_useless_parse_predicates_to_igvn_worklist(); -} - -void PhaseIdealLoop::mark_all_parse_predicates_useless() const { - for (int i = 0; i < C->parse_predicate_count(); i++) { - C->parse_predicate(i)->mark_useless(); - } -} - -void PhaseIdealLoop::mark_loop_associated_parse_predicates_useful() { - for (LoopTreeIterator iterator(_ltree_root); !iterator.done(); iterator.next()) { - IdealLoopTree* loop = iterator.current(); - if (loop->can_apply_loop_predication()) { - mark_useful_parse_predicates_for_loop(loop); - } - } -} - -// This visitor marks all visited Parse Predicates useful. -class ParsePredicateUsefulMarker : public PredicateVisitor { - public: - using PredicateVisitor::visit; - - void visit(const ParsePredicate& parse_predicate) override { - parse_predicate.head()->mark_useful(); - } -}; - -void PhaseIdealLoop::mark_useful_parse_predicates_for_loop(IdealLoopTree* loop) { - Node* entry = loop->_head->as_Loop()->skip_strip_mined()->in(LoopNode::EntryControl); - const PredicateIterator predicate_iterator(entry); - ParsePredicateUsefulMarker useful_marker; - predicate_iterator.for_each(useful_marker); -} - -void PhaseIdealLoop::add_useless_parse_predicates_to_igvn_worklist() { - for (int i = 0; i < C->parse_predicate_count(); i++) { - ParsePredicateNode* parse_predicate_node = C->parse_predicate(i); - if (parse_predicate_node->is_useless()) { - _igvn._worklist.push(parse_predicate_node); - } - } -} - - -// Eliminate all Template Assertion Predicates that do not belong to their originally associated loop anymore by -// replacing the OpaqueTemplateAssertionPredicate node of the If node with true. These nodes will be removed during the -// next round of IGVN. -void PhaseIdealLoop::eliminate_useless_template_assertion_predicates() { - Unique_Node_List useful_predicates; - if (C->has_loops()) { - collect_useful_template_assertion_predicates(useful_predicates); - } - eliminate_useless_template_assertion_predicates(useful_predicates); -} - -void PhaseIdealLoop::collect_useful_template_assertion_predicates(Unique_Node_List& useful_predicates) { - for (LoopTreeIterator iterator(_ltree_root); !iterator.done(); iterator.next()) { - IdealLoopTree* loop = iterator.current(); - if (loop->can_apply_loop_predication()) { - collect_useful_template_assertion_predicates_for_loop(loop, useful_predicates); - } - } -} - -void PhaseIdealLoop::collect_useful_template_assertion_predicates_for_loop(IdealLoopTree* loop, - Unique_Node_List &useful_predicates) { - Node* entry = loop->_head->as_Loop()->skip_strip_mined()->in(LoopNode::EntryControl); - const Predicates predicates(entry); - if (UseProfiledLoopPredicate) { - const PredicateBlock* profiled_loop_predicate_block = predicates.profiled_loop_predicate_block(); - if (profiled_loop_predicate_block->has_parse_predicate()) { - ParsePredicateSuccessProj* parse_predicate_proj = profiled_loop_predicate_block->parse_predicate_success_proj(); - get_opaque_template_assertion_predicate_nodes(parse_predicate_proj, useful_predicates); - } - } - - if (UseLoopPredicate) { - const PredicateBlock* loop_predicate_block = predicates.loop_predicate_block(); - if (loop_predicate_block->has_parse_predicate()) { - ParsePredicateSuccessProj* parse_predicate_proj = loop_predicate_block->parse_predicate_success_proj(); - get_opaque_template_assertion_predicate_nodes(parse_predicate_proj, useful_predicates); - } - } -} - -void PhaseIdealLoop::eliminate_useless_template_assertion_predicates(Unique_Node_List& useful_predicates) const { - for (int i = C->template_assertion_predicate_count(); i > 0; i--) { - OpaqueTemplateAssertionPredicateNode* opaque_node = - C->template_assertion_predicate_opaq_node(i - 1)->as_OpaqueTemplateAssertionPredicate(); - if (!useful_predicates.member(opaque_node)) { // not in the useful list - opaque_node->mark_useless(_igvn); - } - } + EliminateUselessPredicates eliminate_useless_predicates(_igvn, _ltree_root); + eliminate_useless_predicates.eliminate(); } // If a post or main loop is removed due to an assert predicate, the opaque that guards the loop is not needed anymore diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index f28ad41aad5e6..f2ef38c5b6716 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -1442,18 +1442,7 @@ class PhaseIdealLoop : public PhaseTransform { void eliminate_hoisted_range_check(IfTrueNode* hoisted_check_proj, IfTrueNode* template_assertion_predicate_proj); // Helper function to collect predicate for eliminating the useless ones - void eliminate_useless_predicates(); - - void eliminate_useless_parse_predicates(); - void mark_all_parse_predicates_useless() const; - void mark_loop_associated_parse_predicates_useful(); - static void mark_useful_parse_predicates_for_loop(IdealLoopTree* loop); - void add_useless_parse_predicates_to_igvn_worklist(); - - void eliminate_useless_template_assertion_predicates(); - void collect_useful_template_assertion_predicates(Unique_Node_List& useful_predicates); - static void collect_useful_template_assertion_predicates_for_loop(IdealLoopTree* loop, Unique_Node_List& useful_predicates); - void eliminate_useless_template_assertion_predicates(Unique_Node_List& useful_predicates) const; + void eliminate_useless_predicates() const; void eliminate_useless_zero_trip_guard(); void eliminate_useless_multiversion_if(); diff --git a/src/hotspot/share/opto/node.cpp b/src/hotspot/share/opto/node.cpp index d1b3d96b76b84..d31d8a6e212c9 100644 --- a/src/hotspot/share/opto/node.cpp +++ b/src/hotspot/share/opto/node.cpp @@ -511,6 +511,9 @@ Node *Node::clone() const { if (n->is_ParsePredicate()) { C->add_parse_predicate(n->as_ParsePredicate()); } + if (n->is_OpaqueTemplateAssertionPredicate()) { + C->add_template_assertion_predicate_opaque(n->as_OpaqueTemplateAssertionPredicate()); + } BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); bs->register_potential_barrier_node(n); @@ -607,7 +610,7 @@ void Node::destruct(PhaseValues* phase) { compile->remove_expensive_node(this); } if (is_OpaqueTemplateAssertionPredicate()) { - compile->remove_template_assertion_predicate_opaq(this); + compile->remove_template_assertion_predicate_opaque(as_OpaqueTemplateAssertionPredicate()); } if (is_ParsePredicate()) { compile->remove_parse_predicate(as_ParsePredicate()); diff --git a/src/hotspot/share/opto/opaquenode.cpp b/src/hotspot/share/opto/opaquenode.cpp index f0730374045f8..f6ef79a537968 100644 --- a/src/hotspot/share/opto/opaquenode.cpp +++ b/src/hotspot/share/opto/opaquenode.cpp @@ -112,6 +112,11 @@ const Type* OpaqueNotNullNode::Value(PhaseGVN* phase) const { return phase->type(in(1)); } +OpaqueTemplateAssertionPredicateNode::OpaqueTemplateAssertionPredicateNode(BoolNode* bol): Node(nullptr, bol), + _predicate_state(PredicateState::Useful) { + init_class_id(Class_OpaqueTemplateAssertionPredicate); +} + Node* OpaqueTemplateAssertionPredicateNode::Identity(PhaseGVN* phase) { if (!phase->C->post_loop_opts_phase()) { // Record Template Assertion Predicates for post loop opts IGVN. We can remove them when there is no more loop @@ -123,7 +128,9 @@ Node* OpaqueTemplateAssertionPredicateNode::Identity(PhaseGVN* phase) { } const Type* OpaqueTemplateAssertionPredicateNode::Value(PhaseGVN* phase) const { - if (_useless || phase->C->post_loop_opts_phase()) { + assert(_predicate_state != PredicateState::MaybeUseful, "should only be MaybeUseful when eliminating useless " + "predicates during loop opts"); + if (is_useless() || phase->C->post_loop_opts_phase()) { // Template Assertion Predicates only serve as templates to create Initialized Assertion Predicates when splitting // a loop during loop opts. They are not used anymore once loop opts are over and can then be removed. They feed // into the bool input of an If node and can thus be replaced by the success path to let the Template Assertion @@ -135,13 +142,13 @@ const Type* OpaqueTemplateAssertionPredicateNode::Value(PhaseGVN* phase) const { } void OpaqueTemplateAssertionPredicateNode::mark_useless(PhaseIterGVN& igvn) { - _useless = true; + _predicate_state = PredicateState::Useless; igvn._worklist.push(this); } #ifndef PRODUCT void OpaqueTemplateAssertionPredicateNode::dump_spec(outputStream* st) const { - if (_useless) { + if (is_useless()) { st->print("#useless "); } } diff --git a/src/hotspot/share/opto/opaquenode.hpp b/src/hotspot/share/opto/opaquenode.hpp index a8a0781f0cc71..b15bebda63606 100644 --- a/src/hotspot/share/opto/opaquenode.hpp +++ b/src/hotspot/share/opto/opaquenode.hpp @@ -26,8 +26,10 @@ #define SHARE_OPTO_OPAQUENODE_HPP #include "opto/node.hpp" -#include "opto/opcodes.hpp" -#include "subnode.hpp" +#include "opto/predicates_enums.hpp" +#include "opto/subnode.hpp" + +enum class PredicateState; //------------------------------Opaque1Node------------------------------------ // A node to prevent unwanted optimizations. Allows constant folding. @@ -149,10 +151,11 @@ class OpaqueNotNullNode : public Node { // after loop opts and thus is never converted to actual code. In the post loop opts IGVN phase, the // OpaqueTemplateAssertionPredicateNode is replaced by true in order to fold the Template Assertion Predicate away. class OpaqueTemplateAssertionPredicateNode : public Node { + // When splitting a loop or when the associated loop dies, the Template Assertion Predicate with this // OpaqueTemplateAssertionPredicateNode also needs to be removed. We set this flag and then clean this node up in the // next IGVN phase by checking this flag in Value(). - bool _useless; + PredicateState _predicate_state; // OpaqueTemplateAssertionPredicateNodes are unique to a Template Assertion Predicate expression and should never // common up. We still make sure of that by returning NO_HASH here. @@ -161,10 +164,7 @@ class OpaqueTemplateAssertionPredicateNode : public Node { } public: - OpaqueTemplateAssertionPredicateNode(BoolNode* bol) : Node(nullptr, bol), - _useless(false) { - init_class_id(Class_OpaqueTemplateAssertionPredicate); - } + OpaqueTemplateAssertionPredicateNode(BoolNode* bol); virtual int Opcode() const; virtual uint size_of() const { return sizeof(*this); } @@ -172,11 +172,25 @@ class OpaqueTemplateAssertionPredicateNode : public Node { virtual const Type* Value(PhaseGVN* phase) const; virtual const Type* bottom_type() const { return TypeInt::BOOL; } + bool is_useless() const { - return _useless; + return _predicate_state == PredicateState::Useless; } void mark_useless(PhaseIterGVN& igvn); + + void mark_maybe_useful() { + _predicate_state = PredicateState::MaybeUseful; + } + + bool is_useful() const { + return _predicate_state == PredicateState::Useful; + } + + void mark_useful() { + _predicate_state = PredicateState::Useful; + } + NOT_PRODUCT(void dump_spec(outputStream* st) const); }; diff --git a/src/hotspot/share/opto/predicates.cpp b/src/hotspot/share/opto/predicates.cpp index 69559edefb069..03d369f7af07e 100644 --- a/src/hotspot/share/opto/predicates.cpp +++ b/src/hotspot/share/opto/predicates.cpp @@ -51,7 +51,7 @@ bool AssertionPredicate::is_predicate(const Node* maybe_success_proj) { if (!may_be_assertion_predicate_if(maybe_success_proj)) { return false; } - return has_assertion_predicate_opaque(maybe_success_proj) && has_halt(maybe_success_proj); + return has_assertion_predicate_opaque(maybe_success_proj) && has_halt(maybe_success_proj->as_IfTrue()); } // Check if the If node of `predicate_proj` has an OpaqueTemplateAssertionPredicate (Template Assertion Predicate) or @@ -63,8 +63,8 @@ bool AssertionPredicate::has_assertion_predicate_opaque(const Node* predicate_pr } // Check if the other projection (UCT projection) of `success_proj` has a Halt node as output. -bool AssertionPredicate::has_halt(const Node* success_proj) { - ProjNode* other_proj = success_proj->as_IfProj()->other_if_proj(); +bool AssertionPredicate::has_halt(const IfTrueNode* success_proj) { + ProjNode* other_proj = success_proj->other_if_proj(); return other_proj->outcnt() == 1 && other_proj->unique_out()->Opcode() == Op_Halt; } @@ -90,6 +90,11 @@ ParsePredicate ParsePredicate::clone_to_unswitched_loop(Node* new_control, const return ParsePredicate(success_proj, _parse_predicate_node->deopt_reason()); } +// Kills this Parse Predicate by marking it useless. Will be folded away in the next IGVN round. +void ParsePredicate::kill(PhaseIterGVN& igvn) const { + _parse_predicate_node->mark_useless(igvn); +} + #ifndef PRODUCT void ParsePredicate::trace_cloned_parse_predicate(const bool is_false_path_loop, const ParsePredicateSuccessProj* success_proj) { @@ -161,13 +166,16 @@ void TemplateAssertionPredicate::rewire_loop_data_dependencies(IfTrueNode* targe } } -// Template Assertion Predicates always have the dedicated OpaqueTemplateAssertionPredicate to identify them. -bool TemplateAssertionPredicate::is_predicate(const Node* node) { - if (!may_be_assertion_predicate_if(node)) { +// A Template Assertion Predicate always has a dedicated OpaqueTemplateAssertionPredicate to identify it. +bool TemplateAssertionPredicate::is_predicate(const Node* maybe_success_proj) { + if (!may_be_assertion_predicate_if(maybe_success_proj)) { return false; } - IfNode* if_node = node->in(0)->as_If(); - return if_node->in(1)->is_OpaqueTemplateAssertionPredicate(); + IfNode* if_node = maybe_success_proj->in(0)->as_If(); + bool is_template_assertion_predicate = if_node->in(1)->is_OpaqueTemplateAssertionPredicate(); + assert(!is_template_assertion_predicate || AssertionPredicate::has_halt(maybe_success_proj->as_IfTrue()), + "Template Assertion Predicate must have a Halt Node on the failing path"); + return is_template_assertion_predicate; } // Clone this Template Assertion Predicate without modifying any OpaqueLoop*Node inputs. @@ -293,16 +301,43 @@ void InitializedAssertionPredicate::verify() const { } #endif // ASSERT -// Initialized Assertion Predicates always have the dedicated OpaqueInitiailizedAssertionPredicate node to identify -// them. -bool InitializedAssertionPredicate::is_predicate(const Node* node) { - if (!may_be_assertion_predicate_if(node)) { +// An Initialized Assertion Predicate always has a dedicated OpaqueInitializedAssertionPredicate node to identify it. +bool InitializedAssertionPredicate::is_predicate(const Node* maybe_success_proj) { + if (!may_be_assertion_predicate_if(maybe_success_proj)) { return false; } - IfNode* if_node = node->in(0)->as_If(); - return if_node->in(1)->is_OpaqueInitializedAssertionPredicate(); + IfNode* if_node = maybe_success_proj->in(0)->as_If(); + bool is_initialized_assertion_predicate = if_node->in(1)->is_OpaqueInitializedAssertionPredicate(); + assert(!is_initialized_assertion_predicate || has_halt(maybe_success_proj->as_IfTrue()), + "Initialized Assertion Predicate must have a Halt Node on the failing path"); + return is_initialized_assertion_predicate; } +#ifdef ASSERT +bool InitializedAssertionPredicate::has_halt(const IfTrueNode* success_proj) { + ProjNode* other_proj = success_proj->other_if_proj(); + if (other_proj->outcnt() != 1) { + return false; + } + + Node* out = other_proj->unique_out(); + // Either the Halt node is directly the unique output. + if (out->is_Halt()) { + return true; + } + // Or we have a Region that merges serval paths to a single Halt node. Even though OpaqueInitializedAssertionPredicate + // nodes do not common up (i.e. NO_HASH), we could have Initialized Assertion Predicates from already folded loops + // being now part of the innermost loop. When then further splitting this loop, we could be cloning the If node + // of the Initialized Assertion Predicate (part of the loop body) while the OpaqueInitializedAssertionPredicate is not + // cloned because it's outside the loop body. We end up sharing the OpaqueInitializedAssertionPredicate between the + // original and the cloned If. This should be fine. + if (out->is_Region() && out->outcnt() == 2) { + return out->find_out_with(Op_Halt); + } + return false; +} +#endif // ASSERT + // Kills this Initialized Assertion Predicate by marking the associated OpaqueInitializedAssertionPredicate node useless. // It will then be folded away in the next IGVN round. void InitializedAssertionPredicate::kill(PhaseIterGVN& igvn) const { @@ -619,7 +654,7 @@ class AssertionPredicateExpressionCreator : public StackObj { private: OpaqueTemplateAssertionPredicateNode* create_opaque_node(Node* new_control, BoolNode* bool_for_expression) const { OpaqueTemplateAssertionPredicateNode* new_expression = new OpaqueTemplateAssertionPredicateNode(bool_for_expression); - _phase->C->add_template_assertion_predicate_opaq(new_expression); + _phase->C->add_template_assertion_predicate_opaque(new_expression); _phase->register_new_node(new_expression, new_control); return new_expression; } @@ -1135,3 +1170,81 @@ void UpdateStrideForAssertionPredicates::connect_initialized_assertion_predicate _phase->replace_control(new_control_out, initialized_assertion_predicate_success_proj); } } + +// Do the following to find and eliminate useless Parse and Template Assertion Predicates: +// 1. Mark all Parse and Template Assertion Predicates "maybe useful". +// 2. Walk through the loop tree and iterate over all Predicates above each loop head. All found Parse and Template +// Assertion Predicates are marked "useful". +// 3. Those Parse and Template Assertion Predicates that are still marked "maybe useful" are now marked "useless" and +// removed in the next round of IGVN. +// +// Note that we only mark Predicates useless and not actually replace them now. We leave this work for IGVN which is +// better suited for this kind of graph surgery. We also not want to replace conditions with a constant to avoid +// interference with Predicate matching code when iterating through them. +void EliminateUselessPredicates::eliminate() const { + mark_all_predicates_maybe_useful(); + if (C->has_loops()) { + mark_loop_associated_predicates_useful(); + } + mark_maybe_useful_predicates_useless(); +} + +void EliminateUselessPredicates::mark_all_predicates_maybe_useful() const { + mark_predicates_on_list_maybe_useful(_parse_predicates); + mark_predicates_on_list_maybe_useful(_template_assertion_predicate_opaques); +} + +template +void EliminateUselessPredicates::mark_predicates_on_list_maybe_useful(const PredicateList& predicate_list) { + for (int i = 0; i < predicate_list.length(); i++) { + predicate_list.at(i)->mark_maybe_useful(); + } +} + +void EliminateUselessPredicates::mark_loop_associated_predicates_useful() const { + for (LoopTreeIterator iterator(_ltree_root); !iterator.done(); iterator.next()) { + IdealLoopTree* loop = iterator.current(); + if (loop->can_apply_loop_predication()) { + mark_useful_predicates_for_loop(loop); + } + } +} + +// This visitor marks all visited Parse and Template Assertion Predicates useful. +class PredicateUsefulMarkerVisitor : public PredicateVisitor { + public: + using PredicateVisitor::visit; + + void visit(const ParsePredicate& parse_predicate) override { + parse_predicate.head()->mark_useful(); + } + + // We actually mark the associated OpaqueTemplateAssertionPredicate node useful. + void visit(const TemplateAssertionPredicate& template_assertion_predicate) override { + template_assertion_predicate.opaque_node()->mark_useful(); + } +}; + +void EliminateUselessPredicates::mark_useful_predicates_for_loop(IdealLoopTree* loop) { + Node* loop_entry = loop->_head->as_Loop()->skip_strip_mined()->in(LoopNode::EntryControl); + const PredicateIterator predicate_iterator(loop_entry); + PredicateUsefulMarkerVisitor predicate_useful_marker_visitor; + predicate_iterator.for_each(predicate_useful_marker_visitor); +} + +// All Predicates still being marked MaybeUseful could not be found and thus are now marked useless. +void EliminateUselessPredicates::mark_maybe_useful_predicates_useless() const { + mark_maybe_useful_predicates_on_list_useless(_parse_predicates); + mark_maybe_useful_predicates_on_list_useless(_template_assertion_predicate_opaques); +} + +template +void EliminateUselessPredicates::mark_maybe_useful_predicates_on_list_useless( + const PredicateList& predicate_list) const { + for (int i = 0; i < predicate_list.length(); i++) { + auto predicate_node = predicate_list.at(i); + if (!predicate_node->is_useful()) { + predicate_node->mark_useless(_igvn); + } + } +} diff --git a/src/hotspot/share/opto/predicates.hpp b/src/hotspot/share/opto/predicates.hpp index aff72d38831fb..3d1cbe3a89cc0 100644 --- a/src/hotspot/share/opto/predicates.hpp +++ b/src/hotspot/share/opto/predicates.hpp @@ -27,6 +27,7 @@ #include "opto/cfgnode.hpp" #include "opto/opaquenode.hpp" +#include "opto/predicates_enums.hpp" class IdealLoopTree; class InitializedAssertionPredicate; @@ -200,16 +201,6 @@ class TemplateAssertionPredicate; * Main Loop Head */ -// Assertion Predicates are either emitted to check the initial value of a range check in the first iteration or the last -// value of a range check in the last iteration of a loop. -enum class AssertionPredicateType { - None, // Not an Assertion Predicate - InitValue, - LastValue, - // Used for the Initialized Assertion Predicate emitted during Range Check Elimination for the final IV value. - FinalIv -}; - // Interface to represent a C2 predicate. A predicate is always represented by two CFG nodes: // - An If node (head) // - An IfProj node representing the success projection of the If node (tail). @@ -273,9 +264,10 @@ class AssertionPredicates : public StackObj { // - An Initialized Assertion Predicate. class AssertionPredicate : public StackObj { static bool has_assertion_predicate_opaque(const Node* predicate_proj); - static bool has_halt(const Node* success_proj); + public: static bool is_predicate(const Node* maybe_success_proj); + static bool has_halt(const IfTrueNode* success_proj); }; // Utility class representing a Regular Predicate which is either a Runtime Predicate or an Assertion Predicate. @@ -311,10 +303,11 @@ class ParsePredicate : public Predicate { return _entry; } - // This Parse Predicate is valid if the node passed to the constructor is a projection of a ParsePredicateNode and the - // deopt_reason of the uncommon trap of the ParsePredicateNode matches the passed deopt_reason to the constructor. + // This Parse Predicate is valid if the node passed to the constructor is a projection of a ParsePredicateNode, the + // deopt_reason of the uncommon trap of the ParsePredicateNode matches the passed deopt_reason to the constructor and + // the ParsePredicateNode is not marked useless. bool is_valid() const { - return _parse_predicate_node != nullptr; + return _parse_predicate_node != nullptr && !_parse_predicate_node->is_useless(); } ParsePredicateNode* head() const override { @@ -330,11 +323,7 @@ class ParsePredicate : public Predicate { ParsePredicate clone_to_unswitched_loop(Node* new_control, bool is_false_path_loop, PhaseIdealLoop* phase) const; - // Kills this Parse Predicate by marking it useless. Will be folded away in the next IGVN round. - void kill(const PhaseIterGVN& igvn) const { - _parse_predicate_node->mark_useless(); - igvn._worklist.push(_parse_predicate_node); - } + void kill(PhaseIterGVN& igvn) const; }; // Class to represent a Runtime Predicate which always has an associated UCT on the failing path. @@ -433,6 +422,8 @@ class InitializedAssertionPredicate : public Predicate { IfTrueNode* const _success_proj; IfNode* const _if_node; + DEBUG_ONLY(static bool has_halt(const IfTrueNode* success_proj);) + public: explicit InitializedAssertionPredicate(IfTrueNode* success_proj) : _success_proj(success_proj), @@ -1237,4 +1228,36 @@ class UpdateStrideForAssertionPredicates : public PredicateVisitor { void visit(const TemplateAssertionPredicate& template_assertion_predicate) override; void visit(const InitializedAssertionPredicate& initialized_assertion_predicate) override; }; + +// Eliminate all useless Parse and Template Assertion Predicates. They become useless when they can no longer be found +// from a loop head. We mark these useless to clean them up later during IGVN. A Predicate that is marked useless will +// no longer be visited by a PredicateVisitor. +class EliminateUselessPredicates : public StackObj { + Compile* const C; + const GrowableArray& _parse_predicates; + const GrowableArray& _template_assertion_predicate_opaques; + PhaseIterGVN& _igvn; + IdealLoopTree* const _ltree_root; + + void mark_all_predicates_maybe_useful() const; + template + static void mark_predicates_on_list_maybe_useful(const PredicateList& predicate_list); + + void mark_loop_associated_predicates_useful() const; + static void mark_useful_predicates_for_loop(IdealLoopTree* loop); + + void mark_maybe_useful_predicates_useless() const; + template + void mark_maybe_useful_predicates_on_list_useless(const PredicateList& predicate_list) const; + + public: + EliminateUselessPredicates(PhaseIterGVN& igvn, IdealLoopTree* ltree_root) + : C(igvn.C), _parse_predicates(igvn.C->parse_predicates()), + _template_assertion_predicate_opaques(igvn.C->template_assertion_predicate_opaques()), + _igvn(igvn), + _ltree_root(ltree_root) {} + + void eliminate() const; +}; + #endif // SHARE_OPTO_PREDICATES_HPP diff --git a/src/hotspot/share/opto/predicates_enums.hpp b/src/hotspot/share/opto/predicates_enums.hpp new file mode 100644 index 0000000000000..b051b2c416b05 --- /dev/null +++ b/src/hotspot/share/opto/predicates_enums.hpp @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_OPTO_PREDICATES_ENUMS_HPP +#define SHARE_OPTO_PREDICATES_ENUMS_HPP + +// The success projection of a Parse Predicate is always an IfTrueNode and the uncommon projection an IfFalseNode +typedef IfTrueNode ParsePredicateSuccessProj; +typedef IfFalseNode ParsePredicateUncommonProj; + +// Assertion Predicates are either emitted to check the initial value of a range check in the first iteration or the last +// value of a range check in the last iteration of a loop. +enum class AssertionPredicateType { + None, // Not an Assertion Predicate + InitValue, + LastValue, + // Used for the Initialized Assertion Predicate emitted during Range Check Elimination for the final IV value. + FinalIv +}; + +enum class PredicateState { + // The Predicate is useless and will be cleaned up in the next round of IGVN. A useless Predicate is not visited + // anymore by PredicateVisitors. If a Predicate loses its connection to a loop head, it will be marked useless by + // EliminateUselessPredicates and cleaned up by the Value() methods of the associated Predicate IR nodes. + Useless, + // This state is used by EliminateUselessPredicates to temporarily mark a Predicate as neither useless nor useful. + // Outside EliminateUselessPredicates, a Predicate should never be MaybeUseful. + MaybeUseful, + // Default state: The Predicate is useful and will be visited by PredicateVisitors. + Useful +}; + +#endif // SHARE_OPTO_PREDICATES_ENUMS_HPP