diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp index 35ec9ed5c8cab..40fe0c0049010 100644 --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp @@ -1394,7 +1394,7 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) { } if (addr->Opcode() == Op_AddP) { Node* orig_base = addr->in(AddPNode::Base); - Node* base = new CheckCastPPNode(ctrl, orig_base, orig_base->bottom_type(), ConstraintCastNode::StrongDependency); + Node* base = new CheckCastPPNode(ctrl, orig_base, orig_base->bottom_type(), ConstraintCastNode::DependencyType::NonFloatingNarrowing); phase->register_new_node(base, ctrl); if (addr->in(AddPNode::Base) == addr->in((AddPNode::Address))) { // Field access diff --git a/src/hotspot/share/opto/castnode.cpp b/src/hotspot/share/opto/castnode.cpp index 6d899c1f95025..9c764f22e386d 100644 --- a/src/hotspot/share/opto/castnode.cpp +++ b/src/hotspot/share/opto/castnode.cpp @@ -22,7 +22,6 @@ * */ -#include "castnode.hpp" #include "opto/addnode.hpp" #include "opto/callnode.hpp" #include "opto/castnode.hpp" @@ -35,12 +34,22 @@ #include "opto/type.hpp" #include "utilities/checkedCast.hpp" +const ConstraintCastNode::DependencyType ConstraintCastNode::DependencyType::FloatingNarrowing(true, true, "floating narrowing dependency"); // not pinned, narrows type +const ConstraintCastNode::DependencyType ConstraintCastNode::DependencyType::FloatingNonNarrowing(true, false, "floating non-narrowing dependency"); // not pinned, doesn't narrow type +const ConstraintCastNode::DependencyType ConstraintCastNode::DependencyType::NonFloatingNarrowing(false, true, "non-floating narrowing dependency"); // pinned, narrows type +const ConstraintCastNode::DependencyType ConstraintCastNode::DependencyType::NonFloatingNonNarrowing(false, false, "non-floating non-narrowing dependency"); // pinned, doesn't narrow type + //============================================================================= // If input is already higher or equal to cast type, then this is an identity. Node* ConstraintCastNode::Identity(PhaseGVN* phase) { - if (_dependency == UnconditionalDependency) { + if (!_dependency.narrows_type()) { + // If this cast doesn't carry a type dependency (i.e. not used for type narrowing), we cannot optimize it. return this; } + + // This cast node carries a type dependency. We can remove it if: + // - Its input has a narrower type + // - There's a dominating cast with same input but narrower type Node* dom = dominating_cast(phase, phase); if (dom != nullptr) { return dom; @@ -109,7 +118,7 @@ Node* ConstraintCastNode::Ideal(PhaseGVN* phase, bool can_reshape) { } uint ConstraintCastNode::hash() const { - return TypeNode::hash() + (int)_dependency + (_extra_types != nullptr ? _extra_types->hash() : 0); + return TypeNode::hash() + _dependency.hash() + (_extra_types != nullptr ? _extra_types->hash() : 0); } bool ConstraintCastNode::cmp(const Node &n) const { @@ -117,7 +126,7 @@ bool ConstraintCastNode::cmp(const Node &n) const { return false; } ConstraintCastNode& cast = (ConstraintCastNode&) n; - if (cast._dependency != _dependency) { + if (!cast._dependency.cmp(_dependency)) { return false; } if (_extra_types == nullptr || cast._extra_types == nullptr) { @@ -130,7 +139,7 @@ uint ConstraintCastNode::size_of() const { return sizeof(*this); } -Node* ConstraintCastNode::make_cast_for_basic_type(Node* c, Node* n, const Type* t, DependencyType dependency, BasicType bt) { +Node* ConstraintCastNode::make_cast_for_basic_type(Node* c, Node* n, const Type* t, const DependencyType& dependency, BasicType bt) { switch(bt) { case T_INT: return new CastIINode(c, n, t, dependency); @@ -143,9 +152,9 @@ Node* ConstraintCastNode::make_cast_for_basic_type(Node* c, Node* n, const Type* } TypeNode* ConstraintCastNode::dominating_cast(PhaseGVN* gvn, PhaseTransform* pt) const { - if (_dependency == UnconditionalDependency) { - return nullptr; - } + // See discussion at definition of ConstraintCastNode::DependencyType: replacing this cast with a dominating one is + // not safe if _dependency.narrows_type() is not true. + assert(_dependency.narrows_type(), "cast can't be replaced by dominating one"); Node* val = in(1); Node* ctl = in(0); int opc = Opcode(); @@ -205,30 +214,21 @@ void ConstraintCastNode::dump_spec(outputStream *st) const { st->print(" extra types: "); _extra_types->dump_on(st); } - if (_dependency != RegularDependency) { - st->print(" %s dependency", _dependency == StrongDependency ? "strong" : "unconditional"); - } + st->print(" "); + _dependency.dump_on(st); } #endif -const Type* CastIINode::Value(PhaseGVN* phase) const { - const Type *res = ConstraintCastNode::Value(phase); - if (res == Type::TOP) { - return Type::TOP; - } - assert(res->isa_int(), "res must be int"); - - // Similar to ConvI2LNode::Value() for the same reasons - // see if we can remove type assertion after loop opts - res = widen_type(phase, res, T_INT); +CastIINode* CastIINode::make_with(Node* parent, const TypeInteger* type, const DependencyType& dependency) const { + return new CastIINode(in(0), parent, type, dependency, _range_check_dependency, _extra_types); +} - return res; +CastLLNode* CastLLNode::make_with(Node* parent, const TypeInteger* type, const DependencyType& dependency) const { + return new CastLLNode(in(0), parent, type, dependency, _extra_types); } -Node* ConstraintCastNode::find_or_make_integer_cast(PhaseIterGVN* igvn, Node* parent, const TypeInteger* type) const { - Node* n = clone(); - n->set_req(1, parent); - n->as_ConstraintCast()->set_type(type); +Node* ConstraintCastNode::find_or_make_integer_cast(PhaseIterGVN* igvn, Node* parent, const TypeInteger* type, const DependencyType& dependency) const { + Node* n = make_with(parent, type, dependency); Node* existing = igvn->hash_find_insert(n); if (existing != nullptr) { n->destruct(igvn); @@ -242,14 +242,13 @@ Node *CastIINode::Ideal(PhaseGVN *phase, bool can_reshape) { if (progress != nullptr) { return progress; } - if (can_reshape && !phase->C->post_loop_opts_phase()) { - // makes sure we run ::Value to potentially remove type assertion after loop opts + if (!phase->C->post_loop_opts_phase()) { + // makes sure we run widen_type() to potentially common type assertions after loop opts phase->C->record_for_post_loop_opts_igvn(this); } if (!_range_check_dependency || phase->C->post_loop_opts_phase()) { return optimize_integer_cast(phase, T_INT); } - phase->C->record_for_post_loop_opts_igvn(this); return nullptr; } @@ -279,9 +278,9 @@ void CastIINode::dump_spec(outputStream* st) const { #endif CastIINode* CastIINode::pin_array_access_node() const { - assert(_dependency == RegularDependency, "already pinned"); + assert(_dependency.is_floating(), "already pinned"); if (has_range_check()) { - return new CastIINode(in(0), in(1), bottom_type(), StrongDependency, has_range_check()); + return new CastIINode(in(0), in(1), bottom_type(), _dependency.with_pinned_dependency(), has_range_check()); } return nullptr; } @@ -315,16 +314,6 @@ void CastIINode::remove_range_check_cast(Compile* C) { } -const Type* CastLLNode::Value(PhaseGVN* phase) const { - const Type* res = ConstraintCastNode::Value(phase); - if (res == Type::TOP) { - return Type::TOP; - } - assert(res->isa_long(), "res must be long"); - - return widen_type(phase, res, T_LONG); -} - bool CastLLNode::is_inner_loop_backedge(ProjNode* proj) { if (proj != nullptr) { Node* ctrl_use = proj->unique_ctrl_out_or_null(); @@ -392,7 +381,7 @@ Node* CastLLNode::Ideal(PhaseGVN* phase, bool can_reshape) { return progress; } if (!phase->C->post_loop_opts_phase()) { - // makes sure we run ::Value to potentially remove type assertion after loop opts + // makes sure we run widen_type() to potentially common type assertions after loop opts phase->C->record_for_post_loop_opts_igvn(this); } // transform (CastLL (ConvI2L ..)) into (ConvI2L (CastII ..)) if the type of the CastLL is narrower than the type of @@ -543,7 +532,7 @@ Node* CastP2XNode::Identity(PhaseGVN* phase) { return this; } -Node* ConstraintCastNode::make_cast_for_type(Node* c, Node* in, const Type* type, DependencyType dependency, +Node* ConstraintCastNode::make_cast_for_type(Node* c, Node* in, const Type* type, const DependencyType& dependency, const TypeTuple* types) { if (type->isa_int()) { return new CastIINode(c, in, type, dependency, false, types); @@ -564,7 +553,7 @@ Node* ConstraintCastNode::make_cast_for_type(Node* c, Node* in, const Type* type return nullptr; } -Node* ConstraintCastNode::optimize_integer_cast(PhaseGVN* phase, BasicType bt) { +Node* ConstraintCastNode::optimize_integer_cast_of_add(PhaseGVN* phase, BasicType bt) { PhaseIterGVN *igvn = phase->is_IterGVN(); const TypeInteger* this_type = this->type()->isa_integer(bt); if (this_type == nullptr) { @@ -586,8 +575,42 @@ Node* ConstraintCastNode::optimize_integer_cast(PhaseGVN* phase, BasicType bt) { Node* x = z->in(1); Node* y = z->in(2); - Node* cx = find_or_make_integer_cast(igvn, x, rx); - Node* cy = find_or_make_integer_cast(igvn, y, ry); + const TypeInteger* tx = phase->type(x)->is_integer(bt); + const TypeInteger* ty = phase->type(y)->is_integer(bt); + + // (Cast (Add x y) tz) is transformed into (Add (Cast x rx) (Cast y ry)) + // + // tz = [tzlo, tzhi] + // rx = [rxlo, rxhi] + // ry = [rylo, ryhi] + // with type of x, tx = [txlo, txhi] + // with type of y, ty = [tylo, tyhi] + // + // From Compile::push_thru_add(): + // rxlo = max(tzlo - tyhi, txlo) + // rxhi = min(tzhi - tylo, txhi) + // rylo = max(tzlo - txhi, tylo) + // ryhi = min(tzhi - txlo, tyhi) + // + // If x is a constant, then txlo = txhi + // rxlo = txlo, rxhi = txhi + // The bounds of the type of the Add after transformation then is: + // rxlo + rylo >= txlo + tzlo - txhi >= tzlo + // rxhi + ryhi <= txhi + tzhi - txlo <= tzhi + // The resulting type is not wider than the type of the Cast + // before transformation + // + // If neither x nor y are constant then the type of the resulting + // Add can be wider than the type of the type of the Cast before + // transformation. + // For instance, tx = [0, 10], ty = [0, 10], tz = [0, 10] + // then rx = [0, 10], ry = [0, 10] + // and rx + ry = [0, 20] which is wider than tz + // + // Same reasoning applies to (Cast (Sub x y) tz) + const DependencyType& dependency = (!tx->is_con() && !ty->is_con()) ? _dependency.with_non_narrowing() : _dependency; + Node* cx = find_or_make_integer_cast(igvn, x, rx, dependency); + Node* cy = find_or_make_integer_cast(igvn, y, ry, dependency); if (op == Op_Add(bt)) { return AddNode::make(cx, cy, bt); } else { @@ -599,11 +622,26 @@ Node* ConstraintCastNode::optimize_integer_cast(PhaseGVN* phase, BasicType bt) { return nullptr; } -const Type* ConstraintCastNode::widen_type(const PhaseGVN* phase, const Type* res, BasicType bt) const { - if (!phase->C->post_loop_opts_phase()) { +Node* ConstraintCastNode::optimize_integer_cast(PhaseGVN* phase, BasicType bt) { + Node* res = optimize_integer_cast_of_add(phase, bt); + if (res != nullptr) { return res; } + const Type* t = Value(phase); + if (t != Type::TOP && phase->C->post_loop_opts_phase()) { + const Type* bottom_t = bottom_type(); + const TypeInteger* wide_t = widen_type(phase, bottom_t, bt); + if (wide_t != bottom_t) { + // Widening the type of the Cast (to allow some commoning) causes the Cast to change how it can be optimized (if + // type of its input is narrower than the Cast's type, we can't remove it to not loose the control dependency). + return make_with(in(1), wide_t, _dependency.with_non_narrowing()); + } + } + return nullptr; +} +const TypeInteger* ConstraintCastNode::widen_type(const PhaseGVN* phase, const Type* res, BasicType bt) const { + const TypeInteger* this_type = res->is_integer(bt); // At VerifyConstraintCasts == 1, we verify the ConstraintCastNodes that are present during code // emission. This allows us detecting possible mis-scheduling due to these nodes being pinned at // the wrong control nodes. @@ -612,10 +650,9 @@ const Type* ConstraintCastNode::widen_type(const PhaseGVN* phase, const Type* re // mis-transformations that may happen due to these nodes being pinned at the wrong control // nodes. if (VerifyConstraintCasts > 1) { - return res; + return this_type; } - const TypeInteger* this_type = res->is_integer(bt); const TypeInteger* in_type = phase->type(in(1))->isa_integer(bt); if (in_type != nullptr && (in_type->lo_as_long() != this_type->lo_as_long() || @@ -636,5 +673,5 @@ const Type* ConstraintCastNode::widen_type(const PhaseGVN* phase, const Type* re MIN2(in_type->hi_as_long(), hi1), MAX2((int)in_type->_widen, w1), bt); } - return res; + return this_type; } diff --git a/src/hotspot/share/opto/castnode.hpp b/src/hotspot/share/opto/castnode.hpp index 3c6ade64aa85c..2ff13e447801f 100644 --- a/src/hotspot/share/opto/castnode.hpp +++ b/src/hotspot/share/opto/castnode.hpp @@ -33,21 +33,119 @@ // cast to a different range class ConstraintCastNode: public TypeNode { public: - enum DependencyType { - RegularDependency, // if cast doesn't improve input type, cast can be removed - StrongDependency, // leave cast in even if _type doesn't improve input type, can be replaced by stricter dominating cast if one exist - UnconditionalDependency // leave cast in unconditionally + // Cast nodes are subject to a few optimizations: + // + // 1- if the type carried by the Cast doesn't narrow the type of its input, the cast can be replaced by its input. + // Similarly, if a dominating Cast with the same input and a narrower type constraint is found, it can replace the + // current cast. + // + // 2- if the condition that the Cast is control dependent is hoisted, the Cast is hoisted as well + // + // 1- and 2- are not always applied depending on what constraint are applied to the Cast: there are cases where 1- + // and 2- apply, where neither 1- nor 2- apply and where one or the other apply. This class abstract away these + // details. + // + // If _narrows_type is true, the cast carries a type dependency: "after" the control the cast is dependent on, its data + // input is known to have a narrower type (stored in the cast node itself). Optimizations 1- above only apply to cast + // nodes for which _narrows_type is true. + // if _floating is true, the cast only depends on a single control: its control input. Otherwise, it is pinned at its + // current location. Optimizations 2- only apply to cast nodes for which _floating is true. + // _floating here is similar to Node::depends_only_on_test(). + // The 4 combinations of _narrows_types/_floating true/false have some use. See below, at the end of this class + // definition, for examples. + class DependencyType { + private: + const bool _floating; // Does this Cast depends on its control input or is it pinned? + const bool _narrows_type; // Does this Cast narrows the type i.e. if input type is narrower can it be removed? + const char* _desc; + DependencyType(bool depends_on_test, bool narrows_type, const char* desc) + : _floating(depends_on_test), + _narrows_type(narrows_type), + _desc(desc) { + } + NONCOPYABLE(DependencyType); + + public: + + bool is_floating() const { + return _floating; + } + + bool narrows_type() const { + return _narrows_type; + } + + void dump_on(outputStream *st) const { + st->print("%s", _desc); + } + + uint hash() const { + return (_floating ? 1 : 0) + (_narrows_type ? 2 : 0); + } + + bool cmp(const DependencyType& other) const { + return _floating == other._floating && _narrows_type == other._narrows_type; + } + + const DependencyType& with_non_narrowing() const { + if (_floating) { + return FloatingNonNarrowing; + } + return NonFloatingNonNarrowing; + } + + const DependencyType& with_pinned_dependency() const { + if (_narrows_type) { + return NonFloatingNarrowing; + } + return NonFloatingNonNarrowing; + } + + // All the possible combinations of floating/narrowing with example use cases: + + // Use case example: Range Check CastII + // Floating: The Cast is only dependent on the single range check. If the range check was ever to be hoisted it + // would be safe to let the Cast float to where the range check is hoisted up to. + // Narrowing: The Cast narrows the type to a positive index. If the input to the Cast is narrower, we can safely + // remove the cast because the array access will be safe. + static const DependencyType FloatingNarrowing; + // Use case example: Widening Cast nodes' types after loop opts: We want to common Casts with slightly different types. + // Floating: These Casts only depend on the single control. + // NonNarrowing: Even when the input type is narrower, we are not removing the Cast. Otherwise, the dependency + // to the single control is lost, and an array access could float above its range check because we + // just removed the dependency to the range check by removing the Cast. This could lead to an + // out-of-bounds access. + static const DependencyType FloatingNonNarrowing; + // Use case example: An array accesses that is no longer dependent on a single range check (e.g. range check smearing). + // NonFloating: The array access must be pinned below all the checks it depends on. If the check it directly depends + // on with a control input is hoisted, we do not hoist the Cast as well. If we allowed the Cast to float, + // we risk that the array access ends up above another check it depends on (we cannot model two control + // dependencies for a node in the IR). This could lead to an out-of-bounds access. + // Narrowing: If the Cast does not narrow the input type, then it's safe to remove the cast because the array access + // will be safe. + static const DependencyType NonFloatingNarrowing; + // Use case example: Sinking nodes out of a loop + // Non-Floating & Non-Narrowing: We don't want the Cast that forces the node to be out of loop to be removed in any + // case. Otherwise, the sunk node could float back into the loop, undoing the sinking. + // This Cast is only used for pinning without caring about narrowing types. + static const DependencyType NonFloatingNonNarrowing; + }; - protected: - const DependencyType _dependency; +protected: + const DependencyType& _dependency; virtual bool cmp( const Node &n ) const; virtual uint size_of() const; virtual uint hash() const; // Check the type - const Type* widen_type(const PhaseGVN* phase, const Type* res, BasicType bt) const; - Node* find_or_make_integer_cast(PhaseIterGVN* igvn, Node* parent, const TypeInteger* type) const; + const TypeInteger* widen_type(const PhaseGVN* phase, const Type* res, BasicType bt) const; + + virtual ConstraintCastNode* make_with(Node* parent, const TypeInteger* type, const DependencyType& dependency) const { + ShouldNotReachHere(); // Only implemented for CastII and CastLL + return nullptr; + } + + Node* find_or_make_integer_cast(PhaseIterGVN* igvn, Node* parent, const TypeInteger* type, const DependencyType& dependency) const; - private: // PhiNode::Ideal() transforms a Phi that merges a single uncasted value into a single cast pinned at the region. // The types of cast nodes eliminated as a consequence of this transformation are collected and stored here so the // type dependencies carried by the cast are known. The cast can then be eliminated if the type of its input is @@ -55,7 +153,7 @@ class ConstraintCastNode: public TypeNode { const TypeTuple* _extra_types; public: - ConstraintCastNode(Node* ctrl, Node* n, const Type* t, ConstraintCastNode::DependencyType dependency, + ConstraintCastNode(Node* ctrl, Node* n, const Type* t, const DependencyType& dependency, const TypeTuple* extra_types) : TypeNode(t,2), _dependency(dependency), _extra_types(extra_types) { init_class_id(Class_ConstraintCast); @@ -67,18 +165,21 @@ class ConstraintCastNode: public TypeNode { virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); virtual int Opcode() const; virtual uint ideal_reg() const = 0; - virtual bool depends_only_on_test() const { return _dependency == RegularDependency; } - bool carry_dependency() const { return _dependency != RegularDependency; } + bool carry_dependency() const { return !_dependency.cmp(DependencyType::FloatingNarrowing); } + // A cast node depends_only_on_test if and only if it is floating + virtual bool depends_only_on_test() const { return _dependency.is_floating(); } + const DependencyType& dependency() const { return _dependency; } TypeNode* dominating_cast(PhaseGVN* gvn, PhaseTransform* pt) const; - static Node* make_cast_for_basic_type(Node* c, Node* n, const Type* t, DependencyType dependency, BasicType bt); + static Node* make_cast_for_basic_type(Node* c, Node* n, const Type* t, const DependencyType& dependency, BasicType bt); #ifndef PRODUCT virtual void dump_spec(outputStream *st) const; #endif - static Node* make_cast_for_type(Node* c, Node* in, const Type* type, DependencyType dependency, + static Node* make_cast_for_type(Node* c, Node* in, const Type* type, const DependencyType& dependency, const TypeTuple* types); + Node* optimize_integer_cast_of_add(PhaseGVN* phase, BasicType bt); Node* optimize_integer_cast(PhaseGVN* phase, BasicType bt); bool higher_equal_types(PhaseGVN* phase, const Node* other) const; @@ -102,7 +203,7 @@ class CastIINode: public ConstraintCastNode { virtual uint size_of() const; public: - CastIINode(Node* ctrl, Node* n, const Type* t, DependencyType dependency = RegularDependency, bool range_check_dependency = false, const TypeTuple* types = nullptr) + CastIINode(Node* ctrl, Node* n, const Type* t, const DependencyType& dependency = DependencyType::FloatingNarrowing, bool range_check_dependency = false, const TypeTuple* types = nullptr) : ConstraintCastNode(ctrl, n, t, dependency, types), _range_check_dependency(range_check_dependency) { assert(ctrl != nullptr, "control must be set"); init_class_id(Class_CastII); @@ -110,7 +211,7 @@ class CastIINode: public ConstraintCastNode { virtual int Opcode() const; virtual uint ideal_reg() const { return Op_RegI; } virtual Node* Identity(PhaseGVN* phase); - virtual const Type* Value(PhaseGVN* phase) const; + virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); bool has_range_check() const { #ifdef _LP64 @@ -122,6 +223,7 @@ class CastIINode: public ConstraintCastNode { } CastIINode* pin_array_access_node() const; + CastIINode* make_with(Node* parent, const TypeInteger* type, const DependencyType& dependency) const; void remove_range_check_cast(Compile* C); #ifndef PRODUCT @@ -131,14 +233,12 @@ class CastIINode: public ConstraintCastNode { class CastLLNode: public ConstraintCastNode { public: - CastLLNode(Node* ctrl, Node* n, const Type* t, DependencyType dependency = RegularDependency, const TypeTuple* types = nullptr) + CastLLNode(Node* ctrl, Node* n, const Type* t, const DependencyType& dependency = DependencyType::FloatingNarrowing, const TypeTuple* types = nullptr) : ConstraintCastNode(ctrl, n, t, dependency, types) { assert(ctrl != nullptr, "control must be set"); init_class_id(Class_CastLL); } - virtual const Type* Value(PhaseGVN* phase) const; - static bool is_inner_loop_backedge(ProjNode* proj); static bool cmp_used_at_inner_loop_exit_test(CmpNode* cmp); @@ -147,11 +247,12 @@ class CastLLNode: public ConstraintCastNode { virtual Node* Ideal(PhaseGVN* phase, bool can_reshape); virtual int Opcode() const; virtual uint ideal_reg() const { return Op_RegL; } + CastLLNode* make_with(Node* parent, const TypeInteger* type, const DependencyType& dependency) const; }; class CastHHNode: public ConstraintCastNode { public: - CastHHNode(Node* ctrl, Node* n, const Type* t, DependencyType dependency = RegularDependency, const TypeTuple* types = nullptr) + CastHHNode(Node* ctrl, Node* n, const Type* t, const DependencyType& dependency = DependencyType::FloatingNarrowing, const TypeTuple* types = nullptr) : ConstraintCastNode(ctrl, n, t, dependency, types) { assert(ctrl != nullptr, "control must be set"); init_class_id(Class_CastHH); @@ -162,7 +263,7 @@ class CastHHNode: public ConstraintCastNode { class CastFFNode: public ConstraintCastNode { public: - CastFFNode(Node* ctrl, Node* n, const Type* t, DependencyType dependency = RegularDependency, const TypeTuple* types = nullptr) + CastFFNode(Node* ctrl, Node* n, const Type* t, const DependencyType& dependency = DependencyType::FloatingNarrowing, const TypeTuple* types = nullptr) : ConstraintCastNode(ctrl, n, t, dependency, types) { assert(ctrl != nullptr, "control must be set"); init_class_id(Class_CastFF); @@ -173,7 +274,7 @@ class CastFFNode: public ConstraintCastNode { class CastDDNode: public ConstraintCastNode { public: - CastDDNode(Node* ctrl, Node* n, const Type* t, DependencyType dependency = RegularDependency, const TypeTuple* types = nullptr) + CastDDNode(Node* ctrl, Node* n, const Type* t, const DependencyType& dependency = DependencyType::FloatingNarrowing, const TypeTuple* types = nullptr) : ConstraintCastNode(ctrl, n, t, dependency, types) { assert(ctrl != nullptr, "control must be set"); init_class_id(Class_CastDD); @@ -184,7 +285,7 @@ class CastDDNode: public ConstraintCastNode { class CastVVNode: public ConstraintCastNode { public: - CastVVNode(Node* ctrl, Node* n, const Type* t, DependencyType dependency = RegularDependency, const TypeTuple* types = nullptr) + CastVVNode(Node* ctrl, Node* n, const Type* t, const DependencyType& dependency = DependencyType::FloatingNarrowing, const TypeTuple* types = nullptr) : ConstraintCastNode(ctrl, n, t, dependency, types) { assert(ctrl != nullptr, "control must be set"); init_class_id(Class_CastVV); @@ -198,7 +299,7 @@ class CastVVNode: public ConstraintCastNode { // cast pointer to pointer (different type) class CastPPNode: public ConstraintCastNode { public: - CastPPNode (Node* ctrl, Node* n, const Type* t, DependencyType dependency = RegularDependency, const TypeTuple* types = nullptr) + CastPPNode (Node* ctrl, Node* n, const Type* t, const DependencyType& dependency = DependencyType::FloatingNarrowing, const TypeTuple* types = nullptr) : ConstraintCastNode(ctrl, n, t, dependency, types) { init_class_id(Class_CastPP); } @@ -210,7 +311,7 @@ class CastPPNode: public ConstraintCastNode { // for _checkcast, cast pointer to pointer (different type), without JOIN, class CheckCastPPNode: public ConstraintCastNode { public: - CheckCastPPNode(Node* ctrl, Node* n, const Type* t, DependencyType dependency = RegularDependency, const TypeTuple* types = nullptr) + CheckCastPPNode(Node* ctrl, Node* n, const Type* t, const DependencyType& dependency = DependencyType::FloatingNarrowing, const TypeTuple* types = nullptr) : ConstraintCastNode(ctrl, n, t, dependency, types) { assert(ctrl != nullptr, "control must be set"); init_class_id(Class_CheckCastPP); diff --git a/src/hotspot/share/opto/cfgnode.cpp b/src/hotspot/share/opto/cfgnode.cpp index 0293f42d79123..43d96a8cbe595 100644 --- a/src/hotspot/share/opto/cfgnode.cpp +++ b/src/hotspot/share/opto/cfgnode.cpp @@ -2191,7 +2191,7 @@ Node *PhiNode::Ideal(PhaseGVN *phase, bool can_reshape) { if (phi_type->isa_ptr()) { const Type* uin_type = phase->type(uin); if (!phi_type->isa_oopptr() && !uin_type->isa_oopptr()) { - cast = new CastPPNode(r, uin, phi_type, ConstraintCastNode::StrongDependency, extra_types); + cast = new CastPPNode(r, uin, phi_type, ConstraintCastNode::DependencyType::NonFloatingNarrowing, extra_types); } else { // Use a CastPP for a cast to not null and a CheckCastPP for // a cast to a new klass (and both if both null-ness and @@ -2201,7 +2201,7 @@ Node *PhiNode::Ideal(PhaseGVN *phase, bool can_reshape) { // null, uin's type must be casted to not null if (phi_type->join(TypePtr::NOTNULL) == phi_type->remove_speculative() && uin_type->join(TypePtr::NOTNULL) != uin_type->remove_speculative()) { - cast = new CastPPNode(r, uin, TypePtr::NOTNULL, ConstraintCastNode::StrongDependency, extra_types); + cast = new CastPPNode(r, uin, TypePtr::NOTNULL, ConstraintCastNode::DependencyType::NonFloatingNarrowing, extra_types); } // If the type of phi and uin, both casted to not null, @@ -2213,14 +2213,14 @@ Node *PhiNode::Ideal(PhaseGVN *phase, bool can_reshape) { cast = phase->transform(cast); n = cast; } - cast = new CheckCastPPNode(r, n, phi_type, ConstraintCastNode::StrongDependency, extra_types); + cast = new CheckCastPPNode(r, n, phi_type, ConstraintCastNode::DependencyType::NonFloatingNarrowing, extra_types); } if (cast == nullptr) { - cast = new CastPPNode(r, uin, phi_type, ConstraintCastNode::StrongDependency, extra_types); + cast = new CastPPNode(r, uin, phi_type, ConstraintCastNode::DependencyType::NonFloatingNarrowing, extra_types); } } } else { - cast = ConstraintCastNode::make_cast_for_type(r, uin, phi_type, ConstraintCastNode::StrongDependency, extra_types); + cast = ConstraintCastNode::make_cast_for_type(r, uin, phi_type, ConstraintCastNode::DependencyType::NonFloatingNarrowing, extra_types); } assert(cast != nullptr, "cast should be set"); cast = phase->transform(cast); diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index 89b5e36b120f1..9faea10fd9d4d 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -4578,7 +4578,7 @@ Node* Compile::constrained_convI2L(PhaseGVN* phase, Node* value, const TypeInt* // node from floating above the range check during loop optimizations. Otherwise, the // ConvI2L node may be eliminated independently of the range check, causing the data path // to become TOP while the control path is still there (although it's unreachable). - value = new CastIINode(ctrl, value, itype, carry_dependency ? ConstraintCastNode::StrongDependency : ConstraintCastNode::RegularDependency, true /* range check dependency */); + value = new CastIINode(ctrl, value, itype, carry_dependency ? ConstraintCastNode::DependencyType::NonFloatingNarrowing : ConstraintCastNode::DependencyType::FloatingNarrowing, true /* range check dependency */); value = phase->transform(value); } const TypeLong* ltype = TypeLong::make(itype->_lo, itype->_hi, itype->_widen); diff --git a/src/hotspot/share/opto/escape.cpp b/src/hotspot/share/opto/escape.cpp index fb3b5dba42c40..b067d93176a0b 100644 --- a/src/hotspot/share/opto/escape.cpp +++ b/src/hotspot/share/opto/escape.cpp @@ -748,7 +748,7 @@ Node* ConnectionGraph::specialize_castpp(Node* castpp, Node* base, Node* current _igvn->_worklist.push(current_control); _igvn->_worklist.push(control_successor); - return _igvn->transform(ConstraintCastNode::make_cast_for_type(not_eq_control, base, _igvn->type(castpp), ConstraintCastNode::UnconditionalDependency, nullptr)); + return _igvn->transform(ConstraintCastNode::make_cast_for_type(not_eq_control, base, _igvn->type(castpp), ConstraintCastNode::DependencyType::NonFloatingNonNarrowing, nullptr)); } Node* ConnectionGraph::split_castpp_load_through_phi(Node* curr_addp, Node* curr_load, Node* region, GrowableArray* bases_for_loads, GrowableArray &alloc_worklist) { @@ -1235,7 +1235,7 @@ bool ConnectionGraph::reduce_phi_on_safepoints_helper(Node* ophi, Node* cast, No Node* nsr_merge_pointer = ophi; if (cast != nullptr) { const Type* new_t = merge_t->meet(TypePtr::NULL_PTR); - nsr_merge_pointer = _igvn->transform(ConstraintCastNode::make_cast_for_type(cast->in(0), cast->in(1), new_t, ConstraintCastNode::RegularDependency, nullptr)); + nsr_merge_pointer = _igvn->transform(ConstraintCastNode::make_cast_for_type(cast->in(0), cast->in(1), new_t, ConstraintCastNode::DependencyType::FloatingNarrowing, nullptr)); } for (uint spi = 0; spi < safepoints.size(); spi++) { @@ -1376,7 +1376,7 @@ void ConnectionGraph::reset_scalar_replaceable_entries(PhiNode* ophi) { } if (change) { - Node* new_cast = ConstraintCastNode::make_cast_for_type(out->in(0), out->in(1), out_new_t, ConstraintCastNode::StrongDependency, nullptr); + Node* new_cast = ConstraintCastNode::make_cast_for_type(out->in(0), out->in(1), out_new_t, ConstraintCastNode::DependencyType::NonFloatingNarrowing, nullptr); _igvn->replace_node(out, new_cast); _igvn->register_new_node_with_optimizer(new_cast); } diff --git a/src/hotspot/share/opto/library_call.cpp b/src/hotspot/share/opto/library_call.cpp index 7a213102efd15..617dfc0abfb84 100644 --- a/src/hotspot/share/opto/library_call.cpp +++ b/src/hotspot/share/opto/library_call.cpp @@ -1182,7 +1182,7 @@ bool LibraryCallKit::inline_preconditions_checkIndex(BasicType bt) { jlong upper_bound = _gvn.type(length)->is_integer(bt)->hi_as_long(); Node* casted_length = ConstraintCastNode::make_cast_for_basic_type( control(), length, TypeInteger::make(0, upper_bound, Type::WidenMax, bt), - ConstraintCastNode::RegularDependency, bt); + ConstraintCastNode::DependencyType::FloatingNarrowing, bt); casted_length = _gvn.transform(casted_length); replace_in_map(length, casted_length); length = casted_length; @@ -1212,7 +1212,7 @@ bool LibraryCallKit::inline_preconditions_checkIndex(BasicType bt) { // index is now known to be >= 0 and < length, cast it Node* result = ConstraintCastNode::make_cast_for_basic_type( control(), index, TypeInteger::make(0, upper_bound, Type::WidenMax, bt), - ConstraintCastNode::RegularDependency, bt); + ConstraintCastNode::DependencyType::FloatingNarrowing, bt); result = _gvn.transform(result); set_result(result); replace_in_map(index, result); diff --git a/src/hotspot/share/opto/loopTransform.cpp b/src/hotspot/share/opto/loopTransform.cpp index 5c65103677b2a..41bce0fe9b531 100644 --- a/src/hotspot/share/opto/loopTransform.cpp +++ b/src/hotspot/share/opto/loopTransform.cpp @@ -1366,7 +1366,7 @@ Node *PhaseIdealLoop::clone_up_backedge_goo(Node *back_ctrl, Node *preheader_ctr // the backedge of the main or post loop is removed, a Div node won't be able to float above the zero trip guard of the // loop and can't execute even if the loop is not reached. void PhaseIdealLoop::cast_incr_before_loop(Node* incr, Node* ctrl, CountedLoopNode* loop) { - Node* castii = new CastIINode(ctrl, incr, TypeInt::INT, ConstraintCastNode::UnconditionalDependency); + Node* castii = new CastIINode(ctrl, incr, TypeInt::INT, ConstraintCastNode::DependencyType::NonFloatingNonNarrowing); register_new_node(castii, ctrl); Node* phi = loop->phi(); assert(phi->in(LoopNode::EntryControl) == incr, "replacing wrong input?"); @@ -3262,7 +3262,7 @@ bool IdealLoopTree::do_remove_empty_loop(PhaseIdealLoop *phase) { Node* cast_ii = ConstraintCastNode::make_cast_for_basic_type( cl->in(LoopNode::EntryControl), exact_limit, phase->_igvn.type(exact_limit), - ConstraintCastNode::UnconditionalDependency, T_INT); + ConstraintCastNode::DependencyType::NonFloatingNonNarrowing, T_INT); phase->register_new_node(cast_ii, cl->in(LoopNode::EntryControl)); Node* final_iv = new SubINode(cast_ii, cl->stride()); diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 03cc5cbcff662..42d3ee105f8e4 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -1001,7 +1001,7 @@ bool PhaseIdealLoop::create_loop_nest(IdealLoopTree* loop, Node_List &old_new) { // a negative stride). We add a CastII here to guarantee that, when the counted loop is created in a subsequent loop // opts pass, an accurate range of values for the limits is found. const TypeInt* inner_iters_actual_int_range = TypeInt::make(0, iters_limit, Type::WidenMin); - inner_iters_actual_int = new CastIINode(outer_head, inner_iters_actual_int, inner_iters_actual_int_range, ConstraintCastNode::UnconditionalDependency); + inner_iters_actual_int = new CastIINode(outer_head, inner_iters_actual_int, inner_iters_actual_int_range, ConstraintCastNode::DependencyType::NonFloatingNonNarrowing); _igvn.register_new_node_with_optimizer(inner_iters_actual_int); } else { inner_iters_actual_int = inner_iters_actual; @@ -1315,7 +1315,7 @@ bool PhaseIdealLoop::try_make_short_running_loop(IdealLoopTree* loop, jint strid register_new_node(bol, iff->in(0)); new_limit = ConstraintCastNode::make_cast_for_basic_type(new_predicate_proj, new_limit, TypeInteger::make(1, iters_limit_long, Type::WidenMin, bt), - ConstraintCastNode::UnconditionalDependency, bt); + ConstraintCastNode::DependencyType::NonFloatingNonNarrowing, bt); register_new_node(new_limit, new_predicate_proj); #ifndef PRODUCT @@ -1334,7 +1334,7 @@ bool PhaseIdealLoop::try_make_short_running_loop(IdealLoopTree* loop, jint strid const TypeLong* new_limit_t = new_limit->Value(&_igvn)->is_long(); new_limit = ConstraintCastNode::make_cast_for_basic_type(predicates.entry(), new_limit, TypeLong::make(0, new_limit_t->_hi, new_limit_t->_widen), - ConstraintCastNode::UnconditionalDependency, bt); + ConstraintCastNode::DependencyType::NonFloatingNonNarrowing, bt); register_new_node(new_limit, predicates.entry()); } else { assert(bt == T_INT && known_short_running_loop, "only CountedLoop statically known to be short running"); diff --git a/src/hotspot/share/opto/loopopts.cpp b/src/hotspot/share/opto/loopopts.cpp index ee3f138b8afab..b1422d5db20fc 100644 --- a/src/hotspot/share/opto/loopopts.cpp +++ b/src/hotspot/share/opto/loopopts.cpp @@ -1174,7 +1174,7 @@ Node *PhaseIdealLoop::split_if_with_blocks_pre( Node *n ) { if ( nn ) return nn; } - if (n->is_ConstraintCast()) { + if (n->is_ConstraintCast() && n->as_ConstraintCast()->dependency().narrows_type()) { Node* dom_cast = n->as_ConstraintCast()->dominating_cast(&_igvn, this); // ConstraintCastNode::dominating_cast() uses node control input to determine domination. // Node control inputs don't necessarily agree with loop control info (due to @@ -1837,7 +1837,7 @@ void PhaseIdealLoop::try_sink_out_of_loop(Node* n) { if (in != nullptr && ctrl_is_member(n_loop, in)) { const Type* in_t = _igvn.type(in); cast = ConstraintCastNode::make_cast_for_type(x_ctrl, in, in_t, - ConstraintCastNode::UnconditionalDependency, nullptr); + ConstraintCastNode::DependencyType::NonFloatingNonNarrowing, nullptr); } if (cast != nullptr) { Node* prev = _igvn.hash_find_insert(cast); diff --git a/src/hotspot/share/opto/macroArrayCopy.cpp b/src/hotspot/share/opto/macroArrayCopy.cpp index 0ba8ed40c3738..fb4a1842a369f 100644 --- a/src/hotspot/share/opto/macroArrayCopy.cpp +++ b/src/hotspot/share/opto/macroArrayCopy.cpp @@ -233,7 +233,7 @@ void PhaseMacroExpand::generate_partial_inlining_block(Node** ctrl, MergeMemNode Node* inline_block = generate_guard(ctrl, bol_le, nullptr, PROB_FAIR); Node* stub_block = *ctrl; - Node* casted_length = new CastLLNode(inline_block, length, inline_range, ConstraintCastNode::RegularDependency); + Node* casted_length = new CastLLNode(inline_block, length, inline_range, ConstraintCastNode::DependencyType::FloatingNarrowing); transform_later(casted_length); Node* mask_gen = VectorMaskGenNode::make(casted_length, type); transform_later(mask_gen); diff --git a/test/hotspot/jtreg/compiler/c2/irTests/TestPushAddThruCast.java b/test/hotspot/jtreg/compiler/c2/irTests/TestPushAddThruCast.java index 7c5fa05f14788..3aee78e9d6a3f 100644 --- a/test/hotspot/jtreg/compiler/c2/irTests/TestPushAddThruCast.java +++ b/test/hotspot/jtreg/compiler/c2/irTests/TestPushAddThruCast.java @@ -44,13 +44,13 @@ public static void main(String[] args) { TestFramework.run(); } - final static int length = RANDOM.nextInt(Integer.MAX_VALUE); - final static long llength = RANDOM.nextInt(Integer.MAX_VALUE); + final static int length = RANDOM.nextInt(5, Integer.MAX_VALUE); + final static long llength = RANDOM.nextInt(2, Integer.MAX_VALUE); static int i; static long l; @Test - @IR(counts = { IRNode.CAST_II, "1" }) + @IR(counts = { IRNode.CAST_II, "2" }) public static int test1() { int j = Objects.checkIndex(i, length); int k = Objects.checkIndex(i + 1, length); @@ -67,7 +67,7 @@ public static void test1_runner() { } @Test - @IR(counts = { IRNode.CAST_LL, "1" }) + @IR(counts = { IRNode.CAST_LL, "2" }) public static long test2() { long j = Objects.checkIndex(l, llength); long k = Objects.checkIndex(l + 1, llength); @@ -82,4 +82,25 @@ public static void test2_runner() { throw new RuntimeException("incorrect result: " + res); } } + + // Test commoning of Casts after loop opts when they are at the same control + @Test + @IR(phase = CompilePhase.ITER_GVN1, counts = { IRNode.CAST_II, "4" }) + @IR(phase = CompilePhase.OPTIMIZE_FINISHED, counts = { IRNode.CAST_II, "2" }) + public static int test3() { + int j = Objects.checkIndex(i - 3, length); + j += Objects.checkIndex(i, length); + j += Objects.checkIndex(i - 2, length); + j += Objects.checkIndex(i - 1, length); + return j; + } + + @Run(test = "test3") + public static void test3_runner() { + i = RANDOM.nextInt(3, length - 1); + int res = test3(); + if (res != i * 4 - 6) { + throw new RuntimeException("incorrect result: " + res + " for i = " + i); + } + } } diff --git a/test/hotspot/jtreg/compiler/rangechecks/TestArrayAccessAboveRCAfterRCCastIIEliminated.java b/test/hotspot/jtreg/compiler/rangechecks/TestArrayAccessAboveRCAfterRCCastIIEliminated.java index 87e87842af337..b31c142572893 100644 --- a/test/hotspot/jtreg/compiler/rangechecks/TestArrayAccessAboveRCAfterRCCastIIEliminated.java +++ b/test/hotspot/jtreg/compiler/rangechecks/TestArrayAccessAboveRCAfterRCCastIIEliminated.java @@ -44,7 +44,6 @@ public class TestArrayAccessAboveRCAfterRCCastIIEliminated { private static volatile int volatileField; public static void main(String[] args) { - int[] array = new int[100]; for (int i = 0; i < 20_000; i++) { test1(9, 10, 1, true); test1(9, 10, 1, false); @@ -72,6 +71,13 @@ public static void main(String[] args) { test12(9, 10, 1, false); test13(9, 10, 1, true); test13(9, 10, 1, false); + test14(8, 0, 1, true); + test14(8, 0, 1, false); + inlined14(0, 0); + test15(8, 0, 1, true); + test15(8, 0, 1, false); + inlined15(0, 0); + } try { test1(-1, 10, 1, true); @@ -125,6 +131,14 @@ public static void main(String[] args) { test13(-1, 10, 1, true); } catch (ArrayIndexOutOfBoundsException arrayIndexOutOfBoundsException) { } + try { + test14(Integer.MAX_VALUE, 10, 1, true); + } catch (ArrayIndexOutOfBoundsException arrayIndexOutOfBoundsException) { + } + try { + test15(Integer.MAX_VALUE, 10, 1, true); + } catch (ArrayIndexOutOfBoundsException arrayIndexOutOfBoundsException) { + } } private static void test1(int i, int j, int flag, boolean flag2) { @@ -468,6 +482,72 @@ private static void test13(int i, int j, int flag, boolean flag2) { } } + // Range check cast type widen after loop opts causes control dependency to be lost + private static void test14(int i, int j, int flag, boolean flag2) { + int l = 0; + for (; l < 10; l++); + j = inlined14(j, l); + int[] array = new int[10]; + notInlined(array); + if (flag == 0) { + } + if (flag2) { + float[] newArray = new float[10]; + newArray[i+j] = 42; // i+j in [0, 9] + float[] otherArray = new float[i+j]; // i+j in [0, max] + if (flag == 0) { + } + intField = array[otherArray.length]; + } else { + float[] newArray = new float[10]; + newArray[i+j] = 42; // i+j in [0, 9] + float[] otherArray = new float[i+j]; // i+j in [0, max] + if (flag == 0) { + } + intField = array[otherArray.length]; + } + } + + private static int inlined14(int j, int l) { + if (l == 10) { + j = 1; + } + return j; + } + + private static void test15(int i, int j, int flag, boolean flag2) { + i = Integer.max(i, Integer.MIN_VALUE + 1); + int l = 0; + for (; l < 10; l++); + j = inlined15(j, l); + int[] array = new int[10]; + notInlined(array); + if (flag == 0) { + } + if (flag2) { + float[] newArray = new float[10]; + newArray[i+j] = 42; // i+j in [0, 9] + float[] otherArray = new float[i+j]; // i+j in [0, max] + if (flag == 0) { + } + intField = array[otherArray.length]; + } else { + float[] newArray = new float[10]; + newArray[i+j] = 42; // i+j in [0, 9] + float[] otherArray = new float[i+j]; // i+j in [0, max] + if (flag == 0) { + } + intField = array[otherArray.length]; + } + } + + private static int inlined15(int j, int l) { + if (l == 10) { + j = Integer.max(j, Integer.MIN_VALUE + 10); + } + return j; + } + private static void notInlined(int[] array) { }