Skip to content
Permalink
Browse files
8256655: rework long counted loop handling
Reviewed-by: kvn, thartmann, vlivanov
  • Loading branch information
rwestrel committed Dec 1, 2020
1 parent 13bd2e8 commit 3d460bd29566b7b6cdfcc463d5dc4b1354b07258
@@ -109,13 +109,13 @@ static bool commute(Node *add, bool con_left, bool con_right) {
//------------------------------Idealize---------------------------------------
// If we get here, we assume we are associative!
Node *AddNode::Ideal(PhaseGVN *phase, bool can_reshape) {
const Type *t1 = phase->type( in(1) );
const Type *t2 = phase->type( in(2) );
const Type *t1 = phase->type(in(1));
const Type *t2 = phase->type(in(2));
bool con_left = t1->singleton();
bool con_right = t2->singleton();

// Check for commutative operation desired
if( commute(this,con_left,con_right) ) return this;
if (commute(this, con_left, con_right)) return this;

AddNode *progress = NULL; // Progress flag

@@ -126,27 +126,27 @@ Node *AddNode::Ideal(PhaseGVN *phase, bool can_reshape) {
Node *add2 = in(2);
int add1_op = add1->Opcode();
int this_op = Opcode();
if( con_right && t2 != Type::TOP && // Right input is a constant?
add1_op == this_op ) { // Left input is an Add?
if (con_right && t2 != Type::TOP && // Right input is a constant?
add1_op == this_op) { // Left input is an Add?

// Type of left _in right input
const Type *t12 = phase->type( add1->in(2) );
if( t12->singleton() && t12 != Type::TOP ) { // Left input is an add of a constant?
const Type *t12 = phase->type(add1->in(2));
if (t12->singleton() && t12 != Type::TOP) { // Left input is an add of a constant?
// Check for rare case of closed data cycle which can happen inside
// unreachable loops. In these cases the computation is undefined.
#ifdef ASSERT
Node *add11 = add1->in(1);
int add11_op = add11->Opcode();
if( (add1 == add1->in(1))
|| (add11_op == this_op && add11->in(1) == add1) ) {
if ((add1 == add1->in(1))
|| (add11_op == this_op && add11->in(1) == add1)) {
assert(false, "dead loop in AddNode::Ideal");
}
#endif
// The Add of the flattened expression
Node *x1 = add1->in(1);
Node *x2 = phase->makecon( add1->as_Add()->add_ring( t2, t12 ));
Node *x2 = phase->makecon(add1->as_Add()->add_ring(t2, t12));
PhaseIterGVN *igvn = phase->is_IterGVN();
if( igvn ) {
if (igvn) {
set_req_X(2,x2,igvn);
set_req_X(1,x1,igvn);
} else {
@@ -160,11 +160,11 @@ Node *AddNode::Ideal(PhaseGVN *phase, bool can_reshape) {
}

// Convert "(x+1)+y" into "(x+y)+1". Push constants down the expression tree.
if( add1_op == this_op && !con_right ) {
if (add1_op == this_op && !con_right) {
Node *a12 = add1->in(2);
const Type *t12 = phase->type( a12 );
if( t12->singleton() && t12 != Type::TOP && (add1 != add1->in(1)) &&
!(add1->in(1)->is_Phi() && add1->in(1)->as_Phi()->is_tripcount()) ) {
if (t12->singleton() && t12 != Type::TOP && (add1 != add1->in(1)) &&
!(add1->in(1)->is_Phi() && (add1->in(1)->as_Phi()->is_tripcount(T_INT) || add1->in(1)->as_Phi()->is_tripcount(T_LONG)))) {
assert(add1->in(1) != this, "dead loop in AddNode::Ideal");
add2 = add1->clone();
add2->set_req(2, in(2));
@@ -178,11 +178,11 @@ Node *AddNode::Ideal(PhaseGVN *phase, bool can_reshape) {

// Convert "x+(y+1)" into "(x+y)+1". Push constants down the expression tree.
int add2_op = add2->Opcode();
if( add2_op == this_op && !con_left ) {
if (add2_op == this_op && !con_left) {
Node *a22 = add2->in(2);
const Type *t22 = phase->type( a22 );
if( t22->singleton() && t22 != Type::TOP && (add2 != add2->in(1)) &&
!(add2->in(1)->is_Phi() && add2->in(1)->as_Phi()->is_tripcount()) ) {
if (t22->singleton() && t22 != Type::TOP && (add2 != add2->in(1)) &&
!(add2->in(1)->is_Phi() && (add2->in(1)->as_Phi()->is_tripcount(T_INT) || add2->in(1)->as_Phi()->is_tripcount(T_LONG)))) {
assert(add2->in(1) != this, "dead loop in AddNode::Ideal");
Node *addx = add2->clone();
addx->set_req(1, in(1));
@@ -235,6 +235,17 @@ const Type *AddNode::add_of_identity( const Type *t1, const Type *t2 ) const {
return NULL;
}

AddNode* AddNode::make(Node* in1, Node* in2, BasicType bt) {
switch (bt) {
case T_INT:
return new AddINode(in1, in2);
case T_LONG:
return new AddLNode(in1, in2);
default:
fatal("Not implemented for %s", type2name(bt));
}
return NULL;
}

//=============================================================================
//------------------------------Idealize---------------------------------------
@@ -907,51 +918,61 @@ Node* MaxNode::build_min_max(Node* a, Node* b, bool is_max, bool is_unsigned, co
bool is_int = gvn.type(a)->isa_int();
assert(is_int || gvn.type(a)->isa_long(), "int or long inputs");
assert(is_int == (gvn.type(b)->isa_int() != NULL), "inconsistent inputs");
Node* hook = NULL;
if (gvn.is_IterGVN()) {
// Make sure a and b are not destroyed
hook = new Node(2);
hook->init_req(0, a);
hook->init_req(1, b);
}
Node* res = NULL;
if (!is_unsigned) {
if (is_max) {
if (is_int) {
Node* res = gvn.transform(new MaxINode(a, b));
res = gvn.transform(new MaxINode(a, b));
assert(gvn.type(res)->is_int()->_lo >= t->is_int()->_lo && gvn.type(res)->is_int()->_hi <= t->is_int()->_hi, "type doesn't match");
return res;
} else {
Node* cmp = gvn.transform(new CmpLNode(a, b));
Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::lt));
return gvn.transform(new CMoveLNode(bol, a, b, t->is_long()));
res = gvn.transform(new CMoveLNode(bol, a, b, t->is_long()));
}
} else {
if (is_int) {
Node* res = gvn.transform(new MinINode(a, b));
assert(gvn.type(res)->is_int()->_lo >= t->is_int()->_lo && gvn.type(res)->is_int()->_hi <= t->is_int()->_hi, "type doesn't match");
return res;
} else {
Node* cmp = gvn.transform(new CmpLNode(b, a));
Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::lt));
return gvn.transform(new CMoveLNode(bol, a, b, t->is_long()));
res = gvn.transform(new CMoveLNode(bol, a, b, t->is_long()));
}
}
} else {
if (is_max) {
if (is_int) {
Node* cmp = gvn.transform(new CmpUNode(a, b));
Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::lt));
return gvn.transform(new CMoveINode(bol, a, b, t->is_int()));
res = gvn.transform(new CMoveINode(bol, a, b, t->is_int()));
} else {
Node* cmp = gvn.transform(new CmpULNode(a, b));
Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::lt));
return gvn.transform(new CMoveLNode(bol, a, b, t->is_long()));
res = gvn.transform(new CMoveLNode(bol, a, b, t->is_long()));
}
} else {
if (is_int) {
Node* cmp = gvn.transform(new CmpUNode(b, a));
Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::lt));
return gvn.transform(new CMoveINode(bol, a, b, t->is_int()));
res = gvn.transform(new CMoveINode(bol, a, b, t->is_int()));
} else {
Node* cmp = gvn.transform(new CmpULNode(b, a));
Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::lt));
return gvn.transform(new CMoveLNode(bol, a, b, t->is_long()));
res = gvn.transform(new CMoveLNode(bol, a, b, t->is_long()));
}
}
}
if (hook != NULL) {
hook->destruct(&gvn);
}
return res;
}

Node* MaxNode::build_min_max_diff_with_zero(Node* a, Node* b, bool is_max, const Type* t, PhaseGVN& gvn) {
@@ -964,31 +985,43 @@ Node* MaxNode::build_min_max_diff_with_zero(Node* a, Node* b, bool is_max, const
} else {
zero = gvn.longcon(0);
}
Node* hook = NULL;
if (gvn.is_IterGVN()) {
// Make sure a and b are not destroyed
hook = new Node(2);
hook->init_req(0, a);
hook->init_req(1, b);
}
Node* res = NULL;
if (is_max) {
if (is_int) {
Node* cmp = gvn.transform(new CmpINode(a, b));
Node* sub = gvn.transform(new SubINode(a, b));
Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::lt));
return gvn.transform(new CMoveINode(bol, sub, zero, t->is_int()));
res = gvn.transform(new CMoveINode(bol, sub, zero, t->is_int()));
} else {
Node* cmp = gvn.transform(new CmpLNode(a, b));
Node* sub = gvn.transform(new SubLNode(a, b));
Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::lt));
return gvn.transform(new CMoveLNode(bol, sub, zero, t->is_long()));
res = gvn.transform(new CMoveLNode(bol, sub, zero, t->is_long()));
}
} else {
if (is_int) {
Node* cmp = gvn.transform(new CmpINode(b, a));
Node* sub = gvn.transform(new SubINode(a, b));
Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::lt));
return gvn.transform(new CMoveINode(bol, sub, zero, t->is_int()));
res = gvn.transform(new CMoveINode(bol, sub, zero, t->is_int()));
} else {
Node* cmp = gvn.transform(new CmpLNode(b, a));
Node* sub = gvn.transform(new SubLNode(a, b));
Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::lt));
return gvn.transform(new CMoveLNode(bol, sub, zero, t->is_long()));
res = gvn.transform(new CMoveLNode(bol, sub, zero, t->is_long()));
}
}
if (hook != NULL) {
hook->destruct(&gvn);
}
return res;
}

//=============================================================================
@@ -68,6 +68,11 @@ class AddNode : public Node {
// Supplied function to return the additive identity type
virtual const Type *add_id() const = 0;

virtual bool operates_on(BasicType bt, bool signed_int) const {
assert(bt == T_INT || bt == T_LONG, "unsupported");
return false;
}
static AddNode* make(Node* in1, Node* in2, BasicType bt);
};

//------------------------------AddINode---------------------------------------
@@ -81,6 +86,10 @@ class AddINode : public AddNode {
virtual const Type *bottom_type() const { return TypeInt::INT; }
virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
virtual Node* Identity(PhaseGVN* phase);
virtual bool operates_on(BasicType bt, bool signed_int) const {
assert(bt == T_INT || bt == T_LONG, "unsupported");
return bt == T_INT;
}
virtual uint ideal_reg() const { return Op_RegI; }
};

@@ -95,6 +104,10 @@ class AddLNode : public AddNode {
virtual const Type *bottom_type() const { return TypeLong::LONG; }
virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
virtual Node* Identity(PhaseGVN* phase);
virtual bool operates_on(BasicType bt, bool signed_int) const {
assert(bt == T_INT || bt == T_LONG, "unsupported");
return bt == T_LONG;
}
virtual uint ideal_reg() const { return Op_RegL; }
};

@@ -54,6 +54,10 @@ class ConstraintCastNode: public TypeNode {
TypeNode* dominating_cast(PhaseGVN* gvn, PhaseTransform* pt) const;
static Node* make_cast(int opcode, Node* c, Node *n, const Type *t, bool carry_dependency);
static Node* make(Node* c, Node *n, const Type *t, BasicType bt);
virtual bool operates_on(BasicType bt, bool signed_int) const {
assert(bt == T_INT || bt == T_LONG, "unsupported");
return false;
}

#ifndef PRODUCT
virtual void dump_spec(outputStream *st) const;
@@ -87,6 +91,10 @@ class CastIINode: public ConstraintCastNode {
return false;
#endif
}
virtual bool operates_on(BasicType bt, bool signed_int) const {
assert(bt == T_INT || bt == T_LONG, "unsupported");
return bt == T_INT;
}

#ifndef PRODUCT
virtual void dump_spec(outputStream* st) const;
@@ -99,6 +107,10 @@ class CastLLNode: public ConstraintCastNode {
: ConstraintCastNode(n, t, carry_dependency){
init_class_id(Class_CastLL);
}
virtual bool operates_on(BasicType bt, bool signed_int) const {
assert(bt == T_INT || bt == T_LONG, "unsupported");
return bt == T_LONG;
}
virtual int Opcode() const;
virtual uint ideal_reg() const { return Op_RegL; }
};
@@ -1084,30 +1084,32 @@ const Type* PhiNode::Value(PhaseGVN* phase) const {
return Type::TOP;

// Check for trip-counted loop. If so, be smarter.
CountedLoopNode* l = r->is_CountedLoop() ? r->as_CountedLoop() : NULL;
BaseCountedLoopNode* l = r->is_BaseCountedLoop() ? r->as_BaseCountedLoop() : NULL;
if (l && ((const Node*)l->phi() == this)) { // Trip counted loop!
// protect against init_trip() or limit() returning NULL
if (l->can_be_counted_loop(phase)) {
const Node *init = l->init_trip();
const Node *limit = l->limit();
const Node* init = l->init_trip();
const Node* limit = l->limit();
const Node* stride = l->stride();
if (init != NULL && limit != NULL && stride != NULL) {
const TypeInt* lo = phase->type(init)->isa_int();
const TypeInt* hi = phase->type(limit)->isa_int();
const TypeInt* stride_t = phase->type(stride)->isa_int();
const TypeInteger* lo = phase->type(init)->isa_integer(l->bt());
const TypeInteger* hi = phase->type(limit)->isa_integer(l->bt());
const TypeInteger* stride_t = phase->type(stride)->isa_integer(l->bt());
if (lo != NULL && hi != NULL && stride_t != NULL) { // Dying loops might have TOP here
assert(stride_t->_hi >= stride_t->_lo, "bad stride type");
assert(stride_t->hi_as_long() >= stride_t->lo_as_long(), "bad stride type");
BoolTest::mask bt = l->loopexit()->test_trip();
// If the loop exit condition is "not equal", the condition
// would not trigger if init > limit (if stride > 0) or if
// init < limit if (stride > 0) so we can't deduce bounds
// for the iv from the exit condition.
if (bt != BoolTest::ne) {
if (stride_t->_hi < 0) { // Down-counter loop
if (stride_t->hi_as_long() < 0) { // Down-counter loop
swap(lo, hi);
return TypeInt::make(MIN2(lo->_lo, hi->_lo) , hi->_hi, 3)->filter_speculative(_type);
} else if (stride_t->_lo >= 0) {
return TypeInt::make(lo->_lo, MAX2(lo->_hi, hi->_hi), 3)->filter_speculative(_type);
return TypeInteger::make(MIN2(lo->lo_as_long(), hi->lo_as_long()), hi->hi_as_long(), 3,
l->bt())->filter_speculative(_type);
} else if (stride_t->lo_as_long() >= 0) {
return TypeInteger::make(lo->lo_as_long(), MAX2(lo->hi_as_long(), hi->hi_as_long()), 3,
l->bt())->filter_speculative(_type);
}
}
}
@@ -2446,9 +2448,10 @@ bool PhiNode::is_data_loop(RegionNode* r, Node* uin, const PhaseGVN* phase) {
}

//------------------------------is_tripcount-----------------------------------
bool PhiNode::is_tripcount() const {
return (in(0) != NULL && in(0)->is_CountedLoop() &&
in(0)->as_CountedLoop()->phi() == this);
bool PhiNode::is_tripcount(BasicType bt) const {
return (in(0) != NULL && in(0)->is_BaseCountedLoop() &&
in(0)->as_BaseCountedLoop()->operates_on(bt, true) &&
in(0)->as_BaseCountedLoop()->phi() == this);
}

//------------------------------out_RegMask------------------------------------
@@ -2475,7 +2478,7 @@ void PhiNode::related(GrowableArray<Node*> *in_rel, GrowableArray<Node*> *out_re

void PhiNode::dump_spec(outputStream *st) const {
TypeNode::dump_spec(st);
if (is_tripcount()) {
if (is_tripcount(T_INT) || is_tripcount(T_LONG)) {
st->print(" #tripcount");
}
}
@@ -178,7 +178,7 @@ class PhiNode : public TypeNode {
// Accessors
RegionNode* region() const { Node* r = in(Region); assert(!r || r->is_Region(), ""); return (RegionNode*)r; }

bool is_tripcount() const;
bool is_tripcount(BasicType bt) const;

// Determine a unique non-trivial input, if any.
// Ignore casts if it helps. Return NULL on failure.
@@ -144,6 +144,8 @@ macro(CountedLoop)
macro(CountedLoopEnd)
macro(OuterStripMinedLoop)
macro(OuterStripMinedLoopEnd)
macro(LongCountedLoop)
macro(LongCountedLoopEnd)
macro(CountLeadingZerosI)
macro(CountLeadingZerosL)
macro(CountTrailingZerosI)
@@ -3395,6 +3395,7 @@ void Compile::final_graph_reshaping_main_switch(Node* n, Final_Reshape_Counts& f
case Op_Loop:
assert(!n->as_Loop()->is_transformed_long_loop() || _loop_opts_cnt == 0, "should have been turned into a counted loop");
case Op_CountedLoop:
case Op_LongCountedLoop:
case Op_OuterStripMinedLoop:
if (n->as_Loop()->is_inner_loop()) {
frc.inc_inner_loop_count();
@@ -3713,7 +3714,7 @@ bool Compile::final_graph_reshaping() {

// Here so verification code in final_graph_reshaping_walk()
// always see an OuterStripMinedLoopEnd
if (n->is_OuterStripMinedLoopEnd()) {
if (n->is_OuterStripMinedLoopEnd() || n->is_LongCountedLoopEnd()) {
IfNode* init_iff = n->as_If();
Node* iff = new IfNode(init_iff->in(0), init_iff->in(1), init_iff->_prob, init_iff->_fcnt);
n->subsume_by(iff, this);

1 comment on commit 3d460bd

@openjdk-notifier

This comment has been minimized.

Copy link

@openjdk-notifier openjdk-notifier bot commented on 3d460bd Dec 1, 2020

Please sign in to comment.