Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
8259609: C2: optimize long range checks in long counted loops 
Co-authored-by: John R Rose <jrose@openjdk.org>
Reviewed-by: thartmann, jrose
  • Loading branch information
@rwestrel
rwestrel and John R Rose committed Oct 26, 2021
1 parent 574f890 commit 82f4aac
Show file tree
Hide file tree
Showing 13 changed files with 908 additions and 175 deletions.
83 changes: 19 additions & 64 deletions src/hotspot/share/opto/addnode.cpp
View file
Expand Up @@ -1107,6 +1107,7 @@ Node* MaxNode::build_min_max(Node* a, Node* b, bool is_max, bool is_unsigned, co
bool is_int = gvn.type(a)->isa_int();
assert(is_int || gvn.type(a)->isa_long(), "int or long inputs");
assert(is_int == (gvn.type(b)->isa_int() != NULL), "inconsistent inputs");
BasicType bt = is_int ? T_INT: T_LONG;
Node* hook = NULL;
if (gvn.is_IterGVN()) {
// Make sure a and b are not destroyed
Expand All @@ -1115,48 +1116,23 @@ Node* MaxNode::build_min_max(Node* a, Node* b, bool is_max, bool is_unsigned, co
hook->init_req(1, b);
}
Node* res = NULL;
if (!is_unsigned) {
if (is_int && !is_unsigned) {
if (is_max) {
if (is_int) {
res = gvn.transform(new MaxINode(a, b));
assert(gvn.type(res)->is_int()->_lo >= t->is_int()->_lo && gvn.type(res)->is_int()->_hi <= t->is_int()->_hi, "type doesn't match");
} else {
Node* cmp = gvn.transform(new CmpLNode(a, b));
Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::lt));
res = gvn.transform(new CMoveLNode(bol, a, b, t->is_long()));
}
res = gvn.transform(new MaxINode(a, b));
assert(gvn.type(res)->is_int()->_lo >= t->is_int()->_lo && gvn.type(res)->is_int()->_hi <= t->is_int()->_hi, "type doesn't match");
} else {
if (is_int) {
Node* res = gvn.transform(new MinINode(a, b));
assert(gvn.type(res)->is_int()->_lo >= t->is_int()->_lo && gvn.type(res)->is_int()->_hi <= t->is_int()->_hi, "type doesn't match");
} else {
Node* cmp = gvn.transform(new CmpLNode(b, a));
Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::lt));
res = gvn.transform(new CMoveLNode(bol, a, b, t->is_long()));
}
Node* res = gvn.transform(new MinINode(a, b));
assert(gvn.type(res)->is_int()->_lo >= t->is_int()->_lo && gvn.type(res)->is_int()->_hi <= t->is_int()->_hi, "type doesn't match");
}
} else {
Node* cmp = NULL;
if (is_max) {
if (is_int) {
Node* cmp = gvn.transform(new CmpUNode(a, b));
Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::lt));
res = gvn.transform(new CMoveINode(bol, a, b, t->is_int()));
} else {
Node* cmp = gvn.transform(new CmpULNode(a, b));
Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::lt));
res = gvn.transform(new CMoveLNode(bol, a, b, t->is_long()));
}
cmp = gvn.transform(CmpNode::make(a, b, bt, is_unsigned));
} else {
if (is_int) {
Node* cmp = gvn.transform(new CmpUNode(b, a));
Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::lt));
res = gvn.transform(new CMoveINode(bol, a, b, t->is_int()));
} else {
Node* cmp = gvn.transform(new CmpULNode(b, a));
Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::lt));
res = gvn.transform(new CMoveLNode(bol, a, b, t->is_long()));
}
cmp = gvn.transform(CmpNode::make(b, a, bt, is_unsigned));
}
Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::lt));
res = gvn.transform(CMoveNode::make(NULL, bol, a, b, t));
}
if (hook != NULL) {
hook->destruct(&gvn);
Expand All @@ -1168,45 +1144,24 @@ Node* MaxNode::build_min_max_diff_with_zero(Node* a, Node* b, bool is_max, const
bool is_int = gvn.type(a)->isa_int();
assert(is_int || gvn.type(a)->isa_long(), "int or long inputs");
assert(is_int == (gvn.type(b)->isa_int() != NULL), "inconsistent inputs");
Node* zero = NULL;
if (is_int) {
zero = gvn.intcon(0);
} else {
zero = gvn.longcon(0);
}
BasicType bt = is_int ? T_INT: T_LONG;
Node* zero = gvn.integercon(0, bt);
Node* hook = NULL;
if (gvn.is_IterGVN()) {
// Make sure a and b are not destroyed
hook = new Node(2);
hook->init_req(0, a);
hook->init_req(1, b);
}
Node* res = NULL;
Node* cmp = NULL;
if (is_max) {
if (is_int) {
Node* cmp = gvn.transform(new CmpINode(a, b));
Node* sub = gvn.transform(new SubINode(a, b));
Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::lt));
res = gvn.transform(new CMoveINode(bol, sub, zero, t->is_int()));
} else {
Node* cmp = gvn.transform(new CmpLNode(a, b));
Node* sub = gvn.transform(new SubLNode(a, b));
Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::lt));
res = gvn.transform(new CMoveLNode(bol, sub, zero, t->is_long()));
}
cmp = gvn.transform(CmpNode::make(a, b, bt, false));
} else {
if (is_int) {
Node* cmp = gvn.transform(new CmpINode(b, a));
Node* sub = gvn.transform(new SubINode(a, b));
Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::lt));
res = gvn.transform(new CMoveINode(bol, sub, zero, t->is_int()));
} else {
Node* cmp = gvn.transform(new CmpLNode(b, a));
Node* sub = gvn.transform(new SubLNode(a, b));
Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::lt));
res = gvn.transform(new CMoveLNode(bol, sub, zero, t->is_long()));
}
cmp = gvn.transform(CmpNode::make(b, a, bt, false));
}
Node* sub = gvn.transform(SubNode::make(a, b, bt));
Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::lt));
Node* res = gvn.transform(CMoveNode::make(NULL, bol, sub, zero, t));
if (hook != NULL) {
hook->destruct(&gvn);
}
Expand Down
71 changes: 42 additions & 29 deletions src/hotspot/share/opto/loopPredicate.cpp
View file
Expand Up @@ -712,7 +712,8 @@ class Invariance : public StackObj {
// Returns true if the predicate of iff is in "scale*iv + offset u< load_range(ptr)" format
// Note: this function is particularly designed for loop predication. We require load_range
// and offset to be loop invariant computed on the fly by "invar"
bool IdealLoopTree::is_range_check_if(IfNode *iff, PhaseIdealLoop *phase, Invariance& invar DEBUG_ONLY(COMMA ProjNode *predicate_proj)) const {
bool IdealLoopTree::is_range_check_if(IfNode *iff, PhaseIdealLoop *phase, BasicType bt, Node *iv, Node *&range,
Node *&offset, jlong &scale) const {
if (!is_loop_exit(iff)) {
return false;
}
Expand All @@ -727,48 +728,60 @@ bool IdealLoopTree::is_range_check_if(IfNode *iff, PhaseIdealLoop *phase, Invari
return false;
}
const CmpNode *cmp = bol->in(1)->as_Cmp();
if (cmp->Opcode() != Op_CmpU) {
if (!(cmp->is_Cmp() && cmp->operates_on(bt, false))) {
return false;
}
Node* range = cmp->in(2);
if (range->Opcode() != Op_LoadRange && !iff->is_RangeCheck()) {
const TypeInt* tint = phase->_igvn.type(range)->isa_int();
if (tint == NULL || tint->empty() || tint->_lo < 0) {
range = cmp->in(2);
if (range->Opcode() != Op_LoadRange) {
const TypeInteger* tinteger = phase->_igvn.type(range)->isa_integer(bt);
if (tinteger == NULL || tinteger->empty() || tinteger->lo_as_long() < 0) {
// Allow predication on positive values that aren't LoadRanges.
// This allows optimization of loops where the length of the
// array is a known value and doesn't need to be loaded back
// from the array.
return false;
}
} else {
assert(bt == T_INT, "no LoadRange for longs");
}
if (!invar.is_invariant(range)) {
return false;
}
Node *iv = _head->as_CountedLoop()->phi();
int scale = 0;
Node *offset = NULL;
if (!phase->is_scaled_iv_plus_offset(cmp->in(1), iv, &scale, &offset)) {
return false;
}
if (offset && !invar.is_invariant(offset)) { // offset must be invariant
scale = 0;
offset = NULL;
if (!phase->is_scaled_iv_plus_offset(cmp->in(1), iv, &scale, &offset, bt)) {
return false;
}
return true;
}

bool IdealLoopTree::is_range_check_if(IfNode *iff, PhaseIdealLoop *phase, Invariance& invar DEBUG_ONLY(COMMA ProjNode *predicate_proj)) const {
Node* range = NULL;
Node* offset = NULL;
jlong scale = 0;
Node* iv = _head->as_BaseCountedLoop()->phi();
if (is_range_check_if(iff, phase, T_INT, iv, range, offset, scale)) {
if (!invar.is_invariant(range)) {
return false;
}
if (offset && !invar.is_invariant(offset)) { // offset must be invariant
return false;
}
#ifdef ASSERT
if (offset && phase->has_ctrl(offset)) {
Node* offset_ctrl = phase->get_ctrl(offset);
if (phase->get_loop(predicate_proj) == phase->get_loop(offset_ctrl) &&
phase->is_dominator(predicate_proj, offset_ctrl)) {
// If the control of offset is loop predication promoted by previous pass,
// then it will lead to cyclic dependency.
// Previously promoted loop predication is in the same loop of predication
// point.
// This situation can occur when pinning nodes too conservatively - can we do better?
assert(false, "cyclic dependency prevents range check elimination, idx: offset %d, offset_ctrl %d, predicate_proj %d",
offset->_idx, offset_ctrl->_idx, predicate_proj->_idx);
if (offset && phase->has_ctrl(offset)) {
Node* offset_ctrl = phase->get_ctrl(offset);
if (phase->get_loop(predicate_proj) == phase->get_loop(offset_ctrl) &&
phase->is_dominator(predicate_proj, offset_ctrl)) {
// If the control of offset is loop predication promoted by previous pass,
// then it will lead to cyclic dependency.
// Previously promoted loop predication is in the same loop of predication
// point.
// This situation can occur when pinning nodes too conservatively - can we do better?
assert(false, "cyclic dependency prevents range check elimination, idx: offset %d, offset_ctrl %d, predicate_proj %d",
offset->_idx, offset_ctrl->_idx, predicate_proj->_idx);
}
}
}
#endif
return true;
return true;
}
return false;
}

//------------------------------rc_predicate-----------------------------------
Expand Down

1 comment on commit 82f4aac

@openjdk-notifier
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Review

Issues

Please sign in to comment.