Skip to content
Permalink
Browse files

8229495: SIGILL in C2 generated OSR compilation

Reviewed-by: kvn, chagedorn
  • Loading branch information
rwestrel committed Jul 1, 2020
1 parent d7c1bb1 commit b67814941f16e96681fdbfa5354fab612b559c65
@@ -234,6 +234,8 @@ macro(NegF)
macro(NeverBranch)
macro(OnSpinWait)
macro(Opaque1)
macro(OpaqueLoopInit)
macro(OpaqueLoopStride)
macro(Opaque2)
macro(Opaque3)
macro(Opaque4)
@@ -1807,7 +1807,17 @@ void Compile::remove_opaque4_nodes(PhaseIterGVN &igvn) {
for (int i = opaque4_count(); i > 0; i--) {
Node* opaq = opaque4_node(i-1);
assert(opaq->Opcode() == Op_Opaque4, "Opaque4 only");
// With Opaque4 nodes, the expectation is that the test of input 1
// is always equal to the constant value of input 2. So we can
// remove the Opaque4 and replace it by input 2. In debug builds,
// leave the non constant test in instead to sanity check that it
// never fails (if it does, that subgraph was constructed so, at
// runtime, a Halt node is executed).
#ifdef ASSERT
igvn.replace_node(opaq, opaq->in(1));
#else
igvn.replace_node(opaq, opaq->in(2));
#endif
}
assert(opaque4_count() == 0, "should be empty");
}
@@ -1255,15 +1255,34 @@ ProjNode* PhaseIdealLoop::insert_initial_skeleton_predicate(IfNode* iff, IdealLo
Node* init, Node* limit, jint stride,
Node* rng, bool &overflow,
Deoptimization::DeoptReason reason) {
// First predicate for the initial value on first loop iteration
assert(proj->_con && predicate_proj->_con, "not a range check?");
Node* opaque_init = new Opaque1Node(C, init);
Node* opaque_init = new OpaqueLoopInitNode(C, init);
register_new_node(opaque_init, upper_bound_proj);
BoolNode* bol = rc_predicate(loop, upper_bound_proj, scale, offset, opaque_init, limit, stride, rng, (stride > 0) != (scale > 0), overflow);
Node* opaque_bol = new Opaque4Node(C, bol, _igvn.intcon(1)); // This will go away once loop opts are over
register_new_node(opaque_bol, upper_bound_proj);
ProjNode* new_proj = create_new_if_for_predicate(predicate_proj, NULL, reason, overflow ? Op_If : iff->Opcode());
_igvn.replace_input_of(new_proj->in(0), 1, opaque_bol);
assert(opaque_init->outcnt() > 0, "should be used");

// Second predicate for init + (current stride - initial stride)
// This is identical to the previous predicate initially but as
// unrolling proceeds current stride is updated.
Node* init_stride = loop->_head->as_CountedLoop()->stride();
Node* opaque_stride = new OpaqueLoopStrideNode(C, init_stride);
register_new_node(opaque_stride, new_proj);
Node* max_value = new SubINode(opaque_stride, init_stride);
register_new_node(max_value, new_proj);
max_value = new AddINode(opaque_init, max_value);
register_new_node(max_value, new_proj);
bol = rc_predicate(loop, new_proj, scale, offset, max_value, limit, stride, rng, (stride > 0) != (scale > 0), overflow);
opaque_bol = new Opaque4Node(C, bol, _igvn.intcon(1));
register_new_node(opaque_bol, new_proj);
new_proj = create_new_if_for_predicate(predicate_proj, NULL, reason, overflow ? Op_If : iff->Opcode());
_igvn.replace_input_of(new_proj->in(0), 1, opaque_bol);
assert(max_value->outcnt() > 0, "should be used");

return new_proj;
}

@@ -1141,7 +1141,7 @@ void PhaseIdealLoop::ensure_zero_trip_guard_proj(Node* node, bool is_main_loop)
// CastII/ConvI2L nodes cause some data paths to die. For consistency,
// the control paths must die too but the range checks were removed by
// predication. The range checks that we add here guarantee that they do.
void PhaseIdealLoop::copy_skeleton_predicates_to_main_loop_helper(Node* predicate, Node* start, Node* end,
void PhaseIdealLoop::copy_skeleton_predicates_to_main_loop_helper(Node* predicate, Node* init, Node* stride,
IdealLoopTree* outer_loop, LoopNode* outer_main_head,
uint dd_main_head, const uint idx_before_pre_post,
const uint idx_after_post_before_pre, Node* zero_trip_guard_proj_main,
@@ -1159,6 +1159,11 @@ void PhaseIdealLoop::copy_skeleton_predicates_to_main_loop_helper(Node* predicat
predicate = iff->in(0);
Node* current_proj = outer_main_head->in(LoopNode::EntryControl);
Node* prev_proj = current_proj;
Node* opaque_init = new OpaqueLoopInitNode(C, init);
register_new_node(opaque_init, outer_main_head->in(LoopNode::EntryControl));
Node* opaque_stride = new OpaqueLoopStrideNode(C, stride);
register_new_node(opaque_stride, outer_main_head->in(LoopNode::EntryControl));

while (predicate != NULL && predicate->is_Proj() && predicate->in(0)->is_If()) {
iff = predicate->in(0)->as_If();
uncommon_proj = iff->proj_out(1 - predicate->as_Proj()->_con);
@@ -1169,10 +1174,11 @@ void PhaseIdealLoop::copy_skeleton_predicates_to_main_loop_helper(Node* predicat
// Clone the skeleton predicate twice and initialize one with the initial
// value of the loop induction variable. Leave the other predicate
// to be initialized when increasing the stride during loop unrolling.
prev_proj = clone_skeleton_predicate(iff, start, predicate, uncommon_proj, current_proj, outer_loop, prev_proj);
assert(skeleton_predicate_has_opaque(prev_proj->in(0)->as_If()) == (start->Opcode() == Op_Opaque1), "");
prev_proj = clone_skeleton_predicate(iff, end, predicate, uncommon_proj, current_proj, outer_loop, prev_proj);
assert(skeleton_predicate_has_opaque(prev_proj->in(0)->as_If()) == (end->Opcode() == Op_Opaque1), "");
prev_proj = clone_skeleton_predicate(iff, opaque_init, NULL, predicate, uncommon_proj, current_proj, outer_loop, prev_proj);
assert(skeleton_predicate_has_opaque(prev_proj->in(0)->as_If()), "");

prev_proj = clone_skeleton_predicate(iff, init, stride, predicate, uncommon_proj, current_proj, outer_loop, prev_proj);
assert(!skeleton_predicate_has_opaque(prev_proj->in(0)->as_If()), "");

// Rewire any control inputs from the cloned skeleton predicates down to the main and post loop for data nodes that are part of the
// main loop (and were cloned to the pre and post loop).
@@ -1238,14 +1244,14 @@ bool PhaseIdealLoop::skeleton_predicate_has_opaque(IfNode* iff) {
}
continue;
}
if (op == Op_Opaque1) {
if (n->is_Opaque1()) {
return true;
}
}
return false;
}

Node* PhaseIdealLoop::clone_skeleton_predicate(Node* iff, Node* value, Node* predicate, Node* uncommon_proj,
Node* PhaseIdealLoop::clone_skeleton_predicate(Node* iff, Node* new_init, Node* new_stride, Node* predicate, Node* uncommon_proj,
Node* current_proj, IdealLoopTree* outer_loop, Node* prev_proj) {
Node_Stack to_clone(2);
to_clone.push(iff->in(1), 1);
@@ -1265,12 +1271,19 @@ Node* PhaseIdealLoop::clone_skeleton_predicate(Node* iff, Node* value, Node* pre
to_clone.push(m, 1);
continue;
}
if (op == Op_Opaque1) {
if (m->is_Opaque1()) {
if (n->_idx < current) {
n = n->clone();
register_new_node(n, current_proj);
}
if (op == Op_OpaqueLoopInit) {
n->set_req(i, new_init);
} else {
assert(op == Op_OpaqueLoopStride, "unexpected opaque node");
if (new_stride != NULL) {
n->set_req(i, new_stride);
}
}
n->set_req(i, value);
register_new_node(n, current_proj);
to_clone.set_node(n);
}
for (;;) {
@@ -1320,7 +1333,7 @@ Node* PhaseIdealLoop::clone_skeleton_predicate(Node* iff, Node* value, Node* pre
return proj;
}

void PhaseIdealLoop::copy_skeleton_predicates_to_main_loop(CountedLoopNode* pre_head, Node* start, Node* end,
void PhaseIdealLoop::copy_skeleton_predicates_to_main_loop(CountedLoopNode* pre_head, Node* init, Node* stride,
IdealLoopTree* outer_loop, LoopNode* outer_main_head,
uint dd_main_head, const uint idx_before_pre_post,
const uint idx_after_post_before_pre, Node* zero_trip_guard_proj_main,
@@ -1340,10 +1353,10 @@ void PhaseIdealLoop::copy_skeleton_predicates_to_main_loop(CountedLoopNode* pre_
}
}
predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate);
copy_skeleton_predicates_to_main_loop_helper(predicate, start, end, outer_loop, outer_main_head, dd_main_head,
copy_skeleton_predicates_to_main_loop_helper(predicate, init, stride, outer_loop, outer_main_head, dd_main_head,
idx_before_pre_post, idx_after_post_before_pre, zero_trip_guard_proj_main,
zero_trip_guard_proj_post, old_new);
copy_skeleton_predicates_to_main_loop_helper(profile_predicate, start, end, outer_loop, outer_main_head, dd_main_head,
copy_skeleton_predicates_to_main_loop_helper(profile_predicate, init, stride, outer_loop, outer_main_head, dd_main_head,
idx_before_pre_post, idx_after_post_before_pre, zero_trip_guard_proj_main,
zero_trip_guard_proj_post, old_new);
}
@@ -1494,10 +1507,8 @@ void PhaseIdealLoop::insert_pre_post_loops(IdealLoopTree *loop, Node_List &old_n
// CastII for the main loop:
Node* castii = cast_incr_before_loop(pre_incr, min_taken, main_head);
assert(castii != NULL, "no castII inserted");
Node* opaque_castii = new Opaque1Node(C, castii);
register_new_node(opaque_castii, outer_main_head->in(LoopNode::EntryControl));
assert(post_head->in(1)->is_IfProj(), "must be zero-trip guard If node projection of the post loop");
copy_skeleton_predicates_to_main_loop(pre_head, castii, opaque_castii, outer_loop, outer_main_head, dd_main_head,
copy_skeleton_predicates_to_main_loop(pre_head, castii, stride, outer_loop, outer_main_head, dd_main_head,
idx_before_pre_post, idx_after_post_before_pre, min_taken, post_head->in(1), old_new);

// Step B4: Shorten the pre-loop to run only 1 iteration (for now).
@@ -1790,6 +1801,13 @@ void PhaseIdealLoop::update_main_loop_skeleton_predicates(Node* ctrl, CountedLoo
Node* prev_proj = ctrl;
LoopNode* outer_loop_head = loop_head->skip_strip_mined();
IdealLoopTree* outer_loop = get_loop(outer_loop_head);

// Compute the value of the loop induction variable at the end of the
// first iteration of the unrolled loop: init + new_stride_con - init_inc
int new_stride_con = stride_con * 2;
Node* max_value = _igvn.intcon(new_stride_con);
set_ctrl(max_value, C->root());

while (entry != NULL && entry->is_Proj() && entry->in(0)->is_If()) {
IfNode* iff = entry->in(0)->as_If();
ProjNode* proj = iff->proj_out(1 - entry->as_Proj()->_con);
@@ -1805,18 +1823,8 @@ void PhaseIdealLoop::update_main_loop_skeleton_predicates(Node* ctrl, CountedLoo
// tell. Kill it in any case.
_igvn.replace_input_of(iff, 1, iff->in(1)->in(2));
} else {
// Add back the predicate for the value at the beginning of the first entry
prev_proj = clone_skeleton_predicate(iff, init, entry, proj, ctrl, outer_loop, prev_proj);
assert(!skeleton_predicate_has_opaque(prev_proj->in(0)->as_If()), "unexpected");
// Compute the value of the loop induction variable at the end of the
// first iteration of the unrolled loop: init + new_stride_con - init_inc
int init_inc = stride_con/loop_head->unrolled_count();
assert(init_inc != 0, "invalid loop increment");
int new_stride_con = stride_con * 2;
Node* max_value = _igvn.intcon(new_stride_con - init_inc);
max_value = new AddINode(init, max_value);
register_new_node(max_value, get_ctrl(iff->in(1)));
prev_proj = clone_skeleton_predicate(iff, max_value, entry, proj, ctrl, outer_loop, prev_proj);
// Add back predicates updated for the new stride.
prev_proj = clone_skeleton_predicate(iff, init, max_value, entry, proj, ctrl, outer_loop, prev_proj);
assert(!skeleton_predicate_has_opaque(prev_proj->in(0)->as_If()), "unexpected");
}
}
@@ -2670,22 +2678,26 @@ int PhaseIdealLoop::do_range_check(IdealLoopTree *loop, Node_List &old_new) {
// (0-offset)/scale could be outside of loop iterations range.
conditional_rc = true;
Node* init = cl->init_trip();
Node* opaque_init = new Opaque1Node(C, init);
Node* opaque_init = new OpaqueLoopInitNode(C, init);
register_new_node(opaque_init, predicate_proj);
// template predicate so it can be updated on next unrolling
predicate_proj = add_range_check_predicate(loop, cl, predicate_proj, scale_con, offset, limit, stride_con, opaque_init);
assert(skeleton_predicate_has_opaque(predicate_proj->in(0)->as_If()), "unexpected");

// predicate on first value of first iteration
predicate_proj = add_range_check_predicate(loop, cl, predicate_proj, scale_con, offset, limit, stride_con, init);
assert(!skeleton_predicate_has_opaque(predicate_proj->in(0)->as_If()), "unexpected");
int init_inc = stride_con/cl->unrolled_count();
assert(init_inc != 0, "invalid loop increment");
Node* max_value = _igvn.intcon(stride_con - init_inc);
max_value = new AddINode(init, max_value);

// template predicate so it can be updated on next unrolling
predicate_proj = add_range_check_predicate(loop, cl, predicate_proj, scale_con, offset, limit, stride_con, opaque_init);
assert(skeleton_predicate_has_opaque(predicate_proj->in(0)->as_If()), "unexpected");

Node* opaque_stride = new OpaqueLoopStrideNode(C, cl->stride());
register_new_node(opaque_stride, predicate_proj);
Node* max_value = new SubINode(opaque_stride, cl->stride());
register_new_node(max_value, predicate_proj);
max_value = new AddINode(opaque_init, max_value);
register_new_node(max_value, predicate_proj);
// predicate on last value of first iteration (in case unrolling has already happened)
predicate_proj = add_range_check_predicate(loop, cl, predicate_proj, scale_con, offset, limit, stride_con, max_value);
assert(!skeleton_predicate_has_opaque(predicate_proj->in(0)->as_If()), "unexpected");
assert(skeleton_predicate_has_opaque(predicate_proj->in(0)->as_If()), "unexpected");

} else {
if (PrintOpto) {
tty->print_cr("missed RCE opportunity");
@@ -2580,7 +2580,7 @@ uint IdealLoopTree::est_loop_clone_sz(uint factor) const {

uint const bc = 13;
uint const cc = 17;
uint const sz = _body.size() + (_body.size() + 7) / 8;
uint const sz = _body.size() + (_body.size() + 7) / 2;
uint estimate = factor * (sz + bc) + cc;

assert((estimate - cc) / factor == sz + bc, "overflow");
@@ -788,13 +788,13 @@ class PhaseIdealLoop : public PhaseTransform {
#ifdef ASSERT
void ensure_zero_trip_guard_proj(Node* node, bool is_main_loop);
#endif
void copy_skeleton_predicates_to_main_loop_helper(Node* predicate, Node* start, Node* end, IdealLoopTree* outer_loop, LoopNode* outer_main_head,
void copy_skeleton_predicates_to_main_loop_helper(Node* predicate, Node* init, Node* stride, IdealLoopTree* outer_loop, LoopNode* outer_main_head,
uint dd_main_head, const uint idx_before_pre_post, const uint idx_after_post_before_pre,
Node* zero_trip_guard_proj_main, Node* zero_trip_guard_proj_post, const Node_List &old_new);
void copy_skeleton_predicates_to_main_loop(CountedLoopNode* pre_head, Node* start, Node* end, IdealLoopTree* outer_loop, LoopNode* outer_main_head,
void copy_skeleton_predicates_to_main_loop(CountedLoopNode* pre_head, Node* init, Node* stride, IdealLoopTree* outer_loop, LoopNode* outer_main_head,
uint dd_main_head, const uint idx_before_pre_post, const uint idx_after_post_before_pre,
Node* zero_trip_guard_proj_main, Node* zero_trip_guard_proj_post, const Node_List &old_new);
Node* clone_skeleton_predicate(Node* iff, Node* value, Node* predicate, Node* uncommon_proj,
Node* clone_skeleton_predicate(Node* iff, Node* new_init, Node* new_stride, Node* predicate, Node* uncommon_proj,
Node* current_proj, IdealLoopTree* outer_loop, Node* prev_proj);
bool skeleton_predicate_has_opaque(IfNode* iff);
void update_main_loop_skeleton_predicates(Node* ctrl, CountedLoopNode* loop_head, Node* init, int stride_con);
@@ -952,29 +952,43 @@ void PhaseIdealLoop::try_move_store_after_loop(Node* n) {
Node *PhaseIdealLoop::split_if_with_blocks_pre( Node *n ) {
// Cloning these guys is unlikely to win
int n_op = n->Opcode();
if( n_op == Op_MergeMem ) return n;
if( n->is_Proj() ) return n;
if (n_op == Op_MergeMem) {
return n;
}
if (n->is_Proj()) {
return n;
}
// Do not clone-up CmpFXXX variations, as these are always
// followed by a CmpI
if( n->is_Cmp() ) return n;
if (n->is_Cmp()) {
return n;
}
// Attempt to use a conditional move instead of a phi/branch
if( ConditionalMoveLimit > 0 && n_op == Op_Region ) {
if (ConditionalMoveLimit > 0 && n_op == Op_Region) {
Node *cmov = conditional_move( n );
if( cmov ) return cmov;
if (cmov) {
return cmov;
}
}
if( n->is_CFG() || n->is_LoadStore() )
if (n->is_CFG() || n->is_LoadStore()) {
return n;
if( n_op == Op_Opaque1 || // Opaque nodes cannot be mod'd
n_op == Op_Opaque2 ) {
if( !C->major_progress() ) // If chance of no more loop opts...
}
if (n->is_Opaque1() || // Opaque nodes cannot be mod'd
n_op == Op_Opaque2) {
if (!C->major_progress()) { // If chance of no more loop opts...
_igvn._worklist.push(n); // maybe we'll remove them
}
return n;
}

if( n->is_Con() ) return n; // No cloning for Con nodes
if (n->is_Con()) {
return n; // No cloning for Con nodes
}

Node *n_ctrl = get_ctrl(n);
if( !n_ctrl ) return n; // Dead node
if (!n_ctrl) {
return n; // Dead node
}

Node* res = try_move_store_before_loop(n, n_ctrl);
if (res != NULL) {
@@ -2637,9 +2637,10 @@ void PhaseMacroExpand::eliminate_macro_nodes() {
break;
case Node::Class_SubTypeCheck:
break;
case Node::Class_Opaque1:
break;
default:
assert(n->Opcode() == Op_LoopLimit ||
n->Opcode() == Op_Opaque1 ||
n->Opcode() == Op_Opaque2 ||
n->Opcode() == Op_Opaque3 ||
BarrierSet::barrier_set()->barrier_set_c2()->is_gc_barrier_node(n),
@@ -2675,7 +2676,7 @@ bool PhaseMacroExpand::expand_macro_nodes() {
C->remove_macro_node(n);
_igvn._worklist.push(n);
success = true;
} else if (n->Opcode() == Op_Opaque1 || n->Opcode() == Op_Opaque2) {
} else if (n->is_Opaque1() || n->Opcode() == Op_Opaque2) {
_igvn.replace_node(n, n->in(1));
success = true;
#if INCLUDE_RTM_OPT

0 comments on commit b678149

Please sign in to comment.