Skip to content

Commit

Permalink
8253644: C2: assert(skeleton_predicate_has_opaque(iff)) failed: unexp…
Browse files Browse the repository at this point in the history 
…ected

Reviewed-by: roland, kvn
  • Loading branch information
@chhagedorn
chhagedorn committed Dec 8, 2020
1 parent 51ac376 commit 1d0adbb
Show file tree
Hide file tree
Showing 4 changed files with 312 additions and 52 deletions.
80 changes: 45 additions & 35 deletions src/hotspot/share/opto/loopPredicate.cpp
View file
Expand Up @@ -227,19 +227,17 @@ ProjNode* PhaseIdealLoop::clone_predicate_to_unswitched_loop(ProjNode* predicate
return new_predicate_proj;
}

// Clones skeleton predicates starting at 'old_predicate_proj' to
// 'new_predicate_proj' and rewires the control edges of data nodes in
// the loop from the old predicates to the new cloned predicates.
// Clones skeleton predicates starting at 'old_predicate_proj' by following its control inputs and rewires the control edges of in the loop from
// the old predicates to the new cloned predicates.
void PhaseIdealLoop::clone_skeleton_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, Deoptimization::DeoptReason reason,
ProjNode* old_predicate_proj, ProjNode* iffast, ProjNode* ifslow) {
ProjNode* old_predicate_proj, ProjNode* iffast_pred, ProjNode* ifslow_pred) {
IfNode* iff = old_predicate_proj->in(0)->as_If();
assert(iffast_pred->in(0)->is_If() && ifslow_pred->in(0)->is_If(), "sanity check");
ProjNode* uncommon_proj = iff->proj_out(1 - old_predicate_proj->as_Proj()->_con);
Node* rgn = uncommon_proj->unique_ctrl_out();
assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct");
assert(iff->in(1)->in(1)->Opcode() == Op_Opaque1, "unexpected predicate shape");
Node* predicate = iff->in(0);
Node* current_proj = old_predicate_proj;
Node* prev_proj = current_proj;
Unique_Node_List list;
while (predicate != NULL && predicate->is_Proj() && predicate->in(0)->is_If()) {
iff = predicate->in(0)->as_If();
Expand All @@ -256,45 +254,57 @@ void PhaseIdealLoop::clone_skeleton_predicates_to_unswitched_loop(IdealLoopTree*
}

Node_List to_process;
// Process in reverse order such that 'create_new_if_for_predicate' can be used and the original order is maintained
for (int i = list.size()-1; i >= 0; i--) {
// Process in reverse order such that 'create_new_if_for_predicate' can be used in 'clone_skeleton_predicate_for_unswitched_loops'
// and the original order is maintained.
for (int i = list.size() - 1; i >= 0; i--) {
predicate = list.at(i);
assert(predicate->in(0)->is_If(), "must be If node");
iff = predicate->in(0)->as_If();
assert(predicate->is_Proj() && predicate->as_Proj()->is_IfProj(), "predicate must be a projection of an if node");
IfProjNode* predicate_proj = predicate->as_IfProj();

// cloned_proj is the same type of projection as the original predicate projection (IfTrue or IfFalse)
ProjNode* fast_proj = create_new_if_for_predicate(iffast, NULL, reason, iff->Opcode(), predicate_proj->is_IfTrue());
ProjNode* slow_proj = create_new_if_for_predicate(ifslow, NULL, reason, iff->Opcode(), predicate_proj->is_IfTrue());
ProjNode* fast_proj = clone_skeleton_predicate_for_unswitched_loops(iff, predicate_proj, uncommon_proj, reason, iffast_pred, loop);
assert(skeleton_predicate_has_opaque(fast_proj->in(0)->as_If()), "must find skeleton predicate for fast loop");
ProjNode* slow_proj = clone_skeleton_predicate_for_unswitched_loops(iff, predicate_proj, uncommon_proj, reason, ifslow_pred, loop);
assert(skeleton_predicate_has_opaque(slow_proj->in(0)->as_If()), "must find skeleton predicate for slow loop");

// Replace bool input by input from original predicate
_igvn.replace_input_of(fast_proj->in(0), 1, iff->in(1));
_igvn.replace_input_of(slow_proj->in(0), 1, iff->in(1));

for (DUIterator i = predicate->outs(); predicate->has_out(i); i++) {
Node* fast_node = predicate->out(i);
// Update control dependent data nodes.
for (DUIterator j = predicate->outs(); predicate->has_out(j); j++) {
Node* fast_node = predicate->out(j);
if (loop->is_member(get_loop(ctrl_or_self(fast_node)))) {
assert(fast_node->in(0) == predicate, "only control edge");
Node* slow_node = old_new[fast_node->_idx];
assert(slow_node->in(0) == predicate, "only control edge");
_igvn.replace_input_of(fast_node, 0, fast_proj);
to_process.push(slow_node);
--i;
--j;
}
}
// Have to delay updates to the slow loop so uses of predicate are
// not modified while we iterate on them.
// Have to delay updates to the slow loop so uses of predicate are not modified while we iterate on them.
while (to_process.size() > 0) {
Node* slow_node = to_process.pop();
_igvn.replace_input_of(slow_node, 0, slow_proj);
}
}
}

// Clone a skeleton predicate for an unswitched loop. OpaqueLoopInit and OpaqueLoopStride nodes are cloned and uncommon
// traps are kept for the predicate (a Halt node is used later when creating pre/main/post loops and copying this cloned
// predicate again).
ProjNode* PhaseIdealLoop::clone_skeleton_predicate_for_unswitched_loops(Node* iff, ProjNode* predicate, Node* uncommon_proj,
Deoptimization::DeoptReason reason, ProjNode* output_proj,
IdealLoopTree* loop) {
Node* bol = clone_skeleton_predicate_bool(iff, NULL, NULL, predicate, uncommon_proj, output_proj, loop);
ProjNode* proj = create_new_if_for_predicate(output_proj, NULL, reason, iff->Opcode(), predicate->is_IfTrue());
_igvn.replace_input_of(proj->in(0), 1, bol);
_igvn.replace_input_of(output_proj->in(0), 0, proj);
set_idom(output_proj->in(0), proj, dom_depth(proj));
return proj;
}

//--------------------------clone_loop_predicates-----------------------
// Clone loop predicates to cloned loops when unswitching a loop.
void PhaseIdealLoop::clone_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, ProjNode*& iffast, ProjNode*& ifslow) {
void PhaseIdealLoop::clone_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, ProjNode*& iffast_pred, ProjNode*& ifslow_pred) {
LoopNode* head = loop->_head->as_Loop();
bool clone_limit_check = !head->is_CountedLoop();
Node* entry = head->skip_strip_mined()->in(LoopNode::EntryControl);
Expand All @@ -318,31 +328,31 @@ void PhaseIdealLoop::clone_predicates_to_unswitched_loop(IdealLoopTree* loop, co
}
if (predicate_proj != NULL) { // right pattern that can be used by loop predication
// clone predicate
iffast = clone_predicate_to_unswitched_loop(predicate_proj, iffast, Deoptimization::Reason_predicate);
ifslow = clone_predicate_to_unswitched_loop(predicate_proj, ifslow, Deoptimization::Reason_predicate);
clone_skeleton_predicates_to_unswitched_loop(loop, old_new, Deoptimization::Reason_predicate, predicate_proj, iffast, ifslow);
iffast_pred = clone_predicate_to_unswitched_loop(predicate_proj, iffast_pred, Deoptimization::Reason_predicate);
ifslow_pred = clone_predicate_to_unswitched_loop(predicate_proj, ifslow_pred, Deoptimization::Reason_predicate);
clone_skeleton_predicates_to_unswitched_loop(loop, old_new, Deoptimization::Reason_predicate, predicate_proj, iffast_pred, ifslow_pred);

check_created_predicate_for_unswitching(iffast);
check_created_predicate_for_unswitching(ifslow);
check_created_predicate_for_unswitching(iffast_pred);
check_created_predicate_for_unswitching(ifslow_pred);
}
if (profile_predicate_proj != NULL) { // right pattern that can be used by loop predication
// clone predicate
iffast = clone_predicate_to_unswitched_loop(profile_predicate_proj, iffast, Deoptimization::Reason_profile_predicate);
ifslow = clone_predicate_to_unswitched_loop(profile_predicate_proj, ifslow, Deoptimization::Reason_profile_predicate);
clone_skeleton_predicates_to_unswitched_loop(loop, old_new, Deoptimization::Reason_profile_predicate, profile_predicate_proj, iffast, ifslow);
iffast_pred = clone_predicate_to_unswitched_loop(profile_predicate_proj, iffast_pred, Deoptimization::Reason_profile_predicate);
ifslow_pred = clone_predicate_to_unswitched_loop(profile_predicate_proj, ifslow_pred, Deoptimization::Reason_profile_predicate);
clone_skeleton_predicates_to_unswitched_loop(loop, old_new, Deoptimization::Reason_profile_predicate, profile_predicate_proj, iffast_pred, ifslow_pred);

check_created_predicate_for_unswitching(iffast);
check_created_predicate_for_unswitching(ifslow);
check_created_predicate_for_unswitching(iffast_pred);
check_created_predicate_for_unswitching(ifslow_pred);
}
if (limit_check_proj != NULL && clone_limit_check) {
// Clone loop limit check last to insert it before loop.
// Don't clone a limit check which was already finalized
// for this counted loop (only one limit check is needed).
iffast = clone_predicate_to_unswitched_loop(limit_check_proj, iffast, Deoptimization::Reason_loop_limit_check);
ifslow = clone_predicate_to_unswitched_loop(limit_check_proj, ifslow, Deoptimization::Reason_loop_limit_check);
iffast_pred = clone_predicate_to_unswitched_loop(limit_check_proj, iffast_pred, Deoptimization::Reason_loop_limit_check);
ifslow_pred = clone_predicate_to_unswitched_loop(limit_check_proj, ifslow_pred, Deoptimization::Reason_loop_limit_check);

check_created_predicate_for_unswitching(iffast);
check_created_predicate_for_unswitching(ifslow);
check_created_predicate_for_unswitching(iffast_pred);
check_created_predicate_for_unswitching(ifslow_pred);
}
}

Expand Down
47 changes: 34 additions & 13 deletions src/hotspot/share/opto/loopTransform.cpp
View file
Expand Up @@ -1256,10 +1256,10 @@ void PhaseIdealLoop::copy_skeleton_predicates_to_main_loop_helper(Node* predicat
// Clone the skeleton predicate twice and initialize one with the initial
// value of the loop induction variable. Leave the other predicate
// to be initialized when increasing the stride during loop unrolling.
prev_proj = clone_skeleton_predicate(iff, opaque_init, NULL, predicate, uncommon_proj, current_proj, outer_loop, prev_proj);
prev_proj = clone_skeleton_predicate_for_main_loop(iff, opaque_init, NULL, predicate, uncommon_proj, current_proj, outer_loop, prev_proj);
assert(skeleton_predicate_has_opaque(prev_proj->in(0)->as_If()), "");

prev_proj = clone_skeleton_predicate(iff, init, stride, predicate, uncommon_proj, current_proj, outer_loop, prev_proj);
prev_proj = clone_skeleton_predicate_for_main_loop(iff, init, stride, predicate, uncommon_proj, current_proj, outer_loop, prev_proj);
assert(!skeleton_predicate_has_opaque(prev_proj->in(0)->as_If()), "");

// Rewire any control inputs from the cloned skeleton predicates down to the main and post loop for data nodes that are part of the
Expand Down Expand Up @@ -1333,12 +1333,17 @@ bool PhaseIdealLoop::skeleton_predicate_has_opaque(IfNode* iff) {
return false;
}

Node* PhaseIdealLoop::clone_skeleton_predicate(Node* iff, Node* new_init, Node* new_stride, Node* predicate, Node* uncommon_proj,
Node* current_proj, IdealLoopTree* outer_loop, Node* prev_proj) {
// Clone the skeleton predicate bool for a main or unswitched loop:
// Main loop: Set new_init and new_stride nodes as new inputs.
// Unswitched loop: new_init and new_stride are both NULL. Clone OpaqueLoopInit and OpaqueLoopStride instead.
Node* PhaseIdealLoop::clone_skeleton_predicate_bool(Node* iff, Node* new_init, Node* new_stride, Node* predicate, Node* uncommon_proj,
Node* control, IdealLoopTree* outer_loop) {
Node_Stack to_clone(2);
to_clone.push(iff->in(1), 1);
uint current = C->unique();
Node* result = NULL;
bool is_unswitched_loop = new_init == NULL && new_stride == NULL;
assert(new_init != NULL || is_unswitched_loop, "new_init must be set when new_stride is non-null");
// Look for the opaque node to replace with the new value
// and clone everything in between. We keep the Opaque4 node
// so the duplicated predicates are eliminated once loop
Expand All @@ -1350,25 +1355,33 @@ Node* PhaseIdealLoop::clone_skeleton_predicate(Node* iff, Node* new_init, Node*
Node* m = n->in(i);
int op = m->Opcode();
if (skeleton_follow_inputs(m, op)) {
to_clone.push(m, 1);
continue;
to_clone.push(m, 1);
continue;
}
if (m->is_Opaque1()) {
if (n->_idx < current) {
n = n->clone();
register_new_node(n, current_proj);
register_new_node(n, control);
}
if (op == Op_OpaqueLoopInit) {
if (is_unswitched_loop && m->_idx < current && new_init == NULL) {
new_init = m->clone();
register_new_node(new_init, control);
}
n->set_req(i, new_init);
} else {
assert(op == Op_OpaqueLoopStride, "unexpected opaque node");
if (is_unswitched_loop && m->_idx < current && new_stride == NULL) {
new_stride = m->clone();
register_new_node(new_stride, control);
}
if (new_stride != NULL) {
n->set_req(i, new_stride);
}
}
to_clone.set_node(n);
}
for (;;) {
while (true) {
Node* cur = to_clone.node();
uint j = to_clone.index();
if (j+1 < cur->req()) {
Expand All @@ -1386,29 +1399,37 @@ Node* PhaseIdealLoop::clone_skeleton_predicate(Node* iff, Node* new_init, Node*
assert(cur->_idx >= current || next->in(j)->Opcode() == Op_Opaque1, "new node or Opaque1 being replaced");
if (next->_idx < current) {
next = next->clone();
register_new_node(next, current_proj);
register_new_node(next, control);
to_clone.set_node(next);
}
next->set_req(j, cur);
}
}
} while (result == NULL);
assert(result->_idx >= current, "new node expected");
assert(!is_unswitched_loop || new_init != NULL, "new_init must always be found and cloned");
return result;
}

// Clone a skeleton predicate for the main loop. new_init and new_stride are set as new inputs. Since the predicates cannot fail at runtime,
// Halt nodes are inserted instead of uncommon traps.
Node* PhaseIdealLoop::clone_skeleton_predicate_for_main_loop(Node* iff, Node* new_init, Node* new_stride, Node* predicate, Node* uncommon_proj,
Node* control, IdealLoopTree* outer_loop, Node* input_proj) {
Node* result = clone_skeleton_predicate_bool(iff, new_init, new_stride, predicate, uncommon_proj, control, outer_loop);
Node* proj = predicate->clone();
Node* other_proj = uncommon_proj->clone();
Node* new_iff = iff->clone();
new_iff->set_req(1, result);
proj->set_req(0, new_iff);
other_proj->set_req(0, new_iff);
Node *frame = new ParmNode(C->start(), TypeFunc::FramePtr);
Node* frame = new ParmNode(C->start(), TypeFunc::FramePtr);
register_new_node(frame, C->start());
// It's impossible for the predicate to fail at runtime. Use an Halt node.
Node* halt = new HaltNode(other_proj, frame, "duplicated predicate failed which is impossible");
C->root()->add_req(halt);
new_iff->set_req(0, prev_proj);
new_iff->set_req(0, input_proj);

register_control(new_iff, outer_loop->_parent, prev_proj);
register_control(new_iff, outer_loop->_parent, input_proj);
register_control(proj, outer_loop->_parent, new_iff);
register_control(other_proj, _ltree_root, new_iff);
register_control(halt, _ltree_root, other_proj);
Expand Down Expand Up @@ -1904,7 +1925,7 @@ void PhaseIdealLoop::update_main_loop_skeleton_predicates(Node* ctrl, CountedLoo
_igvn.replace_input_of(iff, 1, iff->in(1)->in(2));
} else {
// Add back predicates updated for the new stride.
prev_proj = clone_skeleton_predicate(iff, init, max_value, entry, proj, ctrl, outer_loop, prev_proj);
prev_proj = clone_skeleton_predicate_for_main_loop(iff, init, max_value, entry, proj, ctrl, outer_loop, prev_proj);
assert(!skeleton_predicate_has_opaque(prev_proj->in(0)->as_If()), "unexpected");
}
}
Expand Down
12 changes: 8 additions & 4 deletions src/hotspot/share/opto/loopnode.hpp
View file
Expand Up @@ -910,8 +910,10 @@ class PhaseIdealLoop : public PhaseTransform {
void copy_skeleton_predicates_to_main_loop(CountedLoopNode* pre_head, Node* init, Node* stride, IdealLoopTree* outer_loop, LoopNode* outer_main_head,
uint dd_main_head, const uint idx_before_pre_post, const uint idx_after_post_before_pre,
Node* zero_trip_guard_proj_main, Node* zero_trip_guard_proj_post, const Node_List &old_new);
Node* clone_skeleton_predicate(Node* iff, Node* new_init, Node* new_stride, Node* predicate, Node* uncommon_proj,
Node* current_proj, IdealLoopTree* outer_loop, Node* prev_proj);
Node* clone_skeleton_predicate_for_main_loop(Node* iff, Node* new_init, Node* new_stride, Node* predicate, Node* uncommon_proj, Node* control,
IdealLoopTree* outer_loop, Node* input_proj);
Node* clone_skeleton_predicate_bool(Node* iff, Node* new_init, Node* new_stride, Node* predicate, Node* uncommon_proj, Node* control,
IdealLoopTree* outer_loop);
bool skeleton_predicate_has_opaque(IfNode* iff);
void update_main_loop_skeleton_predicates(Node* ctrl, CountedLoopNode* loop_head, Node* init, int stride_con);
void insert_loop_limit_check(ProjNode* limit_check_proj, Node* cmp_limit, Node* bol);
Expand Down Expand Up @@ -1557,10 +1559,12 @@ class PhaseIdealLoop : public PhaseTransform {
}

// Clone loop predicates to slow and fast loop when unswitching a loop
void clone_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, ProjNode*& iffast, ProjNode*& ifslow);
void clone_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, ProjNode*& iffast_pred, ProjNode*& ifslow_pred);
ProjNode* clone_predicate_to_unswitched_loop(ProjNode* predicate_proj, Node* new_entry, Deoptimization::DeoptReason reason);
void clone_skeleton_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, Deoptimization::DeoptReason reason,
ProjNode* old_predicate_proj, ProjNode* iffast, ProjNode* ifslow);
ProjNode* old_predicate_proj, ProjNode* iffast_pred, ProjNode* ifslow_pred);
ProjNode* clone_skeleton_predicate_for_unswitched_loops(Node* iff, ProjNode* predicate, Node* uncommon_proj, Deoptimization::DeoptReason reason,
ProjNode* output_proj, IdealLoopTree* loop);
void check_created_predicate_for_unswitching(const Node* new_entry) const PRODUCT_RETURN;

bool _created_loop_node;
Expand Down

1 comment on commit 1d0adbb

@openjdk-notifier
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Review

Issues

Please sign in to comment.