Skip to content
Permalink
Browse files
8271954: C2: assert(false) failed: Bad graph detected in build_loop_late
Backport-of: c86e24d4be1e1a26a2a8323ef7ddbab6326bbf3a
  • Loading branch information
TobiHartmann committed Sep 29, 2021
1 parent 056eca0 commit da77acb8f2eb2e2841b0c7137878798fae4bfa62
Showing 3 changed files with 442 additions and 14 deletions.
@@ -107,8 +107,9 @@ void PhaseIdealLoop::register_control(Node* n, IdealLoopTree *loop, Node* pred,
// Otherwise, the continuation projection is set up to be the false
// projection. This code is also used to clone predicates to cloned loops.
ProjNode* PhaseIdealLoop::create_new_if_for_predicate(ProjNode* cont_proj, Node* new_entry,
Deoptimization::DeoptReason reason,
int opcode, bool if_cont_is_true_proj) {
Deoptimization::DeoptReason reason, int opcode,
bool if_cont_is_true_proj, Node_List* old_new,
UnswitchingAction unswitching_action) {
assert(cont_proj->is_uncommon_trap_if_pattern(reason), "must be a uct if pattern!");
IfNode* iff = cont_proj->in(0)->as_If();

@@ -193,11 +194,40 @@ ProjNode* PhaseIdealLoop::create_new_if_for_predicate(ProjNode* cont_proj, Node*
if (use->is_Phi() && use->outcnt() > 0) {
assert(use->in(0) == rgn, "");
_igvn.rehash_node_delayed(use);
use->add_req(use->in(proj_index));
Node* phi_input = use->in(proj_index);
if (unswitching_action == UnswitchingAction::FastLoopCloning
&& !phi_input->is_CFG() && !phi_input->is_Phi() && get_ctrl(phi_input) == uncommon_proj) {
// There are some control dependent nodes on the uncommon projection and we are currently copying predicates
// to the fast loop in loop unswitching (first step, slow loop is processed afterwards). For the fast loop,
// we need to clone all the data nodes in the chain from the phi ('use') up until the node whose control input
// is the uncommon_proj. The slow loop can reuse the old data nodes and thus only needs to update the control
// input to the uncommon_proj (done on the next invocation of this method when 'unswitch_is_slow_loop' is true.
assert(LoopUnswitching, "sanity check");
phi_input = clone_data_nodes_for_fast_loop(phi_input, uncommon_proj, if_uct, old_new);
} else if (unswitching_action == UnswitchingAction::SlowLoopRewiring) {
// Replace phi input for the old predicate path with TOP as the predicate is dying anyways. This avoids the need
// to clone the data nodes again for the slow loop.
assert(LoopUnswitching, "sanity check");
_igvn.replace_input_of(use, proj_index, C->top());
}
use->add_req(phi_input);
has_phi = true;
}
}
assert(!has_phi || rgn->req() > 3, "no phis when region is created");
if (unswitching_action == UnswitchingAction::SlowLoopRewiring) {
// Rewire the control dependent data nodes for the slow loop from the old to the new uncommon projection.
assert(uncommon_proj->outcnt() > 1 && old_new == NULL, "sanity");
for (DUIterator_Fast jmax, j = uncommon_proj->fast_outs(jmax); j < jmax; j++) {
Node* data = uncommon_proj->fast_out(j);
if (!data->is_CFG()) {
_igvn.replace_input_of(data, 0, if_uct);
set_ctrl(data, if_uct);
--j;
--jmax;
}
}
}

if (new_entry == NULL) {
// Attach if_cont to iff
@@ -209,9 +239,70 @@ ProjNode* PhaseIdealLoop::create_new_if_for_predicate(ProjNode* cont_proj, Node*
return if_cont->as_Proj();
}

// Clone data nodes for the fast loop while creating a new If with create_new_if_for_predicate. Returns the node which is
// used for the uncommon trap phi input.
Node* PhaseIdealLoop::clone_data_nodes_for_fast_loop(Node* phi_input, ProjNode* uncommon_proj, Node* if_uct, Node_List* old_new) {
// Step 1: Clone all nodes on the data chain but do not rewire anything, yet. Keep track of the cloned nodes
// by using the old_new mapping. This mapping is then used in step 2 to rewire the cloned nodes accordingly.
DEBUG_ONLY(uint last_idx = C->unique();)
Unique_Node_List list;
list.push(phi_input);
for (uint j = 0; j < list.size(); j++) {
Node* next = list.at(j);
Node* clone = next->clone();
_igvn.register_new_node_with_optimizer(clone);
old_new->map(next->_idx, clone);
for (uint k = 1; k < next->req(); k++) {
Node* in = next->in(k);
if (!in->is_Phi() && get_ctrl(in) == uncommon_proj) {
list.push(in);
}
}
}

// Step 2: All nodes are cloned. Rewire them by using the old_new mapping.
for (uint j = 0; j < list.size(); j++) {
Node* next = list.at(j);
Node* clone = old_new->at(next->_idx);
assert(clone != NULL && clone->_idx >= last_idx, "must exist and be a proper clone");
if (next->in(0) == uncommon_proj) {
// All data nodes with a control input to the uncommon projection in the chain need to be rewired to the new uncommon
// projection (could not only be the last data node in the chain but also, for example, a DivNode within the chain).
_igvn.replace_input_of(clone, 0, if_uct);
set_ctrl(clone, if_uct);
}

// Rewire the inputs of the cloned nodes to the old nodes to the new clones.
for (uint k = 1; k < next->req(); k++) {
Node* in = next->in(k);
if (!in->is_Phi()) {
assert(!in->is_CFG(), "must be data node");
Node* in_clone = old_new->at(in->_idx);
if (in_clone != NULL) {
assert(in_clone->_idx >= last_idx, "must be a valid clone");
_igvn.replace_input_of(clone, k, in_clone);
set_ctrl(clone, if_uct);
}
}
}
}
Node* clone_phi_input = old_new->at(phi_input->_idx);
assert(clone_phi_input != NULL && clone_phi_input->_idx >= last_idx, "must exist and be a proper clone");
return clone_phi_input;
}
//--------------------------clone_predicate-----------------------
ProjNode* PhaseIdealLoop::clone_predicate_to_unswitched_loop(ProjNode* predicate_proj, Node* new_entry, Deoptimization::DeoptReason reason) {
ProjNode* new_predicate_proj = create_new_if_for_predicate(predicate_proj, new_entry, reason, Op_If);
ProjNode* PhaseIdealLoop::clone_predicate_to_unswitched_loop(ProjNode* predicate_proj, Node* new_entry,
Deoptimization::DeoptReason reason, Node_List* old_new) {
UnswitchingAction unswitching_action;
if (predicate_proj->other_if_proj()->outcnt() > 1) {
// There are some data dependencies that need to be taken care of when cloning a predicate.
unswitching_action = old_new == NULL ? UnswitchingAction::SlowLoopRewiring : UnswitchingAction::FastLoopCloning;
} else {
unswitching_action = UnswitchingAction::None;
}

ProjNode* new_predicate_proj = create_new_if_for_predicate(predicate_proj, new_entry, reason, Op_If,
true, old_new, unswitching_action);
IfNode* iff = new_predicate_proj->in(0)->as_If();
Node* ctrl = iff->in(0);

@@ -319,7 +410,7 @@ ProjNode* PhaseIdealLoop::clone_skeleton_predicate_for_unswitched_loops(Node* if

//--------------------------clone_loop_predicates-----------------------
// Clone loop predicates to cloned loops when unswitching a loop.
void PhaseIdealLoop::clone_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, ProjNode*& iffast_pred, ProjNode*& ifslow_pred) {
void PhaseIdealLoop::clone_predicates_to_unswitched_loop(IdealLoopTree* loop, Node_List& old_new, ProjNode*& iffast_pred, ProjNode*& ifslow_pred) {
LoopNode* head = loop->_head->as_Loop();
bool clone_limit_check = !head->is_CountedLoop();
Node* entry = head->skip_strip_mined()->in(LoopNode::EntryControl);
@@ -343,7 +434,7 @@ void PhaseIdealLoop::clone_predicates_to_unswitched_loop(IdealLoopTree* loop, co
}
if (predicate_proj != NULL) { // right pattern that can be used by loop predication
// clone predicate
iffast_pred = clone_predicate_to_unswitched_loop(predicate_proj, iffast_pred, Deoptimization::Reason_predicate);
iffast_pred = clone_predicate_to_unswitched_loop(predicate_proj, iffast_pred, Deoptimization::Reason_predicate, &old_new);
ifslow_pred = clone_predicate_to_unswitched_loop(predicate_proj, ifslow_pred, Deoptimization::Reason_predicate);
clone_skeleton_predicates_to_unswitched_loop(loop, old_new, Deoptimization::Reason_predicate, predicate_proj, iffast_pred, ifslow_pred);

@@ -352,7 +443,7 @@ void PhaseIdealLoop::clone_predicates_to_unswitched_loop(IdealLoopTree* loop, co
}
if (profile_predicate_proj != NULL) { // right pattern that can be used by loop predication
// clone predicate
iffast_pred = clone_predicate_to_unswitched_loop(profile_predicate_proj, iffast_pred, Deoptimization::Reason_profile_predicate);
iffast_pred = clone_predicate_to_unswitched_loop(profile_predicate_proj, iffast_pred, Deoptimization::Reason_profile_predicate, &old_new);
ifslow_pred = clone_predicate_to_unswitched_loop(profile_predicate_proj, ifslow_pred, Deoptimization::Reason_profile_predicate);
clone_skeleton_predicates_to_unswitched_loop(loop, old_new, Deoptimization::Reason_profile_predicate, profile_predicate_proj, iffast_pred, ifslow_pred);

@@ -363,7 +454,7 @@ void PhaseIdealLoop::clone_predicates_to_unswitched_loop(IdealLoopTree* loop, co
// Clone loop limit check last to insert it before loop.
// Don't clone a limit check which was already finalized
// for this counted loop (only one limit check is needed).
iffast_pred = clone_predicate_to_unswitched_loop(limit_check_proj, iffast_pred, Deoptimization::Reason_loop_limit_check);
iffast_pred = clone_predicate_to_unswitched_loop(limit_check_proj, iffast_pred, Deoptimization::Reason_loop_limit_check, &old_new);
ifslow_pred = clone_predicate_to_unswitched_loop(limit_check_proj, ifslow_pred, Deoptimization::Reason_loop_limit_check);

check_created_predicate_for_unswitching(iffast_pred);
@@ -372,7 +463,7 @@ void PhaseIdealLoop::clone_predicates_to_unswitched_loop(IdealLoopTree* loop, co
}

#ifndef PRODUCT
void PhaseIdealLoop::check_created_predicate_for_unswitching(const Node* new_entry) const {
void PhaseIdealLoop::check_created_predicate_for_unswitching(const Node* new_entry) {
assert(new_entry != NULL, "IfTrue or IfFalse after clone predicate");
if (TraceLoopPredicate) {
tty->print("Loop Predicate cloned: ");
@@ -1275,9 +1275,20 @@ class PhaseIdealLoop : public PhaseTransform {
// Return true if exp is a scaled induction var plus (or minus) constant
bool is_scaled_iv_plus_offset(Node* exp, Node* iv, int* p_scale, Node** p_offset, int depth = 0);

// Enum to determine the action to be performed in create_new_if_for_predicate() when processing phis of UCT regions.
enum class UnswitchingAction {
None, // No special action.
FastLoopCloning, // Need to clone nodes for the fast loop.
SlowLoopRewiring // Need to rewire nodes for the slow loop.
};

// Create a new if above the uncommon_trap_if_pattern for the predicate to be promoted
ProjNode* create_new_if_for_predicate(ProjNode* cont_proj, Node* new_entry, Deoptimization::DeoptReason reason,
int opcode, bool if_cont_is_true_proj = true);
int opcode, bool if_cont_is_true_proj = true, Node_List* old_new = NULL,
UnswitchingAction unswitching_action = UnswitchingAction::None);

// Clone data nodes for the fast loop while creating a new If with create_new_if_for_predicate.
Node* clone_data_nodes_for_fast_loop(Node* phi_input, ProjNode* uncommon_proj, Node* if_uct, Node_List* old_new);

void register_control(Node* n, IdealLoopTree *loop, Node* pred, bool update_body = true);

@@ -1563,13 +1574,14 @@ class PhaseIdealLoop : public PhaseTransform {
}

// Clone loop predicates to slow and fast loop when unswitching a loop
void clone_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, ProjNode*& iffast_pred, ProjNode*& ifslow_pred);
ProjNode* clone_predicate_to_unswitched_loop(ProjNode* predicate_proj, Node* new_entry, Deoptimization::DeoptReason reason);
void clone_predicates_to_unswitched_loop(IdealLoopTree* loop, Node_List& old_new, ProjNode*& iffast_pred, ProjNode*& ifslow_pred);
ProjNode* clone_predicate_to_unswitched_loop(ProjNode* predicate_proj, Node* new_entry, Deoptimization::DeoptReason reason,
Node_List* old_new = NULL);
void clone_skeleton_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, Deoptimization::DeoptReason reason,
ProjNode* old_predicate_proj, ProjNode* iffast_pred, ProjNode* ifslow_pred);
ProjNode* clone_skeleton_predicate_for_unswitched_loops(Node* iff, ProjNode* predicate, Node* uncommon_proj, Deoptimization::DeoptReason reason,
ProjNode* output_proj, IdealLoopTree* loop);
void check_created_predicate_for_unswitching(const Node* new_entry) const PRODUCT_RETURN;
static void check_created_predicate_for_unswitching(const Node* new_entry) PRODUCT_RETURN;

bool _created_loop_node;
#ifdef ASSERT

1 comment on commit da77acb

@openjdk-notifier
Copy link

@openjdk-notifier openjdk-notifier bot commented on da77acb Sep 29, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please sign in to comment.