Skip to content

Commit

Permalink
8283466: C2: missing skeleton predicates in peeled loop
Browse files Browse the repository at this point in the history 
Reviewed-by: roland, chagedorn
  • Loading branch information
@eme64
eme64 committed Jun 2, 2022
1 parent a82417f commit 199832a
Show file tree
Hide file tree
Showing 4 changed files with 248 additions and 37 deletions.
55 changes: 39 additions & 16 deletions src/hotspot/share/opto/loopPredicate.cpp
View file
Expand Up @@ -491,24 +491,25 @@ Node* PhaseIdealLoop::skip_loop_predicates(Node* entry) {
}

Node* PhaseIdealLoop::skip_all_loop_predicates(Node* entry) {
Node* predicate = NULL;
predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check);
if (predicate != NULL) {
entry = skip_loop_predicates(entry);
}
if (UseProfiledLoopPredicate) {
predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate);
if (predicate != NULL) { // right pattern that can be used by loop predication
entry = skip_loop_predicates(entry);
}
}
if (UseLoopPredicate) {
predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate);
if (predicate != NULL) { // right pattern that can be used by loop predication
entry = skip_loop_predicates(entry);
Predicates predicates(entry);
return predicates.skip_all();
}

//--------------------------next_predicate---------------------------------
// Find next related predicate, useful for iterating over all related predicates
ProjNode* PhaseIdealLoop::next_predicate(ProjNode* predicate) {
IfNode* iff = predicate->in(0)->as_If();
ProjNode* uncommon_proj = iff->proj_out(1 - predicate->_con);
Node* rgn = uncommon_proj->unique_ctrl_out();
assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct");
Node* next = iff->in(0);
if (next != nullptr && next->is_Proj() && next->in(0)->is_If()) {
uncommon_proj = next->in(0)->as_If()->proj_out(1 - next->as_Proj()->_con);
if (uncommon_proj->unique_ctrl_out() == rgn) { // lead into same region
return next->as_Proj();
}
}
return entry;
return nullptr;
}

//--------------------------find_predicate_insertion_point-------------------
Expand All @@ -522,6 +523,28 @@ ProjNode* PhaseIdealLoop::find_predicate_insertion_point(Node* start_c, Deoptimi
return NULL;
}

//--------------------------Predicates::Predicates--------------------------
// given loop entry, find all predicates above loop
PhaseIdealLoop::Predicates::Predicates(Node* entry) {
_loop_limit_check = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check);
if (_loop_limit_check != nullptr) {
entry = skip_loop_predicates(entry);
}
if (UseProfiledLoopPredicate) {
_profile_predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate);
if (_profile_predicate != nullptr) {
entry = skip_loop_predicates(entry);
}
}
if (UseLoopPredicate) {
_predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate);
if (_predicate != nullptr) {
entry = skip_loop_predicates(entry);
}
}
_entry_to_all_predicates = entry;
}

//--------------------------find_predicate------------------------------------
// Find a predicate
Node* PhaseIdealLoop::find_predicate(Node* entry) {
Expand Down
109 changes: 90 additions & 19 deletions src/hotspot/share/opto/loopTransform.cpp
View file
Expand Up @@ -614,13 +614,16 @@ void PhaseIdealLoop::peeled_dom_test_elim(IdealLoopTree* loop, Node_List& old_ne
// after peel and predicate move
//
// stmt1
// |
// v
// loop predicate
// /
// /
// clone / orig
// /
// / +----------+
// / | |
// / loop predicate |
// / | |
// / | |
// v v |
// TOP-->loop clone loop<----+ |
Expand All @@ -647,7 +650,10 @@ void PhaseIdealLoop::peeled_dom_test_elim(IdealLoopTree* loop, Node_List& old_ne
//
// final graph
//
// stmt1
// stmt1
// |
// v
// loop predicate
// |
// v
// stmt2 clone
Expand All @@ -660,7 +666,7 @@ void PhaseIdealLoop::peeled_dom_test_elim(IdealLoopTree* loop, Node_List& old_ne
// false true
// | |
// | v
// | loop predicate
// | initialized skeleton predicates
// | |
// | v
// | loop<----+
Expand Down Expand Up @@ -714,7 +720,9 @@ void PhaseIdealLoop::do_peeling(IdealLoopTree *loop, Node_List &old_new) {

// Step 1: Clone the loop body. The clone becomes the peeled iteration.
// The pre-loop illegally has 2 control users (old & new loops).
clone_loop(loop, old_new, dom_depth(head->skip_strip_mined()), ControlAroundStripMined);
const uint idx_before_clone = Compile::current()->unique();
LoopNode* outer_loop_head = head->skip_strip_mined();
clone_loop(loop, old_new, dom_depth(outer_loop_head), ControlAroundStripMined);

// Step 2: Make the old-loop fall-in edges point to the peeled iteration.
// Do this by making the old-loop fall-in edges act as if they came
Expand All @@ -723,8 +731,8 @@ void PhaseIdealLoop::do_peeling(IdealLoopTree *loop, Node_List &old_new) {
// the pre-loop with only 1 user (the new peeled iteration), but the
// peeled-loop backedge has 2 users.
Node* new_entry = old_new[head->in(LoopNode::LoopBackControl)->_idx];
_igvn.hash_delete(head->skip_strip_mined());
head->skip_strip_mined()->set_req(LoopNode::EntryControl, new_entry);
_igvn.hash_delete(outer_loop_head);
outer_loop_head->set_req(LoopNode::EntryControl, new_entry);
for (DUIterator_Fast jmax, j = head->fast_outs(jmax); j < jmax; j++) {
Node* old = head->fast_out(j);
if (old->in(0) == loop->_head && old->req() == 3 && old->is_Phi()) {
Expand Down Expand Up @@ -753,16 +761,33 @@ void PhaseIdealLoop::do_peeling(IdealLoopTree *loop, Node_List &old_new) {

// Step 4: Correct dom-depth info. Set to loop-head depth.

int dd = dom_depth(head->skip_strip_mined());
set_idom(head->skip_strip_mined(), head->skip_strip_mined()->in(LoopNode::EntryControl), dd);
int dd_outer_loop_head = dom_depth(outer_loop_head);
set_idom(outer_loop_head, outer_loop_head->in(LoopNode::EntryControl), dd_outer_loop_head);
for (uint j3 = 0; j3 < loop->_body.size(); j3++) {
Node *old = loop->_body.at(j3);
Node *nnn = old_new[old->_idx];
if (!has_ctrl(nnn)) {
set_idom(nnn, idom(nnn), dd-1);
set_idom(nnn, idom(nnn), dd_outer_loop_head-1);
}
}

// Step 5: skeleton_predicates instantiation
if (counted_loop && UseLoopPredicate) {
CountedLoopNode *cl_head = head->as_CountedLoop();
Node* init = cl_head->init_trip();
Node* stride = cl_head->stride();
IdealLoopTree* outer_loop = get_loop(outer_loop_head);
Predicates predicates(new_head->in(LoopNode::EntryControl));
initialize_skeleton_predicates_for_peeled_loop(predicates.predicate(),
outer_loop_head, dd_outer_loop_head,
init, stride, outer_loop,
idx_before_clone, old_new);
initialize_skeleton_predicates_for_peeled_loop(predicates.profile_predicate(),
outer_loop_head, dd_outer_loop_head,
init, stride, outer_loop,
idx_before_clone, old_new);
}

// Now force out all loop-invariant dominating tests. The optimizer
// finds some, but we _know_ they are all useless.
peeled_dom_test_elim(loop,old_new);
Expand Down Expand Up @@ -1318,12 +1343,12 @@ void PhaseIdealLoop::copy_skeleton_predicates_to_main_loop_helper(Node* predicat
// Clone the skeleton predicate twice and initialize one with the initial
// value of the loop induction variable. Leave the other predicate
// to be initialized when increasing the stride during loop unrolling.
prev_proj = clone_skeleton_predicate_for_main_or_post_loop(iff, opaque_init, NULL, predicate, uncommon_proj,
current_proj, outer_loop, prev_proj);
prev_proj = clone_skeleton_predicate_and_initialize(iff, opaque_init, NULL, predicate, uncommon_proj,
current_proj, outer_loop, prev_proj);
assert(skeleton_predicate_has_opaque(prev_proj->in(0)->as_If()), "");

prev_proj = clone_skeleton_predicate_for_main_or_post_loop(iff, init, stride, predicate, uncommon_proj,
current_proj, outer_loop, prev_proj);
prev_proj = clone_skeleton_predicate_and_initialize(iff, init, stride, predicate, uncommon_proj,
current_proj, outer_loop, prev_proj);
assert(!skeleton_predicate_has_opaque(prev_proj->in(0)->as_If()), "");

// Rewire any control inputs from the cloned skeleton predicates down to the main and post loop for data nodes that are part of the
Expand Down Expand Up @@ -1476,8 +1501,8 @@ Node* PhaseIdealLoop::clone_skeleton_predicate_bool(Node* iff, Node* new_init, N

// Clone a skeleton predicate for the main loop. new_init and new_stride are set as new inputs. Since the predicates cannot fail at runtime,
// Halt nodes are inserted instead of uncommon traps.
Node* PhaseIdealLoop::clone_skeleton_predicate_for_main_or_post_loop(Node* iff, Node* new_init, Node* new_stride, Node* predicate, Node* uncommon_proj,
Node* control, IdealLoopTree* outer_loop, Node* input_proj) {
Node* PhaseIdealLoop::clone_skeleton_predicate_and_initialize(Node* iff, Node* new_init, Node* new_stride, Node* predicate, Node* uncommon_proj,
Node* control, IdealLoopTree* outer_loop, Node* input_proj) {
Node* result = clone_skeleton_predicate_bool(iff, new_init, new_stride, control);
Node* proj = predicate->clone();
Node* other_proj = uncommon_proj->clone();
Expand Down Expand Up @@ -2007,8 +2032,8 @@ void PhaseIdealLoop::update_main_loop_skeleton_predicates(Node* ctrl, CountedLoo
_igvn.replace_input_of(iff, 1, iff->in(1)->in(2));
} else {
// Add back predicates updated for the new stride.
prev_proj = clone_skeleton_predicate_for_main_or_post_loop(iff, init, max_value, entry, proj, ctrl, outer_loop,
prev_proj);
prev_proj = clone_skeleton_predicate_and_initialize(iff, init, max_value, entry, proj, ctrl, outer_loop,
prev_proj);
assert(!skeleton_predicate_has_opaque(prev_proj->in(0)->as_If()), "unexpected");
}
}
Expand Down Expand Up @@ -2036,8 +2061,8 @@ void PhaseIdealLoop::copy_skeleton_predicates_to_post_loop(LoopNode* main_loop_h
break;
}
if (iff->in(1)->Opcode() == Op_Opaque4 && skeleton_predicate_has_opaque(iff)) {
prev_proj = clone_skeleton_predicate_for_main_or_post_loop(iff, init, stride, ctrl, proj, post_loop_entry,
post_loop, prev_proj);
prev_proj = clone_skeleton_predicate_and_initialize(iff, init, stride, ctrl, proj, post_loop_entry,
post_loop, prev_proj);
assert(!skeleton_predicate_has_opaque(prev_proj->in(0)->as_If()), "unexpected");
}
ctrl = ctrl->in(0)->in(0);
Expand All @@ -2048,6 +2073,52 @@ void PhaseIdealLoop::copy_skeleton_predicates_to_post_loop(LoopNode* main_loop_h
}
}

void PhaseIdealLoop::initialize_skeleton_predicates_for_peeled_loop(ProjNode* predicate,
LoopNode* outer_loop_head,
int dd_outer_loop_head,
Node* init,
Node* stride,
IdealLoopTree* outer_loop,
const uint idx_before_clone,
const Node_List &old_new) {
if (predicate == nullptr) {
return;
}
Node* control = outer_loop_head->in(LoopNode::EntryControl);
Node* input_proj = control;

predicate = next_predicate(predicate);
while (predicate != nullptr) {
IfNode* iff = predicate->in(0)->as_If();
if (iff->in(1)->Opcode() == Op_Opaque4) {
assert(skeleton_predicate_has_opaque(iff), "unexpected");
ProjNode* uncommon_proj = iff->proj_out(1 - predicate->as_Proj()->_con);
input_proj = clone_skeleton_predicate_and_initialize(iff, init, stride, predicate, uncommon_proj, control, outer_loop, input_proj);

// Rewire any control inputs from the old skeleton predicates above the peeled iteration down to the initialized
// skeleton predicates above the peeled loop.
for (DUIterator i = predicate->outs(); predicate->has_out(i); i++) {
Node* dependent = predicate->out(i);
Node* new_node = old_new[dependent->_idx];

if (!dependent->is_CFG() &&
dependent->_idx < idx_before_clone && // old node
new_node != nullptr && // cloned
new_node->_idx >= idx_before_clone) { // for peeling
// The old nodes from the peeled loop still point to the predicate above the peeled loop.
// We need to rewire the dependencies to the newly initialized skeleton predicates.
_igvn.replace_input_of(dependent, 0, input_proj);
--i; // correct for just deleted predicate->out(i)
}
}
}
predicate = next_predicate(predicate);
}

_igvn.replace_input_of(outer_loop_head, LoopNode::EntryControl, input_proj);
set_idom(outer_loop_head, input_proj, dd_outer_loop_head);
}

//------------------------------do_unroll--------------------------------------
// Unroll the loop body one step - make each trip do 2 iterations.
void PhaseIdealLoop::do_unroll(IdealLoopTree *loop, Node_List &old_new, bool adjust_min_trip) {
Expand Down
41 changes: 39 additions & 2 deletions src/hotspot/share/opto/loopnode.hpp
View file
Expand Up @@ -927,13 +927,16 @@ class PhaseIdealLoop : public PhaseTransform {
void copy_skeleton_predicates_to_main_loop(CountedLoopNode* pre_head, Node* init, Node* stride, IdealLoopTree* outer_loop, LoopNode* outer_main_head,
uint dd_main_head, const uint idx_before_pre_post, const uint idx_after_post_before_pre,
Node* zero_trip_guard_proj_main, Node* zero_trip_guard_proj_post, const Node_List &old_new);
Node* clone_skeleton_predicate_for_main_or_post_loop(Node* iff, Node* new_init, Node* new_stride, Node* predicate, Node* uncommon_proj, Node* control,
IdealLoopTree* outer_loop, Node* input_proj);
Node* clone_skeleton_predicate_and_initialize(Node* iff, Node* new_init, Node* new_stride, Node* predicate, Node* uncommon_proj, Node* control,
IdealLoopTree* outer_loop, Node* input_proj);
Node* clone_skeleton_predicate_bool(Node* iff, Node* new_init, Node* new_stride, Node* control);
static bool skeleton_predicate_has_opaque(IfNode* iff);
static void get_skeleton_predicates(Node* predicate, Unique_Node_List& list, bool get_opaque = false);
void update_main_loop_skeleton_predicates(Node* ctrl, CountedLoopNode* loop_head, Node* init, int stride_con);
void copy_skeleton_predicates_to_post_loop(LoopNode* main_loop_head, CountedLoopNode* post_loop_head, Node* init, Node* stride);
void initialize_skeleton_predicates_for_peeled_loop(ProjNode* predicate, LoopNode* outer_loop_head, int dd_outer_loop_head,
Node* init, Node* stride, IdealLoopTree* outer_loop,
const uint idx_before_clone, const Node_List& old_new);
void insert_loop_limit_check(ProjNode* limit_check_proj, Node* cmp_limit, Node* bol);
#ifdef ASSERT
bool only_has_infinite_loops();
Expand Down Expand Up @@ -1328,9 +1331,43 @@ class PhaseIdealLoop : public PhaseTransform {

static Node* skip_all_loop_predicates(Node* entry);
static Node* skip_loop_predicates(Node* entry);
static ProjNode* next_predicate(ProjNode* predicate);

// Find a good location to insert a predicate
static ProjNode* find_predicate_insertion_point(Node* start_c, Deoptimization::DeoptReason reason);

class Predicates {
public:
// given loop entry, find all predicates above loop
Predicates(Node* entry);

// Proj of empty loop limit check predicate
ProjNode* loop_limit_check() {
return _loop_limit_check;
}

// Proj of empty profile predicate
ProjNode* profile_predicate() {
return _profile_predicate;
}

// Proj of empty predicate
ProjNode* predicate() {
return _predicate;
}

// First control node above all predicates
Node* skip_all() {
return _entry_to_all_predicates;
}

private:
ProjNode*_loop_limit_check = nullptr;
ProjNode* _profile_predicate = nullptr;
ProjNode* _predicate = nullptr;
Node* _entry_to_all_predicates = nullptr;
};

// Find a predicate
static Node* find_predicate(Node* entry);
// Construct a range check for a predicate if
Expand Down

1 comment on commit 199832a

@openjdk-notifier
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Review

Issues

Please sign in to comment.