Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

8277654: Shenandoah: Don't produce new memory state in C2 LRB runtime call #6526

Closed
wants to merge 2 commits into from
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
@@ -54,7 +54,6 @@ bool ShenandoahBarrierC2Support::expand(Compile* C, PhaseIterGVN& igvn) {
PhaseIdealLoop::optimize(igvn, LoopOptsShenandoahExpand);
if (C->failing()) return false;
PhaseIdealLoop::verify(igvn);
DEBUG_ONLY(verify_raw_mem(C->root());)
if (attempt_more_loopopts) {
C->set_major_progress();
if (!C->optimize_loops(igvn, LoopOptsShenandoahPostExpand)) {
@@ -964,18 +963,11 @@ void ShenandoahBarrierC2Support::test_in_cset(Node*& ctrl, Node*& not_cset_ctrl,
phase->register_new_node(cset_bool, old_ctrl);
}

void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, Node*& result_mem, Node* raw_mem,
void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr,
DecoratorSet decorators, PhaseIdealLoop* phase) {
IdealLoopTree*loop = phase->get_loop(ctrl);
const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr();

// The slow path stub consumes and produces raw memory in addition
// to the existing memory edges
Node* base = find_bottom_mem(ctrl, phase);
MergeMemNode* mm = MergeMemNode::make(base);
mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
phase->register_new_node(mm, ctrl);

address calladdr = NULL;
const char* name = NULL;
bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
@@ -1013,16 +1005,14 @@ void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node* lo

call->init_req(TypeFunc::Control, ctrl);
call->init_req(TypeFunc::I_O, phase->C->top());
call->init_req(TypeFunc::Memory, mm);
call->init_req(TypeFunc::Memory, phase->C->top());
call->init_req(TypeFunc::FramePtr, phase->C->top());
call->init_req(TypeFunc::ReturnAdr, phase->C->top());
call->init_req(TypeFunc::Parms, val);
call->init_req(TypeFunc::Parms+1, load_addr);
phase->register_control(call, loop, ctrl);
ctrl = new ProjNode(call, TypeFunc::Control);
phase->register_control(ctrl, loop, call);
result_mem = new ProjNode(call, TypeFunc::Memory);
phase->register_new_node(result_mem, call);
val = new ProjNode(call, TypeFunc::Parms);
phase->register_new_node(val, call);
val = new CheckCastPPNode(ctrl, val, obj_type);
@@ -1341,12 +1331,9 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
Node* ctrl = phase->get_ctrl(lrb);
Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn);


Node* orig_ctrl = ctrl;

Node* raw_mem = fixer.find_mem(ctrl, lrb);
Node* init_raw_mem = raw_mem;
Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, NULL);

IdealLoopTree *loop = phase->get_loop(ctrl);

@@ -1359,7 +1346,6 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
enum { _heap_stable = 1, _evac_path, _not_cset, PATH_LIMIT };
Node* region = new RegionNode(PATH_LIMIT);
Node* val_phi = new PhiNode(region, val->bottom_type()->is_oopptr());
Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);

// Stable path.
int flags = ShenandoahHeap::HAS_FORWARDED;
@@ -1372,7 +1358,6 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
// Heap stable case
region->init_req(_heap_stable, heap_stable_ctrl);
val_phi->init_req(_heap_stable, val);
raw_mem_phi->init_req(_heap_stable, raw_mem);

// Test for in-cset, unless it's a native-LRB. Native LRBs need to return NULL
// even for non-cset objects to prevent ressurrection of such objects.
@@ -1384,11 +1369,9 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
if (not_cset_ctrl != NULL) {
region->init_req(_not_cset, not_cset_ctrl);
val_phi->init_req(_not_cset, val);
raw_mem_phi->init_req(_not_cset, raw_mem);
} else {
region->del_req(_not_cset);
val_phi->del_req(_not_cset);
raw_mem_phi->del_req(_not_cset);
}

// Resolve object when orig-value is in cset.
@@ -1429,15 +1412,13 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
}
}
}
call_lrb_stub(ctrl, val, addr, result_mem, raw_mem, lrb->decorators(), phase);
call_lrb_stub(ctrl, val, addr, lrb->decorators(), phase);
region->init_req(_evac_path, ctrl);
val_phi->init_req(_evac_path, val);
raw_mem_phi->init_req(_evac_path, result_mem);

phase->register_control(region, loop, heap_stable_iff);
Node* out_val = val_phi;
phase->register_new_node(val_phi, region);
phase->register_new_node(raw_mem_phi, region);

fix_ctrl(lrb, region, fixer, uses, uses_to_ignore, last, phase);

@@ -1450,18 +1431,10 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
for(uint next = 0; next < uses.size(); next++ ) {
Node *n = uses.at(next);
assert(phase->get_ctrl(n) == ctrl, "bad control");
assert(n != init_raw_mem, "should leave input raw mem above the barrier");
assert(n != raw_mem, "should leave input raw mem above the barrier");
phase->set_ctrl(n, region);
follow_barrier_uses(n, ctrl, uses, phase);
}

// The slow path call produces memory: hook the raw memory phi
// from the expanded load reference barrier with the rest of the graph
// which may require adding memory phis at every post dominated
// region and at enclosing loop heads. Use the memory state
// collected in memory_nodes to fix the memory graph. Update that
// memory state as we go.
fixer.fix_mem(ctrl, region, init_raw_mem, raw_mem_for_ctrl, raw_mem_phi, uses);
}
// Done expanding load-reference-barriers.
assert(ShenandoahBarrierSetC2::bsc2()->state()->load_reference_barriers_count() == 0, "all load reference barrier nodes should have been replaced");
@@ -1902,105 +1875,6 @@ void ShenandoahBarrierC2Support::optimize_after_expansion(VectorSet &visited, No
}
}

#ifdef ASSERT
void ShenandoahBarrierC2Support::verify_raw_mem(RootNode* root) {
const bool trace = false;
ResourceMark rm;
Unique_Node_List nodes;
Unique_Node_List controls;
Unique_Node_List memories;

nodes.push(root);
for (uint next = 0; next < nodes.size(); next++) {
Node *n = nodes.at(next);
if (ShenandoahBarrierSetC2::is_shenandoah_lrb_call(n)) {
controls.push(n);
if (trace) { tty->print("XXXXXX verifying"); n->dump(); }
for (uint next2 = 0; next2 < controls.size(); next2++) {
Node *m = controls.at(next2);
for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
Node* u = m->fast_out(i);
if (u->is_CFG() && !u->is_Root() &&
!(u->Opcode() == Op_CProj && u->in(0)->Opcode() == Op_NeverBranch && u->as_Proj()->_con == 1) &&
!(u->is_Region() && u->unique_ctrl_out()->Opcode() == Op_Halt)) {
if (trace) { tty->print("XXXXXX pushing control"); u->dump(); }
controls.push(u);
}
}
}
memories.push(n->as_Call()->proj_out(TypeFunc::Memory));
for (uint next2 = 0; next2 < memories.size(); next2++) {
Node *m = memories.at(next2);
assert(m->bottom_type() == Type::MEMORY, "");
for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
Node* u = m->fast_out(i);
if (u->bottom_type() == Type::MEMORY && (u->is_Mem() || u->is_ClearArray())) {
if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); }
memories.push(u);
} else if (u->is_LoadStore()) {
if (trace) { tty->print("XXXXXX pushing memory"); u->find_out_with(Op_SCMemProj)->dump(); }
memories.push(u->find_out_with(Op_SCMemProj));
} else if (u->is_MergeMem() && u->as_MergeMem()->memory_at(Compile::AliasIdxRaw) == m) {
if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); }
memories.push(u);
} else if (u->is_Phi()) {
assert(u->bottom_type() == Type::MEMORY, "");
if (u->adr_type() == TypeRawPtr::BOTTOM || u->adr_type() == TypePtr::BOTTOM) {
assert(controls.member(u->in(0)), "");
if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); }
memories.push(u);
}
} else if (u->is_SafePoint() || u->is_MemBar()) {
for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) {
Node* uu = u->fast_out(j);
if (uu->bottom_type() == Type::MEMORY) {
if (trace) { tty->print("XXXXXX pushing memory"); uu->dump(); }
memories.push(uu);
}
}
}
}
}
for (uint next2 = 0; next2 < controls.size(); next2++) {
Node *m = controls.at(next2);
if (m->is_Region()) {
bool all_in = true;
for (uint i = 1; i < m->req(); i++) {
if (!controls.member(m->in(i))) {
all_in = false;
break;
}
}
if (trace) { tty->print("XXX verifying %s", all_in ? "all in" : ""); m->dump(); }
bool found_phi = false;
for (DUIterator_Fast jmax, j = m->fast_outs(jmax); j < jmax && !found_phi; j++) {
Node* u = m->fast_out(j);
if (u->is_Phi() && memories.member(u)) {
found_phi = true;
for (uint i = 1; i < u->req() && found_phi; i++) {
Node* k = u->in(i);
if (memories.member(k) != controls.member(m->in(i))) {
found_phi = false;
}
}
}
}
assert(found_phi || all_in, "");
}
}
controls.clear();
memories.clear();
}
for( uint i = 0; i < n->len(); ++i ) {
Node *m = n->in(i);
if (m != NULL) {
nodes.push(m);
}
}
}
}
#endif

ShenandoahIUBarrierNode::ShenandoahIUBarrierNode(Node* val) : Node(NULL, val) {
ShenandoahBarrierSetC2::bsc2()->state()->add_iu_barrier(this);
}
@@ -50,7 +50,6 @@ class ShenandoahBarrierC2Support : public AllStatic {

static bool verify_helper(Node* in, Node_Stack& phis, VectorSet& visited, verify_type t, bool trace, Unique_Node_List& barriers_used);
static void report_verify_failure(const char* msg, Node* n1 = NULL, Node* n2 = NULL);
static void verify_raw_mem(RootNode* root);
#endif
static Node* dom_mem(Node* mem, Node* ctrl, int alias, Node*& mem_ctrl, PhaseIdealLoop* phase);
static Node* no_branches(Node* c, Node* dom, bool allow_one_proj, PhaseIdealLoop* phase);
@@ -61,7 +60,7 @@ class ShenandoahBarrierC2Support : public AllStatic {
static void test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase);
static void test_gc_state(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl,
PhaseIdealLoop* phase, int flags);
static void call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, Node*& result_mem, Node* raw_mem,
static void call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr,
DecoratorSet decorators, PhaseIdealLoop* phase);
static void test_in_cset(Node*& ctrl, Node*& not_cset_ctrl, Node* val, Node* raw_mem, PhaseIdealLoop* phase);
static void move_gc_state_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase);