Skip to content

Commit

Permalink
8277654: Shenandoah: Don't produce new memory state in C2 LRB runtime…
Browse files Browse the repository at this point in the history 
… call

Backport-of: 3d39f09c6cdc875b44147b4e84e496b6abf93996
  • Loading branch information
@shipilev
William Kemper authored and shipilev committed Jun 23, 2023
1 parent 832707b commit b8a65e0
Show file tree
Hide file tree
Showing 2 changed files with 5 additions and 132 deletions.
134 changes: 4 additions & 130 deletions src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ bool ShenandoahBarrierC2Support::expand(Compile* C, PhaseIterGVN& igvn) {
PhaseIdealLoop::optimize(igvn, LoopOptsShenandoahExpand);
if (C->failing()) return false;
PhaseIdealLoop::verify(igvn);
DEBUG_ONLY(verify_raw_mem(C->root());)
if (attempt_more_loopopts) {
C->set_major_progress();
if (!C->optimize_loops(igvn, LoopOptsShenandoahPostExpand)) {
Expand Down Expand Up @@ -964,18 +963,11 @@ void ShenandoahBarrierC2Support::test_in_cset(Node*& ctrl, Node*& not_cset_ctrl,
phase->register_new_node(cset_bool, old_ctrl);
}

void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, Node*& result_mem, Node* raw_mem,
void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr,
DecoratorSet decorators, PhaseIdealLoop* phase) {
IdealLoopTree*loop = phase->get_loop(ctrl);
const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr();

// The slow path stub consumes and produces raw memory in addition
// to the existing memory edges
Node* base = find_bottom_mem(ctrl, phase);
MergeMemNode* mm = MergeMemNode::make(base);
mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
phase->register_new_node(mm, ctrl);

address calladdr = NULL;
const char* name = NULL;
bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
Expand Down Expand Up @@ -1013,16 +1005,14 @@ void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node* lo

call->init_req(TypeFunc::Control, ctrl);
call->init_req(TypeFunc::I_O, phase->C->top());
call->init_req(TypeFunc::Memory, mm);
call->init_req(TypeFunc::Memory, phase->C->top());
call->init_req(TypeFunc::FramePtr, phase->C->top());
call->init_req(TypeFunc::ReturnAdr, phase->C->top());
call->init_req(TypeFunc::Parms, val);
call->init_req(TypeFunc::Parms+1, load_addr);
phase->register_control(call, loop, ctrl);
ctrl = new ProjNode(call, TypeFunc::Control);
phase->register_control(ctrl, loop, call);
result_mem = new ProjNode(call, TypeFunc::Memory);
phase->register_new_node(result_mem, call);
val = new ProjNode(call, TypeFunc::Parms);
phase->register_new_node(val, call);
val = new CheckCastPPNode(ctrl, val, obj_type);
Expand Down Expand Up @@ -1341,12 +1331,9 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
Node* ctrl = phase->get_ctrl(lrb);
Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn);


Node* orig_ctrl = ctrl;

Node* raw_mem = fixer.find_mem(ctrl, lrb);
Node* init_raw_mem = raw_mem;
Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, NULL);

IdealLoopTree *loop = phase->get_loop(ctrl);

Expand All @@ -1359,7 +1346,6 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
enum { _heap_stable = 1, _evac_path, _not_cset, PATH_LIMIT };
Node* region = new RegionNode(PATH_LIMIT);
Node* val_phi = new PhiNode(region, val->bottom_type()->is_oopptr());
Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);

// Stable path.
int flags = ShenandoahHeap::HAS_FORWARDED;
Expand All @@ -1372,7 +1358,6 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
// Heap stable case
region->init_req(_heap_stable, heap_stable_ctrl);
val_phi->init_req(_heap_stable, val);
raw_mem_phi->init_req(_heap_stable, raw_mem);

// Test for in-cset, unless it's a native-LRB. Native LRBs need to return NULL
// even for non-cset objects to prevent ressurrection of such objects.
Expand All @@ -1384,11 +1369,9 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
if (not_cset_ctrl != NULL) {
region->init_req(_not_cset, not_cset_ctrl);
val_phi->init_req(_not_cset, val);
raw_mem_phi->init_req(_not_cset, raw_mem);
} else {
region->del_req(_not_cset);
val_phi->del_req(_not_cset);
raw_mem_phi->del_req(_not_cset);
}

// Resolve object when orig-value is in cset.
Expand Down Expand Up @@ -1429,15 +1412,13 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
}
}
}
call_lrb_stub(ctrl, val, addr, result_mem, raw_mem, lrb->decorators(), phase);
call_lrb_stub(ctrl, val, addr, lrb->decorators(), phase);
region->init_req(_evac_path, ctrl);
val_phi->init_req(_evac_path, val);
raw_mem_phi->init_req(_evac_path, result_mem);

phase->register_control(region, loop, heap_stable_iff);
Node* out_val = val_phi;
phase->register_new_node(val_phi, region);
phase->register_new_node(raw_mem_phi, region);

fix_ctrl(lrb, region, fixer, uses, uses_to_ignore, last, phase);

Expand All @@ -1450,18 +1431,10 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
for(uint next = 0; next < uses.size(); next++ ) {
Node *n = uses.at(next);
assert(phase->get_ctrl(n) == ctrl, "bad control");
assert(n != init_raw_mem, "should leave input raw mem above the barrier");
assert(n != raw_mem, "should leave input raw mem above the barrier");
phase->set_ctrl(n, region);
follow_barrier_uses(n, ctrl, uses, phase);
}

// The slow path call produces memory: hook the raw memory phi
// from the expanded load reference barrier with the rest of the graph
// which may require adding memory phis at every post dominated
// region and at enclosing loop heads. Use the memory state
// collected in memory_nodes to fix the memory graph. Update that
// memory state as we go.
fixer.fix_mem(ctrl, region, init_raw_mem, raw_mem_for_ctrl, raw_mem_phi, uses);
}
// Done expanding load-reference-barriers.
assert(ShenandoahBarrierSetC2::bsc2()->state()->load_reference_barriers_count() == 0, "all load reference barrier nodes should have been replaced");
Expand Down Expand Up @@ -1902,105 +1875,6 @@ void ShenandoahBarrierC2Support::optimize_after_expansion(VectorSet &visited, No
}
}

#ifdef ASSERT
void ShenandoahBarrierC2Support::verify_raw_mem(RootNode* root) {
const bool trace = false;
ResourceMark rm;
Unique_Node_List nodes;
Unique_Node_List controls;
Unique_Node_List memories;

nodes.push(root);
for (uint next = 0; next < nodes.size(); next++) {
Node *n = nodes.at(next);
if (ShenandoahBarrierSetC2::is_shenandoah_lrb_call(n)) {
controls.push(n);
if (trace) { tty->print("XXXXXX verifying"); n->dump(); }
for (uint next2 = 0; next2 < controls.size(); next2++) {
Node *m = controls.at(next2);
for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
Node* u = m->fast_out(i);
if (u->is_CFG() && !u->is_Root() &&
!(u->Opcode() == Op_CProj && u->in(0)->Opcode() == Op_NeverBranch && u->as_Proj()->_con == 1) &&
!(u->is_Region() && u->unique_ctrl_out()->Opcode() == Op_Halt)) {
if (trace) { tty->print("XXXXXX pushing control"); u->dump(); }
controls.push(u);
}
}
}
memories.push(n->as_Call()->proj_out(TypeFunc::Memory));
for (uint next2 = 0; next2 < memories.size(); next2++) {
Node *m = memories.at(next2);
assert(m->bottom_type() == Type::MEMORY, "");
for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
Node* u = m->fast_out(i);
if (u->bottom_type() == Type::MEMORY && (u->is_Mem() || u->is_ClearArray())) {
if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); }
memories.push(u);
} else if (u->is_LoadStore()) {
if (trace) { tty->print("XXXXXX pushing memory"); u->find_out_with(Op_SCMemProj)->dump(); }
memories.push(u->find_out_with(Op_SCMemProj));
} else if (u->is_MergeMem() && u->as_MergeMem()->memory_at(Compile::AliasIdxRaw) == m) {
if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); }
memories.push(u);
} else if (u->is_Phi()) {
assert(u->bottom_type() == Type::MEMORY, "");
if (u->adr_type() == TypeRawPtr::BOTTOM || u->adr_type() == TypePtr::BOTTOM) {
assert(controls.member(u->in(0)), "");
if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); }
memories.push(u);
}
} else if (u->is_SafePoint() || u->is_MemBar()) {
for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) {
Node* uu = u->fast_out(j);
if (uu->bottom_type() == Type::MEMORY) {
if (trace) { tty->print("XXXXXX pushing memory"); uu->dump(); }
memories.push(uu);
}
}
}
}
}
for (uint next2 = 0; next2 < controls.size(); next2++) {
Node *m = controls.at(next2);
if (m->is_Region()) {
bool all_in = true;
for (uint i = 1; i < m->req(); i++) {
if (!controls.member(m->in(i))) {
all_in = false;
break;
}
}
if (trace) { tty->print("XXX verifying %s", all_in ? "all in" : ""); m->dump(); }
bool found_phi = false;
for (DUIterator_Fast jmax, j = m->fast_outs(jmax); j < jmax && !found_phi; j++) {
Node* u = m->fast_out(j);
if (u->is_Phi() && memories.member(u)) {
found_phi = true;
for (uint i = 1; i < u->req() && found_phi; i++) {
Node* k = u->in(i);
if (memories.member(k) != controls.member(m->in(i))) {
found_phi = false;
}
}
}
}
assert(found_phi || all_in, "");
}
}
controls.clear();
memories.clear();
}
for( uint i = 0; i < n->len(); ++i ) {
Node *m = n->in(i);
if (m != NULL) {
nodes.push(m);
}
}
}
}
#endif

ShenandoahIUBarrierNode::ShenandoahIUBarrierNode(Node* val) : Node(NULL, val) {
ShenandoahBarrierSetC2::bsc2()->state()->add_iu_barrier(this);
}
Expand Down
3 changes: 1 addition & 2 deletions src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@ class ShenandoahBarrierC2Support : public AllStatic {

static bool verify_helper(Node* in, Node_Stack& phis, VectorSet& visited, verify_type t, bool trace, Unique_Node_List& barriers_used);
static void report_verify_failure(const char* msg, Node* n1 = NULL, Node* n2 = NULL);
static void verify_raw_mem(RootNode* root);
#endif
static Node* dom_mem(Node* mem, Node* ctrl, int alias, Node*& mem_ctrl, PhaseIdealLoop* phase);
static Node* no_branches(Node* c, Node* dom, bool allow_one_proj, PhaseIdealLoop* phase);
Expand All @@ -61,7 +60,7 @@ class ShenandoahBarrierC2Support : public AllStatic {
static void test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase);
static void test_gc_state(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl,
PhaseIdealLoop* phase, int flags);
static void call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, Node*& result_mem, Node* raw_mem,
static void call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr,
DecoratorSet decorators, PhaseIdealLoop* phase);
static void test_in_cset(Node*& ctrl, Node*& not_cset_ctrl, Node* val, Node* raw_mem, PhaseIdealLoop* phase);
static void move_gc_state_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase);
Expand Down

1 comment on commit b8a65e0

@openjdk-notifier
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Review

Issues

Please sign in to comment.