Skip to content
Permalink
Browse files
8252299: [lworld] C2 compilation fails with assert(!use_ReduceInitial…
…CardMarks()) failed: can only happen with card marking
  • Loading branch information
TobiHartmann committed Aug 26, 2020
1 parent acfed88 commit 54b541094689d6244b95307724f9a9698c10e46d
Showing 11 changed files with 45 additions and 30 deletions.
@@ -662,7 +662,7 @@ bool G1BarrierSetC2::is_gc_barrier_node(Node* node) const {
return strcmp(call->_name, "write_ref_field_pre_entry") == 0 || strcmp(call->_name, "write_ref_field_post_entry") == 0;
}

void G1BarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const {
void G1BarrierSetC2::eliminate_gc_barrier(PhaseIterGVN* igvn, Node* node) const {
assert(node->Opcode() == Op_CastP2X, "ConvP2XNode required");
assert(node->outcnt() <= 2, "expects 1 or 2 users: Xor and URShift nodes");
// It could be only one user, URShift node, in Object.clone() intrinsic
@@ -690,7 +690,7 @@ void G1BarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) c
assert(cmpx->is_Cmp() && cmpx->unique_out()->is_Bool() &&
cmpx->unique_out()->as_Bool()->_test._test == BoolTest::ne,
"missing region check in G1 post barrier");
macro->replace_node(cmpx, macro->makecon(TypeInt::CC_EQ));
igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ));

// Remove G1 pre barrier.

@@ -708,14 +708,14 @@ void G1BarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) c
assert(bol->is_Bool(), "");
cmpx = bol->in(1);
if (bol->as_Bool()->_test._test == BoolTest::ne &&
cmpx->is_Cmp() && cmpx->in(2) == macro->intcon(0) &&
cmpx->is_Cmp() && cmpx->in(2) == igvn->intcon(0) &&
cmpx->in(1)->is_Load()) {
Node* adr = cmpx->in(1)->as_Load()->in(MemNode::Address);
const int marking_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset());
if (adr->is_AddP() && adr->in(AddPNode::Base) == macro->top() &&
if (adr->is_AddP() && adr->in(AddPNode::Base) == igvn->C->top() &&
adr->in(AddPNode::Address)->Opcode() == Op_ThreadLocal &&
adr->in(AddPNode::Offset) == macro->MakeConX(marking_offset)) {
macro->replace_node(cmpx, macro->makecon(TypeInt::CC_EQ));
adr->in(AddPNode::Offset) == igvn->MakeConX(marking_offset)) {
igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ));
}
}
}
@@ -734,13 +734,13 @@ void G1BarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) c
assert(cmpx->is_Cmp() && cmpx->unique_out()->is_Bool() &&
cmpx->unique_out()->as_Bool()->_test._test == BoolTest::ne,
"missing card value check in G1 post barrier");
macro->replace_node(cmpx, macro->makecon(TypeInt::CC_EQ));
igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ));
// There is no G1 pre barrier in this case
}
// Now CastP2X can be removed since it is used only on dead path
// which currently still alive until igvn optimize it.
assert(node->outcnt() == 0 || node->unique_out()->Opcode() == Op_URShiftX, "");
macro->replace_node(node, macro->top());
igvn->replace_node(node, igvn->C->top());
}

Node* G1BarrierSetC2::step_over_gc_barrier(Node* c) const {
@@ -86,7 +86,7 @@ class G1BarrierSetC2: public CardTableBarrierSetC2 {

public:
virtual bool is_gc_barrier_node(Node* node) const;
virtual void eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const;
virtual void eliminate_gc_barrier(PhaseIterGVN* igvn, Node* node) const;
virtual Node* step_over_gc_barrier(Node* c) const;

#ifdef ASSERT
@@ -853,5 +853,5 @@ void BarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac
Node* call = phase->make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, payload_src, payload_dst, length XTOP);
phase->transform_later(call);

phase->igvn().replace_node(ac, call);
phase->replace_node(ac, call);
}
@@ -267,7 +267,7 @@ class BarrierSetC2: public CHeapObj<mtGC> {
// Support for macro expanded GC barriers
virtual void register_potential_barrier_node(Node* node) const { }
virtual void unregister_potential_barrier_node(Node* node) const { }
virtual void eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const { }
virtual void eliminate_gc_barrier(PhaseIterGVN* igvn, Node* node) const { }
virtual void enqueue_useful_gc_barrier(PhaseIterGVN* igvn, Node* node) const {}
virtual void eliminate_useless_gc_barriers(Unique_Node_List &useful, Compile* C) const {}

@@ -168,7 +168,7 @@ bool CardTableBarrierSetC2::is_gc_barrier_node(Node* node) const {
return ModRefBarrierSetC2::is_gc_barrier_node(node) || node->Opcode() == Op_StoreCM;
}

void CardTableBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const {
void CardTableBarrierSetC2::eliminate_gc_barrier(PhaseIterGVN* igvn, Node* node) const {
assert(node->Opcode() == Op_CastP2X, "ConvP2XNode required");
Node *shift = node->unique_out();
Node *addp = shift->unique_out();
@@ -178,11 +178,11 @@ void CardTableBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node*
assert(mem->Opcode() == Op_LoadB, "unexpected code shape");
// The load is checking if the card has been written so
// replace it with zero to fold the test.
macro->replace_node(mem, macro->intcon(0));
igvn->replace_node(mem, igvn->intcon(0));
continue;
}
assert(mem->is_Store(), "store required");
macro->replace_node(mem, mem->in(MemNode::Memory));
igvn->replace_node(mem, mem->in(MemNode::Memory));
}
}

@@ -44,7 +44,7 @@ class CardTableBarrierSetC2: public ModRefBarrierSetC2 {
public:
virtual void clone(GraphKit* kit, Node* src_base, Node* dst_base, Node* countx, bool is_array) const;
virtual bool is_gc_barrier_node(Node* node) const;
virtual void eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const;
virtual void eliminate_gc_barrier(PhaseIterGVN* igvn, Node* node) const;
virtual bool array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const;

bool use_ReduceInitialCardMarks() const;
@@ -830,7 +830,7 @@ void ShenandoahBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCo
Node* mem_phi = new PhiNode(region, Type::MEMORY, TypeRawPtr::BOTTOM);

Node* thread = phase->transform_later(new ThreadLocalNode());
Node* offset = phase->igvn().MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
Node* offset = phase->MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
Node* gc_state_addr = phase->transform_later(new AddPNode(phase->C->top(), thread, offset));

uint gc_state_idx = Compile::AliasIdxRaw;
@@ -882,7 +882,7 @@ void ShenandoahBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCo
call = phase->transform_later(call);

// Hook up the whole thing into the graph
phase->igvn().replace_node(ac, call);
phase->replace_node(ac, call);
} else {
BarrierSetC2::clone_at_expansion(phase, ac);
}
@@ -908,9 +908,9 @@ void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const
}
}

void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* n) const {
void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseIterGVN* igvn, Node* n) const {
if (is_shenandoah_wb_pre_call(n)) {
shenandoah_eliminate_wb_pre(n, &macro->igvn());
shenandoah_eliminate_wb_pre(n, igvn);
}
}

@@ -123,7 +123,7 @@ class ShenandoahBarrierSetC2 : public BarrierSetC2 {
// Support for macro expanded GC barriers
virtual void register_potential_barrier_node(Node* node) const;
virtual void unregister_potential_barrier_node(Node* node) const;
virtual void eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const;
virtual void eliminate_gc_barrier(PhaseIterGVN* igvn, Node* node) const;
virtual void enqueue_useful_gc_barrier(PhaseIterGVN* igvn, Node* node) const;
virtual void eliminate_useless_gc_barriers(Unique_Node_List &useful, Compile* C) const;

@@ -267,7 +267,7 @@ void ZBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* a
full_size,
phase->top());
phase->transform_later(call);
phase->igvn().replace_node(ac, call);
phase->replace_node(ac, call);
}

// == Dominating barrier elision ==
@@ -24,6 +24,7 @@

#include "precompiled.hpp"
#include "ci/ciInlineKlass.hpp"
#include "gc/shared/barrierSet.hpp"
#include "opto/addnode.hpp"
#include "opto/castnode.hpp"
#include "opto/graphKit.hpp"
@@ -783,16 +784,27 @@ void InlineTypeNode::initialize_fields(GraphKit* kit, MultiNode* multi, Extended

// Replace a buffer allocation by a dominating allocation
static void replace_allocation(PhaseIterGVN* igvn, Node* res, Node* dom) {
// Remove initializing stores
// Remove initializing stores and GC barriers
for (DUIterator_Fast imax, i = res->fast_outs(imax); i < imax; i++) {
AddPNode* addp = res->fast_out(i)->isa_AddP();
if (addp != NULL) {
for (DUIterator_Fast jmax, j = addp->fast_outs(jmax); j < jmax; j++) {
StoreNode* store = addp->fast_out(j)->isa_Store();
Node* use = res->fast_out(i);
if (use->is_AddP()) {
for (DUIterator_Fast jmax, j = use->fast_outs(jmax); j < jmax; j++) {
Node* store = use->fast_out(j)->isa_Store();
if (store != NULL) {
igvn->rehash_node_delayed(store);
igvn->replace_in_uses(store, store->in(MemNode::Memory));
}
}
} else if (use->Opcode() == Op_CastP2X) {
if (UseG1GC && use->find_out_with(Op_XorX)->in(1) != use) {
// The G1 pre-barrier uses a CastP2X both for the pointer of the object
// we store into, as well as the value we are storing. Skip if this is a
// barrier for storing 'res' into another object.
continue;
}
BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
bs->eliminate_gc_barrier(igvn, use);
--i; --imax;
}
}
igvn->replace_node(res, dom);
@@ -892,7 +904,7 @@ void InlineTypeNode::remove_redundant_allocations(PhaseIterGVN* igvn, PhaseIdeal
Node* out = fast_out(i);
if (out->is_InlineType()) {
// Unlink and recursively process inline type users
igvn->hash_delete(out);
igvn->rehash_node_delayed(out);
int nb = out->replace_edge(this, igvn->C->top());
out->as_InlineType()->remove_redundant_allocations(igvn, phase);
--i; imax -= nb;
@@ -201,7 +201,7 @@ void PhaseMacroExpand::extract_call_projections(CallNode *call) {

void PhaseMacroExpand::eliminate_gc_barrier(Node* p2x) {
BarrierSetC2 *bs = BarrierSet::barrier_set()->barrier_set_c2();
bs->eliminate_gc_barrier(this, p2x);
bs->eliminate_gc_barrier(&_igvn, p2x);
}

// Search for a memory operation for the specified memory slice.
@@ -721,7 +721,8 @@ bool PhaseMacroExpand::can_eliminate_allocation(AllocateNode *alloc, GrowableArr
} else if (use->is_InlineType() && use->isa_InlineType()->get_oop() == res) {
// ok to eliminate
} else if (use->Opcode() == Op_StoreX && use->in(MemNode::Address) == res) {
// store to mark work
// Store to mark word of inline type larval buffer
assert(res_type->is_inlinetypeptr(), "Unexpected store to mark word");
} else if (use->Opcode() != Op_CastP2X) { // CastP2X is used by card mark
if (use->is_Phi()) {
if (use->outcnt() == 1 && use->unique_out()->Opcode() == Op_Return) {
@@ -1052,7 +1053,9 @@ void PhaseMacroExpand::process_users_of_allocation(CallNode *alloc, bool inline_
assert(use->isa_InlineType()->get_oop() == res, "unexpected inline type use");
_igvn.rehash_node_delayed(use);
use->isa_InlineType()->set_oop(_igvn.zerocon(T_INLINE_TYPE));
} else if (use->is_Store()) {
} else if (use->Opcode() == Op_StoreX && use->in(MemNode::Address) == res) {
// Store to mark word of inline type larval buffer
assert(inline_alloc, "Unexpected store to mark word");
_igvn.replace_node(use, use->in(MemNode::Memory));
} else {
eliminate_gc_barrier(use);

0 comments on commit 54b5410

Please sign in to comment.