Skip to content

Commit

Permalink
8315247: GenShen: Condition calls to post-write barrier code generati…
Browse files Browse the repository at this point in the history
…on by a flag

Reviewed-by: wkemper, kdnilsen
  • Loading branch information
Y. Srinivas Ramakrishna committed Aug 29, 2023
1 parent 72009f4 commit ade7796
Show file tree
Hide file tree
Showing 15 changed files with 85 additions and 58 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -90,14 +90,16 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess& access, LI

__ append(new LIR_OpShenandoahCompareAndSwap(addr, cmp_value.result(), new_value.result(), t1, t2, result));

post_barrier(access, access.resolved_addr(), new_value.result());
if (ShenandoahCardBarrier) {
post_barrier(access, access.resolved_addr(), new_value.result());
}
return result;
}
}

LIR_Opr result = BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value);

if (access.is_oop()) {
if (ShenandoahCardBarrier && access.is_oop()) {
post_barrier(access, access.resolved_addr(), new_value.result());
}

Expand Down Expand Up @@ -129,7 +131,9 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRIt
pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr,
result /* pre_val */);
}
post_barrier(access, access.resolved_addr(), result);
if (ShenandoahCardBarrier) {
post_barrier(access, access.resolved_addr(), result);
}
}

return result;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec

void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
Register start, Register count, Register tmp, RegSet saved_regs) {
if (is_oop) {
if (ShenandoahCardBarrier && is_oop) {
gen_write_ref_array_post_barrier(masm, decorators, start, count, tmp, saved_regs);
}
}
Expand Down Expand Up @@ -385,9 +385,7 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet d
}

void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj) {
if (!ShenandoahHeap::heap()->mode()->is_generational()) {
return;
}
assert(ShenandoahCardBarrier, "Did you mean to enable ShenandoahCardBarrier?");

__ lsr(obj, obj, CardTable::card_shift());

Expand Down Expand Up @@ -442,7 +440,9 @@ void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet
__ mov(new_val, val);
}
BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
store_check(masm, r3);
if (ShenandoahCardBarrier) {
store_check(masm, r3);
}
}

}
Expand Down Expand Up @@ -629,9 +629,7 @@ void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm,

void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
Register start, Register count, Register scratch, RegSet saved_regs) {
if (!ShenandoahHeap::heap()->mode()->is_generational()) {
return;
}
assert(ShenandoahCardBarrier, "Did you mean to enable ShenandoahCardBarrier?");

Label L_loop, L_done;
const Register end = count;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,9 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess &access, LI

__ append(new LIR_OpShenandoahCompareAndSwap(addr, cmp_value.result(), new_value.result(), t1, t2, result));

post_barrier(access, access.resolved_addr(), new_value.result());
if (ShenandoahCardBarrier) {
post_barrier(access, access.resolved_addr(), new_value.result());
}

if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
__ membar_acquire();
Expand All @@ -118,7 +120,7 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess &access, LI

LIR_Opr result = BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value);

if (access.is_oop()) {
if (ShenandoahCardBarrier && access.is_oop()) {
post_barrier(access, access.resolved_addr(), new_value.result());
}

Expand Down Expand Up @@ -159,7 +161,9 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess &access, LIRIt
pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr, result);
}

post_barrier(access, access.resolved_addr(), result);
if (ShenandoahCardBarrier) {
post_barrier(access, access.resolved_addr(), result);
}
}

if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, Dec
void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Register dst, Register count,
Register preserve) {
if (is_reference_type(type)) {
if (ShenandoahCardBarrier && is_reference_type(type)) {
__ block_comment("arraycopy_epilogue (shenandoahgc) {");
gen_write_ref_array_post_barrier(masm, decorators, dst, count, preserve);
__ block_comment("} arraycopy_epilogue (shenandoahgc)");
Expand Down Expand Up @@ -597,9 +597,7 @@ void ShenandoahBarrierSetAssembler::load_at(
}

void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register base, RegisterOrConstant ind_or_offs, Register tmp) {
if (!ShenandoahHeap::heap()->mode()->is_generational()) {
return;
}
assert(ShenandoahCardBarrier, "Did you mean to enable ShenandoahCardBarrier?");

ShenandoahBarrierSet* ctbs = ShenandoahBarrierSet::barrier_set();
CardTable* ct = ctbs->card_table();
Expand Down Expand Up @@ -641,7 +639,7 @@ void ShenandoahBarrierSetAssembler::store_at(MacroAssembler *masm, DecoratorSet
preservation_level);

// No need for post barrier if storing NULL
if (is_reference_type(type) && val != noreg) {
if (ShenandoahCardBarrier && is_reference_type(type) && val != noreg) {
store_check(masm, base, ind_or_offs, tmp1);
}
}
Expand Down Expand Up @@ -795,9 +793,7 @@ void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler *masm, Register b

void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
Register addr, Register count, Register preserve) {
if (!ShenandoahHeap::heap()->mode()->is_generational()) {
return;
}
assert(ShenandoahCardBarrier, "Did you mean to enable ShenandoahCardBarrier?");

ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
CardTable* ct = bs->card_table();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -89,14 +89,16 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess& access, LI

__ append(new LIR_OpShenandoahCompareAndSwap(addr, cmp_value.result(), new_value.result(), t1, t2, result));

post_barrier(access, access.resolved_addr(), new_value.result());
if (ShenandoahCardBarrier) {
post_barrier(access, access.resolved_addr(), new_value.result());
}
return result;
}
}

LIR_Opr result = BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value);

if (access.is_oop()) {
if (ShenandoahCardBarrier && access.is_oop()) {
post_barrier(access, access.resolved_addr(), new_value.result());
}

Expand Down Expand Up @@ -130,7 +132,9 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRIt
pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr,
result /* pre_val */);
}
post_barrier(access, access.resolved_addr(), result);
if (ShenandoahCardBarrier) {
post_barrier(access, access.resolved_addr(), result);
}
}

return result;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec
bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;

if (is_reference_type(type)) {
if (ShenandoahHeap::heap()->mode()->is_generational()) {
if (ShenandoahCardBarrier) {
bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
Expand Down Expand Up @@ -208,12 +208,13 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec

void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Register src, Register dst, Register count) {
bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
Register tmp = rax;

if (is_reference_type(type)) {
if (ShenandoahCardBarrier && is_reference_type(type)) {
bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
Register tmp = rax;

#ifdef _LP64
if (!checkcast) {
if (!obj_int) {
Expand Down Expand Up @@ -645,9 +646,7 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet d
}

void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj) {
if (!ShenandoahHeap::heap()->mode()->is_generational()) {
return;
}
assert(ShenandoahCardBarrier, "Did you mean to enable ShenandoahCardBarrier?");

// Does a store check for the oop in register obj. The content of
// register obj is destroyed afterwards.
Expand Down Expand Up @@ -730,9 +729,10 @@ void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet
BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
} else {
iu_barrier(masm, val, tmp3);
// TODO: store_check missing in upstream
BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
store_check(masm, tmp1);
if (ShenandoahCardBarrier) {
store_check(masm, tmp1);
}
}
NOT_LP64(imasm->restore_bcp());
} else {
Expand Down Expand Up @@ -941,9 +941,7 @@ void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm,
void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
Register addr, Register count,
Register tmp) {
if (!ShenandoahHeap::heap()->mode()->is_generational()) {
return;
}
assert(ShenandoahCardBarrier, "Did you mean to enable ShenandoahCardBarrier?");

ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
CardTable* ct = bs->card_table();
Expand Down
6 changes: 2 additions & 4 deletions src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ void ShenandoahBarrierSetC1::store_at_resolved(LIRAccess& access, LIR_Opr value)
}
BarrierSetC1::store_at_resolved(access, value);

if (access.is_oop()) {
if (ShenandoahCardBarrier && access.is_oop()) {
DecoratorSet decorators = access.decorators();
bool is_array = (decorators & IS_ARRAY) != 0;
bool on_anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0;
Expand Down Expand Up @@ -305,9 +305,7 @@ void ShenandoahBarrierSetC1::generate_c1_runtime_stubs(BufferBlob* buffer_blob)
}

void ShenandoahBarrierSetC1::post_barrier(LIRAccess& access, LIR_Opr addr, LIR_Opr new_val) {
if (!ShenandoahHeap::heap()->mode()->is_generational()) {
return;
}
assert(ShenandoahCardBarrier, "Did you mean to enable ShenandoahCardBarrier?");

DecoratorSet decorators = access.decorators();
LIRGenerator* gen = access.gen();
Expand Down
36 changes: 21 additions & 15 deletions src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -472,9 +472,7 @@ void ShenandoahBarrierSetC2::post_barrier(GraphKit* kit,
Node* val,
BasicType bt,
bool use_precise) const {
if (!ShenandoahHeap::heap()->mode()->is_generational()) {
return;
}
assert(ShenandoahCardBarrier, "Did you mean to enable ShenandoahCardBarrier?");

// No store check needed if we're storing a null.
if (val != nullptr && val->is_Con()) {
Expand Down Expand Up @@ -604,11 +602,13 @@ Node* ShenandoahBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue&

Node* result = BarrierSetC2::store_at_resolved(access, val);

bool anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0;
bool is_array = (decorators & IS_ARRAY) != 0;
bool use_precise = is_array || anonymous;
post_barrier(kit, kit->control(), access.raw_access(), access.base(),
adr, adr_idx, val.node(), access.type(), use_precise);
if (ShenandoahCardBarrier) {
const bool anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0;
const bool is_array = (decorators & IS_ARRAY) != 0;
const bool use_precise = is_array || anonymous;
post_barrier(kit, kit->control(), access.raw_access(), access.base(),
adr, adr_idx, val.node(), access.type(), use_precise);
}
return result;
} else {
assert(access.is_opt_access(), "only for optimization passes");
Expand Down Expand Up @@ -734,8 +734,10 @@ Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess
}
#endif
load_store = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(nullptr, load_store, access.decorators()));
post_barrier(kit, kit->control(), access.raw_access(), access.base(),
access.addr().node(), access.alias_idx(), new_val, T_OBJECT, true);
if (ShenandoahCardBarrier) {
post_barrier(kit, kit->control(), access.raw_access(), access.base(),
access.addr().node(), access.alias_idx(), new_val, T_OBJECT, true);
}
return load_store;
}
return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
Expand Down Expand Up @@ -791,8 +793,10 @@ Node* ShenandoahBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAcces
}
access.set_raw_access(load_store);
pin_atomic_op(access);
post_barrier(kit, kit->control(), access.raw_access(), access.base(),
access.addr().node(), access.alias_idx(), new_val, T_OBJECT, true);
if (ShenandoahCardBarrier) {
post_barrier(kit, kit->control(), access.raw_access(), access.base(),
access.addr().node(), access.alias_idx(), new_val, T_OBJECT, true);
}
return load_store;
}
return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
Expand All @@ -809,8 +813,10 @@ Node* ShenandoahBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& acces
shenandoah_write_barrier_pre(kit, false /* do_load */,
nullptr, nullptr, max_juint, nullptr, nullptr,
result /* pre_val */, T_OBJECT);
post_barrier(kit, kit->control(), access.raw_access(), access.base(),
access.addr().node(), access.alias_idx(), val, T_OBJECT, true);
if (ShenandoahCardBarrier) {
post_barrier(kit, kit->control(), access.raw_access(), access.base(),
access.addr().node(), access.alias_idx(), val, T_OBJECT, true);
}
}
return result;
}
Expand Down Expand Up @@ -1013,7 +1019,7 @@ void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node*
if (is_shenandoah_wb_pre_call(node)) {
shenandoah_eliminate_wb_pre(node, &macro->igvn());
}
if (node->Opcode() == Op_CastP2X && ShenandoahHeap::heap()->mode()->is_generational()) {
if (ShenandoahCardBarrier && node->Opcode() == Op_CastP2X) {
Node* shift = node->unique_out();
Node* addp = shift->unique_out();
for (DUIterator_Last jmin, j = addp->last_outs(jmin); j >= jmin; --j) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,14 @@ void ShenandoahGenerationalMode::initialize_flags() const {
vm_exit_during_initialization("Shenandoah Generational GC is not supported on this platform.");
#endif

// Exit if the user has asked ShenandoahCardBarrier to be disabled
if (!FLAG_IS_DEFAULT(ShenandoahCardBarrier)) {
SHENANDOAH_CHECK_FLAG_SET(ShenandoahCardBarrier);
}

// Enable card-marking post-write barrier for tracking old to young pointers
FLAG_SET_DEFAULT(ShenandoahCardBarrier, true);

if (ClassUnloading) {
FLAG_SET_DEFAULT(ShenandoahSuspendibleWorkers, true);
FLAG_SET_DEFAULT(VerifyBeforeExit, false);
Expand All @@ -55,4 +63,5 @@ void ShenandoahGenerationalMode::initialize_flags() const {
SHENANDOAH_CHECK_FLAG_SET(ShenandoahSATBBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahCardBarrier);
}
1 change: 1 addition & 0 deletions src/hotspot/share/gc/shenandoah/mode/shenandoahIUMode.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -58,4 +58,5 @@ void ShenandoahIUMode::initialize_flags() const {
SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahNMethodBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahStackWatermarkBarrier);
SHENANDOAH_CHECK_FLAG_UNSET(ShenandoahCardBarrier);
}
1 change: 1 addition & 0 deletions src/hotspot/share/gc/shenandoah/mode/shenandoahMode.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
#ifndef SHARE_GC_SHENANDOAH_MODE_SHENANDOAHMODE_HPP
#define SHARE_GC_SHENANDOAH_MODE_SHENANDOAHMODE_HPP

#include "gc/shared/gc_globals.hpp"
#include "memory/allocation.hpp"
#include "runtime/java.hpp"
#include "utilities/formatBuffer.hpp"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ void ShenandoahPassiveMode::initialize_flags() const {
SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahCloneBarrier);
SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahNMethodBarrier);
SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahStackWatermarkBarrier);
SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahCardBarrier);

// Final configuration checks
// No barriers are required to run.
Expand Down
2 changes: 2 additions & 0 deletions src/hotspot/share/gc/shenandoah/mode/shenandoahSATBMode.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -46,4 +46,6 @@ void ShenandoahSATBMode::initialize_flags() const {
SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahNMethodBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahStackWatermarkBarrier);
assert(strcmp(ShenandoahGCMode, "generational") != 0, "Error");
SHENANDOAH_CHECK_FLAG_UNSET(ShenandoahCardBarrier);
}
1 change: 1 addition & 0 deletions src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ void ShenandoahArguments::initialize() {
FLAG_SET_DEFAULT(ShenandoahLoadRefBarrier, false);
FLAG_SET_DEFAULT(ShenandoahIUBarrier, false);
FLAG_SET_DEFAULT(ShenandoahCASBarrier, false);
FLAG_SET_DEFAULT(ShenandoahCardBarrier, false);
FLAG_SET_DEFAULT(ShenandoahCloneBarrier, false);

FLAG_SET_DEFAULT(ShenandoahVerifyOptoBarriers, false);
Expand Down

0 comments on commit ade7796

Please sign in to comment.