Skip to content

Commit

Permalink
Ensure correct strength and width of LRB runtime calls
Browse files Browse the repository at this point in the history
  • Loading branch information
Roman Kennke committed Nov 10, 2020
1 parent ab1eda0 commit 8d979ee
Show file tree
Hide file tree
Showing 13 changed files with 174 additions and 184 deletions.
3 changes: 1 addition & 2 deletions src/hotspot/cpu/x86/gc/shenandoah/c1/shenandoahBarrierSetC1_x86.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,8 +111,7 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRIt
__ xchg(access.resolved_addr(), result, result, LIR_OprFact::illegalOpr);

if (access.is_oop()) {
ShenandoahBarrierSet::AccessKind kind = ShenandoahBarrierSet::access_kind(access.decorators(), access.type());
result = load_reference_barrier(access.gen(), result, LIR_OprFact::addressConst(0), kind);
result = load_reference_barrier(access.gen(), result, LIR_OprFact::addressConst(0), access.decorators());
LIR_Opr tmp = gen->new_register(type);
__ move(result, tmp);
result = tmp;
Expand Down
114 changes: 57 additions & 57 deletions src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -268,9 +268,15 @@ void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
__ bind(done);
}

void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address src, ShenandoahBarrierSet::AccessKind kind) {
void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address src, DecoratorSet decorators) {
assert(ShenandoahLoadRefBarrier, "Should be enabled");

bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
bool is_narrow = LP64_ONLY(UseCompressedOops &&) !is_native;

Label heap_stable, not_cset;

__ block_comment("load_reference_barrier { ");
Expand All @@ -292,7 +298,7 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
__ jcc(Assembler::zero, heap_stable);

Register tmp1 = noreg, tmp2 = noreg;
if (kind == ShenandoahBarrierSet::AccessKind::NORMAL) {
if (is_strong) {
// Test for object in cset
// Allocate temporary registers
for (int i = 0; i < 8; i++) {
Expand Down Expand Up @@ -357,26 +363,22 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
}

save_xmm_registers(masm);
switch (kind) {
case ShenandoahBarrierSet::AccessKind::NORMAL:
if (UseCompressedOops) {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow), arg0, arg1);
} else {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), arg0, arg1);
}
break;
case ShenandoahBarrierSet::AccessKind::WEAK:
if (UseCompressedOops) {
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow), arg0, arg1);
} else {
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), arg0, arg1);
}
break;
case ShenandoahBarrierSet::AccessKind::NATIVE:
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), arg0, arg1);
break;
default:
ShouldNotReachHere();
if (is_strong) {
if (is_narrow) {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow), arg0, arg1);
} else {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong), arg0, arg1);
}
} else if (is_weak) {
if (is_narrow) {
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow), arg0, arg1);
} else {
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), arg0, arg1);
}
} else {
assert(is_phantom, "only remaining strength");
assert(!is_narrow, "phantom access cannot be narrow");
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom), arg0, arg1);
}
restore_xmm_registers(masm);

Expand All @@ -401,7 +403,7 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,

__ bind(not_cset);

if (kind == ShenandoahBarrierSet::AccessKind::NORMAL) {
if (is_strong) {
__ pop(tmp2);
__ pop(tmp1);
}
Expand Down Expand Up @@ -499,8 +501,7 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet d

BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);

ShenandoahBarrierSet::AccessKind kind = ShenandoahBarrierSet::access_kind(decorators, type);
load_reference_barrier(masm, dst, src, kind);
load_reference_barrier(masm, dst, src, decorators);

// Move loaded oop to final destination
if (dst != result_dst) {
Expand Down Expand Up @@ -850,18 +851,17 @@ void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assemble
__ bind(slow_path);
ce->store_parameter(res, 0);
ce->store_parameter(addr, 1);
switch (stub->kind()) {
case ShenandoahBarrierSet::AccessKind::NORMAL:
__ call(RuntimeAddress(bs->load_reference_barrier_normal_rt_code_blob()->code_begin()));
break;
case ShenandoahBarrierSet::AccessKind::WEAK:
__ call(RuntimeAddress(bs->load_reference_barrier_weak_rt_code_blob()->code_begin()));
break;
case ShenandoahBarrierSet::AccessKind::NATIVE:
__ call(RuntimeAddress(bs->load_reference_barrier_native_rt_code_blob()->code_begin()));
break;
default:
ShouldNotReachHere();
DecoratorSet decorators = stub->decorators();
bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
if (is_strong) {
__ call(RuntimeAddress(bs->load_reference_barrier_strong_rt_code_blob()->code_begin()));
} else if (is_weak) {
__ call(RuntimeAddress(bs->load_reference_barrier_weak_rt_code_blob()->code_begin()));
} else {
assert(is_phantom, "only remaining strenght");
__ call(RuntimeAddress(bs->load_reference_barrier_phantom_rt_code_blob()->code_begin()));
}
__ jmp(*stub->continuation());
}
Expand Down Expand Up @@ -926,7 +926,7 @@ void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAss
__ epilogue();
}

void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, ShenandoahBarrierSet::AccessKind kind) {
void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, DecoratorSet decorators) {
__ prologue("shenandoah_load_reference_barrier", false);
// arg0 : object to be resolved

Expand All @@ -935,26 +935,26 @@ void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_s
#ifdef _LP64
__ load_parameter(0, c_rarg0);
__ load_parameter(1, c_rarg1);
switch (kind) {
case ShenandoahBarrierSet::AccessKind::NORMAL:
if (UseCompressedOops) {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow), c_rarg0, c_rarg1);
} else {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), c_rarg0, c_rarg1);
}
break;
case ShenandoahBarrierSet::AccessKind::WEAK:
if (UseCompressedOops) {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow), c_rarg0, c_rarg1);
} else {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), c_rarg0, c_rarg1);
}
break;
case ShenandoahBarrierSet::AccessKind::NATIVE:
bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
if (is_strong) {
if (UseCompressedOops) {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow), c_rarg0,
c_rarg1);
} else {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong), c_rarg0, c_rarg1);
}
} else if (is_weak) {
if (UseCompressedOops) {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow), c_rarg0,
c_rarg1);
} else {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), c_rarg0, c_rarg1);
break;
default:
ShouldNotReachHere();
}
} else {
assert(is_phantom, "only remaining strength");
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), c_rarg0, c_rarg1);
}
#else
__ load_parameter(0, rax);
Expand Down
4 changes: 2 additions & 2 deletions src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,10 +64,10 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub);
void gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub);
void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm);
void generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, ShenandoahBarrierSet::AccessKind kind);
void generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, DecoratorSet decorators);
#endif

void load_reference_barrier(MacroAssembler* masm, Register dst, Address src, ShenandoahBarrierSet::AccessKind kind);
void load_reference_barrier(MacroAssembler* masm, Register dst, Address src, DecoratorSet decorators);

void cmpxchg_oop(MacroAssembler* masm,
Register res, Address addr, Register oldval, Register newval,
Expand Down
47 changes: 23 additions & 24 deletions src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,9 @@ void ShenandoahLoadReferenceBarrierStub::emit_code(LIR_Assembler* ce) {

ShenandoahBarrierSetC1::ShenandoahBarrierSetC1() :
_pre_barrier_c1_runtime_code_blob(NULL),
_load_reference_barrier_normal_rt_code_blob(NULL),
_load_reference_barrier_native_rt_code_blob(NULL),
_load_reference_barrier_weak_rt_code_blob(NULL) {}
_load_reference_barrier_strong_rt_code_blob(NULL),
_load_reference_barrier_weak_rt_code_blob(NULL),
_load_reference_barrier_phantom_rt_code_blob(NULL) {}

void ShenandoahBarrierSetC1::pre_barrier(LIRGenerator* gen, CodeEmitInfo* info, DecoratorSet decorators, LIR_Opr addr_opr, LIR_Opr pre_val) {
// First we test whether marking is in progress.
Expand Down Expand Up @@ -109,15 +109,15 @@ void ShenandoahBarrierSetC1::pre_barrier(LIRGenerator* gen, CodeEmitInfo* info,
__ branch_destination(slow->continuation());
}

LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr, ShenandoahBarrierSet::AccessKind kind) {
LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr, DecoratorSet decorators) {
if (ShenandoahLoadRefBarrier) {
return load_reference_barrier_impl(gen, obj, addr, kind);
return load_reference_barrier_impl(gen, obj, addr, decorators);
} else {
return obj;
}
}

LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier_impl(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr, ShenandoahBarrierSet::AccessKind kind) {
LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier_impl(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr, DecoratorSet decorators) {
assert(ShenandoahLoadRefBarrier, "Should be enabled");

obj = ensure_in_register(gen, obj, T_OBJECT);
Expand Down Expand Up @@ -150,7 +150,7 @@ LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier_impl(LIRGenerator* gen, L
}
__ cmp(lir_cond_notEqual, flag_val, LIR_OprFact::intConst(0));

CodeStub* slow = new ShenandoahLoadReferenceBarrierStub(obj, addr, result, tmp1, tmp2, kind);
CodeStub* slow = new ShenandoahLoadReferenceBarrierStub(obj, addr, result, tmp1, tmp2, decorators);
__ branch(lir_cond_notEqual, slow);
__ branch_destination(slow->continuation());

Expand Down Expand Up @@ -213,8 +213,7 @@ void ShenandoahBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result)
if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
LIR_Opr tmp = gen->new_register(T_OBJECT);
BarrierSetC1::load_at_resolved(access, tmp);
ShenandoahBarrierSet::AccessKind kind = ShenandoahBarrierSet::access_kind(decorators, type);
tmp = load_reference_barrier(gen, tmp, access.resolved_addr(), kind);
tmp = load_reference_barrier(gen, tmp, access.resolved_addr(), decorators);
__ move(tmp, result);
} else {
BarrierSetC1::load_at_resolved(access, result);
Expand Down Expand Up @@ -253,14 +252,14 @@ class C1ShenandoahPreBarrierCodeGenClosure : public StubAssemblerCodeGenClosure

class C1ShenandoahLoadReferenceBarrierCodeGenClosure : public StubAssemblerCodeGenClosure {
private:
const ShenandoahBarrierSet::AccessKind _kind;
const DecoratorSet _decorators;

public:
C1ShenandoahLoadReferenceBarrierCodeGenClosure(ShenandoahBarrierSet::AccessKind kind) : _kind(kind) {}
C1ShenandoahLoadReferenceBarrierCodeGenClosure(DecoratorSet decorators) : _decorators(decorators) {}

virtual OopMapSet* generate_code(StubAssembler* sasm) {
ShenandoahBarrierSetAssembler* bs = (ShenandoahBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
bs->generate_c1_load_reference_barrier_runtime_stub(sasm, _kind);
bs->generate_c1_load_reference_barrier_runtime_stub(sasm, _decorators);
return NULL;
}
};
Expand All @@ -271,19 +270,19 @@ void ShenandoahBarrierSetC1::generate_c1_runtime_stubs(BufferBlob* buffer_blob)
"shenandoah_pre_barrier_slow",
false, &pre_code_gen_cl);
if (ShenandoahLoadRefBarrier) {
C1ShenandoahLoadReferenceBarrierCodeGenClosure lrb_code_gen_cl(ShenandoahBarrierSet::AccessKind::NORMAL);
_load_reference_barrier_normal_rt_code_blob = Runtime1::generate_blob(buffer_blob, -1,
"shenandoah_load_reference_barrier_slow",
false, &lrb_code_gen_cl);
C1ShenandoahLoadReferenceBarrierCodeGenClosure lrb_strong_code_gen_cl(ON_STRONG_OOP_REF);
_load_reference_barrier_strong_rt_code_blob = Runtime1::generate_blob(buffer_blob, -1,
"shenandoah_load_reference_barrier_strong_slow",
false, &lrb_strong_code_gen_cl);

C1ShenandoahLoadReferenceBarrierCodeGenClosure lrb_native_code_gen_cl(ShenandoahBarrierSet::AccessKind::NATIVE);
_load_reference_barrier_native_rt_code_blob = Runtime1::generate_blob(buffer_blob, -1,
"shenandoah_load_reference_barrier_native_slow",
false, &lrb_native_code_gen_cl);

C1ShenandoahLoadReferenceBarrierCodeGenClosure lrb_weak_code_gen_cl(ShenandoahBarrierSet::AccessKind::WEAK);
C1ShenandoahLoadReferenceBarrierCodeGenClosure lrb_weak_code_gen_cl(ON_WEAK_OOP_REF);
_load_reference_barrier_weak_rt_code_blob = Runtime1::generate_blob(buffer_blob, -1,
"shenandoah_load_reference_barrier_weak_slow",
false, &lrb_weak_code_gen_cl);
"shenandoah_load_reference_barrier_weak_slow",
false, &lrb_weak_code_gen_cl);

C1ShenandoahLoadReferenceBarrierCodeGenClosure lrb_phantom_code_gen_cl(ON_PHANTOM_OOP_REF);
_load_reference_barrier_phantom_rt_code_blob = Runtime1::generate_blob(buffer_blob, -1,
"shenandoah_load_reference_barrier_phantom_slow",
false, &lrb_phantom_code_gen_cl);
}
}
33 changes: 17 additions & 16 deletions src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,10 +94,10 @@ class ShenandoahLoadReferenceBarrierStub: public CodeStub {
LIR_Opr _result;
LIR_Opr _tmp1;
LIR_Opr _tmp2;
ShenandoahBarrierSet::AccessKind _kind;
DecoratorSet _decorators;
public:
ShenandoahLoadReferenceBarrierStub(LIR_Opr obj, LIR_Opr addr, LIR_Opr result, LIR_Opr tmp1, LIR_Opr tmp2, ShenandoahBarrierSet::AccessKind kind) :
_obj(obj), _addr(addr), _result(result), _tmp1(tmp1), _tmp2(tmp2), _kind(kind)
ShenandoahLoadReferenceBarrierStub(LIR_Opr obj, LIR_Opr addr, LIR_Opr result, LIR_Opr tmp1, LIR_Opr tmp2, DecoratorSet decorators) :
_obj(obj), _addr(addr), _result(result), _tmp1(tmp1), _tmp2(tmp2), _decorators(decorators)
{
assert(_obj->is_register(), "should be register");
assert(_addr->is_register(), "should be register");
Expand All @@ -111,7 +111,7 @@ class ShenandoahLoadReferenceBarrierStub: public CodeStub {
LIR_Opr result() const { return _result; }
LIR_Opr tmp1() const { return _tmp1; }
LIR_Opr tmp2() const { return _tmp2; }
ShenandoahBarrierSet::AccessKind kind() const { return _kind; }
DecoratorSet decorators() const { return _decorators; }

virtual void emit_code(LIR_Assembler* e);
virtual void visit(LIR_OpVisitState* visitor) {
Expand Down Expand Up @@ -190,16 +190,16 @@ class LIR_OpShenandoahCompareAndSwap : public LIR_Op {
class ShenandoahBarrierSetC1 : public BarrierSetC1 {
private:
CodeBlob* _pre_barrier_c1_runtime_code_blob;
CodeBlob* _load_reference_barrier_normal_rt_code_blob;
CodeBlob* _load_reference_barrier_native_rt_code_blob;
CodeBlob* _load_reference_barrier_strong_rt_code_blob;
CodeBlob* _load_reference_barrier_weak_rt_code_blob;
CodeBlob* _load_reference_barrier_phantom_rt_code_blob;

void pre_barrier(LIRGenerator* gen, CodeEmitInfo* info, DecoratorSet decorators, LIR_Opr addr_opr, LIR_Opr pre_val);

LIR_Opr load_reference_barrier(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr, ShenandoahBarrierSet::AccessKind kind);
LIR_Opr load_reference_barrier(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr, DecoratorSet decorators);
LIR_Opr storeval_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, DecoratorSet decorators);

LIR_Opr load_reference_barrier_impl(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr, ShenandoahBarrierSet::AccessKind kind);
LIR_Opr load_reference_barrier_impl(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr, DecoratorSet decorators);

LIR_Opr ensure_in_register(LIRGenerator* gen, LIR_Opr obj, BasicType type);

Expand All @@ -211,20 +211,21 @@ class ShenandoahBarrierSetC1 : public BarrierSetC1 {
return _pre_barrier_c1_runtime_code_blob;
}

CodeBlob* load_reference_barrier_normal_rt_code_blob() {
assert(_load_reference_barrier_normal_rt_code_blob != NULL, "");
return _load_reference_barrier_normal_rt_code_blob;
}

CodeBlob* load_reference_barrier_native_rt_code_blob() {
assert(_load_reference_barrier_native_rt_code_blob != NULL, "");
return _load_reference_barrier_native_rt_code_blob;
CodeBlob* load_reference_barrier_strong_rt_code_blob() {
assert(_load_reference_barrier_strong_rt_code_blob != NULL, "");
return _load_reference_barrier_strong_rt_code_blob;
}

CodeBlob* load_reference_barrier_weak_rt_code_blob() {
assert(_load_reference_barrier_weak_rt_code_blob != NULL, "");
return _load_reference_barrier_weak_rt_code_blob;
}

CodeBlob* load_reference_barrier_phantom_rt_code_blob() {
assert(_load_reference_barrier_phantom_rt_code_blob != NULL, "");
return _load_reference_barrier_phantom_rt_code_blob;
}

protected:

virtual void store_at_resolved(LIRAccess& access, LIR_Opr value);
Expand Down
Loading

0 comments on commit 8d979ee

Please sign in to comment.