From b0c28fadaa235ce9dd0c89fc9ff2998df50473bb Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Fri, 13 Nov 2020 09:47:00 +0000 Subject: [PATCH] 8256011: Shenandoah: Don't resurrect finalizably reachable objects Reviewed-by: shade, zgu --- .../c1/shenandoahBarrierSetC1_aarch64.cpp | 2 +- .../shenandoahBarrierSetAssembler_aarch64.cpp | 144 ++++++++------- .../shenandoahBarrierSetAssembler_aarch64.hpp | 4 +- .../c1/shenandoahBarrierSetC1_x86.cpp | 3 +- .../shenandoahBarrierSetAssembler_x86.cpp | 165 +++++++++--------- .../shenandoahBarrierSetAssembler_x86.hpp | 4 +- .../shenandoah/c1/shenandoahBarrierSetC1.cpp | 51 +++--- .../shenandoah/c1/shenandoahBarrierSetC1.hpp | 35 ++-- .../shenandoah/c2/shenandoahBarrierSetC2.cpp | 22 ++- .../gc/shenandoah/c2/shenandoahSupport.cpp | 79 +++++---- .../gc/shenandoah/c2/shenandoahSupport.hpp | 8 +- .../gc/shenandoah/shenandoahBarrierSet.cpp | 10 -- .../gc/shenandoah/shenandoahBarrierSet.hpp | 29 +-- .../shenandoahBarrierSet.inline.hpp | 12 +- .../share/gc/shenandoah/shenandoahRuntime.cpp | 12 +- .../share/gc/shenandoah/shenandoahRuntime.hpp | 6 +- 16 files changed, 303 insertions(+), 283 deletions(-) diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/c1/shenandoahBarrierSetC1_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shenandoah/c1/shenandoahBarrierSetC1_aarch64.cpp index db9c7577e60..c05b5f99524 100644 --- a/src/hotspot/cpu/aarch64/gc/shenandoah/c1/shenandoahBarrierSetC1_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/gc/shenandoah/c1/shenandoahBarrierSetC1_aarch64.cpp @@ -109,7 +109,7 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRIt __ xchg(access.resolved_addr(), value_opr, result, tmp); if (access.is_oop()) { - result = load_reference_barrier(access.gen(), result, LIR_OprFact::addressConst(0), ShenandoahBarrierSet::AccessKind::NORMAL); + result = load_reference_barrier(access.gen(), result, LIR_OprFact::addressConst(0), access.decorators()); LIR_Opr tmp = gen->new_register(type); __ move(result, tmp); result = tmp; diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp index 840464b251f..151fa2a1b5e 100644 --- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp @@ -225,11 +225,17 @@ void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssemb } } -void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr, ShenandoahBarrierSet::AccessKind kind) { +void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr, DecoratorSet decorators) { assert(ShenandoahLoadRefBarrier, "Should be enabled"); assert(dst != rscratch2, "need rscratch2"); assert_different_registers(load_addr.base(), load_addr.index(), rscratch1, rscratch2); + bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators); + bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators); + bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators); + bool is_native = ShenandoahBarrierSet::is_native_access(decorators); + bool is_narrow = UseCompressedOops && !is_native; + Label heap_stable, not_cset; __ enter(); Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); @@ -252,7 +258,7 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, __ mov(r0, dst); // Test for in-cset - if (kind == ShenandoahBarrierSet::AccessKind::NORMAL) { + if (is_strong) { __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr()); __ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint()); __ ldrb(rscratch2, Address(rscratch2, rscratch1)); @@ -260,26 +266,22 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, } __ push_call_clobbered_registers(); - switch (kind) { - case ShenandoahBarrierSet::AccessKind::NORMAL: - if (UseCompressedOops) { - __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow)); - } else { - __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier)); - } - break; - case ShenandoahBarrierSet::AccessKind::WEAK: - if (UseCompressedOops) { - __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow)); - } else { - __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak)); - } - break; - case ShenandoahBarrierSet::AccessKind::NATIVE: + if (is_strong) { + if (is_narrow) { + __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow)); + } else { + __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong)); + } + } else if (is_weak) { + if (is_narrow) { + __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow)); + } else { __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak)); - break; - default: - ShouldNotReachHere(); + } + } else { + assert(is_phantom, "only remaining strength"); + assert(!is_narrow, "phantom access cannot be narrow"); + __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak)); } __ blr(lr); __ mov(rscratch1, r0); @@ -338,8 +340,7 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet d BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); - ShenandoahBarrierSet::AccessKind kind = ShenandoahBarrierSet::access_kind(decorators, type); - load_reference_barrier(masm, dst, src, kind); + load_reference_barrier(masm, dst, src, decorators); if (dst != result_dst) { __ mov(result_dst, dst); @@ -617,6 +618,12 @@ void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assemble ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); __ bind(*stub->entry()); + DecoratorSet decorators = stub->decorators(); + bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators); + bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators); + bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators); + bool is_native = ShenandoahBarrierSet::is_native_access(decorators); + Register obj = stub->obj()->as_register(); Register res = stub->result()->as_register(); Register addr = stub->addr()->as_pointer_register(); @@ -629,42 +636,27 @@ void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assemble __ mov(res, obj); } - // Check for null. - __ cbz(res, *stub->continuation()); - - // Check for object in cset. - __ mov(tmp2, ShenandoahHeap::in_cset_fast_test_addr()); - __ lsr(tmp1, res, ShenandoahHeapRegion::region_size_bytes_shift_jint()); - __ ldrb(tmp2, Address(tmp2, tmp1)); - __ cbz(tmp2, *stub->continuation()); - - // Check if object is already forwarded. - Label slow_path; - __ ldr(tmp1, Address(res, oopDesc::mark_offset_in_bytes())); - __ eon(tmp1, tmp1, zr); - __ ands(zr, tmp1, markWord::lock_mask_in_place); - __ br(Assembler::NE, slow_path); - - // Decode forwarded object. - __ orr(tmp1, tmp1, markWord::marked_value); - __ eon(res, tmp1, zr); - __ b(*stub->continuation()); + if (is_strong) { + // Check for object in cset. + __ mov(tmp2, ShenandoahHeap::in_cset_fast_test_addr()); + __ lsr(tmp1, res, ShenandoahHeapRegion::region_size_bytes_shift_jint()); + __ ldrb(tmp2, Address(tmp2, tmp1)); + __ cbz(tmp2, *stub->continuation()); + } - __ bind(slow_path); ce->store_parameter(res, 0); ce->store_parameter(addr, 1); - switch (stub->kind()) { - case ShenandoahBarrierSet::AccessKind::NORMAL: - __ far_call(RuntimeAddress(bs->load_reference_barrier_normal_rt_code_blob()->code_begin())); - break; - case ShenandoahBarrierSet::AccessKind::WEAK: - __ far_call(RuntimeAddress(bs->load_reference_barrier_weak_rt_code_blob()->code_begin())); - break; - case ShenandoahBarrierSet::AccessKind::NATIVE: - __ far_call(RuntimeAddress(bs->load_reference_barrier_native_rt_code_blob()->code_begin())); - break; - default: - ShouldNotReachHere(); + if (is_strong) { + if (is_native) { + __ far_call(RuntimeAddress(bs->load_reference_barrier_strong_native_rt_code_blob()->code_begin())); + } else { + __ far_call(RuntimeAddress(bs->load_reference_barrier_strong_rt_code_blob()->code_begin())); + } + } else if (is_weak) { + __ far_call(RuntimeAddress(bs->load_reference_barrier_weak_rt_code_blob()->code_begin())); + } else { + assert(is_phantom, "only remaining strength"); + __ far_call(RuntimeAddress(bs->load_reference_barrier_phantom_rt_code_blob()->code_begin())); } __ b(*stub->continuation()); @@ -720,33 +712,39 @@ void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAss __ epilogue(); } -void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, ShenandoahBarrierSet::AccessKind kind) { +void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, DecoratorSet decorators) { __ prologue("shenandoah_load_reference_barrier", false); // arg0 : object to be resolved __ push_call_clobbered_registers(); __ load_parameter(0, r0); __ load_parameter(1, r1); - switch (kind) { - case ShenandoahBarrierSet::AccessKind::NORMAL: - if (UseCompressedOops) { - __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow)); - } else { - __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier)); - } - break; - case ShenandoahBarrierSet::AccessKind::WEAK: + + bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators); + bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators); + bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators); + bool is_native = ShenandoahBarrierSet::is_native_access(decorators); + if (is_strong) { + if (is_native) { + __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong)); + } else { if (UseCompressedOops) { - __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow)); + __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow)); } else { - __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak)); + __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong)); } - break; - case ShenandoahBarrierSet::AccessKind::NATIVE: + } + } else if (is_weak) { + assert(!is_native, "weak must not be called off-heap"); + if (UseCompressedOops) { + __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow)); + } else { __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak)); - break; - default: - ShouldNotReachHere(); + } + } else { + assert(is_phantom, "only remaining strength"); + assert(is_native, "phantom must only be called off-heap"); + __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom)); } __ blr(lr); __ mov(rscratch1, r0); diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp index 60303725fd8..bbf3fe131f8 100644 --- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp @@ -56,7 +56,7 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler { void resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp = noreg); void resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp = noreg); - void load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr, ShenandoahBarrierSet::AccessKind kind); + void load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr, DecoratorSet decorators); public: @@ -66,7 +66,7 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler { void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub); void gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub); void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm); - void generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, ShenandoahBarrierSet::AccessKind kind); + void generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, DecoratorSet decorators); #endif virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, diff --git a/src/hotspot/cpu/x86/gc/shenandoah/c1/shenandoahBarrierSetC1_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/c1/shenandoahBarrierSetC1_x86.cpp index 2aac0608207..65599ccddf6 100644 --- a/src/hotspot/cpu/x86/gc/shenandoah/c1/shenandoahBarrierSetC1_x86.cpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/c1/shenandoahBarrierSetC1_x86.cpp @@ -111,8 +111,7 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRIt __ xchg(access.resolved_addr(), result, result, LIR_OprFact::illegalOpr); if (access.is_oop()) { - ShenandoahBarrierSet::AccessKind kind = ShenandoahBarrierSet::access_kind(access.decorators(), access.type()); - result = load_reference_barrier(access.gen(), result, LIR_OprFact::addressConst(0), kind); + result = load_reference_barrier(access.gen(), result, LIR_OprFact::addressConst(0), access.decorators()); LIR_Opr tmp = gen->new_register(type); __ move(result, tmp); result = tmp; diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp index 40f16ef2731..b9abe5be130 100644 --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp @@ -268,9 +268,15 @@ void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm, __ bind(done); } -void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address src, ShenandoahBarrierSet::AccessKind kind) { +void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address src, DecoratorSet decorators) { assert(ShenandoahLoadRefBarrier, "Should be enabled"); + bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators); + bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators); + bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators); + bool is_native = ShenandoahBarrierSet::is_native_access(decorators); + bool is_narrow = UseCompressedOops && !is_native; + Label heap_stable, not_cset; __ block_comment("load_reference_barrier { "); @@ -292,7 +298,7 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, __ jcc(Assembler::zero, heap_stable); Register tmp1 = noreg, tmp2 = noreg; - if (kind == ShenandoahBarrierSet::AccessKind::NORMAL) { + if (is_strong) { // Test for object in cset // Allocate temporary registers for (int i = 0; i < 8; i++) { @@ -357,26 +363,22 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, } save_xmm_registers(masm); - switch (kind) { - case ShenandoahBarrierSet::AccessKind::NORMAL: - if (UseCompressedOops) { - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow), arg0, arg1); - } else { - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), arg0, arg1); - } - break; - case ShenandoahBarrierSet::AccessKind::WEAK: - if (UseCompressedOops) { - __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow), arg0, arg1); - } else { - __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), arg0, arg1); - } - break; - case ShenandoahBarrierSet::AccessKind::NATIVE: - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), arg0, arg1); - break; - default: - ShouldNotReachHere(); + if (is_strong) { + if (is_narrow) { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow), arg0, arg1); + } else { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong), arg0, arg1); + } + } else if (is_weak) { + if (is_narrow) { + __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow), arg0, arg1); + } else { + __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), arg0, arg1); + } + } else { + assert(is_phantom, "only remaining strength"); + assert(!is_narrow, "phantom access cannot be narrow"); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom), arg0, arg1); } restore_xmm_registers(masm); @@ -401,7 +403,7 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, __ bind(not_cset); - if (kind == ShenandoahBarrierSet::AccessKind::NORMAL) { + if (is_strong) { __ pop(tmp2); __ pop(tmp1); } @@ -499,8 +501,7 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet d BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); - ShenandoahBarrierSet::AccessKind kind = ShenandoahBarrierSet::access_kind(decorators, type); - load_reference_barrier(masm, dst, src, kind); + load_reference_barrier(masm, dst, src, decorators); // Move loaded oop to final destination if (dst != result_dst) { @@ -813,6 +814,12 @@ void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assemble ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); __ bind(*stub->entry()); + DecoratorSet decorators = stub->decorators(); + bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators); + bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators); + bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators); + bool is_native = ShenandoahBarrierSet::is_native_access(decorators); + Register obj = stub->obj()->as_register(); Register res = stub->result()->as_register(); Register addr = stub->addr()->as_pointer_register(); @@ -828,40 +835,37 @@ void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assemble __ mov(res, obj); } - // Check for null. - __ testptr(res, res); - __ jcc(Assembler::zero, *stub->continuation()); - - // Check for object being in the collection set. - __ mov(tmp1, res); - __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint()); - __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr()); + if (is_strong) { + // Check for object being in the collection set. + __ mov(tmp1, res); + __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint()); + __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr()); #ifdef _LP64 - __ movbool(tmp2, Address(tmp2, tmp1, Address::times_1)); - __ testbool(tmp2); + __ movbool(tmp2, Address(tmp2, tmp1, Address::times_1)); + __ testbool(tmp2); #else - // On x86_32, C1 register allocator can give us the register without 8-bit support. - // Do the full-register access and test to avoid compilation failures. - __ movptr(tmp2, Address(tmp2, tmp1, Address::times_1)); - __ testptr(tmp2, 0xFF); + // On x86_32, C1 register allocator can give us the register without 8-bit support. + // Do the full-register access and test to avoid compilation failures. + __ movptr(tmp2, Address(tmp2, tmp1, Address::times_1)); + __ testptr(tmp2, 0xFF); #endif - __ jcc(Assembler::zero, *stub->continuation()); + __ jcc(Assembler::zero, *stub->continuation()); + } __ bind(slow_path); ce->store_parameter(res, 0); ce->store_parameter(addr, 1); - switch (stub->kind()) { - case ShenandoahBarrierSet::AccessKind::NORMAL: - __ call(RuntimeAddress(bs->load_reference_barrier_normal_rt_code_blob()->code_begin())); - break; - case ShenandoahBarrierSet::AccessKind::WEAK: - __ call(RuntimeAddress(bs->load_reference_barrier_weak_rt_code_blob()->code_begin())); - break; - case ShenandoahBarrierSet::AccessKind::NATIVE: - __ call(RuntimeAddress(bs->load_reference_barrier_native_rt_code_blob()->code_begin())); - break; - default: - ShouldNotReachHere(); + if (is_strong) { + if (is_native) { + __ call(RuntimeAddress(bs->load_reference_barrier_strong_native_rt_code_blob()->code_begin())); + } else { + __ call(RuntimeAddress(bs->load_reference_barrier_strong_rt_code_blob()->code_begin())); + } + } else if (is_weak) { + __ call(RuntimeAddress(bs->load_reference_barrier_weak_rt_code_blob()->code_begin())); + } else { + assert(is_phantom, "only remaining strength"); + __ call(RuntimeAddress(bs->load_reference_barrier_phantom_rt_code_blob()->code_begin())); } __ jmp(*stub->continuation()); } @@ -926,49 +930,52 @@ void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAss __ epilogue(); } -void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, ShenandoahBarrierSet::AccessKind kind) { +void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, DecoratorSet decorators) { __ prologue("shenandoah_load_reference_barrier", false); // arg0 : object to be resolved __ save_live_registers_no_oop_map(true); + bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators); + bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators); + bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators); + bool is_native = ShenandoahBarrierSet::is_native_access(decorators); + #ifdef _LP64 __ load_parameter(0, c_rarg0); __ load_parameter(1, c_rarg1); - switch (kind) { - case ShenandoahBarrierSet::AccessKind::NORMAL: - if (UseCompressedOops) { - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow), c_rarg0, c_rarg1); - } else { - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), c_rarg0, c_rarg1); - } - break; - case ShenandoahBarrierSet::AccessKind::WEAK: + if (is_strong) { + if (is_native) { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong), c_rarg0, c_rarg1); + } else { if (UseCompressedOops) { - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow), c_rarg0, c_rarg1); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow), c_rarg0, c_rarg1); } else { - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), c_rarg0, c_rarg1); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong), c_rarg0, c_rarg1); } - break; - case ShenandoahBarrierSet::AccessKind::NATIVE: + } + } else if (is_weak) { + assert(!is_native, "weak must not be called off-heap"); + if (UseCompressedOops) { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow), c_rarg0, c_rarg1); + } else { __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), c_rarg0, c_rarg1); - break; - default: - ShouldNotReachHere(); + } + } else { + assert(is_phantom, "only remaining strength"); + assert(is_native, "phantom must only be called off-heap"); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom), c_rarg0, c_rarg1); } #else __ load_parameter(0, rax); __ load_parameter(1, rbx); - switch (kind) { - case ShenandoahBarrierSet::AccessKind::NORMAL: - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), rax, rbx); - break; - case ShenandoahBarrierSet::AccessKind::WEAK: - case ShenandoahBarrierSet::AccessKind::NATIVE: - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), rax, rbx); - break; - default: - ShouldNotReachHere(); + if (is_strong) { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong), rax, rbx); + } else if (is_weak) { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), rax, rbx); + } else { + assert(is_phantom, "only remaining strength"); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom), rax, rbx); } #endif diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp index 108b5670206..686389aae66 100644 --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp @@ -64,10 +64,10 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler { void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub); void gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub); void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm); - void generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, ShenandoahBarrierSet::AccessKind kind); + void generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, DecoratorSet decorators); #endif - void load_reference_barrier(MacroAssembler* masm, Register dst, Address src, ShenandoahBarrierSet::AccessKind kind); + void load_reference_barrier(MacroAssembler* masm, Register dst, Address src, DecoratorSet decorators); void cmpxchg_oop(MacroAssembler* masm, Register res, Address addr, Register oldval, Register newval, diff --git a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp index f0c034d1401..7c63902efbd 100644 --- a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp +++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp @@ -51,9 +51,10 @@ void ShenandoahLoadReferenceBarrierStub::emit_code(LIR_Assembler* ce) { ShenandoahBarrierSetC1::ShenandoahBarrierSetC1() : _pre_barrier_c1_runtime_code_blob(NULL), - _load_reference_barrier_normal_rt_code_blob(NULL), - _load_reference_barrier_native_rt_code_blob(NULL), - _load_reference_barrier_weak_rt_code_blob(NULL) {} + _load_reference_barrier_strong_rt_code_blob(NULL), + _load_reference_barrier_strong_native_rt_code_blob(NULL), + _load_reference_barrier_weak_rt_code_blob(NULL), + _load_reference_barrier_phantom_rt_code_blob(NULL) {} void ShenandoahBarrierSetC1::pre_barrier(LIRGenerator* gen, CodeEmitInfo* info, DecoratorSet decorators, LIR_Opr addr_opr, LIR_Opr pre_val) { // First we test whether marking is in progress. @@ -109,15 +110,15 @@ void ShenandoahBarrierSetC1::pre_barrier(LIRGenerator* gen, CodeEmitInfo* info, __ branch_destination(slow->continuation()); } -LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr, ShenandoahBarrierSet::AccessKind kind) { +LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr, DecoratorSet decorators) { if (ShenandoahLoadRefBarrier) { - return load_reference_barrier_impl(gen, obj, addr, kind); + return load_reference_barrier_impl(gen, obj, addr, decorators); } else { return obj; } } -LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier_impl(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr, ShenandoahBarrierSet::AccessKind kind) { +LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier_impl(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr, DecoratorSet decorators) { assert(ShenandoahLoadRefBarrier, "Should be enabled"); obj = ensure_in_register(gen, obj, T_OBJECT); @@ -150,7 +151,7 @@ LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier_impl(LIRGenerator* gen, L } __ cmp(lir_cond_notEqual, flag_val, LIR_OprFact::intConst(0)); - CodeStub* slow = new ShenandoahLoadReferenceBarrierStub(obj, addr, result, tmp1, tmp2, kind); + CodeStub* slow = new ShenandoahLoadReferenceBarrierStub(obj, addr, result, tmp1, tmp2, decorators); __ branch(lir_cond_notEqual, slow); __ branch_destination(slow->continuation()); @@ -213,8 +214,7 @@ void ShenandoahBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) { LIR_Opr tmp = gen->new_register(T_OBJECT); BarrierSetC1::load_at_resolved(access, tmp); - ShenandoahBarrierSet::AccessKind kind = ShenandoahBarrierSet::access_kind(decorators, type); - tmp = load_reference_barrier(gen, tmp, access.resolved_addr(), kind); + tmp = load_reference_barrier(gen, tmp, access.resolved_addr(), decorators); __ move(tmp, result); } else { BarrierSetC1::load_at_resolved(access, result); @@ -253,14 +253,14 @@ class C1ShenandoahPreBarrierCodeGenClosure : public StubAssemblerCodeGenClosure class C1ShenandoahLoadReferenceBarrierCodeGenClosure : public StubAssemblerCodeGenClosure { private: - const ShenandoahBarrierSet::AccessKind _kind; + const DecoratorSet _decorators; public: - C1ShenandoahLoadReferenceBarrierCodeGenClosure(ShenandoahBarrierSet::AccessKind kind) : _kind(kind) {} + C1ShenandoahLoadReferenceBarrierCodeGenClosure(DecoratorSet decorators) : _decorators(decorators) {} virtual OopMapSet* generate_code(StubAssembler* sasm) { ShenandoahBarrierSetAssembler* bs = (ShenandoahBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); - bs->generate_c1_load_reference_barrier_runtime_stub(sasm, _kind); + bs->generate_c1_load_reference_barrier_runtime_stub(sasm, _decorators); return NULL; } }; @@ -271,19 +271,24 @@ void ShenandoahBarrierSetC1::generate_c1_runtime_stubs(BufferBlob* buffer_blob) "shenandoah_pre_barrier_slow", false, &pre_code_gen_cl); if (ShenandoahLoadRefBarrier) { - C1ShenandoahLoadReferenceBarrierCodeGenClosure lrb_code_gen_cl(ShenandoahBarrierSet::AccessKind::NORMAL); - _load_reference_barrier_normal_rt_code_blob = Runtime1::generate_blob(buffer_blob, -1, - "shenandoah_load_reference_barrier_slow", - false, &lrb_code_gen_cl); + C1ShenandoahLoadReferenceBarrierCodeGenClosure lrb_strong_code_gen_cl(ON_STRONG_OOP_REF); + _load_reference_barrier_strong_rt_code_blob = Runtime1::generate_blob(buffer_blob, -1, + "shenandoah_load_reference_barrier_strong_slow", + false, &lrb_strong_code_gen_cl); - C1ShenandoahLoadReferenceBarrierCodeGenClosure lrb_native_code_gen_cl(ShenandoahBarrierSet::AccessKind::NATIVE); - _load_reference_barrier_native_rt_code_blob = Runtime1::generate_blob(buffer_blob, -1, - "shenandoah_load_reference_barrier_native_slow", - false, &lrb_native_code_gen_cl); + C1ShenandoahLoadReferenceBarrierCodeGenClosure lrb_strong_native_code_gen_cl(ON_STRONG_OOP_REF | IN_NATIVE); + _load_reference_barrier_strong_native_rt_code_blob = Runtime1::generate_blob(buffer_blob, -1, + "shenandoah_load_reference_barrier_strong_native_slow", + false, &lrb_strong_native_code_gen_cl); - C1ShenandoahLoadReferenceBarrierCodeGenClosure lrb_weak_code_gen_cl(ShenandoahBarrierSet::AccessKind::WEAK); + C1ShenandoahLoadReferenceBarrierCodeGenClosure lrb_weak_code_gen_cl(ON_WEAK_OOP_REF); _load_reference_barrier_weak_rt_code_blob = Runtime1::generate_blob(buffer_blob, -1, - "shenandoah_load_reference_barrier_weak_slow", - false, &lrb_weak_code_gen_cl); + "shenandoah_load_reference_barrier_weak_slow", + false, &lrb_weak_code_gen_cl); + + C1ShenandoahLoadReferenceBarrierCodeGenClosure lrb_phantom_code_gen_cl(ON_PHANTOM_OOP_REF | IN_NATIVE); + _load_reference_barrier_phantom_rt_code_blob = Runtime1::generate_blob(buffer_blob, -1, + "shenandoah_load_reference_barrier_phantom_slow", + false, &lrb_phantom_code_gen_cl); } } diff --git a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp index eb65475e609..2047ab91ead 100644 --- a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp +++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp @@ -94,10 +94,10 @@ class ShenandoahLoadReferenceBarrierStub: public CodeStub { LIR_Opr _result; LIR_Opr _tmp1; LIR_Opr _tmp2; - ShenandoahBarrierSet::AccessKind _kind; + DecoratorSet _decorators; public: - ShenandoahLoadReferenceBarrierStub(LIR_Opr obj, LIR_Opr addr, LIR_Opr result, LIR_Opr tmp1, LIR_Opr tmp2, ShenandoahBarrierSet::AccessKind kind) : - _obj(obj), _addr(addr), _result(result), _tmp1(tmp1), _tmp2(tmp2), _kind(kind) + ShenandoahLoadReferenceBarrierStub(LIR_Opr obj, LIR_Opr addr, LIR_Opr result, LIR_Opr tmp1, LIR_Opr tmp2, DecoratorSet decorators) : + _obj(obj), _addr(addr), _result(result), _tmp1(tmp1), _tmp2(tmp2), _decorators(decorators) { assert(_obj->is_register(), "should be register"); assert(_addr->is_register(), "should be register"); @@ -111,7 +111,7 @@ class ShenandoahLoadReferenceBarrierStub: public CodeStub { LIR_Opr result() const { return _result; } LIR_Opr tmp1() const { return _tmp1; } LIR_Opr tmp2() const { return _tmp2; } - ShenandoahBarrierSet::AccessKind kind() const { return _kind; } + DecoratorSet decorators() const { return _decorators; } virtual void emit_code(LIR_Assembler* e); virtual void visit(LIR_OpVisitState* visitor) { @@ -190,16 +190,17 @@ class LIR_OpShenandoahCompareAndSwap : public LIR_Op { class ShenandoahBarrierSetC1 : public BarrierSetC1 { private: CodeBlob* _pre_barrier_c1_runtime_code_blob; - CodeBlob* _load_reference_barrier_normal_rt_code_blob; - CodeBlob* _load_reference_barrier_native_rt_code_blob; + CodeBlob* _load_reference_barrier_strong_rt_code_blob; + CodeBlob* _load_reference_barrier_strong_native_rt_code_blob; CodeBlob* _load_reference_barrier_weak_rt_code_blob; + CodeBlob* _load_reference_barrier_phantom_rt_code_blob; void pre_barrier(LIRGenerator* gen, CodeEmitInfo* info, DecoratorSet decorators, LIR_Opr addr_opr, LIR_Opr pre_val); - LIR_Opr load_reference_barrier(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr, ShenandoahBarrierSet::AccessKind kind); + LIR_Opr load_reference_barrier(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr, DecoratorSet decorators); LIR_Opr storeval_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, DecoratorSet decorators); - LIR_Opr load_reference_barrier_impl(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr, ShenandoahBarrierSet::AccessKind kind); + LIR_Opr load_reference_barrier_impl(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr, DecoratorSet decorators); LIR_Opr ensure_in_register(LIRGenerator* gen, LIR_Opr obj, BasicType type); @@ -211,20 +212,26 @@ class ShenandoahBarrierSetC1 : public BarrierSetC1 { return _pre_barrier_c1_runtime_code_blob; } - CodeBlob* load_reference_barrier_normal_rt_code_blob() { - assert(_load_reference_barrier_normal_rt_code_blob != NULL, ""); - return _load_reference_barrier_normal_rt_code_blob; + CodeBlob* load_reference_barrier_strong_rt_code_blob() { + assert(_load_reference_barrier_strong_rt_code_blob != NULL, ""); + return _load_reference_barrier_strong_rt_code_blob; } - CodeBlob* load_reference_barrier_native_rt_code_blob() { - assert(_load_reference_barrier_native_rt_code_blob != NULL, ""); - return _load_reference_barrier_native_rt_code_blob; + CodeBlob* load_reference_barrier_strong_native_rt_code_blob() { + assert(_load_reference_barrier_strong_native_rt_code_blob != NULL, ""); + return _load_reference_barrier_strong_native_rt_code_blob; } CodeBlob* load_reference_barrier_weak_rt_code_blob() { assert(_load_reference_barrier_weak_rt_code_blob != NULL, ""); return _load_reference_barrier_weak_rt_code_blob; } + + CodeBlob* load_reference_barrier_phantom_rt_code_blob() { + assert(_load_reference_barrier_phantom_rt_code_blob != NULL, ""); + return _load_reference_barrier_phantom_rt_code_blob; + } + protected: virtual void store_at_resolved(LIRAccess& access, LIR_Opr value); diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp index e13065e56c2..f901767aeb8 100644 --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp @@ -303,10 +303,11 @@ bool ShenandoahBarrierSetC2::is_shenandoah_lrb_call(Node* call) { } address entry_point = call->as_CallLeaf()->entry_point(); - return (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier)) || - (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow)) || + return (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong)) || + (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow)) || (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak)) || - (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow)); + (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow)) || + (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom)); } bool ShenandoahBarrierSetC2::is_shenandoah_marking_if(PhaseTransform *phase, Node* n) { @@ -546,8 +547,7 @@ Node* ShenandoahBarrierSetC2::load_at_resolved(C2Access& access, const Type* val // 2: apply LRB if needed if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) { - ShenandoahBarrierSet::AccessKind kind = ShenandoahBarrierSet::access_kind(decorators, type); - load = new ShenandoahLoadReferenceBarrierNode(NULL, load, kind); + load = new ShenandoahLoadReferenceBarrierNode(NULL, load, decorators); if (access.is_parse_access()) { load = static_cast(access).kit()->gvn().transform(load); } else { @@ -644,8 +644,7 @@ Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess load_store = kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type())); } #endif - ShenandoahBarrierSet::AccessKind kind = ShenandoahBarrierSet::access_kind(access.decorators(), access.type()); - load_store = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, load_store, kind)); + load_store = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, load_store, access.decorators())); return load_store; } return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type); @@ -713,8 +712,7 @@ Node* ShenandoahBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& acces } Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type); if (access.is_oop()) { - ShenandoahBarrierSet::AccessKind kind = ShenandoahBarrierSet::access_kind(access.decorators(), access.type()); - result = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, result, kind)); + result = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, result, access.decorators())); shenandoah_write_barrier_pre(kit, false /* do_load */, NULL, NULL, max_juint, NULL, NULL, result /* pre_val */, T_OBJECT); @@ -1062,15 +1060,15 @@ Node* ShenandoahBarrierSetC2::ideal_node(PhaseGVN* phase, Node* n, bool can_resh Node* in1 = n->in(1); Node* in2 = n->in(2); - // If one input is NULL, then step over the barriers normal LRB barriers on the other input + // If one input is NULL, then step over the strong LRB barriers on the other input if (in1->bottom_type() == TypePtr::NULL_PTR && !((in2->Opcode() == Op_ShenandoahLoadReferenceBarrier) && - ((ShenandoahLoadReferenceBarrierNode*)in2)->kind() != ShenandoahBarrierSet::AccessKind::NORMAL)) { + !ShenandoahBarrierSet::is_strong_access(((ShenandoahLoadReferenceBarrierNode*)in2)->decorators()))) { in2 = step_over_gc_barrier(in2); } if (in2->bottom_type() == TypePtr::NULL_PTR && !((in1->Opcode() == Op_ShenandoahLoadReferenceBarrier) && - ((ShenandoahLoadReferenceBarrierNode*)in1)->kind() != ShenandoahBarrierSet::AccessKind::NORMAL)) { + !ShenandoahBarrierSet::is_strong_access(((ShenandoahLoadReferenceBarrierNode*)in1)->decorators()))) { in1 = step_over_gc_barrier(in1); } diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp index bd7f450443f..01c81bd1b63 100644 --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp @@ -960,7 +960,7 @@ void ShenandoahBarrierC2Support::test_in_cset(Node*& ctrl, Node*& not_cset_ctrl, } void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, Node*& result_mem, Node* raw_mem, - ShenandoahBarrierSet::AccessKind kind, PhaseIdealLoop* phase) { + DecoratorSet decorators, PhaseIdealLoop* phase) { IdealLoopTree*loop = phase->get_loop(ctrl); const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr(); @@ -973,25 +973,32 @@ void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node* lo address calladdr = NULL; const char* name = NULL; - switch (kind) { - case ShenandoahBarrierSet::AccessKind::NATIVE: + bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators); + bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators); + bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators); + bool is_native = ShenandoahBarrierSet::is_native_access(decorators); + bool is_narrow = UseCompressedOops && !is_native; + if (is_strong) { + if (is_narrow) { + calladdr = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow); + name = "load_reference_barrier_strong_narrow"; + } else { + calladdr = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong); + name = "load_reference_barrier_strong"; + } + } else if (is_weak) { + if (is_narrow) { + calladdr = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow); + name = "load_reference_barrier_weak_narrow"; + } else { calladdr = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak); - name = "load_reference_barrier_native"; - break; - case ShenandoahBarrierSet::AccessKind::WEAK: - calladdr = LP64_ONLY(UseCompressedOops) NOT_LP64(false) ? - CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow) : - CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak); name = "load_reference_barrier_weak"; - break; - case ShenandoahBarrierSet::AccessKind::NORMAL: - calladdr = LP64_ONLY(UseCompressedOops) NOT_LP64(false) ? - CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow) : - CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier); - name = "load_reference_barrier"; - break; - default: - ShouldNotReachHere(); + } + } else { + assert(is_phantom, "only remaining strength"); + assert(!is_narrow, "phantom access cannot be narrow"); + calladdr = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom); + name = "load_reference_barrier_phantom"; } Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type(), calladdr, name, TypeRawPtr::BOTTOM); @@ -1357,7 +1364,7 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) { // even for non-cset objects to prevent ressurrection of such objects. // Wires !in_cset(obj) to slot 2 of region and phis Node* not_cset_ctrl = NULL; - if (lrb->kind() == ShenandoahBarrierSet::AccessKind::NORMAL) { + if (ShenandoahBarrierSet::is_strong_access(lrb->decorators())) { test_in_cset(ctrl, not_cset_ctrl, val, raw_mem, phase); } if (not_cset_ctrl != NULL) { @@ -1408,7 +1415,7 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) { } } } - call_lrb_stub(ctrl, val, addr, result_mem, raw_mem, lrb->kind(), phase); + call_lrb_stub(ctrl, val, addr, result_mem, raw_mem, lrb->decorators(), phase); region->init_req(_evac_path, ctrl); val_phi->init_req(_evac_path, val); raw_mem_phi->init_req(_evac_path, result_mem); @@ -2904,40 +2911,32 @@ void MemoryGraphFixer::fix_memory_uses(Node* mem, Node* replacement, Node* rep_p } } -ShenandoahLoadReferenceBarrierNode::ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* obj, ShenandoahBarrierSet::AccessKind kind) -: Node(ctrl, obj), _kind(kind) { +ShenandoahLoadReferenceBarrierNode::ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* obj, DecoratorSet decorators) +: Node(ctrl, obj), _decorators(decorators) { ShenandoahBarrierSetC2::bsc2()->state()->add_load_reference_barrier(this); } -ShenandoahBarrierSet::AccessKind ShenandoahLoadReferenceBarrierNode::kind() const { - return _kind; +DecoratorSet ShenandoahLoadReferenceBarrierNode::decorators() const { + return _decorators; } uint ShenandoahLoadReferenceBarrierNode::size_of() const { return sizeof(*this); } +static DecoratorSet mask_decorators(DecoratorSet decorators) { + return decorators & (ON_STRONG_OOP_REF | ON_WEAK_OOP_REF | ON_PHANTOM_OOP_REF | ON_UNKNOWN_OOP_REF | IN_NATIVE); +} + uint ShenandoahLoadReferenceBarrierNode::hash() const { uint hash = Node::hash(); - switch (_kind) { - case ShenandoahBarrierSet::AccessKind::NORMAL: - hash += 0; - break; - case ShenandoahBarrierSet::AccessKind::WEAK: - hash += 1; - break; - case ShenandoahBarrierSet::AccessKind::NATIVE: - hash += 2; - break; - default: - ShouldNotReachHere(); - } + hash += mask_decorators(_decorators); return hash; } bool ShenandoahLoadReferenceBarrierNode::cmp( const Node &n ) const { return Node::cmp(n) && n.Opcode() == Op_ShenandoahLoadReferenceBarrier && - _kind == ((const ShenandoahLoadReferenceBarrierNode&)n)._kind; + mask_decorators(_decorators) == mask_decorators(((const ShenandoahLoadReferenceBarrierNode&)n)._decorators); } const Type* ShenandoahLoadReferenceBarrierNode::bottom_type() const { @@ -2949,7 +2948,7 @@ const Type* ShenandoahLoadReferenceBarrierNode::bottom_type() const { return t; } - if (kind() == ShenandoahBarrierSet::AccessKind::NORMAL) { + if (ShenandoahBarrierSet::is_strong_access(decorators())) { return t; } @@ -2965,7 +2964,7 @@ const Type* ShenandoahLoadReferenceBarrierNode::Value(PhaseGVN* phase) const { return t2; } - if (kind() == ShenandoahBarrierSet::AccessKind::NORMAL) { + if (ShenandoahBarrierSet::is_strong_access(decorators())) { return t2; } diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp index c07b4a1ed72..f6a5e324361 100644 --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp @@ -62,7 +62,7 @@ class ShenandoahBarrierC2Support : public AllStatic { static void test_gc_state(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl, PhaseIdealLoop* phase, int flags); static void call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, Node*& result_mem, Node* raw_mem, - ShenandoahBarrierSet::AccessKind kind, PhaseIdealLoop* phase); + DecoratorSet decorators, PhaseIdealLoop* phase); static void test_in_cset(Node*& ctrl, Node*& not_cset_ctrl, Node* val, Node* raw_mem, PhaseIdealLoop* phase); static void move_gc_state_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase); static void merge_back_to_back_tests(Node* n, PhaseIdealLoop* phase); @@ -231,12 +231,12 @@ class ShenandoahLoadReferenceBarrierNode : public Node { }; private: - ShenandoahBarrierSet::AccessKind _kind; + DecoratorSet _decorators; public: - ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* val, ShenandoahBarrierSet::AccessKind kind); + ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* val, DecoratorSet decorators); - ShenandoahBarrierSet::AccessKind kind() const; + DecoratorSet decorators() const; virtual int Opcode() const; virtual const Type* bottom_type() const; virtual const Type* Value(PhaseGVN* phase) const; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp index adf349d3b24..d731a376652 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp @@ -98,16 +98,6 @@ bool ShenandoahBarrierSet::need_keep_alive_barrier(DecoratorSet decorators,Basic return (on_weak_ref || unknown) && keep_alive; } -ShenandoahBarrierSet::AccessKind ShenandoahBarrierSet::access_kind(DecoratorSet decorators, BasicType type) { - if ((decorators & IN_NATIVE) != 0) { - return AccessKind::NATIVE; - } else if ((decorators & (ON_WEAK_OOP_REF | ON_PHANTOM_OOP_REF | ON_UNKNOWN_OOP_REF)) != 0) { - return AccessKind::WEAK; - } else { - return AccessKind::NORMAL; - } -} - void ShenandoahBarrierSet::on_thread_create(Thread* thread) { // Create thread local data ShenandoahThreadLocalData::create(thread); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp index eee66c0c05d..0deda387f15 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp @@ -33,18 +33,6 @@ class ShenandoahBarrierSetAssembler; class ShenandoahBarrierSet: public BarrierSet { -public: - enum class AccessKind { - // Regular in-heap access on reference fields - NORMAL, - - // Off-heap reference access - NATIVE, - - // In-heap reference access on referent fields of j.l.r.Reference objects - WEAK - }; - private: ShenandoahHeap* _heap; BufferNode::Allocator _satb_mark_queue_buffer_allocator; @@ -65,7 +53,22 @@ class ShenandoahBarrierSet: public BarrierSet { static bool need_load_reference_barrier(DecoratorSet decorators, BasicType type); static bool need_keep_alive_barrier(DecoratorSet decorators, BasicType type); - static AccessKind access_kind(DecoratorSet decorators, BasicType type); + + static bool is_strong_access(DecoratorSet decorators) { + return (decorators & (ON_WEAK_OOP_REF | ON_PHANTOM_OOP_REF | ON_UNKNOWN_OOP_REF)) == 0; + } + + static bool is_weak_access(DecoratorSet decorators) { + return (decorators & (ON_WEAK_OOP_REF | ON_UNKNOWN_OOP_REF)) != 0; + } + + static bool is_phantom_access(DecoratorSet decorators) { + return (decorators & ON_PHANTOM_OOP_REF) != 0; + } + + static bool is_native_access(DecoratorSet decorators) { + return (decorators & IN_NATIVE) != 0; + } void print_on(outputStream* st) const; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp index 1388465328d..20370d1828f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp @@ -102,13 +102,21 @@ inline oop ShenandoahBarrierSet::load_reference_barrier(oop obj) { template inline oop ShenandoahBarrierSet::load_reference_barrier(oop obj, T* load_addr) { - // Prevent resurrection of unreachable non-strorg references. - if (!HasDecorator::value && obj != NULL && + // Prevent resurrection of unreachable phantom (i.e. weak-native) references. + if (HasDecorator::value && obj != NULL && _heap->is_concurrent_weak_root_in_progress() && !_heap->marking_context()->is_marked(obj)) { return NULL; } + // Prevent resurrection of unreachable weak references. + if ((HasDecorator::value || HasDecorator::value) && + obj != NULL && _heap->is_concurrent_weak_root_in_progress() && + !_heap->marking_context()->is_marked_strong(obj)) { + assert(Thread::current()->is_Java_thread(), "only Java threads get here"); + return NULL; + } + // Prevent resurrection of unreachable objects that are visited during // concurrent class-unloading. if (HasDecorator::value && obj != NULL && diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp b/src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp index 5fb69eb9b0f..97bfecd7161 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp @@ -50,11 +50,11 @@ JRT_LEAF(void, ShenandoahRuntime::write_ref_field_pre_entry(oopDesc* orig, JavaT ShenandoahThreadLocalData::satb_mark_queue(thread).enqueue_known_active(orig); JRT_END -JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier(oopDesc* src, oop* load_addr)) +JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_strong(oopDesc* src, oop* load_addr)) return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_mutator(src, load_addr); JRT_END -JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_narrow(oopDesc* src, narrowOop* load_addr)) +JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_strong_narrow(oopDesc* src, narrowOop* load_addr)) return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_mutator(src, load_addr); JRT_END @@ -67,9 +67,13 @@ JRT_LEAF(void, ShenandoahRuntime::shenandoah_clone_barrier(oopDesc* src)) JRT_END JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_weak(oopDesc * src, oop* load_addr)) - return (oopDesc*) ShenandoahBarrierSet::barrier_set()->load_reference_barrier(oop(src), load_addr); + return (oopDesc*) ShenandoahBarrierSet::barrier_set()->load_reference_barrier(oop(src), load_addr); JRT_END JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_weak_narrow(oopDesc * src, narrowOop* load_addr)) - return (oopDesc*) ShenandoahBarrierSet::barrier_set()->load_reference_barrier(oop(src), load_addr); + return (oopDesc*) ShenandoahBarrierSet::barrier_set()->load_reference_barrier(oop(src), load_addr); +JRT_END + +JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_phantom(oopDesc * src, oop* load_addr)) + return (oopDesc*) ShenandoahBarrierSet::barrier_set()->load_reference_barrier(oop(src), load_addr); JRT_END diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRuntime.hpp b/src/hotspot/share/gc/shenandoah/shenandoahRuntime.hpp index c0446d32168..9a73493269a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahRuntime.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRuntime.hpp @@ -38,12 +38,14 @@ class ShenandoahRuntime : public AllStatic { static void write_ref_field_pre_entry(oopDesc* orig, JavaThread* thread); - static oopDesc* load_reference_barrier(oopDesc* src, oop* load_addr); - static oopDesc* load_reference_barrier_narrow(oopDesc* src, narrowOop* load_addr); + static oopDesc* load_reference_barrier_strong(oopDesc* src, oop* load_addr); + static oopDesc* load_reference_barrier_strong_narrow(oopDesc* src, narrowOop* load_addr); static oopDesc* load_reference_barrier_weak(oopDesc* src, oop* load_addr); static oopDesc* load_reference_barrier_weak_narrow(oopDesc* src, narrowOop* load_addr); + static oopDesc* load_reference_barrier_phantom(oopDesc* src, oop* load_addr); + static void shenandoah_clone_barrier(oopDesc* src); };