diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 0e0c77a7..6a86e04b 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -129,9 +129,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.37" +version = "1.2.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65193589c6404eb80b450d618eaf9a2cafaaafd57ecce47370519ef674a7bd44" +checksum = "80f41ae168f955c12fb8960b057d70d0ca153fb83182b57d86380443527be7e9" dependencies = [ "find-msvc-tools", "jobserver", @@ -281,9 +281,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fd99930f64d146689264c637b5af2f0233a933bef0d8570e2526bf9e083192d" +checksum = "1ced73b1dacfc750a6db6c0a0c3a3853c8b41997e2e2c563dc90804ae6867959" [[package]] name = "form_urlencoded" @@ -430,9 +430,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.175" +version = "0.2.176" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" +checksum = "58f929b4d672ea937a23a1ab494143d968337a5f47e56d0815df1e0890ddf174" [[package]] name = "libgit2-sys" @@ -492,7 +492,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.31.0" -source = "git+https://github.com/mmtk/mmtk-core.git?rev=a4dd70cb70a116a32b1bbb20501c48f77f49181b#a4dd70cb70a116a32b1bbb20501c48f77f49181b" +source = "git+https://github.com/mmtk/mmtk-core.git?rev=1ffa5b325a07d4fe99fe7e7008b295f8febea407#1ffa5b325a07d4fe99fe7e7008b295f8febea407" dependencies = [ "atomic", "atomic-traits", @@ -531,7 +531,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.31.0" -source = "git+https://github.com/mmtk/mmtk-core.git?rev=a4dd70cb70a116a32b1bbb20501c48f77f49181b#a4dd70cb70a116a32b1bbb20501c48f77f49181b" +source = "git+https://github.com/mmtk/mmtk-core.git?rev=1ffa5b325a07d4fe99fe7e7008b295f8febea407#1ffa5b325a07d4fe99fe7e7008b295f8febea407" dependencies = [ "proc-macro-error", "proc-macro2", @@ -754,9 +754,9 @@ checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" [[package]] name = "serde" -version = "1.0.224" +version = "1.0.226" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6aaeb1e94f53b16384af593c71e20b095e958dab1d26939c1b70645c5cfbcc0b" +checksum = "0dca6411025b24b60bfa7ec1fe1f8e710ac09782dca409ee8237ba74b51295fd" dependencies = [ "serde_core", "serde_derive", @@ -764,18 +764,18 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.224" +version = "1.0.226" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32f39390fa6346e24defbcdd3d9544ba8a19985d0af74df8501fbfe9a64341ab" +checksum = "ba2ba63999edb9dac981fb34b3e5c0d111a69b0924e253ed29d83f7c99e966a4" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.224" +version = "1.0.226" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ff78ab5e8561c9a675bfc1785cb07ae721f0ee53329a595cefd8c04c2ac4e0" +checksum = "8db53ae22f34573731bafa1db20f04027b2d25e02d8205921b569171699cdb33" dependencies = [ "proc-macro2", "quote", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index dee30842..02a6369e 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -36,7 +36,7 @@ probe = "0.5" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "a4dd70cb70a116a32b1bbb20501c48f77f49181b" } +mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "1ffa5b325a07d4fe99fe7e7008b295f8febea407" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } diff --git a/openjdk/barriers/mmtkObjectBarrier.cpp b/openjdk/barriers/mmtkObjectBarrier.cpp index 5f14723c..e11bb770 100644 --- a/openjdk/barriers/mmtkObjectBarrier.cpp +++ b/openjdk/barriers/mmtkObjectBarrier.cpp @@ -2,15 +2,13 @@ #include "mmtkObjectBarrier.hpp" #include "runtime/interfaceSupport.inline.hpp" +//////////////////// Runtime //////////////////// + void MMTkObjectBarrierSetRuntime::object_probable_write(oop new_obj) const { if (mmtk_enable_barrier_fastpath) { // Do fast-path check before entering mmtk rust code, to improve mutator performance. // This is identical to calling `mmtk_object_probable_write` directly without a fast-path. - intptr_t addr = (intptr_t) (void*) new_obj; - uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); - intptr_t shift = (addr >> 3) & 0b111; - uint8_t byte_val = *meta_addr; - if (((byte_val >> shift) & 1) == 1) { + if (is_unlog_bit_set(new_obj)) { // Only promoted objects will reach here. // The duplicated unlog bit check inside slow-path still remains correct. mmtk_object_probable_write((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) new_obj); @@ -23,12 +21,7 @@ void MMTkObjectBarrierSetRuntime::object_probable_write(oop new_obj) const { void MMTkObjectBarrierSetRuntime::object_reference_write_post(oop src, oop* slot, oop target) const { if (mmtk_enable_barrier_fastpath) { - intptr_t addr = (intptr_t) (void*) src; - uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); - intptr_t shift = (addr >> 3) & 0b111; - uint8_t byte_val = *meta_addr; - if (((byte_val >> shift) & 1) == 1) { - // MMTkObjectBarrierSetRuntime::object_reference_write_pre_slow()((void*) src); + if (is_unlog_bit_set(src)) { object_reference_write_slow_call((void*) src, (void*) slot, (void*) target); } } else { @@ -36,57 +29,13 @@ void MMTkObjectBarrierSetRuntime::object_reference_write_post(oop src, oop* slot } } +//////////////////// Assembler //////////////////// + #define __ masm-> void MMTkObjectBarrierSetAssembler::object_reference_write_post(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2, bool compensate_val_reg) const { if (can_remove_barrier(decorators, val, /* skip_const_null */ true)) return; - - bool is_not_null = (decorators & IS_NOT_NULL) != 0; - - Label done; - Register obj = dst.base(); - if (mmtk_enable_barrier_fastpath) { - Register tmp3 = rscratch1; - Register tmp4 = rscratch2; - assert_different_registers(obj, tmp2, tmp3); - assert_different_registers(tmp4, rcx); - - // tmp2 = load-byte (SIDE_METADATA_BASE_ADDRESS + (obj >> 6)); - __ movptr(tmp3, obj); - __ shrptr(tmp3, 6); - __ movptr(tmp2, SIDE_METADATA_BASE_ADDRESS); - __ movb(tmp2, Address(tmp2, tmp3)); - // tmp3 = (obj >> 3) & 7 - __ movptr(tmp3, obj); - __ shrptr(tmp3, 3); - __ andptr(tmp3, 7); - // tmp2 = tmp2 >> tmp3 - __ movptr(tmp4, rcx); - __ movl(rcx, tmp3); - __ shrptr(tmp2); - __ movptr(rcx, tmp4); - // if ((tmp2 & 1) == 1) goto slowpath; - __ andptr(tmp2, 1); - __ cmpptr(tmp2, 1); - __ jcc(Assembler::notEqual, done); - } - - __ movptr(c_rarg0, obj); - __ xorptr(c_rarg1, c_rarg1); - // Note: If `compensate_val_reg == true && UseCompressedOops === true`, the `val` register will be - // holding a compressed pointer to the target object. If the write barrier needs to know the - // target, we will need to decompress it before passing it to the barrier slow path. However, - // since we know the semantics of `mmtk::plan::barriers::ObjectBarrier`, i.e. it logs the object - // without looking at the `slot` or the `target` parameter at all, we simply pass nullptr to both - // parameters. - __ xorptr(c_rarg2, c_rarg2); - - if (mmtk_enable_barrier_fastpath) { - __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), 3); - __ bind(done); - } else { - __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call), 3); - } + object_reference_write_pre_or_post(masm, decorators, dst, val, /* pre = */ false); } void MMTkObjectBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) { @@ -135,56 +84,7 @@ void MMTkObjectBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, Dec #undef __ -#define __ sasm-> - -void MMTkObjectBarrierSetAssembler::generate_c1_post_write_barrier_runtime_stub(StubAssembler* sasm) const { - __ prologue("mmtk_object_barrier", false); - - Label done, runtime; - - __ push(c_rarg0); - __ push(c_rarg1); - __ push(c_rarg2); - __ push(rax); - - __ load_parameter(0, c_rarg0); - __ load_parameter(1, c_rarg1); - __ load_parameter(2, c_rarg2); - - __ bind(runtime); - - __ save_live_registers_no_oop_map(true); - - if (mmtk_enable_barrier_fastpath) { - __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), 3); - } else { - __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call), 3); - } - - __ restore_live_registers(true); - - __ bind(done); - __ pop(rax); - __ pop(c_rarg2); - __ pop(c_rarg1); - __ pop(c_rarg0); - - __ epilogue(); -} - -#undef __ -#define __ ce->masm()-> - -void MMTkObjectBarrierSetAssembler::generate_c1_post_write_barrier_stub(LIR_Assembler* ce, MMTkC1PostBarrierStub* stub) const { - MMTkBarrierSetC1* bs = (MMTkBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1(); - __ bind(*stub->entry()); - ce->store_parameter(stub->src->as_pointer_register(), 0); - ce->store_parameter(stub->slot->as_pointer_register(), 1); - ce->store_parameter(stub->new_val->as_pointer_register(), 2); - __ call(RuntimeAddress(bs->post_barrier_c1_runtime_code_blob()->code_begin())); - __ jmp(*stub->continuation()); -} -#undef __ +//////////////////// C1 //////////////////// #ifdef ASSERT #define __ gen->lir(__FILE__, __LINE__)-> @@ -193,73 +93,13 @@ void MMTkObjectBarrierSetAssembler::generate_c1_post_write_barrier_stub(LIR_Asse #endif void MMTkObjectBarrierSetC1::object_reference_write_post(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val) const { - LIRGenerator* gen = access.gen(); - DecoratorSet decorators = access.decorators(); - if ((decorators & IN_HEAP) == 0) return; - if (!src->is_register()) { - LIR_Opr reg = gen->new_pointer_register(); - if (src->is_constant()) { - __ move(src, reg); - } else { - __ leal(src, reg); - } - src = reg; - } - assert(src->is_register(), "must be a register at this point"); - if (!slot->is_register()) { - LIR_Opr reg = gen->new_pointer_register(); - if (slot->is_constant()) { - __ move(slot, reg); - } else { - __ leal(slot, reg); - } - slot = reg; - } - assert(slot->is_register(), "must be a register at this point"); - if (!new_val->is_register()) { - LIR_Opr new_val_reg = gen->new_register(T_OBJECT); - if (new_val->is_constant()) { - __ move(new_val, new_val_reg); - } else { - __ leal(new_val, new_val_reg); - } - new_val = new_val_reg; - } - assert(new_val->is_register(), "must be a register at this point"); - CodeStub* slow = new MMTkC1PostBarrierStub(src, slot, new_val); - - if (mmtk_enable_barrier_fastpath) { - LIR_Opr addr = src; - // uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); - LIR_Opr offset = gen->new_pointer_register(); - __ move(addr, offset); - __ unsigned_shift_right(offset, 6, offset); - LIR_Opr base = gen->new_pointer_register(); - __ move(LIR_OprFact::longConst(SIDE_METADATA_BASE_ADDRESS), base); - LIR_Address* meta_addr = new LIR_Address(base, offset, T_BYTE); - // uint8_t byte_val = *meta_addr; - LIR_Opr byte_val = gen->new_register(T_INT); - __ move(meta_addr, byte_val); - // intptr_t shift = (addr >> 3) & 0b111; - LIR_Opr shift = gen->new_register(T_INT); - __ move(addr, shift); - __ unsigned_shift_right(shift, 3, shift); - __ logical_and(shift, LIR_OprFact::intConst(0b111), shift); - // if (((byte_val >> shift) & 1) == 1) slow; - LIR_Opr result = byte_val; - __ unsigned_shift_right(result, shift, result, LIR_OprFact::illegalOpr); - __ logical_and(result, LIR_OprFact::intConst(1), result); - __ cmp(lir_cond_equal, result, LIR_OprFact::intConst(1)); - __ branch(lir_cond_equal, T_BYTE, slow); - } else { - __ jump(slow); - } - - __ branch_destination(slow->continuation()); + object_reference_write_pre_or_post(access, src, /* pre = */ false); } #undef __ +//////////////////// C2 //////////////////// + #define __ ideal. void MMTkObjectBarrierSetC2::object_reference_write_post(GraphKit* kit, Node* src, Node* slot, Node* val) const { @@ -267,26 +107,7 @@ void MMTkObjectBarrierSetC2::object_reference_write_post(GraphKit* kit, Node* sr MMTkIdealKit ideal(kit, true); - if (mmtk_enable_barrier_fastpath) { - Node* no_base = __ top(); - float unlikely = PROB_UNLIKELY(0.999); - - Node* zero = __ ConI(0); - Node* addr = __ CastPX(__ ctrl(), src); - Node* meta_addr = __ AddP(no_base, __ ConP(SIDE_METADATA_BASE_ADDRESS), __ URShiftX(addr, __ ConI(6))); - Node* byte = __ load(__ ctrl(), meta_addr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); - Node* shift = __ URShiftX(addr, __ ConI(3)); - shift = __ AndI(__ ConvL2I(shift), __ ConI(7)); - Node* result = __ AndI(__ URShiftI(byte, shift), __ ConI(1)); - - __ if_then(result, BoolTest::ne, zero, unlikely); { - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); - Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), "mmtk_barrier_call", src, slot, val); - } __ end_if(); - } else { - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); - Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call), "mmtk_barrier_call", src, slot, val); - } + object_reference_write_pre_or_post(ideal, src, /* pre = */ false); kit->final_sync(ideal); // Final sync IdealKit and GraphKit. } diff --git a/openjdk/barriers/mmtkObjectBarrier.hpp b/openjdk/barriers/mmtkObjectBarrier.hpp index c04081d3..1735da8f 100644 --- a/openjdk/barriers/mmtkObjectBarrier.hpp +++ b/openjdk/barriers/mmtkObjectBarrier.hpp @@ -6,19 +6,22 @@ #include "../mmtkBarrierSetAssembler_x86.hpp" #include "../mmtkBarrierSetC1.hpp" #include "../mmtkBarrierSetC2.hpp" +#include "mmtkUnlogBitBarrier.hpp" #include "c1/c1_LIRAssembler.hpp" #include "c1/c1_MacroAssembler.hpp" #include "gc/shared/barrierSet.hpp" #include "opto/callnode.hpp" #include "opto/idealKit.hpp" -#define SIDE_METADATA_WORST_CASE_RATIO_LOG 1 -#define LOG_BYTES_IN_CHUNK 22 -#define CHUNK_MASK ((1L << LOG_BYTES_IN_CHUNK) - 1) +/// This file supports the `ObjectBarrier` in MMTk core, +/// i.e. the barrier that remembers the object when it is first modified. +/// Despite the name, `ObjectBarrier` is not the only barrier that uses the object-grained unlogging bit. +/// `SATBBarrier` also uses the object-grained unlog bit. +/// We keep the name in sync with the MMTk core. -const intptr_t SIDE_METADATA_BASE_ADDRESS = (intptr_t) GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS; +//////////////////// Runtime //////////////////// -class MMTkObjectBarrierSetRuntime: public MMTkBarrierSetRuntime { +class MMTkObjectBarrierSetRuntime: public MMTkUnlogBitBarrierSetRuntime { public: // Interfaces called by `MMTkBarrierSet::AccessBarrier` virtual void object_reference_write_post(oop src, oop* slot, oop target) const override; @@ -28,18 +31,19 @@ class MMTkObjectBarrierSetRuntime: public MMTkBarrierSetRuntime { virtual void object_probable_write(oop new_obj) const override; }; -class MMTkObjectBarrierSetAssembler: public MMTkBarrierSetAssembler { +//////////////////// Assembler //////////////////// + +class MMTkObjectBarrierSetAssembler: public MMTkUnlogBitBarrierSetAssembler { protected: virtual void object_reference_write_post(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2, bool compensate_val_reg) const override; - /// Generate C1 write barrier slow-call assembly code - virtual void generate_c1_post_write_barrier_runtime_stub(StubAssembler* sasm) const override; public: - virtual void generate_c1_post_write_barrier_stub(LIR_Assembler* ce, MMTkC1PostBarrierStub* stub) const override; virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) override; virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) override; }; -class MMTkObjectBarrierSetC1: public MMTkBarrierSetC1 { +//////////////////// C1 //////////////////// + +class MMTkObjectBarrierSetC1: public MMTkUnlogBitBarrierSetC1 { protected: virtual void object_reference_write_post(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val) const override; @@ -48,11 +52,15 @@ class MMTkObjectBarrierSetC1: public MMTkBarrierSetC1 { } }; -class MMTkObjectBarrierSetC2: public MMTkBarrierSetC2 { +//////////////////// C2 //////////////////// + +class MMTkObjectBarrierSetC2: public MMTkUnlogBitBarrierSetC2 { protected: virtual void object_reference_write_post(GraphKit* kit, Node* src, Node* slot, Node* val) const override; }; +//////////////////// Impl //////////////////// + struct MMTkObjectBarrier: MMTkBarrierImpl< MMTkObjectBarrierSetRuntime, MMTkObjectBarrierSetAssembler, diff --git a/openjdk/barriers/mmtkSATBBarrier.cpp b/openjdk/barriers/mmtkSATBBarrier.cpp index 3a75ccc5..4a564f9b 100644 --- a/openjdk/barriers/mmtkSATBBarrier.cpp +++ b/openjdk/barriers/mmtkSATBBarrier.cpp @@ -1,23 +1,14 @@ -// Workaround the fact that LIR_Assembler::as_Address is private. -#define private public - #include "precompiled.hpp" #include "mmtkSATBBarrier.hpp" #include "runtime/interfaceSupport.inline.hpp" -#define SOFT_REFERENCE_LOAD_BARRIER true - -constexpr int kUnloggedValue = 1; - -static inline intptr_t side_metadata_base_address() { - return SATB_METADATA_BASE_ADDRESS; -} +//////////////////// Runtime //////////////////// void MMTkSATBBarrierSetRuntime::load_reference(DecoratorSet decorators, oop value) const { -#if SOFT_REFERENCE_LOAD_BARRIER - if (CONCURRENT_MARKING_ACTIVE == 1 && value != NULL) - ::mmtk_load_reference((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) value); -#endif + if (mmtk_enable_reference_load_barrier) { + if (CONCURRENT_MARKING_ACTIVE == 1 && value != NULL) + ::mmtk_load_reference((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) value); + } }; void MMTkSATBBarrierSetRuntime::object_probable_write(oop new_obj) const { @@ -29,13 +20,7 @@ void MMTkSATBBarrierSetRuntime::object_probable_write(oop new_obj) const { void MMTkSATBBarrierSetRuntime::object_reference_write_pre(oop src, oop* slot, oop target) const { if (mmtk_enable_barrier_fastpath) { - // oop pre_val = *slot; - // if (pre_val == NULL) return; - intptr_t addr = ((intptr_t) (void*) src); - const volatile uint8_t * meta_addr = (const volatile uint8_t *) (side_metadata_base_address() + (addr >> 6)); - intptr_t shift = (addr >> 3) & 0b111; - uint8_t byte_val = *meta_addr; - if (((byte_val >> shift) & 1) == kUnloggedValue) { + if (is_unlog_bit_set(src)) { object_reference_write_slow_call((void*) src, (void*) slot, (void*) target); } } else { @@ -43,6 +28,8 @@ void MMTkSATBBarrierSetRuntime::object_reference_write_pre(oop src, oop* slot, o } } +//////////////////// Assembler //////////////////// + #define __ masm-> void MMTkSATBBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp_thread) { @@ -50,96 +37,48 @@ void MMTkSATBBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet dec bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0; bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0; bool on_reference = on_weak || on_phantom; + BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); -#if SOFT_REFERENCE_LOAD_BARRIER - if (on_oop && on_reference) { - Label done; - // No slow-call if SATB is not active - Register tmp = rscratch1; - Register tmp2 = rscratch2; - __ movptr(tmp, intptr_t(&CONCURRENT_MARKING_ACTIVE)); - __ xorq(tmp2, tmp2); - __ movb(tmp2, Address(tmp, 0)); - __ cmpptr(tmp2, 1); - __ jcc(Assembler::notEqual, done); - // No slow-call if dst is NULL - __ cmpptr(dst, 0); - __ jcc(Assembler::equal, done); - // Do slow-call - __ pusha(); - __ mov(c_rarg0, dst); - __ MacroAssembler::call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), 1); - __ popa(); - __ bind(done); + + if (mmtk_enable_reference_load_barrier) { + if (on_oop && on_reference) { + Label done; + + assert_different_registers(dst, tmp1); + + // No slow-call if SATB is not active + // intptr_t tmp1_q = CONCURRENT_MARKING_ACTIVE; + __ movptr(tmp1, intptr_t(&CONCURRENT_MARKING_ACTIVE)); + // Load with zero extension to 32 bits. + // uint32_t tmp1_l = (uint32_t)(*(unt8_t*)tmp1_q); + __ movzbl(tmp1, Address(tmp1, 0)); + // if (tmp1_l == 0) goto done; + __ testl(tmp1, tmp1); + __ jcc(Assembler::zero, done); + // if (dst == 0) goto done; + __ testptr(dst, dst); + __ jcc(Assembler::zero, done); + // Do slow-call + __ pusha(); + __ mov(c_rarg0, dst); + __ MacroAssembler::call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), 1); + __ popa(); + __ bind(done); + } } -#endif } void MMTkSATBBarrierSetAssembler::object_reference_write_pre(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2) const { if (can_remove_barrier(decorators, val, /* skip_const_null */ false)) return; - - if (mmtk_enable_barrier_fastpath) { - Label done; - - Register obj = dst.base(); - Register tmp3 = rscratch1; - Register tmp4 = rscratch2; - Register tmp5 = tmp1 == dst.base() || tmp1 == dst.index() ? tmp2 : tmp1; - - // tmp5 = load-byte (side_metadata_base_address() + (obj >> 6)); - __ movptr(tmp3, obj); - // __ load_heap_oop(tmp3, dst, noreg, noreg, AS_RAW); - // // Is the previous value null? - // __ cmpptr(tmp3, (int32_t) NULL_WORD); - // __ jcc(Assembler::equal, done); - - __ shrptr(tmp3, 6); - __ movptr(tmp5, side_metadata_base_address()); - __ movzbl(tmp5, Address(tmp5, tmp3)); - - // tmp3 = (obj >> 3) & 7 - __ mov(tmp3, obj); - __ shrptr(tmp3, 3); - __ andptr(tmp3, 7); - // tmp5 = tmp5 >> tmp3 - __ movptr(tmp4, rcx); - __ movl(rcx, tmp3); - __ shrptr(tmp5); - __ movptr(rcx, tmp4); - // if ((tmp5 & 1) == 1) goto slowpath; - __ andptr(tmp5, 1); - __ cmpptr(tmp5, kUnloggedValue); - __ jcc(Assembler::notEqual, done); - - // TODO: Spill fewer registers - __ pusha(); - __ movptr(c_rarg0, dst.base()); - __ lea(c_rarg1, dst); - __ movptr(c_rarg2, val == noreg ? (int32_t) NULL_WORD : val); - __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), 3); - __ popa(); - - __ bind(done); - } else { - __ pusha(); - __ movptr(c_rarg0, dst.base()); - __ lea(c_rarg1, dst); - __ movptr(c_rarg2, val == noreg ? (int32_t) NULL_WORD : val); - __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call), 3); - __ popa(); - } + object_reference_write_pre_or_post(masm, decorators, dst, val, /* pre = */ true); } void MMTkSATBBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) { - // `count` or `dst` register values may get overwritten after the array copy, and `arraycopy_epilogue` can receive invalid addresses. - // Save the register values here and restore them in `arraycopy_epilogue`. - // See https://github.com/openjdk/jdk/blob/jdk-11%2B19/src/hotspot/cpu/x86/gc/shared/modRefBarrierSetAssembler_x86.cpp#L37-L50 - if (type == T_OBJECT || type == T_ARRAY) { Label done; - // // Bailout if count is zero - __ cmpptr(count, 0); - __ jcc(Assembler::equal, done); + // Skip the runtime call if count is zero. + __ testptr(count, count); + __ jcc(Assembler::zero, done); __ pusha(); __ movptr(c_rarg0, src); __ movptr(c_rarg1, dst); @@ -152,79 +91,7 @@ void MMTkSATBBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Decor #undef __ -#define __ sasm-> - -void MMTkSATBBarrierSetAssembler::generate_c1_pre_write_barrier_runtime_stub(StubAssembler* sasm) const { - __ prologue("mmtk_satb_barrier", false); - - Label done, runtime; - - __ push(c_rarg0); - __ push(c_rarg1); - __ push(c_rarg2); - __ push(rax); - - __ load_parameter(0, c_rarg0); - __ load_parameter(1, c_rarg1); - __ load_parameter(2, c_rarg2); - - __ bind(runtime); - - __ save_live_registers_no_oop_map(true); - - if (mmtk_enable_barrier_fastpath) { - __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), 3); - } else { - __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call), 3); - } - - __ restore_live_registers(true); - - __ bind(done); - __ pop(rax); - __ pop(c_rarg2); - __ pop(c_rarg1); - __ pop(c_rarg0); - - __ epilogue(); -} - -#undef __ -#define __ ce->masm()-> - -void MMTkSATBBarrierSetAssembler::generate_c1_pre_write_barrier_stub(LIR_Assembler* ce, MMTkC1PreBarrierStub* stub) const { - MMTkBarrierSetC1* bs = (MMTkBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1(); - __ bind(*stub->entry()); - - // For pre-barriers, stub->slot may not be a resolved address. - // Manually patch the address - address runtime_address; - if (stub->patch_code != lir_patch_none) { - // Patch - assert(stub->scratch->is_single_cpu(), "must be"); - assert(stub->scratch->is_register(), "Precondition."); - ce->mem2reg(stub->slot, stub->scratch, T_OBJECT, stub->patch_code, stub->info, false /*wide*/, false /*unaligned*/); - // Now stub->scratch contains the pre_val instead of the slot address - // So the following is to load the slot address into scrach register - // Resolve address - auto masm = ce->masm(); - LIR_Address* addr = stub->slot->as_address_ptr(); - Address from_addr = ce->as_Address(addr); - __ lea(stub->scratch->as_register(), from_addr); - // Store parameter - ce->store_parameter(stub->scratch->as_pointer_register(), 1); - } else { - // Store parameter - ce->store_parameter(stub->slot->as_pointer_register(), 1); - } - - ce->store_parameter(stub->src->as_pointer_register(), 0); - ce->store_parameter(stub->new_val->as_pointer_register(), 2); - __ call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin())); - __ jmp(*stub->continuation()); -} - -#undef __ +//////////////////// C1 //////////////////// #ifdef ASSERT #define __ gen->lir(__FILE__, __LINE__)-> @@ -232,7 +99,6 @@ void MMTkSATBBarrierSetAssembler::generate_c1_pre_write_barrier_stub(LIR_Assembl #define __ gen->lir()-> #endif - void MMTkSATBBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) { DecoratorSet decorators = access.decorators(); bool is_weak = (decorators & ON_WEAK_OOP_REF) != 0; @@ -242,125 +108,42 @@ void MMTkSATBBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) { BarrierSetC1::load_at_resolved(access, result); -#if SOFT_REFERENCE_LOAD_BARRIER - if (access.is_oop() && (is_weak || is_phantom || is_anonymous)) { - // Register the value in the referent field with the pre-barrier - LabelObj *Lcont_anonymous; - if (is_anonymous) { - Lcont_anonymous = new LabelObj(); - generate_referent_check(access, Lcont_anonymous); - } - assert(result->is_register(), "must be"); - assert(result->type() == T_OBJECT, "must be an object"); - auto slow = new MMTkC1ReferenceLoadBarrierStub(result, access.patch_emit_info()); - // Call slow-path only when concurrent marking is active - LIR_Opr cm_flag_addr_opr = gen->new_pointer_register(); - __ move(LIR_OprFact::longConst(uintptr_t(&CONCURRENT_MARKING_ACTIVE)), cm_flag_addr_opr); - LIR_Address* cm_flag_addr = new LIR_Address(cm_flag_addr_opr, T_BYTE); - LIR_Opr cm_flag = gen->new_register(T_INT); - __ move(cm_flag_addr, cm_flag); - // No slow-call if SATB is not active - __ cmp(lir_cond_equal, cm_flag, LIR_OprFact::intConst(1)); - __ branch(lir_cond_equal, T_BYTE, slow); - __ branch_destination(slow->continuation()); - if (is_anonymous) { - __ branch_destination(Lcont_anonymous->label()); - } - } -#endif -} - -void MMTkSATBBarrierSetC1::object_reference_write_pre(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val, CodeEmitInfo* info) const { - LIRGenerator* gen = access.gen(); - DecoratorSet decorators = access.decorators(); - if ((decorators & IN_HEAP) == 0) return; // Not sure if this line is sound - bool needs_patching = (decorators & C1_NEEDS_PATCHING) != 0; - if (!src->is_register()) { - LIR_Opr reg = gen->new_pointer_register(); - if (src->is_constant()) { - __ move(src, reg); - } else { - __ leal(src, reg); - } - src = reg; - } - assert(src->is_register(), "must be a register at this point"); - - if (!slot->is_register() && !needs_patching) { - LIR_Address* address = slot->as_address_ptr(); - LIR_Opr ptr = gen->new_pointer_register(); - if (!address->index()->is_valid() && address->disp() == 0) { - __ move(address->base(), ptr); - } else { - assert(address->disp() != max_jint, "lea doesn't support patched addresses!"); - __ leal(slot, ptr); - } - slot = ptr; - } else if (needs_patching && !slot->is_address()) { - assert(slot->is_register(), "must be"); - slot = LIR_OprFact::address(new LIR_Address(slot, T_OBJECT)); - } - assert(needs_patching || slot->is_register(), "must be a register at this point unless needs_patching"); - if (!new_val->is_register()) { - LIR_Opr new_val_reg = gen->new_register(T_OBJECT); - if (new_val->is_constant()) { - __ move(new_val, new_val_reg); - } else { - __ leal(new_val, new_val_reg); - } - new_val = new_val_reg; - } - assert(new_val->is_register(), "must be a register at this point"); - MMTkC1PreBarrierStub* slow = new MMTkC1PreBarrierStub(src, slot, new_val, info, needs_patching ? lir_patch_normal : lir_patch_none); - if (needs_patching) slow->scratch = gen->new_register(T_OBJECT); - - if (mmtk_enable_barrier_fastpath) { - if (needs_patching) { - // At this stage, slot address is not available, so cannot do the fast-path check until - // its address get resolved - // FIXME: Jump to a medium-path for code patching without entering slow-path - __ jump(slow); - } else { - // // load pre_val - // LIR_Address* slot_addr = new LIR_Address(slot, T_OBJECT); - // LIR_Opr addr = slot; - // __ load(slot_addr, addr); - // // if pre_val == NULL skip the barrier - // __ cmp(lir_cond_equal, addr, LIR_OprFact::oopConst(NULL)); - // __ branch(lir_cond_equal, T_OBJECT, slow->continuation()); - LIR_Opr addr = src; - // uint8_t* meta_addr = (uint8_t*) (side_metadata_base_address() + (addr >> 6)); - LIR_Opr offset = gen->new_pointer_register(); - __ move(addr, offset); - __ unsigned_shift_right(offset, 6, offset); - LIR_Opr base = gen->new_pointer_register(); - __ move(LIR_OprFact::longConst(side_metadata_base_address()), base); - LIR_Address* meta_addr = new LIR_Address(base, offset, T_BYTE); - // uint8_t byte_val = *meta_addr; - LIR_Opr byte_val = gen->new_register(T_INT); - __ move(meta_addr, byte_val); - - // intptr_t shift = (addr >> 3) & 0b111; - LIR_Opr shift = gen->new_register(T_INT); - __ move(addr, shift); - __ unsigned_shift_right(shift, 3, shift); - __ logical_and(shift, LIR_OprFact::intConst(0b111), shift); - // if (((byte_val >> shift) & 1) == 1) slow; - LIR_Opr result = byte_val; - __ unsigned_shift_right(result, shift, result, LIR_OprFact::illegalOpr); - __ logical_and(result, LIR_OprFact::intConst(1), result); - __ cmp(lir_cond_equal, result, LIR_OprFact::intConst(1)); + if (mmtk_enable_reference_load_barrier) { + if (access.is_oop() && (is_weak || is_phantom || is_anonymous)) { + // Register the value in the referent field with the pre-barrier + LabelObj *Lcont_anonymous; + if (is_anonymous) { + Lcont_anonymous = new LabelObj(); + generate_referent_check(access, Lcont_anonymous); + } + assert(result->is_register(), "must be"); + assert(result->type() == T_OBJECT, "must be an object"); + auto slow = new MMTkC1ReferenceLoadBarrierStub(result); + // Call slow-path only when concurrent marking is active + LIR_Opr cm_flag_addr_opr = gen->new_pointer_register(); + __ move(LIR_OprFact::longConst(uintptr_t(&CONCURRENT_MARKING_ACTIVE)), cm_flag_addr_opr); + LIR_Address* cm_flag_addr = new LIR_Address(cm_flag_addr_opr, T_BYTE); + LIR_Opr cm_flag = gen->new_register(T_INT); + __ move(cm_flag_addr, cm_flag); + // No slow-call if SATB is not active + __ cmp(lir_cond_equal, cm_flag, LIR_OprFact::intConst(1)); __ branch(lir_cond_equal, T_BYTE, slow); + __ branch_destination(slow->continuation()); + if (is_anonymous) { + __ branch_destination(Lcont_anonymous->label()); + } } - } else { - __ jump(slow); } +} - __ branch_destination(slow->continuation()); +void MMTkSATBBarrierSetC1::object_reference_write_pre(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val) const { + object_reference_write_pre_or_post(access, src, /* pre = */ true); } #undef __ +//////////////////// C2 //////////////////// + #define __ ideal. void MMTkSATBBarrierSetC2::object_reference_write_pre(GraphKit* kit, Node* src, Node* slot, Node* val) const { @@ -368,30 +151,7 @@ void MMTkSATBBarrierSetC2::object_reference_write_pre(GraphKit* kit, Node* src, MMTkIdealKit ideal(kit, true); - if (mmtk_enable_barrier_fastpath) { - Node* no_base = __ top(); - float unlikely = PROB_UNLIKELY(0.999); - - Node* zero = __ ConI(0); - Node* addr = __ CastPX(__ ctrl(), src); - Node* meta_addr = __ AddP(no_base, __ ConP(side_metadata_base_address()), __ URShiftX(addr, __ ConI(6))); - Node* byte = __ load(__ ctrl(), meta_addr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); - - Node* shift = __ URShiftX(addr, __ ConI(3)); - shift = __ AndI(__ ConvL2I(shift), __ ConI(7)); - Node* result = __ AndI(__ URShiftI(byte, shift), __ ConI(1)); - __ if_then(result, BoolTest::ne, zero, unlikely); { - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); - Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), "mmtk_barrier_call", src, slot, val); - } __ end_if(); - } else { - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); - Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call), "mmtk_barrier_call", src, slot, val); - // Looks like this is necessary - // See https://github.com/mmtk/openjdk/blob/c82e5c44adced4383162826c2c3933a83cfb139b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp#L288-L291 - Node* call = __ ctrl()->in(0); - call->add_req(slot); - } + object_reference_write_pre_or_post(ideal, src, /* pre = */ true); kit->final_sync(ideal); // Final sync IdealKit and GraphKit. } @@ -416,6 +176,12 @@ static void reference_load_barrier(GraphKit* kit, Node* slot, Node* val, bool em } static void reference_load_barrier_for_unknown_load(GraphKit* kit, Node* base_oop, Node* offset, Node* slot, Node* val, bool need_mem_bar) { + // Note: This function is copied from G1BarrierSetC2::insert_pre_barrier, + // and ShenandoahBarrierSetC2::insert_pre_barrier is probably copied from G1 as well. + // It basically implements BarrierSetC1::generate_referent_check in C2 IR. + // TODO: If another barrier needs weak reference load barrier, + // consider hoisting this function to a superclass. + // We could be accessing the referent field of a reference object. If so, when G1 // is enabled, we need to log the value in the referent field in an SATB buffer. // This routine performs some compile time filters and generates suitable @@ -519,13 +285,13 @@ Node* MMTkSATBBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_t return load; } -#if SOFT_REFERENCE_LOAD_BARRIER - if (on_weak) { - reference_load_barrier(kit, adr, load, true); - } else if (unknown) { - reference_load_barrier_for_unknown_load(kit, obj, offset, adr, load, !need_cpu_mem_bar); + if (mmtk_enable_reference_load_barrier) { + if (on_weak) { + reference_load_barrier(kit, adr, load, true); + } else if (unknown) { + reference_load_barrier_for_unknown_load(kit, obj, offset, adr, load, !need_cpu_mem_bar); + } } -#endif return load; } diff --git a/openjdk/barriers/mmtkSATBBarrier.hpp b/openjdk/barriers/mmtkSATBBarrier.hpp index 1f5cf639..b92a39c2 100644 --- a/openjdk/barriers/mmtkSATBBarrier.hpp +++ b/openjdk/barriers/mmtkSATBBarrier.hpp @@ -6,19 +6,20 @@ #include "../mmtkBarrierSetAssembler_x86.hpp" #include "../mmtkBarrierSetC1.hpp" #include "../mmtkBarrierSetC2.hpp" +#include "mmtkUnlogBitBarrier.hpp" #include "c1/c1_LIRAssembler.hpp" #include "c1/c1_MacroAssembler.hpp" #include "gc/shared/barrierSet.hpp" #include "opto/callnode.hpp" #include "opto/idealKit.hpp" -#define SIDE_METADATA_WORST_CASE_RATIO_LOG 1 -#define LOG_BYTES_IN_CHUNK 22 -#define CHUNK_MASK ((1L << LOG_BYTES_IN_CHUNK) - 1) +/// This file supports the `SATBBarrier` in MMTk core, +/// i.e. the barrier that remembers all children before it is first modified, +/// and remembers the target of the weak reference field when loaded. -const intptr_t SATB_METADATA_BASE_ADDRESS = (intptr_t) GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS; +//////////////////// Runtime //////////////////// -class MMTkSATBBarrierSetRuntime: public MMTkBarrierSetRuntime { +class MMTkSATBBarrierSetRuntime: public MMTkUnlogBitBarrierSetRuntime { public: // Interfaces called by `MMTkBarrierSet::AccessBarrier` virtual void object_reference_write_pre(oop src, oop* slot, oop target) const override; @@ -30,20 +31,21 @@ class MMTkSATBBarrierSetRuntime: public MMTkBarrierSetRuntime { virtual void load_reference(DecoratorSet decorators, oop value) const override; }; -class MMTkSATBBarrierSetAssembler: public MMTkBarrierSetAssembler { +//////////////////// Assembler //////////////////// + +class MMTkSATBBarrierSetAssembler: public MMTkUnlogBitBarrierSetAssembler { protected: virtual void object_reference_write_pre(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2) const override; - /// Generate C1 write barrier slow-call assembly code - virtual void generate_c1_pre_write_barrier_runtime_stub(StubAssembler* sasm) const override; public: - virtual void generate_c1_pre_write_barrier_stub(LIR_Assembler* ce, MMTkC1PreBarrierStub* stub) const override; virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) override; virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp_thread) override; }; -class MMTkSATBBarrierSetC1: public MMTkBarrierSetC1 { +//////////////////// C1 //////////////////// + +class MMTkSATBBarrierSetC1: public MMTkUnlogBitBarrierSetC1 { protected: - virtual void object_reference_write_pre(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val, CodeEmitInfo* info) const override; + virtual void object_reference_write_pre(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val) const override; virtual void load_at_resolved(LIRAccess& access, LIR_Opr result) override; @@ -52,7 +54,9 @@ class MMTkSATBBarrierSetC1: public MMTkBarrierSetC1 { } }; -class MMTkSATBBarrierSetC2: public MMTkBarrierSetC2 { +//////////////////// C2 //////////////////// + +class MMTkSATBBarrierSetC2: public MMTkUnlogBitBarrierSetC2 { protected: virtual void object_reference_write_pre(GraphKit* kit, Node* src, Node* slot, Node* val) const override; @@ -63,6 +67,8 @@ class MMTkSATBBarrierSetC2: public MMTkBarrierSetC2 { virtual Node* load_at_resolved(C2Access& access, const Type* val_type) const override; }; +//////////////////// Impl //////////////////// + struct MMTkSATBBarrier: MMTkBarrierImpl< MMTkSATBBarrierSetRuntime, MMTkSATBBarrierSetAssembler, diff --git a/openjdk/barriers/mmtkUnlogBitBarrier.cpp b/openjdk/barriers/mmtkUnlogBitBarrier.cpp new file mode 100644 index 00000000..0f87e3b2 --- /dev/null +++ b/openjdk/barriers/mmtkUnlogBitBarrier.cpp @@ -0,0 +1,226 @@ +#include "precompiled.hpp" +#include "mmtkUnlogBitBarrier.hpp" + +#include "runtime/interfaceSupport.inline.hpp" +#include "c1/c1_LIRAssembler.hpp" +#include "c1/c1_MacroAssembler.hpp" + +//////////////////// Assembler //////////////////// + +#define __ masm-> + +void MMTkUnlogBitBarrierSetAssembler::emit_check_unlog_bit_fast_path(MacroAssembler* masm, Label &done, Register obj, Register tmp1, Register tmp2) { + // Note that `tmp1` and `tmp2` are actual temporary registers available for use, + // not the `tmp1` and `tmp2` from `store_at`. + assert_different_registers(obj, tmp1, tmp2); + + // tmp2 = load-byte (UNLOG_BIT_BASE_ADDRESS + (obj >> 6)); + __ movptr(tmp1, obj); + __ shrptr(tmp1, 6); + __ movptr(tmp2, (intptr_t)UNLOG_BIT_BASE_ADDRESS); + __ movb(tmp2, Address(tmp2, tmp1)); + // tmp1 = (obj >> 3) & 7 + __ movptr(tmp1, obj); + __ shrptr(tmp1, 3); + __ andptr(tmp1, 7); + // tmp2 = tmp2 >> tmp1 + __ xchgptr(tmp1, rcx); + __ shrptr(tmp2); + __ xchgptr(tmp1, rcx); + // if ((tmp2 & 1) == 0) goto done; + __ testptr(tmp2, 1); + __ jcc(Assembler::zero, done); +} + +#undef __ + +#define __ masm-> + +void MMTkUnlogBitBarrierSetAssembler::object_reference_write_pre_or_post(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, bool pre) { + Label done; + Register obj = dst.base(); + if (mmtk_enable_barrier_fastpath) { + // For some instructions in the template table, + // `tmp1` and `tmp2` may overlap with `dst`. + // For example, if the instruction is `aastore`, + // then `dst.base() == tmp1 && dst.index() == tmp2`. + // This is a bug that has been fixed upstream in OpenJDK 21. + // See https://bugs.openjdk.org/browse/JDK-8301371 + // + // We can't overwrite those registers, + // so we steal two scratch register + // and assert they don't overlap with other registers. + Register tmp3 = rscratch1; + Register tmp4 = rscratch2; + assert_different_registers(dst.base(), dst.index(), val, tmp3, tmp4); + + emit_check_unlog_bit_fast_path(masm, done, obj, tmp3, tmp4); + } + + if (pre) { + // This is a pre-barrier. Preserve caller-saved regs for the actual write operation. + __ pusha(); + } + + __ movptr(c_rarg0, obj); + // Neither the ObjectBarrier nor the SATBBarrier need to know the slot or the value. + // We just set both args to nullptr. + // We may need to pass actual arguments if we support other barriers. + // + // Note: If the `compensate_val_reg` parameter in the post barrier is true, and we are using + // compressed oops, the `val` register will be holding a compressed pointer to the target object + // due to the way `BarrierSetAssembler::store_at` works. If the write barrier needs to know the + // target, we will need to decompress it before passing it to the barrier slow path. + __ xorptr(c_rarg1, c_rarg1); + __ xorptr(c_rarg2, c_rarg2); + + address entry_point = mmtk_enable_barrier_fastpath ? FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call) + : pre ? FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call) + : FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call); + + __ call_VM_leaf_base(entry_point, 3); + + if (pre) { + __ popa(); + } + + if (mmtk_enable_barrier_fastpath) { + __ bind(done); + } +} + +#undef __ + +//////////////////// Assembler/C1 //////////////////// + +#define __ ce->masm()-> + +void MMTkUnlogBitBarrierSetAssembler::generate_c1_unlog_bit_barrier_slow_path_stub(LIR_Assembler* ce, MMTkC1UnlogBitBarrierSlowPathStub* stub) const { + MMTkBarrierSetC1* bs = (MMTkBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1(); + __ bind(*stub->entry()); + ce->store_parameter(stub->src->as_pointer_register(), 0); + ce->store_parameter(0, 1); + ce->store_parameter(0, 2); + CodeBlob* code_blob = stub->fast_path_enabled ? bs->object_reference_write_slow_c1_runtime_code_blob() + : stub->pre ? bs->object_reference_write_pre_c1_runtime_code_blob() + : bs->object_reference_write_post_c1_runtime_code_blob(); + __ call(RuntimeAddress(code_blob->code_begin())); + __ jmp(*stub->continuation()); +} + +void MMTkC1UnlogBitBarrierSlowPathStub::emit_code(LIR_Assembler* ce) { + MMTkUnlogBitBarrierSetAssembler* bs = (MMTkUnlogBitBarrierSetAssembler*) BarrierSet::barrier_set()->barrier_set_assembler(); + bs->generate_c1_unlog_bit_barrier_slow_path_stub(ce, this); +} + +#undef __ + +//////////////////// C1 //////////////////// + +#ifdef ASSERT +#define __ gen->lir(__FILE__, __LINE__)-> +#else +#define __ gen->lir()-> +#endif + +void MMTkUnlogBitBarrierSetC1::emit_check_unlog_bit_fast_path(LIRGenerator* gen, LIR_Opr src, CodeStub* slow) { + // We need to do bit operations on the address of `src`. In order to move `src` (`T_OBJECT` or + // `T_ARRAY`) to a pointer regiseter (`T_LONG` on 64 bit), the source operand must be in + // register, in which case `LIR_Assembler::reg2reg` works as expected. Otherwise `stack2ref` + // will complain that the source (`T_OBJECT` or `T_ARRAY` is single-cpu while the destination + // `T_LONG` is double-cpu). + // + // However, checking `src.is_register()` won't work because the same LIR code may be compiled + // again. Even it is register the first time, `src.is_stack()` may instead be true at the second + // time. + // + // So we introduce an intermediate step. We move `src` into `addr` which is a `T_OBJECT` + // register first to make sure it is in register. Then we move `addr` to newly created pointer + // registers. + LIR_Opr addr = gen->new_register(T_OBJECT); + __ move(src, addr); + + // uint8_t* meta_addr = (uint8_t*) (UNLOG_BIT_BASE_ADDRESS + (addr >> 6)); + LIR_Opr offset = gen->new_pointer_register(); + __ move(addr, offset); + __ unsigned_shift_right(offset, 6, offset); + LIR_Opr base = gen->new_pointer_register(); + __ move(LIR_OprFact::longConst(UNLOG_BIT_BASE_ADDRESS), base); + LIR_Address* meta_addr = new LIR_Address(base, offset, T_BYTE); + + // uint8_t byte_val = *meta_addr; + LIR_Opr byte_val = gen->new_register(T_INT); + __ move(meta_addr, byte_val); + + // uint32_t shift = ((uint32_t)addr >> 3) & 0b111; + LIR_Opr shift = gen->new_register(T_INT); + __ move(addr, shift); + __ unsigned_shift_right(shift, 3, shift); + __ logical_and(shift, LIR_OprFact::intConst(0b111), shift); + + // if (((byte_val >> shift) & 1) == 1) slow; + LIR_Opr result = byte_val; + __ unsigned_shift_right(result, shift, result, LIR_OprFact::illegalOpr); + __ logical_and(result, LIR_OprFact::intConst(1), result); + __ cmp(lir_cond_equal, result, LIR_OprFact::intConst(1)); + __ branch(lir_cond_equal, T_BYTE, slow); +} + +void MMTkUnlogBitBarrierSetC1::object_reference_write_pre_or_post(LIRAccess& access, LIR_Opr src, bool pre) { + LIRGenerator* gen = access.gen(); + DecoratorSet decorators = access.decorators(); + if ((decorators & IN_HEAP) == 0) return; + + CodeStub* slow = new MMTkC1UnlogBitBarrierSlowPathStub(src, mmtk_enable_barrier_fastpath, pre); + + if (mmtk_enable_barrier_fastpath) { + emit_check_unlog_bit_fast_path(gen, src, slow); + } else { + __ jump(slow); + } + + __ branch_destination(slow->continuation()); +} + +#undef __ + +//////////////////// C2 //////////////////// + +#define __ ideal. + +Node* MMTkUnlogBitBarrierSetC2::emit_check_unlog_bit_fast_path(MMTkIdealKit& ideal, Node* obj) { + Node* addr = __ CastPX(__ ctrl(), obj); + Node* no_base = __ top(); + Node* meta_addr = __ AddP(no_base, __ ConP(UNLOG_BIT_BASE_ADDRESS), __ URShiftX(addr, __ ConI(6))); + Node* byte = __ load(__ ctrl(), meta_addr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); + Node* shift = __ URShiftX(addr, __ ConI(3)); + shift = __ AndI(__ ConvL2I(shift), __ ConI(7)); + Node* result = __ AndI(__ URShiftI(byte, shift), __ ConI(1)); + + return result; +} + +void MMTkUnlogBitBarrierSetC2::object_reference_write_pre_or_post(MMTkIdealKit& ideal, Node* src, bool pre) { + if (mmtk_enable_barrier_fastpath) { + Node* result = emit_check_unlog_bit_fast_path(ideal, src); + + Node* zero = __ ConI(0); + float unlikely = PROB_UNLIKELY(0.999); + __ if_then(result, BoolTest::ne, zero, unlikely); { + Node* result = emit_check_unlog_bit_fast_path(ideal, src); + + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); + address entry_point = FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call); + Node* null_node = __ NullP(); + Node* x = __ make_leaf_call(tf, entry_point, "mmtk_barrier_call", src, null_node, null_node); + } __ end_if(); + } else { + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); + address entry_point = pre ? FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call) + : FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call); + Node* null_node = __ NullP(); + Node* x = __ make_leaf_call(tf, entry_point, "mmtk_barrier_call", src, null_node, null_node); + } +} + +#undef __ diff --git a/openjdk/barriers/mmtkUnlogBitBarrier.hpp b/openjdk/barriers/mmtkUnlogBitBarrier.hpp new file mode 100644 index 00000000..ec4e0ac2 --- /dev/null +++ b/openjdk/barriers/mmtkUnlogBitBarrier.hpp @@ -0,0 +1,86 @@ +#ifndef MMTK_OPENJDK_BARRIERS_MMTK_UNLOG_BIT_BARRIER_HPP +#define MMTK_OPENJDK_BARRIERS_MMTK_UNLOG_BIT_BARRIER_HPP + +#include "../mmtkBarrierSet.hpp" +#include "../mmtkBarrierSetAssembler_x86.hpp" +#include "../mmtkBarrierSetC1.hpp" +#include "../mmtkBarrierSetC2.hpp" + +/// This file contains abstract barrier sets for barriers based on the (object-grained) unlog bit. + +struct MMTkC1UnlogBitBarrierSlowPathStub; + +const uintptr_t UNLOG_BIT_BASE_ADDRESS = GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS; + +//////////////////// Runtime //////////////////// + +class MMTkUnlogBitBarrierSetRuntime: public MMTkBarrierSetRuntime { +protected: + static bool is_unlog_bit_set(oop obj) { + uintptr_t addr = (uintptr_t) (void*) obj; + uint8_t* meta_addr = (uint8_t*) (UNLOG_BIT_BASE_ADDRESS + (addr >> 6)); + uintptr_t shift = (addr >> 3) & 0b111; + uint8_t byte_val = *meta_addr; + return ((byte_val >> shift) & 1) == 1; + } +}; + +//////////////////// Assembler //////////////////// + +class MMTkUnlogBitBarrierSetAssembler: public MMTkBarrierSetAssembler { +protected: + static void emit_check_unlog_bit_fast_path(MacroAssembler* masm, Label &done, Register obj, Register tmp1, Register tmp2); + static void object_reference_write_pre_or_post(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, bool pre); + +public: + /// Generate C1 barrier slow path stub + void generate_c1_unlog_bit_barrier_slow_path_stub(LIR_Assembler* ce, MMTkC1UnlogBitBarrierSlowPathStub* stub) const; +}; + +//////////////////// C1 //////////////////// + +class MMTkUnlogBitBarrierSetC1: public MMTkBarrierSetC1 { +protected: + static void emit_check_unlog_bit_fast_path(LIRGenerator* gen, LIR_Opr addr, CodeStub* slow); + static void object_reference_write_pre_or_post(LIRAccess& access, LIR_Opr src, bool pre); +}; + +/// C1 write barrier slow path stub. +/// +/// This stub calls `MMTkBarrierSetRuntime::object_reference_write_{slow,pre,post}_call` depending +/// on whether barrier fast paths are enabled and whether it is pre or post barrier, passing the +/// `src` argument, and leaving other arguments as nullptr. This is enough for object-remembering +/// barriers based on the unlog bit, including the ObjectBarrier and the SATBBarrier, because only +/// the `src` argument is significant. +/// +/// Note that this stub cannot be generalized to field-remembering barriers as it does not pass the +/// field or the old/new values. Field-remembering barriers should implement their own slow-path +/// stub(s). +struct MMTkC1UnlogBitBarrierSlowPathStub: CodeStub { + LIR_Opr src; + bool fast_path_enabled; + bool pre; + + MMTkC1UnlogBitBarrierSlowPathStub(LIR_Opr src, bool fast_path_enabled, bool pre): + fast_path_enabled(fast_path_enabled), src(src), pre(pre) {} + + virtual void emit_code(LIR_Assembler* ce) override; + + virtual void visit(LIR_OpVisitState* visitor) override { + visitor->do_slow_case(); + assert(src->is_valid(), "src must be valid"); + visitor->do_input(src); + } + + NOT_PRODUCT(virtual void print_name(outputStream* out) const { out->print("MMTkC1UnlogBitBarrierSlowPathStub"); }); +}; + +//////////////////// C2 //////////////////// + +class MMTkUnlogBitBarrierSetC2: public MMTkBarrierSetC2 { +protected: + static Node* emit_check_unlog_bit_fast_path(MMTkIdealKit& ideal, Node* obj); + static void object_reference_write_pre_or_post(MMTkIdealKit& ideal, Node* src, bool pre); +}; + +#endif // MMTK_OPENJDK_BARRIERS_MMTK_UNLOG_BIT_BARRIER_HPP diff --git a/openjdk/mmtkBarrierSet.cpp b/openjdk/mmtkBarrierSet.cpp index d0c4838a..b6509f05 100644 --- a/openjdk/mmtkBarrierSet.cpp +++ b/openjdk/mmtkBarrierSet.cpp @@ -38,6 +38,7 @@ bool mmtk_enable_allocation_fastpath = true; bool mmtk_enable_barrier_fastpath = true; +bool mmtk_enable_reference_load_barrier = true; MMTkAllocatorOffsets get_tlab_top_and_end_offsets(AllocatorSelector selector) { int tlab_top_offset, tlab_end_offset; diff --git a/openjdk/mmtkBarrierSet.hpp b/openjdk/mmtkBarrierSet.hpp index 37400e2a..d1494307 100644 --- a/openjdk/mmtkBarrierSet.hpp +++ b/openjdk/mmtkBarrierSet.hpp @@ -39,6 +39,7 @@ extern bool mmtk_enable_allocation_fastpath; extern bool mmtk_enable_barrier_fastpath; +extern bool mmtk_enable_reference_load_barrier; const intptr_t VO_BIT_BASE_ADDRESS = VO_BIT_ADDRESS; diff --git a/openjdk/mmtkBarrierSetAssembler_x86.cpp b/openjdk/mmtkBarrierSetAssembler_x86.cpp index 112cf97a..fdb4ae9b 100644 --- a/openjdk/mmtkBarrierSetAssembler_x86.cpp +++ b/openjdk/mmtkBarrierSetAssembler_x86.cpp @@ -142,37 +142,49 @@ void MMTkBarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register threa #undef __ -#define __ sasm-> - -void MMTkBarrierSetAssembler::generate_c1_ref_load_barrier_runtime_stub(StubAssembler* sasm) const { - __ prologue("mmtk_ref_load_barrier", false); +//////////////////// Assembler for C1 //////////////////// - // Address store_addr(rbp, 2*BytesPerWord); +// Generate runtime stubs for the "runtime code blobs" in MMTkBarrierSetC1 - Label done, runtime; +#define __ sasm-> - __ push(c_rarg0); - __ push(rax); +void MMTkBarrierSetAssembler::generate_c1_runtime_stub_general(StubAssembler* sasm, const char* name, address entry_point, int argc) { + __ prologue(name, false); + __ save_live_registers_no_oop_map(true); - __ load_parameter(0, c_rarg0); + if (argc > 0) __ load_parameter(0, c_rarg0); + if (argc > 1) __ load_parameter(1, c_rarg1); + if (argc > 2) __ load_parameter(2, c_rarg2); + if (argc > 3) { + guarantee(false, "Too many args"); + } - __ bind(runtime); + __ call_VM_leaf_base(entry_point, 3); - __ save_live_registers_no_oop_map(true); + __ restore_live_registers(true); + __ epilogue(); +} - __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), 1); +void MMTkBarrierSetAssembler::generate_c1_load_reference_runtime_stub(StubAssembler* sasm) { + generate_c1_runtime_stub_general(sasm, "c1_load_reference_runtime_stub", FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), 1); +} - __ restore_live_registers(true); +void MMTkBarrierSetAssembler::generate_c1_object_reference_write_pre_runtime_stub(StubAssembler* sasm) { + generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_pre_stub", FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call), 3); +} - __ bind(done); - __ pop(rax); - __ pop(c_rarg0); +void MMTkBarrierSetAssembler::generate_c1_object_reference_write_post_runtime_stub(StubAssembler* sasm) { + generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_post_stub", FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call), 3); +} - __ epilogue(); +void MMTkBarrierSetAssembler::generate_c1_object_reference_write_slow_runtime_stub(StubAssembler* sasm) { + generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_slow_stub", FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), 3); } #undef __ +// Generate code stubs + #define __ ce->masm()-> void MMTkBarrierSetAssembler::generate_c1_ref_load_barrier_stub_call(LIR_Assembler* ce, MMTkC1ReferenceLoadBarrierStub* stub) { @@ -186,7 +198,7 @@ void MMTkBarrierSetAssembler::generate_c1_ref_load_barrier_stub_call(LIR_Assembl __ cmpptr(val_reg, (int32_t) NULL_WORD); __ jcc(Assembler::equal, *stub->continuation()); ce->store_parameter(stub->val->as_register(), 0); - __ call(RuntimeAddress(bs->_ref_load_barrier_c1_runtime_code_blob->code_begin())); + __ call(RuntimeAddress(bs->load_reference_c1_runtime_code_blob()->code_begin())); __ jmp(*stub->continuation()); } diff --git a/openjdk/mmtkBarrierSetAssembler_x86.hpp b/openjdk/mmtkBarrierSetAssembler_x86.hpp index 47bade05..8487c7e9 100644 --- a/openjdk/mmtkBarrierSetAssembler_x86.hpp +++ b/openjdk/mmtkBarrierSetAssembler_x86.hpp @@ -5,15 +5,11 @@ #include "gc/shared/barrierSetAssembler.hpp" class MMTkBarrierSetC1; -class MMTkC1PreBarrierStub; -class MMTkC1PostBarrierStub; class MMTkC1ReferenceLoadBarrierStub; class LIR_Assembler; class StubAssembler; class MMTkBarrierSetAssembler: public BarrierSetAssembler { - friend class MMTkBarrierSetC1; - protected: /// Full pre-barrier virtual void object_reference_write_pre(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2) const {} @@ -29,12 +25,6 @@ class MMTkBarrierSetAssembler: public BarrierSetAssembler { return !in_heap || (skip_const_null && val == noreg); } - /// Generate C1 pre write barrier slow-call assembly code - virtual void generate_c1_pre_write_barrier_runtime_stub(StubAssembler* sasm) const {}; - /// Generate C1 post write barrier slow-call assembly code - virtual void generate_c1_post_write_barrier_runtime_stub(StubAssembler* sasm) const {}; - virtual void generate_c1_ref_load_barrier_runtime_stub(StubAssembler* sasm) const; - public: virtual void eden_allocate(MacroAssembler* masm, Register thread, Register obj, Register var_size_in_bytes, int con_size_in_bytes, Register t1, Label& slow_case) override; virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Address dst, Register val, Register tmp1, Register tmp2) override { @@ -45,9 +35,19 @@ class MMTkBarrierSetAssembler: public BarrierSetAssembler { if (type == T_OBJECT || type == T_ARRAY) object_reference_write_post(masm, decorators, dst, val, tmp1, tmp2, true); } - /// Generate C1 write barrier slow-call stub - virtual void generate_c1_pre_write_barrier_stub(LIR_Assembler* ce, MMTkC1PreBarrierStub* stub) const {}; - virtual void generate_c1_post_write_barrier_stub(LIR_Assembler* ce, MMTkC1PostBarrierStub* stub) const {}; + //////////////////// Assembler for C1 //////////////////// + + // Generate runtime stubs for the "runtime code blobs" in MMTkBarrierSetC1 +private: + static void generate_c1_runtime_stub_general(StubAssembler* sasm, const char* name, address func, int argc); +public: + static void generate_c1_load_reference_runtime_stub(StubAssembler* sasm); + static void generate_c1_object_reference_write_pre_runtime_stub(StubAssembler* sasm); + static void generate_c1_object_reference_write_post_runtime_stub(StubAssembler* sasm); + static void generate_c1_object_reference_write_slow_runtime_stub(StubAssembler* sasm); + + // Generate slow-path code stubs +public: static void generate_c1_ref_load_barrier_stub_call(LIR_Assembler* ce, MMTkC1ReferenceLoadBarrierStub* stub); }; #endif // MMTK_OPENJDK_MMTK_BARRIER_SET_ASSEMBLER_X86_HPP diff --git a/openjdk/mmtkBarrierSetC1.cpp b/openjdk/mmtkBarrierSetC1.cpp index 09f6e469..2afc1711 100644 --- a/openjdk/mmtkBarrierSetC1.cpp +++ b/openjdk/mmtkBarrierSetC1.cpp @@ -1,56 +1,31 @@ +#include "precompiled.hpp" #include "c1/c1_CodeStubs.hpp" #include "gc/shared/c1/barrierSetC1.hpp" #include "mmtkBarrierSetAssembler_x86.hpp" #include "mmtkBarrierSetC1.hpp" void MMTkBarrierSetC1::generate_c1_runtime_stubs(BufferBlob* buffer_blob) { - class MMTkPreBarrierCodeGenClosure : public StubAssemblerCodeGenClosure { - virtual OopMapSet* generate_code(StubAssembler* sasm) override { - MMTkBarrierSetAssembler* bs = (MMTkBarrierSetAssembler*) BarrierSet::barrier_set()->barrier_set_assembler(); - bs->generate_c1_pre_write_barrier_runtime_stub(sasm); - return NULL; - } + using GenStubFunc = void(*)(StubAssembler*); + class RuntimeCodeBlobCodeGenClosure : public StubAssemblerCodeGenClosure { + GenStubFunc gen_stub; public: - MMTkPreBarrierCodeGenClosure() {} - }; + RuntimeCodeBlobCodeGenClosure(GenStubFunc gen_stub): gen_stub(gen_stub) {} - class MMTkPostBarrierCodeGenClosure : public StubAssemblerCodeGenClosure { virtual OopMapSet* generate_code(StubAssembler* sasm) override { - MMTkBarrierSetAssembler* bs = (MMTkBarrierSetAssembler*) BarrierSet::barrier_set()->barrier_set_assembler(); - bs->generate_c1_post_write_barrier_runtime_stub(sasm); - return NULL; + gen_stub(sasm); + return nullptr; } - public: - MMTkPostBarrierCodeGenClosure() {} }; - MMTkPreBarrierCodeGenClosure pre_write_code_gen_cl; - _pre_barrier_c1_runtime_code_blob = Runtime1::generate_blob(buffer_blob, -1, "mmtk_pre_write_code_gen_cl", false, &pre_write_code_gen_cl); - MMTkPostBarrierCodeGenClosure post_write_code_gen_cl; - _post_barrier_c1_runtime_code_blob = Runtime1::generate_blob(buffer_blob, -1, "mmtk_post_write_code_gen_cl", false, &post_write_code_gen_cl); - // MMTkBarrierCodeGenClosure write_code_gen_cl_patch_fix(true); - // _write_barrier_c1_runtime_code_blob_with_patch_fix = Runtime1::generate_blob(buffer_blob, -1, "write_code_gen_cl_patch_fix", false, &write_code_gen_cl_patch_fix); - - class MMTkRefLoadBarrierCodeGenClosure : public StubAssemblerCodeGenClosure { - virtual OopMapSet* generate_code(StubAssembler* sasm) override { - MMTkBarrierSetAssembler* bs = (MMTkBarrierSetAssembler*) BarrierSet::barrier_set()->barrier_set_assembler(); - bs->generate_c1_ref_load_barrier_runtime_stub(sasm); - return NULL; - } + auto do_code_blob = [buffer_blob](const char* name, GenStubFunc gen_stub) { + RuntimeCodeBlobCodeGenClosure closure(gen_stub); + return Runtime1::generate_blob(buffer_blob, -1, name, false, &closure); }; - MMTkRefLoadBarrierCodeGenClosure load_code_gen_cl; - _ref_load_barrier_c1_runtime_code_blob = Runtime1::generate_blob(buffer_blob, -1, "load_code_gen_cl", false, &load_code_gen_cl); -} - -void MMTkC1PostBarrierStub::emit_code(LIR_Assembler* ce) { - MMTkBarrierSetAssembler* bs = (MMTkBarrierSetAssembler*) BarrierSet::barrier_set()->barrier_set_assembler(); - bs->generate_c1_post_write_barrier_stub(ce, this); -} - -void MMTkC1PreBarrierStub::emit_code(LIR_Assembler* ce) { - MMTkBarrierSetAssembler* bs = (MMTkBarrierSetAssembler*) BarrierSet::barrier_set()->barrier_set_assembler(); - bs->generate_c1_pre_write_barrier_stub(ce, this); + _load_reference_c1_runtime_code_blob = do_code_blob("c1_load_reference_stub", &MMTkBarrierSetAssembler::generate_c1_load_reference_runtime_stub); + _object_reference_write_pre_c1_runtime_code_blob = do_code_blob("c1_object_reference_write_pre_stub", &MMTkBarrierSetAssembler::generate_c1_object_reference_write_pre_runtime_stub); + _object_reference_write_post_c1_runtime_code_blob = do_code_blob("c1_object_reference_write_post_stub", &MMTkBarrierSetAssembler::generate_c1_object_reference_write_post_runtime_stub); + _object_reference_write_slow_c1_runtime_code_blob = do_code_blob("c1_object_reference_write_slow_stub", &MMTkBarrierSetAssembler::generate_c1_object_reference_write_slow_runtime_stub); } void MMTkC1ReferenceLoadBarrierStub::emit_code(LIR_Assembler* ce) { diff --git a/openjdk/mmtkBarrierSetC1.hpp b/openjdk/mmtkBarrierSetC1.hpp index 8d65b6b3..03e3f2ab 100644 --- a/openjdk/mmtkBarrierSetC1.hpp +++ b/openjdk/mmtkBarrierSetC1.hpp @@ -7,28 +7,38 @@ class MMTkBarrierSetAssembler; class MMTkBarrierSetC1 : public BarrierSetC1 { - friend class MMTkBarrierSetAssembler; +private: + // Code blobs for calling into runtime functions. + // Here in MMTkBarrierSetC1, + // we have one "runtime code blob" for every runtime function we call, + // i.e. `MMTkBarrierSetRuntime::*_call` + // There is no general rules that enfoce this in OpenJDK, + // except that these code blobs are global and implemented in machine-specific assembly. + // Our barrier slow paths are relatively simple, i.e. calling into MMTk-core. + // So we only need such "runtime code blobs" for calling MMTk functions. + // If we want to implement medium paths in machine-specific ways, + // we may consider defining new code blobs for specific barriers. + CodeBlob* _load_reference_c1_runtime_code_blob; + CodeBlob* _object_reference_write_pre_c1_runtime_code_blob; + CodeBlob* _object_reference_write_post_c1_runtime_code_blob; + CodeBlob* _object_reference_write_slow_c1_runtime_code_blob; protected: - CodeBlob* _pre_barrier_c1_runtime_code_blob; - CodeBlob* _post_barrier_c1_runtime_code_blob; - CodeBlob* _ref_load_barrier_c1_runtime_code_blob; - /// Full pre-barrier - virtual void object_reference_write_pre(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val, CodeEmitInfo* info) const {} + virtual void object_reference_write_pre(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val) const {} /// Full post-barrier virtual void object_reference_write_post(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val) const {} /// Substituting write barrier virtual void store_at_resolved(LIRAccess& access, LIR_Opr value) override { - if (access.is_oop()) object_reference_write_pre(access, access.base().opr(), access.resolved_addr(), value, access.patch_emit_info()); + if (access.is_oop()) object_reference_write_pre(access, access.base().opr(), access.resolved_addr(), value); BarrierSetC1::store_at_resolved(access, value); if (access.is_oop()) object_reference_write_post(access, access.base().opr(), access.resolved_addr(), value); } /// Substituting write barrier (cmpxchg) virtual LIR_Opr atomic_cmpxchg_at_resolved(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value) override { - if (access.is_oop()) object_reference_write_pre(access, access.base().opr(), access.resolved_addr(), new_value.result(), NULL); + if (access.is_oop()) object_reference_write_pre(access, access.base().opr(), access.resolved_addr(), new_value.result()); LIR_Opr result = BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value); if (access.is_oop()) object_reference_write_post(access, access.base().opr(), access.resolved_addr(), new_value.result()); return result; @@ -36,7 +46,7 @@ class MMTkBarrierSetC1 : public BarrierSetC1 { /// Substituting write barrier (xchg) virtual LIR_Opr atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value) override { - if (access.is_oop()) object_reference_write_pre(access, access.base().opr(), access.resolved_addr(), value.result(), NULL); + if (access.is_oop()) object_reference_write_pre(access, access.base().opr(), access.resolved_addr(), value.result()); LIR_Opr result = BarrierSetC1::atomic_xchg_at_resolved(access, value); if (access.is_oop()) object_reference_write_post(access, access.base().opr(), access.resolved_addr(), value.result()); return result; @@ -60,83 +70,31 @@ class MMTkBarrierSetC1 : public BarrierSetC1 { public: - MMTkBarrierSetC1() - : _pre_barrier_c1_runtime_code_blob(NULL), - _post_barrier_c1_runtime_code_blob(NULL) {} + MMTkBarrierSetC1() {} - CodeBlob* pre_barrier_c1_runtime_code_blob() { return _pre_barrier_c1_runtime_code_blob; } - CodeBlob* post_barrier_c1_runtime_code_blob() { return _post_barrier_c1_runtime_code_blob; } + CodeBlob* load_reference_c1_runtime_code_blob() { return _load_reference_c1_runtime_code_blob; } + CodeBlob* object_reference_write_pre_c1_runtime_code_blob() { return _object_reference_write_pre_c1_runtime_code_blob; } + CodeBlob* object_reference_write_post_c1_runtime_code_blob() { return _object_reference_write_post_c1_runtime_code_blob; } + CodeBlob* object_reference_write_slow_c1_runtime_code_blob() { return _object_reference_write_slow_c1_runtime_code_blob; } /// Generate C1 write barrier slow-call C1-LIR code virtual void generate_c1_runtime_stubs(BufferBlob* buffer_blob) override; }; -/// C1 pre write barrier slow-call stub. -/// The default behaviour is to call `MMTkBarrierSetRuntime::object_reference_write_pre_call` and pass all the three args. -/// Barrier implementations may inherit from this class, and override `emit_code` to perform a specialized slow-path call. -struct MMTkC1PreBarrierStub: CodeStub { - LIR_Opr src, slot, new_val; - CodeEmitInfo* info; // Code patching info - LIR_PatchCode patch_code; // Enable code patching? - LIR_Opr scratch = NULL; // Scratch register for the resolved field - - MMTkC1PreBarrierStub(LIR_Opr src, LIR_Opr slot, LIR_Opr new_val, CodeEmitInfo* info = NULL, LIR_PatchCode patch_code = lir_patch_none): src(src), slot(slot), new_val(new_val), info(info), patch_code(patch_code) {} - - virtual void emit_code(LIR_Assembler* ce) override; - - virtual void visit(LIR_OpVisitState* visitor) override { - if (info != NULL) { - visitor->do_slow_case(info); - } else { - visitor->do_slow_case(); - } - if (src != NULL) visitor->do_input(src); - if (slot != NULL) visitor->do_input(slot); - if (new_val != NULL) visitor->do_input(new_val); - if (scratch != NULL) { - assert(scratch->is_oop(), "must be"); - visitor->do_temp(scratch); - } - } - - NOT_PRODUCT(virtual void print_name(outputStream* out) const { out->print("MMTkC1PreBarrierStub"); }); -}; - -/// C1 post write barrier slow-call stub. -/// The default behaviour is to call `MMTkBarrierSetRuntime::object_reference_write_post_call` and pass all the three args. -/// Barrier implementations may inherit from this class, and override `emit_code` to perform a specialized slow-path call. -struct MMTkC1PostBarrierStub: CodeStub { - LIR_Opr src, slot, new_val; - - MMTkC1PostBarrierStub(LIR_Opr src, LIR_Opr slot, LIR_Opr new_val): src(src), slot(slot), new_val(new_val) {} - - virtual void emit_code(LIR_Assembler* ce) override; - - virtual void visit(LIR_OpVisitState* visitor) override { - visitor->do_slow_case(); - if (src != NULL) visitor->do_input(src); - if (slot != NULL) visitor->do_input(slot); - if (new_val != NULL) visitor->do_input(new_val); - } - - NOT_PRODUCT(virtual void print_name(outputStream* out) const { out->print("MMTkC1PostBarrierStub"); }); -}; - +/// The code stub for (weak) reference loading barrier slow path. +/// It will call `MMTkBarrierSetRuntime::load_reference_call` if `val` is not null. +/// Currently only the SATB barrier uses this code stub. struct MMTkC1ReferenceLoadBarrierStub: CodeStub { LIR_Opr val; - CodeEmitInfo* info; // Code patching info - MMTkC1ReferenceLoadBarrierStub(LIR_Opr val, CodeEmitInfo* info = NULL): val(val), info(info) {} + MMTkC1ReferenceLoadBarrierStub(LIR_Opr val): val(val) {} virtual void emit_code(LIR_Assembler* ce) override; virtual void visit(LIR_OpVisitState* visitor) override { - if (info != NULL) { - visitor->do_slow_case(info); - } else { - visitor->do_slow_case(); - } - if (val != NULL) visitor->do_input(val); + visitor->do_slow_case(); + assert(val->is_valid(), "val must be valid"); + visitor->do_input(val); } NOT_PRODUCT(virtual void print_name(outputStream* out) const { out->print("MMTkC1ReferenceLoadBarrierStub"); }); diff --git a/openjdk/mmtkBarrierSetC2.hpp b/openjdk/mmtkBarrierSetC2.hpp index d8052d9a..90270d8e 100644 --- a/openjdk/mmtkBarrierSetC2.hpp +++ b/openjdk/mmtkBarrierSetC2.hpp @@ -113,6 +113,7 @@ class MMTkIdealKit: public IdealKit { inline Node* CastXP(Node* x) { return transform(new CastX2PNode(x)); } inline Node* URShiftI(Node* l, Node* r) { return transform(new URShiftINode(l, r)); } inline Node* ConP(intptr_t ptr) { return makecon(TypeRawPtr::make((address) ptr)); } + inline Node* NullP() { return makecon(TypePtr::NULL_PTR); } template inline const TypeFunc* func_type(Types... types) { diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index a6c1c26a..e84bf458 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -94,6 +94,14 @@ jint MMTkHeap::initialize() { set_bool_option_from_env_var("MMTK_ENABLE_ALLOCATION_FASTPATH", &mmtk_enable_allocation_fastpath); set_bool_option_from_env_var("MMTK_ENABLE_BARRIER_FASTPATH", &mmtk_enable_barrier_fastpath); + set_bool_option_from_env_var("MMTK_ENABLE_REFERENCE_LOAD_BARRIER", &mmtk_enable_reference_load_barrier); + + if (!mmtk_enable_reference_load_barrier) { + fprintf(stderr, + "WARNING! Reference load barrier is disabled. " + "The program cannot run correctly unless reference processing is disabled, " + "in which case all java.lang.ref.Reference instances are treated as strong.\n"); + } const size_t min_heap_size = collector_policy()->min_heap_byte_size(); const size_t max_heap_size = collector_policy()->max_heap_byte_size();