From a03671e2b3d8b99303d66a95aa85de95a5fd1268 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Fri, 19 Sep 2025 23:08:06 +0800 Subject: [PATCH 01/46] WIP: MMTkUnlogBitBarrierSetRuntime --- mmtk/Cargo.lock | 2 -- mmtk/Cargo.toml | 3 ++- openjdk/barriers/mmtkObjectBarrier.cpp | 12 ++-------- openjdk/barriers/mmtkObjectBarrier.hpp | 3 ++- openjdk/barriers/mmtkUnlogBitBarrier.cpp | 5 ++++ openjdk/barriers/mmtkUnlogBitBarrier.hpp | 30 ++++++++++++++++++++++++ 6 files changed, 41 insertions(+), 14 deletions(-) create mode 100644 openjdk/barriers/mmtkUnlogBitBarrier.cpp create mode 100644 openjdk/barriers/mmtkUnlogBitBarrier.hpp diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 0e0c77a7..b7b3c7c6 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -492,7 +492,6 @@ dependencies = [ [[package]] name = "mmtk" version = "0.31.0" -source = "git+https://github.com/mmtk/mmtk-core.git?rev=a4dd70cb70a116a32b1bbb20501c48f77f49181b#a4dd70cb70a116a32b1bbb20501c48f77f49181b" dependencies = [ "atomic", "atomic-traits", @@ -531,7 +530,6 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.31.0" -source = "git+https://github.com/mmtk/mmtk-core.git?rev=a4dd70cb70a116a32b1bbb20501c48f77f49181b#a4dd70cb70a116a32b1bbb20501c48f77f49181b" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index dee30842..135dc232 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -36,9 +36,10 @@ probe = "0.5" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "a4dd70cb70a116a32b1bbb20501c48f77f49181b" } +#mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "a4dd70cb70a116a32b1bbb20501c48f77f49181b" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } +mmtk = { path = "../../../../../mmtk-core" } [build-dependencies] built = { version = "0.7.7", features = ["git2"] } diff --git a/openjdk/barriers/mmtkObjectBarrier.cpp b/openjdk/barriers/mmtkObjectBarrier.cpp index 5f14723c..2265794e 100644 --- a/openjdk/barriers/mmtkObjectBarrier.cpp +++ b/openjdk/barriers/mmtkObjectBarrier.cpp @@ -6,11 +6,7 @@ void MMTkObjectBarrierSetRuntime::object_probable_write(oop new_obj) const { if (mmtk_enable_barrier_fastpath) { // Do fast-path check before entering mmtk rust code, to improve mutator performance. // This is identical to calling `mmtk_object_probable_write` directly without a fast-path. - intptr_t addr = (intptr_t) (void*) new_obj; - uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); - intptr_t shift = (addr >> 3) & 0b111; - uint8_t byte_val = *meta_addr; - if (((byte_val >> shift) & 1) == 1) { + if (is_unlog_bit_set(new_obj)) { // Only promoted objects will reach here. // The duplicated unlog bit check inside slow-path still remains correct. mmtk_object_probable_write((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) new_obj); @@ -23,11 +19,7 @@ void MMTkObjectBarrierSetRuntime::object_probable_write(oop new_obj) const { void MMTkObjectBarrierSetRuntime::object_reference_write_post(oop src, oop* slot, oop target) const { if (mmtk_enable_barrier_fastpath) { - intptr_t addr = (intptr_t) (void*) src; - uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); - intptr_t shift = (addr >> 3) & 0b111; - uint8_t byte_val = *meta_addr; - if (((byte_val >> shift) & 1) == 1) { + if (is_unlog_bit_set(src)) { // MMTkObjectBarrierSetRuntime::object_reference_write_pre_slow()((void*) src); object_reference_write_slow_call((void*) src, (void*) slot, (void*) target); } diff --git a/openjdk/barriers/mmtkObjectBarrier.hpp b/openjdk/barriers/mmtkObjectBarrier.hpp index c04081d3..1f27984f 100644 --- a/openjdk/barriers/mmtkObjectBarrier.hpp +++ b/openjdk/barriers/mmtkObjectBarrier.hpp @@ -6,6 +6,7 @@ #include "../mmtkBarrierSetAssembler_x86.hpp" #include "../mmtkBarrierSetC1.hpp" #include "../mmtkBarrierSetC2.hpp" +#include "mmtkUnlogBitBarrier.hpp" #include "c1/c1_LIRAssembler.hpp" #include "c1/c1_MacroAssembler.hpp" #include "gc/shared/barrierSet.hpp" @@ -18,7 +19,7 @@ const intptr_t SIDE_METADATA_BASE_ADDRESS = (intptr_t) GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS; -class MMTkObjectBarrierSetRuntime: public MMTkBarrierSetRuntime { +class MMTkObjectBarrierSetRuntime: public MMTkUnlogBitBarrierSetRuntime { public: // Interfaces called by `MMTkBarrierSet::AccessBarrier` virtual void object_reference_write_post(oop src, oop* slot, oop target) const override; diff --git a/openjdk/barriers/mmtkUnlogBitBarrier.cpp b/openjdk/barriers/mmtkUnlogBitBarrier.cpp new file mode 100644 index 00000000..57858f8d --- /dev/null +++ b/openjdk/barriers/mmtkUnlogBitBarrier.cpp @@ -0,0 +1,5 @@ +#include "precompiled.hpp" +#include "mmtkUnlogBitBarrier.hpp" +#include "runtime/interfaceSupport.inline.hpp" + +#undef __ diff --git a/openjdk/barriers/mmtkUnlogBitBarrier.hpp b/openjdk/barriers/mmtkUnlogBitBarrier.hpp new file mode 100644 index 00000000..7ace08e3 --- /dev/null +++ b/openjdk/barriers/mmtkUnlogBitBarrier.hpp @@ -0,0 +1,30 @@ +#ifndef MMTK_OPENJDK_BARRIERS_MMTK_UNLOG_BIT_BARRIER_HPP +#define MMTK_OPENJDK_BARRIERS_MMTK_UNLOG_BIT_BARRIER_HPP + +#include "../mmtkBarrierSet.hpp" +#include "../mmtkBarrierSetAssembler_x86.hpp" +#include "../mmtkBarrierSetC1.hpp" +#include "../mmtkBarrierSetC2.hpp" + +/// This file contains abstract barrier sets for barriers based on the (object-grained) unlog bit. + +const uintptr_t UNLOG_BIT_BASE_ADDRESS = GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS; + +class MMTkUnlogBitBarrierSetRuntime: public MMTkBarrierSetRuntime { +protected: + static bool is_unlog_bit_set(oop obj) { + uintptr_t addr = (uintptr_t) (void*) obj; + uint8_t* meta_addr = (uint8_t*) (UNLOG_BIT_BASE_ADDRESS + (addr >> 6)); + uintptr_t shift = (addr >> 3) & 0b111; + uint8_t byte_val = *meta_addr; + return ((byte_val >> shift) & 1) == 1; + } +}; + +class MMTkUnlogBitBarrierSetAssembler: public MMTkBarrierSetAssembler {}; + +class MMTkUnlogBitBarrierSetC1: public MMTkBarrierSetC1 {}; + +class MMTkUnlogBitBarrierSetC2: public MMTkBarrierSetC2 {}; + +#endif // MMTK_OPENJDK_BARRIERS_MMTK_UNLOG_BIT_BARRIER_HPP From 6cd0fddf7e52930db5f7fcec6feeae85252e97d1 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Sat, 20 Sep 2025 00:31:20 +0800 Subject: [PATCH 02/46] Use one fewer scratch register --- openjdk/barriers/mmtkObjectBarrier.cpp | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/openjdk/barriers/mmtkObjectBarrier.cpp b/openjdk/barriers/mmtkObjectBarrier.cpp index 2265794e..8b0b4b55 100644 --- a/openjdk/barriers/mmtkObjectBarrier.cpp +++ b/openjdk/barriers/mmtkObjectBarrier.cpp @@ -38,10 +38,11 @@ void MMTkObjectBarrierSetAssembler::object_reference_write_post(MacroAssembler* Label done; Register obj = dst.base(); if (mmtk_enable_barrier_fastpath) { + // For some instructions in the template table, such as aastore, + // we observed that dst.base() == tmp1. + // We steal one more scratch register because one tmp2 is not enough. Register tmp3 = rscratch1; - Register tmp4 = rscratch2; assert_different_registers(obj, tmp2, tmp3); - assert_different_registers(tmp4, rcx); // tmp2 = load-byte (SIDE_METADATA_BASE_ADDRESS + (obj >> 6)); __ movptr(tmp3, obj); @@ -53,14 +54,12 @@ void MMTkObjectBarrierSetAssembler::object_reference_write_post(MacroAssembler* __ shrptr(tmp3, 3); __ andptr(tmp3, 7); // tmp2 = tmp2 >> tmp3 - __ movptr(tmp4, rcx); - __ movl(rcx, tmp3); + __ xchgptr(tmp3, rcx); __ shrptr(tmp2); - __ movptr(rcx, tmp4); - // if ((tmp2 & 1) == 1) goto slowpath; - __ andptr(tmp2, 1); - __ cmpptr(tmp2, 1); - __ jcc(Assembler::notEqual, done); + __ xchgptr(tmp3, rcx); + // if ((tmp2 & 1) == 0) goto done; + __ testptr(tmp2, 1); + __ jcc(Assembler::zero, done); } __ movptr(c_rarg0, obj); From 153a459cd751770d7a868172e7c37903259fd0af Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Sat, 20 Sep 2025 00:37:23 +0800 Subject: [PATCH 03/46] No. We can't. We can't use either tmp1 or tmp2. --- openjdk/barriers/mmtkObjectBarrier.cpp | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/openjdk/barriers/mmtkObjectBarrier.cpp b/openjdk/barriers/mmtkObjectBarrier.cpp index 8b0b4b55..96febed8 100644 --- a/openjdk/barriers/mmtkObjectBarrier.cpp +++ b/openjdk/barriers/mmtkObjectBarrier.cpp @@ -39,26 +39,28 @@ void MMTkObjectBarrierSetAssembler::object_reference_write_post(MacroAssembler* Register obj = dst.base(); if (mmtk_enable_barrier_fastpath) { // For some instructions in the template table, such as aastore, - // we observed that dst.base() == tmp1. - // We steal one more scratch register because one tmp2 is not enough. + // we observed that dst.base() == tmp1 && dst.index() == tmp2. + // We can't afford overwriting any of those registers. + // We steal two scratch register to use. Register tmp3 = rscratch1; - assert_different_registers(obj, tmp2, tmp3); + Register tmp4 = rscratch2; + assert_different_registers(dst.base(), dst.index(), val, tmp3, tmp4); - // tmp2 = load-byte (SIDE_METADATA_BASE_ADDRESS + (obj >> 6)); + // tmp4 = load-byte (SIDE_METADATA_BASE_ADDRESS + (obj >> 6)); __ movptr(tmp3, obj); __ shrptr(tmp3, 6); - __ movptr(tmp2, SIDE_METADATA_BASE_ADDRESS); - __ movb(tmp2, Address(tmp2, tmp3)); + __ movptr(tmp4, SIDE_METADATA_BASE_ADDRESS); + __ movb(tmp4, Address(tmp4, tmp3)); // tmp3 = (obj >> 3) & 7 __ movptr(tmp3, obj); __ shrptr(tmp3, 3); __ andptr(tmp3, 7); - // tmp2 = tmp2 >> tmp3 + // tmp4 = tmp4 >> tmp3 __ xchgptr(tmp3, rcx); - __ shrptr(tmp2); + __ shrptr(tmp4); __ xchgptr(tmp3, rcx); - // if ((tmp2 & 1) == 0) goto done; - __ testptr(tmp2, 1); + // if ((tmp4 & 1) == 0) goto done; + __ testptr(tmp4, 1); __ jcc(Assembler::zero, done); } From b826a0759579b677fd7134ac8285a8072b5e2223 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Sat, 20 Sep 2025 01:16:24 +0800 Subject: [PATCH 04/46] Extract assembler fast path --- openjdk/barriers/mmtkObjectBarrier.cpp | 20 +------------------- openjdk/barriers/mmtkObjectBarrier.hpp | 2 +- openjdk/barriers/mmtkUnlogBitBarrier.cpp | 23 +++++++++++++++++++++++ openjdk/barriers/mmtkUnlogBitBarrier.hpp | 5 ++++- 4 files changed, 29 insertions(+), 21 deletions(-) diff --git a/openjdk/barriers/mmtkObjectBarrier.cpp b/openjdk/barriers/mmtkObjectBarrier.cpp index 96febed8..91ac0e9d 100644 --- a/openjdk/barriers/mmtkObjectBarrier.cpp +++ b/openjdk/barriers/mmtkObjectBarrier.cpp @@ -20,7 +20,6 @@ void MMTkObjectBarrierSetRuntime::object_probable_write(oop new_obj) const { void MMTkObjectBarrierSetRuntime::object_reference_write_post(oop src, oop* slot, oop target) const { if (mmtk_enable_barrier_fastpath) { if (is_unlog_bit_set(src)) { - // MMTkObjectBarrierSetRuntime::object_reference_write_pre_slow()((void*) src); object_reference_write_slow_call((void*) src, (void*) slot, (void*) target); } } else { @@ -33,8 +32,6 @@ void MMTkObjectBarrierSetRuntime::object_reference_write_post(oop src, oop* slot void MMTkObjectBarrierSetAssembler::object_reference_write_post(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2, bool compensate_val_reg) const { if (can_remove_barrier(decorators, val, /* skip_const_null */ true)) return; - bool is_not_null = (decorators & IS_NOT_NULL) != 0; - Label done; Register obj = dst.base(); if (mmtk_enable_barrier_fastpath) { @@ -46,22 +43,7 @@ void MMTkObjectBarrierSetAssembler::object_reference_write_post(MacroAssembler* Register tmp4 = rscratch2; assert_different_registers(dst.base(), dst.index(), val, tmp3, tmp4); - // tmp4 = load-byte (SIDE_METADATA_BASE_ADDRESS + (obj >> 6)); - __ movptr(tmp3, obj); - __ shrptr(tmp3, 6); - __ movptr(tmp4, SIDE_METADATA_BASE_ADDRESS); - __ movb(tmp4, Address(tmp4, tmp3)); - // tmp3 = (obj >> 3) & 7 - __ movptr(tmp3, obj); - __ shrptr(tmp3, 3); - __ andptr(tmp3, 7); - // tmp4 = tmp4 >> tmp3 - __ xchgptr(tmp3, rcx); - __ shrptr(tmp4); - __ xchgptr(tmp3, rcx); - // if ((tmp4 & 1) == 0) goto done; - __ testptr(tmp4, 1); - __ jcc(Assembler::zero, done); + emit_check_unlog_bit_fast_path(masm, done, obj, tmp3, tmp4); } __ movptr(c_rarg0, obj); diff --git a/openjdk/barriers/mmtkObjectBarrier.hpp b/openjdk/barriers/mmtkObjectBarrier.hpp index 1f27984f..d58ce62d 100644 --- a/openjdk/barriers/mmtkObjectBarrier.hpp +++ b/openjdk/barriers/mmtkObjectBarrier.hpp @@ -29,7 +29,7 @@ class MMTkObjectBarrierSetRuntime: public MMTkUnlogBitBarrierSetRuntime { virtual void object_probable_write(oop new_obj) const override; }; -class MMTkObjectBarrierSetAssembler: public MMTkBarrierSetAssembler { +class MMTkObjectBarrierSetAssembler: public MMTkUnlogBitBarrierSetAssembler { protected: virtual void object_reference_write_post(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2, bool compensate_val_reg) const override; /// Generate C1 write barrier slow-call assembly code diff --git a/openjdk/barriers/mmtkUnlogBitBarrier.cpp b/openjdk/barriers/mmtkUnlogBitBarrier.cpp index 57858f8d..6a7a1661 100644 --- a/openjdk/barriers/mmtkUnlogBitBarrier.cpp +++ b/openjdk/barriers/mmtkUnlogBitBarrier.cpp @@ -2,4 +2,27 @@ #include "mmtkUnlogBitBarrier.hpp" #include "runtime/interfaceSupport.inline.hpp" +#define __ masm-> + +void MMTkUnlogBitBarrierSetAssembler::emit_check_unlog_bit_fast_path(MacroAssembler* masm, Label &done, Register obj, Register tmp1, Register tmp2) { + assert_different_registers(obj, tmp1, tmp2); + + // tmp2 = load-byte (UNLOG_BIT_BASE_ADDRESS + (obj >> 6)); + __ movptr(tmp1, obj); + __ shrptr(tmp1, 6); + __ movptr(tmp2, (intptr_t)UNLOG_BIT_BASE_ADDRESS); + __ movb(tmp2, Address(tmp2, tmp1)); + // tmp1 = (obj >> 3) & 7 + __ movptr(tmp1, obj); + __ shrptr(tmp1, 3); + __ andptr(tmp1, 7); + // tmp2 = tmp2 >> tmp1 + __ xchgptr(tmp1, rcx); + __ shrptr(tmp2); + __ xchgptr(tmp1, rcx); + // if ((tmp2 & 1) == 0) goto done; + __ testptr(tmp2, 1); + __ jcc(Assembler::zero, done); +} + #undef __ diff --git a/openjdk/barriers/mmtkUnlogBitBarrier.hpp b/openjdk/barriers/mmtkUnlogBitBarrier.hpp index 7ace08e3..b10b319c 100644 --- a/openjdk/barriers/mmtkUnlogBitBarrier.hpp +++ b/openjdk/barriers/mmtkUnlogBitBarrier.hpp @@ -21,7 +21,10 @@ class MMTkUnlogBitBarrierSetRuntime: public MMTkBarrierSetRuntime { } }; -class MMTkUnlogBitBarrierSetAssembler: public MMTkBarrierSetAssembler {}; +class MMTkUnlogBitBarrierSetAssembler: public MMTkBarrierSetAssembler { +protected: + static void emit_check_unlog_bit_fast_path(MacroAssembler* masm, Label &done, Register obj, Register tmp1, Register tmp2); +}; class MMTkUnlogBitBarrierSetC1: public MMTkBarrierSetC1 {}; From 43f1ac5b80ab571e68bf773c7e5caf71f547780a Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Sat, 20 Sep 2025 01:24:34 +0800 Subject: [PATCH 05/46] Apply unlog bit runtime barrier fast path to SATB --- openjdk/barriers/mmtkSATBBarrier.cpp | 8 +------- openjdk/barriers/mmtkSATBBarrier.hpp | 3 ++- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/openjdk/barriers/mmtkSATBBarrier.cpp b/openjdk/barriers/mmtkSATBBarrier.cpp index 3a75ccc5..4b5ccd34 100644 --- a/openjdk/barriers/mmtkSATBBarrier.cpp +++ b/openjdk/barriers/mmtkSATBBarrier.cpp @@ -29,13 +29,7 @@ void MMTkSATBBarrierSetRuntime::object_probable_write(oop new_obj) const { void MMTkSATBBarrierSetRuntime::object_reference_write_pre(oop src, oop* slot, oop target) const { if (mmtk_enable_barrier_fastpath) { - // oop pre_val = *slot; - // if (pre_val == NULL) return; - intptr_t addr = ((intptr_t) (void*) src); - const volatile uint8_t * meta_addr = (const volatile uint8_t *) (side_metadata_base_address() + (addr >> 6)); - intptr_t shift = (addr >> 3) & 0b111; - uint8_t byte_val = *meta_addr; - if (((byte_val >> shift) & 1) == kUnloggedValue) { + if (is_unlog_bit_set(src)) { object_reference_write_slow_call((void*) src, (void*) slot, (void*) target); } } else { diff --git a/openjdk/barriers/mmtkSATBBarrier.hpp b/openjdk/barriers/mmtkSATBBarrier.hpp index 1f5cf639..5b9a1ff1 100644 --- a/openjdk/barriers/mmtkSATBBarrier.hpp +++ b/openjdk/barriers/mmtkSATBBarrier.hpp @@ -6,6 +6,7 @@ #include "../mmtkBarrierSetAssembler_x86.hpp" #include "../mmtkBarrierSetC1.hpp" #include "../mmtkBarrierSetC2.hpp" +#include "mmtkUnlogBitBarrier.hpp" #include "c1/c1_LIRAssembler.hpp" #include "c1/c1_MacroAssembler.hpp" #include "gc/shared/barrierSet.hpp" @@ -18,7 +19,7 @@ const intptr_t SATB_METADATA_BASE_ADDRESS = (intptr_t) GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS; -class MMTkSATBBarrierSetRuntime: public MMTkBarrierSetRuntime { +class MMTkSATBBarrierSetRuntime: public MMTkUnlogBitBarrierSetRuntime { public: // Interfaces called by `MMTkBarrierSet::AccessBarrier` virtual void object_reference_write_pre(oop src, oop* slot, oop target) const override; From 3f3c1937893806db0d8009b66035d550912c4ba4 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Sat, 20 Sep 2025 01:31:57 +0800 Subject: [PATCH 06/46] Apply unlog bit assembler barrier fast path to SATB --- openjdk/barriers/mmtkSATBBarrier.cpp | 65 +++++++++++----------------- openjdk/barriers/mmtkSATBBarrier.hpp | 2 +- 2 files changed, 26 insertions(+), 41 deletions(-) diff --git a/openjdk/barriers/mmtkSATBBarrier.cpp b/openjdk/barriers/mmtkSATBBarrier.cpp index 4b5ccd34..e6fa225a 100644 --- a/openjdk/barriers/mmtkSATBBarrier.cpp +++ b/openjdk/barriers/mmtkSATBBarrier.cpp @@ -70,55 +70,40 @@ void MMTkSATBBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet dec } void MMTkSATBBarrierSetAssembler::object_reference_write_pre(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2) const { - if (can_remove_barrier(decorators, val, /* skip_const_null */ false)) return; + if (can_remove_barrier(decorators, val, /* skip_const_null */ true)) return; + Label done; + Register obj = dst.base(); if (mmtk_enable_barrier_fastpath) { - Label done; - - Register obj = dst.base(); + // For some instructions in the template table, such as aastore, + // we observed that dst.base() == tmp1 && dst.index() == tmp2. + // We can't afford overwriting any of those registers. + // We steal two scratch register to use. Register tmp3 = rscratch1; Register tmp4 = rscratch2; - Register tmp5 = tmp1 == dst.base() || tmp1 == dst.index() ? tmp2 : tmp1; - - // tmp5 = load-byte (side_metadata_base_address() + (obj >> 6)); - __ movptr(tmp3, obj); - // __ load_heap_oop(tmp3, dst, noreg, noreg, AS_RAW); - // // Is the previous value null? - // __ cmpptr(tmp3, (int32_t) NULL_WORD); - // __ jcc(Assembler::equal, done); - - __ shrptr(tmp3, 6); - __ movptr(tmp5, side_metadata_base_address()); - __ movzbl(tmp5, Address(tmp5, tmp3)); - - // tmp3 = (obj >> 3) & 7 - __ mov(tmp3, obj); - __ shrptr(tmp3, 3); - __ andptr(tmp3, 7); - // tmp5 = tmp5 >> tmp3 - __ movptr(tmp4, rcx); - __ movl(rcx, tmp3); - __ shrptr(tmp5); - __ movptr(rcx, tmp4); - // if ((tmp5 & 1) == 1) goto slowpath; - __ andptr(tmp5, 1); - __ cmpptr(tmp5, kUnloggedValue); - __ jcc(Assembler::notEqual, done); + assert_different_registers(dst.base(), dst.index(), val, tmp3, tmp4); - // TODO: Spill fewer registers - __ pusha(); - __ movptr(c_rarg0, dst.base()); - __ lea(c_rarg1, dst); - __ movptr(c_rarg2, val == noreg ? (int32_t) NULL_WORD : val); + emit_check_unlog_bit_fast_path(masm, done, obj, tmp3, tmp4); + } + + // This is a pre-barrier. Preserve caller-saved regs for the actual write operation. + __ pusha(); + + __ movptr(c_rarg0, obj); + __ xorptr(c_rarg1, c_rarg1); + // Note: If `compensate_val_reg == true && UseCompressedOops === true`, the `val` register will be + // holding a compressed pointer to the target object. If the write barrier needs to know the + // target, we will need to decompress it before passing it to the barrier slow path. However, + // since we know the semantics of `mmtk::plan::barriers::ObjectBarrier`, i.e. it logs the object + // without looking at the `slot` or the `target` parameter at all, we simply pass nullptr to both + // parameters. + __ xorptr(c_rarg2, c_rarg2); + + if (mmtk_enable_barrier_fastpath) { __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), 3); __ popa(); - __ bind(done); } else { - __ pusha(); - __ movptr(c_rarg0, dst.base()); - __ lea(c_rarg1, dst); - __ movptr(c_rarg2, val == noreg ? (int32_t) NULL_WORD : val); __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call), 3); __ popa(); } diff --git a/openjdk/barriers/mmtkSATBBarrier.hpp b/openjdk/barriers/mmtkSATBBarrier.hpp index 5b9a1ff1..62c51cc4 100644 --- a/openjdk/barriers/mmtkSATBBarrier.hpp +++ b/openjdk/barriers/mmtkSATBBarrier.hpp @@ -31,7 +31,7 @@ class MMTkSATBBarrierSetRuntime: public MMTkUnlogBitBarrierSetRuntime { virtual void load_reference(DecoratorSet decorators, oop value) const override; }; -class MMTkSATBBarrierSetAssembler: public MMTkBarrierSetAssembler { +class MMTkSATBBarrierSetAssembler: public MMTkUnlogBitBarrierSetAssembler { protected: virtual void object_reference_write_pre(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2) const override; /// Generate C1 write barrier slow-call assembly code From 9be6ae1df1bf9f9930a9dbef77a39aa1feb390e2 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Sat, 20 Sep 2025 01:56:29 +0800 Subject: [PATCH 07/46] Extract common part of pre and post from assembler --- openjdk/barriers/mmtkObjectBarrier.cpp | 32 +--------------- openjdk/barriers/mmtkSATBBarrier.cpp | 37 +----------------- openjdk/barriers/mmtkUnlogBitBarrier.cpp | 49 ++++++++++++++++++++++++ openjdk/barriers/mmtkUnlogBitBarrier.hpp | 1 + 4 files changed, 52 insertions(+), 67 deletions(-) diff --git a/openjdk/barriers/mmtkObjectBarrier.cpp b/openjdk/barriers/mmtkObjectBarrier.cpp index 91ac0e9d..cfbe702e 100644 --- a/openjdk/barriers/mmtkObjectBarrier.cpp +++ b/openjdk/barriers/mmtkObjectBarrier.cpp @@ -31,37 +31,7 @@ void MMTkObjectBarrierSetRuntime::object_reference_write_post(oop src, oop* slot void MMTkObjectBarrierSetAssembler::object_reference_write_post(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2, bool compensate_val_reg) const { if (can_remove_barrier(decorators, val, /* skip_const_null */ true)) return; - - Label done; - Register obj = dst.base(); - if (mmtk_enable_barrier_fastpath) { - // For some instructions in the template table, such as aastore, - // we observed that dst.base() == tmp1 && dst.index() == tmp2. - // We can't afford overwriting any of those registers. - // We steal two scratch register to use. - Register tmp3 = rscratch1; - Register tmp4 = rscratch2; - assert_different_registers(dst.base(), dst.index(), val, tmp3, tmp4); - - emit_check_unlog_bit_fast_path(masm, done, obj, tmp3, tmp4); - } - - __ movptr(c_rarg0, obj); - __ xorptr(c_rarg1, c_rarg1); - // Note: If `compensate_val_reg == true && UseCompressedOops === true`, the `val` register will be - // holding a compressed pointer to the target object. If the write barrier needs to know the - // target, we will need to decompress it before passing it to the barrier slow path. However, - // since we know the semantics of `mmtk::plan::barriers::ObjectBarrier`, i.e. it logs the object - // without looking at the `slot` or the `target` parameter at all, we simply pass nullptr to both - // parameters. - __ xorptr(c_rarg2, c_rarg2); - - if (mmtk_enable_barrier_fastpath) { - __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), 3); - __ bind(done); - } else { - __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call), 3); - } + object_reference_write_pre_or_post(masm, decorators, dst, val, /* pre = */ false); } void MMTkObjectBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) { diff --git a/openjdk/barriers/mmtkSATBBarrier.cpp b/openjdk/barriers/mmtkSATBBarrier.cpp index e6fa225a..9602fc5c 100644 --- a/openjdk/barriers/mmtkSATBBarrier.cpp +++ b/openjdk/barriers/mmtkSATBBarrier.cpp @@ -71,42 +71,7 @@ void MMTkSATBBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet dec void MMTkSATBBarrierSetAssembler::object_reference_write_pre(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2) const { if (can_remove_barrier(decorators, val, /* skip_const_null */ true)) return; - - Label done; - Register obj = dst.base(); - if (mmtk_enable_barrier_fastpath) { - // For some instructions in the template table, such as aastore, - // we observed that dst.base() == tmp1 && dst.index() == tmp2. - // We can't afford overwriting any of those registers. - // We steal two scratch register to use. - Register tmp3 = rscratch1; - Register tmp4 = rscratch2; - assert_different_registers(dst.base(), dst.index(), val, tmp3, tmp4); - - emit_check_unlog_bit_fast_path(masm, done, obj, tmp3, tmp4); - } - - // This is a pre-barrier. Preserve caller-saved regs for the actual write operation. - __ pusha(); - - __ movptr(c_rarg0, obj); - __ xorptr(c_rarg1, c_rarg1); - // Note: If `compensate_val_reg == true && UseCompressedOops === true`, the `val` register will be - // holding a compressed pointer to the target object. If the write barrier needs to know the - // target, we will need to decompress it before passing it to the barrier slow path. However, - // since we know the semantics of `mmtk::plan::barriers::ObjectBarrier`, i.e. it logs the object - // without looking at the `slot` or the `target` parameter at all, we simply pass nullptr to both - // parameters. - __ xorptr(c_rarg2, c_rarg2); - - if (mmtk_enable_barrier_fastpath) { - __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), 3); - __ popa(); - __ bind(done); - } else { - __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call), 3); - __ popa(); - } + object_reference_write_pre_or_post(masm, decorators, dst, val, /* pre = */ true); } void MMTkSATBBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) { diff --git a/openjdk/barriers/mmtkUnlogBitBarrier.cpp b/openjdk/barriers/mmtkUnlogBitBarrier.cpp index 6a7a1661..28ca54c4 100644 --- a/openjdk/barriers/mmtkUnlogBitBarrier.cpp +++ b/openjdk/barriers/mmtkUnlogBitBarrier.cpp @@ -25,4 +25,53 @@ void MMTkUnlogBitBarrierSetAssembler::emit_check_unlog_bit_fast_path(MacroAssemb __ jcc(Assembler::zero, done); } +void MMTkUnlogBitBarrierSetAssembler::object_reference_write_pre_or_post(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, bool pre) { + Label done; + Register obj = dst.base(); + if (mmtk_enable_barrier_fastpath) { + // For some instructions in the template table, such as aastore, + // we observed that in BarrierSetAssembler::store_at + // which calls `object_reference_write_pre` or `object_reference_write_post`, + // dst.base() == tmp1 && dst.index() == tmp2. + // We can't overwrite those registers, + // so we don't use tmp1 or tmp2 passed to store_at. + // Instead, we steal two scratch register to use. + Register tmp3 = rscratch1; + Register tmp4 = rscratch2; + assert_different_registers(dst.base(), dst.index(), val, tmp3, tmp4); + + emit_check_unlog_bit_fast_path(masm, done, obj, tmp3, tmp4); + } + + if (pre) { + // This is a pre-barrier. Preserve caller-saved regs for the actual write operation. + __ pusha(); + } + + __ movptr(c_rarg0, obj); + // Neither the ObjectBarrier nor the SATBBarrier need to know the slot or the value. + // We just set both args to nullptr. + // We may need to pass actual arguments if we support other barriers. + // + // Note: If the `compensate_val_reg` parameter in the post barrier is true, and we are using + // compressed oops, the `val` register will be holding a compressed pointer to the target object + // due to the way `BarrierSetAssembler::store_at` works. If the write barrier needs to know the + // target, we will need to decompress it before passing it to the barrier slow path. + __ xorptr(c_rarg1, c_rarg1); + __ xorptr(c_rarg2, c_rarg2); + + address entry_point = mmtk_enable_barrier_fastpath ? FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call) + : pre ? FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call) + : FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call); + + __ call_VM_leaf_base(entry_point, 3); + + if (pre) { + __ popa(); + } + + if (mmtk_enable_barrier_fastpath) { + __ bind(done); + } +} #undef __ diff --git a/openjdk/barriers/mmtkUnlogBitBarrier.hpp b/openjdk/barriers/mmtkUnlogBitBarrier.hpp index b10b319c..3bea0fc3 100644 --- a/openjdk/barriers/mmtkUnlogBitBarrier.hpp +++ b/openjdk/barriers/mmtkUnlogBitBarrier.hpp @@ -24,6 +24,7 @@ class MMTkUnlogBitBarrierSetRuntime: public MMTkBarrierSetRuntime { class MMTkUnlogBitBarrierSetAssembler: public MMTkBarrierSetAssembler { protected: static void emit_check_unlog_bit_fast_path(MacroAssembler* masm, Label &done, Register obj, Register tmp1, Register tmp2); + static void object_reference_write_pre_or_post(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, bool pre); }; class MMTkUnlogBitBarrierSetC1: public MMTkBarrierSetC1 {}; From efc92a6ac6d146af06562757ddec776c55935e04 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Sat, 20 Sep 2025 10:31:13 +0800 Subject: [PATCH 08/46] Define __ separately for each function impl --- openjdk/barriers/mmtkUnlogBitBarrier.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/openjdk/barriers/mmtkUnlogBitBarrier.cpp b/openjdk/barriers/mmtkUnlogBitBarrier.cpp index 28ca54c4..62441d5a 100644 --- a/openjdk/barriers/mmtkUnlogBitBarrier.cpp +++ b/openjdk/barriers/mmtkUnlogBitBarrier.cpp @@ -25,6 +25,10 @@ void MMTkUnlogBitBarrierSetAssembler::emit_check_unlog_bit_fast_path(MacroAssemb __ jcc(Assembler::zero, done); } +#undef __ + +#define __ masm-> + void MMTkUnlogBitBarrierSetAssembler::object_reference_write_pre_or_post(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, bool pre) { Label done; Register obj = dst.base(); @@ -74,4 +78,5 @@ void MMTkUnlogBitBarrierSetAssembler::object_reference_write_pre_or_post(MacroAs __ bind(done); } } + #undef __ From 5d49e81cf30f5d18afff23604efea5490e7b2de5 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Sat, 20 Sep 2025 11:43:31 +0800 Subject: [PATCH 09/46] Extract c1 pre/post write barrier runtime stub --- openjdk/barriers/mmtkObjectBarrier.cpp | 33 +-------------------- openjdk/barriers/mmtkSATBBarrier.cpp | 33 +-------------------- openjdk/barriers/mmtkUnlogBitBarrier.cpp | 37 ++++++++++++++++++++++++ openjdk/barriers/mmtkUnlogBitBarrier.hpp | 1 + 4 files changed, 40 insertions(+), 64 deletions(-) diff --git a/openjdk/barriers/mmtkObjectBarrier.cpp b/openjdk/barriers/mmtkObjectBarrier.cpp index cfbe702e..37f8f2dd 100644 --- a/openjdk/barriers/mmtkObjectBarrier.cpp +++ b/openjdk/barriers/mmtkObjectBarrier.cpp @@ -83,38 +83,7 @@ void MMTkObjectBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, Dec #define __ sasm-> void MMTkObjectBarrierSetAssembler::generate_c1_post_write_barrier_runtime_stub(StubAssembler* sasm) const { - __ prologue("mmtk_object_barrier", false); - - Label done, runtime; - - __ push(c_rarg0); - __ push(c_rarg1); - __ push(c_rarg2); - __ push(rax); - - __ load_parameter(0, c_rarg0); - __ load_parameter(1, c_rarg1); - __ load_parameter(2, c_rarg2); - - __ bind(runtime); - - __ save_live_registers_no_oop_map(true); - - if (mmtk_enable_barrier_fastpath) { - __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), 3); - } else { - __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call), 3); - } - - __ restore_live_registers(true); - - __ bind(done); - __ pop(rax); - __ pop(c_rarg2); - __ pop(c_rarg1); - __ pop(c_rarg0); - - __ epilogue(); + generate_c1_pre_or_post_write_barrier_runtime_stub(sasm, "mmtk_object_barrier", /* pre = */ false); } #undef __ diff --git a/openjdk/barriers/mmtkSATBBarrier.cpp b/openjdk/barriers/mmtkSATBBarrier.cpp index 9602fc5c..b9cfefbb 100644 --- a/openjdk/barriers/mmtkSATBBarrier.cpp +++ b/openjdk/barriers/mmtkSATBBarrier.cpp @@ -99,38 +99,7 @@ void MMTkSATBBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Decor #define __ sasm-> void MMTkSATBBarrierSetAssembler::generate_c1_pre_write_barrier_runtime_stub(StubAssembler* sasm) const { - __ prologue("mmtk_satb_barrier", false); - - Label done, runtime; - - __ push(c_rarg0); - __ push(c_rarg1); - __ push(c_rarg2); - __ push(rax); - - __ load_parameter(0, c_rarg0); - __ load_parameter(1, c_rarg1); - __ load_parameter(2, c_rarg2); - - __ bind(runtime); - - __ save_live_registers_no_oop_map(true); - - if (mmtk_enable_barrier_fastpath) { - __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), 3); - } else { - __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call), 3); - } - - __ restore_live_registers(true); - - __ bind(done); - __ pop(rax); - __ pop(c_rarg2); - __ pop(c_rarg1); - __ pop(c_rarg0); - - __ epilogue(); + generate_c1_pre_or_post_write_barrier_runtime_stub(sasm, "mmtk_satb_barrier", /* pre = */ true); } #undef __ diff --git a/openjdk/barriers/mmtkUnlogBitBarrier.cpp b/openjdk/barriers/mmtkUnlogBitBarrier.cpp index 62441d5a..0bf12e7f 100644 --- a/openjdk/barriers/mmtkUnlogBitBarrier.cpp +++ b/openjdk/barriers/mmtkUnlogBitBarrier.cpp @@ -80,3 +80,40 @@ void MMTkUnlogBitBarrierSetAssembler::object_reference_write_pre_or_post(MacroAs } #undef __ + +#define __ sasm-> + +void MMTkUnlogBitBarrierSetAssembler::generate_c1_pre_or_post_write_barrier_runtime_stub(StubAssembler* sasm, const char* name, bool pre) { + __ prologue(name, false); + + Label done, runtime; + + __ push(c_rarg0); + __ push(c_rarg1); + __ push(c_rarg2); + __ push(rax); + + __ load_parameter(0, c_rarg0); + __ load_parameter(1, c_rarg1); + __ load_parameter(2, c_rarg2); + + __ bind(runtime); + + __ save_live_registers_no_oop_map(true); + + address entry_point = mmtk_enable_barrier_fastpath ? FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call) + : pre ? FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call) + : FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call); + + __ call_VM_leaf_base(entry_point, 3); + + __ restore_live_registers(true); + + __ bind(done); + __ pop(rax); + __ pop(c_rarg2); + __ pop(c_rarg1); + __ pop(c_rarg0); + + __ epilogue(); +} diff --git a/openjdk/barriers/mmtkUnlogBitBarrier.hpp b/openjdk/barriers/mmtkUnlogBitBarrier.hpp index 3bea0fc3..7148a4e3 100644 --- a/openjdk/barriers/mmtkUnlogBitBarrier.hpp +++ b/openjdk/barriers/mmtkUnlogBitBarrier.hpp @@ -25,6 +25,7 @@ class MMTkUnlogBitBarrierSetAssembler: public MMTkBarrierSetAssembler { protected: static void emit_check_unlog_bit_fast_path(MacroAssembler* masm, Label &done, Register obj, Register tmp1, Register tmp2); static void object_reference_write_pre_or_post(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, bool pre); + static void generate_c1_pre_or_post_write_barrier_runtime_stub(StubAssembler* sasm, const char* name, bool pre); }; class MMTkUnlogBitBarrierSetC1: public MMTkBarrierSetC1 {}; From 0b0915fc1e3de9072186282c5627ea007938d269 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Sat, 20 Sep 2025 12:36:48 +0800 Subject: [PATCH 10/46] Hoist C1 runtime stubs up to BarrierSetAssembler They are not specific to concrete barriers, although each barrier only uses some but not all of them. --- openjdk/barriers/mmtkObjectBarrier.cpp | 7 --- openjdk/barriers/mmtkObjectBarrier.hpp | 2 - openjdk/barriers/mmtkSATBBarrier.cpp | 7 --- openjdk/barriers/mmtkSATBBarrier.hpp | 2 - openjdk/barriers/mmtkUnlogBitBarrier.cpp | 37 ------------ openjdk/barriers/mmtkUnlogBitBarrier.hpp | 1 - openjdk/mmtkBarrierSetAssembler_x86.cpp | 71 ++++++++++++++++++------ openjdk/mmtkBarrierSetAssembler_x86.hpp | 12 ++-- openjdk/mmtkBarrierSetC1.cpp | 21 +++---- 9 files changed, 69 insertions(+), 91 deletions(-) diff --git a/openjdk/barriers/mmtkObjectBarrier.cpp b/openjdk/barriers/mmtkObjectBarrier.cpp index 37f8f2dd..24b88769 100644 --- a/openjdk/barriers/mmtkObjectBarrier.cpp +++ b/openjdk/barriers/mmtkObjectBarrier.cpp @@ -80,13 +80,6 @@ void MMTkObjectBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, Dec #undef __ -#define __ sasm-> - -void MMTkObjectBarrierSetAssembler::generate_c1_post_write_barrier_runtime_stub(StubAssembler* sasm) const { - generate_c1_pre_or_post_write_barrier_runtime_stub(sasm, "mmtk_object_barrier", /* pre = */ false); -} - -#undef __ #define __ ce->masm()-> void MMTkObjectBarrierSetAssembler::generate_c1_post_write_barrier_stub(LIR_Assembler* ce, MMTkC1PostBarrierStub* stub) const { diff --git a/openjdk/barriers/mmtkObjectBarrier.hpp b/openjdk/barriers/mmtkObjectBarrier.hpp index d58ce62d..db8fb45c 100644 --- a/openjdk/barriers/mmtkObjectBarrier.hpp +++ b/openjdk/barriers/mmtkObjectBarrier.hpp @@ -32,8 +32,6 @@ class MMTkObjectBarrierSetRuntime: public MMTkUnlogBitBarrierSetRuntime { class MMTkObjectBarrierSetAssembler: public MMTkUnlogBitBarrierSetAssembler { protected: virtual void object_reference_write_post(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2, bool compensate_val_reg) const override; - /// Generate C1 write barrier slow-call assembly code - virtual void generate_c1_post_write_barrier_runtime_stub(StubAssembler* sasm) const override; public: virtual void generate_c1_post_write_barrier_stub(LIR_Assembler* ce, MMTkC1PostBarrierStub* stub) const override; virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) override; diff --git a/openjdk/barriers/mmtkSATBBarrier.cpp b/openjdk/barriers/mmtkSATBBarrier.cpp index b9cfefbb..044f9870 100644 --- a/openjdk/barriers/mmtkSATBBarrier.cpp +++ b/openjdk/barriers/mmtkSATBBarrier.cpp @@ -96,13 +96,6 @@ void MMTkSATBBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Decor #undef __ -#define __ sasm-> - -void MMTkSATBBarrierSetAssembler::generate_c1_pre_write_barrier_runtime_stub(StubAssembler* sasm) const { - generate_c1_pre_or_post_write_barrier_runtime_stub(sasm, "mmtk_satb_barrier", /* pre = */ true); -} - -#undef __ #define __ ce->masm()-> void MMTkSATBBarrierSetAssembler::generate_c1_pre_write_barrier_stub(LIR_Assembler* ce, MMTkC1PreBarrierStub* stub) const { diff --git a/openjdk/barriers/mmtkSATBBarrier.hpp b/openjdk/barriers/mmtkSATBBarrier.hpp index 62c51cc4..bc960dbe 100644 --- a/openjdk/barriers/mmtkSATBBarrier.hpp +++ b/openjdk/barriers/mmtkSATBBarrier.hpp @@ -34,8 +34,6 @@ class MMTkSATBBarrierSetRuntime: public MMTkUnlogBitBarrierSetRuntime { class MMTkSATBBarrierSetAssembler: public MMTkUnlogBitBarrierSetAssembler { protected: virtual void object_reference_write_pre(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2) const override; - /// Generate C1 write barrier slow-call assembly code - virtual void generate_c1_pre_write_barrier_runtime_stub(StubAssembler* sasm) const override; public: virtual void generate_c1_pre_write_barrier_stub(LIR_Assembler* ce, MMTkC1PreBarrierStub* stub) const override; virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) override; diff --git a/openjdk/barriers/mmtkUnlogBitBarrier.cpp b/openjdk/barriers/mmtkUnlogBitBarrier.cpp index 0bf12e7f..62441d5a 100644 --- a/openjdk/barriers/mmtkUnlogBitBarrier.cpp +++ b/openjdk/barriers/mmtkUnlogBitBarrier.cpp @@ -80,40 +80,3 @@ void MMTkUnlogBitBarrierSetAssembler::object_reference_write_pre_or_post(MacroAs } #undef __ - -#define __ sasm-> - -void MMTkUnlogBitBarrierSetAssembler::generate_c1_pre_or_post_write_barrier_runtime_stub(StubAssembler* sasm, const char* name, bool pre) { - __ prologue(name, false); - - Label done, runtime; - - __ push(c_rarg0); - __ push(c_rarg1); - __ push(c_rarg2); - __ push(rax); - - __ load_parameter(0, c_rarg0); - __ load_parameter(1, c_rarg1); - __ load_parameter(2, c_rarg2); - - __ bind(runtime); - - __ save_live_registers_no_oop_map(true); - - address entry_point = mmtk_enable_barrier_fastpath ? FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call) - : pre ? FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call) - : FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call); - - __ call_VM_leaf_base(entry_point, 3); - - __ restore_live_registers(true); - - __ bind(done); - __ pop(rax); - __ pop(c_rarg2); - __ pop(c_rarg1); - __ pop(c_rarg0); - - __ epilogue(); -} diff --git a/openjdk/barriers/mmtkUnlogBitBarrier.hpp b/openjdk/barriers/mmtkUnlogBitBarrier.hpp index 7148a4e3..3bea0fc3 100644 --- a/openjdk/barriers/mmtkUnlogBitBarrier.hpp +++ b/openjdk/barriers/mmtkUnlogBitBarrier.hpp @@ -25,7 +25,6 @@ class MMTkUnlogBitBarrierSetAssembler: public MMTkBarrierSetAssembler { protected: static void emit_check_unlog_bit_fast_path(MacroAssembler* masm, Label &done, Register obj, Register tmp1, Register tmp2); static void object_reference_write_pre_or_post(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, bool pre); - static void generate_c1_pre_or_post_write_barrier_runtime_stub(StubAssembler* sasm, const char* name, bool pre); }; class MMTkUnlogBitBarrierSetC1: public MMTkBarrierSetC1 {}; diff --git a/openjdk/mmtkBarrierSetAssembler_x86.cpp b/openjdk/mmtkBarrierSetAssembler_x86.cpp index 112cf97a..f26f0f35 100644 --- a/openjdk/mmtkBarrierSetAssembler_x86.cpp +++ b/openjdk/mmtkBarrierSetAssembler_x86.cpp @@ -142,30 +142,57 @@ void MMTkBarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register threa #undef __ -#define __ sasm-> +#define __ ce->masm()-> -void MMTkBarrierSetAssembler::generate_c1_ref_load_barrier_runtime_stub(StubAssembler* sasm) const { - __ prologue("mmtk_ref_load_barrier", false); +void MMTkBarrierSetAssembler::generate_c1_ref_load_barrier_stub_call(LIR_Assembler* ce, MMTkC1ReferenceLoadBarrierStub* stub) { + MMTkBarrierSetC1* bs = (MMTkBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); - // Address store_addr(rbp, 2*BytesPerWord); + __ bind(*stub->entry()); + assert(stub->val->is_register(), "Precondition."); + + Register val_reg = stub->val->as_register(); + + __ cmpptr(val_reg, (int32_t) NULL_WORD); + __ jcc(Assembler::equal, *stub->continuation()); + ce->store_parameter(stub->val->as_register(), 0); + __ call(RuntimeAddress(bs->_ref_load_barrier_c1_runtime_code_blob->code_begin())); + __ jmp(*stub->continuation()); +} + +#undef __ + +#define __ sasm-> + +void MMTkBarrierSetAssembler::generate_c1_pre_or_post_write_barrier_runtime_stub(StubAssembler* sasm, bool pre) { + __ prologue(pre ? "mmtk_pre_write_barrier" : "mmtk_post_write_barrier", false); Label done, runtime; __ push(c_rarg0); + __ push(c_rarg1); + __ push(c_rarg2); __ push(rax); __ load_parameter(0, c_rarg0); + __ load_parameter(1, c_rarg1); + __ load_parameter(2, c_rarg2); __ bind(runtime); __ save_live_registers_no_oop_map(true); - __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), 1); + address entry_point = mmtk_enable_barrier_fastpath ? FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call) + : pre ? FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call) + : FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call); + + __ call_VM_leaf_base(entry_point, 3); __ restore_live_registers(true); __ bind(done); __ pop(rax); + __ pop(c_rarg2); + __ pop(c_rarg1); __ pop(c_rarg0); __ epilogue(); @@ -173,21 +200,33 @@ void MMTkBarrierSetAssembler::generate_c1_ref_load_barrier_runtime_stub(StubAsse #undef __ -#define __ ce->masm()-> +#define __ sasm-> -void MMTkBarrierSetAssembler::generate_c1_ref_load_barrier_stub_call(LIR_Assembler* ce, MMTkC1ReferenceLoadBarrierStub* stub) { - MMTkBarrierSetC1* bs = (MMTkBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); +void MMTkBarrierSetAssembler::generate_c1_ref_load_barrier_runtime_stub(StubAssembler* sasm) const { + __ prologue("mmtk_ref_load_barrier", false); - __ bind(*stub->entry()); - assert(stub->val->is_register(), "Precondition."); + // Address store_addr(rbp, 2*BytesPerWord); - Register val_reg = stub->val->as_register(); + Label done, runtime; - __ cmpptr(val_reg, (int32_t) NULL_WORD); - __ jcc(Assembler::equal, *stub->continuation()); - ce->store_parameter(stub->val->as_register(), 0); - __ call(RuntimeAddress(bs->_ref_load_barrier_c1_runtime_code_blob->code_begin())); - __ jmp(*stub->continuation()); + __ push(c_rarg0); + __ push(rax); + + __ load_parameter(0, c_rarg0); + + __ bind(runtime); + + __ save_live_registers_no_oop_map(true); + + __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), 1); + + __ restore_live_registers(true); + + __ bind(done); + __ pop(rax); + __ pop(c_rarg0); + + __ epilogue(); } #undef __ diff --git a/openjdk/mmtkBarrierSetAssembler_x86.hpp b/openjdk/mmtkBarrierSetAssembler_x86.hpp index 47bade05..32e43ebd 100644 --- a/openjdk/mmtkBarrierSetAssembler_x86.hpp +++ b/openjdk/mmtkBarrierSetAssembler_x86.hpp @@ -14,6 +14,12 @@ class StubAssembler; class MMTkBarrierSetAssembler: public BarrierSetAssembler { friend class MMTkBarrierSetC1; +private: + /// Generate C1 pre or post write barrier slow-call assembly code + void generate_c1_pre_or_post_write_barrier_runtime_stub(StubAssembler* sasm, bool pre); + /// Generate C1 weak reference load slow-call assembly code + void generate_c1_ref_load_barrier_runtime_stub(StubAssembler* sasm) const; + protected: /// Full pre-barrier virtual void object_reference_write_pre(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2) const {} @@ -29,12 +35,6 @@ class MMTkBarrierSetAssembler: public BarrierSetAssembler { return !in_heap || (skip_const_null && val == noreg); } - /// Generate C1 pre write barrier slow-call assembly code - virtual void generate_c1_pre_write_barrier_runtime_stub(StubAssembler* sasm) const {}; - /// Generate C1 post write barrier slow-call assembly code - virtual void generate_c1_post_write_barrier_runtime_stub(StubAssembler* sasm) const {}; - virtual void generate_c1_ref_load_barrier_runtime_stub(StubAssembler* sasm) const; - public: virtual void eden_allocate(MacroAssembler* masm, Register thread, Register obj, Register var_size_in_bytes, int con_size_in_bytes, Register t1, Label& slow_case) override; virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Address dst, Register val, Register tmp1, Register tmp2) override { diff --git a/openjdk/mmtkBarrierSetC1.cpp b/openjdk/mmtkBarrierSetC1.cpp index 09f6e469..1d9a7b84 100644 --- a/openjdk/mmtkBarrierSetC1.cpp +++ b/openjdk/mmtkBarrierSetC1.cpp @@ -7,30 +7,19 @@ void MMTkBarrierSetC1::generate_c1_runtime_stubs(BufferBlob* buffer_blob) { class MMTkPreBarrierCodeGenClosure : public StubAssemblerCodeGenClosure { virtual OopMapSet* generate_code(StubAssembler* sasm) override { MMTkBarrierSetAssembler* bs = (MMTkBarrierSetAssembler*) BarrierSet::barrier_set()->barrier_set_assembler(); - bs->generate_c1_pre_write_barrier_runtime_stub(sasm); + bs->generate_c1_pre_or_post_write_barrier_runtime_stub(sasm, true); return NULL; } - public: - MMTkPreBarrierCodeGenClosure() {} }; class MMTkPostBarrierCodeGenClosure : public StubAssemblerCodeGenClosure { virtual OopMapSet* generate_code(StubAssembler* sasm) override { MMTkBarrierSetAssembler* bs = (MMTkBarrierSetAssembler*) BarrierSet::barrier_set()->barrier_set_assembler(); - bs->generate_c1_post_write_barrier_runtime_stub(sasm); + bs->generate_c1_pre_or_post_write_barrier_runtime_stub(sasm, false); return NULL; } - public: - MMTkPostBarrierCodeGenClosure() {} }; - MMTkPreBarrierCodeGenClosure pre_write_code_gen_cl; - _pre_barrier_c1_runtime_code_blob = Runtime1::generate_blob(buffer_blob, -1, "mmtk_pre_write_code_gen_cl", false, &pre_write_code_gen_cl); - MMTkPostBarrierCodeGenClosure post_write_code_gen_cl; - _post_barrier_c1_runtime_code_blob = Runtime1::generate_blob(buffer_blob, -1, "mmtk_post_write_code_gen_cl", false, &post_write_code_gen_cl); - // MMTkBarrierCodeGenClosure write_code_gen_cl_patch_fix(true); - // _write_barrier_c1_runtime_code_blob_with_patch_fix = Runtime1::generate_blob(buffer_blob, -1, "write_code_gen_cl_patch_fix", false, &write_code_gen_cl_patch_fix); - class MMTkRefLoadBarrierCodeGenClosure : public StubAssemblerCodeGenClosure { virtual OopMapSet* generate_code(StubAssembler* sasm) override { MMTkBarrierSetAssembler* bs = (MMTkBarrierSetAssembler*) BarrierSet::barrier_set()->barrier_set_assembler(); @@ -39,6 +28,12 @@ void MMTkBarrierSetC1::generate_c1_runtime_stubs(BufferBlob* buffer_blob) { } }; + MMTkPreBarrierCodeGenClosure pre_write_code_gen_cl; + _pre_barrier_c1_runtime_code_blob = Runtime1::generate_blob(buffer_blob, -1, "mmtk_pre_write_code_gen_cl", false, &pre_write_code_gen_cl); + + MMTkPostBarrierCodeGenClosure post_write_code_gen_cl; + _post_barrier_c1_runtime_code_blob = Runtime1::generate_blob(buffer_blob, -1, "mmtk_post_write_code_gen_cl", false, &post_write_code_gen_cl); + MMTkRefLoadBarrierCodeGenClosure load_code_gen_cl; _ref_load_barrier_c1_runtime_code_blob = Runtime1::generate_blob(buffer_blob, -1, "load_code_gen_cl", false, &load_code_gen_cl); } From 19df558e88f496f94468d4e610388fa35051caca Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Sat, 20 Sep 2025 12:45:42 +0800 Subject: [PATCH 11/46] Adjust visibility --- openjdk/mmtkBarrierSetAssembler_x86.cpp | 2 +- openjdk/mmtkBarrierSetC1.hpp | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/openjdk/mmtkBarrierSetAssembler_x86.cpp b/openjdk/mmtkBarrierSetAssembler_x86.cpp index f26f0f35..01857a91 100644 --- a/openjdk/mmtkBarrierSetAssembler_x86.cpp +++ b/openjdk/mmtkBarrierSetAssembler_x86.cpp @@ -155,7 +155,7 @@ void MMTkBarrierSetAssembler::generate_c1_ref_load_barrier_stub_call(LIR_Assembl __ cmpptr(val_reg, (int32_t) NULL_WORD); __ jcc(Assembler::equal, *stub->continuation()); ce->store_parameter(stub->val->as_register(), 0); - __ call(RuntimeAddress(bs->_ref_load_barrier_c1_runtime_code_blob->code_begin())); + __ call(RuntimeAddress(bs->ref_load_barrier_c1_runtime_code_blob()->code_begin())); __ jmp(*stub->continuation()); } diff --git a/openjdk/mmtkBarrierSetC1.hpp b/openjdk/mmtkBarrierSetC1.hpp index 8d65b6b3..23ce53d3 100644 --- a/openjdk/mmtkBarrierSetC1.hpp +++ b/openjdk/mmtkBarrierSetC1.hpp @@ -7,13 +7,12 @@ class MMTkBarrierSetAssembler; class MMTkBarrierSetC1 : public BarrierSetC1 { - friend class MMTkBarrierSetAssembler; - -protected: +private: CodeBlob* _pre_barrier_c1_runtime_code_blob; CodeBlob* _post_barrier_c1_runtime_code_blob; CodeBlob* _ref_load_barrier_c1_runtime_code_blob; +protected: /// Full pre-barrier virtual void object_reference_write_pre(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val, CodeEmitInfo* info) const {} /// Full post-barrier @@ -66,6 +65,7 @@ class MMTkBarrierSetC1 : public BarrierSetC1 { CodeBlob* pre_barrier_c1_runtime_code_blob() { return _pre_barrier_c1_runtime_code_blob; } CodeBlob* post_barrier_c1_runtime_code_blob() { return _post_barrier_c1_runtime_code_blob; } + CodeBlob* ref_load_barrier_c1_runtime_code_blob() { return _ref_load_barrier_c1_runtime_code_blob; } /// Generate C1 write barrier slow-call C1-LIR code virtual void generate_c1_runtime_stubs(BufferBlob* buffer_blob) override; From a6cb14d19bd7d1e1d29e4d4b028fe4040656b55e Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Sat, 20 Sep 2025 16:06:31 +0800 Subject: [PATCH 12/46] Object barrier C1 no use slot or new_val --- openjdk/barriers/mmtkObjectBarrier.cpp | 32 ++++++++------------------ openjdk/mmtkBarrierSetC1.hpp | 8 ++++--- 2 files changed, 15 insertions(+), 25 deletions(-) diff --git a/openjdk/barriers/mmtkObjectBarrier.cpp b/openjdk/barriers/mmtkObjectBarrier.cpp index 24b88769..c95bcc0e 100644 --- a/openjdk/barriers/mmtkObjectBarrier.cpp +++ b/openjdk/barriers/mmtkObjectBarrier.cpp @@ -86,8 +86,10 @@ void MMTkObjectBarrierSetAssembler::generate_c1_post_write_barrier_stub(LIR_Asse MMTkBarrierSetC1* bs = (MMTkBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1(); __ bind(*stub->entry()); ce->store_parameter(stub->src->as_pointer_register(), 0); - ce->store_parameter(stub->slot->as_pointer_register(), 1); - ce->store_parameter(stub->new_val->as_pointer_register(), 2); + assert(stub->slot->is_illegal(), "The slot arg must be illegal"); + assert(stub->new_val->is_illegal(), "The new_val arg must be illegal"); + ce->store_parameter(0, 1); + ce->store_parameter(0, 2); __ call(RuntimeAddress(bs->post_barrier_c1_runtime_code_blob()->code_begin())); __ jmp(*stub->continuation()); } @@ -113,26 +115,12 @@ void MMTkObjectBarrierSetC1::object_reference_write_post(LIRAccess& access, LIR_ src = reg; } assert(src->is_register(), "must be a register at this point"); - if (!slot->is_register()) { - LIR_Opr reg = gen->new_pointer_register(); - if (slot->is_constant()) { - __ move(slot, reg); - } else { - __ leal(slot, reg); - } - slot = reg; - } - assert(slot->is_register(), "must be a register at this point"); - if (!new_val->is_register()) { - LIR_Opr new_val_reg = gen->new_register(T_OBJECT); - if (new_val->is_constant()) { - __ move(new_val, new_val_reg); - } else { - __ leal(new_val, new_val_reg); - } - new_val = new_val_reg; - } - assert(new_val->is_register(), "must be a register at this point"); + + // The object barrier doesn't need the slot or the new_val arguments. + // We won't bother preparing those registers. + slot = LIR_OprFact::illegal(); + new_val = LIR_OprFact::illegal(); + CodeStub* slow = new MMTkC1PostBarrierStub(src, slot, new_val); if (mmtk_enable_barrier_fastpath) { diff --git a/openjdk/mmtkBarrierSetC1.hpp b/openjdk/mmtkBarrierSetC1.hpp index 23ce53d3..b42fd960 100644 --- a/openjdk/mmtkBarrierSetC1.hpp +++ b/openjdk/mmtkBarrierSetC1.hpp @@ -114,9 +114,11 @@ struct MMTkC1PostBarrierStub: CodeStub { virtual void visit(LIR_OpVisitState* visitor) override { visitor->do_slow_case(); - if (src != NULL) visitor->do_input(src); - if (slot != NULL) visitor->do_input(slot); - if (new_val != NULL) visitor->do_input(new_val); + assert(src->is_valid(), "src must be valid"); + visitor->do_input(src); + // Some post barrier stubs (such as object barrier and SATB barrier) don't use slot or new_val. + if (slot->is_valid()) visitor->do_input(slot); + if (slot->is_valid()) visitor->do_input(new_val); } NOT_PRODUCT(virtual void print_name(outputStream* out) const { out->print("MMTkC1PostBarrierStub"); }); From 344f5464759d173e053562357bda8d12670029ab Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Sun, 21 Sep 2025 01:35:58 +0800 Subject: [PATCH 13/46] MMTkC1PreBarrierStub: only src --- openjdk/barriers/mmtkSATBBarrier.cpp | 29 +++------------------------- openjdk/mmtkBarrierSetC1.hpp | 22 +++++---------------- 2 files changed, 8 insertions(+), 43 deletions(-) diff --git a/openjdk/barriers/mmtkSATBBarrier.cpp b/openjdk/barriers/mmtkSATBBarrier.cpp index 044f9870..44eef41f 100644 --- a/openjdk/barriers/mmtkSATBBarrier.cpp +++ b/openjdk/barriers/mmtkSATBBarrier.cpp @@ -101,31 +101,9 @@ void MMTkSATBBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Decor void MMTkSATBBarrierSetAssembler::generate_c1_pre_write_barrier_stub(LIR_Assembler* ce, MMTkC1PreBarrierStub* stub) const { MMTkBarrierSetC1* bs = (MMTkBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1(); __ bind(*stub->entry()); - - // For pre-barriers, stub->slot may not be a resolved address. - // Manually patch the address - address runtime_address; - if (stub->patch_code != lir_patch_none) { - // Patch - assert(stub->scratch->is_single_cpu(), "must be"); - assert(stub->scratch->is_register(), "Precondition."); - ce->mem2reg(stub->slot, stub->scratch, T_OBJECT, stub->patch_code, stub->info, false /*wide*/, false /*unaligned*/); - // Now stub->scratch contains the pre_val instead of the slot address - // So the following is to load the slot address into scrach register - // Resolve address - auto masm = ce->masm(); - LIR_Address* addr = stub->slot->as_address_ptr(); - Address from_addr = ce->as_Address(addr); - __ lea(stub->scratch->as_register(), from_addr); - // Store parameter - ce->store_parameter(stub->scratch->as_pointer_register(), 1); - } else { - // Store parameter - ce->store_parameter(stub->slot->as_pointer_register(), 1); - } - ce->store_parameter(stub->src->as_pointer_register(), 0); - ce->store_parameter(stub->new_val->as_pointer_register(), 2); + ce->store_parameter(0, 1); + ce->store_parameter(0, 2); __ call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin())); __ jmp(*stub->continuation()); } @@ -217,8 +195,7 @@ void MMTkSATBBarrierSetC1::object_reference_write_pre(LIRAccess& access, LIR_Opr new_val = new_val_reg; } assert(new_val->is_register(), "must be a register at this point"); - MMTkC1PreBarrierStub* slow = new MMTkC1PreBarrierStub(src, slot, new_val, info, needs_patching ? lir_patch_normal : lir_patch_none); - if (needs_patching) slow->scratch = gen->new_register(T_OBJECT); + MMTkC1PreBarrierStub* slow = new MMTkC1PreBarrierStub(src); if (mmtk_enable_barrier_fastpath) { if (needs_patching) { diff --git a/openjdk/mmtkBarrierSetC1.hpp b/openjdk/mmtkBarrierSetC1.hpp index b42fd960..03382237 100644 --- a/openjdk/mmtkBarrierSetC1.hpp +++ b/openjdk/mmtkBarrierSetC1.hpp @@ -75,28 +75,16 @@ class MMTkBarrierSetC1 : public BarrierSetC1 { /// The default behaviour is to call `MMTkBarrierSetRuntime::object_reference_write_pre_call` and pass all the three args. /// Barrier implementations may inherit from this class, and override `emit_code` to perform a specialized slow-path call. struct MMTkC1PreBarrierStub: CodeStub { - LIR_Opr src, slot, new_val; - CodeEmitInfo* info; // Code patching info - LIR_PatchCode patch_code; // Enable code patching? - LIR_Opr scratch = NULL; // Scratch register for the resolved field + LIR_Opr src; - MMTkC1PreBarrierStub(LIR_Opr src, LIR_Opr slot, LIR_Opr new_val, CodeEmitInfo* info = NULL, LIR_PatchCode patch_code = lir_patch_none): src(src), slot(slot), new_val(new_val), info(info), patch_code(patch_code) {} + MMTkC1PreBarrierStub(LIR_Opr src): src(src) {} virtual void emit_code(LIR_Assembler* ce) override; virtual void visit(LIR_OpVisitState* visitor) override { - if (info != NULL) { - visitor->do_slow_case(info); - } else { - visitor->do_slow_case(); - } - if (src != NULL) visitor->do_input(src); - if (slot != NULL) visitor->do_input(slot); - if (new_val != NULL) visitor->do_input(new_val); - if (scratch != NULL) { - assert(scratch->is_oop(), "must be"); - visitor->do_temp(scratch); - } + visitor->do_slow_case(); + assert(src->is_valid(), "src must be valid"); + visitor->do_input(src); } NOT_PRODUCT(virtual void print_name(outputStream* out) const { out->print("MMTkC1PreBarrierStub"); }); From 236b578adbae4fe20f91f8139ea4c1daa4e4edfb Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Sun, 21 Sep 2025 01:38:10 +0800 Subject: [PATCH 14/46] No needs_patching --- openjdk/barriers/mmtkSATBBarrier.cpp | 67 +++++++++++++--------------- 1 file changed, 30 insertions(+), 37 deletions(-) diff --git a/openjdk/barriers/mmtkSATBBarrier.cpp b/openjdk/barriers/mmtkSATBBarrier.cpp index 44eef41f..dfd1d509 100644 --- a/openjdk/barriers/mmtkSATBBarrier.cpp +++ b/openjdk/barriers/mmtkSATBBarrier.cpp @@ -198,43 +198,36 @@ void MMTkSATBBarrierSetC1::object_reference_write_pre(LIRAccess& access, LIR_Opr MMTkC1PreBarrierStub* slow = new MMTkC1PreBarrierStub(src); if (mmtk_enable_barrier_fastpath) { - if (needs_patching) { - // At this stage, slot address is not available, so cannot do the fast-path check until - // its address get resolved - // FIXME: Jump to a medium-path for code patching without entering slow-path - __ jump(slow); - } else { - // // load pre_val - // LIR_Address* slot_addr = new LIR_Address(slot, T_OBJECT); - // LIR_Opr addr = slot; - // __ load(slot_addr, addr); - // // if pre_val == NULL skip the barrier - // __ cmp(lir_cond_equal, addr, LIR_OprFact::oopConst(NULL)); - // __ branch(lir_cond_equal, T_OBJECT, slow->continuation()); - LIR_Opr addr = src; - // uint8_t* meta_addr = (uint8_t*) (side_metadata_base_address() + (addr >> 6)); - LIR_Opr offset = gen->new_pointer_register(); - __ move(addr, offset); - __ unsigned_shift_right(offset, 6, offset); - LIR_Opr base = gen->new_pointer_register(); - __ move(LIR_OprFact::longConst(side_metadata_base_address()), base); - LIR_Address* meta_addr = new LIR_Address(base, offset, T_BYTE); - // uint8_t byte_val = *meta_addr; - LIR_Opr byte_val = gen->new_register(T_INT); - __ move(meta_addr, byte_val); - - // intptr_t shift = (addr >> 3) & 0b111; - LIR_Opr shift = gen->new_register(T_INT); - __ move(addr, shift); - __ unsigned_shift_right(shift, 3, shift); - __ logical_and(shift, LIR_OprFact::intConst(0b111), shift); - // if (((byte_val >> shift) & 1) == 1) slow; - LIR_Opr result = byte_val; - __ unsigned_shift_right(result, shift, result, LIR_OprFact::illegalOpr); - __ logical_and(result, LIR_OprFact::intConst(1), result); - __ cmp(lir_cond_equal, result, LIR_OprFact::intConst(1)); - __ branch(lir_cond_equal, T_BYTE, slow); - } + // // load pre_val + // LIR_Address* slot_addr = new LIR_Address(slot, T_OBJECT); + // LIR_Opr addr = slot; + // __ load(slot_addr, addr); + // // if pre_val == NULL skip the barrier + // __ cmp(lir_cond_equal, addr, LIR_OprFact::oopConst(NULL)); + // __ branch(lir_cond_equal, T_OBJECT, slow->continuation()); + LIR_Opr addr = src; + // uint8_t* meta_addr = (uint8_t*) (side_metadata_base_address() + (addr >> 6)); + LIR_Opr offset = gen->new_pointer_register(); + __ move(addr, offset); + __ unsigned_shift_right(offset, 6, offset); + LIR_Opr base = gen->new_pointer_register(); + __ move(LIR_OprFact::longConst(side_metadata_base_address()), base); + LIR_Address* meta_addr = new LIR_Address(base, offset, T_BYTE); + // uint8_t byte_val = *meta_addr; + LIR_Opr byte_val = gen->new_register(T_INT); + __ move(meta_addr, byte_val); + + // intptr_t shift = (addr >> 3) & 0b111; + LIR_Opr shift = gen->new_register(T_INT); + __ move(addr, shift); + __ unsigned_shift_right(shift, 3, shift); + __ logical_and(shift, LIR_OprFact::intConst(0b111), shift); + // if (((byte_val >> shift) & 1) == 1) slow; + LIR_Opr result = byte_val; + __ unsigned_shift_right(result, shift, result, LIR_OprFact::illegalOpr); + __ logical_and(result, LIR_OprFact::intConst(1), result); + __ cmp(lir_cond_equal, result, LIR_OprFact::intConst(1)); + __ branch(lir_cond_equal, T_BYTE, slow); } else { __ jump(slow); } From 4c3847e20cc81f6867ecb1d782126f4f93499529 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Sun, 21 Sep 2025 01:39:40 +0800 Subject: [PATCH 15/46] No touching new_val --- openjdk/barriers/mmtkSATBBarrier.cpp | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/openjdk/barriers/mmtkSATBBarrier.cpp b/openjdk/barriers/mmtkSATBBarrier.cpp index dfd1d509..667788b0 100644 --- a/openjdk/barriers/mmtkSATBBarrier.cpp +++ b/openjdk/barriers/mmtkSATBBarrier.cpp @@ -185,16 +185,6 @@ void MMTkSATBBarrierSetC1::object_reference_write_pre(LIRAccess& access, LIR_Opr slot = LIR_OprFact::address(new LIR_Address(slot, T_OBJECT)); } assert(needs_patching || slot->is_register(), "must be a register at this point unless needs_patching"); - if (!new_val->is_register()) { - LIR_Opr new_val_reg = gen->new_register(T_OBJECT); - if (new_val->is_constant()) { - __ move(new_val, new_val_reg); - } else { - __ leal(new_val, new_val_reg); - } - new_val = new_val_reg; - } - assert(new_val->is_register(), "must be a register at this point"); MMTkC1PreBarrierStub* slow = new MMTkC1PreBarrierStub(src); if (mmtk_enable_barrier_fastpath) { From 7d72ed32d667d667b4ecca4ca2b712a33e8781a0 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Sun, 21 Sep 2025 19:38:52 +0800 Subject: [PATCH 16/46] No touching slot operand in SATB C1 barrier And no define private as public. --- openjdk/barriers/mmtkSATBBarrier.cpp | 52 +++++++++------------------- openjdk/mmtkBarrierSetC1.hpp | 12 +++---- 2 files changed, 23 insertions(+), 41 deletions(-) diff --git a/openjdk/barriers/mmtkSATBBarrier.cpp b/openjdk/barriers/mmtkSATBBarrier.cpp index 667788b0..2f8d5e80 100644 --- a/openjdk/barriers/mmtkSATBBarrier.cpp +++ b/openjdk/barriers/mmtkSATBBarrier.cpp @@ -1,5 +1,5 @@ // Workaround the fact that LIR_Assembler::as_Address is private. -#define private public +//#define private public #include "precompiled.hpp" #include "mmtkSATBBarrier.hpp" @@ -159,43 +159,25 @@ void MMTkSATBBarrierSetC1::object_reference_write_pre(LIRAccess& access, LIR_Opr DecoratorSet decorators = access.decorators(); if ((decorators & IN_HEAP) == 0) return; // Not sure if this line is sound bool needs_patching = (decorators & C1_NEEDS_PATCHING) != 0; - if (!src->is_register()) { - LIR_Opr reg = gen->new_pointer_register(); - if (src->is_constant()) { - __ move(src, reg); - } else { - __ leal(src, reg); - } - src = reg; - } - assert(src->is_register(), "must be a register at this point"); - - if (!slot->is_register() && !needs_patching) { - LIR_Address* address = slot->as_address_ptr(); - LIR_Opr ptr = gen->new_pointer_register(); - if (!address->index()->is_valid() && address->disp() == 0) { - __ move(address->base(), ptr); - } else { - assert(address->disp() != max_jint, "lea doesn't support patched addresses!"); - __ leal(slot, ptr); - } - slot = ptr; - } else if (needs_patching && !slot->is_address()) { - assert(slot->is_register(), "must be"); - slot = LIR_OprFact::address(new LIR_Address(slot, T_OBJECT)); - } - assert(needs_patching || slot->is_register(), "must be a register at this point unless needs_patching"); + MMTkC1PreBarrierStub* slow = new MMTkC1PreBarrierStub(src); if (mmtk_enable_barrier_fastpath) { - // // load pre_val - // LIR_Address* slot_addr = new LIR_Address(slot, T_OBJECT); - // LIR_Opr addr = slot; - // __ load(slot_addr, addr); - // // if pre_val == NULL skip the barrier - // __ cmp(lir_cond_equal, addr, LIR_OprFact::oopConst(NULL)); - // __ branch(lir_cond_equal, T_OBJECT, slow->continuation()); - LIR_Opr addr = src; + // We need to do bit operations on the address of `src`. In order to move `src` (`T_OBJECT` or + // `T_ARRAY`) to a pointer regiseter (`T_LONG` on 64 bit), the source operand must be in + // register, in which case `LIR_Assembler::reg2reg` works as expected. Otherwise `stack2ref` + // will complain that the source (`T_OBJECT` or `T_ARRAY` is single-cpu while the destination + // `T_LONG` is double-cpu). + // + // However, checking `src.is_register()` won't work because the same LIR code may be compiled + // again. Even it is register the first time, `src.is_stack()` may instead be true at the second + // time. + // + // So we introduce an intermediate step. We move `src` into `addr` which is a `T_OBJECT` + // register first to make sure it is in register. Then we move `addr` to newly created pointer + // registers. + LIR_Opr addr = gen->new_register(T_OBJECT); + __ move(src, addr); // uint8_t* meta_addr = (uint8_t*) (side_metadata_base_address() + (addr >> 6)); LIR_Opr offset = gen->new_pointer_register(); __ move(addr, offset); diff --git a/openjdk/mmtkBarrierSetC1.hpp b/openjdk/mmtkBarrierSetC1.hpp index 03382237..5dff09e1 100644 --- a/openjdk/mmtkBarrierSetC1.hpp +++ b/openjdk/mmtkBarrierSetC1.hpp @@ -20,24 +20,24 @@ class MMTkBarrierSetC1 : public BarrierSetC1 { /// Substituting write barrier virtual void store_at_resolved(LIRAccess& access, LIR_Opr value) override { - if (access.is_oop()) object_reference_write_pre(access, access.base().opr(), access.resolved_addr(), value, access.patch_emit_info()); + if (access.is_oop()) object_reference_write_pre(access, access.base().opr(), LIR_OprFact::illegal(), value, access.patch_emit_info()); BarrierSetC1::store_at_resolved(access, value); - if (access.is_oop()) object_reference_write_post(access, access.base().opr(), access.resolved_addr(), value); + if (access.is_oop()) object_reference_write_post(access, access.base().opr(), LIR_OprFact::illegal(), value); } /// Substituting write barrier (cmpxchg) virtual LIR_Opr atomic_cmpxchg_at_resolved(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value) override { - if (access.is_oop()) object_reference_write_pre(access, access.base().opr(), access.resolved_addr(), new_value.result(), NULL); + if (access.is_oop()) object_reference_write_pre(access, access.base().opr(), LIR_OprFact::illegal(), new_value.result(), NULL); LIR_Opr result = BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value); - if (access.is_oop()) object_reference_write_post(access, access.base().opr(), access.resolved_addr(), new_value.result()); + if (access.is_oop()) object_reference_write_post(access, access.base().opr(), LIR_OprFact::illegal(), new_value.result()); return result; } /// Substituting write barrier (xchg) virtual LIR_Opr atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value) override { - if (access.is_oop()) object_reference_write_pre(access, access.base().opr(), access.resolved_addr(), value.result(), NULL); + if (access.is_oop()) object_reference_write_pre(access, access.base().opr(), LIR_OprFact::illegal(), value.result(), NULL); LIR_Opr result = BarrierSetC1::atomic_xchg_at_resolved(access, value); - if (access.is_oop()) object_reference_write_post(access, access.base().opr(), access.resolved_addr(), value.result()); + if (access.is_oop()) object_reference_write_post(access, access.base().opr(), LIR_OprFact::illegal(), value.result()); return result; } From 822c58edd8e077de5c459157ebc30b8c7d90a21e Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Sun, 21 Sep 2025 19:43:38 +0800 Subject: [PATCH 17/46] C1 pre/post barrier method only access arg --- openjdk/barriers/mmtkObjectBarrier.cpp | 7 ++++--- openjdk/barriers/mmtkObjectBarrier.hpp | 2 +- openjdk/barriers/mmtkSATBBarrier.cpp | 4 +++- openjdk/barriers/mmtkSATBBarrier.hpp | 2 +- openjdk/mmtkBarrierSetC1.hpp | 16 ++++++++-------- 5 files changed, 17 insertions(+), 14 deletions(-) diff --git a/openjdk/barriers/mmtkObjectBarrier.cpp b/openjdk/barriers/mmtkObjectBarrier.cpp index c95bcc0e..a933c956 100644 --- a/openjdk/barriers/mmtkObjectBarrier.cpp +++ b/openjdk/barriers/mmtkObjectBarrier.cpp @@ -101,10 +101,11 @@ void MMTkObjectBarrierSetAssembler::generate_c1_post_write_barrier_stub(LIR_Asse #define __ gen->lir()-> #endif -void MMTkObjectBarrierSetC1::object_reference_write_post(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val) const { +void MMTkObjectBarrierSetC1::object_reference_write_post(LIRAccess& access) const { LIRGenerator* gen = access.gen(); DecoratorSet decorators = access.decorators(); if ((decorators & IN_HEAP) == 0) return; + LIR_Opr src = access.base().opr(); if (!src->is_register()) { LIR_Opr reg = gen->new_pointer_register(); if (src->is_constant()) { @@ -118,8 +119,8 @@ void MMTkObjectBarrierSetC1::object_reference_write_post(LIRAccess& access, LIR_ // The object barrier doesn't need the slot or the new_val arguments. // We won't bother preparing those registers. - slot = LIR_OprFact::illegal(); - new_val = LIR_OprFact::illegal(); + LIR_Opr slot = LIR_OprFact::illegal(); + LIR_Opr new_val = LIR_OprFact::illegal(); CodeStub* slow = new MMTkC1PostBarrierStub(src, slot, new_val); diff --git a/openjdk/barriers/mmtkObjectBarrier.hpp b/openjdk/barriers/mmtkObjectBarrier.hpp index db8fb45c..fb26125d 100644 --- a/openjdk/barriers/mmtkObjectBarrier.hpp +++ b/openjdk/barriers/mmtkObjectBarrier.hpp @@ -40,7 +40,7 @@ class MMTkObjectBarrierSetAssembler: public MMTkUnlogBitBarrierSetAssembler { class MMTkObjectBarrierSetC1: public MMTkBarrierSetC1 { protected: - virtual void object_reference_write_post(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val) const override; + virtual void object_reference_write_post(LIRAccess& access) const override; virtual LIR_Opr resolve_address(LIRAccess& access, bool resolve_in_register) override { return MMTkBarrierSetC1::resolve_address_in_register(access, resolve_in_register); diff --git a/openjdk/barriers/mmtkSATBBarrier.cpp b/openjdk/barriers/mmtkSATBBarrier.cpp index 2f8d5e80..9b287b8d 100644 --- a/openjdk/barriers/mmtkSATBBarrier.cpp +++ b/openjdk/barriers/mmtkSATBBarrier.cpp @@ -154,12 +154,14 @@ void MMTkSATBBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) { #endif } -void MMTkSATBBarrierSetC1::object_reference_write_pre(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val, CodeEmitInfo* info) const { +void MMTkSATBBarrierSetC1::object_reference_write_pre(LIRAccess& access) const { LIRGenerator* gen = access.gen(); DecoratorSet decorators = access.decorators(); if ((decorators & IN_HEAP) == 0) return; // Not sure if this line is sound bool needs_patching = (decorators & C1_NEEDS_PATCHING) != 0; + LIR_Opr src = access.base().opr(); + MMTkC1PreBarrierStub* slow = new MMTkC1PreBarrierStub(src); if (mmtk_enable_barrier_fastpath) { diff --git a/openjdk/barriers/mmtkSATBBarrier.hpp b/openjdk/barriers/mmtkSATBBarrier.hpp index bc960dbe..11bcb8d9 100644 --- a/openjdk/barriers/mmtkSATBBarrier.hpp +++ b/openjdk/barriers/mmtkSATBBarrier.hpp @@ -42,7 +42,7 @@ class MMTkSATBBarrierSetAssembler: public MMTkUnlogBitBarrierSetAssembler { class MMTkSATBBarrierSetC1: public MMTkBarrierSetC1 { protected: - virtual void object_reference_write_pre(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val, CodeEmitInfo* info) const override; + virtual void object_reference_write_pre(LIRAccess& access) const override; virtual void load_at_resolved(LIRAccess& access, LIR_Opr result) override; diff --git a/openjdk/mmtkBarrierSetC1.hpp b/openjdk/mmtkBarrierSetC1.hpp index 5dff09e1..e0d4c6c9 100644 --- a/openjdk/mmtkBarrierSetC1.hpp +++ b/openjdk/mmtkBarrierSetC1.hpp @@ -14,30 +14,30 @@ class MMTkBarrierSetC1 : public BarrierSetC1 { protected: /// Full pre-barrier - virtual void object_reference_write_pre(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val, CodeEmitInfo* info) const {} + virtual void object_reference_write_pre(LIRAccess& access) const {} /// Full post-barrier - virtual void object_reference_write_post(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val) const {} + virtual void object_reference_write_post(LIRAccess& access) const {} /// Substituting write barrier virtual void store_at_resolved(LIRAccess& access, LIR_Opr value) override { - if (access.is_oop()) object_reference_write_pre(access, access.base().opr(), LIR_OprFact::illegal(), value, access.patch_emit_info()); + if (access.is_oop()) object_reference_write_pre(access); BarrierSetC1::store_at_resolved(access, value); - if (access.is_oop()) object_reference_write_post(access, access.base().opr(), LIR_OprFact::illegal(), value); + if (access.is_oop()) object_reference_write_post(access); } /// Substituting write barrier (cmpxchg) virtual LIR_Opr atomic_cmpxchg_at_resolved(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value) override { - if (access.is_oop()) object_reference_write_pre(access, access.base().opr(), LIR_OprFact::illegal(), new_value.result(), NULL); + if (access.is_oop()) object_reference_write_pre(access); LIR_Opr result = BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value); - if (access.is_oop()) object_reference_write_post(access, access.base().opr(), LIR_OprFact::illegal(), new_value.result()); + if (access.is_oop()) object_reference_write_post(access); return result; } /// Substituting write barrier (xchg) virtual LIR_Opr atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value) override { - if (access.is_oop()) object_reference_write_pre(access, access.base().opr(), LIR_OprFact::illegal(), value.result(), NULL); + if (access.is_oop()) object_reference_write_pre(access); LIR_Opr result = BarrierSetC1::atomic_xchg_at_resolved(access, value); - if (access.is_oop()) object_reference_write_post(access, access.base().opr(), LIR_OprFact::illegal(), value.result()); + if (access.is_oop()) object_reference_write_post(access); return result; } From cd81d209eac8881616c6dad0bf5300f9cc4b46b0 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Sun, 21 Sep 2025 19:53:42 +0800 Subject: [PATCH 18/46] Extract C1 unlog bit check fast paths --- openjdk/barriers/mmtkObjectBarrier.cpp | 23 +----------- openjdk/barriers/mmtkObjectBarrier.hpp | 2 +- openjdk/barriers/mmtkSATBBarrier.cpp | 38 +------------------ openjdk/barriers/mmtkSATBBarrier.hpp | 2 +- openjdk/barriers/mmtkUnlogBitBarrier.cpp | 48 ++++++++++++++++++++++++ openjdk/barriers/mmtkUnlogBitBarrier.hpp | 5 ++- 6 files changed, 56 insertions(+), 62 deletions(-) diff --git a/openjdk/barriers/mmtkObjectBarrier.cpp b/openjdk/barriers/mmtkObjectBarrier.cpp index a933c956..2eb5085b 100644 --- a/openjdk/barriers/mmtkObjectBarrier.cpp +++ b/openjdk/barriers/mmtkObjectBarrier.cpp @@ -125,28 +125,7 @@ void MMTkObjectBarrierSetC1::object_reference_write_post(LIRAccess& access) cons CodeStub* slow = new MMTkC1PostBarrierStub(src, slot, new_val); if (mmtk_enable_barrier_fastpath) { - LIR_Opr addr = src; - // uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); - LIR_Opr offset = gen->new_pointer_register(); - __ move(addr, offset); - __ unsigned_shift_right(offset, 6, offset); - LIR_Opr base = gen->new_pointer_register(); - __ move(LIR_OprFact::longConst(SIDE_METADATA_BASE_ADDRESS), base); - LIR_Address* meta_addr = new LIR_Address(base, offset, T_BYTE); - // uint8_t byte_val = *meta_addr; - LIR_Opr byte_val = gen->new_register(T_INT); - __ move(meta_addr, byte_val); - // intptr_t shift = (addr >> 3) & 0b111; - LIR_Opr shift = gen->new_register(T_INT); - __ move(addr, shift); - __ unsigned_shift_right(shift, 3, shift); - __ logical_and(shift, LIR_OprFact::intConst(0b111), shift); - // if (((byte_val >> shift) & 1) == 1) slow; - LIR_Opr result = byte_val; - __ unsigned_shift_right(result, shift, result, LIR_OprFact::illegalOpr); - __ logical_and(result, LIR_OprFact::intConst(1), result); - __ cmp(lir_cond_equal, result, LIR_OprFact::intConst(1)); - __ branch(lir_cond_equal, T_BYTE, slow); + emit_check_unlog_bit_fast_path(gen, src, slow); } else { __ jump(slow); } diff --git a/openjdk/barriers/mmtkObjectBarrier.hpp b/openjdk/barriers/mmtkObjectBarrier.hpp index fb26125d..cc000e4e 100644 --- a/openjdk/barriers/mmtkObjectBarrier.hpp +++ b/openjdk/barriers/mmtkObjectBarrier.hpp @@ -38,7 +38,7 @@ class MMTkObjectBarrierSetAssembler: public MMTkUnlogBitBarrierSetAssembler { virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) override; }; -class MMTkObjectBarrierSetC1: public MMTkBarrierSetC1 { +class MMTkObjectBarrierSetC1: public MMTkUnlogBitBarrierSetC1 { protected: virtual void object_reference_write_post(LIRAccess& access) const override; diff --git a/openjdk/barriers/mmtkSATBBarrier.cpp b/openjdk/barriers/mmtkSATBBarrier.cpp index 9b287b8d..90f2805b 100644 --- a/openjdk/barriers/mmtkSATBBarrier.cpp +++ b/openjdk/barriers/mmtkSATBBarrier.cpp @@ -165,43 +165,7 @@ void MMTkSATBBarrierSetC1::object_reference_write_pre(LIRAccess& access) const { MMTkC1PreBarrierStub* slow = new MMTkC1PreBarrierStub(src); if (mmtk_enable_barrier_fastpath) { - // We need to do bit operations on the address of `src`. In order to move `src` (`T_OBJECT` or - // `T_ARRAY`) to a pointer regiseter (`T_LONG` on 64 bit), the source operand must be in - // register, in which case `LIR_Assembler::reg2reg` works as expected. Otherwise `stack2ref` - // will complain that the source (`T_OBJECT` or `T_ARRAY` is single-cpu while the destination - // `T_LONG` is double-cpu). - // - // However, checking `src.is_register()` won't work because the same LIR code may be compiled - // again. Even it is register the first time, `src.is_stack()` may instead be true at the second - // time. - // - // So we introduce an intermediate step. We move `src` into `addr` which is a `T_OBJECT` - // register first to make sure it is in register. Then we move `addr` to newly created pointer - // registers. - LIR_Opr addr = gen->new_register(T_OBJECT); - __ move(src, addr); - // uint8_t* meta_addr = (uint8_t*) (side_metadata_base_address() + (addr >> 6)); - LIR_Opr offset = gen->new_pointer_register(); - __ move(addr, offset); - __ unsigned_shift_right(offset, 6, offset); - LIR_Opr base = gen->new_pointer_register(); - __ move(LIR_OprFact::longConst(side_metadata_base_address()), base); - LIR_Address* meta_addr = new LIR_Address(base, offset, T_BYTE); - // uint8_t byte_val = *meta_addr; - LIR_Opr byte_val = gen->new_register(T_INT); - __ move(meta_addr, byte_val); - - // intptr_t shift = (addr >> 3) & 0b111; - LIR_Opr shift = gen->new_register(T_INT); - __ move(addr, shift); - __ unsigned_shift_right(shift, 3, shift); - __ logical_and(shift, LIR_OprFact::intConst(0b111), shift); - // if (((byte_val >> shift) & 1) == 1) slow; - LIR_Opr result = byte_val; - __ unsigned_shift_right(result, shift, result, LIR_OprFact::illegalOpr); - __ logical_and(result, LIR_OprFact::intConst(1), result); - __ cmp(lir_cond_equal, result, LIR_OprFact::intConst(1)); - __ branch(lir_cond_equal, T_BYTE, slow); + emit_check_unlog_bit_fast_path(gen, src, slow); } else { __ jump(slow); } diff --git a/openjdk/barriers/mmtkSATBBarrier.hpp b/openjdk/barriers/mmtkSATBBarrier.hpp index 11bcb8d9..50697b74 100644 --- a/openjdk/barriers/mmtkSATBBarrier.hpp +++ b/openjdk/barriers/mmtkSATBBarrier.hpp @@ -40,7 +40,7 @@ class MMTkSATBBarrierSetAssembler: public MMTkUnlogBitBarrierSetAssembler { virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp_thread) override; }; -class MMTkSATBBarrierSetC1: public MMTkBarrierSetC1 { +class MMTkSATBBarrierSetC1: public MMTkUnlogBitBarrierSetC1 { protected: virtual void object_reference_write_pre(LIRAccess& access) const override; diff --git a/openjdk/barriers/mmtkUnlogBitBarrier.cpp b/openjdk/barriers/mmtkUnlogBitBarrier.cpp index 62441d5a..fcfdca03 100644 --- a/openjdk/barriers/mmtkUnlogBitBarrier.cpp +++ b/openjdk/barriers/mmtkUnlogBitBarrier.cpp @@ -80,3 +80,51 @@ void MMTkUnlogBitBarrierSetAssembler::object_reference_write_pre_or_post(MacroAs } #undef __ + +#ifdef ASSERT +#define __ gen->lir(__FILE__, __LINE__)-> +#else +#define __ gen->lir()-> +#endif + +void MMTkUnlogBitBarrierSetC1::emit_check_unlog_bit_fast_path(LIRGenerator* gen, LIR_Opr src, CodeStub* slow) { + // We need to do bit operations on the address of `src`. In order to move `src` (`T_OBJECT` or + // `T_ARRAY`) to a pointer regiseter (`T_LONG` on 64 bit), the source operand must be in + // register, in which case `LIR_Assembler::reg2reg` works as expected. Otherwise `stack2ref` + // will complain that the source (`T_OBJECT` or `T_ARRAY` is single-cpu while the destination + // `T_LONG` is double-cpu). + // + // However, checking `src.is_register()` won't work because the same LIR code may be compiled + // again. Even it is register the first time, `src.is_stack()` may instead be true at the second + // time. + // + // So we introduce an intermediate step. We move `src` into `addr` which is a `T_OBJECT` + // register first to make sure it is in register. Then we move `addr` to newly created pointer + // registers. + LIR_Opr addr = gen->new_register(T_OBJECT); + __ move(src, addr); + // uint8_t* meta_addr = (uint8_t*) (side_metadata_base_address() + (addr >> 6)); + LIR_Opr offset = gen->new_pointer_register(); + __ move(addr, offset); + __ unsigned_shift_right(offset, 6, offset); + LIR_Opr base = gen->new_pointer_register(); + __ move(LIR_OprFact::longConst(UNLOG_BIT_BASE_ADDRESS), base); + LIR_Address* meta_addr = new LIR_Address(base, offset, T_BYTE); + // uint8_t byte_val = *meta_addr; + LIR_Opr byte_val = gen->new_register(T_INT); + __ move(meta_addr, byte_val); + + // intptr_t shift = (addr >> 3) & 0b111; + LIR_Opr shift = gen->new_register(T_INT); + __ move(addr, shift); + __ unsigned_shift_right(shift, 3, shift); + __ logical_and(shift, LIR_OprFact::intConst(0b111), shift); + // if (((byte_val >> shift) & 1) == 1) slow; + LIR_Opr result = byte_val; + __ unsigned_shift_right(result, shift, result, LIR_OprFact::illegalOpr); + __ logical_and(result, LIR_OprFact::intConst(1), result); + __ cmp(lir_cond_equal, result, LIR_OprFact::intConst(1)); + __ branch(lir_cond_equal, T_BYTE, slow); +} + +#undef __ diff --git a/openjdk/barriers/mmtkUnlogBitBarrier.hpp b/openjdk/barriers/mmtkUnlogBitBarrier.hpp index 3bea0fc3..eaf00a16 100644 --- a/openjdk/barriers/mmtkUnlogBitBarrier.hpp +++ b/openjdk/barriers/mmtkUnlogBitBarrier.hpp @@ -27,7 +27,10 @@ class MMTkUnlogBitBarrierSetAssembler: public MMTkBarrierSetAssembler { static void object_reference_write_pre_or_post(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, bool pre); }; -class MMTkUnlogBitBarrierSetC1: public MMTkBarrierSetC1 {}; +class MMTkUnlogBitBarrierSetC1: public MMTkBarrierSetC1 { +protected: + static void emit_check_unlog_bit_fast_path(LIRGenerator* gen, LIR_Opr addr, CodeStub* slow); +}; class MMTkUnlogBitBarrierSetC2: public MMTkBarrierSetC2 {}; From 53d68a05f18310ced92e2c9b65338eb946f7f669 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Mon, 22 Sep 2025 00:09:07 +0800 Subject: [PATCH 19/46] Extract C1 pre/post barrier to UnlogBarrierSet and move the slow-path stub to UnlogBarrierSet, too. And combine the pre and post barrier stub. --- openjdk/barriers/mmtkObjectBarrier.cpp | 45 +----------------------- openjdk/barriers/mmtkObjectBarrier.hpp | 1 - openjdk/barriers/mmtkSATBBarrier.cpp | 32 ++--------------- openjdk/barriers/mmtkSATBBarrier.hpp | 1 - openjdk/barriers/mmtkUnlogBitBarrier.cpp | 42 ++++++++++++++++++++++ openjdk/barriers/mmtkUnlogBitBarrier.hpp | 27 ++++++++++++++ openjdk/mmtkBarrierSetAssembler_x86.hpp | 5 --- openjdk/mmtkBarrierSetC1.cpp | 10 ------ openjdk/mmtkBarrierSetC1.hpp | 41 --------------------- 9 files changed, 73 insertions(+), 131 deletions(-) diff --git a/openjdk/barriers/mmtkObjectBarrier.cpp b/openjdk/barriers/mmtkObjectBarrier.cpp index 2eb5085b..222dd42f 100644 --- a/openjdk/barriers/mmtkObjectBarrier.cpp +++ b/openjdk/barriers/mmtkObjectBarrier.cpp @@ -80,21 +80,6 @@ void MMTkObjectBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, Dec #undef __ -#define __ ce->masm()-> - -void MMTkObjectBarrierSetAssembler::generate_c1_post_write_barrier_stub(LIR_Assembler* ce, MMTkC1PostBarrierStub* stub) const { - MMTkBarrierSetC1* bs = (MMTkBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1(); - __ bind(*stub->entry()); - ce->store_parameter(stub->src->as_pointer_register(), 0); - assert(stub->slot->is_illegal(), "The slot arg must be illegal"); - assert(stub->new_val->is_illegal(), "The new_val arg must be illegal"); - ce->store_parameter(0, 1); - ce->store_parameter(0, 2); - __ call(RuntimeAddress(bs->post_barrier_c1_runtime_code_blob()->code_begin())); - __ jmp(*stub->continuation()); -} -#undef __ - #ifdef ASSERT #define __ gen->lir(__FILE__, __LINE__)-> #else @@ -102,35 +87,7 @@ void MMTkObjectBarrierSetAssembler::generate_c1_post_write_barrier_stub(LIR_Asse #endif void MMTkObjectBarrierSetC1::object_reference_write_post(LIRAccess& access) const { - LIRGenerator* gen = access.gen(); - DecoratorSet decorators = access.decorators(); - if ((decorators & IN_HEAP) == 0) return; - LIR_Opr src = access.base().opr(); - if (!src->is_register()) { - LIR_Opr reg = gen->new_pointer_register(); - if (src->is_constant()) { - __ move(src, reg); - } else { - __ leal(src, reg); - } - src = reg; - } - assert(src->is_register(), "must be a register at this point"); - - // The object barrier doesn't need the slot or the new_val arguments. - // We won't bother preparing those registers. - LIR_Opr slot = LIR_OprFact::illegal(); - LIR_Opr new_val = LIR_OprFact::illegal(); - - CodeStub* slow = new MMTkC1PostBarrierStub(src, slot, new_val); - - if (mmtk_enable_barrier_fastpath) { - emit_check_unlog_bit_fast_path(gen, src, slow); - } else { - __ jump(slow); - } - - __ branch_destination(slow->continuation()); + object_reference_write_pre_or_post(access, false); } #undef __ diff --git a/openjdk/barriers/mmtkObjectBarrier.hpp b/openjdk/barriers/mmtkObjectBarrier.hpp index cc000e4e..f5d3a861 100644 --- a/openjdk/barriers/mmtkObjectBarrier.hpp +++ b/openjdk/barriers/mmtkObjectBarrier.hpp @@ -33,7 +33,6 @@ class MMTkObjectBarrierSetAssembler: public MMTkUnlogBitBarrierSetAssembler { protected: virtual void object_reference_write_post(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2, bool compensate_val_reg) const override; public: - virtual void generate_c1_post_write_barrier_stub(LIR_Assembler* ce, MMTkC1PostBarrierStub* stub) const override; virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) override; virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) override; }; diff --git a/openjdk/barriers/mmtkSATBBarrier.cpp b/openjdk/barriers/mmtkSATBBarrier.cpp index 90f2805b..3384d569 100644 --- a/openjdk/barriers/mmtkSATBBarrier.cpp +++ b/openjdk/barriers/mmtkSATBBarrier.cpp @@ -96,20 +96,6 @@ void MMTkSATBBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Decor #undef __ -#define __ ce->masm()-> - -void MMTkSATBBarrierSetAssembler::generate_c1_pre_write_barrier_stub(LIR_Assembler* ce, MMTkC1PreBarrierStub* stub) const { - MMTkBarrierSetC1* bs = (MMTkBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1(); - __ bind(*stub->entry()); - ce->store_parameter(stub->src->as_pointer_register(), 0); - ce->store_parameter(0, 1); - ce->store_parameter(0, 2); - __ call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin())); - __ jmp(*stub->continuation()); -} - -#undef __ - #ifdef ASSERT #define __ gen->lir(__FILE__, __LINE__)-> #else @@ -155,22 +141,10 @@ void MMTkSATBBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) { } void MMTkSATBBarrierSetC1::object_reference_write_pre(LIRAccess& access) const { - LIRGenerator* gen = access.gen(); - DecoratorSet decorators = access.decorators(); - if ((decorators & IN_HEAP) == 0) return; // Not sure if this line is sound - bool needs_patching = (decorators & C1_NEEDS_PATCHING) != 0; - - LIR_Opr src = access.base().opr(); - - MMTkC1PreBarrierStub* slow = new MMTkC1PreBarrierStub(src); - - if (mmtk_enable_barrier_fastpath) { - emit_check_unlog_bit_fast_path(gen, src, slow); - } else { - __ jump(slow); - } + // TODO: Should be able to skip weak field writes, too, because that doesn't change strong + // reachability during concurrent marking. - __ branch_destination(slow->continuation()); + object_reference_write_pre_or_post(access, true); } #undef __ diff --git a/openjdk/barriers/mmtkSATBBarrier.hpp b/openjdk/barriers/mmtkSATBBarrier.hpp index 50697b74..d82f45ff 100644 --- a/openjdk/barriers/mmtkSATBBarrier.hpp +++ b/openjdk/barriers/mmtkSATBBarrier.hpp @@ -35,7 +35,6 @@ class MMTkSATBBarrierSetAssembler: public MMTkUnlogBitBarrierSetAssembler { protected: virtual void object_reference_write_pre(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2) const override; public: - virtual void generate_c1_pre_write_barrier_stub(LIR_Assembler* ce, MMTkC1PreBarrierStub* stub) const override; virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) override; virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp_thread) override; }; diff --git a/openjdk/barriers/mmtkUnlogBitBarrier.cpp b/openjdk/barriers/mmtkUnlogBitBarrier.cpp index fcfdca03..beab4cac 100644 --- a/openjdk/barriers/mmtkUnlogBitBarrier.cpp +++ b/openjdk/barriers/mmtkUnlogBitBarrier.cpp @@ -1,6 +1,9 @@ #include "precompiled.hpp" #include "mmtkUnlogBitBarrier.hpp" + #include "runtime/interfaceSupport.inline.hpp" +#include "c1/c1_LIRAssembler.hpp" +#include "c1/c1_MacroAssembler.hpp" #define __ masm-> @@ -81,6 +84,27 @@ void MMTkUnlogBitBarrierSetAssembler::object_reference_write_pre_or_post(MacroAs #undef __ +#define __ ce->masm()-> + +void MMTkUnlogBitBarrierSetAssembler::generate_c1_unlog_bit_barrier_slow_path_stub(LIR_Assembler* ce, MMTkC1UnlogBitBarrierSlowPathStub* stub) const { + MMTkBarrierSetC1* bs = (MMTkBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1(); + __ bind(*stub->entry()); + ce->store_parameter(stub->src->as_pointer_register(), 0); + ce->store_parameter(0, 1); + ce->store_parameter(0, 2); + CodeBlob* code_blob = stub->pre ? bs->pre_barrier_c1_runtime_code_blob() + : bs->post_barrier_c1_runtime_code_blob(); + __ call(RuntimeAddress(code_blob->code_begin())); + __ jmp(*stub->continuation()); +} + +void MMTkC1UnlogBitBarrierSlowPathStub::emit_code(LIR_Assembler* ce) { + MMTkUnlogBitBarrierSetAssembler* bs = (MMTkUnlogBitBarrierSetAssembler*) BarrierSet::barrier_set()->barrier_set_assembler(); + bs->generate_c1_unlog_bit_barrier_slow_path_stub(ce, this); +} + +#undef __ + #ifdef ASSERT #define __ gen->lir(__FILE__, __LINE__)-> #else @@ -127,4 +151,22 @@ void MMTkUnlogBitBarrierSetC1::emit_check_unlog_bit_fast_path(LIRGenerator* gen, __ branch(lir_cond_equal, T_BYTE, slow); } +void MMTkUnlogBitBarrierSetC1::object_reference_write_pre_or_post(LIRAccess& access, bool pre) { + LIRGenerator* gen = access.gen(); + DecoratorSet decorators = access.decorators(); + if ((decorators & IN_HEAP) == 0) return; + + LIR_Opr src = access.base().opr(); + + CodeStub* slow = new MMTkC1UnlogBitBarrierSlowPathStub(src, pre); + + if (mmtk_enable_barrier_fastpath) { + emit_check_unlog_bit_fast_path(gen, src, slow); + } else { + __ jump(slow); + } + + __ branch_destination(slow->continuation()); +} + #undef __ diff --git a/openjdk/barriers/mmtkUnlogBitBarrier.hpp b/openjdk/barriers/mmtkUnlogBitBarrier.hpp index eaf00a16..4d693afc 100644 --- a/openjdk/barriers/mmtkUnlogBitBarrier.hpp +++ b/openjdk/barriers/mmtkUnlogBitBarrier.hpp @@ -8,6 +8,8 @@ /// This file contains abstract barrier sets for barriers based on the (object-grained) unlog bit. +struct MMTkC1UnlogBitBarrierSlowPathStub; + const uintptr_t UNLOG_BIT_BASE_ADDRESS = GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS; class MMTkUnlogBitBarrierSetRuntime: public MMTkBarrierSetRuntime { @@ -25,11 +27,36 @@ class MMTkUnlogBitBarrierSetAssembler: public MMTkBarrierSetAssembler { protected: static void emit_check_unlog_bit_fast_path(MacroAssembler* masm, Label &done, Register obj, Register tmp1, Register tmp2); static void object_reference_write_pre_or_post(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, bool pre); + +public: + /// Generate C1 barrier slow path stub + void generate_c1_unlog_bit_barrier_slow_path_stub(LIR_Assembler* ce, MMTkC1UnlogBitBarrierSlowPathStub* stub) const; }; class MMTkUnlogBitBarrierSetC1: public MMTkBarrierSetC1 { protected: static void emit_check_unlog_bit_fast_path(LIRGenerator* gen, LIR_Opr addr, CodeStub* slow); + static void object_reference_write_pre_or_post(LIRAccess& access, bool pre); +}; + +/// C1 pre write barrier slow-call stub. +/// The default behaviour is to call `MMTkBarrierSetRuntime::object_reference_write_pre_call` and pass all the three args. +/// Barrier implementations may inherit from this class, and override `emit_code` to perform a specialized slow-path call. +struct MMTkC1UnlogBitBarrierSlowPathStub: CodeStub { + LIR_Opr src; + bool pre; + + MMTkC1UnlogBitBarrierSlowPathStub(LIR_Opr src, bool pre): src(src), pre(pre) {} + + virtual void emit_code(LIR_Assembler* ce) override; + + virtual void visit(LIR_OpVisitState* visitor) override { + visitor->do_slow_case(); + assert(src->is_valid(), "src must be valid"); + visitor->do_input(src); + } + + NOT_PRODUCT(virtual void print_name(outputStream* out) const { out->print("MMTkC1PreBarrierStub"); }); }; class MMTkUnlogBitBarrierSetC2: public MMTkBarrierSetC2 {}; diff --git a/openjdk/mmtkBarrierSetAssembler_x86.hpp b/openjdk/mmtkBarrierSetAssembler_x86.hpp index 32e43ebd..5c55b75d 100644 --- a/openjdk/mmtkBarrierSetAssembler_x86.hpp +++ b/openjdk/mmtkBarrierSetAssembler_x86.hpp @@ -5,8 +5,6 @@ #include "gc/shared/barrierSetAssembler.hpp" class MMTkBarrierSetC1; -class MMTkC1PreBarrierStub; -class MMTkC1PostBarrierStub; class MMTkC1ReferenceLoadBarrierStub; class LIR_Assembler; class StubAssembler; @@ -45,9 +43,6 @@ class MMTkBarrierSetAssembler: public BarrierSetAssembler { if (type == T_OBJECT || type == T_ARRAY) object_reference_write_post(masm, decorators, dst, val, tmp1, tmp2, true); } - /// Generate C1 write barrier slow-call stub - virtual void generate_c1_pre_write_barrier_stub(LIR_Assembler* ce, MMTkC1PreBarrierStub* stub) const {}; - virtual void generate_c1_post_write_barrier_stub(LIR_Assembler* ce, MMTkC1PostBarrierStub* stub) const {}; static void generate_c1_ref_load_barrier_stub_call(LIR_Assembler* ce, MMTkC1ReferenceLoadBarrierStub* stub); }; #endif // MMTK_OPENJDK_MMTK_BARRIER_SET_ASSEMBLER_X86_HPP diff --git a/openjdk/mmtkBarrierSetC1.cpp b/openjdk/mmtkBarrierSetC1.cpp index 1d9a7b84..f6e9c657 100644 --- a/openjdk/mmtkBarrierSetC1.cpp +++ b/openjdk/mmtkBarrierSetC1.cpp @@ -38,16 +38,6 @@ void MMTkBarrierSetC1::generate_c1_runtime_stubs(BufferBlob* buffer_blob) { _ref_load_barrier_c1_runtime_code_blob = Runtime1::generate_blob(buffer_blob, -1, "load_code_gen_cl", false, &load_code_gen_cl); } -void MMTkC1PostBarrierStub::emit_code(LIR_Assembler* ce) { - MMTkBarrierSetAssembler* bs = (MMTkBarrierSetAssembler*) BarrierSet::barrier_set()->barrier_set_assembler(); - bs->generate_c1_post_write_barrier_stub(ce, this); -} - -void MMTkC1PreBarrierStub::emit_code(LIR_Assembler* ce) { - MMTkBarrierSetAssembler* bs = (MMTkBarrierSetAssembler*) BarrierSet::barrier_set()->barrier_set_assembler(); - bs->generate_c1_pre_write_barrier_stub(ce, this); -} - void MMTkC1ReferenceLoadBarrierStub::emit_code(LIR_Assembler* ce) { MMTkBarrierSetAssembler* bs = (MMTkBarrierSetAssembler*) BarrierSet::barrier_set()->barrier_set_assembler(); bs->generate_c1_ref_load_barrier_stub_call(ce, this); diff --git a/openjdk/mmtkBarrierSetC1.hpp b/openjdk/mmtkBarrierSetC1.hpp index e0d4c6c9..2653cae9 100644 --- a/openjdk/mmtkBarrierSetC1.hpp +++ b/openjdk/mmtkBarrierSetC1.hpp @@ -71,47 +71,6 @@ class MMTkBarrierSetC1 : public BarrierSetC1 { virtual void generate_c1_runtime_stubs(BufferBlob* buffer_blob) override; }; -/// C1 pre write barrier slow-call stub. -/// The default behaviour is to call `MMTkBarrierSetRuntime::object_reference_write_pre_call` and pass all the three args. -/// Barrier implementations may inherit from this class, and override `emit_code` to perform a specialized slow-path call. -struct MMTkC1PreBarrierStub: CodeStub { - LIR_Opr src; - - MMTkC1PreBarrierStub(LIR_Opr src): src(src) {} - - virtual void emit_code(LIR_Assembler* ce) override; - - virtual void visit(LIR_OpVisitState* visitor) override { - visitor->do_slow_case(); - assert(src->is_valid(), "src must be valid"); - visitor->do_input(src); - } - - NOT_PRODUCT(virtual void print_name(outputStream* out) const { out->print("MMTkC1PreBarrierStub"); }); -}; - -/// C1 post write barrier slow-call stub. -/// The default behaviour is to call `MMTkBarrierSetRuntime::object_reference_write_post_call` and pass all the three args. -/// Barrier implementations may inherit from this class, and override `emit_code` to perform a specialized slow-path call. -struct MMTkC1PostBarrierStub: CodeStub { - LIR_Opr src, slot, new_val; - - MMTkC1PostBarrierStub(LIR_Opr src, LIR_Opr slot, LIR_Opr new_val): src(src), slot(slot), new_val(new_val) {} - - virtual void emit_code(LIR_Assembler* ce) override; - - virtual void visit(LIR_OpVisitState* visitor) override { - visitor->do_slow_case(); - assert(src->is_valid(), "src must be valid"); - visitor->do_input(src); - // Some post barrier stubs (such as object barrier and SATB barrier) don't use slot or new_val. - if (slot->is_valid()) visitor->do_input(slot); - if (slot->is_valid()) visitor->do_input(new_val); - } - - NOT_PRODUCT(virtual void print_name(outputStream* out) const { out->print("MMTkC1PostBarrierStub"); }); -}; - struct MMTkC1ReferenceLoadBarrierStub: CodeStub { LIR_Opr val; CodeEmitInfo* info; // Code patching info From fbdba71cc7707c72dbc9d9729b4fff35e2a741b4 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Mon, 22 Sep 2025 00:30:45 +0800 Subject: [PATCH 20/46] Comments --- openjdk/barriers/mmtkUnlogBitBarrier.cpp | 6 ++++++ openjdk/barriers/mmtkUnlogBitBarrier.hpp | 22 +++++++++++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/openjdk/barriers/mmtkUnlogBitBarrier.cpp b/openjdk/barriers/mmtkUnlogBitBarrier.cpp index beab4cac..8d9eb20c 100644 --- a/openjdk/barriers/mmtkUnlogBitBarrier.cpp +++ b/openjdk/barriers/mmtkUnlogBitBarrier.cpp @@ -5,6 +5,8 @@ #include "c1/c1_LIRAssembler.hpp" #include "c1/c1_MacroAssembler.hpp" +//////////////////// Assembler //////////////////// + #define __ masm-> void MMTkUnlogBitBarrierSetAssembler::emit_check_unlog_bit_fast_path(MacroAssembler* masm, Label &done, Register obj, Register tmp1, Register tmp2) { @@ -84,6 +86,8 @@ void MMTkUnlogBitBarrierSetAssembler::object_reference_write_pre_or_post(MacroAs #undef __ +//////////////////// Assembler/C1 //////////////////// + #define __ ce->masm()-> void MMTkUnlogBitBarrierSetAssembler::generate_c1_unlog_bit_barrier_slow_path_stub(LIR_Assembler* ce, MMTkC1UnlogBitBarrierSlowPathStub* stub) const { @@ -105,6 +109,8 @@ void MMTkC1UnlogBitBarrierSlowPathStub::emit_code(LIR_Assembler* ce) { #undef __ +//////////////////// C1 //////////////////// + #ifdef ASSERT #define __ gen->lir(__FILE__, __LINE__)-> #else diff --git a/openjdk/barriers/mmtkUnlogBitBarrier.hpp b/openjdk/barriers/mmtkUnlogBitBarrier.hpp index 4d693afc..f28853b9 100644 --- a/openjdk/barriers/mmtkUnlogBitBarrier.hpp +++ b/openjdk/barriers/mmtkUnlogBitBarrier.hpp @@ -12,6 +12,8 @@ struct MMTkC1UnlogBitBarrierSlowPathStub; const uintptr_t UNLOG_BIT_BASE_ADDRESS = GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS; +//////////////////// Runtime //////////////////// + class MMTkUnlogBitBarrierSetRuntime: public MMTkBarrierSetRuntime { protected: static bool is_unlog_bit_set(oop obj) { @@ -23,6 +25,8 @@ class MMTkUnlogBitBarrierSetRuntime: public MMTkBarrierSetRuntime { } }; +//////////////////// Assembler //////////////////// + class MMTkUnlogBitBarrierSetAssembler: public MMTkBarrierSetAssembler { protected: static void emit_check_unlog_bit_fast_path(MacroAssembler* masm, Label &done, Register obj, Register tmp1, Register tmp2); @@ -33,15 +37,25 @@ class MMTkUnlogBitBarrierSetAssembler: public MMTkBarrierSetAssembler { void generate_c1_unlog_bit_barrier_slow_path_stub(LIR_Assembler* ce, MMTkC1UnlogBitBarrierSlowPathStub* stub) const; }; +//////////////////// C1 //////////////////// + class MMTkUnlogBitBarrierSetC1: public MMTkBarrierSetC1 { protected: static void emit_check_unlog_bit_fast_path(LIRGenerator* gen, LIR_Opr addr, CodeStub* slow); static void object_reference_write_pre_or_post(LIRAccess& access, bool pre); }; -/// C1 pre write barrier slow-call stub. -/// The default behaviour is to call `MMTkBarrierSetRuntime::object_reference_write_pre_call` and pass all the three args. -/// Barrier implementations may inherit from this class, and override `emit_code` to perform a specialized slow-path call. +/// C1 write barrier slow path stub. +/// +/// This stub calls `MMTkBarrierSetRuntime::object_reference_write_{slow,pre,post}_call` depending +/// on whether barrier fast paths are enabled and whether it is pre or post barrier, passing the +/// `src` argument, and leaving other arguments as nullptr. This is enough for object-remembering +/// barriers based on the unlog bit, including the ObjectBarrier and the SATBBarrier, because only +/// the `src` argument is significant. +/// +/// Note that this stub cannot be generalized to field-remembering barriers as it does not pass the +/// field or the old/new values. Field-remembering barriers should implement their own slow-path +/// stub(s). struct MMTkC1UnlogBitBarrierSlowPathStub: CodeStub { LIR_Opr src; bool pre; @@ -59,6 +73,8 @@ struct MMTkC1UnlogBitBarrierSlowPathStub: CodeStub { NOT_PRODUCT(virtual void print_name(outputStream* out) const { out->print("MMTkC1PreBarrierStub"); }); }; +//////////////////// C2 //////////////////// + class MMTkUnlogBitBarrierSetC2: public MMTkBarrierSetC2 {}; #endif // MMTK_OPENJDK_BARRIERS_MMTK_UNLOG_BIT_BARRIER_HPP From 24b82ab1abf9ef36ed47cc352fc47021558cfdbd Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Mon, 22 Sep 2025 10:16:03 +0800 Subject: [PATCH 21/46] Revert Cargo.toml to online mmtk-core repo --- mmtk/Cargo.lock | 22 ++++++++++++---------- mmtk/Cargo.toml | 3 +-- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index b7b3c7c6..6025343c 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -129,9 +129,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.37" +version = "1.2.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65193589c6404eb80b450d618eaf9a2cafaaafd57ecce47370519ef674a7bd44" +checksum = "80f41ae168f955c12fb8960b057d70d0ca153fb83182b57d86380443527be7e9" dependencies = [ "find-msvc-tools", "jobserver", @@ -281,9 +281,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fd99930f64d146689264c637b5af2f0233a933bef0d8570e2526bf9e083192d" +checksum = "1ced73b1dacfc750a6db6c0a0c3a3853c8b41997e2e2c563dc90804ae6867959" [[package]] name = "form_urlencoded" @@ -492,6 +492,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.31.0" +source = "git+https://github.com/mmtk/mmtk-core.git?rev=27b4521ca73c2addbafe7ccf374c607d61f3ba51#27b4521ca73c2addbafe7ccf374c607d61f3ba51" dependencies = [ "atomic", "atomic-traits", @@ -530,6 +531,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.31.0" +source = "git+https://github.com/mmtk/mmtk-core.git?rev=27b4521ca73c2addbafe7ccf374c607d61f3ba51#27b4521ca73c2addbafe7ccf374c607d61f3ba51" dependencies = [ "proc-macro-error", "proc-macro2", @@ -752,9 +754,9 @@ checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" [[package]] name = "serde" -version = "1.0.224" +version = "1.0.226" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6aaeb1e94f53b16384af593c71e20b095e958dab1d26939c1b70645c5cfbcc0b" +checksum = "0dca6411025b24b60bfa7ec1fe1f8e710ac09782dca409ee8237ba74b51295fd" dependencies = [ "serde_core", "serde_derive", @@ -762,18 +764,18 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.224" +version = "1.0.226" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32f39390fa6346e24defbcdd3d9544ba8a19985d0af74df8501fbfe9a64341ab" +checksum = "ba2ba63999edb9dac981fb34b3e5c0d111a69b0924e253ed29d83f7c99e966a4" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.224" +version = "1.0.226" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ff78ab5e8561c9a675bfc1785cb07ae721f0ee53329a595cefd8c04c2ac4e0" +checksum = "8db53ae22f34573731bafa1db20f04027b2d25e02d8205921b569171699cdb33" dependencies = [ "proc-macro2", "quote", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 135dc232..21eab63d 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -36,10 +36,9 @@ probe = "0.5" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -#mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "a4dd70cb70a116a32b1bbb20501c48f77f49181b" } +mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "27b4521ca73c2addbafe7ccf374c607d61f3ba51" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } -mmtk = { path = "../../../../../mmtk-core" } [build-dependencies] built = { version = "0.7.7", features = ["git2"] } From 5969878be2398d5eb2b40bcddfaf05a79c7627aa Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Mon, 22 Sep 2025 10:41:34 +0800 Subject: [PATCH 22/46] Remove commented-out hack --- openjdk/barriers/mmtkSATBBarrier.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/openjdk/barriers/mmtkSATBBarrier.cpp b/openjdk/barriers/mmtkSATBBarrier.cpp index 3384d569..70ac21f1 100644 --- a/openjdk/barriers/mmtkSATBBarrier.cpp +++ b/openjdk/barriers/mmtkSATBBarrier.cpp @@ -1,6 +1,3 @@ -// Workaround the fact that LIR_Assembler::as_Address is private. -//#define private public - #include "precompiled.hpp" #include "mmtkSATBBarrier.hpp" #include "runtime/interfaceSupport.inline.hpp" From 684bfa3923408da8a6a471b908913beda943d815 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Mon, 22 Sep 2025 12:43:48 +0800 Subject: [PATCH 23/46] Update comments to link to bug tracker --- openjdk/barriers/mmtkUnlogBitBarrier.cpp | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/openjdk/barriers/mmtkUnlogBitBarrier.cpp b/openjdk/barriers/mmtkUnlogBitBarrier.cpp index 8d9eb20c..0ecb7bef 100644 --- a/openjdk/barriers/mmtkUnlogBitBarrier.cpp +++ b/openjdk/barriers/mmtkUnlogBitBarrier.cpp @@ -10,6 +10,8 @@ #define __ masm-> void MMTkUnlogBitBarrierSetAssembler::emit_check_unlog_bit_fast_path(MacroAssembler* masm, Label &done, Register obj, Register tmp1, Register tmp2) { + // Note that `tmp1` and `tmp2` are actual temporary registers available for use, + // not the `tmp1` and `tmp2` from `store_at`. assert_different_registers(obj, tmp1, tmp2); // tmp2 = load-byte (UNLOG_BIT_BASE_ADDRESS + (obj >> 6)); @@ -38,13 +40,16 @@ void MMTkUnlogBitBarrierSetAssembler::object_reference_write_pre_or_post(MacroAs Label done; Register obj = dst.base(); if (mmtk_enable_barrier_fastpath) { - // For some instructions in the template table, such as aastore, - // we observed that in BarrierSetAssembler::store_at - // which calls `object_reference_write_pre` or `object_reference_write_post`, - // dst.base() == tmp1 && dst.index() == tmp2. + // For some instructions in the template table, + // `tmp1` and `tmp2` may overlap with `dst`. + // For example, if the instruction is `aastore`, + // then `dst.base() == tmp1 && dst.index() == tmp2`. + // This is a bug that has been fixed upstream in OpenJDK 21. + // See https://bugs.openjdk.org/browse/JDK-8301371 + // // We can't overwrite those registers, - // so we don't use tmp1 or tmp2 passed to store_at. - // Instead, we steal two scratch register to use. + // so we steal two scratch register + // and assert they don't overlap with other registers. Register tmp3 = rscratch1; Register tmp4 = rscratch2; assert_different_registers(dst.base(), dst.index(), val, tmp3, tmp4); From 244325210c9d9d3090d654322424d1a017e139da Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Mon, 22 Sep 2025 16:01:14 +0800 Subject: [PATCH 24/46] Comments and spacing --- openjdk/barriers/mmtkUnlogBitBarrier.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/openjdk/barriers/mmtkUnlogBitBarrier.cpp b/openjdk/barriers/mmtkUnlogBitBarrier.cpp index 0ecb7bef..baa7a06f 100644 --- a/openjdk/barriers/mmtkUnlogBitBarrier.cpp +++ b/openjdk/barriers/mmtkUnlogBitBarrier.cpp @@ -138,22 +138,25 @@ void MMTkUnlogBitBarrierSetC1::emit_check_unlog_bit_fast_path(LIRGenerator* gen, // registers. LIR_Opr addr = gen->new_register(T_OBJECT); __ move(src, addr); - // uint8_t* meta_addr = (uint8_t*) (side_metadata_base_address() + (addr >> 6)); + + // uint8_t* meta_addr = (uint8_t*) (UNLOG_BIT_BASE_ADDRESS + (addr >> 6)); LIR_Opr offset = gen->new_pointer_register(); __ move(addr, offset); __ unsigned_shift_right(offset, 6, offset); LIR_Opr base = gen->new_pointer_register(); __ move(LIR_OprFact::longConst(UNLOG_BIT_BASE_ADDRESS), base); LIR_Address* meta_addr = new LIR_Address(base, offset, T_BYTE); + // uint8_t byte_val = *meta_addr; LIR_Opr byte_val = gen->new_register(T_INT); __ move(meta_addr, byte_val); - // intptr_t shift = (addr >> 3) & 0b111; + // uint32_t shift = ((uint32_t)addr >> 3) & 0b111; LIR_Opr shift = gen->new_register(T_INT); __ move(addr, shift); __ unsigned_shift_right(shift, 3, shift); __ logical_and(shift, LIR_OprFact::intConst(0b111), shift); + // if (((byte_val >> shift) & 1) == 1) slow; LIR_Opr result = byte_val; __ unsigned_shift_right(result, shift, result, LIR_OprFact::illegalOpr); From 4c2eddb7b2a6b3a4c424268cd0ab5a7f5a0b839f Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Mon, 22 Sep 2025 16:52:45 +0800 Subject: [PATCH 25/46] Ensure all cpp files include precompiled.hpp This is the basic rule for using pre-compiled headers. --- openjdk/mmtkBarrierSetC1.cpp | 1 + openjdk/mmtkBarrierSetC1.hpp | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/openjdk/mmtkBarrierSetC1.cpp b/openjdk/mmtkBarrierSetC1.cpp index f6e9c657..8badb651 100644 --- a/openjdk/mmtkBarrierSetC1.cpp +++ b/openjdk/mmtkBarrierSetC1.cpp @@ -1,3 +1,4 @@ +#include "precompiled.hpp" #include "c1/c1_CodeStubs.hpp" #include "gc/shared/c1/barrierSetC1.hpp" #include "mmtkBarrierSetAssembler_x86.hpp" diff --git a/openjdk/mmtkBarrierSetC1.hpp b/openjdk/mmtkBarrierSetC1.hpp index 2653cae9..b45ffaf0 100644 --- a/openjdk/mmtkBarrierSetC1.hpp +++ b/openjdk/mmtkBarrierSetC1.hpp @@ -8,6 +8,11 @@ class MMTkBarrierSetAssembler; class MMTkBarrierSetC1 : public BarrierSetC1 { private: + // Code blobs for runtime functions. + // Here we have one code blob for every runtime function we call, + // i.e. MMTkBarrierSetRuntime::*_call + // There is no general rules for this in OpenJDK, + // except that these code blobs are CodeBlob* _pre_barrier_c1_runtime_code_blob; CodeBlob* _post_barrier_c1_runtime_code_blob; CodeBlob* _ref_load_barrier_c1_runtime_code_blob; From e0ae0bcf8490dee2a53ae3dbc4342f6726e95efa Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Mon, 22 Sep 2025 18:22:32 +0800 Subject: [PATCH 26/46] One runtime code blob per function --- openjdk/barriers/mmtkUnlogBitBarrier.cpp | 7 ++-- openjdk/barriers/mmtkUnlogBitBarrier.hpp | 4 +- openjdk/mmtkBarrierSetAssembler_x86.cpp | 53 +++++++++--------------- openjdk/mmtkBarrierSetAssembler_x86.hpp | 21 ++++++---- openjdk/mmtkBarrierSetC1.cpp | 39 +++++++---------- openjdk/mmtkBarrierSetC1.hpp | 33 ++++++++------- 6 files changed, 73 insertions(+), 84 deletions(-) diff --git a/openjdk/barriers/mmtkUnlogBitBarrier.cpp b/openjdk/barriers/mmtkUnlogBitBarrier.cpp index baa7a06f..a3abcc6c 100644 --- a/openjdk/barriers/mmtkUnlogBitBarrier.cpp +++ b/openjdk/barriers/mmtkUnlogBitBarrier.cpp @@ -101,8 +101,9 @@ void MMTkUnlogBitBarrierSetAssembler::generate_c1_unlog_bit_barrier_slow_path_st ce->store_parameter(stub->src->as_pointer_register(), 0); ce->store_parameter(0, 1); ce->store_parameter(0, 2); - CodeBlob* code_blob = stub->pre ? bs->pre_barrier_c1_runtime_code_blob() - : bs->post_barrier_c1_runtime_code_blob(); + CodeBlob* code_blob = stub->fast_path_enabled ? bs->object_reference_write_slow_c1_runtime_code_blob() + : stub->pre ? bs->object_reference_write_pre_c1_runtime_code_blob() + : bs->object_reference_write_post_c1_runtime_code_blob(); __ call(RuntimeAddress(code_blob->code_begin())); __ jmp(*stub->continuation()); } @@ -172,7 +173,7 @@ void MMTkUnlogBitBarrierSetC1::object_reference_write_pre_or_post(LIRAccess& acc LIR_Opr src = access.base().opr(); - CodeStub* slow = new MMTkC1UnlogBitBarrierSlowPathStub(src, pre); + CodeStub* slow = new MMTkC1UnlogBitBarrierSlowPathStub(src, mmtk_enable_barrier_fastpath, pre); if (mmtk_enable_barrier_fastpath) { emit_check_unlog_bit_fast_path(gen, src, slow); diff --git a/openjdk/barriers/mmtkUnlogBitBarrier.hpp b/openjdk/barriers/mmtkUnlogBitBarrier.hpp index f28853b9..01a5b9dc 100644 --- a/openjdk/barriers/mmtkUnlogBitBarrier.hpp +++ b/openjdk/barriers/mmtkUnlogBitBarrier.hpp @@ -58,9 +58,11 @@ class MMTkUnlogBitBarrierSetC1: public MMTkBarrierSetC1 { /// stub(s). struct MMTkC1UnlogBitBarrierSlowPathStub: CodeStub { LIR_Opr src; + bool fast_path_enabled; bool pre; - MMTkC1UnlogBitBarrierSlowPathStub(LIR_Opr src, bool pre): src(src), pre(pre) {} + MMTkC1UnlogBitBarrierSlowPathStub(LIR_Opr src, bool fast_path_enabled, bool pre): + fast_path_enabled(fast_path_enabled), src(src), pre(pre) {} virtual void emit_code(LIR_Assembler* ce) override; diff --git a/openjdk/mmtkBarrierSetAssembler_x86.cpp b/openjdk/mmtkBarrierSetAssembler_x86.cpp index 01857a91..94145562 100644 --- a/openjdk/mmtkBarrierSetAssembler_x86.cpp +++ b/openjdk/mmtkBarrierSetAssembler_x86.cpp @@ -142,6 +142,10 @@ void MMTkBarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register threa #undef __ +//////////////////// Assembler for C1 //////////////////// + +// Generate code stubs + #define __ ce->masm()-> void MMTkBarrierSetAssembler::generate_c1_ref_load_barrier_stub_call(LIR_Assembler* ce, MMTkC1ReferenceLoadBarrierStub* stub) { @@ -155,16 +159,18 @@ void MMTkBarrierSetAssembler::generate_c1_ref_load_barrier_stub_call(LIR_Assembl __ cmpptr(val_reg, (int32_t) NULL_WORD); __ jcc(Assembler::equal, *stub->continuation()); ce->store_parameter(stub->val->as_register(), 0); - __ call(RuntimeAddress(bs->ref_load_barrier_c1_runtime_code_blob()->code_begin())); + __ call(RuntimeAddress(bs->load_reference_c1_runtime_code_blob()->code_begin())); __ jmp(*stub->continuation()); } #undef __ +// Generate runtime stubs for the "runtime code blobs" in MMTkBarrierSetC1 + #define __ sasm-> -void MMTkBarrierSetAssembler::generate_c1_pre_or_post_write_barrier_runtime_stub(StubAssembler* sasm, bool pre) { - __ prologue(pre ? "mmtk_pre_write_barrier" : "mmtk_post_write_barrier", false); +void MMTkBarrierSetAssembler::generate_c1_runtime_stub_general(StubAssembler* sasm, const char* name, address entry_point, int argc) { + __ prologue(name, false); Label done, runtime; @@ -181,10 +187,6 @@ void MMTkBarrierSetAssembler::generate_c1_pre_or_post_write_barrier_runtime_stub __ save_live_registers_no_oop_map(true); - address entry_point = mmtk_enable_barrier_fastpath ? FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call) - : pre ? FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call) - : FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call); - __ call_VM_leaf_base(entry_point, 3); __ restore_live_registers(true); @@ -198,35 +200,20 @@ void MMTkBarrierSetAssembler::generate_c1_pre_or_post_write_barrier_runtime_stub __ epilogue(); } -#undef __ - -#define __ sasm-> - -void MMTkBarrierSetAssembler::generate_c1_ref_load_barrier_runtime_stub(StubAssembler* sasm) const { - __ prologue("mmtk_ref_load_barrier", false); - - // Address store_addr(rbp, 2*BytesPerWord); - - Label done, runtime; - - __ push(c_rarg0); - __ push(rax); - - __ load_parameter(0, c_rarg0); - - __ bind(runtime); - - __ save_live_registers_no_oop_map(true); - - __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), 1); +void MMTkBarrierSetAssembler::generate_c1_load_reference_runtime_stub(StubAssembler* sasm) { + generate_c1_runtime_stub_general(sasm, "c1_load_reference_runtime_stub", FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), 1); +} - __ restore_live_registers(true); +void MMTkBarrierSetAssembler::generate_c1_object_reference_write_pre_runtime_stub(StubAssembler* sasm) { + generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_pre_stub", FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call), 3); +} - __ bind(done); - __ pop(rax); - __ pop(c_rarg0); +void MMTkBarrierSetAssembler::generate_c1_object_reference_write_post_runtime_stub(StubAssembler* sasm) { + generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_post_stub", FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call), 3); +} - __ epilogue(); +void MMTkBarrierSetAssembler::generate_c1_object_reference_write_slow_runtime_stub(StubAssembler* sasm) { + generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_slow_stub", FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), 3); } #undef __ diff --git a/openjdk/mmtkBarrierSetAssembler_x86.hpp b/openjdk/mmtkBarrierSetAssembler_x86.hpp index 5c55b75d..7f068c8b 100644 --- a/openjdk/mmtkBarrierSetAssembler_x86.hpp +++ b/openjdk/mmtkBarrierSetAssembler_x86.hpp @@ -10,14 +10,6 @@ class LIR_Assembler; class StubAssembler; class MMTkBarrierSetAssembler: public BarrierSetAssembler { - friend class MMTkBarrierSetC1; - -private: - /// Generate C1 pre or post write barrier slow-call assembly code - void generate_c1_pre_or_post_write_barrier_runtime_stub(StubAssembler* sasm, bool pre); - /// Generate C1 weak reference load slow-call assembly code - void generate_c1_ref_load_barrier_runtime_stub(StubAssembler* sasm) const; - protected: /// Full pre-barrier virtual void object_reference_write_pre(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2) const {} @@ -43,6 +35,19 @@ class MMTkBarrierSetAssembler: public BarrierSetAssembler { if (type == T_OBJECT || type == T_ARRAY) object_reference_write_post(masm, decorators, dst, val, tmp1, tmp2, true); } + //////////////////// Assembler for C1 //////////////////// + + // Generate code stubs +public: static void generate_c1_ref_load_barrier_stub_call(LIR_Assembler* ce, MMTkC1ReferenceLoadBarrierStub* stub); + + // Generate runtime stubs for the "runtime code blobs" in MMTkBarrierSetC1 +private: + static void generate_c1_runtime_stub_general(StubAssembler* sasm, const char* name, address func, int argc); +public: + static void generate_c1_load_reference_runtime_stub(StubAssembler* sasm); + static void generate_c1_object_reference_write_pre_runtime_stub(StubAssembler* sasm); + static void generate_c1_object_reference_write_post_runtime_stub(StubAssembler* sasm); + static void generate_c1_object_reference_write_slow_runtime_stub(StubAssembler* sasm); }; #endif // MMTK_OPENJDK_MMTK_BARRIER_SET_ASSEMBLER_X86_HPP diff --git a/openjdk/mmtkBarrierSetC1.cpp b/openjdk/mmtkBarrierSetC1.cpp index 8badb651..9d2dbdd9 100644 --- a/openjdk/mmtkBarrierSetC1.cpp +++ b/openjdk/mmtkBarrierSetC1.cpp @@ -5,38 +5,27 @@ #include "mmtkBarrierSetC1.hpp" void MMTkBarrierSetC1::generate_c1_runtime_stubs(BufferBlob* buffer_blob) { - class MMTkPreBarrierCodeGenClosure : public StubAssemblerCodeGenClosure { - virtual OopMapSet* generate_code(StubAssembler* sasm) override { - MMTkBarrierSetAssembler* bs = (MMTkBarrierSetAssembler*) BarrierSet::barrier_set()->barrier_set_assembler(); - bs->generate_c1_pre_or_post_write_barrier_runtime_stub(sasm, true); - return NULL; - } - }; + using GenStubFunc = void(*)(StubAssembler*); + class RuntimeCodeBlobCodeGenClosure : public StubAssemblerCodeGenClosure { + GenStubFunc gen_stub; + public: + RuntimeCodeBlobCodeGenClosure(GenStubFunc gen_stub): gen_stub(gen_stub) {} - class MMTkPostBarrierCodeGenClosure : public StubAssemblerCodeGenClosure { virtual OopMapSet* generate_code(StubAssembler* sasm) override { - MMTkBarrierSetAssembler* bs = (MMTkBarrierSetAssembler*) BarrierSet::barrier_set()->barrier_set_assembler(); - bs->generate_c1_pre_or_post_write_barrier_runtime_stub(sasm, false); - return NULL; + gen_stub(sasm); + return nullptr; } }; - class MMTkRefLoadBarrierCodeGenClosure : public StubAssemblerCodeGenClosure { - virtual OopMapSet* generate_code(StubAssembler* sasm) override { - MMTkBarrierSetAssembler* bs = (MMTkBarrierSetAssembler*) BarrierSet::barrier_set()->barrier_set_assembler(); - bs->generate_c1_ref_load_barrier_runtime_stub(sasm); - return NULL; - } + auto do_code_blob = [buffer_blob](const char* name, GenStubFunc gen_stub) { + RuntimeCodeBlobCodeGenClosure closure(gen_stub); + return Runtime1::generate_blob(buffer_blob, -1, name, false, &closure); }; - MMTkPreBarrierCodeGenClosure pre_write_code_gen_cl; - _pre_barrier_c1_runtime_code_blob = Runtime1::generate_blob(buffer_blob, -1, "mmtk_pre_write_code_gen_cl", false, &pre_write_code_gen_cl); - - MMTkPostBarrierCodeGenClosure post_write_code_gen_cl; - _post_barrier_c1_runtime_code_blob = Runtime1::generate_blob(buffer_blob, -1, "mmtk_post_write_code_gen_cl", false, &post_write_code_gen_cl); - - MMTkRefLoadBarrierCodeGenClosure load_code_gen_cl; - _ref_load_barrier_c1_runtime_code_blob = Runtime1::generate_blob(buffer_blob, -1, "load_code_gen_cl", false, &load_code_gen_cl); + _load_reference_c1_runtime_code_blob = do_code_blob("load_reference", &MMTkBarrierSetAssembler::generate_c1_load_reference_runtime_stub); + _object_reference_write_pre_c1_runtime_code_blob = do_code_blob("object_reference_write_pre", &MMTkBarrierSetAssembler::generate_c1_object_reference_write_pre_runtime_stub); + _object_reference_write_post_c1_runtime_code_blob = do_code_blob("object_reference_write_post", &MMTkBarrierSetAssembler::generate_c1_object_reference_write_post_runtime_stub); + _object_reference_write_slow_c1_runtime_code_blob = do_code_blob("object_reference_write_slow", &MMTkBarrierSetAssembler::generate_c1_object_reference_write_slow_runtime_stub); } void MMTkC1ReferenceLoadBarrierStub::emit_code(LIR_Assembler* ce) { diff --git a/openjdk/mmtkBarrierSetC1.hpp b/openjdk/mmtkBarrierSetC1.hpp index b45ffaf0..546e74de 100644 --- a/openjdk/mmtkBarrierSetC1.hpp +++ b/openjdk/mmtkBarrierSetC1.hpp @@ -8,14 +8,20 @@ class MMTkBarrierSetAssembler; class MMTkBarrierSetC1 : public BarrierSetC1 { private: - // Code blobs for runtime functions. - // Here we have one code blob for every runtime function we call, - // i.e. MMTkBarrierSetRuntime::*_call - // There is no general rules for this in OpenJDK, - // except that these code blobs are - CodeBlob* _pre_barrier_c1_runtime_code_blob; - CodeBlob* _post_barrier_c1_runtime_code_blob; - CodeBlob* _ref_load_barrier_c1_runtime_code_blob; + // Code blobs for calling into runtime functions. + // Here in MMTkBarrierSetC1, + // we have one "runtime code blob" for every runtime function we call, + // i.e. `MMTkBarrierSetRuntime::*_call` + // There is no general rules that enfoce this in OpenJDK, + // except that these code blobs are global and implemented in machine-specific assembly. + // Our barrier slow paths are relatively simple, i.e. calling into MMTk-core. + // So we only need such "runtime code blobs" for calling MMTk functions. + // If we want to implement medium paths in machine-specific ways, + // we may consider defining new code blobs for specific barriers. + CodeBlob* _load_reference_c1_runtime_code_blob; + CodeBlob* _object_reference_write_pre_c1_runtime_code_blob; + CodeBlob* _object_reference_write_post_c1_runtime_code_blob; + CodeBlob* _object_reference_write_slow_c1_runtime_code_blob; protected: /// Full pre-barrier @@ -64,13 +70,12 @@ class MMTkBarrierSetC1 : public BarrierSetC1 { public: - MMTkBarrierSetC1() - : _pre_barrier_c1_runtime_code_blob(NULL), - _post_barrier_c1_runtime_code_blob(NULL) {} + MMTkBarrierSetC1() {} - CodeBlob* pre_barrier_c1_runtime_code_blob() { return _pre_barrier_c1_runtime_code_blob; } - CodeBlob* post_barrier_c1_runtime_code_blob() { return _post_barrier_c1_runtime_code_blob; } - CodeBlob* ref_load_barrier_c1_runtime_code_blob() { return _ref_load_barrier_c1_runtime_code_blob; } + CodeBlob* load_reference_c1_runtime_code_blob() { return _load_reference_c1_runtime_code_blob; } + CodeBlob* object_reference_write_pre_c1_runtime_code_blob() { return _object_reference_write_pre_c1_runtime_code_blob; } + CodeBlob* object_reference_write_post_c1_runtime_code_blob() { return _object_reference_write_post_c1_runtime_code_blob; } + CodeBlob* object_reference_write_slow_c1_runtime_code_blob() { return _object_reference_write_slow_c1_runtime_code_blob; } /// Generate C1 write barrier slow-call C1-LIR code virtual void generate_c1_runtime_stubs(BufferBlob* buffer_blob) override; From 4abf58a42cce8c2b25c739ace910ac0de7e6a77f Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Mon, 22 Sep 2025 18:26:31 +0800 Subject: [PATCH 27/46] Why do we push registers when we can save_live_registers_no_oop_map? --- openjdk/mmtkBarrierSetAssembler_x86.cpp | 28 +++++++------------------ 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/openjdk/mmtkBarrierSetAssembler_x86.cpp b/openjdk/mmtkBarrierSetAssembler_x86.cpp index 94145562..de235c59 100644 --- a/openjdk/mmtkBarrierSetAssembler_x86.cpp +++ b/openjdk/mmtkBarrierSetAssembler_x86.cpp @@ -171,32 +171,18 @@ void MMTkBarrierSetAssembler::generate_c1_ref_load_barrier_stub_call(LIR_Assembl void MMTkBarrierSetAssembler::generate_c1_runtime_stub_general(StubAssembler* sasm, const char* name, address entry_point, int argc) { __ prologue(name, false); - - Label done, runtime; - - __ push(c_rarg0); - __ push(c_rarg1); - __ push(c_rarg2); - __ push(rax); - - __ load_parameter(0, c_rarg0); - __ load_parameter(1, c_rarg1); - __ load_parameter(2, c_rarg2); - - __ bind(runtime); - __ save_live_registers_no_oop_map(true); + if (argc > 0) __ load_parameter(0, c_rarg0); + if (argc > 1) __ load_parameter(1, c_rarg1); + if (argc > 2) __ load_parameter(2, c_rarg2); + if (argc > 3) { + guarantee(false, "Too many args"); + } + __ call_VM_leaf_base(entry_point, 3); __ restore_live_registers(true); - - __ bind(done); - __ pop(rax); - __ pop(c_rarg2); - __ pop(c_rarg1); - __ pop(c_rarg0); - __ epilogue(); } From 0ebac369eaadd509cbac210a1982872471d6f6c5 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Mon, 22 Sep 2025 18:33:44 +0800 Subject: [PATCH 28/46] Name --- openjdk/mmtkBarrierSetC1.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/openjdk/mmtkBarrierSetC1.cpp b/openjdk/mmtkBarrierSetC1.cpp index 9d2dbdd9..2afc1711 100644 --- a/openjdk/mmtkBarrierSetC1.cpp +++ b/openjdk/mmtkBarrierSetC1.cpp @@ -22,10 +22,10 @@ void MMTkBarrierSetC1::generate_c1_runtime_stubs(BufferBlob* buffer_blob) { return Runtime1::generate_blob(buffer_blob, -1, name, false, &closure); }; - _load_reference_c1_runtime_code_blob = do_code_blob("load_reference", &MMTkBarrierSetAssembler::generate_c1_load_reference_runtime_stub); - _object_reference_write_pre_c1_runtime_code_blob = do_code_blob("object_reference_write_pre", &MMTkBarrierSetAssembler::generate_c1_object_reference_write_pre_runtime_stub); - _object_reference_write_post_c1_runtime_code_blob = do_code_blob("object_reference_write_post", &MMTkBarrierSetAssembler::generate_c1_object_reference_write_post_runtime_stub); - _object_reference_write_slow_c1_runtime_code_blob = do_code_blob("object_reference_write_slow", &MMTkBarrierSetAssembler::generate_c1_object_reference_write_slow_runtime_stub); + _load_reference_c1_runtime_code_blob = do_code_blob("c1_load_reference_stub", &MMTkBarrierSetAssembler::generate_c1_load_reference_runtime_stub); + _object_reference_write_pre_c1_runtime_code_blob = do_code_blob("c1_object_reference_write_pre_stub", &MMTkBarrierSetAssembler::generate_c1_object_reference_write_pre_runtime_stub); + _object_reference_write_post_c1_runtime_code_blob = do_code_blob("c1_object_reference_write_post_stub", &MMTkBarrierSetAssembler::generate_c1_object_reference_write_post_runtime_stub); + _object_reference_write_slow_c1_runtime_code_blob = do_code_blob("c1_object_reference_write_slow_stub", &MMTkBarrierSetAssembler::generate_c1_object_reference_write_slow_runtime_stub); } void MMTkC1ReferenceLoadBarrierStub::emit_code(LIR_Assembler* ce) { From 8bc5eab1efe65475b60b590bc6360c8263378b4d Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Mon, 22 Sep 2025 23:31:11 +0800 Subject: [PATCH 29/46] Rewrite some assembly in SATB barrier --- openjdk/barriers/mmtkSATBBarrier.cpp | 33 ++++++++++++++++------------ 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/openjdk/barriers/mmtkSATBBarrier.cpp b/openjdk/barriers/mmtkSATBBarrier.cpp index 70ac21f1..c1a2aede 100644 --- a/openjdk/barriers/mmtkSATBBarrier.cpp +++ b/openjdk/barriers/mmtkSATBBarrier.cpp @@ -41,21 +41,27 @@ void MMTkSATBBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet dec bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0; bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0; bool on_reference = on_weak || on_phantom; + BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); + #if SOFT_REFERENCE_LOAD_BARRIER if (on_oop && on_reference) { Label done; + + assert_different_registers(dst, tmp1); + // No slow-call if SATB is not active - Register tmp = rscratch1; - Register tmp2 = rscratch2; - __ movptr(tmp, intptr_t(&CONCURRENT_MARKING_ACTIVE)); - __ xorq(tmp2, tmp2); - __ movb(tmp2, Address(tmp, 0)); - __ cmpptr(tmp2, 1); - __ jcc(Assembler::notEqual, done); - // No slow-call if dst is NULL - __ cmpptr(dst, 0); - __ jcc(Assembler::equal, done); + // intptr_t tmp1_q = CONCURRENT_MARKING_ACTIVE; + __ movptr(tmp1, intptr_t(&CONCURRENT_MARKING_ACTIVE)); + // Load with zero extension to 32 bits. + // uint32_t tmp1_l = (uint32_t)(*(unt8_t*)tmp1_q); + __ movzbl(tmp1, Address(tmp1, 0)); + // if (tmp1_l == 0) goto done; + __ testl(tmp1, tmp1); + __ jcc(Assembler::zero, done); + // if (dst == 0) goto done; + __ testptr(dst, dst); + __ jcc(Assembler::zero, done); // Do slow-call __ pusha(); __ mov(c_rarg0, dst); @@ -78,9 +84,9 @@ void MMTkSATBBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Decor if (type == T_OBJECT || type == T_ARRAY) { Label done; - // // Bailout if count is zero - __ cmpptr(count, 0); - __ jcc(Assembler::equal, done); + // Skip the runtime call if count is zero. + __ testptr(count, count); + __ jcc(Assembler::zero, done); __ pusha(); __ movptr(c_rarg0, src); __ movptr(c_rarg1, dst); @@ -99,7 +105,6 @@ void MMTkSATBBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Decor #define __ gen->lir()-> #endif - void MMTkSATBBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) { DecoratorSet decorators = access.decorators(); bool is_weak = (decorators & ON_WEAK_OOP_REF) != 0; From de68de8e24ac60fe01dc8d6260b14290401c71c3 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Mon, 22 Sep 2025 23:35:15 +0800 Subject: [PATCH 30/46] Remove irrelevant comment. SATB barrier does not have arraycopy epilogue. --- openjdk/barriers/mmtkSATBBarrier.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/openjdk/barriers/mmtkSATBBarrier.cpp b/openjdk/barriers/mmtkSATBBarrier.cpp index c1a2aede..1f4a4824 100644 --- a/openjdk/barriers/mmtkSATBBarrier.cpp +++ b/openjdk/barriers/mmtkSATBBarrier.cpp @@ -78,10 +78,6 @@ void MMTkSATBBarrierSetAssembler::object_reference_write_pre(MacroAssembler* mas } void MMTkSATBBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) { - // `count` or `dst` register values may get overwritten after the array copy, and `arraycopy_epilogue` can receive invalid addresses. - // Save the register values here and restore them in `arraycopy_epilogue`. - // See https://github.com/openjdk/jdk/blob/jdk-11%2B19/src/hotspot/cpu/x86/gc/shared/modRefBarrierSetAssembler_x86.cpp#L37-L50 - if (type == T_OBJECT || type == T_ARRAY) { Label done; // Skip the runtime call if count is zero. From 43de7469baabba0b975a36a46705ba5d0497713a Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Mon, 22 Sep 2025 23:54:24 +0800 Subject: [PATCH 31/46] MMTkC1ReferenceLoadBarrierStub no code emit info --- openjdk/barriers/mmtkSATBBarrier.cpp | 2 +- openjdk/mmtkBarrierSetC1.hpp | 12 ++++-------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/openjdk/barriers/mmtkSATBBarrier.cpp b/openjdk/barriers/mmtkSATBBarrier.cpp index 1f4a4824..95786eb3 100644 --- a/openjdk/barriers/mmtkSATBBarrier.cpp +++ b/openjdk/barriers/mmtkSATBBarrier.cpp @@ -120,7 +120,7 @@ void MMTkSATBBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) { } assert(result->is_register(), "must be"); assert(result->type() == T_OBJECT, "must be an object"); - auto slow = new MMTkC1ReferenceLoadBarrierStub(result, access.patch_emit_info()); + auto slow = new MMTkC1ReferenceLoadBarrierStub(result); // Call slow-path only when concurrent marking is active LIR_Opr cm_flag_addr_opr = gen->new_pointer_register(); __ move(LIR_OprFact::longConst(uintptr_t(&CONCURRENT_MARKING_ACTIVE)), cm_flag_addr_opr); diff --git a/openjdk/mmtkBarrierSetC1.hpp b/openjdk/mmtkBarrierSetC1.hpp index 546e74de..d1522ad7 100644 --- a/openjdk/mmtkBarrierSetC1.hpp +++ b/openjdk/mmtkBarrierSetC1.hpp @@ -83,19 +83,15 @@ class MMTkBarrierSetC1 : public BarrierSetC1 { struct MMTkC1ReferenceLoadBarrierStub: CodeStub { LIR_Opr val; - CodeEmitInfo* info; // Code patching info - MMTkC1ReferenceLoadBarrierStub(LIR_Opr val, CodeEmitInfo* info = NULL): val(val), info(info) {} + MMTkC1ReferenceLoadBarrierStub(LIR_Opr val): val(val) {} virtual void emit_code(LIR_Assembler* ce) override; virtual void visit(LIR_OpVisitState* visitor) override { - if (info != NULL) { - visitor->do_slow_case(info); - } else { - visitor->do_slow_case(); - } - if (val != NULL) visitor->do_input(val); + visitor->do_slow_case(); + assert(val->is_valid(), "val must be valid"); + visitor->do_input(val); } NOT_PRODUCT(virtual void print_name(outputStream* out) const { out->print("MMTkC1ReferenceLoadBarrierStub"); }); From f6e0659b5505248990ceb1a96a572d865072bfa5 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Mon, 22 Sep 2025 23:58:12 +0800 Subject: [PATCH 32/46] Comment and minor fix for code stubs --- openjdk/barriers/mmtkUnlogBitBarrier.hpp | 2 +- openjdk/mmtkBarrierSetC1.hpp | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/openjdk/barriers/mmtkUnlogBitBarrier.hpp b/openjdk/barriers/mmtkUnlogBitBarrier.hpp index 01a5b9dc..585355e7 100644 --- a/openjdk/barriers/mmtkUnlogBitBarrier.hpp +++ b/openjdk/barriers/mmtkUnlogBitBarrier.hpp @@ -72,7 +72,7 @@ struct MMTkC1UnlogBitBarrierSlowPathStub: CodeStub { visitor->do_input(src); } - NOT_PRODUCT(virtual void print_name(outputStream* out) const { out->print("MMTkC1PreBarrierStub"); }); + NOT_PRODUCT(virtual void print_name(outputStream* out) const { out->print("MMTkC1UnlogBitBarrierSlowPathStub"); }); }; //////////////////// C2 //////////////////// diff --git a/openjdk/mmtkBarrierSetC1.hpp b/openjdk/mmtkBarrierSetC1.hpp index d1522ad7..ec0bc7ef 100644 --- a/openjdk/mmtkBarrierSetC1.hpp +++ b/openjdk/mmtkBarrierSetC1.hpp @@ -81,6 +81,9 @@ class MMTkBarrierSetC1 : public BarrierSetC1 { virtual void generate_c1_runtime_stubs(BufferBlob* buffer_blob) override; }; +/// The code stub for (weak) reference loading barrier slow path. +/// It will call `MMTkBarrierSetRuntime::load_reference_call` if `val` is not null. +/// Currently only the SATB barrier uses this code stub. struct MMTkC1ReferenceLoadBarrierStub: CodeStub { LIR_Opr val; From cdc645529660a91d445897bf455b68474f7e07f7 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Tue, 23 Sep 2025 16:38:33 +0800 Subject: [PATCH 33/46] Merged C2 pre/post barriers for unlog barriers --- openjdk/barriers/mmtkObjectBarrier.cpp | 21 +----------- openjdk/barriers/mmtkObjectBarrier.hpp | 2 +- openjdk/barriers/mmtkSATBBarrier.cpp | 25 +-------------- openjdk/barriers/mmtkSATBBarrier.hpp | 2 +- openjdk/barriers/mmtkUnlogBitBarrier.cpp | 41 ++++++++++++++++++++++++ openjdk/barriers/mmtkUnlogBitBarrier.hpp | 6 +++- openjdk/mmtkBarrierSetC2.hpp | 1 + 7 files changed, 51 insertions(+), 47 deletions(-) diff --git a/openjdk/barriers/mmtkObjectBarrier.cpp b/openjdk/barriers/mmtkObjectBarrier.cpp index 222dd42f..28cf262c 100644 --- a/openjdk/barriers/mmtkObjectBarrier.cpp +++ b/openjdk/barriers/mmtkObjectBarrier.cpp @@ -99,26 +99,7 @@ void MMTkObjectBarrierSetC2::object_reference_write_post(GraphKit* kit, Node* sr MMTkIdealKit ideal(kit, true); - if (mmtk_enable_barrier_fastpath) { - Node* no_base = __ top(); - float unlikely = PROB_UNLIKELY(0.999); - - Node* zero = __ ConI(0); - Node* addr = __ CastPX(__ ctrl(), src); - Node* meta_addr = __ AddP(no_base, __ ConP(SIDE_METADATA_BASE_ADDRESS), __ URShiftX(addr, __ ConI(6))); - Node* byte = __ load(__ ctrl(), meta_addr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); - Node* shift = __ URShiftX(addr, __ ConI(3)); - shift = __ AndI(__ ConvL2I(shift), __ ConI(7)); - Node* result = __ AndI(__ URShiftI(byte, shift), __ ConI(1)); - - __ if_then(result, BoolTest::ne, zero, unlikely); { - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); - Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), "mmtk_barrier_call", src, slot, val); - } __ end_if(); - } else { - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); - Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call), "mmtk_barrier_call", src, slot, val); - } + object_reference_write_pre_or_post(ideal, src, /* pre = */ false); kit->final_sync(ideal); // Final sync IdealKit and GraphKit. } diff --git a/openjdk/barriers/mmtkObjectBarrier.hpp b/openjdk/barriers/mmtkObjectBarrier.hpp index f5d3a861..254d09b8 100644 --- a/openjdk/barriers/mmtkObjectBarrier.hpp +++ b/openjdk/barriers/mmtkObjectBarrier.hpp @@ -46,7 +46,7 @@ class MMTkObjectBarrierSetC1: public MMTkUnlogBitBarrierSetC1 { } }; -class MMTkObjectBarrierSetC2: public MMTkBarrierSetC2 { +class MMTkObjectBarrierSetC2: public MMTkUnlogBitBarrierSetC2 { protected: virtual void object_reference_write_post(GraphKit* kit, Node* src, Node* slot, Node* val) const override; }; diff --git a/openjdk/barriers/mmtkSATBBarrier.cpp b/openjdk/barriers/mmtkSATBBarrier.cpp index 95786eb3..170ad688 100644 --- a/openjdk/barriers/mmtkSATBBarrier.cpp +++ b/openjdk/barriers/mmtkSATBBarrier.cpp @@ -154,30 +154,7 @@ void MMTkSATBBarrierSetC2::object_reference_write_pre(GraphKit* kit, Node* src, MMTkIdealKit ideal(kit, true); - if (mmtk_enable_barrier_fastpath) { - Node* no_base = __ top(); - float unlikely = PROB_UNLIKELY(0.999); - - Node* zero = __ ConI(0); - Node* addr = __ CastPX(__ ctrl(), src); - Node* meta_addr = __ AddP(no_base, __ ConP(side_metadata_base_address()), __ URShiftX(addr, __ ConI(6))); - Node* byte = __ load(__ ctrl(), meta_addr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); - - Node* shift = __ URShiftX(addr, __ ConI(3)); - shift = __ AndI(__ ConvL2I(shift), __ ConI(7)); - Node* result = __ AndI(__ URShiftI(byte, shift), __ ConI(1)); - __ if_then(result, BoolTest::ne, zero, unlikely); { - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); - Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), "mmtk_barrier_call", src, slot, val); - } __ end_if(); - } else { - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); - Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call), "mmtk_barrier_call", src, slot, val); - // Looks like this is necessary - // See https://github.com/mmtk/openjdk/blob/c82e5c44adced4383162826c2c3933a83cfb139b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp#L288-L291 - Node* call = __ ctrl()->in(0); - call->add_req(slot); - } + object_reference_write_pre_or_post(ideal, src, /* pre = */ true); kit->final_sync(ideal); // Final sync IdealKit and GraphKit. } diff --git a/openjdk/barriers/mmtkSATBBarrier.hpp b/openjdk/barriers/mmtkSATBBarrier.hpp index d82f45ff..296bfddf 100644 --- a/openjdk/barriers/mmtkSATBBarrier.hpp +++ b/openjdk/barriers/mmtkSATBBarrier.hpp @@ -50,7 +50,7 @@ class MMTkSATBBarrierSetC1: public MMTkUnlogBitBarrierSetC1 { } }; -class MMTkSATBBarrierSetC2: public MMTkBarrierSetC2 { +class MMTkSATBBarrierSetC2: public MMTkUnlogBitBarrierSetC2 { protected: virtual void object_reference_write_pre(GraphKit* kit, Node* src, Node* slot, Node* val) const override; diff --git a/openjdk/barriers/mmtkUnlogBitBarrier.cpp b/openjdk/barriers/mmtkUnlogBitBarrier.cpp index a3abcc6c..49892a3c 100644 --- a/openjdk/barriers/mmtkUnlogBitBarrier.cpp +++ b/openjdk/barriers/mmtkUnlogBitBarrier.cpp @@ -185,3 +185,44 @@ void MMTkUnlogBitBarrierSetC1::object_reference_write_pre_or_post(LIRAccess& acc } #undef __ + +//////////////////// C2 //////////////////// + +#define __ ideal. + +Node* MMTkUnlogBitBarrierSetC2::emit_check_unlog_bit_fast_path(MMTkIdealKit& ideal, Node* obj) { + Node* addr = __ CastPX(__ ctrl(), obj); + Node* no_base = __ top(); + Node* meta_addr = __ AddP(no_base, __ ConP(UNLOG_BIT_BASE_ADDRESS), __ URShiftX(addr, __ ConI(6))); + Node* byte = __ load(__ ctrl(), meta_addr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); + Node* shift = __ URShiftX(addr, __ ConI(3)); + shift = __ AndI(__ ConvL2I(shift), __ ConI(7)); + Node* result = __ AndI(__ URShiftI(byte, shift), __ ConI(1)); + + return result; +} + +void MMTkUnlogBitBarrierSetC2::object_reference_write_pre_or_post(MMTkIdealKit& ideal, Node* src, bool pre) { + if (mmtk_enable_barrier_fastpath) { + Node* result = emit_check_unlog_bit_fast_path(ideal, src); + + Node* zero = __ ConI(0); + float unlikely = PROB_UNLIKELY(0.999); + __ if_then(result, BoolTest::ne, zero, unlikely); { + Node* result = emit_check_unlog_bit_fast_path(ideal, src); + + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); + address entry_point = FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call); + Node* null_node = __ NullP(); + Node* x = __ make_leaf_call(tf, entry_point, "mmtk_barrier_call", src, null_node, null_node); + } __ end_if(); + } else { + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); + address entry_point = pre ? FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call) + : FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call); + Node* null_node = __ NullP(); + Node* x = __ make_leaf_call(tf, entry_point, "mmtk_barrier_call", src, null_node, null_node); + } +} + +#undef __ diff --git a/openjdk/barriers/mmtkUnlogBitBarrier.hpp b/openjdk/barriers/mmtkUnlogBitBarrier.hpp index 585355e7..2080439e 100644 --- a/openjdk/barriers/mmtkUnlogBitBarrier.hpp +++ b/openjdk/barriers/mmtkUnlogBitBarrier.hpp @@ -77,6 +77,10 @@ struct MMTkC1UnlogBitBarrierSlowPathStub: CodeStub { //////////////////// C2 //////////////////// -class MMTkUnlogBitBarrierSetC2: public MMTkBarrierSetC2 {}; +class MMTkUnlogBitBarrierSetC2: public MMTkBarrierSetC2 { +protected: + static Node* emit_check_unlog_bit_fast_path(MMTkIdealKit& ideal, Node* obj); + static void object_reference_write_pre_or_post(MMTkIdealKit& ideal, Node* src, bool pre); +}; #endif // MMTK_OPENJDK_BARRIERS_MMTK_UNLOG_BIT_BARRIER_HPP diff --git a/openjdk/mmtkBarrierSetC2.hpp b/openjdk/mmtkBarrierSetC2.hpp index d8052d9a..90270d8e 100644 --- a/openjdk/mmtkBarrierSetC2.hpp +++ b/openjdk/mmtkBarrierSetC2.hpp @@ -113,6 +113,7 @@ class MMTkIdealKit: public IdealKit { inline Node* CastXP(Node* x) { return transform(new CastX2PNode(x)); } inline Node* URShiftI(Node* l, Node* r) { return transform(new URShiftINode(l, r)); } inline Node* ConP(intptr_t ptr) { return makecon(TypeRawPtr::make((address) ptr)); } + inline Node* NullP() { return makecon(TypePtr::NULL_PTR); } template inline const TypeFunc* func_type(Types... types) { From 3af966597aaf6f72854a0b3d2ef8e1c3594d079a Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Tue, 23 Sep 2025 17:06:01 +0800 Subject: [PATCH 34/46] Remove unused macros. --- openjdk/barriers/mmtkSATBBarrier.cpp | 16 ++++------------ openjdk/barriers/mmtkSATBBarrier.hpp | 10 +++++----- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/openjdk/barriers/mmtkSATBBarrier.cpp b/openjdk/barriers/mmtkSATBBarrier.cpp index 170ad688..f2e47efa 100644 --- a/openjdk/barriers/mmtkSATBBarrier.cpp +++ b/openjdk/barriers/mmtkSATBBarrier.cpp @@ -2,16 +2,8 @@ #include "mmtkSATBBarrier.hpp" #include "runtime/interfaceSupport.inline.hpp" -#define SOFT_REFERENCE_LOAD_BARRIER true - -constexpr int kUnloggedValue = 1; - -static inline intptr_t side_metadata_base_address() { - return SATB_METADATA_BASE_ADDRESS; -} - void MMTkSATBBarrierSetRuntime::load_reference(DecoratorSet decorators, oop value) const { -#if SOFT_REFERENCE_LOAD_BARRIER +#if WEAK_REFERENCE_LOAD_BARRIER if (CONCURRENT_MARKING_ACTIVE == 1 && value != NULL) ::mmtk_load_reference((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) value); #endif @@ -44,7 +36,7 @@ void MMTkSATBBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet dec BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); -#if SOFT_REFERENCE_LOAD_BARRIER +#if WEAK_REFERENCE_LOAD_BARRIER if (on_oop && on_reference) { Label done; @@ -110,7 +102,7 @@ void MMTkSATBBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) { BarrierSetC1::load_at_resolved(access, result); -#if SOFT_REFERENCE_LOAD_BARRIER +#if WEAK_REFERENCE_LOAD_BARRIER if (access.is_oop() && (is_weak || is_phantom || is_anonymous)) { // Register the value in the referent field with the pre-barrier LabelObj *Lcont_anonymous; @@ -282,7 +274,7 @@ Node* MMTkSATBBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_t return load; } -#if SOFT_REFERENCE_LOAD_BARRIER +#if WEAK_REFERENCE_LOAD_BARRIER if (on_weak) { reference_load_barrier(kit, adr, load, true); } else if (unknown) { diff --git a/openjdk/barriers/mmtkSATBBarrier.hpp b/openjdk/barriers/mmtkSATBBarrier.hpp index 296bfddf..6319cc51 100644 --- a/openjdk/barriers/mmtkSATBBarrier.hpp +++ b/openjdk/barriers/mmtkSATBBarrier.hpp @@ -13,11 +13,11 @@ #include "opto/callnode.hpp" #include "opto/idealKit.hpp" -#define SIDE_METADATA_WORST_CASE_RATIO_LOG 1 -#define LOG_BYTES_IN_CHUNK 22 -#define CHUNK_MASK ((1L << LOG_BYTES_IN_CHUNK) - 1) - -const intptr_t SATB_METADATA_BASE_ADDRESS = (intptr_t) GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS; +// Debug flag for enabling/disabling all weak reference loading barriers. +// Note that the progran can't run correctly without weak reference load barrier, +// unless we disable reference process, too, +// in which case all references are considered strong. +#define WEAK_REFERENCE_LOAD_BARRIER true class MMTkSATBBarrierSetRuntime: public MMTkUnlogBitBarrierSetRuntime { public: From 2e80f07afb3b1c855e923dafe95e4d12987be494 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Tue, 23 Sep 2025 17:25:55 +0800 Subject: [PATCH 35/46] Replace WEAK_REFERENCE_LOAD_BARRIER with a runtime flag ... so that we can debug ref-load barriers without recompilation. --- openjdk/barriers/mmtkSATBBarrier.cpp | 118 +++++++++++++-------------- openjdk/barriers/mmtkSATBBarrier.hpp | 6 -- openjdk/mmtkBarrierSet.cpp | 1 + openjdk/mmtkBarrierSet.hpp | 1 + openjdk/mmtkHeap.cpp | 8 ++ 5 files changed, 69 insertions(+), 65 deletions(-) diff --git a/openjdk/barriers/mmtkSATBBarrier.cpp b/openjdk/barriers/mmtkSATBBarrier.cpp index f2e47efa..1271f6c9 100644 --- a/openjdk/barriers/mmtkSATBBarrier.cpp +++ b/openjdk/barriers/mmtkSATBBarrier.cpp @@ -3,10 +3,10 @@ #include "runtime/interfaceSupport.inline.hpp" void MMTkSATBBarrierSetRuntime::load_reference(DecoratorSet decorators, oop value) const { -#if WEAK_REFERENCE_LOAD_BARRIER - if (CONCURRENT_MARKING_ACTIVE == 1 && value != NULL) - ::mmtk_load_reference((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) value); -#endif + if (mmtk_enable_reference_load_barrier) { + if (CONCURRENT_MARKING_ACTIVE == 1 && value != NULL) + ::mmtk_load_reference((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) value); + } }; void MMTkSATBBarrierSetRuntime::object_probable_write(oop new_obj) const { @@ -36,32 +36,32 @@ void MMTkSATBBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet dec BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); -#if WEAK_REFERENCE_LOAD_BARRIER - if (on_oop && on_reference) { - Label done; - - assert_different_registers(dst, tmp1); - - // No slow-call if SATB is not active - // intptr_t tmp1_q = CONCURRENT_MARKING_ACTIVE; - __ movptr(tmp1, intptr_t(&CONCURRENT_MARKING_ACTIVE)); - // Load with zero extension to 32 bits. - // uint32_t tmp1_l = (uint32_t)(*(unt8_t*)tmp1_q); - __ movzbl(tmp1, Address(tmp1, 0)); - // if (tmp1_l == 0) goto done; - __ testl(tmp1, tmp1); - __ jcc(Assembler::zero, done); - // if (dst == 0) goto done; - __ testptr(dst, dst); - __ jcc(Assembler::zero, done); - // Do slow-call - __ pusha(); - __ mov(c_rarg0, dst); - __ MacroAssembler::call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), 1); - __ popa(); - __ bind(done); + if (mmtk_enable_reference_load_barrier) { + if (on_oop && on_reference) { + Label done; + + assert_different_registers(dst, tmp1); + + // No slow-call if SATB is not active + // intptr_t tmp1_q = CONCURRENT_MARKING_ACTIVE; + __ movptr(tmp1, intptr_t(&CONCURRENT_MARKING_ACTIVE)); + // Load with zero extension to 32 bits. + // uint32_t tmp1_l = (uint32_t)(*(unt8_t*)tmp1_q); + __ movzbl(tmp1, Address(tmp1, 0)); + // if (tmp1_l == 0) goto done; + __ testl(tmp1, tmp1); + __ jcc(Assembler::zero, done); + // if (dst == 0) goto done; + __ testptr(dst, dst); + __ jcc(Assembler::zero, done); + // Do slow-call + __ pusha(); + __ mov(c_rarg0, dst); + __ MacroAssembler::call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), 1); + __ popa(); + __ bind(done); + } } -#endif } void MMTkSATBBarrierSetAssembler::object_reference_write_pre(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2) const { @@ -102,32 +102,32 @@ void MMTkSATBBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) { BarrierSetC1::load_at_resolved(access, result); -#if WEAK_REFERENCE_LOAD_BARRIER - if (access.is_oop() && (is_weak || is_phantom || is_anonymous)) { - // Register the value in the referent field with the pre-barrier - LabelObj *Lcont_anonymous; - if (is_anonymous) { - Lcont_anonymous = new LabelObj(); - generate_referent_check(access, Lcont_anonymous); - } - assert(result->is_register(), "must be"); - assert(result->type() == T_OBJECT, "must be an object"); - auto slow = new MMTkC1ReferenceLoadBarrierStub(result); - // Call slow-path only when concurrent marking is active - LIR_Opr cm_flag_addr_opr = gen->new_pointer_register(); - __ move(LIR_OprFact::longConst(uintptr_t(&CONCURRENT_MARKING_ACTIVE)), cm_flag_addr_opr); - LIR_Address* cm_flag_addr = new LIR_Address(cm_flag_addr_opr, T_BYTE); - LIR_Opr cm_flag = gen->new_register(T_INT); - __ move(cm_flag_addr, cm_flag); - // No slow-call if SATB is not active - __ cmp(lir_cond_equal, cm_flag, LIR_OprFact::intConst(1)); - __ branch(lir_cond_equal, T_BYTE, slow); - __ branch_destination(slow->continuation()); - if (is_anonymous) { - __ branch_destination(Lcont_anonymous->label()); + if (mmtk_enable_reference_load_barrier) { + if (access.is_oop() && (is_weak || is_phantom || is_anonymous)) { + // Register the value in the referent field with the pre-barrier + LabelObj *Lcont_anonymous; + if (is_anonymous) { + Lcont_anonymous = new LabelObj(); + generate_referent_check(access, Lcont_anonymous); + } + assert(result->is_register(), "must be"); + assert(result->type() == T_OBJECT, "must be an object"); + auto slow = new MMTkC1ReferenceLoadBarrierStub(result); + // Call slow-path only when concurrent marking is active + LIR_Opr cm_flag_addr_opr = gen->new_pointer_register(); + __ move(LIR_OprFact::longConst(uintptr_t(&CONCURRENT_MARKING_ACTIVE)), cm_flag_addr_opr); + LIR_Address* cm_flag_addr = new LIR_Address(cm_flag_addr_opr, T_BYTE); + LIR_Opr cm_flag = gen->new_register(T_INT); + __ move(cm_flag_addr, cm_flag); + // No slow-call if SATB is not active + __ cmp(lir_cond_equal, cm_flag, LIR_OprFact::intConst(1)); + __ branch(lir_cond_equal, T_BYTE, slow); + __ branch_destination(slow->continuation()); + if (is_anonymous) { + __ branch_destination(Lcont_anonymous->label()); + } } } -#endif } void MMTkSATBBarrierSetC1::object_reference_write_pre(LIRAccess& access) const { @@ -274,13 +274,13 @@ Node* MMTkSATBBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_t return load; } -#if WEAK_REFERENCE_LOAD_BARRIER - if (on_weak) { - reference_load_barrier(kit, adr, load, true); - } else if (unknown) { - reference_load_barrier_for_unknown_load(kit, obj, offset, adr, load, !need_cpu_mem_bar); + if (mmtk_enable_reference_load_barrier) { + if (on_weak) { + reference_load_barrier(kit, adr, load, true); + } else if (unknown) { + reference_load_barrier_for_unknown_load(kit, obj, offset, adr, load, !need_cpu_mem_bar); + } } -#endif return load; } diff --git a/openjdk/barriers/mmtkSATBBarrier.hpp b/openjdk/barriers/mmtkSATBBarrier.hpp index 6319cc51..fe377f4e 100644 --- a/openjdk/barriers/mmtkSATBBarrier.hpp +++ b/openjdk/barriers/mmtkSATBBarrier.hpp @@ -13,12 +13,6 @@ #include "opto/callnode.hpp" #include "opto/idealKit.hpp" -// Debug flag for enabling/disabling all weak reference loading barriers. -// Note that the progran can't run correctly without weak reference load barrier, -// unless we disable reference process, too, -// in which case all references are considered strong. -#define WEAK_REFERENCE_LOAD_BARRIER true - class MMTkSATBBarrierSetRuntime: public MMTkUnlogBitBarrierSetRuntime { public: // Interfaces called by `MMTkBarrierSet::AccessBarrier` diff --git a/openjdk/mmtkBarrierSet.cpp b/openjdk/mmtkBarrierSet.cpp index d0c4838a..b6509f05 100644 --- a/openjdk/mmtkBarrierSet.cpp +++ b/openjdk/mmtkBarrierSet.cpp @@ -38,6 +38,7 @@ bool mmtk_enable_allocation_fastpath = true; bool mmtk_enable_barrier_fastpath = true; +bool mmtk_enable_reference_load_barrier = true; MMTkAllocatorOffsets get_tlab_top_and_end_offsets(AllocatorSelector selector) { int tlab_top_offset, tlab_end_offset; diff --git a/openjdk/mmtkBarrierSet.hpp b/openjdk/mmtkBarrierSet.hpp index 37400e2a..d1494307 100644 --- a/openjdk/mmtkBarrierSet.hpp +++ b/openjdk/mmtkBarrierSet.hpp @@ -39,6 +39,7 @@ extern bool mmtk_enable_allocation_fastpath; extern bool mmtk_enable_barrier_fastpath; +extern bool mmtk_enable_reference_load_barrier; const intptr_t VO_BIT_BASE_ADDRESS = VO_BIT_ADDRESS; diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index a6c1c26a..e84bf458 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -94,6 +94,14 @@ jint MMTkHeap::initialize() { set_bool_option_from_env_var("MMTK_ENABLE_ALLOCATION_FASTPATH", &mmtk_enable_allocation_fastpath); set_bool_option_from_env_var("MMTK_ENABLE_BARRIER_FASTPATH", &mmtk_enable_barrier_fastpath); + set_bool_option_from_env_var("MMTK_ENABLE_REFERENCE_LOAD_BARRIER", &mmtk_enable_reference_load_barrier); + + if (!mmtk_enable_reference_load_barrier) { + fprintf(stderr, + "WARNING! Reference load barrier is disabled. " + "The program cannot run correctly unless reference processing is disabled, " + "in which case all java.lang.ref.Reference instances are treated as strong.\n"); + } const size_t min_heap_size = collector_policy()->min_heap_byte_size(); const size_t max_heap_size = collector_policy()->max_heap_byte_size(); From e5071952b8916528381517275b2b09314d32ed5d Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Wed, 24 Sep 2025 15:58:20 +0800 Subject: [PATCH 36/46] Remove unused macros --- openjdk/barriers/mmtkObjectBarrier.hpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/openjdk/barriers/mmtkObjectBarrier.hpp b/openjdk/barriers/mmtkObjectBarrier.hpp index 254d09b8..c2c35aff 100644 --- a/openjdk/barriers/mmtkObjectBarrier.hpp +++ b/openjdk/barriers/mmtkObjectBarrier.hpp @@ -13,12 +13,6 @@ #include "opto/callnode.hpp" #include "opto/idealKit.hpp" -#define SIDE_METADATA_WORST_CASE_RATIO_LOG 1 -#define LOG_BYTES_IN_CHUNK 22 -#define CHUNK_MASK ((1L << LOG_BYTES_IN_CHUNK) - 1) - -const intptr_t SIDE_METADATA_BASE_ADDRESS = (intptr_t) GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS; - class MMTkObjectBarrierSetRuntime: public MMTkUnlogBitBarrierSetRuntime { public: // Interfaces called by `MMTkBarrierSet::AccessBarrier` From e9fa0d06563405f21a0d93ce9c2d0361693da5d7 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Wed, 24 Sep 2025 17:29:18 +0800 Subject: [PATCH 37/46] Revert changes to C1 write barrier signatures. Restore src, slot and new_val, even if they are unused. --- openjdk/barriers/mmtkObjectBarrier.cpp | 4 ++-- openjdk/barriers/mmtkObjectBarrier.hpp | 2 +- openjdk/barriers/mmtkSATBBarrier.cpp | 4 ++-- openjdk/barriers/mmtkSATBBarrier.hpp | 2 +- openjdk/barriers/mmtkUnlogBitBarrier.cpp | 4 +--- openjdk/barriers/mmtkUnlogBitBarrier.hpp | 2 +- openjdk/mmtkBarrierSetC1.hpp | 16 ++++++++-------- 7 files changed, 16 insertions(+), 18 deletions(-) diff --git a/openjdk/barriers/mmtkObjectBarrier.cpp b/openjdk/barriers/mmtkObjectBarrier.cpp index 28cf262c..e522df30 100644 --- a/openjdk/barriers/mmtkObjectBarrier.cpp +++ b/openjdk/barriers/mmtkObjectBarrier.cpp @@ -86,8 +86,8 @@ void MMTkObjectBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, Dec #define __ gen->lir()-> #endif -void MMTkObjectBarrierSetC1::object_reference_write_post(LIRAccess& access) const { - object_reference_write_pre_or_post(access, false); +void MMTkObjectBarrierSetC1::object_reference_write_post(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val) const { + object_reference_write_pre_or_post(access, src, false); } #undef __ diff --git a/openjdk/barriers/mmtkObjectBarrier.hpp b/openjdk/barriers/mmtkObjectBarrier.hpp index c2c35aff..d33e22ec 100644 --- a/openjdk/barriers/mmtkObjectBarrier.hpp +++ b/openjdk/barriers/mmtkObjectBarrier.hpp @@ -33,7 +33,7 @@ class MMTkObjectBarrierSetAssembler: public MMTkUnlogBitBarrierSetAssembler { class MMTkObjectBarrierSetC1: public MMTkUnlogBitBarrierSetC1 { protected: - virtual void object_reference_write_post(LIRAccess& access) const override; + virtual void object_reference_write_post(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val) const override; virtual LIR_Opr resolve_address(LIRAccess& access, bool resolve_in_register) override { return MMTkBarrierSetC1::resolve_address_in_register(access, resolve_in_register); diff --git a/openjdk/barriers/mmtkSATBBarrier.cpp b/openjdk/barriers/mmtkSATBBarrier.cpp index 1271f6c9..c567da9b 100644 --- a/openjdk/barriers/mmtkSATBBarrier.cpp +++ b/openjdk/barriers/mmtkSATBBarrier.cpp @@ -130,11 +130,11 @@ void MMTkSATBBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) { } } -void MMTkSATBBarrierSetC1::object_reference_write_pre(LIRAccess& access) const { +void MMTkSATBBarrierSetC1::object_reference_write_pre(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val) const { // TODO: Should be able to skip weak field writes, too, because that doesn't change strong // reachability during concurrent marking. - object_reference_write_pre_or_post(access, true); + object_reference_write_pre_or_post(access, src, true); } #undef __ diff --git a/openjdk/barriers/mmtkSATBBarrier.hpp b/openjdk/barriers/mmtkSATBBarrier.hpp index fe377f4e..db8fea1a 100644 --- a/openjdk/barriers/mmtkSATBBarrier.hpp +++ b/openjdk/barriers/mmtkSATBBarrier.hpp @@ -35,7 +35,7 @@ class MMTkSATBBarrierSetAssembler: public MMTkUnlogBitBarrierSetAssembler { class MMTkSATBBarrierSetC1: public MMTkUnlogBitBarrierSetC1 { protected: - virtual void object_reference_write_pre(LIRAccess& access) const override; + virtual void object_reference_write_pre(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val) const override; virtual void load_at_resolved(LIRAccess& access, LIR_Opr result) override; diff --git a/openjdk/barriers/mmtkUnlogBitBarrier.cpp b/openjdk/barriers/mmtkUnlogBitBarrier.cpp index 49892a3c..0f87e3b2 100644 --- a/openjdk/barriers/mmtkUnlogBitBarrier.cpp +++ b/openjdk/barriers/mmtkUnlogBitBarrier.cpp @@ -166,13 +166,11 @@ void MMTkUnlogBitBarrierSetC1::emit_check_unlog_bit_fast_path(LIRGenerator* gen, __ branch(lir_cond_equal, T_BYTE, slow); } -void MMTkUnlogBitBarrierSetC1::object_reference_write_pre_or_post(LIRAccess& access, bool pre) { +void MMTkUnlogBitBarrierSetC1::object_reference_write_pre_or_post(LIRAccess& access, LIR_Opr src, bool pre) { LIRGenerator* gen = access.gen(); DecoratorSet decorators = access.decorators(); if ((decorators & IN_HEAP) == 0) return; - LIR_Opr src = access.base().opr(); - CodeStub* slow = new MMTkC1UnlogBitBarrierSlowPathStub(src, mmtk_enable_barrier_fastpath, pre); if (mmtk_enable_barrier_fastpath) { diff --git a/openjdk/barriers/mmtkUnlogBitBarrier.hpp b/openjdk/barriers/mmtkUnlogBitBarrier.hpp index 2080439e..ec4e0ac2 100644 --- a/openjdk/barriers/mmtkUnlogBitBarrier.hpp +++ b/openjdk/barriers/mmtkUnlogBitBarrier.hpp @@ -42,7 +42,7 @@ class MMTkUnlogBitBarrierSetAssembler: public MMTkBarrierSetAssembler { class MMTkUnlogBitBarrierSetC1: public MMTkBarrierSetC1 { protected: static void emit_check_unlog_bit_fast_path(LIRGenerator* gen, LIR_Opr addr, CodeStub* slow); - static void object_reference_write_pre_or_post(LIRAccess& access, bool pre); + static void object_reference_write_pre_or_post(LIRAccess& access, LIR_Opr src, bool pre); }; /// C1 write barrier slow path stub. diff --git a/openjdk/mmtkBarrierSetC1.hpp b/openjdk/mmtkBarrierSetC1.hpp index ec0bc7ef..03e3f2ab 100644 --- a/openjdk/mmtkBarrierSetC1.hpp +++ b/openjdk/mmtkBarrierSetC1.hpp @@ -25,30 +25,30 @@ class MMTkBarrierSetC1 : public BarrierSetC1 { protected: /// Full pre-barrier - virtual void object_reference_write_pre(LIRAccess& access) const {} + virtual void object_reference_write_pre(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val) const {} /// Full post-barrier - virtual void object_reference_write_post(LIRAccess& access) const {} + virtual void object_reference_write_post(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val) const {} /// Substituting write barrier virtual void store_at_resolved(LIRAccess& access, LIR_Opr value) override { - if (access.is_oop()) object_reference_write_pre(access); + if (access.is_oop()) object_reference_write_pre(access, access.base().opr(), access.resolved_addr(), value); BarrierSetC1::store_at_resolved(access, value); - if (access.is_oop()) object_reference_write_post(access); + if (access.is_oop()) object_reference_write_post(access, access.base().opr(), access.resolved_addr(), value); } /// Substituting write barrier (cmpxchg) virtual LIR_Opr atomic_cmpxchg_at_resolved(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value) override { - if (access.is_oop()) object_reference_write_pre(access); + if (access.is_oop()) object_reference_write_pre(access, access.base().opr(), access.resolved_addr(), new_value.result()); LIR_Opr result = BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value); - if (access.is_oop()) object_reference_write_post(access); + if (access.is_oop()) object_reference_write_post(access, access.base().opr(), access.resolved_addr(), new_value.result()); return result; } /// Substituting write barrier (xchg) virtual LIR_Opr atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value) override { - if (access.is_oop()) object_reference_write_pre(access); + if (access.is_oop()) object_reference_write_pre(access, access.base().opr(), access.resolved_addr(), value.result()); LIR_Opr result = BarrierSetC1::atomic_xchg_at_resolved(access, value); - if (access.is_oop()) object_reference_write_post(access); + if (access.is_oop()) object_reference_write_post(access, access.base().opr(), access.resolved_addr(), value.result()); return result; } From 73fa6c4c3cf4eee5838d495db69370297ee13bff Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Wed, 24 Sep 2025 17:39:31 +0800 Subject: [PATCH 38/46] Section separators and file header comments --- openjdk/barriers/mmtkObjectBarrier.cpp | 8 ++++++++ openjdk/barriers/mmtkObjectBarrier.hpp | 16 ++++++++++++++++ openjdk/barriers/mmtkSATBBarrier.cpp | 8 ++++++++ openjdk/barriers/mmtkSATBBarrier.hpp | 14 ++++++++++++++ 4 files changed, 46 insertions(+) diff --git a/openjdk/barriers/mmtkObjectBarrier.cpp b/openjdk/barriers/mmtkObjectBarrier.cpp index e522df30..7f67da1d 100644 --- a/openjdk/barriers/mmtkObjectBarrier.cpp +++ b/openjdk/barriers/mmtkObjectBarrier.cpp @@ -2,6 +2,8 @@ #include "mmtkObjectBarrier.hpp" #include "runtime/interfaceSupport.inline.hpp" +//////////////////// Runtime //////////////////// + void MMTkObjectBarrierSetRuntime::object_probable_write(oop new_obj) const { if (mmtk_enable_barrier_fastpath) { // Do fast-path check before entering mmtk rust code, to improve mutator performance. @@ -27,6 +29,8 @@ void MMTkObjectBarrierSetRuntime::object_reference_write_post(oop src, oop* slot } } +//////////////////// Assembler //////////////////// + #define __ masm-> void MMTkObjectBarrierSetAssembler::object_reference_write_post(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2, bool compensate_val_reg) const { @@ -80,6 +84,8 @@ void MMTkObjectBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, Dec #undef __ +//////////////////// C1 //////////////////// + #ifdef ASSERT #define __ gen->lir(__FILE__, __LINE__)-> #else @@ -92,6 +98,8 @@ void MMTkObjectBarrierSetC1::object_reference_write_post(LIRAccess& access, LIR_ #undef __ +//////////////////// C2 //////////////////// + #define __ ideal. void MMTkObjectBarrierSetC2::object_reference_write_post(GraphKit* kit, Node* src, Node* slot, Node* val) const { diff --git a/openjdk/barriers/mmtkObjectBarrier.hpp b/openjdk/barriers/mmtkObjectBarrier.hpp index d33e22ec..1735da8f 100644 --- a/openjdk/barriers/mmtkObjectBarrier.hpp +++ b/openjdk/barriers/mmtkObjectBarrier.hpp @@ -13,6 +13,14 @@ #include "opto/callnode.hpp" #include "opto/idealKit.hpp" +/// This file supports the `ObjectBarrier` in MMTk core, +/// i.e. the barrier that remembers the object when it is first modified. +/// Despite the name, `ObjectBarrier` is not the only barrier that uses the object-grained unlogging bit. +/// `SATBBarrier` also uses the object-grained unlog bit. +/// We keep the name in sync with the MMTk core. + +//////////////////// Runtime //////////////////// + class MMTkObjectBarrierSetRuntime: public MMTkUnlogBitBarrierSetRuntime { public: // Interfaces called by `MMTkBarrierSet::AccessBarrier` @@ -23,6 +31,8 @@ class MMTkObjectBarrierSetRuntime: public MMTkUnlogBitBarrierSetRuntime { virtual void object_probable_write(oop new_obj) const override; }; +//////////////////// Assembler //////////////////// + class MMTkObjectBarrierSetAssembler: public MMTkUnlogBitBarrierSetAssembler { protected: virtual void object_reference_write_post(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2, bool compensate_val_reg) const override; @@ -31,6 +41,8 @@ class MMTkObjectBarrierSetAssembler: public MMTkUnlogBitBarrierSetAssembler { virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) override; }; +//////////////////// C1 //////////////////// + class MMTkObjectBarrierSetC1: public MMTkUnlogBitBarrierSetC1 { protected: virtual void object_reference_write_post(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val) const override; @@ -40,11 +52,15 @@ class MMTkObjectBarrierSetC1: public MMTkUnlogBitBarrierSetC1 { } }; +//////////////////// C2 //////////////////// + class MMTkObjectBarrierSetC2: public MMTkUnlogBitBarrierSetC2 { protected: virtual void object_reference_write_post(GraphKit* kit, Node* src, Node* slot, Node* val) const override; }; +//////////////////// Impl //////////////////// + struct MMTkObjectBarrier: MMTkBarrierImpl< MMTkObjectBarrierSetRuntime, MMTkObjectBarrierSetAssembler, diff --git a/openjdk/barriers/mmtkSATBBarrier.cpp b/openjdk/barriers/mmtkSATBBarrier.cpp index c567da9b..bd829b55 100644 --- a/openjdk/barriers/mmtkSATBBarrier.cpp +++ b/openjdk/barriers/mmtkSATBBarrier.cpp @@ -2,6 +2,8 @@ #include "mmtkSATBBarrier.hpp" #include "runtime/interfaceSupport.inline.hpp" +//////////////////// Runtime //////////////////// + void MMTkSATBBarrierSetRuntime::load_reference(DecoratorSet decorators, oop value) const { if (mmtk_enable_reference_load_barrier) { if (CONCURRENT_MARKING_ACTIVE == 1 && value != NULL) @@ -26,6 +28,8 @@ void MMTkSATBBarrierSetRuntime::object_reference_write_pre(oop src, oop* slot, o } } +//////////////////// Assembler //////////////////// + #define __ masm-> void MMTkSATBBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp_thread) { @@ -87,6 +91,8 @@ void MMTkSATBBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Decor #undef __ +//////////////////// Assembler/C1 //////////////////// + #ifdef ASSERT #define __ gen->lir(__FILE__, __LINE__)-> #else @@ -139,6 +145,8 @@ void MMTkSATBBarrierSetC1::object_reference_write_pre(LIRAccess& access, LIR_Opr #undef __ +//////////////////// C2 //////////////////// + #define __ ideal. void MMTkSATBBarrierSetC2::object_reference_write_pre(GraphKit* kit, Node* src, Node* slot, Node* val) const { diff --git a/openjdk/barriers/mmtkSATBBarrier.hpp b/openjdk/barriers/mmtkSATBBarrier.hpp index db8fea1a..b92a39c2 100644 --- a/openjdk/barriers/mmtkSATBBarrier.hpp +++ b/openjdk/barriers/mmtkSATBBarrier.hpp @@ -13,6 +13,12 @@ #include "opto/callnode.hpp" #include "opto/idealKit.hpp" +/// This file supports the `SATBBarrier` in MMTk core, +/// i.e. the barrier that remembers all children before it is first modified, +/// and remembers the target of the weak reference field when loaded. + +//////////////////// Runtime //////////////////// + class MMTkSATBBarrierSetRuntime: public MMTkUnlogBitBarrierSetRuntime { public: // Interfaces called by `MMTkBarrierSet::AccessBarrier` @@ -25,6 +31,8 @@ class MMTkSATBBarrierSetRuntime: public MMTkUnlogBitBarrierSetRuntime { virtual void load_reference(DecoratorSet decorators, oop value) const override; }; +//////////////////// Assembler //////////////////// + class MMTkSATBBarrierSetAssembler: public MMTkUnlogBitBarrierSetAssembler { protected: virtual void object_reference_write_pre(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2) const override; @@ -33,6 +41,8 @@ class MMTkSATBBarrierSetAssembler: public MMTkUnlogBitBarrierSetAssembler { virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp_thread) override; }; +//////////////////// C1 //////////////////// + class MMTkSATBBarrierSetC1: public MMTkUnlogBitBarrierSetC1 { protected: virtual void object_reference_write_pre(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val) const override; @@ -44,6 +54,8 @@ class MMTkSATBBarrierSetC1: public MMTkUnlogBitBarrierSetC1 { } }; +//////////////////// C2 //////////////////// + class MMTkSATBBarrierSetC2: public MMTkUnlogBitBarrierSetC2 { protected: virtual void object_reference_write_pre(GraphKit* kit, Node* src, Node* slot, Node* val) const override; @@ -55,6 +67,8 @@ class MMTkSATBBarrierSetC2: public MMTkUnlogBitBarrierSetC2 { virtual Node* load_at_resolved(C2Access& access, const Type* val_type) const override; }; +//////////////////// Impl //////////////////// + struct MMTkSATBBarrier: MMTkBarrierImpl< MMTkSATBBarrierSetRuntime, MMTkSATBBarrierSetAssembler, From 1c6990b92d1e5b322866316b63be098c058ebb5f Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Wed, 24 Sep 2025 17:42:30 +0800 Subject: [PATCH 39/46] Bump mmtk-core revision --- mmtk/Cargo.lock | 8 ++++---- mmtk/Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 6025343c..d89d564d 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -430,9 +430,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.175" +version = "0.2.176" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" +checksum = "58f929b4d672ea937a23a1ab494143d968337a5f47e56d0815df1e0890ddf174" [[package]] name = "libgit2-sys" @@ -492,7 +492,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.31.0" -source = "git+https://github.com/mmtk/mmtk-core.git?rev=27b4521ca73c2addbafe7ccf374c607d61f3ba51#27b4521ca73c2addbafe7ccf374c607d61f3ba51" +source = "git+https://github.com/mmtk/mmtk-core.git?rev=a2cf101bc1841a6e3856262d6f757469ff2aa359#a2cf101bc1841a6e3856262d6f757469ff2aa359" dependencies = [ "atomic", "atomic-traits", @@ -531,7 +531,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.31.0" -source = "git+https://github.com/mmtk/mmtk-core.git?rev=27b4521ca73c2addbafe7ccf374c607d61f3ba51#27b4521ca73c2addbafe7ccf374c607d61f3ba51" +source = "git+https://github.com/mmtk/mmtk-core.git?rev=a2cf101bc1841a6e3856262d6f757469ff2aa359#a2cf101bc1841a6e3856262d6f757469ff2aa359" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 21eab63d..896a5029 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -36,7 +36,7 @@ probe = "0.5" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "27b4521ca73c2addbafe7ccf374c607d61f3ba51" } +mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "a2cf101bc1841a6e3856262d6f757469ff2aa359" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } From 1770104a0d25803c306364829f54ad5494a2fd62 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Wed, 24 Sep 2025 17:48:05 +0800 Subject: [PATCH 40/46] Remove new-idea comment Maybe start an issue instead. --- openjdk/barriers/mmtkSATBBarrier.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/openjdk/barriers/mmtkSATBBarrier.cpp b/openjdk/barriers/mmtkSATBBarrier.cpp index bd829b55..83d957bf 100644 --- a/openjdk/barriers/mmtkSATBBarrier.cpp +++ b/openjdk/barriers/mmtkSATBBarrier.cpp @@ -137,9 +137,6 @@ void MMTkSATBBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) { } void MMTkSATBBarrierSetC1::object_reference_write_pre(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val) const { - // TODO: Should be able to skip weak field writes, too, because that doesn't change strong - // reachability during concurrent marking. - object_reference_write_pre_or_post(access, src, true); } From 3ee91126dcdf3c9b2f032cddcbe16d2fe89b691c Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Wed, 24 Sep 2025 17:49:25 +0800 Subject: [PATCH 41/46] Don't disrupt the code order too much for no reason --- openjdk/mmtkBarrierSetAssembler_x86.cpp | 42 ++++++++++++------------- openjdk/mmtkBarrierSetAssembler_x86.hpp | 8 ++--- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/openjdk/mmtkBarrierSetAssembler_x86.cpp b/openjdk/mmtkBarrierSetAssembler_x86.cpp index de235c59..fdb4ae9b 100644 --- a/openjdk/mmtkBarrierSetAssembler_x86.cpp +++ b/openjdk/mmtkBarrierSetAssembler_x86.cpp @@ -144,27 +144,6 @@ void MMTkBarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register threa //////////////////// Assembler for C1 //////////////////// -// Generate code stubs - -#define __ ce->masm()-> - -void MMTkBarrierSetAssembler::generate_c1_ref_load_barrier_stub_call(LIR_Assembler* ce, MMTkC1ReferenceLoadBarrierStub* stub) { - MMTkBarrierSetC1* bs = (MMTkBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); - - __ bind(*stub->entry()); - assert(stub->val->is_register(), "Precondition."); - - Register val_reg = stub->val->as_register(); - - __ cmpptr(val_reg, (int32_t) NULL_WORD); - __ jcc(Assembler::equal, *stub->continuation()); - ce->store_parameter(stub->val->as_register(), 0); - __ call(RuntimeAddress(bs->load_reference_c1_runtime_code_blob()->code_begin())); - __ jmp(*stub->continuation()); -} - -#undef __ - // Generate runtime stubs for the "runtime code blobs" in MMTkBarrierSetC1 #define __ sasm-> @@ -203,3 +182,24 @@ void MMTkBarrierSetAssembler::generate_c1_object_reference_write_slow_runtime_st } #undef __ + +// Generate code stubs + +#define __ ce->masm()-> + +void MMTkBarrierSetAssembler::generate_c1_ref_load_barrier_stub_call(LIR_Assembler* ce, MMTkC1ReferenceLoadBarrierStub* stub) { + MMTkBarrierSetC1* bs = (MMTkBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); + + __ bind(*stub->entry()); + assert(stub->val->is_register(), "Precondition."); + + Register val_reg = stub->val->as_register(); + + __ cmpptr(val_reg, (int32_t) NULL_WORD); + __ jcc(Assembler::equal, *stub->continuation()); + ce->store_parameter(stub->val->as_register(), 0); + __ call(RuntimeAddress(bs->load_reference_c1_runtime_code_blob()->code_begin())); + __ jmp(*stub->continuation()); +} + +#undef __ diff --git a/openjdk/mmtkBarrierSetAssembler_x86.hpp b/openjdk/mmtkBarrierSetAssembler_x86.hpp index 7f068c8b..8487c7e9 100644 --- a/openjdk/mmtkBarrierSetAssembler_x86.hpp +++ b/openjdk/mmtkBarrierSetAssembler_x86.hpp @@ -37,10 +37,6 @@ class MMTkBarrierSetAssembler: public BarrierSetAssembler { //////////////////// Assembler for C1 //////////////////// - // Generate code stubs -public: - static void generate_c1_ref_load_barrier_stub_call(LIR_Assembler* ce, MMTkC1ReferenceLoadBarrierStub* stub); - // Generate runtime stubs for the "runtime code blobs" in MMTkBarrierSetC1 private: static void generate_c1_runtime_stub_general(StubAssembler* sasm, const char* name, address func, int argc); @@ -49,5 +45,9 @@ class MMTkBarrierSetAssembler: public BarrierSetAssembler { static void generate_c1_object_reference_write_pre_runtime_stub(StubAssembler* sasm); static void generate_c1_object_reference_write_post_runtime_stub(StubAssembler* sasm); static void generate_c1_object_reference_write_slow_runtime_stub(StubAssembler* sasm); + + // Generate slow-path code stubs +public: + static void generate_c1_ref_load_barrier_stub_call(LIR_Assembler* ce, MMTkC1ReferenceLoadBarrierStub* stub); }; #endif // MMTK_OPENJDK_MMTK_BARRIER_SET_ASSEMBLER_X86_HPP From 85c61b30e0777d3b35fa5aefc7a8cf334399b38c Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Thu, 25 Sep 2025 11:04:17 +0800 Subject: [PATCH 42/46] Cite the source of reference_load_barrier_for_unknown_load --- openjdk/barriers/mmtkSATBBarrier.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/openjdk/barriers/mmtkSATBBarrier.cpp b/openjdk/barriers/mmtkSATBBarrier.cpp index 83d957bf..bfe59e3f 100644 --- a/openjdk/barriers/mmtkSATBBarrier.cpp +++ b/openjdk/barriers/mmtkSATBBarrier.cpp @@ -176,6 +176,12 @@ static void reference_load_barrier(GraphKit* kit, Node* slot, Node* val, bool em } static void reference_load_barrier_for_unknown_load(GraphKit* kit, Node* base_oop, Node* offset, Node* slot, Node* val, bool need_mem_bar) { + // Note: This function is copied from G1BarrierSetC2::insert_pre_barrier, + // and ShenandoahBarrierSetC2::insert_pre_barrier is probably copied from G1 as well. + // It basically implements BarrierSetC1::generate_referent_check in C2 IR. + // TODO: If another barrier needs weak reference load barrier, + // consider hoisting this function to a superclass. + // We could be accessing the referent field of a reference object. If so, when G1 // is enabled, we need to log the value in the referent field in an SATB buffer. // This routine performs some compile time filters and generates suitable From a31a3bae53ad8d49fd874c4a0bd5c581ea904f21 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Thu, 25 Sep 2025 15:13:22 +0800 Subject: [PATCH 43/46] Add missing /* pre = */ comment --- openjdk/barriers/mmtkObjectBarrier.cpp | 2 +- openjdk/barriers/mmtkSATBBarrier.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openjdk/barriers/mmtkObjectBarrier.cpp b/openjdk/barriers/mmtkObjectBarrier.cpp index 7f67da1d..e11bb770 100644 --- a/openjdk/barriers/mmtkObjectBarrier.cpp +++ b/openjdk/barriers/mmtkObjectBarrier.cpp @@ -93,7 +93,7 @@ void MMTkObjectBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, Dec #endif void MMTkObjectBarrierSetC1::object_reference_write_post(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val) const { - object_reference_write_pre_or_post(access, src, false); + object_reference_write_pre_or_post(access, src, /* pre = */ false); } #undef __ diff --git a/openjdk/barriers/mmtkSATBBarrier.cpp b/openjdk/barriers/mmtkSATBBarrier.cpp index bfe59e3f..6823815a 100644 --- a/openjdk/barriers/mmtkSATBBarrier.cpp +++ b/openjdk/barriers/mmtkSATBBarrier.cpp @@ -137,7 +137,7 @@ void MMTkSATBBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) { } void MMTkSATBBarrierSetC1::object_reference_write_pre(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val) const { - object_reference_write_pre_or_post(access, src, true); + object_reference_write_pre_or_post(access, src, /* pre = */ true); } #undef __ From 63bffddaad4f69f47e9bd14bac146cb5ec7406dd Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Thu, 25 Sep 2025 16:19:00 +0800 Subject: [PATCH 44/46] Fix comment The code following the comment is just for C1, not assembler that emits stubs for C1. --- openjdk/barriers/mmtkSATBBarrier.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openjdk/barriers/mmtkSATBBarrier.cpp b/openjdk/barriers/mmtkSATBBarrier.cpp index 6823815a..960fc33e 100644 --- a/openjdk/barriers/mmtkSATBBarrier.cpp +++ b/openjdk/barriers/mmtkSATBBarrier.cpp @@ -91,7 +91,7 @@ void MMTkSATBBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Decor #undef __ -//////////////////// Assembler/C1 //////////////////// +//////////////////// C1 //////////////////// #ifdef ASSERT #define __ gen->lir(__FILE__, __LINE__)-> From 940c6915a8d39661cb78112afc97ef4b80cb1305 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Thu, 16 Oct 2025 16:40:11 +0800 Subject: [PATCH 45/46] Shouldn't remove SATB barrier if val is null --- openjdk/barriers/mmtkSATBBarrier.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openjdk/barriers/mmtkSATBBarrier.cpp b/openjdk/barriers/mmtkSATBBarrier.cpp index 960fc33e..4a564f9b 100644 --- a/openjdk/barriers/mmtkSATBBarrier.cpp +++ b/openjdk/barriers/mmtkSATBBarrier.cpp @@ -69,7 +69,7 @@ void MMTkSATBBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet dec } void MMTkSATBBarrierSetAssembler::object_reference_write_pre(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2) const { - if (can_remove_barrier(decorators, val, /* skip_const_null */ true)) return; + if (can_remove_barrier(decorators, val, /* skip_const_null */ false)) return; object_reference_write_pre_or_post(masm, decorators, dst, val, /* pre = */ true); } From 9f5efe0c8644bbd07faf314b4c5ffd58d822f189 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Fri, 17 Oct 2025 18:28:34 +0800 Subject: [PATCH 46/46] Bump mmtk-core revision --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index d89d564d..6a86e04b 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -492,7 +492,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.31.0" -source = "git+https://github.com/mmtk/mmtk-core.git?rev=a2cf101bc1841a6e3856262d6f757469ff2aa359#a2cf101bc1841a6e3856262d6f757469ff2aa359" +source = "git+https://github.com/mmtk/mmtk-core.git?rev=1ffa5b325a07d4fe99fe7e7008b295f8febea407#1ffa5b325a07d4fe99fe7e7008b295f8febea407" dependencies = [ "atomic", "atomic-traits", @@ -531,7 +531,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.31.0" -source = "git+https://github.com/mmtk/mmtk-core.git?rev=a2cf101bc1841a6e3856262d6f757469ff2aa359#a2cf101bc1841a6e3856262d6f757469ff2aa359" +source = "git+https://github.com/mmtk/mmtk-core.git?rev=1ffa5b325a07d4fe99fe7e7008b295f8febea407#1ffa5b325a07d4fe99fe7e7008b295f8febea407" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 896a5029..02a6369e 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -36,7 +36,7 @@ probe = "0.5" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "a2cf101bc1841a6e3856262d6f757469ff2aa359" } +mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "1ffa5b325a07d4fe99fe7e7008b295f8febea407" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" }