From 431ef1c008954367bdc191b16bd4329d38d461a6 Mon Sep 17 00:00:00 2001 From: Evgeny Astigeevich Date: Fri, 14 Nov 2025 17:57:11 +0000 Subject: [PATCH 01/18] 8370947: Mitigate Neoverse-N1 erratum 1542419 negative impact on GenZGC performance --- .../gc/z/zBarrierSetAssembler_aarch64.cpp | 8 ++++- .../gc/z/zBarrierSetAssembler_aarch64.hpp | 2 +- src/hotspot/cpu/aarch64/globals_aarch64.hpp | 2 ++ src/hotspot/cpu/aarch64/icache_aarch64.hpp | 36 +++++++++++++++++++ src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp | 7 ++++ .../cpu/aarch64/vm_version_aarch64.cpp | 11 ++++++ src/hotspot/share/code/nmethod.cpp | 3 +- src/hotspot/share/code/nmethod.hpp | 3 +- src/hotspot/share/code/relocInfo.hpp | 7 ++++ src/hotspot/share/gc/z/zMark.cpp | 19 ++++++---- src/hotspot/share/gc/z/zNMethod.cpp | 8 ++--- src/hotspot/share/gc/z/zNMethod.hpp | 4 +-- src/hotspot/share/runtime/icache.hpp | 32 +++++++++++++++++ 13 files changed, 125 insertions(+), 17 deletions(-) diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp index 5d4f0801ec62f..08814b3e6983f 100644 --- a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp @@ -862,7 +862,7 @@ static void change_immediate(uint32_t& instr, uint32_t imm, uint32_t start, uint instr |= imm << start; } -void ZBarrierSetAssembler::patch_barrier_relocation(address addr, int format) { +void ZBarrierSetAssembler::patch_barrier_relocation(address addr, int format, bool defer_icache_invalidation) { const uint16_t value = patch_barrier_relocation_value(format); uint32_t* const patch_addr = (uint32_t*)addr; @@ -879,6 +879,12 @@ void ZBarrierSetAssembler::patch_barrier_relocation(address addr, int format) { ShouldNotReachHere(); } + if (defer_icache_invalidation) { + // Instruction cache invalidation per barrier can be expensive, e.g. on Neoverse N1 having erratum 1542419. + // Defer the ICache invalidation to a later point where multiple patches can be handled together. + return; + } + OrderAccess::fence(); ICache::invalidate_word((address)patch_addr); } diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp index ae2819e78cad2..c852c567d1d8b 100644 --- a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp @@ -154,7 +154,7 @@ class ZBarrierSetAssembler : public ZBarrierSetAssemblerBase { virtual NMethodPatchingType nmethod_patching_type() { return NMethodPatchingType::conc_instruction_and_data_patch; } - void patch_barrier_relocation(address addr, int format); + void patch_barrier_relocation(address addr, int format, bool defer_icache_invalidation = false); void patch_barriers() {} diff --git a/src/hotspot/cpu/aarch64/globals_aarch64.hpp b/src/hotspot/cpu/aarch64/globals_aarch64.hpp index 8e520314c8b6f..430316360f6fe 100644 --- a/src/hotspot/cpu/aarch64/globals_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/globals_aarch64.hpp @@ -127,6 +127,8 @@ define_pd_global(intx, InlineSmallCode, 1000); "Branch Protection to use: none, standard, pac-ret") \ product(bool, AlwaysMergeDMB, true, DIAGNOSTIC, \ "Always merge DMB instructions in code emission") \ + product(bool, NeoverseN1Errata1542419, false, DIAGNOSTIC, \ + "Enable workaround for Neoverse N1 erratum 1542419") \ // end of ARCH_FLAGS diff --git a/src/hotspot/cpu/aarch64/icache_aarch64.hpp b/src/hotspot/cpu/aarch64/icache_aarch64.hpp index 9fd0e8575cbe7..03a0157685145 100644 --- a/src/hotspot/cpu/aarch64/icache_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/icache_aarch64.hpp @@ -28,4 +28,40 @@ #include OS_CPU_HEADER(icache) +#include "utilities/globalDefinitions.hpp" + +#define PD_ICACHE_INVALIDATION_CONTEXT + +inline void ICacheInvalidationContext::pd_init(nmethod* nm) { + if (NeoverseN1Errata1542419) { + _nm = nm; + } +} + +inline void ICacheInvalidationContext::pd_invalidate_icache() { + if (_nm != nullptr) { + assert(NeoverseN1Errata1542419, "Should only be set for Neoverse N1 erratum"); + // Neoverse-N1 implementation mitigates erratum 1542419 with a workaround: + // - Disable coherent icache. + // - Trap IC IVAU instructions. + // - Execute: + // - tlbi vae3is, xzr + // - dsb sy + // + // `tlbi vae3is, xzr` invalidates translations for all address spaces (global for address). + // It waits for all memory accesses using in-scope old translation information to complete + // before it is considered complete. + // + // As this workaround has significant overhead, Arm Neoverse N1 (MP050) Software Developer + // Errata Notice version 29.0 suggests: + // + // "Since one TLB inner-shareable invalidation is enough to avoid this erratum, the number + // of injected TLB invalidations should be minimized in the trap handler to mitigate + // the performance impact due to this workaround." + // + // As the addrress for icache invalidation is not relevant, we use the nmethod's code start address. + ICache::invalidate_word(_nm->code_begin()); + } +} + #endif // CPU_AARCH64_ICACHE_AARCH64_HPP diff --git a/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp b/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp index 94694b58d2fae..f939c18c6f78e 100644 --- a/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp @@ -54,6 +54,13 @@ void Relocation::pd_set_data_value(address x, bool verify_only) { bytes = MacroAssembler::pd_patch_instruction_size(addr(), x); break; } + + if (binding() != nullptr && binding()->deferred_icache_invalidation()) { + // Instruction cache invalidation per relocation can be expensive, e.g. on Neoverse N1 having erratum 1542419. + // Defer the ICache invalidation to a later point where multiple patches can be handled together. + return; + } + ICache::invalidate_range(addr(), bytes); } diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp index a04e9defa4b09..a4327f5327590 100644 --- a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp @@ -641,6 +641,17 @@ void VM_Version::initialize() { clear_feature(CPU_SVE); } + // Neoverse N1: 0xd0c + if (_cpu == CPU_ARM && model_is(0xd0c) && FLAG_IS_DEFAULT(NeoverseN1Errata1542419)) { + const int major_rev_num = cpu_variant(); + const int minor_rev_num = cpu_revision(); + if (!(major_rev_num >= 4 && minor_rev_num >= 1)) { + // As Neoverse N1 r4p1 and later are not affected by the erratum, + // enable the workaround by default for earlier revisions. + FLAG_SET_DEFAULT(NeoverseN1Errata1542419, true); + } + } + // Construct the "features" string stringStream ss(512); ss.print("0x%02x:0x%x:0x%03x:%d", _cpu, _variant, _model, _revision); diff --git a/src/hotspot/share/code/nmethod.cpp b/src/hotspot/share/code/nmethod.cpp index d91af9b4991e1..eab506ab660bc 100644 --- a/src/hotspot/share/code/nmethod.cpp +++ b/src/hotspot/share/code/nmethod.cpp @@ -2032,9 +2032,10 @@ void nmethod::copy_values(GrowableArray* array) { } } -void nmethod::fix_oop_relocations(address begin, address end, bool initialize_immediates) { +void nmethod::fix_oop_relocations(address begin, address end, bool initialize_immediates, bool defer_icache_invalidation) { // re-patch all oop-bearing instructions, just in case some oops moved RelocIterator iter(this, begin, end); + AARCH64_ONLY(iter.set_deferred_icache_invalidation(defer_icache_invalidation);) while (iter.next()) { if (iter.type() == relocInfo::oop_type) { oop_Relocation* reloc = iter.oop_reloc(); diff --git a/src/hotspot/share/code/nmethod.hpp b/src/hotspot/share/code/nmethod.hpp index 34accf428b690..fbc092a4814c2 100644 --- a/src/hotspot/share/code/nmethod.hpp +++ b/src/hotspot/share/code/nmethod.hpp @@ -799,7 +799,7 @@ class nmethod : public CodeBlob { // Relocation support private: - void fix_oop_relocations(address begin, address end, bool initialize_immediates); + void fix_oop_relocations(address begin, address end, bool initialize_immediates, bool defer_icache_invalidation = false); inline void initialize_immediate_oop(oop* dest, jobject handle); protected: @@ -808,6 +808,7 @@ class nmethod : public CodeBlob { public: void fix_oop_relocations(address begin, address end) { fix_oop_relocations(begin, end, false); } void fix_oop_relocations() { fix_oop_relocations(nullptr, nullptr, false); } + void fix_oop_relocations(bool defer_icache_invalidation) { fix_oop_relocations(nullptr, nullptr, false, defer_icache_invalidation); } bool is_at_poll_return(address pc); bool is_at_poll_or_poll_return(address pc); diff --git a/src/hotspot/share/code/relocInfo.hpp b/src/hotspot/share/code/relocInfo.hpp index a6a08815d1055..dad0cf8c300df 100644 --- a/src/hotspot/share/code/relocInfo.hpp +++ b/src/hotspot/share/code/relocInfo.hpp @@ -569,6 +569,8 @@ class RelocIterator : public StackObj { short* _data; // pointer to the relocation's data short _datalen; // number of halfwords in _data + AARCH64_ONLY(bool _deferred_icache_invalidation;) + // Base addresses needed to compute targets of section_word_type relocs. address _section_start[SECT_LIMIT]; address _section_end [SECT_LIMIT]; @@ -639,6 +641,11 @@ class RelocIterator : public StackObj { bool has_current() const { return _datalen >= 0; } bool addr_in_const() const; +#ifdef AARCH64 + bool deferred_icache_invalidation() const { return _deferred_icache_invalidation; } + void set_deferred_icache_invalidation(bool b) { _deferred_icache_invalidation = b; } +#endif + address section_start(int n) const { assert(_section_start[n], "section %d must be initialized", n); return _section_start[n]; diff --git a/src/hotspot/share/gc/z/zMark.cpp b/src/hotspot/share/gc/z/zMark.cpp index 3b247fdd35e45..7d0164747c35c 100644 --- a/src/hotspot/share/gc/z/zMark.cpp +++ b/src/hotspot/share/gc/z/zMark.cpp @@ -60,6 +60,7 @@ #include "runtime/atomicAccess.hpp" #include "runtime/continuation.hpp" #include "runtime/handshake.hpp" +#include "runtime/icache.hpp" #include "runtime/javaThread.hpp" #include "runtime/prefetch.inline.hpp" #include "runtime/safepointMechanism.hpp" @@ -753,10 +754,6 @@ class ZMarkYoungNMethodClosure : public NMethodClosure { if (_bs_nm->is_armed(nm)) { const uintptr_t prev_color = ZNMethod::color(nm); - // Heal oops - ZUncoloredRootMarkYoungOopClosure cl(prev_color); - ZNMethod::nmethod_oops_do_inner(nm, &cl); - // Disarm only the young marking, not any potential old marking cycle const uintptr_t old_marked_mask = ZPointerMarkedMask ^ (ZPointerMarkedYoung0 | ZPointerMarkedYoung1); @@ -767,9 +764,17 @@ class ZMarkYoungNMethodClosure : public NMethodClosure { // Check if disarming for young mark, completely disarms the nmethod entry barrier const bool complete_disarm = ZPointer::is_store_good(new_disarm_value_ptr); - if (complete_disarm) { - // We are about to completely disarm the nmethod, must take responsibility to patch all barriers before disarming - ZNMethod::nmethod_patch_barriers(nm); + { + ICacheInvalidationContext icic(nm); + + if (complete_disarm) { + // We are about to completely disarm the nmethod, must take responsibility to patch all barriers before disarming + ZNMethod::nmethod_patch_barriers(nm, icic.deferred_invalidation()); + } + + // Heal oops + ZUncoloredRootMarkYoungOopClosure cl(prev_color); + ZNMethod::nmethod_oops_do_inner(nm, &cl, icic.deferred_invalidation()); } _bs_nm->guard_with(nm, (int)untype(new_disarm_value_ptr)); diff --git a/src/hotspot/share/gc/z/zNMethod.cpp b/src/hotspot/share/gc/z/zNMethod.cpp index 780bc9e3bf759..c743cb2496ca5 100644 --- a/src/hotspot/share/gc/z/zNMethod.cpp +++ b/src/hotspot/share/gc/z/zNMethod.cpp @@ -244,11 +244,11 @@ void ZNMethod::set_guard_value(nmethod* nm, int value) { bs->guard_with(nm, value); } -void ZNMethod::nmethod_patch_barriers(nmethod* nm) { +void ZNMethod::nmethod_patch_barriers(nmethod* nm, bool defer_icache_invalidation) { ZBarrierSetAssembler* const bs_asm = ZBarrierSet::assembler(); ZArrayIterator iter(gc_data(nm)->barriers()); for (ZNMethodDataBarrier barrier; iter.next(&barrier);) { - bs_asm->patch_barrier_relocation(barrier._reloc_addr, barrier._reloc_format); + bs_asm->patch_barrier_relocation(barrier._reloc_addr, barrier._reloc_format AARCH64_ONLY(COMMA defer_icache_invalidation)); } } @@ -257,7 +257,7 @@ void ZNMethod::nmethod_oops_do(nmethod* nm, OopClosure* cl) { ZNMethod::nmethod_oops_do_inner(nm, cl); } -void ZNMethod::nmethod_oops_do_inner(nmethod* nm, OopClosure* cl) { +void ZNMethod::nmethod_oops_do_inner(nmethod* nm, OopClosure* cl, bool defer_icache_invalidation) { // Process oops table { oop* const begin = nm->oops_begin(); @@ -283,7 +283,7 @@ void ZNMethod::nmethod_oops_do_inner(nmethod* nm, OopClosure* cl) { // Process non-immediate oops if (data->has_non_immediate_oops()) { - nm->fix_oop_relocations(); + nm->fix_oop_relocations(defer_icache_invalidation); } } diff --git a/src/hotspot/share/gc/z/zNMethod.hpp b/src/hotspot/share/gc/z/zNMethod.hpp index 5d797f3fd5542..bc1e7c005c0f8 100644 --- a/src/hotspot/share/gc/z/zNMethod.hpp +++ b/src/hotspot/share/gc/z/zNMethod.hpp @@ -55,10 +55,10 @@ class ZNMethod : public AllStatic { static void disarm(nmethod* nm); static void set_guard_value(nmethod* nm, int value); - static void nmethod_patch_barriers(nmethod* nm); + static void nmethod_patch_barriers(nmethod* nm, bool defer_icache_invalidation = false); static void nmethod_oops_do(nmethod* nm, OopClosure* cl); - static void nmethod_oops_do_inner(nmethod* nm, OopClosure* cl); + static void nmethod_oops_do_inner(nmethod* nm, OopClosure* cl, bool defer_icache_invalidation = false); static void nmethods_do_begin(bool secondary); static void nmethods_do_end(bool secondary); diff --git a/src/hotspot/share/runtime/icache.hpp b/src/hotspot/share/runtime/icache.hpp index e442c4d88e00a..2128d4b88d161 100644 --- a/src/hotspot/share/runtime/icache.hpp +++ b/src/hotspot/share/runtime/icache.hpp @@ -71,6 +71,32 @@ class AbstractICache : AllStatic { static void invalidate_range(address start, int nbytes); }; +class nmethod; + +class ICacheInvalidationContext : StackObj { + NONCOPYABLE(ICacheInvalidationContext); + +private: + nmethod* _nm; + + void pd_init(nmethod* nm); + void pd_invalidate_icache(); + +public: + ICacheInvalidationContext(nmethod* nm) : _nm(nullptr) { + pd_init(nm); + } + + ~ICacheInvalidationContext() { + if (_nm != nullptr) { + pd_invalidate_icache(); + } + } + + bool deferred_invalidation() const { + return _nm != nullptr; + } +}; // Must be included before the definition of ICacheStubGenerator // because ICacheStubGenerator uses ICache definitions. @@ -129,4 +155,10 @@ class ICacheStubGenerator : public StubCodeGenerator { void generate_icache_flush(ICache::flush_icache_stub_t* flush_icache_stub); }; +#ifndef PD_ICACHE_INVALIDATION_CONTEXT + // Default implementation: do nothing + inline void ICacheInvalidationContext::pd_init(nmethod*) {} + inline void ICacheInvalidationContext::pd_invalidate_icache() {} +#endif + #endif // SHARE_RUNTIME_ICACHE_HPP From 2094d1937202bba322b68f86b7381e217dfb7cf3 Mon Sep 17 00:00:00 2001 From: Evgeny Astigeevich Date: Thu, 20 Nov 2025 14:29:27 +0000 Subject: [PATCH 02/18] Add deferred icache invalidation to all places; Add JMH microbench --- src/hotspot/cpu/aarch64/icache_aarch64.hpp | 3 +- src/hotspot/share/gc/z/zBarrierSetNMethod.cpp | 15 +- src/hotspot/share/gc/z/zGeneration.cpp | 16 +- src/hotspot/share/gc/z/zMark.cpp | 13 +- src/hotspot/share/gc/z/zNMethod.cpp | 18 +- src/hotspot/share/gc/z/zUnload.cpp | 4 +- .../bench/vm/gc/GCPatchingNmethodCost.java | 205 ++++++++++++++++++ 7 files changed, 251 insertions(+), 23 deletions(-) create mode 100644 test/micro/org/openjdk/bench/vm/gc/GCPatchingNmethodCost.java diff --git a/src/hotspot/cpu/aarch64/icache_aarch64.hpp b/src/hotspot/cpu/aarch64/icache_aarch64.hpp index 03a0157685145..5095512c1ec5d 100644 --- a/src/hotspot/cpu/aarch64/icache_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/icache_aarch64.hpp @@ -28,6 +28,7 @@ #include OS_CPU_HEADER(icache) +#include "code/nmethod.hpp" #include "utilities/globalDefinitions.hpp" #define PD_ICACHE_INVALIDATION_CONTEXT @@ -59,7 +60,7 @@ inline void ICacheInvalidationContext::pd_invalidate_icache() { // of injected TLB invalidations should be minimized in the trap handler to mitigate // the performance impact due to this workaround." // - // As the addrress for icache invalidation is not relevant, we use the nmethod's code start address. + // As the address for icache invalidation is not relevant, we use the nmethod's code start address. ICache::invalidate_word(_nm->code_begin()); } } diff --git a/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp b/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp index d80ce4e149d09..eecdcedf0ee46 100644 --- a/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp +++ b/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp @@ -33,6 +33,7 @@ #include "gc/z/zThreadLocalData.hpp" #include "gc/z/zUncoloredRoot.inline.hpp" #include "logging/log.hpp" +#include "runtime/icache.hpp" #include "runtime/threadWXSetters.inline.hpp" bool ZBarrierSetNMethod::nmethod_entry_barrier(nmethod* nm) { @@ -70,12 +71,16 @@ bool ZBarrierSetNMethod::nmethod_entry_barrier(nmethod* nm) { return false; } - // Heal barriers - ZNMethod::nmethod_patch_barriers(nm); + { + ICacheInvalidationContext icic(nm); - // Heal oops - ZUncoloredRootProcessWeakOopClosure cl(ZNMethod::color(nm)); - ZNMethod::nmethod_oops_do_inner(nm, &cl); + // Heal barriers + ZNMethod::nmethod_patch_barriers(nm, icic.deferred_invalidation()); + + // Heal oops + ZUncoloredRootProcessWeakOopClosure cl(ZNMethod::color(nm)); + ZNMethod::nmethod_oops_do_inner(nm, &cl, icic.deferred_invalidation()); + } const uintptr_t prev_color = ZNMethod::color(nm); const uintptr_t new_color = *ZPointerStoreGoodMaskLowOrderBitsAddr; diff --git a/src/hotspot/share/gc/z/zGeneration.cpp b/src/hotspot/share/gc/z/zGeneration.cpp index d1680b6c33650..1f63188a4ad14 100644 --- a/src/hotspot/share/gc/z/zGeneration.cpp +++ b/src/hotspot/share/gc/z/zGeneration.cpp @@ -59,6 +59,7 @@ #include "runtime/atomicAccess.hpp" #include "runtime/continuation.hpp" #include "runtime/handshake.hpp" +#include "runtime/icache.hpp" #include "runtime/safepoint.hpp" #include "runtime/threads.hpp" #include "runtime/vmOperations.hpp" @@ -1424,12 +1425,15 @@ class ZRemapNMethodClosure : public NMethodClosure { virtual void do_nmethod(nmethod* nm) { ZLocker locker(ZNMethod::lock_for_nmethod(nm)); if (_bs_nm->is_armed(nm)) { - // Heal barriers - ZNMethod::nmethod_patch_barriers(nm); - - // Heal oops - ZUncoloredRootProcessOopClosure cl(ZNMethod::color(nm)); - ZNMethod::nmethod_oops_do_inner(nm, &cl); + { + ICacheInvalidationContext icic(nm); + // Heal barriers + ZNMethod::nmethod_patch_barriers(nm, icic.deferred_invalidation()); + + // Heal oops + ZUncoloredRootProcessOopClosure cl(ZNMethod::color(nm)); + ZNMethod::nmethod_oops_do_inner(nm, &cl, icic.deferred_invalidation()); + } log_trace(gc, nmethod)("nmethod: " PTR_FORMAT " visited by old remapping", p2i(nm)); diff --git a/src/hotspot/share/gc/z/zMark.cpp b/src/hotspot/share/gc/z/zMark.cpp index 7d0164747c35c..48bde6db2195a 100644 --- a/src/hotspot/share/gc/z/zMark.cpp +++ b/src/hotspot/share/gc/z/zMark.cpp @@ -719,12 +719,15 @@ class ZMarkNMethodClosure : public NMethodClosure { virtual void do_nmethod(nmethod* nm) { ZLocker locker(ZNMethod::lock_for_nmethod(nm)); if (_bs_nm->is_armed(nm)) { - // Heal barriers - ZNMethod::nmethod_patch_barriers(nm); + { + ICacheInvalidationContext icic(nm); + // Heal barriers + ZNMethod::nmethod_patch_barriers(nm, icic.deferred_invalidation()); - // Heal oops - ZUncoloredRootMarkOopClosure cl(ZNMethod::color(nm)); - ZNMethod::nmethod_oops_do_inner(nm, &cl); + // Heal oops + ZUncoloredRootMarkOopClosure cl(ZNMethod::color(nm)); + ZNMethod::nmethod_oops_do_inner(nm, &cl, icic.deferred_invalidation()); + } // CodeCache unloading support nm->mark_as_maybe_on_stack(); diff --git a/src/hotspot/share/gc/z/zNMethod.cpp b/src/hotspot/share/gc/z/zNMethod.cpp index c743cb2496ca5..a3ee5465a01e6 100644 --- a/src/hotspot/share/gc/z/zNMethod.cpp +++ b/src/hotspot/share/gc/z/zNMethod.cpp @@ -50,6 +50,7 @@ #include "oops/oop.inline.hpp" #include "runtime/atomicAccess.hpp" #include "runtime/continuation.hpp" +#include "runtime/icache.hpp" #include "utilities/debug.hpp" static ZNMethodData* gc_data(const nmethod* nm) { @@ -201,8 +202,12 @@ void ZNMethod::register_nmethod(nmethod* nm) { log_register(nm); - // Patch nmethod barriers - nmethod_patch_barriers(nm); + { + ICacheInvalidationContext icic(nm); + + // Patch nmethod barriers + nmethod_patch_barriers(nm, icic.deferred_invalidation()); + } // Register nmethod ZNMethodTable::register_nmethod(nm); @@ -366,9 +371,12 @@ class ZNMethodUnlinkClosure : public NMethodClosure { const uintptr_t prev_color = ZNMethod::color(nm); assert(prev_color != ZPointerStoreGoodMask, "Potentially non-monotonic transition"); - // Heal oops and potentially mark young objects if there is a concurrent young collection. - ZUncoloredRootProcessOopClosure cl(prev_color); - ZNMethod::nmethod_oops_do_inner(nm, &cl); + { + ICacheInvalidationContext icic(nm); + // Heal oops and potentially mark young objects if there is a concurrent young collection. + ZUncoloredRootProcessOopClosure cl(prev_color); + ZNMethod::nmethod_oops_do_inner(nm, &cl, icic.deferred_invalidation()); + } // Disarm for marking and relocation, but leave the remset bits so this isn't store good. // This makes sure the mutator still takes a slow path to fill in the nmethod epoch for diff --git a/src/hotspot/share/gc/z/zUnload.cpp b/src/hotspot/share/gc/z/zUnload.cpp index 5c50b3077dd65..ee32983f2492f 100644 --- a/src/hotspot/share/gc/z/zUnload.cpp +++ b/src/hotspot/share/gc/z/zUnload.cpp @@ -38,6 +38,7 @@ #include "gc/z/zUnload.hpp" #include "memory/metaspaceUtils.hpp" #include "oops/access.inline.hpp" +#include "runtime/icache.hpp" static const ZStatSubPhase ZSubPhaseConcurrentClassesUnlink("Concurrent Classes Unlink", ZGenerationId::old); static const ZStatSubPhase ZSubPhaseConcurrentClassesPurge("Concurrent Classes Purge", ZGenerationId::old); @@ -81,7 +82,8 @@ class ZIsUnloadingBehaviour : public IsUnloadingBehaviour { return false; } ZIsUnloadingOopClosure cl(nm); - ZNMethod::nmethod_oops_do_inner(nm, &cl); + ICacheInvalidationContext icic(nm); + ZNMethod::nmethod_oops_do_inner(nm, &cl, icic.deferred_invalidation()); return cl.is_unloading(); } }; diff --git a/test/micro/org/openjdk/bench/vm/gc/GCPatchingNmethodCost.java b/test/micro/org/openjdk/bench/vm/gc/GCPatchingNmethodCost.java new file mode 100644 index 0000000000000..3e1542bdb51d4 --- /dev/null +++ b/test/micro/org/openjdk/bench/vm/gc/GCPatchingNmethodCost.java @@ -0,0 +1,205 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +package org.openjdk.bench.vm.gc; + +import java.lang.reflect.Method; +import java.util.*; +import java.util.concurrent.TimeUnit; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.CompilerControl; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +import org.openjdk.bench.util.InMemoryJavaCompiler; + +import jdk.test.whitebox.WhiteBox; +import jdk.test.whitebox.code.NMethod; + +/* + * Nmethods have OOPs and GC barriers emmedded into their code. + * GCs patch them which causes invalidation of nmethods' code. + * + * This benchmark can be used to estimate the cost of patching + * OOPs and GC barriers. + * + * We create 5000 nmethods which access fields of a class. + * We measure the time of different GC cycles to see + * the impact of patching nmethods. + * + * The benchmark parameters are method count and accessed field count. + */ + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +@Fork(value = 3, jvmArgsAppend = { + "-XX:+UnlockDiagnosticVMOptions", + "-XX:+UnlockExperimentalVMOptions", + "-XX:+WhiteBoxAPI", + "-Xbootclasspath/a:lib-test/wb.jar", + "-XX:-UseCodeCacheFlushing" +}) +public class GCPatchingNmethodCost { + + private static final int COMP_LEVEL = 1; + private static final String FIELD_USER = "FieldUser"; + + public static Fields fields; + + private static TestMethod[] methods = {}; + private static byte[] BYTE_CODE; + private static WhiteBox WB; + + @Param({"5000"}) + public int methodCount; + + @Param({"0", "2", "4", "8"}) + public int accessedFieldCount; + + public static class Fields { + public String f1; + public String f2; + public String f3; + public String f4; + public String f5; + public String f6; + public String f7; + public String f8; + public String f9; + } + + private static final class TestMethod { + private final Method method; + + public TestMethod(Method method) throws Exception { + this.method = method; + WB.testSetDontInlineMethod(method, true); + } + + public void profile() throws Exception { + method.invoke(null); + WB.markMethodProfiled(method); + } + + public void invoke() throws Exception { + method.invoke(null); + } + + public void compile() throws Exception { + WB.enqueueMethodForCompilation(method, COMP_LEVEL); + while (WB.isMethodQueuedForCompilation(method)) { + Thread.onSpinWait(); + } + if (WB.getMethodCompilationLevel(method) != COMP_LEVEL) { + throw new IllegalStateException("Method " + method + " is not compiled at the compilation level: " + COMP_LEVEL + ". Got: " + WB.getMethodCompilationLevel(method)); + } + } + + public NMethod getNMethod() { + return NMethod.get(method, false); + } + } + + private static ClassLoader createClassLoader() { + return new ClassLoader() { + @Override + public Class loadClass(String name) throws ClassNotFoundException { + if (!name.equals(FIELD_USER)) { + return super.loadClass(name); + } + + return defineClass(name, BYTE_CODE, 0, BYTE_CODE.length); + } + }; + } + + private static void createTestMethods(int accessedFieldCount, int count) throws Exception { + String javaCode = "public class " + FIELD_USER + " {"; + String field = GCPatchingNmethodCost.class.getName() + ".fields.f"; + javaCode += "public static void accessFields() {"; + for (int i = 1; i <= accessedFieldCount; i++) { + javaCode += field + i + "= " + field + i + " + " + i + ";"; + } + javaCode += "}}"; + + BYTE_CODE = InMemoryJavaCompiler.compile(FIELD_USER, javaCode); + + fields = new Fields(); + + methods = new TestMethod[count]; + for (int i = 0; i < count; i++) { + var cl = createClassLoader().loadClass(FIELD_USER); + Method method = cl.getMethod("accessFields"); + methods[i] = new TestMethod(method); + methods[i].profile(); + methods[i].compile(); + } + } + + private static void initWhiteBox() { + WB = WhiteBox.getWhiteBox(); + } + + @Setup(Level.Trial) + public void setupCodeCache() throws Exception { + initWhiteBox(); + createTestMethods(accessedFieldCount, methodCount); + Thread.sleep(1000); + } + + @Benchmark + @Warmup(iterations = 0) + @Measurement(iterations = 1) + public void youngGC() throws Exception { + fields = null; + WB.youngGC(); + } + + @Benchmark + @Warmup(iterations = 0) + @Measurement(iterations = 1) + public void fullGC() throws Exception { + fields = null; + WB.fullGC(); + } + + @Benchmark + @Warmup(iterations = 0) + @Measurement(iterations = 1) + public void systemGC() throws Exception { + fields = null; + System.gc(); + } +} From b60317e947bbff1d52cc62b4ed966089b6acc0db Mon Sep 17 00:00:00 2001 From: Evgeny Astigeevich Date: Fri, 21 Nov 2025 22:17:00 +0000 Subject: [PATCH 03/18] Use THREAD_LOCAL deferred_icache_invalidation instead of parameter --- .../aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp | 10 +++++++--- .../aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp | 2 +- src/hotspot/cpu/aarch64/icache_aarch64.hpp | 6 ++++++ src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp | 2 +- src/hotspot/share/code/nmethod.cpp | 3 +-- src/hotspot/share/code/nmethod.hpp | 3 +-- src/hotspot/share/code/relocInfo.hpp | 7 ------- src/hotspot/share/gc/z/zBarrierSetNMethod.cpp | 4 ++-- src/hotspot/share/gc/z/zGeneration.cpp | 4 ++-- src/hotspot/share/gc/z/zMark.cpp | 8 ++++---- src/hotspot/share/gc/z/zNMethod.cpp | 12 ++++++------ src/hotspot/share/gc/z/zNMethod.hpp | 4 ++-- src/hotspot/share/gc/z/zUnload.cpp | 2 +- src/hotspot/share/runtime/icache.cpp | 2 ++ src/hotspot/share/runtime/icache.hpp | 13 +++++++------ 15 files changed, 43 insertions(+), 39 deletions(-) diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp index 693132d3d7589..d32f96f38ba58 100644 --- a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp @@ -862,7 +862,7 @@ static void change_immediate(uint32_t& instr, uint32_t imm, uint32_t start, uint instr |= imm << start; } -void ZBarrierSetAssembler::patch_barrier_relocation(address addr, int format, bool defer_icache_invalidation) { +void ZBarrierSetAssembler::patch_barrier_relocation(address addr, int format) { const uint16_t value = patch_barrier_relocation_value(format); uint32_t* const patch_addr = (uint32_t*)addr; @@ -879,9 +879,13 @@ void ZBarrierSetAssembler::patch_barrier_relocation(address addr, int format, bo ShouldNotReachHere(); } - if (defer_icache_invalidation) { - // Instruction cache invalidation per barrier can be expensive, e.g. on Neoverse N1 having erratum 1542419. + if (NeoverseN1Errata1542419) { + // Instruction cache invalidation per barrier is expensive on Neoverse N1 having erratum 1542419. // Defer the ICache invalidation to a later point where multiple patches can be handled together. + // + // Note: We rely on the fact that this function is only called from places where deferred invalidation + // is safe. This assumption helps to avoid overhead of accessing thread-local data here. + assert(ICacheInvalidationContext::deferred_invalidation(), "ICache invalidation should be deferred"); return; } diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp index c852c567d1d8b..ae2819e78cad2 100644 --- a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp @@ -154,7 +154,7 @@ class ZBarrierSetAssembler : public ZBarrierSetAssemblerBase { virtual NMethodPatchingType nmethod_patching_type() { return NMethodPatchingType::conc_instruction_and_data_patch; } - void patch_barrier_relocation(address addr, int format, bool defer_icache_invalidation = false); + void patch_barrier_relocation(address addr, int format); void patch_barriers() {} diff --git a/src/hotspot/cpu/aarch64/icache_aarch64.hpp b/src/hotspot/cpu/aarch64/icache_aarch64.hpp index 5095512c1ec5d..ece2d766b674a 100644 --- a/src/hotspot/cpu/aarch64/icache_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/icache_aarch64.hpp @@ -36,6 +36,7 @@ inline void ICacheInvalidationContext::pd_init(nmethod* nm) { if (NeoverseN1Errata1542419) { _nm = nm; + _deferred_icache_invalidation = (_nm != nullptr); } } @@ -62,7 +63,12 @@ inline void ICacheInvalidationContext::pd_invalidate_icache() { // // As the address for icache invalidation is not relevant, we use the nmethod's code start address. ICache::invalidate_word(_nm->code_begin()); + _deferred_icache_invalidation = false; } } +inline bool ICacheInvalidationContext::deferred_invalidation() { + return _deferred_icache_invalidation; +} + #endif // CPU_AARCH64_ICACHE_AARCH64_HPP diff --git a/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp b/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp index f939c18c6f78e..bc88a033f8a6d 100644 --- a/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp @@ -55,7 +55,7 @@ void Relocation::pd_set_data_value(address x, bool verify_only) { break; } - if (binding() != nullptr && binding()->deferred_icache_invalidation()) { + if (ICacheInvalidationContext::deferred_invalidation()) { // Instruction cache invalidation per relocation can be expensive, e.g. on Neoverse N1 having erratum 1542419. // Defer the ICache invalidation to a later point where multiple patches can be handled together. return; diff --git a/src/hotspot/share/code/nmethod.cpp b/src/hotspot/share/code/nmethod.cpp index eab506ab660bc..d91af9b4991e1 100644 --- a/src/hotspot/share/code/nmethod.cpp +++ b/src/hotspot/share/code/nmethod.cpp @@ -2032,10 +2032,9 @@ void nmethod::copy_values(GrowableArray* array) { } } -void nmethod::fix_oop_relocations(address begin, address end, bool initialize_immediates, bool defer_icache_invalidation) { +void nmethod::fix_oop_relocations(address begin, address end, bool initialize_immediates) { // re-patch all oop-bearing instructions, just in case some oops moved RelocIterator iter(this, begin, end); - AARCH64_ONLY(iter.set_deferred_icache_invalidation(defer_icache_invalidation);) while (iter.next()) { if (iter.type() == relocInfo::oop_type) { oop_Relocation* reloc = iter.oop_reloc(); diff --git a/src/hotspot/share/code/nmethod.hpp b/src/hotspot/share/code/nmethod.hpp index fbc092a4814c2..34accf428b690 100644 --- a/src/hotspot/share/code/nmethod.hpp +++ b/src/hotspot/share/code/nmethod.hpp @@ -799,7 +799,7 @@ class nmethod : public CodeBlob { // Relocation support private: - void fix_oop_relocations(address begin, address end, bool initialize_immediates, bool defer_icache_invalidation = false); + void fix_oop_relocations(address begin, address end, bool initialize_immediates); inline void initialize_immediate_oop(oop* dest, jobject handle); protected: @@ -808,7 +808,6 @@ class nmethod : public CodeBlob { public: void fix_oop_relocations(address begin, address end) { fix_oop_relocations(begin, end, false); } void fix_oop_relocations() { fix_oop_relocations(nullptr, nullptr, false); } - void fix_oop_relocations(bool defer_icache_invalidation) { fix_oop_relocations(nullptr, nullptr, false, defer_icache_invalidation); } bool is_at_poll_return(address pc); bool is_at_poll_or_poll_return(address pc); diff --git a/src/hotspot/share/code/relocInfo.hpp b/src/hotspot/share/code/relocInfo.hpp index 6bd22073ce89a..6f1778ef479c4 100644 --- a/src/hotspot/share/code/relocInfo.hpp +++ b/src/hotspot/share/code/relocInfo.hpp @@ -568,8 +568,6 @@ class RelocIterator : public StackObj { short* _data; // pointer to the relocation's data short _datalen; // number of halfwords in _data - AARCH64_ONLY(bool _deferred_icache_invalidation;) - // Base addresses needed to compute targets of section_word_type relocs. address _section_start[SECT_LIMIT]; address _section_end [SECT_LIMIT]; @@ -640,11 +638,6 @@ class RelocIterator : public StackObj { bool has_current() const { return _datalen >= 0; } bool addr_in_const() const; -#ifdef AARCH64 - bool deferred_icache_invalidation() const { return _deferred_icache_invalidation; } - void set_deferred_icache_invalidation(bool b) { _deferred_icache_invalidation = b; } -#endif - address section_start(int n) const { assert(_section_start[n], "section %d must be initialized", n); return _section_start[n]; diff --git a/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp b/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp index eecdcedf0ee46..218f0c97244bd 100644 --- a/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp +++ b/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp @@ -75,11 +75,11 @@ bool ZBarrierSetNMethod::nmethod_entry_barrier(nmethod* nm) { ICacheInvalidationContext icic(nm); // Heal barriers - ZNMethod::nmethod_patch_barriers(nm, icic.deferred_invalidation()); + ZNMethod::nmethod_patch_barriers(nm); // Heal oops ZUncoloredRootProcessWeakOopClosure cl(ZNMethod::color(nm)); - ZNMethod::nmethod_oops_do_inner(nm, &cl, icic.deferred_invalidation()); + ZNMethod::nmethod_oops_do_inner(nm, &cl); } const uintptr_t prev_color = ZNMethod::color(nm); diff --git a/src/hotspot/share/gc/z/zGeneration.cpp b/src/hotspot/share/gc/z/zGeneration.cpp index 717d393c5411c..00624f16bd467 100644 --- a/src/hotspot/share/gc/z/zGeneration.cpp +++ b/src/hotspot/share/gc/z/zGeneration.cpp @@ -1438,11 +1438,11 @@ class ZRemapNMethodClosure : public NMethodClosure { { ICacheInvalidationContext icic(nm); // Heal barriers - ZNMethod::nmethod_patch_barriers(nm, icic.deferred_invalidation()); + ZNMethod::nmethod_patch_barriers(nm); // Heal oops ZUncoloredRootProcessOopClosure cl(ZNMethod::color(nm)); - ZNMethod::nmethod_oops_do_inner(nm, &cl, icic.deferred_invalidation()); + ZNMethod::nmethod_oops_do_inner(nm, &cl); } log_trace(gc, nmethod)("nmethod: " PTR_FORMAT " visited by old remapping", p2i(nm)); diff --git a/src/hotspot/share/gc/z/zMark.cpp b/src/hotspot/share/gc/z/zMark.cpp index 48bde6db2195a..bf95f782938cf 100644 --- a/src/hotspot/share/gc/z/zMark.cpp +++ b/src/hotspot/share/gc/z/zMark.cpp @@ -722,11 +722,11 @@ class ZMarkNMethodClosure : public NMethodClosure { { ICacheInvalidationContext icic(nm); // Heal barriers - ZNMethod::nmethod_patch_barriers(nm, icic.deferred_invalidation()); + ZNMethod::nmethod_patch_barriers(nm); // Heal oops ZUncoloredRootMarkOopClosure cl(ZNMethod::color(nm)); - ZNMethod::nmethod_oops_do_inner(nm, &cl, icic.deferred_invalidation()); + ZNMethod::nmethod_oops_do_inner(nm, &cl); } // CodeCache unloading support @@ -772,12 +772,12 @@ class ZMarkYoungNMethodClosure : public NMethodClosure { if (complete_disarm) { // We are about to completely disarm the nmethod, must take responsibility to patch all barriers before disarming - ZNMethod::nmethod_patch_barriers(nm, icic.deferred_invalidation()); + ZNMethod::nmethod_patch_barriers(nm); } // Heal oops ZUncoloredRootMarkYoungOopClosure cl(prev_color); - ZNMethod::nmethod_oops_do_inner(nm, &cl, icic.deferred_invalidation()); + ZNMethod::nmethod_oops_do_inner(nm, &cl); } _bs_nm->guard_with(nm, (int)untype(new_disarm_value_ptr)); diff --git a/src/hotspot/share/gc/z/zNMethod.cpp b/src/hotspot/share/gc/z/zNMethod.cpp index a3ee5465a01e6..1db7cf9a66c52 100644 --- a/src/hotspot/share/gc/z/zNMethod.cpp +++ b/src/hotspot/share/gc/z/zNMethod.cpp @@ -206,7 +206,7 @@ void ZNMethod::register_nmethod(nmethod* nm) { ICacheInvalidationContext icic(nm); // Patch nmethod barriers - nmethod_patch_barriers(nm, icic.deferred_invalidation()); + nmethod_patch_barriers(nm); } // Register nmethod @@ -249,11 +249,11 @@ void ZNMethod::set_guard_value(nmethod* nm, int value) { bs->guard_with(nm, value); } -void ZNMethod::nmethod_patch_barriers(nmethod* nm, bool defer_icache_invalidation) { +void ZNMethod::nmethod_patch_barriers(nmethod* nm) { ZBarrierSetAssembler* const bs_asm = ZBarrierSet::assembler(); ZArrayIterator iter(gc_data(nm)->barriers()); for (ZNMethodDataBarrier barrier; iter.next(&barrier);) { - bs_asm->patch_barrier_relocation(barrier._reloc_addr, barrier._reloc_format AARCH64_ONLY(COMMA defer_icache_invalidation)); + bs_asm->patch_barrier_relocation(barrier._reloc_addr, barrier._reloc_format); } } @@ -262,7 +262,7 @@ void ZNMethod::nmethod_oops_do(nmethod* nm, OopClosure* cl) { ZNMethod::nmethod_oops_do_inner(nm, cl); } -void ZNMethod::nmethod_oops_do_inner(nmethod* nm, OopClosure* cl, bool defer_icache_invalidation) { +void ZNMethod::nmethod_oops_do_inner(nmethod* nm, OopClosure* cl) { // Process oops table { oop* const begin = nm->oops_begin(); @@ -288,7 +288,7 @@ void ZNMethod::nmethod_oops_do_inner(nmethod* nm, OopClosure* cl, bool defer_ica // Process non-immediate oops if (data->has_non_immediate_oops()) { - nm->fix_oop_relocations(defer_icache_invalidation); + nm->fix_oop_relocations(); } } @@ -375,7 +375,7 @@ class ZNMethodUnlinkClosure : public NMethodClosure { ICacheInvalidationContext icic(nm); // Heal oops and potentially mark young objects if there is a concurrent young collection. ZUncoloredRootProcessOopClosure cl(prev_color); - ZNMethod::nmethod_oops_do_inner(nm, &cl, icic.deferred_invalidation()); + ZNMethod::nmethod_oops_do_inner(nm, &cl); } // Disarm for marking and relocation, but leave the remset bits so this isn't store good. diff --git a/src/hotspot/share/gc/z/zNMethod.hpp b/src/hotspot/share/gc/z/zNMethod.hpp index bc1e7c005c0f8..5d797f3fd5542 100644 --- a/src/hotspot/share/gc/z/zNMethod.hpp +++ b/src/hotspot/share/gc/z/zNMethod.hpp @@ -55,10 +55,10 @@ class ZNMethod : public AllStatic { static void disarm(nmethod* nm); static void set_guard_value(nmethod* nm, int value); - static void nmethod_patch_barriers(nmethod* nm, bool defer_icache_invalidation = false); + static void nmethod_patch_barriers(nmethod* nm); static void nmethod_oops_do(nmethod* nm, OopClosure* cl); - static void nmethod_oops_do_inner(nmethod* nm, OopClosure* cl, bool defer_icache_invalidation = false); + static void nmethod_oops_do_inner(nmethod* nm, OopClosure* cl); static void nmethods_do_begin(bool secondary); static void nmethods_do_end(bool secondary); diff --git a/src/hotspot/share/gc/z/zUnload.cpp b/src/hotspot/share/gc/z/zUnload.cpp index ee32983f2492f..b13247de82f22 100644 --- a/src/hotspot/share/gc/z/zUnload.cpp +++ b/src/hotspot/share/gc/z/zUnload.cpp @@ -83,7 +83,7 @@ class ZIsUnloadingBehaviour : public IsUnloadingBehaviour { } ZIsUnloadingOopClosure cl(nm); ICacheInvalidationContext icic(nm); - ZNMethod::nmethod_oops_do_inner(nm, &cl, icic.deferred_invalidation()); + ZNMethod::nmethod_oops_do_inner(nm, &cl); return cl.is_unloading(); } }; diff --git a/src/hotspot/share/runtime/icache.cpp b/src/hotspot/share/runtime/icache.cpp index 34d5a7e055b47..9d7adcd43820b 100644 --- a/src/hotspot/share/runtime/icache.cpp +++ b/src/hotspot/share/runtime/icache.cpp @@ -125,3 +125,5 @@ void icache_init() { void icache_init2() { ICache::initialize(2); } + +THREAD_LOCAL bool ICacheInvalidationContext::_deferred_icache_invalidation = false; diff --git a/src/hotspot/share/runtime/icache.hpp b/src/hotspot/share/runtime/icache.hpp index 2128d4b88d161..f8991a8ef3bec 100644 --- a/src/hotspot/share/runtime/icache.hpp +++ b/src/hotspot/share/runtime/icache.hpp @@ -77,6 +77,8 @@ class ICacheInvalidationContext : StackObj { NONCOPYABLE(ICacheInvalidationContext); private: + static THREAD_LOCAL bool _deferred_icache_invalidation; + nmethod* _nm; void pd_init(nmethod* nm); @@ -88,14 +90,10 @@ class ICacheInvalidationContext : StackObj { } ~ICacheInvalidationContext() { - if (_nm != nullptr) { - pd_invalidate_icache(); - } + pd_invalidate_icache(); } - bool deferred_invalidation() const { - return _nm != nullptr; - } + static bool deferred_invalidation(); }; // Must be included before the definition of ICacheStubGenerator @@ -159,6 +157,9 @@ class ICacheStubGenerator : public StubCodeGenerator { // Default implementation: do nothing inline void ICacheInvalidationContext::pd_init(nmethod*) {} inline void ICacheInvalidationContext::pd_invalidate_icache() {} + inline bool ICacheInvalidationContext::deferred_invalidation() { + return false; + } #endif #endif // SHARE_RUNTIME_ICACHE_HPP From 20480771fd8f1cb40e9d0dd22a511f70899e075a Mon Sep 17 00:00:00 2001 From: Evgeny Astigeevich Date: Mon, 24 Nov 2025 17:40:48 +0000 Subject: [PATCH 04/18] Explicitly check Neoverse N1 revisions affected by errata 1542419 --- .../cpu/aarch64/vm_version_aarch64.cpp | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp index d793d2094d9ce..496838d226824 100644 --- a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp @@ -63,6 +63,18 @@ static SpinWait get_spin_wait_desc() { return spin_wait; } +static bool has_neoverse_n1_errata_1542419() { + const int major_rev_num = VM_Version::cpu_variant(); + const int minor_rev_num = VM_Version::cpu_revision(); + // Neoverse N1: 0xd0c + // Erratum 1542419 affects r3p0, r3p1 and r4p0. + return (VM_Version::cpu_family() == VM_Version::CPU_ARM && + VM_Version::model_is(0xd0c) && + ((major_rev_num == 3 && minor_rev_num == 0) || + (major_rev_num == 3 && minor_rev_num == 1) || + (major_rev_num == 4 && minor_rev_num == 0))); +} + void VM_Version::initialize() { #define SET_CPU_FEATURE_NAME(id, name, bit) \ _features_names[bit] = XSTR(name); @@ -641,15 +653,12 @@ void VM_Version::initialize() { clear_feature(CPU_SVE); } - // Neoverse N1: 0xd0c - if (_cpu == CPU_ARM && model_is(0xd0c) && FLAG_IS_DEFAULT(NeoverseN1Errata1542419)) { - const int major_rev_num = cpu_variant(); - const int minor_rev_num = cpu_revision(); - if (!(major_rev_num >= 4 && minor_rev_num >= 1)) { - // As Neoverse N1 r4p1 and later are not affected by the erratum, - // enable the workaround by default for earlier revisions. - FLAG_SET_DEFAULT(NeoverseN1Errata1542419, true); - } + if (FLAG_IS_DEFAULT(NeoverseN1Errata1542419) && has_neoverse_n1_errata_1542419()) { + FLAG_SET_DEFAULT(NeoverseN1Errata1542419, true); + } + + if (NeoverseN1Errata1542419 && !has_neoverse_n1_errata_1542419()) { + warning("NeoverseN1Errata1542419 is set for the CPU not having Neoverse N1 errata 1542419"); } // Construct the "features" string From 79297bd4332fd27eb6a5c3c0c1d3e6185981e121 Mon Sep 17 00:00:00 2001 From: Evgeny Astigeevich Date: Tue, 25 Nov 2025 16:33:07 +0000 Subject: [PATCH 05/18] Add inline assembly to ICacheInvalidationContext::pd_invalidate_icache not using nmethod --- src/hotspot/cpu/aarch64/icache_aarch64.cpp | 3 + src/hotspot/cpu/aarch64/icache_aarch64.hpp | 55 +++++++++++++------ src/hotspot/share/gc/z/zBarrierSetNMethod.cpp | 2 +- src/hotspot/share/gc/z/zGeneration.cpp | 2 +- src/hotspot/share/gc/z/zMark.cpp | 4 +- src/hotspot/share/gc/z/zNMethod.cpp | 4 +- src/hotspot/share/gc/z/zUnload.cpp | 2 +- src/hotspot/share/runtime/icache.cpp | 2 - src/hotspot/share/runtime/icache.hpp | 10 +--- 9 files changed, 50 insertions(+), 34 deletions(-) diff --git a/src/hotspot/cpu/aarch64/icache_aarch64.cpp b/src/hotspot/cpu/aarch64/icache_aarch64.cpp index a942406f45ee9..539171e68c734 100644 --- a/src/hotspot/cpu/aarch64/icache_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/icache_aarch64.cpp @@ -24,6 +24,7 @@ */ #include "runtime/icache.hpp" +#include "utilities/globalDefinitions.hpp" void ICacheStubGenerator::generate_icache_flush( ICache::flush_icache_stub_t* flush_icache_stub) { @@ -32,3 +33,5 @@ void ICacheStubGenerator::generate_icache_flush( } void ICache::initialize(int phase) {} + +THREAD_LOCAL bool deferred_icache_invalidation = false; diff --git a/src/hotspot/cpu/aarch64/icache_aarch64.hpp b/src/hotspot/cpu/aarch64/icache_aarch64.hpp index ece2d766b674a..4e23025b027c0 100644 --- a/src/hotspot/cpu/aarch64/icache_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/icache_aarch64.hpp @@ -33,26 +33,36 @@ #define PD_ICACHE_INVALIDATION_CONTEXT -inline void ICacheInvalidationContext::pd_init(nmethod* nm) { - if (NeoverseN1Errata1542419) { - _nm = nm; - _deferred_icache_invalidation = (_nm != nullptr); +extern THREAD_LOCAL bool deferred_icache_invalidation; + +inline void ICacheInvalidationContext::pd_init() { + if (NeoverseN1Errata1542419) { + deferred_icache_invalidation = true; } } +inline bool ICacheInvalidationContext::deferred_invalidation() { + return deferred_icache_invalidation; +} + inline void ICacheInvalidationContext::pd_invalidate_icache() { - if (_nm != nullptr) { - assert(NeoverseN1Errata1542419, "Should only be set for Neoverse N1 erratum"); - // Neoverse-N1 implementation mitigates erratum 1542419 with a workaround: + if (NeoverseN1Errata1542419) { + assert(deferred_icache_invalidation, "Deferred icache invalidation must be enabled"); + // Errata 1542419: Neoverse N1 cores with the 'COHERENT_ICACHE' feature may fetch stale + // instructions when software depends on prefetch-speculation-protection + // instead of explicit synchronization. + // + // Neoverse-N1 implementation mitigates the errata 1542419 with a workaround: // - Disable coherent icache. // - Trap IC IVAU instructions. // - Execute: // - tlbi vae3is, xzr // - dsb sy + // - Ignore trapped IC IVAU instructions. // - // `tlbi vae3is, xzr` invalidates translations for all address spaces (global for address). - // It waits for all memory accesses using in-scope old translation information to complete - // before it is considered complete. + // `tlbi vae3is, xzr` invalidates all translation entries (all VAs, all possible levels). + // It waits for all memory accesses using in-scope old translation information to complete + // before it is considered complete. // // As this workaround has significant overhead, Arm Neoverse N1 (MP050) Software Developer // Errata Notice version 29.0 suggests: @@ -60,15 +70,24 @@ inline void ICacheInvalidationContext::pd_invalidate_icache() { // "Since one TLB inner-shareable invalidation is enough to avoid this erratum, the number // of injected TLB invalidations should be minimized in the trap handler to mitigate // the performance impact due to this workaround." - // - // As the address for icache invalidation is not relevant, we use the nmethod's code start address. - ICache::invalidate_word(_nm->code_begin()); - _deferred_icache_invalidation = false; - } -} +#ifndef PRODUCT + unsigned int cache_info = 0; + asm volatile ("mrs\t%0, ctr_el0":"=r" (cache_info)); + constexpr unsigned int CTR_IDC_SHIFT = 28; + constexpr unsigned int CTR_DIC_SHIFT = 29; + assert(((cache_info >> CTR_IDC_SHIFT) & 0x1) != 0x0, "Expect CTR_EL0.IDC to be enabled"); + assert(((cache_info >> CTR_DIC_SHIFT) & 0x1) == 0x0, "Expect CTR_EL0.DIC to be disabled"); +#endif -inline bool ICacheInvalidationContext::deferred_invalidation() { - return _deferred_icache_invalidation; + // As the address for icache invalidation is not relevant + // and IC IVAU instruction is ignored, we use XZR in it. + asm volatile("dsb ish \n" + "ic ivau, xzr \n" + "isb \n" + : : : "memory"); + + deferred_icache_invalidation = false; + } } #endif // CPU_AARCH64_ICACHE_AARCH64_HPP diff --git a/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp b/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp index 218f0c97244bd..143c0580811aa 100644 --- a/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp +++ b/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp @@ -72,7 +72,7 @@ bool ZBarrierSetNMethod::nmethod_entry_barrier(nmethod* nm) { } { - ICacheInvalidationContext icic(nm); + ICacheInvalidationContext icic; // Heal barriers ZNMethod::nmethod_patch_barriers(nm); diff --git a/src/hotspot/share/gc/z/zGeneration.cpp b/src/hotspot/share/gc/z/zGeneration.cpp index 00624f16bd467..0b5105abbf2ad 100644 --- a/src/hotspot/share/gc/z/zGeneration.cpp +++ b/src/hotspot/share/gc/z/zGeneration.cpp @@ -1436,7 +1436,7 @@ class ZRemapNMethodClosure : public NMethodClosure { ZLocker locker(ZNMethod::lock_for_nmethod(nm)); if (_bs_nm->is_armed(nm)) { { - ICacheInvalidationContext icic(nm); + ICacheInvalidationContext icic; // Heal barriers ZNMethod::nmethod_patch_barriers(nm); diff --git a/src/hotspot/share/gc/z/zMark.cpp b/src/hotspot/share/gc/z/zMark.cpp index bf95f782938cf..2f965fa575dce 100644 --- a/src/hotspot/share/gc/z/zMark.cpp +++ b/src/hotspot/share/gc/z/zMark.cpp @@ -720,7 +720,7 @@ class ZMarkNMethodClosure : public NMethodClosure { ZLocker locker(ZNMethod::lock_for_nmethod(nm)); if (_bs_nm->is_armed(nm)) { { - ICacheInvalidationContext icic(nm); + ICacheInvalidationContext icic; // Heal barriers ZNMethod::nmethod_patch_barriers(nm); @@ -768,7 +768,7 @@ class ZMarkYoungNMethodClosure : public NMethodClosure { const bool complete_disarm = ZPointer::is_store_good(new_disarm_value_ptr); { - ICacheInvalidationContext icic(nm); + ICacheInvalidationContext icic; if (complete_disarm) { // We are about to completely disarm the nmethod, must take responsibility to patch all barriers before disarming diff --git a/src/hotspot/share/gc/z/zNMethod.cpp b/src/hotspot/share/gc/z/zNMethod.cpp index 1db7cf9a66c52..4d3abab111b61 100644 --- a/src/hotspot/share/gc/z/zNMethod.cpp +++ b/src/hotspot/share/gc/z/zNMethod.cpp @@ -203,7 +203,7 @@ void ZNMethod::register_nmethod(nmethod* nm) { log_register(nm); { - ICacheInvalidationContext icic(nm); + ICacheInvalidationContext icic; // Patch nmethod barriers nmethod_patch_barriers(nm); @@ -372,7 +372,7 @@ class ZNMethodUnlinkClosure : public NMethodClosure { assert(prev_color != ZPointerStoreGoodMask, "Potentially non-monotonic transition"); { - ICacheInvalidationContext icic(nm); + ICacheInvalidationContext icic; // Heal oops and potentially mark young objects if there is a concurrent young collection. ZUncoloredRootProcessOopClosure cl(prev_color); ZNMethod::nmethod_oops_do_inner(nm, &cl); diff --git a/src/hotspot/share/gc/z/zUnload.cpp b/src/hotspot/share/gc/z/zUnload.cpp index b13247de82f22..e05eb15e22bd1 100644 --- a/src/hotspot/share/gc/z/zUnload.cpp +++ b/src/hotspot/share/gc/z/zUnload.cpp @@ -82,7 +82,7 @@ class ZIsUnloadingBehaviour : public IsUnloadingBehaviour { return false; } ZIsUnloadingOopClosure cl(nm); - ICacheInvalidationContext icic(nm); + ICacheInvalidationContext icic; ZNMethod::nmethod_oops_do_inner(nm, &cl); return cl.is_unloading(); } diff --git a/src/hotspot/share/runtime/icache.cpp b/src/hotspot/share/runtime/icache.cpp index 9d7adcd43820b..34d5a7e055b47 100644 --- a/src/hotspot/share/runtime/icache.cpp +++ b/src/hotspot/share/runtime/icache.cpp @@ -125,5 +125,3 @@ void icache_init() { void icache_init2() { ICache::initialize(2); } - -THREAD_LOCAL bool ICacheInvalidationContext::_deferred_icache_invalidation = false; diff --git a/src/hotspot/share/runtime/icache.hpp b/src/hotspot/share/runtime/icache.hpp index f8991a8ef3bec..15e8c26036b10 100644 --- a/src/hotspot/share/runtime/icache.hpp +++ b/src/hotspot/share/runtime/icache.hpp @@ -77,16 +77,12 @@ class ICacheInvalidationContext : StackObj { NONCOPYABLE(ICacheInvalidationContext); private: - static THREAD_LOCAL bool _deferred_icache_invalidation; - - nmethod* _nm; - - void pd_init(nmethod* nm); + void pd_init(); void pd_invalidate_icache(); public: - ICacheInvalidationContext(nmethod* nm) : _nm(nullptr) { - pd_init(nm); + ICacheInvalidationContext() { + pd_init(); } ~ICacheInvalidationContext() { From e774cda1dd8e0efb67e6e151a5ea8ef9527fc47f Mon Sep 17 00:00:00 2001 From: Evgeny Astigeevich Date: Tue, 25 Nov 2025 16:48:27 +0000 Subject: [PATCH 06/18] Fix build issue on non aarch64 platforms --- src/hotspot/share/runtime/icache.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/runtime/icache.hpp b/src/hotspot/share/runtime/icache.hpp index 15e8c26036b10..efb7f89173556 100644 --- a/src/hotspot/share/runtime/icache.hpp +++ b/src/hotspot/share/runtime/icache.hpp @@ -151,7 +151,7 @@ class ICacheStubGenerator : public StubCodeGenerator { #ifndef PD_ICACHE_INVALIDATION_CONTEXT // Default implementation: do nothing - inline void ICacheInvalidationContext::pd_init(nmethod*) {} + inline void ICacheInvalidationContext::pd_init() {} inline void ICacheInvalidationContext::pd_invalidate_icache() {} inline bool ICacheInvalidationContext::deferred_invalidation() { return false; From 0c38d6b907738508d72ae95a40abe32c23193684 Mon Sep 17 00:00:00 2001 From: Evgeny Astigeevich Date: Tue, 25 Nov 2025 17:21:31 +0000 Subject: [PATCH 07/18] Move ICacheInvalidationContext::pd_ to icache_linux_aarch64 --- src/hotspot/cpu/aarch64/icache_aarch64.cpp | 3 - src/hotspot/cpu/aarch64/icache_aarch64.hpp | 60 ------------------ .../linux_aarch64/icache_linux_aarch64.cpp | 27 ++++++++ .../linux_aarch64/icache_linux_aarch64.hpp | 61 +++++++++++++++++++ src/hotspot/share/runtime/icache.hpp | 2 - 5 files changed, 88 insertions(+), 65 deletions(-) create mode 100644 src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp diff --git a/src/hotspot/cpu/aarch64/icache_aarch64.cpp b/src/hotspot/cpu/aarch64/icache_aarch64.cpp index 539171e68c734..a942406f45ee9 100644 --- a/src/hotspot/cpu/aarch64/icache_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/icache_aarch64.cpp @@ -24,7 +24,6 @@ */ #include "runtime/icache.hpp" -#include "utilities/globalDefinitions.hpp" void ICacheStubGenerator::generate_icache_flush( ICache::flush_icache_stub_t* flush_icache_stub) { @@ -33,5 +32,3 @@ void ICacheStubGenerator::generate_icache_flush( } void ICache::initialize(int phase) {} - -THREAD_LOCAL bool deferred_icache_invalidation = false; diff --git a/src/hotspot/cpu/aarch64/icache_aarch64.hpp b/src/hotspot/cpu/aarch64/icache_aarch64.hpp index 4e23025b027c0..46689a23d8095 100644 --- a/src/hotspot/cpu/aarch64/icache_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/icache_aarch64.hpp @@ -29,65 +29,5 @@ #include OS_CPU_HEADER(icache) #include "code/nmethod.hpp" -#include "utilities/globalDefinitions.hpp" - -#define PD_ICACHE_INVALIDATION_CONTEXT - -extern THREAD_LOCAL bool deferred_icache_invalidation; - -inline void ICacheInvalidationContext::pd_init() { - if (NeoverseN1Errata1542419) { - deferred_icache_invalidation = true; - } -} - -inline bool ICacheInvalidationContext::deferred_invalidation() { - return deferred_icache_invalidation; -} - -inline void ICacheInvalidationContext::pd_invalidate_icache() { - if (NeoverseN1Errata1542419) { - assert(deferred_icache_invalidation, "Deferred icache invalidation must be enabled"); - // Errata 1542419: Neoverse N1 cores with the 'COHERENT_ICACHE' feature may fetch stale - // instructions when software depends on prefetch-speculation-protection - // instead of explicit synchronization. - // - // Neoverse-N1 implementation mitigates the errata 1542419 with a workaround: - // - Disable coherent icache. - // - Trap IC IVAU instructions. - // - Execute: - // - tlbi vae3is, xzr - // - dsb sy - // - Ignore trapped IC IVAU instructions. - // - // `tlbi vae3is, xzr` invalidates all translation entries (all VAs, all possible levels). - // It waits for all memory accesses using in-scope old translation information to complete - // before it is considered complete. - // - // As this workaround has significant overhead, Arm Neoverse N1 (MP050) Software Developer - // Errata Notice version 29.0 suggests: - // - // "Since one TLB inner-shareable invalidation is enough to avoid this erratum, the number - // of injected TLB invalidations should be minimized in the trap handler to mitigate - // the performance impact due to this workaround." -#ifndef PRODUCT - unsigned int cache_info = 0; - asm volatile ("mrs\t%0, ctr_el0":"=r" (cache_info)); - constexpr unsigned int CTR_IDC_SHIFT = 28; - constexpr unsigned int CTR_DIC_SHIFT = 29; - assert(((cache_info >> CTR_IDC_SHIFT) & 0x1) != 0x0, "Expect CTR_EL0.IDC to be enabled"); - assert(((cache_info >> CTR_DIC_SHIFT) & 0x1) == 0x0, "Expect CTR_EL0.DIC to be disabled"); -#endif - - // As the address for icache invalidation is not relevant - // and IC IVAU instruction is ignored, we use XZR in it. - asm volatile("dsb ish \n" - "ic ivau, xzr \n" - "isb \n" - : : : "memory"); - - deferred_icache_invalidation = false; - } -} #endif // CPU_AARCH64_ICACHE_AARCH64_HPP diff --git a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp new file mode 100644 index 0000000000000..4dcb552f5798b --- /dev/null +++ b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp @@ -0,0 +1,27 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "utilities/globalDefinitions.hpp" + +THREAD_LOCAL bool deferred_icache_invalidation = false; diff --git a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp index 957be26670229..a9ea416cf567f 100644 --- a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp +++ b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp @@ -26,6 +26,67 @@ #ifndef OS_CPU_LINUX_AARCH64_ICACHE_AARCH64_HPP #define OS_CPU_LINUX_AARCH64_ICACHE_AARCH64_HPP +#include "utilities/globalDefinitions.hpp" + +#define PD_ICACHE_INVALIDATION_CONTEXT + +extern THREAD_LOCAL bool deferred_icache_invalidation; + +inline void ICacheInvalidationContext::pd_init() { + if (NeoverseN1Errata1542419) { + deferred_icache_invalidation = true; + } +} + +inline bool ICacheInvalidationContext::deferred_invalidation() { + return deferred_icache_invalidation; +} + +inline void ICacheInvalidationContext::pd_invalidate_icache() { + if (NeoverseN1Errata1542419) { + assert(deferred_icache_invalidation, "Deferred icache invalidation must be enabled"); + // Errata 1542419: Neoverse N1 cores with the 'COHERENT_ICACHE' feature may fetch stale + // instructions when software depends on prefetch-speculation-protection + // instead of explicit synchronization. + // + // Neoverse-N1 implementation mitigates the errata 1542419 with a workaround: + // - Disable coherent icache. + // - Trap IC IVAU instructions. + // - Execute: + // - tlbi vae3is, xzr + // - dsb sy + // - Ignore trapped IC IVAU instructions. + // + // `tlbi vae3is, xzr` invalidates all translation entries (all VAs, all possible levels). + // It waits for all memory accesses using in-scope old translation information to complete + // before it is considered complete. + // + // As this workaround has significant overhead, Arm Neoverse N1 (MP050) Software Developer + // Errata Notice version 29.0 suggests: + // + // "Since one TLB inner-shareable invalidation is enough to avoid this erratum, the number + // of injected TLB invalidations should be minimized in the trap handler to mitigate + // the performance impact due to this workaround." +#ifndef PRODUCT + unsigned int cache_info = 0; + asm volatile ("mrs\t%0, ctr_el0":"=r" (cache_info)); + constexpr unsigned int CTR_IDC_SHIFT = 28; + constexpr unsigned int CTR_DIC_SHIFT = 29; + assert(((cache_info >> CTR_IDC_SHIFT) & 0x1) != 0x0, "Expect CTR_EL0.IDC to be enabled"); + assert(((cache_info >> CTR_DIC_SHIFT) & 0x1) == 0x0, "Expect CTR_EL0.DIC to be disabled"); +#endif + + // As the address for icache invalidation is not relevant + // and IC IVAU instruction is ignored, we use XZR in it. + asm volatile("dsb ish \n" + "ic ivau, xzr \n" + "isb \n" + : : : "memory"); + + deferred_icache_invalidation = false; + } +} + // Interface for updating the instruction cache. Whenever the VM // modifies code, part of the processor instruction cache potentially // has to be flushed. diff --git a/src/hotspot/share/runtime/icache.hpp b/src/hotspot/share/runtime/icache.hpp index efb7f89173556..88f9c20d9f22a 100644 --- a/src/hotspot/share/runtime/icache.hpp +++ b/src/hotspot/share/runtime/icache.hpp @@ -71,8 +71,6 @@ class AbstractICache : AllStatic { static void invalidate_range(address start, int nbytes); }; -class nmethod; - class ICacheInvalidationContext : StackObj { NONCOPYABLE(ICacheInvalidationContext); From 42745e5636d6897df0635aa8fcac847e4bc465b2 Mon Sep 17 00:00:00 2001 From: Evgeny Astigeevich Date: Tue, 25 Nov 2025 17:25:13 +0000 Subject: [PATCH 08/18] Remove redundant include --- src/hotspot/cpu/aarch64/icache_aarch64.hpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/hotspot/cpu/aarch64/icache_aarch64.hpp b/src/hotspot/cpu/aarch64/icache_aarch64.hpp index 46689a23d8095..9fd0e8575cbe7 100644 --- a/src/hotspot/cpu/aarch64/icache_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/icache_aarch64.hpp @@ -28,6 +28,4 @@ #include OS_CPU_HEADER(icache) -#include "code/nmethod.hpp" - #endif // CPU_AARCH64_ICACHE_AARCH64_HPP From 01a7ad295935ec602417336b809f9f276d32af7b Mon Sep 17 00:00:00 2001 From: Evgeny Astigeevich Date: Wed, 26 Nov 2025 10:31:24 +0000 Subject: [PATCH 09/18] Correct ifdef; Add dsb after ic --- src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp index a9ea416cf567f..024aaf375edd5 100644 --- a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp +++ b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp @@ -67,7 +67,7 @@ inline void ICacheInvalidationContext::pd_invalidate_icache() { // "Since one TLB inner-shareable invalidation is enough to avoid this erratum, the number // of injected TLB invalidations should be minimized in the trap handler to mitigate // the performance impact due to this workaround." -#ifndef PRODUCT +#ifdef ASSERT unsigned int cache_info = 0; asm volatile ("mrs\t%0, ctr_el0":"=r" (cache_info)); constexpr unsigned int CTR_IDC_SHIFT = 28; @@ -80,6 +80,7 @@ inline void ICacheInvalidationContext::pd_invalidate_icache() { // and IC IVAU instruction is ignored, we use XZR in it. asm volatile("dsb ish \n" "ic ivau, xzr \n" + "dsb ish \n" "isb \n" : : : "memory"); From 17456558ac6e3bc370cb7e686d876d2d39357f76 Mon Sep 17 00:00:00 2001 From: Evgeny Astigeevich Date: Wed, 26 Nov 2025 10:31:54 +0000 Subject: [PATCH 10/18] Fix code style --- src/hotspot/share/gc/z/zGeneration.cpp | 1 + src/hotspot/share/gc/z/zMark.cpp | 1 + src/hotspot/share/gc/z/zNMethod.cpp | 1 + src/hotspot/share/gc/z/zUnload.cpp | 3 ++- src/hotspot/share/runtime/icache.hpp | 4 ++-- 5 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/hotspot/share/gc/z/zGeneration.cpp b/src/hotspot/share/gc/z/zGeneration.cpp index 0b5105abbf2ad..ea4cd357ed257 100644 --- a/src/hotspot/share/gc/z/zGeneration.cpp +++ b/src/hotspot/share/gc/z/zGeneration.cpp @@ -1437,6 +1437,7 @@ class ZRemapNMethodClosure : public NMethodClosure { if (_bs_nm->is_armed(nm)) { { ICacheInvalidationContext icic; + // Heal barriers ZNMethod::nmethod_patch_barriers(nm); diff --git a/src/hotspot/share/gc/z/zMark.cpp b/src/hotspot/share/gc/z/zMark.cpp index 2f965fa575dce..5fe1fd3afd9b8 100644 --- a/src/hotspot/share/gc/z/zMark.cpp +++ b/src/hotspot/share/gc/z/zMark.cpp @@ -721,6 +721,7 @@ class ZMarkNMethodClosure : public NMethodClosure { if (_bs_nm->is_armed(nm)) { { ICacheInvalidationContext icic; + // Heal barriers ZNMethod::nmethod_patch_barriers(nm); diff --git a/src/hotspot/share/gc/z/zNMethod.cpp b/src/hotspot/share/gc/z/zNMethod.cpp index 4d3abab111b61..df12225eeab98 100644 --- a/src/hotspot/share/gc/z/zNMethod.cpp +++ b/src/hotspot/share/gc/z/zNMethod.cpp @@ -373,6 +373,7 @@ class ZNMethodUnlinkClosure : public NMethodClosure { { ICacheInvalidationContext icic; + // Heal oops and potentially mark young objects if there is a concurrent young collection. ZUncoloredRootProcessOopClosure cl(prev_color); ZNMethod::nmethod_oops_do_inner(nm, &cl); diff --git a/src/hotspot/share/gc/z/zUnload.cpp b/src/hotspot/share/gc/z/zUnload.cpp index e05eb15e22bd1..0afb09bb799ae 100644 --- a/src/hotspot/share/gc/z/zUnload.cpp +++ b/src/hotspot/share/gc/z/zUnload.cpp @@ -81,8 +81,9 @@ class ZIsUnloadingBehaviour : public IsUnloadingBehaviour { // Disarmed nmethods are alive return false; } - ZIsUnloadingOopClosure cl(nm); ICacheInvalidationContext icic; + + ZIsUnloadingOopClosure cl(nm); ZNMethod::nmethod_oops_do_inner(nm, &cl); return cl.is_unloading(); } diff --git a/src/hotspot/share/runtime/icache.hpp b/src/hotspot/share/runtime/icache.hpp index 88f9c20d9f22a..9d901c5ec29d4 100644 --- a/src/hotspot/share/runtime/icache.hpp +++ b/src/hotspot/share/runtime/icache.hpp @@ -72,13 +72,13 @@ class AbstractICache : AllStatic { }; class ICacheInvalidationContext : StackObj { + private: NONCOPYABLE(ICacheInvalidationContext); -private: void pd_init(); void pd_invalidate_icache(); -public: + public: ICacheInvalidationContext() { pd_init(); } From d36be373337822937899590624f6e133d41eeb70 Mon Sep 17 00:00:00 2001 From: Evgeny Astigeevich Date: Wed, 26 Nov 2025 16:53:11 +0000 Subject: [PATCH 11/18] Fix regressions for Java methods without field accesses --- .../linux_aarch64/icache_linux_aarch64.cpp | 2 +- .../linux_aarch64/icache_linux_aarch64.hpp | 17 ++++++++++------- src/hotspot/share/gc/z/zBarrierSetNMethod.cpp | 2 +- src/hotspot/share/gc/z/zGeneration.cpp | 2 +- src/hotspot/share/gc/z/zMark.cpp | 9 +++++++-- src/hotspot/share/gc/z/zNMethod.cpp | 16 ++++++++++++++-- src/hotspot/share/gc/z/zNMethod.hpp | 4 ++++ src/hotspot/share/gc/z/zUnload.cpp | 2 +- src/hotspot/share/runtime/icache.hpp | 4 +++- 9 files changed, 42 insertions(+), 16 deletions(-) diff --git a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp index 4dcb552f5798b..d0f05fd0652d1 100644 --- a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp +++ b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp @@ -24,4 +24,4 @@ #include "utilities/globalDefinitions.hpp" -THREAD_LOCAL bool deferred_icache_invalidation = false; +THREAD_LOCAL ICacheInvalidationContext* current_icache_invalidation_context = nullptr; diff --git a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp index 024aaf375edd5..7c3ed230ac767 100644 --- a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp +++ b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp @@ -30,21 +30,24 @@ #define PD_ICACHE_INVALIDATION_CONTEXT -extern THREAD_LOCAL bool deferred_icache_invalidation; +extern THREAD_LOCAL ICacheInvalidationContext* current_icache_invalidation_context; inline void ICacheInvalidationContext::pd_init() { - if (NeoverseN1Errata1542419) { - deferred_icache_invalidation = true; + if (NeoverseN1Errata1542419 && _needs_invalidation) { + current_icache_invalidation_context = this; } } inline bool ICacheInvalidationContext::deferred_invalidation() { - return deferred_icache_invalidation; + if (NeoverseN1Errata1542419 && current_icache_invalidation_context != nullptr) { + assert(current_icache_invalidation_context->_needs_invalidation, "ICacheInvalidationContext::deferred_invalidation must be invoked when icache invalidation is needed"); + return true; + } + return false; } inline void ICacheInvalidationContext::pd_invalidate_icache() { - if (NeoverseN1Errata1542419) { - assert(deferred_icache_invalidation, "Deferred icache invalidation must be enabled"); + if (NeoverseN1Errata1542419 && _needs_invalidation) { // Errata 1542419: Neoverse N1 cores with the 'COHERENT_ICACHE' feature may fetch stale // instructions when software depends on prefetch-speculation-protection // instead of explicit synchronization. @@ -84,7 +87,7 @@ inline void ICacheInvalidationContext::pd_invalidate_icache() { "isb \n" : : : "memory"); - deferred_icache_invalidation = false; + current_icache_invalidation_context = nullptr; } } diff --git a/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp b/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp index 143c0580811aa..858d97ceb30c7 100644 --- a/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp +++ b/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp @@ -72,7 +72,7 @@ bool ZBarrierSetNMethod::nmethod_entry_barrier(nmethod* nm) { } { - ICacheInvalidationContext icic; + ICacheInvalidationContext icic(ZNMethod::needs_icache_invalidation(nm)); // Heal barriers ZNMethod::nmethod_patch_barriers(nm); diff --git a/src/hotspot/share/gc/z/zGeneration.cpp b/src/hotspot/share/gc/z/zGeneration.cpp index ea4cd357ed257..09393cef82f25 100644 --- a/src/hotspot/share/gc/z/zGeneration.cpp +++ b/src/hotspot/share/gc/z/zGeneration.cpp @@ -1436,7 +1436,7 @@ class ZRemapNMethodClosure : public NMethodClosure { ZLocker locker(ZNMethod::lock_for_nmethod(nm)); if (_bs_nm->is_armed(nm)) { { - ICacheInvalidationContext icic; + ICacheInvalidationContext icic(ZNMethod::needs_icache_invalidation(nm)); // Heal barriers ZNMethod::nmethod_patch_barriers(nm); diff --git a/src/hotspot/share/gc/z/zMark.cpp b/src/hotspot/share/gc/z/zMark.cpp index 5fe1fd3afd9b8..68ced4212344a 100644 --- a/src/hotspot/share/gc/z/zMark.cpp +++ b/src/hotspot/share/gc/z/zMark.cpp @@ -720,7 +720,7 @@ class ZMarkNMethodClosure : public NMethodClosure { ZLocker locker(ZNMethod::lock_for_nmethod(nm)); if (_bs_nm->is_armed(nm)) { { - ICacheInvalidationContext icic; + ICacheInvalidationContext icic(ZNMethod::needs_icache_invalidation(nm)); // Heal barriers ZNMethod::nmethod_patch_barriers(nm); @@ -769,7 +769,12 @@ class ZMarkYoungNMethodClosure : public NMethodClosure { const bool complete_disarm = ZPointer::is_store_good(new_disarm_value_ptr); { - ICacheInvalidationContext icic; + bool needs_icache_invalidation = ZNMethod::needs_non_immediate_oops_patching(nm); + if (complete_disarm && ZNMethod::needs_barrier_patching(nm)) { + needs_icache_invalidation = true; + } + + ICacheInvalidationContext icic(needs_icache_invalidation); if (complete_disarm) { // We are about to completely disarm the nmethod, must take responsibility to patch all barriers before disarming diff --git a/src/hotspot/share/gc/z/zNMethod.cpp b/src/hotspot/share/gc/z/zNMethod.cpp index df12225eeab98..0411d259677c7 100644 --- a/src/hotspot/share/gc/z/zNMethod.cpp +++ b/src/hotspot/share/gc/z/zNMethod.cpp @@ -203,7 +203,7 @@ void ZNMethod::register_nmethod(nmethod* nm) { log_register(nm); { - ICacheInvalidationContext icic; + ICacheInvalidationContext icic(needs_barrier_patching(nm)); // Patch nmethod barriers nmethod_patch_barriers(nm); @@ -372,7 +372,7 @@ class ZNMethodUnlinkClosure : public NMethodClosure { assert(prev_color != ZPointerStoreGoodMask, "Potentially non-monotonic transition"); { - ICacheInvalidationContext icic; + ICacheInvalidationContext icic(ZNMethod::needs_non_immediate_oops_patching(nm)); // Heal oops and potentially mark young objects if there is a concurrent young collection. ZUncoloredRootProcessOopClosure cl(prev_color); @@ -428,3 +428,15 @@ void ZNMethod::unlink(ZWorkers* workers, bool unloading_occurred) { void ZNMethod::purge() { ClassUnloadingContext::context()->purge_and_free_nmethods(); } + +bool ZNMethod::needs_icache_invalidation(nmethod* nm) { + return needs_barrier_patching(nm) || needs_non_immediate_oops_patching(nm); +} + +bool ZNMethod::needs_barrier_patching(nmethod* nm) { + return gc_data(nm)->barriers()->is_nonempty(); +} + +bool ZNMethod::needs_non_immediate_oops_patching(nmethod* nm) { + return gc_data(nm)->has_non_immediate_oops(); +} diff --git a/src/hotspot/share/gc/z/zNMethod.hpp b/src/hotspot/share/gc/z/zNMethod.hpp index 5d797f3fd5542..6048e94b3734f 100644 --- a/src/hotspot/share/gc/z/zNMethod.hpp +++ b/src/hotspot/share/gc/z/zNMethod.hpp @@ -74,6 +74,10 @@ class ZNMethod : public AllStatic { static oop oop_load_no_keepalive(const nmethod* nm, int index); static oop oop_load_phantom(const nmethod* nm, int index); + + static bool needs_icache_invalidation(nmethod* nm); + static bool needs_barrier_patching(nmethod* nm); + static bool needs_non_immediate_oops_patching(nmethod* nm); }; #endif // SHARE_GC_Z_ZNMETHOD_HPP diff --git a/src/hotspot/share/gc/z/zUnload.cpp b/src/hotspot/share/gc/z/zUnload.cpp index 0afb09bb799ae..3d69bcb875516 100644 --- a/src/hotspot/share/gc/z/zUnload.cpp +++ b/src/hotspot/share/gc/z/zUnload.cpp @@ -81,7 +81,7 @@ class ZIsUnloadingBehaviour : public IsUnloadingBehaviour { // Disarmed nmethods are alive return false; } - ICacheInvalidationContext icic; + ICacheInvalidationContext icic(ZNMethod::needs_non_immediate_oops_patching(nm)); ZIsUnloadingOopClosure cl(nm); ZNMethod::nmethod_oops_do_inner(nm, &cl); diff --git a/src/hotspot/share/runtime/icache.hpp b/src/hotspot/share/runtime/icache.hpp index 9d901c5ec29d4..46a9e57db3618 100644 --- a/src/hotspot/share/runtime/icache.hpp +++ b/src/hotspot/share/runtime/icache.hpp @@ -75,11 +75,13 @@ class ICacheInvalidationContext : StackObj { private: NONCOPYABLE(ICacheInvalidationContext); + bool _needs_invalidation; + void pd_init(); void pd_invalidate_icache(); public: - ICacheInvalidationContext() { + ICacheInvalidationContext(bool needs_invalidation) : _needs_invalidation(needs_invalidation) { pd_init(); } From ae3b97e8ac31721a732c8f9a7ec7ca7daaaf3b4e Mon Sep 17 00:00:00 2001 From: Evgeny Astigeevich Date: Wed, 26 Nov 2025 18:25:37 +0000 Subject: [PATCH 12/18] Fix linux-cross-compile aarch64 build --- src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp index d0f05fd0652d1..98f09de736a3a 100644 --- a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp +++ b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp @@ -24,4 +24,6 @@ #include "utilities/globalDefinitions.hpp" +class ICacheInvalidationContext; + THREAD_LOCAL ICacheInvalidationContext* current_icache_invalidation_context = nullptr; From daf6cb70ab161a7cd6fc2d579e863605c400ed73 Mon Sep 17 00:00:00 2001 From: Evgeny Astigeevich Date: Fri, 28 Nov 2025 10:34:47 +0000 Subject: [PATCH 13/18] Add jtreg test --- .../gc/TestDeferredICacheInvalidation.java | 254 ++++++++++++++++++ 1 file changed, 254 insertions(+) create mode 100644 test/hotspot/jtreg/gc/TestDeferredICacheInvalidation.java diff --git a/test/hotspot/jtreg/gc/TestDeferredICacheInvalidation.java b/test/hotspot/jtreg/gc/TestDeferredICacheInvalidation.java new file mode 100644 index 0000000000000..eb6fc8b4a9c46 --- /dev/null +++ b/test/hotspot/jtreg/gc/TestDeferredICacheInvalidation.java @@ -0,0 +1,254 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +package gc; + +/* + * @test id=ParallelGC + * @bug 8370947 + * @summary Check no assertion is triggered when UseDeferredICacheInvalidation is enabled for ParallelGC + * @library /test/lib + * @requires vm.debug + * @requires os.family=="linux" + * @requires os.arch=="aarch64" + * @requires vm.gc.Parallel + * @build jdk.test.whitebox.WhiteBox + * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseParallelGC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation youngGC C1 + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseParallelGC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation fullGC C1 + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseParallelGC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation youngGC C2 + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseParallelGC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation fullGC C2 + */ + +/* + * @test id=G1GC + * @bug 8370947 + * @summary Check no assertion is triggered when UseDeferredICacheInvalidation is enabled for G1GC + * @library /test/lib + * @requires vm.debug + * @requires os.family=="linux" + * @requires os.arch=="aarch64" + * @requires vm.gc.G1 + * @build jdk.test.whitebox.WhiteBox + * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseG1GC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation youngGC C1 + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseG1GC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation fullGC C1 + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseG1GC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation youngGC C2 + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseG1GC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation fullGC C2 + */ + +/* + * @test id=ShenandoahGC + * @bug 8370947 + * @summary Check no assertion is triggered when UseDeferredICacheInvalidation is enabled for ShenandoahGC + * @library /test/lib + * @requires vm.debug + * @requires os.family=="linux" + * @requires os.arch=="aarch64" + * @requires vm.gc.Shenandoah + * @build jdk.test.whitebox.WhiteBox + * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseShenandoahGC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation fullGC C1 + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseShenandoahGC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation fullGC C2 + */ + +/* + * @test id=GenShenGC + * @bug 8370947 + * @summary Check no assertion is triggered when UseDeferredICacheInvalidation is enabled for generational ShenandoahGC + * @library /test/lib + * @requires vm.debug + * @requires os.family=="linux" + * @requires os.arch=="aarch64" + * @requires vm.gc.Shenandoah + * @build jdk.test.whitebox.WhiteBox + * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseShenandoahGC -XX:+UnlockExperimentalVMOptions -XX:ShenandoahGCMode=generational -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation youngGC C1 + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseShenandoahGC -XX:+UnlockExperimentalVMOptions -XX:ShenandoahGCMode=generational -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation fullGC C1 + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseShenandoahGC -XX:+UnlockExperimentalVMOptions -XX:ShenandoahGCMode=generational -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation youngGC C2 + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseShenandoahGC -XX:+UnlockExperimentalVMOptions -XX:ShenandoahGCMode=generational -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation fullGC C2 + */ + +/* + * @test id=ZGC + * @bug 8370947 + * @summary Check no assertion is triggered when UseDeferredICacheInvalidation is enabled for ZGC + * @library /test/lib + * @requires vm.debug + * @requires os.family=="linux" + * @requires os.arch=="aarch64" + * @requires vm.gc.Z + * @build jdk.test.whitebox.WhiteBox + * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseZGC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation youngGC C1 + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseZGC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation fullGC C1 + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseZGC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation youngGC C2 + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseZGC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation fullGC C2 + */ + +/* + * Nmethods have GC barriers and OOPs embedded into their code. GCs can patch nmethod's code + * which requires icache invalidation. Doing invalidation per instruction can be expensive. + * CPU can support hardware dcache and icache coherence. This would allow to defer cache + * invalidation. + * + * There are assertions for deferred cache invalidation. This test checks that all of them + * are passed. + */ + +import jdk.test.whitebox.WhiteBox; + +public class TestDeferredICacheInvalidation { + + private static final WhiteBox WB = WhiteBox.getWhiteBox(); + + public static class A { + public String s1; + public String s2; + public String s3; + public String s4; + public String s5; + public String s6; + public String s7; + public String s8; + public String s9; + } + + public static A a = new A(); + + private static int compLevel; + + public static class B { + public static void test0() { + } + + public static void test1() { + a.s1 = a.s1 + "1"; + } + + public static void test2() { + a.s1 = a.s1 + "1"; + a.s2 = a.s2 + "2"; + } + + public static void test3() { + a.s1 = a.s1 + "1"; + a.s2 = a.s2 + "2"; + a.s3 = a.s3 + "3"; + } + + public static void test4() { + a.s1 = a.s1 + "1"; + a.s2 = a.s2 + "2"; + a.s3 = a.s3 + "3"; + a.s4 = a.s4 + "4"; + } + + public static void test5() { + a.s1 = a.s1 + "1"; + a.s2 = a.s2 + "2"; + a.s3 = a.s3 + "3"; + a.s4 = a.s4 + "4"; + a.s5 = a.s5 + "5"; + } + + public static void test6() { + a.s1 = a.s1 + "1"; + a.s2 = a.s2 + "2"; + a.s3 = a.s3 + "3"; + a.s4 = a.s4 + "4"; + a.s5 = a.s5 + "5"; + a.s6 = a.s6 + "6"; + } + + public static void test7() { + a.s1 = a.s1 + "1"; + a.s2 = a.s2 + "2"; + a.s3 = a.s3 + "3"; + a.s4 = a.s4 + "4"; + a.s5 = a.s5 + "5"; + a.s6 = a.s6 + "6"; + a.s7 = a.s7 + "7"; + } + + public static void test8() { + a.s1 = a.s1 + "1"; + a.s2 = a.s2 + "2"; + a.s3 = a.s3 + "3"; + a.s4 = a.s4 + "4"; + a.s5 = a.s5 + "5"; + a.s6 = a.s6 + "6"; + a.s7 = a.s7 + "7"; + a.s8 = a.s8 + "8"; + } + + public static void test9() { + a.s1 = a.s1 + "1"; + a.s2 = a.s2 + "2"; + a.s3 = a.s3 + "3"; + a.s4 = a.s4 + "4"; + a.s5 = a.s5 + "5"; + a.s6 = a.s6 + "6"; + a.s7 = a.s7 + "7"; + a.s8 = a.s8 + "8"; + a.s9 = a.s9 + "9"; + } + } + + private static void compileMethods() throws Exception { + for (var m : B.class.getDeclaredMethods()) { + if (!m.getName().startsWith("test")) { + continue; + } + m.invoke(null); + WB.markMethodProfiled(m); + WB.enqueueMethodForCompilation(m, compLevel); + while (WB.isMethodQueuedForCompilation(m)) { + Thread.onSpinWait(); + } + if (WB.getMethodCompilationLevel(m) != compLevel) { + throw new IllegalStateException("Method " + m + " is not compiled at the compilation level: " + compLevel + ". Got: " + WB.getMethodCompilationLevel(m)); + } + } + } + + public static void youngGC() throws Exception { + a = null; + WB.youngGC(); + } + + public static void fullGC() throws Exception { + a = null; + WB.fullGC(); + } + + public static void main(String[] args) throws Exception { + if (!Boolean.TRUE.equals(WB.getBooleanVMFlag("UseDeferredICacheInvalidation"))) { + System.out.println("Skip. Test requires UseDeferredICacheInvalidation enabled."); + } + compLevel = (args[1].equals("C1")) ? 1 : 4; + compileMethods(); + TestDeferredICacheInvalidation.class.getMethod(args[0]).invoke(null); + } +} From dbeeecf139314ee07ae1d72c40b0694bca602bd4 Mon Sep 17 00:00:00 2001 From: Evgeny Astigeevich Date: Fri, 28 Nov 2025 10:36:38 +0000 Subject: [PATCH 14/18] Add UseDeferredICacheInvalidation to defer invalidation on CPU with hardware cache coherence --- .../gc/z/zBarrierSetAssembler_aarch64.cpp | 3 +- src/hotspot/cpu/aarch64/globals_aarch64.hpp | 2 + .../cpu/aarch64/vm_version_aarch64.cpp | 9 +- .../linux_aarch64/icache_linux_aarch64.hpp | 92 +++++++++++-------- 4 files changed, 66 insertions(+), 40 deletions(-) diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp index d32f96f38ba58..a3b6f8b16a02f 100644 --- a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp @@ -879,8 +879,7 @@ void ZBarrierSetAssembler::patch_barrier_relocation(address addr, int format) { ShouldNotReachHere(); } - if (NeoverseN1Errata1542419) { - // Instruction cache invalidation per barrier is expensive on Neoverse N1 having erratum 1542419. + if (UseDeferredICacheInvalidation) { // Defer the ICache invalidation to a later point where multiple patches can be handled together. // // Note: We rely on the fact that this function is only called from places where deferred invalidation diff --git a/src/hotspot/cpu/aarch64/globals_aarch64.hpp b/src/hotspot/cpu/aarch64/globals_aarch64.hpp index 430316360f6fe..a805c00d386e9 100644 --- a/src/hotspot/cpu/aarch64/globals_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/globals_aarch64.hpp @@ -129,6 +129,8 @@ define_pd_global(intx, InlineSmallCode, 1000); "Always merge DMB instructions in code emission") \ product(bool, NeoverseN1Errata1542419, false, DIAGNOSTIC, \ "Enable workaround for Neoverse N1 erratum 1542419") \ + product(bool, UseDeferredICacheInvalidation, false, DIAGNOSTIC, \ + "Defer multiple ICache invalidation to single invalidation") \ // end of ARCH_FLAGS diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp index 496838d226824..b598d71b40af9 100644 --- a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp @@ -657,8 +657,13 @@ void VM_Version::initialize() { FLAG_SET_DEFAULT(NeoverseN1Errata1542419, true); } - if (NeoverseN1Errata1542419 && !has_neoverse_n1_errata_1542419()) { - warning("NeoverseN1Errata1542419 is set for the CPU not having Neoverse N1 errata 1542419"); + if (NeoverseN1Errata1542419) { + if (!has_neoverse_n1_errata_1542419()) { + warning("NeoverseN1Errata1542419 is set for the CPU not having Neoverse N1 errata 1542419"); + } + if (FLAG_IS_DEFAULT(UseDeferredICacheInvalidation)) { + FLAG_SET_DEFAULT(UseDeferredICacheInvalidation, true); + } } // Construct the "features" string diff --git a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp index 7c3ed230ac767..65d6d16ee0590 100644 --- a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp +++ b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp @@ -33,59 +33,79 @@ extern THREAD_LOCAL ICacheInvalidationContext* current_icache_invalidation_context; inline void ICacheInvalidationContext::pd_init() { - if (NeoverseN1Errata1542419 && _needs_invalidation) { + if (UseDeferredICacheInvalidation && _needs_invalidation) { current_icache_invalidation_context = this; } } inline bool ICacheInvalidationContext::deferred_invalidation() { - if (NeoverseN1Errata1542419 && current_icache_invalidation_context != nullptr) { - assert(current_icache_invalidation_context->_needs_invalidation, "ICacheInvalidationContext::deferred_invalidation must be invoked when icache invalidation is needed"); + if (UseDeferredICacheInvalidation && current_icache_invalidation_context != nullptr) { + assert(current_icache_invalidation_context->_needs_invalidation, + "ICacheInvalidationContext::deferred_invalidation must be invoked " + "when icache invalidation is needed"); return true; } return false; } inline void ICacheInvalidationContext::pd_invalidate_icache() { - if (NeoverseN1Errata1542419 && _needs_invalidation) { - // Errata 1542419: Neoverse N1 cores with the 'COHERENT_ICACHE' feature may fetch stale - // instructions when software depends on prefetch-speculation-protection - // instead of explicit synchronization. - // - // Neoverse-N1 implementation mitigates the errata 1542419 with a workaround: - // - Disable coherent icache. - // - Trap IC IVAU instructions. - // - Execute: - // - tlbi vae3is, xzr - // - dsb sy - // - Ignore trapped IC IVAU instructions. - // - // `tlbi vae3is, xzr` invalidates all translation entries (all VAs, all possible levels). - // It waits for all memory accesses using in-scope old translation information to complete - // before it is considered complete. - // - // As this workaround has significant overhead, Arm Neoverse N1 (MP050) Software Developer - // Errata Notice version 29.0 suggests: - // - // "Since one TLB inner-shareable invalidation is enough to avoid this erratum, the number - // of injected TLB invalidations should be minimized in the trap handler to mitigate - // the performance impact due to this workaround." + if (UseDeferredICacheInvalidation && _needs_invalidation) { + + // For deferred icache invalidation, we expect hardware dcache + // and icache to be coherent: CTR_EL0.IDC == 1 and CTR_EL0.DIC == 1 + // An exception is Neoverse N1 with erratum 1542419, which requires + // a use of 'IC IVAU' instruction. In such a case, we expect + // CTR_EL0.DIC == 0. #ifdef ASSERT - unsigned int cache_info = 0; - asm volatile ("mrs\t%0, ctr_el0":"=r" (cache_info)); + static unsigned int cache_info = 0; + if (cache_info == 0) { + asm volatile ("mrs\t%0, ctr_el0":"=r" (cache_info)); + } constexpr unsigned int CTR_IDC_SHIFT = 28; constexpr unsigned int CTR_DIC_SHIFT = 29; assert(((cache_info >> CTR_IDC_SHIFT) & 0x1) != 0x0, "Expect CTR_EL0.IDC to be enabled"); - assert(((cache_info >> CTR_DIC_SHIFT) & 0x1) == 0x0, "Expect CTR_EL0.DIC to be disabled"); + if (NeoverseN1Errata1542419) { + assert(((cache_info >> CTR_DIC_SHIFT) & 0x1) == 0x0, "Expect CTR_EL0.DIC to be disabled for Neoverse N1 with erratum 1542419"); + } else { + assert(((cache_info >> CTR_DIC_SHIFT) & 0x1) != 0x0, "Expect CTR_EL0.DIC to be enabled"); + } #endif - // As the address for icache invalidation is not relevant - // and IC IVAU instruction is ignored, we use XZR in it. - asm volatile("dsb ish \n" - "ic ivau, xzr \n" - "dsb ish \n" - "isb \n" - : : : "memory"); + asm volatile("dsb ish" : : : "memory"); + + if (NeoverseN1Errata1542419) { + // Errata 1542419: Neoverse N1 cores with the 'COHERENT_ICACHE' feature + // may fetch stale instructions when software depends on + // prefetch-speculation-protection instead of explicit synchronization. + // + // Neoverse-N1 implementation mitigates the errata 1542419 with a + // workaround: + // - Disable coherent icache. + // - Trap IC IVAU instructions. + // - Execute: + // - tlbi vae3is, xzr + // - dsb sy + // - Ignore trapped IC IVAU instructions. + // + // `tlbi vae3is, xzr` invalidates all translation entries (all VAs, all + // possible levels). It waits for all memory accesses using in-scope old + // translation information to complete before it is considered complete. + // + // As this workaround has significant overhead, Arm Neoverse N1 (MP050) + // Software Developer Errata Notice version 29.0 suggests: + // + // "Since one TLB inner-shareable invalidation is enough to avoid this + // erratum, the number of injected TLB invalidations should be minimized + // in the trap handler to mitigate the performance impact due to this + // workaround." + // As the address for icache invalidation is not relevant and + // IC IVAU instruction is ignored, we use XZR in it. + asm volatile("ic ivau, xzr \n" + "dsb ish \n" + : : : "memory"); + } + + asm volatile("isb" : : : "memory"); current_icache_invalidation_context = nullptr; } From 79f9a2a0e01038843e331912c930b01f1e08b169 Mon Sep 17 00:00:00 2001 From: Evgeny Astigeevich Date: Wed, 3 Dec 2025 14:35:27 +0000 Subject: [PATCH 15/18] Add support of deferred icache invalidation to other GCs and JIT --- .../gc/z/zBarrierSetAssembler_aarch64.cpp | 5 +- src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp | 25 +++++++- .../linux_aarch64/icache_linux_aarch64.cpp | 4 +- .../linux_aarch64/icache_linux_aarch64.hpp | 56 +++++++++++------- src/hotspot/share/asm/codeBuffer.cpp | 15 +++-- src/hotspot/share/asm/codeBuffer.hpp | 12 ++++ src/hotspot/share/code/codeBlob.cpp | 4 ++ src/hotspot/share/code/codeCache.cpp | 2 + src/hotspot/share/code/nmethod.cpp | 30 ++++++++-- src/hotspot/share/code/nmethod.hpp | 5 +- src/hotspot/share/gc/g1/g1NMethodClosure.cpp | 3 + .../share/gc/shared/scavengableNMethods.cpp | 3 + .../gc/shenandoah/shenandoahCodeRoots.cpp | 1 + .../shenandoah/shenandoahNMethod.inline.hpp | 3 + src/hotspot/share/gc/z/zNMethod.cpp | 1 + src/hotspot/share/runtime/icache.hpp | 58 ++++++++++++++++--- 16 files changed, 179 insertions(+), 48 deletions(-) diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp index a3b6f8b16a02f..5cc08f3f44258 100644 --- a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp @@ -884,7 +884,10 @@ void ZBarrierSetAssembler::patch_barrier_relocation(address addr, int format) { // // Note: We rely on the fact that this function is only called from places where deferred invalidation // is safe. This assumption helps to avoid overhead of accessing thread-local data here. - assert(ICacheInvalidationContext::deferred_invalidation(), "ICache invalidation should be deferred"); + assert(ICacheInvalidationContext::current() != nullptr, "ICache invalidation context should be set"); + assert(ICacheInvalidationContext::current()->mode() == ICacheInvalidation::DEFERRED || + ICacheInvalidationContext::current()->mode() == ICacheInvalidation::NOT_NEEDED, + "ICache invalidation should be deferred or unneeded"); return; } diff --git a/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp b/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp index bc88a033f8a6d..a7460cd7e2dce 100644 --- a/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp @@ -26,6 +26,7 @@ #include "asm/macroAssembler.hpp" #include "code/nmethod.hpp" #include "code/relocInfo.hpp" +#include "gc/shared/barrierSetNMethod.hpp" #include "nativeInst_aarch64.hpp" #include "oops/oop.inline.hpp" #include "runtime/safepoint.hpp" @@ -55,9 +56,29 @@ void Relocation::pd_set_data_value(address x, bool verify_only) { break; } - if (ICacheInvalidationContext::deferred_invalidation()) { - // Instruction cache invalidation per relocation can be expensive, e.g. on Neoverse N1 having erratum 1542419. + if (UseDeferredICacheInvalidation) { // Defer the ICache invalidation to a later point where multiple patches can be handled together. + // + // Note: We rely on the fact that this function is only called from places where deferred invalidation + // is safe. This assumption helps to avoid overhead of accessing thread-local data here. + assert(ICacheInvalidationContext::current() != nullptr, "ICache invalidation context should be set"); + assert(ICacheInvalidationContext::current()->mode() == ICacheInvalidation::DEFERRED || + ICacheInvalidationContext::current()->mode() == ICacheInvalidation::NOT_NEEDED, + "ICache invalidation should be deferred or unneeded"); +#ifdef ASSERT + if (_binding != nullptr && _binding->code() != nullptr) { + nmethod *nm = _binding->code(); + if (!(BarrierSet::barrier_set()->barrier_set_nmethod()->is_armed(nm) || + SafepointSynchronize::is_at_safepoint() || + nm->is_unloading())) { + ConditionalMutexLocker ml(NMethodState_lock, !NMethodState_lock->owned_by_self(), Mutex::_no_safepoint_check_flag); + assert(nm->is_not_installed(), + "ICache invalidation context should only be used for armed " + "or unloading or not installed nmethod or at safepoint"); + + } + } +#endif return; } diff --git a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp index 98f09de736a3a..a85d462d39b19 100644 --- a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp +++ b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp @@ -24,6 +24,4 @@ #include "utilities/globalDefinitions.hpp" -class ICacheInvalidationContext; - -THREAD_LOCAL ICacheInvalidationContext* current_icache_invalidation_context = nullptr; +NOT_PRODUCT(THREAD_LOCAL ICacheInvalidationContext* current_icache_invalidation_context = nullptr;) diff --git a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp index 65d6d16ee0590..5a934a5c9e233 100644 --- a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp +++ b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp @@ -30,32 +30,23 @@ #define PD_ICACHE_INVALIDATION_CONTEXT -extern THREAD_LOCAL ICacheInvalidationContext* current_icache_invalidation_context; +NOT_PRODUCT(extern THREAD_LOCAL ICacheInvalidationContext* current_icache_invalidation_context;) inline void ICacheInvalidationContext::pd_init() { - if (UseDeferredICacheInvalidation && _needs_invalidation) { - current_icache_invalidation_context = this; + assert(current_icache_invalidation_context == nullptr, "nested ICacheInvalidationContext not supported"); + NOT_PRODUCT(current_icache_invalidation_context = this); + if (_mode == ICacheInvalidation::DEFERRED && _code == nullptr && !UseDeferredICacheInvalidation) { + _mode = ICacheInvalidation::IMMEDIATE; } } -inline bool ICacheInvalidationContext::deferred_invalidation() { - if (UseDeferredICacheInvalidation && current_icache_invalidation_context != nullptr) { - assert(current_icache_invalidation_context->_needs_invalidation, - "ICacheInvalidationContext::deferred_invalidation must be invoked " - "when icache invalidation is needed"); - return true; - } - return false; +#ifdef ASSERT +inline ICacheInvalidationContext* ICacheInvalidationContext::pd_current() { + return current_icache_invalidation_context; } +#endif -inline void ICacheInvalidationContext::pd_invalidate_icache() { - if (UseDeferredICacheInvalidation && _needs_invalidation) { - - // For deferred icache invalidation, we expect hardware dcache - // and icache to be coherent: CTR_EL0.IDC == 1 and CTR_EL0.DIC == 1 - // An exception is Neoverse N1 with erratum 1542419, which requires - // a use of 'IC IVAU' instruction. In such a case, we expect - // CTR_EL0.DIC == 0. +inline void assert_hardware_cache_coherency() { #ifdef ASSERT static unsigned int cache_info = 0; if (cache_info == 0) { @@ -70,6 +61,16 @@ inline void ICacheInvalidationContext::pd_invalidate_icache() { assert(((cache_info >> CTR_DIC_SHIFT) & 0x1) != 0x0, "Expect CTR_EL0.DIC to be enabled"); } #endif +} + +inline void ICacheInvalidationContext::pd_invalidate_icache() { + if (_mode == ICacheInvalidation::DEFERRED && UseDeferredICacheInvalidation) { + // For deferred icache invalidation, we expect hardware dcache + // and icache to be coherent: CTR_EL0.IDC == 1 and CTR_EL0.DIC == 1 + // An exception is Neoverse N1 with erratum 1542419, which requires + // a use of 'IC IVAU' instruction. In such a case, we expect + // CTR_EL0.DIC == 0. + assert_hardware_cache_coherency(); asm volatile("dsb ish" : : : "memory"); @@ -106,9 +107,11 @@ inline void ICacheInvalidationContext::pd_invalidate_icache() { } asm volatile("isb" : : : "memory"); - - current_icache_invalidation_context = nullptr; } + NOT_PRODUCT(current_icache_invalidation_context = nullptr); + _code = nullptr; + _size = 0; + _mode = ICacheInvalidation::NOT_NEEDED; } // Interface for updating the instruction cache. Whenever the VM @@ -122,7 +125,16 @@ class ICache : public AbstractICache { __builtin___clear_cache((char *)addr, (char *)(addr + 4)); } static void invalidate_range(address start, int nbytes) { - __builtin___clear_cache((char *)start, (char *)(start + nbytes)); + if (NeoverseN1Errata1542419) { + assert_hardware_cache_coherency(); + asm volatile("dsb ish \n" + "ic ivau, xzr \n" + "dsb ish \n" + "isb \n" + : : : "memory"); + } else { + __builtin___clear_cache((char *)start, (char *)(start + nbytes)); + } } }; diff --git a/src/hotspot/share/asm/codeBuffer.cpp b/src/hotspot/share/asm/codeBuffer.cpp index 2c6b7d7e96ec0..c9484881404d6 100644 --- a/src/hotspot/share/asm/codeBuffer.cpp +++ b/src/hotspot/share/asm/codeBuffer.cpp @@ -365,6 +365,10 @@ void CodeSection::relocate(address at, RelocationHolder const& spec, int format) // If it has data, insert the prefix, as (data_prefix_tag | data1), data2. end->initialize(this, reloc); + if (rtype == relocInfo::oop_type && + !((oop_Relocation*)reloc)->oop_is_immediate()) { + _has_non_immediate_oops = true; + } } void CodeSection::initialize_locs(int locs_capacity) { @@ -745,9 +749,6 @@ void CodeBuffer::copy_code_to(CodeBlob* dest_blob) { // Done moving code bytes; were they the right size? assert((int)align_up(dest.total_content_size(), oopSize) == dest_blob->content_size(), "sanity"); - - // Flush generated code - ICache::invalidate_range(dest_blob->code_begin(), dest_blob->code_size()); } // Move all my code into another code buffer. Consult applicable @@ -930,8 +931,12 @@ void CodeBuffer::expand(CodeSection* which_cs, csize_t amount) { // Needs to be initialized when calling fix_relocation_after_move. cb.blob()->set_ctable_begin(cb.consts()->start()); - // Move all the code and relocations to the new blob: - relocate_code_to(&cb); + { + ICacheInvalidationContext icic(ICacheInvalidation::NOT_NEEDED); + + // Move all the code and relocations to the new blob: + relocate_code_to(&cb); + } // some internal addresses, _last_insn _last_label, are used during code emission, // adjust them in expansion diff --git a/src/hotspot/share/asm/codeBuffer.hpp b/src/hotspot/share/asm/codeBuffer.hpp index 430d4949467d4..b6e8b76097b97 100644 --- a/src/hotspot/share/asm/codeBuffer.hpp +++ b/src/hotspot/share/asm/codeBuffer.hpp @@ -106,6 +106,7 @@ class CodeSection { int _skipped_instructions_size; int8_t _index; // my section number (SECT_INST, etc.) CodeBuffer* _outer; // enclosing CodeBuffer + bool _has_non_immediate_oops; // (Note: _locs_point used to be called _last_reloc_offset.) @@ -121,6 +122,7 @@ class CodeSection { _locs_own = false; _scratch_emit = false; _skipped_instructions_size = 0; + _has_non_immediate_oops = false; DEBUG_ONLY(_index = -1); DEBUG_ONLY(_outer = (CodeBuffer*)badAddress); } @@ -281,6 +283,10 @@ class CodeSection { // Return true if there was an expansion. bool maybe_expand_to_ensure_remaining(csize_t amount); + bool has_non_immediate_oops() const { + return _has_non_immediate_oops; + } + #ifndef PRODUCT void decode(); void print_on(outputStream* st, const char* name); @@ -852,6 +858,12 @@ class CodeBuffer: public StackObj DEBUG_ONLY(COMMA private Scrubber) { } } + bool has_non_immediate_oops() const { + return _consts.has_non_immediate_oops() + || _insts.has_non_immediate_oops() + || _stubs.has_non_immediate_oops(); + } + void block_comment(ptrdiff_t offset, const char* comment) PRODUCT_RETURN; const char* code_string(const char* str) PRODUCT_RETURN_(return nullptr;); diff --git a/src/hotspot/share/code/codeBlob.cpp b/src/hotspot/share/code/codeBlob.cpp index a0a34ec23fad6..d5bb6fa1a1361 100644 --- a/src/hotspot/share/code/codeBlob.cpp +++ b/src/hotspot/share/code/codeBlob.cpp @@ -331,6 +331,10 @@ RuntimeBlob::RuntimeBlob( : CodeBlob(name, kind, cb, size, header_size, frame_complete, frame_size, oop_maps, caller_must_gc_arguments, align_up(cb->total_relocation_size(), oopSize)) { + // Optimize ICache invalidation by batching it for the whole blob if + // possible. + ICacheInvalidationContext icic(code_begin(), code_size()); + cb->copy_code_and_locs_to(this); } diff --git a/src/hotspot/share/code/codeCache.cpp b/src/hotspot/share/code/codeCache.cpp index beca3bcbcf5d8..56a7b8f8e092b 100644 --- a/src/hotspot/share/code/codeCache.cpp +++ b/src/hotspot/share/code/codeCache.cpp @@ -681,6 +681,8 @@ void CodeCache::nmethods_do(NMethodClosure* cl) { assert_locked_or_safepoint(CodeCache_lock); NMethodIterator iter(NMethodIterator::all); while(iter.next()) { + ICacheInvalidationContext icic(iter.method()->has_non_immediate_oops()); + cl->do_nmethod(iter.method()); } } diff --git a/src/hotspot/share/code/nmethod.cpp b/src/hotspot/share/code/nmethod.cpp index d91af9b4991e1..8c80ed1f8477e 100644 --- a/src/hotspot/share/code/nmethod.cpp +++ b/src/hotspot/share/code/nmethod.cpp @@ -1231,6 +1231,7 @@ void nmethod::init_defaults(CodeBuffer *code_buffer, CodeOffsets* offsets) { _has_flushed_dependencies = 0; _is_unlinked = 0; _load_reported = 0; // jvmti state + _has_non_immediate_oops = 0; _deoptimization_status = not_marked; @@ -1324,9 +1325,16 @@ nmethod::nmethod( _speculations_offset = 0; #endif _immutable_data_ref_count_offset = 0; + _has_non_immediate_oops = code_buffer->has_non_immediate_oops(); - code_buffer->copy_code_and_locs_to(this); - code_buffer->copy_values_to(this); + { + // Optimize ICache invalidation by batching it for the whole blob if + // possible. + ICacheInvalidationContext icic(code_begin(), code_size()); + + code_buffer->copy_code_and_locs_to(this); + code_buffer->copy_values_to(this); + } post_init(); } @@ -1480,6 +1488,7 @@ nmethod::nmethod(const nmethod &nm) : CodeBlob(nm._name, nm._kind, nm._size, nm. _has_flushed_dependencies = nm._has_flushed_dependencies; _is_unlinked = nm._is_unlinked; _load_reported = nm._load_reported; + _has_non_immediate_oops = nm._has_non_immediate_oops; _deoptimization_status = nm._deoptimization_status; @@ -1693,6 +1702,8 @@ nmethod::nmethod( _num_stack_arg_slots = entry_bci != InvocationEntryBci ? 0 : _method->constMethod()->num_stack_arg_slots(); + _has_non_immediate_oops = code_buffer->has_non_immediate_oops(); + set_ctable_begin(header_begin() + content_offset()); #if INCLUDE_JVMCI @@ -1762,10 +1773,17 @@ nmethod::nmethod( assert(immutable_data_end_offset <= immutable_data_size, "wrong read-only data size: %d > %d", immutable_data_end_offset, immutable_data_size); - // Copy code and relocation info - code_buffer->copy_code_and_locs_to(this); - // Copy oops and metadata - code_buffer->copy_values_to(this); + { + // Optimize ICache invalidation by batching it for the whole blob if + // possible. + ICacheInvalidationContext icic(code_begin(), code_size()); + + // Copy code and relocation info + code_buffer->copy_code_and_locs_to(this); + // Copy oops and metadata + code_buffer->copy_values_to(this); + } + dependencies->copy_to(this); // Copy PcDesc and ScopeDesc data debug_info->copy_to(this); diff --git a/src/hotspot/share/code/nmethod.hpp b/src/hotspot/share/code/nmethod.hpp index 34accf428b690..ea9801e9e22dc 100644 --- a/src/hotspot/share/code/nmethod.hpp +++ b/src/hotspot/share/code/nmethod.hpp @@ -274,7 +274,8 @@ class nmethod : public CodeBlob { _has_scoped_access:1, // used by for shared scope closure (scopedMemoryAccess.cpp) _has_flushed_dependencies:1, // Used for maintenance of dependencies (under CodeCache_lock) _is_unlinked:1, // mark during class unloading - _load_reported:1; // used by jvmti to track if an event has been posted for this nmethod + _load_reported:1, // used by jvmti to track if an event has been posted for this nmethod + _has_non_immediate_oops:1; enum DeoptimizationStatus : u1 { not_marked, @@ -774,6 +775,8 @@ class nmethod : public CodeBlob { int comp_level() const { return _comp_level; } + bool has_non_immediate_oops() const { return _has_non_immediate_oops; } + // Support for oops in scopes and relocs: // Note: index 0 is reserved for null. oop oop_at(int index) const; diff --git a/src/hotspot/share/gc/g1/g1NMethodClosure.cpp b/src/hotspot/share/gc/g1/g1NMethodClosure.cpp index d74aa5eae1d01..a327fcd57743f 100644 --- a/src/hotspot/share/gc/g1/g1NMethodClosure.cpp +++ b/src/hotspot/share/gc/g1/g1NMethodClosure.cpp @@ -32,6 +32,7 @@ #include "oops/access.inline.hpp" #include "oops/compressedOops.inline.hpp" #include "oops/oop.inline.hpp" +#include "runtime/icache.hpp" template void G1NMethodClosure::HeapRegionGatheringOopClosure::do_oop_work(T* p) { @@ -87,6 +88,8 @@ void G1NMethodClosure::do_evacuation_and_fixup(nmethod* nm) { bs_nm->disarm(nm); } + ICacheInvalidationContext icic(nm->has_non_immediate_oops()); + nm->fix_oop_relocations(); } diff --git a/src/hotspot/share/gc/shared/scavengableNMethods.cpp b/src/hotspot/share/gc/shared/scavengableNMethods.cpp index 887ac5f43a28a..a0fe7521fefc6 100644 --- a/src/hotspot/share/gc/shared/scavengableNMethods.cpp +++ b/src/hotspot/share/gc/shared/scavengableNMethods.cpp @@ -26,6 +26,7 @@ #include "code/nmethod.hpp" #include "gc/shared/scavengableNMethods.hpp" #include "gc/shared/scavengableNMethodsData.hpp" +#include "runtime/icache.hpp" #include "runtime/mutexLocker.hpp" #include "utilities/debug.hpp" @@ -141,6 +142,8 @@ void ScavengableNMethods::nmethods_do_and_prune(NMethodToOopClosure* cl) { assert(data.on_list(), "else shouldn't be on this list"); if (cl != nullptr) { + ICacheInvalidationContext icic(cur->has_non_immediate_oops()); + cl->do_nmethod(cur); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp index ec39e0c0ccb0a..6931ddf347ac8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp @@ -34,6 +34,7 @@ #include "memory/resourceArea.hpp" #include "memory/universe.hpp" #include "runtime/atomicAccess.hpp" +#include "runtime/icache.hpp" #include "utilities/powerOfTwo.hpp" diff --git a/src/hotspot/share/gc/shenandoah/shenandoahNMethod.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahNMethod.inline.hpp index 6758298675ba5..16e5d1ad8b2d7 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahNMethod.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahNMethod.inline.hpp @@ -67,6 +67,9 @@ void ShenandoahNMethod::oops_do(OopClosure* oops, bool fix_relocations) { void ShenandoahNMethod::heal_nmethod_metadata(ShenandoahNMethod* nmethod_data) { ShenandoahEvacuateUpdateMetadataClosure cl; + ICacheInvalidationContext icic(nmethod_data->_has_non_immed_oops); + + assert(nmethod_data->_has_non_immed_oops == nmethod_data->_nm->has_non_immediate_oops(), "Inconsistent non-immed oops state"); nmethod_data->oops_do(&cl, true /*fix relocation*/); } diff --git a/src/hotspot/share/gc/z/zNMethod.cpp b/src/hotspot/share/gc/z/zNMethod.cpp index 0411d259677c7..470b54da9f47c 100644 --- a/src/hotspot/share/gc/z/zNMethod.cpp +++ b/src/hotspot/share/gc/z/zNMethod.cpp @@ -438,5 +438,6 @@ bool ZNMethod::needs_barrier_patching(nmethod* nm) { } bool ZNMethod::needs_non_immediate_oops_patching(nmethod* nm) { + assert(gc_data(nm)->has_non_immediate_oops() == nm->has_non_immediate_oops(), "Inconsistent non-immed oops state"); return gc_data(nm)->has_non_immediate_oops(); } diff --git a/src/hotspot/share/runtime/icache.hpp b/src/hotspot/share/runtime/icache.hpp index 46a9e57db3618..21dfd65588378 100644 --- a/src/hotspot/share/runtime/icache.hpp +++ b/src/hotspot/share/runtime/icache.hpp @@ -71,25 +71,53 @@ class AbstractICache : AllStatic { static void invalidate_range(address start, int nbytes); }; +enum class ICacheInvalidation : uint8_t { + NOT_NEEDED = 0, + IMMEDIATE = 1, + DEFERRED = 2 +}; + class ICacheInvalidationContext : StackObj { private: NONCOPYABLE(ICacheInvalidationContext); - bool _needs_invalidation; + address _code; + int _size; + ICacheInvalidation _mode; void pd_init(); void pd_invalidate_icache(); + NOT_PRODUCT(static ICacheInvalidationContext* pd_current()); + public: - ICacheInvalidationContext(bool needs_invalidation) : _needs_invalidation(needs_invalidation) { + // Hardware optimized icache invalidation which does not depend on what code is invalidated. + ICacheInvalidationContext(ICacheInvalidation mode) : _code(nullptr), _size(0), _mode(mode) { + pd_init(); + } + + ICacheInvalidationContext(bool needs_invalidation) + : ICacheInvalidationContext(needs_invalidation + ? ICacheInvalidation::DEFERRED + : ICacheInvalidation::NOT_NEEDED) {} + + // Use hardware optimized icache invalidation if possible, + // Otherwise invalidate the specified code range. + ICacheInvalidationContext(address code, int size) : _code(code), _size(size), _mode(ICacheInvalidation::DEFERRED) { pd_init(); } - ~ICacheInvalidationContext() { - pd_invalidate_icache(); + ~ICacheInvalidationContext(); + +#ifdef ASSERT + static ICacheInvalidationContext* current() { + return pd_current(); } +#endif - static bool deferred_invalidation(); + ICacheInvalidation mode() const { + return _mode; + } }; // Must be included before the definition of ICacheStubGenerator @@ -153,9 +181,23 @@ class ICacheStubGenerator : public StubCodeGenerator { // Default implementation: do nothing inline void ICacheInvalidationContext::pd_init() {} inline void ICacheInvalidationContext::pd_invalidate_icache() {} - inline bool ICacheInvalidationContext::deferred_invalidation() { - return false; +#ifdef ASSERT + inline ICacheInvalidationContext* ICacheInvalidationContext::pd_current() { + Unimplemented(); + return nullptr; } -#endif +#endif // ASSERT +#endif // PD_ICACHE_INVALIDATION_CONTEXT + +inline ICacheInvalidationContext::~ICacheInvalidationContext() { + pd_invalidate_icache(); + if (_code != nullptr) { + assert(_mode == ICacheInvalidation::DEFERRED, "sanity"); + assert(_size > 0, "size must be positive"); + ICache::invalidate_range(_code, _size); + _code = nullptr; + _size = 0; + } +} #endif // SHARE_RUNTIME_ICACHE_HPP From 8c5ef0e8f6a60505e514edc8799b072d5318732c Mon Sep 17 00:00:00 2001 From: Evgeny Astigeevich Date: Wed, 3 Dec 2025 14:49:50 +0000 Subject: [PATCH 16/18] Remove trailing whitespaces --- src/hotspot/share/gc/shenandoah/shenandoahNMethod.inline.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahNMethod.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahNMethod.inline.hpp index 16e5d1ad8b2d7..d98708cf6f133 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahNMethod.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahNMethod.inline.hpp @@ -68,7 +68,7 @@ void ShenandoahNMethod::oops_do(OopClosure* oops, bool fix_relocations) { void ShenandoahNMethod::heal_nmethod_metadata(ShenandoahNMethod* nmethod_data) { ShenandoahEvacuateUpdateMetadataClosure cl; ICacheInvalidationContext icic(nmethod_data->_has_non_immed_oops); - + assert(nmethod_data->_has_non_immed_oops == nmethod_data->_nm->has_non_immediate_oops(), "Inconsistent non-immed oops state"); nmethod_data->oops_do(&cl, true /*fix relocation*/); } From 4b04496f852f30fa5f68e01c85b9c5f64218fd87 Mon Sep 17 00:00:00 2001 From: Evgeny Astigeevich Date: Wed, 3 Dec 2025 15:39:24 +0000 Subject: [PATCH 17/18] Fix linux-cross-compile build aarch64 --- src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp index a85d462d39b19..0011be2fc4851 100644 --- a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp +++ b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp @@ -24,4 +24,6 @@ #include "utilities/globalDefinitions.hpp" +class ICacheInvalidationContext; + NOT_PRODUCT(THREAD_LOCAL ICacheInvalidationContext* current_icache_invalidation_context = nullptr;) From b9380fd8fb8393c9a1b11ec9dbb64154a7fad41b Mon Sep 17 00:00:00 2001 From: Evgeny Astigeevich Date: Fri, 5 Dec 2025 17:49:37 +0000 Subject: [PATCH 18/18] Implement nested ICacheInvalidationContext --- src/hotspot/cpu/aarch64/globals_aarch64.hpp | 2 - src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp | 2 +- .../cpu/aarch64/vm_version_aarch64.cpp | 1 + .../linux_aarch64/icache_linux_aarch64.cpp | 5 +- .../linux_aarch64/icache_linux_aarch64.hpp | 194 ++++++++++++------ src/hotspot/share/asm/codeBuffer.cpp | 10 +- src/hotspot/share/asm/codeBuffer.hpp | 12 -- src/hotspot/share/code/codeBlob.cpp | 1 - src/hotspot/share/code/codeCache.cpp | 3 +- src/hotspot/share/code/nmethod.cpp | 17 +- src/hotspot/share/code/nmethod.hpp | 5 +- src/hotspot/share/code/relocInfo.cpp | 8 +- src/hotspot/share/code/relocInfo.hpp | 4 +- src/hotspot/share/gc/g1/g1NMethodClosure.cpp | 3 - .../share/gc/shared/scavengableNMethods.cpp | 3 - .../gc/shenandoah/shenandoahCodeRoots.cpp | 1 - .../shenandoah/shenandoahNMethod.inline.hpp | 3 - src/hotspot/share/gc/z/zBarrierSetNMethod.cpp | 3 +- src/hotspot/share/gc/z/zGeneration.cpp | 3 +- src/hotspot/share/gc/z/zMark.cpp | 11 +- src/hotspot/share/gc/z/zNMethod.cpp | 35 +--- src/hotspot/share/gc/z/zNMethod.hpp | 4 - src/hotspot/share/gc/z/zUnload.cpp | 3 - src/hotspot/share/runtime/globals.hpp | 2 + src/hotspot/share/runtime/icache.cpp | 2 + src/hotspot/share/runtime/icache.hpp | 118 +++++------ .../gc/TestDeferredICacheInvalidation.java | 20 +- 27 files changed, 235 insertions(+), 240 deletions(-) diff --git a/src/hotspot/cpu/aarch64/globals_aarch64.hpp b/src/hotspot/cpu/aarch64/globals_aarch64.hpp index a805c00d386e9..430316360f6fe 100644 --- a/src/hotspot/cpu/aarch64/globals_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/globals_aarch64.hpp @@ -129,8 +129,6 @@ define_pd_global(intx, InlineSmallCode, 1000); "Always merge DMB instructions in code emission") \ product(bool, NeoverseN1Errata1542419, false, DIAGNOSTIC, \ "Enable workaround for Neoverse N1 erratum 1542419") \ - product(bool, UseDeferredICacheInvalidation, false, DIAGNOSTIC, \ - "Defer multiple ICache invalidation to single invalidation") \ // end of ARCH_FLAGS diff --git a/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp b/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp index a7460cd7e2dce..7bd5625e3430f 100644 --- a/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp @@ -64,7 +64,7 @@ void Relocation::pd_set_data_value(address x, bool verify_only) { assert(ICacheInvalidationContext::current() != nullptr, "ICache invalidation context should be set"); assert(ICacheInvalidationContext::current()->mode() == ICacheInvalidation::DEFERRED || ICacheInvalidationContext::current()->mode() == ICacheInvalidation::NOT_NEEDED, - "ICache invalidation should be deferred or unneeded"); + "ICache invalidation should be deferred or unneeded."); #ifdef ASSERT if (_binding != nullptr && _binding->code() != nullptr) { nmethod *nm = _binding->code(); diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp index b598d71b40af9..6de8de3688b86 100644 --- a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp @@ -68,6 +68,7 @@ static bool has_neoverse_n1_errata_1542419() { const int minor_rev_num = VM_Version::cpu_revision(); // Neoverse N1: 0xd0c // Erratum 1542419 affects r3p0, r3p1 and r4p0. + // It is fixed in r4p1 and later revisions, which are not affected. return (VM_Version::cpu_family() == VM_Version::CPU_ARM && VM_Version::model_is(0xd0c) && ((major_rev_num == 3 && minor_rev_num == 0) || diff --git a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp index 0011be2fc4851..4ec720293b552 100644 --- a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp +++ b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp @@ -22,8 +22,7 @@ * */ +#include "runtime/icache.hpp" #include "utilities/globalDefinitions.hpp" -class ICacheInvalidationContext; - -NOT_PRODUCT(THREAD_LOCAL ICacheInvalidationContext* current_icache_invalidation_context = nullptr;) +THREAD_LOCAL AArch64ICacheInvalidationContext* AArch64ICacheInvalidationContext::_current_context = nullptr; diff --git a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp index 5a934a5c9e233..33892eb052d9a 100644 --- a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp +++ b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2014, Red Hat Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -26,50 +26,132 @@ #ifndef OS_CPU_LINUX_AARCH64_ICACHE_AARCH64_HPP #define OS_CPU_LINUX_AARCH64_ICACHE_AARCH64_HPP +#include "memory/allocation.hpp" #include "utilities/globalDefinitions.hpp" -#define PD_ICACHE_INVALIDATION_CONTEXT - -NOT_PRODUCT(extern THREAD_LOCAL ICacheInvalidationContext* current_icache_invalidation_context;) - -inline void ICacheInvalidationContext::pd_init() { - assert(current_icache_invalidation_context == nullptr, "nested ICacheInvalidationContext not supported"); - NOT_PRODUCT(current_icache_invalidation_context = this); - if (_mode == ICacheInvalidation::DEFERRED && _code == nullptr && !UseDeferredICacheInvalidation) { - _mode = ICacheInvalidation::IMMEDIATE; +inline void assert_hardware_cache_coherency() { + // For deferred icache invalidation, we expect hardware dcache + // and icache to be coherent: CTR_EL0.IDC == 1 and CTR_EL0.DIC == 1 + // An exception is Neoverse N1 with erratum 1542419, which requires + // a use of 'IC IVAU' instruction. In such a case, we expect + // CTR_EL0.DIC == 0. +#ifdef ASSERT + static unsigned int cache_info = 0; + if (cache_info == 0) { + asm volatile("mrs\t%0, ctr_el0" : "=r"(cache_info)); + } + constexpr unsigned int CTR_IDC_SHIFT = 28; + constexpr unsigned int CTR_DIC_SHIFT = 29; + assert(((cache_info >> CTR_IDC_SHIFT) & 0x1) != 0x0, + "Expect CTR_EL0.IDC to be enabled"); + if (NeoverseN1Errata1542419) { + assert(((cache_info >> CTR_DIC_SHIFT) & 0x1) == 0x0, + "Expect CTR_EL0.DIC to be disabled for Neoverse N1 with erratum " + "1542419"); + } else { + assert(((cache_info >> CTR_DIC_SHIFT) & 0x1) != 0x0, + "Expect CTR_EL0.DIC to be enabled"); } +#endif } -#ifdef ASSERT -inline ICacheInvalidationContext* ICacheInvalidationContext::pd_current() { - return current_icache_invalidation_context; -} -#endif +// Interface for updating the instruction cache. Whenever the VM +// modifies code, part of the processor instruction cache potentially +// has to be flushed. -inline void assert_hardware_cache_coherency() { -#ifdef ASSERT - static unsigned int cache_info = 0; - if (cache_info == 0) { - asm volatile ("mrs\t%0, ctr_el0":"=r" (cache_info)); - } - constexpr unsigned int CTR_IDC_SHIFT = 28; - constexpr unsigned int CTR_DIC_SHIFT = 29; - assert(((cache_info >> CTR_IDC_SHIFT) & 0x1) != 0x0, "Expect CTR_EL0.IDC to be enabled"); +class ICache : public AbstractICache { + public: + static void initialize(int phase); + static void invalidate_word(address addr) { + __builtin___clear_cache((char *)addr, (char *)(addr + 4)); + } + static void invalidate_range(address start, int nbytes) { if (NeoverseN1Errata1542419) { - assert(((cache_info >> CTR_DIC_SHIFT) & 0x1) == 0x0, "Expect CTR_EL0.DIC to be disabled for Neoverse N1 with erratum 1542419"); + assert_hardware_cache_coherency(); + asm volatile("dsb ish \n" + "ic ivau, xzr \n" + "dsb ish \n" + "isb \n" + : : : "memory"); } else { - assert(((cache_info >> CTR_DIC_SHIFT) & 0x1) != 0x0, "Expect CTR_EL0.DIC to be enabled"); + __builtin___clear_cache((char *)start, (char *)(start + nbytes)); + } + } +}; + +class AArch64ICacheInvalidationContext final : StackObj { + private: + NONCOPYABLE(AArch64ICacheInvalidationContext); + + static THREAD_LOCAL AArch64ICacheInvalidationContext* _current_context; + + AArch64ICacheInvalidationContext* _parent; + address _code; + int _size; + ICacheInvalidation _mode; + bool _has_modified_code; + + public: + AArch64ICacheInvalidationContext(ICacheInvalidation mode) + : _parent(nullptr), _code(nullptr), _size(0), _mode(mode), _has_modified_code(false) { + _parent = _current_context; + _current_context = this; + if (_parent != nullptr) { + // The parent context is in charge of icache invalidation. + _mode = (_parent->mode() == ICacheInvalidation::IMMEDIATE) ? ICacheInvalidation::IMMEDIATE : ICacheInvalidation::NOT_NEEDED; + } + } + + AArch64ICacheInvalidationContext() + : AArch64ICacheInvalidationContext(UseDeferredICacheInvalidation + ? ICacheInvalidation::DEFERRED + : ICacheInvalidation::IMMEDIATE) {} + + AArch64ICacheInvalidationContext(address code, int size) + : _parent(nullptr), + _code(code), + _size(size), + _mode(ICacheInvalidation::DEFERRED), + _has_modified_code(true) { + assert(_current_context == nullptr, + "nested ICacheInvalidationContext(code, size) not supported"); + assert(code != nullptr, "code must not be null for deferred invalidation"); + assert(size > 0, "size must be positive for deferred invalidation"); + + _current_context = this; + + if (UseDeferredICacheInvalidation) { + // With hardware dcache and icache coherency, we don't need _code. + _code = nullptr; + _size = 0; + } + } + + ~AArch64ICacheInvalidationContext() { + _current_context = _parent; + + if (_code != nullptr) { + assert(_size > 0, "size must be positive for deferred invalidation"); + assert(_mode == ICacheInvalidation::DEFERRED, "sanity"); + assert(_has_modified_code, "sanity"); + assert(_parent == nullptr, "sanity"); + + ICache::invalidate_range(_code, _size); + return; + } + + if (!_has_modified_code) { + return; + } + + if (_parent != nullptr) { + _parent->set_has_modified_code(); + } + + if (_mode != ICacheInvalidation::DEFERRED) { + return; } -#endif -} -inline void ICacheInvalidationContext::pd_invalidate_icache() { - if (_mode == ICacheInvalidation::DEFERRED && UseDeferredICacheInvalidation) { - // For deferred icache invalidation, we expect hardware dcache - // and icache to be coherent: CTR_EL0.IDC == 1 and CTR_EL0.DIC == 1 - // An exception is Neoverse N1 with erratum 1542419, which requires - // a use of 'IC IVAU' instruction. In such a case, we expect - // CTR_EL0.DIC == 0. assert_hardware_cache_coherency(); asm volatile("dsb ish" : : : "memory"); @@ -101,41 +183,27 @@ inline void ICacheInvalidationContext::pd_invalidate_icache() { // workaround." // As the address for icache invalidation is not relevant and // IC IVAU instruction is ignored, we use XZR in it. - asm volatile("ic ivau, xzr \n" - "dsb ish \n" - : : : "memory"); + asm volatile( + "ic ivau, xzr \n" + "dsb ish \n" + : + : + : "memory"); } - asm volatile("isb" : : : "memory"); } - NOT_PRODUCT(current_icache_invalidation_context = nullptr); - _code = nullptr; - _size = 0; - _mode = ICacheInvalidation::NOT_NEEDED; -} -// Interface for updating the instruction cache. Whenever the VM -// modifies code, part of the processor instruction cache potentially -// has to be flushed. + ICacheInvalidation mode() const { return _mode; } -class ICache : public AbstractICache { - public: - static void initialize(int phase); - static void invalidate_word(address addr) { - __builtin___clear_cache((char *)addr, (char *)(addr + 4)); + void set_has_modified_code() { + _has_modified_code = true; } - static void invalidate_range(address start, int nbytes) { - if (NeoverseN1Errata1542419) { - assert_hardware_cache_coherency(); - asm volatile("dsb ish \n" - "ic ivau, xzr \n" - "dsb ish \n" - "isb \n" - : : : "memory"); - } else { - __builtin___clear_cache((char *)start, (char *)(start + nbytes)); - } + + static AArch64ICacheInvalidationContext* current() { + return _current_context; } }; +#define PD_ICACHE_INVALIDATION_CONTEXT AArch64ICacheInvalidationContext + #endif // OS_CPU_LINUX_AARCH64_ICACHE_AARCH64_HPP diff --git a/src/hotspot/share/asm/codeBuffer.cpp b/src/hotspot/share/asm/codeBuffer.cpp index c9484881404d6..cdbc310e20771 100644 --- a/src/hotspot/share/asm/codeBuffer.cpp +++ b/src/hotspot/share/asm/codeBuffer.cpp @@ -365,10 +365,6 @@ void CodeSection::relocate(address at, RelocationHolder const& spec, int format) // If it has data, insert the prefix, as (data_prefix_tag | data1), data2. end->initialize(this, reloc); - if (rtype == relocInfo::oop_type && - !((oop_Relocation*)reloc)->oop_is_immediate()) { - _has_non_immediate_oops = true; - } } void CodeSection::initialize_locs(int locs_capacity) { @@ -727,6 +723,11 @@ csize_t CodeBuffer::copy_relocations_to(CodeBlob* dest) const { } void CodeBuffer::copy_code_to(CodeBlob* dest_blob) { + assert(ICacheInvalidationContext::current() != nullptr, + "ICache invalidation context should be set"); + assert(ICacheInvalidationContext::current()->mode() == ICacheInvalidation::DEFERRED || + ICacheInvalidationContext::current()->mode() == ICacheInvalidation::NOT_NEEDED, + "ICache invalidation should be deferred or unneeded"); #ifndef PRODUCT if (PrintNMethods && (WizardMode || Verbose)) { tty->print("done with CodeBuffer:"); @@ -933,7 +934,6 @@ void CodeBuffer::expand(CodeSection* which_cs, csize_t amount) { { ICacheInvalidationContext icic(ICacheInvalidation::NOT_NEEDED); - // Move all the code and relocations to the new blob: relocate_code_to(&cb); } diff --git a/src/hotspot/share/asm/codeBuffer.hpp b/src/hotspot/share/asm/codeBuffer.hpp index b6e8b76097b97..430d4949467d4 100644 --- a/src/hotspot/share/asm/codeBuffer.hpp +++ b/src/hotspot/share/asm/codeBuffer.hpp @@ -106,7 +106,6 @@ class CodeSection { int _skipped_instructions_size; int8_t _index; // my section number (SECT_INST, etc.) CodeBuffer* _outer; // enclosing CodeBuffer - bool _has_non_immediate_oops; // (Note: _locs_point used to be called _last_reloc_offset.) @@ -122,7 +121,6 @@ class CodeSection { _locs_own = false; _scratch_emit = false; _skipped_instructions_size = 0; - _has_non_immediate_oops = false; DEBUG_ONLY(_index = -1); DEBUG_ONLY(_outer = (CodeBuffer*)badAddress); } @@ -283,10 +281,6 @@ class CodeSection { // Return true if there was an expansion. bool maybe_expand_to_ensure_remaining(csize_t amount); - bool has_non_immediate_oops() const { - return _has_non_immediate_oops; - } - #ifndef PRODUCT void decode(); void print_on(outputStream* st, const char* name); @@ -858,12 +852,6 @@ class CodeBuffer: public StackObj DEBUG_ONLY(COMMA private Scrubber) { } } - bool has_non_immediate_oops() const { - return _consts.has_non_immediate_oops() - || _insts.has_non_immediate_oops() - || _stubs.has_non_immediate_oops(); - } - void block_comment(ptrdiff_t offset, const char* comment) PRODUCT_RETURN; const char* code_string(const char* str) PRODUCT_RETURN_(return nullptr;); diff --git a/src/hotspot/share/code/codeBlob.cpp b/src/hotspot/share/code/codeBlob.cpp index d5bb6fa1a1361..a02ee04597730 100644 --- a/src/hotspot/share/code/codeBlob.cpp +++ b/src/hotspot/share/code/codeBlob.cpp @@ -334,7 +334,6 @@ RuntimeBlob::RuntimeBlob( // Optimize ICache invalidation by batching it for the whole blob if // possible. ICacheInvalidationContext icic(code_begin(), code_size()); - cb->copy_code_and_locs_to(this); } diff --git a/src/hotspot/share/code/codeCache.cpp b/src/hotspot/share/code/codeCache.cpp index 56a7b8f8e092b..911fe5625af85 100644 --- a/src/hotspot/share/code/codeCache.cpp +++ b/src/hotspot/share/code/codeCache.cpp @@ -681,8 +681,7 @@ void CodeCache::nmethods_do(NMethodClosure* cl) { assert_locked_or_safepoint(CodeCache_lock); NMethodIterator iter(NMethodIterator::all); while(iter.next()) { - ICacheInvalidationContext icic(iter.method()->has_non_immediate_oops()); - + ICacheInvalidationContext icic; cl->do_nmethod(iter.method()); } } diff --git a/src/hotspot/share/code/nmethod.cpp b/src/hotspot/share/code/nmethod.cpp index 5c009c19e49de..b61c825ca6695 100644 --- a/src/hotspot/share/code/nmethod.cpp +++ b/src/hotspot/share/code/nmethod.cpp @@ -1231,7 +1231,6 @@ void nmethod::init_defaults(CodeBuffer *code_buffer, CodeOffsets* offsets) { _has_flushed_dependencies = 0; _is_unlinked = 0; _load_reported = 0; // jvmti state - _has_non_immediate_oops = 0; _deoptimization_status = not_marked; @@ -1325,13 +1324,11 @@ nmethod::nmethod( _speculations_offset = 0; #endif _immutable_data_ref_count_offset = 0; - _has_non_immediate_oops = code_buffer->has_non_immediate_oops(); { // Optimize ICache invalidation by batching it for the whole blob if // possible. ICacheInvalidationContext icic(code_begin(), code_size()); - code_buffer->copy_code_and_locs_to(this); code_buffer->copy_values_to(this); } @@ -1488,7 +1485,6 @@ nmethod::nmethod(const nmethod &nm) : CodeBlob(nm._name, nm._kind, nm._size, nm. _has_flushed_dependencies = nm._has_flushed_dependencies; _is_unlinked = nm._is_unlinked; _load_reported = nm._load_reported; - _has_non_immediate_oops = nm._has_non_immediate_oops; _deoptimization_status = nm._deoptimization_status; @@ -1702,8 +1698,6 @@ nmethod::nmethod( _num_stack_arg_slots = entry_bci != InvocationEntryBci ? 0 : _method->constMethod()->num_stack_arg_slots(); - _has_non_immediate_oops = code_buffer->has_non_immediate_oops(); - set_ctable_begin(header_begin() + content_offset()); #if INCLUDE_JVMCI @@ -1784,7 +1778,6 @@ nmethod::nmethod( // Optimize ICache invalidation by batching it for the whole blob if // possible. ICacheInvalidationContext icic(code_begin(), code_size()); - // Copy code and relocation info code_buffer->copy_code_and_locs_to(this); // Copy oops and metadata @@ -2060,19 +2053,25 @@ void nmethod::copy_values(GrowableArray* array) { void nmethod::fix_oop_relocations(address begin, address end, bool initialize_immediates) { // re-patch all oop-bearing instructions, just in case some oops moved RelocIterator iter(this, begin, end); + ICacheInvalidationContext icic; while (iter.next()) { + bool modified_code = false; if (iter.type() == relocInfo::oop_type) { oop_Relocation* reloc = iter.oop_reloc(); if (initialize_immediates && reloc->oop_is_immediate()) { oop* dest = reloc->oop_addr(); jobject obj = *reinterpret_cast(dest); initialize_immediate_oop(dest, obj); + icic.set_has_modified_code(); } // Refresh the oop-related bits of this instruction. - reloc->fix_oop_relocation(); + modified_code = reloc->fix_oop_relocation(); } else if (iter.type() == relocInfo::metadata_type) { metadata_Relocation* reloc = iter.metadata_reloc(); - reloc->fix_metadata_relocation(); + modified_code = reloc->fix_metadata_relocation(); + } + if (modified_code) { + icic.set_has_modified_code(); } } } diff --git a/src/hotspot/share/code/nmethod.hpp b/src/hotspot/share/code/nmethod.hpp index 4488dfebd24ca..0fa9d7fda9ebb 100644 --- a/src/hotspot/share/code/nmethod.hpp +++ b/src/hotspot/share/code/nmethod.hpp @@ -274,8 +274,7 @@ class nmethod : public CodeBlob { _has_scoped_access:1, // used by for shared scope closure (scopedMemoryAccess.cpp) _has_flushed_dependencies:1, // Used for maintenance of dependencies (under CodeCache_lock) _is_unlinked:1, // mark during class unloading - _load_reported:1, // used by jvmti to track if an event has been posted for this nmethod - _has_non_immediate_oops:1; + _load_reported:1; // used by jvmti to track if an event has been posted for this nmethod enum DeoptimizationStatus : u1 { not_marked, @@ -775,8 +774,6 @@ class nmethod : public CodeBlob { int comp_level() const { return _comp_level; } - bool has_non_immediate_oops() const { return _has_non_immediate_oops; } - // Support for oops in scopes and relocs: // Note: index 0 is reserved for null. oop oop_at(int index) const; diff --git a/src/hotspot/share/code/relocInfo.cpp b/src/hotspot/share/code/relocInfo.cpp index 2a6335e21182b..342ba23bd8445 100644 --- a/src/hotspot/share/code/relocInfo.cpp +++ b/src/hotspot/share/code/relocInfo.cpp @@ -591,11 +591,13 @@ oop oop_Relocation::oop_value() { } -void oop_Relocation::fix_oop_relocation() { +bool oop_Relocation::fix_oop_relocation() { if (!oop_is_immediate()) { // get the oop from the pool, and re-insert it into the instruction: set_value(value()); + return true; } + return false; } @@ -627,11 +629,13 @@ Metadata* metadata_Relocation::metadata_value() { } -void metadata_Relocation::fix_metadata_relocation() { +bool metadata_Relocation::fix_metadata_relocation() { if (!metadata_is_immediate()) { // get the metadata from the pool, and re-insert it into the instruction: pd_fix_value(value()); + return true; } + return false; } address virtual_call_Relocation::cached_value() { diff --git a/src/hotspot/share/code/relocInfo.hpp b/src/hotspot/share/code/relocInfo.hpp index 6f1778ef479c4..62d32a896bbe4 100644 --- a/src/hotspot/share/code/relocInfo.hpp +++ b/src/hotspot/share/code/relocInfo.hpp @@ -988,7 +988,7 @@ class oop_Relocation : public DataRelocation { void pack_data_to(CodeSection* dest) override; void unpack_data() override; - void fix_oop_relocation(); // reasserts oop value + bool fix_oop_relocation(); // reasserts oop value void verify_oop_relocation(); @@ -1040,7 +1040,7 @@ class metadata_Relocation : public DataRelocation { void pack_data_to(CodeSection* dest) override; void unpack_data() override; - void fix_metadata_relocation(); // reasserts metadata value + bool fix_metadata_relocation(); // reasserts metadata value address value() override { return (address) *metadata_addr(); } diff --git a/src/hotspot/share/gc/g1/g1NMethodClosure.cpp b/src/hotspot/share/gc/g1/g1NMethodClosure.cpp index a327fcd57743f..d74aa5eae1d01 100644 --- a/src/hotspot/share/gc/g1/g1NMethodClosure.cpp +++ b/src/hotspot/share/gc/g1/g1NMethodClosure.cpp @@ -32,7 +32,6 @@ #include "oops/access.inline.hpp" #include "oops/compressedOops.inline.hpp" #include "oops/oop.inline.hpp" -#include "runtime/icache.hpp" template void G1NMethodClosure::HeapRegionGatheringOopClosure::do_oop_work(T* p) { @@ -88,8 +87,6 @@ void G1NMethodClosure::do_evacuation_and_fixup(nmethod* nm) { bs_nm->disarm(nm); } - ICacheInvalidationContext icic(nm->has_non_immediate_oops()); - nm->fix_oop_relocations(); } diff --git a/src/hotspot/share/gc/shared/scavengableNMethods.cpp b/src/hotspot/share/gc/shared/scavengableNMethods.cpp index a0fe7521fefc6..887ac5f43a28a 100644 --- a/src/hotspot/share/gc/shared/scavengableNMethods.cpp +++ b/src/hotspot/share/gc/shared/scavengableNMethods.cpp @@ -26,7 +26,6 @@ #include "code/nmethod.hpp" #include "gc/shared/scavengableNMethods.hpp" #include "gc/shared/scavengableNMethodsData.hpp" -#include "runtime/icache.hpp" #include "runtime/mutexLocker.hpp" #include "utilities/debug.hpp" @@ -142,8 +141,6 @@ void ScavengableNMethods::nmethods_do_and_prune(NMethodToOopClosure* cl) { assert(data.on_list(), "else shouldn't be on this list"); if (cl != nullptr) { - ICacheInvalidationContext icic(cur->has_non_immediate_oops()); - cl->do_nmethod(cur); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp index 6931ddf347ac8..ec39e0c0ccb0a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp @@ -34,7 +34,6 @@ #include "memory/resourceArea.hpp" #include "memory/universe.hpp" #include "runtime/atomicAccess.hpp" -#include "runtime/icache.hpp" #include "utilities/powerOfTwo.hpp" diff --git a/src/hotspot/share/gc/shenandoah/shenandoahNMethod.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahNMethod.inline.hpp index d98708cf6f133..6758298675ba5 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahNMethod.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahNMethod.inline.hpp @@ -67,9 +67,6 @@ void ShenandoahNMethod::oops_do(OopClosure* oops, bool fix_relocations) { void ShenandoahNMethod::heal_nmethod_metadata(ShenandoahNMethod* nmethod_data) { ShenandoahEvacuateUpdateMetadataClosure cl; - ICacheInvalidationContext icic(nmethod_data->_has_non_immed_oops); - - assert(nmethod_data->_has_non_immed_oops == nmethod_data->_nm->has_non_immediate_oops(), "Inconsistent non-immed oops state"); nmethod_data->oops_do(&cl, true /*fix relocation*/); } diff --git a/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp b/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp index 858d97ceb30c7..50663eeec53e6 100644 --- a/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp +++ b/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp @@ -72,8 +72,7 @@ bool ZBarrierSetNMethod::nmethod_entry_barrier(nmethod* nm) { } { - ICacheInvalidationContext icic(ZNMethod::needs_icache_invalidation(nm)); - + ICacheInvalidationContext icic; // Heal barriers ZNMethod::nmethod_patch_barriers(nm); diff --git a/src/hotspot/share/gc/z/zGeneration.cpp b/src/hotspot/share/gc/z/zGeneration.cpp index 51764109b6e5b..3085d4e35344f 100644 --- a/src/hotspot/share/gc/z/zGeneration.cpp +++ b/src/hotspot/share/gc/z/zGeneration.cpp @@ -1437,8 +1437,7 @@ class ZRemapNMethodClosure : public NMethodClosure { ZLocker locker(ZNMethod::lock_for_nmethod(nm)); if (_bs_nm->is_armed(nm)) { { - ICacheInvalidationContext icic(ZNMethod::needs_icache_invalidation(nm)); - + ICacheInvalidationContext icic; // Heal barriers ZNMethod::nmethod_patch_barriers(nm); diff --git a/src/hotspot/share/gc/z/zMark.cpp b/src/hotspot/share/gc/z/zMark.cpp index 68ced4212344a..ca755c47b8229 100644 --- a/src/hotspot/share/gc/z/zMark.cpp +++ b/src/hotspot/share/gc/z/zMark.cpp @@ -720,8 +720,7 @@ class ZMarkNMethodClosure : public NMethodClosure { ZLocker locker(ZNMethod::lock_for_nmethod(nm)); if (_bs_nm->is_armed(nm)) { { - ICacheInvalidationContext icic(ZNMethod::needs_icache_invalidation(nm)); - + ICacheInvalidationContext icic; // Heal barriers ZNMethod::nmethod_patch_barriers(nm); @@ -769,13 +768,7 @@ class ZMarkYoungNMethodClosure : public NMethodClosure { const bool complete_disarm = ZPointer::is_store_good(new_disarm_value_ptr); { - bool needs_icache_invalidation = ZNMethod::needs_non_immediate_oops_patching(nm); - if (complete_disarm && ZNMethod::needs_barrier_patching(nm)) { - needs_icache_invalidation = true; - } - - ICacheInvalidationContext icic(needs_icache_invalidation); - + ICacheInvalidationContext icic; if (complete_disarm) { // We are about to completely disarm the nmethod, must take responsibility to patch all barriers before disarming ZNMethod::nmethod_patch_barriers(nm); diff --git a/src/hotspot/share/gc/z/zNMethod.cpp b/src/hotspot/share/gc/z/zNMethod.cpp index 470b54da9f47c..4db5ccf97658b 100644 --- a/src/hotspot/share/gc/z/zNMethod.cpp +++ b/src/hotspot/share/gc/z/zNMethod.cpp @@ -202,12 +202,8 @@ void ZNMethod::register_nmethod(nmethod* nm) { log_register(nm); - { - ICacheInvalidationContext icic(needs_barrier_patching(nm)); - - // Patch nmethod barriers - nmethod_patch_barriers(nm); - } + // Patch nmethod barriers + nmethod_patch_barriers(nm); // Register nmethod ZNMethodTable::register_nmethod(nm); @@ -252,6 +248,10 @@ void ZNMethod::set_guard_value(nmethod* nm, int value) { void ZNMethod::nmethod_patch_barriers(nmethod* nm) { ZBarrierSetAssembler* const bs_asm = ZBarrierSet::assembler(); ZArrayIterator iter(gc_data(nm)->barriers()); + ICacheInvalidationContext icic; + if (gc_data(nm)->barriers()->is_nonempty()) { + icic.set_has_modified_code(); + } for (ZNMethodDataBarrier barrier; iter.next(&barrier);) { bs_asm->patch_barrier_relocation(barrier._reloc_addr, barrier._reloc_format); } @@ -371,13 +371,9 @@ class ZNMethodUnlinkClosure : public NMethodClosure { const uintptr_t prev_color = ZNMethod::color(nm); assert(prev_color != ZPointerStoreGoodMask, "Potentially non-monotonic transition"); - { - ICacheInvalidationContext icic(ZNMethod::needs_non_immediate_oops_patching(nm)); - - // Heal oops and potentially mark young objects if there is a concurrent young collection. - ZUncoloredRootProcessOopClosure cl(prev_color); - ZNMethod::nmethod_oops_do_inner(nm, &cl); - } + // Heal oops and potentially mark young objects if there is a concurrent young collection. + ZUncoloredRootProcessOopClosure cl(prev_color); + ZNMethod::nmethod_oops_do_inner(nm, &cl); // Disarm for marking and relocation, but leave the remset bits so this isn't store good. // This makes sure the mutator still takes a slow path to fill in the nmethod epoch for @@ -428,16 +424,3 @@ void ZNMethod::unlink(ZWorkers* workers, bool unloading_occurred) { void ZNMethod::purge() { ClassUnloadingContext::context()->purge_and_free_nmethods(); } - -bool ZNMethod::needs_icache_invalidation(nmethod* nm) { - return needs_barrier_patching(nm) || needs_non_immediate_oops_patching(nm); -} - -bool ZNMethod::needs_barrier_patching(nmethod* nm) { - return gc_data(nm)->barriers()->is_nonempty(); -} - -bool ZNMethod::needs_non_immediate_oops_patching(nmethod* nm) { - assert(gc_data(nm)->has_non_immediate_oops() == nm->has_non_immediate_oops(), "Inconsistent non-immed oops state"); - return gc_data(nm)->has_non_immediate_oops(); -} diff --git a/src/hotspot/share/gc/z/zNMethod.hpp b/src/hotspot/share/gc/z/zNMethod.hpp index 6048e94b3734f..5d797f3fd5542 100644 --- a/src/hotspot/share/gc/z/zNMethod.hpp +++ b/src/hotspot/share/gc/z/zNMethod.hpp @@ -74,10 +74,6 @@ class ZNMethod : public AllStatic { static oop oop_load_no_keepalive(const nmethod* nm, int index); static oop oop_load_phantom(const nmethod* nm, int index); - - static bool needs_icache_invalidation(nmethod* nm); - static bool needs_barrier_patching(nmethod* nm); - static bool needs_non_immediate_oops_patching(nmethod* nm); }; #endif // SHARE_GC_Z_ZNMETHOD_HPP diff --git a/src/hotspot/share/gc/z/zUnload.cpp b/src/hotspot/share/gc/z/zUnload.cpp index 3d69bcb875516..5c50b3077dd65 100644 --- a/src/hotspot/share/gc/z/zUnload.cpp +++ b/src/hotspot/share/gc/z/zUnload.cpp @@ -38,7 +38,6 @@ #include "gc/z/zUnload.hpp" #include "memory/metaspaceUtils.hpp" #include "oops/access.inline.hpp" -#include "runtime/icache.hpp" static const ZStatSubPhase ZSubPhaseConcurrentClassesUnlink("Concurrent Classes Unlink", ZGenerationId::old); static const ZStatSubPhase ZSubPhaseConcurrentClassesPurge("Concurrent Classes Purge", ZGenerationId::old); @@ -81,8 +80,6 @@ class ZIsUnloadingBehaviour : public IsUnloadingBehaviour { // Disarmed nmethods are alive return false; } - ICacheInvalidationContext icic(ZNMethod::needs_non_immediate_oops_patching(nm)); - ZIsUnloadingOopClosure cl(nm); ZNMethod::nmethod_oops_do_inner(nm, &cl); return cl.is_unloading(); diff --git a/src/hotspot/share/runtime/globals.hpp b/src/hotspot/share/runtime/globals.hpp index 68b5d8254fd33..721b0bc30384e 100644 --- a/src/hotspot/share/runtime/globals.hpp +++ b/src/hotspot/share/runtime/globals.hpp @@ -2003,6 +2003,8 @@ const int ObjectAlignmentInBytes = 8; develop(uint, BinarySearchThreshold, 16, \ "Minimal number of elements in a sorted collection to prefer" \ "binary search over simple linear search." ) \ + product(bool, UseDeferredICacheInvalidation, false, DIAGNOSTIC, \ + "Defer multiple ICache invalidation to single invalidation") \ \ // end of RUNTIME_FLAGS diff --git a/src/hotspot/share/runtime/icache.cpp b/src/hotspot/share/runtime/icache.cpp index 34d5a7e055b47..4a4162b41ce07 100644 --- a/src/hotspot/share/runtime/icache.cpp +++ b/src/hotspot/share/runtime/icache.cpp @@ -31,6 +31,8 @@ // The flush stub function address AbstractICache::flush_icache_stub_t AbstractICache::_flush_icache_stub = nullptr; +NOT_PRODUCT(THREAD_LOCAL DefaultICacheInvalidationContext* DefaultICacheInvalidationContext::_current_context = nullptr;) + void AbstractICache::initialize(int phase) { // Making this stub must be FIRST use of assembler ResourceMark rm; diff --git a/src/hotspot/share/runtime/icache.hpp b/src/hotspot/share/runtime/icache.hpp index 21dfd65588378..88d9cce71e444 100644 --- a/src/hotspot/share/runtime/icache.hpp +++ b/src/hotspot/share/runtime/icache.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -77,49 +77,6 @@ enum class ICacheInvalidation : uint8_t { DEFERRED = 2 }; -class ICacheInvalidationContext : StackObj { - private: - NONCOPYABLE(ICacheInvalidationContext); - - address _code; - int _size; - ICacheInvalidation _mode; - - void pd_init(); - void pd_invalidate_icache(); - - NOT_PRODUCT(static ICacheInvalidationContext* pd_current()); - - public: - // Hardware optimized icache invalidation which does not depend on what code is invalidated. - ICacheInvalidationContext(ICacheInvalidation mode) : _code(nullptr), _size(0), _mode(mode) { - pd_init(); - } - - ICacheInvalidationContext(bool needs_invalidation) - : ICacheInvalidationContext(needs_invalidation - ? ICacheInvalidation::DEFERRED - : ICacheInvalidation::NOT_NEEDED) {} - - // Use hardware optimized icache invalidation if possible, - // Otherwise invalidate the specified code range. - ICacheInvalidationContext(address code, int size) : _code(code), _size(size), _mode(ICacheInvalidation::DEFERRED) { - pd_init(); - } - - ~ICacheInvalidationContext(); - -#ifdef ASSERT - static ICacheInvalidationContext* current() { - return pd_current(); - } -#endif - - ICacheInvalidation mode() const { - return _mode; - } -}; - // Must be included before the definition of ICacheStubGenerator // because ICacheStubGenerator uses ICache definitions. @@ -177,27 +134,62 @@ class ICacheStubGenerator : public StubCodeGenerator { void generate_icache_flush(ICache::flush_icache_stub_t* flush_icache_stub); }; -#ifndef PD_ICACHE_INVALIDATION_CONTEXT - // Default implementation: do nothing - inline void ICacheInvalidationContext::pd_init() {} - inline void ICacheInvalidationContext::pd_invalidate_icache() {} -#ifdef ASSERT - inline ICacheInvalidationContext* ICacheInvalidationContext::pd_current() { - Unimplemented(); - return nullptr; +class DefaultICacheInvalidationContext final : StackObj { + private: + NONCOPYABLE(DefaultICacheInvalidationContext); + + NOT_PRODUCT(static THREAD_LOCAL DefaultICacheInvalidationContext* _current_context;) + + address _code; + int _size; + ICacheInvalidation _mode; + + public: + DefaultICacheInvalidationContext(ICacheInvalidation mode) + : _code(nullptr), _size(0), _mode(mode) { + NOT_PRODUCT(_current_context = this); + } + + DefaultICacheInvalidationContext() : DefaultICacheInvalidationContext(ICacheInvalidation::IMMEDIATE) {} + + DefaultICacheInvalidationContext(address code, int size) + : _code(code), _size(size), _mode(ICacheInvalidation::DEFERRED) { + NOT_PRODUCT(_current_context = this); + assert(code != nullptr, "code must not be null for deferred invalidation"); + assert(size > 0, "size must be positive for deferred invalidation"); } -#endif // ASSERT -#endif // PD_ICACHE_INVALIDATION_CONTEXT -inline ICacheInvalidationContext::~ICacheInvalidationContext() { - pd_invalidate_icache(); - if (_code != nullptr) { - assert(_mode == ICacheInvalidation::DEFERRED, "sanity"); - assert(_size > 0, "size must be positive"); - ICache::invalidate_range(_code, _size); - _code = nullptr; - _size = 0; + ~DefaultICacheInvalidationContext() { + NOT_PRODUCT(_current_context = nullptr); + if (_code != nullptr) { + assert(_mode == ICacheInvalidation::DEFERRED, "sanity"); + assert(_size > 0, "size must be positive for deferred invalidation"); + ICache::invalidate_range(_code, _size); + _code = nullptr; + _size = 0; + _mode = ICacheInvalidation::NOT_NEEDED; + } } -} + + ICacheInvalidation mode() const { + return _mode; + } + + void set_has_modified_code() { + // No-op for the default implementation. + } + +#ifdef ASSERT + static DefaultICacheInvalidationContext* current() { + return _current_context; + } +#endif +}; + +#ifdef PD_ICACHE_INVALIDATION_CONTEXT +using ICacheInvalidationContext = PD_ICACHE_INVALIDATION_CONTEXT; +#else +using ICacheInvalidationContext = DefaultICacheInvalidationContext; +#endif // PD_ICACHE_INVALIDATION_CONTEXT #endif // SHARE_RUNTIME_ICACHE_HPP diff --git a/test/hotspot/jtreg/gc/TestDeferredICacheInvalidation.java b/test/hotspot/jtreg/gc/TestDeferredICacheInvalidation.java index eb6fc8b4a9c46..c104d367ac2bc 100644 --- a/test/hotspot/jtreg/gc/TestDeferredICacheInvalidation.java +++ b/test/hotspot/jtreg/gc/TestDeferredICacheInvalidation.java @@ -25,13 +25,11 @@ package gc; /* - * @test id=ParallelGC + * @test id=parallel * @bug 8370947 * @summary Check no assertion is triggered when UseDeferredICacheInvalidation is enabled for ParallelGC * @library /test/lib * @requires vm.debug - * @requires os.family=="linux" - * @requires os.arch=="aarch64" * @requires vm.gc.Parallel * @build jdk.test.whitebox.WhiteBox * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox @@ -42,13 +40,11 @@ */ /* - * @test id=G1GC + * @test id=g1 * @bug 8370947 * @summary Check no assertion is triggered when UseDeferredICacheInvalidation is enabled for G1GC * @library /test/lib * @requires vm.debug - * @requires os.family=="linux" - * @requires os.arch=="aarch64" * @requires vm.gc.G1 * @build jdk.test.whitebox.WhiteBox * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox @@ -59,13 +55,11 @@ */ /* - * @test id=ShenandoahGC + * @test id=shenandoah * @bug 8370947 * @summary Check no assertion is triggered when UseDeferredICacheInvalidation is enabled for ShenandoahGC * @library /test/lib * @requires vm.debug - * @requires os.family=="linux" - * @requires os.arch=="aarch64" * @requires vm.gc.Shenandoah * @build jdk.test.whitebox.WhiteBox * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox @@ -74,13 +68,11 @@ */ /* - * @test id=GenShenGC + * @test id=genshen * @bug 8370947 * @summary Check no assertion is triggered when UseDeferredICacheInvalidation is enabled for generational ShenandoahGC * @library /test/lib * @requires vm.debug - * @requires os.family=="linux" - * @requires os.arch=="aarch64" * @requires vm.gc.Shenandoah * @build jdk.test.whitebox.WhiteBox * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox @@ -91,13 +83,11 @@ */ /* - * @test id=ZGC + * @test id=z * @bug 8370947 * @summary Check no assertion is triggered when UseDeferredICacheInvalidation is enabled for ZGC * @library /test/lib * @requires vm.debug - * @requires os.family=="linux" - * @requires os.arch=="aarch64" * @requires vm.gc.Z * @build jdk.test.whitebox.WhiteBox * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox