diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp index 07a2d6fbfa0be..5cc08f3f44258 100644 --- a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp @@ -879,6 +879,18 @@ void ZBarrierSetAssembler::patch_barrier_relocation(address addr, int format) { ShouldNotReachHere(); } + if (UseDeferredICacheInvalidation) { + // Defer the ICache invalidation to a later point where multiple patches can be handled together. + // + // Note: We rely on the fact that this function is only called from places where deferred invalidation + // is safe. This assumption helps to avoid overhead of accessing thread-local data here. + assert(ICacheInvalidationContext::current() != nullptr, "ICache invalidation context should be set"); + assert(ICacheInvalidationContext::current()->mode() == ICacheInvalidation::DEFERRED || + ICacheInvalidationContext::current()->mode() == ICacheInvalidation::NOT_NEEDED, + "ICache invalidation should be deferred or unneeded"); + return; + } + ICache::invalidate_word((address)patch_addr); } diff --git a/src/hotspot/cpu/aarch64/globals_aarch64.hpp b/src/hotspot/cpu/aarch64/globals_aarch64.hpp index 8e520314c8b6f..430316360f6fe 100644 --- a/src/hotspot/cpu/aarch64/globals_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/globals_aarch64.hpp @@ -127,6 +127,8 @@ define_pd_global(intx, InlineSmallCode, 1000); "Branch Protection to use: none, standard, pac-ret") \ product(bool, AlwaysMergeDMB, true, DIAGNOSTIC, \ "Always merge DMB instructions in code emission") \ + product(bool, NeoverseN1Errata1542419, false, DIAGNOSTIC, \ + "Enable workaround for Neoverse N1 erratum 1542419") \ // end of ARCH_FLAGS diff --git a/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp b/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp index 94694b58d2fae..7bd5625e3430f 100644 --- a/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp @@ -26,6 +26,7 @@ #include "asm/macroAssembler.hpp" #include "code/nmethod.hpp" #include "code/relocInfo.hpp" +#include "gc/shared/barrierSetNMethod.hpp" #include "nativeInst_aarch64.hpp" #include "oops/oop.inline.hpp" #include "runtime/safepoint.hpp" @@ -54,6 +55,33 @@ void Relocation::pd_set_data_value(address x, bool verify_only) { bytes = MacroAssembler::pd_patch_instruction_size(addr(), x); break; } + + if (UseDeferredICacheInvalidation) { + // Defer the ICache invalidation to a later point where multiple patches can be handled together. + // + // Note: We rely on the fact that this function is only called from places where deferred invalidation + // is safe. This assumption helps to avoid overhead of accessing thread-local data here. + assert(ICacheInvalidationContext::current() != nullptr, "ICache invalidation context should be set"); + assert(ICacheInvalidationContext::current()->mode() == ICacheInvalidation::DEFERRED || + ICacheInvalidationContext::current()->mode() == ICacheInvalidation::NOT_NEEDED, + "ICache invalidation should be deferred or unneeded."); +#ifdef ASSERT + if (_binding != nullptr && _binding->code() != nullptr) { + nmethod *nm = _binding->code(); + if (!(BarrierSet::barrier_set()->barrier_set_nmethod()->is_armed(nm) || + SafepointSynchronize::is_at_safepoint() || + nm->is_unloading())) { + ConditionalMutexLocker ml(NMethodState_lock, !NMethodState_lock->owned_by_self(), Mutex::_no_safepoint_check_flag); + assert(nm->is_not_installed(), + "ICache invalidation context should only be used for armed " + "or unloading or not installed nmethod or at safepoint"); + + } + } +#endif + return; + } + ICache::invalidate_range(addr(), bytes); } diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp index 659c231464a43..6de8de3688b86 100644 --- a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp @@ -63,6 +63,19 @@ static SpinWait get_spin_wait_desc() { return spin_wait; } +static bool has_neoverse_n1_errata_1542419() { + const int major_rev_num = VM_Version::cpu_variant(); + const int minor_rev_num = VM_Version::cpu_revision(); + // Neoverse N1: 0xd0c + // Erratum 1542419 affects r3p0, r3p1 and r4p0. + // It is fixed in r4p1 and later revisions, which are not affected. + return (VM_Version::cpu_family() == VM_Version::CPU_ARM && + VM_Version::model_is(0xd0c) && + ((major_rev_num == 3 && minor_rev_num == 0) || + (major_rev_num == 3 && minor_rev_num == 1) || + (major_rev_num == 4 && minor_rev_num == 0))); +} + void VM_Version::initialize() { #define SET_CPU_FEATURE_NAME(id, name, bit) \ _features_names[bit] = XSTR(name); @@ -641,6 +654,19 @@ void VM_Version::initialize() { clear_feature(CPU_SVE); } + if (FLAG_IS_DEFAULT(NeoverseN1Errata1542419) && has_neoverse_n1_errata_1542419()) { + FLAG_SET_DEFAULT(NeoverseN1Errata1542419, true); + } + + if (NeoverseN1Errata1542419) { + if (!has_neoverse_n1_errata_1542419()) { + warning("NeoverseN1Errata1542419 is set for the CPU not having Neoverse N1 errata 1542419"); + } + if (FLAG_IS_DEFAULT(UseDeferredICacheInvalidation)) { + FLAG_SET_DEFAULT(UseDeferredICacheInvalidation, true); + } + } + // Construct the "features" string stringStream ss(512); ss.print("0x%02x:0x%x:0x%03x:%d", _cpu, _variant, _model, _revision); diff --git a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp new file mode 100644 index 0000000000000..4ec720293b552 --- /dev/null +++ b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp @@ -0,0 +1,28 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "runtime/icache.hpp" +#include "utilities/globalDefinitions.hpp" + +THREAD_LOCAL AArch64ICacheInvalidationContext* AArch64ICacheInvalidationContext::_current_context = nullptr; diff --git a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp index 957be26670229..33892eb052d9a 100644 --- a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp +++ b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2014, Red Hat Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -26,6 +26,35 @@ #ifndef OS_CPU_LINUX_AARCH64_ICACHE_AARCH64_HPP #define OS_CPU_LINUX_AARCH64_ICACHE_AARCH64_HPP +#include "memory/allocation.hpp" +#include "utilities/globalDefinitions.hpp" + +inline void assert_hardware_cache_coherency() { + // For deferred icache invalidation, we expect hardware dcache + // and icache to be coherent: CTR_EL0.IDC == 1 and CTR_EL0.DIC == 1 + // An exception is Neoverse N1 with erratum 1542419, which requires + // a use of 'IC IVAU' instruction. In such a case, we expect + // CTR_EL0.DIC == 0. +#ifdef ASSERT + static unsigned int cache_info = 0; + if (cache_info == 0) { + asm volatile("mrs\t%0, ctr_el0" : "=r"(cache_info)); + } + constexpr unsigned int CTR_IDC_SHIFT = 28; + constexpr unsigned int CTR_DIC_SHIFT = 29; + assert(((cache_info >> CTR_IDC_SHIFT) & 0x1) != 0x0, + "Expect CTR_EL0.IDC to be enabled"); + if (NeoverseN1Errata1542419) { + assert(((cache_info >> CTR_DIC_SHIFT) & 0x1) == 0x0, + "Expect CTR_EL0.DIC to be disabled for Neoverse N1 with erratum " + "1542419"); + } else { + assert(((cache_info >> CTR_DIC_SHIFT) & 0x1) != 0x0, + "Expect CTR_EL0.DIC to be enabled"); + } +#endif +} + // Interface for updating the instruction cache. Whenever the VM // modifies code, part of the processor instruction cache potentially // has to be flushed. @@ -37,8 +66,144 @@ class ICache : public AbstractICache { __builtin___clear_cache((char *)addr, (char *)(addr + 4)); } static void invalidate_range(address start, int nbytes) { - __builtin___clear_cache((char *)start, (char *)(start + nbytes)); + if (NeoverseN1Errata1542419) { + assert_hardware_cache_coherency(); + asm volatile("dsb ish \n" + "ic ivau, xzr \n" + "dsb ish \n" + "isb \n" + : : : "memory"); + } else { + __builtin___clear_cache((char *)start, (char *)(start + nbytes)); + } + } +}; + +class AArch64ICacheInvalidationContext final : StackObj { + private: + NONCOPYABLE(AArch64ICacheInvalidationContext); + + static THREAD_LOCAL AArch64ICacheInvalidationContext* _current_context; + + AArch64ICacheInvalidationContext* _parent; + address _code; + int _size; + ICacheInvalidation _mode; + bool _has_modified_code; + + public: + AArch64ICacheInvalidationContext(ICacheInvalidation mode) + : _parent(nullptr), _code(nullptr), _size(0), _mode(mode), _has_modified_code(false) { + _parent = _current_context; + _current_context = this; + if (_parent != nullptr) { + // The parent context is in charge of icache invalidation. + _mode = (_parent->mode() == ICacheInvalidation::IMMEDIATE) ? ICacheInvalidation::IMMEDIATE : ICacheInvalidation::NOT_NEEDED; + } + } + + AArch64ICacheInvalidationContext() + : AArch64ICacheInvalidationContext(UseDeferredICacheInvalidation + ? ICacheInvalidation::DEFERRED + : ICacheInvalidation::IMMEDIATE) {} + + AArch64ICacheInvalidationContext(address code, int size) + : _parent(nullptr), + _code(code), + _size(size), + _mode(ICacheInvalidation::DEFERRED), + _has_modified_code(true) { + assert(_current_context == nullptr, + "nested ICacheInvalidationContext(code, size) not supported"); + assert(code != nullptr, "code must not be null for deferred invalidation"); + assert(size > 0, "size must be positive for deferred invalidation"); + + _current_context = this; + + if (UseDeferredICacheInvalidation) { + // With hardware dcache and icache coherency, we don't need _code. + _code = nullptr; + _size = 0; + } + } + + ~AArch64ICacheInvalidationContext() { + _current_context = _parent; + + if (_code != nullptr) { + assert(_size > 0, "size must be positive for deferred invalidation"); + assert(_mode == ICacheInvalidation::DEFERRED, "sanity"); + assert(_has_modified_code, "sanity"); + assert(_parent == nullptr, "sanity"); + + ICache::invalidate_range(_code, _size); + return; + } + + if (!_has_modified_code) { + return; + } + + if (_parent != nullptr) { + _parent->set_has_modified_code(); + } + + if (_mode != ICacheInvalidation::DEFERRED) { + return; + } + + assert_hardware_cache_coherency(); + + asm volatile("dsb ish" : : : "memory"); + + if (NeoverseN1Errata1542419) { + // Errata 1542419: Neoverse N1 cores with the 'COHERENT_ICACHE' feature + // may fetch stale instructions when software depends on + // prefetch-speculation-protection instead of explicit synchronization. + // + // Neoverse-N1 implementation mitigates the errata 1542419 with a + // workaround: + // - Disable coherent icache. + // - Trap IC IVAU instructions. + // - Execute: + // - tlbi vae3is, xzr + // - dsb sy + // - Ignore trapped IC IVAU instructions. + // + // `tlbi vae3is, xzr` invalidates all translation entries (all VAs, all + // possible levels). It waits for all memory accesses using in-scope old + // translation information to complete before it is considered complete. + // + // As this workaround has significant overhead, Arm Neoverse N1 (MP050) + // Software Developer Errata Notice version 29.0 suggests: + // + // "Since one TLB inner-shareable invalidation is enough to avoid this + // erratum, the number of injected TLB invalidations should be minimized + // in the trap handler to mitigate the performance impact due to this + // workaround." + // As the address for icache invalidation is not relevant and + // IC IVAU instruction is ignored, we use XZR in it. + asm volatile( + "ic ivau, xzr \n" + "dsb ish \n" + : + : + : "memory"); + } + asm volatile("isb" : : : "memory"); + } + + ICacheInvalidation mode() const { return _mode; } + + void set_has_modified_code() { + _has_modified_code = true; + } + + static AArch64ICacheInvalidationContext* current() { + return _current_context; } }; +#define PD_ICACHE_INVALIDATION_CONTEXT AArch64ICacheInvalidationContext + #endif // OS_CPU_LINUX_AARCH64_ICACHE_AARCH64_HPP diff --git a/src/hotspot/share/asm/codeBuffer.cpp b/src/hotspot/share/asm/codeBuffer.cpp index 2c6b7d7e96ec0..cdbc310e20771 100644 --- a/src/hotspot/share/asm/codeBuffer.cpp +++ b/src/hotspot/share/asm/codeBuffer.cpp @@ -723,6 +723,11 @@ csize_t CodeBuffer::copy_relocations_to(CodeBlob* dest) const { } void CodeBuffer::copy_code_to(CodeBlob* dest_blob) { + assert(ICacheInvalidationContext::current() != nullptr, + "ICache invalidation context should be set"); + assert(ICacheInvalidationContext::current()->mode() == ICacheInvalidation::DEFERRED || + ICacheInvalidationContext::current()->mode() == ICacheInvalidation::NOT_NEEDED, + "ICache invalidation should be deferred or unneeded"); #ifndef PRODUCT if (PrintNMethods && (WizardMode || Verbose)) { tty->print("done with CodeBuffer:"); @@ -745,9 +750,6 @@ void CodeBuffer::copy_code_to(CodeBlob* dest_blob) { // Done moving code bytes; were they the right size? assert((int)align_up(dest.total_content_size(), oopSize) == dest_blob->content_size(), "sanity"); - - // Flush generated code - ICache::invalidate_range(dest_blob->code_begin(), dest_blob->code_size()); } // Move all my code into another code buffer. Consult applicable @@ -930,8 +932,11 @@ void CodeBuffer::expand(CodeSection* which_cs, csize_t amount) { // Needs to be initialized when calling fix_relocation_after_move. cb.blob()->set_ctable_begin(cb.consts()->start()); - // Move all the code and relocations to the new blob: - relocate_code_to(&cb); + { + ICacheInvalidationContext icic(ICacheInvalidation::NOT_NEEDED); + // Move all the code and relocations to the new blob: + relocate_code_to(&cb); + } // some internal addresses, _last_insn _last_label, are used during code emission, // adjust them in expansion diff --git a/src/hotspot/share/code/codeBlob.cpp b/src/hotspot/share/code/codeBlob.cpp index a0a34ec23fad6..a02ee04597730 100644 --- a/src/hotspot/share/code/codeBlob.cpp +++ b/src/hotspot/share/code/codeBlob.cpp @@ -331,6 +331,9 @@ RuntimeBlob::RuntimeBlob( : CodeBlob(name, kind, cb, size, header_size, frame_complete, frame_size, oop_maps, caller_must_gc_arguments, align_up(cb->total_relocation_size(), oopSize)) { + // Optimize ICache invalidation by batching it for the whole blob if + // possible. + ICacheInvalidationContext icic(code_begin(), code_size()); cb->copy_code_and_locs_to(this); } diff --git a/src/hotspot/share/code/codeCache.cpp b/src/hotspot/share/code/codeCache.cpp index beca3bcbcf5d8..911fe5625af85 100644 --- a/src/hotspot/share/code/codeCache.cpp +++ b/src/hotspot/share/code/codeCache.cpp @@ -681,6 +681,7 @@ void CodeCache::nmethods_do(NMethodClosure* cl) { assert_locked_or_safepoint(CodeCache_lock); NMethodIterator iter(NMethodIterator::all); while(iter.next()) { + ICacheInvalidationContext icic; cl->do_nmethod(iter.method()); } } diff --git a/src/hotspot/share/code/nmethod.cpp b/src/hotspot/share/code/nmethod.cpp index c2f8b46f00e95..b61c825ca6695 100644 --- a/src/hotspot/share/code/nmethod.cpp +++ b/src/hotspot/share/code/nmethod.cpp @@ -1325,8 +1325,13 @@ nmethod::nmethod( #endif _immutable_data_ref_count_offset = 0; - code_buffer->copy_code_and_locs_to(this); - code_buffer->copy_values_to(this); + { + // Optimize ICache invalidation by batching it for the whole blob if + // possible. + ICacheInvalidationContext icic(code_begin(), code_size()); + code_buffer->copy_code_and_locs_to(this); + code_buffer->copy_values_to(this); + } post_init(); } @@ -1769,10 +1774,16 @@ nmethod::nmethod( assert(immutable_data_end_offset <= immutable_data_size, "wrong read-only data size: %d > %d", immutable_data_end_offset, immutable_data_size); - // Copy code and relocation info - code_buffer->copy_code_and_locs_to(this); - // Copy oops and metadata - code_buffer->copy_values_to(this); + { + // Optimize ICache invalidation by batching it for the whole blob if + // possible. + ICacheInvalidationContext icic(code_begin(), code_size()); + // Copy code and relocation info + code_buffer->copy_code_and_locs_to(this); + // Copy oops and metadata + code_buffer->copy_values_to(this); + } + dependencies->copy_to(this); // Copy PcDesc and ScopeDesc data debug_info->copy_to(this); @@ -2042,19 +2053,25 @@ void nmethod::copy_values(GrowableArray* array) { void nmethod::fix_oop_relocations(address begin, address end, bool initialize_immediates) { // re-patch all oop-bearing instructions, just in case some oops moved RelocIterator iter(this, begin, end); + ICacheInvalidationContext icic; while (iter.next()) { + bool modified_code = false; if (iter.type() == relocInfo::oop_type) { oop_Relocation* reloc = iter.oop_reloc(); if (initialize_immediates && reloc->oop_is_immediate()) { oop* dest = reloc->oop_addr(); jobject obj = *reinterpret_cast(dest); initialize_immediate_oop(dest, obj); + icic.set_has_modified_code(); } // Refresh the oop-related bits of this instruction. - reloc->fix_oop_relocation(); + modified_code = reloc->fix_oop_relocation(); } else if (iter.type() == relocInfo::metadata_type) { metadata_Relocation* reloc = iter.metadata_reloc(); - reloc->fix_metadata_relocation(); + modified_code = reloc->fix_metadata_relocation(); + } + if (modified_code) { + icic.set_has_modified_code(); } } } diff --git a/src/hotspot/share/code/relocInfo.cpp b/src/hotspot/share/code/relocInfo.cpp index 2a6335e21182b..342ba23bd8445 100644 --- a/src/hotspot/share/code/relocInfo.cpp +++ b/src/hotspot/share/code/relocInfo.cpp @@ -591,11 +591,13 @@ oop oop_Relocation::oop_value() { } -void oop_Relocation::fix_oop_relocation() { +bool oop_Relocation::fix_oop_relocation() { if (!oop_is_immediate()) { // get the oop from the pool, and re-insert it into the instruction: set_value(value()); + return true; } + return false; } @@ -627,11 +629,13 @@ Metadata* metadata_Relocation::metadata_value() { } -void metadata_Relocation::fix_metadata_relocation() { +bool metadata_Relocation::fix_metadata_relocation() { if (!metadata_is_immediate()) { // get the metadata from the pool, and re-insert it into the instruction: pd_fix_value(value()); + return true; } + return false; } address virtual_call_Relocation::cached_value() { diff --git a/src/hotspot/share/code/relocInfo.hpp b/src/hotspot/share/code/relocInfo.hpp index 6f1778ef479c4..62d32a896bbe4 100644 --- a/src/hotspot/share/code/relocInfo.hpp +++ b/src/hotspot/share/code/relocInfo.hpp @@ -988,7 +988,7 @@ class oop_Relocation : public DataRelocation { void pack_data_to(CodeSection* dest) override; void unpack_data() override; - void fix_oop_relocation(); // reasserts oop value + bool fix_oop_relocation(); // reasserts oop value void verify_oop_relocation(); @@ -1040,7 +1040,7 @@ class metadata_Relocation : public DataRelocation { void pack_data_to(CodeSection* dest) override; void unpack_data() override; - void fix_metadata_relocation(); // reasserts metadata value + bool fix_metadata_relocation(); // reasserts metadata value address value() override { return (address) *metadata_addr(); } diff --git a/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp b/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp index d80ce4e149d09..50663eeec53e6 100644 --- a/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp +++ b/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp @@ -33,6 +33,7 @@ #include "gc/z/zThreadLocalData.hpp" #include "gc/z/zUncoloredRoot.inline.hpp" #include "logging/log.hpp" +#include "runtime/icache.hpp" #include "runtime/threadWXSetters.inline.hpp" bool ZBarrierSetNMethod::nmethod_entry_barrier(nmethod* nm) { @@ -70,12 +71,15 @@ bool ZBarrierSetNMethod::nmethod_entry_barrier(nmethod* nm) { return false; } - // Heal barriers - ZNMethod::nmethod_patch_barriers(nm); + { + ICacheInvalidationContext icic; + // Heal barriers + ZNMethod::nmethod_patch_barriers(nm); - // Heal oops - ZUncoloredRootProcessWeakOopClosure cl(ZNMethod::color(nm)); - ZNMethod::nmethod_oops_do_inner(nm, &cl); + // Heal oops + ZUncoloredRootProcessWeakOopClosure cl(ZNMethod::color(nm)); + ZNMethod::nmethod_oops_do_inner(nm, &cl); + } const uintptr_t prev_color = ZNMethod::color(nm); const uintptr_t new_color = *ZPointerStoreGoodMaskLowOrderBitsAddr; diff --git a/src/hotspot/share/gc/z/zGeneration.cpp b/src/hotspot/share/gc/z/zGeneration.cpp index a8c646bf895ee..3085d4e35344f 100644 --- a/src/hotspot/share/gc/z/zGeneration.cpp +++ b/src/hotspot/share/gc/z/zGeneration.cpp @@ -59,6 +59,7 @@ #include "runtime/atomicAccess.hpp" #include "runtime/continuation.hpp" #include "runtime/handshake.hpp" +#include "runtime/icache.hpp" #include "runtime/safepoint.hpp" #include "runtime/threads.hpp" #include "runtime/vmOperations.hpp" @@ -1435,12 +1436,15 @@ class ZRemapNMethodClosure : public NMethodClosure { virtual void do_nmethod(nmethod* nm) { ZLocker locker(ZNMethod::lock_for_nmethod(nm)); if (_bs_nm->is_armed(nm)) { - // Heal barriers - ZNMethod::nmethod_patch_barriers(nm); - - // Heal oops - ZUncoloredRootProcessOopClosure cl(ZNMethod::color(nm)); - ZNMethod::nmethod_oops_do_inner(nm, &cl); + { + ICacheInvalidationContext icic; + // Heal barriers + ZNMethod::nmethod_patch_barriers(nm); + + // Heal oops + ZUncoloredRootProcessOopClosure cl(ZNMethod::color(nm)); + ZNMethod::nmethod_oops_do_inner(nm, &cl); + } log_trace(gc, nmethod)("nmethod: " PTR_FORMAT " visited by old remapping", p2i(nm)); diff --git a/src/hotspot/share/gc/z/zMark.cpp b/src/hotspot/share/gc/z/zMark.cpp index 3b247fdd35e45..ca755c47b8229 100644 --- a/src/hotspot/share/gc/z/zMark.cpp +++ b/src/hotspot/share/gc/z/zMark.cpp @@ -60,6 +60,7 @@ #include "runtime/atomicAccess.hpp" #include "runtime/continuation.hpp" #include "runtime/handshake.hpp" +#include "runtime/icache.hpp" #include "runtime/javaThread.hpp" #include "runtime/prefetch.inline.hpp" #include "runtime/safepointMechanism.hpp" @@ -718,12 +719,15 @@ class ZMarkNMethodClosure : public NMethodClosure { virtual void do_nmethod(nmethod* nm) { ZLocker locker(ZNMethod::lock_for_nmethod(nm)); if (_bs_nm->is_armed(nm)) { - // Heal barriers - ZNMethod::nmethod_patch_barriers(nm); - - // Heal oops - ZUncoloredRootMarkOopClosure cl(ZNMethod::color(nm)); - ZNMethod::nmethod_oops_do_inner(nm, &cl); + { + ICacheInvalidationContext icic; + // Heal barriers + ZNMethod::nmethod_patch_barriers(nm); + + // Heal oops + ZUncoloredRootMarkOopClosure cl(ZNMethod::color(nm)); + ZNMethod::nmethod_oops_do_inner(nm, &cl); + } // CodeCache unloading support nm->mark_as_maybe_on_stack(); @@ -753,10 +757,6 @@ class ZMarkYoungNMethodClosure : public NMethodClosure { if (_bs_nm->is_armed(nm)) { const uintptr_t prev_color = ZNMethod::color(nm); - // Heal oops - ZUncoloredRootMarkYoungOopClosure cl(prev_color); - ZNMethod::nmethod_oops_do_inner(nm, &cl); - // Disarm only the young marking, not any potential old marking cycle const uintptr_t old_marked_mask = ZPointerMarkedMask ^ (ZPointerMarkedYoung0 | ZPointerMarkedYoung1); @@ -767,9 +767,16 @@ class ZMarkYoungNMethodClosure : public NMethodClosure { // Check if disarming for young mark, completely disarms the nmethod entry barrier const bool complete_disarm = ZPointer::is_store_good(new_disarm_value_ptr); - if (complete_disarm) { - // We are about to completely disarm the nmethod, must take responsibility to patch all barriers before disarming - ZNMethod::nmethod_patch_barriers(nm); + { + ICacheInvalidationContext icic; + if (complete_disarm) { + // We are about to completely disarm the nmethod, must take responsibility to patch all barriers before disarming + ZNMethod::nmethod_patch_barriers(nm); + } + + // Heal oops + ZUncoloredRootMarkYoungOopClosure cl(prev_color); + ZNMethod::nmethod_oops_do_inner(nm, &cl); } _bs_nm->guard_with(nm, (int)untype(new_disarm_value_ptr)); diff --git a/src/hotspot/share/gc/z/zNMethod.cpp b/src/hotspot/share/gc/z/zNMethod.cpp index 780bc9e3bf759..4db5ccf97658b 100644 --- a/src/hotspot/share/gc/z/zNMethod.cpp +++ b/src/hotspot/share/gc/z/zNMethod.cpp @@ -50,6 +50,7 @@ #include "oops/oop.inline.hpp" #include "runtime/atomicAccess.hpp" #include "runtime/continuation.hpp" +#include "runtime/icache.hpp" #include "utilities/debug.hpp" static ZNMethodData* gc_data(const nmethod* nm) { @@ -247,6 +248,10 @@ void ZNMethod::set_guard_value(nmethod* nm, int value) { void ZNMethod::nmethod_patch_barriers(nmethod* nm) { ZBarrierSetAssembler* const bs_asm = ZBarrierSet::assembler(); ZArrayIterator iter(gc_data(nm)->barriers()); + ICacheInvalidationContext icic; + if (gc_data(nm)->barriers()->is_nonempty()) { + icic.set_has_modified_code(); + } for (ZNMethodDataBarrier barrier; iter.next(&barrier);) { bs_asm->patch_barrier_relocation(barrier._reloc_addr, barrier._reloc_format); } diff --git a/src/hotspot/share/runtime/globals.hpp b/src/hotspot/share/runtime/globals.hpp index 68b5d8254fd33..721b0bc30384e 100644 --- a/src/hotspot/share/runtime/globals.hpp +++ b/src/hotspot/share/runtime/globals.hpp @@ -2003,6 +2003,8 @@ const int ObjectAlignmentInBytes = 8; develop(uint, BinarySearchThreshold, 16, \ "Minimal number of elements in a sorted collection to prefer" \ "binary search over simple linear search." ) \ + product(bool, UseDeferredICacheInvalidation, false, DIAGNOSTIC, \ + "Defer multiple ICache invalidation to single invalidation") \ \ // end of RUNTIME_FLAGS diff --git a/src/hotspot/share/runtime/icache.cpp b/src/hotspot/share/runtime/icache.cpp index 34d5a7e055b47..4a4162b41ce07 100644 --- a/src/hotspot/share/runtime/icache.cpp +++ b/src/hotspot/share/runtime/icache.cpp @@ -31,6 +31,8 @@ // The flush stub function address AbstractICache::flush_icache_stub_t AbstractICache::_flush_icache_stub = nullptr; +NOT_PRODUCT(THREAD_LOCAL DefaultICacheInvalidationContext* DefaultICacheInvalidationContext::_current_context = nullptr;) + void AbstractICache::initialize(int phase) { // Making this stub must be FIRST use of assembler ResourceMark rm; diff --git a/src/hotspot/share/runtime/icache.hpp b/src/hotspot/share/runtime/icache.hpp index e442c4d88e00a..88d9cce71e444 100644 --- a/src/hotspot/share/runtime/icache.hpp +++ b/src/hotspot/share/runtime/icache.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -71,6 +71,11 @@ class AbstractICache : AllStatic { static void invalidate_range(address start, int nbytes); }; +enum class ICacheInvalidation : uint8_t { + NOT_NEEDED = 0, + IMMEDIATE = 1, + DEFERRED = 2 +}; // Must be included before the definition of ICacheStubGenerator // because ICacheStubGenerator uses ICache definitions. @@ -129,4 +134,62 @@ class ICacheStubGenerator : public StubCodeGenerator { void generate_icache_flush(ICache::flush_icache_stub_t* flush_icache_stub); }; +class DefaultICacheInvalidationContext final : StackObj { + private: + NONCOPYABLE(DefaultICacheInvalidationContext); + + NOT_PRODUCT(static THREAD_LOCAL DefaultICacheInvalidationContext* _current_context;) + + address _code; + int _size; + ICacheInvalidation _mode; + + public: + DefaultICacheInvalidationContext(ICacheInvalidation mode) + : _code(nullptr), _size(0), _mode(mode) { + NOT_PRODUCT(_current_context = this); + } + + DefaultICacheInvalidationContext() : DefaultICacheInvalidationContext(ICacheInvalidation::IMMEDIATE) {} + + DefaultICacheInvalidationContext(address code, int size) + : _code(code), _size(size), _mode(ICacheInvalidation::DEFERRED) { + NOT_PRODUCT(_current_context = this); + assert(code != nullptr, "code must not be null for deferred invalidation"); + assert(size > 0, "size must be positive for deferred invalidation"); + } + + ~DefaultICacheInvalidationContext() { + NOT_PRODUCT(_current_context = nullptr); + if (_code != nullptr) { + assert(_mode == ICacheInvalidation::DEFERRED, "sanity"); + assert(_size > 0, "size must be positive for deferred invalidation"); + ICache::invalidate_range(_code, _size); + _code = nullptr; + _size = 0; + _mode = ICacheInvalidation::NOT_NEEDED; + } + } + + ICacheInvalidation mode() const { + return _mode; + } + + void set_has_modified_code() { + // No-op for the default implementation. + } + +#ifdef ASSERT + static DefaultICacheInvalidationContext* current() { + return _current_context; + } +#endif +}; + +#ifdef PD_ICACHE_INVALIDATION_CONTEXT +using ICacheInvalidationContext = PD_ICACHE_INVALIDATION_CONTEXT; +#else +using ICacheInvalidationContext = DefaultICacheInvalidationContext; +#endif // PD_ICACHE_INVALIDATION_CONTEXT + #endif // SHARE_RUNTIME_ICACHE_HPP diff --git a/test/hotspot/jtreg/gc/TestDeferredICacheInvalidation.java b/test/hotspot/jtreg/gc/TestDeferredICacheInvalidation.java new file mode 100644 index 0000000000000..c104d367ac2bc --- /dev/null +++ b/test/hotspot/jtreg/gc/TestDeferredICacheInvalidation.java @@ -0,0 +1,244 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +package gc; + +/* + * @test id=parallel + * @bug 8370947 + * @summary Check no assertion is triggered when UseDeferredICacheInvalidation is enabled for ParallelGC + * @library /test/lib + * @requires vm.debug + * @requires vm.gc.Parallel + * @build jdk.test.whitebox.WhiteBox + * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseParallelGC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation youngGC C1 + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseParallelGC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation fullGC C1 + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseParallelGC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation youngGC C2 + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseParallelGC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation fullGC C2 + */ + +/* + * @test id=g1 + * @bug 8370947 + * @summary Check no assertion is triggered when UseDeferredICacheInvalidation is enabled for G1GC + * @library /test/lib + * @requires vm.debug + * @requires vm.gc.G1 + * @build jdk.test.whitebox.WhiteBox + * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseG1GC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation youngGC C1 + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseG1GC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation fullGC C1 + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseG1GC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation youngGC C2 + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseG1GC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation fullGC C2 + */ + +/* + * @test id=shenandoah + * @bug 8370947 + * @summary Check no assertion is triggered when UseDeferredICacheInvalidation is enabled for ShenandoahGC + * @library /test/lib + * @requires vm.debug + * @requires vm.gc.Shenandoah + * @build jdk.test.whitebox.WhiteBox + * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseShenandoahGC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation fullGC C1 + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseShenandoahGC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation fullGC C2 + */ + +/* + * @test id=genshen + * @bug 8370947 + * @summary Check no assertion is triggered when UseDeferredICacheInvalidation is enabled for generational ShenandoahGC + * @library /test/lib + * @requires vm.debug + * @requires vm.gc.Shenandoah + * @build jdk.test.whitebox.WhiteBox + * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseShenandoahGC -XX:+UnlockExperimentalVMOptions -XX:ShenandoahGCMode=generational -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation youngGC C1 + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseShenandoahGC -XX:+UnlockExperimentalVMOptions -XX:ShenandoahGCMode=generational -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation fullGC C1 + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseShenandoahGC -XX:+UnlockExperimentalVMOptions -XX:ShenandoahGCMode=generational -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation youngGC C2 + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseShenandoahGC -XX:+UnlockExperimentalVMOptions -XX:ShenandoahGCMode=generational -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation fullGC C2 + */ + +/* + * @test id=z + * @bug 8370947 + * @summary Check no assertion is triggered when UseDeferredICacheInvalidation is enabled for ZGC + * @library /test/lib + * @requires vm.debug + * @requires vm.gc.Z + * @build jdk.test.whitebox.WhiteBox + * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseZGC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation youngGC C1 + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseZGC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation fullGC C1 + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseZGC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation youngGC C2 + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseZGC -XX:-UseCodeCacheFlushing gc.TestDeferredICacheInvalidation fullGC C2 + */ + +/* + * Nmethods have GC barriers and OOPs embedded into their code. GCs can patch nmethod's code + * which requires icache invalidation. Doing invalidation per instruction can be expensive. + * CPU can support hardware dcache and icache coherence. This would allow to defer cache + * invalidation. + * + * There are assertions for deferred cache invalidation. This test checks that all of them + * are passed. + */ + +import jdk.test.whitebox.WhiteBox; + +public class TestDeferredICacheInvalidation { + + private static final WhiteBox WB = WhiteBox.getWhiteBox(); + + public static class A { + public String s1; + public String s2; + public String s3; + public String s4; + public String s5; + public String s6; + public String s7; + public String s8; + public String s9; + } + + public static A a = new A(); + + private static int compLevel; + + public static class B { + public static void test0() { + } + + public static void test1() { + a.s1 = a.s1 + "1"; + } + + public static void test2() { + a.s1 = a.s1 + "1"; + a.s2 = a.s2 + "2"; + } + + public static void test3() { + a.s1 = a.s1 + "1"; + a.s2 = a.s2 + "2"; + a.s3 = a.s3 + "3"; + } + + public static void test4() { + a.s1 = a.s1 + "1"; + a.s2 = a.s2 + "2"; + a.s3 = a.s3 + "3"; + a.s4 = a.s4 + "4"; + } + + public static void test5() { + a.s1 = a.s1 + "1"; + a.s2 = a.s2 + "2"; + a.s3 = a.s3 + "3"; + a.s4 = a.s4 + "4"; + a.s5 = a.s5 + "5"; + } + + public static void test6() { + a.s1 = a.s1 + "1"; + a.s2 = a.s2 + "2"; + a.s3 = a.s3 + "3"; + a.s4 = a.s4 + "4"; + a.s5 = a.s5 + "5"; + a.s6 = a.s6 + "6"; + } + + public static void test7() { + a.s1 = a.s1 + "1"; + a.s2 = a.s2 + "2"; + a.s3 = a.s3 + "3"; + a.s4 = a.s4 + "4"; + a.s5 = a.s5 + "5"; + a.s6 = a.s6 + "6"; + a.s7 = a.s7 + "7"; + } + + public static void test8() { + a.s1 = a.s1 + "1"; + a.s2 = a.s2 + "2"; + a.s3 = a.s3 + "3"; + a.s4 = a.s4 + "4"; + a.s5 = a.s5 + "5"; + a.s6 = a.s6 + "6"; + a.s7 = a.s7 + "7"; + a.s8 = a.s8 + "8"; + } + + public static void test9() { + a.s1 = a.s1 + "1"; + a.s2 = a.s2 + "2"; + a.s3 = a.s3 + "3"; + a.s4 = a.s4 + "4"; + a.s5 = a.s5 + "5"; + a.s6 = a.s6 + "6"; + a.s7 = a.s7 + "7"; + a.s8 = a.s8 + "8"; + a.s9 = a.s9 + "9"; + } + } + + private static void compileMethods() throws Exception { + for (var m : B.class.getDeclaredMethods()) { + if (!m.getName().startsWith("test")) { + continue; + } + m.invoke(null); + WB.markMethodProfiled(m); + WB.enqueueMethodForCompilation(m, compLevel); + while (WB.isMethodQueuedForCompilation(m)) { + Thread.onSpinWait(); + } + if (WB.getMethodCompilationLevel(m) != compLevel) { + throw new IllegalStateException("Method " + m + " is not compiled at the compilation level: " + compLevel + ". Got: " + WB.getMethodCompilationLevel(m)); + } + } + } + + public static void youngGC() throws Exception { + a = null; + WB.youngGC(); + } + + public static void fullGC() throws Exception { + a = null; + WB.fullGC(); + } + + public static void main(String[] args) throws Exception { + if (!Boolean.TRUE.equals(WB.getBooleanVMFlag("UseDeferredICacheInvalidation"))) { + System.out.println("Skip. Test requires UseDeferredICacheInvalidation enabled."); + } + compLevel = (args[1].equals("C1")) ? 1 : 4; + compileMethods(); + TestDeferredICacheInvalidation.class.getMethod(args[0]).invoke(null); + } +} diff --git a/test/micro/org/openjdk/bench/vm/gc/GCPatchingNmethodCost.java b/test/micro/org/openjdk/bench/vm/gc/GCPatchingNmethodCost.java new file mode 100644 index 0000000000000..3e1542bdb51d4 --- /dev/null +++ b/test/micro/org/openjdk/bench/vm/gc/GCPatchingNmethodCost.java @@ -0,0 +1,205 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +package org.openjdk.bench.vm.gc; + +import java.lang.reflect.Method; +import java.util.*; +import java.util.concurrent.TimeUnit; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.CompilerControl; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +import org.openjdk.bench.util.InMemoryJavaCompiler; + +import jdk.test.whitebox.WhiteBox; +import jdk.test.whitebox.code.NMethod; + +/* + * Nmethods have OOPs and GC barriers emmedded into their code. + * GCs patch them which causes invalidation of nmethods' code. + * + * This benchmark can be used to estimate the cost of patching + * OOPs and GC barriers. + * + * We create 5000 nmethods which access fields of a class. + * We measure the time of different GC cycles to see + * the impact of patching nmethods. + * + * The benchmark parameters are method count and accessed field count. + */ + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +@Fork(value = 3, jvmArgsAppend = { + "-XX:+UnlockDiagnosticVMOptions", + "-XX:+UnlockExperimentalVMOptions", + "-XX:+WhiteBoxAPI", + "-Xbootclasspath/a:lib-test/wb.jar", + "-XX:-UseCodeCacheFlushing" +}) +public class GCPatchingNmethodCost { + + private static final int COMP_LEVEL = 1; + private static final String FIELD_USER = "FieldUser"; + + public static Fields fields; + + private static TestMethod[] methods = {}; + private static byte[] BYTE_CODE; + private static WhiteBox WB; + + @Param({"5000"}) + public int methodCount; + + @Param({"0", "2", "4", "8"}) + public int accessedFieldCount; + + public static class Fields { + public String f1; + public String f2; + public String f3; + public String f4; + public String f5; + public String f6; + public String f7; + public String f8; + public String f9; + } + + private static final class TestMethod { + private final Method method; + + public TestMethod(Method method) throws Exception { + this.method = method; + WB.testSetDontInlineMethod(method, true); + } + + public void profile() throws Exception { + method.invoke(null); + WB.markMethodProfiled(method); + } + + public void invoke() throws Exception { + method.invoke(null); + } + + public void compile() throws Exception { + WB.enqueueMethodForCompilation(method, COMP_LEVEL); + while (WB.isMethodQueuedForCompilation(method)) { + Thread.onSpinWait(); + } + if (WB.getMethodCompilationLevel(method) != COMP_LEVEL) { + throw new IllegalStateException("Method " + method + " is not compiled at the compilation level: " + COMP_LEVEL + ". Got: " + WB.getMethodCompilationLevel(method)); + } + } + + public NMethod getNMethod() { + return NMethod.get(method, false); + } + } + + private static ClassLoader createClassLoader() { + return new ClassLoader() { + @Override + public Class loadClass(String name) throws ClassNotFoundException { + if (!name.equals(FIELD_USER)) { + return super.loadClass(name); + } + + return defineClass(name, BYTE_CODE, 0, BYTE_CODE.length); + } + }; + } + + private static void createTestMethods(int accessedFieldCount, int count) throws Exception { + String javaCode = "public class " + FIELD_USER + " {"; + String field = GCPatchingNmethodCost.class.getName() + ".fields.f"; + javaCode += "public static void accessFields() {"; + for (int i = 1; i <= accessedFieldCount; i++) { + javaCode += field + i + "= " + field + i + " + " + i + ";"; + } + javaCode += "}}"; + + BYTE_CODE = InMemoryJavaCompiler.compile(FIELD_USER, javaCode); + + fields = new Fields(); + + methods = new TestMethod[count]; + for (int i = 0; i < count; i++) { + var cl = createClassLoader().loadClass(FIELD_USER); + Method method = cl.getMethod("accessFields"); + methods[i] = new TestMethod(method); + methods[i].profile(); + methods[i].compile(); + } + } + + private static void initWhiteBox() { + WB = WhiteBox.getWhiteBox(); + } + + @Setup(Level.Trial) + public void setupCodeCache() throws Exception { + initWhiteBox(); + createTestMethods(accessedFieldCount, methodCount); + Thread.sleep(1000); + } + + @Benchmark + @Warmup(iterations = 0) + @Measurement(iterations = 1) + public void youngGC() throws Exception { + fields = null; + WB.youngGC(); + } + + @Benchmark + @Warmup(iterations = 0) + @Measurement(iterations = 1) + public void fullGC() throws Exception { + fields = null; + WB.fullGC(); + } + + @Benchmark + @Warmup(iterations = 0) + @Measurement(iterations = 1) + public void systemGC() throws Exception { + fields = null; + System.gc(); + } +}