8370947: Mitigate Neoverse-N1 erratum 1542419 negative impact on GCs and JIT performance

src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp

@@ -879,6 +879,18 @@ void ZBarrierSetAssembler::patch_barrier_relocation(address addr, int format) {
     ShouldNotReachHere();
   }
 
+  if (UseDeferredICacheInvalidation) {
+    // Defer the ICache invalidation to a later point where multiple patches can be handled together.
+    //
+    // Note: We rely on the fact that this function is only called from places where deferred invalidation
+    // is safe. This assumption helps to avoid overhead of accessing thread-local data here.
+    assert(ICacheInvalidationContext::current() != nullptr, "ICache invalidation context should be set");
+    assert(ICacheInvalidationContext::current()->mode() == ICacheInvalidation::DEFERRED ||
+           ICacheInvalidationContext::current()->mode() == ICacheInvalidation::NOT_NEEDED,
+           "ICache invalidation should be deferred or unneeded");
+    return;
+  }
+
   ICache::invalidate_word((address)patch_addr);
 }
 

src/hotspot/cpu/aarch64/globals_aarch64.hpp

@@ -127,6 +127,8 @@ define_pd_global(intx, InlineSmallCode,          1000);
           "Branch Protection to use: none, standard, pac-ret")          \
   product(bool, AlwaysMergeDMB, true, DIAGNOSTIC,                       \
           "Always merge DMB instructions in code emission")             \
+  product(bool, NeoverseN1Errata1542419, false, DIAGNOSTIC,             \
+          "Enable workaround for Neoverse N1 erratum 1542419")          \
 
 // end of ARCH_FLAGS
 

src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp

@@ -26,6 +26,7 @@
 #include "asm/macroAssembler.hpp"
 #include "code/nmethod.hpp"
 #include "code/relocInfo.hpp"
+#include "gc/shared/barrierSetNMethod.hpp"
 #include "nativeInst_aarch64.hpp"
 #include "oops/oop.inline.hpp"
 #include "runtime/safepoint.hpp"
@@ -54,6 +55,33 @@ void Relocation::pd_set_data_value(address x, bool verify_only) {
     bytes = MacroAssembler::pd_patch_instruction_size(addr(), x);
     break;
   }
+
+  if (UseDeferredICacheInvalidation) {
+    // Defer the ICache invalidation to a later point where multiple patches can be handled together.
+    //
+    // Note: We rely on the fact that this function is only called from places where deferred invalidation
+    // is safe. This assumption helps to avoid overhead of accessing thread-local data here.
+    assert(ICacheInvalidationContext::current() != nullptr, "ICache invalidation context should be set");
+    assert(ICacheInvalidationContext::current()->mode() == ICacheInvalidation::DEFERRED ||
+           ICacheInvalidationContext::current()->mode() == ICacheInvalidation::NOT_NEEDED,
+           "ICache invalidation should be deferred or unneeded.");
+#ifdef ASSERT
+    if (_binding != nullptr && _binding->code() != nullptr) {
+      nmethod *nm = _binding->code();
+      if (!(BarrierSet::barrier_set()->barrier_set_nmethod()->is_armed(nm) ||
+            SafepointSynchronize::is_at_safepoint() ||
+            nm->is_unloading())) {
+        ConditionalMutexLocker ml(NMethodState_lock, !NMethodState_lock->owned_by_self(), Mutex::_no_safepoint_check_flag);
+        assert(nm->is_not_installed(),
+             "ICache invalidation context should only be used for armed "
+             "or unloading or not installed nmethod or at safepoint");
+
+      }
+    }
+#endif
+    return;
+  }
+
   ICache::invalidate_range(addr(), bytes);
 }
 

src/hotspot/cpu/aarch64/vm_version_aarch64.cpp

@@ -63,6 +63,19 @@ static SpinWait get_spin_wait_desc() {
   return spin_wait;
 }
 
+static bool has_neoverse_n1_errata_1542419() {
+  const int major_rev_num = VM_Version::cpu_variant();
+  const int minor_rev_num = VM_Version::cpu_revision();
+  // Neoverse N1: 0xd0c
+  // Erratum 1542419 affects r3p0, r3p1 and r4p0.
+  // It is fixed in r4p1 and later revisions, which are not affected.
+  return (VM_Version::cpu_family() == VM_Version::CPU_ARM &&
+          VM_Version::model_is(0xd0c) &&
+          ((major_rev_num == 3 && minor_rev_num == 0) ||
+           (major_rev_num == 3 && minor_rev_num == 1) ||
+           (major_rev_num == 4 && minor_rev_num == 0)));
+}
+
 void VM_Version::initialize() {
 #define SET_CPU_FEATURE_NAME(id, name, bit) \
   _features_names[bit] = XSTR(name);
@@ -641,6 +654,19 @@ void VM_Version::initialize() {
     clear_feature(CPU_SVE);
   }
 
+  if (FLAG_IS_DEFAULT(NeoverseN1Errata1542419) && has_neoverse_n1_errata_1542419()) {
+    FLAG_SET_DEFAULT(NeoverseN1Errata1542419, true);
+  }
+
+  if (NeoverseN1Errata1542419) {
+    if (!has_neoverse_n1_errata_1542419()) {
+      warning("NeoverseN1Errata1542419 is set for the CPU not having Neoverse N1 errata 1542419");
+    }
+    if (FLAG_IS_DEFAULT(UseDeferredICacheInvalidation)) {
+      FLAG_SET_DEFAULT(UseDeferredICacheInvalidation, true);
+    }
+  }
+
   // Construct the "features" string
   stringStream ss(512);
   ss.print("0x%02x:0x%x:0x%03x:%d", _cpu, _variant, _model, _revision);

src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp

@@ -0,0 +1,28 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "runtime/icache.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+THREAD_LOCAL AArch64ICacheInvalidationContext* AArch64ICacheInvalidationContext::_current_context = nullptr;

src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1999, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -26,6 +26,35 @@
 #ifndef OS_CPU_LINUX_AARCH64_ICACHE_AARCH64_HPP
 #define OS_CPU_LINUX_AARCH64_ICACHE_AARCH64_HPP
 
+#include "memory/allocation.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+inline void assert_hardware_cache_coherency() {
+  // For deferred icache invalidation, we expect hardware dcache
+  // and icache to be coherent: CTR_EL0.IDC == 1 and CTR_EL0.DIC == 1
+  // An exception is Neoverse N1 with erratum 1542419, which requires
+  // a use of 'IC IVAU' instruction. In such a case, we expect
+  // CTR_EL0.DIC == 0.
+#ifdef ASSERT
+  static unsigned int cache_info = 0;
+  if (cache_info == 0) {
+    asm volatile("mrs\t%0, ctr_el0" : "=r"(cache_info));
+  }
+  constexpr unsigned int CTR_IDC_SHIFT = 28;
+  constexpr unsigned int CTR_DIC_SHIFT = 29;
+  assert(((cache_info >> CTR_IDC_SHIFT) & 0x1) != 0x0,
+         "Expect CTR_EL0.IDC to be enabled");
+  if (NeoverseN1Errata1542419) {
+    assert(((cache_info >> CTR_DIC_SHIFT) & 0x1) == 0x0,
+           "Expect CTR_EL0.DIC to be disabled for Neoverse N1 with erratum "
+           "1542419");
+  } else {
+    assert(((cache_info >> CTR_DIC_SHIFT) & 0x1) != 0x0,
+           "Expect CTR_EL0.DIC to be enabled");
+  }
+#endif
+}
+
 // Interface for updating the instruction cache.  Whenever the VM
 // modifies code, part of the processor instruction cache potentially
 // has to be flushed.
@@ -37,8 +66,144 @@ class ICache : public AbstractICache {
     __builtin___clear_cache((char *)addr, (char *)(addr + 4));
   }
   static void invalidate_range(address start, int nbytes) {
-    __builtin___clear_cache((char *)start, (char *)(start + nbytes));
+    if (NeoverseN1Errata1542419) {
+      assert_hardware_cache_coherency();
+      asm volatile("dsb ish       \n"
+                   "ic  ivau, xzr \n"
+                   "dsb ish       \n"
+                   "isb           \n"
+                   : : : "memory");
+    } else {
+      __builtin___clear_cache((char *)start, (char *)(start + nbytes));
+    }
+  }
+};
+
+class AArch64ICacheInvalidationContext final : StackObj {
+ private:
+  NONCOPYABLE(AArch64ICacheInvalidationContext);
+
+  static THREAD_LOCAL AArch64ICacheInvalidationContext* _current_context;
+
+  AArch64ICacheInvalidationContext* _parent;
+  address                           _code;
+  int                               _size;
+  ICacheInvalidation                _mode;
+  bool                              _has_modified_code;
+
+ public:
+  AArch64ICacheInvalidationContext(ICacheInvalidation mode)
+      : _parent(nullptr), _code(nullptr), _size(0), _mode(mode), _has_modified_code(false) {
+    _parent = _current_context;
+    _current_context = this;
+    if (_parent != nullptr) {
+      // The parent context is in charge of icache invalidation.
+      _mode = (_parent->mode() == ICacheInvalidation::IMMEDIATE) ? ICacheInvalidation::IMMEDIATE : ICacheInvalidation::NOT_NEEDED;
+    }
+  }
+
+  AArch64ICacheInvalidationContext()
+      : AArch64ICacheInvalidationContext(UseDeferredICacheInvalidation
+                                            ? ICacheInvalidation::DEFERRED
+                                            : ICacheInvalidation::IMMEDIATE) {}
+
+  AArch64ICacheInvalidationContext(address code, int size)
+      : _parent(nullptr),
+        _code(code),
+        _size(size),
+        _mode(ICacheInvalidation::DEFERRED),
+        _has_modified_code(true) {
+    assert(_current_context == nullptr,
+           "nested ICacheInvalidationContext(code, size) not supported");
+    assert(code != nullptr, "code must not be null for deferred invalidation");
+    assert(size > 0, "size must be positive for deferred invalidation");
+
+    _current_context = this;
+
+    if (UseDeferredICacheInvalidation) {
+      // With hardware dcache and icache coherency, we don't need _code.
+      _code = nullptr;
+      _size = 0;
+    }
+  }
+
+  ~AArch64ICacheInvalidationContext() {
+    _current_context = _parent;
+
+    if (_code != nullptr) {
+      assert(_size > 0, "size must be positive for deferred invalidation");
+      assert(_mode == ICacheInvalidation::DEFERRED, "sanity");
+      assert(_has_modified_code, "sanity");
+      assert(_parent == nullptr, "sanity");
+
+      ICache::invalidate_range(_code, _size);
+      return;
+    }
+
+    if (!_has_modified_code) {
+      return;
+    }
+
+    if (_parent != nullptr) {
+      _parent->set_has_modified_code();
+    }
+
+    if (_mode != ICacheInvalidation::DEFERRED) {
+      return;
+    }
+
+    assert_hardware_cache_coherency();
+
+    asm volatile("dsb ish" : : : "memory");
+
+    if (NeoverseN1Errata1542419) {
+      // Errata 1542419: Neoverse N1 cores with the 'COHERENT_ICACHE' feature
+      // may fetch stale instructions when software depends on
+      // prefetch-speculation-protection instead of explicit synchronization.
+      //
+      // Neoverse-N1 implementation mitigates the errata 1542419 with a
+      // workaround:
+      // - Disable coherent icache.
+      // - Trap IC IVAU instructions.
+      // - Execute:
+      //   - tlbi vae3is, xzr
+      //   - dsb sy
+      // - Ignore trapped IC IVAU instructions.
+      //
+      // `tlbi vae3is, xzr` invalidates all translation entries (all VAs, all
+      // possible levels). It waits for all memory accesses using in-scope old
+      // translation information to complete before it is considered complete.
+      //
+      // As this workaround has significant overhead, Arm Neoverse N1 (MP050)
+      // Software Developer Errata Notice version 29.0 suggests:
+      //
+      // "Since one TLB inner-shareable invalidation is enough to avoid this
+      // erratum, the number of injected TLB invalidations should be minimized
+      // in the trap handler to mitigate the performance impact due to this
+      // workaround."
+      // As the address for icache invalidation is not relevant and
+      // IC IVAU instruction is ignored, we use XZR in it.
+      asm volatile(
+          "ic  ivau, xzr \n"
+          "dsb ish       \n"
+          :
+          :
+          : "memory");
+    }
+    asm volatile("isb" : : : "memory");
+  }
+
+  ICacheInvalidation mode() const { return _mode; }
+
+  void set_has_modified_code() {
+    _has_modified_code = true;
+  }
+
+  static AArch64ICacheInvalidationContext* current() {
+    return _current_context;
   }
 };
 
+#define PD_ICACHE_INVALIDATION_CONTEXT AArch64ICacheInvalidationContext
+
 #endif // OS_CPU_LINUX_AARCH64_ICACHE_AARCH64_HPP

src/hotspot/share/asm/codeBuffer.cpp

@@ -723,6 +723,11 @@ csize_t CodeBuffer::copy_relocations_to(CodeBlob* dest) const {
 }
 
 void CodeBuffer::copy_code_to(CodeBlob* dest_blob) {
+  assert(ICacheInvalidationContext::current() != nullptr,
+         "ICache invalidation context should be set");
+  assert(ICacheInvalidationContext::current()->mode() == ICacheInvalidation::DEFERRED ||
+         ICacheInvalidationContext::current()->mode() == ICacheInvalidation::NOT_NEEDED,
+         "ICache invalidation should be deferred or unneeded");
 #ifndef PRODUCT
   if (PrintNMethods && (WizardMode || Verbose)) {
     tty->print("done with CodeBuffer:");
@@ -745,9 +750,6 @@ void CodeBuffer::copy_code_to(CodeBlob* dest_blob) {
 
   // Done moving code bytes; were they the right size?
   assert((int)align_up(dest.total_content_size(), oopSize) == dest_blob->content_size(), "sanity");
-
-  // Flush generated code
-  ICache::invalidate_range(dest_blob->code_begin(), dest_blob->code_size());
 }
 
 // Move all my code into another code buffer.  Consult applicable
@@ -930,8 +932,11 @@ void CodeBuffer::expand(CodeSection* which_cs, csize_t amount) {
   // Needs to be initialized when calling fix_relocation_after_move.
   cb.blob()->set_ctable_begin(cb.consts()->start());
 
-  // Move all the code and relocations to the new blob:
-  relocate_code_to(&cb);
+  {
+    ICacheInvalidationContext icic(ICacheInvalidation::NOT_NEEDED);
+    // Move all the code and relocations to the new blob:
+    relocate_code_to(&cb);
+  }
 
   // some internal addresses, _last_insn _last_label, are used during code emission,
   // adjust them in expansion

src/hotspot/share/code/codeBlob.cpp

@@ -331,6 +331,9 @@ RuntimeBlob::RuntimeBlob(
   : CodeBlob(name, kind, cb, size, header_size, frame_complete, frame_size, oop_maps, caller_must_gc_arguments,
              align_up(cb->total_relocation_size(), oopSize))
 {
+  // Optimize ICache invalidation by batching it for the whole blob if
+  // possible.
+  ICacheInvalidationContext icic(code_begin(), code_size());
   cb->copy_code_and_locs_to(this);
 }
 

src/hotspot/share/code/codeCache.cpp

@@ -681,6 +681,7 @@ void CodeCache::nmethods_do(NMethodClosure* cl) {
   assert_locked_or_safepoint(CodeCache_lock);
   NMethodIterator iter(NMethodIterator::all);
   while(iter.next()) {
+    ICacheInvalidationContext icic;
     cl->do_nmethod(iter.method());
   }
 }