8322630: Remove ICStubs and related safepoints

src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

@@ -1236,7 +1236,7 @@ void MacroAssembler::lookup_interface_method(Register recv_klass,
 }
 
 // Look up the method for a megamorphic invokeinterface call in a single pass over itable:
-// - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICHolder
+// - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICData
 // - find a holder_klass (class that implements the method) vtable offset and get the method from vtable by index
 // The target method is determined by <holder_klass, itable_index>.
 // The receiver klass is in recv_klass.

src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp

@@ -738,7 +738,7 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
   address c2i_unverified_entry = __ pc();
   Label skip_fixup;
 
-  Register holder = rscratch2;
+  Register data = rscratch2;
   Register receiver = j_rarg0;
   Register tmp = r10;  // A call-clobbered register not used for arg passing
 
@@ -754,7 +754,7 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
   {
     __ block_comment("c2i_unverified_entry {");
     __ ic_check(1 /* end_alignment */);
-    __ ldr(rmethod, Address(holder, CompiledICData::speculated_method_offset()));
+    __ ldr(rmethod, Address(data, CompiledICData::speculated_method_offset()));
 
     __ ldr(rscratch1, Address(rmethod, in_bytes(Method::code_offset())));
     __ cbz(rscratch1, skip_fixup);

src/hotspot/cpu/aarch64/vtableStubs_aarch64.cpp

@@ -174,16 +174,16 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
   // This stub is called from compiled code which has no callee-saved registers,
   // so all registers except arguments are free at this point.
   const Register recv_klass_reg     = r10;
-  const Register holder_klass_reg   = r16; // declaring interface klass (DECC)
+  const Register holder_klass_reg   = r16; // declaring interface klass (DEFC)
   const Register resolved_klass_reg = r17; // resolved interface klass (REFC)
   const Register temp_reg           = r11;
   const Register temp_reg2          = r15;
-  const Register icholder_reg       = rscratch2;
+  const Register icdata_reg         = rscratch2;
 
   Label L_no_such_interface;
 
-  __ ldr(resolved_klass_reg, Address(icholder_reg, CompiledICData::itable_refc_klass_offset()));
-  __ ldr(holder_klass_reg,   Address(icholder_reg, CompiledICData::itable_defc_klass_offset()));
+  __ ldr(resolved_klass_reg, Address(icdata_reg, CompiledICData::itable_refc_klass_offset()));
+  __ ldr(holder_klass_reg,   Address(icdata_reg, CompiledICData::itable_defc_klass_offset()));
 
   start_pc = __ pc();
 

src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp

@@ -1172,8 +1172,8 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
   BLOCK_COMMENT("c2i unverified entry");
   c2i_unverified_entry = __ pc();
 
-  // inline_cache contains a compiledICHolder
-  const Register ic             = R19_method;
+  // inline_cache contains a CompiledICData
+  const Register ic             = R19_inline_cache_reg;
   const Register ic_klass       = R11_scratch1;
   const Register receiver_klass = R12_scratch2;
   const Register code           = R21_tmp1;

src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp

@@ -51,7 +51,6 @@
 #endif
 
 NEEDS_CLEANUP // remove this definitions ?
-const Register IC_Klass    = t1;    // where the IC klass is cached
 const Register SYNC_header = x10;   // synchronization header
 const Register SHIFT_count = x10;   // where count for shift operations must be
 

src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.hpp

@@ -68,7 +68,7 @@ friend class ArrayCopyStub;
 
   enum {
     // See emit_static_call_stub for detail
-    // CompiledDirectCall::to_interp_stub_size() (14) + CompiledStaticCall::to_trampoline_stub_size() (1 + 3 + address)
+    // CompiledDirectCall::to_interp_stub_size() (14) + CompiledDirectCall::to_trampoline_stub_size() (1 + 3 + address)
     _call_stub_size = 14 * NativeInstruction::instruction_size +
                       (NativeInstruction::instruction_size + NativeCallTrampolineStub::instruction_size),
     // See emit_exception_handler for detail

src/hotspot/cpu/riscv/macroAssembler_riscv.cpp

@@ -2543,7 +2543,7 @@ void MacroAssembler::lookup_interface_method(Register recv_klass,
 }
 
 // Look up the method for a megamorphic invokeinterface call in a single pass over itable:
-// - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICHolder
+// - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICData
 // - find a holder_klass (class that implements the method) vtable offset and get the method from vtable by index
 // The target method is determined by <holder_klass, itable_index>.
 // The receiver klass is in recv_klass.

src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp

@@ -620,10 +620,8 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
   address c2i_unverified_entry = __ pc();
   Label skip_fixup;
 
-  Label ok;
-
   const Register receiver = j_rarg0;
-  const Register holder = t1;
+  const Register data = t1;
   const Register tmp = t2;  // A call-clobbered register not used for arg passing
 
   // -------------------------------------------------------------------------
@@ -639,12 +637,8 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
     __ block_comment("c2i_unverified_entry {");
 
     __ ic_check();
-    __ ld(xmethod, Address(holder, CompiledICData::speculated_method_offset()));
+    __ ld(xmethod, Address(data, CompiledICData::speculated_method_offset()));
 
-    __ bind(ok);
-    // Method might have been compiled since the call site was patched to
-    // interpreted; if that is the case treat it as a miss so we can get
-    // the call site corrected.
     __ ld(t0, Address(xmethod, in_bytes(Method::code_offset())));
     __ beqz(t0, skip_fixup);
     __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));

src/hotspot/cpu/riscv/vtableStubs_riscv.cpp

@@ -171,22 +171,22 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
   assert(VtableStub::receiver_location() == j_rarg0->as_VMReg(), "receiver expected in j_rarg0");
 
   // Entry arguments:
-  //  t1: CompiledICHolder
+  //  t1: CompiledICData
   //  j_rarg0: Receiver
 
   // This stub is called from compiled code which has no callee-saved registers,
   // so all registers except arguments are free at this point.
   const Register recv_klass_reg     = x18;
-  const Register holder_klass_reg   = x19; // declaring interface klass (DECC)
+  const Register holder_klass_reg   = x19; // declaring interface klass (DEFC)
   const Register resolved_klass_reg = x30; // resolved interface klass (REFC)
   const Register temp_reg           = x28;
   const Register temp_reg2          = x29;
-  const Register icholder_reg       = t1;
+  const Register icdata_reg         = t1;
 
   Label L_no_such_interface;
 
-  __ ld(resolved_klass_reg, Address(icholder_reg, CompiledICData::itable_refc_klass_offset()));
-  __ ld(holder_klass_reg,   Address(icholder_reg, CompiledICData::itable_defc_klass_offset()));
+  __ ld(resolved_klass_reg, Address(icdata_reg, CompiledICData::itable_refc_klass_offset()));
+  __ ld(holder_klass_reg,   Address(icdata_reg, CompiledICData::itable_defc_klass_offset()));
 
   start_pc = __ pc();
 

src/hotspot/cpu/s390/sharedRuntime_s390.cpp

@@ -2318,7 +2318,7 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
     c2i_unverified_entry = __ pc();
 
     __ ic_check(2);
-    __ z_lg(Z_method, Address(Z_method, CompiledICData::speculated_method_offset()));
+    __ z_lg(Z_method, Address(Z_inline_cache, CompiledICData::speculated_method_offset()));
     // This def MUST MATCH code in gen_c2i_adapter!
     const Register code = Z_R11;
 

src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp

@@ -943,13 +943,13 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
   address c2i_unverified_entry = __ pc();
   Label skip_fixup;
 
-  Register holder = rax;
+  Register data = rax;
   Register receiver = rcx;
   Register temp = rbx;
 
   {
     __ ic_check(1 /* end_alignment */);
-    __ movptr(rbx, Address(holder, CompiledICData::speculated_method_offset()));
+    __ movptr(rbx, Address(data, CompiledICData::speculated_method_offset()));
     // Method might have been compiled since the call site was patched to
     // interpreted if that is the case treat it as a miss so we can get
     // the call site corrected.

src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp

@@ -999,13 +999,13 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
   address c2i_unverified_entry = __ pc();
   Label skip_fixup;
 
-  Register holder = rax;
+  Register data = rax;
   Register receiver = j_rarg0;
   Register temp = rbx;
 
   {
     __ ic_check(1 /* end_alignment */);
-    __ movptr(rbx, Address(holder, CompiledICData::speculated_method_offset()));
+    __ movptr(rbx, Address(data, CompiledICData::speculated_method_offset()));
     // Method might have been compiled since the call site was patched to
     // interpreted if that is the case treat it as a miss so we can get
     // the call site corrected.

src/hotspot/cpu/x86/vtableStubs_x86_32.cpp

@@ -176,21 +176,21 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
 #endif /* PRODUCT */
 
   // Entry arguments:
-  //  rax: CompiledICHolder
+  //  rax: CompiledICData
   //  rcx: Receiver
 
   // Most registers are in use; we'll use rax, rbx, rcx, rdx, rsi, rdi
   // (If we need to make rsi, rdi callee-save, do a push/pop here.)
   const Register recv_klass_reg     = rsi;
-  const Register holder_klass_reg   = rax; // declaring interface klass (DECC)
+  const Register holder_klass_reg   = rax; // declaring interface klass (DEFC)
   const Register resolved_klass_reg = rdi; // resolved interface klass (REFC)
   const Register temp_reg           = rdx;
   const Register method             = rbx;
-  const Register icholder_reg       = rax;
+  const Register icdata_reg         = rax;
   const Register receiver           = rcx;
 
-  __ movptr(resolved_klass_reg, Address(icholder_reg, CompiledICData::itable_refc_klass_offset()));
-  __ movptr(holder_klass_reg,   Address(icholder_reg, CompiledICData::itable_defc_klass_offset()));
+  __ movptr(resolved_klass_reg, Address(icdata_reg, CompiledICData::itable_refc_klass_offset()));
+  __ movptr(holder_klass_reg,   Address(icdata_reg, CompiledICData::itable_defc_klass_offset()));
 
   Label L_no_such_interface;
 

src/hotspot/cpu/x86/vtableStubs_x86_64.cpp

@@ -179,10 +179,10 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
   const Register temp_reg           = r11;
   const Register temp_reg2          = r13;
   const Register method             = rbx;
-  const Register icholder_reg       = rax;
+  const Register icdata_reg         = rax;
 
-  __ movptr(resolved_klass_reg, Address(icholder_reg, CompiledICData::itable_refc_klass_offset()));
-  __ movptr(holder_klass_reg,   Address(icholder_reg, CompiledICData::itable_defc_klass_offset()));
+  __ movptr(resolved_klass_reg, Address(icdata_reg, CompiledICData::itable_refc_klass_offset()));
+  __ movptr(holder_klass_reg,   Address(icdata_reg, CompiledICData::itable_defc_klass_offset()));
 
   Label L_no_such_interface;
 

src/hotspot/share/code/compiledIC.cpp

@@ -169,29 +169,19 @@ CompiledIC::CompiledIC(RelocIterator* iter)
 
 CompiledIC* CompiledIC_before(CompiledMethod* nm, address return_addr) {
   address call_site = nativeCall_before(return_addr)->instruction_address();
-  RelocIterator iter(nm, call_site, call_site + 1);
-  iter.next();
-  CompiledIC* c_ic = new CompiledIC(&iter);
-  c_ic->verify();
-  return c_ic;
+  return CompiledIC_at(nm, call_site);
 }
 
 CompiledIC* CompiledIC_at(CompiledMethod* nm, address call_site) {
   RelocIterator iter(nm, call_site, call_site + 1);
   iter.next();
-  CompiledIC* c_ic = new CompiledIC(&iter);
-  c_ic->verify();
-  return c_ic;
+  return CompiledIC_at(&iter);
 }
 
 CompiledIC* CompiledIC_at(Relocation* call_reloc) {
   address call_site = call_reloc->addr();
   CompiledMethod* cm = CodeCache::find_blob(call_reloc->addr())->as_compiled_method();
-  RelocIterator iter(cm, call_site, call_site + 1);
-  iter.next();
-  CompiledIC* c_ic = new CompiledIC(&iter);
-  c_ic->verify();
-  return c_ic;
+  return CompiledIC_at(cm, call_site);
 }
 
 CompiledIC* CompiledIC_at(RelocIterator* reloc_iter) {

src/hotspot/share/code/compiledIC.hpp

@@ -59,8 +59,9 @@ class CompiledICLocker: public StackObj {
 // (1) The first receiver klass and its selected method
 // (2) Itable call metadata
 
-class CompiledICData : public CHeapObj<mtCompiler> {
+class CompiledICData : public CHeapObj<mtCode> {
   friend class VMStructs;
+  friend class JVMCIVMStructs;
 
   Method*   volatile _speculated_method;
   uintptr_t volatile _speculated_klass;

src/hotspot/share/code/compiledMethod.cpp

@@ -407,19 +407,6 @@ void CompiledMethod::clear_inline_caches() {
   }
 }
 
-// Clear IC callsites
-// as well as any associated CompiledICHolders.
-void CompiledMethod::purge_ic_callsites() {
-  ResourceMark rm;
-  RelocIterator iter(this);
-  while(iter.next()) {
-    if (iter.type() == relocInfo::virtual_call_type) {
-      CompiledIC* ic = CompiledIC_at(&iter);
-      delete ic->data();
-    }
-  }
-}
-
 #ifdef ASSERT
 // Check class_loader is alive for this bit of metadata.
 class CheckClass : public MetadataClosure {

src/hotspot/share/code/compiledMethod.hpp

@@ -373,12 +373,10 @@ class CompiledMethod : public CodeBlob {
   void cleanup_inline_caches_whitebox();
 
   virtual void clear_inline_caches();
-  void purge_ic_callsites();
 
   // Execute nmethod barrier code, as if entering through nmethod call.
   void run_nmethod_entry_barrier();
 
-  // Verify and count cached icholder relocations.
   void verify_oop_relocations();
 
   bool has_evol_metadata();

src/hotspot/share/code/nmethod.cpp

@@ -640,6 +640,7 @@ nmethod::nmethod(
   ByteSize basic_lock_sp_offset,
   OopMapSet* oop_maps )
   : CompiledMethod(method, "native nmethod", type, nmethod_size, sizeof(nmethod), code_buffer, offsets->value(CodeOffsets::Frame_Complete), frame_size, oop_maps, false, true),
+  _compiled_ic_data(nullptr),
   _is_unlinked(false),
   _native_receiver_sp_offset(basic_lock_owner_sp_offset),
   _native_basic_lock_sp_offset(basic_lock_sp_offset),
@@ -784,6 +785,7 @@ nmethod::nmethod(
 #endif
   )
   : CompiledMethod(method, "nmethod", type, nmethod_size, sizeof(nmethod), code_buffer, offsets->value(CodeOffsets::Frame_Complete), frame_size, oop_maps, false, true),
+  _compiled_ic_data(nullptr),
   _is_unlinked(false),
   _native_receiver_sp_offset(in_ByteSize(-1)),
   _native_basic_lock_sp_offset(in_ByteSize(-1)),
@@ -1145,20 +1147,33 @@ static void install_post_call_nop_displacement(nmethod* nm, address pc) {
 void nmethod::finalize_relocations() {
   NoSafepointVerifier nsv;
 
+  GrowableArray<NativeMovConstReg*> virtual_call_data;
+
   // Make sure that post call nops fill in nmethod offsets eagerly so
   // we don't have to race with deoptimization
   RelocIterator iter(this);
   while (iter.next()) {
     if (iter.type() == relocInfo::virtual_call_type) {
       virtual_call_Relocation* r = iter.virtual_call_reloc();
       NativeMovConstReg* value = nativeMovConstReg_at(r->cached_value());
-      value->set_data((intptr_t)new CompiledICData());
+      virtual_call_data.append(value);
     } else if (iter.type() == relocInfo::post_call_nop_type) {
       post_call_nop_Relocation* const reloc = iter.post_call_nop_reloc();
       address pc = reloc->addr();
       install_post_call_nop_displacement(this, pc);
     }
   }
+
+  if (virtual_call_data.length() > 0) {
+    // We allocate a block of CompiledICData per nmethod so the GC can purge this faster.
+    _compiled_ic_data = new CompiledICData[virtual_call_data.length()];
+    CompiledICData* next_data = _compiled_ic_data;
+
+    for (NativeMovConstReg* value : virtual_call_data) {
+      value->set_data((intptr_t)next_data);
+      next_data++;
+    }
+  }
 }
 
 void nmethod::make_deoptimized() {
@@ -1464,7 +1479,7 @@ void nmethod::purge(bool free_code_cache_data, bool unregister_nmethod) {
     ec = next;
   }
 
-  purge_ic_callsites();
+  delete[] _compiled_ic_data;
 
   if (unregister_nmethod) {
     Universe::heap()->unregister_nmethod(this);

src/hotspot/share/code/nmethod.hpp

@@ -27,6 +27,7 @@
 
 #include "code/compiledMethod.hpp"
 
+class CompiledICData;
 class CompileTask;
 class DepChange;
 class DirectiveSet;
@@ -196,6 +197,7 @@ class nmethod : public CompiledMethod {
   address _verified_entry_point;             // entry point without class check
   address _osr_entry_point;                  // entry point for on stack replacement
 
+  CompiledICData* _compiled_ic_data;
   bool _is_unlinked;
 
   // Shared fields for all nmethod's

src/hotspot/share/code/vtableStubs.cpp

@@ -283,13 +283,6 @@ VtableStub* VtableStubs::entry_point(address pc) {
   return (s == stub) ? s : nullptr;
 }
 
-bool VtableStubs::is_icholder_entry(address pc) {
-  assert(contains(pc), "must contain all vtable blobs");
-  VtableStub* stub = (VtableStub*)(pc - VtableStub::entry_offset());
-  // itable stubs use CompiledICHolder.
-  return stub->is_itable_stub();
-}
-
 bool VtableStubs::contains(address pc) {
   // simple solution for now - we may want to use
   // a faster way if this function is called often

src/hotspot/share/code/vtableStubs.hpp

@@ -107,7 +107,6 @@ class VtableStubs : AllStatic {
   static address     find_itable_stub(int itable_index) { return find_stub(false, itable_index); }
 
   static VtableStub* entry_point(address pc);                        // vtable stub entry point for a pc
-  static bool        is_icholder_entry(address pc);                  // is the blob containing pc (which must be a vtable blob) an icholder?
   static bool        contains(address pc);                           // is pc within any stub?
   static VtableStub* stub_containing(address pc);                    // stub containing pc or nullptr
   static void        initialize();