8293989: [JVMCI] re-use cleared oop handles

Reviewed-by: never

Author: Doug Simon

src/hotspot/share/jvmci/jvmciCompilerToVM.cpp

@@ -2208,11 +2208,9 @@ C2V_VMENTRY_0(jint, arrayIndexScale, (JNIEnv* env, jobject, jchar type_char))
   return type2aelembytes(type);
 C2V_END
 
-C2V_VMENTRY(void, deleteGlobalHandle, (JNIEnv* env, jobject, jlong handle))
-  if (handle != 0) {
-    JVMCIENV->runtime()->destroy_oop_handle(handle);
-  }
-}
+C2V_VMENTRY(void, releaseClearedOopHandles, (JNIEnv* env, jobject))
+  JVMCIENV->runtime()->release_cleared_oop_handles();
+C2V_END
 
 static void requireJVMCINativeLibrary(JVMCI_TRAPS) {
   if (!UseJVMCINativeLibrary) {
@@ -2903,7 +2901,7 @@ JNINativeMethod CompilerToVM::methods[] = {
   {CC "readArrayElement",                             CC "(" OBJECTCONSTANT "I)Ljava/lang/Object;",                                         FN_PTR(readArrayElement)},
   {CC "arrayBaseOffset",                              CC "(C)I",                                                                            FN_PTR(arrayBaseOffset)},
   {CC "arrayIndexScale",                              CC "(C)I",                                                                            FN_PTR(arrayIndexScale)},
-  {CC "deleteGlobalHandle",                           CC "(J)V",                                                                            FN_PTR(deleteGlobalHandle)},
+  {CC "releaseClearedOopHandles",                     CC "()V",                                                                             FN_PTR(releaseClearedOopHandles)},
   {CC "registerNativeMethods",                        CC "(" CLASS ")[J",                                                                   FN_PTR(registerNativeMethods)},
   {CC "isCurrentThreadAttached",                      CC "()Z",                                                                             FN_PTR(isCurrentThreadAttached)},
   {CC "getCurrentJavaThread",                         CC "()J",                                                                             FN_PTR(getCurrentJavaThread)},

src/hotspot/share/jvmci/jvmciRuntime.cpp

@@ -843,94 +843,107 @@ static OopStorage* object_handles() {
 jlong JVMCIRuntime::make_oop_handle(const Handle& obj) {
   assert(!Universe::heap()->is_gc_active(), "can't extend the root set during GC");
   assert(oopDesc::is_oop(obj()), "not an oop");
-  oop* ptr = object_handles()->allocate();
-  jlong res = 0;
-  if (ptr != nullptr) {
-    assert(*ptr == nullptr, "invariant");
-    NativeAccess<>::oop_store(ptr, obj());
-    res = (jlong) ptr;
-  } else {
-    vm_exit_out_of_memory(sizeof(oop), OOM_MALLOC_ERROR,
-                          "Cannot create JVMCI oop handle");
-  }
+
+  oop* ptr = OopHandle(object_handles(), obj()).ptr_raw();
   MutexLocker ml(_lock);
   _oop_handles.append(ptr);
-  return res;
+  return (jlong) ptr;
 }
 
-bool JVMCIRuntime::probe_oop_handle(jlong handle, int index) {
-  oop* key = (oop*) handle;
-  if (key == _oop_handles.at(index)) {
-    _last_found_oop_handle_index = index;
-    return true;
+int JVMCIRuntime::release_and_clear_oop_handles() {
+  guarantee(_num_attached_threads == cannot_be_attached, "only call during JVMCI runtime shutdown");
+  int released = release_cleared_oop_handles();
+  if (_oop_handles.length() != 0) {
+    for (int i = 0; i < _oop_handles.length(); i++) {
+      oop* oop_ptr = _oop_handles.at(i);
+      guarantee(oop_ptr != nullptr, "release_cleared_oop_handles left null entry in _oop_handles");
+      guarantee(*oop_ptr != nullptr, "unexpected cleared handle");
+      // Satisfy OopHandles::release precondition that all
+      // handles being released are null.
+      NativeAccess<>::oop_store(oop_ptr, (oop) NULL);
+    }
+
+    // Do the bulk release
+    object_handles()->release(_oop_handles.adr_at(0), _oop_handles.length());
+    released += _oop_handles.length();
   }
-  return false;
+  _oop_handles.clear();
+  return released;
 }
 
-int JVMCIRuntime::find_oop_handle(jlong handle) {
-  int len = _oop_handles.length();
-  int next = _last_found_oop_handle_index + 1;
-  int prev = MAX2(_last_found_oop_handle_index, 0) - 1;
+static bool is_referent_non_null(oop* handle) {
+  return handle != nullptr && *handle != nullptr;
+}
 
-  // Search "outwards" from the index of the last found
-  // entry. Experimentation shows that this significantly
-  // reduces the amount of searching performed.
-  do {
-    if (next < len) {
-      if (probe_oop_handle(handle, next)) {
-        return next;
-      }
-      next++;
-    }
-    if (prev >= 0) {
-      if (probe_oop_handle(handle, prev)) {
-        return prev;
-      }
-      prev--;
-    }
-  } while (next - (prev + 1) < len);
-  return -1;
+// Swaps the elements in `array` at index `a` and index `b`
+static void swap(GrowableArray<oop*>* array, int a, int b) {
+  oop* tmp = array->at(a);
+  array->at_put(a, array->at(b));
+  array->at_put(b, tmp);
 }
 
-int JVMCIRuntime::release_and_clear_globals() {
-  int released = 0;
+int JVMCIRuntime::release_cleared_oop_handles() {
+  // Despite this lock, it's possible for another thread
+  // to clear a handle's referent concurrently (e.g., a thread
+  // executing IndirectHotSpotObjectConstantImpl.clear()).
+  // This is benign - it means there can still be cleared
+  // handles in _oop_handles when this method returns.
+  MutexLocker ml(_lock);
+
+  int next = 0;
   if (_oop_handles.length() != 0) {
-    // Squash non-null JNI handles to front of _oop_handles for
-    // the bulk release operation
+    // Key for _oop_handles contents in example below:
+    //   H: handle with non-null referent
+    //   h: handle with clear (i.e., null) referent
+    //   -: null entry
+
+    // Shuffle all handles with non-null referents to the front of the list
+    // Example: Before: 0HHh-Hh-
+    //           After: HHHh--h-
     for (int i = 0; i < _oop_handles.length(); i++) {
-      oop* oop_ptr = _oop_handles.at(i);
-      if (oop_ptr != nullptr) {
-        // Satisfy OopHandles::release precondition that all
-        // handles being released are null.
-        NativeAccess<>::oop_store(oop_ptr, (oop) NULL);
+      oop* handle = _oop_handles.at(i);
+      if (is_referent_non_null(handle)) {
+        if (i != next && !is_referent_non_null(_oop_handles.at(next))) {
+          // Swap elements at index `next` and `i`
+          swap(&_oop_handles, next, i);
+        }
+        next++;
+      }
+    }
 
-        _oop_handles.at_put(released++, oop_ptr);
+    // `next` is now the index of the first null handle or handle with a null referent
+    int num_alive = next;
+
+    // Shuffle all null handles to the end of the list
+    // Example: Before: HHHh--h-
+    //           After: HHHhh---
+    //       num_alive: 3
+    for (int i = next; i < _oop_handles.length(); i++) {
+      oop* handle = _oop_handles.at(i);
+      if (handle != nullptr) {
+        if (i != next && _oop_handles.at(next) == nullptr) {
+          // Swap elements at index `next` and `i`
+          swap(&_oop_handles, next, i);
+        }
+        next++;
       }
     }
-    // Do the bulk release
-    object_handles()->release(_oop_handles.adr_at(0), released);
-  }
-  _oop_handles.clear();
-  _last_found_oop_handle_index = -1;
-  return released;
-}
+    int to_release = next - num_alive;
 
-void JVMCIRuntime::destroy_oop_handle(jlong handle) {
-  // Assert before nulling out, for better debugging.
-  assert(is_oop_handle(handle), "precondition");
-  oop* oop_ptr = (oop*) handle;
-  NativeAccess<>::oop_store(oop_ptr, (oop) nullptr);
-  object_handles()->release(oop_ptr);
+    // `next` is now the index of the first null handle
+    // Example: to_release: 2
 
-  MutexLocker ml(_lock);
-  int index = find_oop_handle(handle);
-  guarantee(index != -1, "global not allocated in JVMCI runtime %d: " INTPTR_FORMAT, id(), handle);
-  _oop_handles.at_put(index, nullptr);
-}
+    // Bulk release the handles with a null referent
+    object_handles()->release(_oop_handles.adr_at(num_alive), to_release);
 
-bool JVMCIRuntime::is_oop_handle(jlong handle) {
-  const oop* ptr = (oop*) handle;
-  return object_handles()->allocation_status(ptr) == OopStorage::ALLOCATED_ENTRY;
+    // Truncate oop handles to only those with a non-null referent
+    JVMCI_event_1("compacted oop handles in JVMCI runtime %d from %d to %d", _id, _oop_handles.length(), num_alive);
+    _oop_handles.trunc_to(num_alive);
+    // Example: HHH
+
+    return to_release;
+  }
+  return 0;
 }
 
 jmetadata JVMCIRuntime::allocate_handle(const methodHandle& handle) {
@@ -988,8 +1001,7 @@ JVMCIRuntime::JVMCIRuntime(JVMCIRuntime* next, int id, bool for_compile_broker)
   _metadata_handles(new MetadataHandles()),
   _oop_handles(100, mtJVMCI),
   _num_attached_threads(0),
-  _for_compile_broker(for_compile_broker),
-  _last_found_oop_handle_index(-1)
+  _for_compile_broker(for_compile_broker)
 {
   if (id == -1) {
     _lock = JVMCIRuntime_lock;
@@ -1169,7 +1181,7 @@ bool JVMCIRuntime::detach_thread(JavaThread* thread, const char* reason, bool ca
         // that could be using them. Handles for the Java JVMCI runtime
         // are never released as we cannot guarantee all compiler threads
         // using it have been stopped.
-        int released = release_and_clear_globals();
+        int released = release_and_clear_oop_handles();
         JVMCI_event_1("releasing handles for JVMCI runtime %d: oop handles=%d, metadata handles={total=%d, live=%d, blocks=%d}",
             _id,
             released,

src/hotspot/share/jvmci/jvmciRuntime.hpp

@@ -223,14 +223,9 @@ class JVMCIRuntime: public CHeapObj<mtJVMCI> {
   // JVMCI_lock must be held by current thread
   static JVMCIRuntime* select_runtime_in_shutdown(JavaThread* thread);
 
-  // Helpers for destroy_oop_handle
-  int _last_found_oop_handle_index;
-  bool probe_oop_handle(jlong handle, int index);
-  int find_oop_handle(jlong handle);
-
   // Releases all the non-null entries in _oop_handles and then clears
-  // the list. Returns the number of non-null entries prior to clearing.
-  int release_and_clear_globals();
+  // the list. Returns the number released handles.
+  int release_and_clear_oop_handles();
 
  public:
   JVMCIRuntime(JVMCIRuntime* next, int id, bool for_compile_broker);
@@ -277,10 +272,12 @@ class JVMCIRuntime: public CHeapObj<mtJVMCI> {
   // used when creating an IndirectHotSpotObjectConstantImpl in the
   // shared library JavaVM.
   jlong make_oop_handle(const Handle& obj);
-  bool is_oop_handle(jlong handle);
 
-  // Called from IndirectHotSpotObjectConstantImpl.clear(Object)
-  void destroy_oop_handle(jlong handle);
+  // Releases all the non-null entries in _oop_handles whose referent is null.
+  // Returns the number of handles released by this call.
+  // The method also resets _last_found_oop_handle_index to -1
+  // and _null_oop_handles to 0.
+  int release_cleared_oop_handles();
 
   // Allocation and management of metadata handles.
   jmetadata allocate_handle(const methodHandle& handle);

src/jdk.internal.vm.ci/share/classes/jdk.vm.ci.hotspot/src/jdk/vm/ci/hotspot/Cleaner.java

@@ -97,19 +97,29 @@ private static synchronized void remove(Cleaner cl) {
 
     /**
      * Performs the cleanup action now that this object's referent has become weakly reachable.
+     *
+     * @returns true if the clean up action cleared the referent of an oop handle and requires a
+     *          subsequent call to {@link CompilerToVM#releaseClearedOopHandles()} to reclaim the
+     *          resources of the handle itself
      */
-    abstract void doCleanup();
+    abstract boolean doCleanup();
 
     /**
      * Remove the cleaners whose referents have become weakly reachable.
      */
     static void clean() {
         Cleaner c = (Cleaner) queue.poll();
+        boolean oopHandleCleared = false;
         while (c != null) {
             remove(c);
-            c.doCleanup();
+            if (c.doCleanup()) {
+                oopHandleCleared = true;
+            }
             c = (Cleaner) queue.poll();
         }
+        if (oopHandleCleared) {
+            CompilerToVM.compilerToVM().releaseClearedOopHandles();
+        }
     }
 
     /**

src/jdk.internal.vm.ci/share/classes/jdk.vm.ci.hotspot/src/jdk/vm/ci/hotspot/CompilerToVM.java

@@ -1175,10 +1175,9 @@ boolean isTrustedForIntrinsics(HotSpotResolvedObjectTypeImpl klass) {
     native boolean isTrustedForIntrinsics(HotSpotResolvedObjectTypeImpl klass, long klassPointer);
 
     /**
-     * Releases the resources backing the global JNI {@code handle}. This is equivalent to the
-     * {@code DeleteGlobalRef} JNI function.
+     * Releases all oop handles whose referent is null.
      */
-    native void deleteGlobalHandle(long handle);
+    native void releaseClearedOopHandles();
 
     /**
      * Gets the failed speculations pointed to by {@code *failedSpeculationsAddress}.

src/jdk.internal.vm.ci/share/classes/jdk.vm.ci.hotspot/src/jdk/vm/ci/hotspot/HandleCleaner.java

@@ -57,18 +57,17 @@ private HandleCleaner(Object wrapper, long handle, boolean isJObject) {
      * Releases the resource associated with {@code this.handle}.
      */
     @Override
-    void doCleanup() {
+    boolean doCleanup() {
         if (isJObject) {
-            // The sentinel value used to denote a free handle is
-            // an object on the HotSpot heap so we call into the
-            // VM to set the target of an object handle to this value.
-            CompilerToVM.compilerToVM().deleteGlobalHandle(handle);
+            IndirectHotSpotObjectConstantImpl.clearHandle(handle);
+            return true;
         } else {
             // Setting the target of a jmetadata handle to 0 enables
             // the handle to be reused. See MetadataHandles in
             // metadataHandles.hpp for more info.
             long value = UNSAFE.getLong(null, handle);
             UNSAFE.compareAndSetLong(null, handle, value, 0);
+            return false;
         }
     }
 

src/jdk.internal.vm.ci/share/classes/jdk.vm.ci.hotspot/src/jdk/vm/ci/hotspot/HotSpotObjectConstantScope.java

@@ -115,6 +115,7 @@ public void close() {
                 obj.clear(localScopeDescription);
             }
             foreignObjects = null;
+            CompilerToVM.compilerToVM().releaseClearedOopHandles();
         }
         CURRENT.set(parent);
     }

src/jdk.internal.vm.ci/share/classes/jdk.vm.ci.hotspot/src/jdk/vm/ci/hotspot/HotSpotSpeculationLog.java

@@ -354,12 +354,13 @@ private static final class LogCleaner extends Cleaner {
         }
 
         @Override
-        void doCleanup() {
+        boolean doCleanup() {
             long pointer = UnsafeAccess.UNSAFE.getAddress(address);
             if (pointer != 0) {
                 compilerToVM().releaseFailedSpeculations(address);
             }
             UnsafeAccess.UNSAFE.freeMemory(address);
+            return false;
         }
 
         final long address;

src/jdk.internal.vm.ci/share/classes/jdk.vm.ci.hotspot/src/jdk/vm/ci/hotspot/IndirectHotSpotObjectConstantImpl.java

@@ -23,6 +23,7 @@
 package jdk.vm.ci.hotspot;
 
 import static jdk.vm.ci.hotspot.HotSpotJVMCIRuntime.runtime;
+import static jdk.vm.ci.hotspot.UnsafeAccess.UNSAFE;
 
 import java.io.ByteArrayOutputStream;
 import java.io.PrintStream;
@@ -144,13 +145,22 @@ public HotSpotResolvedObjectType getType() {
      */
     void clear(Object scopeDescription) {
         checkHandle();
-        CompilerToVM.compilerToVM().deleteGlobalHandle(objectHandle);
         if (rawAudit == null) {
             rawAudit = scopeDescription;
         }
+
+        clearHandle(objectHandle);
         objectHandle = 0L;
     }
 
+    /**
+     * Sets the referent of {@code handle} to 0 so that it will be reclaimed when calling
+     * {@link CompilerToVM#releaseClearedOopHandles}.
+     */
+    static void clearHandle(long handle) {
+        UNSAFE.putLong(handle, 0);
+    }
+
     @Override
     public JavaConstant compress() {
         assert !compressed;