8319797: Recursive lightweight locking: Runtime implementation

Reviewed-by: shade
Backport-of: 5dbf13730ee2b57f089c57e9e7ee8ab65d4a67af

Author: rkennke

src/hotspot/share/prims/whitebox.cpp

@@ -78,6 +78,7 @@
 #include "runtime/interfaceSupport.inline.hpp"
 #include "runtime/javaCalls.hpp"
 #include "runtime/jniHandles.inline.hpp"
+#include "runtime/lockStack.hpp"
 #include "runtime/os.hpp"
 #include "runtime/stackFrameStream.inline.hpp"
 #include "runtime/sweeper.hpp"
@@ -1754,6 +1755,14 @@ WB_ENTRY(jboolean, WB_IsMonitorInflated(JNIEnv* env, jobject wb, jobject obj))
   return (jboolean) obj_oop->mark().has_monitor();
 WB_END
 
+WB_ENTRY(jint, WB_getLockStackCapacity(JNIEnv* env))
+  return (jint) LockStack::CAPACITY;
+WB_END
+
+WB_ENTRY(jboolean, WB_supportsRecursiveLightweightLocking(JNIEnv* env))
+  return (jboolean) VM_Version::supports_recursive_lightweight_locking();
+WB_END
+
 WB_ENTRY(jboolean, WB_DeflateIdleMonitors(JNIEnv* env, jobject wb))
   log_info(monitorinflation)("WhiteBox initiated DeflateIdleMonitors");
   return ObjectSynchronizer::request_deflate_idle_monitors();
@@ -2549,6 +2558,8 @@ static JNINativeMethod methods[] = {
                                                       (void*)&WB_AddModuleExportsToAll },
   {CC"deflateIdleMonitors", CC"()Z",                  (void*)&WB_DeflateIdleMonitors },
   {CC"isMonitorInflated0", CC"(Ljava/lang/Object;)Z", (void*)&WB_IsMonitorInflated  },
+  {CC"getLockStackCapacity", CC"()I",                 (void*)&WB_getLockStackCapacity },
+  {CC"supportsRecursiveLightweightLocking", CC"()Z",  (void*)&WB_supportsRecursiveLightweightLocking },
   {CC"forceSafepoint",     CC"()V",                   (void*)&WB_ForceSafepoint     },
   {CC"getConstantPool0",   CC"(Ljava/lang/Class;)J",  (void*)&WB_GetConstantPool    },
   {CC"getConstantPoolCacheIndexTag0", CC"()I",  (void*)&WB_GetConstantPoolCacheIndexTag},

src/hotspot/share/runtime/abstract_vm_version.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -190,6 +190,9 @@ class Abstract_VM_Version: AllStatic {
   // Does platform support stack watermark barriers for concurrent stack processing?
   constexpr static bool supports_stack_watermark_barrier() { return false; }
 
+  // Is recursive lightweight locking implemented for this platform?
+  constexpr static bool supports_recursive_lightweight_locking() { return false; }
+
   static bool print_matching_lines_from_file(const char* filename, outputStream* st, const char* keywords_to_match[]);
 };
 

src/hotspot/share/runtime/lockStack.cpp

@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2022, Red Hat, Inc. All rights reserved.
  * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,20 +26,30 @@
 
 #include "precompiled.hpp"
 #include "memory/allocation.hpp"
+#include "runtime/globals.hpp"
 #include "runtime/lockStack.inline.hpp"
 #include "runtime/safepoint.hpp"
 #include "runtime/stackWatermark.hpp"
 #include "runtime/stackWatermarkSet.inline.hpp"
 #include "runtime/thread.hpp"
 #include "utilities/copy.hpp"
+#include "utilities/debug.hpp"
+#include "utilities/globalDefinitions.hpp"
 #include "utilities/ostream.hpp"
 
+#include <type_traits>
+
 const int LockStack::lock_stack_offset =      in_bytes(JavaThread::lock_stack_offset());
 const int LockStack::lock_stack_top_offset =  in_bytes(JavaThread::lock_stack_top_offset());
 const int LockStack::lock_stack_base_offset = in_bytes(JavaThread::lock_stack_base_offset());
 
 LockStack::LockStack(JavaThread* jt) :
   _top(lock_stack_base_offset), _base() {
+  // Make sure the layout of the object is compatible with the emitted code's assumptions.
+  STATIC_ASSERT(sizeof(_bad_oop_sentinel) == oopSize);
+  STATIC_ASSERT(sizeof(_base[0]) == oopSize);
+  STATIC_ASSERT(std::is_standard_layout<LockStack>::value);
+  STATIC_ASSERT(offsetof(LockStack, _bad_oop_sentinel) == offsetof(LockStack, _base) - oopSize);
 #ifdef ASSERT
   for (int i = 0; i < CAPACITY; i++) {
     _base[i] = NULL;
@@ -62,11 +73,21 @@ uint32_t LockStack::end_offset() {
 void LockStack::verify(const char* msg) const {
   assert(LockingMode == LM_LIGHTWEIGHT, "never use lock-stack when light weight locking is disabled");
   assert((_top <= end_offset()), "lockstack overflow: _top %d end_offset %d", _top, end_offset());
-  assert((_top >= start_offset()), "lockstack underflow: _top %d end_offset %d", _top, start_offset());
+  assert((_top >= start_offset()), "lockstack underflow: _top %d start_offset %d", _top, start_offset());
   if (SafepointSynchronize::is_at_safepoint() || (Thread::current()->is_Java_thread() && is_owning_thread())) {
     int top = to_index(_top);
     for (int i = 0; i < top; i++) {
       assert(_base[i] != NULL, "no zapped before top");
+      if (VM_Version::supports_recursive_lightweight_locking()) {
+        oop o = _base[i];
+        for (; i < top - 1; i++) {
+          // Consecutive entries may be the same
+          if (_base[i + 1] != o) {
+            break;
+          }
+        }
+      }
+
       for (int j = i + 1; j < top; j++) {
         assert(_base[i] != _base[j], "entries must be unique: %s", msg);
       }

src/hotspot/share/runtime/lockStack.hpp

@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2022, Red Hat, Inc. All rights reserved.
  * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -35,10 +36,12 @@ class OopClosure;
 class outputStream;
 
 class LockStack {
+  friend class LockStackTest;
   friend class VMStructs;
   JVMCI_ONLY(friend class JVMCIVMStructs;)
-private:
+public:
   static const int CAPACITY = 8;
+private:
 
   // TODO: It would be very useful if JavaThread::lock_stack_offset() and friends were constexpr,
   // but this is currently not the case because we're using offset_of() which is non-constexpr,
@@ -51,6 +54,9 @@ class LockStack {
   // We do this instead of a simple index into the array because this allows for
   // efficient addressing in generated code.
   uint32_t _top;
+  // The _bad_oop_sentinel acts as a sentinel value to elide underflow checks in generated code.
+  // The correct layout is statically asserted in the constructor.
+  const uintptr_t _bad_oop_sentinel = badOopVal;
   oop _base[CAPACITY];
 
   // Get the owning thread of this lock-stack.
@@ -75,14 +81,35 @@ class LockStack {
   static uint32_t start_offset();
   static uint32_t end_offset();
 
-  // Return true if we have room to push onto this lock-stack, false otherwise.
-  inline bool can_push() const;
+  // Returns true if the lock-stack is full. False otherwise.
+  inline bool is_full() const;
 
   // Pushes an oop on this lock-stack.
   inline void push(oop o);
 
+  // Get the oldest oop from this lock-stack.
+  // Precondition: This lock-stack must not be empty.
+  inline oop bottom() const;
+
+  // Is the lock-stack empty.
+  inline bool is_empty() const;
+
+  // Check if object is recursive.
+  // Precondition: This lock-stack must contain the oop.
+  inline bool is_recursive(oop o) const;
+
+  // Try recursive enter.
+  // Precondition: This lock-stack must not be full.
+  inline bool try_recursive_enter(oop o);
+
+  // Try recursive exit.
+  // Precondition: This lock-stack must contain the oop.
+  inline bool try_recursive_exit(oop o);
+
   // Removes an oop from an arbitrary location of this lock-stack.
-  inline void remove(oop o);
+  // Precondition: This lock-stack must contain the oop.
+  // Returns the number of oops removed.
+  inline size_t remove(oop o);
 
   // Tests whether the oop is on this lock-stack.
   inline bool contains(oop o) const;

src/hotspot/share/runtime/lockStack.inline.hpp

@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2022, Red Hat, Inc. All rights reserved.
  * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,14 +27,21 @@
 #ifndef SHARE_RUNTIME_LOCKSTACK_INLINE_HPP
 #define SHARE_RUNTIME_LOCKSTACK_INLINE_HPP
 
-#include "memory/iterator.hpp"
 #include "runtime/lockStack.hpp"
+
+#include "memory/iterator.hpp"
 #include "runtime/safepoint.hpp"
 #include "runtime/stackWatermark.hpp"
 #include "runtime/stackWatermarkSet.inline.hpp"
 #include "runtime/thread.hpp"
+#include "runtime/vm_version.hpp"
+#include "utilities/align.hpp"
+#include "utilities/globalDefinitions.hpp"
 
 inline int LockStack::to_index(uint32_t offset) {
+  assert(is_aligned(offset, oopSize), "Bad alignment: %u", offset);
+  assert((offset <= end_offset()), "lockstack overflow: offset %d end_offset %d", offset, end_offset());
+  assert((offset >= start_offset()), "lockstack underflow: offset %d start_offset %d", offset, start_offset());
   return (offset - lock_stack_base_offset) / oopSize;
 }
 
@@ -42,8 +50,8 @@ JavaThread* LockStack::get_thread() const {
   return reinterpret_cast<JavaThread*>(addr - lock_stack_offset);
 }
 
-inline bool LockStack::can_push() const {
-  return to_index(_top) < CAPACITY;
+inline bool LockStack::is_full() const {
+  return to_index(_top) == CAPACITY;
 }
 
 inline bool LockStack::is_owning_thread() const {
@@ -61,32 +69,132 @@ inline void LockStack::push(oop o) {
   verify("pre-push");
   assert(oopDesc::is_oop(o), "must be");
   assert(!contains(o), "entries must be unique");
-  assert(can_push(), "must have room");
+  assert(!is_full(), "must have room");
   assert(_base[to_index(_top)] == NULL, "expect zapped entry");
   _base[to_index(_top)] = o;
   _top += oopSize;
   verify("post-push");
 }
 
-inline void LockStack::remove(oop o) {
+inline oop LockStack::bottom() const {
+  assert(to_index(_top) > 0, "must contain an oop");
+  return _base[0];
+}
+
+inline bool LockStack::is_empty() const {
+  return to_index(_top) == 0;
+}
+
+inline bool LockStack::is_recursive(oop o) const {
+  if (!VM_Version::supports_recursive_lightweight_locking()) {
+    return false;
+  }
+  verify("pre-is_recursive");
+
+  // This will succeed iff there is a consecutive run of oops on the
+  // lock-stack with a length of at least 2.
+
+  assert(contains(o), "at least one entry must exist");
+  int end = to_index(_top);
+  // Start iterating from the top because the runtime code is more
+  // interested in the balanced locking case when the top oop on the
+  // lock-stack matches o. This will cause the for loop to break out
+  // in the first loop iteration if it is non-recursive.
+  for (int i = end - 1; i > 0; i--) {
+    if (_base[i - 1] == o && _base[i] == o) {
+      verify("post-is_recursive");
+      return true;
+    }
+    if (_base[i] == o) {
+      // o can only occur in one consecutive run on the lock-stack.
+      // Only one of the two oops checked matched o, so this run
+      // must be of length 1 and thus not be recursive. Stop the search.
+      break;
+    }
+  }
+
+  verify("post-is_recursive");
+  return false;
+}
+
+inline bool LockStack::try_recursive_enter(oop o) {
+  if (!VM_Version::supports_recursive_lightweight_locking()) {
+    return false;
+  }
+  verify("pre-try_recursive_enter");
+
+  // This will succeed iff the top oop on the stack matches o.
+  // When successful o will be pushed to the lock-stack creating
+  // a consecutive run at least 2 oops that matches o on top of
+  // the lock-stack.
+
+  assert(!is_full(), "precond");
+
+  int end = to_index(_top);
+  if (end == 0 || _base[end - 1] != o) {
+    // Topmost oop does not match o.
+    verify("post-try_recursive_enter");
+    return false;
+  }
+
+  _base[end] = o;
+  _top += oopSize;
+  verify("post-try_recursive_enter");
+  return true;
+}
+
+inline bool LockStack::try_recursive_exit(oop o) {
+  if (!VM_Version::supports_recursive_lightweight_locking()) {
+    return false;
+  }
+  verify("pre-try_recursive_exit");
+
+  // This will succeed iff the top two oops on the stack matches o.
+  // When successful the top oop will be popped of the lock-stack.
+  // When unsuccessful the lock may still be recursive, in which
+  // case the locking is unbalanced. This case is handled externally.
+
+  assert(contains(o), "entries must exist");
+
+  int end = to_index(_top);
+  if (end <= 1 || _base[end - 1] != o || _base[end - 2] != o) {
+    // The two topmost oops do not match o.
+    verify("post-try_recursive_exit");
+    return false;
+  }
+
+  _top -= oopSize;
+  DEBUG_ONLY(_base[to_index(_top)] = nullptr;)
+  verify("post-try_recursive_exit");
+  return true;
+}
+
+inline size_t LockStack::remove(oop o) {
   verify("pre-remove");
   assert(contains(o), "entry must be present: " PTR_FORMAT, p2i(o));
+
   int end = to_index(_top);
+  int inserted = 0;
   for (int i = 0; i < end; i++) {
-    if (_base[i] == o) {
-      int last = end - 1;
-      for (; i < last; i++) {
-        _base[i] = _base[i + 1];
+    if (_base[i] != o) {
+      if (inserted != i) {
+        _base[inserted] = _base[i];
       }
-      _top -= oopSize;
-#ifdef ASSERT
-      _base[to_index(_top)] = NULL;
-#endif
-      break;
+      inserted++;
     }
   }
-  assert(!contains(o), "entries must be unique: " PTR_FORMAT, p2i(o));
+
+#ifdef ASSERT
+  for (int i = inserted; i < end; i++) {
+    _base[i] = nullptr;
+  }
+#endif
+
+  uint32_t removed = end - inserted;
+  _top -= removed * oopSize;
+  assert(!contains(o), "entry must have been removed: " PTR_FORMAT, p2i(o));
   verify("post-remove");
+  return removed;
 }
 
 inline bool LockStack::contains(oop o) const {

src/hotspot/share/runtime/objectMonitor.hpp

@@ -308,6 +308,7 @@ class ObjectMonitor : public CHeapObj<mtObjectMonitor> {
   jint      contentions() const;
   void      add_to_contentions(jint value);
   intx      recursions() const                                         { return _recursions; }
+  void      set_recursions(size_t recursions);
 
   // JVM/TI GetObjectMonitorUsage() needs this:
   ObjectWaiter* first_waiter()                                         { return _WaitSet; }

src/hotspot/share/runtime/objectMonitor.inline.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -102,6 +102,12 @@ inline void ObjectMonitor::add_to_contentions(jint value) {
   Atomic::add(&_contentions, value);
 }
 
+inline void ObjectMonitor::set_recursions(size_t recursions) {
+  assert(_recursions == 0, "must be");
+  assert(has_owner(), "must be owned");
+  _recursions = checked_cast<intx>(recursions);
+}
+
 // Clear _owner field; current value must match old_value.
 inline void ObjectMonitor::release_clear_owner(void* old_value) {
 #ifdef ASSERT