8265298: Hard VM crash when deadlock between "access" and higher ranked lock is detected

Reviewed-by: pchilanomate, dholmes

Author: shipilev

src/hotspot/share/runtime/mutex.cpp

@@ -412,7 +412,12 @@ void Mutex::check_rank(Thread* thread) {
     // to acquire, then deadlock prevention rules require that the rank
     // of m2 be less than the rank of m1. This prevents circular waits.
     if (least != NULL && least->rank() <= this->rank()) {
-      thread->print_owned_locks();
+      if (least->rank() > Mutex::tty) {
+        // Printing owned locks acquires tty lock. If the least rank was below or equal
+        // tty, then deadlock detection code would circle back here, until we run
+        // out of stack and crash hard. Print locks only when it is safe.
+        thread->print_owned_locks();
+      }
       assert(false, "Attempting to acquire lock %s/%d out of order with lock %s/%d -- "
              "possible deadlock", this->name(), this->rank(), least->name(), least->rank());
     }

test/hotspot/gtest/runtime/test_mutex_rank.cpp

@@ -104,6 +104,36 @@ TEST_VM_ASSERT_MSG(MutexRank, mutex_trylock_rank_out_of_orderB,
   mutex_rankA->unlock();
 }
 
+TEST_VM_ASSERT_MSG(MutexRank, mutex_lock_access_leaf,
+                   ".* Attempting to acquire lock mutex_rank_leaf/11 out of order with lock mutex_rank_access/1 "
+                   "-- possible deadlock") {
+  JavaThread* THREAD = JavaThread::current();
+  ThreadInVMfromNative invm(THREAD);
+
+  Mutex* mutex_rank_access = new Mutex(Mutex::access, "mutex_rank_access", false, Mutex::_safepoint_check_never);
+  Mutex* mutex_rank_leaf = new Mutex(Mutex::leaf, "mutex_rank_leaf", false, Mutex::_safepoint_check_never);
+
+  mutex_rank_access->lock_without_safepoint_check();
+  mutex_rank_leaf->lock_without_safepoint_check();
+  mutex_rank_leaf->unlock();
+  mutex_rank_access->unlock();
+}
+
+TEST_VM_ASSERT_MSG(MutexRank, mutex_lock_tty_special,
+                   ".* Attempting to acquire lock mutex_rank_special/6 out of order with lock mutex_rank_tty/3 "
+                   "-- possible deadlock") {
+  JavaThread* THREAD = JavaThread::current();
+  ThreadInVMfromNative invm(THREAD);
+
+  Mutex* mutex_rank_tty = new Mutex(Mutex::tty, "mutex_rank_tty", false, Mutex::_safepoint_check_never);
+  Mutex* mutex_rank_special = new Mutex(Mutex::special, "mutex_rank_special", false, Mutex::_safepoint_check_never);
+
+  mutex_rank_tty->lock_without_safepoint_check();
+  mutex_rank_special->lock_without_safepoint_check();
+  mutex_rank_special->unlock();
+  mutex_rank_tty->unlock();
+}
+
 TEST_OTHER_VM(MutexRank, monitor_wait_rank_in_order) {
   JavaThread* THREAD = JavaThread::current();
   ThreadInVMfromNative invm(THREAD);
@@ -165,4 +195,36 @@ TEST_VM_ASSERT_MSG(MutexRank, monitor_wait_rank_special,
   monitor_rank_special_minus_one->unlock();
   monitor_rank_special->unlock();
 }
+
+TEST_VM_ASSERT_MSG(MutexRank, monitor_wait_access_leaf,
+                   ".* Attempting to wait on monitor monitor_rank_access/1 while holding lock monitor_rank_tty/3 "
+                   "-- possible deadlock. Should not block\\(wait\\) while holding a lock of rank special.") {
+  JavaThread* THREAD = JavaThread::current();
+  ThreadInVMfromNative invm(THREAD);
+
+  Monitor* monitor_rank_tty = new Monitor(Mutex::tty, "monitor_rank_tty", false, Mutex::_safepoint_check_never);
+  Monitor* monitor_rank_access = new Monitor(Mutex::access, "monitor_rank_access", false, Mutex::_safepoint_check_never);
+
+  monitor_rank_tty->lock_without_safepoint_check();
+  monitor_rank_access->lock_without_safepoint_check();
+  monitor_rank_access->wait_without_safepoint_check(1);
+  monitor_rank_access->unlock();
+  monitor_rank_tty->unlock();
+}
+
+TEST_VM_ASSERT_MSG(MutexRank, monitor_wait_tty_special,
+                   ".* Attempting to wait on monitor monitor_rank_tty/3 while holding lock monitor_rank_special/6 "
+                   "-- possible deadlock. Should not block\\(wait\\) while holding a lock of rank special.") {
+  JavaThread* THREAD = JavaThread::current();
+  ThreadInVMfromNative invm(THREAD);
+
+  Monitor* monitor_rank_special = new Monitor(Mutex::special, "monitor_rank_special", false, Mutex::_safepoint_check_never);
+  Monitor* monitor_rank_tty = new Monitor(Mutex::tty, "monitor_rank_tty", false, Mutex::_safepoint_check_never);
+
+  monitor_rank_special->lock_without_safepoint_check();
+  monitor_rank_tty->lock_without_safepoint_check();
+  monitor_rank_tty->wait_without_safepoint_check(1);
+  monitor_rank_tty->unlock();
+  monitor_rank_special->unlock();
+}
 #endif // ASSERT