8311981: Test gc/stringdedup/TestStringDeduplicationAgeThreshold.java#ZGenerational timed out

Backport-of: f142470deaebbf9cff7f6f28842972393506b7c4

Author: David Holmes

src/hotspot/share/runtime/handshake.cpp

@@ -497,8 +497,17 @@ HandshakeOperation* HandshakeState::get_op_for_self(bool allow_suspend, bool che
 }
 
 bool HandshakeState::has_operation(bool allow_suspend, bool check_async_exception) {
-  MutexLocker ml(&_lock, Mutex::_no_safepoint_check_flag);
-  return get_op_for_self(allow_suspend, check_async_exception) != nullptr;
+  // We must not block here as that could lead to deadlocks if we already hold an
+  // "external" mutex. If the try_lock fails then we assume that there is an operation
+  // and force the caller to check more carefully in a safer context. If we can't get
+  // the lock it means another thread is trying to handshake with us, so it can't
+  // happen during thread termination and destruction.
+  bool ret = true;
+  if (_lock.try_lock()) {
+    ret = get_op_for_self(allow_suspend, check_async_exception) != nullptr;
+    _lock.unlock();
+  }
+  return ret;
 }
 
 bool HandshakeState::has_async_exception_operation() {