8252543: [JVMCI] Libgraal can deadlock in blocking compilation mode

Reviewed-by: kvn

src/hotspot/share/compiler/compileBroker.cpp

@@ -249,11 +249,6 @@ bool compileBroker_init() {
 CompileTaskWrapper::CompileTaskWrapper(CompileTask* task) {
   CompilerThread* thread = CompilerThread::current();
   thread->set_task(task);
-#if INCLUDE_JVMCI
-  if (task->is_blocking() && CompileBroker::compiler(task->comp_level())->is_jvmci()) {
-    task->set_jvmci_compiler_thread(thread);
-  }
-#endif
   CompileLog*     log  = thread->log();
   if (log != NULL && !task->is_unloaded())  task->log_task_start(log);
 }
@@ -277,7 +272,7 @@ CompileTaskWrapper::~CompileTaskWrapper() {
           // The waiting thread timed out and thus did not free the task.
           free_task = true;
         }
-        task->set_jvmci_compiler_thread(NULL);
+        task->set_blocking_jvmci_compile_state(NULL);
       }
 #endif
       if (!free_task) {
@@ -1604,36 +1599,39 @@ bool CompileBroker::wait_for_jvmci_completion(JVMCICompiler* jvmci, CompileTask*
   assert(UseJVMCICompiler, "sanity");
   MonitorLocker ml(thread, task->lock());
   int progress_wait_attempts = 0;
-  int methods_compiled = jvmci->methods_compiled();
+  jint thread_jvmci_compilation_ticks = 0;
+  jint global_jvmci_compilation_ticks = jvmci->global_compilation_ticks();
   while (!task->is_complete() && !is_compilation_disabled_forever() &&
          ml.wait(JVMCI_COMPILATION_PROGRESS_WAIT_TIMESLICE)) {
-    CompilerThread* jvmci_compiler_thread = task->jvmci_compiler_thread();
+    JVMCICompileState* jvmci_compile_state = task->blocking_jvmci_compile_state();
 
     bool progress;
-    if (jvmci_compiler_thread != NULL) {
-      // If the JVMCI compiler thread is not blocked or suspended, we deem it to be making progress.
-      progress = jvmci_compiler_thread->thread_state() != _thread_blocked &&
-        !jvmci_compiler_thread->is_external_suspend();
+    if (jvmci_compile_state != NULL) {
+      jint ticks = jvmci_compile_state->compilation_ticks();
+      progress = (ticks - thread_jvmci_compilation_ticks) != 0;
+      JVMCI_event_1("waiting on compilation %d [ticks=%d]", task->compile_id(), ticks);
+      thread_jvmci_compilation_ticks = ticks;
     } else {
       // Still waiting on JVMCI compiler queue. This thread may be holding a lock
-      // that all JVMCI compiler threads are blocked on. We use the counter for
-      // successful JVMCI compilations to determine whether JVMCI compilation
+      // that all JVMCI compiler threads are blocked on. We use the global JVMCI
+      // compilation ticks to determine whether JVMCI compilation
       // is still making progress through the JVMCI compiler queue.
-      progress = jvmci->methods_compiled() != methods_compiled;
+      jint ticks = jvmci->global_compilation_ticks();
+      progress = (ticks - global_jvmci_compilation_ticks) != 0;
+      JVMCI_event_1("waiting on compilation %d to be queued [ticks=%d]", task->compile_id(), ticks);
+      global_jvmci_compilation_ticks = ticks;
     }
 
     if (!progress) {
       if (++progress_wait_attempts == JVMCI_COMPILATION_PROGRESS_WAIT_ATTEMPTS) {
         if (PrintCompilation) {
           task->print(tty, "wait for blocking compilation timed out");
         }
+        JVMCI_event_1("waiting on compilation %d timed out", task->compile_id());
         break;
       }
     } else {
       progress_wait_attempts = 0;
-      if (jvmci_compiler_thread == NULL) {
-        methods_compiled = jvmci->methods_compiled();
-      }
     }
   }
   task->clear_waiter();
@@ -2152,7 +2150,7 @@ void CompileBroker::invoke_compiler_on_method(CompileTask* task) {
 
     TraceTime t1("compilation", &time);
     EventCompilation event;
-    JVMCICompileState compile_state(task);
+    JVMCICompileState compile_state(task, jvmci);
     JVMCIRuntime *runtime = NULL;
 
     if (JVMCI::in_shutdown()) {

src/hotspot/share/compiler/compileTask.cpp

@@ -100,7 +100,7 @@ void CompileTask::initialize(int compile_id,
   _osr_bci = osr_bci;
   _is_blocking = is_blocking;
   JVMCI_ONLY(_has_waiter = CompileBroker::compiler(comp_level)->is_jvmci();)
-  JVMCI_ONLY(_jvmci_compiler_thread = NULL;)
+  JVMCI_ONLY(_blocking_jvmci_compile_state = NULL;)
   _comp_level = comp_level;
   _num_inlined_bytecodes = 0;
 

src/hotspot/share/compiler/compileTask.hpp

@@ -31,6 +31,8 @@
 #include "memory/allocation.hpp"
 #include "utilities/xmlstream.hpp"
 
+JVMCI_ONLY(class JVMCICompileState;)
+
 // CompileTask
 //
 // An entry in the compile queue.  It represents a pending or current
@@ -81,8 +83,8 @@ class CompileTask : public CHeapObj<mtCompiler> {
   bool         _is_blocking;
 #if INCLUDE_JVMCI
   bool         _has_waiter;
-  // Compiler thread for a blocking JVMCI compilation
-  CompilerThread* _jvmci_compiler_thread;
+  // Compilation state for a blocking JVMCI compilation
+  JVMCICompileState* _blocking_jvmci_compile_state;
 #endif
   int          _comp_level;
   int          _num_inlined_bytecodes;
@@ -144,11 +146,9 @@ class CompileTask : public CHeapObj<mtCompiler> {
 
   bool         has_waiter() const                { return _has_waiter; }
   void         clear_waiter()                    { _has_waiter = false; }
-  CompilerThread* jvmci_compiler_thread() const  { return _jvmci_compiler_thread; }
-  void         set_jvmci_compiler_thread(CompilerThread* t) {
-    assert(is_blocking(), "must be");
-    assert((t == NULL) != (_jvmci_compiler_thread == NULL), "must be");
-    _jvmci_compiler_thread = t;
+  JVMCICompileState* blocking_jvmci_compile_state() const { return _blocking_jvmci_compile_state; }
+  void         set_blocking_jvmci_compile_state(JVMCICompileState* state) {
+    _blocking_jvmci_compile_state = state;
   }
 #endif
 

src/hotspot/share/jvmci/jvmci.cpp

@@ -23,9 +23,11 @@
 
 #include "precompiled.hpp"
 #include "classfile/systemDictionary.hpp"
+#include "compiler/compileTask.hpp"
 #include "gc/shared/collectedHeap.hpp"
 #include "jvmci/jvmci.hpp"
 #include "jvmci/jvmciJavaClasses.hpp"
+#include "jvmci/jvmciEnv.hpp"
 #include "jvmci/jvmciRuntime.hpp"
 #include "jvmci/metadataHandles.hpp"
 #include "memory/resourceArea.hpp"
@@ -123,6 +125,18 @@ void JVMCI::initialize_globals() {
   }
 }
 
+JavaThread* JVMCI::compilation_tick(JavaThread* thread) {
+  if (thread->is_Compiler_thread()) {
+    CompileTask *task = thread->as_CompilerThread()->task();
+    if (task != NULL) {
+      JVMCICompileState *state = task->blocking_jvmci_compile_state();
+      if (state != NULL) {
+        state->inc_compilation_ticks();
+      }
+    }
+  }
+  return thread;
+}
 
 void JVMCI::metadata_do(void f(Metadata*)) {
   if (_java_runtime != NULL) {

src/hotspot/share/jvmci/jvmci.hpp

@@ -110,6 +110,11 @@ class JVMCI : public AllStatic {
 
   static void initialize_compiler(TRAPS);
 
+  // Increments a value indicating some JVMCI compilation activity
+  // happened on `thread` if it is a CompilerThread.
+  // Returns `thread`.
+  static JavaThread* compilation_tick(JavaThread* thread);
+
   static JVMCIRuntime* compiler_runtime() { return _compiler_runtime; }
   // Gets the single runtime for JVMCI on the Java heap. This is the only
   // JVMCI runtime available when !UseJVMCINativeLibrary.

src/hotspot/share/jvmci/jvmciCompiler.cpp

@@ -37,6 +37,7 @@ JVMCICompiler::JVMCICompiler() : AbstractCompiler(compiler_jvmci) {
   _bootstrapping = false;
   _bootstrap_compilation_request_handled = false;
   _methods_compiled = 0;
+  _global_compilation_ticks = 0;
   assert(_instance == NULL, "only one instance allowed");
   _instance = this;
 }
@@ -154,3 +155,12 @@ void JVMCICompiler::print_compilation_timers() {
     tty->print_cr("    JVMCI code install time:        %6.3f s", code_install_time);
   }
 }
+
+void JVMCICompiler::inc_methods_compiled() {
+  Atomic::inc(&_methods_compiled);
+  Atomic::inc(&_global_compilation_ticks);
+}
+
+void JVMCICompiler::inc_global_compilation_ticks() {
+  Atomic::inc(&_global_compilation_ticks);
+}

src/hotspot/share/jvmci/jvmciCompiler.hpp

@@ -42,6 +42,10 @@ class JVMCICompiler : public AbstractCompiler {
    */
   volatile int _methods_compiled;
 
+  // Incremented periodically by JVMCI compiler threads
+  // to indicate JVMCI compilation activity.
+  volatile int _global_compilation_ticks;
+
   static JVMCICompiler* _instance;
 
   static elapsedTimer _codeInstallTimer;
@@ -99,15 +103,16 @@ class JVMCICompiler : public AbstractCompiler {
   // Print compilation timers and statistics
   virtual void print_timers();
 
-  /**
-   * Gets the number of methods that have been successfully compiled by
-   * a call to JVMCICompiler::compile_method().
-   */
+  // Gets the number of methods that have been successfully compiled by
+  // a call to JVMCICompiler::compile_method().
   int methods_compiled() { return _methods_compiled; }
+  void inc_methods_compiled();
 
-  void inc_methods_compiled() {
-    Atomic::inc(&_methods_compiled);
-  }
+  // Gets a value indicating JVMCI compilation activity on any thread.
+  // If successive calls to this method return a different value, then
+  // some degree of JVMCI compilation occurred between the calls.
+  int global_compilation_ticks() const { return _global_compilation_ticks; }
+  void inc_global_compilation_ticks();
 
   // Print compilation timers and statistics
   static void print_compilation_timers();

src/hotspot/share/jvmci/jvmciCompilerToVM.cpp

@@ -132,7 +132,7 @@ Handle JavaArgumentUnboxer::next_arg(BasicType expectedType) {
   TRACE_CALL(result_type, jvmci_ ## name signature)  \
   JVMCI_VM_ENTRY_MARK;                               \
   ResourceMark rm;                                   \
-  JNI_JVMCIENV(thread, env);
+  JNI_JVMCIENV(JVMCI::compilation_tick(thread), env);
 
 static JavaThread* get_current_thread(bool allow_null=true) {
   Thread* thread = Thread::current_or_null_safe();