Merge pull request #16394 from mpirvu/connmessage

Improve client JVM disconnect messages
eclipse-openj9 · Dec 5, 2022 · 9c79e90 · 9c79e90
2 parents 4fb37ad + ef6c16e
commit 9c79e90
Show file tree

Hide file tree

Showing 4 changed files with 47 additions and 30 deletions.
diff --git a/runtime/compiler/control/CompilationThread.cpp b/runtime/compiler/control/CompilationThread.cpp
@@ -3771,7 +3771,7 @@ void TR::CompilationInfo::stopCompilationThreads()
          }
       catch (const JITServer::StreamFailure &e)
          {
-         JITServerHelpers::postStreamFailure(OMRPORT_FROM_J9PORT(_jitConfig->javaVM->portLibrary), this, e.retryConnectionImmediately());
+         JITServerHelpers::postStreamFailure(OMRPORT_FROM_J9PORT(_jitConfig->javaVM->portLibrary), this, e.retryConnectionImmediately(), true);
          // catch the stream failure exception if the server dies before the dummy message is send for termination.
          if (TR::Options::getVerboseOption(TR_VerboseJITServer))
             TR_VerboseLog::writeLineLocked(TR_Vlog_JITServer, "JITServer StreamFailure (server unreachable before the termination message was sent): %s", e.what());

diff --git a/runtime/compiler/control/JITClientCompilationThread.cpp b/runtime/compiler/control/JITClientCompilationThread.cpp
@@ -3090,12 +3090,13 @@ remoteCompile(J9VMThread *vmThread, TR::Compilation *compiler, TR_ResolvedMethod
          }
       catch (const JITServer::StreamFailure &e)
          {
-         JITServerHelpers::postStreamFailure(OMRPORT_FROM_J9PORT(compInfoPT->getJitConfig()->javaVM->portLibrary), compInfo,
-                                             e.retryConnectionImmediately());
          if (TR::Options::isAnyVerboseOptionSet(TR_VerboseJITServer, TR_VerboseCompilationDispatch))
             TR_VerboseLog::writeLineLocked(TR_Vlog_FAILURE,
                "JITServer::StreamFailure: %s for %s @ %s", e.what(), compiler->signature(), compiler->getHotnessName());
 
+         JITServerHelpers::postStreamFailure(OMRPORT_FROM_J9PORT(compInfoPT->getJitConfig()->javaVM->portLibrary), compInfo,
+                                             e.retryConnectionImmediately(), true);
+
          Trc_JITServerStreamFailure(vmThread, compInfoPT->getCompThreadId(), __FUNCTION__,
                compiler->signature(), compiler->getHotnessName(), e.what());
 
@@ -3357,7 +3358,7 @@ remoteCompile(J9VMThread *vmThread, TR::Compilation *compiler, TR_ResolvedMethod
             }
 
          // Since server has crashed, all compilations will switch to local
-         JITServerHelpers::postStreamFailure(OMRPORT_FROM_J9PORT(compInfoPT->getJitConfig()->javaVM->portLibrary), compInfo, false);
+         JITServerHelpers::postStreamFailure(OMRPORT_FROM_J9PORT(compInfoPT->getJitConfig()->javaVM->portLibrary), compInfo, false, false);
          entry->_compErrCode = compilationFailure;
          compiler->failCompilation<JITServer::ServerCompilationFailure>("JITServer compilation thread has crashed.");
          }
@@ -3387,8 +3388,12 @@ remoteCompile(J9VMThread *vmThread, TR::Compilation *compiler, TR_ResolvedMethod
       }
    catch (const JITServer::StreamFailure &e)
       {
+      if (TR::Options::isAnyVerboseOptionSet(TR_VerboseJITServer, TR_VerboseCompilationDispatch))
+          TR_VerboseLog::writeLineLocked(TR_Vlog_FAILURE,
+            "JITServer::StreamFailure: %s for %s @ %s", e.what(), compiler->signature(), compiler->getHotnessName());
+
       JITServerHelpers::postStreamFailure(OMRPORT_FROM_J9PORT(compInfoPT->getJitConfig()->javaVM->portLibrary), compInfo,
-                                          e.retryConnectionImmediately());
+                                          e.retryConnectionImmediately(), false);
 
       if (!details.isJitDumpMethod())
          {
@@ -3397,10 +3402,6 @@ remoteCompile(J9VMThread *vmThread, TR::Compilation *compiler, TR_ResolvedMethod
          compInfoPT->setClientStream(NULL);
          }
 
-      if (TR::Options::isAnyVerboseOptionSet(TR_VerboseJITServer, TR_VerboseCompilationDispatch))
-          TR_VerboseLog::writeLineLocked(TR_Vlog_FAILURE,
-            "JITServer::StreamFailure: %s for %s @ %s", e.what(), compiler->signature(), compiler->getHotnessName());
-
       Trc_JITServerStreamFailure(vmThread, compInfoPT->getCompThreadId(), __FUNCTION__,
                compiler->signature(), compiler->getHotnessName(), e.what());
 

diff --git a/runtime/compiler/control/JITServerHelpers.cpp b/runtime/compiler/control/JITServerHelpers.cpp
@@ -896,7 +896,7 @@ JITServerHelpers::romMethodOfRamMethod(J9Method* method)
    }
 
 void
-JITServerHelpers::postStreamFailure(OMRPortLibrary *portLibrary, TR::CompilationInfo *compInfo, bool retryConnectionImmediately)
+JITServerHelpers::postStreamFailure(OMRPortLibrary *portLibrary, TR::CompilationInfo *compInfo, bool retryConnectionImmediately, bool connectionFailure)
    {
    OMR::CriticalSection postStreamFailure(getClientStreamMonitor());
 
@@ -915,28 +915,44 @@ JITServerHelpers::postStreamFailure(OMRPortLibrary *portLibrary, TR::Compilation
       _nextConnectionRetryTime = current_time + _waitTimeMs;
       }
 
-
-   if (_serverAvailable && TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerboseJITServerConns))
+   // If this is a connection failure, set the global flag _serverAvailable to false
+   // so that other threads can see it and avoid trying too often.
+   // If there was another networking problem (read/write error), the client stream
+   // will be deleted, the socket will be closed and the client will be forced to
+   // reconnect again. The client will discover if the server is down during the connection attempt.
+   if (connectionFailure && !retryConnectionImmediately)
       {
-      TR_VerboseLog::writeLineLocked(TR_Vlog_JITServer,
-                                     "t=%6u Lost connection to the server (serverUID=%llu). Retry immediately: %d.",
-                                     (uint32_t)compInfo->getPersistentInfo()->getElapsedTime(),
-                                     (unsigned long long)compInfo->getPersistentInfo()->getServerUID(),
-                                     retryConnectionImmediately);
+      if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerboseJITServerConns))
+         {
+         if (compInfo->getPersistentInfo()->getServerUID() != 0)
+            {
+            TR_VerboseLog::writeLineLocked(TR_Vlog_JITServer,
+                                           "t=%6u Lost connection to the server (serverUID=%llu). Next attempt in %llu ms.",
+                                           (uint32_t)compInfo->getPersistentInfo()->getElapsedTime(),
+                                           (unsigned long long)compInfo->getPersistentInfo()->getServerUID(),
+                                           _waitTimeMs);
+            }
+         else // Was not connected to a server
+            {
+            TR_VerboseLog::writeLineLocked(TR_Vlog_JITServer,
+                                           "t=%6u Could not connect to a server. Next attempt in %llu ms.",
+                                           (uint32_t)compInfo->getPersistentInfo()->getElapsedTime(),
+                                           _waitTimeMs);
+            }
+         }
       compInfo->getPersistentInfo()->setServerUID(0);
-      }
-
-   _serverAvailable = false;
+      _serverAvailable = false;
 
-   // Reset the activation policy flag in case we never reconnect to the server
-   // and client compiles locally or connects to a new server
-   compInfo->setCompThreadActivationPolicy(JITServer::CompThreadActivationPolicy::AGGRESSIVE);
-   if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerboseCompilationThreads) ||
-       TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerboseJITServer))
-      {
-      TR_VerboseLog::writeLineLocked(TR_Vlog_JITServer,
-                                     "t=%6u client has lost connection, resetting activation policy to AGGRESSIVE",
-                                     (uint32_t)compInfo->getPersistentInfo()->getElapsedTime());
+      // Reset the activation policy flag in case we never reconnect to the server
+      // and client compiles locally or connects to a new server
+      compInfo->setCompThreadActivationPolicy(JITServer::CompThreadActivationPolicy::AGGRESSIVE);
+      if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerboseCompilationThreads) ||
+          TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerboseJITServer))
+         {
+         TR_VerboseLog::writeLineLocked(TR_Vlog_JITServer,
+                                        "t=%6u Resetting activation policy to AGGRESSIVE because client has lost connection to server",
+                                        (uint32_t)compInfo->getPersistentInfo()->getElapsedTime());
+         }
       }
    }
 

diff --git a/runtime/compiler/control/JITServerHelpers.hpp b/runtime/compiler/control/JITServerHelpers.hpp
@@ -120,7 +120,7 @@ class JITServerHelpers
 
    // Functions used for allowing the client to compile locally when server is unavailable.
    // Should be used only on the client side.
-   static void postStreamFailure(OMRPortLibrary *portLibrary, TR::CompilationInfo *compInfo, bool retryConnectionImmediately);
+   static void postStreamFailure(OMRPortLibrary *portLibrary, TR::CompilationInfo *compInfo, bool retryConnectionImmediately, bool connectionFailure);
    static bool shouldRetryConnection(OMRPortLibrary *portLibrary);
    static void postStreamConnectionSuccess();
    static bool isServerAvailable() { return _serverAvailable; }