Merge pull request #11323 from dmitry-ten/server-low-memory

Use fewer compilation threads at client when JITServer is low on memory

runtime/compiler/control/CompilationRuntime.hpp

@@ -1048,6 +1048,9 @@ class CompilationInfo
    void  addJITServerSslCert(const std::string &cert) { _sslCerts.push_back(cert); }
    const std::string &getJITServerSslRootCerts() const { return _sslRootCerts; }
    void  setJITServerSslRootCerts(const std::string &cert) { _sslRootCerts = cert; }
+
+   bool serverHasLowPhysicalMemory() const { return _serverHasLowPhysicalMemory; }
+   void setServerHasLowPhysicalMemory(bool isLowMemory) { _serverHasLowPhysicalMemory = isLowMemory; }
 #endif /* defined(J9VM_OPT_JITSERVER) */
 
    static void replenishInvocationCount(J9Method* method, TR::Compilation* comp);
@@ -1265,6 +1268,7 @@ class CompilationInfo
    std::string                   _sslRootCerts;
    PersistentVector<std::string> _sslKeys;
    PersistentVector<std::string> _sslCerts;
+   bool _serverHasLowPhysicalMemory;
 #endif /* defined(J9VM_OPT_JITSERVER) */
    }; // CompilationInfo
 }

runtime/compiler/control/CompilationThread.cpp

@@ -504,7 +504,16 @@ TR_YesNoMaybe TR::CompilationInfo::shouldActivateNewCompThread()
 #ifdef J9VM_OPT_JITSERVER
    // Always activate in JITServer server mode
    if (getPersistentInfo()->getRemoteCompilationMode() == JITServer::SERVER)
+      {
       return TR_yes;
+      }
+   else if (getPersistentInfo()->getRemoteCompilationMode() == JITServer::CLIENT &&
+            JITServerHelpers::isServerAvailable() &&
+            serverHasLowPhysicalMemory())
+      {
+      // If the available memory on the server is low, do not activate more client threads
+      return TR_no;
+      }
 #endif
 
    // Do not activate if we already exceed the CPU enablement for compilation threads
@@ -549,6 +558,7 @@ TR_YesNoMaybe TR::CompilationInfo::shouldActivateNewCompThread()
 #if defined(J9VM_OPT_JITSERVER)
       else if (getPersistentInfo()->getRemoteCompilationMode() == JITServer::CLIENT && JITServerHelpers::isServerAvailable())
          {
+
          // For JITClient let's be more agressive with compilation thread activation
          // because the latencies are larger. Beyond 'numProc-1' we will use the
          // 'starvation activation schedule', but accelerated (divide those thresholds by 2)
@@ -1164,6 +1174,7 @@ TR::CompilationInfo::CompilationInfo(J9JITConfig *jitConfig) :
    _compReqSeqNo = 0;
    _chTableUpdateFlags = 0;
    _localGCCounter = 0;
+   _serverHasLowPhysicalMemory = false;
 #endif /* defined(J9VM_OPT_JITSERVER) */
    }
 
@@ -3554,7 +3565,7 @@ void TR::CompilationInfo::stopCompilationThreads()
          }
       catch (const JITServer::StreamFailure &e)
          {
-         JITServerHelpers::postStreamFailure(OMRPORT_FROM_J9PORT(_jitConfig->javaVM->portLibrary));
+         JITServerHelpers::postStreamFailure(OMRPORT_FROM_J9PORT(_jitConfig->javaVM->portLibrary), this);
          // catch the stream failure exception if the server dies before the dummy message is send for termination.
          if (TR::Options::getVerboseOption(TR_VerboseJITServer))
             TR_VerboseLog::writeLineLocked(TR_Vlog_JITServer, "JITServer StreamFailure (server unreachable before the termination message was sent): %s", e.what());
@@ -4392,6 +4403,10 @@ TR::CompilationInfoPerThread::processEntry(TR_MethodToBeCompiled &entry, J9::J9S
             && TR::Options::getCmdLineOptions()->getOption(TR_SuspendEarly)
             && compInfo->getQueueWeight() < TR::CompilationInfo::getCompThreadSuspensionThreshold(compInfo->getNumCompThreadsActive())
             )
+#if defined(J9VM_OPT_JITSERVER)
+         || (compInfo->getPersistentInfo()->getRemoteCompilationMode() == JITServer::CLIENT
+            && compInfo->serverHasLowPhysicalMemory()) // keep suspending threads until server space frees up
+#endif
          )
       )
       {
@@ -4400,13 +4415,18 @@ TR::CompilationInfoPerThread::processEntry(TR_MethodToBeCompiled &entry, J9::J9S
       compInfo->decNumCompThreadsActive();
       if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerboseCompilationThreads))
          {
-         TR_VerboseLog::writeLineLocked(TR_Vlog_INFO, "t=%6u Suspend compThread %d Qweight=%d active=%d %s %s",
+         TR_VerboseLog::writeLineLocked(TR_Vlog_INFO, "t=%6u Suspend compThread %d Qweight=%d active=%d %s %s %s",
             (uint32_t)compInfo->getPersistentInfo()->getElapsedTime(),
             getCompThreadId(),
             compInfo->getQueueWeight(),
             compInfo->getNumCompThreadsActive(),
             compInfo->getRampDownMCT() ? "RampDownMCT" : "",
-            compInfo->getSuspendThreadDueToLowPhysicalMemory() ? "LowPhysicalMem" : "");
+            compInfo->getSuspendThreadDueToLowPhysicalMemory() ? "LowPhysicalMem" : "",
+#if defined(J9VM_OPT_JITSERVER)
+            compInfo->serverHasLowPhysicalMemory() ? "ServerLowPhysicalMem" :
+#endif
+            ""
+            );
          }
       // If the other remaining active thread(s) are sleeping (maybe because
       // we wanted to avoid two concurrent hot requests) we need to wake them

runtime/compiler/control/JITClientCompilationThread.cpp

@@ -3220,7 +3220,7 @@ remoteCompile(
          }
       catch (const JITServer::StreamFailure &e)
          {
-         JITServerHelpers::postStreamFailure(OMRPORT_FROM_J9PORT(compInfoPT->getJitConfig()->javaVM->portLibrary));
+         JITServerHelpers::postStreamFailure(OMRPORT_FROM_J9PORT(compInfoPT->getJitConfig()->javaVM->portLibrary), compInfo);
          if (TR::Options::isAnyVerboseOptionSet(TR_VerboseJITServer, TR_VerboseCompilationDispatch))
             TR_VerboseLog::writeLineLocked(TR_Vlog_FAILURE,
                "JITServer::StreamFailure: %s for %s @ %s", e.what(), compiler->signature(), compiler->getHotnessName());
@@ -3322,7 +3322,7 @@ remoteCompile(
          {
          auto recv = client->getRecvData<std::string, std::string, CHTableCommitData, std::vector<TR_OpaqueClassBlock*>,
                                          std::string, std::string, std::vector<TR_ResolvedJ9Method*>,
-                                         TR_OptimizationPlan, std::vector<SerializedRuntimeAssumption>>();
+                                         TR_OptimizationPlan, std::vector<SerializedRuntimeAssumption>, bool>();
          statusCode = compilationOK;
          codeCacheStr = std::get<0>(recv);
          dataCacheStr = std::get<1>(recv);
@@ -3333,6 +3333,8 @@ remoteCompile(
          resolvedMirrorMethodsPersistIPInfo = std::get<6>(recv);
          modifiedOptPlan = std::get<7>(recv);
          serializedRuntimeAssumptions = std::get<8>(recv);
+
+         compInfoPT->getCompilationInfo()->setServerHasLowPhysicalMemory(std::get<9>(recv));
          }
       else
          {
@@ -3355,7 +3357,7 @@ remoteCompile(
       }
    catch (const JITServer::StreamFailure &e)
       {
-      JITServerHelpers::postStreamFailure(OMRPORT_FROM_J9PORT(compInfoPT->getJitConfig()->javaVM->portLibrary));
+      JITServerHelpers::postStreamFailure(OMRPORT_FROM_J9PORT(compInfoPT->getJitConfig()->javaVM->portLibrary), compInfo);
 
       client->~ClientStream();
       TR_Memory::jitPersistentFree(client);

runtime/compiler/control/JITServerCompilationThread.cpp

@@ -103,13 +103,20 @@ outOfProcessCompilationEnd(
       }
 
    auto resolvedMirrorMethodsPersistIPInfo = compInfoPT->getCachedResolvedMirrorMethodsPersistIPInfo();
+
+   // If the server is running low on memory, tell clients to reduce the number of active compilation threads
+   bool incompleteInfo;
+   uint64_t freePhysicalMemorySizeB = compInfoPT->getCompilationInfo()->computeAndCacheFreePhysicalMemory(incompleteInfo);
+   bool serverHasLowMemory = (freePhysicalMemorySizeB != OMRPORT_MEMINFO_NOT_AVAILABLE &&
+       freePhysicalMemorySizeB <= (uint64_t)TR::Options::getSafeReservePhysicalMemoryValue() + TR::Options::getScratchSpaceLowerBound());
+
    entry->_stream->finishCompilation(codeCacheStr, dataCacheStr, chTableData,
                                      std::vector<TR_OpaqueClassBlock*>(classesThatShouldNotBeNewlyExtended->begin(), classesThatShouldNotBeNewlyExtended->end()),
                                      logFileStr, svmSymbolToIdStr,
                                      (resolvedMirrorMethodsPersistIPInfo) ?
                                                          std::vector<TR_ResolvedJ9Method*>(resolvedMirrorMethodsPersistIPInfo->begin(), resolvedMirrorMethodsPersistIPInfo->end()) :
                                                          std::vector<TR_ResolvedJ9Method*>(),
-                                     *entry->_optimizationPlan, serializedRuntimeAssumptions
+                                     *entry->_optimizationPlan, serializedRuntimeAssumptions, serverHasLowMemory
                                      );
    compInfoPT->clearPerCompilationCaches();
 

runtime/compiler/control/JITServerHelpers.cpp

@@ -613,7 +613,7 @@ JITServerHelpers::romMethodOfRamMethod(J9Method* method)
    }
 
 void
-JITServerHelpers::postStreamFailure(OMRPortLibrary *portLibrary)
+JITServerHelpers::postStreamFailure(OMRPortLibrary *portLibrary, TR::CompilationInfo *compInfo)
    {
    OMR::CriticalSection postStreamFailure(getClientStreamMonitor());
 
@@ -625,6 +625,10 @@ JITServerHelpers::postStreamFailure(OMRPortLibrary *portLibrary)
       }
    _nextConnectionRetryTime = current_time + _waitTimeMs;
    _serverAvailable = false;
+
+   // Reset the low memory flag in case we never reconnect to the server
+   // and client compiles locally or connects to a new server
+   compInfo->setServerHasLowPhysicalMemory(false);
    }
 
 void

runtime/compiler/control/JITServerHelpers.hpp

@@ -89,7 +89,7 @@ class JITServerHelpers
 
    // Functions used for allowing the client to compile locally when server is unavailable.
    // Should be used only on the client side.
-   static void postStreamFailure(OMRPortLibrary *portLibrary);
+   static void postStreamFailure(OMRPortLibrary *portLibrary, TR::CompilationInfo *compInfo);
    static bool shouldRetryConnection(OMRPortLibrary *portLibrary);
    static void postStreamConnectionSuccess();
    static bool isServerAvailable() { return _serverAvailable; }