Disclaim Cold Code Cache

- use the same heuristcs for code cache disclaim as for data cache
- disclaim starting from the cold code
- move stack overflow outline instructions into the warm area to
  increase disclaim efficiency
- code is enabled with -Xjit:enableCodeCacheDisclaiming

runtime/compiler/control/HookedByTheJit.cpp

@@ -4610,10 +4610,22 @@ void disclaimIProfilerSegments(uint64_t crtElapsedTime)
       }
    }
 
+void disclaimCodeCaches(uint64_t crtElapsedTime)
+   {
+   size_t rssBefore = getRSS_Kb();
+   int numDisclaimed = TR::CodeCacheManager::instance()->disclaimAllCodeCaches();
+   size_t rssAfter = getRSS_Kb();
+   if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerbosePerformance))
+      TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "t=%u JIT disclaimed %d Code Caches RSS before=%zu KB, RSS after=%zu KB, delta=%zu KB = %5.2f%%",
+                                     (uint32_t)crtElapsedTime, numDisclaimed, rssBefore, rssAfter, rssBefore - rssAfter, ((long)(rssAfter - rssBefore) * 100.0 / rssBefore));
+   }
+
 void memoryDisclaimLogic(TR::CompilationInfo *compInfo, uint64_t crtElapsedTime, uint8_t jitState)
    {
    static uint64_t lastDataCacheDisclaimTime = 0;
    static int32_t  lastNumAllocatedDataCaches = 0;
+   static uint64_t lastCodeCacheDisclaimTime = 0;
+   static int32_t  lastNumAllocatedCodeCaches = 0;
    static uint64_t lastIProfilerDisclaimTime = 0;
    static uint32_t lastNumCompilationsDuringIProfilerDisclaim = 0;
 
@@ -4644,6 +4656,25 @@ void memoryDisclaimLogic(TR::CompilationInfo *compInfo, uint64_t crtElapsedTime,
             }
          }
       }
+
+   // Use logic similar to Data caches above
+   if (TR::CodeCacheManager::instance()->isDisclaimEnabled())
+      {
+      // Ensure we don't do it too often
+      if (crtElapsedTime > lastCodeCacheDisclaimTime + TR::Options::_minTimeBetweenMemoryDisclaims)
+         {
+         // Disclaim if at least one code cache has been allocated since the last disclaim
+         // or if there was a large time interval since the last disclaim
+         if (TR::CodeCacheManager::instance()->getCurrentNumberOfCodeCaches() > lastNumAllocatedCodeCaches ||
+             crtElapsedTime > lastCodeCacheDisclaimTime + 12 * TR::Options::_minTimeBetweenMemoryDisclaims)
+            {
+            disclaimCodeCaches(crtElapsedTime);
+            lastCodeCacheDisclaimTime = crtElapsedTime; // Update the time when disclaim was last performed
+            lastNumAllocatedCodeCaches = TR::CodeCacheManager::instance()->getCurrentNumberOfCodeCaches();
+            }
+         }
+      }
+
 #if defined(J9VM_INTERP_PROFILING_BYTECODES)
    if (!TR::Options::getCmdLineOptions()->getOption(TR_DisableInterpreterProfiling))
       {
@@ -5344,6 +5375,17 @@ static void jitStateLogic(J9JITConfig * jitConfig, TR::CompilationInfo * compInf
          }
       }
 
+#ifdef DEBUG_CODE_DISCLAIM
+   static int printRSS = 0;
+   printRSS++;
+   if (printRSS == 4 &&   // ~every 2s
+       TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerbosePerformance))
+      {
+      TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "Current RSS %zuKB", getRSS_Kb());
+      printRSS = 0;
+      }
+#endif
+
    if (lateDisclaimNeeded)
       {
       CpuUtilization *cpuUtil = compInfo->getCpuUtil();

runtime/compiler/control/J9Options.cpp

@@ -2736,6 +2736,7 @@ J9::Options::fePreProcess(void * base)
       {
       self()->setOption(TR_DisableDataCacheDisclaiming);
       self()->setOption(TR_DisableIProfilerDataDisclaiming);
+      self()->setOption(TR_EnableCodeCacheDisclaiming, false);
       }
 
    return true;
@@ -2900,12 +2901,28 @@ J9::Options::fePostProcessJIT(void * base)
       }
 
    if (!self()->getOption(TR_DisableDataCacheDisclaiming) ||
-       !self()->getOption(TR_DisableIProfilerDataDisclaiming))
+       !self()->getOption(TR_DisableIProfilerDataDisclaiming) ||
+       self()->getOption(TR_EnableCodeCacheDisclaiming))
       {
       // Check requirements for memory disclaiming (Linux kernel and default page size)
       TR::Options::disableMemoryDisclaimIfNeeded(jitConfig);
       }
 
+   const char *ccOption = J9::Options::_externalOptionStrings[J9::ExternalOptions::Xcodecache];
+   J9JavaVM *vm = javaVM; // needed by FIND_ARG_IN_VMARGS macro
+   int32_t argIndex = FIND_ARG_IN_VMARGS(EXACT_MEMORY_MATCH, ccOption, 0);
+
+   if (argIndex >= 0)
+      {
+      if (jitConfig->codeCacheKB < 4*1024*1024)
+         self()->setOption(TR_EnableCodeCacheDisclaiming, false);
+      }
+   else if (TR::Compiler->target.isLinux() &&
+            self()->getOption(TR_EnableCodeCacheDisclaiming))
+      {
+      jitConfig->codeCacheKB *= 2;
+      }
+
 #if defined(J9VM_OPT_JITSERVER)
    self()->setupJITServerOptions();
 #endif /* defined(J9VM_OPT_JITSERVER) */
@@ -2957,6 +2974,7 @@ J9::Options::disableMemoryDisclaimIfNeeded(J9JITConfig *jitConfig)
       {
       TR::Options::getCmdLineOptions()->setOption(TR_DisableDataCacheDisclaiming);
       TR::Options::getCmdLineOptions()->setOption(TR_DisableIProfilerDataDisclaiming);
+      TR::Options::getCmdLineOptions()->setOption(TR_EnableCodeCacheDisclaiming, false);
       }
    return shouldDisableMemoryDisclaim;
    }

runtime/compiler/control/OptionsPostRestore.cpp

@@ -90,6 +90,8 @@ J9::OptionsPostRestore::OptionsPostRestore(J9VMThread *vmThread, J9JITConfig *ji
       = J9::Options::_xrsSync
         || options->getOption(TR_NoResumableTrapHandler)
         || options->getOption(TR_DisableTraps);
+
+   _enableCodeCacheDisclaimingPreCheckpoint = options->getOption(TR_EnableCodeCacheDisclaiming);
    }
 
 void
@@ -804,8 +806,19 @@ J9::OptionsPostRestore::postProcessInternalCompilerOptions()
          }
       }
 
+
+   if (!_enableCodeCacheDisclaimingPreCheckpoint &&
+       TR::Options::getCmdLineOptions()->getOption(TR_EnableCodeCacheDisclaiming))
+      {
+      TR::Options::getCmdLineOptions()->setOption(TR_EnableCodeCacheDisclaiming, false);
+
+      if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerbosePerformance))
+          TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "WARNING: Code Cache disclaiming disabled since it was disabled before checkpoint");
+      }
+
    if (!TR::Options::getCmdLineOptions()->getOption(TR_DisableDataCacheDisclaiming) ||
-       !TR::Options::getCmdLineOptions()->getOption(TR_DisableIProfilerDataDisclaiming))
+       !TR::Options::getCmdLineOptions()->getOption(TR_DisableIProfilerDataDisclaiming) ||
+       TR::Options::getCmdLineOptions()->getOption(TR_EnableCodeCacheDisclaiming))
       {
       TR::Options::disableMemoryDisclaimIfNeeded(_jitConfig);
       }

runtime/compiler/control/OptionsPostRestore.hpp

@@ -164,6 +164,7 @@ class OptionsPostRestore
    bool _asyncCompilationPreCheckpoint;
    bool _disableTrapsPreCheckpoint;
    bool _disableAOTPostRestore;
+   bool _enableCodeCacheDisclaimingPreCheckpoint;
 
    int32_t _argIndexXjit;
    int32_t _argIndexXjitcolon;

runtime/compiler/env/J2IThunk.cpp

@@ -62,7 +62,8 @@ TR_MHJ2IThunk::allocate(
    else
 #endif /* defined(J9VM_OPT_JITSERVER) */
       {
-      result = (TR_MHJ2IThunk*)cg->allocateCodeMemory(totalSize, true, false);
+      bool disclaim = TR::Options::getCmdLineOptions()->getOption(TR_EnableCodeCacheDisclaiming);
+      result = (TR_MHJ2IThunk*)cg->allocateCodeMemory(totalSize, !disclaim, false);
       }
    omrthread_jit_write_protect_disable();
    result->_codeSize  = codeSize;

runtime/compiler/runtime/CRRuntime.cpp

@@ -654,9 +654,10 @@ TR::CRRuntime::prepareForCheckpoint()
 #endif
 
    // Make sure the limit for the ghost files is at least as big as the data cache size
-   if (!TR::Options::getCmdLineOptions()->getOption(TR_DisableDataCacheDisclaiming))
+   if (!TR::Options::getCmdLineOptions()->getOption(TR_DisableDataCacheDisclaiming) ||
+       TR::Options::getCmdLineOptions()->getOption(TR_EnableCodeCacheDisclaiming))
       {
-      U_32 ghostFileLimit = vm->jitConfig->dataCacheKB * 1024; // convert to bytes
+      U_32 ghostFileLimit = std::max(vm->jitConfig->dataCacheKB, vm->jitConfig->codeCacheTotalKB) * 1024; // convert to bytes
       vm->internalVMFunctions->setRequiredGhostFileLimit(vmThread, ghostFileLimit);
       }
 

runtime/compiler/runtime/J9CodeCache.cpp

@@ -27,6 +27,7 @@
 #include <algorithm>
 #include <stdlib.h>
 #include <string.h>
+#include <errno.h>
 #include "j9.h"
 #include "j9protos.h"
 #include "j9thread.h"
@@ -55,11 +56,21 @@
 #include "env/VerboseLog.hpp"
 #include "omrformatconsts.h"
 
+// for madvise
+#ifdef LINUX
+#include <sys/mman.h>
+#ifndef MADV_NOHUGEPAGE
+#define MADV_NOHUGEPAGE  15
+#endif // MADV_NOHUGEPAGE
+#ifndef MADV_PAGEOUT
+#define MADV_PAGEOUT     21
+#endif // MADV_PAGEOUT
+#endif
+
 OMR::CodeCacheMethodHeader *getCodeCacheMethodHeader(char *p, int searchLimit, J9JITExceptionTable * metaData);
 
 #define addFreeBlock2(start, end) addFreeBlock2WithCallSite((start), (end), __FILE__, __LINE__)
 
-
 TR::CodeCache *
 J9::CodeCache::self()
    {
@@ -134,8 +145,48 @@ J9::CodeCache::initialize(TR::CodeCacheManager *manager,
 
    if (!self()->OMR::CodeCache::initialize(manager, codeCacheSegment, allocatedCodeCacheSizeInBytes))
       return false;
+
    self()->setInitialAllocationPointers();
 
+#ifdef LINUX
+   if (manager->isDisclaimEnabled())
+      {
+      J9JavaVM * javaVM = jitConfig->javaVM;
+      PORT_ACCESS_FROM_JAVAVM(javaVM); // for j9vmem_supported_page_sizes
+
+      uint8_t *middle  = _warmCodeAlloc + (_coldCodeAllocBase - _warmCodeAlloc) / 2;
+      size_t round = j9vmem_supported_page_sizes()[0] - 1;
+      middle = (uint8_t *)(((size_t)(middle + round)) & ~round);
+
+      TR_ASSERT_FATAL(_coldCodeAlloc > middle, "A code cache can't be smaller than a page");
+
+      size_t coldCacheSize = _coldCodeAlloc - middle;
+      coldCacheSize = (coldCacheSize + round) & ~round;
+
+      if (madvise(middle, coldCacheSize, MADV_NOHUGEPAGE) != 0)
+         {
+         const char *error = strerror(errno);
+         if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerbosePerformance))
+            TR_VerboseLog::writeLineLocked(TR_Vlog_INFO, "Failed to set MADV_NOHUGEPAGE for code cache: %s: %p %zu", error, middle, coldCacheSize);
+         }
+      else if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerbosePerformance))
+         {
+         TR_VerboseLog::writeLineLocked(TR_Vlog_INFO, "Forcing code cache cold region %p-%p to use default size memory pages",                                          middle, middle + coldCacheSize);
+         }
+
+      // If the memory segment is backed by a file, disable read-ahead
+      // so that touching one byte brings a single page in
+      if (codeCacheSegment->j9segment()->vmemIdentifier.allocator == OMRPORT_VMEM_RESERVE_USED_MMAP_SHM)
+         {
+         if (madvise(middle, coldCacheSize, MADV_RANDOM) != 0)
+            {
+            if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerbosePerformance))
+                TR_VerboseLog::writeLineLocked(TR_Vlog_INFO, "Failed to set MADV_RANDOM for cold code cache");
+            }
+         }
+      }
+#endif // ifdef LINUX
+
    _manager->reportCodeLoadEvents();
 
    return true;
@@ -763,3 +814,57 @@ extern "C"
       }
 
    }
+
+
+int32_t
+J9::CodeCache::disclaim(TR::CodeCacheManager *manager, bool canDisclaimOnSwap)
+   {
+   int32_t disclaimDone = 0;
+
+#ifdef LINUX
+   J9JavaVM * javaVM = jitConfig->javaVM;
+   PORT_ACCESS_FROM_JAVAVM(javaVM); // for j9vmem_supported_page_sizes
+
+   bool trace = TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerbosePerformance);
+   uint8_t *disclaim_start = _coldCodeAlloc;
+   size_t pageSize = j9vmem_supported_page_sizes()[0];
+   size_t round = pageSize - 1;
+   disclaim_start = (uint8_t *)(((size_t)(disclaim_start + round)) & ~round);
+
+   if (_coldCodeAllocBase <= disclaim_start)
+      return 0;
+
+   size_t disclaim_size = (_coldCodeAllocBase - disclaim_start + round) & ~round;
+
+   if (trace)
+      {
+      size_t warm_size = _warmCodeAlloc - _segment->segmentBase() + sizeof(this);
+      size_t cold_size = _coldCodeAllocBase - _coldCodeAlloc;
+
+      TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "Will disclaim cold code cache %p : coldStart=%p coldBase=%p warm_size=%zuB cold_size=%zuB cold_size/(cold_size + warm_size)=%5.2f%%\n",
+                               this, _coldCodeAlloc, _coldCodeAllocBase,
+                               warm_size, cold_size, cold_size * 100.0/(cold_size + warm_size));
+      }
+
+   int32_t ret = madvise((void *)disclaim_start, disclaim_size, MADV_PAGEOUT);
+
+   if (ret != 0)
+      {
+      if (trace)
+         TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "WARNING: Failed to use madvise to disclaim memory for code cache");
+
+      if (ret == EINVAL)
+         {
+         manager->setDisclaimEnabled(false); // Don't try to disclaim again, since support seems to be missing
+         if (trace)
+            TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "WARNING: Disabling data cache disclaiming from now on");
+         }
+      }
+   else
+      {
+      disclaimDone = 1;
+      }
+#endif // ifdef LINUX
+
+   return disclaimDone;
+   }

runtime/compiler/runtime/J9CodeCache.hpp

@@ -103,6 +103,8 @@ class OMR_EXTENSIBLE CodeCache : public OMR::CodeCacheConnector
    */
    void resetCodeCache();
 
+   int32_t disclaim(TR::CodeCacheManager *manager, bool canDisclaimOnSwap);
+
    private:
    /**
     * @brief Restore trampoline pointers to their initial positions

runtime/compiler/runtime/J9CodeCacheManager.cpp

@@ -385,6 +385,19 @@ J9::CodeCacheManager::allocateCodeCacheSegment(size_t segmentSize,
       }
 #endif
 
+#ifdef LINUX
+      if (_disclaimEnabled)
+         {
+         // If swap is enabled, we can allocate memory with mmap(MAP_ANOYNMOUS|MAP_PRIVATE) and disclaim to swap
+         // If swap is not enabled we can disclaim to a backing file
+         TR::CompilationInfo * compInfo = TR::CompilationInfo::get(_jitConfig);
+         if (!TR::Options::getCmdLineOptions()->getOption(TR_DisclaimMemoryOnSwap) || compInfo->isSwapMemoryDisabled())
+            {
+            segmentType |= MEMORY_TYPE_DISCLAIMABLE_TO_FILE;
+            }
+         }
+#endif
+
    mcc_printf("TR::CodeCache::allocate : requesting %d bytes\n", codeCacheSizeToAllocate);
    mcc_printf("TR::CodeCache::allocate : javaVM = %p\n", javaVM);
    mcc_printf("TR::CodeCache::allocate : codeCacheList = %p\n", jitConfig->codeCacheList);
@@ -488,11 +501,21 @@ J9::CodeCacheManager::allocateCodeCacheSegment(size_t segmentSize,
             vmemParams.startAddress,
             vmemParams.endAddress);
          }
+
+      if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerbosePerformance))
+         TR_VerboseLog::writeLineLocked(TR_Vlog_INFO, "Allocated new code cache segment %p starting at address %p %d Kb",
+                                        codeCacheSegment,
+                                        codeCacheSegment->heapBase,
+                                        _jitConfig->codeCacheKB);
       }
    else
       {
       // TODO: we should generate a trace point
       mcc_printf("TR::CodeCache::allocate : codeCacheSegment is NULL, %p\n",codeCacheSegment);
+
+      if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerbosePerformance))
+         TR_VerboseLog::writeLineLocked(TR_Vlog_INFO, "Failed to allocate new code cache segment of %d Kb", _jitConfig->codeCacheKB);
+
       return 0;
       }
 
@@ -779,3 +802,26 @@ J9::CodeCacheManager::printOccupancyStats()
       codeCache->printOccupancyStats();
       }
    }
+
+
+int32_t
+J9::CodeCacheManager::disclaimAllCodeCaches()
+   {
+   if (!_disclaimEnabled)
+      return 0;
+
+   int32_t numDisclaimed = 0;
+
+#ifdef LINUX
+   TR::CompilationInfo *compInfo = TR::CompilationInfo::get(_jitConfig);
+   bool canDisclaimOnSwap = TR::Options::getCmdLineOptions()->getOption(TR_DisclaimMemoryOnSwap) && !compInfo->isSwapMemoryDisabled();
+
+   CacheListCriticalSection scanCacheList(self());
+   for (TR::CodeCache *codeCache = self()->getFirstCodeCache(); codeCache; codeCache = codeCache->next())
+      {
+      numDisclaimed += codeCache->disclaim(self(), canDisclaimOnSwap);
+      }
+#endif // LINUX
+
+   return numDisclaimed;
+   }