From fddac4d9e4d662652e13821df1ad99061dd8aebb Mon Sep 17 00:00:00 2001
From: Irwin D'Souza <dsouzai.gh@gmail.com>
Date: Mon, 29 Aug 2022 10:42:22 -0400
Subject: [PATCH] Add ability to sleep before checkpoint

This commit adds the ability to sleep before a checkpoint to allow the
compilation threads to compile the methods already present in the
compilation queue. It is important that the hook thread releases both
VM Access and the Compilation Monitor before sleeping; otherwise, the
compilation threads will not be able to empty the queue.

Signed-off-by: Irwin D'Souza <dsouzai.gh@gmail.com>
---
 .../compiler/control/CompilationThread.cpp    | 20 +++++++++++++++++++
 runtime/compiler/control/J9Options.cpp        |  8 ++++++++
 runtime/compiler/control/J9Options.hpp        |  4 ++++
 3 files changed, 32 insertions(+)

diff --git a/runtime/compiler/control/CompilationThread.cpp b/runtime/compiler/control/CompilationThread.cpp
index 150eaee2965..96a2bd84470 100644
--- a/runtime/compiler/control/CompilationThread.cpp
+++ b/runtime/compiler/control/CompilationThread.cpp
@@ -2646,9 +2646,23 @@ void TR::CompilationInfo::prepareForCheckpoint()
    J9JavaVM   *vm       = _jitConfig->javaVM;
    J9VMThread *vmThread = vm->internalVMFunctions->currentVMThread(vm);
 
+   if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerboseCheckpointRestore))
+      TR_VerboseLog::writeLineLocked(TR_Vlog_CHECKPOINT_RESTORE, "Preparing for checkpoint");
+
    {
    ReleaseVMAccessAndAcquireMonitor suspendCompThreadsForCheckpoint(vmThread, getCompilationMonitor());
 
+   if (TR::Options::_sleepMsBeforeCheckpoint)
+      {
+      if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerboseCheckpointRestore))
+         TR_VerboseLog::writeLineLocked(TR_Vlog_CHECKPOINT_RESTORE, "Sleeping for %d ms", TR::Options::_sleepMsBeforeCheckpoint);
+
+      releaseCompMonitor(vmThread);
+      j9thread_sleep(static_cast<int64_t>(TR::Options::_sleepMsBeforeCheckpoint));
+      acquireCompMonitor(vmThread);
+      }
+
+
    if (shouldCheckpointBeInterrupted())
       return;
 
@@ -2713,10 +2727,14 @@ void TR::CompilationInfo::prepareForCheckpoint()
       }
    }
 
+   if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerboseCheckpointRestore))
+      TR_VerboseLog::writeLineLocked(TR_Vlog_CHECKPOINT_RESTORE, "Ready for checkpoint");
    }
 
 void TR::CompilationInfo::prepareForRestore()
    {
+   if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerboseCheckpointRestore))
+      TR_VerboseLog::writeLineLocked(TR_Vlog_CHECKPOINT_RESTORE, "Preparing for restore");
 
    {
    OMR::CriticalSection resumeCompThreadsForRestore(getCompilationMonitor());
@@ -2728,6 +2746,8 @@ void TR::CompilationInfo::prepareForRestore()
    resumeCompilationThread();
    }
 
+   if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerboseCheckpointRestore))
+      TR_VerboseLog::writeLineLocked(TR_Vlog_CHECKPOINT_RESTORE, "Ready for restore");
    }
 #endif // #if defined(J9VM_OPT_CRIU_SUPPORT)
 
diff --git a/runtime/compiler/control/J9Options.cpp b/runtime/compiler/control/J9Options.cpp
index d46da272cd6..395da172537 100644
--- a/runtime/compiler/control/J9Options.cpp
+++ b/runtime/compiler/control/J9Options.cpp
@@ -135,6 +135,10 @@ int32_t J9::Options::_highActiveThreadThreshold = -1;
 int32_t J9::Options::_veryHighActiveThreadThreshold = -1;
 #endif /* defined(J9VM_OPT_JITSERVER) */
 
+#if defined(J9VM_OPT_CRIU_SUPPORT)
+int32_t J9::Options::_sleepMsBeforeCheckpoint = 1000; // 1 second
+#endif
+
 int32_t J9::Options::_interpreterSamplingThreshold = 300;
 int32_t J9::Options::_interpreterSamplingDivisor = TR_DEFAULT_INTERPRETER_SAMPLING_DIVISOR;
 int32_t J9::Options::_interpreterSamplingThresholdInStartupMode = TR_DEFAULT_INITIAL_BCOUNT; // 3000
@@ -991,6 +995,10 @@ TR::OptionTable OMR::Options::_feOptions[] = {
         TR::Options::setStaticNumeric, (intptr_t)&TR::Options::_sharedROMClassCacheNumPartitions, 0, "F%d", NOT_IN_SUBSET},
 #endif /* defined(J9VM_OPT_JITSERVER) */
    {"singleCache", "C\tallow only one code cache and one data cache to be allocated", RESET_JITCONFIG_RUNTIME_FLAG(J9JIT_GROW_CACHES) },
+#if defined(J9VM_OPT_CRIU_SUPPORT)
+   {"sleepMsBeforeCheckpoint=", " O<nnn>\tNumber of milliseconds to sleep before a checkpoint",
+        TR::Options::setStaticNumeric, (intptr_t)&TR::Options::_sleepMsBeforeCheckpoint,  0, "F%d" },
+#endif
    {"smallMethodBytecodeSizeThreshold=", "O<nnn> \tThreshold for determining small methods "
                                          "(measured in number of bytecodes)",
         TR::Options::setStaticNumeric, (intptr_t)&TR::Options::_smallMethodBytecodeSizeThreshold, 0, "F%d", NOT_IN_SUBSET},
diff --git a/runtime/compiler/control/J9Options.hpp b/runtime/compiler/control/J9Options.hpp
index 2b299b4d41e..ac0b76bf66a 100644
--- a/runtime/compiler/control/J9Options.hpp
+++ b/runtime/compiler/control/J9Options.hpp
@@ -279,6 +279,10 @@ class OMR_EXTENSIBLE Options : public OMR::OptionsConnector
    static const uint32_t DEFAULT_JITSERVER_TIMEOUT = 30000; // ms
 #endif /* defined(J9VM_OPT_JITSERVER) */
 
+#if defined(J9VM_OPT_CRIU_SUPPORT)
+   static int32_t _sleepMsBeforeCheckpoint;
+#endif
+
    static int32_t _waitTimeToEnterIdleMode;
    static int32_t _waitTimeToEnterDeepIdleMode;
    static int32_t _waitTimeToExitStartupMode;