Merge pull request #18252 from tajila/criu_3

Introduce JVMPortableRestoreMode

runtime/compiler/compile/J9Compilation.cpp

@@ -857,8 +857,9 @@ bool
 J9::Compilation::compilePortableCode()
    {
    return (self()->fej9()->inSnapshotMode() ||
-             (self()->compileRelocatableCode() &&
-                self()->fej9()->isPortableSCCEnabled()));
+             self()->fej9()->isPortableRestoreModeEnabled() ||
+                (self()->compileRelocatableCode() &&
+                   self()->fej9()->isPortableSCCEnabled()));
    }
 
 

runtime/compiler/control/HookedByTheJit.cpp

@@ -1856,7 +1856,7 @@ static void jitHookPrepareRestore(J9HookInterface * * hookInterface, UDATA event
     * remove the portability restrictions on the target CPU (used
     * for JIT compiles) to allow optimal code generation
     */
-   if (!javaVM->internalVMFunctions->isCheckpointAllowed(vmThread))
+   if (!javaVM->internalVMFunctions->isJVMInPortableRestoreMode(vmThread))
       {
       TR::Compiler->target.cpu = TR::CPU::detect(TR::Compiler->omrPortLib);
       jitConfig->targetProcessor = TR::Compiler->target.cpu.getProcessorDescription();

runtime/compiler/control/J9Options.cpp

@@ -1496,7 +1496,7 @@ void J9::Options::preProcessMmf(J9JavaVM *vm, J9JITConfig *jitConfig)
 
    if (J9_ARE_ANY_BITS_SET(vm->extendedRuntimeFlags2, J9_EXTENDED_RUNTIME2_ENABLE_PORTABLE_SHARED_CACHE)
 #if defined(J9VM_OPT_CRIU_SUPPORT)
-       || vm->internalVMFunctions->isCheckpointAllowed(vmThread)
+       || vm->internalVMFunctions->isJVMInPortableRestoreMode(vmThread)
 #endif /* defined(J9VM_OPT_CRIU_SUPPORT) */
        )
       {
@@ -2375,7 +2375,7 @@ bool J9::Options::preProcessJitServer(J9JavaVM *vm, J9JITConfig *jitConfig)
             if (implicitClientMode && useJitServerExplicitlySpecified)
                {
                compInfo->setRemoteCompilationRequestedAtBootstrap(true);
-               if (!ifuncs->isNonPortableRestoreMode(currentThread))
+               if (ifuncs->isJVMInPortableRestoreMode(currentThread))
                    compInfo->setCanPerformRemoteCompilationInCRIUMode(true);
                }
 #endif

runtime/compiler/control/JITClientCompilationThread.cpp

@@ -524,9 +524,10 @@ handleServerMessage(JITServer::ClientStream *client, TR_J9VM *fe, JITServer::Mes
             vmInfo._aotHeader = *header;
             }
          vmInfo._inSnapshotMode = fe->inSnapshotMode();
+         vmInfo._isPortableRestoreMode = fe->isPortableRestoreModeEnabled();
          vmInfo._isSnapshotModeEnabled = fe->isSnapshotModeEnabled();
 #if defined(J9VM_OPT_CRIU_SUPPORT)
-         vmInfo._isNonPortableRestoreMode = javaVM->internalVMFunctions->isNonPortableRestoreMode(vmThread);
+         vmInfo._isNonPortableRestoreMode = !javaVM->internalVMFunctions->isJVMInPortableRestoreMode(vmThread);
 #else
          vmInfo._isNonPortableRestoreMode = false;
 #endif /* defined(J9VM_OPT_CRIU_SUPPORT) */

runtime/compiler/control/rossa.cpp

@@ -1999,7 +1999,7 @@ aboutToBootstrap(J9JavaVM * javaVM, J9JITConfig * jitConfig)
     * is because, the restore run may not be on the same machine as the one that created
     * the snapshot; thus the JIT code must be portable.
     */
-   if (javaVM->internalVMFunctions->isCheckpointAllowed(curThread))
+   if (javaVM->internalVMFunctions->isJVMInPortableRestoreMode(curThread))
       {
       TR::Compiler->target.cpu = TR::CPU::detectRelocatable(TR::Compiler->omrPortLib);
       if (!J9_ARE_ANY_BITS_SET(javaVM->extendedRuntimeFlags2, J9_EXTENDED_RUNTIME2_ENABLE_PORTABLE_SHARED_CACHE))

runtime/compiler/env/VMJ9.cpp

@@ -9377,6 +9377,16 @@ TR_J9VMBase::inSnapshotMode()
 #endif /* defined(J9VM_OPT_CRIU_SUPPORT) */
    }
 
+bool
+TR_J9VMBase::isPortableRestoreModeEnabled()
+   {
+#if defined(J9VM_OPT_CRIU_SUPPORT)
+   return getJ9JITConfig()->javaVM->internalVMFunctions->isJVMInPortableRestoreMode(vmThread());
+#else /* defined(J9VM_OPT_CRIU_SUPPORT) */
+   return false;
+#endif /* defined(J9VM_OPT_CRIU_SUPPORT) */
+   }
+
 bool
 TR_J9VMBase::isSnapshotModeEnabled()
    {

runtime/compiler/env/VMJ9.h

@@ -1398,6 +1398,13 @@ class TR_J9VMBase : public TR_FrontEnd
     */
    virtual bool inSnapshotMode();
 
+   /**
+    * \brief Answers whether the JIT should generate portable restore code.
+    *
+    * \return True if portable restore code should be generated, false otherwise.
+    */
+   virtual bool isPortableRestoreModeEnabled();
+
    /**
     * \brief Answers whether checkpoint and restore mode is enabled (but not necessarily
     * whether snapshots can be taken or if any restores have already occurred).

runtime/compiler/env/VMJ9Server.cpp

@@ -2481,6 +2481,14 @@ TR_J9ServerVM::isSnapshotModeEnabled()
    return vmInfo->_isSnapshotModeEnabled;
    }
 
+bool
+TR_J9ServerVM::isPortableRestoreModeEnabled()
+   {
+   JITServer::ServerStream *stream = _compInfoPT->getMethodBeingCompiled()->_stream;
+   auto *vmInfo = _compInfoPT->getClientData()->getOrCacheVMInfo(stream);
+   return vmInfo->_isPortableRestoreMode;
+   }
+
 bool
 TR_J9SharedCacheServerVM::isClassLibraryMethod(TR_OpaqueMethodBlock *method, bool vettedForAOT)
    {

runtime/compiler/env/VMJ9Server.hpp

@@ -247,6 +247,7 @@ class TR_J9ServerVM: public TR_J9VM
    virtual bool isMethodHandleExpectedType(TR::Compilation *comp, TR::KnownObjectTable::Index mhIndex, TR::KnownObjectTable::Index expectedTypeIndex) override;
    virtual bool inSnapshotMode() override;
    virtual bool isSnapshotModeEnabled() override;
+   virtual bool isPortableRestoreModeEnabled() override;
 
 private:
    bool instanceOfOrCheckCastHelper(J9Class *instanceClass, J9Class* castClass, bool cacheUpdate);

runtime/compiler/net/CommunicationStream.hpp

@@ -118,7 +118,7 @@ class CommunicationStream
    // likely to lose an increment when merging/rebasing/etc.
    //
    static const uint8_t MAJOR_NUMBER = 1;
-   static const uint16_t MINOR_NUMBER = 52; // ID: vKDVVxR2fImMeQKnbv6I
+   static const uint16_t MINOR_NUMBER = 53; // ID: 7dZlozupV5RwUvR62RqE
    static const uint8_t PATCH_NUMBER = 0;
    static uint32_t CONFIGURATION_FLAGS;
 

runtime/compiler/runtime/JITClientSession.hpp

@@ -326,6 +326,7 @@ class ClientSessionData
       // Do not protect them with #if defined(J9VM_OPT_CRIU_SUPPORT) because we want JITServer to be
       // able to handle all clients whether or not they have CRIU support enabled
       bool _inSnapshotMode;
+      bool _isPortableRestoreMode;
       bool _isSnapshotModeEnabled;
       bool _isNonPortableRestoreMode;
       }; // struct VMInfo

runtime/oti/j9nonbuilder.h

@@ -4200,6 +4200,7 @@ typedef struct J9DelayedLockingOpertionsRecord {
 #define J9VM_CRIU_IS_NON_PORTABLE_RESTORE_MODE 0x4
 #define J9VM_CRIU_IS_JDWP_ENABLED 0x8
 #define J9VM_CRIU_IS_THROW_ON_DELAYED_CHECKPOINT_ENABLED 0x10
+#define J9VM_CRIU_IS_PORTABLE_JVM_RESTORE_MODE 0x20
 
 typedef struct J9CRIUCheckpointState {
 	U_32 flags;
@@ -5021,6 +5022,7 @@ typedef struct J9InternalVMFunctions {
 	BOOLEAN (*isCRIUSupportEnabled_VM)(struct J9JavaVM *vm);
 	BOOLEAN (*isCheckpointAllowed)(struct J9VMThread *currentThread);
 	BOOLEAN (*isNonPortableRestoreMode)(struct J9VMThread *currentThread);
+	BOOLEAN (*isJVMInPortableRestoreMode)(struct J9VMThread *currentThread);
 	BOOLEAN (*runInternalJVMCheckpointHooks)(struct J9VMThread *currentThread, const char **nlsMsgFormat);
 	BOOLEAN (*runInternalJVMRestoreHooks)(struct J9VMThread *currentThread, const char **nlsMsgFormat);
 	BOOLEAN (*runDelayedLockRelatedOperations)(struct J9VMThread *currentThread);

runtime/oti/jvminit.h

@@ -426,6 +426,8 @@ enum INIT_STAGE {
 #define VMOPT_XXDISABLECRIU "-XX:-EnableCRIUSupport"
 #define VMOPT_XXENABLECRIUNONPORTABLEMODE "-XX:+CRIURestoreNonPortableMode"
 #define VMOPT_XXDISABLECRIUNONPORTABLEMODE "-XX:-CRIURestoreNonPortableMode"
+#define VMOPT_XXENABLEJVMRESTOREPORTABLEMODE "-XX:+JVMPortableRestoreMode"
+#define VMOPT_XXDISABLEJVMRESTOREPORTABLEMODE "-XX:-JVMPortableRestoreMode"
 #define VMOPT_XSHARECLASSES_DISABLEONRESTORE "-Xshareclasses:disableOnRestore"
 #define VMOPT_XXENABLETHROWONDELAYECHECKPOINTOPERATION "-XX:+ThrowOnDelayedCheckpointOperation"
 #define VMOPT_XXDISABLETHROWONDELAYECHECKPOINTOPERATION "-XX:-ThrowOnDelayedCheckpointOperation"

runtime/oti/vm_api.h

@@ -541,6 +541,19 @@ isCheckpointAllowed(J9VMThread *currentThread);
 BOOLEAN
 isNonPortableRestoreMode(J9VMThread *currentThread);
 
+/**
+ * @brief Queries if portable restore mode (specified via
+ * -XX:+JVMPortableRestoreMode) is enabled. If so, the JVM
+ * will remain in portable mode after restore. However, this
+ * will have no impact on the JCL and taking another checkpoint
+ * will still not be permitted.
+ *
+ * @param currentThread vmthread token
+ * @return TRUE if enabled, FALSE otherwise
+ */
+BOOLEAN
+isJVMInPortableRestoreMode(J9VMThread *currentThread);
+
 /**
  * @brief JVM hooks to run before performing a JVM checkpoint
  *

runtime/vm/CRIUHelpers.cpp

@@ -134,6 +134,12 @@ isNonPortableRestoreMode(J9VMThread *currentThread)
 	return J9_ARE_ALL_BITS_SET(currentThread->javaVM->checkpointState.flags, J9VM_CRIU_IS_NON_PORTABLE_RESTORE_MODE);
 }
 
+BOOLEAN
+isJVMInPortableRestoreMode(J9VMThread *currentThread)
+{
+	return (!isNonPortableRestoreMode(currentThread) || J9_ARE_ALL_BITS_SET(currentThread->javaVM->checkpointState.flags, J9VM_CRIU_IS_PORTABLE_JVM_RESTORE_MODE)) && isCRIUSupportEnabled(currentThread);
+}
+
 /**
  * This adds an internal CRIU hook to trace all heap objects of instanceType and its subclasses if specified.
  *

runtime/vm/intfunc.c

@@ -411,6 +411,7 @@ J9InternalVMFunctions J9InternalFunctions = {
 	isCRIUSupportEnabled_VM,
 	isCheckpointAllowed,
 	isNonPortableRestoreMode,
+	isJVMInPortableRestoreMode,
 	runInternalJVMCheckpointHooks,
 	runInternalJVMRestoreHooks,
 	runDelayedLockRelatedOperations,

runtime/vm/jvminit.c

@@ -3893,6 +3893,16 @@ processVMArgsFromFirstToLast(J9JavaVM * vm)
 		}
 	}
 
+	{
+		IDATA enableJVMRestorePortableeMode = FIND_AND_CONSUME_VMARG(EXACT_MATCH, VMOPT_XXENABLEJVMRESTOREPORTABLEMODE, NULL);
+		IDATA disableJVMRestorePortableMode = FIND_AND_CONSUME_VMARG(EXACT_MATCH, VMOPT_XXDISABLEJVMRESTOREPORTABLEMODE, NULL);
+		if (enableJVMRestorePortableeMode > disableJVMRestorePortableMode) {
+			if (J9_ARE_ALL_BITS_SET(vm->checkpointState.flags, J9VM_CRIU_IS_CHECKPOINT_ENABLED)) {
+				vm->checkpointState.flags |= J9VM_CRIU_IS_PORTABLE_JVM_RESTORE_MODE;
+			}
+		}
+	}
+
 	{
 		IDATA enableThrowOnDelayedCheckpointOperation = FIND_AND_CONSUME_VMARG(EXACT_MATCH, VMOPT_XXENABLETHROWONDELAYECHECKPOINTOPERATION, NULL);
 		IDATA disableThrowOnDelayedCheckpointOperation = FIND_AND_CONSUME_VMARG(EXACT_MATCH, VMOPT_XXDISABLETHROWONDELAYECHECKPOINTOPERATION, NULL);

test/functional/cmdLineTests/criu/playlist.xml

@@ -91,6 +91,8 @@
 		<variations>
 			<variation>-Xjit -XX:+CRIURestoreNonPortableMode</variation>
 			<variation>-Xint -XX:+CRIURestoreNonPortableMode</variation>
+			<variation>-XX:+JVMPortableRestoreMode</variation>
+			<variation>-Xjit:count=0 -XX:+JVMPortableRestoreMode</variation>
 			<variation>-Xjit:count=0 -XX:+CRIURestoreNonPortableMode</variation>
 			<variation>-Xgcpolicy:optthruput</variation>
 			<variation>-Xgcpolicy:optavgpause</variation>
@@ -191,6 +193,7 @@
 			<variation>-Xjit</variation>
 			<variation>-Xjit:count=0</variation>
 			<variation>-Xjit:vlog=vlog</variation>
+			<variation>-XX:+JVMPortableRestoreMode</variation>
 		</variations>
 		<command>
 			if [ -x $(Q)$(TEST_JDK_BIN)$(D)jitserver$(Q) ]; \
@@ -352,6 +355,7 @@
 			<variation>-Xjit -XX:+CRIURestoreNonPortableMode</variation>
 			<variation>-Xint -XX:+CRIURestoreNonPortableMode</variation>
 			<variation>-Xjit:count=0 -XX:+CRIURestoreNonPortableMode</variation>
+			<variation>-XX:+JVMPortableRestoreMode</variation>
 		</variations>
 		<command>
 			$(JAVA_COMMAND) $(CMDLINETESTER_JVM_OPTIONS) -Xdump \

test/functional/cmdLineTests/criu/src/org/openj9/criu/CRIUSimpleTest.java

@@ -25,6 +25,11 @@
 import java.nio.file.Paths;
 import java.nio.file.Path;
 
+import javax.swing.SwingUtilities;
+import javax.swing.JFrame;
+import java.lang.management.*;
+
+
 public class CRIUSimpleTest {
 
 	public static void main(String args[]) {
@@ -36,12 +41,24 @@ public static void main(String args[]) {
 		}
 	}
 
+	private static void loadNewClasses() {
+		try {
+			SwingUtilities.isEventDispatchThread();
+			JFrame frame = new JFrame("Test code");
+		} catch (Throwable t) {
+			//ignore
+		}
+
+		ThreadMXBean mxb = ManagementFactory.getThreadMXBean();
+	}
+
 	public static void checkpoints(int num_checkpoints) {
 		Path path = Paths.get("cpData");
 		System.out.println("Total checkpoint(s) " + num_checkpoints + ":\nPre-checkpoint");
 		for (int cur_checkpint = 1; cur_checkpint <= num_checkpoints; ++cur_checkpint) {
 			CRIUTestUtils.checkPointJVM(path);
 			System.out.println("Post-checkpoint " + cur_checkpint);
+			loadNewClasses();
 		}
 	}
 }