Merge pull request #17652 from tajila/criu_4

(0.40) Add checkpoint delay when clinit is occuring

jcl/src/openj9.criu/share/classes/org/eclipse/openj9/criu/CRIUSupport.java

@@ -257,7 +257,7 @@ public static String getErrorMessage() {
 	public CRIUSupport setImageDir(Path imageDir) {
 		Objects.requireNonNull(imageDir, "Image directory cannot be null"); //$NON-NLS-1$
 		if (!Files.isDirectory(imageDir)) {
-			throw new IllegalArgumentException("imageDir is not a valid directory"); //$NON-NLS-1$
+			throw new IllegalArgumentException(imageDir.toAbsolutePath() + " is not a valid directory"); //$NON-NLS-1$
 		}
 		String dir = imageDir.toAbsolutePath().toString();
 

runtime/criusupport/criusupport.cpp

@@ -51,6 +51,9 @@ extern "C" {
 #define RESTORE_ARGS_RETURN_OPTIONS_FILE_FAILED 2
 #define RESTORE_ARGS_RETURN_ENV_VAR_FILE_FAILED 3
 
+#define J9VM_DELAYCHECKPOINT_NOTCHECKPOINTSAFE 0x1
+#define J9VM_DELAYCHECKPOINT_CLINIT 0x2
+
 #define OPENJ9_RESTORE_OPTS_VAR "OPENJ9_RESTORE_JAVA_OPTIONS="
 
 static bool
@@ -149,8 +152,11 @@ setupJNIFieldIDsAndCRIUAPI(JNIEnv *env, jclass *currentExceptionClass, IDATA *sy
 		goto done;
 	}
 
-	vm->checkpointState.isJdwpEnabled = (FIND_ARG_IN_VMARGS(STARTSWITH_MATCH, MAPOPT_AGENTLIB_JDWP_EQUALS, NULL) >= 0)
-		|| (FIND_ARG_IN_VMARGS(STARTSWITH_MATCH, MAPOPT_XRUNJDWP, NULL) >= 0);
+	if ((FIND_ARG_IN_VMARGS(STARTSWITH_MATCH, MAPOPT_AGENTLIB_JDWP_EQUALS, NULL) >= 0)
+		|| (FIND_ARG_IN_VMARGS(STARTSWITH_MATCH, MAPOPT_XRUNJDWP, NULL) >= 0)
+	) {
+		vm->checkpointState.flags |= J9VM_CRIU_IS_JDWP_ENABLED;
+	}
 
 done:
 	return returnCode;
@@ -269,7 +275,7 @@ shouldToggleJavaThread(J9VMThread *currentThread, BOOLEAN toggleDebugThreads)
 {
 	J9JavaVM *vm = currentThread->javaVM;
 	bool result = true;
-	if (vm->checkpointState.isJdwpEnabled) {
+	if (J9_ARE_ALL_BITS_SET(vm->checkpointState.flags, J9VM_CRIU_IS_JDWP_ENABLED)) {
 		char *threadName = getOMRVMThreadName(currentThread->omrVMThread);
 		releaseOMRVMThreadName(currentThread->omrVMThread);
 		/* all threads started by JDWP begin with "JDWP" in their name */
@@ -313,33 +319,47 @@ toggleSuspendOnJavaThreads(J9VMThread *currentThread, BOOLEAN suspend, BOOLEAN t
 }
 
 static UDATA
-notCheckpointSafeFrameWalkFunction(J9VMThread *vmThread, J9StackWalkState *walkState)
+notCheckpointSafeOrClinitFrameWalkFunction(J9VMThread *vmThread, J9StackWalkState *walkState)
 {
 	J9Method *method = walkState->method;
+	UDATA returnCode = J9_STACKWALK_KEEP_ITERATING;
+
 	if (NULL != method) {
+		J9ROMMethod *romMethod = J9_ROM_METHOD_FROM_RAM_METHOD(method);
 		J9ClassLoader *methodLoader = J9_CLASS_FROM_METHOD(method)->classLoader;
+		J9UTF8 *romMethodName = J9ROMMETHOD_NAME(romMethod);
+		/* only method names that start with '<' are <init>, <vnew> and <clinit>  */
+		if (0 == strncmp((char*)J9UTF8_DATA(romMethodName), "<c", 2)) {
+			*(UDATA*)walkState->userData1 = J9VM_DELAYCHECKPOINT_CLINIT;
+			goto fail;
+		}
+
 		/* we only enforce this in methods loaded by the bootloader */
 		if (methodLoader == vmThread->javaVM->systemClassLoader) {
-			J9ROMMethod *romMethod = J9_ROM_METHOD_FROM_RAM_METHOD(method);
 			if (J9ROMMETHOD_HAS_EXTENDED_MODIFIERS(romMethod)) {
 				U_32 extraModifiers = getExtendedModifiersDataFromROMMethod(romMethod);
 				if (J9ROMMETHOD_HAS_NOT_CHECKPOINT_SAFE_ANNOTATION(extraModifiers)) {
-					*(bool *)walkState->userData1 = false;
-					walkState->userData2 = (void *)vmThread;
-					walkState->userData3 = (void *)method;
-					return J9_STACKWALK_STOP_ITERATING;
+					*(UDATA*)walkState->userData1 = J9VM_DELAYCHECKPOINT_NOTCHECKPOINTSAFE;
+					goto fail;
 				}
 			}
 		}
 	}
 
-	return J9_STACKWALK_KEEP_ITERATING;
+done:
+	return returnCode;
+
+fail:
+	walkState->userData2 = (void *)vmThread;
+	walkState->userData3 = (void *)method;
+	returnCode = J9_STACKWALK_STOP_ITERATING;
+	goto done;
 }
 
 static bool
 checkIfSafeToCheckpoint(J9VMThread *currentThread)
 {
-	bool isSafe = true;
+	UDATA notSafeToCheckpoint = 0;
 	J9JavaVM *vm = currentThread->javaVM;
 
 	Assert_CRIU_true((J9_XACCESS_EXCLUSIVE == vm->exclusiveAccessState) || (J9_XACCESS_EXCLUSIVE == vm->safePointState));
@@ -353,19 +373,19 @@ checkIfSafeToCheckpoint(J9VMThread *currentThread)
 			walkState.walkThread = walkThread;
 			walkState.flags = J9_STACKWALK_ITERATE_FRAMES | J9_STACKWALK_INCLUDE_NATIVES;
 			walkState.skipCount = 0;
-			walkState.userData1 = (void *)&isSafe;
-			walkState.frameWalkFunction = notCheckpointSafeFrameWalkFunction;
+			walkState.userData1 = (void *)&notSafeToCheckpoint;
+			walkState.frameWalkFunction = notCheckpointSafeOrClinitFrameWalkFunction;
 
 			vm->walkStackFrames(walkThread, &walkState);
-			if (!isSafe) {
-				Trc_CRIU_checkpointJVMImpl_checkIfSafeToCheckpointBlocked(currentThread, walkState.userData2, walkState.userData3);
+			if (0 != notSafeToCheckpoint) {
+				Trc_CRIU_checkpointJVMImpl_checkIfSafeToCheckpointBlockedVer2(currentThread, walkState.userData2, walkState.userData3, *(UDATA*)walkState.userData1);
 				break;
 			}
 		}
 		walkThread = J9_LINKED_LIST_NEXT_DO(vm->mainThread, walkThread);
 	}
 
-	return isSafe;
+	return notSafeToCheckpoint;
 }
 
 static VMINLINE void
@@ -654,11 +674,12 @@ Java_org_eclipse_openj9_criu_CRIUSupport_checkpointJVMImpl(JNIEnv *env,
 		U_64 restoreNanoUTCTime = 0;
 		UDATA success = 0;
 		bool safePoint = J9_ARE_ANY_BITS_SET(vm->extendedRuntimeFlags, J9_EXTENDED_RUNTIME_OSR_SAFE_POINT);
-		bool retryPermitted = vm->checkpointState.maxRetryForNotCheckpointSafe > 0;
+		UDATA maxRetries = vm->checkpointState.maxRetryForNotCheckpointSafe;
 		BOOLEAN syslogFlagNone = TRUE;
 		char *syslogOptions = NULL;
 		I_32 syslogBufferSize = 0;
 		UDATA oldVMState = VM_VMHelpers::setVMState(currentThread, J9VMSTATE_CRIU_SUPPORT_CHECKPOINT_PHASE_START);
+		UDATA notSafeToCheckpoint = 0;
 
 		vmFuncs->internalEnterVMFromJNI(currentThread);
 
@@ -803,18 +824,27 @@ Java_org_eclipse_openj9_criu_CRIUSupport_checkpointJVMImpl(JNIEnv *env,
 
 		acquireSafeOrExcusiveVMAccess(currentThread, vmFuncs, safePoint);
 
-		for (UDATA i = 0; !checkIfSafeToCheckpoint(currentThread) && retryPermitted; i++) {
+		notSafeToCheckpoint = checkIfSafeToCheckpoint(currentThread);
+
+		for (UDATA i = 0; (0 != notSafeToCheckpoint) && (i <= maxRetries); i++) {
 			releaseSafeOrExcusiveVMAccess(currentThread, vmFuncs, safePoint);
 			vmFuncs->internalExitVMToJNI(currentThread);
-			omrthread_nanosleep(1000);
+			omrthread_nanosleep(10000);
 			vmFuncs->internalEnterVMFromJNI(currentThread);
-			if (i == vm->checkpointState.maxRetryForNotCheckpointSafe) {
-				currentExceptionClass = vm->checkpointState.criuJVMCheckpointExceptionClass;
-				systemReturnCode = vm->checkpointState.maxRetryForNotCheckpointSafe;
-				nlsMsgFormat = j9nls_lookup_message(J9NLS_DO_NOT_PRINT_MESSAGE_TAG | J9NLS_DO_NOT_APPEND_NEWLINE, J9NLS_JCL_CRIU_MAX_RETRY_FOR_NOTCHECKPOINTSAFE_REACHED, NULL);
-				goto closeWorkDirFD;
-			}
 			acquireSafeOrExcusiveVMAccess(currentThread, vmFuncs, safePoint);
+			notSafeToCheckpoint = checkIfSafeToCheckpoint(currentThread);
+		}
+
+		if ((J9VM_DELAYCHECKPOINT_NOTCHECKPOINTSAFE == notSafeToCheckpoint)
+			|| ((J9VM_DELAYCHECKPOINT_CLINIT == notSafeToCheckpoint) && J9_ARE_ALL_BITS_SET(vm->checkpointState.flags, J9VM_CRIU_IS_THROW_ON_DELAYED_CHECKPOINT_ENABLED))
+		) {
+			releaseSafeOrExcusiveVMAccess(currentThread, vmFuncs, safePoint);
+			currentExceptionClass = vm->checkpointState.criuJVMCheckpointExceptionClass;
+			systemReturnCode = vm->checkpointState.maxRetryForNotCheckpointSafe;
+			nlsMsgFormat = j9nls_lookup_message(J9NLS_DO_NOT_PRINT_MESSAGE_TAG | J9NLS_DO_NOT_APPEND_NEWLINE, J9NLS_JCL_CRIU_MAX_RETRY_FOR_NOTCHECKPOINTSAFE_REACHED, NULL);
+			goto closeWorkDirFD;
+		} else {
+			Trc_CRIU_checkpointJVMImpl_checkpointWithActiveCLinit(currentThread);
 		}
 
 		toggleSuspendOnJavaThreads(currentThread, TRUE, FALSE);
@@ -856,7 +886,7 @@ Java_org_eclipse_openj9_criu_CRIUSupport_checkpointJVMImpl(JNIEnv *env,
 			goto wakeJavaThreadsWithExclusiveVMAccess;
 		}
 
-		if (vm->checkpointState.isJdwpEnabled) {
+		if (J9_ARE_ALL_BITS_SET(vm->checkpointState.flags, J9VM_CRIU_IS_JDWP_ENABLED)) {
 			toggleSuspendOnJavaThreads(currentThread, TRUE, TRUE);
 		}
 
@@ -954,7 +984,7 @@ Java_org_eclipse_openj9_criu_CRIUSupport_checkpointJVMImpl(JNIEnv *env,
 			goto wakeJavaThreadsWithExclusiveVMAccess;
 		}
 
-		if (vm->checkpointState.isJdwpEnabled) {
+		if (J9_ARE_ALL_BITS_SET(vm->checkpointState.flags, J9VM_CRIU_IS_JDWP_ENABLED)) {
 			toggleSuspendOnJavaThreads(currentThread, FALSE, TRUE);
 		}
 

runtime/criusupport/j9criu.tdf

@@ -37,9 +37,11 @@ TraceEvent=Trc_CRIU_before_checkpoint Overhead=1 Level=2 Template="Before checkp
 TraceEvent=Trc_CRIU_after_checkpoint Obsolete Overhead=1 Level=2 Template="After checkpoint criu_dump(), restoreNanoUTCTime = %llu, checkpointNanoUTCTime = %llu, checkpointRestoreTimeDelta = %lld, restoreNanoTimeMonotonic = %lld, checkpointNanoTimeMonotonic = %lld, nanoTimeMonotonicClockDelta = %lld"
 TraceEntry=Trc_CRIU_checkpointJVMImpl_Entry Overhead=1 Level=2 Template="Java_org_eclipse_openj9_criu_CRIUSupport_checkpointJVMImpl"
 TraceExit=Trc_CRIU_checkpointJVMImpl_Exit Overhead=1 Level=2 Template="Java_org_eclipse_openj9_criu_CRIUSupport_checkpointJVMImpl"
-TraceEvent=Trc_CRIU_checkpointJVMImpl_checkIfSafeToCheckpointBlocked Overhead=1 Level=2 Template="Checkpoint blocked because thread=%p is in method=%p marked as not safe to checkpoint"
+TraceEvent=Trc_CRIU_checkpointJVMImpl_checkIfSafeToCheckpointBlocked Obsolete Overhead=1 Level=2 Template="Checkpoint blocked because thread=%p is in method=%p marked as not safe to checkpoint"
 TraceEvent=Trc_CRIU_checkpointJVMImpl_syslogOptions Overhead=1 Level=3 Template="Current syslogOptions: %s"
 TraceEvent=Trc_CRIU_checkpoint_nano_times Overhead=1 Level=2 Template="Before checkpoint, checkpointNanoTimeMonotonic = %lld, checkpointNanoUTCTime = %llu"
 TraceEvent=Trc_CRIU_restore_nano_times Overhead=1 Level=2 Template="After restore, restoreNanoUTCTime = %llu, checkpointNanoUTCTime = %llu, checkpointRestoreTimeDelta = %lld, restoreNanoTimeMonotonic = %lld, checkpointNanoTimeMonotonic = %lld, nanoTimeMonotonicClockDelta = %lld"
 TraceEvent=Trc_CRIU_after_checkpoint Overhead=1 Level=2 Template="After checkpoint criu_dump(), j9time_nano_time() returns %lld, j9time_current_time_nanos() returns %llu"
 TraceEvent=Trc_CRIU_restoreArg Overhead=1 Level=5 Test Template="Restore arg: %s"
+TraceEvent=Trc_CRIU_checkpointJVMImpl_checkIfSafeToCheckpointBlockedVer2 Overhead=1 Level=2 Template="Checkpoint blocked because thread=%p is in method=%p due to delay code %zu"
+TraceEvent=Trc_CRIU_checkpointJVMImpl_checkpointWithActiveCLinit Overhead=1 Level=2 Template="Taking a checkpoint with active clinit"

runtime/gc_modron_startup/mmparseXgcpolicy.cpp

@@ -85,7 +85,7 @@ isMetronomeGCPolicySupported(MM_GCExtensions *extensions)
 {
 #if defined(J9VM_OPT_CRIU_SUPPORT)
 	J9JavaVM *vm = extensions->getJavaVM();
-	if (TRUE == vm->checkpointState.isCheckPointEnabled) {
+	if (vm->internalVMFunctions->isCRIUSupportEnabled_VM(vm)) {
 		PORT_ACCESS_FROM_JAVAVM(vm);
 		j9nls_printf(PORTLIB, J9NLS_ERROR, J9NLS_GC_POLICY_NOT_SUPPOURTED_CRIU, "metronome");
 		return false;
@@ -107,7 +107,7 @@ isBalancedGCPolicySupported(MM_GCExtensions *extensions)
 {
 #if defined(J9VM_OPT_CRIU_SUPPORT)
 	J9JavaVM *vm = extensions->getJavaVM();
-	if (TRUE == vm->checkpointState.isCheckPointEnabled) {
+	if (vm->internalVMFunctions->isCRIUSupportEnabled_VM(vm)) {
 		PORT_ACCESS_FROM_JAVAVM(vm);
 		j9nls_printf(PORTLIB, J9NLS_ERROR, J9NLS_GC_POLICY_NOT_SUPPOURTED_CRIU, "balanced");
 		return false;

runtime/oti/j9nonbuilder.h

@@ -4178,11 +4178,14 @@ typedef struct J9DelayedLockingOpertionsRecord {
 #define J9_SINGLE_THREAD_MODE_OP_NOTIFY_ALL 0x2
 #define J9_SINGLE_THREAD_MODE_OP_INTERRUPT 0x3
 
+#define J9VM_CRIU_IS_CHECKPOINT_ENABLED 0x1
+#define J9VM_CRIU_IS_CHECKPOINT_ALLOWED 0x2
+#define J9VM_CRIU_IS_NON_PORTABLE_RESTORE_MODE 0x4
+#define J9VM_CRIU_IS_JDWP_ENABLED 0x8
+#define J9VM_CRIU_IS_THROW_ON_DELAYED_CHECKPOINT_ENABLED 0x10
+
 typedef struct J9CRIUCheckpointState {
-	BOOLEAN isCheckPointEnabled;
-	BOOLEAN isCheckPointAllowed;
-	BOOLEAN isNonPortableRestoreMode;
-	BOOLEAN isJdwpEnabled;
+	U_32 flags;
 	struct J9DelayedLockingOpertionsRecord *delayedLockingOperationsRoot;
 	struct J9Pool *hookRecords;
 	struct J9Pool *classIterationRestoreHookRecords;

runtime/oti/jvminit.h

@@ -425,6 +425,8 @@ enum INIT_STAGE {
 #define VMOPT_XXENABLECRIUNONPORTABLEMODE "-XX:+CRIURestoreNonPortableMode"
 #define VMOPT_XXDISABLECRIUNONPORTABLEMODE "-XX:-CRIURestoreNonPortableMode"
 #define VMOPT_XSHARECLASSES_DISABLEONRESTORE "-Xshareclasses:disableOnRestore"
+#define VMOPT_XXENABLETHROWONDELAYECHECKPOINTOPERATION "-XX:+ThrowOnDelayedCheckpointOperation"
+#define VMOPT_XXDISABLETHROWONDELAYECHECKPOINTOPERATION "-XX:-ThrowOnDelayedCheckpointOperation"
 #endif /* defined(J9VM_OPT_CRIU_SUPPORT) */
 
 /*

runtime/vm/CRIUHelpers.cpp

@@ -84,12 +84,12 @@ jvmRestoreHooks(J9VMThread *currentThread)
 	nas.name = (J9UTF8 *)&runPostRestoreHooks_name;
 	nas.signature = (J9UTF8 *)&runPostRestoreHooks_sig;
 
-	Assert_VM_true(vm->checkpointState.isCheckPointEnabled);
+	Assert_VM_true(isCRIUSupportEnabled_VM(vm));
 
-	if (vm->checkpointState.isNonPortableRestoreMode) {
+	if (J9_ARE_ALL_BITS_SET(vm->checkpointState.flags, J9VM_CRIU_IS_NON_PORTABLE_RESTORE_MODE)) {
 		PORT_ACCESS_FROM_JAVAVM(vm);
-		vm->checkpointState.isCheckPointAllowed = FALSE;
 		vm->portLibrary->isCheckPointAllowed = FALSE;
+		vm->checkpointState.flags &= ~J9VM_CRIU_IS_CHECKPOINT_ALLOWED;
 		j9port_control(J9PORT_CTLDATA_CRIU_SUPPORT_FLAGS, OMRPORT_CRIU_SUPPORT_ENABLED | J9OMRPORT_CRIU_SUPPORT_FINAL_RESTORE);
 	}
 
@@ -114,7 +114,7 @@ isCRIUSupportEnabled(J9VMThread *currentThread)
 BOOLEAN
 isCRIUSupportEnabled_VM(J9JavaVM *vm)
 {
-	return vm->checkpointState.isCheckPointEnabled;
+	return J9_ARE_ALL_BITS_SET(vm->checkpointState.flags, J9VM_CRIU_IS_CHECKPOINT_ENABLED);
 }
 
 BOOLEAN
@@ -123,7 +123,7 @@ isCheckpointAllowed(J9VMThread *currentThread)
 	BOOLEAN result = FALSE;
 
 	if (isCRIUSupportEnabled(currentThread)) {
-		result = currentThread->javaVM->checkpointState.isCheckPointAllowed;
+		result = J9_ARE_ALL_BITS_SET(currentThread->javaVM->checkpointState.flags, J9VM_CRIU_IS_CHECKPOINT_ALLOWED);
 	}
 
 	return result;
@@ -132,7 +132,7 @@ isCheckpointAllowed(J9VMThread *currentThread)
 BOOLEAN
 isNonPortableRestoreMode(J9VMThread *currentThread)
 {
-	return currentThread->javaVM->checkpointState.isNonPortableRestoreMode;
+	return J9_ARE_ALL_BITS_SET(currentThread->javaVM->checkpointState.flags, J9VM_CRIU_IS_NON_PORTABLE_RESTORE_MODE);
 }
 
 /**
@@ -402,15 +402,15 @@ cleanupCriuHooks(J9VMThread *currentThread)
 			hookRecord = (J9InternalHookRecord*)pool_nextDo(&walkState);
 		}
 
-		if (vm->checkpointState.isNonPortableRestoreMode) {
+		if (J9_ARE_ALL_BITS_SET(vm->checkpointState.flags, J9VM_CRIU_IS_NON_PORTABLE_RESTORE_MODE)) {
 			/* No more checkpoint, cleanup hook records. */
 			pool_kill(vm->checkpointState.hookRecords);
 			vm->checkpointState.hookRecords = NULL;
 		}
 	}
 
 	J9Pool *classIterationRestoreHookRecords = vm->checkpointState.classIterationRestoreHookRecords;
-	if ((NULL != classIterationRestoreHookRecords) && (vm->checkpointState.isNonPortableRestoreMode)) {
+	if ((NULL != classIterationRestoreHookRecords) && J9_ARE_ALL_BITS_SET(vm->checkpointState.flags, J9VM_CRIU_IS_NON_PORTABLE_RESTORE_MODE)) {
 		/* No more checkpoint, cleanup hook records. */
 		pool_kill(vm->checkpointState.classIterationRestoreHookRecords);
 		vm->checkpointState.classIterationRestoreHookRecords = NULL;

runtime/vm/jvminit.c

@@ -2875,7 +2875,7 @@ VMInitStages(J9JavaVM *vm, IDATA stage, void* reserved)
 				}
 			} else
 #if defined(J9VM_OPT_CRIU_SUPPORT)
-			if (vm->checkpointState.isCheckPointAllowed) {
+			if (J9_ARE_ALL_BITS_SET(vm->checkpointState.flags, J9VM_CRIU_IS_CHECKPOINT_ALLOWED)) {
 				if (J9JAVAVM_COMPRESS_OBJECT_REFERENCES(vm)) {
 #if defined(OMR_GC_COMPRESSED_POINTERS)
 					vm->bytecodeLoop = criuBytecodeLoopCompressed;
@@ -3819,8 +3819,7 @@ processVMArgsFromFirstToLast(J9JavaVM * vm)
 		IDATA disableCRIU = FIND_AND_CONSUME_VMARG(EXACT_MATCH, VMOPT_XXDISABLECRIU, NULL);
 		if (enableCRIU > disableCRIU) {
 			PORT_ACCESS_FROM_JAVAVM(vm);
-			vm->checkpointState.isCheckPointEnabled = TRUE;
-			vm->checkpointState.isCheckPointAllowed = TRUE;
+			vm->checkpointState.flags |= J9VM_CRIU_IS_CHECKPOINT_ENABLED | J9VM_CRIU_IS_CHECKPOINT_ALLOWED;
 			vm->portLibrary->isCheckPointAllowed = TRUE;
 			j9port_control(J9PORT_CTLDATA_CRIU_SUPPORT_FLAGS, OMRPORT_CRIU_SUPPORT_ENABLED);
 		}
@@ -3830,12 +3829,20 @@ processVMArgsFromFirstToLast(J9JavaVM * vm)
 		IDATA enableCRIUNonPortableMode = FIND_AND_CONSUME_VMARG(EXACT_MATCH, VMOPT_XXENABLECRIUNONPORTABLEMODE, NULL);
 		IDATA disableCRIUNonPortableMode = FIND_AND_CONSUME_VMARG(EXACT_MATCH, VMOPT_XXDISABLECRIUNONPORTABLEMODE, NULL);
 		if (enableCRIUNonPortableMode >= disableCRIUNonPortableMode) {
-			if (vm->checkpointState.isCheckPointEnabled) {
-				vm->checkpointState.isNonPortableRestoreMode = TRUE;
+			if (J9_ARE_ALL_BITS_SET(vm->checkpointState.flags, J9VM_CRIU_IS_CHECKPOINT_ENABLED)) {
+				vm->checkpointState.flags |= J9VM_CRIU_IS_NON_PORTABLE_RESTORE_MODE;
 			}
 		}
 	}
 
+	{
+		IDATA enableThrowOnDelayedCheckpointOperation = FIND_AND_CONSUME_VMARG(EXACT_MATCH, VMOPT_XXENABLETHROWONDELAYECHECKPOINTOPERATION, NULL);
+		IDATA disableThrowOnDelayedCheckpointOperation = FIND_AND_CONSUME_VMARG(EXACT_MATCH, VMOPT_XXDISABLETHROWONDELAYECHECKPOINTOPERATION, NULL);
+		if (disableThrowOnDelayedCheckpointOperation >= enableThrowOnDelayedCheckpointOperation) {
+			vm->checkpointState.flags |= J9VM_CRIU_IS_THROW_ON_DELAYED_CHECKPOINT_ENABLED;
+		}
+	}
+
 	/* Its unclear if we need an option for this, so we can keep the init here for the time being */
 	vm->checkpointState.maxRetryForNotCheckpointSafe = 100;
 #endif /* defined(J9VM_OPT_CRIU_SUPPORT) */

test/functional/cmdLineTests/criu/criuScript.sh

@@ -36,7 +36,7 @@ echo "export GLIBC_TUNABLES=glibc.cpu.hwcaps=-XSAVEC,-XSAVE,-AVX2,-ERMS,-AVX,-AV
 export GLIBC_TUNABLES=glibc.pthread.rseq=0:glibc.cpu.hwcaps=-XSAVEC,-XSAVE,-AVX2,-ERMS,-AVX,-AVX_Fast_Unaligned_Load
 echo "export LD_BIND_NOT=on";
 export LD_BIND_NOT=on
-
+echo "$2 -XX:+EnableCRIUSupport $3 -cp "$1/criu.jar" $4 $5 $6"
 $2 -XX:+EnableCRIUSupport $3 -cp "$1/criu.jar" $4 $5 $6 >testOutput 2>&1;
 
 if [ "$7" != true ]; then