Fix up global continuation lists instead of rebuilding for compaction

Once continuation Object is constructed, the Object would be added in
global continuation lists and the "dead" Object would be collected by
GC, since logically compaction would not collect any continuation Object
in the list, so we piggyback heap fix up to rebuild the list to reduce
the extra cycle to walk the old list for fixing up.
But between the Object is created and added into the list, GC could be
triggered and find the live Object, which has not be added in the list,
it could cause a circular linklist case, so changing fix up the existing
list instead of rebuilding the list for compaction.

Signed-off-by: Lin Hu <linhu@ca.ibm.com>

runtime/gc_glue_java/CompactDelegate.cpp

@@ -31,8 +31,6 @@
 #include "MarkMap.hpp"
 #include "OwnableSynchronizerObjectBuffer.hpp"
 #include "OwnableSynchronizerObjectList.hpp"
-#include "ContinuationObjectBuffer.hpp"
-#include "ContinuationObjectList.hpp"
 #include "PointerContiguousArrayIterator.hpp"
 
 #if defined(OMR_GC_MODRON_COMPACTION)
@@ -132,8 +130,6 @@ MM_CompactDelegate::workerCleanupAfterGC(MM_EnvironmentBase *env)
 {
 	/* flush ownable synchronizer object buffer after rebuild the ownableSynchronizerObjectList during fixupObjects */
 	env->getGCEnvironment()->_ownableSynchronizerObjectBuffer->flush(env);
-	/* flush continuation object buffer after rebuild the continuationObjectList during fixupObjects */
-	env->getGCEnvironment()->_continuationObjectBuffer->flush(env);
 }
 
 void
@@ -146,7 +142,6 @@ MM_CompactDelegate::mainSetupForGC(MM_EnvironmentBase *env)
 		MM_HeapRegionDescriptorStandardExtension *regionExtension = MM_ConfigurationDelegate::getHeapRegionDescriptorStandardExtension(env, region);
 		for (uintptr_t i = 0; i < regionExtension->_maxListIndex; i++) {
 			regionExtension->_ownableSynchronizerObjectLists[i].startOwnableSynchronizerProcessing();
-			regionExtension->_continuationObjectLists[i].startProcessing();
 		}
 	}
 }

runtime/gc_glue_java/CompactSchemeFixupObject.cpp

@@ -32,7 +32,6 @@
 #include "MixedObjectIterator.hpp"
 #include "ObjectAccessBarrier.hpp"
 #include "OwnableSynchronizerObjectBuffer.hpp"
-#include "ContinuationObjectBuffer.hpp"
 #include "VMHelpers.hpp"
 #include "ParallelDispatcher.hpp"
 #include "PointerContiguousArrayIterator.hpp"
@@ -126,12 +125,6 @@ MM_CompactSchemeFixupObject::addOwnableSynchronizerObjectInList(MM_EnvironmentBa
 	}
 }
 
-MMINLINE void
-MM_CompactSchemeFixupObject::addContinuationObjectInList(MM_EnvironmentBase *env, omrobjectptr_t objectPtr)
-{
-	env->getGCEnvironment()->_continuationObjectBuffer->add(env, objectPtr);
-}
-
 void
 MM_CompactSchemeFixupObject::fixupObject(MM_EnvironmentStandard *env, omrobjectptr_t objectPtr)
 {
@@ -158,7 +151,6 @@ MM_CompactSchemeFixupObject::fixupObject(MM_EnvironmentStandard *env, omrobjectp
 		fixupMixedObject(objectPtr);
 		break;
 	case GC_ObjectModel::SCAN_CONTINUATION_OBJECT:
-		addContinuationObjectInList(env, objectPtr);
 		fixupContinuationObject(env, objectPtr);
 		break;
 	case GC_ObjectModel::SCAN_FLATTENED_ARRAY_OBJECT:

runtime/gc_glue_java/CompactSchemeFixupObject.hpp

@@ -82,8 +82,6 @@ class MM_CompactSchemeFixupObject {
 	 * @param object -- The object of type or subclass of java.util.concurrent.locks.AbstractOwnableSynchronizer.
 	 */
 	MMINLINE void addOwnableSynchronizerObjectInList(MM_EnvironmentBase *env, omrobjectptr_t objectPtr);
-
-	MMINLINE void addContinuationObjectInList(MM_EnvironmentBase *env, omrobjectptr_t objectPtr);
 };
 
 typedef struct StackIteratorData4CompactSchemeFixupObject {

runtime/gc_glue_java/CompactSchemeFixupRoots.cpp

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2001, 2020 IBM Corp. and others
+ * Copyright (c) 2001, 2022 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -36,6 +36,8 @@
 #include "Task.hpp"
 #include "UnfinalizedObjectBuffer.hpp"
 #include "UnfinalizedObjectList.hpp"
+#include "ContinuationObjectBuffer.hpp"
+#include "ContinuationObjectList.hpp"
 
 
 #if defined(J9VM_GC_FINALIZATION)
@@ -97,7 +99,7 @@ MM_CompactSchemeFixupRoots::fixupUnfinalizedObjects(MM_EnvironmentBase *env)
 		GC_HeapRegionIteratorStandard regionIterator(extensions->getHeap()->getHeapRegionManager());
 		while(NULL != (region = regionIterator.nextRegion())) {
 			MM_HeapRegionDescriptorStandardExtension *regionExtension = MM_ConfigurationDelegate::getHeapRegionDescriptorStandardExtension(env, region);
-			for (UDATA i = 0; i < regionExtension->_maxListIndex; i++) {
+			for (uintptr_t i = 0; i < regionExtension->_maxListIndex; i++) {
 				MM_UnfinalizedObjectList *list = &regionExtension->_unfinalizedObjectLists[i];
 				list->startUnfinalizedProcessing();
 			}
@@ -108,7 +110,7 @@ MM_CompactSchemeFixupRoots::fixupUnfinalizedObjects(MM_EnvironmentBase *env)
 	GC_HeapRegionIteratorStandard regionIterator(extensions->getHeap()->getHeapRegionManager());
 	while(NULL != (region = regionIterator.nextRegion())) {
 		MM_HeapRegionDescriptorStandardExtension *regionExtension = MM_ConfigurationDelegate::getHeapRegionDescriptorStandardExtension(env, region);
-		for (UDATA i = 0; i < regionExtension->_maxListIndex; i++) {
+		for (uintptr_t i = 0; i < regionExtension->_maxListIndex; i++) {
 			MM_UnfinalizedObjectList *list = &regionExtension->_unfinalizedObjectLists[i];
 			if (!list->wasEmpty()) {
 				if(J9MODRON_HANDLE_NEXT_WORK_UNIT(env)) {
@@ -129,3 +131,45 @@ MM_CompactSchemeFixupRoots::fixupUnfinalizedObjects(MM_EnvironmentBase *env)
 	env->getGCEnvironment()->_unfinalizedObjectBuffer->flush(env);
 }
 #endif /* J9VM_GC_FINALIZATION */
+
+
+void
+MM_CompactSchemeFixupRoots::fixupContinuationObjects(MM_EnvironmentBase *env)
+{
+	MM_GCExtensions* extensions = MM_GCExtensions::getExtensions(env);
+	if (env->_currentTask->synchronizeGCThreadsAndReleaseMain(env, UNIQUE_ID)) {
+		MM_HeapRegionDescriptorStandard *region = NULL;
+		GC_HeapRegionIteratorStandard regionIterator(extensions->getHeap()->getHeapRegionManager());
+		while(NULL != (region = regionIterator.nextRegion())) {
+			MM_HeapRegionDescriptorStandardExtension *regionExtension = MM_ConfigurationDelegate::getHeapRegionDescriptorStandardExtension(env, region);
+			for (uintptr_t i = 0; i < regionExtension->_maxListIndex; i++) {
+				MM_ContinuationObjectList *list = &regionExtension->_continuationObjectLists[i];
+				list->startProcessing();
+			}
+		}
+		env->_currentTask->releaseSynchronizedGCThreads(env);
+	}
+	MM_HeapRegionDescriptorStandard *region = NULL;
+	GC_HeapRegionIteratorStandard regionIterator(extensions->getHeap()->getHeapRegionManager());
+	while(NULL != (region = regionIterator.nextRegion())) {
+		MM_HeapRegionDescriptorStandardExtension *regionExtension = MM_ConfigurationDelegate::getHeapRegionDescriptorStandardExtension(env, region);
+		for (uintptr_t i = 0; i < regionExtension->_maxListIndex; i++) {
+			MM_ContinuationObjectList *list = &regionExtension->_continuationObjectLists[i];
+			if (!list->wasEmpty()) {
+				if(J9MODRON_HANDLE_NEXT_WORK_UNIT(env)) {
+					omrobjectptr_t object = list->getPriorList();
+					while (NULL != object) {
+						omrobjectptr_t forwardedPtr = _compactScheme->getForwardingPtr(object);
+						/* read the next link out of the moved copy of the object before we add it to the buffer */
+						object = extensions->accessBarrier->getContinuationLink(forwardedPtr);
+						/* store the object in this thread's buffer. It will be flushed to the appropriate list when necessary. */
+						env->getGCEnvironment()->_continuationObjectBuffer->add(env, forwardedPtr);
+					}
+				}
+			}
+		}
+	}
+
+	/* restore everything to a flushed state before exiting */
+	env->getGCEnvironment()->_continuationObjectBuffer->flush(env);
+}

runtime/gc_glue_java/CompactSchemeFixupRoots.hpp

@@ -73,7 +73,6 @@ class MM_CompactSchemeFixupRoots : public MM_RootScanner {
 	virtual void scanUnfinalizedObjects(MM_EnvironmentBase *env) {
 		/* allow the compact scheme to handle this */
 		reportScanningStarted(RootScannerEntity_UnfinalizedObjects);
-		MM_CompactSchemeFixupObject fixupObject(env, _compactScheme);
 		fixupUnfinalizedObjects(env);
 		reportScanningEnded(RootScannerEntity_UnfinalizedObjects);
 	}
@@ -87,7 +86,6 @@ class MM_CompactSchemeFixupRoots : public MM_RootScanner {
 	virtual void scanFinalizableObjects(MM_EnvironmentBase *env) {
 		if (_singleThread || J9MODRON_HANDLE_NEXT_WORK_UNIT(env)) {
 			reportScanningStarted(RootScannerEntity_FinalizableObjects);
-			MM_CompactSchemeFixupObject fixupObject(env, _compactScheme);
 			fixupFinalizableObjects(env);
 			reportScanningEnded(RootScannerEntity_FinalizableObjects);
 		}
@@ -99,13 +97,17 @@ class MM_CompactSchemeFixupRoots : public MM_RootScanner {
 	}
 
 	virtual void scanContinuationObjects(MM_EnvironmentBase *env) {
-		/* empty, move continuation processing in fixupObject */
+		/* allow the compact scheme to handle this */
+		reportScanningStarted(RootScannerEntity_ContinuationObjects);
+		fixupContinuationObjects(env);
+		reportScanningEnded(RootScannerEntity_ContinuationObjects);
 	}
 
 private:
 #if defined(J9VM_GC_FINALIZATION)
 	void fixupFinalizableObjects(MM_EnvironmentBase *env);
 	void fixupUnfinalizedObjects(MM_EnvironmentBase *env);
 #endif
+	void fixupContinuationObjects(MM_EnvironmentBase *env);
 };
 #endif /* FIXUPROOTS_HPP_ */

runtime/gc_vlhgc/WriteOnceCompactor.cpp

@@ -43,6 +43,8 @@
 #include "ClassLoaderIterator.hpp"
 #include "ClassLoaderRememberedSet.hpp"
 #include "CompactGroupManager.hpp"
+#include "ContinuationObjectBuffer.hpp"
+#include "ContinuationObjectList.hpp"
 #include "VMHelpers.hpp"
 #include "WriteOnceCompactor.hpp"
 #include "Debug.hpp"
@@ -558,7 +560,6 @@ MM_WriteOnceCompactor::compact(MM_EnvironmentVLHGC *env)
 	env->_compactVLHGCStats._moveStartTime = timeTemp;
 	moveObjects(env);
 	env->getGCEnvironment()->_ownableSynchronizerObjectBuffer->flush(env);
-	env->getGCEnvironment()->_continuationObjectBuffer->flush(env);
 	/* Note:  moveObjects implicitly synchronizes threads */
 	timeTemp = j9time_hires_clock();
 	env->_compactVLHGCStats._moveEndTime = timeTemp;
@@ -1456,7 +1457,6 @@ MM_WriteOnceCompactor::fixupObject(MM_EnvironmentVLHGC* env, J9Object *objectPtr
 		fixupMixedObject(env, objectPtr, cache);
 		break;
 	case GC_ObjectModel::SCAN_CONTINUATION_OBJECT:
-		addContinuationObjectInList(env, objectPtr);
 		fixupContinuationObject(env, objectPtr, cache);
 		break;
 	case GC_ObjectModel::SCAN_CLASS_OBJECT:
@@ -2041,11 +2041,27 @@ MM_WriteOnceCompactor::fixupArrayletLeafRegionContentsAndObjectLists(MM_Environm
 					}
 				}
 			}
+			if (!region->getContinuationObjectList()->wasEmpty()) {
+				if (J9MODRON_HANDLE_NEXT_WORK_UNIT(env)) {
+					J9Object *pointer = region->getContinuationObjectList()->getPriorList();
+					while (NULL != pointer) {
+						Assert_MM_true(region->isAddressInRegion(pointer));
+						J9Object* forwardedPtr = getForwardingPtr(pointer);
+
+						/* read the next link out of the moved copy of the object before we add it to the buffer */
+						pointer = _extensions->accessBarrier->getContinuationLink(forwardedPtr);
+
+						/* store the object in this thread's buffer. It will be flushed to the appropriate list when necessary. */
+						env->getGCEnvironment()->_continuationObjectBuffer->add(env, forwardedPtr);
+					}
+				}
+			}
 		}
 	}
 
 	/* restore everything to a flushed state before exiting */
 	env->getGCEnvironment()->_unfinalizedObjectBuffer->flush(env);
+	env->getGCEnvironment()->_continuationObjectBuffer->flush(env);
 }
 
 void

runtime/gc_vlhgc/WriteOnceCompactor.hpp

@@ -514,11 +514,6 @@ class MM_WriteOnceCompactor : public MM_BaseVirtual
 		}
 	}
 
-	MMINLINE void addContinuationObjectInList(MM_EnvironmentVLHGC *env, J9Object *objectPtr)
-	{
-		((MM_ContinuationObjectBufferVLHGC*) env->getGCEnvironment()->_continuationObjectBuffer)->addForOnlyCompactedRegion(env, objectPtr);
-	}
-
 #if defined(J9VM_GC_FINALIZATION)
 	void fixupFinalizableObjects(MM_EnvironmentVLHGC *env);