Merge pull request #16360 from LinHu2016/GC_Loom_fix

Fix up global continuation lists instead of rebuilding for compaction

runtime/gc_glue_java/CompactDelegate.cpp

@@ -31,8 +31,6 @@
 #include "MarkMap.hpp"
 #include "OwnableSynchronizerObjectBuffer.hpp"
 #include "OwnableSynchronizerObjectList.hpp"
-#include "ContinuationObjectBuffer.hpp"
-#include "ContinuationObjectList.hpp"
 #include "PointerContiguousArrayIterator.hpp"
 
 #if defined(OMR_GC_MODRON_COMPACTION)
@@ -132,8 +130,6 @@ MM_CompactDelegate::workerCleanupAfterGC(MM_EnvironmentBase *env)
 {
 	/* flush ownable synchronizer object buffer after rebuild the ownableSynchronizerObjectList during fixupObjects */
 	env->getGCEnvironment()->_ownableSynchronizerObjectBuffer->flush(env);
-	/* flush continuation object buffer after rebuild the continuationObjectList during fixupObjects */
-	env->getGCEnvironment()->_continuationObjectBuffer->flush(env);
 }
 
 void
@@ -146,7 +142,6 @@ MM_CompactDelegate::mainSetupForGC(MM_EnvironmentBase *env)
 		MM_HeapRegionDescriptorStandardExtension *regionExtension = MM_ConfigurationDelegate::getHeapRegionDescriptorStandardExtension(env, region);
 		for (uintptr_t i = 0; i < regionExtension->_maxListIndex; i++) {
 			regionExtension->_ownableSynchronizerObjectLists[i].startOwnableSynchronizerProcessing();
-			regionExtension->_continuationObjectLists[i].startProcessing();
 		}
 	}
 }

runtime/gc_glue_java/CompactSchemeFixupObject.cpp

@@ -32,7 +32,6 @@
 #include "MixedObjectIterator.hpp"
 #include "ObjectAccessBarrier.hpp"
 #include "OwnableSynchronizerObjectBuffer.hpp"
-#include "ContinuationObjectBuffer.hpp"
 #include "VMHelpers.hpp"
 #include "ParallelDispatcher.hpp"
 #include "PointerContiguousArrayIterator.hpp"
@@ -126,12 +125,6 @@ MM_CompactSchemeFixupObject::addOwnableSynchronizerObjectInList(MM_EnvironmentBa
 	}
 }
 
-MMINLINE void
-MM_CompactSchemeFixupObject::addContinuationObjectInList(MM_EnvironmentBase *env, omrobjectptr_t objectPtr)
-{
-	env->getGCEnvironment()->_continuationObjectBuffer->add(env, objectPtr);
-}
-
 void
 MM_CompactSchemeFixupObject::fixupObject(MM_EnvironmentStandard *env, omrobjectptr_t objectPtr)
 {
@@ -158,7 +151,6 @@ MM_CompactSchemeFixupObject::fixupObject(MM_EnvironmentStandard *env, omrobjectp
 		fixupMixedObject(objectPtr);
 		break;
 	case GC_ObjectModel::SCAN_CONTINUATION_OBJECT:
-		addContinuationObjectInList(env, objectPtr);
 		fixupContinuationObject(env, objectPtr);
 		break;
 	case GC_ObjectModel::SCAN_FLATTENED_ARRAY_OBJECT:

runtime/gc_glue_java/CompactSchemeFixupObject.hpp

@@ -82,8 +82,6 @@ class MM_CompactSchemeFixupObject {
 	 * @param object -- The object of type or subclass of java.util.concurrent.locks.AbstractOwnableSynchronizer.
 	 */
 	MMINLINE void addOwnableSynchronizerObjectInList(MM_EnvironmentBase *env, omrobjectptr_t objectPtr);
-
-	MMINLINE void addContinuationObjectInList(MM_EnvironmentBase *env, omrobjectptr_t objectPtr);
 };
 
 typedef struct StackIteratorData4CompactSchemeFixupObject {

runtime/gc_glue_java/CompactSchemeFixupRoots.cpp

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2001, 2020 IBM Corp. and others
+ * Copyright (c) 2001, 2022 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -36,6 +36,8 @@
 #include "Task.hpp"
 #include "UnfinalizedObjectBuffer.hpp"
 #include "UnfinalizedObjectList.hpp"
+#include "ContinuationObjectBuffer.hpp"
+#include "ContinuationObjectList.hpp"
 
 
 #if defined(J9VM_GC_FINALIZATION)
@@ -97,7 +99,7 @@ MM_CompactSchemeFixupRoots::fixupUnfinalizedObjects(MM_EnvironmentBase *env)
 		GC_HeapRegionIteratorStandard regionIterator(extensions->getHeap()->getHeapRegionManager());
 		while(NULL != (region = regionIterator.nextRegion())) {
 			MM_HeapRegionDescriptorStandardExtension *regionExtension = MM_ConfigurationDelegate::getHeapRegionDescriptorStandardExtension(env, region);
-			for (UDATA i = 0; i < regionExtension->_maxListIndex; i++) {
+			for (uintptr_t i = 0; i < regionExtension->_maxListIndex; i++) {
 				MM_UnfinalizedObjectList *list = &regionExtension->_unfinalizedObjectLists[i];
 				list->startUnfinalizedProcessing();
 			}
@@ -108,7 +110,7 @@ MM_CompactSchemeFixupRoots::fixupUnfinalizedObjects(MM_EnvironmentBase *env)
 	GC_HeapRegionIteratorStandard regionIterator(extensions->getHeap()->getHeapRegionManager());
 	while(NULL != (region = regionIterator.nextRegion())) {
 		MM_HeapRegionDescriptorStandardExtension *regionExtension = MM_ConfigurationDelegate::getHeapRegionDescriptorStandardExtension(env, region);
-		for (UDATA i = 0; i < regionExtension->_maxListIndex; i++) {
+		for (uintptr_t i = 0; i < regionExtension->_maxListIndex; i++) {
 			MM_UnfinalizedObjectList *list = &regionExtension->_unfinalizedObjectLists[i];
 			if (!list->wasEmpty()) {
 				if(J9MODRON_HANDLE_NEXT_WORK_UNIT(env)) {
@@ -129,3 +131,45 @@ MM_CompactSchemeFixupRoots::fixupUnfinalizedObjects(MM_EnvironmentBase *env)
 	env->getGCEnvironment()->_unfinalizedObjectBuffer->flush(env);
 }
 #endif /* J9VM_GC_FINALIZATION */
+
+
+void
+MM_CompactSchemeFixupRoots::fixupContinuationObjects(MM_EnvironmentBase *env)
+{
+	MM_GCExtensions* extensions = MM_GCExtensions::getExtensions(env);
+	if (env->_currentTask->synchronizeGCThreadsAndReleaseMain(env, UNIQUE_ID)) {
+		MM_HeapRegionDescriptorStandard *region = NULL;
+		GC_HeapRegionIteratorStandard regionIterator(extensions->getHeap()->getHeapRegionManager());
+		while(NULL != (region = regionIterator.nextRegion())) {
+			MM_HeapRegionDescriptorStandardExtension *regionExtension = MM_ConfigurationDelegate::getHeapRegionDescriptorStandardExtension(env, region);
+			for (uintptr_t i = 0; i < regionExtension->_maxListIndex; i++) {
+				MM_ContinuationObjectList *list = &regionExtension->_continuationObjectLists[i];
+				list->startProcessing();
+			}
+		}
+		env->_currentTask->releaseSynchronizedGCThreads(env);
+	}
+	MM_HeapRegionDescriptorStandard *region = NULL;
+	GC_HeapRegionIteratorStandard regionIterator(extensions->getHeap()->getHeapRegionManager());
+	while(NULL != (region = regionIterator.nextRegion())) {
+		MM_HeapRegionDescriptorStandardExtension *regionExtension = MM_ConfigurationDelegate::getHeapRegionDescriptorStandardExtension(env, region);
+		for (uintptr_t i = 0; i < regionExtension->_maxListIndex; i++) {
+			MM_ContinuationObjectList *list = &regionExtension->_continuationObjectLists[i];
+			if (!list->wasEmpty()) {
+				if(J9MODRON_HANDLE_NEXT_WORK_UNIT(env)) {
+					omrobjectptr_t object = list->getPriorList();
+					while (NULL != object) {
+						omrobjectptr_t forwardedPtr = _compactScheme->getForwardingPtr(object);
+						/* read the next link out of the moved copy of the object before we add it to the buffer */
+						object = extensions->accessBarrier->getContinuationLink(forwardedPtr);
+						/* store the object in this thread's buffer. It will be flushed to the appropriate list when necessary. */
+						env->getGCEnvironment()->_continuationObjectBuffer->add(env, forwardedPtr);
+					}
+				}
+			}
+		}
+	}
+
+	/* restore everything to a flushed state before exiting */
+	env->getGCEnvironment()->_continuationObjectBuffer->flush(env);
+}

runtime/gc_glue_java/CompactSchemeFixupRoots.hpp

@@ -73,7 +73,6 @@ class MM_CompactSchemeFixupRoots : public MM_RootScanner {
 	virtual void scanUnfinalizedObjects(MM_EnvironmentBase *env) {
 		/* allow the compact scheme to handle this */
 		reportScanningStarted(RootScannerEntity_UnfinalizedObjects);
-		MM_CompactSchemeFixupObject fixupObject(env, _compactScheme);
 		fixupUnfinalizedObjects(env);
 		reportScanningEnded(RootScannerEntity_UnfinalizedObjects);
 	}
@@ -87,7 +86,6 @@ class MM_CompactSchemeFixupRoots : public MM_RootScanner {
 	virtual void scanFinalizableObjects(MM_EnvironmentBase *env) {
 		if (_singleThread || J9MODRON_HANDLE_NEXT_WORK_UNIT(env)) {
 			reportScanningStarted(RootScannerEntity_FinalizableObjects);
-			MM_CompactSchemeFixupObject fixupObject(env, _compactScheme);
 			fixupFinalizableObjects(env);
 			reportScanningEnded(RootScannerEntity_FinalizableObjects);
 		}
@@ -99,13 +97,17 @@ class MM_CompactSchemeFixupRoots : public MM_RootScanner {
 	}
 
 	virtual void scanContinuationObjects(MM_EnvironmentBase *env) {
-		/* empty, move continuation processing in fixupObject */
+		/* allow the compact scheme to handle this */
+		reportScanningStarted(RootScannerEntity_ContinuationObjects);
+		fixupContinuationObjects(env);
+		reportScanningEnded(RootScannerEntity_ContinuationObjects);
 	}
 
 private:
 #if defined(J9VM_GC_FINALIZATION)
 	void fixupFinalizableObjects(MM_EnvironmentBase *env);
 	void fixupUnfinalizedObjects(MM_EnvironmentBase *env);
 #endif
+	void fixupContinuationObjects(MM_EnvironmentBase *env);
 };
 #endif /* FIXUPROOTS_HPP_ */

runtime/gc_vlhgc/WriteOnceCompactor.cpp

@@ -43,6 +43,8 @@
 #include "ClassLoaderIterator.hpp"
 #include "ClassLoaderRememberedSet.hpp"
 #include "CompactGroupManager.hpp"
+#include "ContinuationObjectBuffer.hpp"
+#include "ContinuationObjectList.hpp"
 #include "VMHelpers.hpp"
 #include "WriteOnceCompactor.hpp"
 #include "Debug.hpp"
@@ -558,7 +560,6 @@ MM_WriteOnceCompactor::compact(MM_EnvironmentVLHGC *env)
 	env->_compactVLHGCStats._moveStartTime = timeTemp;
 	moveObjects(env);
 	env->getGCEnvironment()->_ownableSynchronizerObjectBuffer->flush(env);
-	env->getGCEnvironment()->_continuationObjectBuffer->flush(env);
 	/* Note:  moveObjects implicitly synchronizes threads */
 	timeTemp = j9time_hires_clock();
 	env->_compactVLHGCStats._moveEndTime = timeTemp;
@@ -1456,7 +1457,6 @@ MM_WriteOnceCompactor::fixupObject(MM_EnvironmentVLHGC* env, J9Object *objectPtr
 		fixupMixedObject(env, objectPtr, cache);
 		break;
 	case GC_ObjectModel::SCAN_CONTINUATION_OBJECT:
-		addContinuationObjectInList(env, objectPtr);
 		fixupContinuationObject(env, objectPtr, cache);
 		break;
 	case GC_ObjectModel::SCAN_CLASS_OBJECT:
@@ -2041,11 +2041,27 @@ MM_WriteOnceCompactor::fixupArrayletLeafRegionContentsAndObjectLists(MM_Environm
 					}
 				}
 			}
+			if (!region->getContinuationObjectList()->wasEmpty()) {
+				if (J9MODRON_HANDLE_NEXT_WORK_UNIT(env)) {
+					J9Object *pointer = region->getContinuationObjectList()->getPriorList();
+					while (NULL != pointer) {
+						Assert_MM_true(region->isAddressInRegion(pointer));
+						J9Object* forwardedPtr = getForwardingPtr(pointer);
+
+						/* read the next link out of the moved copy of the object before we add it to the buffer */
+						pointer = _extensions->accessBarrier->getContinuationLink(forwardedPtr);
+
+						/* store the object in this thread's buffer. It will be flushed to the appropriate list when necessary. */
+						env->getGCEnvironment()->_continuationObjectBuffer->add(env, forwardedPtr);
+					}
+				}
+			}
 		}
 	}
 
 	/* restore everything to a flushed state before exiting */
 	env->getGCEnvironment()->_unfinalizedObjectBuffer->flush(env);
+	env->getGCEnvironment()->_continuationObjectBuffer->flush(env);
 }
 
 void

runtime/gc_vlhgc/WriteOnceCompactor.hpp

@@ -514,11 +514,6 @@ class MM_WriteOnceCompactor : public MM_BaseVirtual
 		}
 	}
 
-	MMINLINE void addContinuationObjectInList(MM_EnvironmentVLHGC *env, J9Object *objectPtr)
-	{
-		((MM_ContinuationObjectBufferVLHGC*) env->getGCEnvironment()->_continuationObjectBuffer)->addForOnlyCompactedRegion(env, objectPtr);
-	}
-
 #if defined(J9VM_GC_FINALIZATION)
 	void fixupFinalizableObjects(MM_EnvironmentVLHGC *env);