Merge pull request #5318 from bragaigor/contiguous

Add double map API for LINUX and balanced GC policy

buildspecs/j9.flags

@@ -334,6 +334,17 @@ Currently only available on linux_x86 and win_x96 32 bit builds.</description>
 			<require flag="gc_realtime"/>
 		</requires>
 	</flag>
+	<flag id="gc_enableDoubleMap">
+		<description>Set on double map. Allows LINUX systems to double map arrays that are stored as arraylets.
+When enabled, a contiguous block of memory is created for each array which data surpasses the size of a region. This contiguous block represents the array as
+if the data was stored in a contiguous region of memory. All of the array data will be stored at their own region (not with spine); hence, all arraylets
+become discontiguous whenever this flag is enabled. Since there won’t be any empty arraylet leaves, then arrayoid NULL pointers are no longer required since
+all data is stored in their own region. It additionaly reduces footprint, mainly for JNI primitive array critical.</description>
+		<ifRemoved></ifRemoved>
+		<requires>
+			<require flag="gc_vlhgc"/>
+		</requires>
+	</flag>
 	<flag id="gc_compressedPointerBarrier">
 		<description>VM performs runtime checks for missed access barriers in a compressed pointer sense</description>
 		<ifRemoved>VM does not check for missed access barriers in a compressed pointer sense</ifRemoved>

buildspecs/linux_390-64.spec

@@ -134,6 +134,7 @@ SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 OR GPL-2.0 WITH Classpath-excepti
 		<flag id="gc_debugAsserts" value="true"/>
 		<flag id="gc_idleHeapManager" value="true"/>
 		<flag id="gc_inlinedAllocFields" value="true"/>
+		<flag id="gc_enableDoubleMap" value="true"/>
 		<flag id="gc_minimumObjectSize" value="true"/>
 		<flag id="gc_subpoolsAlias" value="true"/>
 		<flag id="graph_cmdLineTester" value="true"/>

buildspecs/linux_390-64_cmprssptrs.spec

@@ -137,6 +137,7 @@ SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 OR GPL-2.0 WITH Classpath-excepti
 		<flag id="gc_debugAsserts" value="true"/>
 		<flag id="gc_idleHeapManager" value="true"/>
 		<flag id="gc_inlinedAllocFields" value="true"/>
+		<flag id="gc_enableDoubleMap" value="true"/>
 		<flag id="gc_minimumObjectSize" value="true"/>
 		<flag id="gc_subpoolsAlias" value="true"/>
 		<flag id="graph_cmdLineTester" value="true"/>

buildspecs/linux_ppc-64.spec

@@ -138,6 +138,7 @@ SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 OR GPL-2.0 WITH Classpath-excepti
 		<flag id="gc_debugAsserts" value="true"/>
 		<flag id="gc_idleHeapManager" value="true"/>
 		<flag id="gc_inlinedAllocFields" value="true"/>
+		<flag id="gc_enableDoubleMap" value="true"/>
 		<flag id="gc_minimumObjectSize" value="true"/>
 		<flag id="gc_subpoolsAlias" value="true"/>
 		<flag id="graph_cmdLineTester" value="true"/>

buildspecs/linux_ppc-64_cmprssptrs.spec

@@ -136,6 +136,7 @@ SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 OR GPL-2.0 WITH Classpath-excepti
 		<flag id="gc_debugAsserts" value="true"/>
 		<flag id="gc_idleHeapManager" value="true"/>
 		<flag id="gc_inlinedAllocFields" value="true"/>
+		<flag id="gc_enableDoubleMap" value="true"/>
 		<flag id="gc_minimumObjectSize" value="true"/>
 		<flag id="gc_subpoolsAlias" value="true"/>
 		<flag id="graph_cmdLineTester" value="true"/>

buildspecs/linux_ppc-64_cmprssptrs_le.spec

@@ -138,6 +138,7 @@ SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 OR GPL-2.0 WITH Classpath-excepti
 		<flag id="gc_debugAsserts" value="true"/>
 		<flag id="gc_idleHeapManager" value="true"/>
 		<flag id="gc_inlinedAllocFields" value="true"/>
+		<flag id="gc_enableDoubleMap" value="true"/>
 		<flag id="gc_minimumObjectSize" value="true"/>
 		<flag id="gc_subpoolsAlias" value="true"/>
 		<flag id="graph_cmdLineTester" value="true"/>

buildspecs/linux_ppc-64_cmprssptrs_le_gcc.spec

@@ -140,6 +140,7 @@ SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 OR GPL-2.0 WITH Classpath-excepti
 		<flag id="gc_batchClearTLH" value="true"/>
 		<flag id="gc_debugAsserts" value="true"/>
 		<flag id="gc_idleHeapManager" value="true"/>
+		<flag id="gc_enableDoubleMap" value="true"/>
 		<flag id="gc_inlinedAllocFields" value="true"/>
 		<flag id="gc_minimumObjectSize" value="true"/>
 		<flag id="gc_subpoolsAlias" value="true"/>

buildspecs/linux_ppc-64_le.spec

@@ -139,6 +139,7 @@ SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 OR GPL-2.0 WITH Classpath-excepti
 		<flag id="gc_debugAsserts" value="true"/>
 		<flag id="gc_idleHeapManager" value="true"/>
 		<flag id="gc_inlinedAllocFields" value="true"/>
+		<flag id="gc_enableDoubleMap" value="true"/>
 		<flag id="gc_minimumObjectSize" value="true"/>
 		<flag id="gc_subpoolsAlias" value="true"/>
 		<flag id="graph_cmdLineTester" value="true"/>

buildspecs/linux_ppc-64_le_gcc.spec

@@ -139,6 +139,7 @@ SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 OR GPL-2.0 WITH Classpath-excepti
 		<flag id="gc_debugAsserts" value="true"/>
 		<flag id="gc_idleHeapManager" value="true"/>
 		<flag id="gc_inlinedAllocFields" value="true"/>
+		<flag id="gc_enableDoubleMap" value="true"/>
 		<flag id="gc_minimumObjectSize" value="true"/>
 		<flag id="gc_subpoolsAlias" value="true"/>
 		<flag id="graph_cmdLineTester" value="true"/>

buildspecs/linux_x86-64.spec

@@ -141,6 +141,7 @@ SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 OR GPL-2.0 WITH Classpath-excepti
 		<flag id="gc_debugAsserts" value="true"/>
 		<flag id="gc_idleHeapManager" value="true"/>
 		<flag id="gc_inlinedAllocFields" value="true"/>
+		<flag id="gc_enableDoubleMap" value="true"/>
 		<flag id="gc_minimumObjectSize" value="true"/>
 		<flag id="gc_tlhPrefetchFTA" value="true"/>
 		<flag id="graph_cmdLineTester" value="true"/>

buildspecs/linux_x86-64_cmprssptrs.spec

@@ -138,6 +138,7 @@ SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 OR GPL-2.0 WITH Classpath-excepti
 		<flag id="gc_debugAsserts" value="true"/>
 		<flag id="gc_idleHeapManager" value="true"/>
 		<flag id="gc_inlinedAllocFields" value="true"/>
+		<flag id="gc_enableDoubleMap" value="true"/>
 		<flag id="gc_minimumObjectSize" value="true"/>
 		<flag id="gc_tlhPrefetchFTA" value="true"/>
 		<flag id="graph_cmdLineTester" value="true"/>

runtime/gc_base/IndexableObjectAllocationModel.cpp

@@ -26,6 +26,12 @@
 #include "IndexableObjectAllocationModel.hpp"
 #include "Math.hpp"
 #include "MemorySpace.hpp"
+#if defined(J9VM_GC_ENABLE_DOUBLE_MAP)
+#include "ArrayletLeafIterator.hpp"
+#include "HeapRegionManager.hpp"
+#include "HeapRegionDescriptorVLHGC.hpp"
+#include "Heap.hpp"
+#endif /* J9VM_GC_ENABLE_DOUBLE_MAP */
 
 /**
  * Allocation description and layout initialization. This is called before OMR allocates
@@ -251,6 +257,17 @@ MM_IndexableObjectAllocationModel::layoutDiscontiguousArraylet(MM_EnvironmentBas
 	if (NULL != spine) {
 		switch (_layout) {
 		case GC_ArrayletObjectModel::Discontiguous:
+#if defined(J9VM_GC_ENABLE_DOUBLE_MAP)
+			if (extensions->indexableObjectModel.isDoubleMappingEnabled()) {
+				/**
+				 * There are some special cases where double mapping an arraylet is
+				 * not necessary; isArrayletDataDiscontiguous() details those cases.
+				 */
+				if (extensions->indexableObjectModel.isArrayletDataDiscontiguous(spine)) {
+					doubleMapArraylets(env, (J9Object *)spine);
+				}
+			}
+#endif /* J9VM_GC_ENABLE_DOUBLE_MAP */
 			/* if last arraylet leaf is empty (contains 0 bytes) arrayoid pointer is set to NULL */
 			if (arrayoidIndex == (_numberOfArraylets - 1)) {
 				Assert_MM_true(0 == (_dataSize % arrayletLeafSize));
@@ -263,6 +280,12 @@ MM_IndexableObjectAllocationModel::layoutDiscontiguousArraylet(MM_EnvironmentBas
 			break;
 
 		case GC_ArrayletObjectModel::Hybrid:
+#if defined(J9VM_GC_ENABLE_DOUBLE_MAP)
+			/* Unreachable if double map is enabled */
+			if (extensions->indexableObjectModel.isDoubleMappingEnabled()) {
+				Assert_MM_double_map_unreachable();
+			}
+#endif /* J9VM_GC_ENABLE_DOUBLE_MAP */
 			/* last arrayoid points to end of arrayoid array in spine header (object-aligned if
 			 * required). (data size % leaf size) bytes of data are stored here (may be empty).
 			 */
@@ -287,5 +310,81 @@ MM_IndexableObjectAllocationModel::layoutDiscontiguousArraylet(MM_EnvironmentBas
 	return spine;
 }
 
+#if defined(J9VM_GC_ENABLE_DOUBLE_MAP)
+#if !(defined(LINUX) && defined(J9VM_ENV_DATA64))
+/* Double map is only supported on LINUX 64 bit Systems for now */
+#error "Platform not supported by Double Map API"
+#endif /* !(defined(LINUX) && defined(J9VM_ENV_DATA64)) */
+void * 
+MM_IndexableObjectAllocationModel::doubleMapArraylets(MM_EnvironmentBase *env, J9Object *objectPtr) 
+{
+	MM_GCExtensions *extensions = MM_GCExtensions::getExtensions(env);
+	J9JavaVM *javaVM = extensions->getJavaVM();
+	PORT_ACCESS_FROM_ENVIRONMENT(env);
+
+	GC_ArrayletLeafIterator arrayletLeafIterator(javaVM, (J9IndexableObject *)objectPtr);
+	MM_Heap *heap = extensions->getHeap();
+	UDATA arrayletLeafSize = env->getOmrVM()->_arrayletLeafSize;
+	UDATA arrayletLeafCount = MM_Math::roundToCeiling(arrayletLeafSize, _dataSize) / arrayletLeafSize;
+
+	void *result = NULL;
+
+#define ARRAYLET_ALLOC_THRESHOLD 64
+	void *leaves[ARRAYLET_ALLOC_THRESHOLD];
+	void **arrayletLeaveAddrs = leaves;
+	if (arrayletLeafCount > ARRAYLET_ALLOC_THRESHOLD) {
+		arrayletLeaveAddrs = (void **)env->getForge()->allocate(arrayletLeafCount * sizeof(uintptr_t), MM_AllocationCategory::GC_HEAP, J9_GET_CALLSITE());
+	}
+
+	if (NULL == arrayletLeaveAddrs) {
+		return NULL;
+	}
+
+	GC_SlotObject *slotObject = NULL;
+	uintptr_t count = 0;
+
+	while (NULL != (slotObject = arrayletLeafIterator.nextLeafPointer())) {
+		void *currentLeaf = slotObject->readReferenceFromSlot();
+		/* In some corner cases the last leaf might be NULL therefore we must ignore it */
+		if (NULL == currentLeaf) {
+			break;
+		}
+		arrayletLeaveAddrs[count] = currentLeaf;
+		count++;
+	}
+
+	/* Number of arraylet leaves in the iterator must match the number of leaves calculated */
+	Assert_MM_true(arrayletLeafCount == count);
+
+	GC_SlotObject objectSlot(env->getOmrVM(), &extensions->indexableObjectModel.getArrayoidPointer((J9IndexableObject *)objectPtr)[0]);
+	J9Object *firstLeafSlot = objectSlot.readReferenceFromSlot();
+
+	MM_HeapRegionDescriptorVLHGC *firstLeafRegionDescriptor = (MM_HeapRegionDescriptorVLHGC *)heap->getHeapRegionManager()->tableDescriptorForAddress(firstLeafSlot);
+
+	/* gets pagesize  or j9vmem_supported_page_sizes()[0]? */
+	UDATA pageSize = j9mmap_get_region_granularity(NULL);
 
+	/* Get heap and from there call an OMR API that will doble map everything */
+	result = heap->doubleMapArraylet(env, arrayletLeaveAddrs, count, arrayletLeafSize, _dataSize,
+				&firstLeafRegionDescriptor->_arrayletDoublemapID,
+				pageSize);
+
+	if (arrayletLeafCount > ARRAYLET_ALLOC_THRESHOLD) {
+		env->getForge()->free((void *)arrayletLeaveAddrs);
+	}
+
+	/*
+	 * Double map failed.
+	 * If doublemap fails the caller must handle it appropriately. The only case being
+	 * JNI critical, where it will fall back to copying each element of the array to
+	 * a temporary array (logic handled by JNI Critical). It might hurt performance
+	 * but execution won't halt.
+	 */
+	if (NULL == firstLeafRegionDescriptor->_arrayletDoublemapID.address) {
+		result = NULL;
+	}
+
+	return result;
+}
+#endif /* J9VM_GC_ENABLE_DOUBLE_MAP */
 

runtime/gc_base/IndexableObjectAllocationModel.hpp

@@ -1,6 +1,6 @@
 
 /*******************************************************************************
- * Copyright (c) 1991, 2014 IBM Corp. and others
+ * Copyright (c) 1991, 2019 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -124,6 +124,19 @@ class MM_IndexableObjectAllocationModel : public MM_JavaObjectAllocationModel
 	 */
 	bool initializeAllocateDescription(MM_EnvironmentBase *env);
 
+#if defined(J9VM_GC_ENABLE_DOUBLE_MAP)
+	/**
+	 * For non-contiguous arraylets (discontiguous arraylets, hybrid not allowed
+	 * when double map is enabled), double maps the arraylet leaves to a contiguous
+	 * region outside the heap, making a discontiguous arraylet look contiguous
+	 *
+	 * @param env thread GC Environment
+	 * @param objectPtr indexable object spine
+	 * @return the contiguous address pointer
+	 */
+	void *doubleMapArraylets(MM_EnvironmentBase *env, J9Object *objectPtr);
+#endif /* J9VM_GC_ENABLE_DOUBLE_MAP */
+
 	/**
 	 * Initializer.
 	 */

runtime/gc_base/RootScanner.cpp

@@ -46,6 +46,9 @@
 #include "HeapRegionDescriptor.hpp"
 #include "HeapRegionIterator.hpp"
 #include "HeapRegionManager.hpp"
+#if defined(J9VM_GC_ENABLE_DOUBLE_MAP)
+#include "HeapRegionIteratorVLHGC.hpp"
+#endif /* J9VM_GC_ENABLE_DOUBLE_MAP */
 #include "MemoryPool.hpp"
 #include "MemorySubSpace.hpp"
 #include "MemorySpace.hpp"
@@ -227,6 +230,14 @@ MM_RootScanner::doStringTableSlot(J9Object **slotPtr, GC_StringTableIterator *st
 	doSlot(slotPtr);
 }
 
+#if defined(J9VM_GC_ENABLE_DOUBLE_MAP)
+void
+MM_RootScanner::doDoubleMappedObjectSlot(J9Object *objectPtr, struct J9PortVmemIdentifier *identifier)
+{
+	/* No need to call doSlot() here since there's nothing to update */
+}
+#endif /* J9VM_GC_ENABLE_DOUBLE_MAP */
+
 /**
  * @Perform operation on the given string cache table slot.
  * @String table cache contains cached entries of string table, it's
@@ -856,6 +867,29 @@ MM_RootScanner::scanJVMTIObjectTagTables(MM_EnvironmentBase *env)
 }
 #endif /* J9VM_OPT_JVMTI */
 
+#if defined(J9VM_GC_ENABLE_DOUBLE_MAP)
+void 
+MM_RootScanner::scanDoubleMappedObjects(MM_EnvironmentBase *env)
+{
+	if (_singleThread || J9MODRON_HANDLE_NEXT_WORK_UNIT(env)) {
+		GC_HeapRegionIteratorVLHGC regionIterator(_extensions->heap->getHeapRegionManager());
+		MM_HeapRegionDescriptorVLHGC *region = NULL;
+		reportScanningStarted(RootScannerEntity_DoubleMappedObjects);
+		while (NULL != (region = regionIterator.nextRegion())) {
+			if (region->isArrayletLeaf()) {
+				J9Object *spineObject = (J9Object *)region->_allocateData.getSpine();
+				Assert_MM_true(NULL != spineObject);
+				J9PortVmemIdentifier *arrayletDoublemapID = &region->_arrayletDoublemapID;
+				if (NULL != arrayletDoublemapID->address) {
+					doDoubleMappedObjectSlot(spineObject, arrayletDoublemapID);
+				}
+			}
+		}
+		reportScanningEnded(RootScannerEntity_DoubleMappedObjects);
+	}
+}
+#endif /* J9VM_GC_ENABLE_DOUBLE_MAP */
+
 /**
  * Scan all root set references from the VM into the heap.
  * For all slots that are hard root references into the heap, the appropriate slot handler will be called.
@@ -980,6 +1014,12 @@ MM_RootScanner::scanClearable(MM_EnvironmentBase *env)
 		scanJVMTIObjectTagTables(env);
 	}
 #endif /* J9VM_OPT_JVMTI */
+
+#if defined(J9VM_GC_ENABLE_DOUBLE_MAP)
+	if (_includeDoubleMap) {
+		scanDoubleMappedObjects(env);
+	}
+#endif /* J9VM_GC_ENABLE_DOUBLE_MAP */
 }
 
 /**
@@ -1029,6 +1069,12 @@ MM_RootScanner::scanAllSlots(MM_EnvironmentBase *env)
 	}
 #endif /* J9VM_OPT_JVMTI */
 
+#if defined(J9VM_GC_ENABLE_DOUBLE_MAP)
+        if (_includeDoubleMap) {
+                scanDoubleMappedObjects(env);
+        }
+#endif /* J9VM_GC_ENABLE_DOUBLE_MAP */
+
 	scanOwnableSynchronizerObjects(env);
 }
 

runtime/gc_base/RootScanner.hpp

@@ -91,6 +91,9 @@ class MM_RootScanner : public MM_BaseVirtual
 #endif /* J9VM_GC_MODRON_SCAVENGER */	 	
 	bool _classDataAsRoots; /**< Should all classes (and class loaders) be treated as roots. Default true, should set to false when class unloading */
 	bool _includeJVMTIObjectTagTables; /**< Should the iterator include the JVMTIObjectTagTables. Default true, should set to false when doing JVMTI object walks */
+#if defined(J9VM_GC_ENABLE_DOUBLE_MAP)
+	bool _includeDoubleMap; /**< Enables doublemap should the GC policy be balanced. Default is false. */
+#endif /* J9VM_GC_ENABLE_DOUBLE_MAP */
 	bool _trackVisibleStackFrameDepth; /**< Should the stack walker be told to track the visible frame depth. Default false, should set to true when doing JVMTI walks that report stack slots */
 
 	U_64 _entityStartScanTime; /**< The start time of the scan of the current scanning entity, or 0 if no entity is being scanned.  Defaults to 0. */
@@ -304,6 +307,9 @@ class MM_RootScanner : public MM_BaseVirtual
 #endif /* J9VM_GC_MODRON_SCAVENGER */
 		, _classDataAsRoots(true)
 		, _includeJVMTIObjectTagTables(true)
+#if defined(J9VM_GC_ENABLE_DOUBLE_MAP)
+		, _includeDoubleMap(_extensions->indexableObjectModel.isDoubleMappingEnabled())
+#endif /* J9VM_GC_ENABLE_DOUBLE_MAP */
 		, _trackVisibleStackFrameDepth(false)
 		, _entityStartScanTime(0)
 		, _entityIncrementStartTime(0)
@@ -434,6 +440,19 @@ class MM_RootScanner : public MM_BaseVirtual
 	void scanJVMTIObjectTagTables(MM_EnvironmentBase *env);
 #endif /* J9VM_OPT_JVMTI */
 
+#if defined(J9VM_GC_ENABLE_DOUBLE_MAP)
+	/**
+	 * Scans each heap region for arraylet leaves that contains a not NULL
+	 * contiguous address. This address points to a contiguous representation
+	 * of the arraylet associated with this leaf. Only arraylets that has been
+	 * double mapped will contain such contiguous address, otherwise the
+	 * address will be NULL
+	 * 
+	 * @param env thread GC Environment
+	 */
+	void scanDoubleMappedObjects(MM_EnvironmentBase *env);
+#endif /* J9VM_GC_ENABLE_DOUBLE_MAP */
+
 	virtual void doClassLoader(J9ClassLoader *classLoader);
 
 	virtual void scanWeakReferenceObjects(MM_EnvironmentBase *env);
@@ -487,6 +506,17 @@ class MM_RootScanner : public MM_BaseVirtual
 	virtual void doStringCacheTableSlot(J9Object **slotPtr);
 	virtual void doVMClassSlot(J9Class **slotPtr, GC_VMClassSlotIterator *vmClassSlotIterator);
 	virtual void doVMThreadSlot(J9Object **slotPtr, GC_VMThreadIterator *vmThreadIterator);
+#if defined(J9VM_GC_ENABLE_DOUBLE_MAP)
+	/**
+	 * Frees double mapped region associated to objectPtr (arraylet spine) if objectPtr
+	 * is not live
+	 *
+	 * @param objectPtr[in] indexable object's spine
+	 * @param identifier[in/out] identifier associated with object's spine, which contains
+	 * doble mapped address and size
+	 */
+	virtual void doDoubleMappedObjectSlot(J9Object *objectPtr, struct J9PortVmemIdentifier *identifier);
+#endif /* J9VM_GC_ENABLE_DOUBLE_MAP */
 
 	/**
 	 * Called for each object stack slot. Subclasses may override.