Add double map API for OSX

Update double map API to allow to reserve contiguous address
at a FIXED address.

Add release double map region API to free memory associated
to double map accordingly

Include extra test to test new version of double map API.

Refactor double map API to generalize methods and variables names.
Discontiguous arraylets from balanced GC policy is one use case
of double mapping API, but the API can be used for any consumer
that wants to double map multiple regions of memory into a single
contiguous region.

mmap(2) in OSX is limitted where it won't allow a mapping that's
backed by a file (have a file descriptor) and huge pages to
work together. That's because huge pages flag is specified
using the file descriptor field; therefore, we cannot have
double mapping, which requires the heap to be backed by a
file descriptor, and huge pages since both use the same field.
In such scenario, if huge pages is available and double map
is requested, huge pages takes precedence over double mapping.

Signed-off-by: Igor Braga <higorb1@gmail.com>

gc/base/Heap.hpp

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 1991, 2016 IBM Corp. and others
+ * Copyright (c) 1991, 2020 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -117,6 +117,7 @@ class MM_Heap : public MM_BaseVirtual
 	virtual void *getHeapTop() = 0;
 #if defined(OMR_GC_DOUBLE_MAP_ARRAYLETS)
 	virtual void *doubleMapArraylet(MM_EnvironmentBase *env, void* arrayletLeaves[], UDATA arrayletLeafCount, UDATA arrayletLeafSize, UDATA byteAmount, struct J9PortVmemIdentifier *newIdentifier, UDATA pageSize) = 0;
+	virtual void *doubleMapRegions(MM_EnvironmentBase *env, void* regions[], UDATA regionsCount, UDATA regionSize, UDATA byteAmount, struct J9PortVmemIdentifier *newIdentifier, UDATA pageSize, void *preferredAddress) = 0;
 #endif /* defined(OMR_GC_DOUBLE_MAP_ARRAYLETS) */
 
 	virtual uintptr_t getMaximumPhysicalRange() = 0;

gc/base/HeapSplit.cpp

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 1991, 2015 IBM Corp. and others
+ * Copyright (c) 1991, 2020 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -190,6 +190,13 @@ MM_HeapSplit::doubleMapArraylet(MM_EnvironmentBase *env, void* arrayletLeaves[],
 	/* Unreachable */
 	return NULL;
 }
+
+void*
+MM_HeapSplit::doubleMapRegions(MM_EnvironmentBase *env, void* regions[], UDATA regionsCount, UDATA regionSize, UDATA byteAmount, struct J9PortVmemIdentifier *newIdentifier, UDATA pageSize, void *preferredAddress)
+{
+	/* Unreachable */
+	return NULL;
+}
 #endif /* defined(OMR_GC_DOUBLE_MAP_ARRAYLETS) */
 
 /**

gc/base/HeapSplit.hpp

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 1991, 2015 IBM Corp. and others
+ * Copyright (c) 1991, 2020 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -63,6 +63,7 @@ class MM_HeapSplit : public MM_Heap
 	virtual void *getHeapTop();
 #if defined(OMR_GC_DOUBLE_MAP_ARRAYLETS)
 	virtual void *doubleMapArraylet(MM_EnvironmentBase *env, void* arrayletLeaves[], UDATA arrayletLeafCount, UDATA arrayletLeafSize, UDATA byteAmount, struct J9PortVmemIdentifier *newIdentifier, UDATA pageSize);
+	virtual void *doubleMapRegions(MM_EnvironmentBase *env, void* regions[], UDATA regionsCount, UDATA regionSize, UDATA byteAmount, struct J9PortVmemIdentifier *newIdentifier, UDATA pageSize, void *preferredAddress);
 #endif /* defined(OMR_GC_DOUBLE_MAP_ARRAYLETS) */
 
 	virtual uintptr_t getMaximumPhysicalRange();

gc/base/HeapVirtualMemory.cpp

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 1991, 2018 IBM Corp. and others
+ * Copyright (c) 1991, 2020 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -181,6 +181,13 @@ MM_HeapVirtualMemory::doubleMapArraylet(MM_EnvironmentBase *env, void* arrayletL
 	MM_MemoryManager* memoryManager = MM_GCExtensionsBase::getExtensions(_omrVM)->memoryManager;
 	return memoryManager->doubleMapArraylet(&_vmemHandle, env, arrayletLeaves, arrayletLeafCount, arrayletLeafSize, byteAmount, newIdentifier, pageSize);
 }
+
+void*
+MM_HeapVirtualMemory::doubleMapRegions(MM_EnvironmentBase *env, void* regions[], UDATA regionsCount, UDATA regionSize, UDATA byteAmount, struct J9PortVmemIdentifier *newIdentifier, UDATA pageSize, void *preferredAddress)
+{
+	MM_MemoryManager* memoryManager = MM_GCExtensionsBase::getExtensions(_omrVM)->memoryManager;
+	return memoryManager->doubleMapRegions(&_vmemHandle, env, regions, regionsCount, regionSize, byteAmount, newIdentifier, pageSize, preferredAddress);
+}
 #endif /* defined(OMR_GC_DOUBLE_MAP_ARRAYLETS) */
 
 uintptr_t

gc/base/HeapVirtualMemory.hpp

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 1991, 2015 IBM Corp. and others
+ * Copyright (c) 1991, 2020 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -58,6 +58,7 @@ class MM_HeapVirtualMemory : public MM_Heap {
 	virtual void* getHeapTop();
 #if defined(OMR_GC_DOUBLE_MAP_ARRAYLETS)
 	virtual void *doubleMapArraylet(MM_EnvironmentBase *env, void* arrayletLeaves[], UDATA arrayletLeafCount, UDATA arrayletLeafSize, UDATA byteAmount, struct J9PortVmemIdentifier *newIdentifier, UDATA pageSize);
+	virtual void *doubleMapRegions(MM_EnvironmentBase *env, void* regions[], UDATA regionsCount, UDATA regionSize, UDATA byteAmount, struct J9PortVmemIdentifier *newIdentifier, UDATA pageSize, void *preferredAddress);
 #endif /* defined(OMR_GC_DOUBLE_MAP_ARRAYLETS) */
 
 	virtual uintptr_t getMaximumPhysicalRange();

gc/base/MemoryManager.cpp

@@ -576,6 +576,15 @@ MM_MemoryManager::doubleMapArraylet(MM_MemoryHandle* handle, MM_EnvironmentBase
 	Assert_MM_true(NULL != memory);
 	return memory->doubleMapArraylet(env, arrayletLeaves, arrayletLeafCount, arrayletLeafSize, byteAmount, newIdentifier, pageSize);
 }
+
+void*
+MM_MemoryManager::doubleMapRegions(MM_MemoryHandle* handle, MM_EnvironmentBase *env, void* regions[], UDATA regionsCount, UDATA regionSize, UDATA byteAmount, struct J9PortVmemIdentifier *newIdentifier, UDATA pageSize, void *preferredAddress)
+{
+	Assert_MM_true(NULL != handle);
+	MM_VirtualMemory* memory = handle->getVirtualMemory();
+	Assert_MM_true(NULL != memory);
+	return memory->doubleMapRegions(env, regions, regionsCount, regionSize, byteAmount, newIdentifier, pageSize, preferredAddress);
+}
 #endif /* defined(OMR_GC_DOUBLE_MAP_ARRAYLETS) */
 
 bool

gc/base/MemoryManager.hpp

@@ -176,6 +176,7 @@ class MM_MemoryManager : public MM_BaseNonVirtual {
 	 */
 	void destroyVirtualMemoryForHeap(MM_EnvironmentBase* env, MM_MemoryHandle* handle);
 
+#if defined(OMR_GC_DOUBLE_MAP_ARRAYLETS)
 	/**
  	 * Double maps arraylets, arrays that does not fit into one region are split into leaves, 
  	 * which are then double mapped by this function 
@@ -190,8 +191,24 @@ class MM_MemoryManager : public MM_BaseNonVirtual {
  	 * @param pageSize
  	 * @param category
   	 */
-#if defined(OMR_GC_DOUBLE_MAP_ARRAYLETS)
 	void *doubleMapArraylet(MM_MemoryHandle* handle, MM_EnvironmentBase *env, void* arrayletLeaves[], UDATA arrayletLeafCount, UDATA arrayletLeafSize, UDATA byteAmount, struct J9PortVmemIdentifier *newIdentifier, UDATA pageSize);
+
+	/**
+	 * Double maps regions. Discontiguous regions are double mapped to one contiguous region.
+	 *
+	 * @param pointer to memory handle
+	 * @param env environment
+	 * @param regions list of regions to be double mapped
+	 * @param regionsCount, number of regions to be double mapped
+	 * @param regionSize, size of each region
+	 * @param byteAmount, total byte amount to be allocate contiguous block of meory to double map
+	 * @param newIdentifier, hold information of newly created contiguous block of memory
+	 * @param pageSize
+	 * @param preferredAddress, prefered address of contiguous region to double map
+	 *
+	 * @return pointer to contiguous region to which regions were double mapped into, NULL is returned if unsuccessful
+	 */
+	void *doubleMapRegions(MM_MemoryHandle* handle, MM_EnvironmentBase *env, void* regions[], UDATA regionsCount, UDATA regionSize, UDATA byteAmount, struct J9PortVmemIdentifier *newIdentifier, UDATA pageSize, void *preferredAddress);
 #endif /* defined(OMR_GC_DOUBLE_MAP_ARRAYLETS) */
 
 	/**

gc/base/VirtualMemory.cpp

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 1991, 2016 IBM Corp. and others
+ * Copyright (c) 1991, 2020 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -171,6 +171,16 @@ MM_VirtualMemory::doubleMapArraylet(MM_EnvironmentBase *env, void* arrayletLeave
 
 	return omrvmem_get_contiguous_region_memory(arrayletLeaves, arrayletLeafCount, arrayletLeafSize, byteAmount, oldIdentifier, newIdentifier, mode, pageSize, omrmem_get_category(OMRMEM_CATEGORY_MM));
 }
+
+void*
+MM_VirtualMemory::doubleMapRegions(MM_EnvironmentBase *env, void* regions[], UDATA regionsCount, UDATA regionSize, UDATA byteAmount, struct J9PortVmemIdentifier *newIdentifier, UDATA pageSize, void *preferredAddress)
+{
+	OMRPORT_ACCESS_FROM_OMRVM(_extensions->getOmrVM());
+	struct J9PortVmemIdentifier *oldIdentifier = &_identifier;
+	uintptr_t mode = OMRPORT_VMEM_MEMORY_MODE_READ | OMRPORT_VMEM_MEMORY_MODE_WRITE | OMRPORT_VMEM_MEMORY_MODE_COMMIT;
+
+	return omrvmem_create_double_mapped_region(regions, regionsCount, regionSize, byteAmount, oldIdentifier, newIdentifier, mode, pageSize, omrmem_get_category(OMRMEM_CATEGORY_MM), preferredAddress);
+}
 #endif /* defined(OMR_GC_DOUBLE_MAP_ARRAYLETS) */
 
 bool MM_VirtualMemory::freeMemory()

gc/base/VirtualMemory.hpp

@@ -83,6 +83,7 @@ class MM_VirtualMemory : public MM_BaseVirtual {
 	virtual void* reserveMemory(J9PortVmemParams* params);
 #if defined(OMR_GC_DOUBLE_MAP_ARRAYLETS)
 	virtual void *doubleMapArraylet(MM_EnvironmentBase *env, void* arrayletLeaves[], UDATA arrayletLeafCount, UDATA arrayletLeafSize, UDATA byteAmount, struct J9PortVmemIdentifier *newIdentifier, UDATA pageSize);
+	virtual void *doubleMapRegions(MM_EnvironmentBase *env, void* regions[], UDATA regionsCount, UDATA regionSize, UDATA byteAmount, struct J9PortVmemIdentifier *newIdentifier, UDATA pageSize, void *preferredAddress);
 #endif /* defined(OMR_GC_DOUBLE_MAP_ARRAYLETS) */
 
 	MM_VirtualMemory(MM_EnvironmentBase* env, uintptr_t heapAlignment, uintptr_t pageSize, uintptr_t pageFlags, uintptr_t tailPadding, uintptr_t mode)

include_core/omrport.h

@@ -1129,6 +1129,7 @@ typedef struct J9ProcessorInfos {
 #define OMRPORT_VMEM_RESERVE_USED_J9ALLOCATE_4K_PAGES_BELOW_BAR 10
 #define OMRPORT_VMEM_RESERVE_USED_MOSERVICES 11
 #define OMRPORT_VMEM_RESERVE_USED_MMAP_SHM 12
+#define OMRPORT_VMEM_RESERVE_USED_MMAP_RESTORE_MMAP 13
 
 #define OMRPORT_ENSURE_CAPACITY_FAILED  0
 #define OMRPORT_ENSURE_CAPACITY_SUCCESS  1
@@ -1990,6 +1991,10 @@ typedef struct OMRPortLibrary {
 	void *(*vmem_reserve_memory_ex)(struct OMRPortLibrary *portLibrary, struct J9PortVmemIdentifier *identifier, struct J9PortVmemParams *params) ;
 	/** see @ref omrvmem.c::omrvmem_get_contiguous_region_memory "omrvmem_get_contiguous_region_memory"*/
 	void *(*vmem_get_contiguous_region_memory)(struct OMRPortLibrary *portLibrary, void* addresses[], uintptr_t addressesCount, uintptr_t addressSize, uintptr_t byteAmount, struct J9PortVmemIdentifier *oldIdentifier, struct J9PortVmemIdentifier *newIdentifier, uintptr_t mode, uintptr_t pageSize, OMRMemCategory *category);
+	/** see @ref omrvmem.c::omrvmem_create_double_mapped_region "omrvmem_create_double_mapped_region"*/
+	void *(*vmem_create_double_mapped_region)(struct OMRPortLibrary *portLibrary, void* regionAddresses[], uintptr_t regionsCount, uintptr_t regionSize, uintptr_t byteAmount, struct J9PortVmemIdentifier *oldIdentifier, struct J9PortVmemIdentifier *newIdentifier, uintptr_t mode, uintptr_t pageSize, OMRMemCategory *category, void *preferredAddress);
+	/** see @ref omrvmem.c::omrvmem_release_double_mapped_region "omrvmem_release_double_mapped_region"*/
+	int32_t (*vmem_release_double_mapped_region)(struct OMRPortLibrary *portLibrary, void *address, uintptr_t byteAmount, struct J9PortVmemIdentifier *identifier);
 	/** see @ref omrvmem.c::omrvmem_get_page_size "omrvmem_get_page_size"*/
 	uintptr_t (*vmem_get_page_size)(struct OMRPortLibrary *portLibrary, struct J9PortVmemIdentifier *identifier) ;
 	/** see @ref omrvmem.c::omrvmem_get_page_flags "omrvmem_get_page_flags"*/
@@ -2780,6 +2785,8 @@ extern J9_CFUNC int32_t omrport_getVersion(struct OMRPortLibrary *portLibrary);
 #define omrvmem_reserve_memory(param1,param2,param3,param4,param5,param6) privateOmrPortLibrary->vmem_reserve_memory(privateOmrPortLibrary, (param1), (param2), (param3), (param4), (param5), (param6))
 #define omrvmem_reserve_memory_ex(param1,param2) privateOmrPortLibrary->vmem_reserve_memory_ex(privateOmrPortLibrary, (param1), (param2))
 #define omrvmem_get_contiguous_region_memory(param1, param2, param3, param4, param5, param6, param7, param8, param9) privateOmrPortLibrary->vmem_get_contiguous_region_memory(privateOmrPortLibrary, (param1), (param2), (param3), (param4), (param5), (param6), (param7), (param8), (param9))
+#define omrvmem_create_double_mapped_region(param1, param2, param3, param4, param5, param6, param7, param8, param9, param10) privateOmrPortLibrary->vmem_create_double_mapped_region(privateOmrPortLibrary, (param1), (param2), (param3), (param4), (param5), (param6), (param7), (param8), (param9), (param10))
+#define omrvmem_release_double_mapped_region(param1, param2, param3) privateOmrPortLibrary->vmem_release_double_mapped_region(privateOmrPortLibrary, (param1), (param2), (param3))
 #define omrvmem_get_page_size(param1) privateOmrPortLibrary->vmem_get_page_size(privateOmrPortLibrary, (param1))
 #define omrvmem_get_page_flags(param1) privateOmrPortLibrary->vmem_get_page_flags(privateOmrPortLibrary, (param1))
 #define omrvmem_supported_page_sizes() privateOmrPortLibrary->vmem_supported_page_sizes(privateOmrPortLibrary)

include_core/omrporterror.h

@@ -219,6 +219,8 @@
 #define OMRPORT_ERROR_VMEM_INSUFFICENT_RESOURCES (OMRPORT_ERROR_VMEM_BASE -1)
 #define OMRPORT_ERROR_VMEM_INVALID_PARAMS (OMRPORT_ERROR_VMEM_BASE -2)
 #define OMRPORT_ERROR_VMEM_NOT_SUPPORTED (OMRPORT_ERROR_VMEM_BASE -3)
+#define OMRPORT_ERROR_VMEM_DOUBLE_MAP_FAILED (OMRPORT_ERROR_VMEM_BASE -4)
+#define OMRPORT_ERROR_VMEM_DOUBLE_MAP_ADRESS_RESERVE_FAILED (OMRPORT_ERROR_VMEM_BASE -5)
 
 /** @} */
 

port/aix/omrvmem.c

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 1991, 2019 IBM Corp. and others
+ * Copyright (c) 1991, 2020 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -1532,3 +1532,17 @@ omrvmem_get_contiguous_region_memory(struct OMRPortLibrary *portLibrary, void* a
 	portLibrary->error_set_last_error(portLibrary, errno, OMRPORT_ERROR_VMEM_NOT_SUPPORTED);
 	return NULL;
 }
+
+int32_t
+omrvmem_release_double_mapped_region(struct OMRPortLibrary *portLibrary, void *address, uintptr_t byteAmount, struct J9PortVmemIdentifier *oldIdentifier)
+{
+	portLibrary->error_set_last_error(portLibrary, errno, OMRPORT_ERROR_VMEM_NOT_SUPPORTED);
+	return -1;
+}
+
+void *
+omrvmem_create_double_mapped_region(struct OMRPortLibrary *portLibrary, void* regions[], uintptr_t regionsCount, uintptr_t regionSize, uintptr_t byteAmount, struct J9PortVmemIdentifier *oldIdentifier, struct J9PortVmemIdentifier *newIdentifier, uintptr_t mode, uintptr_t pageSize, OMRMemCategory *category, void *preferredAddress)
+{
+	portLibrary->error_set_last_error(portLibrary, errno, OMRPORT_ERROR_VMEM_NOT_SUPPORTED);
+	return NULL;
+}

port/common/omrport.c

@@ -178,6 +178,8 @@ static OMRPortLibrary MainPortLibraryTable = {
 	omrvmem_reserve_memory, /* vmem_reserve_memory */
 	omrvmem_reserve_memory_ex, /* vmem_reserve_memory_ex */
 	omrvmem_get_contiguous_region_memory, /* vmem_get_contiguous_region_memory */
+	omrvmem_create_double_mapped_region, /* vmem_create_double_mapped_region */
+	omrvmem_release_double_mapped_region, /* omrvmem_release_double_mapped_region */
 	omrvmem_get_page_size, /* vmem_get_page_size */
 	omrvmem_get_page_flags, /* omrvmem_get_page_flags */
 	omrvmem_supported_page_sizes, /* vmem_supported_page_sizes */

port/common/omrport.tdf

@@ -1589,3 +1589,18 @@ TraceEvent=Trc_PRT_shared_omr_cleanSharedMemorySegments_childFailed Group=omrsha
 // sysvipc
 TraceException=Trc_PRT_omr_sysvipc_shmctlWrapper_Failed NoEnv Overhead=1 Level=1 Template="shmctlWrapper failed with platform error code=%d"
 TraceException=Trc_PRT_sysinfo_unrecognized_Windows_version Obsolete NoEnv Overhead=1 Level=1 Template="omrsysinfo_get_OS_type unrecognized Windows version: type=%d version=%d.%d"
+
+TraceEntry=Trc_PRT_double_map_regions_Create_Entry Overhead=1 Level=3 Group=double_map NoEnv Template="omrvmem_create_double_mapped_region. Entered addresses: %p, addressesCount: %zu, addressSize: %zu, byteAmount: %zu, pageSize: %zu, preferredAddress: %p"
+TraceEvent=Trc_PRT_double_map_regions_Create_UsingPreferredAddress Overhead=1 Level=1 Group=double_map NoEnv Template="Using prefered address to double map regions, preferredAddress: %p"
+TraceEvent=Trc_PRT_double_map_regions_Create_GenerateContiguousAddress Overhead=1 Level=1 Group=double_map NoEnv Template="Generating contiguous region of memory, contiguousMap: %p"
+TraceEvent=Trc_PRT_double_map_regions_Create_Success Overhead=1 Level=1 Group=double_map NoEnv Template="mmap contiguous region of memory created successfully. contiguousMap: %p, preferredAddress: %p"
+TraceException=Trc_PRT_double_map_regions_Create_Failure Overhead=1 Level=1 Group=double_map NoEnv Template="Failed to mmap contiguous region of memory"
+TraceException=Trc_PRT_double_map_regions_Create_Failure2 Overhead=1 Level=1 Group=double_map NoEnv Template="Double map failed while double mapping individual leaves."
+TraceException=Trc_PRT_double_map_regions_Create_Failure3 Overhead=1 Level=1 Group=double_map NoEnv Template="Address returned by mmap dpes not match expected. Expected: %p, returned: %p"
+TraceException=Trc_PRT_double_map_regions_Create_Failure4 Overhead=1 Level=1 Group=double_map NoEnv Template="Double map failed. Clean up phase."
+TraceExit=Trc_PRT_double_map_regions_Create_Exit Overhead=1 Level=3 Group=double_map NoEnv Template="omrvmem_create_double_mapped_region. result=%p"
+TraceEntry=Trc_PRT_double_map_regions_Release_Entry Group=double_map Overhead=1 Level=3 NoEnv Template="omrvmem_release_double_mapped_region address: %p, byteAmount: %zu"
+TraceEvent=Trc_PRT_double_map_regions_Release_Restore_Region Overhead=1 Level=1 Group=double_map NoEnv Template="OMRPORT_VMEM_RESERVE_USED_MMAP_RESTORE_MMAP is set. Calling mmap to revert region. address: %p, byteAmount: %zu, fd: %d"
+TraceExit=Trc_PRT_double_map_regions_Release_Exit Group=double_map Overhead=1 Level=5 NoEnv Template="omrvmem_release_double_mapped_region returnCode: %d"
+TraceException=Trc_PRT_double_map_regions_Release_Failure Overhead=1 Level=1 Group=double_map NoEnv Template="Failed to mmap FIXED contiguous region of memory when releasing region"
+TraceException=Trc_PRT_double_map_regions_Release_Failure2 Overhead=1 Level=1 Group=double_map NoEnv Template="Failed to mmap FIXED contiguous region of memory. Expected address: %p, mmap returned: %p"