Cleanup softmx check and apply softmx to nursery expansion

At the moment, heap expansion logic allows nursery size to expand well beyond the softmx limit. Additionally, when the total heap size was above the softmx size, the softmx size was effectively ignored.

This PR fixes both behaviours mentioned above, and makes softmx limit apply to both nursery and tenure (ie, tenure + nursery < softmx).

Additionally, this PR does the following:
- simplified initialization of `desiredContractionFactor` and  `adjustedContractionFactor` in `MemorySubSpaceSemiSpace::checkSubSpaceMemoryPostCollectResize()`. Same idea for `desiredExpansionFactor` and `adjustedExpansionFactor` just above
- refactoring `if(` to `if (`(with space) and `){` to `) {` (with space) in `MemorySubSpaceSemiSpace` and `MemorySubSpaceUnispace`

This PR is a continuation of #5728. Please see it for initial PR feedback on the initial set of changes that was done

Signed-off-by: Cedric Hansen cedric.hansen@ibm.com

gc/base/Heap.cpp

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 1991, 2016 IBM Corp. and others
+ * Copyright (c) 1991, 2021 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -531,7 +531,7 @@ MM_Heap::initializeCommonGCData(MM_EnvironmentBase* env, struct MM_CommonGCData*
  * @return Size of the adjustable heap memory
  */
 uintptr_t
-MM_Heap::getActualSoftMxSize(MM_EnvironmentBase* env)
+MM_Heap::getActualSoftMxSize(MM_EnvironmentBase* env, uintptr_t memoryType)
 {
 	uintptr_t actualSoftMX = 0;
 	MM_GCExtensionsBase* extensions = env->getExtensions();
@@ -544,11 +544,18 @@ MM_Heap::getActualSoftMxSize(MM_EnvironmentBase* env)
 
 		uintptr_t nurserySize = totalHeapSize - tenureSize;
 
-		if (nurserySize <= extensions->softMx) {
-			actualSoftMX = extensions->softMx - nurserySize;
+		if (MEMORY_TYPE_NEW == memoryType) {
+			actualSoftMX = (uintptr_t)(extensions->softMx * ((double)extensions->maxNewSpaceSize / extensions->memoryMax));
+		} else if (MEMORY_TYPE_OLD == memoryType) {
+			if (nurserySize <= extensions->softMx) {
+				actualSoftMX = extensions->softMx - nurserySize;
+			} else {
+				actualSoftMX = 0;
+			}
 		} else {
-			actualSoftMX = 0;
+			Assert_MM_unreachable();
 		}
+
 	} else {
 		actualSoftMX = extensions->softMx;
 	}

gc/base/Heap.hpp

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 1991, 2020 IBM Corp. and others
+ * Copyright (c) 1991, 2021 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -165,7 +165,7 @@ class MM_Heap : public MM_BaseVirtual
 	void initializeCommonGCStartData(MM_EnvironmentBase *env, struct MM_CommonGCStartData *data);
 	void initializeCommonGCEndData(MM_EnvironmentBase *env, struct MM_CommonGCEndData *data);
 
-	uintptr_t getActualSoftMxSize(MM_EnvironmentBase *env);
+	uintptr_t getActualSoftMxSize(MM_EnvironmentBase *env, uintptr_t memoryType = MEMORY_TYPE_OLD);
 
 	/**
 	 * Create a Heap object.

gc/base/MemorySubSpace.cpp

@@ -1143,6 +1143,38 @@ MM_MemorySubSpace::canExpand(MM_EnvironmentBase* env, uintptr_t expandSize)
 	return false;
 }
 
+/**
+ * Compare the specified expand amount with -XsoftMX value
+ * @return Updated expand size
+ */
+uintptr_t
+MM_MemorySubSpace::adjustExpansionWithinSoftMax(MM_EnvironmentBase *env, uintptr_t expandSize, uintptr_t minimumBytesRequired, uintptr_t memoryType)
+{
+	MM_Heap *heap = env->getExtensions()->getHeap();
+
+	uintptr_t actualSoftMx = heap->getActualSoftMxSize(env, memoryType);
+	uintptr_t activeMemorySize = getActiveMemorySize(memoryType);
+	OMRPORT_ACCESS_FROM_OMRPORT(env->getPortLibrary());
+
+	if (0 != actualSoftMx) {
+		if ((0 != minimumBytesRequired) && ((activeMemorySize + minimumBytesRequired) > actualSoftMx)) {
+			if (J9_EVENT_IS_HOOKED(env->getExtensions()->omrHookInterface, J9HOOK_MM_OMR_OOM_DUE_TO_SOFTMX)) {
+				ALWAYS_TRIGGER_J9HOOK_MM_OMR_OOM_DUE_TO_SOFTMX(env->getExtensions()->omrHookInterface, env->getOmrVMThread(), omrtime_hires_clock(),
+						heap->getMaximumMemorySize(), heap->getActiveMemorySize(memoryType), actualSoftMx, minimumBytesRequired);
+				actualSoftMx = heap->getActualSoftMxSize(env, memoryType);
+			}
+		}
+		if (actualSoftMx < activeMemorySize) {
+			/* if our softmx is smaller than our currentsize, we should be contracting not expanding */
+			expandSize = 0;
+		} else if ((activeMemorySize + expandSize) > actualSoftMx) {
+			/* we would go past our -XsoftMx so just expand up to it instead */
+			expandSize = actualSoftMx - activeMemorySize;
+		}
+	}
+	return expandSize;
+}
+
 /**
  * Adjust the specified expansion amount by the specified user increment amount (i.e. -Xmoi)
  * @return the updated expand size

gc/base/MemorySubSpace.hpp

@@ -365,6 +365,7 @@ friend class GC_MemorySubSpaceRegionIterator;
 	bool setResizable(bool resizable);
 
 	bool canExpand(MM_EnvironmentBase *env, uintptr_t expandSize);
+	virtual uintptr_t adjustExpansionWithinSoftMax(MM_EnvironmentBase *env, uintptr_t expandSize, uintptr_t minimumBytesRequired, uintptr_t memoryType = MEMORY_TYPE_OLD);
 	virtual uintptr_t adjustExpansionWithinUserIncrement(MM_EnvironmentBase *env, uintptr_t expandSize);
 	uintptr_t maxExpansion(MM_EnvironmentBase *env);
 	virtual uintptr_t maxExpansionInSpace(MM_EnvironmentBase *env);