Add Dynamic Breadth First Scan Ordering to the GC

Add Dynamic Breadth First Scan Ordering to the GC implemented
using JIT hot fields. This enables the copying of a hot field
marked by the JIT immediately after the object containing the
hot field is copied during a Scavenge.

Signed-off-by: Jonathan Oommen <jon.oommen@gmail.com>

compiler/control/OMROptions.cpp

@@ -801,6 +801,7 @@ TR::OptionTable OMR::Options::_jitOptions[] = {
    {"help",               " \tdisplay this help information", TR::Options::helpOption, 0, 0, NULL, NOT_IN_SUBSET},
    {"help=",              " {regex}\tdisplay help for options whose names match {regex}", TR::Options::helpOption, 1, 0, NULL, NOT_IN_SUBSET},
    {"highOpt",            "O\tdeprecated; equivalent to optLevel=hot", TR::Options::set32BitValue, offsetof(OMR::Options, _optLevel), hot},
+   {"hotFieldReductionAlgorithm=",          "O\tcompilation's hot field combined block frequency reduction algorithm", TR::Options::setHotFieldReductionAlgorithm, 0, 0, "F", NOT_IN_SUBSET},
    {"hotFieldThreshold=", "M<nnn>\t The normalized frequency of a reference to a field to be marked as hot.   Values are 0 to 10000.  Default is 10",
                           TR::Options::setStaticNumeric, (intptr_t)&OMR::Options::_hotFieldThreshold, 0, " %d", NOT_IN_SUBSET},
    {"hotMaxStaticPICSlots=", " <nnn>\tmaximum number of polymorphic inline cache slots pre-populated from profiling info for hot and above.  A negative value -N means use N times the maxStaticPICSlots setting.",
@@ -1558,6 +1559,7 @@ OMR::Options::VerboseOptionFlagArray OMR::Options::_verboseOptionFlags;
 bool          OMR::Options::_quickstartDetected = false;
 
 OMR::Options::SamplingJProfilingOptionFlagArray OMR::Options::_samplingJProfilingOptionFlags;
+OMR::Options::HotFieldReductionAlgorithmArray OMR::Options::_hotFieldReductionAlgorithms;
 
 int32_t       OMR::Options::_samplingFrequency = 2; // ms
 
@@ -1584,7 +1586,7 @@ int32_t       OMR::Options::_interpreterSamplingDivisorInStartupMode = -1; // un
 int32_t       OMR::Options::_numJitEntries = 0;
 int32_t       OMR::Options::_numVmEntries = 0;
 int32_t       OMR::Options::_numVecRegsToLock=0;
-int32_t       OMR::Options::_hotFieldThreshold = 100;
+int32_t       OMR::Options::_hotFieldThreshold = 200;
 int32_t       OMR::Options::_maxNumPrexAssumptions = 209;
 int32_t       OMR::Options::_maxNumVisitedSubclasses = 500;
 
@@ -4964,6 +4966,36 @@ char *OMR::Options::setSamplingJProfilingBits(char *option, void *base, TR::Opti
    return option;
    }
 
+char *OMR::Options::_hotFieldReductionAlgorithmNames[TR_NumReductionAlgorithms] =
+   {
+   "sum",
+   "average",
+   "max",
+   };
+
+char *OMR::Options::setHotFieldReductionAlgorithm(char *option, void *base, TR::OptionTable *entry)
+   {
+   TR::SimpleRegex * regex = TR::SimpleRegex::create(option);
+   bool foundMatch = false;
+   if (regex)
+      {
+      for (int8_t i = 0; i < TR_NumReductionAlgorithms; i++)
+         {
+         if (TR::SimpleRegex::matchIgnoringLocale(regex, _hotFieldReductionAlgorithmNames[i], false))
+            {
+            _hotFieldReductionAlgorithms.set(i);
+            foundMatch = true;
+            }
+         }
+      }
+   if (!foundMatch)
+      {
+      TR_VerboseLog::write("<JIT: Reduction algorithm not found.  Default sum reduction algorithm set.>");
+      _hotFieldReductionAlgorithms.set(TR_HotFieldReductionAlgorithmSum);
+      }
+   return option;
+   }
+
 char *
 OMR::Options::breakOnLoad(char *option, void *base, TR::OptionTable *entry)
    {

compiler/control/OMROptions.hpp

@@ -1105,6 +1105,16 @@ enum TR_ProcessOptionsFlags
    TR_JITProcessErrorUnknown = 0x00000080
    };
 
+enum TR_ReductionAlgorithms
+   {
+   TR_HotFieldReductionAlgorithmSum,
+   TR_HotFieldReductionAlgorithmAverage,
+   TR_HotFieldReductionAlgorithmMax,
+   // Any new option added here must be added also to _hotFieldReductionAlgorithmNames
+
+   // The below must be the last option...
+   TR_NumReductionAlgorithms,
+   };
 
 #define TR_FILTER_EXCLUDE_NAME_ONLY         1
 #define TR_FILTER_EXCLUDE_NAME_AND_SIG      2
@@ -1440,6 +1450,8 @@ class OMR_EXTENSIBLE Options
    static void  resetSamplingJProfilingOption(TR_SamplingJProfilingFlags op) { _samplingJProfilingOptionFlags.reset(op); }
    static bool  isAnySamplingJProfilingOptionSet()                   { return !_samplingJProfilingOptionFlags.isEmpty(); }
 
+   static bool  getReductionAlgorithm(TR_ReductionAlgorithms op)     {  return _hotFieldReductionAlgorithms.isSet(op); }
+   static void  setReductionAlgorithm(TR_ReductionAlgorithms op)     { _hotFieldReductionAlgorithms.set(op); }
    static bool  getVerboseOption(TR_VerboseFlags op)     {  return _verboseOptionFlags.isSet(op); }
    static void  setVerboseOption(TR_VerboseFlags op)     { _verboseOptionFlags.set(op); }
    static void  setVerboseOptions(uint64_t mask)         { _verboseOptionFlags.maskWord(0, mask); }
@@ -1942,6 +1954,10 @@ class OMR_EXTENSIBLE Options
    // Helper method used by the two methods above
    static char *setVerboseBitsHelper(char *option, VerboseOptionFlagArray *verboseOptionFlags, uintptr_t defaultVerboseFlags);
 
+   //set hot field reduction algorithm for dynamicBreadthFirstScanOrdering
+   //
+   static char *setHotFieldReductionAlgorithm(char *option, void *base, TR::OptionTable *entry);
+
    // Set samplingjprofiling bits
    //
    static char *setSamplingJProfilingBits(char* option, void *base, TR::OptionTable *entry);
@@ -2232,6 +2248,9 @@ class OMR_EXTENSIBLE Options
    static SamplingJProfilingOptionFlagArray _samplingJProfilingOptionFlags;
    static char                     *_samplingJProfilingOptionNames[TR_NumSamplingJProfilingFlags];
 
+   typedef OptionFlagArray<TR_ReductionAlgorithms, TR_NumReductionAlgorithms> HotFieldReductionAlgorithmArray;
+   static HotFieldReductionAlgorithmArray  _hotFieldReductionAlgorithms;
+   static char                   *_hotFieldReductionAlgorithmNames[TR_NumReductionAlgorithms];
    // Miscellaneous options
    //
    char *                      _osVersionString;

example/glue/ObjectModelDelegate.hpp

@@ -201,6 +201,45 @@ class GC_ObjectModelDelegate
 		return false;
 	}
 
+	/**
+	 * Returns the field offset of the hottest field of the object referred to by the forwarded header.
+	 * Valid if scavenger dynamicBreadthFirstScanOrdering is enabled.
+	 *
+	 * @param forwardedHeader pointer to the MM_ForwardedHeader instance encapsulating the object
+	 * @return the offset of the hottest field of the given object referred to by the forwarded header, return U_8_MAX if a hot field does not exist
+	 */
+	MMINLINE uint8_t
+	getHotFieldOffset(MM_ForwardedHeader *forwardedHeader)
+	{
+		return U_8_MAX;
+	}
+
+	/**
+	 * Returns the field offset of the second hottest field of the object referred to by the forwarded header.
+	 * Valid if scavenger dynamicBreadthFirstScanOrdering is enabled
+	 *
+	 * @param forwardedHeader pointer to the MM_ForwardedHeader instance encapsulating the object
+	 * @return the offset of the second hottest field of the given object referred to by the forwarded header, return U_8_MAX if the hot field does not exist
+	 */
+	MMINLINE uint8_t
+	getHotFieldOffset2(MM_ForwardedHeader *forwardedHeader)
+	{
+		return U_8_MAX;
+	}
+
+	/**
+	 * Returns the field offset of the third hottest field of the object referred to by the forwarded header.
+	 * Valid if scavenger dynamicBreadthFirstScanOrdering is enabled
+	 *
+	 * @param forwardedHeader pointer to the MM_ForwardedHeader instance encapsulating the object
+	 * @return the offset of the third hottest field of the given object referred to by the forwarded header, return U_8_MAX if the hot field does not exist
+	 */
+	MMINLINE uint8_t
+	getHotFieldOffset3(MM_ForwardedHeader *forwardedHeader)
+	{
+		return U_8_MAX;
+	}
+
 	/**
 	 * Get the instance size (total) of a forwarded object from the forwarding pointer. The  size must
 	 * include the header and any expansion bytes to be allocated if the object will grow when moved.

gc/base/EnvironmentBase.cpp

@@ -99,6 +99,13 @@ MM_EnvironmentBase::initialize(MM_GCExtensionsBase *extensions)
 		}
 	}
 
+#if defined(OMR_GC_MODRON_SCAVENGER)
+	/* Disable dynamic depth copying if scavengerDynamicBreadthFirstScanOrdering is not selected */
+	if (extensions->scavengerScanOrdering != MM_GCExtensionsBase::OMR_GC_SCAVENGER_SCANORDERING_DYNAMIC_BREADTH_FIRST) {
+		disableHotFieldDepthCopy();
+	}
+#endif /* defined(OMR_GC_MODRON_SCAVENGER) */
+
 #if defined(OMR_GC_SEGREGATED_HEAP)
 	if (extensions->isSegregatedHeap()) {
 		_regionWorkList = MM_RegionPoolSegregated::allocateHeapRegionQueue(this, MM_HeapRegionList::HRL_KIND_LOCAL_WORK, true, false, false);

gc/base/EnvironmentBase.hpp

@@ -152,6 +152,10 @@ class MM_EnvironmentBase : public MM_BaseVirtual
 
 	uintptr_t approxScanCacheCount; /**< Local copy of approximate entries in global Cache Scan List. Updated upon allocation of new cache. */
 
+	#if defined(OMR_GC_MODRON_SCAVENGER)
+	uintptr_t _hotFieldCopyDepthCount; /**< Used for dynamic breadth first scan ordering. Counter for the current copying depth based on the initial object copied. */
+	#endif /* defined(OMR_GC_MODRON_SCAVENGER) */
+
 	MM_Validator *_activeValidator; /**< Used to identify and report crashes inside Validators */
 
 	MM_MarkStats _markStats;
@@ -529,6 +533,25 @@ class MM_EnvironmentBase : public MM_BaseVirtual
 	 */
 	void forceOutOfLineVMAccess() { _delegate.forceOutOfLineVMAccess(); }
 
+#if defined(OMR_GC_MODRON_SCAVENGER)
+	/**
+	 * Disable scavenger hot field depth copying for dynamicBreadthFirstScanOrdering
+	 */
+	MMINLINE void disableHotFieldDepthCopy()
+	{
+		_hotFieldCopyDepthCount = getExtensions()->depthCopyMax;
+	}
+	/**
+	 * Enable scavenger hot field depth copying for dynamicBreadthFirstScanOrdering
+	 */
+	MMINLINE void enableHotFieldDepthCopy()
+	{ 
+		if (getExtensions()->scavengerScanOrdering == MM_GCExtensionsBase::OMR_GC_SCAVENGER_SCANORDERING_DYNAMIC_BREADTH_FIRST) {
+			_hotFieldCopyDepthCount = 0;
+		}
+	}
+#endif /* defined(OMR_GC_MODRON_SCAVENGER) */
+
 #if defined (OMR_GC_THREAD_LOCAL_HEAP)
 	/**
 	 * Disable inline TLH allocates by hiding the real heap top address from
@@ -695,6 +718,9 @@ class MM_EnvironmentBase : public MM_BaseVirtual
 		,_traceAllocationBytes(0)
 		,_traceAllocationBytesCurrentTLH(0)
 		,approxScanCacheCount(0)
+#if defined(OMR_GC_MODRON_SCAVENGER)
+		,_hotFieldCopyDepthCount(0)
+#endif /* defined(OMR_GC_MODRON_SCAVENGER) */
 		,_activeValidator(NULL)
 		,_lastSyncPointReached(NULL)
 #if defined(OMR_GC_SEGREGATED_HEAP)
@@ -749,6 +775,9 @@ class MM_EnvironmentBase : public MM_BaseVirtual
 		,_traceAllocationBytes(0)
 		,_traceAllocationBytesCurrentTLH(0)
 		,approxScanCacheCount(0)
+#if defined(OMR_GC_MODRON_SCAVENGER)
+		,_hotFieldCopyDepthCount(0)
+#endif /* defined(OMR_GC_MODRON_SCAVENGER) */
 		,_activeValidator(NULL)
 		,_lastSyncPointReached(NULL)
 #if defined(OMR_GC_SEGREGATED_HEAP)

gc/base/GCExtensionsBase.hpp

@@ -444,10 +444,26 @@ class MM_GCExtensionsBase : public MM_BaseVirtual {
 #if defined(OMR_GC_MODRON_SCAVENGER) || defined(OMR_GC_VLHGC)
 	enum ScavengerScanOrdering {
 		OMR_GC_SCAVENGER_SCANORDERING_BREADTH_FIRST = 0,
+		OMR_GC_SCAVENGER_SCANORDERING_DYNAMIC_BREADTH_FIRST,
 		OMR_GC_SCAVENGER_SCANORDERING_HIERARCHICAL,
 	};
 	ScavengerScanOrdering scavengerScanOrdering; /**< scan ordering in Scavenger */
 #if defined(OMR_GC_MODRON_SCAVENGER)
+	/* Start of options relating to dynamicBreadthFirstScanOrdering */
+	uintptr_t gcCountBetweenHotFieldSort;
+	uintptr_t gcCountBetweenHotFieldSortMax;
+	bool adaptiveGcCountBetweenHotFieldSort;
+	bool depthCopyTwoPaths;
+	bool depthCopyThreePaths;
+	bool alwaysDepthCopyFirstOffset;
+	bool allowPermanantHotFields;
+	bool hotFieldResettingEnabled;
+	uintptr_t maxConsecutiveHotFieldSelections;
+	uintptr_t gcCountBetweenHotFieldReset;
+	uintptr_t depthCopyMax;
+	uint32_t maxHotFieldListLength;
+	uintptr_t minCpuUtil;
+	/* End of options relating to dynamicBreadthFirstScanOrdering */
 	uintptr_t scvTenureRatioHigh;
 	uintptr_t scvTenureRatioLow;
 	uintptr_t scvTenureFixedTenureAge; /**< The tenure age to use for the Fixed scavenger tenure strategy. */
@@ -1523,6 +1539,21 @@ class MM_GCExtensionsBase : public MM_BaseVirtual {
 		, scavengerScanOrdering(OMR_GC_SCAVENGER_SCANORDERING_HIERARCHICAL)
 #endif /* OMR_GC_MODRON_SCAVENGER || OMR_GC_VLHGC */
 #if defined(OMR_GC_MODRON_SCAVENGER)
+		/* Start of options relating to dynamicBreadthFirstScanOrdering */
+		, gcCountBetweenHotFieldSort(1)
+		, gcCountBetweenHotFieldSortMax(6)
+		, adaptiveGcCountBetweenHotFieldSort(true)
+		, depthCopyTwoPaths(true)
+		, depthCopyThreePaths(false)
+		, alwaysDepthCopyFirstOffset(false)
+		, allowPermanantHotFields (false)
+		, hotFieldResettingEnabled (false)
+		, maxConsecutiveHotFieldSelections(10)
+		, gcCountBetweenHotFieldReset(100)
+		, depthCopyMax(3)
+		, maxHotFieldListLength(10)
+		, minCpuUtil (1)
+		/* End of options relating to dynamicBreadthFirstScanOrdering */
 		, scvTenureRatioHigh(OMR_SCV_TENURE_RATIO_HIGH)
 		, scvTenureRatioLow(OMR_SCV_TENURE_RATIO_LOW)
 		, scvTenureFixedTenureAge(OBJECT_HEADER_AGE_MAX)

gc/base/ObjectModelBase.hpp

@@ -826,6 +826,48 @@ class GC_ObjectModelBase : public MM_BaseVirtual
 		return _delegate.getForwardedObjectSizeInBytes(forwardedHeader);
 	}
 
+	/**
+	 * Returns the field offset of the hottest field of the object referred to by the forwarded header.
+	 * Valid if scavenger dynamicBreadthFirstScanOrdering is enabled.
+	 *
+	 * @param forwardedHeader pointer to the MM_ForwardedHeader instance encapsulating the object
+	 * @return the offset of the hottest field of the given object referred to by the forwarded header, return U_8_MAX if a hot field does not exist
+	 */
+	MMINLINE uint8_t
+	getHotFieldOffset(MM_ForwardedHeader *forwardedHeader)
+	{
+		//return _delegate.getHotFieldOffset(forwardedHeader); //TODO: will be populated once Language specific implementation is ready
+		return U_8_MAX;
+	}
+
+	/**
+	 * Returns the field offset of the second hottest field of the object referred to by the forwarded header.
+	 * Valid if scavenger dynamicBreadthFirstScanOrdering is enabled
+	 *
+	 * @param forwardedHeader pointer to the MM_ForwardedHeader instance encapsulating the object
+	 * @return the offset of the second hottest field of the given object referred to by the forwarded header, return U_8_MAX if the hot field does not exist
+	 */
+	MMINLINE uint8_t
+	getHotFieldOffset2(MM_ForwardedHeader *forwardedHeader)
+	{
+		//return _delegate.getHotFieldOffset2(forwardedHeader); //TODO: will be populated once Language specific implementation is ready
+		return U_8_MAX;
+	}
+
+		/**
+	 * Returns the field offset of the third hottest field of the object referred to by the forwarded header.
+	 * Valid if scavenger dynamicBreadthFirstScanOrdering is enabled
+	 *
+	 * @param forwardedHeader pointer to the MM_ForwardedHeader instance encapsulating the object
+	 * @return the offset of the third hottest field of the given object referred to by the forwarded header, return U_8_MAX if the hot field does not exist
+	 */
+	MMINLINE uint8_t
+	getHotFieldOffset3(MM_ForwardedHeader *forwardedHeader)
+	{
+		//return _delegate.getHotFieldOffset3(forwardedHeader); //TODO: will be populated once Language specific implementation is ready
+		return U_8_MAX;
+	}
+
 	/**
 	 * Extract the flag bits from an unforwarded object. Flag bits are returned in the low-order byte of the returned value.
 	 *