Grow slabs by factor.

src/gpgmm/BUILD.gn

@@ -133,7 +133,6 @@ source_set("gpgmm_sources") {
       "d3d12/CapsD3D12.h",
       "d3d12/DebugResourceAllocatorD3D12.cpp",
       "d3d12/DebugResourceAllocatorD3D12.h",
-      "d3d12/DefaultsD3D12.h",
       "d3d12/ErrorD3D12.cpp",
       "d3d12/ErrorD3D12.h",
       "d3d12/FenceD3D12.cpp",

src/gpgmm/CMakeLists.txt

@@ -47,7 +47,6 @@ if (GPGMM_ENABLE_D3D12)
         "d3d12/DebugResourceAllocatorD3D12.h"
         "d3d12/CapsD3D12.cpp"
         "d3d12/CapsD3D12.h"
-        "d3d12/DefaultsD3D12.h"
         "d3d12/ErrorD3D12.cpp"
         "d3d12/ErrorD3D12.h"
         "d3d12/FenceD3D12.cpp"

src/gpgmm/common/Defaults.h

@@ -20,6 +20,7 @@
 namespace gpgmm {
     static constexpr const char* kDefaultTraceFile = "gpgmm_event_trace.json";
     static constexpr double kDefaultFragmentationLimit = 0.125;  // 1/8th or 12.5%
+    static constexpr double kDefaultMemoryGrowthFactor = 1.25;   // 25% growth
 }  // namespace gpgmm
 
 #endif  // GPGMM_COMMON_DEFAULTS_H_

src/gpgmm/common/SlabMemoryAllocator.cpp

@@ -31,21 +31,25 @@ namespace gpgmm {
 
     SlabMemoryAllocator::SlabMemoryAllocator(uint64_t blockSize,
                                              uint64_t maxSlabSize,
-                                             uint64_t slabSize,
+                                             uint64_t minSlabSize,
                                              uint64_t slabAlignment,
                                              double slabFragmentationLimit,
                                              bool prefetchSlab,
+                                             double slabGrowthFactor,
                                              MemoryAllocator* memoryAllocator)
-        : mBlockSize(blockSize),
-          mMaxSlabSize(maxSlabSize),
-          mSlabSize(slabSize),
+        : mLastUsedSlabSize(0),
+          mBlockSize(blockSize),
           mSlabAlignment(slabAlignment),
+          mMaxSlabSize(maxSlabSize),
+          mMinSlabSize(std::max(minSlabSize, mSlabAlignment)),
           mSlabFragmentationLimit(slabFragmentationLimit),
           mPrefetchSlab(prefetchSlab),
+          mSlabGrowthFactor(slabGrowthFactor),
           mMemoryAllocator(memoryAllocator) {
         ASSERT(IsPowerOfTwo(mMaxSlabSize));
         ASSERT(mMemoryAllocator != nullptr);
-        ASSERT(mSlabSize <= mMaxSlabSize);
+        ASSERT(mSlabGrowthFactor >= 1);
+        ASSERT(mSlabAlignment > 0);
     }
 
     SlabMemoryAllocator::~SlabMemoryAllocator() {
@@ -60,18 +64,16 @@ namespace gpgmm {
         }
     }
 
-    uint64_t SlabMemoryAllocator::ComputeSlabSize(uint64_t requestSize) const {
+    // Returns a new slab size of a power-of-two value.
+    uint64_t SlabMemoryAllocator::ComputeSlabSize(uint64_t requestSize, uint64_t slabSize) const {
+        ASSERT(requestSize <= mBlockSize);
+
         // If the left over empty space is less than |mSlabFragmentationLimit| x slab size,
         // then the fragmentation is acceptable and we are done. For example, a 4MB slab and and a
         // 512KB block fits exactly 8 blocks with no wasted space. But a 3MB block has 1MB worth of
         // empty space leftover which exceeds |mSlabFragmentationLimit| x slab size or 500KB.
-        ASSERT(requestSize <= mBlockSize);
-
-        // Slabs are grown in multiple of powers of two of the block size or |mSlabSize|
-        // if specified.
-        uint64_t slabSize = std::max(mSlabSize, mBlockSize);
-        const uint64_t wastedBytes = mBlockSize - requestSize;
-        while (wastedBytes > (mSlabFragmentationLimit * slabSize)) {
+        const uint64_t fragmentedBytes = mBlockSize - requestSize;
+        while (requestSize > slabSize || fragmentedBytes > (mSlabFragmentationLimit * slabSize)) {
             slabSize *= 2;
         }
 
@@ -104,7 +106,7 @@ namespace gpgmm {
             return {};
         }
 
-        const uint64_t slabSize = ComputeSlabSize(requestSize);
+        uint64_t slabSize = ComputeSlabSize(requestSize, std::max(mMinSlabSize, mLastUsedSlabSize));
         if (slabSize > mMaxSlabSize) {
             InfoEvent("SlabMemoryAllocator.TryAllocateMemory", ALLOCATOR_MESSAGE_ID_SIZE_EXCEEDED)
                 << "Slab size exceeded the max slab size (" + std::to_string(slabSize) + " vs " +
@@ -130,14 +132,23 @@ namespace gpgmm {
 
         // Push new free slab at free-list HEAD
         if (cache->FreeList.empty() || pFreeSlab->IsFull()) {
+            // Get the next free slab.
+            if (mLastUsedSlabSize > 0) {
+                uint64_t newSlabSize = std::min(
+                    ComputeSlabSize(requestSize, slabSize * mSlabGrowthFactor), mMaxSlabSize);
+                if (newSlabSize > slabSize) {
+                    cache = GetOrCreateCache(newSlabSize);
+                    slabSize = newSlabSize;
+                }
+            }
+
             Slab* pNewFreeSlab = new Slab(slabSize / mBlockSize, mBlockSize);
             pNewFreeSlab->InsertBefore(cache->FreeList.head());
             pFreeSlab = pNewFreeSlab;
         }
 
         ASSERT(pFreeSlab != nullptr);
         ASSERT(!pFreeSlab->IsFull());
-        ASSERT(!cache->FreeList.empty());
 
         std::unique_ptr<MemoryAllocation> subAllocation;
         GPGMM_TRY_ASSIGN(
@@ -185,6 +196,10 @@ namespace gpgmm {
                 mMemoryAllocator->TryAllocateMemoryAsync(slabSize, mSlabAlignment);
         }
 
+        // Remember the last allocated slab size so if a subsequent allocation requests a new slab,
+        // the new slab size will be slightly larger than the old slab size.
+        mLastUsedSlabSize = slabSize;
+
         // Wrap the block in the containing slab. Since the slab's block could reside in another
         // allocated block, the slab's allocation offset must be made relative to slab's underlying
         // memory and not the slab.
@@ -275,19 +290,22 @@ namespace gpgmm {
 
     SlabCacheAllocator::SlabCacheAllocator(uint64_t minBlockSize,
                                            uint64_t maxSlabSize,
-                                           uint64_t slabSize,
+                                           uint64_t minSlabSize,
                                            uint64_t slabAlignment,
                                            double slabFragmentationLimit,
                                            bool prefetchSlab,
+                                           double slabGrowthFactor,
                                            std::unique_ptr<MemoryAllocator> memoryAllocator)
         : MemoryAllocator(std::move(memoryAllocator)),
           mMinBlockSize(minBlockSize),
           mMaxSlabSize(maxSlabSize),
-          mSlabSize(slabSize),
+          mMinSlabSize(minSlabSize),
           mSlabAlignment(slabAlignment),
           mSlabFragmentationLimit(slabFragmentationLimit),
-          mPrefetchSlab(prefetchSlab) {
+          mPrefetchSlab(prefetchSlab),
+          mSlabGrowthFactor(slabGrowthFactor) {
         ASSERT(IsPowerOfTwo(mMaxSlabSize));
+        ASSERT(mSlabGrowthFactor >= 1);
     }
 
     SlabCacheAllocator::~SlabCacheAllocator() {
@@ -312,9 +330,9 @@ namespace gpgmm {
         auto entry = mSizeCache.GetOrCreate(SlabAllocatorCacheEntry(blockSize), cacheSize);
         SlabMemoryAllocator* slabAllocator = entry->GetValue().pSlabAllocator;
         if (slabAllocator == nullptr) {
-            slabAllocator =
-                new SlabMemoryAllocator(blockSize, mMaxSlabSize, mSlabSize, mSlabAlignment,
-                                        mSlabFragmentationLimit, mPrefetchSlab, GetFirstChild());
+            slabAllocator = new SlabMemoryAllocator(
+                blockSize, mMaxSlabSize, mMinSlabSize, mSlabAlignment, mSlabFragmentationLimit,
+                mPrefetchSlab, mSlabGrowthFactor, GetFirstChild());
             entry->GetValue().pSlabAllocator = slabAllocator;
             mSlabAllocators.Append(slabAllocator);
         }
@@ -329,7 +347,7 @@ namespace gpgmm {
         // Hold onto the cached allocator until the last allocation gets deallocated.
         entry->Ref();
 
-        TRACE_COUNTER1(TraceEventCategory::Default, "GPU slabs allocated (MB)",
+        TRACE_COUNTER1(TraceEventCategory::Default, "GPU slab memory used (MB)",
                        (GetFirstChild()->GetInfo().UsedMemoryUsage) / 1e6);
 
         TRACE_COUNTER1(TraceEventCategory::Default, "GPU slab cache miss-rate (%)",
@@ -374,12 +392,10 @@ namespace gpgmm {
         }
 
         // Memory allocator is common across slab allocators.
-        {
-            const MEMORY_ALLOCATOR_INFO& info = GetFirstChild()->GetInfo();
-            result.FreeMemoryUsage = info.FreeMemoryUsage;
-            result.UsedMemoryCount = info.UsedMemoryCount;
-            result.UsedMemoryUsage = info.UsedMemoryUsage;
-        }
+        const MEMORY_ALLOCATOR_INFO& info = GetFirstChild()->GetInfo();
+        result.FreeMemoryUsage = info.FreeMemoryUsage;
+        result.UsedMemoryCount = info.UsedMemoryCount;
+        result.UsedMemoryUsage = info.UsedMemoryUsage;
 
         return result;
     }

src/gpgmm/common/SlabMemoryAllocator.h

@@ -45,10 +45,11 @@ namespace gpgmm {
       public:
         SlabMemoryAllocator(uint64_t blockSize,
                             uint64_t maxSlabSize,
-                            uint64_t slabSize,
+                            uint64_t minSlabSize,
                             uint64_t slabAlignment,
                             double slabFragmentationLimit,
                             bool prefetchSlab,
+                            double slabGrowthFactor,
                             MemoryAllocator* memoryAllocator);
         ~SlabMemoryAllocator() override;
 
@@ -65,7 +66,7 @@ namespace gpgmm {
         uint64_t GetSlabSizeForTesting() const;
 
       private:
-        uint64_t ComputeSlabSize(uint64_t requestSize) const;
+        uint64_t ComputeSlabSize(uint64_t requestSize, uint64_t slabSize) const;
 
         // Slab is a node in a doubly-linked list that contains a free-list of blocks
         // and a reference to underlying memory.
@@ -112,12 +113,16 @@ namespace gpgmm {
 
         std::vector<SlabCache> mCaches;
 
+        uint64_t mLastUsedSlabSize = 0;
+
         const uint64_t mBlockSize;
-        const uint64_t mMaxSlabSize;
-        const uint64_t mSlabSize;
         const uint64_t mSlabAlignment;
+        const uint64_t mMaxSlabSize;
+        const uint64_t mMinSlabSize;  // Optional size when non-zero.
+
         const double mSlabFragmentationLimit;
         const bool mPrefetchSlab;
+        const double mSlabGrowthFactor;
 
         MemoryAllocator* mMemoryAllocator = nullptr;
         std::shared_ptr<MemoryAllocationEvent> mNextSlabAllocationEvent;
@@ -129,10 +134,11 @@ namespace gpgmm {
       public:
         SlabCacheAllocator(uint64_t minBlockSize,
                            uint64_t maxSlabSize,
-                           uint64_t slabSize,
+                           uint64_t minSlabSize,
                            uint64_t slabAlignment,
                            double slabFragmentationLimit,
                            bool prefetchSlab,
+                           double slabGrowthFactor,
                            std::unique_ptr<MemoryAllocator> memoryAllocator);
 
         ~SlabCacheAllocator() override;
@@ -171,10 +177,12 @@ namespace gpgmm {
 
         const uint64_t mMinBlockSize;
         const uint64_t mMaxSlabSize;
-        const uint64_t mSlabSize;  // Optional size when non-zero.
+        const uint64_t mMinSlabSize;
         const uint64_t mSlabAlignment;
+
         const double mSlabFragmentationLimit;
         const bool mPrefetchSlab;
+        const double mSlabGrowthFactor;
 
         LinkedList<MemoryAllocator> mSlabAllocators;
         MemoryCache<SlabAllocatorCacheEntry> mSizeCache;

src/gpgmm/d3d12/ResidencyManagerD3D12.cpp

@@ -16,7 +16,6 @@
 #include "gpgmm/d3d12/ResidencyManagerD3D12.h"
 
 #include "gpgmm/common/Debug.h"
-#include "gpgmm/d3d12/DefaultsD3D12.h"
 #include "gpgmm/d3d12/ErrorD3D12.h"
 #include "gpgmm/d3d12/FenceD3D12.h"
 #include "gpgmm/d3d12/HeapD3D12.h"

src/gpgmm/d3d12/ResourceAllocatorD3D12.cpp

@@ -18,6 +18,7 @@
 #include "gpgmm/common/BuddyMemoryAllocator.h"
 #include "gpgmm/common/ConditionalMemoryAllocator.h"
 #include "gpgmm/common/Debug.h"
+#include "gpgmm/common/Defaults.h"
 #include "gpgmm/common/MemorySize.h"
 #include "gpgmm/common/SegmentedMemoryAllocator.h"
 #include "gpgmm/common/SlabMemoryAllocator.h"
@@ -26,7 +27,6 @@
 #include "gpgmm/d3d12/BufferAllocatorD3D12.h"
 #include "gpgmm/d3d12/CapsD3D12.h"
 #include "gpgmm/d3d12/DebugResourceAllocatorD3D12.h"
-#include "gpgmm/d3d12/DefaultsD3D12.h"
 #include "gpgmm/d3d12/ErrorD3D12.h"
 #include "gpgmm/d3d12/HeapD3D12.h"
 #include "gpgmm/d3d12/JSONSerializerD3D12.h"
@@ -341,9 +341,9 @@ namespace gpgmm { namespace d3d12 {
         }
 
         ALLOCATOR_DESC newDescriptor = descriptor;
-        newDescriptor.PreferredResourceHeapSize = (descriptor.PreferredResourceHeapSize > 0)
-                                                      ? descriptor.PreferredResourceHeapSize
-                                                      : kDefaultPreferredResourceHeapSize;
+        newDescriptor.MemoryGrowthFactor = (descriptor.MemoryGrowthFactor >= 1.0)
+                                               ? descriptor.MemoryGrowthFactor
+                                               : kDefaultMemoryGrowthFactor;
 
         newDescriptor.MaxResourceHeapSize =
             (descriptor.MaxResourceHeapSize > 0)
@@ -455,21 +455,17 @@ namespace gpgmm { namespace d3d12 {
                     pooledOrNonPooledAllocator = std::move(resourceHeapAllocator);
                 }
 
-                std::unique_ptr<MemoryAllocator> buddyAllocator =
-                    std::make_unique<BuddyMemoryAllocator>(
-                        PrevPowerOfTwo(mMaxResourceHeapSize), descriptor.PreferredResourceHeapSize,
-                        heapAlignment, std::move(pooledOrNonPooledAllocator));
-
-                // TODO: Figure out the optimal slab size to heap ratio.
+                // TODO: Re-enable the buddy allocator?
                 mResourceAllocatorOfType[resourceHeapTypeIndex] = std::make_unique<
                     SlabCacheAllocator>(
                     /*minBlockSize*/ D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT,
                     /*maxSlabSize*/ PrevPowerOfTwo(mMaxResourceHeapSize),
-                    /*slabSize*/ descriptor.PreferredResourceHeapSize,
+                    /*minSlabSize*/ std::max(heapAlignment, descriptor.PreferredResourceHeapSize),
                     /*slabAlignment*/ heapAlignment,
                     /*slabFragmentationLimit*/ descriptor.MemoryFragmentationLimit,
                     /*enablePrefetch*/ !(descriptor.Flags & ALLOCATOR_FLAG_DISABLE_MEMORY_PREFETCH),
-                    std::move(buddyAllocator));
+                    /*slabGrowthFactor*/ descriptor.MemoryGrowthFactor,
+                    std::move(pooledOrNonPooledAllocator));
             }
 
             {
@@ -519,7 +515,8 @@ namespace gpgmm { namespace d3d12 {
                         /*slabSize*/ D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT,
                         /*slabAlignment*/ D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT,
                         /*slabFragmentationLimit*/ 0,
-                        /*enablePrefetch*/ false, std::move(pooledOrNonPooledAllocator));
+                        /*enablePrefetch*/ false,
+                        /*slabMemoryGrowth*/ 1, std::move(pooledOrNonPooledAllocator));
             }
 
             // Cache resource sizes commonly requested.

src/gpgmm/d3d12/ResourceAllocatorD3D12.h

@@ -256,7 +256,7 @@ namespace gpgmm { namespace d3d12 {
         A larger resource heap consumes more memory but could be faster for sub-allocation.
 
         Optional parameter. When 0 is specified, the API will automatically set the preferred
-        resource heap size to the default value of 4MB.
+        resource heap size to be a multiple of minimum resource heap size allowed by D3D12.
         */
         uint64_t PreferredResourceHeapSize;
 
@@ -309,6 +309,23 @@ namespace gpgmm { namespace d3d12 {
         fragmentation limit to 1/8th the resource heap size.
         */
         double MemoryFragmentationLimit;
+
+        /** \brief Memory growth factor, expressed as a multipler of the resource heap size
+        that will monotonically increase.
+
+        A factor value of 1.0 specifies no growth, where the resource heap size is always determined
+        by other limits or constraints. If no factor gets specified (or a value less than 1 is
+        specified), GPGMM will allocate a resource heap size with enough space to fit exactly one
+        resource.
+
+        Memory growth avoids the need to specify |PreferredResourceHeapSize|, which
+        especially helps in situations where the resource size cannot be predicated (eg.
+        user-defined), by allowing the resource heap size to gradually increase in size
+        per demand to achieve a balance of memory usage and performance.
+
+        Optional parameter. When 0 is specified, the default of 1.25 is used (or 25% growth).
+        */
+        double MemoryGrowthFactor;
     };
 
     /** \enum ALLOCATION_FLAGS