Fix ALLOCATION_FLAG_ALWAYS_CACHE_SIZE from disabling sub-allocation.

src/gpgmm/common/MemoryAllocator.h

@@ -63,14 +63,27 @@ namespace gpgmm {
          */
         uint64_t PrefetchedMemoryMissesEliminated;
 
+        /** \brief Requested size was NOT cached.
+         */
+        uint64_t CacheSizeMisses;
+
+        /** \brief Requested size was cached.
+         */
+        uint64_t CacheSizeHits;
+
         MEMORY_ALLOCATOR_INFO& operator+=(const MEMORY_ALLOCATOR_INFO& rhs) {
             UsedBlockCount += rhs.UsedBlockCount;
             UsedBlockUsage += rhs.UsedBlockUsage;
             FreeMemoryUsage += rhs.FreeMemoryUsage;
             UsedMemoryUsage += rhs.UsedMemoryUsage;
             UsedMemoryCount += rhs.UsedMemoryCount;
+
             PrefetchedMemoryMisses += rhs.PrefetchedMemoryMisses;
             PrefetchedMemoryMissesEliminated += rhs.PrefetchedMemoryMissesEliminated;
+
+            CacheSizeMisses += rhs.CacheSizeMisses;
+            CacheSizeHits += rhs.CacheSizeHits;
+
             return *this;
         }
     };

src/gpgmm/common/SlabMemoryAllocator.cpp

@@ -487,6 +487,11 @@ namespace gpgmm {
         result.UsedMemoryCount = info.UsedMemoryCount;
         result.UsedMemoryUsage = info.UsedMemoryUsage;
 
+        // Size cache is common across slab allocators.
+        const CacheStats& sizeCacheStats = mSizeCache.GetStats();
+        result.CacheSizeHits = sizeCacheStats.NumOfHits;
+        result.CacheSizeMisses = sizeCacheStats.NumOfMisses;
+
         return result;
     }
 

src/gpgmm/d3d12/ResourceAllocatorD3D12.cpp

@@ -443,50 +443,51 @@ namespace gpgmm { namespace d3d12 {
                 CreateSmallBufferAllocator(descriptor, heapFlags, heapType, heapAlignment);
 
             // Cache resource sizes commonly requested.
-            // Ensures the next block is always made available upon first request without
-            // increasing the memory footprint. Since resources are always sized-aligned, the
+            // Allows the next memory block to be made available upon request without
+            // increasing memory footprint. Since resources are always sized-aligned, the
             // cached size must be requested per alignment {4KB, 64KB, or 4MB}. To avoid unbounded
             // cache growth, a known set of pre-defined sizes initializes the allocators.
-
 #if defined(GPGMM_ENABLE_SIZE_CACHE)
-            // Temporary suppress log messages emitted from internal cache-miss requests.
             {
+                // Temporary suppress log messages emitted from internal cache-miss requests.
                 ScopedLogLevel scopedLogLevel(LogSeverity::Info);
-                for (uint64_t i = 0; i < MemorySize::kPowerOfTwoClassSize; i++) {
-                    MemoryAllocator* allocator =
-                        mResourceAllocatorOfType[resourceHeapTypeIndex].get();
-                    const uint64_t sizeToCache = MemorySize::kPowerOfTwoCacheSizes[i].SizeInBytes;
-                    if (sizeToCache > allocator->GetMemorySize()) {
-                        continue;
-                    }
 
-                    MEMORY_ALLOCATION_REQUEST cacheRequest = {};
-                    cacheRequest.SizeInBytes = sizeToCache;
-                    cacheRequest.NeverAllocate = true;
-                    cacheRequest.AlwaysCacheSize = true;
-                    cacheRequest.AlwaysPrefetch = false;
-                    cacheRequest.AvailableForAllocation = kInvalidSize;
+                MEMORY_ALLOCATION_REQUEST cacheRequest = {};
+                cacheRequest.NeverAllocate = true;
+                cacheRequest.AlwaysCacheSize = true;
+                cacheRequest.AlwaysPrefetch = false;
+                cacheRequest.AvailableForAllocation = kInvalidSize;
+
+                for (uint64_t i = 0; i < MemorySize::kPowerOfTwoClassSize; i++) {
+                    MemoryAllocator* allocator = nullptr;
+                    cacheRequest.SizeInBytes = MemorySize::kPowerOfTwoCacheSizes[i].SizeInBytes;
 
-                    if (IsAligned(MemorySize::kPowerOfTwoCacheSizes[i].SizeInBytes,
+                    allocator = mSmallBufferAllocatorOfType[resourceHeapTypeIndex].get();
+                    if (cacheRequest.SizeInBytes <= allocator->GetMemorySize() &&
+                        IsAligned(cacheRequest.SizeInBytes,
                                   D3D12_SMALL_RESOURCE_PLACEMENT_ALIGNMENT)) {
                         cacheRequest.Alignment = D3D12_SMALL_RESOURCE_PLACEMENT_ALIGNMENT;
                         allocator->TryAllocateMemory(cacheRequest);
                     }
 
-                    if (IsAligned(MemorySize::kPowerOfTwoCacheSizes[i].SizeInBytes,
+                    allocator = mResourceAllocatorOfType[resourceHeapTypeIndex].get();
+                    if (cacheRequest.SizeInBytes <= allocator->GetMemorySize() &&
+                        IsAligned(cacheRequest.SizeInBytes,
                                   D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT)) {
                         cacheRequest.Alignment = D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT;
                         allocator->TryAllocateMemory(cacheRequest);
                     }
 
-                    if (IsAligned(MemorySize::kPowerOfTwoCacheSizes[i].SizeInBytes,
+                    allocator = mMSAAResourceAllocatorOfType[resourceHeapTypeIndex].get();
+                    if (cacheRequest.SizeInBytes <= allocator->GetMemorySize() &&
+                        IsAligned(cacheRequest.SizeInBytes,
                                   D3D12_DEFAULT_MSAA_RESOURCE_PLACEMENT_ALIGNMENT)) {
                         cacheRequest.Alignment = D3D12_DEFAULT_MSAA_RESOURCE_PLACEMENT_ALIGNMENT;
                         allocator->TryAllocateMemory(cacheRequest);
                     }
                 }
             }
-#endif
+#endif  // defined(GPGMM_ENABLE_SIZE_CACHE)
         }
     }
 
@@ -1111,12 +1112,18 @@ namespace gpgmm { namespace d3d12 {
         TRACE_COUNTER1(TraceEventCategory::Default, "GPU allocation free (MB)",
                        result.FreeMemoryUsage / 1e6);
 
-        TRACE_COUNTER1(TraceEventCategory::Default, "GPU memory prefetch (%)",
+        TRACE_COUNTER1(TraceEventCategory::Default, "GPU prefetch memory miss (%)",
                        SafeDivison(result.PrefetchedMemoryMissesEliminated,
                                    static_cast<double>(result.PrefetchedMemoryMisses +
                                                        result.PrefetchedMemoryMissesEliminated)) *
                            100);
 
+        TRACE_COUNTER1(
+            TraceEventCategory::Default, "GPU size cache miss (%)",
+            SafeDivison(result.CacheSizeMisses,
+                        static_cast<double>(result.CacheSizeMisses + result.CacheSizeHits)) *
+                100);
+
         return result;
     }
 

src/gpgmm/d3d12/ResourceAllocatorD3D12.h

@@ -429,7 +429,7 @@ namespace gpgmm { namespace d3d12 {
 
         Allow internal data structures used for resource allocation to be cached in-memory.
         */
-        ALLOCATION_FLAG_ALWAYS_CACHE_SIZE = 0x16,
+        ALLOCATION_FLAG_ALWAYS_CACHE_SIZE = 0x10,
     };
 
     using ALLOCATION_FLAGS_TYPE = Flags<ALLOCATION_FLAGS>;

src/tests/D3D12Test.cpp

@@ -134,4 +134,12 @@ namespace gpgmm { namespace d3d12 {
         return GPGMMTestBase::GenerateTestAllocations(D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT);
     }
 
+    bool D3D12TestBase::IsSizeCacheEnabled() const {
+#if defined(GPGMM_ENABLE_SIZE_CACHE)
+        return true;
+#else
+        return false;
+#endif
+    }
+
 }}  // namespace gpgmm::d3d12

src/tests/D3D12Test.h

@@ -50,6 +50,8 @@ namespace gpgmm { namespace d3d12 {
 
         static std::vector<MEMORY_ALLOCATION_EXPECT> GenerateBufferAllocations();
 
+        bool IsSizeCacheEnabled() const;
+
       protected:
         ComPtr<IDXGIAdapter3> mAdapter;
         ComPtr<ID3D12Device> mDevice;

src/tests/end2end/D3D12ResourceAllocatorTests.cpp

@@ -29,6 +29,24 @@ static constexpr uint64_t kDefaultBufferSize = 4ll * 1024ll * 1024ll;  // 4MB
 
 #define GPGMM_GET_VAR_NAME(x) (#x)
 
+#define EXPECT_SIZE_CACHE_HIT(allocator, statement)              \
+    do {                                                         \
+        ASSERT_NE(allocator, nullptr);                           \
+        size_t countBefore = allocator->GetInfo().CacheSizeHits; \
+        EXPECT_SUCCEEDED(statement);                             \
+        size_t countAfter = allocator->GetInfo().CacheSizeHits;  \
+        EXPECT_GT(countAfter, countBefore);                      \
+    } while (0)
+
+#define EXPECT_SIZE_CACHE_MISS(allocator, statement)               \
+    do {                                                           \
+        ASSERT_NE(allocator, nullptr);                             \
+        size_t countBefore = allocator->GetInfo().CacheSizeMisses; \
+        EXPECT_SUCCEEDED(statement);                               \
+        size_t countAfter = allocator->GetInfo().CacheSizeMisses;  \
+        EXPECT_GT(countAfter, countBefore);                        \
+    } while (0)
+
 class D3D12ResourceAllocatorTests : public D3D12TestBase, public ::testing::Test {
   protected:
     void SetUp() override {
@@ -1091,6 +1109,158 @@ TEST_F(D3D12ResourceAllocatorTests, CreateBufferWithinManyThreaded) {
     EXPECT_EQ(resourceAllocator->GetInfo().UsedMemoryUsage, 0u);
 }
 
+TEST_F(D3D12ResourceAllocatorTests, CreateBufferCacheSize) {
+    GPGMM_SKIP_TEST_IF(!IsSizeCacheEnabled());
+
+    ComPtr<ResourceAllocator> resourceAllocator;
+    ASSERT_SUCCEEDED(
+        ResourceAllocator::CreateAllocator(CreateBasicAllocatorDesc(), &resourceAllocator));
+    ASSERT_NE(resourceAllocator, nullptr);
+
+    // Upon creating the resource allocator, min. buffer size-alignment is always cached.
+    {
+        ComPtr<ResourceAllocation> cachedAllocation;
+
+        ALLOCATION_DESC smallResourceAllocDesc = {};
+        smallResourceAllocDesc.HeapType = D3D12_HEAP_TYPE_UPLOAD;
+        smallResourceAllocDesc.Flags |= ALLOCATION_FLAG_ALLOW_SUBALLOCATE_WITHIN_RESOURCE;
+
+        EXPECT_SIZE_CACHE_HIT(resourceAllocator,
+                              resourceAllocator->CreateResource(
+                                  smallResourceAllocDesc,
+                                  CreateBasicBufferDesc(D3D12_SMALL_RESOURCE_PLACEMENT_ALIGNMENT),
+                                  D3D12_RESOURCE_STATE_GENERIC_READ, nullptr, &cachedAllocation));
+        ASSERT_NE(cachedAllocation, nullptr);
+        EXPECT_EQ(cachedAllocation->GetSize(),
+                  static_cast<uint64_t>(D3D12_SMALL_RESOURCE_PLACEMENT_ALIGNMENT));
+    }
+    {
+        ComPtr<ResourceAllocation> cachedAllocation;
+        EXPECT_SIZE_CACHE_HIT(
+            resourceAllocator,
+            resourceAllocator->CreateResource(
+                {}, CreateBasicBufferDesc(D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT),
+                D3D12_RESOURCE_STATE_COMMON, nullptr, &cachedAllocation));
+        ASSERT_NE(cachedAllocation, nullptr);
+        EXPECT_EQ(cachedAllocation->GetSize(),
+                  static_cast<uint64_t>(D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT));
+    }
+    {
+        ComPtr<ResourceAllocation> cachedAllocation;
+        EXPECT_SIZE_CACHE_HIT(
+            resourceAllocator,
+            resourceAllocator->CreateResource(
+                {}, CreateBasicBufferDesc(D3D12_DEFAULT_MSAA_RESOURCE_PLACEMENT_ALIGNMENT),
+                D3D12_RESOURCE_STATE_COMMON, nullptr, &cachedAllocation));
+        ASSERT_NE(cachedAllocation, nullptr);
+        EXPECT_EQ(cachedAllocation->GetSize(),
+                  static_cast<uint64_t>(D3D12_DEFAULT_MSAA_RESOURCE_PLACEMENT_ALIGNMENT));
+    }
+
+    // Verify misaligned allocations are uncached.
+    {
+        ComPtr<ResourceAllocation> uncachedAllocation;
+
+        ALLOCATION_DESC smallResourceAllocDesc = {};
+        smallResourceAllocDesc.HeapType = D3D12_HEAP_TYPE_UPLOAD;
+        smallResourceAllocDesc.Flags |= ALLOCATION_FLAG_ALLOW_SUBALLOCATE_WITHIN_RESOURCE;
+
+        EXPECT_SIZE_CACHE_MISS(
+            resourceAllocator,
+            resourceAllocator->CreateResource(
+                smallResourceAllocDesc,
+                CreateBasicBufferDesc(D3D12_SMALL_RESOURCE_PLACEMENT_ALIGNMENT * 3),
+                D3D12_RESOURCE_STATE_GENERIC_READ, nullptr, &uncachedAllocation));
+
+        ASSERT_NE(uncachedAllocation, nullptr);
+        EXPECT_EQ(uncachedAllocation->GetSize(),
+                  static_cast<uint64_t>(D3D12_SMALL_RESOURCE_PLACEMENT_ALIGNMENT * 3));
+    }
+    {
+        ComPtr<ResourceAllocation> uncachedAllocation;
+        EXPECT_SIZE_CACHE_MISS(
+            resourceAllocator,
+            resourceAllocator->CreateResource(
+                {}, CreateBasicBufferDesc(D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT * 3),
+                D3D12_RESOURCE_STATE_COMMON, nullptr, &uncachedAllocation));
+
+        ASSERT_NE(uncachedAllocation, nullptr);
+        EXPECT_EQ(uncachedAllocation->GetSize(),
+                  static_cast<uint64_t>(D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT * 3));
+    }
+    {
+        ComPtr<ResourceAllocation> uncachedAllocation;
+        EXPECT_SIZE_CACHE_MISS(
+            resourceAllocator,
+            resourceAllocator->CreateResource(
+                {}, CreateBasicBufferDesc(D3D12_DEFAULT_MSAA_RESOURCE_PLACEMENT_ALIGNMENT * 3),
+                D3D12_RESOURCE_STATE_COMMON, nullptr, &uncachedAllocation));
+
+        ASSERT_NE(uncachedAllocation, nullptr);
+        EXPECT_EQ(uncachedAllocation->GetSize(),
+                  static_cast<uint64_t>(D3D12_DEFAULT_MSAA_RESOURCE_PLACEMENT_ALIGNMENT * 3));
+    }
+
+    // Verify requesting for cached always caches.
+    ALLOCATION_DESC allocationWithSizeCached = {};
+    allocationWithSizeCached.Flags = ALLOCATION_FLAG_ALWAYS_CACHE_SIZE;
+    {
+        ALLOCATION_DESC smallResourceAllocDesc = allocationWithSizeCached;
+        smallResourceAllocDesc.HeapType = D3D12_HEAP_TYPE_UPLOAD;
+        smallResourceAllocDesc.Flags |= ALLOCATION_FLAG_ALLOW_SUBALLOCATE_WITHIN_RESOURCE;
+
+        ComPtr<ResourceAllocation> uncachedAllocation;
+        EXPECT_SIZE_CACHE_MISS(
+            resourceAllocator,
+            resourceAllocator->CreateResource(
+                smallResourceAllocDesc,
+                CreateBasicBufferDesc(D3D12_SMALL_RESOURCE_PLACEMENT_ALIGNMENT * 3),
+                D3D12_RESOURCE_STATE_GENERIC_READ, nullptr, &uncachedAllocation));
+
+        ComPtr<ResourceAllocation> cachedAllocation;
+        EXPECT_SIZE_CACHE_HIT(
+            resourceAllocator,
+            resourceAllocator->CreateResource(
+                smallResourceAllocDesc,
+                CreateBasicBufferDesc(D3D12_SMALL_RESOURCE_PLACEMENT_ALIGNMENT * 3),
+                D3D12_RESOURCE_STATE_GENERIC_READ, nullptr, &cachedAllocation));
+    }
+    {
+        ComPtr<ResourceAllocation> uncachedAllocation;
+        EXPECT_SIZE_CACHE_MISS(
+            resourceAllocator,
+            resourceAllocator->CreateResource(
+                allocationWithSizeCached,
+                CreateBasicBufferDesc(D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT * 3),
+                D3D12_RESOURCE_STATE_COMMON, nullptr, &uncachedAllocation));
+
+        ComPtr<ResourceAllocation> cachedAllocation;
+        EXPECT_SIZE_CACHE_HIT(
+            resourceAllocator,
+            resourceAllocator->CreateResource(
+                allocationWithSizeCached,
+                CreateBasicBufferDesc(D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT * 3),
+                D3D12_RESOURCE_STATE_COMMON, nullptr, &cachedAllocation));
+    }
+    {
+        ComPtr<ResourceAllocation> uncachedAllocation;
+        EXPECT_SIZE_CACHE_MISS(
+            resourceAllocator,
+            resourceAllocator->CreateResource(
+                allocationWithSizeCached,
+                CreateBasicBufferDesc(D3D12_DEFAULT_MSAA_RESOURCE_PLACEMENT_ALIGNMENT * 3),
+                D3D12_RESOURCE_STATE_COMMON, nullptr, &uncachedAllocation));
+
+        ComPtr<ResourceAllocation> cachedAllocation;
+        EXPECT_SIZE_CACHE_HIT(
+            resourceAllocator,
+            resourceAllocator->CreateResource(
+                allocationWithSizeCached,
+                CreateBasicBufferDesc(D3D12_DEFAULT_MSAA_RESOURCE_PLACEMENT_ALIGNMENT * 3),
+                D3D12_RESOURCE_STATE_COMMON, nullptr, &cachedAllocation));
+    }
+}
+
 TEST_F(D3D12ResourceAllocatorTests, CheckFeatureSupport) {
     ComPtr<ResourceAllocator> resourceAllocator;
     ASSERT_SUCCEEDED(