From 214bae30724487d69eb923a5f77b9df5fe2ca73c Mon Sep 17 00:00:00 2001 From: Bryan Bernhart Date: Mon, 13 Jun 2022 14:35:42 -0700 Subject: [PATCH] Fix ALLOCATION_FLAG_ALWAYS_CACHE_SIZE from disabling sub-allocation. Flag value specified by GPGMM was wrong, which resulted in sub-allocation being disabled. Also, cached sizes did not apply to sub-allocation within which may be unexpected. --- src/gpgmm/common/MemoryAllocator.h | 13 ++ src/gpgmm/common/SlabMemoryAllocator.cpp | 5 + src/gpgmm/d3d12/ResourceAllocatorD3D12.cpp | 51 +++--- src/gpgmm/d3d12/ResourceAllocatorD3D12.h | 2 +- src/tests/D3D12Test.cpp | 8 + src/tests/D3D12Test.h | 2 + .../end2end/D3D12ResourceAllocatorTests.cpp | 170 ++++++++++++++++++ 7 files changed, 228 insertions(+), 23 deletions(-) diff --git a/src/gpgmm/common/MemoryAllocator.h b/src/gpgmm/common/MemoryAllocator.h index 9002c8dbb..a61a3cbe7 100644 --- a/src/gpgmm/common/MemoryAllocator.h +++ b/src/gpgmm/common/MemoryAllocator.h @@ -63,14 +63,27 @@ namespace gpgmm { */ uint64_t PrefetchedMemoryMissesEliminated; + /** \brief Requested size was NOT cached. + */ + uint64_t CacheSizeMisses; + + /** \brief Requested size was cached. + */ + uint64_t CacheSizeHits; + MEMORY_ALLOCATOR_INFO& operator+=(const MEMORY_ALLOCATOR_INFO& rhs) { UsedBlockCount += rhs.UsedBlockCount; UsedBlockUsage += rhs.UsedBlockUsage; FreeMemoryUsage += rhs.FreeMemoryUsage; UsedMemoryUsage += rhs.UsedMemoryUsage; UsedMemoryCount += rhs.UsedMemoryCount; + PrefetchedMemoryMisses += rhs.PrefetchedMemoryMisses; PrefetchedMemoryMissesEliminated += rhs.PrefetchedMemoryMissesEliminated; + + CacheSizeMisses += rhs.CacheSizeMisses; + CacheSizeHits += rhs.CacheSizeHits; + return *this; } }; diff --git a/src/gpgmm/common/SlabMemoryAllocator.cpp b/src/gpgmm/common/SlabMemoryAllocator.cpp index 21f61dde7..4ea1d8b71 100644 --- a/src/gpgmm/common/SlabMemoryAllocator.cpp +++ b/src/gpgmm/common/SlabMemoryAllocator.cpp @@ -487,6 +487,11 @@ namespace gpgmm { result.UsedMemoryCount = info.UsedMemoryCount; result.UsedMemoryUsage = info.UsedMemoryUsage; + // Size cache is common across slab allocators. + const CacheStats& sizeCacheStats = mSizeCache.GetStats(); + result.CacheSizeHits = sizeCacheStats.NumOfHits; + result.CacheSizeMisses = sizeCacheStats.NumOfMisses; + return result; } diff --git a/src/gpgmm/d3d12/ResourceAllocatorD3D12.cpp b/src/gpgmm/d3d12/ResourceAllocatorD3D12.cpp index 41475a178..6bbedd0fa 100644 --- a/src/gpgmm/d3d12/ResourceAllocatorD3D12.cpp +++ b/src/gpgmm/d3d12/ResourceAllocatorD3D12.cpp @@ -443,50 +443,51 @@ namespace gpgmm { namespace d3d12 { CreateSmallBufferAllocator(descriptor, heapFlags, heapType, heapAlignment); // Cache resource sizes commonly requested. - // Ensures the next block is always made available upon first request without - // increasing the memory footprint. Since resources are always sized-aligned, the + // Allows the next memory block to be made available upon request without + // increasing memory footprint. Since resources are always sized-aligned, the // cached size must be requested per alignment {4KB, 64KB, or 4MB}. To avoid unbounded // cache growth, a known set of pre-defined sizes initializes the allocators. - #if defined(GPGMM_ENABLE_SIZE_CACHE) - // Temporary suppress log messages emitted from internal cache-miss requests. { + // Temporary suppress log messages emitted from internal cache-miss requests. ScopedLogLevel scopedLogLevel(LogSeverity::Info); - for (uint64_t i = 0; i < MemorySize::kPowerOfTwoClassSize; i++) { - MemoryAllocator* allocator = - mResourceAllocatorOfType[resourceHeapTypeIndex].get(); - const uint64_t sizeToCache = MemorySize::kPowerOfTwoCacheSizes[i].SizeInBytes; - if (sizeToCache > allocator->GetMemorySize()) { - continue; - } - MEMORY_ALLOCATION_REQUEST cacheRequest = {}; - cacheRequest.SizeInBytes = sizeToCache; - cacheRequest.NeverAllocate = true; - cacheRequest.AlwaysCacheSize = true; - cacheRequest.AlwaysPrefetch = false; - cacheRequest.AvailableForAllocation = kInvalidSize; + MEMORY_ALLOCATION_REQUEST cacheRequest = {}; + cacheRequest.NeverAllocate = true; + cacheRequest.AlwaysCacheSize = true; + cacheRequest.AlwaysPrefetch = false; + cacheRequest.AvailableForAllocation = kInvalidSize; + + for (uint64_t i = 0; i < MemorySize::kPowerOfTwoClassSize; i++) { + MemoryAllocator* allocator = nullptr; + cacheRequest.SizeInBytes = MemorySize::kPowerOfTwoCacheSizes[i].SizeInBytes; - if (IsAligned(MemorySize::kPowerOfTwoCacheSizes[i].SizeInBytes, + allocator = mSmallBufferAllocatorOfType[resourceHeapTypeIndex].get(); + if (cacheRequest.SizeInBytes <= allocator->GetMemorySize() && + IsAligned(cacheRequest.SizeInBytes, D3D12_SMALL_RESOURCE_PLACEMENT_ALIGNMENT)) { cacheRequest.Alignment = D3D12_SMALL_RESOURCE_PLACEMENT_ALIGNMENT; allocator->TryAllocateMemory(cacheRequest); } - if (IsAligned(MemorySize::kPowerOfTwoCacheSizes[i].SizeInBytes, + allocator = mResourceAllocatorOfType[resourceHeapTypeIndex].get(); + if (cacheRequest.SizeInBytes <= allocator->GetMemorySize() && + IsAligned(cacheRequest.SizeInBytes, D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT)) { cacheRequest.Alignment = D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT; allocator->TryAllocateMemory(cacheRequest); } - if (IsAligned(MemorySize::kPowerOfTwoCacheSizes[i].SizeInBytes, + allocator = mMSAAResourceAllocatorOfType[resourceHeapTypeIndex].get(); + if (cacheRequest.SizeInBytes <= allocator->GetMemorySize() && + IsAligned(cacheRequest.SizeInBytes, D3D12_DEFAULT_MSAA_RESOURCE_PLACEMENT_ALIGNMENT)) { cacheRequest.Alignment = D3D12_DEFAULT_MSAA_RESOURCE_PLACEMENT_ALIGNMENT; allocator->TryAllocateMemory(cacheRequest); } } } -#endif +#endif // defined(GPGMM_ENABLE_SIZE_CACHE) } } @@ -1111,12 +1112,18 @@ namespace gpgmm { namespace d3d12 { TRACE_COUNTER1(TraceEventCategory::Default, "GPU allocation free (MB)", result.FreeMemoryUsage / 1e6); - TRACE_COUNTER1(TraceEventCategory::Default, "GPU memory prefetch (%)", + TRACE_COUNTER1(TraceEventCategory::Default, "GPU prefetch memory miss (%)", SafeDivison(result.PrefetchedMemoryMissesEliminated, static_cast(result.PrefetchedMemoryMisses + result.PrefetchedMemoryMissesEliminated)) * 100); + TRACE_COUNTER1( + TraceEventCategory::Default, "GPU size cache miss (%)", + SafeDivison(result.CacheSizeMisses, + static_cast(result.CacheSizeMisses + result.CacheSizeHits)) * + 100); + return result; } diff --git a/src/gpgmm/d3d12/ResourceAllocatorD3D12.h b/src/gpgmm/d3d12/ResourceAllocatorD3D12.h index af00158d8..5b18945e1 100644 --- a/src/gpgmm/d3d12/ResourceAllocatorD3D12.h +++ b/src/gpgmm/d3d12/ResourceAllocatorD3D12.h @@ -429,7 +429,7 @@ namespace gpgmm { namespace d3d12 { Allow internal data structures used for resource allocation to be cached in-memory. */ - ALLOCATION_FLAG_ALWAYS_CACHE_SIZE = 0x16, + ALLOCATION_FLAG_ALWAYS_CACHE_SIZE = 0x10, }; using ALLOCATION_FLAGS_TYPE = Flags; diff --git a/src/tests/D3D12Test.cpp b/src/tests/D3D12Test.cpp index 5cec92ea4..cc44891c7 100644 --- a/src/tests/D3D12Test.cpp +++ b/src/tests/D3D12Test.cpp @@ -134,4 +134,12 @@ namespace gpgmm { namespace d3d12 { return GPGMMTestBase::GenerateTestAllocations(D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT); } + bool D3D12TestBase::IsSizeCacheEnabled() const { +#if defined(GPGMM_ENABLE_SIZE_CACHE) + return true; +#else + return false; +#endif + } + }} // namespace gpgmm::d3d12 diff --git a/src/tests/D3D12Test.h b/src/tests/D3D12Test.h index b6fdc9225..6fdd08300 100644 --- a/src/tests/D3D12Test.h +++ b/src/tests/D3D12Test.h @@ -50,6 +50,8 @@ namespace gpgmm { namespace d3d12 { static std::vector GenerateBufferAllocations(); + bool IsSizeCacheEnabled() const; + protected: ComPtr mAdapter; ComPtr mDevice; diff --git a/src/tests/end2end/D3D12ResourceAllocatorTests.cpp b/src/tests/end2end/D3D12ResourceAllocatorTests.cpp index 814acc4fb..aa1ffdf4f 100644 --- a/src/tests/end2end/D3D12ResourceAllocatorTests.cpp +++ b/src/tests/end2end/D3D12ResourceAllocatorTests.cpp @@ -29,6 +29,24 @@ static constexpr uint64_t kDefaultBufferSize = 4ll * 1024ll * 1024ll; // 4MB #define GPGMM_GET_VAR_NAME(x) (#x) +#define EXPECT_SIZE_CACHE_HIT(allocator, statement) \ + do { \ + ASSERT_NE(allocator, nullptr); \ + size_t countBefore = allocator->GetInfo().CacheSizeHits; \ + EXPECT_SUCCEEDED(statement); \ + size_t countAfter = allocator->GetInfo().CacheSizeHits; \ + EXPECT_GT(countAfter, countBefore); \ + } while (0) + +#define EXPECT_SIZE_CACHE_MISS(allocator, statement) \ + do { \ + ASSERT_NE(allocator, nullptr); \ + size_t countBefore = allocator->GetInfo().CacheSizeMisses; \ + EXPECT_SUCCEEDED(statement); \ + size_t countAfter = allocator->GetInfo().CacheSizeMisses; \ + EXPECT_GT(countAfter, countBefore); \ + } while (0) + class D3D12ResourceAllocatorTests : public D3D12TestBase, public ::testing::Test { protected: void SetUp() override { @@ -1091,6 +1109,158 @@ TEST_F(D3D12ResourceAllocatorTests, CreateBufferWithinManyThreaded) { EXPECT_EQ(resourceAllocator->GetInfo().UsedMemoryUsage, 0u); } +TEST_F(D3D12ResourceAllocatorTests, CreateBufferCacheSize) { + GPGMM_SKIP_TEST_IF(!IsSizeCacheEnabled()); + + ComPtr resourceAllocator; + ASSERT_SUCCEEDED( + ResourceAllocator::CreateAllocator(CreateBasicAllocatorDesc(), &resourceAllocator)); + ASSERT_NE(resourceAllocator, nullptr); + + // Upon creating the resource allocator, min. buffer size-alignment is always cached. + { + ComPtr cachedAllocation; + + ALLOCATION_DESC smallResourceAllocDesc = {}; + smallResourceAllocDesc.HeapType = D3D12_HEAP_TYPE_UPLOAD; + smallResourceAllocDesc.Flags |= ALLOCATION_FLAG_ALLOW_SUBALLOCATE_WITHIN_RESOURCE; + + EXPECT_SIZE_CACHE_HIT(resourceAllocator, + resourceAllocator->CreateResource( + smallResourceAllocDesc, + CreateBasicBufferDesc(D3D12_SMALL_RESOURCE_PLACEMENT_ALIGNMENT), + D3D12_RESOURCE_STATE_GENERIC_READ, nullptr, &cachedAllocation)); + ASSERT_NE(cachedAllocation, nullptr); + EXPECT_EQ(cachedAllocation->GetSize(), + static_cast(D3D12_SMALL_RESOURCE_PLACEMENT_ALIGNMENT)); + } + { + ComPtr cachedAllocation; + EXPECT_SIZE_CACHE_HIT( + resourceAllocator, + resourceAllocator->CreateResource( + {}, CreateBasicBufferDesc(D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT), + D3D12_RESOURCE_STATE_COMMON, nullptr, &cachedAllocation)); + ASSERT_NE(cachedAllocation, nullptr); + EXPECT_EQ(cachedAllocation->GetSize(), + static_cast(D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT)); + } + { + ComPtr cachedAllocation; + EXPECT_SIZE_CACHE_HIT( + resourceAllocator, + resourceAllocator->CreateResource( + {}, CreateBasicBufferDesc(D3D12_DEFAULT_MSAA_RESOURCE_PLACEMENT_ALIGNMENT), + D3D12_RESOURCE_STATE_COMMON, nullptr, &cachedAllocation)); + ASSERT_NE(cachedAllocation, nullptr); + EXPECT_EQ(cachedAllocation->GetSize(), + static_cast(D3D12_DEFAULT_MSAA_RESOURCE_PLACEMENT_ALIGNMENT)); + } + + // Verify misaligned allocations are uncached. + { + ComPtr uncachedAllocation; + + ALLOCATION_DESC smallResourceAllocDesc = {}; + smallResourceAllocDesc.HeapType = D3D12_HEAP_TYPE_UPLOAD; + smallResourceAllocDesc.Flags |= ALLOCATION_FLAG_ALLOW_SUBALLOCATE_WITHIN_RESOURCE; + + EXPECT_SIZE_CACHE_MISS( + resourceAllocator, + resourceAllocator->CreateResource( + smallResourceAllocDesc, + CreateBasicBufferDesc(D3D12_SMALL_RESOURCE_PLACEMENT_ALIGNMENT * 3), + D3D12_RESOURCE_STATE_GENERIC_READ, nullptr, &uncachedAllocation)); + + ASSERT_NE(uncachedAllocation, nullptr); + EXPECT_EQ(uncachedAllocation->GetSize(), + static_cast(D3D12_SMALL_RESOURCE_PLACEMENT_ALIGNMENT * 3)); + } + { + ComPtr uncachedAllocation; + EXPECT_SIZE_CACHE_MISS( + resourceAllocator, + resourceAllocator->CreateResource( + {}, CreateBasicBufferDesc(D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT * 3), + D3D12_RESOURCE_STATE_COMMON, nullptr, &uncachedAllocation)); + + ASSERT_NE(uncachedAllocation, nullptr); + EXPECT_EQ(uncachedAllocation->GetSize(), + static_cast(D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT * 3)); + } + { + ComPtr uncachedAllocation; + EXPECT_SIZE_CACHE_MISS( + resourceAllocator, + resourceAllocator->CreateResource( + {}, CreateBasicBufferDesc(D3D12_DEFAULT_MSAA_RESOURCE_PLACEMENT_ALIGNMENT * 3), + D3D12_RESOURCE_STATE_COMMON, nullptr, &uncachedAllocation)); + + ASSERT_NE(uncachedAllocation, nullptr); + EXPECT_EQ(uncachedAllocation->GetSize(), + static_cast(D3D12_DEFAULT_MSAA_RESOURCE_PLACEMENT_ALIGNMENT * 3)); + } + + // Verify requesting for cached always caches. + ALLOCATION_DESC allocationWithSizeCached = {}; + allocationWithSizeCached.Flags = ALLOCATION_FLAG_ALWAYS_CACHE_SIZE; + { + ALLOCATION_DESC smallResourceAllocDesc = allocationWithSizeCached; + smallResourceAllocDesc.HeapType = D3D12_HEAP_TYPE_UPLOAD; + smallResourceAllocDesc.Flags |= ALLOCATION_FLAG_ALLOW_SUBALLOCATE_WITHIN_RESOURCE; + + ComPtr uncachedAllocation; + EXPECT_SIZE_CACHE_MISS( + resourceAllocator, + resourceAllocator->CreateResource( + smallResourceAllocDesc, + CreateBasicBufferDesc(D3D12_SMALL_RESOURCE_PLACEMENT_ALIGNMENT * 3), + D3D12_RESOURCE_STATE_GENERIC_READ, nullptr, &uncachedAllocation)); + + ComPtr cachedAllocation; + EXPECT_SIZE_CACHE_HIT( + resourceAllocator, + resourceAllocator->CreateResource( + smallResourceAllocDesc, + CreateBasicBufferDesc(D3D12_SMALL_RESOURCE_PLACEMENT_ALIGNMENT * 3), + D3D12_RESOURCE_STATE_GENERIC_READ, nullptr, &cachedAllocation)); + } + { + ComPtr uncachedAllocation; + EXPECT_SIZE_CACHE_MISS( + resourceAllocator, + resourceAllocator->CreateResource( + allocationWithSizeCached, + CreateBasicBufferDesc(D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT * 3), + D3D12_RESOURCE_STATE_COMMON, nullptr, &uncachedAllocation)); + + ComPtr cachedAllocation; + EXPECT_SIZE_CACHE_HIT( + resourceAllocator, + resourceAllocator->CreateResource( + allocationWithSizeCached, + CreateBasicBufferDesc(D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT * 3), + D3D12_RESOURCE_STATE_COMMON, nullptr, &cachedAllocation)); + } + { + ComPtr uncachedAllocation; + EXPECT_SIZE_CACHE_MISS( + resourceAllocator, + resourceAllocator->CreateResource( + allocationWithSizeCached, + CreateBasicBufferDesc(D3D12_DEFAULT_MSAA_RESOURCE_PLACEMENT_ALIGNMENT * 3), + D3D12_RESOURCE_STATE_COMMON, nullptr, &uncachedAllocation)); + + ComPtr cachedAllocation; + EXPECT_SIZE_CACHE_HIT( + resourceAllocator, + resourceAllocator->CreateResource( + allocationWithSizeCached, + CreateBasicBufferDesc(D3D12_DEFAULT_MSAA_RESOURCE_PLACEMENT_ALIGNMENT * 3), + D3D12_RESOURCE_STATE_COMMON, nullptr, &cachedAllocation)); + } +} + TEST_F(D3D12ResourceAllocatorTests, CheckFeatureSupport) { ComPtr resourceAllocator; ASSERT_SUCCEEDED(