diff --git a/src/gpgmm/d3d12/BufferAllocatorD3D12.cpp b/src/gpgmm/d3d12/BufferAllocatorD3D12.cpp index 145f37efc..3516789e2 100644 --- a/src/gpgmm/d3d12/BufferAllocatorD3D12.cpp +++ b/src/gpgmm/d3d12/BufferAllocatorD3D12.cpp @@ -14,11 +14,13 @@ #include "gpgmm/d3d12/BufferAllocatorD3D12.h" +#include "gpgmm/common/EventMessage.h" #include "gpgmm/common/TraceEvent.h" #include "gpgmm/d3d12/BackendD3D12.h" #include "gpgmm/d3d12/HeapD3D12.h" #include "gpgmm/d3d12/ResourceAllocationD3D12.h" #include "gpgmm/d3d12/ResourceAllocatorD3D12.h" +#include "gpgmm/utils/Math.h" namespace gpgmm::d3d12 { @@ -26,16 +28,12 @@ namespace gpgmm::d3d12 { D3D12_HEAP_PROPERTIES heapProperties, D3D12_HEAP_FLAGS heapFlags, D3D12_RESOURCE_FLAGS resourceFlags, - D3D12_RESOURCE_STATES initialResourceState, - uint64_t bufferSize, - uint64_t bufferAlignment) + D3D12_RESOURCE_STATES initialResourceState) : mResourceAllocator(resourceAllocator), mHeapProperties(heapProperties), mHeapFlags(heapFlags), mResourceFlags(resourceFlags), - mInitialResourceState(initialResourceState), - mBufferSize(bufferSize), - mBufferAlignment(bufferAlignment) { + mInitialResourceState(initialResourceState) { } std::unique_ptr BufferAllocator::TryAllocateMemory( @@ -44,13 +42,20 @@ namespace gpgmm::d3d12 { std::lock_guard lock(mMutex); - if (GetMemorySize() != request.SizeInBytes || GetMemoryAlignment() != request.Alignment || - request.NeverAllocate) { + if (request.NeverAllocate) { return {}; } + const uint64_t heapSize = AlignTo(request.SizeInBytes, request.Alignment); + if (heapSize > request.SizeInBytes) { + DebugEvent(GetTypename(), EventMessageId::AlignmentMismatch) + << "Resource heap size is larger then the requested size (" + + std::to_string(heapSize) + " vs " + std::to_string(request.SizeInBytes) + + " bytes)."; + } + D3D12_RESOURCE_ALLOCATION_INFO info = {}; - info.SizeInBytes = request.SizeInBytes; + info.SizeInBytes = heapSize; info.Alignment = request.Alignment; D3D12_RESOURCE_DESC resourceDescriptor; @@ -90,13 +95,4 @@ namespace gpgmm::d3d12 { SafeRelease(allocation); } - - uint64_t BufferAllocator::GetMemorySize() const { - return mBufferSize; - } - - uint64_t BufferAllocator::GetMemoryAlignment() const { - return mBufferAlignment; - } - } // namespace gpgmm::d3d12 diff --git a/src/gpgmm/d3d12/BufferAllocatorD3D12.h b/src/gpgmm/d3d12/BufferAllocatorD3D12.h index dfcbf6798..6a7a2141f 100644 --- a/src/gpgmm/d3d12/BufferAllocatorD3D12.h +++ b/src/gpgmm/d3d12/BufferAllocatorD3D12.h @@ -29,9 +29,7 @@ namespace gpgmm::d3d12 { D3D12_HEAP_PROPERTIES heapProperties, D3D12_HEAP_FLAGS heapFlags, D3D12_RESOURCE_FLAGS resourceFlags, - D3D12_RESOURCE_STATES initialResourceState, - uint64_t bufferSize, - uint64_t bufferAlignment); + D3D12_RESOURCE_STATES initialResourceState); ~BufferAllocator() override = default; // MemoryAllocator interface @@ -39,9 +37,6 @@ namespace gpgmm::d3d12 { const MemoryAllocationRequest& request) override; void DeallocateMemory(std::unique_ptr allocation) override; - uint64_t GetMemorySize() const override; - uint64_t GetMemoryAlignment() const override; - private: ResourceAllocator* const mResourceAllocator; @@ -49,8 +44,6 @@ namespace gpgmm::d3d12 { const D3D12_HEAP_FLAGS mHeapFlags; const D3D12_RESOURCE_FLAGS mResourceFlags; const D3D12_RESOURCE_STATES mInitialResourceState; - const uint64_t mBufferSize; - const uint64_t mBufferAlignment; }; } // namespace gpgmm::d3d12 diff --git a/src/gpgmm/d3d12/ResourceAllocatorD3D12.cpp b/src/gpgmm/d3d12/ResourceAllocatorD3D12.cpp index 0d9ceed65..34f363fef 100644 --- a/src/gpgmm/d3d12/ResourceAllocatorD3D12.cpp +++ b/src/gpgmm/d3d12/ResourceAllocatorD3D12.cpp @@ -508,17 +508,22 @@ namespace gpgmm::d3d12 { // General-purpose allocators. // Used for dynamic resource allocation or when the resource size is not known at // compile-time. - mResourceAllocatorOfType[resourceHeapTypeIndex] = CreateResourceHeapSubAllocator( - descriptor, heapFlags, heapProperties, heapAlignment); + mResourceAllocatorOfType[resourceHeapTypeIndex] = + CreateResourceAllocator(descriptor, heapFlags, heapProperties, heapAlignment); - mMSAAResourceAllocatorOfType[resourceHeapTypeIndex] = CreateResourceHeapSubAllocator( - descriptor, heapFlags, heapProperties, msaaHeapAlignment); + mMSAAResourceAllocatorOfType[resourceHeapTypeIndex] = + CreateResourceAllocator(descriptor, heapFlags, heapProperties, msaaHeapAlignment); - mResourceHeapAllocatorOfType[resourceHeapTypeIndex] = - CreateResourceHeapAllocator(descriptor, heapFlags, heapProperties, heapAlignment); + // Dedicated allocators are used when sub-allocation cannot but heaps could still be + // recycled. + ALLOCATOR_DESC dedicatedDescriptor = descriptor; + dedicatedDescriptor.SubAllocationAlgorithm = ALLOCATOR_ALGORITHM_DEDICATED; - mMSAAResourceHeapAllocatorOfType[resourceHeapTypeIndex] = CreateResourceHeapAllocator( - descriptor, heapFlags, heapProperties, msaaHeapAlignment); + mDedicatedResourceAllocatorOfType[resourceHeapTypeIndex] = CreateResourceAllocator( + dedicatedDescriptor, heapFlags, heapProperties, heapAlignment); + + mMSAADedicatedResourceAllocatorOfType[resourceHeapTypeIndex] = CreateResourceAllocator( + dedicatedDescriptor, heapFlags, heapProperties, msaaHeapAlignment); // Resource specific allocators. mSmallBufferAllocatorOfType[resourceHeapTypeIndex] = @@ -569,37 +574,63 @@ namespace gpgmm::d3d12 { } } - std::unique_ptr ResourceAllocator::CreateResourceHeapSubAllocator( - const ALLOCATOR_DESC& descriptor, - D3D12_HEAP_FLAGS heapFlags, - D3D12_HEAP_PROPERTIES heapProperties, - uint64_t heapAlignment) { - std::unique_ptr pooledOrNonPooledAllocator = - CreateResourceHeapAllocator(descriptor, heapFlags, heapProperties, heapAlignment); + std::unique_ptr ResourceAllocator::CreatePoolAllocator( + ALLOCATOR_ALGORITHM algorithm, + uint64_t memorySize, + uint64_t memoryAlignment, + bool isAlwaysOnDemand, + std::unique_ptr underlyingAllocator) { + if (isAlwaysOnDemand) { + return underlyingAllocator; + } + switch (algorithm) { + case ALLOCATOR_ALGORITHM_FIXED_POOL: { + return std::make_unique(memorySize, memoryAlignment, + std::move(underlyingAllocator)); + } + case ALLOCATOR_ALGORITHM_SEGMENTED_POOL: { + return std::make_unique(std::move(underlyingAllocator), + memoryAlignment); + } + default: { + UNREACHABLE(); + return {}; + } + } + } + + std::unique_ptr ResourceAllocator::CreateSubAllocator( + ALLOCATOR_ALGORITHM algorithm, + uint64_t memorySize, + uint64_t memoryAlignment, + double memoryFragmentationLimit, + double memoryGrowthFactor, + bool isPrefetchAllowed, + std::unique_ptr underlyingAllocator) { const uint64_t maxResourceHeapSize = mCaps->GetMaxResourceHeapSize(); - switch (descriptor.SubAllocationAlgorithm) { + switch (algorithm) { case ALLOCATOR_ALGORITHM_BUDDY_SYSTEM: { return std::make_unique( /*systemSize*/ PrevPowerOfTwo(maxResourceHeapSize), - /*memorySize*/ std::max(heapAlignment, descriptor.PreferredResourceHeapSize), - /*memoryAlignment*/ heapAlignment, - /*memoryAllocator*/ std::move(pooledOrNonPooledAllocator)); + /*memorySize*/ std::max(memoryAlignment, memorySize), + /*memoryAlignment*/ memoryAlignment, + /*memoryAllocator*/ std::move(underlyingAllocator)); } case ALLOCATOR_ALGORITHM_SLAB: { return std::make_unique( /*maxSlabSize*/ PrevPowerOfTwo(maxResourceHeapSize), - /*minSlabSize*/ std::max(heapAlignment, descriptor.PreferredResourceHeapSize), - /*slabAlignment*/ heapAlignment, - /*slabFragmentationLimit*/ descriptor.MemoryFragmentationLimit, - /*allowSlabPrefetch*/ - !(descriptor.Flags & ALLOCATOR_FLAG_DISABLE_MEMORY_PREFETCH), - /*slabGrowthFactor*/ descriptor.MemoryGrowthFactor, - /*memoryAllocator*/ std::move(pooledOrNonPooledAllocator)); + /*minSlabSize*/ std::max(memoryAlignment, memorySize), + /*slabAlignment*/ memoryAlignment, + /*slabFragmentationLimit*/ memoryFragmentationLimit, + /*allowSlabPrefetch*/ isPrefetchAllowed, + /*slabGrowthFactor*/ memoryGrowthFactor, + /*memoryAllocator*/ std::move(underlyingAllocator)); } - case ALLOCATOR_ALGORITHM_DEDICATED: + case ALLOCATOR_ALGORITHM_DEDICATED: { return std::make_unique( - /*memoryAllocator*/ std::move(pooledOrNonPooledAllocator)); + /*memoryAllocator*/ std::move(underlyingAllocator)); + } default: { UNREACHABLE(); return {}; @@ -607,7 +638,7 @@ namespace gpgmm::d3d12 { } } - std::unique_ptr ResourceAllocator::CreateResourceHeapAllocator( + std::unique_ptr ResourceAllocator::CreateResourceAllocator( const ALLOCATOR_DESC& descriptor, D3D12_HEAP_FLAGS heapFlags, D3D12_HEAP_PROPERTIES heapProperties, @@ -616,25 +647,17 @@ namespace gpgmm::d3d12 { std::make_unique(mResidencyManager.Get(), mDevice.Get(), heapProperties, heapFlags, mIsAlwaysInBudget); - if (!(descriptor.Flags & ALLOCATOR_FLAG_ALWAYS_ON_DEMAND)) { - switch (descriptor.PoolAlgorithm) { - case ALLOCATOR_ALGORITHM_FIXED_POOL: { - return std::make_unique( - descriptor.PreferredResourceHeapSize, heapAlignment, - std::move(resourceHeapAllocator)); - } - case ALLOCATOR_ALGORITHM_SEGMENTED_POOL: { - return std::make_unique( - std::move(resourceHeapAllocator), heapAlignment); - } - default: { - UNREACHABLE(); - return {}; - } - } - } - - return resourceHeapAllocator; + std::unique_ptr pooledOrNonPooledAllocator = CreatePoolAllocator( + descriptor.PoolAlgorithm, /*memorySize*/ descriptor.PreferredResourceHeapSize, + heapAlignment, (descriptor.Flags & ALLOCATOR_FLAG_ALWAYS_ON_DEMAND), + std::move(resourceHeapAllocator)); + + return CreateSubAllocator( + descriptor.SubAllocationAlgorithm, + /*memorySize*/ std::max(heapAlignment, descriptor.PreferredResourceHeapSize), + heapAlignment, descriptor.MemoryFragmentationLimit, descriptor.MemoryGrowthFactor, + /*allowSlabPrefetch*/ !(descriptor.Flags & ALLOCATOR_FLAG_DISABLE_MEMORY_PREFETCH), + std::move(pooledOrNonPooledAllocator)); } std::unique_ptr ResourceAllocator::CreateSmallBufferAllocator( @@ -647,46 +670,21 @@ namespace gpgmm::d3d12 { // https://docs.microsoft.com/en-us/windows/win32/api/d3d12/ns-d3d12-d3d12_resource_desc std::unique_ptr smallBufferOnlyAllocator = std::make_unique(this, heapProperties, heapFlags, - D3D12_RESOURCE_FLAG_NONE, initialResourceState, - /*bufferSize*/ heapAlignment, - /*bufferAlignment*/ heapAlignment); - - std::unique_ptr pooledOrNonPooledAllocator; - if (!(descriptor.Flags & ALLOCATOR_FLAG_ALWAYS_ON_DEMAND)) { - // Small buffers always use a 64KB heap. - pooledOrNonPooledAllocator = std::make_unique( - heapAlignment, heapAlignment, std::move(smallBufferOnlyAllocator)); - } else { - pooledOrNonPooledAllocator = std::move(smallBufferOnlyAllocator); - } + D3D12_RESOURCE_FLAG_NONE, initialResourceState); - switch (descriptor.SubAllocationAlgorithm) { - case ALLOCATOR_ALGORITHM_BUDDY_SYSTEM: { - return std::make_unique( - /*systemSize*/ heapAlignment, - /*memorySize*/ heapAlignment, - /*memoryAlignment*/ heapAlignment, - /*memoryAllocator*/ std::move(pooledOrNonPooledAllocator)); - } - case ALLOCATOR_ALGORITHM_SLAB: { - // Any amount of fragmentation must be allowed for small buffers since the resource - // heap size cannot change. - return std::make_unique( - /*maxSlabSize*/ heapAlignment, - /*slabSize*/ heapAlignment, - /*slabAlignment*/ heapAlignment, - /*slabFragmentationLimit*/ 1, - /*allowSlabPrefetch*/ false, - /*slabMemoryGrowth*/ 1, - /*memoryAllocator*/ std::move(pooledOrNonPooledAllocator)); - } - case ALLOCATOR_ALGORITHM_DEDICATED: - return std::make_unique( - /*memoryAllocator*/ std::move(pooledOrNonPooledAllocator)); - default: - UNREACHABLE(); - return {}; - } + std::unique_ptr pooledOrNonPooledAllocator = + CreatePoolAllocator(descriptor.PoolAlgorithm, heapAlignment, heapAlignment, + (descriptor.Flags & ALLOCATOR_FLAG_ALWAYS_ON_DEMAND), + std::move(smallBufferOnlyAllocator)); + + // Any amount of fragmentation must be allowed for small buffers since the allocation can + // be smaller then the resource heap alignment. + return CreateSubAllocator( + descriptor.SubAllocationAlgorithm, + /*memorySize*/ std::max(heapAlignment, descriptor.PreferredResourceHeapSize), + heapAlignment, + /*memoryFragmentationLimit*/ 1, descriptor.MemoryGrowthFactor, + /*allowSlabPrefetch*/ false, std::move(pooledOrNonPooledAllocator)); } ResourceAllocator::~ResourceAllocator() { @@ -701,11 +699,11 @@ namespace gpgmm::d3d12 { // before event tracer shutdown. mSmallBufferAllocatorOfType = {}; - mMSAAResourceHeapAllocatorOfType = {}; + mMSAADedicatedResourceAllocatorOfType = {}; mMSAAResourceAllocatorOfType = {}; mResourceAllocatorOfType = {}; - mResourceHeapAllocatorOfType = {}; + mDedicatedResourceAllocatorOfType = {}; #if defined(GPGMM_ENABLE_DEVICE_LEAK_CHECKS) ReportLiveDeviceObjects(mDevice); @@ -737,7 +735,8 @@ namespace gpgmm::d3d12 { } bytesReleased += - mResourceHeapAllocatorOfType[resourceHeapTypeIndex]->ReleaseMemory(bytesToRelease); + mDedicatedResourceAllocatorOfType[resourceHeapTypeIndex]->ReleaseMemory( + bytesToRelease); if (bytesReleased >= bytesToRelease) { break; } @@ -748,8 +747,9 @@ namespace gpgmm::d3d12 { break; } - bytesReleased += mMSAAResourceHeapAllocatorOfType[resourceHeapTypeIndex]->ReleaseMemory( - bytesToRelease); + bytesReleased += + mMSAADedicatedResourceAllocatorOfType[resourceHeapTypeIndex]->ReleaseMemory( + bytesToRelease); if (bytesReleased >= bytesToRelease) { break; } @@ -1050,10 +1050,11 @@ namespace gpgmm::d3d12 { if (!isAlwaysCommitted) { if (isMSAA) { allocator = - mMSAAResourceHeapAllocatorOfType[static_cast(resourceHeapType)].get(); + mMSAADedicatedResourceAllocatorOfType[static_cast(resourceHeapType)] + .get(); } else { allocator = - mResourceHeapAllocatorOfType[static_cast(resourceHeapType)].get(); + mDedicatedResourceAllocatorOfType[static_cast(resourceHeapType)].get(); } request.Alignment = allocator->GetMemoryAlignment(); @@ -1282,11 +1283,11 @@ namespace gpgmm::d3d12 { resourceHeapTypeIndex++) { result += mSmallBufferAllocatorOfType[resourceHeapTypeIndex]->GetInfo(); - result += mMSAAResourceHeapAllocatorOfType[resourceHeapTypeIndex]->GetInfo(); + result += mMSAADedicatedResourceAllocatorOfType[resourceHeapTypeIndex]->GetInfo(); result += mMSAAResourceAllocatorOfType[resourceHeapTypeIndex]->GetInfo(); result += mResourceAllocatorOfType[resourceHeapTypeIndex]->GetInfo(); - result += mResourceHeapAllocatorOfType[resourceHeapTypeIndex]->GetInfo(); + result += mDedicatedResourceAllocatorOfType[resourceHeapTypeIndex]->GetInfo(); } GPGMM_TRACE_EVENT_METRIC( diff --git a/src/gpgmm/d3d12/ResourceAllocatorD3D12.h b/src/gpgmm/d3d12/ResourceAllocatorD3D12.h index 74eb1ac39..cab43e6d9 100644 --- a/src/gpgmm/d3d12/ResourceAllocatorD3D12.h +++ b/src/gpgmm/d3d12/ResourceAllocatorD3D12.h @@ -612,13 +612,7 @@ namespace gpgmm::d3d12 { ComPtr residencyManager, std::unique_ptr caps); - std::unique_ptr CreateResourceHeapSubAllocator( - const ALLOCATOR_DESC& descriptor, - D3D12_HEAP_FLAGS heapFlags, - D3D12_HEAP_PROPERTIES heapProperties, - uint64_t heapAlignment); - - std::unique_ptr CreateResourceHeapAllocator( + std::unique_ptr CreateResourceAllocator( const ALLOCATOR_DESC& descriptor, D3D12_HEAP_FLAGS heapFlags, D3D12_HEAP_PROPERTIES heapProperties, @@ -631,6 +625,22 @@ namespace gpgmm::d3d12 { uint64_t heapAlignment, D3D12_RESOURCE_STATES initialResourceState); + std::unique_ptr CreatePoolAllocator( + ALLOCATOR_ALGORITHM algorithm, + uint64_t memorySize, + uint64_t memoryAlignment, + bool isAlwaysOnDemand, + std::unique_ptr underlyingAllocator); + + std::unique_ptr CreateSubAllocator( + ALLOCATOR_ALGORITHM algorithm, + uint64_t memorySize, + uint64_t memoryAlignment, + double memoryFragmentationLimit, + double memoryGrowthFactor, + bool isPrefetchAllowed, + std::unique_ptr underlyingAllocator); + HRESULT CreatePlacedResource(Heap* const resourceHeap, uint64_t resourceOffset, const D3D12_RESOURCE_DESC* resourceDescriptor, @@ -672,12 +682,12 @@ namespace gpgmm::d3d12 { static constexpr uint64_t kNumOfResourceHeapTypes = 8u; std::array, kNumOfResourceHeapTypes> - mResourceHeapAllocatorOfType; + mDedicatedResourceAllocatorOfType; std::array, kNumOfResourceHeapTypes> mResourceAllocatorOfType; std::array, kNumOfResourceHeapTypes> - mMSAAResourceHeapAllocatorOfType; + mMSAADedicatedResourceAllocatorOfType; std::array, kNumOfResourceHeapTypes> mMSAAResourceAllocatorOfType;