Reland "Add option to disable custom heaps. (#602) (#603)" again.

src/gpgmm/d3d12/ResidencyManagerD3D12.cpp

@@ -762,9 +762,8 @@ namespace gpgmm::d3d12 {
         mBudgetNotificationUpdateEvent = nullptr;
     }
 
-    DXGI_MEMORY_SEGMENT_GROUP ResidencyManager::GetMemorySegmentGroup(
-        D3D12_MEMORY_POOL memoryPool) const {
-        return d3d12::GetMemorySegmentGroup(memoryPool, mIsUMA);
+    bool ResidencyManager::IsUMA() const {
+        return mIsUMA;
     }
 
 }  // namespace gpgmm::d3d12

src/gpgmm/d3d12/ResidencyManagerD3D12.h

@@ -256,7 +256,7 @@ namespace gpgmm::d3d12 {
         friend BudgetUpdateTask;
         HRESULT UpdateMemorySegments();
 
-        DXGI_MEMORY_SEGMENT_GROUP GetMemorySegmentGroup(D3D12_MEMORY_POOL memoryPool) const;
+        bool IsUMA() const;
 
         const char* GetTypename() const;
 

src/gpgmm/d3d12/ResourceAllocatorD3D12.cpp

@@ -292,6 +292,40 @@ namespace gpgmm::d3d12 {
             return hr;
         }
 
+        D3D12_HEAP_PROPERTIES GetHeapProperties(ID3D12Device* device,
+                                                D3D12_HEAP_TYPE heapType,
+                                                bool isCustomHeapDisabled) {
+            // Produces the corresponding properties from the corresponding heap type per this table
+            // https://docs.microsoft.com/en-us/windows/win32/api/d3d12/nf-d3d12-id3d12device-getcustomheapproperties
+            if (!isCustomHeapDisabled) {
+                return device->GetCustomHeapProperties(0, heapType);
+            }
+
+            D3D12_HEAP_PROPERTIES heapProperties = {};
+            heapProperties.Type = heapType;
+            heapProperties.CPUPageProperty = D3D12_CPU_PAGE_PROPERTY_UNKNOWN;
+            heapProperties.MemoryPoolPreference = D3D12_MEMORY_POOL_UNKNOWN;
+
+            return heapProperties;
+        }
+
+        D3D12_MEMORY_POOL GetMemoryPool(const D3D12_HEAP_PROPERTIES& heapProperties, bool isUMA) {
+            // Custom heap types are required to specify a non-unknown pool.
+            // https://docs.microsoft.com/en-us/windows/win32/api/d3d12/ns-d3d12-d3d12_heap_properties
+            if (heapProperties.MemoryPoolPreference != D3D12_MEMORY_POOL_UNKNOWN) {
+                return heapProperties.MemoryPoolPreference;
+            }
+
+            // Otherwise, a unknown pool corresponds to the custom heap type properties and only L1
+            // exists when non-UMA adapter.
+            // https://docs.microsoft.com/en-us/windows/win32/api/d3d12/nf-d3d12-id3d12device-getcustomheapproperties
+            if (!isUMA && heapProperties.Type == D3D12_HEAP_TYPE_DEFAULT) {
+                return D3D12_MEMORY_POOL_L1;
+            } else {
+                return D3D12_MEMORY_POOL_L0;  // Physical system
+            }
+        }
+
     }  // namespace
 
     // static
@@ -444,46 +478,52 @@ namespace gpgmm::d3d12 {
           mIsAlwaysInBudget(descriptor.Flags & ALLOCATOR_FLAG_ALWAYS_IN_BUDGET),
           mFlushEventBuffersOnDestruct(descriptor.RecordOptions.EventScope &
                                        EVENT_RECORD_SCOPE_PER_INSTANCE),
-          mUseDetailedTimingEvents(descriptor.RecordOptions.UseDetailedTimingEvents) {
+          mUseDetailedTimingEvents(descriptor.RecordOptions.UseDetailedTimingEvents),
+          mIsCustomHeapsDisabled(descriptor.Flags & ALLOCATOR_FLAG_DISABLE_CUSTOM_HEAPS) {
         GPGMM_TRACE_EVENT_OBJECT_NEW(this);
 
 #if defined(GPGMM_ENABLE_ALLOCATOR_LEAK_CHECKS)
         mDebugAllocator = std::make_unique<DebugResourceAllocator>();
 #endif
 
+        const bool isUMA =
+            (IsResidencyEnabled()) ? mResidencyManager->IsUMA() : mCaps->IsAdapterUMA();
+
         for (uint32_t resourceHeapTypeIndex = 0; resourceHeapTypeIndex < kNumOfResourceHeapTypes;
              resourceHeapTypeIndex++) {
             const RESOURCE_HEAP_TYPE& resourceHeapType =
                 static_cast<RESOURCE_HEAP_TYPE>(resourceHeapTypeIndex);
 
             const D3D12_HEAP_FLAGS& heapFlags =
                 GetHeapFlags(resourceHeapType, IsCreateHeapNotResident());
+            const D3D12_HEAP_TYPE heapType = GetHeapType(resourceHeapType);
 
             const uint64_t msaaHeapAlignment = GetHeapAlignment(heapFlags, true);
             const uint64_t heapAlignment = GetHeapAlignment(heapFlags, false);
 
-            const D3D12_HEAP_PROPERTIES customHeapProperties =
-                mDevice->GetCustomHeapProperties(0, GetHeapType(resourceHeapType));
+            D3D12_HEAP_PROPERTIES heapProperties =
+                GetHeapProperties(mDevice.Get(), heapType, mIsCustomHeapsDisabled);
+            heapProperties.MemoryPoolPreference = GetMemoryPool(heapProperties, isUMA);
 
             // General-purpose allocators.
             // Used for dynamic resource allocation or when the resource size is not known at
             // compile-time.
             mResourceAllocatorOfType[resourceHeapTypeIndex] = CreateResourceHeapSubAllocator(
-                descriptor, heapFlags, customHeapProperties, heapAlignment);
+                descriptor, heapFlags, heapProperties, heapAlignment);
 
             mMSAAResourceAllocatorOfType[resourceHeapTypeIndex] = CreateResourceHeapSubAllocator(
-                descriptor, heapFlags, customHeapProperties, msaaHeapAlignment);
+                descriptor, heapFlags, heapProperties, msaaHeapAlignment);
 
-            mResourceHeapAllocatorOfType[resourceHeapTypeIndex] = CreateResourceHeapAllocator(
-                descriptor, heapFlags, customHeapProperties, heapAlignment);
+            mResourceHeapAllocatorOfType[resourceHeapTypeIndex] =
+                CreateResourceHeapAllocator(descriptor, heapFlags, heapProperties, heapAlignment);
 
             mMSAAResourceHeapAllocatorOfType[resourceHeapTypeIndex] = CreateResourceHeapAllocator(
-                descriptor, heapFlags, customHeapProperties, msaaHeapAlignment);
+                descriptor, heapFlags, heapProperties, msaaHeapAlignment);
 
             // Resource specific allocators.
-            mSmallBufferAllocatorOfType[resourceHeapTypeIndex] = CreateSmallBufferAllocator(
-                descriptor, heapFlags, customHeapProperties, heapAlignment,
-                GetInitialResourceState(GetHeapType(resourceHeapType)));
+            mSmallBufferAllocatorOfType[resourceHeapTypeIndex] =
+                CreateSmallBufferAllocator(descriptor, heapFlags, heapProperties, heapAlignment,
+                                           GetInitialResourceState(heapType));
 
             // Cache resource sizes commonly requested.
             // Allows the next memory block to be made available upon request without
@@ -864,12 +904,17 @@ namespace gpgmm::d3d12 {
             }
         }
 
-        D3D12_HEAP_PROPERTIES customHeapProperties = mDevice->GetCustomHeapProperties(0, heapType);
+        D3D12_HEAP_PROPERTIES heapProperties =
+            GetHeapProperties(mDevice.Get(), heapType, mIsCustomHeapsDisabled);
 
         // Limit available memory to unused budget when residency is enabled.
-        if (mResidencyManager != nullptr) {
-            const DXGI_MEMORY_SEGMENT_GROUP segment =
-                mResidencyManager->GetMemorySegmentGroup(customHeapProperties.MemoryPoolPreference);
+        if (IsResidencyEnabled()) {
+            heapProperties.MemoryPoolPreference =
+                GetMemoryPool(heapProperties, mResidencyManager->IsUMA());
+
+            const DXGI_MEMORY_SEGMENT_GROUP segment = GetMemorySegmentGroup(
+                heapProperties.MemoryPoolPreference, mResidencyManager->IsUMA());
+
             DXGI_QUERY_VIDEO_MEMORY_INFO* currentVideoInfo =
                 mResidencyManager->GetVideoMemoryInfo(segment);
 
@@ -1040,7 +1085,7 @@ namespace gpgmm::d3d12 {
 
         ComPtr<ID3D12Resource> committedResource;
         Heap* resourceHeap = nullptr;
-        ReturnIfFailed(CreateCommittedResource(customHeapProperties, heapFlags, resourceInfo,
+        ReturnIfFailed(CreateCommittedResource(heapProperties, heapFlags, resourceInfo,
                                                &newResourceDesc, clearValue, initialResourceState,
                                                &committedResource, &resourceHeap));
 
@@ -1161,9 +1206,9 @@ namespace gpgmm::d3d12 {
         resourceHeapDesc.Alignment = info.Alignment;
         resourceHeapDesc.AlwaysInBudget = mIsAlwaysInBudget;
 
-        if (mResidencyManager != nullptr) {
-            resourceHeapDesc.MemorySegmentGroup =
-                mResidencyManager->GetMemorySegmentGroup(heapProperties.MemoryPoolPreference);
+        if (IsResidencyEnabled()) {
+            resourceHeapDesc.MemorySegmentGroup = GetMemorySegmentGroup(
+                heapProperties.MemoryPoolPreference, mResidencyManager->IsUMA());
         }
 
         // Since residency is per heap, every committed resource is wrapped in a heap object.
@@ -1178,6 +1223,11 @@ namespace gpgmm::d3d12 {
                 heapFlags &= ~(D3D12_HEAP_FLAG_DENY_NON_RT_DS_TEXTURES |
                                D3D12_HEAP_FLAG_DENY_RT_DS_TEXTURES | D3D12_HEAP_FLAG_DENY_BUFFERS);
 
+                // Non-custom heaps are not allowed to have the pool-specified.
+                if (heapProperties.Type != D3D12_HEAP_TYPE_CUSTOM) {
+                    heapProperties.MemoryPoolPreference = D3D12_MEMORY_POOL_UNKNOWN;
+                }
+
                 ReturnIfFailed(mDevice->CreateCommittedResource(
                     &heapProperties, heapFlags, resourceDescriptor, initialResourceState,
                     clearValue, IID_PPV_ARGS(&committedResource)));
@@ -1295,7 +1345,11 @@ namespace gpgmm::d3d12 {
     }
 
     bool ResourceAllocator::IsCreateHeapNotResident() const {
-        return mResidencyManager != nullptr && !mIsAlwaysInBudget;
+        return IsResidencyEnabled() && !mIsAlwaysInBudget;
+    }
+
+    bool ResourceAllocator::IsResidencyEnabled() const {
+        return mResidencyManager != nullptr;
     }
 
     HRESULT ResourceAllocator::CheckFeatureSupport(FEATURE feature,

src/gpgmm/d3d12/ResourceAllocatorD3D12.h

@@ -76,6 +76,13 @@ namespace gpgmm::d3d12 {
         minimal possible GPU memory footprint or debugging OOM failures.
         */
         ALLOCATOR_FLAG_ALWAYS_ON_DEMAND = 0x8,
+
+        /** \brief Disables use of D3D12_HEAP_TYPE_CUSTOM.
+
+        Used to workaround issues when a custom-equivalent heap is not considered equal to
+        the corresponding heap type.
+        */
+        ALLOCATOR_FLAG_DISABLE_CUSTOM_HEAPS = 0x10,
     };
 
     DEFINE_ENUM_FLAG_OPERATORS(ALLOCATOR_FLAGS)
@@ -585,6 +592,7 @@ namespace gpgmm::d3d12 {
         static HRESULT ReportLiveDeviceObjects(ComPtr<ID3D12Device> device);
 
         bool IsCreateHeapNotResident() const;
+        bool IsResidencyEnabled() const;
 
         // MemoryAllocator interface
         void DeallocateMemory(std::unique_ptr<MemoryAllocation> allocation) override;
@@ -601,6 +609,7 @@ namespace gpgmm::d3d12 {
         const bool mIsAlwaysInBudget;
         const bool mFlushEventBuffersOnDestruct;
         const bool mUseDetailedTimingEvents;
+        const bool mIsCustomHeapsDisabled;
 
         static constexpr uint64_t kNumOfResourceHeapTypes = 8u;
 

src/gpgmm/d3d12/ResourceHeapAllocatorD3D12.cpp

@@ -66,8 +66,8 @@ namespace gpgmm::d3d12 {
         resourceHeapDesc.AlwaysInBudget = mAlwaysInBudget;
 
         if (mResidencyManager != nullptr) {
-            resourceHeapDesc.MemorySegmentGroup =
-                mResidencyManager->GetMemorySegmentGroup(mHeapProperties.MemoryPoolPreference);
+            resourceHeapDesc.MemorySegmentGroup = GetMemorySegmentGroup(
+                mHeapProperties.MemoryPoolPreference, mResidencyManager->IsUMA());
         }
 
         Heap* resourceHeap = nullptr;
@@ -80,6 +80,11 @@ namespace gpgmm::d3d12 {
                     heapDesc.Alignment = resourceHeapDesc.Alignment;
                     heapDesc.Flags = mHeapFlags;
 
+                    // Non-custom heaps are not allowed to have the pool-specified.
+                    if (heapDesc.Properties.Type != D3D12_HEAP_TYPE_CUSTOM) {
+                        heapDesc.Properties.MemoryPoolPreference = D3D12_MEMORY_POOL_UNKNOWN;
+                    }
+
                     ComPtr<ID3D12Heap> heap;
                     ReturnIfFailed(mDevice->CreateHeap(&heapDesc, IID_PPV_ARGS(&heap)));
 

src/include/min/gpgmm_d3d12.h

@@ -224,6 +224,7 @@ namespace gpgmm::d3d12 {
         ALLOCATOR_FLAG_ALWAYS_IN_BUDGET = 0x2,
         ALLOCATOR_FLAG_DISABLE_MEMORY_PREFETCH = 0x4,
         ALLOCATOR_FLAG_ALWAYS_ON_DEMAND = 0x8,
+        ALLOCATOR_FLAG_DISABLE_CUSTOM_HEAPS = 0x10,
     };
 
     DEFINE_ENUM_FLAG_OPERATORS(ALLOCATOR_FLAGS)

src/tests/end2end/D3D12ResourceAllocatorTests.cpp

@@ -456,6 +456,23 @@ TEST_F(D3D12ResourceAllocatorTests, CreateBufferUMA) {
     EXPECT_EQ(resourceAllocator->GetInfo().FreeMemoryUsage, kDefaultBufferSize * 2);
 }
 
+TEST_F(D3D12ResourceAllocatorTests, CreateBufferDisableCustomHeaps) {
+    ALLOCATOR_DESC allocatorDesc = CreateBasicAllocatorDesc();
+    allocatorDesc.Flags |= ALLOCATOR_FLAG_DISABLE_CUSTOM_HEAPS;
+
+    ComPtr<ResourceAllocator> resourceAllocator;
+    ASSERT_SUCCEEDED(ResourceAllocator::CreateAllocator(allocatorDesc, &resourceAllocator));
+
+    ComPtr<ResourceAllocation> allocation;
+    ASSERT_SUCCEEDED(
+        resourceAllocator->CreateResource({}, CreateBasicBufferDesc(kDefaultBufferSize),
+                                          D3D12_RESOURCE_STATE_COPY_DEST, nullptr, &allocation));
+
+    D3D12_HEAP_PROPERTIES heapProperties = {};
+    ASSERT_SUCCEEDED(allocation->GetResource()->GetHeapProperties(&heapProperties, nullptr));
+    EXPECT_NE(heapProperties.Type, D3D12_HEAP_TYPE_CUSTOM);
+}
+
 TEST_F(D3D12ResourceAllocatorTests, CreateSmallTexture) {
     ComPtr<ResourceAllocator> resourceAllocator;
     ASSERT_SUCCEEDED(