intel · bbernhar · Jul 29, 2022 · Jul 29, 2022
diff --git a/src/gpgmm/d3d12/HeapD3D12.cpp b/src/gpgmm/d3d12/HeapD3D12.cpp
@@ -44,7 +44,8 @@ namespace gpgmm::d3d12 {
         // Ensure enough free memory exists before allocating to avoid an out-of-memory error
         // when over budget.
         if (pResidencyManager != nullptr && descriptor.AlwaysInBudget) {
-            ReturnIfFailed(pResidencyManager->Evict(descriptor.SizeInBytes, memorySegmentGroup));
+            ReturnIfFailed(
+                pResidencyManager->EnsureInBudget(descriptor.SizeInBytes, memorySegmentGroup));
         }
 
         ComPtr<ID3D12Pageable> pageable;

diff --git a/src/gpgmm/d3d12/ResidencyManagerD3D12.cpp b/src/gpgmm/d3d12/ResidencyManagerD3D12.cpp
@@ -383,8 +383,8 @@ namespace gpgmm::d3d12 {
 
         ReturnIfFailed(QueryVideoMemoryInfo(memorySegmentGroup, videoMemorySegmentInfo));
 
-        if (pCurrentReservationOut != nullptr){
-          *pCurrentReservationOut = videoMemorySegmentInfo->CurrentReservation;
+        if (pCurrentReservationOut != nullptr) {
+            *pCurrentReservationOut = videoMemorySegmentInfo->CurrentReservation;
         }
 
         return S_OK;
@@ -466,17 +466,19 @@ namespace gpgmm::d3d12 {
         return S_OK;
     }
 
-    HRESULT ResidencyManager::Evict(uint64_t evictSizeInBytes,
-                                    const DXGI_MEMORY_SEGMENT_GROUP& memorySegmentGroup) {
+    HRESULT ResidencyManager::EnsureInBudget(uint64_t bytesInBudget,
+                                             const DXGI_MEMORY_SEGMENT_GROUP& memorySegmentGroup) {
         std::lock_guard<std::mutex> lock(mMutex);
-        return EvictInternal(evictSizeInBytes, memorySegmentGroup);
+        uint64_t bytesEvicted = bytesInBudget;
+        ReturnIfFailed(EvictInternal(bytesInBudget, memorySegmentGroup, &bytesEvicted));
+        return (bytesEvicted >= bytesInBudget) ? S_OK : E_FAIL;
     }
 
     // Evicts |evictSizeInBytes| bytes of memory in |memorySegmentGroup| and returns the number of
     // bytes evicted.
-    HRESULT ResidencyManager::EvictInternal(uint64_t evictSizeInBytes,
+    HRESULT ResidencyManager::EvictInternal(uint64_t bytesToEvict,
                                             const DXGI_MEMORY_SEGMENT_GROUP& memorySegmentGroup,
-                                            uint64_t* evictedSizeInBytesOut) {
+                                            uint64_t* bytesEvictedOut) {
         TRACE_EVENT0(TraceEventCategory::Default, "ResidencyManager.Evict");
 
         DXGI_QUERY_VIDEO_MEMORY_INFO* videoMemorySegmentInfo =
@@ -486,22 +488,29 @@ namespace gpgmm::d3d12 {
             ReturnIfFailed(QueryVideoMemoryInfo(memorySegmentGroup, videoMemorySegmentInfo));
         }
 
-        const uint64_t currentUsageAfterEvict =
-            evictSizeInBytes + videoMemorySegmentInfo->CurrentUsage;
+        const uint64_t currentUsageAfterEvict = bytesToEvict + videoMemorySegmentInfo->CurrentUsage;
 
-        // Return if we will remain under budget after evict.
-        if (currentUsageAfterEvict < videoMemorySegmentInfo->Budget) {
+        // Return if we will remain under budget after evict or there is no budget to evict from.
+        // The latter happens if the first budget change event hasn't happened yet.
+        if (videoMemorySegmentInfo->Budget == 0 ||
+            currentUsageAfterEvict < videoMemorySegmentInfo->Budget) {
             return S_OK;
         }
 
         // Any time we need to make something resident, we must check that we have enough free
         // memory to make the new object resident while also staying within budget. If there isn't
         // enough memory, we should evict until there is.
         std::vector<ID3D12Pageable*> objectsToEvict;
-        const uint64_t sizeNeededToBeUnderBudget =
+        const uint64_t bytesNeededToBeUnderBudget =
             currentUsageAfterEvict - videoMemorySegmentInfo->Budget;
-        uint64_t evictedSizeInBytes = 0;
-        while (evictedSizeInBytes < sizeNeededToBeUnderBudget) {
+
+        // Return if nothing needs to be evicted to stay within budget.
+        if (bytesNeededToBeUnderBudget == 0) {
+            return S_OK;
+        }
+
+        uint64_t bytesEvicted = 0;
+        while (bytesEvicted < bytesNeededToBeUnderBudget) {
             // If the cache is empty, allow execution to continue. Note that fully
             // emptying the cache is undesirable, because it can mean either 1) the cache is not
             // accurately accounting for GPU allocations, or 2) an external component is
@@ -530,7 +539,7 @@ namespace gpgmm::d3d12 {
 
             heap->RemoveFromList();
 
-            evictedSizeInBytes += heap->GetSize();
+            bytesEvicted += heap->GetSize();
 
             ComPtr<ID3D12Pageable> pageable;
             ReturnIfFailed(heap->QueryInterface(IID_PPV_ARGS(&pageable)));
@@ -539,20 +548,20 @@ namespace gpgmm::d3d12 {
         }
 
         if (objectsToEvict.size() > 0) {
-            GPGMM_TRACE_EVENT_METRIC("GPU memory page-out (MB)",
-                                     GPGMM_BYTES_TO_MB(evictedSizeInBytes));
+            GPGMM_TRACE_EVENT_METRIC("GPU memory page-out (MB)", GPGMM_BYTES_TO_MB(bytesEvicted));
 
             const uint32_t objectEvictCount = static_cast<uint32_t>(objectsToEvict.size());
             ReturnIfFailed(mDevice->Evict(objectEvictCount, objectsToEvict.data()));
 
             DebugEvent("GPU page-out", EventMessageId::BudgetExceeded)
-                << "Number of allocations: " << objectsToEvict.size() << " (" << evictedSizeInBytes
+                << "Number of allocations: " << objectsToEvict.size() << " (" << bytesEvicted
                 << " bytes).";
         }
 
-        if (evictedSizeInBytesOut != nullptr) {
-            *evictedSizeInBytesOut = evictedSizeInBytes;
+        if (bytesEvictedOut != nullptr) {
+            *bytesEvictedOut = bytesEvicted;
         }
+
         return S_OK;
     }
 

diff --git a/src/gpgmm/d3d12/ResidencyManagerD3D12.h b/src/gpgmm/d3d12/ResidencyManagerD3D12.h
@@ -241,12 +241,12 @@ namespace gpgmm::d3d12 {
 
         ResidencyManager(const RESIDENCY_DESC& descriptor, std::unique_ptr<Fence> fence);
 
-        HRESULT Evict(uint64_t evictSizeInBytes,
-                      const DXGI_MEMORY_SEGMENT_GROUP& memorySegmentGroup);
+        HRESULT EnsureInBudget(uint64_t bytesToEvict,
+                               const DXGI_MEMORY_SEGMENT_GROUP& memorySegmentGroup);
 
-        HRESULT EvictInternal(uint64_t evictSizeInBytes,
+        HRESULT EvictInternal(uint64_t bytesToEvict,
                               const DXGI_MEMORY_SEGMENT_GROUP& memorySegmentGroup,
-                              uint64_t* evictedSizeInBytesOut = nullptr);
+                              uint64_t* bytesEvictedOut = nullptr);
 
         HRESULT InsertHeap(Heap* heap);
 

diff --git a/src/gpgmm/d3d12/ResourceAllocatorD3D12.cpp b/src/gpgmm/d3d12/ResourceAllocatorD3D12.cpp
@@ -338,6 +338,14 @@ namespace gpgmm::d3d12 {
                                                ? allocatorDescriptor.MemoryGrowthFactor
                                                : kDefaultMemoryGrowthFactor;
 
+        // ID3D12Device::CreateCommittedResource and ID3D12Device::CreateHeap implicity
+        // call ID3D12Device::MakeResident, requiring resource heaps to be "created in budget".
+        // But this can be disabled if D3D12_HEAP_FLAG_CREATE_NOT_RESIDENT is supported.
+        if (!(allocatorDescriptor.Flags & ALLOCATOR_FLAG_ALWAYS_IN_BUDGET) &&
+            !caps->IsCreateHeapNotResidentSupported()) {
+            newDescriptor.Flags |= ALLOCATOR_FLAG_ALWAYS_IN_BUDGET;
+        }
+
         newDescriptor.MaxResourceHeapSize =
             (allocatorDescriptor.MaxResourceHeapSize > 0)
                 ? std::min(allocatorDescriptor.MaxResourceHeapSize, caps->GetMaxResourceHeapSize())
@@ -531,7 +539,7 @@ namespace gpgmm::d3d12 {
         uint64_t heapAlignment) {
         std::unique_ptr<MemoryAllocator> resourceHeapAllocator =
             std::make_unique<ResourceHeapAllocator>(mResidencyManager.Get(), mDevice.Get(),
-                                                    heapType, heapFlags);
+                                                    heapType, heapFlags, mIsAlwaysInBudget);
 
         if (!(descriptor.Flags & ALLOCATOR_FLAG_ALWAYS_ON_DEMAND)) {
             switch (descriptor.PoolAlgorithm) {
@@ -1078,10 +1086,9 @@ namespace gpgmm::d3d12 {
 
         HEAP_DESC resourceHeapDesc = {};
         resourceHeapDesc.SizeInBytes = info.SizeInBytes;
-        resourceHeapDesc.IsExternal = false;
         resourceHeapDesc.DebugName = "Resource heap (committed)";
         resourceHeapDesc.Alignment = info.Alignment;
-        resourceHeapDesc.AlwaysInBudget = !(heapFlags & D3D12_HEAP_FLAG_CREATE_NOT_RESIDENT);
+        resourceHeapDesc.AlwaysInBudget = mIsAlwaysInBudget;
         resourceHeapDesc.HeapType = heapType;
 
         // Since residency is per heap, every committed resource is wrapped in a heap object.
@@ -1216,13 +1223,6 @@ namespace gpgmm::d3d12 {
     }
 
     bool ResourceAllocator::IsCreateHeapNotResident() const {
-        // By default, ID3D12Device::CreateCommittedResource and ID3D12Device::CreateHeap implicity
-        // call MakeResident(). This can be disabled when residency exists and resources are not
-        // required to be "created in budget".
-        if (!mCaps->IsCreateHeapNotResidentSupported()) {
-            return false;
-        }
-
         return mResidencyManager != nullptr && !mIsAlwaysInBudget;
     }
 

diff --git a/src/gpgmm/d3d12/ResourceHeapAllocatorD3D12.cpp b/src/gpgmm/d3d12/ResourceHeapAllocatorD3D12.cpp
@@ -29,11 +29,13 @@ namespace gpgmm::d3d12 {
     ResourceHeapAllocator::ResourceHeapAllocator(ResidencyManager* residencyManager,
                                                  ID3D12Device* device,
                                                  D3D12_HEAP_TYPE heapType,
-                                                 D3D12_HEAP_FLAGS heapFlags)
+                                                 D3D12_HEAP_FLAGS heapFlags,
+                                                 bool alwaysInBudget)
         : mResidencyManager(residencyManager),
           mDevice(device),
           mHeapType(heapType),
-          mHeapFlags(heapFlags) {
+          mHeapFlags(heapFlags),
+          mAlwaysInBudget(alwaysInBudget) {
     }
 
     std::unique_ptr<MemoryAllocation> ResourceHeapAllocator::TryAllocateMemory(
@@ -59,10 +61,9 @@ namespace gpgmm::d3d12 {
 
         HEAP_DESC resourceHeapDesc = {};
         resourceHeapDesc.SizeInBytes = heapSize;
-        resourceHeapDesc.IsExternal = false;
         resourceHeapDesc.DebugName = "Resource heap";
         resourceHeapDesc.Alignment = request.Alignment;
-        resourceHeapDesc.AlwaysInBudget = !(mHeapFlags & D3D12_HEAP_FLAG_CREATE_NOT_RESIDENT);
+        resourceHeapDesc.AlwaysInBudget = mAlwaysInBudget;
         resourceHeapDesc.HeapType = mHeapType;
 
         Heap* resourceHeap = nullptr;

diff --git a/src/gpgmm/d3d12/ResourceHeapAllocatorD3D12.h b/src/gpgmm/d3d12/ResourceHeapAllocatorD3D12.h
@@ -29,7 +29,8 @@ namespace gpgmm::d3d12 {
         ResourceHeapAllocator(ResidencyManager* residencyManager,
                               ID3D12Device* device,
                               D3D12_HEAP_TYPE heapType,
-                              D3D12_HEAP_FLAGS heapFlags);
+                              D3D12_HEAP_FLAGS heapFlags,
+                              bool alwaysInBudget);
         ~ResourceHeapAllocator() override = default;
 
         // MemoryAllocator interface
@@ -44,6 +45,7 @@ namespace gpgmm::d3d12 {
         ID3D12Device* const mDevice;
         const D3D12_HEAP_TYPE mHeapType;
         const D3D12_HEAP_FLAGS mHeapFlags;
+        const bool mAlwaysInBudget;
     };
 
 }  // namespace gpgmm::d3d12