intel · bbernhar · May 11, 2022 · May 10, 2022
diff --git a/patches/gpgmm_dawn.diff b/patches/gpgmm_dawn.diff
@@ -1,4 +1,4 @@
-From beb21d1d2baa8e0147d4d8ef6a0c46e44e89b811 Mon Sep 17 00:00:00 2001
+From e9627e75c5a3baf15f02fff11838115ada0e1947 Mon Sep 17 00:00:00 2001
 From: Bryan Bernhart <bryan.bernhart@intel.com>
 Date: Tue, 15 Feb 2022 17:25:29 -0800
 Subject: [PATCH] Use GPGMM for D3D12 backend.
@@ -46,15 +46,15 @@ index 91b835033..6f17c2ec3 100644
  /third_party/gpuweb-cts
  /third_party/jinja2
 diff --git a/DEPS b/DEPS
-index e5a8b0da6..e666940a1 100644
+index e5a8b0da6..740aee04f 100644
 --- a/DEPS
 +++ b/DEPS
 @@ -146,6 +146,10 @@ deps = {
      'condition': 'dawn_standalone',
    },
 
 +  'third_party/gpgmm': {
-+    'url': '{github_git}/intel/gpgmm.git@7690524f05784c2eac2938c506b48179b7bc387c',
++    'url': '{github_git}/intel/gpgmm.git@8d8ecf73a0aa3235163ff55f5def13f7b15b36e7',
 +  },
 +
    'third_party/abseil-cpp': {
@@ -437,7 +437,7 @@ index 18d7145c8..098254e5c 100644
 
      AdapterDiscoveryOptions::AdapterDiscoveryOptions()
 diff --git a/src/dawn/native/d3d12/DeviceD3D12.cpp b/src/dawn/native/d3d12/DeviceD3D12.cpp
-index 6b77b3a07..b06def815 100644
+index 6b77b3a07..8e7e55e84 100644
 --- a/src/dawn/native/d3d12/DeviceD3D12.cpp
 +++ b/src/dawn/native/d3d12/DeviceD3D12.cpp
 @@ -127,8 +127,33 @@ namespace dawn::native::d3d12 {
@@ -457,12 +457,12 @@ index 6b77b3a07..b06def815 100644
 +        allocatorDesc.PreferredResourceHeapSize = 4ll * 1024ll * 1024ll;  // 4MB
 +
 +        if (IsToggleEnabled(Toggle::UseD3D12ResidencyManagement)) {
-+            allocatorDesc.Flags |= gpgmm::d3d12::ALLOCATOR_FLAG_ALWAYS_IN_BUDGET;
 +            allocatorDesc.MaxVideoMemoryBudget = 0.95;  // Use up to 95%.
 +        }
 +
 +        if (IsToggleEnabled(Toggle::UseD3D12SmallResidencyBudgetForTesting)) {
 +            allocatorDesc.Budget = 100000000;  // 100MB
++            allocatorDesc.Flags |= gpgmm::d3d12::ALLOCATOR_FLAG_DISABLE_MEMORY_PREFETCH;
 +        }
 +
 +        if (IsToggleEnabled(Toggle::DumpResourceAllocator)) {

diff --git a/src/gpgmm/d3d12/BufferAllocatorD3D12.cpp b/src/gpgmm/d3d12/BufferAllocatorD3D12.cpp
@@ -24,12 +24,14 @@ namespace gpgmm { namespace d3d12 {
 
     BufferAllocator::BufferAllocator(ResourceAllocator* resourceAllocator,
                                      D3D12_HEAP_TYPE heapType,
+                                     D3D12_HEAP_FLAGS heapFlags,
                                      D3D12_RESOURCE_FLAGS resourceFlags,
                                      D3D12_RESOURCE_STATES initialResourceState,
                                      uint64_t bufferSize,
                                      uint64_t bufferAlignment)
         : mResourceAllocator(resourceAllocator),
           mHeapType(heapType),
+          mHeapFlags(heapFlags),
           mResourceFlags(resourceFlags),
           mInitialResourceState(initialResourceState),
           mBufferSize(bufferSize),
@@ -63,7 +65,7 @@ namespace gpgmm { namespace d3d12 {
         // Optimized clear is not supported for buffers.
         Heap* resourceHeap = nullptr;
         if (FAILED(mResourceAllocator->CreateCommittedResource(
-                mHeapType, D3D12_HEAP_FLAG_NONE, request.SizeInBytes, &resourceDescriptor,
+                mHeapType, mHeapFlags, request.SizeInBytes, &resourceDescriptor,
                 /*pOptimizedClearValue*/ nullptr, mInitialResourceState, /*resourceOut*/ nullptr,
                 &resourceHeap))) {
             return {};

diff --git a/src/gpgmm/d3d12/BufferAllocatorD3D12.h b/src/gpgmm/d3d12/BufferAllocatorD3D12.h
@@ -27,6 +27,7 @@ namespace gpgmm { namespace d3d12 {
       public:
         BufferAllocator(ResourceAllocator* resourceAllocator,
                         D3D12_HEAP_TYPE heapType,
+                        D3D12_HEAP_FLAGS heapFlags,
                         D3D12_RESOURCE_FLAGS resourceFlags,
                         D3D12_RESOURCE_STATES initialResourceState,
                         uint64_t bufferSize,
@@ -45,6 +46,7 @@ namespace gpgmm { namespace d3d12 {
         ResourceAllocator* const mResourceAllocator;
 
         const D3D12_HEAP_TYPE mHeapType;
+        const D3D12_HEAP_FLAGS mHeapFlags;
         const D3D12_RESOURCE_FLAGS mResourceFlags;
         const D3D12_RESOURCE_STATES mInitialResourceState;
         const uint64_t mBufferSize;

diff --git a/src/gpgmm/d3d12/CapsD3D12.cpp b/src/gpgmm/d3d12/CapsD3D12.cpp
@@ -41,16 +41,33 @@ namespace gpgmm { namespace d3d12 {
         return S_OK;
     }
 
+    HRESULT SetCreateHeapNotResidentSupported(ID3D12Device* device,
+                                              bool* createHeapNotResidencySupported) {
+        *createHeapNotResidencySupported = false;
+
+        // Only Windows 10 Build 20348 and later support creating non-resident heaps.
+#ifdef D3D12_FEATURE_D3D12_OPTIONS7
+        D3D12_FEATURE_DATA_D3D12_OPTIONS7 options7 = {};
+        if (SUCCEEDED(device->CheckFeatureSupport(D3D12_FEATURE_D3D12_OPTIONS7, &options7,
+                                                  sizeof(options7)))) {
+            *createHeapNotResidencySupported = true;
+        }
+#endif
+        return S_OK;
+    }
+
     // static
     HRESULT Caps::CreateCaps(ID3D12Device* device, IDXGIAdapter* adapter, Caps** capsOut) {
         DXGI_ADAPTER_DESC adapterDesc;
         ReturnIfFailed(adapter->GetDesc(&adapterDesc));
 
-        Caps* caps = new Caps();
+        std::unique_ptr<Caps> caps(new Caps());
         ReturnIfFailed(SetMaxResourceSize(device, &caps->mMaxResourceSize));
         ReturnIfFailed(SetMaxResourceHeapSize(device, &caps->mMaxResourceHeapSize));
+        ReturnIfFailed(
+            SetCreateHeapNotResidentSupported(device, &caps->mIsCreateHeapNotResidentSupported));
 
-        *capsOut = caps;
+        *capsOut = caps.release();
         return S_OK;
     }
 
@@ -62,4 +79,8 @@ namespace gpgmm { namespace d3d12 {
         return mMaxResourceHeapSize;
     }
 
+    bool Caps::IsCreateHeapNotResidentSupported() const {
+        return mIsCreateHeapNotResidentSupported;
+    }
+
 }}  // namespace gpgmm::d3d12
diff --git a/src/gpgmm/d3d12/CapsD3D12.h b/src/gpgmm/d3d12/CapsD3D12.h
@@ -32,11 +32,15 @@ namespace gpgmm { namespace d3d12 {
         // Largest resource heap that this device can make available.
         uint64_t GetMaxResourceHeapSize() const;
 
+        // Allows a resource heap to be created without being resident.
+        bool IsCreateHeapNotResidentSupported() const;
+
       private:
         Caps() = default;
 
         uint64_t mMaxResourceSize = 0;
         uint64_t mMaxResourceHeapSize = 0;
+        bool mIsCreateHeapNotResidentSupported = false;
     };
 
 }}  // namespace gpgmm::d3d12

diff --git a/src/gpgmm/d3d12/ResourceAllocatorD3D12.cpp b/src/gpgmm/d3d12/ResourceAllocatorD3D12.cpp
@@ -158,20 +158,22 @@ namespace gpgmm { namespace d3d12 {
             }
         }
 
-        D3D12_HEAP_FLAGS GetHeapFlags(RESOURCE_HEAP_TYPE resourceHeapType) {
+        D3D12_HEAP_FLAGS GetHeapFlags(RESOURCE_HEAP_TYPE resourceHeapType, bool createNotResident) {
+            const D3D12_HEAP_FLAGS createHeapFlags =
+                (createNotResident) ? D3D12_HEAP_FLAG_CREATE_NOT_RESIDENT : D3D12_HEAP_FLAG_NONE;
             switch (resourceHeapType) {
                 case RESOURCE_HEAP_TYPE_DEFAULT_ALLOW_ALL_BUFFERS_AND_TEXTURES:
                 case RESOURCE_HEAP_TYPE_READBACK_ALLOW_ALL_BUFFERS_AND_TEXTURES:
                 case RESOURCE_HEAP_TYPE_UPLOAD_ALLOW_ALL_BUFFERS_AND_TEXTURES:
-                    return D3D12_HEAP_FLAG_ALLOW_ALL_BUFFERS_AND_TEXTURES;
+                    return createHeapFlags | D3D12_HEAP_FLAG_ALLOW_ALL_BUFFERS_AND_TEXTURES;
                 case RESOURCE_HEAP_TYPE_DEFAULT_ALLOW_ONLY_BUFFERS:
                 case RESOURCE_HEAP_TYPE_READBACK_ALLOW_ONLY_BUFFERS:
                 case RESOURCE_HEAP_TYPE_UPLOAD_ALLOW_ONLY_BUFFERS:
-                    return D3D12_HEAP_FLAG_ALLOW_ONLY_BUFFERS;
+                    return createHeapFlags | D3D12_HEAP_FLAG_ALLOW_ONLY_BUFFERS;
                 case RESOURCE_HEAP_TYPE_DEFAULT_ALLOW_ONLY_NON_RT_OR_DS_TEXTURES:
-                    return D3D12_HEAP_FLAG_ALLOW_ONLY_NON_RT_DS_TEXTURES;
+                    return createHeapFlags | D3D12_HEAP_FLAG_ALLOW_ONLY_NON_RT_DS_TEXTURES;
                 case RESOURCE_HEAP_TYPE_DEFAULT_ALLOW_ONLY_RT_OR_DS_TEXTURES:
-                    return D3D12_HEAP_FLAG_ALLOW_ONLY_RT_DS_TEXTURES;
+                    return createHeapFlags | D3D12_HEAP_FLAG_ALLOW_ONLY_RT_DS_TEXTURES;
                 default:
                     UNREACHABLE();
                     return D3D12_HEAP_FLAG_NONE;
@@ -445,7 +447,8 @@ namespace gpgmm { namespace d3d12 {
             const RESOURCE_HEAP_TYPE& resourceHeapType =
                 static_cast<RESOURCE_HEAP_TYPE>(resourceHeapTypeIndex);
 
-            const D3D12_HEAP_FLAGS& heapFlags = GetHeapFlags(resourceHeapType);
+            const D3D12_HEAP_FLAGS& heapFlags =
+                GetHeapFlags(resourceHeapType, IsCreateHeapNotResident());
             const D3D12_HEAP_TYPE& heapType = GetHeapType(resourceHeapType);
 
             // General-purpose allocators.
@@ -470,9 +473,10 @@ namespace gpgmm { namespace d3d12 {
             // https://docs.microsoft.com/en-us/windows/win32/api/d3d12/ns-d3d12-d3d12_resource_desc
             std::unique_ptr<MemoryAllocator> bufferOnlyAllocator =
                 std::make_unique<BufferAllocator>(
-                    this, heapType, D3D12_RESOURCE_FLAG_NONE, GetInitialResourceState(heapType),
-                    /*resourceSize*/ D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT,
-                    /*resourceAlignment*/ D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT);
+                    this, heapType, heapFlags, D3D12_RESOURCE_FLAG_NONE,
+                    GetInitialResourceState(heapType),
+                    /*bufferSize*/ D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT,
+                    /*bufferAlignment*/ D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT);
 
             std::unique_ptr<MemoryAllocator> pooledOrNonPooledAllocator;
             if (!(descriptor.Flags & ALLOCATOR_FLAG_ALWAYS_ON_DEMAND)) {
@@ -549,7 +553,7 @@ namespace gpgmm { namespace d3d12 {
 
         std::unique_ptr<MemoryAllocator> resourceHeapAllocator =
             std::make_unique<ResourceHeapAllocator>(mResidencyManager.Get(), mDevice.Get(),
-                                                    heapType, heapFlags, mIsUMA, mIsAlwaysInBudget);
+                                                    heapType, heapFlags, mIsUMA);
 
         std::unique_ptr<MemoryAllocator> pooledOrNonPooledAllocator;
         if (!(descriptor.Flags & ALLOCATOR_FLAG_ALWAYS_ON_DEMAND)) {
@@ -578,7 +582,7 @@ namespace gpgmm { namespace d3d12 {
         bool allowMSAA) {
         std::unique_ptr<MemoryAllocator> resourceHeapAllocator =
             std::make_unique<ResourceHeapAllocator>(mResidencyManager.Get(), mDevice.Get(),
-                                                    heapType, heapFlags, mIsUMA, mIsAlwaysInBudget);
+                                                    heapType, heapFlags, mIsUMA);
 
         std::unique_ptr<MemoryAllocator> pooledOrNonPooledAllocator;
         if (!(descriptor.Flags & ALLOCATOR_FLAG_ALWAYS_ON_DEMAND)) {
@@ -812,7 +816,8 @@ namespace gpgmm { namespace d3d12 {
                 }));
         }
 
-        const D3D12_HEAP_FLAGS& heapFlags = GetHeapFlags(resourceHeapType);
+        const D3D12_HEAP_FLAGS& heapFlags =
+            GetHeapFlags(resourceHeapType, IsCreateHeapNotResident());
 
         // Attempt to create a resource allocation by placing a single resource fully contained
         // in a resource heap. This strategy is slightly better then creating a committed
@@ -986,7 +991,7 @@ namespace gpgmm { namespace d3d12 {
         const DXGI_MEMORY_SEGMENT_GROUP memorySegmentGroup =
             GetPreferredMemorySegmentGroup(mDevice.Get(), mIsUMA, heapType);
 
-        if (mIsAlwaysInBudget && mResidencyManager != nullptr) {
+        if (!(heapFlags & D3D12_HEAP_FLAG_CREATE_NOT_RESIDENT) && mResidencyManager != nullptr) {
             ReturnIfFailed(mResidencyManager->Evict(resourceSize, memorySegmentGroup));
         }
 
@@ -1108,4 +1113,15 @@ namespace gpgmm { namespace d3d12 {
         SafeRelease(allocation);
     }
 
+    bool ResourceAllocator::IsCreateHeapNotResident() const {
+        // By default, ID3D12Device::CreateCommittedResource and ID3D12Device::CreateHeap implicity
+        // call MakeResident(). This can be disabled when residency exists and resources are not
+        // required to be "created in budget".
+        if (!mCaps->IsCreateHeapNotResidentSupported()) {
+            return false;
+        }
+
+        return mResidencyManager != nullptr && !mIsAlwaysInBudget;
+    }
+
 }}  // namespace gpgmm::d3d12
diff --git a/src/gpgmm/d3d12/ResourceAllocatorD3D12.h b/src/gpgmm/d3d12/ResourceAllocatorD3D12.h
@@ -45,15 +45,20 @@ namespace gpgmm { namespace d3d12 {
          */
         ALLOCATOR_FLAG_NONE = 0x0,
 
-        /** \brief Disable reuse of resource memory.
+        /** \brief Disable re-use of resource heap.
 
-        Should only be used for debugging and testing purposes.
+        Mostly used for debugging and testing purposes.
         */
         ALLOCATOR_FLAG_ALWAYS_COMMITED = 0x1,
 
-        /** \brief Ensures resources are always within the resource budget at creation time.
+        /** \brief Creates resource within budget.
 
-        Mostly used to debug with residency being over committed.
+        By default (and when residency is used), resources will not be created resident unless an
+        operation is performed on the allocation that requires them to be (ex. Map). Otherwise, the
+        resource will become resident once ExecuteCommandList() is called. However, this flag can be
+        used to change this behavior by requiring resource heaps to be always resident at resource
+        creation. When residency is not used, ALLOCATOR_FLAG_ALWAYS_IN_BUDGET is implicitly enabled
+        through the GPU/driver instead of explicitly through GPGMM.
         */
         ALLOCATOR_FLAG_ALWAYS_IN_BUDGET = 0x2,
 
@@ -500,6 +505,8 @@ namespace gpgmm { namespace d3d12 {
 
         static HRESULT ReportLiveDeviceObjects(ComPtr<ID3D12Device> device);
 
+        bool IsCreateHeapNotResident() const;
+
         // MemoryAllocator interface
         void DeallocateMemory(std::unique_ptr<MemoryAllocation> allocation) override;
 

diff --git a/src/gpgmm/d3d12/ResourceHeapAllocatorD3D12.cpp b/src/gpgmm/d3d12/ResourceHeapAllocatorD3D12.cpp
@@ -30,14 +30,12 @@ namespace gpgmm { namespace d3d12 {
                                                  ID3D12Device* device,
                                                  D3D12_HEAP_TYPE heapType,
                                                  D3D12_HEAP_FLAGS heapFlags,
-                                                 bool isUMA,
-                                                 bool isAlwaysInBudget)
+                                                 bool isUMA)
         : mResidencyManager(residencyManager),
           mDevice(device),
           mHeapType(heapType),
           mHeapFlags(heapFlags),
-          mIsUMA(isUMA),
-          mIsAlwaysInBudget(isAlwaysInBudget) {
+          mIsUMA(isUMA) {
     }
 
     std::unique_ptr<MemoryAllocation> ResourceHeapAllocator::TryAllocateMemory(
@@ -65,9 +63,10 @@ namespace gpgmm { namespace d3d12 {
         const DXGI_MEMORY_SEGMENT_GROUP memorySegmentGroup =
             GetPreferredMemorySegmentGroup(mDevice, mIsUMA, mHeapType);
 
-        // CreateHeap will implicitly make the created heap resident. We must ensure enough free
-        // memory exists before allocating to avoid an out-of-memory error when overcommitted.
-        if (mIsAlwaysInBudget && mResidencyManager != nullptr) {
+        // CreateHeap will implicitly make the created heap resident unless
+        // D3D12_HEAP_FLAG_CREATE_NOT_RESIDENT is set. Otherwise, CreateHeap could return
+        // out-of-memory when overcommitted if Evict() is not called first.
+        if (!(mHeapFlags & D3D12_HEAP_FLAG_CREATE_NOT_RESIDENT) && mResidencyManager != nullptr) {
             mResidencyManager->Evict(heapSize, memorySegmentGroup);
         }
 

diff --git a/src/gpgmm/d3d12/ResourceHeapAllocatorD3D12.h b/src/gpgmm/d3d12/ResourceHeapAllocatorD3D12.h
@@ -30,8 +30,7 @@ namespace gpgmm { namespace d3d12 {
                               ID3D12Device* device,
                               D3D12_HEAP_TYPE heapType,
                               D3D12_HEAP_FLAGS heapFlags,
-                              bool isUMA,
-                              bool isAlwaysInBudget);
+                              bool isUMA);
         ~ResourceHeapAllocator() override = default;
 
         // MemoryAllocator interface
@@ -47,7 +46,6 @@ namespace gpgmm { namespace d3d12 {
         const D3D12_HEAP_TYPE mHeapType;
         const D3D12_HEAP_FLAGS mHeapFlags;
         const bool mIsUMA;
-        const bool mIsAlwaysInBudget;
     };
 
 }}  // namespace gpgmm::d3d12

diff --git a/src/gpgmm/d3d12/d3d12_platform.h b/src/gpgmm/d3d12/d3d12_platform.h
@@ -20,6 +20,14 @@
 #include <dxgi1_4.h>
 #include <wrl.h>
 
+// Keep backwards compatibility when using D3D12 feature flags that are only defined in a newer
+// D3D12.h versions.
+// Only once ALL builds upgrade to the newer D3D12.h version, can these defines be safely
+// removed.
+#ifndef D3D12_FEATURE_D3D12_OPTIONS7
+#    define D3D12_HEAP_FLAG_CREATE_NOT_RESIDENT static_cast<D3D12_HEAP_FLAGS>(0x800)
+#endif
+
 using Microsoft::WRL::ComPtr;
 
 #endif  // GPGMM_D3D12_D3D12PLATFORM_H_