diff --git a/include/gpgmm_d3d12.h b/include/gpgmm_d3d12.h index 9d0c8e456..772795752 100644 --- a/include/gpgmm_d3d12.h +++ b/include/gpgmm_d3d12.h @@ -814,6 +814,18 @@ namespace gpgmm::d3d12 { operations at resource creation. */ RESOURCE_ALLOCATOR_FLAG_CREATE_NOT_RESIDENT = 0x40, + + /** \brief Never allow creation of resources when out of memory. + + By default, unused heaps will be freed if there is not enough available memory for + allocation. This prevents allocation from failing should the application forgo calling + ReleaseResourceHeaps(). It is recommended for application developers to periodically call + ReleaseResourceHeaps() when pooling is enabled or when the working set size changes + significantly. + + With this flag, there will be no attempt to free unused heaps when E_OUTOFMEMORY. + */ + RESOURCE_ALLOCATOR_FLAG_NEVER_OVER_ALLOCATE = 0x80, }; DEFINE_ENUM_FLAG_OPERATORS(RESOURCE_ALLOCATOR_FLAGS) @@ -1004,6 +1016,16 @@ namespace gpgmm::d3d12 { Optional parameter. When 0 is specified, the default of 1.25 is used (or 25% growth). */ FLOAT ResourceHeapGrowthFactor; + + /** \brief Size of memory, in bytes, to release from the resource allocator at once, + should there not be enough memory left. + + A release size of UINT64_MAX releases ALL free memory held by the resource allocator. + + Optional parameter. When 0 is specified, the API will use the size of the current + allocation. + */ + UINT64 ReleaseSizeInBytes; }; /** \enum RESOURCE_ALLOCATION_FLAGS diff --git a/src/gpgmm/common/Error.cpp b/src/gpgmm/common/Error.cpp index 8a7675375..62c893deb 100644 --- a/src/gpgmm/common/Error.cpp +++ b/src/gpgmm/common/Error.cpp @@ -36,6 +36,9 @@ namespace gpgmm { return "INVALID_ARGUMENT"; case ErrorCode::kBadOperation: return "BAD_OPERATION"; + case ErrorCode::kOutOfMemory: + case ErrorCode::kOutOfMemoryAndFatal: + return "OUT_OF_MEMORY"; default: UNREACHABLE(); return ""; @@ -45,10 +48,38 @@ namespace gpgmm { bool IsErrorCodeFatal(ErrorCode errorCode) { switch (errorCode) { case ErrorCode::kBadOperation: + case ErrorCode::kOutOfMemoryAndFatal: return true; default: return false; } } + std::string GetErrorCodeToString(ErrorCode errorCode) noexcept { + switch (errorCode) { + case ErrorCode::kNone: + return ""; + case ErrorCode::kUnknown: + return "Error could not be determined"; + case ErrorCode::kSizeExceeded: + return "Size was invalid"; + case ErrorCode::kAllocationFailed: + return "Failed to allocate memory"; + case ErrorCode::kPrefetchFailed: + return "Failed to pre-fetch memory"; + case ErrorCode::kBudgetInvalid: + return "Budget was invalid"; + case ErrorCode::kInvalidArgument: + return "Argument was invalid"; + case ErrorCode::kBadOperation: + return "Operation was illegal"; + case ErrorCode::kOutOfMemory: + case ErrorCode::kOutOfMemoryAndFatal: + return "Not enough memory to complete operation"; + default: + UNREACHABLE(); + return ""; + } + } + } // namespace gpgmm diff --git a/src/gpgmm/common/Error.h b/src/gpgmm/common/Error.h index 8f557c74c..f56abb47a 100644 --- a/src/gpgmm/common/Error.h +++ b/src/gpgmm/common/Error.h @@ -17,6 +17,7 @@ #include "gpgmm/utils/Assert.h" +#include #include // Generates a unique variable name to avoid variable shadowing with result variables. @@ -76,13 +77,16 @@ namespace gpgmm { kInvalidArgument, kBadOperation, kUnsupported, - kOutOfMemory, + kOutOfMemoryAndFatal, + kOutOfMemory }; const char* GetErrorCodeToChar(ErrorCode errorCode); bool IsErrorCodeFatal(ErrorCode errorCode); + std::string GetErrorCodeToString(ErrorCode error) noexcept; + // Wraps a backend error code with a result object. // Use Result::IsSuccess then Result::AcquireResult to use or else, use Result::GetErrorCode to // return the error for backend-specific handling. diff --git a/src/gpgmm/d3d12/ErrorD3D12.cpp b/src/gpgmm/d3d12/ErrorD3D12.cpp index 3dbcd3cd7..2fbf23a14 100644 --- a/src/gpgmm/d3d12/ErrorD3D12.cpp +++ b/src/gpgmm/d3d12/ErrorD3D12.cpp @@ -53,6 +53,7 @@ namespace gpgmm::d3d12 { case ErrorCode::kUnsupported: return E_NOTIMPL; case ErrorCode::kOutOfMemory: + case ErrorCode::kOutOfMemoryAndFatal: return E_OUTOFMEMORY; case ErrorCode::kUnknown: case ErrorCode::kAllocationFailed: diff --git a/src/gpgmm/d3d12/ResourceAllocatorD3D12.cpp b/src/gpgmm/d3d12/ResourceAllocatorD3D12.cpp index b0dc535ca..6a2f27292 100644 --- a/src/gpgmm/d3d12/ResourceAllocatorD3D12.cpp +++ b/src/gpgmm/d3d12/ResourceAllocatorD3D12.cpp @@ -606,7 +606,10 @@ namespace gpgmm::d3d12 { mIsCustomHeapsEnabled(descriptor.Flags & RESOURCE_ALLOCATOR_FLAG_ALLOW_UNIFIED_MEMORY), mIsCreateNotResidentEnabled(descriptor.Flags & RESOURCE_ALLOCATOR_FLAG_CREATE_NOT_RESIDENT), - mMaxResourceHeapSize(descriptor.MaxResourceHeapSize) { + mMaxResourceHeapSize(descriptor.MaxResourceHeapSize), + mIsNeverOverAllocateEnabled(descriptor.Flags & + RESOURCE_ALLOCATOR_FLAG_NEVER_OVER_ALLOCATE), + mReleaseSizeInBytes(descriptor.ReleaseSizeInBytes) { ASSERT(mDevice != nullptr); GPGMM_TRACE_EVENT_OBJECT_NEW(this); @@ -868,6 +871,11 @@ namespace gpgmm::d3d12 { HRESULT ResourceAllocator::ReleaseResourceHeaps(uint64_t bytesToRelease, uint64_t* pBytesReleased) { std::lock_guard lock(mMutex); + return ReleaseResourceHeapsInternal(bytesToRelease, pBytesReleased); + } + + HRESULT ResourceAllocator::ReleaseResourceHeapsInternal(uint64_t bytesToRelease, + uint64_t* pBytesReleased) { uint64_t bytesReleased = 0; for (uint32_t resourceHeapTypeIndex = 0; resourceHeapTypeIndex < kNumOfResourceHeapTypes; resourceHeapTypeIndex++) { @@ -911,6 +919,8 @@ namespace gpgmm::d3d12 { // Update allocation metrics. if (bytesReleased > 0) { GetStats(); + DebugLog(MessageId::kMemoryUsageUpdated, this) + << "Number of bytes freed: " << GetBytesToSizeInUnits(bytesReleased) << "."; } if (pBytesReleased != nullptr) { @@ -984,15 +994,59 @@ namespace gpgmm::d3d12 { // when the allocation never calls Detach() below and calls release which // re-enters |this| upon DeallocateMemory(). std::lock_guard lock(mMutex); - const MaybeError result = - CreateResourceInternal(allocationDescriptor, resourceDescriptor, + + const D3D12_RESOURCE_ALLOCATION_INFO resourceInfo = + GetResourceAllocationInfo(resourceDescriptor); + + MaybeError result = + CreateResourceInternal(allocationDescriptor, resourceInfo, resourceDescriptor, initialResourceState, pClearValue, &allocation); + + // CreateResource() may fail if there is not enough available memory. This could occur + // if the working set size changes significantly or if the resource allocation sizes + // change. We may be able to continue creation by releasing some unused memory and + // calling CreateResource() again. + uint64_t freedSizeInBytes = 0; + while (result.GetErrorCode() == ErrorCode::kOutOfMemory && + !mIsNeverOverAllocateEnabled) { + const uint64_t bytesToRelease = + std::max(mReleaseSizeInBytes, resourceInfo.SizeInBytes); + GPGMM_RETURN_IF_FAILED( + ReleaseResourceHeapsInternal(bytesToRelease, &freedSizeInBytes), mDevice); + // If not enough memory can be freed after ReleaseResourceHeaps(), we cannot + // continue creation and must return E_OUTOFMEMORY for real. + if (freedSizeInBytes < resourceInfo.SizeInBytes) { + MemoryAllocatorStats stats = GetStats(); + ErrorLog(result.GetErrorCode(), this) + << GetBytesToSizeInUnits(freedSizeInBytes + stats.FreeMemoryUsage) + + " free vs " + GetBytesToSizeInUnits(resourceInfo.SizeInBytes) + + " required."; + return E_OUTOFMEMORY; + } + + result = + CreateResourceInternal(allocationDescriptor, resourceInfo, resourceDescriptor, + initialResourceState, pClearValue, &allocation); + + // Do not repeatedly attempt re-creation since there is no guarantee creation + // can be successful after E_OUTOFMEMORY. + // TODO: consider retries or backoff strategy if one-time trimming was insufficent. + break; + } + if (!result.IsSuccess()) { ErrorLog(result.GetErrorCode(), this) - << "Failed to create resource for allocation."; + << "Failed to create resource for allocation: " + + GetErrorCodeToString(result.GetErrorCode()); return GetErrorResult(result.GetErrorCode()); } + if (freedSizeInBytes > 0) { + WarnLog(MessageId::kPerformanceWarning, this) + << "Resource could not be created without freeing " + + GetBytesToSizeInUnits(freedSizeInBytes) + " of unused heaps."; + } + ASSERT(allocation->GetResource() != nullptr); if (GPGMM_UNLIKELY(mTrackingAllocator)) { @@ -1059,6 +1113,7 @@ namespace gpgmm::d3d12 { MaybeError ResourceAllocator::CreateResourceInternal( const RESOURCE_ALLOCATION_DESC& allocationDescriptor, + const D3D12_RESOURCE_ALLOCATION_INFO& resourceInfo, const D3D12_RESOURCE_DESC& resourceDescriptor, D3D12_RESOURCE_STATES initialResourceState, const D3D12_CLEAR_VALUE* clearValue, @@ -1068,8 +1123,6 @@ namespace gpgmm::d3d12 { // If d3d tells us the resource size is invalid, treat the error as OOM. // Otherwise, creating a very large resource could overflow the allocator. - const D3D12_RESOURCE_ALLOCATION_INFO resourceInfo = - GetResourceAllocationInfo(resourceDescriptor); if (resourceInfo.SizeInBytes > mMaxResourceHeapSize) { ErrorLog(ErrorCode::kSizeExceeded, this) << "Unable to create resource allocation because the resource size exceeded " diff --git a/src/gpgmm/d3d12/ResourceAllocatorD3D12.h b/src/gpgmm/d3d12/ResourceAllocatorD3D12.h index 5b010b466..55f61febe 100644 --- a/src/gpgmm/d3d12/ResourceAllocatorD3D12.h +++ b/src/gpgmm/d3d12/ResourceAllocatorD3D12.h @@ -89,6 +89,7 @@ namespace gpgmm::d3d12 { CreateResourceFn&& createResourceFn); MaybeError CreateResourceInternal(const RESOURCE_ALLOCATION_DESC& allocationDescriptor, + const D3D12_RESOURCE_ALLOCATION_INFO& resourceInfo, const D3D12_RESOURCE_DESC& resourceDescriptor, D3D12_RESOURCE_STATES initialResourceState, const D3D12_CLEAR_VALUE* clearValue, @@ -139,6 +140,8 @@ namespace gpgmm::d3d12 { ID3D12Resource** committedResourceOut, ResidencyHeap** resourceHeapOut); + HRESULT ReleaseResourceHeapsInternal(uint64_t bytesToRelease, uint64_t* pBytesReleased); + HRESULT ReportLiveDeviceObjects() const; bool IsCreateHeapNotResidentEnabled() const; @@ -168,6 +171,8 @@ namespace gpgmm::d3d12 { const bool mIsCustomHeapsEnabled; const bool mIsCreateNotResidentEnabled; const uint64_t mMaxResourceHeapSize; + const bool mIsNeverOverAllocateEnabled = false; + const uint64_t mReleaseSizeInBytes; static constexpr uint64_t kNumOfResourceHeapTypes = 12u; diff --git a/src/tests/end2end/D3D12ResourceAllocatorTests.cpp b/src/tests/end2end/D3D12ResourceAllocatorTests.cpp index c0b067c09..06484c8f2 100644 --- a/src/tests/end2end/D3D12ResourceAllocatorTests.cpp +++ b/src/tests/end2end/D3D12ResourceAllocatorTests.cpp @@ -68,6 +68,9 @@ class D3D12ResourceAllocatorTests : public D3D12TestBase, public ::testing::Test // Make sure leak detection is always enabled. desc.Flags |= gpgmm::d3d12::RESOURCE_ALLOCATOR_FLAG_NEVER_LEAK; + // Make sure E_OUTOFMEMORY is always possible at first chance. + desc.Flags |= gpgmm::d3d12::RESOURCE_ALLOCATOR_FLAG_NEVER_OVER_ALLOCATE; + return desc; } };