From 6b4b281734c8a4470ad75dda5f68fa4c8e635abe Mon Sep 17 00:00:00 2001 From: Bryan Bernhart Date: Mon, 18 Jul 2022 15:37:01 -0700 Subject: [PATCH] Enable pool-allocation for VK backend. --- README.md | 2 +- src/gpgmm/common/MemoryAllocator.cpp | 7 +- src/gpgmm/utils/Limits.h | 1 + src/gpgmm/vk/DeviceMemoryAllocatorVk.cpp | 12 +- src/gpgmm/vk/DeviceMemoryAllocatorVk.h | 7 +- src/gpgmm/vk/FunctionsVk.cpp | 24 ++-- src/gpgmm/vk/FunctionsVk.h | 8 +- src/gpgmm/vk/ResourceAllocatorVk.cpp | 49 ++++++-- src/gpgmm/vk/ResourceAllocatorVk.h | 114 ++++++++++++++++-- src/tests/VKTest.cpp | 4 +- src/tests/VKTest.h | 4 +- .../end2end/VKResourceAllocatorTests.cpp | 3 +- 12 files changed, 174 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index f4714db22..ab576e98f 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ To clean-up, simply call `Release()` once the is GPU is finished. ```cpp #include -gpgmm::vk::GpCreateAllocatorInfo allocatorInfo = {}; +gpgmm::vk::GpAllocatorCreateInfo allocatorInfo = {}; gpgmm::vk::GpResourceAllocator resourceAllocator; gpgmm::vk::gpCreateResourceAllocator(allocatorInfo, &resourceAllocator) diff --git a/src/gpgmm/common/MemoryAllocator.cpp b/src/gpgmm/common/MemoryAllocator.cpp index 973a69c3c..990e037a8 100644 --- a/src/gpgmm/common/MemoryAllocator.cpp +++ b/src/gpgmm/common/MemoryAllocator.cpp @@ -110,7 +110,7 @@ namespace gpgmm { } uint64_t MemoryAllocator::GetMemoryAlignment() const { - return kInvalidOffset; + return kNoRequiredAlignment; } MemoryAllocatorInfo MemoryAllocator::GetInfo() const { @@ -143,8 +143,9 @@ namespace gpgmm { } // Check request size has compatible alignment with |this| memory allocator. - if (GetMemoryAlignment() != kInvalidSize && - !IsAligned(GetMemoryAlignment(), request.Alignment)) { + // Alignment value of 1 means no alignment required. + if (GetMemoryAlignment() == 0 || + (GetMemoryAlignment() > 1 && !IsAligned(GetMemoryAlignment(), request.Alignment))) { DebugEvent(GetTypename(), EventMessageId::AlignmentMismatch) << "Requested alignment exceeds memory alignment (" + std::to_string(request.Alignment) + " vs " + diff --git a/src/gpgmm/utils/Limits.h b/src/gpgmm/utils/Limits.h index 23011d394..7791bd060 100644 --- a/src/gpgmm/utils/Limits.h +++ b/src/gpgmm/utils/Limits.h @@ -21,6 +21,7 @@ namespace gpgmm { + static constexpr uint64_t kNoRequiredAlignment = 1u; static constexpr uint64_t kInvalidOffset = std::numeric_limits::max(); static constexpr uint64_t kInvalidSize = std::numeric_limits::max(); static constexpr uint64_t kInvalidIndex = std::numeric_limits::max(); diff --git a/src/gpgmm/vk/DeviceMemoryAllocatorVk.cpp b/src/gpgmm/vk/DeviceMemoryAllocatorVk.cpp index 6f8eb4773..c64a48ccd 100644 --- a/src/gpgmm/vk/DeviceMemoryAllocatorVk.cpp +++ b/src/gpgmm/vk/DeviceMemoryAllocatorVk.cpp @@ -24,11 +24,8 @@ namespace gpgmm::vk { DeviceMemoryAllocator::DeviceMemoryAllocator(GpResourceAllocator resourceAllocator, - uint32_t memoryTypeIndex, - VkDeviceSize memorySize) - : mResourceAllocator(resourceAllocator), - mMemoryTypeIndex(memoryTypeIndex), - mMemorySize(memorySize) { + uint32_t memoryTypeIndex) + : mResourceAllocator(resourceAllocator), mMemoryTypeIndex(memoryTypeIndex) { } std::unique_ptr DeviceMemoryAllocator::TryAllocateMemory( @@ -86,9 +83,4 @@ namespace gpgmm::vk { SafeRelease(allocation); } - - uint64_t DeviceMemoryAllocator::GetMemorySize() const { - return mMemorySize; - } - } // namespace gpgmm::vk diff --git a/src/gpgmm/vk/DeviceMemoryAllocatorVk.h b/src/gpgmm/vk/DeviceMemoryAllocatorVk.h index fbd401df1..97e74118a 100644 --- a/src/gpgmm/vk/DeviceMemoryAllocatorVk.h +++ b/src/gpgmm/vk/DeviceMemoryAllocatorVk.h @@ -24,9 +24,7 @@ namespace gpgmm::vk { class DeviceMemoryAllocator final : public MemoryAllocator { public: - DeviceMemoryAllocator(GpResourceAllocator resourceAllocator, - uint32_t memoryTypeIndex, - VkDeviceSize memorySize); + DeviceMemoryAllocator(GpResourceAllocator resourceAllocator, uint32_t memoryTypeIndex); ~DeviceMemoryAllocator() override = default; // MemoryAllocator interface @@ -34,12 +32,9 @@ namespace gpgmm::vk { const MemoryAllocationRequest& request) override; void DeallocateMemory(std::unique_ptr allocation) override; - uint64_t GetMemorySize() const override; - private: GpResourceAllocator mResourceAllocator; uint32_t mMemoryTypeIndex; - VkDeviceSize mMemorySize; }; } // namespace gpgmm::vk diff --git a/src/gpgmm/vk/FunctionsVk.cpp b/src/gpgmm/vk/FunctionsVk.cpp index 16a7857e8..fa08bb341 100644 --- a/src/gpgmm/vk/FunctionsVk.cpp +++ b/src/gpgmm/vk/FunctionsVk.cpp @@ -83,18 +83,18 @@ namespace gpgmm::vk { DestroyImage = vkFunctions->DestroyImage; } - void VulkanFunctions::AssertVulkanFunctionsAreValid() { - ASSERT(GetPhysicalDeviceMemoryProperties != nullptr); - ASSERT(GetPhysicalDeviceProperties != nullptr); - ASSERT(AllocateMemory != nullptr); - ASSERT(FreeMemory != nullptr); - ASSERT(BindBufferMemory != nullptr); - ASSERT(GetBufferMemoryRequirements != nullptr); - ASSERT(GetImageMemoryRequirements != nullptr); - ASSERT(CreateBuffer != nullptr); - ASSERT(DestroyBuffer != nullptr); - ASSERT(CreateImage != nullptr); - ASSERT(DestroyImage != nullptr); + void AssertVulkanFunctionsExist(const VulkanFunctions& vkFunctions) { + ASSERT(vkFunctions.GetPhysicalDeviceMemoryProperties != nullptr); + ASSERT(vkFunctions.GetPhysicalDeviceProperties != nullptr); + ASSERT(vkFunctions.AllocateMemory != nullptr); + ASSERT(vkFunctions.FreeMemory != nullptr); + ASSERT(vkFunctions.BindBufferMemory != nullptr); + ASSERT(vkFunctions.GetBufferMemoryRequirements != nullptr); + ASSERT(vkFunctions.GetImageMemoryRequirements != nullptr); + ASSERT(vkFunctions.CreateBuffer != nullptr); + ASSERT(vkFunctions.DestroyBuffer != nullptr); + ASSERT(vkFunctions.CreateImage != nullptr); + ASSERT(vkFunctions.DestroyImage != nullptr); } } // namespace gpgmm::vk diff --git a/src/gpgmm/vk/FunctionsVk.h b/src/gpgmm/vk/FunctionsVk.h index 4e0865faf..c95dc2608 100644 --- a/src/gpgmm/vk/FunctionsVk.h +++ b/src/gpgmm/vk/FunctionsVk.h @@ -25,12 +25,9 @@ namespace gpgmm::vk { // Used to statically set functions from a static library (Vulkan loader). void ImportDeviceFunctions(); - // Used to import functions pre-specified by user. + // Used to import pre-loaded functions set by the user. void ImportDeviceFunctions(const VulkanFunctions* vkFunctions); - // ASSERTs if any Vulkan function is left unset. - void AssertVulkanFunctionsAreValid(); - // Order is important: instance must be loaded before device. PFN_vkGetInstanceProcAddr GetInstanceProcAddr = nullptr; @@ -50,4 +47,7 @@ namespace gpgmm::vk { PFN_vkDestroyImage DestroyImage = nullptr; }; + // ASSERTs if any Vulkan function is left unset. + void AssertVulkanFunctionsExist(const VulkanFunctions& vkFunctions); + } // namespace gpgmm::vk diff --git a/src/gpgmm/vk/ResourceAllocatorVk.cpp b/src/gpgmm/vk/ResourceAllocatorVk.cpp index fc9352470..78a540729 100644 --- a/src/gpgmm/vk/ResourceAllocatorVk.cpp +++ b/src/gpgmm/vk/ResourceAllocatorVk.cpp @@ -15,6 +15,8 @@ #include "gpgmm/vk/ResourceAllocatorVk.h" #include "gpgmm/common/EventMessage.h" +#include "gpgmm/common/PooledMemoryAllocator.h" +#include "gpgmm/common/SegmentedMemoryAllocator.h" #include "gpgmm/vk/BackendVk.h" #include "gpgmm/vk/CapsVk.h" #include "gpgmm/vk/DeviceMemoryAllocatorVk.h" @@ -23,7 +25,7 @@ namespace gpgmm::vk { - VkResult gpCreateResourceAllocator(const GpCreateAllocatorInfo& info, + VkResult gpCreateResourceAllocator(const GpAllocatorCreateInfo& info, GpResourceAllocator* allocatorOut) { return GpResourceAllocator_T::CreateAllocator(info, allocatorOut); } @@ -109,7 +111,7 @@ namespace gpgmm::vk { // GpResourceAllocator_T // static - VkResult GpResourceAllocator_T::CreateAllocator(const GpCreateAllocatorInfo& info, + VkResult GpResourceAllocator_T::CreateAllocator(const GpAllocatorCreateInfo& info, GpResourceAllocator* allocatorOut) { VulkanFunctions vulkanFunctions = {}; { @@ -125,7 +127,7 @@ namespace gpgmm::vk { } #ifndef NDEBUG - vulkanFunctions.AssertVulkanFunctionsAreValid(); + AssertVulkanFunctionsExist(vulkanFunctions); #endif } @@ -144,7 +146,7 @@ namespace gpgmm::vk { return VK_SUCCESS; } - GpResourceAllocator_T::GpResourceAllocator_T(const GpCreateAllocatorInfo& info, + GpResourceAllocator_T::GpResourceAllocator_T(const GpAllocatorCreateInfo& info, const VulkanFunctions& vulkanFunctions, std::unique_ptr caps) : mDevice(info.device), mVulkanFunctions(vulkanFunctions), mCaps(std::move(caps)) { @@ -160,9 +162,8 @@ namespace gpgmm::vk { for (uint32_t memoryTypeIndex = 0; memoryTypeIndex < mMemoryTypes.size(); memoryTypeIndex++) { - mDeviceAllocatorsPerType.emplace_back(std::make_unique( - this, memoryTypeIndex, - memoryHeaps[mMemoryTypes[memoryTypeIndex].heapIndex].size)); + mDeviceAllocatorsPerType.emplace_back( + CreateDeviceMemoryAllocator(info, memoryTypeIndex, kNoRequiredAlignment)); } } } @@ -225,9 +226,10 @@ namespace gpgmm::vk { MemoryAllocationRequest request = {}; request.SizeInBytes = requirements.size; request.Alignment = requirements.alignment; - request.NeverAllocate = (allocationInfo.flags & GP_ALLOCATION_FLAG_NEVER_ALLOCATE_MEMORY); + request.NeverAllocate = (allocationInfo.flags & GP_ALLOCATION_CREATE_NEVER_ALLOCATE_MEMORY); request.AlwaysCacheSize = false; - request.AlwaysPrefetch = (allocationInfo.flags & GP_ALLOCATION_FLAG_ALWAYS_PREFETCH_MEMORY); + request.AlwaysPrefetch = + (allocationInfo.flags & GP_ALLOCATION_CREATE_ALWAYS_PREFETCH_MEMORY); std::unique_ptr memoryAllocation = allocator->TryAllocateMemory(request); if (memoryAllocation == nullptr) { @@ -261,4 +263,33 @@ namespace gpgmm::vk { Caps* GpResourceAllocator_T::GetCaps() const { return mCaps.get(); } + + std::unique_ptr GpResourceAllocator_T::CreateDeviceMemoryAllocator( + const GpAllocatorCreateInfo& info, + uint64_t memoryTypeIndex, + uint64_t memoryAlignment) { + std::unique_ptr deviceMemoryAllocator = + std::make_unique(this, memoryTypeIndex); + + if (!(info.flags & GP_ALLOCATOR_CREATE_ALWAYS_ON_DEMAND)) { + switch (info.poolAlgorithm) { + case GP_ALLOCATOR_ALGORITHM_FIXED_POOL: { + return std::make_unique( + info.preferredDeviceMemorySize, memoryAlignment, + std::move(deviceMemoryAllocator)); + } + case GP_ALLOCATOR_ALGORITHM_SEGMENTED_POOL: { + return std::make_unique( + std::move(deviceMemoryAllocator), memoryAlignment); + } + default: { + UNREACHABLE(); + return {}; + } + } + } + + return deviceMemoryAllocator; + } + } // namespace gpgmm::vk diff --git a/src/gpgmm/vk/ResourceAllocatorVk.h b/src/gpgmm/vk/ResourceAllocatorVk.h index 9c5b11bb1..e95ed0473 100644 --- a/src/gpgmm/vk/ResourceAllocatorVk.h +++ b/src/gpgmm/vk/ResourceAllocatorVk.h @@ -38,10 +38,71 @@ namespace gpgmm::vk { */ VK_DEFINE_HANDLE(GpResourceAllocation) - /** \struct GpCreateAllocatorInfo - \brief Specifies how allocators should be created. + /** \enum GpAllocatorCreateFlags + \brief Configures how allocators should be created. */ - struct GpCreateAllocatorInfo { + enum GpAllocatorCreateFlags { + /** \brief Disables all allocator flags. + */ + GP_ALLOCATOR_CREATE_NONE = 0x0, + + /** \brief Tell GPGMM to allocate exactly what is needed, and to de-allocate + memory immediately once no longer needed (instead of re-using it). + + This is very slow and not recommended for general use but may be useful for running with the + minimal possible GPU memory footprint or debugging OOM failures. + */ + GP_ALLOCATOR_CREATE_ALWAYS_ON_DEMAND = 0x8, + }; + + /** \enum GpAllocatorAlgorithm + Specify the algorithms used for allocation. + */ + enum GpAllocatorAlgorithm { + /** \brief Use the slab allocation mechanism. + + Slab allocation allocates/deallocates in O(1) time using O(N * pageSize) space. + + Slab allocation does not suffer from internal fragmentation but could externally fragment + when many unique request sizes are used. + */ + GP_ALLOCATOR_ALGORITHM_SLAB = 0x0, + + /** \brief Use the buddy system mechanism. + + Buddy system allocate/deallocates in O(Log2) time using O(1) space. + + Buddy system suffers from internal fragmentation (ie. resources are not a power-of-two) but + does not suffer from external fragmentation as much since the device memory size does not + change. + + It is recommend to specify a preferredDeviceMemorySize large enough such that multiple + requests can fit within the specified preferredDeviceMemorySize but not too large where + creating the larger device memory becomes a bigger bottleneck. + */ + GP_ALLOCATOR_ALGORITHM_BUDDY_SYSTEM = 0x1, + + /** \brief Recycles device memory of a size being specified. + + Fixed pools allocate/deallocate in O(1) time using O(N) space. + + Fixed-size pool limits recycling to device memorys equal to + preferredDeviceMemorySize. A preferredDeviceMemorySize of zero is effectively + equivelent to ALLOCATOR_FLAG_ALWAYS_ON_DEMAND. + */ + GP_ALLOCATOR_ALGORITHM_FIXED_POOL = 0x2, + + /** \brief Recycles device memory of any size using multiple pools. + + Segmented pool allocate/deallocates in O(Log2) time using O(N * K) space. + */ + GP_ALLOCATOR_ALGORITHM_SEGMENTED_POOL = 0x3, + }; + + /** \struct GpAllocatorCreateInfo + \brief Used to create allocator. + */ + struct GpAllocatorCreateInfo { /** \brief Function pointer to Vulkan functions. There are 3 ways to specify Vulkan functions. @@ -69,15 +130,41 @@ namespace gpgmm::vk { /** \brief Vulkan version return by VK_MAKE_VERSION. */ uint32_t vulkanApiVersion; + + /** \brief Flags used to configure allocator. + */ + GpAllocatorCreateFlags flags; + + /** \brief Specifies the algorithm to use for device memory pooling. + + Used to evaluate how allocation implementations perform with various algorithms that + sub-divide device memorys. + + Optional parameter. By default, the slab allocator is used. + */ + GpAllocatorAlgorithm poolAlgorithm = GP_ALLOCATOR_ALGORITHM_SEGMENTED_POOL; + + /** \brief Specifies the preferred size of device memory. + + The preferred size of the device memory is the minimum memory size to sub-allocate from. + A larger device memory consumes more memory but could be faster for sub-allocation. + + Optional parameter. When 0 is specified, the API will automatically set the preferred + device memory size to be a multiple of minimum device memory size allowed by Vulkan. + */ + uint64_t preferredDeviceMemorySize; }; + /** \enum GpResourceAllocationCreateFlags + Additional controls that modify allocations. + */ enum GpResourceAllocationCreateFlags { /** \brief Disables all allocation flags. Enabled by default. */ - GP_ALLOCATION_FLAG_NONE = 0x0, + GP_ALLOCATION_CREATE_NONE = 0x0, /** \brief Disallow creating new device memory when creating a resource. @@ -85,13 +172,13 @@ namespace gpgmm::vk { must use existing device memory or error. Effectively disables creating standalone allocations whose memory cannot be reused. */ - GP_ALLOCATION_FLAG_NEVER_ALLOCATE_MEMORY = 0x1, + GP_ALLOCATION_CREATE_NEVER_ALLOCATE_MEMORY = 0x1, /** \brief Disallow creating multiple resource allocations from the same device memory. The created resource will always be allocated with it's own device memory. */ - GP_ALLOCATION_FLAG_NEVER_SUBALLOCATE_MEMORY = 0x4, + GP_ALLOCATION_CREATE_NEVER_SUBALLOCATE_MEMORY = 0x4, /** \brief Prefetch memory for the next resource allocation. @@ -101,7 +188,7 @@ namespace gpgmm::vk { allocating for large contiguous allocations. Should not be used with ALLOCATION_FLAG_NEVER_ALLOCATE_MEMORY. */ - GP_ALLOCATION_FLAG_ALWAYS_PREFETCH_MEMORY = 0x8, + GP_ALLOCATION_CREATE_ALWAYS_PREFETCH_MEMORY = 0x8, }; /** \struct GpResourceAllocationCreateInfo @@ -120,13 +207,13 @@ namespace gpgmm::vk { /** \brief Create allocator used to create and manage video memory for the App specified device and instance. - @param info A reference to GpCreateAllocatorInfo structure that describes the allocator. + @param info A reference to GpAllocatorCreateInfo structure that describes the allocator. @param[out] allocatorOut Pointer to a memory block that recieves a pointer to the resource allocator. Pass NULL to test if allocator creation would succeed, but not actually create the allocator. If NULL is passed and allocator creating would succeed, VK_INCOMPLETE is returned. */ - GPGMM_EXPORT VkResult gpCreateResourceAllocator(const GpCreateAllocatorInfo& info, + GPGMM_EXPORT VkResult gpCreateResourceAllocator(const GpAllocatorCreateInfo& info, GpResourceAllocator* allocatorOut); /** \brief Destroy allocator. @@ -169,7 +256,7 @@ namespace gpgmm::vk { class Caps; struct GpResourceAllocator_T { public: - static VkResult CreateAllocator(const GpCreateAllocatorInfo& info, + static VkResult CreateAllocator(const GpAllocatorCreateInfo& info, GpResourceAllocator* allocatorOut); VkResult TryAllocateMemory(const VkMemoryRequirements& requirements, @@ -185,7 +272,7 @@ namespace gpgmm::vk { Caps* GetCaps() const; private: - GpResourceAllocator_T(const GpCreateAllocatorInfo& info, + GpResourceAllocator_T(const GpAllocatorCreateInfo& info, const VulkanFunctions& vulkanFunctions, std::unique_ptr caps); @@ -193,6 +280,11 @@ namespace gpgmm::vk { const GpResourceAllocationCreateInfo& allocationInfo, uint32_t* memoryTypeIndexOut); + std::unique_ptr CreateDeviceMemoryAllocator( + const GpAllocatorCreateInfo& info, + uint64_t memoryTypeIndex, + uint64_t memoryAlignment); + VkDevice mDevice; VulkanFunctions mVulkanFunctions; std::unique_ptr mCaps; diff --git a/src/tests/VKTest.cpp b/src/tests/VKTest.cpp index 269599144..df092094d 100644 --- a/src/tests/VKTest.cpp +++ b/src/tests/VKTest.cpp @@ -131,8 +131,8 @@ namespace gpgmm::vk { } } - GpCreateAllocatorInfo VKTestBase::CreateBasicAllocatorInfo() const { - GpCreateAllocatorInfo allocatorInfo = {}; + GpAllocatorCreateInfo VKTestBase::CreateBasicAllocatorInfo() const { + GpAllocatorCreateInfo allocatorInfo = {}; allocatorInfo.device = mDevice; allocatorInfo.instance = mInstance; allocatorInfo.physicalDevice = mPhysicalDevice; diff --git a/src/tests/VKTest.h b/src/tests/VKTest.h index ec503d27c..cfc225a92 100644 --- a/src/tests/VKTest.h +++ b/src/tests/VKTest.h @@ -24,14 +24,14 @@ namespace gpgmm::vk { - struct GpCreateAllocatorInfo; + struct GpAllocatorCreateInfo; class VKTestBase : public GPGMMTestBase { public: void SetUp(); void TearDown(); - GpCreateAllocatorInfo CreateBasicAllocatorInfo() const; + GpAllocatorCreateInfo CreateBasicAllocatorInfo() const; protected: VkDevice mDevice = VK_NULL_HANDLE; diff --git a/src/tests/end2end/VKResourceAllocatorTests.cpp b/src/tests/end2end/VKResourceAllocatorTests.cpp index c76d09afd..719c70276 100644 --- a/src/tests/end2end/VKResourceAllocatorTests.cpp +++ b/src/tests/end2end/VKResourceAllocatorTests.cpp @@ -12,13 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "gpgmm/common/SizeClass.h" #include "tests/VKTest.h" #include using namespace gpgmm::vk; -static constexpr uint64_t kDefaultBufferSize = 4ll * 1024ll * 1024ll; // 4MB +static constexpr uint64_t kDefaultBufferSize = GPGMM_MB_TO_BYTES(4); class VKResourceAllocatorTests : public VKTestBase, public ::testing::Test { protected: