From 364ad974ff1c16a94eaf5d02902092d4b45b2bd7 Mon Sep 17 00:00:00 2001 From: Bryan Bernhart Date: Thu, 5 May 2022 11:51:04 -0700 Subject: [PATCH] Specify slab block size based on request. Removes the min. block size from the SlabCacheAllocator by using the requested size-aligned value instead. This allows backends to use smaller (or larger) page sizes as requested. This optimization specifically helps allocating "small" resources that can use smaller page sizes (4KB vs 64KB). --- src/gpgmm/common/SlabMemoryAllocator.cpp | 6 +- src/gpgmm/common/SlabMemoryAllocator.h | 4 +- src/gpgmm/d3d12/ResourceAllocatorD3D12.cpp | 2 - .../unittests/SlabMemoryAllocatorTests.cpp | 89 +++++++++++-------- 4 files changed, 56 insertions(+), 45 deletions(-) diff --git a/src/gpgmm/common/SlabMemoryAllocator.cpp b/src/gpgmm/common/SlabMemoryAllocator.cpp index 53f1cec8f..ba2eadcd6 100644 --- a/src/gpgmm/common/SlabMemoryAllocator.cpp +++ b/src/gpgmm/common/SlabMemoryAllocator.cpp @@ -311,8 +311,7 @@ namespace gpgmm { // SlabCacheAllocator - SlabCacheAllocator::SlabCacheAllocator(uint64_t minBlockSize, - uint64_t maxSlabSize, + SlabCacheAllocator::SlabCacheAllocator(uint64_t maxSlabSize, uint64_t minSlabSize, uint64_t slabAlignment, double slabFragmentationLimit, @@ -320,7 +319,6 @@ namespace gpgmm { double slabGrowthFactor, std::unique_ptr memoryAllocator) : MemoryAllocator(std::move(memoryAllocator)), - mMinBlockSize(minBlockSize), mMaxSlabSize(maxSlabSize), mMinSlabSize(minSlabSize), mSlabAlignment(slabAlignment), @@ -347,7 +345,7 @@ namespace gpgmm { GPGMM_CHECK_NONZERO(requestSize); - const uint64_t blockSize = AlignTo(requestSize, mMinBlockSize); + const uint64_t blockSize = AlignTo(requestSize, alignment); // Create a slab allocator for the new entry. auto entry = mSizeCache.GetOrCreate(SlabAllocatorCacheEntry(blockSize), cacheSize); diff --git a/src/gpgmm/common/SlabMemoryAllocator.h b/src/gpgmm/common/SlabMemoryAllocator.h index 96851a834..b0ddec522 100644 --- a/src/gpgmm/common/SlabMemoryAllocator.h +++ b/src/gpgmm/common/SlabMemoryAllocator.h @@ -134,8 +134,7 @@ namespace gpgmm { // fixed-sized slabs. class SlabCacheAllocator : public MemoryAllocator { public: - SlabCacheAllocator(uint64_t minBlockSize, - uint64_t maxSlabSize, + SlabCacheAllocator(uint64_t maxSlabSize, uint64_t minSlabSize, uint64_t slabAlignment, double slabFragmentationLimit, @@ -177,7 +176,6 @@ namespace gpgmm { const uint64_t mBlockSize; }; - const uint64_t mMinBlockSize; const uint64_t mMaxSlabSize; const uint64_t mMinSlabSize; const uint64_t mSlabAlignment; diff --git a/src/gpgmm/d3d12/ResourceAllocatorD3D12.cpp b/src/gpgmm/d3d12/ResourceAllocatorD3D12.cpp index b5c1bf5cb..693b2b7d8 100644 --- a/src/gpgmm/d3d12/ResourceAllocatorD3D12.cpp +++ b/src/gpgmm/d3d12/ResourceAllocatorD3D12.cpp @@ -459,7 +459,6 @@ namespace gpgmm { namespace d3d12 { // TODO: Re-enable the buddy allocator? mResourceAllocatorOfType[resourceHeapTypeIndex] = std::make_unique< SlabCacheAllocator>( - /*minBlockSize*/ D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT, /*maxSlabSize*/ PrevPowerOfTwo(mMaxResourceHeapSize), /*minSlabSize*/ std::max(heapAlignment, descriptor.PreferredResourceHeapSize), /*slabAlignment*/ heapAlignment, @@ -511,7 +510,6 @@ namespace gpgmm { namespace d3d12 { // fragment by definition. mBufferAllocatorOfType[resourceHeapTypeIndex] = std::make_unique( - /*minBlockSize*/ 1, /*maxSlabSize*/ D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT, /*slabSize*/ D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT, /*slabAlignment*/ D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT, diff --git a/src/tests/unittests/SlabMemoryAllocatorTests.cpp b/src/tests/unittests/SlabMemoryAllocatorTests.cpp index 4e9ddc0e9..807a98215 100644 --- a/src/tests/unittests/SlabMemoryAllocatorTests.cpp +++ b/src/tests/unittests/SlabMemoryAllocatorTests.cpp @@ -631,25 +631,49 @@ TEST(SlabMemoryAllocatorTests, SlabGrowthLimit) { } TEST(SlabCacheAllocatorTests, SingleSlabMultipleSize) { - constexpr uint64_t kMinBlockSize = 4; constexpr uint64_t kMaxSlabSize = 256; constexpr uint64_t kSlabSize = 0; // deduce slab size from allocation size. - SlabCacheAllocator allocator(kMinBlockSize, kMaxSlabSize, kSlabSize, kDefaultSlabAlignment, + SlabCacheAllocator allocator(kMaxSlabSize, kSlabSize, kDefaultSlabAlignment, kDefaultSlabFragmentationLimit, kDefaultPrefetchSlab, kNoSlabGrowthFactor, std::make_unique()); // Verify requesting an allocation without memory will not return a valid allocation. { - EXPECT_EQ(allocator.TryAllocateMemory(kMinBlockSize, 1, true, false, false), nullptr); - EXPECT_EQ(allocator.TryAllocateMemory(kMinBlockSize * 2, 1, true, false, false), nullptr); + constexpr uint64_t kBlockSize = 4; + EXPECT_EQ(allocator.TryAllocateMemory(kBlockSize, 1, true, false, false), nullptr); + EXPECT_EQ(allocator.TryAllocateMemory(kBlockSize * 2, 1, true, false, false), nullptr); + } +} + +TEST(SlabCacheAllocatorTests, SingleSlabMultipleAlignments) { + constexpr uint64_t kMaxSlabSize = 256; + constexpr uint64_t kSlabSize = 0; // deduce slab size from allocation size. + SlabCacheAllocator allocator(kMaxSlabSize, kSlabSize, kDefaultSlabAlignment, + kDefaultSlabFragmentationLimit, kDefaultPrefetchSlab, + kNoSlabGrowthFactor, std::make_unique()); + + // Verify requesting an allocation of same size using multiple alignment succeeds. + { + constexpr uint64_t kBlockSize = 4; + std::unique_ptr allocationWithAlignmentA = + allocator.TryAllocateMemory(kBlockSize, 1, false, false, false); + ASSERT_NE(allocationWithAlignmentA, nullptr); + EXPECT_EQ(allocationWithAlignmentA->GetSize(), AlignTo(kBlockSize, 1)); + + std::unique_ptr allocationWithAlignmentB = + allocator.TryAllocateMemory(kBlockSize, 16, false, false, false); + ASSERT_NE(allocationWithAlignmentB, nullptr); + EXPECT_EQ(allocationWithAlignmentB->GetSize(), AlignTo(kBlockSize, 16)); + + allocator.DeallocateMemory(std::move(allocationWithAlignmentB)); + allocator.DeallocateMemory(std::move(allocationWithAlignmentA)); } } TEST(SlabCacheAllocatorTests, MultipleSlabsSameSize) { - constexpr uint64_t kMinBlockSize = 4; constexpr uint64_t kMaxSlabSize = 256; constexpr uint64_t kSlabSize = 0; // deduce slab size from allocation size. - SlabCacheAllocator allocator(kMinBlockSize, kMaxSlabSize, kSlabSize, kDefaultSlabAlignment, + SlabCacheAllocator allocator(kMaxSlabSize, kSlabSize, kDefaultSlabAlignment, kDefaultSlabFragmentationLimit, kDefaultPrefetchSlab, kNoSlabGrowthFactor, std::make_unique()); @@ -677,10 +701,9 @@ TEST(SlabCacheAllocatorTests, MultipleSlabsSameSize) { } TEST(SlabCacheAllocatorTests, MultipleSlabsVariableSizes) { - constexpr uint64_t kMinBlockSize = 4; constexpr uint64_t kMaxSlabSize = 256; constexpr uint64_t kSlabSize = 0; // deduce slab size from allocation size. - SlabCacheAllocator allocator(kMinBlockSize, kMaxSlabSize, kSlabSize, kDefaultSlabAlignment, + SlabCacheAllocator allocator(kMaxSlabSize, kSlabSize, kDefaultSlabAlignment, kDefaultSlabFragmentationLimit, kDefaultPrefetchSlab, kNoSlabGrowthFactor, std::make_unique()); { @@ -690,7 +713,7 @@ TEST(SlabCacheAllocatorTests, MultipleSlabsVariableSizes) { ASSERT_NE(allocation, nullptr); EXPECT_EQ(allocation->GetOffset(), 0u); EXPECT_EQ(allocation->GetMethod(), AllocationMethod::kSubAllocated); - EXPECT_GE(allocation->GetSize(), AlignTo(allocationSize, kMinBlockSize)); + EXPECT_GE(allocation->GetSize(), allocationSize); allocator.DeallocateMemory(std::move(allocation)); } @@ -701,7 +724,7 @@ TEST(SlabCacheAllocatorTests, MultipleSlabsVariableSizes) { ASSERT_NE(allocation, nullptr); EXPECT_EQ(allocation->GetOffset(), 0u); EXPECT_EQ(allocation->GetMethod(), AllocationMethod::kSubAllocated); - EXPECT_GE(allocation->GetSize(), AlignTo(allocationSize, kMinBlockSize)); + EXPECT_GE(allocation->GetSize(), allocationSize); allocator.DeallocateMemory(std::move(allocation)); } @@ -712,7 +735,7 @@ TEST(SlabCacheAllocatorTests, MultipleSlabsVariableSizes) { ASSERT_NE(allocation, nullptr); EXPECT_EQ(allocation->GetOffset(), 0u); EXPECT_EQ(allocation->GetMethod(), AllocationMethod::kSubAllocated); - EXPECT_GE(allocation->GetSize(), AlignTo(allocationSize, kMinBlockSize)); + EXPECT_GE(allocation->GetSize(), allocationSize); allocator.DeallocateMemory(std::move(allocation)); } @@ -724,22 +747,22 @@ TEST(SlabCacheAllocatorTests, SingleSlabInBuddy) { // 1. Create a buddy allocator as the back-end allocator. // 2. Create a slab allocator as the front-end allocator. constexpr uint64_t kMaxBlockSize = 256; - constexpr uint64_t kMinBlockSize = 4; constexpr uint64_t kMaxSlabSize = kMaxBlockSize; constexpr uint64_t kSlabSize = kDefaultSlabSize / 8; - SlabCacheAllocator allocator(kMinBlockSize, kMaxSlabSize, kSlabSize, kDefaultSlabAlignment, + SlabCacheAllocator allocator(kMaxSlabSize, kSlabSize, kDefaultSlabAlignment, kDefaultSlabFragmentationLimit, kDefaultPrefetchSlab, kNoSlabGrowthFactor, std::make_unique( kMaxBlockSize, kDefaultSlabSize, kDefaultSlabAlignment, std::make_unique())); + constexpr uint64_t kBlockSize = 4; std::unique_ptr allocation = - allocator.TryAllocateMemory(kMinBlockSize, 1, false, false, false); + allocator.TryAllocateMemory(kBlockSize, 1, false, false, false); ASSERT_NE(allocation, nullptr); EXPECT_EQ(allocation->GetOffset(), 0u); EXPECT_EQ(allocation->GetMethod(), AllocationMethod::kSubAllocated); - EXPECT_GE(allocation->GetSize(), kMinBlockSize); + EXPECT_GE(allocation->GetSize(), kBlockSize); allocator.DeallocateMemory(std::move(allocation)); } @@ -748,10 +771,9 @@ TEST(SlabCacheAllocatorTests, MultipleSlabsInBuddy) { // 1. Create a buddy allocator as the back-end allocator. // 2. Create a slab allocator as the front-end allocator. constexpr uint64_t kMaxBlockSize = 256; - constexpr uint64_t kMinBlockSize = 4; constexpr uint64_t kMaxSlabSize = kMaxBlockSize; constexpr uint64_t kSlabSize = kDefaultSlabSize / 8; - SlabCacheAllocator allocator(kMinBlockSize, kMaxSlabSize, kSlabSize, kDefaultSlabAlignment, + SlabCacheAllocator allocator(kMaxSlabSize, kSlabSize, kDefaultSlabAlignment, kDefaultSlabFragmentationLimit, kDefaultPrefetchSlab, kNoSlabGrowthFactor, std::make_unique( @@ -760,7 +782,7 @@ TEST(SlabCacheAllocatorTests, MultipleSlabsInBuddy) { // Verify multiple slab-buddy sub-allocation in the same slab are allocated contigiously. { - constexpr uint64_t allocationSize = kMinBlockSize * 2; + constexpr uint64_t allocationSize = 8; std::unique_ptr firstAllocation = allocator.TryAllocateMemory(allocationSize, 1, false, false, false); ASSERT_NE(firstAllocation, nullptr); @@ -826,13 +848,11 @@ TEST(SlabCacheAllocatorTests, MultipleSlabsInBuddy) { TEST(SlabCacheAllocatorTests, GetInfo) { // Test Slab allocator. { - constexpr uint64_t kMinBlockSize = 4; constexpr uint64_t kBlockSize = 32; constexpr uint64_t kMaxSlabSize = 512; - SlabCacheAllocator allocator(kMinBlockSize, kMaxSlabSize, kDefaultSlabSize, - kDefaultSlabAlignment, kDefaultSlabFragmentationLimit, - kDefaultPrefetchSlab, kNoSlabGrowthFactor, - std::make_unique()); + SlabCacheAllocator allocator(kMaxSlabSize, kDefaultSlabSize, kDefaultSlabAlignment, + kDefaultSlabFragmentationLimit, kDefaultPrefetchSlab, + kNoSlabGrowthFactor, std::make_unique()); std::unique_ptr allocation = allocator.TryAllocateMemory(kBlockSize, 1, false, false, false); @@ -858,12 +878,11 @@ TEST(SlabCacheAllocatorTests, GetInfo) { // Test Slab + pooled allocator. { LIFOMemoryPool pool(kDefaultSlabSize); - constexpr uint64_t kMinBlockSize = 4; constexpr uint64_t kBlockSize = 32; constexpr uint64_t kMaxSlabSize = 512; - SlabCacheAllocator allocator(kMinBlockSize, kMaxSlabSize, kDefaultSlabSize, - kDefaultSlabAlignment, kDefaultSlabFragmentationLimit, - kDefaultPrefetchSlab, kNoSlabGrowthFactor, + SlabCacheAllocator allocator(kMaxSlabSize, kDefaultSlabSize, kDefaultSlabAlignment, + kDefaultSlabFragmentationLimit, kDefaultPrefetchSlab, + kNoSlabGrowthFactor, std::make_unique( std::make_unique(), &pool)); @@ -891,23 +910,23 @@ TEST(SlabCacheAllocatorTests, GetInfo) { // Test Slab-Buddy allocator. { constexpr uint64_t kMaxBlockSize = 256; - constexpr uint64_t kMinBlockSize = 4; constexpr uint64_t kMaxSlabSize = kMaxBlockSize; constexpr uint64_t kSlabSize = kDefaultSlabSize / 8; - SlabCacheAllocator allocator(kMinBlockSize, kMaxSlabSize, kSlabSize, kDefaultSlabAlignment, + SlabCacheAllocator allocator(kMaxSlabSize, kSlabSize, kDefaultSlabAlignment, kDefaultSlabFragmentationLimit, kDefaultPrefetchSlab, kNoSlabGrowthFactor, std::make_unique( kMaxBlockSize, kDefaultSlabSize, kDefaultSlabAlignment, std::make_unique())); + constexpr uint64_t kBlockSize = 4; std::unique_ptr allocation = - allocator.TryAllocateMemory(kMinBlockSize, 1, false, false, false); + allocator.TryAllocateMemory(kBlockSize, 1, false, false, false); EXPECT_NE(allocation, nullptr); // Single slab block within buddy memory should be used. EXPECT_EQ(allocator.GetInfo().UsedBlockCount, 1u); - EXPECT_EQ(allocator.GetInfo().UsedBlockUsage, kMinBlockSize); + EXPECT_EQ(allocator.GetInfo().UsedBlockUsage, kBlockSize); EXPECT_EQ(allocator.GetInfo().UsedMemoryCount, 1u); EXPECT_EQ(allocator.GetInfo().UsedMemoryUsage, kDefaultSlabSize); EXPECT_EQ(allocator.GetInfo().FreeMemoryUsage, 0u); @@ -926,13 +945,11 @@ TEST(SlabCacheAllocatorTests, GetInfo) { // Pre-fetch |kNumOfSlabs| slabs worth of sub-allocations of various sizes. TEST(SlabCacheAllocatorTests, SlabPrefetch) { constexpr uint64_t kBlockSize = 32; - constexpr uint64_t kMinBlockSize = 4; constexpr uint64_t kMaxSlabSize = 512; - SlabCacheAllocator allocator(kMinBlockSize, kMaxSlabSize, kDefaultSlabSize, - kDefaultSlabAlignment, kDefaultSlabFragmentationLimit, - /*prefetchSlab*/ true, kNoSlabGrowthFactor, - std::make_unique()); + SlabCacheAllocator allocator( + kMaxSlabSize, kDefaultSlabSize, kDefaultSlabAlignment, kDefaultSlabFragmentationLimit, + /*prefetchSlab*/ true, kNoSlabGrowthFactor, std::make_unique()); constexpr uint64_t kNumOfSlabs = 10u; std::vector> allocations = {};