24 changes: 12 additions & 12 deletions libc/src/__support/freelist_heap.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ class FreeListHeap {

void *allocate_impl(size_t alignment, size_t size);

span<cpp::byte> block_to_span(Block<> *block) {
span<cpp::byte> block_to_span(Block *block) {
return span<cpp::byte>(block->usable_space(), block->inner_size());
}

Expand All @@ -75,8 +75,8 @@ template <size_t BUFF_SIZE> class FreeListHeapBuffer : public FreeListHeap {

LIBC_INLINE void FreeListHeap::init() {
LIBC_ASSERT(!is_initialized && "duplicate initialization");
auto result = Block<>::init(region());
Block<> *block = *result;
auto result = Block::init(region());
Block *block = *result;
free_store.set_range({0, cpp::bit_ceil(block->inner_size())});
free_store.insert(block);
is_initialized = true;
Expand All @@ -93,25 +93,25 @@ LIBC_INLINE void *FreeListHeap::allocate_impl(size_t alignment, size_t size) {

// TODO: usable_space should always be aligned to max_align_t.
if (alignment > alignof(max_align_t) ||
(Block<>::BLOCK_OVERHEAD % alignof(max_align_t) != 0)) {
(Block::BLOCK_OVERHEAD % alignof(max_align_t) != 0)) {
// TODO: This bound isn't precisely calculated yet. It assumes one extra
// Block<>::ALIGNMENT to accomodate the possibility for padding block
// Block::ALIGNMENT to accomodate the possibility for padding block
// overhead. (alignment - 1) ensures that there is an aligned point
// somewhere in usable_space, but this isn't tight either, since
// usable_space is also already somewhat aligned.
if (add_overflow(size, (alignment - 1) + Block<>::ALIGNMENT, request_size))
if (add_overflow(size, (alignment - 1) + Block::ALIGNMENT, request_size))
return nullptr;
}

Block<> *block = free_store.remove_best_fit(request_size);
Block *block = free_store.remove_best_fit(request_size);
if (!block)
return nullptr;

LIBC_ASSERT(block->can_allocate(alignment, size) &&
"block should always be large enough to allocate at the correct "
"alignment");

auto block_info = Block<>::allocate(block, alignment, size);
auto block_info = Block::allocate(block, alignment, size);
if (block_info.next)
free_store.insert(block_info.next);
if (block_info.prev)
Expand Down Expand Up @@ -143,14 +143,14 @@ LIBC_INLINE void FreeListHeap::free(void *ptr) {

LIBC_ASSERT(is_valid_ptr(bytes) && "Invalid pointer");

Block<> *block = Block<>::from_usable_space(bytes);
Block *block = Block::from_usable_space(bytes);
LIBC_ASSERT(block->next() && "sentinel last block cannot be freed");
LIBC_ASSERT(block->used() && "double free");
block->mark_free();

// Can we combine with the left or right blocks?
Block<> *prev_free = block->prev_free();
Block<> *next = block->next();
Block *prev_free = block->prev_free();
Block *next = block->next();

if (prev_free != nullptr) {
// Remove from free store and merge.
Expand Down Expand Up @@ -183,7 +183,7 @@ LIBC_INLINE void *FreeListHeap::realloc(void *ptr, size_t size) {
if (!is_valid_ptr(bytes))
return nullptr;

Block<> *block = Block<>::from_usable_space(bytes);
Block *block = Block::from_usable_space(bytes);
if (!block->used())
return nullptr;
size_t old_size = block->inner_size();
Expand Down
28 changes: 14 additions & 14 deletions libc/src/__support/freestore.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,40 +29,40 @@ class FreeStore {

/// Insert a free block. If the block is too small to be tracked, nothing
/// happens.
void insert(Block<> *block);
void insert(Block *block);

/// Remove a free block. If the block is too small to be tracked, nothing
/// happens.
void remove(Block<> *block);
void remove(Block *block);

/// Remove a best-fit free block that can contain the given size when
/// allocated. Returns nullptr if there is no such block.
Block<> *remove_best_fit(size_t size);
Block *remove_best_fit(size_t size);

private:
static constexpr size_t ALIGNMENT = alignof(max_align_t);
static constexpr size_t MIN_OUTER_SIZE =
align_up(Block<>::BLOCK_OVERHEAD + sizeof(FreeList::Node), ALIGNMENT);
align_up(Block::BLOCK_OVERHEAD + sizeof(FreeList::Node), ALIGNMENT);
static constexpr size_t MIN_LARGE_OUTER_SIZE =
align_up(Block<>::BLOCK_OVERHEAD + sizeof(FreeTrie::Node), ALIGNMENT);
align_up(Block::BLOCK_OVERHEAD + sizeof(FreeTrie::Node), ALIGNMENT);
static constexpr size_t NUM_SMALL_SIZES =
(MIN_LARGE_OUTER_SIZE - MIN_OUTER_SIZE) / ALIGNMENT;

LIBC_INLINE static bool too_small(Block<> *block) {
LIBC_INLINE static bool too_small(Block *block) {
return block->outer_size() < MIN_OUTER_SIZE;
}
LIBC_INLINE static bool is_small(Block<> *block) {
LIBC_INLINE static bool is_small(Block *block) {
return block->outer_size() < MIN_LARGE_OUTER_SIZE;
}

FreeList &small_list(Block<> *block);
FreeList &small_list(Block *block);
FreeList *find_best_small_fit(size_t size);

cpp::array<FreeList, NUM_SMALL_SIZES> small_lists;
FreeTrie large_trie;
};

LIBC_INLINE void FreeStore::insert(Block<> *block) {
LIBC_INLINE void FreeStore::insert(Block *block) {
if (too_small(block))
return;
if (is_small(block))
Expand All @@ -71,7 +71,7 @@ LIBC_INLINE void FreeStore::insert(Block<> *block) {
large_trie.push(block);
}

LIBC_INLINE void FreeStore::remove(Block<> *block) {
LIBC_INLINE void FreeStore::remove(Block *block) {
if (too_small(block))
return;
if (is_small(block)) {
Expand All @@ -83,21 +83,21 @@ LIBC_INLINE void FreeStore::remove(Block<> *block) {
}
}

LIBC_INLINE Block<> *FreeStore::remove_best_fit(size_t size) {
LIBC_INLINE Block *FreeStore::remove_best_fit(size_t size) {
if (FreeList *list = find_best_small_fit(size)) {
Block<> *block = list->front();
Block *block = list->front();
list->pop();
return block;
}
if (FreeTrie::Node *best_fit = large_trie.find_best_fit(size)) {
Block<> *block = best_fit->block();
Block *block = best_fit->block();
large_trie.remove(best_fit);
return block;
}
return nullptr;
}

LIBC_INLINE FreeList &FreeStore::small_list(Block<> *block) {
LIBC_INLINE FreeList &FreeStore::small_list(Block *block) {
LIBC_ASSERT(is_small(block) && "only legal for small blocks");
return small_lists[(block->outer_size() - MIN_OUTER_SIZE) / ALIGNMENT];
}
Expand Down
4 changes: 2 additions & 2 deletions libc/src/__support/freetrie.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ class FreeTrie {
LIBC_INLINE bool empty() const { return !root; }

/// Push a block to the trie.
void push(Block<> *block);
void push(Block *block);

/// Remove a node from this trie node's free list.
void remove(Node *node);
Expand All @@ -117,7 +117,7 @@ class FreeTrie {
SizeRange range;
};

LIBC_INLINE void FreeTrie::push(Block<> *block) {
LIBC_INLINE void FreeTrie::push(Block *block) {
LIBC_ASSERT(block->inner_size_free() >= sizeof(Node) &&
"block too small to accomodate free trie node");
size_t size = block->inner_size();
Expand Down
344 changes: 153 additions & 191 deletions libc/test/src/__support/block_test.cpp
View file

Large diffs are not rendered by default.

12 changes: 6 additions & 6 deletions libc/test/src/__support/freelist_heap_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ using LIBC_NAMESPACE::cpp::span;
void RunTest(FreeListHeap &allocator, [[maybe_unused]] size_t N); \
}; \
TEST_F(LlvmLibcFreeListHeapTest##TestCase, TestCase) { \
alignas(Block<>) byte buf[BufferSize] = {byte(0)}; \
alignas(Block) byte buf[BufferSize] = {byte(0)}; \
FreeListHeap allocator(buf); \
RunTest(allocator, BufferSize); \
RunTest(*freelist_heap, freelist_heap->region().size()); \
Expand Down Expand Up @@ -95,13 +95,13 @@ TEST_FOR_EACH_ALLOCATOR(ReturnsNullWhenAllocationTooLarge, 2048) {
// is used for other test cases and we don't explicitly free them.
TEST(LlvmLibcFreeListHeap, ReturnsNullWhenFull) {
constexpr size_t N = 2048;
alignas(Block<>) byte buf[N] = {byte(0)};
alignas(Block) byte buf[N] = {byte(0)};

FreeListHeap allocator(buf);

// Use aligned_allocate so we don't need to worry about ensuring the `buf`
// being aligned to max_align_t.
EXPECT_NE(allocator.aligned_allocate(1, N - 2 * Block<>::BLOCK_OVERHEAD),
EXPECT_NE(allocator.aligned_allocate(1, N - 2 * Block::BLOCK_OVERHEAD),
static_cast<void *>(nullptr));
EXPECT_EQ(allocator.allocate(1), static_cast<void *>(nullptr));
}
Expand Down Expand Up @@ -241,16 +241,16 @@ TEST_FOR_EACH_ALLOCATOR(AlignedAlloc, 2048) {

// This test is not part of the TEST_FOR_EACH_ALLOCATOR since we want to
// explicitly ensure that the buffer can still return aligned allocations even
// if the underlying buffer is at most aligned to the Block<> alignment. This
// if the underlying buffer is at most aligned to the Block alignment. This
// is so we can check that we can still get aligned allocations even if the
// underlying buffer is not aligned to the alignments we request.
TEST(LlvmLibcFreeListHeap, AlignedAllocOnlyBlockAligned) {
constexpr size_t BUFFER_SIZE = 4096;
constexpr size_t BUFFER_ALIGNMENT = alignof(Block<>) * 2;
constexpr size_t BUFFER_ALIGNMENT = alignof(Block) * 2;
alignas(BUFFER_ALIGNMENT) byte buf[BUFFER_SIZE] = {byte(0)};

// Ensure the underlying buffer is at most aligned to the block type.
FreeListHeap allocator(span<byte>(buf).subspan(alignof(Block<>)));
FreeListHeap allocator(span<byte>(buf).subspan(alignof(Block)));

constexpr size_t ALIGNMENTS[] = {1, 2, 4, 8, 16, 32, 64, 128, 256};
constexpr size_t SIZE_SCALES[] = {1, 2, 3, 4, 5};
Expand Down
8 changes: 4 additions & 4 deletions libc/test/src/__support/freelist_malloc_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,12 @@ TEST(LlvmLibcFreeListMalloc, Malloc) {
constexpr size_t kCallocSize = 64;

void *ptr1 = LIBC_NAMESPACE::malloc(kAllocSize);
auto *block = Block<>::from_usable_space(ptr1);
auto *block = Block::from_usable_space(ptr1);
EXPECT_GE(block->inner_size(), kAllocSize);

LIBC_NAMESPACE::free(ptr1);
ASSERT_NE(block->next(), static_cast<Block<> *>(nullptr));
ASSERT_EQ(block->next()->next(), static_cast<Block<> *>(nullptr));
ASSERT_NE(block->next(), static_cast<Block *>(nullptr));
ASSERT_EQ(block->next()->next(), static_cast<Block *>(nullptr));
size_t heap_size = block->inner_size();

void *ptr2 = LIBC_NAMESPACE::calloc(kCallocNum, kCallocSize);
Expand All @@ -46,7 +46,7 @@ TEST(LlvmLibcFreeListMalloc, Malloc) {
void *ptr3 = LIBC_NAMESPACE::aligned_alloc(ALIGN, kAllocSize);
EXPECT_NE(ptr3, static_cast<void *>(nullptr));
EXPECT_EQ(reinterpret_cast<uintptr_t>(ptr3) % ALIGN, size_t(0));
auto *aligned_block = reinterpret_cast<Block<> *>(ptr3);
auto *aligned_block = reinterpret_cast<Block *>(ptr3);
EXPECT_GE(aligned_block->inner_size(), kAllocSize);

LIBC_NAMESPACE::free(ptr3);
Expand Down
6 changes: 3 additions & 3 deletions libc/test/src/__support/freelist_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,13 @@ using LIBC_NAMESPACE::cpp::optional;

TEST(LlvmLibcFreeList, FreeList) {
byte mem[1024];
optional<Block<> *> maybeBlock = Block<>::init(mem);
optional<Block *> maybeBlock = Block::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
Block<> *block1 = *maybeBlock;
Block *block1 = *maybeBlock;

maybeBlock = block1->split(128);
ASSERT_TRUE(maybeBlock.has_value());
Block<> *block2 = *maybeBlock;
Block *block2 = *maybeBlock;

maybeBlock = block2->split(128);
ASSERT_TRUE(maybeBlock.has_value());
Expand Down
40 changes: 18 additions & 22 deletions libc/test/src/__support/freestore_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,12 @@ using LIBC_NAMESPACE::cpp::optional;
// Inserting or removing blocks too small to be tracked does nothing.
TEST(LlvmLibcFreeStore, TooSmall) {
byte mem[1024];
optional<Block<> *> maybeBlock = Block<>::init(mem);
optional<Block *> maybeBlock = Block::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
Block<> *too_small = *maybeBlock;
maybeBlock = too_small->split(sizeof(Block<>::offset_type));
Block *too_small = *maybeBlock;
maybeBlock = too_small->split(sizeof(size_t));
ASSERT_TRUE(maybeBlock.has_value());
Block<> *remainder = *maybeBlock;
Block *remainder = *maybeBlock;

FreeStore store;
store.set_range({0, 4096});
Expand All @@ -39,24 +39,22 @@ TEST(LlvmLibcFreeStore, TooSmall) {

TEST(LlvmLibcFreeStore, RemoveBestFit) {
byte mem[1024];
optional<Block<> *> maybeBlock = Block<>::init(mem);
optional<Block *> maybeBlock = Block::init(mem);
ASSERT_TRUE(maybeBlock.has_value());

Block<> *smallest = *maybeBlock;
maybeBlock =
smallest->split(sizeof(FreeList::Node) + sizeof(Block<>::offset_type));
Block *smallest = *maybeBlock;
maybeBlock = smallest->split(sizeof(FreeList::Node) + sizeof(size_t));
ASSERT_TRUE(maybeBlock.has_value());

Block<> *largest_small = *maybeBlock;
maybeBlock =
largest_small->split(sizeof(FreeTrie::Node) +
sizeof(Block<>::offset_type) - alignof(max_align_t));
Block *largest_small = *maybeBlock;
maybeBlock = largest_small->split(sizeof(FreeTrie::Node) + sizeof(size_t) -
alignof(max_align_t));
ASSERT_TRUE(maybeBlock.has_value());
if (largest_small->inner_size() == smallest->inner_size())
largest_small = smallest;
ASSERT_GE(largest_small->inner_size(), smallest->inner_size());

Block<> *remainder = *maybeBlock;
Block *remainder = *maybeBlock;

FreeStore store;
store.set_range({0, 4096});
Expand All @@ -74,8 +72,7 @@ TEST(LlvmLibcFreeStore, RemoveBestFit) {
store.insert(largest_small);

// Search small list for best fit.
Block<> *next_smallest =
largest_small == smallest ? remainder : largest_small;
Block *next_smallest = largest_small == smallest ? remainder : largest_small;
ASSERT_EQ(store.remove_best_fit(smallest->inner_size() + 1), next_smallest);
store.insert(next_smallest);

Expand All @@ -85,15 +82,14 @@ TEST(LlvmLibcFreeStore, RemoveBestFit) {

TEST(LlvmLibcFreeStore, Remove) {
byte mem[1024];
optional<Block<> *> maybeBlock = Block<>::init(mem);
optional<Block *> maybeBlock = Block::init(mem);
ASSERT_TRUE(maybeBlock.has_value());

Block<> *small = *maybeBlock;
maybeBlock =
small->split(sizeof(FreeList::Node) + sizeof(Block<>::offset_type));
Block *small = *maybeBlock;
maybeBlock = small->split(sizeof(FreeList::Node) + sizeof(size_t));
ASSERT_TRUE(maybeBlock.has_value());

Block<> *remainder = *maybeBlock;
Block *remainder = *maybeBlock;

FreeStore store;
store.set_range({0, 4096});
Expand All @@ -102,8 +98,8 @@ TEST(LlvmLibcFreeStore, Remove) {

store.remove(remainder);
ASSERT_EQ(store.remove_best_fit(remainder->inner_size()),
static_cast<Block<> *>(nullptr));
static_cast<Block *>(nullptr));
store.remove(small);
ASSERT_EQ(store.remove_best_fit(small->inner_size()),
static_cast<Block<> *>(nullptr));
static_cast<Block *>(nullptr));
}
32 changes: 16 additions & 16 deletions libc/test/src/__support/freetrie_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,9 @@ TEST(LlvmLibcFreeTrie, FindBestFitRoot) {
EXPECT_EQ(trie.find_best_fit(123), static_cast<FreeTrie::Node *>(nullptr));

byte mem[1024];
optional<Block<> *> maybeBlock = Block<>::init(mem);
optional<Block *> maybeBlock = Block::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
Block<> *block = *maybeBlock;
Block *block = *maybeBlock;
trie.push(block);

FreeTrie::Node *root = trie.find_best_fit(0);
Expand All @@ -37,12 +37,12 @@ TEST(LlvmLibcFreeTrie, FindBestFitRoot) {

TEST(LlvmLibcFreeTrie, FindBestFitLower) {
byte mem[4096];
optional<Block<> *> maybeBlock = Block<>::init(mem);
optional<Block *> maybeBlock = Block::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
Block<> *lower = *maybeBlock;
Block *lower = *maybeBlock;
maybeBlock = lower->split(512);
ASSERT_TRUE(maybeBlock.has_value());
Block<> *root = *maybeBlock;
Block *root = *maybeBlock;

FreeTrie trie({0, 4096});
trie.push(root);
Expand All @@ -53,12 +53,12 @@ TEST(LlvmLibcFreeTrie, FindBestFitLower) {

TEST(LlvmLibcFreeTrie, FindBestFitUpper) {
byte mem[4096];
optional<Block<> *> maybeBlock = Block<>::init(mem);
optional<Block *> maybeBlock = Block::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
Block<> *root = *maybeBlock;
Block *root = *maybeBlock;
maybeBlock = root->split(512);
ASSERT_TRUE(maybeBlock.has_value());
Block<> *upper = *maybeBlock;
Block *upper = *maybeBlock;

FreeTrie trie({0, 4096});
trie.push(root);
Expand All @@ -71,15 +71,15 @@ TEST(LlvmLibcFreeTrie, FindBestFitUpper) {

TEST(LlvmLibcFreeTrie, FindBestFitLowerAndUpper) {
byte mem[4096];
optional<Block<> *> maybeBlock = Block<>::init(mem);
optional<Block *> maybeBlock = Block::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
Block<> *root = *maybeBlock;
Block *root = *maybeBlock;
maybeBlock = root->split(1024);
ASSERT_TRUE(maybeBlock.has_value());
Block<> *lower = *maybeBlock;
Block *lower = *maybeBlock;
maybeBlock = lower->split(128);
ASSERT_TRUE(maybeBlock.has_value());
Block<> *upper = *maybeBlock;
Block *upper = *maybeBlock;

FreeTrie trie({0, 4096});
trie.push(root);
Expand All @@ -95,16 +95,16 @@ TEST(LlvmLibcFreeTrie, FindBestFitLowerAndUpper) {

TEST(LlvmLibcFreeTrie, Remove) {
byte mem[4096];
optional<Block<> *> maybeBlock = Block<>::init(mem);
optional<Block *> maybeBlock = Block::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
Block<> *small1 = *maybeBlock;
Block *small1 = *maybeBlock;
maybeBlock = small1->split(512);
ASSERT_TRUE(maybeBlock.has_value());
Block<> *small2 = *maybeBlock;
Block *small2 = *maybeBlock;
maybeBlock = small2->split(512);
ASSERT_TRUE(maybeBlock.has_value());
ASSERT_EQ(small1->inner_size(), small2->inner_size());
Block<> *large = *maybeBlock;
Block *large = *maybeBlock;

// Removing the root empties the trie.
FreeTrie trie({0, 4096});
Expand Down