228 changes: 174 additions & 54 deletions libc/src/__support/freelist.h
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//===-- Interface for freelist --------------------------------------------===//
//===-- Interface for freelist_malloc -------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
Expand All @@ -9,80 +9,200 @@
#ifndef LLVM_LIBC_SRC___SUPPORT_FREELIST_H
#define LLVM_LIBC_SRC___SUPPORT_FREELIST_H

#include "block.h"
#include "src/__support/CPP/array.h"
#include "src/__support/CPP/cstddef.h"
#include "src/__support/CPP/new.h"
#include "src/__support/CPP/span.h"
#include "src/__support/fixedvector.h"
#include "src/__support/macros/config.h"

namespace LIBC_NAMESPACE_DECL {

/// A circularly-linked FIFO list storing free Blocks. All Blocks on a list
/// are the same size. The blocks are referenced by Nodes in the list; the list
/// refers to these, but it does not own them.
using cpp::span;

/// Basic [freelist](https://en.wikipedia.org/wiki/Free_list) implementation
/// for an allocator. This implementation buckets by chunk size, with a list
/// of user-provided buckets. Each bucket is a linked list of storage chunks.
/// Because this freelist uses the added chunks themselves as list nodes, there
/// is a lower bound of `sizeof(FreeList.FreeListNode)` bytes for chunks which
/// can be added to this freelist. There is also an implicit bucket for
/// "everything else", for chunks which do not fit into a bucket.
///
/// Each added chunk will be added to the smallest bucket under which it fits.
/// If it does not fit into any user-provided bucket, it will be added to the
/// default bucket.
///
/// As an example, assume that the `FreeList` is configured with buckets of
/// sizes {64, 128, 256, and 512} bytes. The internal state may look like the
/// following:
///
/// Allocating free blocks in FIFO order maximizes the amount of time before a
/// free block is reused. This in turn maximizes the number of opportunities for
/// it to be coalesced with an adjacent block, which tends to reduce heap
/// fragmentation.
class FreeList {
/// @code{.unparsed}
/// bucket[0] (64B) --> chunk[12B] --> chunk[42B] --> chunk[64B] --> NULL
/// bucket[1] (128B) --> chunk[65B] --> chunk[72B] --> NULL
/// bucket[2] (256B) --> NULL
/// bucket[3] (512B) --> chunk[312B] --> chunk[512B] --> chunk[416B] --> NULL
/// bucket[4] (implicit) --> chunk[1024B] --> chunk[513B] --> NULL
/// @endcode
///
/// Note that added chunks should be aligned to a 4-byte boundary.
template <size_t NUM_BUCKETS = 6> class FreeList {
public:
class Node {
public:
/// @returns The block containing this node.
LIBC_INLINE const Block<> *block() const {
return Block<>::from_usable_space(this);
// Remove copy/move ctors
FreeList(const FreeList &other) = delete;
FreeList(FreeList &&other) = delete;
FreeList &operator=(const FreeList &other) = delete;
FreeList &operator=(FreeList &&other) = delete;

/// Adds a chunk to this freelist.
bool add_chunk(cpp::span<cpp::byte> chunk);

/// Finds an eligible chunk for an allocation of size `size`.
///
/// @note This returns the first allocation possible within a given bucket;
/// It does not currently optimize for finding the smallest chunk.
///
/// @returns
/// * On success - A span representing the chunk.
/// * On failure (e.g. there were no chunks available for that allocation) -
/// A span with a size of 0.
cpp::span<cpp::byte> find_chunk(size_t size) const;

template <typename Cond> cpp::span<cpp::byte> find_chunk_if(Cond op) const;

/// Removes a chunk from this freelist.
bool remove_chunk(cpp::span<cpp::byte> chunk);

/// For a given size, find which index into chunks_ the node should be written
/// to.
constexpr size_t find_chunk_ptr_for_size(size_t size, bool non_null) const;

struct FreeListNode {
FreeListNode *next;
size_t size;
};

constexpr void set_freelist_node(FreeListNode &node,
cpp::span<cpp::byte> chunk);

constexpr explicit FreeList(const cpp::array<size_t, NUM_BUCKETS> &sizes)
: chunks_(NUM_BUCKETS + 1, 0), sizes_(sizes.begin(), sizes.end()) {}

private:
FixedVector<FreeList::FreeListNode *, NUM_BUCKETS + 1> chunks_;
FixedVector<size_t, NUM_BUCKETS> sizes_;
};

template <size_t NUM_BUCKETS>
constexpr void FreeList<NUM_BUCKETS>::set_freelist_node(FreeListNode &node,
span<cpp::byte> chunk) {
// Add it to the correct list.
size_t chunk_ptr = find_chunk_ptr_for_size(chunk.size(), false);
node.size = chunk.size();
node.next = chunks_[chunk_ptr];
chunks_[chunk_ptr] = &node;
}

template <size_t NUM_BUCKETS>
bool FreeList<NUM_BUCKETS>::add_chunk(span<cpp::byte> chunk) {
// Check that the size is enough to actually store what we need
if (chunk.size() < sizeof(FreeListNode))
return false;

FreeListNode *node = ::new (chunk.data()) FreeListNode;
set_freelist_node(*node, chunk);

return true;
}

template <size_t NUM_BUCKETS>
template <typename Cond>
span<cpp::byte> FreeList<NUM_BUCKETS>::find_chunk_if(Cond op) const {
for (FreeListNode *node : chunks_) {
while (node != nullptr) {
span<cpp::byte> chunk(reinterpret_cast<cpp::byte *>(node), node->size);
if (op(chunk))
return chunk;

node = node->next;
}
}

/// @returns The block containing this node.
LIBC_INLINE Block<> *block() { return Block<>::from_usable_space(this); }
return {};
}

/// @returns The inner size of blocks in the list containing this node.
LIBC_INLINE size_t size() const { return block()->inner_size(); }
template <size_t NUM_BUCKETS>
span<cpp::byte> FreeList<NUM_BUCKETS>::find_chunk(size_t size) const {
if (size == 0)
return span<cpp::byte>();

private:
// Circularly linked pointers to adjacent nodes.
Node *prev;
Node *next;
friend class FreeList;
};
size_t chunk_ptr = find_chunk_ptr_for_size(size, true);

// Check that there's data. This catches the case where we run off the
// end of the array
if (chunks_[chunk_ptr] == nullptr)
return span<cpp::byte>();

LIBC_INLINE constexpr FreeList() : FreeList(nullptr) {}
LIBC_INLINE constexpr FreeList(Node *begin) : begin_(begin) {}
// Now iterate up the buckets, walking each list to find a good candidate
for (size_t i = chunk_ptr; i < chunks_.size(); i++) {
FreeListNode *node = chunks_[static_cast<unsigned short>(i)];

LIBC_INLINE bool empty() const { return !begin_; }
while (node != nullptr) {
if (node->size >= size)
return span<cpp::byte>(reinterpret_cast<cpp::byte *>(node), node->size);

/// @returns The inner size of blocks in the list.
LIBC_INLINE size_t size() const {
LIBC_ASSERT(begin_ && "empty lists have no size");
return begin_->size();
node = node->next;
}
}

/// @returns The first node in the list.
LIBC_INLINE Node *begin() { return begin_; }
// If we get here, we've checked every block in every bucket. There's
// nothing that can support this allocation.
return span<cpp::byte>();
}

/// @returns The first block in the list.
LIBC_INLINE Block<> *front() { return begin_->block(); }
template <size_t NUM_BUCKETS>
bool FreeList<NUM_BUCKETS>::remove_chunk(span<cpp::byte> chunk) {
size_t chunk_ptr = find_chunk_ptr_for_size(chunk.size(), true);

/// Push a block to the back of the list.
/// The block must be large enough to contain a node.
LIBC_INLINE void push(Block<> *block) {
LIBC_ASSERT(!block->used() &&
"only free blocks can be placed on free lists");
LIBC_ASSERT(block->inner_size_free() >= sizeof(FreeList) &&
"block too small to accomodate free list node");
push(new (block->usable_space()) Node);
// Check head first.
if (chunks_[chunk_ptr] == nullptr)
return false;

FreeListNode *node = chunks_[chunk_ptr];
if (reinterpret_cast<cpp::byte *>(node) == chunk.data()) {
chunks_[chunk_ptr] = node->next;
return true;
}

/// Push an already-constructed node to the back of the list.
/// This allows pushing derived node types with additional data.
void push(Node *node);
// No? Walk the nodes.
node = chunks_[chunk_ptr];

/// Pop the first node from the list.
LIBC_INLINE void pop() { remove(begin_); }
while (node->next != nullptr) {
if (reinterpret_cast<cpp::byte *>(node->next) == chunk.data()) {
// Found it, remove this node out of the chain
node->next = node->next->next;
return true;
}

/// Remove an arbitrary node from the list.
void remove(Node *node);
node = node->next;
}

private:
Node *begin_;
};
return false;
}

template <size_t NUM_BUCKETS>
constexpr size_t
FreeList<NUM_BUCKETS>::find_chunk_ptr_for_size(size_t size,
bool non_null) const {
size_t chunk_ptr = 0;
for (chunk_ptr = 0u; chunk_ptr < sizes_.size(); chunk_ptr++) {
if (sizes_[chunk_ptr] >= size &&
(!non_null || chunks_[chunk_ptr] != nullptr)) {
break;
}
}

return chunk_ptr;
}

} // namespace LIBC_NAMESPACE_DECL

Expand Down
153 changes: 85 additions & 68 deletions libc/src/__support/freelist_heap.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,11 @@
#include <stddef.h>

#include "block.h"
#include "freestore.h"
#include "freelist.h"
#include "src/__support/CPP/optional.h"
#include "src/__support/CPP/span.h"
#include "src/__support/libc_assert.h"
#include "src/__support/macros/config.h"
#include "src/__support/math_extras.h"
#include "src/string/memory_utils/inline_memcpy.h"
#include "src/string/memory_utils/inline_memset.h"

Expand All @@ -29,14 +28,23 @@ extern "C" cpp::byte __llvm_libc_heap_limit;
using cpp::optional;
using cpp::span;

LIBC_INLINE constexpr bool IsPow2(size_t x) { return x && (x & (x - 1)) == 0; }
inline constexpr bool IsPow2(size_t x) { return x && (x & (x - 1)) == 0; }

class FreeListHeap {
static constexpr cpp::array<size_t, 6> DEFAULT_BUCKETS{16, 32, 64,
128, 256, 512};

template <size_t NUM_BUCKETS = DEFAULT_BUCKETS.size()> class FreeListHeap {
public:
constexpr FreeListHeap() : begin(&_end), end(&__llvm_libc_heap_limit) {}
using BlockType = Block<>;
using FreeListType = FreeList<NUM_BUCKETS>;

static constexpr size_t MIN_ALIGNMENT =
cpp::max(BlockType::ALIGNMENT, alignof(max_align_t));

constexpr FreeListHeap() : begin_(&_end), end_(&__llvm_libc_heap_limit) {}

constexpr FreeListHeap(span<cpp::byte> region)
: begin(region.begin()), end(region.end()) {}
: begin_(region.begin()), end_(region.end()) {}

void *allocate(size_t size);
void *aligned_allocate(size_t alignment, size_t size);
Expand All @@ -46,79 +54,89 @@ class FreeListHeap {
void *realloc(void *ptr, size_t size);
void *calloc(size_t num, size_t size);

cpp::span<cpp::byte> region() const { return {begin, end}; }
cpp::span<cpp::byte> region() const { return {begin_, end_}; }

private:
void init();

void *allocate_impl(size_t alignment, size_t size);

span<cpp::byte> block_to_span(Block<> *block) {
span<cpp::byte> block_to_span(BlockType *block) {
return span<cpp::byte>(block->usable_space(), block->inner_size());
}

bool is_valid_ptr(void *ptr) { return ptr >= begin && ptr < end; }
bool is_valid_ptr(void *ptr) { return ptr >= begin_ && ptr < end_; }

cpp::byte *begin;
cpp::byte *end;
bool is_initialized = false;
FreeStore free_store;
bool is_initialized_ = false;
cpp::byte *begin_;
cpp::byte *end_;
FreeListType freelist_{DEFAULT_BUCKETS};
};

template <size_t BUFF_SIZE> class FreeListHeapBuffer : public FreeListHeap {
template <size_t BUFF_SIZE, size_t NUM_BUCKETS = DEFAULT_BUCKETS.size()>
class FreeListHeapBuffer : public FreeListHeap<NUM_BUCKETS> {
using parent = FreeListHeap<NUM_BUCKETS>;
using FreeListNode = typename parent::FreeListType::FreeListNode;

public:
constexpr FreeListHeapBuffer() : FreeListHeap{buffer}, buffer{} {}
constexpr FreeListHeapBuffer()
: FreeListHeap<NUM_BUCKETS>{buffer}, buffer{} {}

private:
cpp::byte buffer[BUFF_SIZE];
};

LIBC_INLINE void FreeListHeap::init() {
LIBC_ASSERT(!is_initialized && "duplicate initialization");
auto result = Block<>::init(region());
Block<> *block = *result;
free_store.set_range({0, cpp::bit_ceil(block->inner_size())});
free_store.insert(block);
is_initialized = true;
template <size_t NUM_BUCKETS> void FreeListHeap<NUM_BUCKETS>::init() {
LIBC_ASSERT(!is_initialized_ && "duplicate initialization");
auto result = BlockType::init(region());
BlockType *block = *result;
freelist_.add_chunk(block_to_span(block));
is_initialized_ = true;
}

LIBC_INLINE void *FreeListHeap::allocate_impl(size_t alignment, size_t size) {
template <size_t NUM_BUCKETS>
void *FreeListHeap<NUM_BUCKETS>::allocate_impl(size_t alignment, size_t size) {
if (size == 0)
return nullptr;

if (!is_initialized)
if (!is_initialized_)
init();

size_t request_size = size;
if (alignment > alignof(max_align_t)) {
if (add_overflow(size, alignment - 1, request_size))
return nullptr;
}
// Find a chunk in the freelist. Split it if needed, then return.
auto chunk =
freelist_.find_chunk_if([alignment, size](span<cpp::byte> chunk) {
BlockType *block = BlockType::from_usable_space(chunk.data());
return block->can_allocate(alignment, size);
});

Block<> *block = free_store.remove_best_fit(request_size);
if (!block)
if (chunk.data() == nullptr)
return nullptr;
freelist_.remove_chunk(chunk);

LIBC_ASSERT(block->can_allocate(alignment, size) &&
"block should always be large enough to allocate at the correct "
"alignment");
BlockType *chunk_block = BlockType::from_usable_space(chunk.data());
LIBC_ASSERT(!chunk_block->used());

auto block_info = Block<>::allocate(block, alignment, size);
// Split that chunk. If there's a leftover chunk, add it to the freelist
auto block_info = BlockType::allocate(chunk_block, alignment, size);
if (block_info.next)
free_store.insert(block_info.next);
freelist_.add_chunk(block_to_span(block_info.next));
if (block_info.prev)
free_store.insert(block_info.prev);
freelist_.add_chunk(block_to_span(block_info.prev));
chunk_block = block_info.block;

chunk_block->mark_used();

block_info.block->mark_used();
return block_info.block->usable_space();
return chunk_block->usable_space();
}

LIBC_INLINE void *FreeListHeap::allocate(size_t size) {
return allocate_impl(alignof(max_align_t), size);
template <size_t NUM_BUCKETS>
void *FreeListHeap<NUM_BUCKETS>::allocate(size_t size) {
return allocate_impl(MIN_ALIGNMENT, size);
}

LIBC_INLINE void *FreeListHeap::aligned_allocate(size_t alignment,
size_t size) {
template <size_t NUM_BUCKETS>
void *FreeListHeap<NUM_BUCKETS>::aligned_allocate(size_t alignment,
size_t size) {
// The alignment must be an integral power of two.
if (!IsPow2(alignment))
return nullptr;
Expand All @@ -130,37 +148,38 @@ LIBC_INLINE void *FreeListHeap::aligned_allocate(size_t alignment,
return allocate_impl(alignment, size);
}

LIBC_INLINE void FreeListHeap::free(void *ptr) {
template <size_t NUM_BUCKETS> void FreeListHeap<NUM_BUCKETS>::free(void *ptr) {
cpp::byte *bytes = static_cast<cpp::byte *>(ptr);

LIBC_ASSERT(is_valid_ptr(bytes) && "Invalid pointer");

Block<> *block = Block<>::from_usable_space(bytes);
LIBC_ASSERT(block->next() && "sentinel last block cannot be freed");
LIBC_ASSERT(block->used() && "double free");
block->mark_free();
BlockType *chunk_block = BlockType::from_usable_space(bytes);
LIBC_ASSERT(chunk_block->next() && "sentinel last block cannot be freed");
LIBC_ASSERT(chunk_block->used() && "The block is not in-use");
chunk_block->mark_free();

// Can we combine with the left or right blocks?
Block<> *prev_free = block->prev_free();
Block<> *next = block->next();
BlockType *prev_free = chunk_block->prev_free();
BlockType *next = chunk_block->next();

if (prev_free != nullptr) {
// Remove from free store and merge.
free_store.remove(prev_free);
block = prev_free;
block->merge_next();
// Remove from freelist and merge
freelist_.remove_chunk(block_to_span(prev_free));
chunk_block = prev_free;
chunk_block->merge_next();
}
if (!next->used()) {
free_store.remove(next);
block->merge_next();
freelist_.remove_chunk(block_to_span(next));
chunk_block->merge_next();
}
// Add back to the freelist
free_store.insert(block);
freelist_.add_chunk(block_to_span(chunk_block));
}

// Follows constract of the C standard realloc() function
// If ptr is free'd, will return nullptr.
LIBC_INLINE void *FreeListHeap::realloc(void *ptr, size_t size) {
template <size_t NUM_BUCKETS>
void *FreeListHeap<NUM_BUCKETS>::realloc(void *ptr, size_t size) {
if (size == 0) {
free(ptr);
return nullptr;
Expand All @@ -175,10 +194,10 @@ LIBC_INLINE void *FreeListHeap::realloc(void *ptr, size_t size) {
if (!is_valid_ptr(bytes))
return nullptr;

Block<> *block = Block<>::from_usable_space(bytes);
if (!block->used())
BlockType *chunk_block = BlockType::from_usable_space(bytes);
if (!chunk_block->used())
return nullptr;
size_t old_size = block->inner_size();
size_t old_size = chunk_block->inner_size();

// Do nothing and return ptr if the required memory size is smaller than
// the current size.
Expand All @@ -195,17 +214,15 @@ LIBC_INLINE void *FreeListHeap::realloc(void *ptr, size_t size) {
return new_ptr;
}

LIBC_INLINE void *FreeListHeap::calloc(size_t num, size_t size) {
size_t bytes;
if (__builtin_mul_overflow(num, size, &bytes))
return nullptr;
void *ptr = allocate(bytes);
template <size_t NUM_BUCKETS>
void *FreeListHeap<NUM_BUCKETS>::calloc(size_t num, size_t size) {
void *ptr = allocate(num * size);
if (ptr != nullptr)
LIBC_NAMESPACE::inline_memset(ptr, 0, bytes);
LIBC_NAMESPACE::inline_memset(ptr, 0, num * size);
return ptr;
}

extern FreeListHeap *freelist_heap;
extern FreeListHeap<> *freelist_heap;

} // namespace LIBC_NAMESPACE_DECL

Expand Down
114 changes: 0 additions & 114 deletions libc/src/__support/freestore.h
View file

This file was deleted.

64 changes: 0 additions & 64 deletions libc/src/__support/freetrie.cpp
View file

This file was deleted.

237 changes: 0 additions & 237 deletions libc/src/__support/freetrie.h
View file

This file was deleted.

4 changes: 2 additions & 2 deletions libc/src/stdlib/freelist_malloc.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@

namespace LIBC_NAMESPACE_DECL {

static LIBC_CONSTINIT FreeListHeap freelist_heap_symbols;
FreeListHeap *freelist_heap = &freelist_heap_symbols;
static LIBC_CONSTINIT FreeListHeap<> freelist_heap_symbols;
FreeListHeap<> *freelist_heap = &freelist_heap_symbols;

LLVM_LIBC_FUNCTION(void *, malloc, (size_t size)) {
return freelist_heap->allocate(size);
Expand Down
27 changes: 0 additions & 27 deletions libc/test/src/__support/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,34 +24,7 @@ if(NOT LIBC_TARGET_OS_IS_GPU)
DEPENDS
libc.src.__support.CPP.array
libc.src.__support.CPP.span
libc.src.__support.block
libc.src.__support.freelist
)

add_libc_test(
freetrie_test
SUITE
libc-support-tests
SRCS
freetrie_test.cpp
DEPENDS
libc.src.__support.CPP.optional
libc.src.__support.block
libc.src.__support.freetrie
)

add_libc_test(
freestore_test
SUITE
libc-support-tests
SRCS
freestore_test.cpp
DEPENDS
libc.src.__support.CPP.optional
libc.src.__support.block
libc.src.__support.freelist
libc.src.__support.freestore
libc.src.__support.freetrie
)
endif()

Expand Down
14 changes: 14 additions & 0 deletions libc/test/src/__support/block_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -238,6 +238,20 @@ TEST_FOR_EACH_BLOCK_TYPE(CannotMakeSecondBlockLargerInSplit) {
ASSERT_FALSE(result.has_value());
}

TEST_FOR_EACH_BLOCK_TYPE(CannotMakeZeroSizeFirstBlock) {
// This block doesn't support splitting with zero payload size, since the
// prev_ field of the next block is always available.
constexpr size_t kN = 1024;

alignas(BlockType::ALIGNMENT) array<byte, kN> bytes;
auto result = BlockType::init(bytes);
ASSERT_TRUE(result.has_value());
BlockType *block = *result;

result = block->split(0);
EXPECT_FALSE(result.has_value());
}

TEST_FOR_EACH_BLOCK_TYPE(CanMakeMinimalSizeFirstBlock) {
// This block does support splitting with minimal payload size.
constexpr size_t kN = 1024;
Expand Down
53 changes: 28 additions & 25 deletions libc/test/src/__support/freelist_heap_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,9 @@
#include "src/string/memcpy.h"
#include "test/UnitTest/Test.h"

using LIBC_NAMESPACE::Block;
namespace LIBC_NAMESPACE_DECL {

using LIBC_NAMESPACE::freelist_heap;
using LIBC_NAMESPACE::FreeListHeap;
using LIBC_NAMESPACE::FreeListHeapBuffer;
using LIBC_NAMESPACE::cpp::byte;
using LIBC_NAMESPACE::cpp::span;

// Similar to `LlvmLibcBlockTest` in block_test.cpp, we'd like to run the same
// tests independently for different parameters. In this case, we'd like to test
Expand All @@ -31,23 +28,23 @@ using LIBC_NAMESPACE::cpp::span;
// made in tests leak and aren't free'd. This is fine for the purposes of this
// test file.
#define TEST_FOR_EACH_ALLOCATOR(TestCase, BufferSize) \
class LlvmLibcFreeListHeapTest##TestCase \
: public LIBC_NAMESPACE::testing::Test { \
class LlvmLibcFreeListHeapTest##TestCase : public testing::Test { \
public: \
FreeListHeapBuffer<BufferSize> fake_global_buffer; \
void SetUp() override { \
freelist_heap = \
new (&fake_global_buffer) FreeListHeapBuffer<BufferSize>; \
} \
void RunTest(FreeListHeap &allocator, [[maybe_unused]] size_t N); \
void RunTest(FreeListHeap<> &allocator, [[maybe_unused]] size_t N); \
}; \
TEST_F(LlvmLibcFreeListHeapTest##TestCase, TestCase) { \
alignas(Block<>) byte buf[BufferSize] = {byte(0)}; \
FreeListHeap allocator(buf); \
alignas(FreeListHeap<>::BlockType) \
cpp::byte buf[BufferSize] = {cpp::byte(0)}; \
FreeListHeap<> allocator(buf); \
RunTest(allocator, BufferSize); \
RunTest(*freelist_heap, freelist_heap->region().size()); \
} \
void LlvmLibcFreeListHeapTest##TestCase::RunTest(FreeListHeap &allocator, \
void LlvmLibcFreeListHeapTest##TestCase::RunTest(FreeListHeap<> &allocator, \
size_t N)

TEST_FOR_EACH_ALLOCATOR(CanAllocate, 2048) {
Expand Down Expand Up @@ -95,13 +92,14 @@ TEST_FOR_EACH_ALLOCATOR(ReturnsNullWhenAllocationTooLarge, 2048) {
// is used for other test cases and we don't explicitly free them.
TEST(LlvmLibcFreeListHeap, ReturnsNullWhenFull) {
constexpr size_t N = 2048;
alignas(Block<>) byte buf[N] = {byte(0)};
alignas(FreeListHeap<>::BlockType) cpp::byte buf[N] = {cpp::byte(0)};

FreeListHeap allocator(buf);
FreeListHeap<> allocator(buf);

// Use aligned_allocate so we don't need to worry about ensuring the `buf`
// being aligned to max_align_t.
EXPECT_NE(allocator.aligned_allocate(1, N - 2 * Block<>::BLOCK_OVERHEAD),
EXPECT_NE(allocator.aligned_allocate(
1, N - 2 * FreeListHeap<>::BlockType::BLOCK_OVERHEAD),
static_cast<void *>(nullptr));
EXPECT_EQ(allocator.allocate(1), static_cast<void *>(nullptr));
}
Expand Down Expand Up @@ -136,9 +134,9 @@ TEST_FOR_EACH_ALLOCATOR(ReallocHasSameContent, 2048) {
constexpr size_t ALLOC_SIZE = sizeof(int);
constexpr size_t kNewAllocSize = sizeof(int) * 2;
// Data inside the allocated block.
byte data1[ALLOC_SIZE];
cpp::byte data1[ALLOC_SIZE];
// Data inside the reallocated block.
byte data2[ALLOC_SIZE];
cpp::byte data2[ALLOC_SIZE];

int *ptr1 = reinterpret_cast<int *>(allocator.allocate(ALLOC_SIZE));
*ptr1 = 42;
Expand Down Expand Up @@ -190,9 +188,10 @@ TEST_FOR_EACH_ALLOCATOR(CanCalloc, 2048) {
constexpr size_t ALLOC_SIZE = 128;
constexpr size_t NUM = 4;
constexpr int size = NUM * ALLOC_SIZE;
constexpr byte zero{0};
constexpr cpp::byte zero{0};

byte *ptr1 = reinterpret_cast<byte *>(allocator.calloc(NUM, ALLOC_SIZE));
cpp::byte *ptr1 =
reinterpret_cast<cpp::byte *>(allocator.calloc(NUM, ALLOC_SIZE));

// calloc'd content is zero.
for (int i = 0; i < size; i++) {
Expand All @@ -204,9 +203,10 @@ TEST_FOR_EACH_ALLOCATOR(CanCallocWeirdSize, 2048) {
constexpr size_t ALLOC_SIZE = 143;
constexpr size_t NUM = 3;
constexpr int size = NUM * ALLOC_SIZE;
constexpr byte zero{0};
constexpr cpp::byte zero{0};

byte *ptr1 = reinterpret_cast<byte *>(allocator.calloc(NUM, ALLOC_SIZE));
cpp::byte *ptr1 =
reinterpret_cast<cpp::byte *>(allocator.calloc(NUM, ALLOC_SIZE));

// calloc'd content is zero.
for (int i = 0; i < size; i++) {
Expand Down Expand Up @@ -241,16 +241,17 @@ TEST_FOR_EACH_ALLOCATOR(AlignedAlloc, 2048) {

// This test is not part of the TEST_FOR_EACH_ALLOCATOR since we want to
// explicitly ensure that the buffer can still return aligned allocations even
// if the underlying buffer is at most aligned to the Block<> alignment. This
// if the underlying buffer is at most aligned to the BlockType alignment. This
// is so we can check that we can still get aligned allocations even if the
// underlying buffer is not aligned to the alignments we request.
TEST(LlvmLibcFreeListHeap, AlignedAllocOnlyBlockAligned) {
TEST(LlvmLibcFreeListHeap, AlignedAllocOnlyBlockTypeAligned) {
constexpr size_t BUFFER_SIZE = 4096;
constexpr size_t BUFFER_ALIGNMENT = alignof(Block<>) * 2;
alignas(BUFFER_ALIGNMENT) byte buf[BUFFER_SIZE] = {byte(0)};
constexpr size_t BUFFER_ALIGNMENT = alignof(FreeListHeap<>::BlockType) * 2;
alignas(BUFFER_ALIGNMENT) cpp::byte buf[BUFFER_SIZE] = {cpp::byte(0)};

// Ensure the underlying buffer is at most aligned to the block type.
FreeListHeap allocator(span<byte>(buf).subspan(alignof(Block<>)));
FreeListHeap<> allocator(
span<cpp::byte>(buf).subspan(alignof(FreeListHeap<>::BlockType)));

constexpr size_t ALIGNMENTS[] = {1, 2, 4, 8, 16, 32, 64, 128, 256};
constexpr size_t SIZE_SCALES[] = {1, 2, 3, 4, 5};
Expand Down Expand Up @@ -288,3 +289,5 @@ TEST_FOR_EACH_ALLOCATOR(InvalidAlignedAllocAlignment, 2048) {
ptr = allocator.aligned_allocate(0, 8);
EXPECT_EQ(ptr, static_cast<void *>(nullptr));
}

} // namespace LIBC_NAMESPACE_DECL
11 changes: 6 additions & 5 deletions libc/test/src/__support/freelist_malloc_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
#include "src/stdlib/malloc.h"
#include "test/UnitTest/Test.h"

using LIBC_NAMESPACE::Block;
using LIBC_NAMESPACE::freelist_heap;
using LIBC_NAMESPACE::FreeListHeap;
using LIBC_NAMESPACE::FreeListHeapBuffer;
Expand All @@ -23,13 +22,15 @@ TEST(LlvmLibcFreeListMalloc, Malloc) {
constexpr size_t kCallocNum = 4;
constexpr size_t kCallocSize = 64;

typedef FreeListHeap<>::BlockType Block;

void *ptr1 = LIBC_NAMESPACE::malloc(kAllocSize);
auto *block = Block<>::from_usable_space(ptr1);
auto *block = Block::from_usable_space(ptr1);
EXPECT_GE(block->inner_size(), kAllocSize);

LIBC_NAMESPACE::free(ptr1);
ASSERT_NE(block->next(), static_cast<Block<> *>(nullptr));
ASSERT_EQ(block->next()->next(), static_cast<Block<> *>(nullptr));
ASSERT_NE(block->next(), static_cast<Block *>(nullptr));
ASSERT_EQ(block->next()->next(), static_cast<Block *>(nullptr));
size_t heap_size = block->inner_size();

void *ptr2 = LIBC_NAMESPACE::calloc(kCallocNum, kCallocSize);
Expand All @@ -46,7 +47,7 @@ TEST(LlvmLibcFreeListMalloc, Malloc) {
void *ptr3 = LIBC_NAMESPACE::aligned_alloc(ALIGN, kAllocSize);
EXPECT_NE(ptr3, static_cast<void *>(nullptr));
EXPECT_EQ(reinterpret_cast<uintptr_t>(ptr3) % ALIGN, size_t(0));
auto *aligned_block = reinterpret_cast<Block<> *>(ptr3);
auto *aligned_block = reinterpret_cast<Block *>(ptr3);
EXPECT_GE(aligned_block->inner_size(), kAllocSize);

LIBC_NAMESPACE::free(ptr3);
Expand Down
187 changes: 150 additions & 37 deletions libc/test/src/__support/freelist_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,46 +8,159 @@

#include <stddef.h>

#include "src/__support/CPP/array.h"
#include "src/__support/CPP/span.h"
#include "src/__support/freelist.h"
#include "test/UnitTest/Test.h"

using LIBC_NAMESPACE::Block;
using LIBC_NAMESPACE::FreeList;
using LIBC_NAMESPACE::cpp::array;
using LIBC_NAMESPACE::cpp::byte;
using LIBC_NAMESPACE::cpp::optional;

TEST(LlvmLibcFreeList, FreeList) {
byte mem[1024];
optional<Block<> *> maybeBlock = Block<>::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
Block<> *block1 = *maybeBlock;

maybeBlock = block1->split(128);
ASSERT_TRUE(maybeBlock.has_value());
Block<> *block2 = *maybeBlock;

maybeBlock = block2->split(128);
ASSERT_TRUE(maybeBlock.has_value());

FreeList list;
list.push(block1);
ASSERT_FALSE(list.empty());
EXPECT_EQ(list.front(), block1);

list.push(block2);
EXPECT_EQ(list.front(), block1);

list.pop();
ASSERT_FALSE(list.empty());
EXPECT_EQ(list.front(), block2);

list.pop();
ASSERT_TRUE(list.empty());

list.push(block1);
list.push(block2);
list.remove(reinterpret_cast<FreeList::Node *>(block2->usable_space()));
EXPECT_EQ(list.front(), block1);
list.pop();
ASSERT_TRUE(list.empty());
using LIBC_NAMESPACE::cpp::span;

static constexpr size_t SIZE = 8;
static constexpr array<size_t, SIZE> example_sizes = {64, 128, 256, 512,
1024, 2048, 4096, 8192};

TEST(LlvmLibcFreeList, EmptyListHasNoMembers) {
FreeList<SIZE> list(example_sizes);

auto item = list.find_chunk(4);
EXPECT_EQ(item.size(), static_cast<size_t>(0));
item = list.find_chunk(128);
EXPECT_EQ(item.size(), static_cast<size_t>(0));
}

TEST(LlvmLibcFreeList, CanRetrieveAddedMember) {
FreeList<SIZE> list(example_sizes);
constexpr size_t N = 512;

byte data[N] = {byte(0)};

bool ok = list.add_chunk(span<byte>(data, N));
EXPECT_TRUE(ok);

auto item = list.find_chunk(N);
EXPECT_EQ(item.size(), N);
EXPECT_EQ(item.data(), data);
}

TEST(LlvmLibcFreeList, CanRetrieveAddedMemberForSmallerSize) {
FreeList<SIZE> list(example_sizes);
constexpr size_t N = 512;

byte data[N] = {byte(0)};

ASSERT_TRUE(list.add_chunk(span<byte>(data, N)));
auto item = list.find_chunk(N / 2);
EXPECT_EQ(item.size(), N);
EXPECT_EQ(item.data(), data);
}

TEST(LlvmLibcFreeList, CanRemoveItem) {
FreeList<SIZE> list(example_sizes);
constexpr size_t N = 512;

byte data[N] = {byte(0)};

ASSERT_TRUE(list.add_chunk(span<byte>(data, N)));
EXPECT_TRUE(list.remove_chunk(span<byte>(data, N)));

auto item = list.find_chunk(N);
EXPECT_EQ(item.size(), static_cast<size_t>(0));
}

TEST(LlvmLibcFreeList, FindReturnsSmallestChunk) {
FreeList<SIZE> list(example_sizes);
constexpr size_t kN1 = 512;
constexpr size_t kN2 = 1024;

byte data1[kN1] = {byte(0)};
byte data2[kN2] = {byte(0)};

ASSERT_TRUE(list.add_chunk(span<byte>(data1, kN1)));
ASSERT_TRUE(list.add_chunk(span<byte>(data2, kN2)));

auto chunk = list.find_chunk(kN1 / 2);
EXPECT_EQ(chunk.size(), kN1);
EXPECT_EQ(chunk.data(), data1);

chunk = list.find_chunk(kN1);
EXPECT_EQ(chunk.size(), kN1);
EXPECT_EQ(chunk.data(), data1);

chunk = list.find_chunk(kN1 + 1);
EXPECT_EQ(chunk.size(), kN2);
EXPECT_EQ(chunk.data(), data2);
}

TEST(LlvmLibcFreeList, FindReturnsCorrectChunkInSameBucket) {
// If we have two values in the same bucket, ensure that the allocation will
// pick an appropriately sized one.
FreeList<SIZE> list(example_sizes);
constexpr size_t kN1 = 512;
constexpr size_t kN2 = 257;

byte data1[kN1] = {byte(0)};
byte data2[kN2] = {byte(0)};

// List should now be 257 -> 512 -> NULL
ASSERT_TRUE(list.add_chunk(span<byte>(data1, kN1)));
ASSERT_TRUE(list.add_chunk(span<byte>(data2, kN2)));

auto chunk = list.find_chunk(kN2 + 1);
EXPECT_EQ(chunk.size(), kN1);
}

TEST(LlvmLibcFreeList, FindCanMoveUpThroughBuckets) {
// Ensure that finding a chunk will move up through buckets if no appropriate
// chunks were found in a given bucket
FreeList<SIZE> list(example_sizes);
constexpr size_t kN1 = 257;
constexpr size_t kN2 = 513;

byte data1[kN1] = {byte(0)};
byte data2[kN2] = {byte(0)};

// List should now be:
// bkt[3] (257 bytes up to 512 bytes) -> 257 -> NULL
// bkt[4] (513 bytes up to 1024 bytes) -> 513 -> NULL
ASSERT_TRUE(list.add_chunk(span<byte>(data1, kN1)));
ASSERT_TRUE(list.add_chunk(span<byte>(data2, kN2)));

// Request a 300 byte chunk. This should return the 513 byte one
auto chunk = list.find_chunk(kN1 + 1);
EXPECT_EQ(chunk.size(), kN2);
}

TEST(LlvmLibcFreeList, RemoveUnknownChunkReturnsNotFound) {
FreeList<SIZE> list(example_sizes);
constexpr size_t N = 512;

byte data[N] = {byte(0)};
byte data2[N] = {byte(0)};

ASSERT_TRUE(list.add_chunk(span<byte>(data, N)));
EXPECT_FALSE(list.remove_chunk(span<byte>(data2, N)));
}

TEST(LlvmLibcFreeList, CanStoreMultipleChunksPerBucket) {
FreeList<SIZE> list(example_sizes);
constexpr size_t N = 512;

byte data1[N] = {byte(0)};
byte data2[N] = {byte(0)};

ASSERT_TRUE(list.add_chunk(span<byte>(data1, N)));
ASSERT_TRUE(list.add_chunk(span<byte>(data2, N)));

auto chunk1 = list.find_chunk(N);
ASSERT_TRUE(list.remove_chunk(chunk1));
auto chunk2 = list.find_chunk(N);
ASSERT_TRUE(list.remove_chunk(chunk2));

// Ordering of the chunks doesn't matter
EXPECT_TRUE(chunk1.data() != chunk2.data());
EXPECT_TRUE(chunk1.data() == data1 || chunk1.data() == data2);
EXPECT_TRUE(chunk2.data() == data1 || chunk2.data() == data2);
}
103 changes: 0 additions & 103 deletions libc/test/src/__support/freestore_test.cpp
View file

This file was deleted.

125 changes: 0 additions & 125 deletions libc/test/src/__support/freetrie_test.cpp
View file

This file was deleted.