228 changes: 54 additions & 174 deletions libc/src/__support/freelist.h
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//===-- Interface for freelist_malloc -------------------------------------===//
//===-- Interface for freelist --------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
Expand All @@ -9,200 +9,80 @@
#ifndef LLVM_LIBC_SRC___SUPPORT_FREELIST_H
#define LLVM_LIBC_SRC___SUPPORT_FREELIST_H

#include "src/__support/CPP/array.h"
#include "src/__support/CPP/cstddef.h"
#include "src/__support/CPP/new.h"
#include "src/__support/CPP/span.h"
#include "src/__support/fixedvector.h"
#include "src/__support/macros/config.h"
#include "block.h"

namespace LIBC_NAMESPACE_DECL {

using cpp::span;

/// Basic [freelist](https://en.wikipedia.org/wiki/Free_list) implementation
/// for an allocator. This implementation buckets by chunk size, with a list
/// of user-provided buckets. Each bucket is a linked list of storage chunks.
/// Because this freelist uses the added chunks themselves as list nodes, there
/// is a lower bound of `sizeof(FreeList.FreeListNode)` bytes for chunks which
/// can be added to this freelist. There is also an implicit bucket for
/// "everything else", for chunks which do not fit into a bucket.
///
/// Each added chunk will be added to the smallest bucket under which it fits.
/// If it does not fit into any user-provided bucket, it will be added to the
/// default bucket.
///
/// As an example, assume that the `FreeList` is configured with buckets of
/// sizes {64, 128, 256, and 512} bytes. The internal state may look like the
/// following:
/// A circularly-linked FIFO list storing free Blocks. All Blocks on a list
/// are the same size. The blocks are referenced by Nodes in the list; the list
/// refers to these, but it does not own them.
///
/// @code{.unparsed}
/// bucket[0] (64B) --> chunk[12B] --> chunk[42B] --> chunk[64B] --> NULL
/// bucket[1] (128B) --> chunk[65B] --> chunk[72B] --> NULL
/// bucket[2] (256B) --> NULL
/// bucket[3] (512B) --> chunk[312B] --> chunk[512B] --> chunk[416B] --> NULL
/// bucket[4] (implicit) --> chunk[1024B] --> chunk[513B] --> NULL
/// @endcode
///
/// Note that added chunks should be aligned to a 4-byte boundary.
template <size_t NUM_BUCKETS = 6> class FreeList {
/// Allocating free blocks in FIFO order maximizes the amount of time before a
/// free block is reused. This in turn maximizes the number of opportunities for
/// it to be coalesced with an adjacent block, which tends to reduce heap
/// fragmentation.
class FreeList {
public:
// Remove copy/move ctors
FreeList(const FreeList &other) = delete;
FreeList(FreeList &&other) = delete;
FreeList &operator=(const FreeList &other) = delete;
FreeList &operator=(FreeList &&other) = delete;

/// Adds a chunk to this freelist.
bool add_chunk(cpp::span<cpp::byte> chunk);

/// Finds an eligible chunk for an allocation of size `size`.
///
/// @note This returns the first allocation possible within a given bucket;
/// It does not currently optimize for finding the smallest chunk.
///
/// @returns
/// * On success - A span representing the chunk.
/// * On failure (e.g. there were no chunks available for that allocation) -
/// A span with a size of 0.
cpp::span<cpp::byte> find_chunk(size_t size) const;

template <typename Cond> cpp::span<cpp::byte> find_chunk_if(Cond op) const;

/// Removes a chunk from this freelist.
bool remove_chunk(cpp::span<cpp::byte> chunk);

/// For a given size, find which index into chunks_ the node should be written
/// to.
constexpr size_t find_chunk_ptr_for_size(size_t size, bool non_null) const;

struct FreeListNode {
FreeListNode *next;
size_t size;
};

constexpr void set_freelist_node(FreeListNode &node,
cpp::span<cpp::byte> chunk);

constexpr explicit FreeList(const cpp::array<size_t, NUM_BUCKETS> &sizes)
: chunks_(NUM_BUCKETS + 1, 0), sizes_(sizes.begin(), sizes.end()) {}

private:
FixedVector<FreeList::FreeListNode *, NUM_BUCKETS + 1> chunks_;
FixedVector<size_t, NUM_BUCKETS> sizes_;
};

template <size_t NUM_BUCKETS>
constexpr void FreeList<NUM_BUCKETS>::set_freelist_node(FreeListNode &node,
span<cpp::byte> chunk) {
// Add it to the correct list.
size_t chunk_ptr = find_chunk_ptr_for_size(chunk.size(), false);
node.size = chunk.size();
node.next = chunks_[chunk_ptr];
chunks_[chunk_ptr] = &node;
}

template <size_t NUM_BUCKETS>
bool FreeList<NUM_BUCKETS>::add_chunk(span<cpp::byte> chunk) {
// Check that the size is enough to actually store what we need
if (chunk.size() < sizeof(FreeListNode))
return false;

FreeListNode *node = ::new (chunk.data()) FreeListNode;
set_freelist_node(*node, chunk);

return true;
}

template <size_t NUM_BUCKETS>
template <typename Cond>
span<cpp::byte> FreeList<NUM_BUCKETS>::find_chunk_if(Cond op) const {
for (FreeListNode *node : chunks_) {
while (node != nullptr) {
span<cpp::byte> chunk(reinterpret_cast<cpp::byte *>(node), node->size);
if (op(chunk))
return chunk;

node = node->next;
class Node {
public:
/// @returns The block containing this node.
LIBC_INLINE const Block<> *block() const {
return Block<>::from_usable_space(this);
}
}

return {};
}
/// @returns The block containing this node.
LIBC_INLINE Block<> *block() { return Block<>::from_usable_space(this); }

template <size_t NUM_BUCKETS>
span<cpp::byte> FreeList<NUM_BUCKETS>::find_chunk(size_t size) const {
if (size == 0)
return span<cpp::byte>();
/// @returns The inner size of blocks in the list containing this node.
LIBC_INLINE size_t size() const { return block()->inner_size(); }

size_t chunk_ptr = find_chunk_ptr_for_size(size, true);

// Check that there's data. This catches the case where we run off the
// end of the array
if (chunks_[chunk_ptr] == nullptr)
return span<cpp::byte>();
private:
// Circularly linked pointers to adjacent nodes.
Node *prev;
Node *next;
friend class FreeList;
};

// Now iterate up the buckets, walking each list to find a good candidate
for (size_t i = chunk_ptr; i < chunks_.size(); i++) {
FreeListNode *node = chunks_[static_cast<unsigned short>(i)];
LIBC_INLINE constexpr FreeList() : FreeList(nullptr) {}
LIBC_INLINE constexpr FreeList(Node *begin) : begin_(begin) {}

while (node != nullptr) {
if (node->size >= size)
return span<cpp::byte>(reinterpret_cast<cpp::byte *>(node), node->size);
LIBC_INLINE bool empty() const { return !begin_; }

node = node->next;
}
/// @returns The inner size of blocks in the list.
LIBC_INLINE size_t size() const {
LIBC_ASSERT(begin_ && "empty lists have no size");
return begin_->size();
}

// If we get here, we've checked every block in every bucket. There's
// nothing that can support this allocation.
return span<cpp::byte>();
}
/// @returns The first node in the list.
LIBC_INLINE Node *begin() { return begin_; }

template <size_t NUM_BUCKETS>
bool FreeList<NUM_BUCKETS>::remove_chunk(span<cpp::byte> chunk) {
size_t chunk_ptr = find_chunk_ptr_for_size(chunk.size(), true);
/// @returns The first block in the list.
LIBC_INLINE Block<> *front() { return begin_->block(); }

// Check head first.
if (chunks_[chunk_ptr] == nullptr)
return false;

FreeListNode *node = chunks_[chunk_ptr];
if (reinterpret_cast<cpp::byte *>(node) == chunk.data()) {
chunks_[chunk_ptr] = node->next;
return true;
/// Push a block to the back of the list.
/// The block must be large enough to contain a node.
LIBC_INLINE void push(Block<> *block) {
LIBC_ASSERT(!block->used() &&
"only free blocks can be placed on free lists");
LIBC_ASSERT(block->inner_size_free() >= sizeof(FreeList) &&
"block too small to accomodate free list node");
push(new (block->usable_space()) Node);
}

// No? Walk the nodes.
node = chunks_[chunk_ptr];
/// Push an already-constructed node to the back of the list.
/// This allows pushing derived node types with additional data.
void push(Node *node);

while (node->next != nullptr) {
if (reinterpret_cast<cpp::byte *>(node->next) == chunk.data()) {
// Found it, remove this node out of the chain
node->next = node->next->next;
return true;
}
/// Pop the first node from the list.
LIBC_INLINE void pop() { remove(begin_); }

node = node->next;
}
/// Remove an arbitrary node from the list.
void remove(Node *node);

return false;
}

template <size_t NUM_BUCKETS>
constexpr size_t
FreeList<NUM_BUCKETS>::find_chunk_ptr_for_size(size_t size,
bool non_null) const {
size_t chunk_ptr = 0;
for (chunk_ptr = 0u; chunk_ptr < sizes_.size(); chunk_ptr++) {
if (sizes_[chunk_ptr] >= size &&
(!non_null || chunks_[chunk_ptr] != nullptr)) {
break;
}
}

return chunk_ptr;
}
private:
Node *begin_;
};

} // namespace LIBC_NAMESPACE_DECL

Expand Down
153 changes: 68 additions & 85 deletions libc/src/__support/freelist_heap.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,12 @@
#include <stddef.h>

#include "block.h"
#include "freelist.h"
#include "freestore.h"
#include "src/__support/CPP/optional.h"
#include "src/__support/CPP/span.h"
#include "src/__support/libc_assert.h"
#include "src/__support/macros/config.h"
#include "src/__support/math_extras.h"
#include "src/string/memory_utils/inline_memcpy.h"
#include "src/string/memory_utils/inline_memset.h"

Expand All @@ -28,23 +29,14 @@ extern "C" cpp::byte __llvm_libc_heap_limit;
using cpp::optional;
using cpp::span;

inline constexpr bool IsPow2(size_t x) { return x && (x & (x - 1)) == 0; }
LIBC_INLINE constexpr bool IsPow2(size_t x) { return x && (x & (x - 1)) == 0; }

static constexpr cpp::array<size_t, 6> DEFAULT_BUCKETS{16, 32, 64,
128, 256, 512};

template <size_t NUM_BUCKETS = DEFAULT_BUCKETS.size()> class FreeListHeap {
class FreeListHeap {
public:
using BlockType = Block<>;
using FreeListType = FreeList<NUM_BUCKETS>;

static constexpr size_t MIN_ALIGNMENT =
cpp::max(BlockType::ALIGNMENT, alignof(max_align_t));

constexpr FreeListHeap() : begin_(&_end), end_(&__llvm_libc_heap_limit) {}
constexpr FreeListHeap() : begin(&_end), end(&__llvm_libc_heap_limit) {}

constexpr FreeListHeap(span<cpp::byte> region)
: begin_(region.begin()), end_(region.end()) {}
: begin(region.begin()), end(region.end()) {}

void *allocate(size_t size);
void *aligned_allocate(size_t alignment, size_t size);
Expand All @@ -54,89 +46,79 @@ template <size_t NUM_BUCKETS = DEFAULT_BUCKETS.size()> class FreeListHeap {
void *realloc(void *ptr, size_t size);
void *calloc(size_t num, size_t size);

cpp::span<cpp::byte> region() const { return {begin_, end_}; }
cpp::span<cpp::byte> region() const { return {begin, end}; }

private:
void init();

void *allocate_impl(size_t alignment, size_t size);

span<cpp::byte> block_to_span(BlockType *block) {
span<cpp::byte> block_to_span(Block<> *block) {
return span<cpp::byte>(block->usable_space(), block->inner_size());
}

bool is_valid_ptr(void *ptr) { return ptr >= begin_ && ptr < end_; }
bool is_valid_ptr(void *ptr) { return ptr >= begin && ptr < end; }

bool is_initialized_ = false;
cpp::byte *begin_;
cpp::byte *end_;
FreeListType freelist_{DEFAULT_BUCKETS};
cpp::byte *begin;
cpp::byte *end;
bool is_initialized = false;
FreeStore free_store;
};

template <size_t BUFF_SIZE, size_t NUM_BUCKETS = DEFAULT_BUCKETS.size()>
class FreeListHeapBuffer : public FreeListHeap<NUM_BUCKETS> {
using parent = FreeListHeap<NUM_BUCKETS>;
using FreeListNode = typename parent::FreeListType::FreeListNode;

template <size_t BUFF_SIZE> class FreeListHeapBuffer : public FreeListHeap {
public:
constexpr FreeListHeapBuffer()
: FreeListHeap<NUM_BUCKETS>{buffer}, buffer{} {}
constexpr FreeListHeapBuffer() : FreeListHeap{buffer}, buffer{} {}

private:
cpp::byte buffer[BUFF_SIZE];
};

template <size_t NUM_BUCKETS> void FreeListHeap<NUM_BUCKETS>::init() {
LIBC_ASSERT(!is_initialized_ && "duplicate initialization");
auto result = BlockType::init(region());
BlockType *block = *result;
freelist_.add_chunk(block_to_span(block));
is_initialized_ = true;
LIBC_INLINE void FreeListHeap::init() {
LIBC_ASSERT(!is_initialized && "duplicate initialization");
auto result = Block<>::init(region());
Block<> *block = *result;
free_store.set_range({0, cpp::bit_ceil(block->inner_size())});
free_store.insert(block);
is_initialized = true;
}

template <size_t NUM_BUCKETS>
void *FreeListHeap<NUM_BUCKETS>::allocate_impl(size_t alignment, size_t size) {
LIBC_INLINE void *FreeListHeap::allocate_impl(size_t alignment, size_t size) {
if (size == 0)
return nullptr;

if (!is_initialized_)
if (!is_initialized)
init();

// Find a chunk in the freelist. Split it if needed, then return.
auto chunk =
freelist_.find_chunk_if([alignment, size](span<cpp::byte> chunk) {
BlockType *block = BlockType::from_usable_space(chunk.data());
return block->can_allocate(alignment, size);
});
size_t request_size = size;
if (alignment > alignof(max_align_t)) {
if (add_overflow(size, alignment - 1, request_size))
return nullptr;
}

if (chunk.data() == nullptr)
Block<> *block = free_store.remove_best_fit(request_size);
if (!block)
return nullptr;
freelist_.remove_chunk(chunk);

BlockType *chunk_block = BlockType::from_usable_space(chunk.data());
LIBC_ASSERT(!chunk_block->used());
LIBC_ASSERT(block->can_allocate(alignment, size) &&
"block should always be large enough to allocate at the correct "
"alignment");

// Split that chunk. If there's a leftover chunk, add it to the freelist
auto block_info = BlockType::allocate(chunk_block, alignment, size);
auto block_info = Block<>::allocate(block, alignment, size);
if (block_info.next)
freelist_.add_chunk(block_to_span(block_info.next));
free_store.insert(block_info.next);
if (block_info.prev)
freelist_.add_chunk(block_to_span(block_info.prev));
chunk_block = block_info.block;

chunk_block->mark_used();
free_store.insert(block_info.prev);

return chunk_block->usable_space();
block_info.block->mark_used();
return block_info.block->usable_space();
}

template <size_t NUM_BUCKETS>
void *FreeListHeap<NUM_BUCKETS>::allocate(size_t size) {
return allocate_impl(MIN_ALIGNMENT, size);
LIBC_INLINE void *FreeListHeap::allocate(size_t size) {
return allocate_impl(alignof(max_align_t), size);
}

template <size_t NUM_BUCKETS>
void *FreeListHeap<NUM_BUCKETS>::aligned_allocate(size_t alignment,
size_t size) {
LIBC_INLINE void *FreeListHeap::aligned_allocate(size_t alignment,
size_t size) {
// The alignment must be an integral power of two.
if (!IsPow2(alignment))
return nullptr;
Expand All @@ -148,38 +130,37 @@ void *FreeListHeap<NUM_BUCKETS>::aligned_allocate(size_t alignment,
return allocate_impl(alignment, size);
}

template <size_t NUM_BUCKETS> void FreeListHeap<NUM_BUCKETS>::free(void *ptr) {
LIBC_INLINE void FreeListHeap::free(void *ptr) {
cpp::byte *bytes = static_cast<cpp::byte *>(ptr);

LIBC_ASSERT(is_valid_ptr(bytes) && "Invalid pointer");

BlockType *chunk_block = BlockType::from_usable_space(bytes);
LIBC_ASSERT(chunk_block->next() && "sentinel last block cannot be freed");
LIBC_ASSERT(chunk_block->used() && "The block is not in-use");
chunk_block->mark_free();
Block<> *block = Block<>::from_usable_space(bytes);
LIBC_ASSERT(block->next() && "sentinel last block cannot be freed");
LIBC_ASSERT(block->used() && "double free");
block->mark_free();

// Can we combine with the left or right blocks?
BlockType *prev_free = chunk_block->prev_free();
BlockType *next = chunk_block->next();
Block<> *prev_free = block->prev_free();
Block<> *next = block->next();

if (prev_free != nullptr) {
// Remove from freelist and merge
freelist_.remove_chunk(block_to_span(prev_free));
chunk_block = prev_free;
chunk_block->merge_next();
// Remove from free store and merge.
free_store.remove(prev_free);
block = prev_free;
block->merge_next();
}
if (!next->used()) {
freelist_.remove_chunk(block_to_span(next));
chunk_block->merge_next();
free_store.remove(next);
block->merge_next();
}
// Add back to the freelist
freelist_.add_chunk(block_to_span(chunk_block));
free_store.insert(block);
}

// Follows constract of the C standard realloc() function
// If ptr is free'd, will return nullptr.
template <size_t NUM_BUCKETS>
void *FreeListHeap<NUM_BUCKETS>::realloc(void *ptr, size_t size) {
LIBC_INLINE void *FreeListHeap::realloc(void *ptr, size_t size) {
if (size == 0) {
free(ptr);
return nullptr;
Expand All @@ -194,10 +175,10 @@ void *FreeListHeap<NUM_BUCKETS>::realloc(void *ptr, size_t size) {
if (!is_valid_ptr(bytes))
return nullptr;

BlockType *chunk_block = BlockType::from_usable_space(bytes);
if (!chunk_block->used())
Block<> *block = Block<>::from_usable_space(bytes);
if (!block->used())
return nullptr;
size_t old_size = chunk_block->inner_size();
size_t old_size = block->inner_size();

// Do nothing and return ptr if the required memory size is smaller than
// the current size.
Expand All @@ -214,15 +195,17 @@ void *FreeListHeap<NUM_BUCKETS>::realloc(void *ptr, size_t size) {
return new_ptr;
}

template <size_t NUM_BUCKETS>
void *FreeListHeap<NUM_BUCKETS>::calloc(size_t num, size_t size) {
void *ptr = allocate(num * size);
LIBC_INLINE void *FreeListHeap::calloc(size_t num, size_t size) {
size_t bytes;
if (__builtin_mul_overflow(num, size, &bytes))
return nullptr;
void *ptr = allocate(bytes);
if (ptr != nullptr)
LIBC_NAMESPACE::inline_memset(ptr, 0, num * size);
LIBC_NAMESPACE::inline_memset(ptr, 0, bytes);
return ptr;
}

extern FreeListHeap<> *freelist_heap;
extern FreeListHeap *freelist_heap;

} // namespace LIBC_NAMESPACE_DECL

Expand Down
114 changes: 114 additions & 0 deletions libc/src/__support/freestore.h
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
//===-- Interface for freestore ------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#ifndef LLVM_LIBC_SRC___SUPPORT_FREESTORE_H
#define LLVM_LIBC_SRC___SUPPORT_FREESTORE_H

#include "freetrie.h"

namespace LIBC_NAMESPACE_DECL {

/// A best-fit store of variously-sized free blocks. Blocks can be inserted and
/// removed in logarithmic time.
class FreeStore {
public:
FreeStore() = default;
FreeStore(const FreeStore &other) = delete;
FreeStore &operator=(const FreeStore &other) = delete;

/// Sets the range of possible block sizes. This can only be called when the
/// trie is empty.
LIBC_INLINE void set_range(FreeTrie::SizeRange range) {
large_trie.set_range(range);
}

/// Insert a free block. If the block is too small to be tracked, nothing
/// happens.
void insert(Block<> *block);

/// Remove a free block. If the block is too small to be tracked, nothing
/// happens.
void remove(Block<> *block);

/// Remove a best-fit free block that can contain the given size when
/// allocated. Returns nullptr if there is no such block.
Block<> *remove_best_fit(size_t size);

private:
static constexpr size_t ALIGNMENT = alignof(max_align_t);
static constexpr size_t MIN_OUTER_SIZE =
align_up(Block<>::BLOCK_OVERHEAD + sizeof(FreeList::Node), ALIGNMENT);
static constexpr size_t MIN_LARGE_OUTER_SIZE =
align_up(Block<>::BLOCK_OVERHEAD + sizeof(FreeTrie::Node), ALIGNMENT);
static constexpr size_t NUM_SMALL_SIZES =
(MIN_LARGE_OUTER_SIZE - MIN_OUTER_SIZE) / ALIGNMENT;

LIBC_INLINE static bool too_small(Block<> *block) {
return block->outer_size() < MIN_OUTER_SIZE;
}
LIBC_INLINE static bool is_small(Block<> *block) {
return block->outer_size() < MIN_LARGE_OUTER_SIZE;
}

FreeList &small_list(Block<> *block);
FreeList *find_best_small_fit(size_t size);

cpp::array<FreeList, NUM_SMALL_SIZES> small_lists;
FreeTrie large_trie;
};

LIBC_INLINE void FreeStore::insert(Block<> *block) {
if (too_small(block))
return;
if (is_small(block))
small_list(block).push(block);
else
large_trie.push(block);
}

LIBC_INLINE void FreeStore::remove(Block<> *block) {
if (too_small(block))
return;
if (is_small(block)) {
small_list(block).remove(
reinterpret_cast<FreeList::Node *>(block->usable_space()));
} else {
large_trie.remove(
reinterpret_cast<FreeTrie::Node *>(block->usable_space()));
}
}

LIBC_INLINE Block<> *FreeStore::remove_best_fit(size_t size) {
if (FreeList *list = find_best_small_fit(size)) {
Block<> *block = list->front();
list->pop();
return block;
}
if (FreeTrie::Node *best_fit = large_trie.find_best_fit(size)) {
Block<> *block = best_fit->block();
large_trie.remove(best_fit);
return block;
}
return nullptr;
}

LIBC_INLINE FreeList &FreeStore::small_list(Block<> *block) {
LIBC_ASSERT(is_small(block) && "only legal for small blocks");
return small_lists[(block->outer_size() - MIN_OUTER_SIZE) / ALIGNMENT];
}

LIBC_INLINE FreeList *FreeStore::find_best_small_fit(size_t size) {
for (FreeList &list : small_lists)
if (!list.empty() && list.size() >= size)
return &list;
return nullptr;
}

} // namespace LIBC_NAMESPACE_DECL

#endif // LLVM_LIBC_SRC___SUPPORT_FREESTORE_H
64 changes: 64 additions & 0 deletions libc/src/__support/freetrie.cpp
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
//===-- Implementation for freetrie ---------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include "freetrie.h"

namespace LIBC_NAMESPACE_DECL {

void FreeTrie::remove(Node *node) {
LIBC_ASSERT(!empty() && "cannot remove from empty trie");
FreeList list = node;
list.pop();
Node *new_node = static_cast<Node *>(list.begin());
if (!new_node) {
// The freelist is empty. Replace the subtrie root with an arbitrary leaf.
// This is legal because there is no relationship between the size of the
// root and its children.
Node *leaf = node;
while (leaf->lower || leaf->upper)
leaf = leaf->lower ? leaf->lower : leaf->upper;
if (leaf == node) {
// If the root is a leaf, then removing it empties the subtrie.
replace_node(node, nullptr);
return;
}

replace_node(leaf, nullptr);
new_node = leaf;
}

if (!is_head(node))
return;

// Copy the trie links to the new head.
new_node->lower = node->lower;
new_node->upper = node->upper;
new_node->parent = node->parent;
replace_node(node, new_node);
}

void FreeTrie::replace_node(Node *node, Node *new_node) {
LIBC_ASSERT(is_head(node) && "only head nodes contain trie links");

if (node->parent) {
Node *&parent_child =
node->parent->lower == node ? node->parent->lower : node->parent->upper;
LIBC_ASSERT(parent_child == node &&
"no reference to child node found in parent");
parent_child = new_node;
} else {
LIBC_ASSERT(root == node && "non-root node had no parent");
root = new_node;
}
if (node->lower)
node->lower->parent = new_node;
if (node->upper)
node->upper->parent = new_node;
}

} // namespace LIBC_NAMESPACE_DECL
237 changes: 237 additions & 0 deletions libc/src/__support/freetrie.h
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,237 @@
//===-- Interface for freetrie --------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#ifndef LLVM_LIBC_SRC___SUPPORT_FREETRIE_H
#define LLVM_LIBC_SRC___SUPPORT_FREETRIE_H

#include "freelist.h"

namespace LIBC_NAMESPACE_DECL {

/// A trie of free lists.
///
/// This is an unusual little data structure originally from Doug Lea's malloc.
/// Finding the best fit from a set of differently-sized free list typically
/// required some kind of ordered map, and these are typically implemented using
/// a self-balancing binary search tree. Those are notorious for having a
/// relatively large number of special cases, while this trie has relatively
/// few, which helps with code size.
///
/// Operations on the trie are logarithmic not on the number of nodes within it,
/// but rather the fixed range of possible sizes that the trie can contain. This
/// means that the data structure would likely actually perform worse than an
/// e.g. red-black tree, but its implementation is still much simpler.
///
/// Each trie node's children subdivide the range of possible sizes into two
/// halves: a lower and an upper. The node itself holds a free list of some size
/// within its range. This makes it possible to summarily replace any node with
/// any leaf within its subtrie, which makes it very straightforward to remove a
/// node. Insertion is also simple; the only real complexity lies with finding
/// the best fit. This can still be done in logarithmic time with only a few
/// cases to consider.
///
/// The trie refers to, but does not own, the Nodes that comprise it.
class FreeTrie {
public:
/// A trie node that is also a free list. Only the head node of each list is
/// actually part of the trie. The subtrie contains a continous SizeRange of
/// free lists. The lower and upper subtrie's contain the lower and upper half
/// of the subtries range. There is no direct relationship between the size of
/// this node's free list and the contents of the lower and upper subtries.
class Node : public FreeList::Node {
/// The child subtrie covering the lower half of this subtrie's size range.
/// Undefined if this is not the head of the list.
Node *lower;
/// The child subtrie covering the upper half of this subtrie's size range.
/// Undefined if this is not the head of the list.
Node *upper;
/// The parent subtrie. nullptr if this is the root or not the head of the
/// list.
Node *parent;

friend class FreeTrie;
};

/// Power-of-two range of sizes covered by a subtrie.
struct SizeRange {
size_t min;
size_t width;

LIBC_INLINE constexpr SizeRange(size_t min, size_t width)
: min(min), width(width) {
LIBC_ASSERT(!(width & (width - 1)) && "width must be a power of two");
}

/// @returns The lower half of the size range.
LIBC_INLINE SizeRange lower() const { return {min, width / 2}; }

/// @returns The upper half of the size range.
LIBC_INLINE SizeRange upper() const { return {min + width / 2, width / 2}; }

/// @returns The largest size in this range.
LIBC_INLINE size_t max() const { return min + (width - 1); }

/// @returns Whether the range contains the given size.
LIBC_INLINE bool contains(size_t size) const {
return min <= size && size < min + width;
}
};

LIBC_INLINE constexpr FreeTrie() : FreeTrie(SizeRange{0, 0}) {}
LIBC_INLINE constexpr FreeTrie(SizeRange range) : range(range) {}

/// Sets the range of possible block sizes. This can only be called when the
/// trie is empty.
LIBC_INLINE void set_range(FreeTrie::SizeRange range) {
LIBC_ASSERT(empty() && "cannot change the range of a preexisting trie");
this->range = range;
}

/// @returns Whether the trie contains any blocks.
LIBC_INLINE bool empty() const { return !root; }

/// Push a block to the trie.
void push(Block<> *block);

/// Remove a node from this trie node's free list.
void remove(Node *node);

/// @returns A smallest node that can allocate the given size; otherwise
/// nullptr.
Node *find_best_fit(size_t size);

private:
/// @returns Whether a node is the head of its containing freelist.
bool is_head(Node *node) const { return node->parent || node == root; }

/// Replaces references to one node with another (or nullptr) in all adjacent
/// parent and child nodes.
void replace_node(Node *node, Node *new_node);

Node *root = nullptr;
SizeRange range;
};

LIBC_INLINE void FreeTrie::push(Block<> *block) {
LIBC_ASSERT(block->inner_size_free() >= sizeof(Node) &&
"block too small to accomodate free trie node");
size_t size = block->inner_size();
LIBC_ASSERT(range.contains(size) && "requested size out of trie range");

// Find the position in the tree to push to.
Node **cur = &root;
Node *parent = nullptr;
SizeRange cur_range = range;
while (*cur && (*cur)->size() != size) {
LIBC_ASSERT(cur_range.contains(size) && "requested size out of trie range");
parent = *cur;
if (size <= cur_range.lower().max()) {
cur = &(*cur)->lower;
cur_range = cur_range.lower();
} else {
cur = &(*cur)->upper;
cur_range = cur_range.upper();
}
}

Node *node = new (block->usable_space()) Node;
FreeList list = *cur;
if (list.empty()) {
node->parent = parent;
node->lower = node->upper = nullptr;
} else {
node->parent = nullptr;
}
list.push(node);
*cur = static_cast<Node *>(list.begin());
}

LIBC_INLINE FreeTrie::Node *FreeTrie::find_best_fit(size_t size) {
if (empty() || range.max() < size)
return nullptr;

Node *cur = root;
SizeRange cur_range = range;
Node *best_fit = nullptr;
Node *deferred_upper_trie = nullptr;
FreeTrie::SizeRange deferred_upper_range{0, 0};

while (true) {
LIBC_ASSERT(cur_range.contains(cur->size()) &&
"trie node size out of range");
LIBC_ASSERT(cur_range.max() >= size &&
"range could not fit requested size");
LIBC_ASSERT((!best_fit || cur_range.min < best_fit->size()) &&
"range could not contain a best fit");

// If the current node is an exact fit, it is a best fit.
if (cur->size() == size)
return cur;

if (cur->size() > size && (!best_fit || cur->size() < best_fit->size())) {
// The current node is a better fit.
best_fit = cur;

// If there is a deferred upper subtrie, then the current node is
// somewhere in its lower sibling subtrie. That means that the new best
// fit is better than the best fit in the deferred subtrie.
LIBC_ASSERT(
!deferred_upper_trie ||
deferred_upper_range.min > best_fit->size() &&
"deferred upper subtrie should be outclassed by new best fit");
deferred_upper_trie = nullptr;
}

// Determine which subtries might contain the best fit.
bool lower_impossible = !cur->lower || cur_range.lower().max() < size;
bool upper_impossible =
!cur->upper ||
// If every node in the lower trie fits
(!lower_impossible && cur_range.min >= size) ||
// If every node in the upper trie is worse than the current best
(best_fit && cur_range.upper().min >= best_fit->size());

if (lower_impossible && upper_impossible) {
if (!deferred_upper_trie)
return best_fit;
// Scan the deferred upper subtrie and consider whether any element within
// provides a better fit.
//
// This can only ever be reached once. In a deferred upper subtrie, every
// node fits, so the higher of two subtries can never contain a best fit.
cur = deferred_upper_trie;
cur_range = deferred_upper_range;
deferred_upper_trie = nullptr;
continue;
}

if (lower_impossible) {
cur = cur->upper;
cur_range = cur_range.upper();
} else if (upper_impossible) {
cur = cur->lower;
cur_range = cur_range.lower();
} else {
// Both subtries might contain a better fit. Any fit in the lower subtrie
// is better than the any fit in the upper subtrie, so scan the lower
// and return to the upper only if no better fits were found. (Any better
// fit found clears the deferred upper subtrie.)
LIBC_ASSERT(!deferred_upper_trie ||
cur_range.upper().max() < deferred_upper_range.min &&
"old deferred upper subtrie should be outclassed by new");
deferred_upper_trie = cur->upper;
deferred_upper_range = cur_range.upper();
cur = cur->lower;
cur_range = cur_range.lower();
}
}
}

} // namespace LIBC_NAMESPACE_DECL

#endif // LLVM_LIBC_SRC___SUPPORT_FREETRIE_H
4 changes: 2 additions & 2 deletions libc/src/stdlib/freelist_malloc.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@

namespace LIBC_NAMESPACE_DECL {

static LIBC_CONSTINIT FreeListHeap<> freelist_heap_symbols;
FreeListHeap<> *freelist_heap = &freelist_heap_symbols;
static LIBC_CONSTINIT FreeListHeap freelist_heap_symbols;
FreeListHeap *freelist_heap = &freelist_heap_symbols;

LLVM_LIBC_FUNCTION(void *, malloc, (size_t size)) {
return freelist_heap->allocate(size);
Expand Down
27 changes: 27 additions & 0 deletions libc/test/src/__support/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,34 @@ if(NOT LIBC_TARGET_OS_IS_GPU)
DEPENDS
libc.src.__support.CPP.array
libc.src.__support.CPP.span
libc.src.__support.block
libc.src.__support.freelist
)

add_libc_test(
freetrie_test
SUITE
libc-support-tests
SRCS
freetrie_test.cpp
DEPENDS
libc.src.__support.CPP.optional
libc.src.__support.block
libc.src.__support.freetrie
)

add_libc_test(
freestore_test
SUITE
libc-support-tests
SRCS
freestore_test.cpp
DEPENDS
libc.src.__support.CPP.optional
libc.src.__support.block
libc.src.__support.freelist
libc.src.__support.freestore
libc.src.__support.freetrie
)
endif()

Expand Down
14 changes: 0 additions & 14 deletions libc/test/src/__support/block_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -238,20 +238,6 @@ TEST_FOR_EACH_BLOCK_TYPE(CannotMakeSecondBlockLargerInSplit) {
ASSERT_FALSE(result.has_value());
}

TEST_FOR_EACH_BLOCK_TYPE(CannotMakeZeroSizeFirstBlock) {
// This block doesn't support splitting with zero payload size, since the
// prev_ field of the next block is always available.
constexpr size_t kN = 1024;

alignas(BlockType::ALIGNMENT) array<byte, kN> bytes;
auto result = BlockType::init(bytes);
ASSERT_TRUE(result.has_value());
BlockType *block = *result;

result = block->split(0);
EXPECT_FALSE(result.has_value());
}

TEST_FOR_EACH_BLOCK_TYPE(CanMakeMinimalSizeFirstBlock) {
// This block does support splitting with minimal payload size.
constexpr size_t kN = 1024;
Expand Down
53 changes: 25 additions & 28 deletions libc/test/src/__support/freelist_heap_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,12 @@
#include "src/string/memcpy.h"
#include "test/UnitTest/Test.h"

namespace LIBC_NAMESPACE_DECL {

using LIBC_NAMESPACE::Block;
using LIBC_NAMESPACE::freelist_heap;
using LIBC_NAMESPACE::FreeListHeap;
using LIBC_NAMESPACE::FreeListHeapBuffer;
using LIBC_NAMESPACE::cpp::byte;
using LIBC_NAMESPACE::cpp::span;

// Similar to `LlvmLibcBlockTest` in block_test.cpp, we'd like to run the same
// tests independently for different parameters. In this case, we'd like to test
Expand All @@ -28,23 +31,23 @@ using LIBC_NAMESPACE::freelist_heap;
// made in tests leak and aren't free'd. This is fine for the purposes of this
// test file.
#define TEST_FOR_EACH_ALLOCATOR(TestCase, BufferSize) \
class LlvmLibcFreeListHeapTest##TestCase : public testing::Test { \
class LlvmLibcFreeListHeapTest##TestCase \
: public LIBC_NAMESPACE::testing::Test { \
public: \
FreeListHeapBuffer<BufferSize> fake_global_buffer; \
void SetUp() override { \
freelist_heap = \
new (&fake_global_buffer) FreeListHeapBuffer<BufferSize>; \
} \
void RunTest(FreeListHeap<> &allocator, [[maybe_unused]] size_t N); \
void RunTest(FreeListHeap &allocator, [[maybe_unused]] size_t N); \
}; \
TEST_F(LlvmLibcFreeListHeapTest##TestCase, TestCase) { \
alignas(FreeListHeap<>::BlockType) \
cpp::byte buf[BufferSize] = {cpp::byte(0)}; \
FreeListHeap<> allocator(buf); \
alignas(Block<>) byte buf[BufferSize] = {byte(0)}; \
FreeListHeap allocator(buf); \
RunTest(allocator, BufferSize); \
RunTest(*freelist_heap, freelist_heap->region().size()); \
} \
void LlvmLibcFreeListHeapTest##TestCase::RunTest(FreeListHeap<> &allocator, \
void LlvmLibcFreeListHeapTest##TestCase::RunTest(FreeListHeap &allocator, \
size_t N)

TEST_FOR_EACH_ALLOCATOR(CanAllocate, 2048) {
Expand Down Expand Up @@ -92,14 +95,13 @@ TEST_FOR_EACH_ALLOCATOR(ReturnsNullWhenAllocationTooLarge, 2048) {
// is used for other test cases and we don't explicitly free them.
TEST(LlvmLibcFreeListHeap, ReturnsNullWhenFull) {
constexpr size_t N = 2048;
alignas(FreeListHeap<>::BlockType) cpp::byte buf[N] = {cpp::byte(0)};
alignas(Block<>) byte buf[N] = {byte(0)};

FreeListHeap<> allocator(buf);
FreeListHeap allocator(buf);

// Use aligned_allocate so we don't need to worry about ensuring the `buf`
// being aligned to max_align_t.
EXPECT_NE(allocator.aligned_allocate(
1, N - 2 * FreeListHeap<>::BlockType::BLOCK_OVERHEAD),
EXPECT_NE(allocator.aligned_allocate(1, N - 2 * Block<>::BLOCK_OVERHEAD),
static_cast<void *>(nullptr));
EXPECT_EQ(allocator.allocate(1), static_cast<void *>(nullptr));
}
Expand Down Expand Up @@ -134,9 +136,9 @@ TEST_FOR_EACH_ALLOCATOR(ReallocHasSameContent, 2048) {
constexpr size_t ALLOC_SIZE = sizeof(int);
constexpr size_t kNewAllocSize = sizeof(int) * 2;
// Data inside the allocated block.
cpp::byte data1[ALLOC_SIZE];
byte data1[ALLOC_SIZE];
// Data inside the reallocated block.
cpp::byte data2[ALLOC_SIZE];
byte data2[ALLOC_SIZE];

int *ptr1 = reinterpret_cast<int *>(allocator.allocate(ALLOC_SIZE));
*ptr1 = 42;
Expand Down Expand Up @@ -188,10 +190,9 @@ TEST_FOR_EACH_ALLOCATOR(CanCalloc, 2048) {
constexpr size_t ALLOC_SIZE = 128;
constexpr size_t NUM = 4;
constexpr int size = NUM * ALLOC_SIZE;
constexpr cpp::byte zero{0};
constexpr byte zero{0};

cpp::byte *ptr1 =
reinterpret_cast<cpp::byte *>(allocator.calloc(NUM, ALLOC_SIZE));
byte *ptr1 = reinterpret_cast<byte *>(allocator.calloc(NUM, ALLOC_SIZE));

// calloc'd content is zero.
for (int i = 0; i < size; i++) {
Expand All @@ -203,10 +204,9 @@ TEST_FOR_EACH_ALLOCATOR(CanCallocWeirdSize, 2048) {
constexpr size_t ALLOC_SIZE = 143;
constexpr size_t NUM = 3;
constexpr int size = NUM * ALLOC_SIZE;
constexpr cpp::byte zero{0};
constexpr byte zero{0};

cpp::byte *ptr1 =
reinterpret_cast<cpp::byte *>(allocator.calloc(NUM, ALLOC_SIZE));
byte *ptr1 = reinterpret_cast<byte *>(allocator.calloc(NUM, ALLOC_SIZE));

// calloc'd content is zero.
for (int i = 0; i < size; i++) {
Expand Down Expand Up @@ -241,17 +241,16 @@ TEST_FOR_EACH_ALLOCATOR(AlignedAlloc, 2048) {

// This test is not part of the TEST_FOR_EACH_ALLOCATOR since we want to
// explicitly ensure that the buffer can still return aligned allocations even
// if the underlying buffer is at most aligned to the BlockType alignment. This
// if the underlying buffer is at most aligned to the Block<> alignment. This
// is so we can check that we can still get aligned allocations even if the
// underlying buffer is not aligned to the alignments we request.
TEST(LlvmLibcFreeListHeap, AlignedAllocOnlyBlockTypeAligned) {
TEST(LlvmLibcFreeListHeap, AlignedAllocOnlyBlockAligned) {
constexpr size_t BUFFER_SIZE = 4096;
constexpr size_t BUFFER_ALIGNMENT = alignof(FreeListHeap<>::BlockType) * 2;
alignas(BUFFER_ALIGNMENT) cpp::byte buf[BUFFER_SIZE] = {cpp::byte(0)};
constexpr size_t BUFFER_ALIGNMENT = alignof(Block<>) * 2;
alignas(BUFFER_ALIGNMENT) byte buf[BUFFER_SIZE] = {byte(0)};

// Ensure the underlying buffer is at most aligned to the block type.
FreeListHeap<> allocator(
span<cpp::byte>(buf).subspan(alignof(FreeListHeap<>::BlockType)));
FreeListHeap allocator(span<byte>(buf).subspan(alignof(Block<>)));

constexpr size_t ALIGNMENTS[] = {1, 2, 4, 8, 16, 32, 64, 128, 256};
constexpr size_t SIZE_SCALES[] = {1, 2, 3, 4, 5};
Expand Down Expand Up @@ -289,5 +288,3 @@ TEST_FOR_EACH_ALLOCATOR(InvalidAlignedAllocAlignment, 2048) {
ptr = allocator.aligned_allocate(0, 8);
EXPECT_EQ(ptr, static_cast<void *>(nullptr));
}

} // namespace LIBC_NAMESPACE_DECL
11 changes: 5 additions & 6 deletions libc/test/src/__support/freelist_malloc_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
#include "src/stdlib/malloc.h"
#include "test/UnitTest/Test.h"

using LIBC_NAMESPACE::Block;
using LIBC_NAMESPACE::freelist_heap;
using LIBC_NAMESPACE::FreeListHeap;
using LIBC_NAMESPACE::FreeListHeapBuffer;
Expand All @@ -22,15 +23,13 @@ TEST(LlvmLibcFreeListMalloc, Malloc) {
constexpr size_t kCallocNum = 4;
constexpr size_t kCallocSize = 64;

typedef FreeListHeap<>::BlockType Block;

void *ptr1 = LIBC_NAMESPACE::malloc(kAllocSize);
auto *block = Block::from_usable_space(ptr1);
auto *block = Block<>::from_usable_space(ptr1);
EXPECT_GE(block->inner_size(), kAllocSize);

LIBC_NAMESPACE::free(ptr1);
ASSERT_NE(block->next(), static_cast<Block *>(nullptr));
ASSERT_EQ(block->next()->next(), static_cast<Block *>(nullptr));
ASSERT_NE(block->next(), static_cast<Block<> *>(nullptr));
ASSERT_EQ(block->next()->next(), static_cast<Block<> *>(nullptr));
size_t heap_size = block->inner_size();

void *ptr2 = LIBC_NAMESPACE::calloc(kCallocNum, kCallocSize);
Expand All @@ -47,7 +46,7 @@ TEST(LlvmLibcFreeListMalloc, Malloc) {
void *ptr3 = LIBC_NAMESPACE::aligned_alloc(ALIGN, kAllocSize);
EXPECT_NE(ptr3, static_cast<void *>(nullptr));
EXPECT_EQ(reinterpret_cast<uintptr_t>(ptr3) % ALIGN, size_t(0));
auto *aligned_block = reinterpret_cast<Block *>(ptr3);
auto *aligned_block = reinterpret_cast<Block<> *>(ptr3);
EXPECT_GE(aligned_block->inner_size(), kAllocSize);

LIBC_NAMESPACE::free(ptr3);
Expand Down
185 changes: 35 additions & 150 deletions libc/test/src/__support/freelist_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,159 +8,44 @@

#include <stddef.h>

#include "src/__support/CPP/array.h"
#include "src/__support/CPP/span.h"
#include "src/__support/freelist.h"
#include "test/UnitTest/Test.h"

using LIBC_NAMESPACE::Block;
using LIBC_NAMESPACE::FreeList;
using LIBC_NAMESPACE::cpp::array;
using LIBC_NAMESPACE::cpp::byte;
using LIBC_NAMESPACE::cpp::span;

static constexpr size_t SIZE = 8;
static constexpr array<size_t, SIZE> example_sizes = {64, 128, 256, 512,
1024, 2048, 4096, 8192};

TEST(LlvmLibcFreeList, EmptyListHasNoMembers) {
FreeList<SIZE> list(example_sizes);

auto item = list.find_chunk(4);
EXPECT_EQ(item.size(), static_cast<size_t>(0));
item = list.find_chunk(128);
EXPECT_EQ(item.size(), static_cast<size_t>(0));
}

TEST(LlvmLibcFreeList, CanRetrieveAddedMember) {
FreeList<SIZE> list(example_sizes);
constexpr size_t N = 512;

byte data[N] = {byte(0)};

bool ok = list.add_chunk(span<byte>(data, N));
EXPECT_TRUE(ok);

auto item = list.find_chunk(N);
EXPECT_EQ(item.size(), N);
EXPECT_EQ(item.data(), data);
}

TEST(LlvmLibcFreeList, CanRetrieveAddedMemberForSmallerSize) {
FreeList<SIZE> list(example_sizes);
constexpr size_t N = 512;

byte data[N] = {byte(0)};

ASSERT_TRUE(list.add_chunk(span<byte>(data, N)));
auto item = list.find_chunk(N / 2);
EXPECT_EQ(item.size(), N);
EXPECT_EQ(item.data(), data);
}

TEST(LlvmLibcFreeList, CanRemoveItem) {
FreeList<SIZE> list(example_sizes);
constexpr size_t N = 512;

byte data[N] = {byte(0)};

ASSERT_TRUE(list.add_chunk(span<byte>(data, N)));
EXPECT_TRUE(list.remove_chunk(span<byte>(data, N)));

auto item = list.find_chunk(N);
EXPECT_EQ(item.size(), static_cast<size_t>(0));
}

TEST(LlvmLibcFreeList, FindReturnsSmallestChunk) {
FreeList<SIZE> list(example_sizes);
constexpr size_t kN1 = 512;
constexpr size_t kN2 = 1024;

byte data1[kN1] = {byte(0)};
byte data2[kN2] = {byte(0)};

ASSERT_TRUE(list.add_chunk(span<byte>(data1, kN1)));
ASSERT_TRUE(list.add_chunk(span<byte>(data2, kN2)));

auto chunk = list.find_chunk(kN1 / 2);
EXPECT_EQ(chunk.size(), kN1);
EXPECT_EQ(chunk.data(), data1);

chunk = list.find_chunk(kN1);
EXPECT_EQ(chunk.size(), kN1);
EXPECT_EQ(chunk.data(), data1);

chunk = list.find_chunk(kN1 + 1);
EXPECT_EQ(chunk.size(), kN2);
EXPECT_EQ(chunk.data(), data2);
}

TEST(LlvmLibcFreeList, FindReturnsCorrectChunkInSameBucket) {
// If we have two values in the same bucket, ensure that the allocation will
// pick an appropriately sized one.
FreeList<SIZE> list(example_sizes);
constexpr size_t kN1 = 512;
constexpr size_t kN2 = 257;

byte data1[kN1] = {byte(0)};
byte data2[kN2] = {byte(0)};

// List should now be 257 -> 512 -> NULL
ASSERT_TRUE(list.add_chunk(span<byte>(data1, kN1)));
ASSERT_TRUE(list.add_chunk(span<byte>(data2, kN2)));

auto chunk = list.find_chunk(kN2 + 1);
EXPECT_EQ(chunk.size(), kN1);
}

TEST(LlvmLibcFreeList, FindCanMoveUpThroughBuckets) {
// Ensure that finding a chunk will move up through buckets if no appropriate
// chunks were found in a given bucket
FreeList<SIZE> list(example_sizes);
constexpr size_t kN1 = 257;
constexpr size_t kN2 = 513;

byte data1[kN1] = {byte(0)};
byte data2[kN2] = {byte(0)};

// List should now be:
// bkt[3] (257 bytes up to 512 bytes) -> 257 -> NULL
// bkt[4] (513 bytes up to 1024 bytes) -> 513 -> NULL
ASSERT_TRUE(list.add_chunk(span<byte>(data1, kN1)));
ASSERT_TRUE(list.add_chunk(span<byte>(data2, kN2)));

// Request a 300 byte chunk. This should return the 513 byte one
auto chunk = list.find_chunk(kN1 + 1);
EXPECT_EQ(chunk.size(), kN2);
}

TEST(LlvmLibcFreeList, RemoveUnknownChunkReturnsNotFound) {
FreeList<SIZE> list(example_sizes);
constexpr size_t N = 512;

byte data[N] = {byte(0)};
byte data2[N] = {byte(0)};

ASSERT_TRUE(list.add_chunk(span<byte>(data, N)));
EXPECT_FALSE(list.remove_chunk(span<byte>(data2, N)));
}

TEST(LlvmLibcFreeList, CanStoreMultipleChunksPerBucket) {
FreeList<SIZE> list(example_sizes);
constexpr size_t N = 512;

byte data1[N] = {byte(0)};
byte data2[N] = {byte(0)};

ASSERT_TRUE(list.add_chunk(span<byte>(data1, N)));
ASSERT_TRUE(list.add_chunk(span<byte>(data2, N)));

auto chunk1 = list.find_chunk(N);
ASSERT_TRUE(list.remove_chunk(chunk1));
auto chunk2 = list.find_chunk(N);
ASSERT_TRUE(list.remove_chunk(chunk2));

// Ordering of the chunks doesn't matter
EXPECT_TRUE(chunk1.data() != chunk2.data());
EXPECT_TRUE(chunk1.data() == data1 || chunk1.data() == data2);
EXPECT_TRUE(chunk2.data() == data1 || chunk2.data() == data2);
using LIBC_NAMESPACE::cpp::optional;

TEST(LlvmLibcFreeList, FreeList) {
byte mem1[1024];
optional<Block<> *> maybeBlock = Block<>::init(mem1);
ASSERT_TRUE(maybeBlock.has_value());
Block<> *block1 = *maybeBlock;

byte mem2[1024];
maybeBlock = Block<>::init(mem2);
ASSERT_TRUE(maybeBlock.has_value());
Block<> *block2 = *maybeBlock;

FreeList list;
list.push(block1);
ASSERT_FALSE(list.empty());
EXPECT_EQ(list.front(), block1);

list.push(block2);
EXPECT_EQ(list.front(), block1);

list.pop();
ASSERT_FALSE(list.empty());
EXPECT_EQ(list.front(), block2);

list.pop();
ASSERT_TRUE(list.empty());

list.push(block1);
list.push(block2);
list.remove(reinterpret_cast<FreeList::Node *>(block2->usable_space()));
EXPECT_EQ(list.front(), block1);
list.pop();
ASSERT_TRUE(list.empty());
}
101 changes: 101 additions & 0 deletions libc/test/src/__support/freestore_test.cpp
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
//===-- Unittests for a freestore -------------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include <stddef.h>

#include "src/__support/freestore.h"
#include "test/UnitTest/Test.h"

using LIBC_NAMESPACE::Block;
using LIBC_NAMESPACE::FreeList;
using LIBC_NAMESPACE::FreeStore;
using LIBC_NAMESPACE::FreeTrie;
using LIBC_NAMESPACE::cpp::byte;
using LIBC_NAMESPACE::cpp::optional;

// Inserting or removing blocks too small to be tracked does nothing.
TEST(LlvmLibcFreeStore, TooSmall) {
byte mem[1024];
optional<Block<> *> maybeBlock = Block<>::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
Block<> *too_small = *maybeBlock;
maybeBlock = too_small->split(sizeof(Block<>::offset_type));
ASSERT_TRUE(maybeBlock.has_value());
Block<> *remainder = *maybeBlock;

FreeStore store;
store.set_range({0, 4096});
store.insert(too_small);
store.insert(remainder);

EXPECT_EQ(store.remove_best_fit(too_small->inner_size()), remainder);
store.remove(too_small);
}

TEST(LlvmLibcFreeStore, RemoveBestFit) {
byte mem[1024];
optional<Block<> *> maybeBlock = Block<>::init(mem);
ASSERT_TRUE(maybeBlock.has_value());

Block<> *smallest = *maybeBlock;
maybeBlock = smallest->split(sizeof(FreeList) + sizeof(Block<>::offset_type));
ASSERT_TRUE(maybeBlock.has_value());
Block<> *largest_small = *maybeBlock;
maybeBlock =
largest_small->split(sizeof(FreeTrie::Node) +
sizeof(Block<>::offset_type) - alignof(max_align_t));
ASSERT_TRUE(maybeBlock.has_value());
ASSERT_GT(largest_small->inner_size(), smallest->inner_size());

Block<> *remainder = *maybeBlock;

FreeStore store;
store.set_range({0, 4096});
store.insert(smallest);
store.insert(largest_small);
store.insert(remainder);

// Find exact match for smallest.
ASSERT_EQ(store.remove_best_fit(smallest->inner_size()), smallest);
store.insert(smallest);

// Find exact match for largest.
ASSERT_EQ(store.remove_best_fit(largest_small->inner_size()), largest_small);
store.insert(largest_small);

// Search smallest for best fit.
ASSERT_EQ(store.remove_best_fit(smallest->inner_size() + 1), largest_small);
store.insert(largest_small);

// Continue search for best fit to large blocks.
EXPECT_EQ(store.remove_best_fit(largest_small->inner_size() + 1), remainder);
}

TEST(LlvmLibcFreeStore, Remove) {
byte mem[1024];
optional<Block<> *> maybeBlock = Block<>::init(mem);
ASSERT_TRUE(maybeBlock.has_value());

Block<> *small = *maybeBlock;
maybeBlock = small->split(sizeof(FreeList) + sizeof(Block<>::offset_type));
ASSERT_TRUE(maybeBlock.has_value());

Block<> *remainder = *maybeBlock;

FreeStore store;
store.set_range({0, 4096});
store.insert(small);
store.insert(remainder);

store.remove(remainder);
ASSERT_EQ(store.remove_best_fit(remainder->inner_size()),
static_cast<Block<> *>(nullptr));
store.remove(small);
ASSERT_EQ(store.remove_best_fit(small->inner_size()),
static_cast<Block<> *>(nullptr));
}
125 changes: 125 additions & 0 deletions libc/test/src/__support/freetrie_test.cpp
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
//===-- Unittests for a freetrie --------------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include <stddef.h>

#include "src/__support/freetrie.h"
#include "test/UnitTest/Test.h"

using LIBC_NAMESPACE::Block;
using LIBC_NAMESPACE::FreeTrie;
using LIBC_NAMESPACE::cpp::byte;
using LIBC_NAMESPACE::cpp::optional;

TEST(LlvmLibcFreeTrie, FindBestFitRoot) {
FreeTrie trie({0, 4096});
EXPECT_EQ(trie.find_best_fit(123), static_cast<FreeTrie::Node *>(nullptr));

byte mem[1024];
optional<Block<> *> maybeBlock = Block<>::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
Block<> *block = *maybeBlock;
trie.push(block);

FreeTrie::Node *root = trie.find_best_fit(0);
ASSERT_EQ(root->block(), block);
EXPECT_EQ(trie.find_best_fit(block->inner_size() - 1), root);
EXPECT_EQ(trie.find_best_fit(block->inner_size()), root);
EXPECT_EQ(trie.find_best_fit(block->inner_size() + 1),
static_cast<FreeTrie::Node *>(nullptr));
EXPECT_EQ(trie.find_best_fit(4095), static_cast<FreeTrie::Node *>(nullptr));
}

TEST(LlvmLibcFreeTrie, FindBestFitLower) {
byte mem[4096];
optional<Block<> *> maybeBlock = Block<>::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
Block<> *lower = *maybeBlock;
maybeBlock = lower->split(512);
ASSERT_TRUE(maybeBlock.has_value());
Block<> *root = *maybeBlock;

FreeTrie trie({0, 4096});
trie.push(root);
trie.push(lower);

EXPECT_EQ(trie.find_best_fit(0)->block(), lower);
}

TEST(LlvmLibcFreeTrie, FindBestFitUpper) {
byte mem[4096];
optional<Block<> *> maybeBlock = Block<>::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
Block<> *root = *maybeBlock;
maybeBlock = root->split(512);
ASSERT_TRUE(maybeBlock.has_value());
Block<> *upper = *maybeBlock;

FreeTrie trie({0, 4096});
trie.push(root);
trie.push(upper);

EXPECT_EQ(trie.find_best_fit(root->inner_size() + 1)->block(), upper);
// The upper subtrie should be skipped if it could not contain a better fit.
EXPECT_EQ(trie.find_best_fit(root->inner_size() - 1)->block(), root);
}

TEST(LlvmLibcFreeTrie, FindBestFitLowerAndUpper) {
byte mem[4096];
optional<Block<> *> maybeBlock = Block<>::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
Block<> *root = *maybeBlock;
maybeBlock = root->split(1024);
ASSERT_TRUE(maybeBlock.has_value());
Block<> *lower = *maybeBlock;
maybeBlock = lower->split(128);
ASSERT_TRUE(maybeBlock.has_value());
Block<> *upper = *maybeBlock;

FreeTrie trie({0, 4096});
trie.push(root);
trie.push(lower);
trie.push(upper);

// The lower subtrie is examined first.
EXPECT_EQ(trie.find_best_fit(0)->block(), lower);
// The upper subtrie is examined if there are no fits found in the upper
// subtrie.
EXPECT_EQ(trie.find_best_fit(2048)->block(), upper);
}

TEST(LlvmLibcFreeTrie, Remove) {
byte mem[4096];
optional<Block<> *> maybeBlock = Block<>::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
Block<> *small1 = *maybeBlock;
maybeBlock = small1->split(512);
ASSERT_TRUE(maybeBlock.has_value());
Block<> *small2 = *maybeBlock;
maybeBlock = small2->split(512);
ASSERT_TRUE(maybeBlock.has_value());
ASSERT_EQ(small1->inner_size(), small2->inner_size());
Block<> *large = *maybeBlock;

// Removing the root empties the trie.
FreeTrie trie({0, 4096});
trie.push(large);
FreeTrie::Node *large_node = trie.find_best_fit(0);
ASSERT_EQ(large_node->block(), large);
trie.remove(large_node);
ASSERT_TRUE(trie.empty());

// Removing the head of a trie list preserves the trie structure.
trie.push(small1);
trie.push(small2);
trie.push(large);
trie.remove(trie.find_best_fit(small1->inner_size()));
EXPECT_EQ(trie.find_best_fit(large->inner_size())->block(), large);
trie.remove(trie.find_best_fit(small1->inner_size()));
EXPECT_EQ(trie.find_best_fit(large->inner_size())->block(), large);
}