181 changes: 53 additions & 128 deletions libc/src/__support/block.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,13 +159,10 @@ class Block {
return reinterpret_cast<const cpp::byte *>(this) + BLOCK_OVERHEAD;
}

/// Marks the block as free and merges it with any free neighbors.
///
/// This method is static in order to consume and replace the given block
/// pointer. If neither member is free, the returned pointer will point to the
/// original block. Otherwise, it will point to the new, larger block created
/// by merging adjacent free blocks together.
static void free(Block *&block);
// @returns The region of memory the block manages, including the header.
ByteSpan region() {
return {reinterpret_cast<cpp::byte *>(this), outer_size()};
}

/// Attempts to split this block.
///
Expand All @@ -176,75 +173,34 @@ class Block {
/// This method may fail if the remaining space is too small to hold a new
/// block. If this method fails for any reason, the original block is
/// unmodified.
///
/// This method is static in order to consume and replace the given block
/// pointer with a pointer to the new, smaller block.
static optional<Block *> split(Block *&block, size_t new_inner_size);
optional<Block *> split(size_t new_inner_size);

/// Merges this block with the one that comes after it.
///
/// This method is static in order to consume and replace the given block
/// pointer with a pointer to the new, larger block.
static bool merge_next(Block *&block);
bool merge_next();

/// Fetches the block immediately after this one.
///
/// For performance, this always returns a block pointer, even if the returned
/// pointer is invalid. The pointer is valid if and only if `last()` is false.
///
/// Typically, after calling `Init` callers may save a pointer past the end of
/// the list using `next()`. This makes it easy to subsequently iterate over
/// the list:
/// @code{.cpp}
/// auto result = Block<>::init(byte_span);
/// Block<>* begin = *result;
/// Block<>* end = begin->next();
/// ...
/// for (auto* block = begin; block != end; block = block->next()) {
/// // Do something which each block.
/// }
/// @endcode
/// @returns The block immediately after this one, or a null pointer if this
/// is the last block.
Block *next() const;

/// @copydoc `next`.
static Block *next_block(const Block *block) {
return block == nullptr ? nullptr : block->next();
}

/// @returns The block immediately before this one, or a null pointer if this
/// is the first block.
Block *prev() const;

/// @copydoc `prev`.
static Block *prev_block(const Block *block) {
return block == nullptr ? nullptr : block->prev();
}

/// Indicates whether the block is in use.
///
/// @returns `true` if the block is in use or `false` if not.
bool used() const { return next_ & USED_MASK; }

/// Indicates whether this block is the last block or not (i.e. whether
/// `next()` points to a valid block or not). This is needed because
/// `next()` points to the end of this block, whether there is a valid
/// block there or not.
///
/// @returns `true` is this is the last block or `false` if not.
bool last() const { return next_ & LAST_MASK; }

/// Marks this block as in use.
void mark_used() { next_ |= USED_MASK; }

/// Marks this block as free.
void mark_free() { next_ &= ~USED_MASK; }

/// Marks this block as the last one in the chain.
/// Marks this block as the last one in the chain. Makes next() return
/// nullptr.
constexpr void mark_last() { next_ |= LAST_MASK; }

/// Clears the last bit from this block.
void clear_last() { next_ &= ~LAST_MASK; }

/// @brief Checks if a block is valid.
///
/// @returns `true` if and only if the following conditions are met:
Expand Down Expand Up @@ -314,10 +270,8 @@ class Block {
static BlockInfo allocate(Block *block, size_t alignment, size_t size);

private:
/// Consumes the block and returns as a span of bytes.
static ByteSpan as_bytes(Block *&&block);

/// Consumes the span of bytes and uses it to construct and return a block.
/// Construct a block to represent a span of bytes. Overwrites only enough
/// memory for the block header; the rest of the span is left alone.
static Block *as_block(size_t prev_outer_size, ByteSpan bytes);

/// Returns a `BlockStatus` that is either VALID or indicates the reason why
Expand All @@ -329,7 +283,7 @@ class Block {

/// Like `split`, but assumes the caller has already checked to parameters to
/// ensure the split will succeed.
static Block *split_impl(Block *&block, size_t new_inner_size);
Block *split_impl(size_t new_inner_size);

/// Offset from this block to the previous block. 0 if this is the first
/// block.
Expand Down Expand Up @@ -389,20 +343,6 @@ Block<OffsetType, kAlign>::init(ByteSpan region) {
return block;
}

template <typename OffsetType, size_t kAlign>
void Block<OffsetType, kAlign>::free(Block *&block) {
if (block == nullptr)
return;

block->mark_free();
Block *prev = block->prev();

if (merge_next(prev))
block = prev;

merge_next(block);
}

template <typename OffsetType, size_t kAlign>
bool Block<OffsetType, kAlign>::can_allocate(size_t alignment,
size_t size) const {
Expand Down Expand Up @@ -436,7 +376,7 @@ Block<OffsetType, kAlign>::allocate(Block *block, size_t alignment,

Block *original = info.block;
optional<Block *> maybe_aligned_block =
Block::split(original, adjustment - BLOCK_OVERHEAD);
original->split(adjustment - BLOCK_OVERHEAD);
LIBC_ASSERT(maybe_aligned_block.has_value() &&
"This split should always result in a new block. The check in "
"`can_allocate` ensures that we have enough space here to make "
Expand All @@ -445,7 +385,7 @@ Block<OffsetType, kAlign>::allocate(Block *block, size_t alignment,
if (Block *prev = original->prev()) {
// If there is a block before this, we can merge the current one with the
// newly created one.
merge_next(prev);
prev->merge_next();
} else {
// Otherwise, this was the very first block in the chain. Now we can make
// it the new first block.
Expand All @@ -459,84 +399,76 @@ Block<OffsetType, kAlign>::allocate(Block *block, size_t alignment,
}

// Now get a block for the requested size.
if (optional<Block *> next = Block::split(info.block, size))
if (optional<Block *> next = info.block->split(size))
info.next = *next;

return info;
}

template <typename OffsetType, size_t kAlign>
optional<Block<OffsetType, kAlign> *>
Block<OffsetType, kAlign>::split(Block *&block, size_t new_inner_size) {
if (block == nullptr)
Block<OffsetType, kAlign>::split(size_t new_inner_size) {
if (used())
return {};

if (block->used())
return {};

size_t old_inner_size = block->inner_size();
size_t old_inner_size = inner_size();
new_inner_size = align_up(new_inner_size, ALIGNMENT);
if (old_inner_size < new_inner_size)
return {};

if (old_inner_size - new_inner_size < BLOCK_OVERHEAD)
return {};

return split_impl(block, new_inner_size);
return split_impl(new_inner_size);
}

template <typename OffsetType, size_t kAlign>
Block<OffsetType, kAlign> *
Block<OffsetType, kAlign>::split_impl(Block *&block, size_t new_inner_size) {
size_t prev_outer_size = block->prev_;
Block<OffsetType, kAlign>::split_impl(size_t new_inner_size) {
size_t outer_size1 = new_inner_size + BLOCK_OVERHEAD;
bool is_last = block->last();
ByteSpan bytes = as_bytes(cpp::move(block));
Block *block1 = as_block(prev_outer_size, bytes.subspan(0, outer_size1));
Block *block2 = as_block(outer_size1, bytes.subspan(outer_size1));

if (is_last)
block2->mark_last();
else
block2->next()->prev_ = block2->next_;

block = cpp::move(block1);
return block2;
bool has_next = next();
ByteSpan new_region = region().subspan(outer_size1);
LIBC_ASSERT(!used() && "used blocks cannot be split");
// The low order bits of outer_size1 should both be zero, and is the correct
// value for the flags is false.
next_ = outer_size1;
LIBC_ASSERT(!used() && next() && "incorrect first split flags");
Block *new_block = as_block(outer_size1, new_region);

if (has_next) {
// The two flags are both false, so next_ is a plain size.
LIBC_ASSERT(!new_block->used() && next() && "flags disrupt use of size");
new_block->next()->prev_ = new_block->next_;
} else {
new_block->mark_last();
}
return new_block;
}

template <typename OffsetType, size_t kAlign>
bool Block<OffsetType, kAlign>::merge_next(Block *&block) {
if (block == nullptr)
return false;

if (block->last())
return false;

Block *next = block->next();
if (block->used() || next->used())
bool Block<OffsetType, kAlign>::merge_next() {
if (used() || !next() || next()->used())
return false;

size_t prev_outer_size = block->prev_;
bool is_last = next->last();
ByteSpan prev_bytes = as_bytes(cpp::move(block));
ByteSpan next_bytes = as_bytes(cpp::move(next));
size_t outer_size = prev_bytes.size() + next_bytes.size();
cpp::byte *merged = ::new (prev_bytes.data()) cpp::byte[outer_size];
block = as_block(prev_outer_size, ByteSpan(merged, outer_size));
// Extend the size and copy the last() flag from the next block to this one.
next_ &= SIZE_MASK;
next_ += next()->next_;

if (is_last)
block->mark_last();
else
block->next()->prev_ = block->next_;
if (next()) {
// The two flags are both false, so next_ is a plain size.
LIBC_ASSERT(!used() && next() && "flags disrupt use of size");
next()->prev_ = next_;
}

return true;
}

template <typename OffsetType, size_t kAlign>
Block<OffsetType, kAlign> *Block<OffsetType, kAlign>::next() const {
uintptr_t addr =
last() ? 0 : reinterpret_cast<uintptr_t>(this) + outer_size();
return reinterpret_cast<Block *>(addr);
if (next_ & LAST_MASK)
return nullptr;
return reinterpret_cast<Block *>(reinterpret_cast<uintptr_t>(this) +
outer_size());
}

template <typename OffsetType, size_t kAlign>
Expand All @@ -555,13 +487,6 @@ constexpr Block<OffsetType, kAlign>::Block(size_t prev_outer_size,
next_ = outer_size;
}

template <typename OffsetType, size_t kAlign>
ByteSpan Block<OffsetType, kAlign>::as_bytes(Block *&&block) {
size_t block_size = block->outer_size();
cpp::byte *bytes = new (cpp::move(block)) cpp::byte[block_size];
return {bytes, block_size};
}

template <typename OffsetType, size_t kAlign>
Block<OffsetType, kAlign> *
Block<OffsetType, kAlign>::as_block(size_t prev_outer_size, ByteSpan bytes) {
Expand All @@ -573,7 +498,7 @@ internal::BlockStatus Block<OffsetType, kAlign>::check_status() const {
if (reinterpret_cast<uintptr_t>(this) % ALIGNMENT != 0)
return internal::BlockStatus::MISALIGNED;

if (!last() && (this >= next() || this != next()->prev()))
if (next() && (this >= next() || this != next()->prev()))
return internal::BlockStatus::NEXT_MISMATCHED;

if (prev() && (this <= prev() || this != prev()->next()))
Expand Down
6 changes: 3 additions & 3 deletions libc/src/__support/freelist_heap.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,19 +189,19 @@ template <size_t NUM_BUCKETS> void FreeListHeap<NUM_BUCKETS>::free(void *ptr) {
BlockType *prev = chunk_block->prev();
BlockType *next = nullptr;

if (!chunk_block->last())
if (chunk_block->next())
next = chunk_block->next();

if (prev != nullptr && !prev->used()) {
// Remove from freelist and merge
freelist_.remove_chunk(block_to_span(prev));
chunk_block = chunk_block->prev();
BlockType::merge_next(chunk_block);
chunk_block->merge_next();
}

if (next != nullptr && !next->used()) {
freelist_.remove_chunk(block_to_span(next));
BlockType::merge_next(chunk_block);
chunk_block->merge_next();
}
// Add back to the freelist
freelist_.add_chunk(block_to_span(chunk_block));
Expand Down
193 changes: 26 additions & 167 deletions libc/test/src/__support/block_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ TEST_FOR_EACH_BLOCK_TYPE(CanCreateSingleAlignedBlock) {
EXPECT_EQ(block->prev(), static_cast<BlockType *>(nullptr));
EXPECT_EQ(block->next(), static_cast<BlockType *>(nullptr));
EXPECT_FALSE(block->used());
EXPECT_TRUE(block->last());
}

TEST_FOR_EACH_BLOCK_TYPE(CanCreateUnalignedSingleBlock) {
Expand Down Expand Up @@ -96,18 +95,16 @@ TEST_FOR_EACH_BLOCK_TYPE(CanSplitBlock) {
ASSERT_TRUE(result.has_value());
auto *block1 = *result;

result = BlockType::split(block1, kSplitN);
result = block1->split(kSplitN);
ASSERT_TRUE(result.has_value());

auto *block2 = *result;

EXPECT_EQ(block1->inner_size(), kSplitN);
EXPECT_EQ(block1->outer_size(), kSplitN + BlockType::BLOCK_OVERHEAD);
EXPECT_FALSE(block1->last());

EXPECT_EQ(block2->outer_size(), kN - kSplitN - BlockType::BLOCK_OVERHEAD);
EXPECT_FALSE(block2->used());
EXPECT_TRUE(block2->last());

EXPECT_EQ(block1->next(), block2);
EXPECT_EQ(block2->prev(), block1);
Expand All @@ -128,7 +125,7 @@ TEST_FOR_EACH_BLOCK_TYPE(CanSplitBlockUnaligned) {
split_addr += alignof(BlockType) - (split_addr % alignof(BlockType));
uintptr_t split_len = split_addr - (uintptr_t)&bytes;

result = BlockType::split(block1, kSplitN);
result = block1->split(kSplitN);
ASSERT_TRUE(result.has_value());
BlockType *block2 = *result;

Expand Down Expand Up @@ -161,11 +158,11 @@ TEST_FOR_EACH_BLOCK_TYPE(CanSplitMidBlock) {
ASSERT_TRUE(result.has_value());
BlockType *block1 = *result;

result = BlockType::split(block1, kSplit1);
result = block1->split(kSplit1);
ASSERT_TRUE(result.has_value());
BlockType *block2 = *result;

result = BlockType::split(block1, kSplit2);
result = block1->split(kSplit2);
ASSERT_TRUE(result.has_value());
BlockType *block3 = *result;

Expand All @@ -184,7 +181,7 @@ TEST_FOR_EACH_BLOCK_TYPE(CannotSplitTooSmallBlock) {
ASSERT_TRUE(result.has_value());
BlockType *block = *result;

result = BlockType::split(block, kSplitN);
result = block->split(kSplitN);
ASSERT_FALSE(result.has_value());
}

Expand All @@ -197,13 +194,7 @@ TEST_FOR_EACH_BLOCK_TYPE(CannotSplitBlockWithoutHeaderSpace) {
ASSERT_TRUE(result.has_value());
BlockType *block = *result;

result = BlockType::split(block, kSplitN);
ASSERT_FALSE(result.has_value());
}

TEST_FOR_EACH_BLOCK_TYPE(CannotSplitNull) {
BlockType *block = nullptr;
auto result = BlockType::split(block, 1);
result = block->split(kSplitN);
ASSERT_FALSE(result.has_value());
}

Expand All @@ -216,7 +207,7 @@ TEST_FOR_EACH_BLOCK_TYPE(CannotMakeBlockLargerInSplit) {
ASSERT_TRUE(result.has_value());
BlockType *block = *result;

result = BlockType::split(block, block->inner_size() + 1);
result = block->split(block->inner_size() + 1);
ASSERT_FALSE(result.has_value());
}

Expand All @@ -229,8 +220,7 @@ TEST_FOR_EACH_BLOCK_TYPE(CannotMakeSecondBlockLargerInSplit) {
ASSERT_TRUE(result.has_value());
BlockType *block = *result;

result = BlockType::split(block, block->inner_size() -
BlockType::BLOCK_OVERHEAD + 1);
result = block->split(block->inner_size() - BlockType::BLOCK_OVERHEAD + 1);
ASSERT_FALSE(result.has_value());
}

Expand All @@ -243,7 +233,7 @@ TEST_FOR_EACH_BLOCK_TYPE(CanMakeZeroSizeFirstBlock) {
ASSERT_TRUE(result.has_value());
BlockType *block = *result;

result = BlockType::split(block, 0);
result = block->split(0);
ASSERT_TRUE(result.has_value());
EXPECT_EQ(block->inner_size(), static_cast<size_t>(0));
}
Expand All @@ -257,8 +247,7 @@ TEST_FOR_EACH_BLOCK_TYPE(CanMakeZeroSizeSecondBlock) {
ASSERT_TRUE(result.has_value());
BlockType *block1 = *result;

result = BlockType::split(block1,
block1->inner_size() - BlockType::BLOCK_OVERHEAD);
result = block1->split(block1->inner_size() - BlockType::BLOCK_OVERHEAD);
ASSERT_TRUE(result.has_value());
BlockType *block2 = *result;

Expand Down Expand Up @@ -293,7 +282,7 @@ TEST_FOR_EACH_BLOCK_TYPE(CannotSplitUsedBlock) {
BlockType *block = *result;

block->mark_used();
result = BlockType::split(block, kSplitN);
result = block->split(kSplitN);
ASSERT_FALSE(result.has_value());
}

Expand All @@ -309,14 +298,14 @@ TEST_FOR_EACH_BLOCK_TYPE(CanMergeWithNextBlock) {
ASSERT_TRUE(result.has_value());
BlockType *block1 = *result;

result = BlockType::split(block1, kSplit1);
result = block1->split(kSplit1);
ASSERT_TRUE(result.has_value());

result = BlockType::split(block1, kSplit2);
result = block1->split(kSplit2);
ASSERT_TRUE(result.has_value());
BlockType *block3 = *result;

EXPECT_TRUE(BlockType::merge_next(block3));
EXPECT_TRUE(block3->merge_next());

EXPECT_EQ(block1->next(), block3);
EXPECT_EQ(block3->prev(), block1);
Expand All @@ -334,16 +323,11 @@ TEST_FOR_EACH_BLOCK_TYPE(CannotMergeWithFirstOrLastBlock) {
BlockType *block1 = *result;

// Do a split, just to check that the checks on next/prev are different...
result = BlockType::split(block1, kSplitN);
result = block1->split(kSplitN);
ASSERT_TRUE(result.has_value());
BlockType *block2 = *result;

EXPECT_FALSE(BlockType::merge_next(block2));
}

TEST_FOR_EACH_BLOCK_TYPE(CannotMergeNull) {
BlockType *block = nullptr;
EXPECT_FALSE(BlockType::merge_next(block));
EXPECT_FALSE(block2->merge_next());
}

TEST_FOR_EACH_BLOCK_TYPE(CannotMergeUsedBlock) {
Expand All @@ -356,131 +340,11 @@ TEST_FOR_EACH_BLOCK_TYPE(CannotMergeUsedBlock) {
BlockType *block = *result;

// Do a split, just to check that the checks on next/prev are different...
result = BlockType::split(block, kSplitN);
ASSERT_TRUE(result.has_value());

block->mark_used();
EXPECT_FALSE(BlockType::merge_next(block));
}

TEST_FOR_EACH_BLOCK_TYPE(CanFreeSingleBlock) {
constexpr size_t kN = 1024;
alignas(BlockType::ALIGNMENT) array<byte, kN> bytes;

auto result = BlockType::init(bytes);
result = block->split(kSplitN);
ASSERT_TRUE(result.has_value());
BlockType *block = *result;

block->mark_used();
BlockType::free(block);
EXPECT_FALSE(block->used());
EXPECT_EQ(block->outer_size(), kN);
}

TEST_FOR_EACH_BLOCK_TYPE(CanFreeBlockWithoutMerging) {
constexpr size_t kN = 1024;
constexpr size_t kSplit1 = 512;
constexpr size_t kSplit2 = 256;

alignas(BlockType::ALIGNMENT) array<byte, kN> bytes;
auto result = BlockType::init(bytes);
ASSERT_TRUE(result.has_value());
BlockType *block1 = *result;

result = BlockType::split(block1, kSplit1);
ASSERT_TRUE(result.has_value());
BlockType *block2 = *result;

result = BlockType::split(block2, kSplit2);
ASSERT_TRUE(result.has_value());
BlockType *block3 = *result;

block1->mark_used();
block2->mark_used();
block3->mark_used();

BlockType::free(block2);
EXPECT_FALSE(block2->used());
EXPECT_NE(block2->prev(), static_cast<BlockType *>(nullptr));
EXPECT_FALSE(block2->last());
}

TEST_FOR_EACH_BLOCK_TYPE(CanFreeBlockAndMergeWithPrev) {
constexpr size_t kN = 1024;
constexpr size_t kSplit1 = 512;
constexpr size_t kSplit2 = 256;

alignas(BlockType::ALIGNMENT) array<byte, kN> bytes;
auto result = BlockType::init(bytes);
ASSERT_TRUE(result.has_value());
BlockType *block1 = *result;

result = BlockType::split(block1, kSplit1);
ASSERT_TRUE(result.has_value());
BlockType *block2 = *result;

result = BlockType::split(block2, kSplit2);
ASSERT_TRUE(result.has_value());
BlockType *block3 = *result;

block2->mark_used();
block3->mark_used();

BlockType::free(block2);
EXPECT_FALSE(block2->used());
EXPECT_EQ(block2->prev(), static_cast<BlockType *>(nullptr));
EXPECT_FALSE(block2->last());
}

TEST_FOR_EACH_BLOCK_TYPE(CanFreeBlockAndMergeWithNext) {
constexpr size_t kN = 1024;
constexpr size_t kSplit1 = 512;
constexpr size_t kSplit2 = 256;

alignas(BlockType::ALIGNMENT) array<byte, kN> bytes;
auto result = BlockType::init(bytes);
ASSERT_TRUE(result.has_value());
BlockType *block1 = *result;

result = BlockType::split(block1, kSplit1);
ASSERT_TRUE(result.has_value());
BlockType *block2 = *result;

result = BlockType::split(block2, kSplit2);
ASSERT_TRUE(result.has_value());

block1->mark_used();
block2->mark_used();

BlockType::free(block2);
EXPECT_FALSE(block2->used());
EXPECT_NE(block2->prev(), static_cast<BlockType *>(nullptr));
EXPECT_TRUE(block2->last());
}

TEST_FOR_EACH_BLOCK_TYPE(CanFreeUsedBlockAndMergeWithBoth) {
constexpr size_t kN = 1024;
constexpr size_t kSplit1 = 512;
constexpr size_t kSplit2 = 256;

alignas(BlockType::ALIGNMENT) array<byte, kN> bytes;
auto result = BlockType::init(bytes);
ASSERT_TRUE(result.has_value());
BlockType *block1 = *result;

result = BlockType::split(block1, kSplit1);
ASSERT_TRUE(result.has_value());
BlockType *block2 = *result;

result = BlockType::split(block2, kSplit2);
ASSERT_TRUE(result.has_value());

block2->mark_used();

BlockType::free(block2);
EXPECT_FALSE(block2->used());
EXPECT_EQ(block2->prev(), static_cast<BlockType *>(nullptr));
EXPECT_TRUE(block2->last());
EXPECT_FALSE(block->merge_next());
}

TEST_FOR_EACH_BLOCK_TYPE(CanCheckValidBlock) {
Expand All @@ -493,11 +357,11 @@ TEST_FOR_EACH_BLOCK_TYPE(CanCheckValidBlock) {
ASSERT_TRUE(result.has_value());
BlockType *block1 = *result;

result = BlockType::split(block1, kSplit1);
result = block1->split(kSplit1);
ASSERT_TRUE(result.has_value());
BlockType *block2 = *result;

result = BlockType::split(block2, kSplit2);
result = block2->split(kSplit2);
ASSERT_TRUE(result.has_value());
BlockType *block3 = *result;

Expand All @@ -517,15 +381,15 @@ TEST_FOR_EACH_BLOCK_TYPE(CanCheckInvalidBlock) {
ASSERT_TRUE(result.has_value());
BlockType *block1 = *result;

result = BlockType::split(block1, kSplit1);
result = block1->split(kSplit1);
ASSERT_TRUE(result.has_value());
BlockType *block2 = *result;

result = BlockType::split(block2, kSplit2);
result = block2->split(kSplit2);
ASSERT_TRUE(result.has_value());
BlockType *block3 = *result;

result = BlockType::split(block3, kSplit3);
result = block3->split(kSplit3);
ASSERT_TRUE(result.has_value());

// Corrupt a Block header.
Expand Down Expand Up @@ -630,7 +494,6 @@ TEST_FOR_EACH_BLOCK_TYPE(AllocateAlreadyAligned) {
// Check the next block.
EXPECT_NE(next, static_cast<BlockType *>(nullptr));
EXPECT_EQ(aligned_block->next(), next);
EXPECT_EQ(next->next(), static_cast<BlockType *>(nullptr));
EXPECT_EQ(reinterpret_cast<byte *>(next) + next->outer_size(),
bytes.data() + bytes.size());
}
Expand Down Expand Up @@ -674,7 +537,6 @@ TEST_FOR_EACH_BLOCK_TYPE(AllocateNeedsAlignment) {
// Check the next block.
EXPECT_NE(next, static_cast<BlockType *>(nullptr));
EXPECT_EQ(aligned_block->next(), next);
EXPECT_EQ(next->next(), static_cast<BlockType *>(nullptr));
EXPECT_EQ(reinterpret_cast<byte *>(next) + next->outer_size(), &*bytes.end());
}

Expand All @@ -687,7 +549,7 @@ TEST_FOR_EACH_BLOCK_TYPE(PreviousBlockMergedIfNotFirst) {
BlockType *block = *result;

// Split the block roughly halfway and work on the second half.
auto result2 = BlockType::split(block, kN / 2);
auto result2 = block->split(kN / 2);
ASSERT_TRUE(result2.has_value());
BlockType *newblock = *result2;
ASSERT_EQ(newblock->prev(), block);
Expand Down Expand Up @@ -744,20 +606,17 @@ TEST_FOR_EACH_BLOCK_TYPE(CanRemergeBlockAllocations) {

// Check we have the appropriate blocks.
ASSERT_NE(prev, static_cast<BlockType *>(nullptr));
ASSERT_FALSE(prev->last());
ASSERT_EQ(aligned_block->prev(), prev);
EXPECT_NE(next, static_cast<BlockType *>(nullptr));
EXPECT_NE(next, static_cast<BlockType *>(nullptr));
EXPECT_EQ(aligned_block->next(), next);
EXPECT_EQ(next->next(), static_cast<BlockType *>(nullptr));
ASSERT_TRUE(next->last());

// Now check for successful merges.
EXPECT_TRUE(BlockType::merge_next(prev));
EXPECT_TRUE(prev->merge_next());
EXPECT_EQ(prev->next(), next);
EXPECT_TRUE(BlockType::merge_next(prev));
EXPECT_TRUE(prev->merge_next());
EXPECT_EQ(prev->next(), static_cast<BlockType *>(nullptr));
EXPECT_TRUE(prev->last());

// We should have the original buffer.
EXPECT_EQ(reinterpret_cast<byte *>(prev), &*bytes.begin());
Expand Down
2 changes: 1 addition & 1 deletion lldb/test/API/assert_messages_test/TestAssertMessages.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ def assert_expect_fails_with(self, cmd, expect_args, expected_msg):
else:
self.fail("Initial expect should have raised AssertionError!")

@expectedFailureAll(remote=True)
@expectedFailureAll(oslist=no_match(["linux"]), remote=True)
def test_createTestTarget(self):
try:
self.createTestTarget("doesnt_exist")
Expand Down
27 changes: 23 additions & 4 deletions llvm/lib/CodeGen/StackFrameLayoutAnalysisPass.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ struct StackFrameLayoutAnalysisPass : public MachineFunctionPass {

enum SlotType {
Spill, // a Spill slot
Fixed, // a Fixed slot (e.g. arguments passed on the stack)
VariableSized, // a variable sized object
StackProtector, // Stack Protector slot
Variable, // a slot used to store a local data (could be a tmp)
Invalid // It's an error for a slot to have this type
Expand All @@ -72,17 +74,30 @@ struct StackFrameLayoutAnalysisPass : public MachineFunctionPass {
Scalable = MFI.getStackID(Idx) == TargetStackID::ScalableVector;
if (MFI.isSpillSlotObjectIndex(Idx))
SlotTy = SlotType::Spill;
else if (Idx == MFI.getStackProtectorIndex())
else if (MFI.isFixedObjectIndex(Idx))
SlotTy = SlotType::Fixed;
else if (MFI.isVariableSizedObjectIndex(Idx))
SlotTy = SlotType::VariableSized;
else if (MFI.hasStackProtectorIndex() &&
Idx == MFI.getStackProtectorIndex())
SlotTy = SlotType::StackProtector;
else
SlotTy = SlotType::Variable;
}

bool isVarSize() const { return SlotTy == SlotType::VariableSized; }

// We use this to sort in reverse order, so that the layout is displayed
// correctly.
// correctly. Variable sized slots are sorted to the end of the list, as
// offsets are currently incorrect for these but they reside at the end of
// the stack frame. The Slot index is used to ensure deterministic order
// when offsets are equal.
bool operator<(const SlotData &Rhs) const {
return (Offset.getFixed() + Offset.getScalable()) >
(Rhs.Offset.getFixed() + Rhs.Offset.getScalable());
return std::make_tuple(!isVarSize(),
Offset.getFixed() + Offset.getScalable(), Slot) >
std::make_tuple(!Rhs.isVarSize(),
Rhs.Offset.getFixed() + Rhs.Offset.getScalable(),
Rhs.Slot);
}
};

Expand Down Expand Up @@ -121,6 +136,10 @@ struct StackFrameLayoutAnalysisPass : public MachineFunctionPass {
switch (Ty) {
case SlotType::Spill:
return "Spill";
case SlotType::Fixed:
return "Fixed";
case SlotType::VariableSized:
return "VariableSized";
case SlotType::StackProtector:
return "Protector";
case SlotType::Variable:
Expand Down
25 changes: 16 additions & 9 deletions llvm/test/CodeGen/AArch64/sve-stack-frame-layout.ll
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,10 +147,11 @@ entry:
; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-8], Type: Spill, Align: 8, Size: 8
; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-16], Type: Spill, Align: 8, Size: 8
; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-24], Type: Spill, Align: 8, Size: 8
; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-32], Type: Variable, Align: 1, Size: 0
; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-32], Type: Spill, Align: 8, Size: 8
; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-32-16 x vscale], Type: Variable, Align: 16, Size: vscale x 16
; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-40-16 x vscale], Type: Variable, Align: 8, Size: 8
; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-32], Type: VariableSized, Align: 1, Size: 0
; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-32], Type: VariableSized, Align: 1, Size: 0

define i32 @csr_d8_allocnxv4i32i32f64_vla(double %d, i32 %i) "aarch64_pstate_sm_compatible" {
; CHECK-LABEL: csr_d8_allocnxv4i32i32f64_vla:
Expand All @@ -172,7 +173,10 @@ define i32 @csr_d8_allocnxv4i32i32f64_vla(double %d, i32 %i) "aarch64_pstate_sm_
; CHECK-NEXT: mov x9, sp
; CHECK-NEXT: add x8, x8, #15
; CHECK-NEXT: and x8, x8, #0x7fffffff0
; CHECK-NEXT: sub x8, x9, x8
; CHECK-NEXT: sub x9, x9, x8
; CHECK-NEXT: mov sp, x9
; CHECK-NEXT: mov x10, sp
; CHECK-NEXT: sub x8, x10, x8
; CHECK-NEXT: mov sp, x8
; CHECK-NEXT: mov z1.s, #0 // =0x0
; CHECK-NEXT: ptrue p0.s
Expand All @@ -181,8 +185,9 @@ define i32 @csr_d8_allocnxv4i32i32f64_vla(double %d, i32 %i) "aarch64_pstate_sm_
; CHECK-NEXT: str wzr, [x8]
; CHECK-NEXT: sub x8, x29, #8
; CHECK-NEXT: mov w0, wzr
; CHECK-NEXT: str d0, [x19, #8]
; CHECK-NEXT: str wzr, [x9]
; CHECK-NEXT: st1w { z1.s }, p0, [x8, #-1, mul vl]
; CHECK-NEXT: str d0, [x19, #8]
; CHECK-NEXT: sub sp, x29, #8
; CHECK-NEXT: ldp x29, x30, [sp, #8] // 16-byte Folded Reload
; CHECK-NEXT: ldr x19, [sp, #24] // 8-byte Folded Reload
Expand All @@ -191,18 +196,20 @@ define i32 @csr_d8_allocnxv4i32i32f64_vla(double %d, i32 %i) "aarch64_pstate_sm_
entry:
%a = alloca <vscale x 4 x i32>
%0 = zext i32 %i to i64
%b = alloca i32, i64 %0
%vla0 = alloca i32, i64 %0
%vla1 = alloca i32, i64 %0
%c = alloca double
tail call void asm sideeffect "", "~{d8}"() #1
store <vscale x 4 x i32> zeroinitializer, ptr %a
store i32 zeroinitializer, ptr %b
store i32 zeroinitializer, ptr %vla0
store i32 zeroinitializer, ptr %vla1
store double %d, ptr %c
ret i32 0
}

; CHECK-FRAMELAYOUT-LABEL: Function: csr_d8_allocnxv4i32i32f64_stackargsi32f64
; CHECK-FRAMELAYOUT-NEXT: Offset: [SP+8], Type: Variable, Align: 8, Size: 4
; CHECK-FRAMELAYOUT-NEXT: Offset: [SP+0], Type: Protector, Align: 16, Size: 8
; CHECK-FRAMELAYOUT-NEXT: Offset: [SP+8], Type: Fixed, Align: 8, Size: 4
; CHECK-FRAMELAYOUT-NEXT: Offset: [SP+0], Type: Fixed, Align: 16, Size: 8
; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-8], Type: Spill, Align: 8, Size: 8
; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-16], Type: Spill, Align: 8, Size: 8
; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-16-16 x vscale], Type: Variable, Align: 16, Size: vscale x 16
Expand Down Expand Up @@ -289,7 +296,7 @@ entry:
}

; CHECK-FRAMELAYOUT-LABEL: Function: svecc_z8_allocnxv4i32i32f64_stackargsi32_fp
; CHECK-FRAMELAYOUT-NEXT: Offset: [SP+0], Type: Protector, Align: 16, Size: 4
; CHECK-FRAMELAYOUT-NEXT: Offset: [SP+0], Type: Fixed, Align: 16, Size: 4
; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-8], Type: Spill, Align: 8, Size: 8
; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-16], Type: Spill, Align: 8, Size: 8
; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-16-16 x vscale], Type: Spill, Align: 16, Size: vscale x 16
Expand Down Expand Up @@ -514,7 +521,7 @@ declare ptr @memset(ptr, i32, i32)
; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-104], Type: Spill, Align: 8, Size: 8
; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-112], Type: Spill, Align: 8, Size: 8
; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-128], Type: Variable, Align: 16, Size: 16
; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-128], Type: Variable, Align: 16, Size: 0
; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-128], Type: VariableSized, Align: 16, Size: 0

define i32 @vastate(i32 %x) "aarch64_inout_za" "aarch64_pstate_sm_enabled" "target-features"="+sme" {
; CHECK-LABEL: vastate:
Expand Down
12 changes: 6 additions & 6 deletions llvm/test/CodeGen/X86/stack-frame-layout-remarks.ll
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ entry:
declare void @llvm.dbg.declare(metadata, metadata, metadata) #0

; BOTH: Function: cleanup_array
; BOTH-NEXT: Offset: [SP+4], Type: Protector, Align: 16, Size: 4
; BOTH-NEXT: Offset: [SP+4], Type: Fixed, Align: 16, Size: 4
; DEBUG: a @ dot.c:13
; STRIPPED-NOT: a @ dot.c:13
; BOTH: Offset: [SP-4], Type: Spill, Align: 8, Size: 4
Expand All @@ -47,7 +47,7 @@ define void @cleanup_array(ptr %0) #1 {
}

; BOTH: Function: cleanup_result
; BOTH: Offset: [SP+4], Type: Protector, Align: 16, Size: 4
; BOTH: Offset: [SP+4], Type: Fixed, Align: 16, Size: 4
; DEBUG: res @ dot.c:21
; STRIPPED-NOT: res @ dot.c:21
; BOTH: Offset: [SP-4], Type: Spill, Align: 8, Size: 4
Expand All @@ -59,11 +59,11 @@ define void @cleanup_result(ptr %0) #1 {
}

; BOTH: Function: do_work
; BOTH: Offset: [SP+12], Type: Variable, Align: 8, Size: 4
; BOTH: Offset: [SP+12], Type: Fixed, Align: 8, Size: 4
; DEBUG: out @ dot.c:32
; STRIPPED-NOT: out @ dot.c:32
; BOTH: Offset: [SP+8], Type: Variable, Align: 4, Size: 4
; BOTH: Offset: [SP+4], Type: Protector, Align: 16, Size: 4
; BOTH: Offset: [SP+8], Type: Fixed, Align: 4, Size: 4
; BOTH: Offset: [SP+4], Type: Fixed, Align: 16, Size: 4
; DEBUG: A @ dot.c:32
; STRIPPED-NOT: A @ dot.c:32
; BOTH: Offset: [SP-4], Type: Spill, Align: 8, Size: 4
Expand Down Expand Up @@ -125,7 +125,7 @@ define i32 @do_work(ptr %0, ptr %1, ptr %2) #2 {
}

; BOTH: Function: gen_array
; BOTH: Offset: [SP+4], Type: Protector, Align: 16, Size: 4
; BOTH: Offset: [SP+4], Type: Fixed, Align: 16, Size: 4
; DEBUG: size @ dot.c:62
; STRIPPED-NOT: size @ dot.c:62
; BOTH: Offset: [SP-4], Type: Spill, Align: 8, Size: 4
Expand Down
4 changes: 4 additions & 0 deletions llvm/utils/gn/secondary/bolt/lib/RuntimeLibs/BUILD.gn
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,8 @@ static_library("RuntimeLibs") {
"InstrumentationRuntimeLibrary.cpp",
"RuntimeLibrary.cpp",
]

# FIXME: Hopefully change this, see
# https://github.com/llvm/llvm-project/pull/97130/files#r1691863361
defines = [ "CMAKE_INSTALL_FULL_LIBDIR=\"\"" ]
}
17 changes: 17 additions & 0 deletions llvm/utils/gn/secondary/bolt/lib/Utils/BUILD.gn
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,24 @@
import("//llvm/utils/gn/build/write_vcsrevision.gni")

# Configure the VCSVersion.inc file
config("write_vcsrevision_config") {
# To pick up the generated inc file.
include_dirs = [ target_gen_dir ]
visibility = [ ":write_vcsversion" ]
}

write_vcsrevision("write_vcsversion") {
visibility = [ ":Utils" ]
header = "$target_gen_dir/VCSVersion.inc"
names = [ "LLDB" ]
public_configs = [ ":write_vcsrevision_config" ]
}

static_library("Utils") {
output_name = "LLVMBOLTUtils"
configs += [ "//llvm/utils/gn/build:bolt_code" ]
deps = [
":write_vcsversion",
"//llvm/lib/Support",
"//llvm/utils/gn/build/libs/pthread",
]
Expand Down
2 changes: 2 additions & 0 deletions llvm/utils/gn/secondary/bolt/test/BUILD.gn
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@ write_lit_config("lit_site_cfg") {
"BOLT_ENABLE_RUNTIME=0", # FIXME: enable runtime
"BOLT_TARGETS_TO_BUILD=$bolt_targets_to_build_string",
"GNU_LD_EXECUTABLE=", # FIXME: set sometimes?
"LIBBOLT_RT_HUGIFY=",
"LIBBOLT_RT_INSTR=",
"LLVM_HOST_TRIPLE=$llvm_current_triple",
"LLVM_USE_SANITIZER=",
"Python3_EXECUTABLE=$python_path",
Expand Down
1 change: 1 addition & 0 deletions llvm/utils/gn/secondary/lldb/source/Interpreter/BUILD.gn
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ static_library("Interpreter") {
deps = [
":InterpreterProperties",
":InterpreterPropertiesEnum",
"Interfaces",
"//lldb/source/Commands",
"//lldb/source/Core",
"//lldb/source/DataFormatters",
Expand Down
9 changes: 9 additions & 0 deletions llvm/utils/gn/secondary/lldb/source/Interpreter/Interfaces/BUILD.gn
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
static_library("Interfaces") {
output_name = "lldbInterpreterInterfaces"
configs += [ "//llvm/utils/gn/build:lldb_code" ]
deps = [
"//lldb/source/Utility",
"//llvm/lib/Support",
]
sources = [ "ScriptedInterfaceUsages.cpp" ]
}