93 changes: 63 additions & 30 deletions compiler-rt/lib/scudo/standalone/release.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ namespace scudo {

class ReleaseRecorder {
public:
ReleaseRecorder(uptr Base, MapPlatformData *Data = nullptr)
: Base(Base), Data(Data) {}
ReleaseRecorder(uptr Base, uptr Offset = 0, MapPlatformData *Data = nullptr)
: Base(Base), Offset(Offset), Data(Data) {}

uptr getReleasedRangesCount() const { return ReleasedRangesCount; }

Expand All @@ -30,15 +30,22 @@ class ReleaseRecorder {
// Releases [From, To) range of pages back to OS.
void releasePageRangeToOS(uptr From, uptr To) {
const uptr Size = To - From;
releasePagesToOS(Base, From, Size, Data);
releasePagesToOS(Base, From + Offset, Size, Data);
ReleasedRangesCount++;
ReleasedBytes += Size;
}

private:
uptr ReleasedRangesCount = 0;
uptr ReleasedBytes = 0;
// The starting address to release. Note that we may want to combine (Base +
// Offset) as a new Base. However, the Base is retrieved from
// `MapPlatformData` on Fuchsia, which means the offset won't be aware.
// Therefore, store them separately to make it work on all the platforms.
uptr Base = 0;
// The release offset from Base. This is used when we know a given range after
// Base will not be released.
uptr Offset = 0;
MapPlatformData *Data = nullptr;
};

Expand Down Expand Up @@ -259,10 +266,10 @@ template <class ReleaseRecorderT> class FreePagesRangeTracker {
};

struct PageReleaseContext {
PageReleaseContext(uptr BlockSize, uptr RegionSize, uptr NumberOfRegions) :
BlockSize(BlockSize),
RegionSize(RegionSize),
NumberOfRegions(NumberOfRegions) {
PageReleaseContext(uptr BlockSize, uptr RegionSize, uptr NumberOfRegions,
uptr ReleaseSize, uptr ReleaseOffset = 0)
: BlockSize(BlockSize), RegionSize(RegionSize),
NumberOfRegions(NumberOfRegions) {
PageSize = getPageSizeCached();
if (BlockSize <= PageSize) {
if (PageSize % BlockSize == 0) {
Expand Down Expand Up @@ -294,10 +301,20 @@ struct PageReleaseContext {
}
}

PagesCount = roundUp(RegionSize, PageSize) / PageSize;
// TODO: For multiple regions, it's more complicated to support partial
// region marking (which includes the complexity of how to handle the last
// block in a region). We may consider this after markFreeBlocks() accepts
// only free blocks from the same region.
if (NumberOfRegions != 1) {
DCHECK_EQ(ReleaseSize, RegionSize);
DCHECK_EQ(ReleaseOffset, 0U);
}

PagesCount = roundUp(ReleaseSize, PageSize) / PageSize;
PageSizeLog = getLog2(PageSize);
RoundedRegionSize = PagesCount << PageSizeLog;
RoundedRegionSize = roundUp(RegionSize, PageSize);
RoundedSize = NumberOfRegions * RoundedRegionSize;
ReleasePageOffset = ReleaseOffset >> PageSizeLog;
}

// PageMap is lazily allocated when markFreeBlocks() is invoked.
Expand Down Expand Up @@ -364,7 +381,7 @@ struct PageReleaseContext {
uptr NumBlocksInFirstPage =
(FromInRegion + PageSize - FirstBlockInRange + BlockSize - 1) /
BlockSize;
PageMap.incN(RegionIndex, FromInRegion >> PageSizeLog,
PageMap.incN(RegionIndex, getPageIndex(FromInRegion),
NumBlocksInFirstPage);
FromInRegion = roundUp(FromInRegion + 1, PageSize);
}
Expand Down Expand Up @@ -392,8 +409,8 @@ struct PageReleaseContext {
// The last block is not aligned to `To`, we need to increment the
// counter of `next page` by 1.
if (LastBlockInRange + BlockSize != ToInRegion) {
PageMap.incRange(RegionIndex, ToInRegion >> PageSizeLog,
(LastBlockInRange + BlockSize - 1) >> PageSizeLog);
PageMap.incRange(RegionIndex, getPageIndex(ToInRegion),
getPageIndex(LastBlockInRange + BlockSize - 1));
}
} else {
ToInRegion = RegionSize;
Expand All @@ -402,8 +419,8 @@ struct PageReleaseContext {
// After handling the first page and the last block, it's safe to mark any
// page in between the range [From, To).
if (FromInRegion < ToInRegion) {
PageMap.setAsAllCountedRange(RegionIndex, FromInRegion >> PageSizeLog,
(ToInRegion - 1) >> PageSizeLog);
PageMap.setAsAllCountedRange(RegionIndex, getPageIndex(FromInRegion),
getPageIndex(ToInRegion - 1));
}
}

Expand All @@ -412,6 +429,19 @@ struct PageReleaseContext {
DecompactPtrT DecompactPtr, uptr Base) {
ensurePageMapAllocated();

const uptr LastBlockInRegion = ((RegionSize / BlockSize) - 1U) * BlockSize;

// The last block in a region may not use the entire page, so if it's free,
// we mark the following "pretend" memory block(s) as free.
auto markLastBlock = [this, LastBlockInRegion](const uptr RegionIndex) {
uptr PInRegion = LastBlockInRegion + BlockSize;
while (PInRegion < RoundedRegionSize) {
PageMap.incRange(RegionIndex, getPageIndex(PInRegion),
getPageIndex(PInRegion + BlockSize - 1));
PInRegion += BlockSize;
}
};

// Iterate over free chunks and count how many free chunks affect each
// allocated page.
if (BlockSize <= PageSize && PageSize % BlockSize == 0) {
Expand All @@ -423,41 +453,38 @@ struct PageReleaseContext {
continue;
const uptr RegionIndex = NumberOfRegions == 1U ? 0 : P / RegionSize;
const uptr PInRegion = P - RegionIndex * RegionSize;
PageMap.inc(RegionIndex, PInRegion >> PageSizeLog);
PageMap.inc(RegionIndex, getPageIndex(PInRegion));
if (PInRegion == LastBlockInRegion)
markLastBlock(RegionIndex);
}
}
} else {
// In all other cases chunks might affect more than one page.
DCHECK_GE(RegionSize, BlockSize);
const uptr LastBlockInRegion =
((RegionSize / BlockSize) - 1U) * BlockSize;
for (const auto &It : FreeList) {
for (u16 I = 0; I < It.getCount(); I++) {
const uptr P = DecompactPtr(It.get(I)) - Base;
if (P >= RoundedSize)
continue;
const uptr RegionIndex = NumberOfRegions == 1U ? 0 : P / RegionSize;
uptr PInRegion = P - RegionIndex * RegionSize;
PageMap.incRange(RegionIndex, PInRegion >> PageSizeLog,
(PInRegion + BlockSize - 1) >> PageSizeLog);
// The last block in a region might straddle a page, so if it's
// free, we mark the following "pretend" memory block(s) as free.
if (PInRegion == LastBlockInRegion) {
PInRegion += BlockSize;
while (PInRegion < RoundedRegionSize) {
PageMap.incRange(RegionIndex, PInRegion >> PageSizeLog,
(PInRegion + BlockSize - 1) >> PageSizeLog);
PInRegion += BlockSize;
}
}
PageMap.incRange(RegionIndex, getPageIndex(PInRegion),
getPageIndex(PInRegion + BlockSize - 1));
if (PInRegion == LastBlockInRegion)
markLastBlock(RegionIndex);
}
}
}
}

uptr getPageIndex(uptr P) { return (P >> PageSizeLog) - ReleasePageOffset; }

uptr BlockSize;
uptr RegionSize;
uptr NumberOfRegions;
// For partial region marking, some pages in front are not needed to be
// counted.
uptr ReleasePageOffset;
uptr PageSize;
uptr PagesCount;
uptr PageSizeLog;
Expand All @@ -479,6 +506,7 @@ releaseFreeMemoryToOS(PageReleaseContext &Context,
const uptr BlockSize = Context.BlockSize;
const uptr PagesCount = Context.PagesCount;
const uptr NumberOfRegions = Context.NumberOfRegions;
const uptr ReleasePageOffset = Context.ReleasePageOffset;
const uptr FullPagesBlockCountMax = Context.FullPagesBlockCountMax;
const bool SameBlockCountPerPage = Context.SameBlockCountPerPage;
RegionPageMap &PageMap = Context.PageMap;
Expand Down Expand Up @@ -516,6 +544,10 @@ releaseFreeMemoryToOS(PageReleaseContext &Context,
}
uptr PrevPageBoundary = 0;
uptr CurrentBoundary = 0;
if (ReleasePageOffset > 0) {
PrevPageBoundary = ReleasePageOffset * PageSize;
CurrentBoundary = roundUpSlow(PrevPageBoundary, BlockSize);
}
for (uptr J = 0; J < PagesCount; J++) {
const uptr PageBoundary = PrevPageBoundary + PageSize;
uptr BlocksPerPage = Pn;
Expand Down Expand Up @@ -547,7 +579,8 @@ releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> &FreeList,
uptr RegionSize, uptr NumberOfRegions, uptr BlockSize,
ReleaseRecorderT &Recorder, DecompactPtrT DecompactPtr,
SkipRegionT SkipRegion) {
PageReleaseContext Context(BlockSize, RegionSize, NumberOfRegions);
PageReleaseContext Context(BlockSize, /*ReleaseSize=*/RegionSize, RegionSize,
NumberOfRegions);
Context.markFreeBlocks(FreeList, DecompactPtr, Recorder.getBase());
releaseFreeMemoryToOS(Context, Recorder, SkipRegion);
}
Expand Down
10 changes: 6 additions & 4 deletions compiler-rt/lib/scudo/standalone/tests/common_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,9 +69,11 @@ TEST(ScudoCommonTest, Zeros) {
unmap(P, Size, 0, &Data);
}

#if SCUDO_LINUX && !defined(__powerpc__)
// This test fails intermediately on PPC, which is why this test is disabled
// for now on this platform.
#if 0
// This test is temorarily disabled because it may not work as expected. E.g.,
// it doesn't dirty the pages so the pages may not be commited and it may only
// work on the single thread environment. As a result, this test is flaky and is
// impacting many test scenarios.
TEST(ScudoCommonTest, GetRssFromBuffer) {
constexpr int64_t AllocSize = 10000000;
constexpr int64_t Error = 3000000;
Expand All @@ -88,6 +90,6 @@ TEST(ScudoCommonTest, GetRssFromBuffer) {
EXPECT_LE(std::abs(Rss - AllocSize - Prev), Error);
}
}
#endif // SCUDO_LINUX
#endif

} // namespace scudo
153 changes: 147 additions & 6 deletions compiler-rt/lib/scudo/standalone/tests/release_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,15 +138,18 @@ TEST(ScudoReleaseTest, FreePagesRangeTracker) {

class ReleasedPagesRecorder {
public:
ReleasedPagesRecorder() = default;
explicit ReleasedPagesRecorder(scudo::uptr Base) : Base(Base) {}
std::set<scudo::uptr> ReportedPages;

void releasePageRangeToOS(scudo::uptr From, scudo::uptr To) {
const scudo::uptr PageSize = scudo::getPageSizeCached();
for (scudo::uptr I = From; I < To; I += PageSize)
ReportedPages.insert(I);
ReportedPages.insert(I + getBase());
}

scudo::uptr getBase() const { return 0; }
scudo::uptr getBase() const { return Base; }
scudo::uptr Base = 0;
};

// Simplified version of a TransferBatch.
Expand Down Expand Up @@ -219,7 +222,8 @@ template <class SizeClassMap> void testReleaseFreeMemoryToOS() {
ReleasedPagesRecorder Recorder;
scudo::PageReleaseContext Context(BlockSize,
/*RegionSize=*/MaxBlocks * BlockSize,
/*NumberOfRegions=*/1U);
/*NumberOfRegions=*/1U,
/*ReleaseSize=*/MaxBlocks * BlockSize);
ASSERT_FALSE(Context.hasBlockMarked());
Context.markFreeBlocks(FreeList, DecompactPtr, Recorder.getBase());
ASSERT_TRUE(Context.hasBlockMarked());
Expand Down Expand Up @@ -325,8 +329,9 @@ template <class SizeClassMap> void testPageMapMarkRange() {
const scudo::uptr GroupEnd = GroupBeg + GroupSize;

scudo::PageReleaseContext Context(BlockSize, RegionSize,
/*NumberOfRegions=*/1U);
Context.markRangeAsAllCounted(GroupBeg, GroupEnd, /*Base=*/0);
/*NumberOfRegions=*/1U,
/*ReleaseSize=*/RegionSize);
Context.markRangeAsAllCounted(GroupBeg, GroupEnd, /*Base=*/0U);

scudo::uptr FirstBlock =
((GroupBeg + BlockSize - 1) / BlockSize) * BlockSize;
Expand Down Expand Up @@ -394,13 +399,142 @@ template <class SizeClassMap> void testPageMapMarkRange() {

// Release the entire region. This is to ensure the last page is counted.
scudo::PageReleaseContext Context(BlockSize, RegionSize,
/*NumberOfRegions=*/1U);
/*NumberOfRegions=*/1U,
/*ReleaseSize=*/RegionSize);
Context.markRangeAsAllCounted(/*From=*/0U, /*To=*/RegionSize, /*Base=*/0);
for (scudo::uptr Page = 0; Page < RoundedRegionSize / PageSize; ++Page)
EXPECT_TRUE(Context.PageMap.isAllCounted(/*Region=*/0, Page));
} // Iterate each size class
}

template <class SizeClassMap> void testReleasePartialRegion() {
typedef FreeBatch<SizeClassMap> Batch;
const scudo::uptr PageSize = scudo::getPageSizeCached();
const scudo::uptr ReleaseBase = PageSize;
const scudo::uptr BasePageOffset = ReleaseBase / PageSize;

for (scudo::uptr I = 1; I <= SizeClassMap::LargestClassId; I++) {
// In the following, we want to ensure the region includes at least 2 pages
// and we will release all the pages except the first one. The handling of
// the last block is tricky, so we always test the case that includes the
// last block.
const scudo::uptr BlockSize = SizeClassMap::getSizeByClassId(I);
const scudo::uptr RegionSize =
scudo::roundUpSlow(scudo::roundUp(BlockSize, PageSize) * 2, BlockSize) +
BlockSize;
const scudo::uptr RoundedRegionSize = scudo::roundUp(RegionSize, PageSize);

scudo::SinglyLinkedList<Batch> FreeList;
FreeList.clear();

// Skip the blocks in the first page and add the remaining.
std::vector<scudo::uptr> Pages(RoundedRegionSize / PageSize, 0);
for (scudo::uptr Block = scudo::roundUpSlow(PageSize, BlockSize);
Block + BlockSize <= RoundedRegionSize; Block += BlockSize) {
for (scudo::uptr Page = Block / PageSize;
Page <= (Block + BlockSize - 1) / PageSize; ++Page) {
ASSERT_LT(Page, Pages.size());
++Pages[Page];
}
}

// This follows the logic how we count the last page. It should be
// consistent with how markFreeBlocks() handles the last block.
if (RoundedRegionSize % BlockSize != 0)
++Pages.back();

Batch *CurrentBatch = nullptr;
for (scudo::uptr Block = scudo::roundUpSlow(PageSize, BlockSize);
Block < RegionSize; Block += BlockSize) {
if (CurrentBatch == nullptr ||
CurrentBatch->getCount() == Batch::MaxCount) {
CurrentBatch = new Batch;
CurrentBatch->clear();
FreeList.push_back(CurrentBatch);
}
CurrentBatch->add(Block);
}

auto VerifyReleaseToOs = [&](scudo::PageReleaseContext &Context) {
auto SkipRegion = [](UNUSED scudo::uptr RegionIndex) { return false; };
ReleasedPagesRecorder Recorder(ReleaseBase);
releaseFreeMemoryToOS(Context, Recorder, SkipRegion);
const scudo::uptr FirstBlock = scudo::roundUpSlow(PageSize, BlockSize);

for (scudo::uptr P = 0; P < RoundedRegionSize; P += PageSize) {
if (P < FirstBlock) {
// If FirstBlock is not aligned with page boundary, the first touched
// page will not be released either.
EXPECT_TRUE(Recorder.ReportedPages.find(P) ==
Recorder.ReportedPages.end());
} else {
EXPECT_TRUE(Recorder.ReportedPages.find(P) !=
Recorder.ReportedPages.end());
}
}
};

// Test marking by visiting each block.
{
auto DecompactPtr = [](scudo::uptr P) { return P; };
scudo::PageReleaseContext Context(
BlockSize, RegionSize, /*NumberOfRegions=*/1U,
/*ReleaseSize=*/RegionSize - PageSize, ReleaseBase);
Context.markFreeBlocks(FreeList, DecompactPtr, /*Base=*/0U);
for (const Batch &It : FreeList) {
for (scudo::u16 I = 0; I < It.getCount(); I++) {
scudo::uptr Block = It.get(I);
for (scudo::uptr Page = Block / PageSize;
Page <= (Block + BlockSize - 1) / PageSize; ++Page) {
EXPECT_EQ(Pages[Page], Context.PageMap.get(/*Region=*/0U,
Page - BasePageOffset));
}
}
}

VerifyReleaseToOs(Context);
}

// Test range marking.
{
scudo::PageReleaseContext Context(
BlockSize, RegionSize, /*NumberOfRegions=*/1U,
/*ReleaseSize=*/RegionSize - PageSize, ReleaseBase);
Context.markRangeAsAllCounted(ReleaseBase, RegionSize, /*Base=*/0U);
for (scudo::uptr Page = ReleaseBase / PageSize;
Page < RoundedRegionSize / PageSize; ++Page) {
if (Context.PageMap.get(/*Region=*/0, Page - BasePageOffset) !=
Pages[Page]) {
EXPECT_TRUE(Context.PageMap.isAllCounted(/*Region=*/0,
Page - BasePageOffset));
}
}

VerifyReleaseToOs(Context);
}

// Check the buffer size of PageMap.
{
scudo::PageReleaseContext Full(BlockSize, RegionSize,
/*NumberOfRegions=*/1U,
/*ReleaseSize=*/RegionSize);
Full.ensurePageMapAllocated();
scudo::PageReleaseContext Partial(
BlockSize, RegionSize, /*NumberOfRegions=*/1U,
/*ReleaseSize=*/RegionSize - PageSize, ReleaseBase);
Partial.ensurePageMapAllocated();

EXPECT_GE(Full.PageMap.getBufferSize(), Partial.PageMap.getBufferSize());
}

while (!FreeList.empty()) {
CurrentBatch = FreeList.front();
FreeList.pop_front();
delete CurrentBatch;
}
} // Iterate each size class
}

TEST(ScudoReleaseTest, ReleaseFreeMemoryToOSDefault) {
testReleaseFreeMemoryToOS<scudo::DefaultSizeClassMap>();
}
Expand All @@ -419,3 +553,10 @@ TEST(ScudoReleaseTest, PageMapMarkRange) {
testPageMapMarkRange<scudo::FuchsiaSizeClassMap>();
testPageMapMarkRange<scudo::SvelteSizeClassMap>();
}

TEST(ScudoReleaseTest, ReleasePartialRegion) {
testReleasePartialRegion<scudo::DefaultSizeClassMap>();
testReleasePartialRegion<scudo::AndroidSizeClassMap>();
testReleasePartialRegion<scudo::FuchsiaSizeClassMap>();
testReleasePartialRegion<scudo::SvelteSizeClassMap>();
}