diff --git a/compiler-rt/lib/scudo/standalone/combined.h b/compiler-rt/lib/scudo/standalone/combined.h index fae71ba1b84ff4..911c49dcc3fd7c 100644 --- a/compiler-rt/lib/scudo/standalone/combined.h +++ b/compiler-rt/lib/scudo/standalone/combined.h @@ -100,7 +100,7 @@ class Allocator { // Reset tag to 0 as this chunk may have been previously used for a tagged // user allocation. - if (UNLIKELY(Allocator.useMemoryTagging())) + if (UNLIKELY(useMemoryTagging(Allocator.Primary.Options.load()))) storeTags(reinterpret_cast(Ptr), reinterpret_cast(Ptr) + sizeof(QuarantineBatch)); @@ -161,6 +161,9 @@ class Allocator { Primary.Options.set(OptionBit::DeallocTypeMismatch); if (getFlags()->delete_size_mismatch) Primary.Options.set(OptionBit::DeleteSizeMismatch); + if (allocatorSupportsMemoryTagging() && + systemSupportsMemoryTagging()) + Primary.Options.set(OptionBit::UseMemoryTagging); Primary.Options.set(OptionBit::UseOddEvenTags); QuarantineMaxChunkSize = @@ -240,7 +243,7 @@ class Allocator { } ALWAYS_INLINE void *untagPointerMaybe(void *Ptr) { - if (Primary.SupportsMemoryTagging) + if (allocatorSupportsMemoryTagging()) return reinterpret_cast( untagPointer(reinterpret_cast(Ptr))); return Ptr; @@ -367,7 +370,7 @@ class Allocator { // // When memory tagging is enabled, zeroing the contents is done as part of // setting the tag. - if (UNLIKELY(useMemoryTagging(Options))) { + if (UNLIKELY(useMemoryTagging(Options))) { uptr PrevUserPtr; Chunk::UnpackedHeader Header; const uptr BlockSize = PrimaryT::getSizeByClassId(ClassId); @@ -594,7 +597,7 @@ class Allocator { : BlockEnd - (reinterpret_cast(OldPtr) + NewSize)) & Chunk::SizeOrUnusedBytesMask; Chunk::compareExchangeHeader(Cookie, OldPtr, &NewHeader, &OldHeader); - if (UNLIKELY(ClassId && useMemoryTagging(Options))) { + if (UNLIKELY(ClassId && useMemoryTagging(Options))) { resizeTaggedChunk(reinterpret_cast(OldTaggedPtr) + OldSize, reinterpret_cast(OldTaggedPtr) + NewSize, BlockEnd); @@ -692,7 +695,7 @@ class Allocator { if (getChunkFromBlock(Block, &Chunk, &Header) && Header.State == Chunk::State::Allocated) { uptr TaggedChunk = Chunk; - if (useMemoryTagging(Primary.Options.load())) + if (useMemoryTagging(Primary.Options.load())) TaggedChunk = loadTag(Chunk); Callback(TaggedChunk, getSize(reinterpret_cast(Chunk), &Header), Arg); @@ -783,15 +786,14 @@ class Allocator { Header.State == Chunk::State::Allocated; } - bool useMemoryTagging() const { - return useMemoryTagging(Primary.Options.load()); + bool useMemoryTaggingTestOnly() const { + return useMemoryTagging(Primary.Options.load()); } - static bool useMemoryTagging(Options Options) { - return PrimaryT::useMemoryTagging(Options); + void disableMemoryTagging() { + if (allocatorSupportsMemoryTagging()) + Primary.Options.clear(OptionBit::UseMemoryTagging); } - void disableMemoryTagging() { Primary.disableMemoryTagging(); } - void setTrackAllocationStacks(bool Track) { initThreadMaybe(); if (Track) @@ -823,7 +825,7 @@ class Allocator { const char *MemoryTags, uintptr_t MemoryAddr, size_t MemorySize) { *ErrorInfo = {}; - if (!PrimaryT::SupportsMemoryTagging || + if (!allocatorSupportsMemoryTagging() || MemoryAddr + MemorySize < MemoryAddr) return; @@ -942,7 +944,7 @@ class Allocator { static_assert(MinAlignment >= sizeof(Chunk::PackedHeader), "Minimal alignment must at least cover a chunk header."); - static_assert(!PrimaryT::SupportsMemoryTagging || + static_assert(!allocatorSupportsMemoryTagging() || MinAlignment >= archMemoryTagGranuleSize(), ""); @@ -1037,7 +1039,7 @@ class Allocator { void quarantineOrDeallocateChunk(Options Options, void *Ptr, Chunk::UnpackedHeader *Header, uptr Size) { Chunk::UnpackedHeader NewHeader = *Header; - if (UNLIKELY(NewHeader.ClassId && useMemoryTagging(Options))) { + if (UNLIKELY(NewHeader.ClassId && useMemoryTagging(Options))) { u8 PrevTag = extractTag(loadTag(reinterpret_cast(Ptr))); if (!TSDRegistry.getDisableMemInit()) { uptr TaggedBegin, TaggedEnd; diff --git a/compiler-rt/lib/scudo/standalone/options.h b/compiler-rt/lib/scudo/standalone/options.h index 2cffc4d75c38c0..91301bf5ec9c2c 100644 --- a/compiler-rt/lib/scudo/standalone/options.h +++ b/compiler-rt/lib/scudo/standalone/options.h @@ -11,6 +11,7 @@ #include "atomic_helpers.h" #include "common.h" +#include "memtag.h" namespace scudo { @@ -36,6 +37,11 @@ struct Options { } }; +template bool useMemoryTagging(Options Options) { + return allocatorSupportsMemoryTagging() && + Options.get(OptionBit::UseMemoryTagging); +} + struct AtomicOptions { atomic_u32 Val; diff --git a/compiler-rt/lib/scudo/standalone/primary32.h b/compiler-rt/lib/scudo/standalone/primary32.h index c744670b439266..a88a2a67e95159 100644 --- a/compiler-rt/lib/scudo/standalone/primary32.h +++ b/compiler-rt/lib/scudo/standalone/primary32.h @@ -50,7 +50,6 @@ template class SizeClassAllocator32 { typedef SizeClassAllocator32 ThisT; typedef SizeClassAllocatorLocalCache CacheT; typedef typename CacheT::TransferBatch TransferBatch; - static const bool SupportsMemoryTagging = false; static uptr getSizeByClassId(uptr ClassId) { return (ClassId == SizeClassMap::BatchClassId) @@ -216,9 +215,6 @@ template class SizeClassAllocator32 { return TotalReleasedBytes; } - static bool useMemoryTagging(UNUSED Options Options) { return false; } - void disableMemoryTagging() {} - const char *getRegionInfoArrayAddress() const { return nullptr; } static uptr getRegionInfoArraySize() { return 0; } diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h index df1310aa8e9595..2724a2529f7595 100644 --- a/compiler-rt/lib/scudo/standalone/primary64.h +++ b/compiler-rt/lib/scudo/standalone/primary64.h @@ -46,8 +46,6 @@ template class SizeClassAllocator64 { typedef SizeClassAllocator64 ThisT; typedef SizeClassAllocatorLocalCache CacheT; typedef typename CacheT::TransferBatch TransferBatch; - static const bool SupportsMemoryTagging = - allocatorSupportsMemoryTagging(); static uptr getSizeByClassId(uptr ClassId) { return (ClassId == SizeClassMap::BatchClassId) @@ -76,9 +74,6 @@ template class SizeClassAllocator64 { Region->ReleaseInfo.LastReleaseAtNs = Time; } setOption(Option::ReleaseInterval, static_cast(ReleaseToOsInterval)); - - if (SupportsMemoryTagging && systemSupportsMemoryTagging()) - Options.set(OptionBit::UseMemoryTagging); } void init(s32 ReleaseToOsInterval) { memset(this, 0, sizeof(*this)); @@ -193,11 +188,6 @@ template class SizeClassAllocator64 { return TotalReleasedBytes; } - static bool useMemoryTagging(Options Options) { - return SupportsMemoryTagging && Options.get(OptionBit::UseMemoryTagging); - } - void disableMemoryTagging() { Options.clear(OptionBit::UseMemoryTagging); } - const char *getRegionInfoArrayAddress() const { return reinterpret_cast(RegionInfoArray); } @@ -335,7 +325,7 @@ template class SizeClassAllocator64 { if (!map(reinterpret_cast(RegionBeg + MappedUser), UserMapSize, "scudo:primary", MAP_ALLOWNOMEM | MAP_RESIZABLE | - (useMemoryTagging(Options.load()) ? MAP_MEMTAG : 0), + (useMemoryTagging(Options.load()) ? MAP_MEMTAG : 0), &Region->Data)) return nullptr; Region->MappedUser += UserMapSize; diff --git a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp index 53874933e7ac3e..7bb6725d3a5202 100644 --- a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp @@ -47,7 +47,7 @@ bool isPrimaryAllocation(scudo::uptr Size, scudo::uptr Alignment) { template bool isTaggedAllocation(AllocatorT *Allocator, scudo::uptr Size, scudo::uptr Alignment) { - return Allocator->useMemoryTagging() && + return Allocator->useMemoryTaggingTestOnly() && scudo::systemDetectsMemoryTagFaultsTestOnly() && isPrimaryAllocation(Size, Alignment); } @@ -162,7 +162,7 @@ template static void testAllocator() { for (scudo::uptr I = 0; I < Size; I++) { unsigned char V = (reinterpret_cast(P))[I]; if (isPrimaryAllocation(Size, 1U << MinAlignLog) && - !Allocator->useMemoryTagging()) + !Allocator->useMemoryTaggingTestOnly()) ASSERT_EQ(V, scudo::PatternFillByte); else ASSERT_TRUE(V == scudo::PatternFillByte || V == 0); @@ -248,7 +248,7 @@ template static void testAllocator() { Allocator->releaseToOS(); - if (Allocator->useMemoryTagging() && + if (Allocator->useMemoryTaggingTestOnly() && scudo::systemDetectsMemoryTagFaultsTestOnly()) { // Check that use-after-free is detected. for (scudo::uptr SizeLog = 0U; SizeLog <= 20U; SizeLog++) { @@ -493,7 +493,7 @@ TEST(ScudoCombinedTest, OddEven) { using SizeClassMap = AllocatorT::PrimaryT::SizeClassMap; auto Allocator = std::unique_ptr(new AllocatorT()); - if (!Allocator->useMemoryTagging()) + if (!Allocator->useMemoryTaggingTestOnly()) return; auto CheckOddEven = [](scudo::uptr P1, scudo::uptr P2) {