diff --git a/compiler-rt/lib/scudo/CMakeLists.txt b/compiler-rt/lib/scudo/CMakeLists.txt new file mode 100644 index 00000000000000..c75ba2540afc43 --- /dev/null +++ b/compiler-rt/lib/scudo/CMakeLists.txt @@ -0,0 +1,160 @@ +add_compiler_rt_component(scudo) + +include_directories(..) + +set(SCUDO_CFLAGS ${SANITIZER_COMMON_CFLAGS}) +# SANITIZER_COMMON_CFLAGS include -fno-builtin, but we actually want builtins! +list(APPEND SCUDO_CFLAGS -fbuiltin) +append_rtti_flag(OFF SCUDO_CFLAGS) + +# Too many existing bugs, needs cleanup. +append_list_if(COMPILER_RT_HAS_WNO_FORMAT -Wno-format SCUDO_CFLAGS) + +set(SCUDO_MINIMAL_DYNAMIC_LIBS ${SANITIZER_COMMON_LINK_LIBS}) +append_list_if(COMPILER_RT_HAS_LIBDL dl SCUDO_MINIMAL_DYNAMIC_LIBS) +append_list_if(COMPILER_RT_HAS_LIBRT rt SCUDO_MINIMAL_DYNAMIC_LIBS) +append_list_if(COMPILER_RT_HAS_LIBPTHREAD pthread SCUDO_MINIMAL_DYNAMIC_LIBS) +append_list_if(COMPILER_RT_HAS_LIBLOG log SCUDO_MINIMAL_DYNAMIC_LIBS) +append_list_if(COMPILER_RT_HAS_OMIT_FRAME_POINTER_FLAG -fno-omit-frame-pointer + SCUDO_CFLAGS) + +set(SCUDO_DYNAMIC_LINK_FLAGS ${SANITIZER_COMMON_LINK_FLAGS}) +# Use gc-sections by default to avoid unused code being pulled in. +list(APPEND SCUDO_DYNAMIC_LINK_FLAGS -Wl,--gc-sections) + +if(ANDROID) +# Put most Sanitizer shared libraries in the global group. For more details, see +# android-changes-for-ndk-developers.md#changes-to-library-search-order + if (COMPILER_RT_HAS_Z_GLOBAL) + list(APPEND SCUDO_DYNAMIC_LINK_FLAGS -Wl,-z,global) + endif() +endif() + +# The minimal Scudo runtime does not include the UBSan runtime. +set(SCUDO_MINIMAL_OBJECT_LIBS + RTSanitizerCommonNoTermination + RTSanitizerCommonLibc + RTInterception) + +if (COMPILER_RT_HAS_GWP_ASAN) + list(APPEND SCUDO_MINIMAL_OBJECT_LIBS + RTGwpAsan RTGwpAsanOptionsParser RTGwpAsanBacktraceLibc + RTGwpAsanSegvHandler) + list(APPEND SCUDO_CFLAGS -DGWP_ASAN_HOOKS) +endif() + +set(SCUDO_OBJECT_LIBS ${SCUDO_MINIMAL_OBJECT_LIBS}) +set(SCUDO_DYNAMIC_LIBS ${SCUDO_MINIMAL_DYNAMIC_LIBS}) + +if (FUCHSIA) + list(APPEND SCUDO_CFLAGS -nostdinc++) + list(APPEND SCUDO_DYNAMIC_LINK_FLAGS -nostdlib++) +else() + list(APPEND SCUDO_DYNAMIC_LIBS + ${COMPILER_RT_UNWINDER_LINK_LIBS} + ${SANITIZER_CXX_ABI_LIBRARIES}) + list(APPEND SCUDO_OBJECT_LIBS + RTSanitizerCommonCoverage + RTSanitizerCommonSymbolizer + RTUbsan) +endif() + +set(SCUDO_SOURCES + scudo_allocator.cpp + scudo_crc32.cpp + scudo_errors.cpp + scudo_flags.cpp + scudo_malloc.cpp + scudo_termination.cpp + scudo_tsd_exclusive.cpp + scudo_tsd_shared.cpp + scudo_utils.cpp) + +set(SCUDO_CXX_SOURCES + scudo_new_delete.cpp) + +set(SCUDO_HEADERS + scudo_allocator.h + scudo_allocator_combined.h + scudo_allocator_secondary.h + scudo_crc32.h + scudo_errors.h + scudo_flags.h + scudo_flags.inc + scudo_interface_internal.h + scudo_platform.h + scudo_tsd.h + scudo_tsd_exclusive.inc + scudo_tsd_shared.inc + scudo_utils.h) + +# Enable the necessary instruction set for scudo_crc32.cpp, if available. +# Newer compiler versions use -mcrc32 rather than -msse4.2. +if (COMPILER_RT_HAS_MCRC32_FLAG) + set_source_files_properties(scudo_crc32.cpp PROPERTIES COMPILE_FLAGS -mcrc32) +elseif (COMPILER_RT_HAS_MSSE4_2_FLAG) + set_source_files_properties(scudo_crc32.cpp PROPERTIES COMPILE_FLAGS -msse4.2) +endif() + +# Enable the AArch64 CRC32 feature for scudo_crc32.cpp, if available. +# Note that it is enabled by default starting with armv8.1-a. +if (COMPILER_RT_HAS_MCRC_FLAG) + set_source_files_properties(scudo_crc32.cpp PROPERTIES COMPILE_FLAGS -mcrc) +endif() + +if(COMPILER_RT_HAS_SCUDO) + add_compiler_rt_runtime(clang_rt.scudo_minimal + STATIC + ARCHS ${SCUDO_SUPPORTED_ARCH} + SOURCES ${SCUDO_SOURCES} + ADDITIONAL_HEADERS ${SCUDO_HEADERS} + OBJECT_LIBS ${SCUDO_MINIMAL_OBJECT_LIBS} + CFLAGS ${SCUDO_CFLAGS} + PARENT_TARGET scudo) + add_compiler_rt_runtime(clang_rt.scudo_cxx_minimal + STATIC + ARCHS ${SCUDO_SUPPORTED_ARCH} + SOURCES ${SCUDO_CXX_SOURCES} + ADDITIONAL_HEADERS ${SCUDO_HEADERS} + CFLAGS ${SCUDO_CFLAGS} + PARENT_TARGET scudo) + + add_compiler_rt_runtime(clang_rt.scudo + STATIC + ARCHS ${SCUDO_SUPPORTED_ARCH} + SOURCES ${SCUDO_SOURCES} + ADDITIONAL_HEADERS ${SCUDO_HEADERS} + OBJECT_LIBS ${SCUDO_OBJECT_LIBS} + CFLAGS ${SCUDO_CFLAGS} + PARENT_TARGET scudo) + add_compiler_rt_runtime(clang_rt.scudo_cxx + STATIC + ARCHS ${SCUDO_SUPPORTED_ARCH} + SOURCES ${SCUDO_CXX_SOURCES} + ADDITIONAL_HEADERS ${SCUDO_HEADERS} + OBJECT_LIBS RTUbsan_cxx + CFLAGS ${SCUDO_CFLAGS} + PARENT_TARGET scudo) + + add_compiler_rt_runtime(clang_rt.scudo_minimal + SHARED + ARCHS ${SCUDO_SUPPORTED_ARCH} + SOURCES ${SCUDO_SOURCES} ${SCUDO_CXX_SOURCES} + ADDITIONAL_HEADERS ${SCUDO_HEADERS} + OBJECT_LIBS ${SCUDO_MINIMAL_OBJECT_LIBS} + CFLAGS ${SCUDO_CFLAGS} + LINK_FLAGS ${SCUDO_DYNAMIC_LINK_FLAGS} + LINK_LIBS ${SCUDO_MINIMAL_DYNAMIC_LIBS} + PARENT_TARGET scudo) + + add_compiler_rt_runtime(clang_rt.scudo + SHARED + ARCHS ${SCUDO_SUPPORTED_ARCH} + SOURCES ${SCUDO_SOURCES} ${SCUDO_CXX_SOURCES} + ADDITIONAL_HEADERS ${SCUDO_HEADERS} + OBJECT_LIBS ${SCUDO_OBJECT_LIBS} + CFLAGS ${SCUDO_CFLAGS} + LINK_FLAGS ${SCUDO_DYNAMIC_LINK_FLAGS} + LINK_LIBS ${SCUDO_DYNAMIC_LIBS} + PARENT_TARGET scudo) +endif() diff --git a/compiler-rt/lib/scudo/scudo_allocator.cpp b/compiler-rt/lib/scudo/scudo_allocator.cpp new file mode 100644 index 00000000000000..6a6b577ab00251 --- /dev/null +++ b/compiler-rt/lib/scudo/scudo_allocator.cpp @@ -0,0 +1,831 @@ +//===-- scudo_allocator.cpp -------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// Scudo Hardened Allocator implementation. +/// It uses the sanitizer_common allocator as a base and aims at mitigating +/// heap corruption vulnerabilities. It provides a checksum-guarded chunk +/// header, a delayed free list, and additional sanity checks. +/// +//===----------------------------------------------------------------------===// + +#include "scudo_allocator.h" +#include "scudo_crc32.h" +#include "scudo_errors.h" +#include "scudo_flags.h" +#include "scudo_interface_internal.h" +#include "scudo_tsd.h" +#include "scudo_utils.h" + +#include "sanitizer_common/sanitizer_allocator_checks.h" +#include "sanitizer_common/sanitizer_allocator_interface.h" +#include "sanitizer_common/sanitizer_quarantine.h" + +#ifdef GWP_ASAN_HOOKS +# include "gwp_asan/guarded_pool_allocator.h" +# include "gwp_asan/optional/backtrace.h" +# include "gwp_asan/optional/options_parser.h" +#include "gwp_asan/optional/segv_handler.h" +#endif // GWP_ASAN_HOOKS + +#include +#include + +namespace __scudo { + +// Global static cookie, initialized at start-up. +static u32 Cookie; + +// We default to software CRC32 if the alternatives are not supported, either +// at compilation or at runtime. +static atomic_uint8_t HashAlgorithm = { CRC32Software }; + +inline u32 computeCRC32(u32 Crc, uptr Value, uptr *Array, uptr ArraySize) { + // If the hardware CRC32 feature is defined here, it was enabled everywhere, + // as opposed to only for scudo_crc32.cpp. This means that other hardware + // specific instructions were likely emitted at other places, and as a + // result there is no reason to not use it here. +#if defined(__CRC32__) || defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) + Crc = CRC32_INTRINSIC(Crc, Value); + for (uptr i = 0; i < ArraySize; i++) + Crc = CRC32_INTRINSIC(Crc, Array[i]); + return Crc; +#else + if (atomic_load_relaxed(&HashAlgorithm) == CRC32Hardware) { + Crc = computeHardwareCRC32(Crc, Value); + for (uptr i = 0; i < ArraySize; i++) + Crc = computeHardwareCRC32(Crc, Array[i]); + return Crc; + } + Crc = computeSoftwareCRC32(Crc, Value); + for (uptr i = 0; i < ArraySize; i++) + Crc = computeSoftwareCRC32(Crc, Array[i]); + return Crc; +#endif // defined(__CRC32__) || defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) +} + +static BackendT &getBackend(); + +namespace Chunk { + static inline AtomicPackedHeader *getAtomicHeader(void *Ptr) { + return reinterpret_cast(reinterpret_cast(Ptr) - + getHeaderSize()); + } + static inline + const AtomicPackedHeader *getConstAtomicHeader(const void *Ptr) { + return reinterpret_cast( + reinterpret_cast(Ptr) - getHeaderSize()); + } + + static inline bool isAligned(const void *Ptr) { + return IsAligned(reinterpret_cast(Ptr), MinAlignment); + } + + // We can't use the offset member of the chunk itself, as we would double + // fetch it without any warranty that it wouldn't have been tampered. To + // prevent this, we work with a local copy of the header. + static inline void *getBackendPtr(const void *Ptr, UnpackedHeader *Header) { + return reinterpret_cast(reinterpret_cast(Ptr) - + getHeaderSize() - (Header->Offset << MinAlignmentLog)); + } + + // Returns the usable size for a chunk, meaning the amount of bytes from the + // beginning of the user data to the end of the backend allocated chunk. + static inline uptr getUsableSize(const void *Ptr, UnpackedHeader *Header) { + const uptr ClassId = Header->ClassId; + if (ClassId) + return PrimaryT::ClassIdToSize(ClassId) - getHeaderSize() - + (Header->Offset << MinAlignmentLog); + return SecondaryT::GetActuallyAllocatedSize( + getBackendPtr(Ptr, Header)) - getHeaderSize(); + } + + // Returns the size the user requested when allocating the chunk. + static inline uptr getSize(const void *Ptr, UnpackedHeader *Header) { + const uptr SizeOrUnusedBytes = Header->SizeOrUnusedBytes; + if (Header->ClassId) + return SizeOrUnusedBytes; + return SecondaryT::GetActuallyAllocatedSize( + getBackendPtr(Ptr, Header)) - getHeaderSize() - SizeOrUnusedBytes; + } + + // Compute the checksum of the chunk pointer and its header. + static inline u16 computeChecksum(const void *Ptr, UnpackedHeader *Header) { + UnpackedHeader ZeroChecksumHeader = *Header; + ZeroChecksumHeader.Checksum = 0; + uptr HeaderHolder[sizeof(UnpackedHeader) / sizeof(uptr)]; + memcpy(&HeaderHolder, &ZeroChecksumHeader, sizeof(HeaderHolder)); + const u32 Crc = computeCRC32(Cookie, reinterpret_cast(Ptr), + HeaderHolder, ARRAY_SIZE(HeaderHolder)); + return static_cast(Crc); + } + + // Checks the validity of a chunk by verifying its checksum. It doesn't + // incur termination in the event of an invalid chunk. + static inline bool isValid(const void *Ptr) { + PackedHeader NewPackedHeader = + atomic_load_relaxed(getConstAtomicHeader(Ptr)); + UnpackedHeader NewUnpackedHeader = + bit_cast(NewPackedHeader); + return (NewUnpackedHeader.Checksum == + computeChecksum(Ptr, &NewUnpackedHeader)); + } + + // Ensure that ChunkAvailable is 0, so that if a 0 checksum is ever valid + // for a fully nulled out header, its state will be available anyway. + COMPILER_CHECK(ChunkAvailable == 0); + + // Loads and unpacks the header, verifying the checksum in the process. + static inline + void loadHeader(const void *Ptr, UnpackedHeader *NewUnpackedHeader) { + PackedHeader NewPackedHeader = + atomic_load_relaxed(getConstAtomicHeader(Ptr)); + *NewUnpackedHeader = bit_cast(NewPackedHeader); + if (UNLIKELY(NewUnpackedHeader->Checksum != + computeChecksum(Ptr, NewUnpackedHeader))) + dieWithMessage("corrupted chunk header at address %p\n", Ptr); + } + + // Packs and stores the header, computing the checksum in the process. + static inline void storeHeader(void *Ptr, UnpackedHeader *NewUnpackedHeader) { + NewUnpackedHeader->Checksum = computeChecksum(Ptr, NewUnpackedHeader); + PackedHeader NewPackedHeader = bit_cast(*NewUnpackedHeader); + atomic_store_relaxed(getAtomicHeader(Ptr), NewPackedHeader); + } + + // Packs and stores the header, computing the checksum in the process. We + // compare the current header with the expected provided one to ensure that + // we are not being raced by a corruption occurring in another thread. + static inline void compareExchangeHeader(void *Ptr, + UnpackedHeader *NewUnpackedHeader, + UnpackedHeader *OldUnpackedHeader) { + NewUnpackedHeader->Checksum = computeChecksum(Ptr, NewUnpackedHeader); + PackedHeader NewPackedHeader = bit_cast(*NewUnpackedHeader); + PackedHeader OldPackedHeader = bit_cast(*OldUnpackedHeader); + if (UNLIKELY(!atomic_compare_exchange_strong( + getAtomicHeader(Ptr), &OldPackedHeader, NewPackedHeader, + memory_order_relaxed))) + dieWithMessage("race on chunk header at address %p\n", Ptr); + } +} // namespace Chunk + +struct QuarantineCallback { + explicit QuarantineCallback(AllocatorCacheT *Cache) + : Cache_(Cache) {} + + // Chunk recycling function, returns a quarantined chunk to the backend, + // first making sure it hasn't been tampered with. + void Recycle(void *Ptr) { + UnpackedHeader Header; + Chunk::loadHeader(Ptr, &Header); + if (UNLIKELY(Header.State != ChunkQuarantine)) + dieWithMessage("invalid chunk state when recycling address %p\n", Ptr); + UnpackedHeader NewHeader = Header; + NewHeader.State = ChunkAvailable; + Chunk::compareExchangeHeader(Ptr, &NewHeader, &Header); + void *BackendPtr = Chunk::getBackendPtr(Ptr, &Header); + if (Header.ClassId) + getBackend().deallocatePrimary(Cache_, BackendPtr, Header.ClassId); + else + getBackend().deallocateSecondary(BackendPtr); + } + + // Internal quarantine allocation and deallocation functions. We first check + // that the batches are indeed serviced by the Primary. + // TODO(kostyak): figure out the best way to protect the batches. + void *Allocate(uptr Size) { + const uptr BatchClassId = SizeClassMap::ClassID(sizeof(QuarantineBatch)); + return getBackend().allocatePrimary(Cache_, BatchClassId); + } + + void Deallocate(void *Ptr) { + const uptr BatchClassId = SizeClassMap::ClassID(sizeof(QuarantineBatch)); + getBackend().deallocatePrimary(Cache_, Ptr, BatchClassId); + } + + AllocatorCacheT *Cache_; + COMPILER_CHECK(sizeof(QuarantineBatch) < SizeClassMap::kMaxSize); +}; + +typedef Quarantine QuarantineT; +typedef QuarantineT::Cache QuarantineCacheT; +COMPILER_CHECK(sizeof(QuarantineCacheT) <= + sizeof(ScudoTSD::QuarantineCachePlaceHolder)); + +QuarantineCacheT *getQuarantineCache(ScudoTSD *TSD) { + return reinterpret_cast(TSD->QuarantineCachePlaceHolder); +} + +#ifdef GWP_ASAN_HOOKS +static gwp_asan::GuardedPoolAllocator GuardedAlloc; +#endif // GWP_ASAN_HOOKS + +struct Allocator { + static const uptr MaxAllowedMallocSize = + FIRST_32_SECOND_64(2UL << 30, 1ULL << 40); + + BackendT Backend; + QuarantineT Quarantine; + + u32 QuarantineChunksUpToSize; + + bool DeallocationTypeMismatch; + bool ZeroContents; + bool DeleteSizeMismatch; + + bool CheckRssLimit; + uptr HardRssLimitMb; + uptr SoftRssLimitMb; + atomic_uint8_t RssLimitExceeded; + atomic_uint64_t RssLastCheckedAtNS; + + explicit Allocator(LinkerInitialized) + : Quarantine(LINKER_INITIALIZED) {} + + NOINLINE void performSanityChecks(); + + void init() { + SanitizerToolName = "Scudo"; + PrimaryAllocatorName = "ScudoPrimary"; + SecondaryAllocatorName = "ScudoSecondary"; + + initFlags(); + + performSanityChecks(); + + // Check if hardware CRC32 is supported in the binary and by the platform, + // if so, opt for the CRC32 hardware version of the checksum. + if (&computeHardwareCRC32 && hasHardwareCRC32()) + atomic_store_relaxed(&HashAlgorithm, CRC32Hardware); + + SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null); + Backend.init(common_flags()->allocator_release_to_os_interval_ms); + HardRssLimitMb = common_flags()->hard_rss_limit_mb; + SoftRssLimitMb = common_flags()->soft_rss_limit_mb; + Quarantine.Init( + static_cast(getFlags()->QuarantineSizeKb) << 10, + static_cast(getFlags()->ThreadLocalQuarantineSizeKb) << 10); + QuarantineChunksUpToSize = (Quarantine.GetCacheSize() == 0) ? 0 : + getFlags()->QuarantineChunksUpToSize; + DeallocationTypeMismatch = getFlags()->DeallocationTypeMismatch; + DeleteSizeMismatch = getFlags()->DeleteSizeMismatch; + ZeroContents = getFlags()->ZeroContents; + + if (UNLIKELY(!GetRandom(reinterpret_cast(&Cookie), sizeof(Cookie), + /*blocking=*/false))) { + Cookie = static_cast((NanoTime() >> 12) ^ + (reinterpret_cast(this) >> 4)); + } + + CheckRssLimit = HardRssLimitMb || SoftRssLimitMb; + if (CheckRssLimit) + atomic_store_relaxed(&RssLastCheckedAtNS, MonotonicNanoTime()); + } + + // Helper function that checks for a valid Scudo chunk. nullptr isn't. + bool isValidPointer(const void *Ptr) { + initThreadMaybe(); + if (UNLIKELY(!Ptr)) + return false; + if (!Chunk::isAligned(Ptr)) + return false; + return Chunk::isValid(Ptr); + } + + NOINLINE bool isRssLimitExceeded(); + + // Allocates a chunk. + void * + allocate(uptr Size, uptr Alignment, AllocType Type, + bool ForceZeroContents = false) SANITIZER_NO_THREAD_SAFETY_ANALYSIS { + initThreadMaybe(); + + if (UNLIKELY(Alignment > MaxAlignment)) { + if (AllocatorMayReturnNull()) + return nullptr; + reportAllocationAlignmentTooBig(Alignment, MaxAlignment); + } + if (UNLIKELY(Alignment < MinAlignment)) + Alignment = MinAlignment; + +#ifdef GWP_ASAN_HOOKS + if (UNLIKELY(GuardedAlloc.shouldSample())) { + if (void *Ptr = GuardedAlloc.allocate(Size, Alignment)) { + if (SCUDO_CAN_USE_HOOKS && &__sanitizer_malloc_hook) + __sanitizer_malloc_hook(Ptr, Size); + return Ptr; + } + } +#endif // GWP_ASAN_HOOKS + + const uptr NeededSize = RoundUpTo(Size ? Size : 1, MinAlignment) + + Chunk::getHeaderSize(); + const uptr AlignedSize = (Alignment > MinAlignment) ? + NeededSize + (Alignment - Chunk::getHeaderSize()) : NeededSize; + if (UNLIKELY(Size >= MaxAllowedMallocSize) || + UNLIKELY(AlignedSize >= MaxAllowedMallocSize)) { + if (AllocatorMayReturnNull()) + return nullptr; + reportAllocationSizeTooBig(Size, AlignedSize, MaxAllowedMallocSize); + } + + if (CheckRssLimit && UNLIKELY(isRssLimitExceeded())) { + if (AllocatorMayReturnNull()) + return nullptr; + reportRssLimitExceeded(); + } + + // Primary and Secondary backed allocations have a different treatment. We + // deal with alignment requirements of Primary serviced allocations here, + // but the Secondary will take care of its own alignment needs. + void *BackendPtr; + uptr BackendSize; + u8 ClassId; + if (PrimaryT::CanAllocate(AlignedSize, MinAlignment)) { + BackendSize = AlignedSize; + ClassId = SizeClassMap::ClassID(BackendSize); + bool UnlockRequired; + ScudoTSD *TSD = getTSDAndLock(&UnlockRequired); + BackendPtr = Backend.allocatePrimary(&TSD->Cache, ClassId); + if (UnlockRequired) + TSD->unlock(); + } else { + BackendSize = NeededSize; + ClassId = 0; + BackendPtr = Backend.allocateSecondary(BackendSize, Alignment); + } + if (UNLIKELY(!BackendPtr)) { + SetAllocatorOutOfMemory(); + if (AllocatorMayReturnNull()) + return nullptr; + reportOutOfMemory(Size); + } + + // If requested, we will zero out the entire contents of the returned chunk. + if ((ForceZeroContents || ZeroContents) && ClassId) + memset(BackendPtr, 0, PrimaryT::ClassIdToSize(ClassId)); + + UnpackedHeader Header = {}; + uptr UserPtr = reinterpret_cast(BackendPtr) + Chunk::getHeaderSize(); + if (UNLIKELY(!IsAligned(UserPtr, Alignment))) { + // Since the Secondary takes care of alignment, a non-aligned pointer + // means it is from the Primary. It is also the only case where the offset + // field of the header would be non-zero. + DCHECK(ClassId); + const uptr AlignedUserPtr = RoundUpTo(UserPtr, Alignment); + Header.Offset = (AlignedUserPtr - UserPtr) >> MinAlignmentLog; + UserPtr = AlignedUserPtr; + } + DCHECK_LE(UserPtr + Size, reinterpret_cast(BackendPtr) + BackendSize); + Header.State = ChunkAllocated; + Header.AllocType = Type; + if (ClassId) { + Header.ClassId = ClassId; + Header.SizeOrUnusedBytes = Size; + } else { + // The secondary fits the allocations to a page, so the amount of unused + // bytes is the difference between the end of the user allocation and the + // next page boundary. + const uptr PageSize = GetPageSizeCached(); + const uptr TrailingBytes = (UserPtr + Size) & (PageSize - 1); + if (TrailingBytes) + Header.SizeOrUnusedBytes = PageSize - TrailingBytes; + } + void *Ptr = reinterpret_cast(UserPtr); + Chunk::storeHeader(Ptr, &Header); + if (SCUDO_CAN_USE_HOOKS && &__sanitizer_malloc_hook) + __sanitizer_malloc_hook(Ptr, Size); + return Ptr; + } + + // Place a chunk in the quarantine or directly deallocate it in the event of + // a zero-sized quarantine, or if the size of the chunk is greater than the + // quarantine chunk size threshold. + void quarantineOrDeallocateChunk(void *Ptr, UnpackedHeader *Header, uptr Size) + SANITIZER_NO_THREAD_SAFETY_ANALYSIS { + const bool BypassQuarantine = !Size || (Size > QuarantineChunksUpToSize); + if (BypassQuarantine) { + UnpackedHeader NewHeader = *Header; + NewHeader.State = ChunkAvailable; + Chunk::compareExchangeHeader(Ptr, &NewHeader, Header); + void *BackendPtr = Chunk::getBackendPtr(Ptr, Header); + if (Header->ClassId) { + bool UnlockRequired; + ScudoTSD *TSD = getTSDAndLock(&UnlockRequired); + getBackend().deallocatePrimary(&TSD->Cache, BackendPtr, + Header->ClassId); + if (UnlockRequired) + TSD->unlock(); + } else { + getBackend().deallocateSecondary(BackendPtr); + } + } else { + // If a small memory amount was allocated with a larger alignment, we want + // to take that into account. Otherwise the Quarantine would be filled + // with tiny chunks, taking a lot of VA memory. This is an approximation + // of the usable size, that allows us to not call + // GetActuallyAllocatedSize. + const uptr EstimatedSize = Size + (Header->Offset << MinAlignmentLog); + UnpackedHeader NewHeader = *Header; + NewHeader.State = ChunkQuarantine; + Chunk::compareExchangeHeader(Ptr, &NewHeader, Header); + bool UnlockRequired; + ScudoTSD *TSD = getTSDAndLock(&UnlockRequired); + Quarantine.Put(getQuarantineCache(TSD), QuarantineCallback(&TSD->Cache), + Ptr, EstimatedSize); + if (UnlockRequired) + TSD->unlock(); + } + } + + // Deallocates a Chunk, which means either adding it to the quarantine or + // directly returning it to the backend if criteria are met. + void deallocate(void *Ptr, uptr DeleteSize, uptr DeleteAlignment, + AllocType Type) { + // For a deallocation, we only ensure minimal initialization, meaning thread + // local data will be left uninitialized for now (when using ELF TLS). The + // fallback cache will be used instead. This is a workaround for a situation + // where the only heap operation performed in a thread would be a free past + // the TLS destructors, ending up in initialized thread specific data never + // being destroyed properly. Any other heap operation will do a full init. + initThreadMaybe(/*MinimalInit=*/true); + if (SCUDO_CAN_USE_HOOKS && &__sanitizer_free_hook) + __sanitizer_free_hook(Ptr); + if (UNLIKELY(!Ptr)) + return; + +#ifdef GWP_ASAN_HOOKS + if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr))) { + GuardedAlloc.deallocate(Ptr); + return; + } +#endif // GWP_ASAN_HOOKS + + if (UNLIKELY(!Chunk::isAligned(Ptr))) + dieWithMessage("misaligned pointer when deallocating address %p\n", Ptr); + UnpackedHeader Header; + Chunk::loadHeader(Ptr, &Header); + if (UNLIKELY(Header.State != ChunkAllocated)) + dieWithMessage("invalid chunk state when deallocating address %p\n", Ptr); + if (DeallocationTypeMismatch) { + // The deallocation type has to match the allocation one. + if (Header.AllocType != Type) { + // With the exception of memalign'd Chunks, that can be still be free'd. + if (Header.AllocType != FromMemalign || Type != FromMalloc) + dieWithMessage("allocation type mismatch when deallocating address " + "%p\n", Ptr); + } + } + const uptr Size = Chunk::getSize(Ptr, &Header); + if (DeleteSizeMismatch) { + if (DeleteSize && DeleteSize != Size) + dieWithMessage("invalid sized delete when deallocating address %p\n", + Ptr); + } + (void)DeleteAlignment; // TODO(kostyak): verify that the alignment matches. + quarantineOrDeallocateChunk(Ptr, &Header, Size); + } + + // Reallocates a chunk. We can save on a new allocation if the new requested + // size still fits in the chunk. + void *reallocate(void *OldPtr, uptr NewSize) { + initThreadMaybe(); + +#ifdef GWP_ASAN_HOOKS + if (UNLIKELY(GuardedAlloc.pointerIsMine(OldPtr))) { + size_t OldSize = GuardedAlloc.getSize(OldPtr); + void *NewPtr = allocate(NewSize, MinAlignment, FromMalloc); + if (NewPtr) + memcpy(NewPtr, OldPtr, (NewSize < OldSize) ? NewSize : OldSize); + GuardedAlloc.deallocate(OldPtr); + return NewPtr; + } +#endif // GWP_ASAN_HOOKS + + if (UNLIKELY(!Chunk::isAligned(OldPtr))) + dieWithMessage("misaligned address when reallocating address %p\n", + OldPtr); + UnpackedHeader OldHeader; + Chunk::loadHeader(OldPtr, &OldHeader); + if (UNLIKELY(OldHeader.State != ChunkAllocated)) + dieWithMessage("invalid chunk state when reallocating address %p\n", + OldPtr); + if (DeallocationTypeMismatch) { + if (UNLIKELY(OldHeader.AllocType != FromMalloc)) + dieWithMessage("allocation type mismatch when reallocating address " + "%p\n", OldPtr); + } + const uptr UsableSize = Chunk::getUsableSize(OldPtr, &OldHeader); + // The new size still fits in the current chunk, and the size difference + // is reasonable. + if (NewSize <= UsableSize && + (UsableSize - NewSize) < (SizeClassMap::kMaxSize / 2)) { + UnpackedHeader NewHeader = OldHeader; + NewHeader.SizeOrUnusedBytes = + OldHeader.ClassId ? NewSize : UsableSize - NewSize; + Chunk::compareExchangeHeader(OldPtr, &NewHeader, &OldHeader); + return OldPtr; + } + // Otherwise, we have to allocate a new chunk and copy the contents of the + // old one. + void *NewPtr = allocate(NewSize, MinAlignment, FromMalloc); + if (NewPtr) { + const uptr OldSize = OldHeader.ClassId ? OldHeader.SizeOrUnusedBytes : + UsableSize - OldHeader.SizeOrUnusedBytes; + memcpy(NewPtr, OldPtr, Min(NewSize, UsableSize)); + quarantineOrDeallocateChunk(OldPtr, &OldHeader, OldSize); + } + return NewPtr; + } + + // Helper function that returns the actual usable size of a chunk. + uptr getUsableSize(const void *Ptr) { + initThreadMaybe(); + if (UNLIKELY(!Ptr)) + return 0; + +#ifdef GWP_ASAN_HOOKS + if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr))) + return GuardedAlloc.getSize(Ptr); +#endif // GWP_ASAN_HOOKS + + UnpackedHeader Header; + Chunk::loadHeader(Ptr, &Header); + // Getting the usable size of a chunk only makes sense if it's allocated. + if (UNLIKELY(Header.State != ChunkAllocated)) + dieWithMessage("invalid chunk state when sizing address %p\n", Ptr); + return Chunk::getUsableSize(Ptr, &Header); + } + + void *calloc(uptr NMemB, uptr Size) { + initThreadMaybe(); + if (UNLIKELY(CheckForCallocOverflow(NMemB, Size))) { + if (AllocatorMayReturnNull()) + return nullptr; + reportCallocOverflow(NMemB, Size); + } + return allocate(NMemB * Size, MinAlignment, FromMalloc, true); + } + + void commitBack(ScudoTSD *TSD) { + Quarantine.Drain(getQuarantineCache(TSD), QuarantineCallback(&TSD->Cache)); + Backend.destroyCache(&TSD->Cache); + } + + uptr getStats(AllocatorStat StatType) { + initThreadMaybe(); + uptr stats[AllocatorStatCount]; + Backend.getStats(stats); + return stats[StatType]; + } + + bool canReturnNull() { + initThreadMaybe(); + return AllocatorMayReturnNull(); + } + + void setRssLimit(uptr LimitMb, bool HardLimit) { + if (HardLimit) + HardRssLimitMb = LimitMb; + else + SoftRssLimitMb = LimitMb; + CheckRssLimit = HardRssLimitMb || SoftRssLimitMb; + } + + void printStats() { + initThreadMaybe(); + Backend.printStats(); + } +}; + +NOINLINE void Allocator::performSanityChecks() { + // Verify that the header offset field can hold the maximum offset. In the + // case of the Secondary allocator, it takes care of alignment and the + // offset will always be 0. In the case of the Primary, the worst case + // scenario happens in the last size class, when the backend allocation + // would already be aligned on the requested alignment, which would happen + // to be the maximum alignment that would fit in that size class. As a + // result, the maximum offset will be at most the maximum alignment for the + // last size class minus the header size, in multiples of MinAlignment. + UnpackedHeader Header = {}; + const uptr MaxPrimaryAlignment = + 1 << MostSignificantSetBitIndex(SizeClassMap::kMaxSize - MinAlignment); + const uptr MaxOffset = + (MaxPrimaryAlignment - Chunk::getHeaderSize()) >> MinAlignmentLog; + Header.Offset = MaxOffset; + if (Header.Offset != MaxOffset) + dieWithMessage("maximum possible offset doesn't fit in header\n"); + // Verify that we can fit the maximum size or amount of unused bytes in the + // header. Given that the Secondary fits the allocation to a page, the worst + // case scenario happens in the Primary. It will depend on the second to + // last and last class sizes, as well as the dynamic base for the Primary. + // The following is an over-approximation that works for our needs. + const uptr MaxSizeOrUnusedBytes = SizeClassMap::kMaxSize - 1; + Header.SizeOrUnusedBytes = MaxSizeOrUnusedBytes; + if (Header.SizeOrUnusedBytes != MaxSizeOrUnusedBytes) + dieWithMessage("maximum possible unused bytes doesn't fit in header\n"); + + const uptr LargestClassId = SizeClassMap::kLargestClassID; + Header.ClassId = LargestClassId; + if (Header.ClassId != LargestClassId) + dieWithMessage("largest class ID doesn't fit in header\n"); +} + +// Opportunistic RSS limit check. This will update the RSS limit status, if +// it can, every 250ms, otherwise it will just return the current one. +NOINLINE bool Allocator::isRssLimitExceeded() { + u64 LastCheck = atomic_load_relaxed(&RssLastCheckedAtNS); + const u64 CurrentCheck = MonotonicNanoTime(); + if (LIKELY(CurrentCheck < LastCheck + (250ULL * 1000000ULL))) + return atomic_load_relaxed(&RssLimitExceeded); + if (!atomic_compare_exchange_weak(&RssLastCheckedAtNS, &LastCheck, + CurrentCheck, memory_order_relaxed)) + return atomic_load_relaxed(&RssLimitExceeded); + // TODO(kostyak): We currently use sanitizer_common's GetRSS which reads the + // RSS from /proc/self/statm by default. We might want to + // call getrusage directly, even if it's less accurate. + const uptr CurrentRssMb = GetRSS() >> 20; + if (HardRssLimitMb && UNLIKELY(HardRssLimitMb < CurrentRssMb)) + dieWithMessage("hard RSS limit exhausted (%zdMb vs %zdMb)\n", + HardRssLimitMb, CurrentRssMb); + if (SoftRssLimitMb) { + if (atomic_load_relaxed(&RssLimitExceeded)) { + if (CurrentRssMb <= SoftRssLimitMb) + atomic_store_relaxed(&RssLimitExceeded, false); + } else { + if (CurrentRssMb > SoftRssLimitMb) { + atomic_store_relaxed(&RssLimitExceeded, true); + Printf("Scudo INFO: soft RSS limit exhausted (%zdMb vs %zdMb)\n", + SoftRssLimitMb, CurrentRssMb); + } + } + } + return atomic_load_relaxed(&RssLimitExceeded); +} + +static Allocator Instance(LINKER_INITIALIZED); + +static BackendT &getBackend() { + return Instance.Backend; +} + +void initScudo() { + Instance.init(); +#ifdef GWP_ASAN_HOOKS + gwp_asan::options::initOptions(__sanitizer::GetEnv("GWP_ASAN_OPTIONS"), + Printf); + gwp_asan::options::Options &Opts = gwp_asan::options::getOptions(); + Opts.Backtrace = gwp_asan::backtrace::getBacktraceFunction(); + GuardedAlloc.init(Opts); + + if (Opts.InstallSignalHandlers) + gwp_asan::segv_handler::installSignalHandlers( + &GuardedAlloc, __sanitizer::Printf, + gwp_asan::backtrace::getPrintBacktraceFunction(), + gwp_asan::backtrace::getSegvBacktraceFunction()); +#endif // GWP_ASAN_HOOKS +} + +void ScudoTSD::init() { + getBackend().initCache(&Cache); + memset(QuarantineCachePlaceHolder, 0, sizeof(QuarantineCachePlaceHolder)); +} + +void ScudoTSD::commitBack() { + Instance.commitBack(this); +} + +void *scudoAllocate(uptr Size, uptr Alignment, AllocType Type) { + if (Alignment && UNLIKELY(!IsPowerOfTwo(Alignment))) { + errno = EINVAL; + if (Instance.canReturnNull()) + return nullptr; + reportAllocationAlignmentNotPowerOfTwo(Alignment); + } + return SetErrnoOnNull(Instance.allocate(Size, Alignment, Type)); +} + +void scudoDeallocate(void *Ptr, uptr Size, uptr Alignment, AllocType Type) { + Instance.deallocate(Ptr, Size, Alignment, Type); +} + +void *scudoRealloc(void *Ptr, uptr Size) { + if (!Ptr) + return SetErrnoOnNull(Instance.allocate(Size, MinAlignment, FromMalloc)); + if (Size == 0) { + Instance.deallocate(Ptr, 0, 0, FromMalloc); + return nullptr; + } + return SetErrnoOnNull(Instance.reallocate(Ptr, Size)); +} + +void *scudoCalloc(uptr NMemB, uptr Size) { + return SetErrnoOnNull(Instance.calloc(NMemB, Size)); +} + +void *scudoValloc(uptr Size) { + return SetErrnoOnNull( + Instance.allocate(Size, GetPageSizeCached(), FromMemalign)); +} + +void *scudoPvalloc(uptr Size) { + const uptr PageSize = GetPageSizeCached(); + if (UNLIKELY(CheckForPvallocOverflow(Size, PageSize))) { + errno = ENOMEM; + if (Instance.canReturnNull()) + return nullptr; + reportPvallocOverflow(Size); + } + // pvalloc(0) should allocate one page. + Size = Size ? RoundUpTo(Size, PageSize) : PageSize; + return SetErrnoOnNull(Instance.allocate(Size, PageSize, FromMemalign)); +} + +int scudoPosixMemalign(void **MemPtr, uptr Alignment, uptr Size) { + if (UNLIKELY(!CheckPosixMemalignAlignment(Alignment))) { + if (!Instance.canReturnNull()) + reportInvalidPosixMemalignAlignment(Alignment); + return EINVAL; + } + void *Ptr = Instance.allocate(Size, Alignment, FromMemalign); + if (UNLIKELY(!Ptr)) + return ENOMEM; + *MemPtr = Ptr; + return 0; +} + +void *scudoAlignedAlloc(uptr Alignment, uptr Size) { + if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(Alignment, Size))) { + errno = EINVAL; + if (Instance.canReturnNull()) + return nullptr; + reportInvalidAlignedAllocAlignment(Size, Alignment); + } + return SetErrnoOnNull(Instance.allocate(Size, Alignment, FromMalloc)); +} + +uptr scudoMallocUsableSize(void *Ptr) { + return Instance.getUsableSize(Ptr); +} + +} // namespace __scudo + +using namespace __scudo; + +// MallocExtension helper functions + +uptr __sanitizer_get_current_allocated_bytes() { + return Instance.getStats(AllocatorStatAllocated); +} + +uptr __sanitizer_get_heap_size() { + return Instance.getStats(AllocatorStatMapped); +} + +uptr __sanitizer_get_free_bytes() { + return 1; +} + +uptr __sanitizer_get_unmapped_bytes() { + return 1; +} + +uptr __sanitizer_get_estimated_allocated_size(uptr Size) { + return Size; +} + +int __sanitizer_get_ownership(const void *Ptr) { + return Instance.isValidPointer(Ptr); +} + +uptr __sanitizer_get_allocated_size(const void *Ptr) { + return Instance.getUsableSize(Ptr); +} + +#if !SANITIZER_SUPPORTS_WEAK_HOOKS +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_malloc_hook, + void *Ptr, uptr Size) { + (void)Ptr; + (void)Size; +} + +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_free_hook, void *Ptr) { + (void)Ptr; +} +#endif + +// Interface functions + +void __scudo_set_rss_limit(uptr LimitMb, s32 HardLimit) { + if (!SCUDO_CAN_USE_PUBLIC_INTERFACE) + return; + Instance.setRssLimit(LimitMb, !!HardLimit); +} + +void __scudo_print_stats() { + Instance.printStats(); +} diff --git a/compiler-rt/lib/scudo/scudo_allocator.h b/compiler-rt/lib/scudo/scudo_allocator.h new file mode 100644 index 00000000000000..0efa5c52029613 --- /dev/null +++ b/compiler-rt/lib/scudo/scudo_allocator.h @@ -0,0 +1,125 @@ +//===-- scudo_allocator.h ---------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// Header for scudo_allocator.cpp. +/// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_ALLOCATOR_H_ +#define SCUDO_ALLOCATOR_H_ + +#include "scudo_platform.h" + +namespace __scudo { + +enum AllocType : u8 { + FromMalloc = 0, // Memory block came from malloc, realloc, calloc, etc. + FromNew = 1, // Memory block came from operator new. + FromNewArray = 2, // Memory block came from operator new []. + FromMemalign = 3, // Memory block came from memalign, posix_memalign, etc. +}; + +enum ChunkState : u8 { + ChunkAvailable = 0, + ChunkAllocated = 1, + ChunkQuarantine = 2 +}; + +// Our header requires 64 bits of storage. Having the offset saves us from +// using functions such as GetBlockBegin, that is fairly costly. Our first +// implementation used the MetaData as well, which offers the advantage of +// being stored away from the chunk itself, but accessing it was costly as +// well. The header will be atomically loaded and stored. +typedef u64 PackedHeader; +struct UnpackedHeader { + u64 Checksum : 16; + u64 ClassId : 8; + u64 SizeOrUnusedBytes : 20; // Size for Primary backed allocations, amount of + // unused bytes in the chunk for Secondary ones. + u64 State : 2; // available, allocated, or quarantined + u64 AllocType : 2; // malloc, new, new[], or memalign + u64 Offset : 16; // Offset from the beginning of the backend + // allocation to the beginning of the chunk + // itself, in multiples of MinAlignment. See + // comment about its maximum value and in init(). +}; + +typedef atomic_uint64_t AtomicPackedHeader; +COMPILER_CHECK(sizeof(UnpackedHeader) == sizeof(PackedHeader)); + +// Minimum alignment of 8 bytes for 32-bit, 16 for 64-bit +const uptr MinAlignmentLog = FIRST_32_SECOND_64(3, 4); +const uptr MaxAlignmentLog = 24; // 16 MB +const uptr MinAlignment = 1 << MinAlignmentLog; +const uptr MaxAlignment = 1 << MaxAlignmentLog; + +// constexpr version of __sanitizer::RoundUp without the extraneous CHECK. +// This way we can use it in constexpr variables and functions declarations. +constexpr uptr RoundUpTo(uptr Size, uptr Boundary) { + return (Size + Boundary - 1) & ~(Boundary - 1); +} + +namespace Chunk { + constexpr uptr getHeaderSize() { + return RoundUpTo(sizeof(PackedHeader), MinAlignment); + } +} + +#if SANITIZER_CAN_USE_ALLOCATOR64 +const uptr AllocatorSpace = ~0ULL; +struct AP64 { + static const uptr kSpaceBeg = AllocatorSpace; + static const uptr kSpaceSize = AllocatorSize; + static const uptr kMetadataSize = 0; + typedef __scudo::SizeClassMap SizeClassMap; + typedef NoOpMapUnmapCallback MapUnmapCallback; + static const uptr kFlags = + SizeClassAllocator64FlagMasks::kRandomShuffleChunks; + using AddressSpaceView = LocalAddressSpaceView; +}; +typedef SizeClassAllocator64 PrimaryT; +#else +struct AP32 { + static const uptr kSpaceBeg = 0; + static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; + static const uptr kMetadataSize = 0; + typedef __scudo::SizeClassMap SizeClassMap; + static const uptr kRegionSizeLog = RegionSizeLog; + using AddressSpaceView = LocalAddressSpaceView; + typedef NoOpMapUnmapCallback MapUnmapCallback; + static const uptr kFlags = + SizeClassAllocator32FlagMasks::kRandomShuffleChunks | + SizeClassAllocator32FlagMasks::kUseSeparateSizeClassForBatch; +}; +typedef SizeClassAllocator32 PrimaryT; +#endif // SANITIZER_CAN_USE_ALLOCATOR64 + +#include "scudo_allocator_secondary.h" + +typedef LargeMmapAllocator SecondaryT; + +#include "scudo_allocator_combined.h" + +typedef CombinedAllocator BackendT; +typedef CombinedAllocator::AllocatorCache AllocatorCacheT; + +void initScudo(); + +void *scudoAllocate(uptr Size, uptr Alignment, AllocType Type); +void scudoDeallocate(void *Ptr, uptr Size, uptr Alignment, AllocType Type); +void *scudoRealloc(void *Ptr, uptr Size); +void *scudoCalloc(uptr NMemB, uptr Size); +void *scudoValloc(uptr Size); +void *scudoPvalloc(uptr Size); +int scudoPosixMemalign(void **MemPtr, uptr Alignment, uptr Size); +void *scudoAlignedAlloc(uptr Alignment, uptr Size); +uptr scudoMallocUsableSize(void *Ptr); + +} // namespace __scudo + +#endif // SCUDO_ALLOCATOR_H_ diff --git a/compiler-rt/lib/scudo/scudo_allocator_combined.h b/compiler-rt/lib/scudo/scudo_allocator_combined.h new file mode 100644 index 00000000000000..d61cc9ec1a5285 --- /dev/null +++ b/compiler-rt/lib/scudo/scudo_allocator_combined.h @@ -0,0 +1,75 @@ +//===-- scudo_allocator_combined.h ------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// Scudo Combined Allocator, dispatches allocation & deallocation requests to +/// the Primary or the Secondary backend allocators. +/// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_ALLOCATOR_COMBINED_H_ +#define SCUDO_ALLOCATOR_COMBINED_H_ + +#ifndef SCUDO_ALLOCATOR_H_ +# error "This file must be included inside scudo_allocator.h." +#endif + +class CombinedAllocator { + public: + using PrimaryAllocator = PrimaryT; + using SecondaryAllocator = SecondaryT; + using AllocatorCache = typename PrimaryAllocator::AllocatorCache; + void init(s32 ReleaseToOSIntervalMs) { + Primary.Init(ReleaseToOSIntervalMs); + Secondary.Init(); + Stats.Init(); + } + + // Primary allocations are always MinAlignment aligned, and as such do not + // require an Alignment parameter. + void *allocatePrimary(AllocatorCache *Cache, uptr ClassId) { + return Cache->Allocate(&Primary, ClassId); + } + + // Secondary allocations do not require a Cache, but do require an Alignment + // parameter. + void *allocateSecondary(uptr Size, uptr Alignment) { + return Secondary.Allocate(&Stats, Size, Alignment); + } + + void deallocatePrimary(AllocatorCache *Cache, void *Ptr, uptr ClassId) { + Cache->Deallocate(&Primary, ClassId, Ptr); + } + + void deallocateSecondary(void *Ptr) { + Secondary.Deallocate(&Stats, Ptr); + } + + void initCache(AllocatorCache *Cache) { + Cache->Init(&Stats); + } + + void destroyCache(AllocatorCache *Cache) { + Cache->Destroy(&Primary, &Stats); + } + + void getStats(AllocatorStatCounters StatType) const { + Stats.Get(StatType); + } + + void printStats() { + Primary.PrintStats(); + Secondary.PrintStats(); + } + + private: + PrimaryAllocator Primary; + SecondaryAllocator Secondary; + AllocatorGlobalStats Stats; +}; + +#endif // SCUDO_ALLOCATOR_COMBINED_H_ diff --git a/compiler-rt/lib/scudo/scudo_allocator_secondary.h b/compiler-rt/lib/scudo/scudo_allocator_secondary.h new file mode 100644 index 00000000000000..80198c4aebf5bf --- /dev/null +++ b/compiler-rt/lib/scudo/scudo_allocator_secondary.h @@ -0,0 +1,192 @@ +//===-- scudo_allocator_secondary.h -----------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// Scudo Secondary Allocator. +/// This services allocation that are too large to be serviced by the Primary +/// Allocator. It is directly backed by the memory mapping functions of the +/// operating system. +/// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_ALLOCATOR_SECONDARY_H_ +#define SCUDO_ALLOCATOR_SECONDARY_H_ + +#ifndef SCUDO_ALLOCATOR_H_ +# error "This file must be included inside scudo_allocator.h." +#endif + +// Secondary backed allocations are standalone chunks that contain extra +// information stored in a LargeChunk::Header prior to the frontend's header. +// +// The secondary takes care of alignment requirements (so that it can release +// unnecessary pages in the rare event of larger alignments), and as such must +// know about the frontend's header size. +// +// Since Windows doesn't support partial releasing of a reserved memory region, +// we have to keep track of both the reserved and the committed memory. +// +// The resulting chunk resembles the following: +// +// +--------------------+ +// | Guard page(s) | +// +--------------------+ +// | Unused space* | +// +--------------------+ +// | LargeChunk::Header | +// +--------------------+ +// | {Unp,P}ackedHeader | +// +--------------------+ +// | Data (aligned) | +// +--------------------+ +// | Unused space** | +// +--------------------+ +// | Guard page(s) | +// +--------------------+ + +namespace LargeChunk { +struct Header { + ReservedAddressRange StoredRange; + uptr CommittedSize; + uptr Size; +}; +constexpr uptr getHeaderSize() { + return RoundUpTo(sizeof(Header), MinAlignment); +} +static Header *getHeader(uptr Ptr) { + return reinterpret_cast
(Ptr - getHeaderSize()); +} +static Header *getHeader(const void *Ptr) { + return getHeader(reinterpret_cast(Ptr)); +} +} // namespace LargeChunk + +class LargeMmapAllocator { + public: + void Init() { + internal_memset(this, 0, sizeof(*this)); + } + + void *Allocate(AllocatorStats *Stats, uptr Size, uptr Alignment) { + const uptr UserSize = Size - Chunk::getHeaderSize(); + // The Scudo frontend prevents us from allocating more than + // MaxAllowedMallocSize, so integer overflow checks would be superfluous. + uptr ReservedSize = Size + LargeChunk::getHeaderSize(); + if (UNLIKELY(Alignment > MinAlignment)) + ReservedSize += Alignment; + const uptr PageSize = GetPageSizeCached(); + ReservedSize = RoundUpTo(ReservedSize, PageSize); + // Account for 2 guard pages, one before and one after the chunk. + ReservedSize += 2 * PageSize; + + ReservedAddressRange AddressRange; + uptr ReservedBeg = AddressRange.Init(ReservedSize, SecondaryAllocatorName); + if (UNLIKELY(ReservedBeg == ~static_cast(0))) + return nullptr; + // A page-aligned pointer is assumed after that, so check it now. + DCHECK(IsAligned(ReservedBeg, PageSize)); + uptr ReservedEnd = ReservedBeg + ReservedSize; + // The beginning of the user area for that allocation comes after the + // initial guard page, and both headers. This is the pointer that has to + // abide by alignment requirements. + uptr CommittedBeg = ReservedBeg + PageSize; + uptr UserBeg = CommittedBeg + HeadersSize; + uptr UserEnd = UserBeg + UserSize; + uptr CommittedEnd = RoundUpTo(UserEnd, PageSize); + + // In the rare event of larger alignments, we will attempt to fit the mmap + // area better and unmap extraneous memory. This will also ensure that the + // offset and unused bytes field of the header stay small. + if (UNLIKELY(Alignment > MinAlignment)) { + if (!IsAligned(UserBeg, Alignment)) { + UserBeg = RoundUpTo(UserBeg, Alignment); + CommittedBeg = RoundDownTo(UserBeg - HeadersSize, PageSize); + const uptr NewReservedBeg = CommittedBeg - PageSize; + DCHECK_GE(NewReservedBeg, ReservedBeg); + if (!SANITIZER_WINDOWS && NewReservedBeg != ReservedBeg) { + AddressRange.Unmap(ReservedBeg, NewReservedBeg - ReservedBeg); + ReservedBeg = NewReservedBeg; + } + UserEnd = UserBeg + UserSize; + CommittedEnd = RoundUpTo(UserEnd, PageSize); + } + const uptr NewReservedEnd = CommittedEnd + PageSize; + DCHECK_LE(NewReservedEnd, ReservedEnd); + if (!SANITIZER_WINDOWS && NewReservedEnd != ReservedEnd) { + AddressRange.Unmap(NewReservedEnd, ReservedEnd - NewReservedEnd); + ReservedEnd = NewReservedEnd; + } + } + + DCHECK_LE(UserEnd, CommittedEnd); + const uptr CommittedSize = CommittedEnd - CommittedBeg; + // Actually mmap the memory, preserving the guard pages on either sides. + CHECK_EQ(CommittedBeg, AddressRange.Map(CommittedBeg, CommittedSize)); + const uptr Ptr = UserBeg - Chunk::getHeaderSize(); + LargeChunk::Header *H = LargeChunk::getHeader(Ptr); + H->StoredRange = AddressRange; + H->Size = CommittedEnd - Ptr; + H->CommittedSize = CommittedSize; + + // The primary adds the whole class size to the stats when allocating a + // chunk, so we will do something similar here. But we will not account for + // the guard pages. + { + SpinMutexLock l(&StatsMutex); + Stats->Add(AllocatorStatAllocated, CommittedSize); + Stats->Add(AllocatorStatMapped, CommittedSize); + AllocatedBytes += CommittedSize; + if (LargestSize < CommittedSize) + LargestSize = CommittedSize; + NumberOfAllocs++; + } + + return reinterpret_cast(Ptr); + } + + void Deallocate(AllocatorStats *Stats, void *Ptr) { + LargeChunk::Header *H = LargeChunk::getHeader(Ptr); + // Since we're unmapping the entirety of where the ReservedAddressRange + // actually is, copy onto the stack. + ReservedAddressRange AddressRange = H->StoredRange; + const uptr Size = H->CommittedSize; + { + SpinMutexLock l(&StatsMutex); + Stats->Sub(AllocatorStatAllocated, Size); + Stats->Sub(AllocatorStatMapped, Size); + FreedBytes += Size; + NumberOfFrees++; + } + AddressRange.Unmap(reinterpret_cast(AddressRange.base()), + AddressRange.size()); + } + + static uptr GetActuallyAllocatedSize(void *Ptr) { + return LargeChunk::getHeader(Ptr)->Size; + } + + void PrintStats() { + Printf("Stats: LargeMmapAllocator: allocated %zd times (%zd K), " + "freed %zd times (%zd K), remains %zd (%zd K) max %zd M\n", + NumberOfAllocs, AllocatedBytes >> 10, NumberOfFrees, + FreedBytes >> 10, NumberOfAllocs - NumberOfFrees, + (AllocatedBytes - FreedBytes) >> 10, LargestSize >> 20); + } + + private: + static constexpr uptr HeadersSize = + LargeChunk::getHeaderSize() + Chunk::getHeaderSize(); + + StaticSpinMutex StatsMutex; + u32 NumberOfAllocs; + u32 NumberOfFrees; + uptr AllocatedBytes; + uptr FreedBytes; + uptr LargestSize; +}; + +#endif // SCUDO_ALLOCATOR_SECONDARY_H_ diff --git a/compiler-rt/lib/scudo/scudo_crc32.cpp b/compiler-rt/lib/scudo/scudo_crc32.cpp new file mode 100644 index 00000000000000..137c44c5c1cda4 --- /dev/null +++ b/compiler-rt/lib/scudo/scudo_crc32.cpp @@ -0,0 +1,24 @@ +//===-- scudo_crc32.cpp -----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// CRC32 function leveraging hardware specific instructions. This has to be +/// kept separated to restrict the use of compiler specific flags to this file. +/// +//===----------------------------------------------------------------------===// + +#include "scudo_crc32.h" + +namespace __scudo { + +#if defined(__CRC32__) || defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) +u32 computeHardwareCRC32(u32 Crc, uptr Data) { + return CRC32_INTRINSIC(Crc, Data); +} +#endif // defined(__CRC32__) || defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) + +} // namespace __scudo diff --git a/compiler-rt/lib/scudo/scudo_crc32.h b/compiler-rt/lib/scudo/scudo_crc32.h new file mode 100644 index 00000000000000..4314d30e929fea --- /dev/null +++ b/compiler-rt/lib/scudo/scudo_crc32.h @@ -0,0 +1,104 @@ +//===-- scudo_crc32.h -------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// Scudo chunk header checksum related definitions. +/// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_CRC32_H_ +#define SCUDO_CRC32_H_ + +#include "sanitizer_common/sanitizer_internal_defs.h" + +// Hardware CRC32 is supported at compilation via the following: +// - for i386 & x86_64: -mcrc32 (earlier: -msse4.2) +// - for ARM & AArch64: -march=armv8-a+crc or -mcrc +// An additional check must be performed at runtime as well to make sure the +// emitted instructions are valid on the target host. + +#if defined(__CRC32__) || defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) +# if defined(__CRC32__) +// NB: clang has but GCC does not +# include +# define CRC32_INTRINSIC FIRST_32_SECOND_64(__builtin_ia32_crc32si, __builtin_ia32_crc32di) +# elif defined(__SSE4_2__) +# include +# define CRC32_INTRINSIC FIRST_32_SECOND_64(_mm_crc32_u32, _mm_crc32_u64) +# endif +# ifdef __ARM_FEATURE_CRC32 +# include +# define CRC32_INTRINSIC FIRST_32_SECOND_64(__crc32cw, __crc32cd) +# endif +#endif // defined(__CRC32__) || defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) + +namespace __scudo { + +enum : u8 { + CRC32Software = 0, + CRC32Hardware = 1, +}; + +static const u32 CRC32Table[] = { + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, + 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, + 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2, + 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, + 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, + 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, + 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c, + 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, + 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, + 0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, + 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106, + 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, + 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, + 0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, + 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, + 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, + 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, + 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, + 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, + 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, + 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, + 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84, + 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, + 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, + 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, + 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e, + 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, + 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, + 0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, + 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28, + 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, + 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, + 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, + 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, + 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, + 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, + 0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, + 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, + 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, + 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, + 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d +}; + +inline u32 computeSoftwareCRC32(u32 Crc, uptr Data) { + for (uptr i = 0; i < sizeof(Data); i++) { + Crc = CRC32Table[(Crc ^ Data) & 0xff] ^ (Crc >> 8); + Data >>= 8; + } + return Crc; +} + +SANITIZER_WEAK_ATTRIBUTE u32 computeHardwareCRC32(u32 Crc, uptr Data); + +} // namespace __scudo + +#endif // SCUDO_CRC32_H_ diff --git a/compiler-rt/lib/scudo/scudo_errors.cpp b/compiler-rt/lib/scudo/scudo_errors.cpp new file mode 100644 index 00000000000000..4bea9ebc6ab056 --- /dev/null +++ b/compiler-rt/lib/scudo/scudo_errors.cpp @@ -0,0 +1,77 @@ +//===-- scudo_errors.cpp ----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// Verbose termination functions. +/// +//===----------------------------------------------------------------------===// + +#include "scudo_utils.h" + +#include "sanitizer_common/sanitizer_flags.h" + +namespace __scudo { + +void NORETURN reportCallocOverflow(uptr Count, uptr Size) { + dieWithMessage("calloc parameters overflow: count * size (%zd * %zd) cannot " + "be represented with type size_t\n", Count, Size); +} + +void NORETURN reportPvallocOverflow(uptr Size) { + dieWithMessage("pvalloc parameters overflow: size 0x%zx rounded up to system " + "page size 0x%zx cannot be represented in type size_t\n", Size, + GetPageSizeCached()); +} + +void NORETURN reportAllocationAlignmentTooBig(uptr Alignment, + uptr MaxAlignment) { + dieWithMessage("invalid allocation alignment: %zd exceeds maximum supported " + "allocation of %zd\n", Alignment, MaxAlignment); +} + +void NORETURN reportAllocationAlignmentNotPowerOfTwo(uptr Alignment) { + dieWithMessage("invalid allocation alignment: %zd, alignment must be a power " + "of two\n", Alignment); +} + +void NORETURN reportInvalidPosixMemalignAlignment(uptr Alignment) { + dieWithMessage( + "invalid alignment requested in posix_memalign: %zd, alignment" + " must be a power of two and a multiple of sizeof(void *) == %zd\n", + Alignment, sizeof(void *)); +} + +void NORETURN reportInvalidAlignedAllocAlignment(uptr Size, uptr Alignment) { +#if SANITIZER_POSIX + dieWithMessage("invalid alignment requested in aligned_alloc: %zd, alignment " + "must be a power of two and the requested size 0x%zx must be a multiple " + "of alignment\n", Alignment, Size); +#else + dieWithMessage("invalid alignment requested in aligned_alloc: %zd, the " + "requested size 0x%zx must be a multiple of alignment\n", Alignment, + Size); +#endif +} + +void NORETURN reportAllocationSizeTooBig(uptr UserSize, uptr TotalSize, + uptr MaxSize) { + dieWithMessage("requested allocation size 0x%zx (0x%zx after adjustments) " + "exceeds maximum supported size of 0x%zx\n", UserSize, TotalSize, + MaxSize); +} + +void NORETURN reportRssLimitExceeded() { + dieWithMessage("specified RSS limit exceeded, currently set to " + "soft_rss_limit_mb=%zd\n", common_flags()->soft_rss_limit_mb); +} + +void NORETURN reportOutOfMemory(uptr RequestedSize) { + dieWithMessage("allocator is out of memory trying to allocate 0x%zx bytes\n", + RequestedSize); +} + +} // namespace __scudo diff --git a/compiler-rt/lib/scudo/scudo_errors.h b/compiler-rt/lib/scudo/scudo_errors.h new file mode 100644 index 00000000000000..258695c2c02c7b --- /dev/null +++ b/compiler-rt/lib/scudo/scudo_errors.h @@ -0,0 +1,34 @@ +//===-- scudo_errors.h ------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// Header for scudo_errors.cpp. +/// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_ERRORS_H_ +#define SCUDO_ERRORS_H_ + +#include "sanitizer_common/sanitizer_internal_defs.h" + +namespace __scudo { + +void NORETURN reportCallocOverflow(uptr Count, uptr Size); +void NORETURN reportPvallocOverflow(uptr Size); +void NORETURN reportAllocationAlignmentTooBig(uptr Alignment, + uptr MaxAlignment); +void NORETURN reportAllocationAlignmentNotPowerOfTwo(uptr Alignment); +void NORETURN reportInvalidPosixMemalignAlignment(uptr Alignment); +void NORETURN reportInvalidAlignedAllocAlignment(uptr Size, uptr Alignment); +void NORETURN reportAllocationSizeTooBig(uptr UserSize, uptr TotalSize, + uptr MaxSize); +void NORETURN reportRssLimitExceeded(); +void NORETURN reportOutOfMemory(uptr RequestedSize); + +} // namespace __scudo + +#endif // SCUDO_ERRORS_H_ diff --git a/compiler-rt/lib/scudo/scudo_flags.cpp b/compiler-rt/lib/scudo/scudo_flags.cpp new file mode 100644 index 00000000000000..c564e217b35bff --- /dev/null +++ b/compiler-rt/lib/scudo/scudo_flags.cpp @@ -0,0 +1,136 @@ +//===-- scudo_flags.cpp -----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// Hardened Allocator flag parsing logic. +/// +//===----------------------------------------------------------------------===// + +#include "scudo_flags.h" +#include "scudo_interface_internal.h" +#include "scudo_utils.h" + +#include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_flag_parser.h" + +namespace __scudo { + +static Flags ScudoFlags; // Use via getFlags(). + +void Flags::setDefaults() { +#define SCUDO_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue; +#include "scudo_flags.inc" +#undef SCUDO_FLAG +} + +static void RegisterScudoFlags(FlagParser *parser, Flags *f) { +#define SCUDO_FLAG(Type, Name, DefaultValue, Description) \ + RegisterFlag(parser, #Name, Description, &f->Name); +#include "scudo_flags.inc" +#undef SCUDO_FLAG +} + +static const char *getCompileDefinitionScudoDefaultOptions() { +#ifdef SCUDO_DEFAULT_OPTIONS + return SANITIZER_STRINGIFY(SCUDO_DEFAULT_OPTIONS); +#else + return ""; +#endif +} + +static const char *getScudoDefaultOptions() { + return (&__scudo_default_options) ? __scudo_default_options() : ""; +} + +void initFlags() { + SetCommonFlagsDefaults(); + { + CommonFlags cf; + cf.CopyFrom(*common_flags()); + cf.exitcode = 1; + OverrideCommonFlags(cf); + } + Flags *f = getFlags(); + f->setDefaults(); + + FlagParser ScudoParser; + RegisterScudoFlags(&ScudoParser, f); + RegisterCommonFlags(&ScudoParser); + + // Override from compile definition. + ScudoParser.ParseString(getCompileDefinitionScudoDefaultOptions()); + + // Override from user-specified string. + ScudoParser.ParseString(getScudoDefaultOptions()); + + // Override from environment. + ScudoParser.ParseStringFromEnv("SCUDO_OPTIONS"); + + InitializeCommonFlags(); + + // Sanity checks and default settings for the Quarantine parameters. + + if (f->QuarantineSizeMb >= 0) { + // Backward compatible logic if QuarantineSizeMb is set. + if (f->QuarantineSizeKb >= 0) { + dieWithMessage("ERROR: please use either QuarantineSizeMb (deprecated) " + "or QuarantineSizeKb, but not both\n"); + } + if (f->QuarantineChunksUpToSize >= 0) { + dieWithMessage("ERROR: QuarantineChunksUpToSize cannot be used in " + " conjunction with the deprecated QuarantineSizeMb option\n"); + } + // If everything is in order, update QuarantineSizeKb accordingly. + f->QuarantineSizeKb = f->QuarantineSizeMb * 1024; + } else { + // Otherwise proceed with the new options. + if (f->QuarantineSizeKb < 0) { + const int DefaultQuarantineSizeKb = FIRST_32_SECOND_64(64, 256); + f->QuarantineSizeKb = DefaultQuarantineSizeKb; + } + if (f->QuarantineChunksUpToSize < 0) { + const int DefaultQuarantineChunksUpToSize = FIRST_32_SECOND_64(512, 2048); + f->QuarantineChunksUpToSize = DefaultQuarantineChunksUpToSize; + } + } + + // We enforce an upper limit for the chunk quarantine threshold of 4Mb. + if (f->QuarantineChunksUpToSize > (4 * 1024 * 1024)) { + dieWithMessage("ERROR: the chunk quarantine threshold is too large\n"); + } + + // We enforce an upper limit for the quarantine size of 32Mb. + if (f->QuarantineSizeKb > (32 * 1024)) { + dieWithMessage("ERROR: the quarantine size is too large\n"); + } + + if (f->ThreadLocalQuarantineSizeKb < 0) { + const int DefaultThreadLocalQuarantineSizeKb = FIRST_32_SECOND_64(16, 64); + f->ThreadLocalQuarantineSizeKb = DefaultThreadLocalQuarantineSizeKb; + } + // And an upper limit of 8Mb for the thread quarantine cache. + if (f->ThreadLocalQuarantineSizeKb > (8 * 1024)) { + dieWithMessage("ERROR: the per thread quarantine cache size is too " + "large\n"); + } + if (f->ThreadLocalQuarantineSizeKb == 0 && f->QuarantineSizeKb > 0) { + dieWithMessage("ERROR: ThreadLocalQuarantineSizeKb can be set to 0 only " + "when QuarantineSizeKb is set to 0\n"); + } +} + +Flags *getFlags() { + return &ScudoFlags; +} + +} // namespace __scudo + +#if !SANITIZER_SUPPORTS_WEAK_HOOKS +SANITIZER_INTERFACE_WEAK_DEF(const char*, __scudo_default_options, void) { + return ""; +} +#endif diff --git a/compiler-rt/lib/scudo/scudo_flags.h b/compiler-rt/lib/scudo/scudo_flags.h new file mode 100644 index 00000000000000..483c79621cbf45 --- /dev/null +++ b/compiler-rt/lib/scudo/scudo_flags.h @@ -0,0 +1,32 @@ +//===-- scudo_flags.h -------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// Header for scudo_flags.cpp. +/// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_FLAGS_H_ +#define SCUDO_FLAGS_H_ + +namespace __scudo { + +struct Flags { +#define SCUDO_FLAG(Type, Name, DefaultValue, Description) Type Name; +#include "scudo_flags.inc" +#undef SCUDO_FLAG + + void setDefaults(); +}; + +Flags *getFlags(); + +void initFlags(); + +} // namespace __scudo + +#endif // SCUDO_FLAGS_H_ diff --git a/compiler-rt/lib/scudo/scudo_flags.inc b/compiler-rt/lib/scudo/scudo_flags.inc new file mode 100644 index 00000000000000..c124738c1f3a71 --- /dev/null +++ b/compiler-rt/lib/scudo/scudo_flags.inc @@ -0,0 +1,48 @@ +//===-- scudo_flags.inc -----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// Hardened Allocator runtime flags. +/// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_FLAG +# error "Define SCUDO_FLAG prior to including this file!" +#endif + +SCUDO_FLAG(int, QuarantineSizeMb, -1, + "Deprecated. Please use QuarantineSizeKb.") + +// Default value is set in scudo_flags.cpp based on architecture. +SCUDO_FLAG(int, QuarantineSizeKb, -1, + "Size in KB of quarantine used to delay the actual deallocation of " + "chunks. Lower value may reduce memory usage but decrease the " + "effectiveness of the mitigation. Defaults to 64KB (32-bit) or " + "256KB (64-bit)") + +// Default value is set in scudo_flags.cpp based on architecture. +SCUDO_FLAG(int, ThreadLocalQuarantineSizeKb, -1, + "Size in KB of per-thread cache used to offload the global " + "quarantine. Lower value may reduce memory usage but might increase " + "the contention on the global quarantine. Defaults to 16KB (32-bit) " + "or 64KB (64-bit)") + +// Default value is set in scudo_flags.cpp based on architecture. +SCUDO_FLAG(int, QuarantineChunksUpToSize, -1, + "Size in bytes up to which chunks will be quarantined (if lower than" + "or equal to). Defaults to 256 (32-bit) or 2048 (64-bit)") + +// Disable the deallocation type check by default on Android, it causes too many +// issues with third party libraries. +SCUDO_FLAG(bool, DeallocationTypeMismatch, !SANITIZER_ANDROID, + "Report errors on malloc/delete, new/free, new/delete[], etc.") + +SCUDO_FLAG(bool, DeleteSizeMismatch, true, + "Report errors on mismatch between size of new and delete.") + +SCUDO_FLAG(bool, ZeroContents, false, + "Zero chunk contents on allocation and deallocation.") diff --git a/compiler-rt/lib/scudo/scudo_interface_internal.h b/compiler-rt/lib/scudo/scudo_interface_internal.h new file mode 100644 index 00000000000000..75c63aa6d48936 --- /dev/null +++ b/compiler-rt/lib/scudo/scudo_interface_internal.h @@ -0,0 +1,32 @@ +//===-- scudo_interface_internal.h ------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// Private Scudo interface header. +/// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_INTERFACE_INTERNAL_H_ +#define SCUDO_INTERFACE_INTERNAL_H_ + +#include "sanitizer_common/sanitizer_internal_defs.h" + +using __sanitizer::uptr; +using __sanitizer::s32; + +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE +const char* __scudo_default_options(); + +SANITIZER_INTERFACE_ATTRIBUTE +void __scudo_set_rss_limit(uptr LimitMb, s32 HardLimit); + +SANITIZER_INTERFACE_ATTRIBUTE +void __scudo_print_stats(); +} // extern "C" + +#endif // SCUDO_INTERFACE_INTERNAL_H_ diff --git a/compiler-rt/lib/scudo/scudo_malloc.cpp b/compiler-rt/lib/scudo/scudo_malloc.cpp new file mode 100644 index 00000000000000..a72b861e28ee42 --- /dev/null +++ b/compiler-rt/lib/scudo/scudo_malloc.cpp @@ -0,0 +1,84 @@ +//===-- scudo_malloc.cpp ----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// Interceptors for malloc related functions. +/// +//===----------------------------------------------------------------------===// + +#include "scudo_allocator.h" + +#include "interception/interception.h" +#include "sanitizer_common/sanitizer_platform_interceptors.h" + +#include + +using namespace __scudo; + +extern "C" { +INTERCEPTOR_ATTRIBUTE void free(void *ptr) { + scudoDeallocate(ptr, 0, 0, FromMalloc); +} + +INTERCEPTOR_ATTRIBUTE void *malloc(size_t size) { + return scudoAllocate(size, 0, FromMalloc); +} + +INTERCEPTOR_ATTRIBUTE void *realloc(void *ptr, size_t size) { + return scudoRealloc(ptr, size); +} + +INTERCEPTOR_ATTRIBUTE void *calloc(size_t nmemb, size_t size) { + return scudoCalloc(nmemb, size); +} + +INTERCEPTOR_ATTRIBUTE void *valloc(size_t size) { + return scudoValloc(size); +} + +INTERCEPTOR_ATTRIBUTE +int posix_memalign(void **memptr, size_t alignment, size_t size) { + return scudoPosixMemalign(memptr, alignment, size); +} + +#if SANITIZER_INTERCEPT_CFREE +INTERCEPTOR_ATTRIBUTE void cfree(void *ptr) ALIAS("free"); +#endif + +#if SANITIZER_INTERCEPT_MEMALIGN +INTERCEPTOR_ATTRIBUTE void *memalign(size_t alignment, size_t size) { + return scudoAllocate(size, alignment, FromMemalign); +} + +INTERCEPTOR_ATTRIBUTE +void *__libc_memalign(size_t alignment, size_t size) ALIAS("memalign"); +#endif + +#if SANITIZER_INTERCEPT_PVALLOC +INTERCEPTOR_ATTRIBUTE void *pvalloc(size_t size) { + return scudoPvalloc(size); +} +#endif + +#if SANITIZER_INTERCEPT_ALIGNED_ALLOC +INTERCEPTOR_ATTRIBUTE void *aligned_alloc(size_t alignment, size_t size) { + return scudoAlignedAlloc(alignment, size); +} +#endif + +#if SANITIZER_INTERCEPT_MALLOC_USABLE_SIZE +INTERCEPTOR_ATTRIBUTE size_t malloc_usable_size(void *ptr) { + return scudoMallocUsableSize(ptr); +} +#endif + +#if SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO +INTERCEPTOR_ATTRIBUTE int mallopt(int cmd, int value) { + return 0; +} +#endif +} // extern "C" diff --git a/compiler-rt/lib/scudo/scudo_new_delete.cpp b/compiler-rt/lib/scudo/scudo_new_delete.cpp new file mode 100644 index 00000000000000..03eef7f28bb92a --- /dev/null +++ b/compiler-rt/lib/scudo/scudo_new_delete.cpp @@ -0,0 +1,107 @@ +//===-- scudo_new_delete.cpp ------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// Interceptors for operators new and delete. +/// +//===----------------------------------------------------------------------===// + +#include "scudo_allocator.h" +#include "scudo_errors.h" + +#include "interception/interception.h" + +#include + +using namespace __scudo; + +#define CXX_OPERATOR_ATTRIBUTE INTERCEPTOR_ATTRIBUTE + +// Fake std::nothrow_t to avoid including . +namespace std { +struct nothrow_t {}; +enum class align_val_t: size_t {}; +} // namespace std + +// TODO(alekseys): throw std::bad_alloc instead of dying on OOM. +#define OPERATOR_NEW_BODY_ALIGN(Type, Align, NoThrow) \ + void *Ptr = scudoAllocate(size, static_cast(Align), Type); \ + if (!NoThrow && UNLIKELY(!Ptr)) reportOutOfMemory(size); \ + return Ptr; +#define OPERATOR_NEW_BODY(Type, NoThrow) \ + OPERATOR_NEW_BODY_ALIGN(Type, 0, NoThrow) + +CXX_OPERATOR_ATTRIBUTE +void *operator new(size_t size) +{ OPERATOR_NEW_BODY(FromNew, /*NoThrow=*/false); } +CXX_OPERATOR_ATTRIBUTE +void *operator new[](size_t size) +{ OPERATOR_NEW_BODY(FromNewArray, /*NoThrow=*/false); } +CXX_OPERATOR_ATTRIBUTE +void *operator new(size_t size, std::nothrow_t const&) +{ OPERATOR_NEW_BODY(FromNew, /*NoThrow=*/true); } +CXX_OPERATOR_ATTRIBUTE +void *operator new[](size_t size, std::nothrow_t const&) +{ OPERATOR_NEW_BODY(FromNewArray, /*NoThrow=*/true); } +CXX_OPERATOR_ATTRIBUTE +void *operator new(size_t size, std::align_val_t align) +{ OPERATOR_NEW_BODY_ALIGN(FromNew, align, /*NoThrow=*/false); } +CXX_OPERATOR_ATTRIBUTE +void *operator new[](size_t size, std::align_val_t align) +{ OPERATOR_NEW_BODY_ALIGN(FromNewArray, align, /*NoThrow=*/false); } +CXX_OPERATOR_ATTRIBUTE +void *operator new(size_t size, std::align_val_t align, std::nothrow_t const&) +{ OPERATOR_NEW_BODY_ALIGN(FromNew, align, /*NoThrow=*/true); } +CXX_OPERATOR_ATTRIBUTE +void *operator new[](size_t size, std::align_val_t align, std::nothrow_t const&) +{ OPERATOR_NEW_BODY_ALIGN(FromNewArray, align, /*NoThrow=*/true); } + +#define OPERATOR_DELETE_BODY(Type) \ + scudoDeallocate(ptr, 0, 0, Type); +#define OPERATOR_DELETE_BODY_SIZE(Type) \ + scudoDeallocate(ptr, size, 0, Type); +#define OPERATOR_DELETE_BODY_ALIGN(Type) \ + scudoDeallocate(ptr, 0, static_cast(align), Type); +#define OPERATOR_DELETE_BODY_SIZE_ALIGN(Type) \ + scudoDeallocate(ptr, size, static_cast(align), Type); + +CXX_OPERATOR_ATTRIBUTE +void operator delete(void *ptr) NOEXCEPT +{ OPERATOR_DELETE_BODY(FromNew); } +CXX_OPERATOR_ATTRIBUTE +void operator delete[](void *ptr) NOEXCEPT +{ OPERATOR_DELETE_BODY(FromNewArray); } +CXX_OPERATOR_ATTRIBUTE +void operator delete(void *ptr, std::nothrow_t const&) +{ OPERATOR_DELETE_BODY(FromNew); } +CXX_OPERATOR_ATTRIBUTE +void operator delete[](void *ptr, std::nothrow_t const&) +{ OPERATOR_DELETE_BODY(FromNewArray); } +CXX_OPERATOR_ATTRIBUTE +void operator delete(void *ptr, size_t size) NOEXCEPT +{ OPERATOR_DELETE_BODY_SIZE(FromNew); } +CXX_OPERATOR_ATTRIBUTE +void operator delete[](void *ptr, size_t size) NOEXCEPT +{ OPERATOR_DELETE_BODY_SIZE(FromNewArray); } +CXX_OPERATOR_ATTRIBUTE +void operator delete(void *ptr, std::align_val_t align) NOEXCEPT +{ OPERATOR_DELETE_BODY_ALIGN(FromNew); } +CXX_OPERATOR_ATTRIBUTE +void operator delete[](void *ptr, std::align_val_t align) NOEXCEPT +{ OPERATOR_DELETE_BODY_ALIGN(FromNewArray); } +CXX_OPERATOR_ATTRIBUTE +void operator delete(void *ptr, std::align_val_t align, std::nothrow_t const&) +{ OPERATOR_DELETE_BODY_ALIGN(FromNew); } +CXX_OPERATOR_ATTRIBUTE +void operator delete[](void *ptr, std::align_val_t align, std::nothrow_t const&) +{ OPERATOR_DELETE_BODY_ALIGN(FromNewArray); } +CXX_OPERATOR_ATTRIBUTE +void operator delete(void *ptr, size_t size, std::align_val_t align) NOEXCEPT +{ OPERATOR_DELETE_BODY_SIZE_ALIGN(FromNew); } +CXX_OPERATOR_ATTRIBUTE +void operator delete[](void *ptr, size_t size, std::align_val_t align) NOEXCEPT +{ OPERATOR_DELETE_BODY_SIZE_ALIGN(FromNewArray); } diff --git a/compiler-rt/lib/scudo/scudo_platform.h b/compiler-rt/lib/scudo/scudo_platform.h new file mode 100644 index 00000000000000..07d4b70fc8e9bc --- /dev/null +++ b/compiler-rt/lib/scudo/scudo_platform.h @@ -0,0 +1,93 @@ +//===-- scudo_platform.h ----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// Scudo platform specific definitions. +/// TODO(kostyak): add tests for the compile time defines. +/// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_PLATFORM_H_ +#define SCUDO_PLATFORM_H_ + +#include "sanitizer_common/sanitizer_allocator.h" + +#if !SANITIZER_LINUX && !SANITIZER_FUCHSIA +# error "The Scudo hardened allocator is not supported on this platform." +#endif + +#define SCUDO_TSD_EXCLUSIVE_SUPPORTED (!SANITIZER_ANDROID && !SANITIZER_FUCHSIA) + +#ifndef SCUDO_TSD_EXCLUSIVE +// SCUDO_TSD_EXCLUSIVE wasn't defined, use a default TSD model for the platform. +# if SANITIZER_ANDROID || SANITIZER_FUCHSIA +// Android and Fuchsia use a pool of TSDs shared between threads. +# define SCUDO_TSD_EXCLUSIVE 0 +# elif SANITIZER_LINUX && !SANITIZER_ANDROID +// Non-Android Linux use an exclusive TSD per thread. +# define SCUDO_TSD_EXCLUSIVE 1 +# else +# error "No default TSD model defined for this platform." +# endif // SANITIZER_ANDROID || SANITIZER_FUCHSIA +#endif // SCUDO_TSD_EXCLUSIVE + +// If the exclusive TSD model is chosen, make sure the platform supports it. +#if SCUDO_TSD_EXCLUSIVE && !SCUDO_TSD_EXCLUSIVE_SUPPORTED +# error "The exclusive TSD model is not supported on this platform." +#endif + +// Maximum number of TSDs that can be created for the Shared model. +#ifndef SCUDO_SHARED_TSD_POOL_SIZE +# if SANITIZER_ANDROID +# define SCUDO_SHARED_TSD_POOL_SIZE 2U +# else +# define SCUDO_SHARED_TSD_POOL_SIZE 32U +# endif // SANITIZER_ANDROID +#endif // SCUDO_SHARED_TSD_POOL_SIZE + +// The following allows the public interface functions to be disabled. +#ifndef SCUDO_CAN_USE_PUBLIC_INTERFACE +# define SCUDO_CAN_USE_PUBLIC_INTERFACE 1 +#endif + +// Hooks in the allocation & deallocation paths can become a security concern if +// implemented improperly, or if overwritten by an attacker. Use with caution. +#ifndef SCUDO_CAN_USE_HOOKS +# if SANITIZER_FUCHSIA +# define SCUDO_CAN_USE_HOOKS 1 +# else +# define SCUDO_CAN_USE_HOOKS 0 +# endif // SANITIZER_FUCHSIA +#endif // SCUDO_CAN_USE_HOOKS + +namespace __scudo { + +#if SANITIZER_CAN_USE_ALLOCATOR64 +# if defined(__aarch64__) && SANITIZER_ANDROID +const uptr AllocatorSize = 0x4000000000ULL; // 256G. +# elif defined(__aarch64__) +const uptr AllocatorSize = 0x10000000000ULL; // 1T. +# else +const uptr AllocatorSize = 0x40000000000ULL; // 4T. +# endif +#else +const uptr RegionSizeLog = SANITIZER_ANDROID ? 19 : 20; +#endif // SANITIZER_CAN_USE_ALLOCATOR64 + +#if !defined(SCUDO_SIZE_CLASS_MAP) +# define SCUDO_SIZE_CLASS_MAP Dense +#endif + +#define SIZE_CLASS_MAP_TYPE SIZE_CLASS_MAP_TYPE_(SCUDO_SIZE_CLASS_MAP) +#define SIZE_CLASS_MAP_TYPE_(T) SIZE_CLASS_MAP_TYPE__(T) +#define SIZE_CLASS_MAP_TYPE__(T) T##SizeClassMap + +typedef SIZE_CLASS_MAP_TYPE SizeClassMap; + +} // namespace __scudo + +#endif // SCUDO_PLATFORM_H_ diff --git a/compiler-rt/lib/scudo/scudo_termination.cpp b/compiler-rt/lib/scudo/scudo_termination.cpp new file mode 100644 index 00000000000000..5f1337efaca0dc --- /dev/null +++ b/compiler-rt/lib/scudo/scudo_termination.cpp @@ -0,0 +1,41 @@ +//===-- scudo_termination.cpp -----------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// This file contains bare-bones termination functions to replace the +/// __sanitizer ones, in order to avoid any potential abuse of the callbacks +/// functionality. +/// +//===----------------------------------------------------------------------===// + +#include "scudo_utils.h" + +#include "sanitizer_common/sanitizer_common.h" + +namespace __sanitizer { + +bool AddDieCallback(DieCallbackType Callback) { return true; } + +bool RemoveDieCallback(DieCallbackType Callback) { return true; } + +void SetUserDieCallback(DieCallbackType Callback) {} + +void NORETURN Die() { + if (common_flags()->abort_on_error) + Abort(); + internal__exit(common_flags()->exitcode); +} + +void SetCheckUnwindCallback(void (*callback)()) {} + +void NORETURN CheckFailed(const char *File, int Line, const char *Condition, + u64 Value1, u64 Value2) { + __scudo::dieWithMessage("CHECK failed at %s:%d %s (%lld, %lld)\n", + File, Line, Condition, Value1, Value2); +} + +} // namespace __sanitizer diff --git a/compiler-rt/lib/scudo/scudo_tsd.h b/compiler-rt/lib/scudo/scudo_tsd.h new file mode 100644 index 00000000000000..eef4a7ba1e656e --- /dev/null +++ b/compiler-rt/lib/scudo/scudo_tsd.h @@ -0,0 +1,65 @@ +//===-- scudo_tsd.h ---------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// Scudo thread specific data definition. +/// Implementation will differ based on the thread local storage primitives +/// offered by the underlying platform. +/// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_TSD_H_ +#define SCUDO_TSD_H_ + +#include "scudo_allocator.h" +#include "scudo_utils.h" + +#include + +namespace __scudo { + +struct ALIGNED(SANITIZER_CACHE_LINE_SIZE) ScudoTSD { + AllocatorCacheT Cache; + uptr QuarantineCachePlaceHolder[4]; + + void init(); + void commitBack(); + + inline bool tryLock() SANITIZER_TRY_ACQUIRE(true, Mutex) { + if (Mutex.TryLock()) { + atomic_store_relaxed(&Precedence, 0); + return true; + } + if (atomic_load_relaxed(&Precedence) == 0) + atomic_store_relaxed(&Precedence, static_cast( + MonotonicNanoTime() >> FIRST_32_SECOND_64(16, 0))); + return false; + } + + inline void lock() SANITIZER_ACQUIRE(Mutex) { + atomic_store_relaxed(&Precedence, 0); + Mutex.Lock(); + } + + inline void unlock() SANITIZER_RELEASE(Mutex) { Mutex.Unlock(); } + + inline uptr getPrecedence() { return atomic_load_relaxed(&Precedence); } + + private: + StaticSpinMutex Mutex; + atomic_uintptr_t Precedence; +}; + +void initThread(bool MinimalInit); + +// TSD model specific fastpath functions definitions. +#include "scudo_tsd_exclusive.inc" +#include "scudo_tsd_shared.inc" + +} // namespace __scudo + +#endif // SCUDO_TSD_H_ diff --git a/compiler-rt/lib/scudo/scudo_tsd_exclusive.cpp b/compiler-rt/lib/scudo/scudo_tsd_exclusive.cpp new file mode 100644 index 00000000000000..a203a74bbcf8e8 --- /dev/null +++ b/compiler-rt/lib/scudo/scudo_tsd_exclusive.cpp @@ -0,0 +1,67 @@ +//===-- scudo_tsd_exclusive.cpp ---------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// Scudo exclusive TSD implementation. +/// +//===----------------------------------------------------------------------===// + +#include "scudo_tsd.h" + +#if SCUDO_TSD_EXCLUSIVE + +namespace __scudo { + +static pthread_once_t GlobalInitialized = PTHREAD_ONCE_INIT; +static pthread_key_t PThreadKey; + +__attribute__((tls_model("initial-exec"))) +THREADLOCAL ThreadState ScudoThreadState = ThreadNotInitialized; +__attribute__((tls_model("initial-exec"))) +THREADLOCAL ScudoTSD TSD; + +// Fallback TSD for when the thread isn't initialized yet or is torn down. It +// can be shared between multiple threads and as such must be locked. +ScudoTSD FallbackTSD; + +static void teardownThread(void *Ptr) { + uptr I = reinterpret_cast(Ptr); + // The glibc POSIX thread-local-storage deallocation routine calls user + // provided destructors in a loop of PTHREAD_DESTRUCTOR_ITERATIONS. + // We want to be called last since other destructors might call free and the + // like, so we wait until PTHREAD_DESTRUCTOR_ITERATIONS before draining the + // quarantine and swallowing the cache. + if (I > 1) { + // If pthread_setspecific fails, we will go ahead with the teardown. + if (LIKELY(pthread_setspecific(PThreadKey, + reinterpret_cast(I - 1)) == 0)) + return; + } + TSD.commitBack(); + ScudoThreadState = ThreadTornDown; +} + + +static void initOnce() { + CHECK_EQ(pthread_key_create(&PThreadKey, teardownThread), 0); + initScudo(); + FallbackTSD.init(); +} + +void initThread(bool MinimalInit) { + CHECK_EQ(pthread_once(&GlobalInitialized, initOnce), 0); + if (UNLIKELY(MinimalInit)) + return; + CHECK_EQ(pthread_setspecific(PThreadKey, reinterpret_cast( + GetPthreadDestructorIterations())), 0); + TSD.init(); + ScudoThreadState = ThreadInitialized; +} + +} // namespace __scudo + +#endif // SCUDO_TSD_EXCLUSIVE diff --git a/compiler-rt/lib/scudo/scudo_tsd_exclusive.inc b/compiler-rt/lib/scudo/scudo_tsd_exclusive.inc new file mode 100644 index 00000000000000..dc4d982f2fa8b0 --- /dev/null +++ b/compiler-rt/lib/scudo/scudo_tsd_exclusive.inc @@ -0,0 +1,47 @@ +//===-- scudo_tsd_exclusive.inc ---------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// Scudo exclusive TSD fastpath functions implementation. +/// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_TSD_H_ +#error "This file must be included inside scudo_tsd.h." +#endif // SCUDO_TSD_H_ + +#if SCUDO_TSD_EXCLUSIVE + +enum ThreadState : u8 { + ThreadNotInitialized = 0, + ThreadInitialized, + ThreadTornDown, +}; +__attribute__(( + tls_model("initial-exec"))) extern THREADLOCAL ThreadState ScudoThreadState; +__attribute__((tls_model("initial-exec"))) extern THREADLOCAL ScudoTSD TSD; + +extern ScudoTSD FallbackTSD; + +ALWAYS_INLINE void initThreadMaybe(bool MinimalInit = false) { + if (LIKELY(ScudoThreadState != ThreadNotInitialized)) + return; + initThread(MinimalInit); +} + +ALWAYS_INLINE ScudoTSD * +getTSDAndLock(bool *UnlockRequired) SANITIZER_NO_THREAD_SAFETY_ANALYSIS { + if (UNLIKELY(ScudoThreadState != ThreadInitialized)) { + FallbackTSD.lock(); + *UnlockRequired = true; + return &FallbackTSD; + } + *UnlockRequired = false; + return &TSD; +} + +#endif // SCUDO_TSD_EXCLUSIVE diff --git a/compiler-rt/lib/scudo/scudo_tsd_shared.cpp b/compiler-rt/lib/scudo/scudo_tsd_shared.cpp new file mode 100644 index 00000000000000..fc691b21a21356 --- /dev/null +++ b/compiler-rt/lib/scudo/scudo_tsd_shared.cpp @@ -0,0 +1,107 @@ +//===-- scudo_tsd_shared.cpp ------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// Scudo shared TSD implementation. +/// +//===----------------------------------------------------------------------===// + +#include "scudo_tsd.h" + +#if !SCUDO_TSD_EXCLUSIVE + +namespace __scudo { + +static pthread_once_t GlobalInitialized = PTHREAD_ONCE_INIT; +pthread_key_t PThreadKey; + +static atomic_uint32_t CurrentIndex; +static ScudoTSD *TSDs; +static u32 NumberOfTSDs; +static u32 CoPrimes[SCUDO_SHARED_TSD_POOL_SIZE]; +static u32 NumberOfCoPrimes = 0; + +#if SANITIZER_LINUX && !SANITIZER_ANDROID +__attribute__((tls_model("initial-exec"))) +THREADLOCAL ScudoTSD *CurrentTSD; +#endif + +static void initOnce() { + CHECK_EQ(pthread_key_create(&PThreadKey, NULL), 0); + initScudo(); + NumberOfTSDs = Min(Max(1U, GetNumberOfCPUsCached()), + static_cast(SCUDO_SHARED_TSD_POOL_SIZE)); + TSDs = reinterpret_cast( + MmapOrDie(sizeof(ScudoTSD) * NumberOfTSDs, "ScudoTSDs")); + for (u32 I = 0; I < NumberOfTSDs; I++) { + TSDs[I].init(); + u32 A = I + 1; + u32 B = NumberOfTSDs; + while (B != 0) { const u32 T = A; A = B; B = T % B; } + if (A == 1) + CoPrimes[NumberOfCoPrimes++] = I + 1; + } +} + +ALWAYS_INLINE void setCurrentTSD(ScudoTSD *TSD) { +#if SANITIZER_ANDROID + *get_android_tls_ptr() = reinterpret_cast(TSD); +#elif SANITIZER_LINUX + CurrentTSD = TSD; +#else + CHECK_EQ(pthread_setspecific(PThreadKey, reinterpret_cast(TSD)), 0); +#endif // SANITIZER_ANDROID +} + +void initThread(bool MinimalInit) { + pthread_once(&GlobalInitialized, initOnce); + // Initial context assignment is done in a plain round-robin fashion. + u32 Index = atomic_fetch_add(&CurrentIndex, 1, memory_order_relaxed); + setCurrentTSD(&TSDs[Index % NumberOfTSDs]); +} + +ScudoTSD *getTSDAndLockSlow(ScudoTSD *TSD) SANITIZER_NO_THREAD_SAFETY_ANALYSIS { + if (NumberOfTSDs > 1) { + // Use the Precedence of the current TSD as our random seed. Since we are in + // the slow path, it means that tryLock failed, and as a result it's very + // likely that said Precedence is non-zero. + u32 RandState = static_cast(TSD->getPrecedence()); + const u32 R = Rand(&RandState); + const u32 Inc = CoPrimes[R % NumberOfCoPrimes]; + u32 Index = R % NumberOfTSDs; + uptr LowestPrecedence = UINTPTR_MAX; + ScudoTSD *CandidateTSD = nullptr; + // Go randomly through at most 4 contexts and find a candidate. + for (u32 I = 0; I < Min(4U, NumberOfTSDs); I++) { + if (TSDs[Index].tryLock()) { + setCurrentTSD(&TSDs[Index]); + return &TSDs[Index]; + } + const uptr Precedence = TSDs[Index].getPrecedence(); + // A 0 precedence here means another thread just locked this TSD. + if (Precedence && Precedence < LowestPrecedence) { + CandidateTSD = &TSDs[Index]; + LowestPrecedence = Precedence; + } + Index += Inc; + if (Index >= NumberOfTSDs) + Index -= NumberOfTSDs; + } + if (CandidateTSD) { + CandidateTSD->lock(); + setCurrentTSD(CandidateTSD); + return CandidateTSD; + } + } + // Last resort, stick with the current one. + TSD->lock(); + return TSD; +} + +} // namespace __scudo + +#endif // !SCUDO_TSD_EXCLUSIVE diff --git a/compiler-rt/lib/scudo/scudo_tsd_shared.inc b/compiler-rt/lib/scudo/scudo_tsd_shared.inc new file mode 100644 index 00000000000000..b25392a9630ef3 --- /dev/null +++ b/compiler-rt/lib/scudo/scudo_tsd_shared.inc @@ -0,0 +1,56 @@ +//===-- scudo_tsd_shared.inc ------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// Scudo shared TSD fastpath functions implementation. +/// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_TSD_H_ +# error "This file must be included inside scudo_tsd.h." +#endif // SCUDO_TSD_H_ + +#if !SCUDO_TSD_EXCLUSIVE + +extern pthread_key_t PThreadKey; + +#if SANITIZER_LINUX && !SANITIZER_ANDROID +__attribute__((tls_model("initial-exec"))) +extern THREADLOCAL ScudoTSD *CurrentTSD; +#endif + +ALWAYS_INLINE ScudoTSD* getCurrentTSD() { +#if SANITIZER_ANDROID + return reinterpret_cast(*get_android_tls_ptr()); +#elif SANITIZER_LINUX + return CurrentTSD; +#else + return reinterpret_cast(pthread_getspecific(PThreadKey)); +#endif // SANITIZER_ANDROID +} + +ALWAYS_INLINE void initThreadMaybe(bool MinimalInit = false) { + if (LIKELY(getCurrentTSD())) + return; + initThread(MinimalInit); +} + +ScudoTSD *getTSDAndLockSlow(ScudoTSD *TSD); + +ALWAYS_INLINE ScudoTSD * +getTSDAndLock(bool *UnlockRequired) SANITIZER_NO_THREAD_SAFETY_ANALYSIS { + ScudoTSD *TSD = getCurrentTSD(); + DCHECK(TSD && "No TSD associated with the current thread!"); + *UnlockRequired = true; + // Try to lock the currently associated context. + if (TSD->tryLock()) + return TSD; + // If it failed, go the slow path. + return getTSDAndLockSlow(TSD); +} + +#endif // !SCUDO_TSD_EXCLUSIVE diff --git a/compiler-rt/lib/scudo/scudo_utils.cpp b/compiler-rt/lib/scudo/scudo_utils.cpp new file mode 100644 index 00000000000000..b0aef752c67937 --- /dev/null +++ b/compiler-rt/lib/scudo/scudo_utils.cpp @@ -0,0 +1,145 @@ +//===-- scudo_utils.cpp -----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// Platform specific utility functions. +/// +//===----------------------------------------------------------------------===// + +#include "scudo_utils.h" + +#if defined(__x86_64__) || defined(__i386__) +# include +#elif defined(__arm__) || defined(__aarch64__) +# include "sanitizer_common/sanitizer_getauxval.h" +# if SANITIZER_FUCHSIA +# include +# include +# elif SANITIZER_POSIX +# include "sanitizer_common/sanitizer_posix.h" +# include +# endif +#endif + +#include + +// TODO(kostyak): remove __sanitizer *Printf uses in favor for our own less +// complicated string formatting code. The following is a +// temporary workaround to be able to use __sanitizer::VSNPrintf. +namespace __sanitizer { + +extern int VSNPrintf(char *buff, int buff_length, const char *format, + va_list args); + +} // namespace __sanitizer + +namespace __scudo { + +void dieWithMessage(const char *Format, ...) { + static const char ScudoError[] = "Scudo ERROR: "; + static constexpr uptr PrefixSize = sizeof(ScudoError) - 1; + // Our messages are tiny, 256 characters is more than enough. + char Message[256]; + va_list Args; + va_start(Args, Format); + internal_memcpy(Message, ScudoError, PrefixSize); + VSNPrintf(Message + PrefixSize, sizeof(Message) - PrefixSize, Format, Args); + va_end(Args); + LogMessageOnPrintf(Message); + if (common_flags()->abort_on_error) + SetAbortMessage(Message); + RawWrite(Message); + Die(); +} + +#if defined(__x86_64__) || defined(__i386__) +// i386 and x86_64 specific code to detect CRC32 hardware support via CPUID. +// CRC32 requires the SSE 4.2 instruction set. +# ifndef bit_SSE4_2 +# define bit_SSE4_2 bit_SSE42 // clang and gcc have different defines. +# endif + +#ifndef signature_HYGON_ebx // They are not defined in gcc. +// HYGON: "HygonGenuine". +#define signature_HYGON_ebx 0x6f677948 +#define signature_HYGON_edx 0x6e65476e +#define signature_HYGON_ecx 0x656e6975 +#endif + +bool hasHardwareCRC32() { + u32 Eax, Ebx, Ecx, Edx; + __get_cpuid(0, &Eax, &Ebx, &Ecx, &Edx); + const bool IsIntel = (Ebx == signature_INTEL_ebx) && + (Edx == signature_INTEL_edx) && + (Ecx == signature_INTEL_ecx); + const bool IsAMD = (Ebx == signature_AMD_ebx) && + (Edx == signature_AMD_edx) && + (Ecx == signature_AMD_ecx); + const bool IsHygon = (Ebx == signature_HYGON_ebx) && + (Edx == signature_HYGON_edx) && + (Ecx == signature_HYGON_ecx); + if (!IsIntel && !IsAMD && !IsHygon) + return false; + __get_cpuid(1, &Eax, &Ebx, &Ecx, &Edx); + return !!(Ecx & bit_SSE4_2); +} +#elif defined(__arm__) || defined(__aarch64__) +// For ARM and AArch64, hardware CRC32 support is indicated in the AT_HWCAP +// auxiliary vector. +# ifndef AT_HWCAP +# define AT_HWCAP 16 +# endif +# ifndef HWCAP_CRC32 +# define HWCAP_CRC32 (1 << 7) // HWCAP_CRC32 is missing on older platforms. +# endif +# if SANITIZER_POSIX +bool hasHardwareCRC32ARMPosix() { + uptr F = internal_open("/proc/self/auxv", O_RDONLY); + if (internal_iserror(F)) + return false; + struct { uptr Tag; uptr Value; } Entry = { 0, 0 }; + for (;;) { + uptr N = internal_read(F, &Entry, sizeof(Entry)); + if (internal_iserror(N) || N != sizeof(Entry) || + (Entry.Tag == 0 && Entry.Value == 0) || Entry.Tag == AT_HWCAP) + break; + } + internal_close(F); + return (Entry.Tag == AT_HWCAP && (Entry.Value & HWCAP_CRC32) != 0); +} +# else +bool hasHardwareCRC32ARMPosix() { return false; } +# endif // SANITIZER_POSIX + +// Bionic doesn't initialize its globals early enough. This causes issues when +// trying to access them from a preinit_array (b/25751302) or from another +// constructor called before the libc one (b/68046352). __progname is +// initialized after the other globals, so we can check its value to know if +// calling getauxval is safe. +extern "C" SANITIZER_WEAK_ATTRIBUTE char *__progname; +inline bool areBionicGlobalsInitialized() { + return !SANITIZER_ANDROID || (&__progname && __progname); +} + +bool hasHardwareCRC32() { +#if SANITIZER_FUCHSIA + u32 HWCap; + zx_status_t Status = zx_system_get_features(ZX_FEATURE_KIND_CPU, &HWCap); + if (Status != ZX_OK || (HWCap & ZX_ARM64_FEATURE_ISA_CRC32) == 0) + return false; + return true; +#else + if (&getauxval && areBionicGlobalsInitialized()) + return !!(getauxval(AT_HWCAP) & HWCAP_CRC32); + return hasHardwareCRC32ARMPosix(); +#endif // SANITIZER_FUCHSIA +} +#else +bool hasHardwareCRC32() { return false; } +#endif // defined(__x86_64__) || defined(__i386__) + +} // namespace __scudo diff --git a/compiler-rt/lib/scudo/scudo_utils.h b/compiler-rt/lib/scudo/scudo_utils.h new file mode 100644 index 00000000000000..5a9b32f0b234e6 --- /dev/null +++ b/compiler-rt/lib/scudo/scudo_utils.h @@ -0,0 +1,36 @@ +//===-- scudo_utils.h -------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// Header for scudo_utils.cpp. +/// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_UTILS_H_ +#define SCUDO_UTILS_H_ + +#include "sanitizer_common/sanitizer_common.h" + +#include + +namespace __scudo { + +template +inline Dest bit_cast(const Source& source) { + static_assert(sizeof(Dest) == sizeof(Source), "Sizes are not equal!"); + Dest dest; + memcpy(&dest, &source, sizeof(dest)); + return dest; +} + +void dieWithMessage(const char *Format, ...) NORETURN FORMAT(1, 2); + +bool hasHardwareCRC32(); + +} // namespace __scudo + +#endif // SCUDO_UTILS_H_