[scudo] Manage pages with MemMap in SizeClassAllocator64

ChiaHungDuan · ChiaHungDuan · commit 405ceaa09de0 · 2023-04-05T21:28:54.000Z
Introduce a new data structure to manage the allocated pages from the system. This is meant to deprecate certain memory system call wrappers in Scudo, e.g., map()/unmap(). Besides, we would like to make MapPlatformData to be appeared in platform specific data structure only. Given that there are several allocators in Scudo and each of them has different way of page management. The deprecation will be done in several CLs. In this commit, we start from SizeClassAllocator64. Reviewed By: cferris Differential Revision: https://reviews.llvm.org/D146009
diff --git a/compiler-rt/lib/scudo/standalone/CMakeLists.txt b/compiler-rt/lib/scudo/standalone/CMakeLists.txt
@@ -71,6 +71,8 @@ set(SCUDO_HEADERS
   list.h
   local_cache.h
   memtag.h
+  mem_map.h
+  mem_map_base.h
   mutex.h
   options.h
   platform.h
@@ -104,6 +106,7 @@ set(SCUDO_SOURCES
   flags.cpp
   fuchsia.cpp
   linux.cpp
+  mem_map.cpp
   release.cpp
   report.cpp
   rss_limit_checker.cpp
diff --git a/compiler-rt/lib/scudo/standalone/mem_map.cpp b/compiler-rt/lib/scudo/standalone/mem_map.cpp
@@ -0,0 +1,73 @@
+//===-- mem_map.cpp ---------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mem_map.h"
+
+#include "common.h"
+
+namespace scudo {
+
+bool MemMapDefault::mapImpl(uptr Addr, uptr Size, const char *Name,
+                            uptr Flags) {
+  void *MappedAddr =
+      ::scudo::map(reinterpret_cast<void *>(Addr), Size, Name, Flags, &Data);
+  if (MappedAddr == nullptr)
+    return false;
+  Base = reinterpret_cast<uptr>(MappedAddr);
+  Capacity = Size;
+  return true;
+}
+
+void MemMapDefault::unmapImpl(uptr Addr, uptr Size) {
+  if (Size == Capacity) {
+    Base = Capacity = 0;
+  } else {
+    if (Base == Addr)
+      Base = Addr + Size;
+    Capacity -= Size;
+  }
+
+  ::scudo::unmap(reinterpret_cast<void *>(Addr), Size, UNMAP_ALL, &Data);
+}
+
+bool MemMapDefault::remapImpl(uptr Addr, uptr Size, const char *Name,
+                              uptr Flags) {
+  void *RemappedAddr =
+      ::scudo::map(reinterpret_cast<void *>(Addr), Size, Name, Flags, &Data);
+  return reinterpret_cast<uptr>(RemappedAddr) == Addr;
+}
+
+void MemMapDefault::releaseAndZeroPagesToOSImpl(uptr From, uptr Size) {
+  return ::scudo::releasePagesToOS(Base, From - Base, Size, &Data);
+}
+
+void ReservedMemoryDefault::releaseImpl() {
+  ::scudo::unmap(reinterpret_cast<void *>(Base), Capacity, UNMAP_ALL, &Data);
+}
+
+bool ReservedMemoryDefault::createImpl(uptr Addr, uptr Size, const char *Name,
+                                       uptr Flags) {
+  void *Reserved = ::scudo::map(reinterpret_cast<void *>(Addr), Size, Name,
+                                Flags | MAP_NOACCESS, &Data);
+  if (Reserved == nullptr)
+    return false;
+
+  Base = reinterpret_cast<uptr>(Reserved);
+  Capacity = Size;
+
+  return true;
+}
+
+ReservedMemoryDefault::MemMapT ReservedMemoryDefault::dispatchImpl(uptr Addr,
+                                                                   uptr Size) {
+  ReservedMemoryDefault::MemMapT NewMap(Addr, Size);
+  NewMap.setMapPlatformData(Data);
+  return NewMap;
+}
+
+} // namespace scudo
diff --git a/compiler-rt/lib/scudo/standalone/mem_map.h b/compiler-rt/lib/scudo/standalone/mem_map.h
@@ -0,0 +1,87 @@
+//===-- mem_map.h -----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SCUDO_MEM_MAP_H_
+#define SCUDO_MEM_MAP_H_
+
+#include "mem_map_base.h"
+
+#include "common.h"
+#include "internal_defs.h"
+
+// TODO: This is only used for `MapPlatformData`. Remove these includes when we
+// have all three platform specific `MemMap` and `ReservedMemory`
+// implementations.
+#include "fuchsia.h"
+#include "linux.h"
+#include "trusty.h"
+
+namespace scudo {
+
+// This will be deprecated when every allocator has been supported by each
+// platform's `MemMap` implementation.
+class MemMapDefault final : public MemMapBase<MemMapDefault> {
+public:
+  constexpr MemMapDefault() = default;
+  MemMapDefault(uptr Base, uptr Capacity) : Base(Base), Capacity(Capacity) {}
+
+  // Impls for base functions.
+  bool mapImpl(uptr Addr, uptr Size, const char *Name, uptr Flags);
+  void unmapImpl(uptr Addr, uptr Size);
+  bool remapImpl(uptr Addr, uptr Size, const char *Name, uptr Flags);
+  void releasePagesToOSImpl(uptr From, uptr Size) {
+    return releaseAndZeroPagesToOSImpl(From, Size);
+  }
+  void releaseAndZeroPagesToOSImpl(uptr From, uptr Size);
+  uptr getBaseImpl() { return Base; }
+  uptr getCapacityImpl() { return Capacity; }
+
+  void setMapPlatformData(MapPlatformData &NewData) { Data = NewData; }
+
+private:
+  uptr Base = 0;
+  uptr Capacity = 0;
+  MapPlatformData Data = {};
+};
+
+// This will be deprecated when every allocator has been supported by each
+// platform's `MemMap` implementation.
+class ReservedMemoryDefault final
+    : public ReservedMemory<ReservedMemoryDefault, MemMapDefault> {
+public:
+  constexpr ReservedMemoryDefault() = default;
+
+  bool createImpl(uptr Addr, uptr Size, const char *Name, uptr Flags);
+  void releaseImpl();
+  MemMapT dispatchImpl(uptr Addr, uptr Size);
+  uptr getBaseImpl() { return Base; }
+  uptr getCapacityImpl() { return Capacity; }
+
+private:
+  uptr Base = 0;
+  uptr Capacity = 0;
+  MapPlatformData Data = {};
+};
+
+#if SCUDO_LINUX
+using ReservedMemoryT = ReservedMemoryDefault;
+using MemMapT = ReservedMemoryT::MemMapT;
+#elif SCUDO_FUCHSIA
+using ReservedMemoryT = ReservedMemoryDefault;
+using MemMapT = ReservedMemoryT::MemMapT;
+#elif SCUDO_TRUSTY
+using ReservedMemoryT = ReservedMemoryDefault;
+using MemMapT = ReservedMemoryT::MemMapT;
+#else
+#error                                                                         \
+    "Unsupported platform, please implement the ReservedMemory for your platform!"
+#endif
+
+} // namespace scudo
+
+#endif // SCUDO_MEM_MAP_H_
diff --git a/compiler-rt/lib/scudo/standalone/mem_map_base.h b/compiler-rt/lib/scudo/standalone/mem_map_base.h
@@ -0,0 +1,121 @@
+//===-- mem_map_base.h ------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SCUDO_MEM_MAP_BASE_H_
+#define SCUDO_MEM_MAP_BASE_H_
+
+#include "common.h"
+
+namespace scudo {
+
+// In Scudo, every memory operation will be fulfilled through a
+// platform-specific `MemMap` instance. The essential APIs are listed in the
+// `MemMapBase` below. This is implemented in CRTP, so for each implementation,
+// it has to implement all of the 'Impl' named functions.
+template <class Derived> class MemMapBase {
+public:
+  constexpr MemMapBase() = default;
+
+  // This is used to map a new set of contiguous pages. Note that the `Addr` is
+  // only a suggestion to the system.
+  bool map(uptr Addr, uptr Size, const char *Name, uptr Flags = 0) {
+    DCHECK(!isAllocated());
+    return invokeImpl(&Derived::mapImpl, Addr, Size, Name, Flags);
+  }
+
+  // This is used to unmap partial/full pages from the beginning or the end.
+  // I.e., the result pages are expected to be still contiguous.
+  void unmap(uptr Addr, uptr Size) {
+    DCHECK(isAllocated());
+    DCHECK((Addr == getBase()) || (Addr + Size == getBase() + getCapacity()));
+    invokeImpl(&Derived::unmapImpl, Addr, Size);
+  }
+
+  // This is used to remap a mapped range (either from map() or dispatched from
+  // ReservedMemory). For example, we have reserved several pages and then we
+  // want to remap them with different accessibility.
+  bool remap(uptr Addr, uptr Size, const char *Name, uptr Flags = 0) {
+    DCHECK(isAllocated());
+    DCHECK((Addr >= getBase()) || (Addr + Size <= getBase() + getCapacity()));
+    return invokeImpl(&Derived::remapImpl, Addr, Size, Name, Flags);
+  }
+
+  // Suggest releasing a set of contiguous physical pages back to the OS. Note
+  // that only physical pages are supposed to be released. Any release of
+  // virtual pages may lead to undefined behavior.
+  void releasePagesToOS(uptr From, uptr Size) {
+    DCHECK(isAllocated());
+    DCHECK((From >= getBase()) || (From + Size <= getBase() + getCapacity()));
+    invokeImpl(&Derived::releasePagesToOSImpl, From, Size);
+  }
+  // This is similar to the above one except that any subsequent access to the
+  // released pages will return with zero-filled pages.
+  void releaseAndZeroPagesToOS(uptr From, uptr Size) {
+    DCHECK(isAllocated());
+    DCHECK((From >= getBase()) || (From + Size <= getBase() + getCapacity()));
+    invokeImpl(&Derived::releaseAndZeroPagesToOSImpl, From, Size);
+  }
+
+  uptr getBase() { return invokeImpl(&Derived::getBaseImpl); }
+  uptr getCapacity() { return invokeImpl(&Derived::getCapacityImpl); }
+
+  bool isAllocated() { return getBase() != 0U; }
+
+protected:
+  template <typename R, typename... Args>
+  R invokeImpl(R (Derived::*MemFn)(Args...), Args... args) {
+    return (static_cast<Derived *>(this)->*MemFn)(args...);
+  }
+};
+
+// `ReservedMemory` is a special memory handle which can be viewed as a page
+// allocator. `ReservedMemory` will reserve a contiguous pages and the later
+// page request can be fulfilled at the designated address. This is used when
+// we want to ensure the virtual address of the MemMap will be in a known range.
+// This is implemented in CRTP, so for each
+// implementation, it has to implement all of the 'Impl' named functions.
+template <class Derived, typename MemMapTy> class ReservedMemory {
+public:
+  using MemMapT = MemMapTy;
+  constexpr ReservedMemory() = default;
+
+  // Reserve a chunk of memory at a suggested address.
+  bool create(uptr Addr, uptr Size, const char *Name, uptr Flags = 0) {
+    DCHECK(!isCreated());
+    return invokeImpl(&Derived::createImpl, Addr, Size, Name, Flags);
+  }
+
+  // Release the entire reserved memory.
+  void release() {
+    DCHECK(isCreated());
+    invokeImpl(&Derived::releaseImpl);
+  }
+
+  // Dispatch a sub-range of reserved memory. Note that any fragmentation of
+  // the reserved pages is managed by each implementation.
+  MemMapT dispatch(uptr Addr, uptr Size) {
+    DCHECK(isCreated());
+    DCHECK((Addr >= getBase()) || (Addr + Size <= getBase() + getCapacity()));
+    return invokeImpl(&Derived::dispatchImpl, Addr, Size);
+  }
+
+  uptr getBase() { return invokeImpl(&Derived::getBaseImpl); }
+  uptr getCapacity() { return invokeImpl(&Derived::getCapacityImpl); }
+
+  bool isCreated() { return getBase() != 0U; }
+
+protected:
+  template <typename R, typename... Args>
+  R invokeImpl(R (Derived::*MemFn)(Args...), Args... args) {
+    return (static_cast<Derived *>(this)->*MemFn)(args...);
+  }
+};
+
+} // namespace scudo
+
+#endif // SCUDO_MEM_MAP_BASE_H_
diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h
@@ -13,6 +13,7 @@
 #include "common.h"
 #include "list.h"
 #include "local_cache.h"
+#include "mem_map.h"
 #include "memtag.h"
 #include "options.h"
 #include "release.h"
@@ -103,10 +104,11 @@ template <typename Config> class SizeClassAllocator64 {
     SmallerBlockReleasePageDelta =
         PagesInGroup * (1 + MinSizeClass / 16U) / 100;
 
-    DCHECK_EQ(PrimaryBase, 0U);
     // Reserve the space required for the Primary.
-    PrimaryBase = reinterpret_cast<uptr>(map(
-        nullptr, PrimarySize, "scudo:primary_reserve", MAP_NOACCESS, &Data));
+    CHECK(ReservedMemory.create(/*Addr=*/0U, PrimarySize,
+                                "scudo:primary_reserve"));
+    PrimaryBase = ReservedMemory.getBase();
+    DCHECK_NE(PrimaryBase, 0U);
 
     u32 Seed;
     const u64 Time = getMonotonicTimeFast();
@@ -141,8 +143,7 @@ template <typename Config> class SizeClassAllocator64 {
       *Region = {};
     }
     if (PrimaryBase)
-      unmap(reinterpret_cast<void *>(PrimaryBase), PrimarySize, UNMAP_ALL,
-            &Data);
+      ReservedMemory.release();
     PrimaryBase = 0U;
   }
 
@@ -449,7 +450,7 @@ template <typename Config> class SizeClassAllocator64 {
     uptr AllocatedUser GUARDED_BY(Mutex) = 0;
     // The minimum size of pushed blocks to trigger page release.
     uptr TryReleaseThreshold GUARDED_BY(Mutex) = 0;
-    MapPlatformData Data GUARDED_BY(Mutex) = {};
+    MemMapT MemMap = {};
     ReleaseToOsInfo ReleaseInfo GUARDED_BY(Mutex) = {};
     bool Exhausted GUARDED_BY(Mutex) = false;
   };
@@ -459,11 +460,13 @@ template <typename Config> class SizeClassAllocator64 {
   };
   static_assert(sizeof(RegionInfo) % SCUDO_CACHE_LINE_SIZE == 0, "");
 
+  // TODO: `PrimaryBase` can be obtained from ReservedMemory. This needs to be
+  // deprecated.
   uptr PrimaryBase = 0;
+  ReservedMemoryT ReservedMemory = {};
   // The minimum size of pushed blocks that we will try to release the pages in
   // that size class.
   uptr SmallerBlockReleasePageDelta = 0;
-  MapPlatformData Data = {};
   atomic_s32 ReleaseToOsIntervalMs = {};
   alignas(SCUDO_CACHE_LINE_SIZE) RegionInfo RegionInfoArray[NumClasses];
 
@@ -742,14 +745,18 @@ template <typename Config> class SizeClassAllocator64 {
         Region->Exhausted = true;
         return false;
       }
-      if (MappedUser == 0)
-        Region->Data = Data;
-      if (UNLIKELY(!map(
-              reinterpret_cast<void *>(RegionBeg + MappedUser), MapSize,
-              "scudo:primary",
+      // TODO: Consider allocating MemMap in init().
+      if (!Region->MemMap.isAllocated()) {
+        Region->MemMap = ReservedMemory.dispatch(
+            getRegionBaseByClassId(ClassId), RegionSize);
+      }
+      DCHECK(Region->MemMap.isAllocated());
+
+      if (UNLIKELY(!Region->MemMap.remap(
+              RegionBeg + MappedUser, MapSize, "scudo:primary",
               MAP_ALLOWNOMEM | MAP_RESIZABLE |
-                  (useMemoryTagging<Config>(Options.load()) ? MAP_MEMTAG : 0),
-              &Region->Data))) {
+                  (useMemoryTagging<Config>(Options.load()) ? MAP_MEMTAG
+                                                            : 0)))) {
         return false;
       }
       Region->MappedUser += MapSize;
@@ -1082,7 +1089,8 @@ template <typename Config> class SizeClassAllocator64 {
     const uptr ReleaseRangeSize = ReleaseEnd - ReleaseBase;
     const uptr ReleaseOffset = ReleaseBase - Region->RegionBeg;
 
-    ReleaseRecorder Recorder(Region->RegionBeg, ReleaseOffset, &Region->Data);
+    RegionReleaseRecorder<MemMapT> Recorder(&Region->MemMap, Region->RegionBeg,
+                                            ReleaseOffset);
     PageReleaseContext Context(BlockSize, /*NumberOfRegions=*/1U,
                                ReleaseRangeSize, ReleaseOffset);
 
diff --git a/compiler-rt/lib/scudo/standalone/release.h b/compiler-rt/lib/scudo/standalone/release.h
