[scudo] Manage pages with MemMap in Secondary Allocator

Replace the uses of raw map()/unmap(), .etc calls with MemMap. Also
remove the direct use of MapPlatformData in the secondary allocator.

Also add setMemoryPermission() in MemMap.

Reviewed By: cryptoad

Differential Revision: https://reviews.llvm.org/D146454

Author: ChiaHungDuan

compiler-rt/lib/scudo/standalone/mem_map.cpp

@@ -46,6 +46,10 @@ void MemMapDefault::releaseAndZeroPagesToOSImpl(uptr From, uptr Size) {
   return ::scudo::releasePagesToOS(Base, From - Base, Size, &Data);
 }
 
+void MemMapDefault::setMemoryPermissionImpl(uptr Addr, uptr Size, uptr Flags) {
+  return ::scudo::setMemoryPermission(Addr, Size, Flags);
+}
+
 void ReservedMemoryDefault::releaseImpl() {
   ::scudo::unmap(reinterpret_cast<void *>(Base), Capacity, UNMAP_ALL, &Data);
 }

compiler-rt/lib/scudo/standalone/mem_map.h

@@ -34,6 +34,7 @@ class MemMapDefault final : public MemMapBase<MemMapDefault> {
   bool mapImpl(uptr Addr, uptr Size, const char *Name, uptr Flags);
   void unmapImpl(uptr Addr, uptr Size);
   bool remapImpl(uptr Addr, uptr Size, const char *Name, uptr Flags);
+  void setMemoryPermissionImpl(uptr Addr, uptr Size, uptr Flags);
   void releasePagesToOSImpl(uptr From, uptr Size) {
     return releaseAndZeroPagesToOSImpl(From, Size);
   }

compiler-rt/lib/scudo/standalone/mem_map_base.h

@@ -45,6 +45,15 @@ template <class Derived> class MemMapBase {
     return invokeImpl(&Derived::remapImpl, Addr, Size, Name, Flags);
   }
 
+  // This is used to update the pages' access permission. For example, mark
+  // pages as no read/write permission.
+  void setMemoryPermission(uptr Addr, uptr Size, uptr Flags) {
+    DCHECK(isAllocated());
+    DCHECK((Addr >= getBase()) || (Addr + Size <= getBase() + getCapacity()));
+    return static_cast<Derived *>(this)->setMemoryPermissionImpl(Addr, Size,
+                                                                 Flags);
+  }
+
   // Suggest releasing a set of contiguous physical pages back to the OS. Note
   // that only physical pages are supposed to be released. Any release of
   // virtual pages may lead to undefined behavior.

compiler-rt/lib/scudo/standalone/secondary.h

@@ -12,6 +12,7 @@
 #include "chunk.h"
 #include "common.h"
 #include "list.h"
+#include "mem_map.h"
 #include "memtag.h"
 #include "mutex.h"
 #include "options.h"
@@ -37,9 +38,7 @@ struct alignas(Max<uptr>(archSupportsMemoryTagging()
   LargeBlock::Header *Next;
   uptr CommitBase;
   uptr CommitSize;
-  uptr MapBase;
-  uptr MapSize;
-  [[no_unique_address]] MapPlatformData Data;
+  MemMapT MemMap;
 };
 
 static_assert(sizeof(Header) % (1U << SCUDO_MIN_ALIGNMENT_LOG) == 0, "");
@@ -66,8 +65,11 @@ template <typename Config> static Header *getHeader(const void *Ptr) {
 } // namespace LargeBlock
 
 static void unmap(LargeBlock::Header *H) {
-  MapPlatformData Data = H->Data;
-  unmap(reinterpret_cast<void *>(H->MapBase), H->MapSize, UNMAP_ALL, &Data);
+  // Note that the `H->MapMap` is stored on the pages managed by itself. Take
+  // over the ownership before unmap() so that any operation along with unmap()
+  // won't touch inaccessible pages.
+  MemMapT MemMap = H->MemMap;
+  MemMap.unmap(MemMap.getBase(), MemMap.getCapacity());
 }
 
 class MapAllocatorNoCache {
@@ -97,20 +99,19 @@ static const uptr MaxUnusedCachePages = 4U;
 
 template <typename Config>
 void mapSecondary(Options Options, uptr CommitBase, uptr CommitSize,
-                  uptr AllocPos, uptr Flags, MapPlatformData *Data) {
+                  uptr AllocPos, uptr Flags, MemMapT &MemMap) {
   const uptr MaxUnusedCacheBytes = MaxUnusedCachePages * getPageSizeCached();
   if (useMemoryTagging<Config>(Options) && CommitSize > MaxUnusedCacheBytes) {
     const uptr UntaggedPos = Max(AllocPos, CommitBase + MaxUnusedCacheBytes);
-    map(reinterpret_cast<void *>(CommitBase), UntaggedPos - CommitBase,
-        "scudo:secondary", MAP_RESIZABLE | MAP_MEMTAG | Flags, Data);
-    map(reinterpret_cast<void *>(UntaggedPos),
-        CommitBase + CommitSize - UntaggedPos, "scudo:secondary",
-        MAP_RESIZABLE | Flags, Data);
+    MemMap.remap(CommitBase, UntaggedPos - CommitBase, "scudo:secondary",
+                 MAP_RESIZABLE | MAP_MEMTAG | Flags);
+    MemMap.remap(UntaggedPos, CommitBase + CommitSize - UntaggedPos,
+                 "scudo:secondary", MAP_RESIZABLE | Flags);
   } else {
-    map(reinterpret_cast<void *>(CommitBase), CommitSize, "scudo:secondary",
+    const uptr RemapFlags =
         MAP_RESIZABLE | (useMemoryTagging<Config>(Options) ? MAP_MEMTAG : 0) |
-            Flags,
-        Data);
+        Flags;
+    MemMap.remap(CommitBase, CommitSize, "scudo:secondary", RemapFlags);
   }
 }
 
@@ -155,10 +156,8 @@ template <typename Config> class MapAllocatorCache {
     CachedBlock Entry;
     Entry.CommitBase = H->CommitBase;
     Entry.CommitSize = H->CommitSize;
-    Entry.MapBase = H->MapBase;
-    Entry.MapSize = H->MapSize;
     Entry.BlockBegin = reinterpret_cast<uptr>(H + 1);
-    Entry.Data = H->Data;
+    Entry.MemMap = H->MemMap;
     Entry.Time = Time;
     if (useMemoryTagging<Config>(Options)) {
       if (Interval == 0 && !SCUDO_FUCHSIA) {
@@ -168,13 +167,13 @@ template <typename Config> class MapAllocatorCache {
         // on top so we just do the two syscalls there.
         Entry.Time = 0;
         mapSecondary<Config>(Options, Entry.CommitBase, Entry.CommitSize,
-                             Entry.CommitBase, MAP_NOACCESS, &Entry.Data);
+                             Entry.CommitBase, MAP_NOACCESS, Entry.MemMap);
       } else {
-        setMemoryPermission(Entry.CommitBase, Entry.CommitSize, MAP_NOACCESS,
-                            &Entry.Data);
+        Entry.MemMap.setMemoryPermission(Entry.CommitBase, Entry.CommitSize,
+                                         MAP_NOACCESS);
       }
     } else if (Interval == 0) {
-      releasePagesToOS(Entry.CommitBase, 0, Entry.CommitSize, &Entry.Data);
+      Entry.MemMap.releasePagesToOS(Entry.CommitBase, Entry.CommitSize);
       Entry.Time = 0;
     }
     do {
@@ -223,8 +222,7 @@ template <typename Config> class MapAllocatorCache {
     else if (Interval >= 0)
       releaseOlderThan(Time - static_cast<u64>(Interval) * 1000000);
     if (!EntryCached)
-      unmap(reinterpret_cast<void *>(Entry.MapBase), Entry.MapSize, UNMAP_ALL,
-            &Entry.Data);
+      Entry.MemMap.unmap(Entry.MemMap.getBase(), Entry.MemMap.getCapacity());
   }
 
   bool retrieve(Options Options, uptr Size, uptr Alignment,
@@ -267,7 +265,7 @@ template <typename Config> class MapAllocatorCache {
         LargeBlock::addHeaderTag<Config>(HeaderPos));
     *Zeroed = Entry.Time == 0;
     if (useMemoryTagging<Config>(Options))
-      setMemoryPermission(Entry.CommitBase, Entry.CommitSize, 0, &Entry.Data);
+      Entry.MemMap.setMemoryPermission(Entry.CommitBase, Entry.CommitSize, 0);
     uptr NewBlockBegin = reinterpret_cast<uptr>(*H + 1);
     if (useMemoryTagging<Config>(Options)) {
       if (*Zeroed) {
@@ -282,9 +280,7 @@ template <typename Config> class MapAllocatorCache {
     }
     (*H)->CommitBase = Entry.CommitBase;
     (*H)->CommitSize = Entry.CommitSize;
-    (*H)->MapBase = Entry.MapBase;
-    (*H)->MapSize = Entry.MapSize;
-    (*H)->Data = Entry.Data;
+    (*H)->MemMap = Entry.MemMap;
     return true;
   }
 
@@ -323,16 +319,18 @@ template <typename Config> class MapAllocatorCache {
     ScopedLock L(Mutex);
     for (u32 I = 0; I != Config::SecondaryCacheQuarantineSize; ++I) {
       if (Quarantine[I].CommitBase) {
-        unmap(reinterpret_cast<void *>(Quarantine[I].MapBase),
-              Quarantine[I].MapSize, UNMAP_ALL, &Quarantine[I].Data);
+        MemMapT &MemMap = Quarantine[I].MemMap;
+        MemMap.unmap(MemMap.getBase(), MemMap.getCapacity());
         Quarantine[I].CommitBase = 0;
       }
     }
     const u32 MaxCount = atomic_load_relaxed(&MaxEntriesCount);
-    for (u32 I = 0; I < MaxCount; I++)
-      if (Entries[I].CommitBase)
-        setMemoryPermission(Entries[I].CommitBase, Entries[I].CommitSize, 0,
-                            &Entries[I].Data);
+    for (u32 I = 0; I < MaxCount; I++) {
+      if (Entries[I].CommitBase) {
+        Entries[I].MemMap.setMemoryPermission(Entries[I].CommitBase,
+                                              Entries[I].CommitSize, 0);
+      }
+    }
     QuarantinePos = -1U;
   }
 
@@ -344,38 +342,31 @@ template <typename Config> class MapAllocatorCache {
 
 private:
   void empty() {
-    struct {
-      void *MapBase;
-      uptr MapSize;
-      MapPlatformData Data;
-    } MapInfo[Config::SecondaryCacheEntriesArraySize];
+    MemMapT MapInfo[Config::SecondaryCacheEntriesArraySize];
     uptr N = 0;
     {
       ScopedLock L(Mutex);
       for (uptr I = 0; I < Config::SecondaryCacheEntriesArraySize; I++) {
         if (!Entries[I].CommitBase)
           continue;
-        MapInfo[N].MapBase = reinterpret_cast<void *>(Entries[I].MapBase);
-        MapInfo[N].MapSize = Entries[I].MapSize;
-        MapInfo[N].Data = Entries[I].Data;
+        MapInfo[N] = Entries[I].MemMap;
         Entries[I].CommitBase = 0;
         N++;
       }
       EntriesCount = 0;
       IsFullEvents = 0;
     }
-    for (uptr I = 0; I < N; I++)
-      unmap(MapInfo[I].MapBase, MapInfo[I].MapSize, UNMAP_ALL,
-            &MapInfo[I].Data);
+    for (uptr I = 0; I < N; I++) {
+      MemMapT &MemMap = MapInfo[I];
+      MemMap.unmap(MemMap.getBase(), MemMap.getCapacity());
+    }
   }
 
   struct CachedBlock {
     uptr CommitBase;
     uptr CommitSize;
-    uptr MapBase;
-    uptr MapSize;
     uptr BlockBegin;
-    [[no_unique_address]] MapPlatformData Data;
+    MemMapT MemMap;
     u64 Time;
   };
 
@@ -387,7 +378,7 @@ template <typename Config> class MapAllocatorCache {
         OldestTime = Entry.Time;
       return;
     }
-    releasePagesToOS(Entry.CommitBase, 0, Entry.CommitSize, &Entry.Data);
+    Entry.MemMap.releasePagesToOS(Entry.CommitBase, Entry.CommitSize);
     Entry.Time = 0;
   }
 
@@ -538,16 +529,20 @@ void *MapAllocator<Config>::allocate(Options Options, uptr Size, uptr Alignment,
         AllocatedBytes += BlockSize;
         NumberOfAllocs++;
         Stats.add(StatAllocated, BlockSize);
-        Stats.add(StatMapped, H->MapSize);
+        Stats.add(StatMapped, H->MemMap.getCapacity());
       }
       return Ptr;
     }
   }
 
-  MapPlatformData Data = {};
+  ReservedMemoryT ReservedMemory;
   const uptr MapSize = RoundedSize + 2 * PageSize;
-  uptr MapBase = reinterpret_cast<uptr>(
-      map(nullptr, MapSize, nullptr, MAP_NOACCESS | MAP_ALLOWNOMEM, &Data));
+  ReservedMemory.create(/*Addr=*/0U, MapSize, nullptr, MAP_ALLOWNOMEM);
+
+  // Take the entire ownership of reserved region.
+  MemMapT MemMap = ReservedMemory.dispatch(ReservedMemory.getBase(),
+                                           ReservedMemory.getCapacity());
+  uptr MapBase = MemMap.getBase();
   if (UNLIKELY(!MapBase))
     return nullptr;
   uptr CommitBase = MapBase + PageSize;
@@ -565,33 +560,31 @@ void *MapAllocator<Config>::allocate(Options Options, uptr Size, uptr Alignment,
     // We only trim the extra memory on 32-bit platforms: 64-bit platforms
     // are less constrained memory wise, and that saves us two syscalls.
     if (SCUDO_WORDSIZE == 32U && NewMapBase != MapBase) {
-      unmap(reinterpret_cast<void *>(MapBase), NewMapBase - MapBase, 0, &Data);
+      MemMap.unmap(MapBase, NewMapBase - MapBase);
       MapBase = NewMapBase;
     }
     const uptr NewMapEnd =
         CommitBase + PageSize + roundUp(Size, PageSize) + PageSize;
     DCHECK_LE(NewMapEnd, MapEnd);
     if (SCUDO_WORDSIZE == 32U && NewMapEnd != MapEnd) {
-      unmap(reinterpret_cast<void *>(NewMapEnd), MapEnd - NewMapEnd, 0, &Data);
+      MemMap.unmap(NewMapEnd, MapEnd - NewMapEnd);
       MapEnd = NewMapEnd;
     }
   }
 
   const uptr CommitSize = MapEnd - PageSize - CommitBase;
   const uptr AllocPos = roundDown(CommitBase + CommitSize - Size, Alignment);
-  mapSecondary<Config>(Options, CommitBase, CommitSize, AllocPos, 0, &Data);
+  mapSecondary<Config>(Options, CommitBase, CommitSize, AllocPos, 0, MemMap);
   const uptr HeaderPos =
       AllocPos - Chunk::getHeaderSize() - LargeBlock::getHeaderSize();
   LargeBlock::Header *H = reinterpret_cast<LargeBlock::Header *>(
       LargeBlock::addHeaderTag<Config>(HeaderPos));
   if (useMemoryTagging<Config>(Options))
     storeTags(LargeBlock::addHeaderTag<Config>(CommitBase),
               reinterpret_cast<uptr>(H + 1));
-  H->MapBase = MapBase;
-  H->MapSize = MapEnd - MapBase;
   H->CommitBase = CommitBase;
   H->CommitSize = CommitSize;
-  H->Data = Data;
+  H->MemMap = MemMap;
   if (BlockEndPtr)
     *BlockEndPtr = CommitBase + CommitSize;
   {
@@ -602,7 +595,7 @@ void *MapAllocator<Config>::allocate(Options Options, uptr Size, uptr Alignment,
       LargestSize = CommitSize;
     NumberOfAllocs++;
     Stats.add(StatAllocated, CommitSize);
-    Stats.add(StatMapped, H->MapSize);
+    Stats.add(StatMapped, H->MemMap.getCapacity());
   }
   return reinterpret_cast<void *>(HeaderPos + LargeBlock::getHeaderSize());
 }
@@ -618,7 +611,7 @@ void MapAllocator<Config>::deallocate(Options Options, void *Ptr)
     FreedBytes += CommitSize;
     NumberOfFrees++;
     Stats.sub(StatAllocated, CommitSize);
-    Stats.sub(StatMapped, H->MapSize);
+    Stats.sub(StatMapped, H->MemMap.getCapacity());
   }
   Cache.store(Options, H);
 }