[scudo][standalone] Introduce the Secondary allocator

Summary:
The Secondary allocator wraps the platform allocation primitives. It is
meant to be used for larger sizes that the Primary can't fullfill, as
it will be slower, and sizes are multiple of the system page size.

This also changes some of the existing code, notably the opaque
platform data being passed to the platform specific functions: we can
shave a couple of syscalls on Fuchsia by storing additional data (this
addresses a TODO).

Reviewers: eugenis, vitalybuka, hctim, morehouse

Reviewed By: morehouse

Subscribers: mgorny, delcypher, jfb, #sanitizers, llvm-commits

Tags: #llvm, #sanitizers

Differential Revision: https://reviews.llvm.org/D60787

llvm-svn: 359097

compiler-rt/lib/scudo/standalone/CMakeLists.txt

@@ -42,6 +42,7 @@ set(SCUDO_SOURCES
   fuchsia.cc
   linux.cc
   report.cc
+  secondary.cc
   string_utils.cc)
 
 # Enable the SSE 4.2 instruction set for crc32_hw.cc, if available.
@@ -61,13 +62,15 @@ set(SCUDO_HEADERS
   checksum.h
   flags.h
   flags_parser.h
+  fuchsia.h
   interface.h
   internal_defs.h
   linux.h
   list.h
   mutex.h
   platform.h
   report.h
+  secondary.h
   stats.h
   string_utils.h
   vector.h)

compiler-rt/lib/scudo/standalone/common.h

@@ -11,6 +11,9 @@
 
 #include "internal_defs.h"
 
+#include "fuchsia.h"
+#include "linux.h"
+
 #include <stddef.h>
 #include <string.h>
 
@@ -144,20 +147,21 @@ bool getRandom(void *Buffer, uptr Length, bool Blocking = false);
 // - commit memory in a previously reserved space;
 // - commit memory at a random address.
 // As such, only a subset of parameters combinations is valid, which is checked
-// by the function implementation. The Extra parameter allows to pass opaque
+// by the function implementation. The Data parameter allows to pass opaque
 // platform specific data to the function.
 // Returns nullptr on error or dies if MAP_ALLOWNOMEM is not specified.
 void *map(void *Addr, uptr Size, const char *Name, uptr Flags = 0,
-          u64 *Extra = nullptr);
+          MapPlatformData *Data = nullptr);
 
 // Indicates that we are getting rid of the whole mapping, which might have
-// further consequences on Extra, depending on the platform.
+// further consequences on Data, depending on the platform.
 #define UNMAP_ALL (1U << 0)
 
-void unmap(void *Addr, uptr Size, uptr Flags = 0, u64 *Extra = nullptr);
+void unmap(void *Addr, uptr Size, uptr Flags = 0,
+           MapPlatformData *Data = nullptr);
 
 void releasePagesToOS(uptr BaseAddress, uptr Offset, uptr Size,
-                      u64 *Extra = nullptr);
+                      MapPlatformData *Data = nullptr);
 
 // Internal map & unmap fatal error. This must not call map().
 void NORETURN dieOnMapUnmapError(bool OutOfMemory = false);

compiler-rt/lib/scudo/standalone/fuchsia.cc

@@ -15,8 +15,7 @@
 #include "string_utils.h"
 
 #include <limits.h> // for PAGE_SIZE
-#include <stdlib.h> // for abort()
-#include <zircon/process.h>
+#include <stdlib.h> // for getenv()
 #include <zircon/sanitizer.h>
 #include <zircon/syscalls.h>
 
@@ -35,69 +34,45 @@ void NORETURN die() { __builtin_trap(); }
 // with ZX_HANDLE_INVALID.
 COMPILER_CHECK(ZX_HANDLE_INVALID == 0);
 
-struct MapInfo {
-  zx_handle_t Vmar;
-  zx_handle_t Vmo;
-};
-COMPILER_CHECK(sizeof(MapInfo) == sizeof(u64));
-
-static void *allocateVmar(uptr Size, MapInfo *Info, bool AllowNoMem) {
+static void *allocateVmar(uptr Size, MapPlatformData *Data, bool AllowNoMem) {
   // Only scenario so far.
-  DCHECK(Info);
-  DCHECK_EQ(Info->Vmar, ZX_HANDLE_INVALID);
+  DCHECK(Data);
+  DCHECK_EQ(Data->Vmar, ZX_HANDLE_INVALID);
 
-  uintptr_t P;
   const zx_status_t Status = _zx_vmar_allocate(
       _zx_vmar_root_self(),
       ZX_VM_CAN_MAP_READ | ZX_VM_CAN_MAP_WRITE | ZX_VM_CAN_MAP_SPECIFIC, 0,
-      Size, &Info->Vmar, &P);
+      Size, &Data->Vmar, &Data->VmarBase);
   if (Status != ZX_OK) {
     if (Status != ZX_ERR_NO_MEMORY || !AllowNoMem)
       dieOnMapUnmapError(Status == ZX_ERR_NO_MEMORY);
     return nullptr;
   }
-  return reinterpret_cast<void *>(P);
-}
-
-// Returns the offset of an address range in a Vmar, while checking that said
-// range fully belongs to the Vmar. An alternative would be to keep track of
-// both the base & length to avoid calling this. The tradeoff being a system
-// call vs two extra uptr of storage.
-// TODO(kostyak): revisit the implications of both options.
-static uint64_t getOffsetInVmar(zx_handle_t Vmar, void *Addr, uintptr_t Size) {
-  zx_info_vmar_t Info;
-  const zx_status_t Status = _zx_object_get_info(
-      Vmar, ZX_INFO_VMAR, &Info, sizeof(Info), nullptr, nullptr);
-  CHECK_EQ(Status, ZX_OK);
-  const uint64_t Offset = reinterpret_cast<uintptr_t>(Addr) - Info.base;
-  CHECK_LE(Offset, Info.base + Info.len);
-  CHECK_LE(Offset + Size, Info.base + Info.len);
-  return Offset;
+  return reinterpret_cast<void *>(Data->VmarBase);
 }
 
-void *map(void *Addr, uptr Size, const char *Name, uptr Flags, u64 *Extra) {
+void *map(void *Addr, uptr Size, const char *Name, uptr Flags,
+          MapPlatformData *Data) {
   DCHECK_EQ(Size % PAGE_SIZE, 0);
   const bool AllowNoMem = !!(Flags & MAP_ALLOWNOMEM);
-  MapInfo *Info = reinterpret_cast<MapInfo *>(Extra);
 
   // For MAP_NOACCESS, just allocate a Vmar and return.
   if (Flags & MAP_NOACCESS)
-    return allocateVmar(Size, Info, AllowNoMem);
+    return allocateVmar(Size, Data, AllowNoMem);
 
-  const zx_handle_t Vmar = Info ? Info->Vmar : _zx_vmar_root_self();
+  const zx_handle_t Vmar = Data ? Data->Vmar : _zx_vmar_root_self();
   CHECK_NE(Vmar, ZX_HANDLE_INVALID);
 
   zx_status_t Status;
   zx_handle_t Vmo;
   uint64_t VmoSize = 0;
-  if (Info && Info->Vmo != ZX_HANDLE_INVALID) {
+  if (Data && Data->Vmo != ZX_HANDLE_INVALID) {
     // If a Vmo was specified, it's a resize operation.
     CHECK(Addr);
     DCHECK(Flags & MAP_RESIZABLE);
-    Vmo = Info->Vmo;
-    Status = _zx_vmo_get_size(Vmo, &VmoSize);
-    if (Status == ZX_OK)
-      Status = _zx_vmo_set_size(Vmo, VmoSize + Size);
+    Vmo = Data->Vmo;
+    VmoSize = Data->VmoSize;
+    Status = _zx_vmo_set_size(Vmo, VmoSize + Size);
     if (Status != ZX_OK) {
       if (Status != ZX_ERR_NO_MEMORY || !AllowNoMem)
         dieOnMapUnmapError(Status == ZX_ERR_NO_MEMORY);
@@ -116,15 +91,16 @@ void *map(void *Addr, uptr Size, const char *Name, uptr Flags, u64 *Extra) {
 
   uintptr_t P;
   zx_vm_option_t MapFlags = ZX_VM_PERM_READ | ZX_VM_PERM_WRITE;
-  const uint64_t Offset = Addr ? getOffsetInVmar(Vmar, Addr, Size) : 0;
+  const uint64_t Offset =
+      Addr ? reinterpret_cast<uintptr_t>(Addr) - Data->VmarBase : 0;
   if (Offset)
     MapFlags |= ZX_VM_SPECIFIC;
   Status = _zx_vmar_map(Vmar, MapFlags, Offset, Vmo, VmoSize, Size, &P);
   // No need to track the Vmo if we don't intend on resizing it. Close it.
   if (Flags & MAP_RESIZABLE) {
-    DCHECK(Info);
-    DCHECK_EQ(Info->Vmo, ZX_HANDLE_INVALID);
-    Info->Vmo = Vmo;
+    DCHECK(Data);
+    DCHECK_EQ(Data->Vmo, ZX_HANDLE_INVALID);
+    Data->Vmo = Vmo;
   } else {
     CHECK_EQ(_zx_handle_close(Vmo), ZX_OK);
   }
@@ -133,42 +109,41 @@ void *map(void *Addr, uptr Size, const char *Name, uptr Flags, u64 *Extra) {
       dieOnMapUnmapError(Status == ZX_ERR_NO_MEMORY);
     return nullptr;
   }
+  if (Data)
+    Data->VmoSize += Size;
 
   return reinterpret_cast<void *>(P);
 }
 
-void unmap(void *Addr, uptr Size, uptr Flags, u64 *Extra) {
-  MapInfo *Info = reinterpret_cast<MapInfo *>(Extra);
+void unmap(void *Addr, uptr Size, uptr Flags, MapPlatformData *Data) {
   if (Flags & UNMAP_ALL) {
-    DCHECK_NE(Info, nullptr);
-    const zx_handle_t Vmar = Info->Vmar;
+    DCHECK_NE(Data, nullptr);
+    const zx_handle_t Vmar = Data->Vmar;
     DCHECK_NE(Vmar, _zx_vmar_root_self());
     // Destroying the vmar effectively unmaps the whole mapping.
     CHECK_EQ(_zx_vmar_destroy(Vmar), ZX_OK);
     CHECK_EQ(_zx_handle_close(Vmar), ZX_OK);
   } else {
-    const zx_handle_t Vmar = Info ? Info->Vmar : _zx_vmar_root_self();
+    const zx_handle_t Vmar = Data ? Data->Vmar : _zx_vmar_root_self();
     const zx_status_t Status =
         _zx_vmar_unmap(Vmar, reinterpret_cast<uintptr_t>(Addr), Size);
     if (Status != ZX_OK)
       dieOnMapUnmapError();
   }
-  if (Info) {
-    if (Info->Vmo != ZX_HANDLE_INVALID)
-      CHECK_EQ(_zx_handle_close(Info->Vmo), ZX_OK);
-    Info->Vmo = ZX_HANDLE_INVALID;
-    Info->Vmar = ZX_HANDLE_INVALID;
+  if (Data) {
+    if (Data->Vmo != ZX_HANDLE_INVALID)
+      CHECK_EQ(_zx_handle_close(Data->Vmo), ZX_OK);
+    memset(Data, 0, sizeof(*Data));
   }
 }
 
 void releasePagesToOS(UNUSED uptr BaseAddress, uptr Offset, uptr Size,
-                      u64 *Extra) {
-  MapInfo *Info = reinterpret_cast<MapInfo *>(Extra);
-  DCHECK(Info);
-  DCHECK_NE(Info->Vmar, ZX_HANDLE_INVALID);
-  DCHECK_NE(Info->Vmo, ZX_HANDLE_INVALID);
+                      MapPlatformData *Data) {
+  DCHECK(Data);
+  DCHECK_NE(Data->Vmar, ZX_HANDLE_INVALID);
+  DCHECK_NE(Data->Vmo, ZX_HANDLE_INVALID);
   const zx_status_t Status =
-      _zx_vmo_op_range(Info->Vmo, ZX_VMO_OP_DECOMMIT, Offset, Size, NULL, 0);
+      _zx_vmo_op_range(Data->Vmo, ZX_VMO_OP_DECOMMIT, Offset, Size, NULL, 0);
   CHECK_EQ(Status, ZX_OK);
 }
 
@@ -188,7 +163,7 @@ void BlockingMutex::wake() {
   CHECK_EQ(Status, ZX_OK);
 }
 
-u64 getMonotonicTime() { return _zx_clock_get(ZX_CLOCK_MONOTONIC); }
+u64 getMonotonicTime() { return _zx_clock_get_monotonic(); }
 
 u32 getNumberOfCPUs() { return _zx_system_get_num_cpus(); }
 

compiler-rt/lib/scudo/standalone/fuchsia.h

@@ -0,0 +1,31 @@
+//===-- fuchsia.h -----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SCUDO_FUCHSIA_H_
+#define SCUDO_FUCHSIA_H_
+
+#include "platform.h"
+
+#if SCUDO_FUCHSIA
+
+#include <zircon/process.h>
+
+namespace scudo {
+
+struct MapPlatformData {
+  zx_handle_t Vmar;
+  zx_handle_t Vmo;
+  uintptr_t VmarBase;
+  uint64_t VmoSize;
+};
+
+} // namespace scudo
+
+#endif // SCUDO_FUCHSIA
+
+#endif // SCUDO_FUCHSIA_H_

compiler-rt/lib/scudo/standalone/linux.cc

@@ -44,7 +44,7 @@ uptr getPageSize() { return static_cast<uptr>(sysconf(_SC_PAGESIZE)); }
 void NORETURN die() { abort(); }
 
 void *map(void *Addr, uptr Size, UNUSED const char *Name, uptr Flags,
-          UNUSED u64 *Extra) {
+          UNUSED MapPlatformData *Data) {
   int MmapFlags = MAP_PRIVATE | MAP_ANON;
   if (Flags & MAP_NOACCESS)
     MmapFlags |= MAP_NORESERVE;
@@ -68,13 +68,14 @@ void *map(void *Addr, uptr Size, UNUSED const char *Name, uptr Flags,
   return P;
 }
 
-void unmap(void *Addr, uptr Size, UNUSED uptr Flags, UNUSED u64 *Extra) {
+void unmap(void *Addr, uptr Size, UNUSED uptr Flags,
+           UNUSED MapPlatformData *Data) {
   if (munmap(Addr, Size) != 0)
     dieOnMapUnmapError();
 }
 
 void releasePagesToOS(uptr BaseAddress, uptr Offset, uptr Size,
-                      UNUSED u64 *Extra) {
+                      UNUSED MapPlatformData *Data) {
   void *Addr = reinterpret_cast<void *>(BaseAddress + Offset);
   while (madvise(Addr, Size, MADV_DONTNEED) == -1 && errno == EAGAIN) {
   }

compiler-rt/lib/scudo/standalone/linux.h

@@ -15,6 +15,9 @@
 
 namespace scudo {
 
+// MapPlatformData is unused on Linux, define it as a minimally sized structure.
+struct MapPlatformData {};
+
 #if SCUDO_ANDROID
 
 #if defined(__aarch64__)

compiler-rt/lib/scudo/standalone/list.h

@@ -110,9 +110,9 @@ template <class Item> struct IntrusiveList {
       CHECK_EQ(Last, 0);
     } else {
       uptr count = 0;
-      for (Item *i = First;; i = i->Next) {
+      for (Item *I = First;; I = I->Next) {
         count++;
-        if (i == Last)
+        if (I == Last)
           break;
       }
       CHECK_EQ(size(), count);