[NFC][ASAN] Replace AsanInitIsRunning with TryAsanInitFromRtl

[vitalybuka]

No description provided.

[llvmbot]

@llvm/pr-subscribers-compiler-rt-sanitizer

Author: Vitaly Buka (vitalybuka)

Changes

---

Full diff: https://github.com/llvm/llvm-project/pull/74171.diff

4 Files Affected:

• (modified) compiler-rt/lib/asan/asan_interceptors.cpp (+19-17)
• (modified) compiler-rt/lib/asan/asan_internal.h (+1-1)
• (modified) compiler-rt/lib/asan/asan_malloc_linux.cpp (+1-4)
• (modified) compiler-rt/lib/asan/asan_rtl.cpp (+9-1)

diff --git a/compiler-rt/lib/asan/asan_interceptors.cpp b/compiler-rt/lib/asan/asan_interceptors.cpp
index e80f66142b7a2fb..a364b971bda8f32 100644
--- a/compiler-rt/lib/asan/asan_interceptors.cpp
+++ b/compiler-rt/lib/asan/asan_interceptors.cpp
@@ -96,14 +96,16 @@ DECLARE_REAL_AND_INTERCEPTOR(void, free, void *)
   ASAN_WRITE_RANGE(ctx, ptr, size)
 #define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \
   ASAN_READ_RANGE(ctx, ptr, size)
-#  define COMMON_INTERCEPTOR_ENTER(ctx, func, ...)    \
-    ASAN_INTERCEPTOR_ENTER(ctx, func);                \
-    do {                                              \
-      if (AsanInitIsRunning())                        \
-        return REAL(func)(__VA_ARGS__);               \
-      if (SANITIZER_APPLE && UNLIKELY(!AsanInited())) \
-        return REAL(func)(__VA_ARGS__);               \
-      ENSURE_ASAN_INITED();                           \
+#  define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \
+    ASAN_INTERCEPTOR_ENTER(ctx, func);             \
+    do {                                           \
+      if constexpr (SANITIZER_APPLE) {             \
+        if (UNLIKELY(!AsanInited()))               \
+          return REAL(func)(__VA_ARGS__);          \
+      } else {                                     \
+        if (!TryAsanInitFromRtl())                 \
+          return REAL(func)(__VA_ARGS__);          \
+      }                                            \
     } while (false)
 #define COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path) \
   do {                                            \
@@ -534,16 +536,16 @@ INTERCEPTOR(char*, strncat, char *to, const char *from, uptr size) {
 INTERCEPTOR(char *, strcpy, char *to, const char *from) {
   void *ctx;
   ASAN_INTERCEPTOR_ENTER(ctx, strcpy);
-#if SANITIZER_APPLE
-  if (UNLIKELY(!AsanInited()))
-    return REAL(strcpy)(to, from);
-#endif
-  // strcpy is called from malloc_default_purgeable_zone()
-  // in __asan::ReplaceSystemAlloc() on Mac.
-  if (AsanInitIsRunning()) {
-    return REAL(strcpy)(to, from);
+  if constexpr (SANITIZER_APPLE) {
+    // strcpy is called from malloc_default_purgeable_zone()
+    // in __asan::ReplaceSystemAlloc() on Mac.
+    if (UNLIKELY(!AsanInited()))
+      return REAL(strcpy)(to, from);
+  } else {
+    if (!TryAsanInitFromRtl())
+      return REAL(strcpy)(to, from);
   }
-  ENSURE_ASAN_INITED();
+
   if (flags()->replace_str) {
     uptr from_size = internal_strlen(from) + 1;
     CHECK_RANGES_OVERLAP("strcpy", to, from_size, from, from_size);
diff --git a/compiler-rt/lib/asan/asan_internal.h b/compiler-rt/lib/asan/asan_internal.h
index e2b1e9800f5be62..5b97e77882cd674 100644
--- a/compiler-rt/lib/asan/asan_internal.h
+++ b/compiler-rt/lib/asan/asan_internal.h
@@ -60,6 +60,7 @@ class AsanThread;
 using __sanitizer::StackTrace;
 
 void AsanInitFromRtl();
+bool TryAsanInitFromRtl();
 
 // asan_win.cpp
 void InitializePlatformExceptionHandlers();
@@ -131,7 +132,6 @@ void InstallAtExitCheckLeaks();
   __asan_on_error()
 
 bool AsanInited();
-bool AsanInitIsRunning();  // Used to avoid infinite recursion in __asan_init().
 extern bool replace_intrin_cached;
 extern void (*death_callback)(void);
 // These magic values are written to shadow for better error
diff --git a/compiler-rt/lib/asan/asan_malloc_linux.cpp b/compiler-rt/lib/asan/asan_malloc_linux.cpp
index 0ba74c5d71432bd..eb29233c3cf82ff 100644
--- a/compiler-rt/lib/asan/asan_malloc_linux.cpp
+++ b/compiler-rt/lib/asan/asan_malloc_linux.cpp
@@ -31,7 +31,7 @@
 using namespace __asan;
 
 struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> {
-  static bool UseImpl() { return AsanInitIsRunning(); }
+  static bool UseImpl() { return !TryAsanInitFromRtl(); }
   static void OnAllocate(const void *ptr, uptr size) {
 #  if CAN_SANITIZE_LEAKS
     // Suppress leaks from dlerror(). Previously dlsym hack on global array was
@@ -65,7 +65,6 @@ INTERCEPTOR(void, cfree, void *ptr) {
 INTERCEPTOR(void*, malloc, uptr size) {
   if (DlsymAlloc::Use())
     return DlsymAlloc::Allocate(size);
-  ENSURE_ASAN_INITED();
   GET_STACK_TRACE_MALLOC;
   return asan_malloc(size, &stack);
 }
@@ -73,7 +72,6 @@ INTERCEPTOR(void*, malloc, uptr size) {
 INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) {
   if (DlsymAlloc::Use())
     return DlsymAlloc::Callocate(nmemb, size);
-  ENSURE_ASAN_INITED();
   GET_STACK_TRACE_MALLOC;
   return asan_calloc(nmemb, size, &stack);
 }
@@ -81,7 +79,6 @@ INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) {
 INTERCEPTOR(void*, realloc, void *ptr, uptr size) {
   if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr))
     return DlsymAlloc::Realloc(ptr, size);
-  ENSURE_ASAN_INITED();
   GET_STACK_TRACE_MALLOC;
   return asan_realloc(ptr, size, &stack);
 }
diff --git a/compiler-rt/lib/asan/asan_rtl.cpp b/compiler-rt/lib/asan/asan_rtl.cpp
index b129e22813dac99..45bd4e3c18f0fc4 100644
--- a/compiler-rt/lib/asan/asan_rtl.cpp
+++ b/compiler-rt/lib/asan/asan_rtl.cpp
@@ -80,7 +80,7 @@ static void SetAsanInitIsRunning(u32 val) { asan_init_is_running = val; }
 
 bool AsanInited() { return asan_inited == 1; }
 
-bool AsanInitIsRunning() { return asan_init_is_running == 1; }
+static bool AsanInitIsRunning() { return asan_init_is_running == 1; }
 
 bool replace_intrin_cached;
 
@@ -525,6 +525,14 @@ void AsanInitFromRtl() {
     AsanInitInternal();
 }
 
+bool TryAsanInitFromRtl() {
+  if (UNLIKELY(AsanInitIsRunning()))
+    return false;
+  if (UNLIKELY(!AsanInited()))
+    AsanInitInternal();
+  return true;
+}
+
 #if ASAN_DYNAMIC
 // Initialize runtime in case it's LD_PRELOAD-ed into unsanitized executable
 // (and thus normal initializers from .preinit_array or modules haven't run).