[TSan] Add lock_during_write flag on Apple platforms to avoid deadlock #157928

DanBlackwell · 2025-09-10T18:46:20Z

There is a commonly used library that interposes the write call, and is interposed itself. When running with TSAN_OPTIONS=verbosity=(1|2), this leads to a thread locking itself, resulting in the program hanging.

This patch adds a new flag lock_during_write to TSan (on Apple platforms only) that, when set, allows interceptors to be bypassed during calls to write. The flag can be inherited by children, or not, depending on the value.

rdar://157565672

llvmbot · 2025-09-10T18:46:56Z

@llvm/pr-subscribers-compiler-rt-sanitizer

Author: Dan Blackwell (DanBlackwell)

Changes

There is a commonly used library that interposes the write call, and is interposed itself. When running with TSAN_OPTIONS=verbosity=(1|2), this leads to a thread locking itself, resulting in the program hanging.

This patch adds a new flag lock_during_write to TSan (on Apple platforms only) that, when set, allows interceptors to be bypassed during calls to write. The flag can be inherited by children, or not, depending on the value.

rdar://157565672

Full diff: https://github.com/llvm/llvm-project/pull/157928.diff

9 Files Affected:

(modified) compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp (+12-1)
(modified) compiler-rt/lib/tsan/rtl/tsan_flags.cpp (+37)
(modified) compiler-rt/lib/tsan/rtl/tsan_flags.h (+8)
(modified) compiler-rt/lib/tsan/rtl/tsan_flags.inc (+12)
(modified) compiler-rt/lib/tsan/rtl/tsan_interceptors.h (+9-1)
(modified) compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp (+10)
(modified) compiler-rt/lib/tsan/rtl/tsan_rtl.cpp (+14)
(modified) compiler-rt/lib/tsan/rtl/tsan_rtl.h (+4)
(added) compiler-rt/test/tsan/Darwin/write-interpose.c (+40)

diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
index d4811ff4ed217..dfc7562f66848 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
@@ -167,8 +167,19 @@ uptr internal_read(fd_t fd, void *buf, uptr count) {
   return read(fd, buf, count);
 }
 
+}  // namespace __sanitizer
+
+// Weak symbol no-op when TSan is not linked
+SANITIZER_WEAK_ATTRIBUTE void __tsan_set_in_internal_write_call(bool value) {}
+
+namespace __sanitizer {
+
 uptr internal_write(fd_t fd, const void *buf, uptr count) {
-  return write(fd, buf, count);
+  // We need to disable interceptors when writing in TSan
+  __tsan_set_in_internal_write_call(true);
+  uptr res = write(fd, buf, count);
+  __tsan_set_in_internal_write_call(false);
+  return res;
 }
 
 uptr internal_stat(const char *path, void *buf) {
diff --git a/compiler-rt/lib/tsan/rtl/tsan_flags.cpp b/compiler-rt/lib/tsan/rtl/tsan_flags.cpp
index 3fd58f46983fd..50632d2016376 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_flags.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_flags.cpp
@@ -20,6 +20,43 @@
 #include "tsan_rtl.h"
 #include "ubsan/ubsan_flags.h"
 
+#if SANITIZER_APPLE
+namespace __sanitizer {
+
+template <>
+inline bool FlagHandler<LockDuringWriteSetting>::Parse(const char *value) {
+  if (internal_strcmp(value, "on") == 0) {
+    *t_ = kLockDuringAllWrites;
+    return true;
+  }
+  if (internal_strcmp(value, "disable_for_current_process") == 0) {
+    *t_ = kNoLockDuringWritesCurrentProcess;
+    return true;
+  }
+  if (internal_strcmp(value, "disable_for_all_processes") == 0) {
+    *t_ = kNoLockDuringWritesAllProcesses;
+    return true;
+  }
+  Printf("ERROR: Invalid value for signal handler option: '%s'\n", value);
+  return false;
+}
+
+template <>
+inline bool FlagHandler<LockDuringWriteSetting>::Format(char *buffer,
+                                                        uptr size) {
+  switch (*t_) {
+    case kLockDuringAllWrites:
+      return FormatString(buffer, size, "on");
+    case kNoLockDuringWritesCurrentProcess:
+      return FormatString(buffer, size, "disable_for_current_process");
+    case kNoLockDuringWritesAllProcesses:
+      return FormatString(buffer, size, "disable_for_all_processes");
+  }
+}
+
+}  // namespace __sanitizer
+#endif
+
 namespace __tsan {
 
 // Can be overriden in frontend.
diff --git a/compiler-rt/lib/tsan/rtl/tsan_flags.h b/compiler-rt/lib/tsan/rtl/tsan_flags.h
index da27d5b992bcb..477d08d334605 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_flags.h
+++ b/compiler-rt/lib/tsan/rtl/tsan_flags.h
@@ -16,6 +16,14 @@
 #include "sanitizer_common/sanitizer_flags.h"
 #include "sanitizer_common/sanitizer_deadlock_detector_interface.h"
 
+#if SANITIZER_APPLE
+enum LockDuringWriteSetting {
+  kLockDuringAllWrites,
+  kNoLockDuringWritesCurrentProcess,
+  kNoLockDuringWritesAllProcesses,
+};
+#endif
+
 namespace __tsan {
 
 struct Flags : DDFlags {
diff --git a/compiler-rt/lib/tsan/rtl/tsan_flags.inc b/compiler-rt/lib/tsan/rtl/tsan_flags.inc
index 731d776cc893e..64cc0919c0090 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_flags.inc
+++ b/compiler-rt/lib/tsan/rtl/tsan_flags.inc
@@ -80,3 +80,15 @@ TSAN_FLAG(bool, shared_ptr_interceptor, true,
 TSAN_FLAG(bool, print_full_thread_history, false,
           "If set, prints thread creation stacks for the threads involved in "
           "the report and their ancestors up to the main thread.")
+
+#if SANITIZER_APPLE
+TSAN_FLAG(LockDuringWriteSetting, lock_during_write, kLockDuringAllWrites,
+          "Determines whether to obtain a lock while writing logs or error "
+          "reports. "
+          "\"on\" - [default] lock during all writes. "
+          "\"disable_for_current_process\" - don't lock during all writes in "
+          "the current process, but do lock for all writes in child "
+          "processes."
+          "\"disable_for_all_processes\" - don't lock during all writes in "
+          "the current process and it's children processes.")
+#endif
diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors.h b/compiler-rt/lib/tsan/rtl/tsan_interceptors.h
index a357a870fdf8e..d4b65ab1aaa6a 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_interceptors.h
+++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors.h
@@ -1,6 +1,9 @@
 #ifndef TSAN_INTERCEPTORS_H
 #define TSAN_INTERCEPTORS_H
 
+#if SANITIZER_APPLE
+#  include "sanitizer_common/sanitizer_mac.h"
+#endif
 #include "sanitizer_common/sanitizer_stacktrace.h"
 #include "tsan_rtl.h"
 
@@ -43,7 +46,12 @@ inline bool in_symbolizer() {
 #endif
 
 inline bool MustIgnoreInterceptor(ThreadState *thr) {
-  return !thr->is_inited || thr->ignore_interceptors || thr->in_ignored_lib;
+  return !thr->is_inited || thr->ignore_interceptors || thr->in_ignored_lib
+#if SANITIZER_APPLE
+         || (flags()->lock_during_write != kLockDuringAllWrites &&
+             thr->in_internal_write_call)
+#endif
+      ;
 }
 
 }  // namespace __tsan
diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
index b46a81031258c..5de97ff549209 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
@@ -31,6 +31,9 @@
 #include "sanitizer_common/sanitizer_tls_get_addr.h"
 #include "sanitizer_common/sanitizer_vector.h"
 #include "tsan_fd.h"
+#if SANITIZER_APPLE
+#  include "tsan_flags.h"
+#endif
 #include "tsan_interceptors.h"
 #include "tsan_interface.h"
 #include "tsan_mman.h"
@@ -1649,6 +1652,13 @@ TSAN_INTERCEPTOR(int, pthread_barrier_wait, void *b) {
 
 TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) {
   SCOPED_INTERCEPTOR_RAW(pthread_once, o, f);
+#if SANITIZER_APPLE
+  if (flags()->lock_during_write != kLockDuringAllWrites &&
+      cur_thread_init()->in_internal_write_call) {
+    f();
+    return 0;
+  }
+#endif
   if (o == 0 || f == 0)
     return errno_EINVAL;
   atomic_uint32_t *a;
diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp
index 0d7247a56a4c2..b8041d724d342 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp
@@ -40,6 +40,13 @@ SANITIZER_WEAK_DEFAULT_IMPL
 void __tsan_test_only_on_fork() {}
 #endif
 
+#if SANITIZER_APPLE
+// Override weak symbol from sanitizer_common
+extern void __tsan_set_in_internal_write_call(bool value) {
+  __tsan::cur_thread_init()->in_internal_write_call = value;
+}
+#endif
+
 namespace __tsan {
 
 #if !SANITIZER_GO
@@ -893,6 +900,13 @@ void ForkChildAfter(ThreadState* thr, uptr pc, bool start_thread) {
     ThreadIgnoreBegin(thr, pc);
     ThreadIgnoreSyncBegin(thr, pc);
   }
+
+#  if SANITIZER_APPLE
+  // This flag can have inheritance disabled - we are the child so act
+  // accordingly
+  if (flags()->lock_during_write == kNoLockDuringWritesCurrentProcess)
+    flags()->lock_during_write = kLockDuringAllWrites;
+#  endif
 }
 #endif
 
diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl.h b/compiler-rt/lib/tsan/rtl/tsan_rtl.h
index 0b6d5f088b142..77390f090f8af 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_rtl.h
+++ b/compiler-rt/lib/tsan/rtl/tsan_rtl.h
@@ -236,6 +236,10 @@ struct alignas(SANITIZER_CACHE_LINE_SIZE) ThreadState {
 
   const ReportDesc *current_report;
 
+#if SANITIZER_APPLE
+  bool in_internal_write_call;
+#endif
+
   explicit ThreadState(Tid tid);
 };
 
diff --git a/compiler-rt/test/tsan/Darwin/write-interpose.c b/compiler-rt/test/tsan/Darwin/write-interpose.c
new file mode 100644
index 0000000000000..7e4d50694bdb4
--- /dev/null
+++ b/compiler-rt/test/tsan/Darwin/write-interpose.c
@@ -0,0 +1,40 @@
+// Test that dylibs interposing write, and then calling functions intercepted
+// by TSan don't deadlock (self-lock)
+
+// RUN: %clang_tsan %s -o %t
+// RUN: %clang_tsan %s -o %t.dylib -fno-sanitize=thread -dynamiclib -DSHARED_LIB
+
+// Note that running the below command with out `lock_during_write` should
+// deadlock (self-lock)
+// RUN: env DYLD_INSERT_LIBRARIES=%t.dylib TSAN_OPTIONS=verbosity=2:lock_during_write=disable_for_current_process %run %t 2>&1 | FileCheck %s
+
+#include <stdio.h>
+
+#if defined(SHARED_LIB)
+
+// dylib implementation - interposes write() calls
+#  include <mach-o/dyld-interposing.h>
+#  include <os/lock.h>
+#  include <unistd.h>
+
+static ssize_t my_write(int fd, const void *buf, size_t count) {
+  struct os_unfair_lock_s lock = OS_UNFAIR_LOCK_INIT;
+  os_unfair_lock_lock(&lock);
+  printf("Interposed write called: fd=%d, count=%zu\n", fd, count);
+  ssize_t res = write(fd, buf, count);
+  os_unfair_lock_unlock(&lock);
+  return res;
+}
+DYLD_INTERPOSE(my_write, write);
+
+#else // defined(SHARED_LIB)
+
+int main() {
+  printf("Write test completed\n");
+  return 0;
+}
+
+#endif // defined(SHARED_LIB)
+
+// CHECK: Interposed write called: fd={{[0-9]+}}, count={{[0-9]+}}
+// CHECK: Write test completed

compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp

github-actions · 2025-09-10T18:49:31Z

⚠️ C/C++ code formatter, clang-format found issues in your code. ⚠️

You can test this locally with the following command:

git-clang-format --diff origin/main HEAD --extensions cpp,inc,c,h -- compiler-rt/test/tsan/Darwin/write-interpose.c compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp compiler-rt/lib/tsan/rtl/tsan_flags.cpp compiler-rt/lib/tsan/rtl/tsan_flags.h compiler-rt/lib/tsan/rtl/tsan_flags.inc compiler-rt/lib/tsan/rtl/tsan_interceptors.h compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp compiler-rt/lib/tsan/rtl/tsan_rtl.cpp compiler-rt/lib/tsan/rtl/tsan_rtl.h

⚠️
The reproduction instructions above might return results for more than one PR
in a stack if you are using a stacked PR workflow. You can limit the results by
changing origin/main to the base branch/commit you want to compare against.
⚠️

View the diff from clang-format here.

diff --git a/compiler-rt/lib/tsan/rtl/tsan_flags.cpp b/compiler-rt/lib/tsan/rtl/tsan_flags.cpp
index 50632d201..c7413b0cc 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_flags.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_flags.cpp
@@ -24,7 +24,7 @@
 namespace __sanitizer {
 
 template <>
-inline bool FlagHandler<LockDuringWriteSetting>::Parse(const char *value) {
+inline bool FlagHandler<LockDuringWriteSetting>::Parse(const char* value) {
   if (internal_strcmp(value, "on") == 0) {
     *t_ = kLockDuringAllWrites;
     return true;
@@ -42,7 +42,7 @@ inline bool FlagHandler<LockDuringWriteSetting>::Parse(const char *value) {
 }
 
 template <>
-inline bool FlagHandler<LockDuringWriteSetting>::Format(char *buffer,
+inline bool FlagHandler<LockDuringWriteSetting>::Format(char* buffer,
                                                         uptr size) {
   switch (*t_) {
     case kLockDuringAllWrites:

wrotki · 2025-09-10T19:16:56Z

Maybe the new flag should be documented in clang/docs/ThreadSanitizer.rst . The doc is sparse there (actually it doesn't mention any TSAN_OPTIONS). But AddressSanitizer.rst has some, with usage examples, so maybe we should follow the pattern.

wrotki

LGTM

thurstond · 2025-09-10T20:39:45Z

Maybe the new flag should be documented in clang/docs/ThreadSanitizer.rst . The doc is sparse there (actually it doesn't mention any TSAN_OPTIONS).

Historically, the TSan flags were documented at https://github.com/google/sanitizers/wiki/ThreadSanitizerFlags
However, that repository has now been archived (that page in particular has not been updated in 10 years).

DanBlackwell · 2025-09-11T14:54:02Z

Note: I've tested the following on Ubuntu to see if the same thing can happen:

#include <stdio.h>
#include <string.h>
#include <unistd.h>

#ifdef SHARED_LIB
#include <dlfcn.h>
#include <stdlib.h>
#include <pthread.h>

pthread_mutex_t mtx;

typedef ssize_t (*func_t)(int, const void *, size_t);

extern "C" ssize_t write(int fd, const void *buf, size_t count) {
    static func_t write_real = NULL;
    write_real = (func_t)dlsym(RTLD_NEXT, "write");
    pthread_mutex_lock(&mtx);
    printf("Intercepted\n");
    pthread_mutex_unlock(&mtx);
    return write_real(fd, buf, count);
}

#else

int main() {
    write(1, "test", 4);
    return 0;
}

#endif

From what I can tell, any calls within the write wrapper here (pthread_mutex_lock) don't get intercepted by TSan; thus it can't deadlock in the same way as Apple can with interposition. I will keep this flag as Apple only, as it solves a problem that is Apple specific.

compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp

…lock There is a common library that interposes the write call, and is interposed itself. When running with TSAN_OPTIONS=verbosity=(1|2), this leads to a thread locking itself, resulting in a hang. This patch adds a new flag `lock_during_write` to TSan that allows interceptors to be bypassed during calls to write when set. The flag can be inherited by children, or not, depending on the value. rdar://157565672

#157928) There is a commonly used library that interposes the write call, and is interposed itself. When running with `TSAN_OPTIONS=verbosity=(1|2)`, this leads to a thread locking itself, resulting in the program hanging. This patch adds a new flag `lock_during_write` to TSan (on Apple platforms only) that, when set, allows interceptors to be bypassed during calls to write. The flag can be inherited by children, or not, depending on the value. rdar://157565672

llvm#157928) There is a commonly used library that interposes the write call, and is interposed itself. When running with `TSAN_OPTIONS=verbosity=(1|2)`, this leads to a thread locking itself, resulting in the program hanging. This patch adds a new flag `lock_during_write` to TSan (on Apple platforms only) that, when set, allows interceptors to be bypassed during calls to write. The flag can be inherited by children, or not, depending on the value. rdar://157565672

DanBlackwell requested review from thetruestblue, wrotki and yln September 10, 2025 18:46

llvmbot added compiler-rt compiler-rt:tsan Thread sanitizer compiler-rt:sanitizer labels Sep 10, 2025

DanBlackwell commented Sep 10, 2025

View reviewed changes

compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp Show resolved Hide resolved

wrotki approved these changes Sep 10, 2025

View reviewed changes

DanBlackwell commented Sep 11, 2025

View reviewed changes

compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp Outdated Show resolved Hide resolved

DanBlackwell force-pushed the tsan-write-lock-optional-bypass branch from 82db9fc to bd55046 Compare September 24, 2025 15:09

DanBlackwell force-pushed the tsan-write-lock-optional-bypass branch from bd55046 to c2c6fed Compare September 29, 2025 13:45

DanBlackwell merged commit 9abd6f2 into llvm:main Oct 9, 2025
8 of 9 checks passed

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

[TSan] Add lock_during_write flag on Apple platforms to avoid deadlock #157928

[TSan] Add lock_during_write flag on Apple platforms to avoid deadlock #157928

Uh oh!

DanBlackwell commented Sep 10, 2025

Uh oh!

llvmbot commented Sep 10, 2025

Uh oh!

Uh oh!

github-actions bot commented Sep 10, 2025 •

edited

Loading

Uh oh!

wrotki commented Sep 10, 2025

Uh oh!

wrotki left a comment

Uh oh!

thurstond commented Sep 10, 2025

Uh oh!

DanBlackwell commented Sep 11, 2025

Uh oh!

Uh oh!

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

4 participants

[TSan] Add lock_during_write flag on Apple platforms to avoid deadlock #157928

[TSan] Add lock_during_write flag on Apple platforms to avoid deadlock #157928

Uh oh!

Conversation

DanBlackwell commented Sep 10, 2025

Uh oh!

llvmbot commented Sep 10, 2025

Uh oh!

Uh oh!

github-actions bot commented Sep 10, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

wrotki commented Sep 10, 2025

Uh oh!

wrotki left a comment

Choose a reason for hiding this comment

Uh oh!

thurstond commented Sep 10, 2025

Uh oh!

DanBlackwell commented Sep 11, 2025

Uh oh!

Uh oh!

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

4 participants

github-actions bot commented Sep 10, 2025 •

edited

Loading