i#95 Linux detach: detaching via dr_app_stop_and_cleanup()

Modeled on Peter Goodman's code at https://codereview.appspot.com/13314047
but with many changes, including:
+ not including any of the re-takeover changes: global var resets, etc.
+ using a futex instead of sleeping and checking a volatile bool.

This does retain the duplication of Windows detach code from the patch.
That will be addressed in an upcoming CL.

Adds a new app API routine dr_app_stop_and_cleanup() which triggers a
detach that can assume it's on an app thread and app stack.

Implements detach cleanup for Linux by splitting the portions that must be
executed by each thread (mainly segment restoration) from the rest of the
cleanup that the detaching thread can perform.

The final go-native step is accomplished via sigreturn using a generalized
dynamorio_futex_wake_and_exit/dynamorio_semaphore_signal_all routine,
renamed to dynamorio_condvar_wake_and_jmp and taking in a target routine to
jump to after the wake.  This target is dynamorio_sigreturn for detach.

Adds two app-detach tests.

Review-URL: https://codereview.appspot.com/302710043

core/arch/aarchxx/aarchxx.asm

@@ -89,12 +89,12 @@ GLOBAL_LABEL(xfer_to_new_libdr:)
 # endif /* !STANDALONE_UNIT_TEST && !STATIC_LIBRARY */
 #endif /* UNIX */
 
-/* we need to call futex_wakeall without using any stack, to support
- * THREAD_SYNCH_TERMINATED_AND_CLEANED.
- * takes int* futex in r0.
+/* We need to call futex_wakeall without using any stack.
+ * Takes KSYNCH_TYPE* in r0 and the post-syscall jump target in r1.
  */
-        DECLARE_FUNC(dynamorio_futex_wake_and_exit)
-GLOBAL_LABEL(dynamorio_futex_wake_and_exit:)
+        DECLARE_FUNC(dynamorio_condvar_wake_and_jmp)
+GLOBAL_LABEL(dynamorio_condvar_wake_and_jmp:)
+        mov      REG_R12, REG_R1 /* save across syscall */
         mov      REG_R5, #0 /* arg6 */
         mov      REG_R4, #0 /* arg5 */
         mov      REG_R3, #0 /* arg4 */
@@ -107,7 +107,7 @@ GLOBAL_LABEL(dynamorio_futex_wake_and_exit:)
         mov      r7, #240 /* SYS_futex */
 #endif
         svc      #0
-        b        GLOBAL_REF(dynamorio_sys_exit)
-        END_FUNC(dynamorio_futex_wake_and_exit)
+        INDJMP   REG_R12
+        END_FUNC(dynamorio_condvar_wake_and_jmp)
 
 END_FILE

core/arch/arch_exports.h

@@ -1146,16 +1146,12 @@ void dynamorio_earliest_init_takeover(void);
 void client_int_syscall(void);
 void dynamorio_sigreturn(void);
 void dynamorio_sys_exit(void);
-# ifdef MACOS
-void dynamorio_semaphore_signal_all(KSYNCH_TYPE *ksynch/*in xax*/);
-# endif
+void dynamorio_condvar_wake_and_jmp(KSYNCH_TYPE *ksynch/*in xax/r0*/,
+                                    byte *jmp_tgt/*in xcx/r1*/);
 # ifdef LINUX
-void dynamorio_futex_wake_and_exit(volatile int *futex/* in xax*/);
 #  ifndef X64
 void dynamorio_nonrt_sigreturn(void);
 #  endif
-# endif
-# ifdef LINUX
 thread_id_t dynamorio_clone(uint flags, byte *newsp, void *ptid, void *tls,
                             void *ctid, void (*func)(void));
 void xfer_to_new_libdr(app_pc entry, void **init_sp, byte *cur_dr_map,

core/arch/x86/x86.asm

@@ -1299,14 +1299,17 @@ dynamorio_sys_exit_failed:
         jmp      GLOBAL_REF(unexpected_return)
         END_FUNC(dynamorio_sys_exit)
 
-#ifdef LINUX
-/* we need to call futex_wakeall without using any stack, to support
- * THREAD_SYNCH_TERMINATED_AND_CLEANED.
- * takes int* futex in xax.
+#ifdef UNIX
+/* We need to signal a futex or semaphore without using our dstack, to support
+ * THREAD_SYNCH_TERMINATED_AND_CLEANED and detach.
+ * Takes KSYNCH_TYPE* in xax and the post-syscall jump target in xcx.
  */
-        DECLARE_FUNC(dynamorio_futex_wake_and_exit)
-GLOBAL_LABEL(dynamorio_futex_wake_and_exit:)
-#ifdef X64
+        DECLARE_FUNC(dynamorio_condvar_wake_and_jmp)
+GLOBAL_LABEL(dynamorio_condvar_wake_and_jmp:)
+# ifdef LINUX
+        /* We call futex_wakeall */
+#  ifdef X64
+        mov      r12, rcx /* save across syscall */
         mov      ARG6, 0
         mov      ARG5, 0
         mov      ARG4, 0
@@ -1316,7 +1319,10 @@ GLOBAL_LABEL(dynamorio_futex_wake_and_exit:)
         mov      rax, 202 /* SYS_futex */
         mov      r10, rcx
         syscall
-#else
+        jmp      r12
+#  else
+        /* We use the stack, which should be the app stack: see the MacOS args below. */
+        push     ecx /* save across syscall */
         mov      ebp, 0 /* arg6 */
         mov      edi, 0 /* arg5 */
         mov      esi, 0 /* arg4 */
@@ -1326,28 +1332,23 @@ GLOBAL_LABEL(dynamorio_futex_wake_and_exit:)
         mov      eax, 240 /* SYS_futex */
         /* PR 254280: we assume int$80 is ok even for LOL64 */
         int      HEX(80)
-#endif
-        jmp      GLOBAL_REF(dynamorio_sys_exit)
-        END_FUNC(dynamorio_futex_wake_and_exit)
-#endif /* LINUX */
-
-#ifdef MACOS
-/* We need to call semaphore_signal_all without using dstack, to support
- * THREAD_SYNCH_TERMINATED_AND_CLEANED.  We have to put syscall args on
- * the stack for 32-bit, and we use the stack for call;pop for
- * sysenter -- so we use the app stack, which we assume the caller has
- * put us on.  We're only called when terminating a thread so transparency
- * should be ok so long as the app's stack is valid.
- * Takes KSYNCH_TYPE* in xax.
- */
-        DECLARE_FUNC(dynamorio_semaphore_signal_all)
-GLOBAL_LABEL(dynamorio_semaphore_signal_all:)
+        pop      ecx
+        jmp      ecx
+#  endif
+# elif defined(MACOS)
+       /* We call semaphore_signal_all.  We have to put syscall args on
+        * the stack for 32-bit, and we use the stack for call;pop for
+        * sysenter -- so we use the app stack, which we assume the caller has
+        * put us on.  We're only called when terminating a thread or detaching
+        * so transparency should be ok so long as the app's stack is valid.
+        */
+        mov      REG_XDI, REG_XCX /* save across syscall */
         mov      REG_XAX, DWORD [REG_XAX] /* load mach_synch_t->sem */
-# ifdef X64
+#  ifdef X64
         mov      ARG1, REG_XAX
         mov      eax, MACH_semaphore_signal_all_trap
         or       eax, SYSCALL_NUM_MARKER_MACH
-# else
+#  else
         push     REG_XAX
         mov      eax, MACH_semaphore_signal_all_trap
         neg      eax
@@ -1362,13 +1363,14 @@ dynamorio_semaphore_next:
         lea      REG_XDX, [1/*pop*/ + 3/*lea*/ + 2/*sysenter*/ + 2/*mov*/ + REG_XDX]
         mov      REG_XCX, REG_XSP
         sysenter
-# ifndef X64
+#  ifndef X64
         lea      esp, [2*ARG_SZ + esp] /* must not change flags */
-# endif
+#  endif
         /* we ignore return val */
-        jmp      GLOBAL_REF(dynamorio_sys_exit)
-        END_FUNC(dynamorio_semaphore_signal_all)
-#endif /* MACOS */
+        jmp      REG_XDI
+# endif /* MACOS */
+        END_FUNC(dynamorio_condvar_wake_and_jmp)
+#endif /* UNIX */
 
 /* exit entire group without using any stack, in case something like
  * SYS_kill via cleanup_and_terminate fails.

core/dispatch.c

@@ -255,7 +255,8 @@ is_stopping_point(dcontext_t *dcontext, app_pc pc)
               * should not be called from the cache.
               */
              pc == (app_pc)dynamo_thread_exit ||
-             pc == (app_pc)dr_app_stop))
+             pc == (app_pc)dr_app_stop ||
+             pc == (app_pc)dr_app_stop_and_cleanup))
 #endif
 #ifdef WINDOWS
         /* we go all the way to NtTerminateThread/NtTerminateProcess */
@@ -595,15 +596,20 @@ dispatch_at_stopping_point(dcontext_t *dcontext)
         LOG(THREAD, LOG_INTERP, 1, "\t==dynamo_thread_exit\n");
     else if (dcontext->next_tag == (app_pc)dynamorio_app_exit)
         LOG(THREAD, LOG_INTERP, 1, "\t==dynamorio_app_exit\n");
-    else if (dcontext->next_tag == (app_pc)dr_app_stop) {
+    else if (dcontext->next_tag == (app_pc)dr_app_stop)
         LOG(THREAD, LOG_INTERP, 1, "\t==dr_app_stop\n");
-    }
+    else if (dcontext->next_tag == (app_pc)dr_app_stop_and_cleanup)
+        LOG(THREAD, LOG_INTERP, 1, "\t==dr_app_stop_and_cleanup\n");
 #  endif
 # endif
 
     /* XXX i#95: should we add an instrument_thread_detach_event()? */
 
-    dynamo_thread_not_under_dynamo(dcontext);
+#ifdef DR_APP_EXPORTS
+    /* not_under will be called by dynamo_shared_exit so skip it here. */
+    if (dcontext->next_tag != (app_pc)dr_app_stop_and_cleanup)
+#endif
+        dynamo_thread_not_under_dynamo(dcontext);
     dcontext->go_native = false;
 }
 #endif

core/dynamo.c

@@ -916,8 +916,8 @@ dynamo_process_exit_with_thread_info(void)
 
 /* shared between app_exit and detach */
 int
-dynamo_shared_exit(IF_WINDOWS_(thread_record_t *toexit)
-                   IF_WINDOWS_ELSE_NP(bool detach_stacked_callbacks, void))
+dynamo_shared_exit(thread_record_t *toexit /* must ==cur thread for Linux */
+                   _IF_WINDOWS(bool detach_stacked_callbacks))
 {
     DEBUG_DECLARE(uint endtime);
     /* set this now, could already be set */
@@ -979,14 +979,23 @@ dynamo_shared_exit(IF_WINDOWS_(thread_record_t *toexit)
         loader_thread_exit(get_thread_private_dcontext());
     loader_exit();
 
-#ifdef WINDOWS
     if (toexit != NULL) {
         /* free detaching thread's dcontext */
+#ifdef WINDOWS
+        /* If we use dynamo_thread_exit() when toexit is the current thread,
+         * it results in asserts in the win32.tls test, so we stick with this.
+         */
         mutex_lock(&thread_initexit_lock);
         dynamo_other_thread_exit(toexit, false);
         mutex_unlock(&thread_initexit_lock);
-    }
+#else
+        /* On Linux, restoring segment registers can only be done
+         * on the current thread, which must be toexit.
+         */
+        ASSERT(toexit->id == get_thread_id());
+        dynamo_thread_exit();
 #endif
+    }
 
     if (IF_WINDOWS_ELSE(!detach_stacked_callbacks, true)) {
         /* We don't fully free cur thread until after client exit event (PR 536058) */
@@ -1239,8 +1248,8 @@ dynamo_process_exit_cleanup(void)
         unhook_vsyscall();
 #endif /* UNIX */
 
-        return dynamo_shared_exit(IF_WINDOWS_(NULL) /* not detaching */
-                                  IF_WINDOWS(false /* not detaching */));
+        return dynamo_shared_exit(NULL /* not detaching */
+                                  _IF_WINDOWS(false /* not detaching */));
     }
     return SUCCESS;
 }
@@ -2604,6 +2613,15 @@ dr_app_stop(void)
     /* the application regains control in here */
 }
 
+DR_APP_API void
+dr_app_stop_and_cleanup(void)
+{
+    if (dynamo_initialized && !dynamo_exited && !doing_detach) {
+        detach_called_from_app_thread();
+    }
+    /* the application regains control in here */
+}
+
 DR_APP_API int
 dr_app_setup_and_start(void)
 {
@@ -2614,7 +2632,7 @@ dr_app_setup_and_start(void)
 }
 #endif
 
-/* For use by threads that start and stop whether dynamo controls them
+/* For use by threads that start and stop whether dynamo controls them.
  */
 void
 dynamo_thread_under_dynamo(dcontext_t *dcontext)
@@ -2637,8 +2655,6 @@ dynamo_thread_under_dynamo(dcontext_t *dcontext)
 /* For use by threads that start and stop whether dynamo controls them.
  * This must be called by the owner of dcontext and not another
  * non-executing thread.
- * XXX i#95: for detach we'll need to send a signal and have the
- * target thread run this on its own (ditto for os_tls_exit()).
  */
 void
 dynamo_thread_not_under_dynamo(dcontext_t *dcontext)

core/globals.h

@@ -447,6 +447,10 @@ extern bool dynamo_exited_log_and_stats; /* are stats and logfile shut down? */
 #endif
 extern bool dynamo_resetting;    /* in middle of global reset? */
 extern bool dynamo_all_threads_synched; /* are all other threads suspended safely? */
+/* Not guarded by DR_APP_EXPORTS because later detach implementations might not
+ * go through the app interface.
+ */
+extern bool doing_detach;
 
 #if defined(CLIENT_INTERFACE) || defined(STANDALONE_UNIT_TEST)
 extern bool standalone_library;  /* used as standalone library */
@@ -557,8 +561,8 @@ void dynamorio_take_over_threads(dcontext_t *dcontext);
 dr_statistics_t * get_dr_stats(void);
 
 /* functions needed by detach */
-int dynamo_shared_exit(IF_WINDOWS_(thread_record_t *toexit)
-                       IF_WINDOWS_ELSE_NP(bool detach_stacked_callbacks, void));
+int dynamo_shared_exit(thread_record_t *toexit
+                       _IF_WINDOWS(bool detach_stacked_callbacks));
 /* perform exit tasks that require full thread data structs */
 void dynamo_process_exit_with_thread_info(void);
 /* thread cleanup prior to clean exit event */

core/heap.c

@@ -1,5 +1,5 @@
 /* **********************************************************
- * Copyright (c) 2010-2015 Google, Inc.  All rights reserved.
+ * Copyright (c) 2010-2016 Google, Inc.  All rights reserved.
  * Copyright (c) 2001-2010 VMware, Inc.  All rights reserved.
  * **********************************************************/
 
@@ -1405,8 +1405,7 @@ vmm_heap_exit()
                 VMM_BLOCK_SIZE;
             uint unfreed_blocks = perstack * 1 /* initstack */ +
                 /* current stack */
-                perstack * ((IF_WINDOWS_ELSE(doing_detach, false)
-                             IF_APP_EXPORTS(|| dr_api_exit)) ? 0 : 1);
+                perstack * ((doing_detach IF_APP_EXPORTS(|| dr_api_exit)) ? 0 : 1);
             /* FIXME: on detach arch_thread_exit should explicitly mark as
                left behind all TPCs needed so then we can assert even for
                detach

core/lib/dr_app.h

@@ -67,6 +67,7 @@ dr_app_setup(void);
 
 /**
  * Application-wide cleanup.  Prints statistics. Returns zero on success.
+ * Once this is invoked, calling dr_app_start() or dr_app_setup() is not supported.
  */
 DR_APP_API int
 dr_app_cleanup(void);
@@ -109,6 +110,16 @@ dr_app_take_over(void);
 DR_APP_API int
 dr_app_setup_and_start(void);
 
+/**
+ * Causes all of the application's threads to run directly on the machine upon
+ * return from this call, and additionally frees the resources used by DR.
+ * Once this is invoked, calling dr_app_start() or dr_app_setup() is not supported.
+ * This call has no effect if the application is not currently running
+ * under DR control.
+ */
+DR_APP_API void
+dr_app_stop_and_cleanup(void);
+
 /**
  * Indicates whether the current thread is running within the DynamoRIO code
  * cache. Returns \p true only if the current thread is running within the

core/optionsx.h

@@ -1920,14 +1920,14 @@ IF_RCT_IND_BRANCH(options->rct_ind_jump = OPTION_DISABLED;)
 
     OPTION_DEFAULT(bool, track_module_filenames, true,
         "track module file names by watching section creation")
+#endif
 
-    /* FIXME: since we have dynamic options this option can be false for most of the time,
+    /* XXX: since we have dynamic options this option can be false for most of the time,
      * and the gui should set true only when going to detach to prevent a security risk.
      * The setting should be removed when detach is complete.
      * In vault mode: -no_allow_detach -no_dynamic_options
      */
     DYNAMIC_OPTION_DEFAULT(bool, allow_detach, true, "allow detaching from process")
-#endif
 
     /* turn off critical features, right now for experimentation only */
 #ifdef WINDOWS

core/rct.c

@@ -1,5 +1,5 @@
 /* **********************************************************
- * Copyright (c) 2012 Google, Inc.  All rights reserved.
+ * Copyright (c) 2012-2016 Google, Inc.  All rights reserved.
  * Copyright (c) 2004-2009 VMware, Inc.  All rights reserved.
  * **********************************************************/
 
@@ -320,6 +320,7 @@ rct_ind_branch_check(dcontext_t *dcontext, app_pc target_addr, app_pc src_addr)
         if (target_addr == (app_pc) dr_app_start ||
             target_addr == (app_pc) dr_app_take_over ||
             target_addr == (app_pc) dr_app_stop ||
+            target_addr == (app_pc) dr_app_stop_and_cleanup ||
             target_addr == (app_pc) dr_app_cleanup)
             goto good;
 #endif