i#2039 trace trim, part 1: Add post-attach and pre-detach events

Adds two new DR events: post-attach and pre-detach.  These help tools
align instrumentation with points where all threads are under DR
control, avoiding uneven thread execution during the incremental
staggered attach and detach processes.

Adds some sanity tests that the events are called where they should
be.  Adding them to drmemtrace will be done separately.

Issue: #2039

api/docs/release.dox

@@ -169,6 +169,9 @@ Further non-compatibility-affecting changes include:
  - Changed the default drmemtrace offline file format from .gz to .zip and
    added the option -chunk_instr_count to control the split of a file within
    the .zip, which sets the granularity of a fast seek.
+ - Added dr_register_post_attach_event(), dr_unregister_post_attach_event(),
+   dr_register_pre_detach_event(), and dr_unregister_pre_detach_event().
+
 
 The changes between version 9.0.1 and 9.0.0 include the following compatibility
 changes:

core/dynamo.c

@@ -99,6 +99,7 @@ bool dynamo_heap_initialized = false;
 bool dynamo_started = false;
 bool automatic_startup = false;
 bool control_all_threads = false;
+bool dynamo_control_via_attach = false;
 #ifdef WINDOWS
 bool dr_early_injected = false;
 int dr_early_injected_location = INJECT_LOCATION_Invalid;
@@ -2736,6 +2737,7 @@ dr_app_setup(void)
     if (DATASEC_WRITABLE(DATASEC_RARELY_PROT) == 0)
         SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
     dr_api_entry = true;
+    dynamo_control_via_attach = true;
     res = dynamorio_app_init();
     /* For dr_api_entry, we do not install all our signal handlers during init (to avoid
      * races: i#2335): we delay until dr_app_start().  Plus the vsyscall hook is
@@ -2924,6 +2926,9 @@ dynamorio_take_over_threads(dcontext_t *dcontext)
             os_thread_sleep(1);
     } while (found_threads && attempts < max_takeover_attempts);
     os_process_under_dynamorio_complete(dcontext);
+
+    instrument_post_attach_event();
+
     /* End the barrier to new threads. */
     signal_event(dr_attach_finished);
 

core/globals.h

@@ -1,5 +1,5 @@
 /* **********************************************************
- * Copyright (c) 2011-2021 Google, Inc.  All rights reserved.
+ * Copyright (c) 2011-2022 Google, Inc.  All rights reserved.
  * Copyright (c) 2000-2010 VMware, Inc.  All rights reserved.
  * **********************************************************/
 
@@ -431,6 +431,7 @@ extern bool dynamo_exited_log_and_stats; /* are stats and logfile shut down? */
 #endif
 extern bool dynamo_resetting;           /* in middle of global reset? */
 extern bool dynamo_all_threads_synched; /* are all other threads suspended safely? */
+extern bool dynamo_control_via_attach;  /* Attached (vs launched by DR). */
 /* Not guarded by DR_APP_EXPORTS because later detach implementations might not
  * go through the app interface.
  */

core/lib/dr_events.h

@@ -73,6 +73,60 @@ DR_API
 bool
 dr_unregister_exit_event(void (*func)(void));
 
+DR_API
+/**
+ * Registers a function which is called after all other threads have been taken over
+ * during a process attach event, whether externally triggered or internally triggered
+ * (via dr_app_start() or related functions).  If this process instance was not
+ * initiated by an attach, this registration function returns false.
+ *
+ * The attach methodology operates in a staggered fashion, with each thread being taken
+ * over and executed under DR control in turn.  If the application has many threads,
+ * threads taken over early in this process may execute substantial amounts of
+ * instrumented code before the threads taken over last start executing instrumented
+ * code.  The purpose of this event is to provide a single control point where all
+ * threads are known to be under DR control and running instrumented code.
+ */
+bool
+dr_register_post_attach_event(void (*func)(void));
+
+DR_API
+/**
+ * Unregister a callback function for the post-attach event (see
+ * dr_register_post_attach_event()).  \return true if unregistration is successful and
+ * false if it is not (e.g., \p func was not registered).
+ */
+bool
+dr_unregister_post_attach_event(void (*func)(void));
+
+DR_API
+/**
+ * Registers a function which is called at the start of a full detach of DR from the
+ * current process, whether externally triggered or internally triggered (via
+ * dr_app_stop_and_cleanup() or related functions), as well as at the start of DR
+ * sending all threads native but not cleaning up its own state (through dr_app_stop()).
+ *
+ * The detach methodology operates in a staggered fashion, with each thread being return
+ * to native control in turn.  If the application has many threads, threads detached
+ * late in this process may execute substantial amounts of instrumented code before the
+ * full detach process is complete, while threads detached early are running natively.
+ * The purpose of this event is to provide a single final control point where all
+ * threads are known to be under DR control and running instrumented code.  The exit
+ * event (see dr_register_exit_event()) is not called until after all other threads have
+ * been detached.
+ */
+void
+dr_register_pre_detach_event(void (*func)(void));
+
+DR_API
+/**
+ * Unregister a callback function for the post-attach event (see
+ * dr_register_pre_detach_event()).  \return true if unregistration is successful and
+ * false if it is not (e.g., \p func was not registered).
+ */
+bool
+dr_unregister_pre_detach_event(void (*func)(void));
+
 /**
  * Flags controlling the behavior of basic blocks and traces when emitted
  * into the code cache.  These flags are bitmasks that can be combined by

core/lib/instrument.c

@@ -198,6 +198,12 @@ typedef struct _callback_list_t {
 static callback_list_t exit_callbacks = {
     0,
 };
+static callback_list_t post_attach_callbacks = {
+    0,
+};
+static callback_list_t pre_detach_callbacks = {
+    0,
+};
 static callback_list_t thread_init_callbacks = {
     0,
 };
@@ -816,6 +822,8 @@ static void
 free_all_callback_lists()
 {
     free_callback_list(&exit_callbacks);
+    free_callback_list(&post_attach_callbacks);
+    free_callback_list(&pre_detach_callbacks);
     free_callback_list(&thread_init_callbacks);
     free_callback_list(&thread_exit_callbacks);
 #ifdef UNIX
@@ -869,6 +877,20 @@ instrument_exit_event(void)
              NULL);
 }
 
+void
+instrument_post_attach_event(void)
+{
+    if (!dynamo_control_via_attach)
+        return;
+    call_all(post_attach_callbacks, int (*)(), NULL);
+}
+
+void
+instrument_pre_detach_event(void)
+{
+    call_all(pre_detach_callbacks, int (*)(), NULL);
+}
+
 void
 instrument_exit(void)
 {
@@ -971,6 +993,33 @@ dr_unregister_exit_event(void (*func)(void))
     return remove_callback(&exit_callbacks, (void (*)(void))func, true);
 }
 
+bool
+dr_register_post_attach_event(void (*func)(void))
+{
+    if (!dynamo_control_via_attach)
+        return false;
+    add_callback(&post_attach_callbacks, (void (*)(void))func, true);
+    return true;
+}
+
+bool
+dr_unregister_post_attach_event(void (*func)(void))
+{
+    return remove_callback(&post_attach_callbacks, (void (*)(void))func, true);
+}
+
+void
+dr_register_pre_detach_event(void (*func)(void))
+{
+    add_callback(&pre_detach_callbacks, (void (*)(void))func, true);
+}
+
+bool
+dr_unregister_pre_detach_event(void (*func)(void))
+{
+    return remove_callback(&pre_detach_callbacks, (void (*)(void))func, true);
+}
+
 void
 dr_register_bb_event(dr_emit_flags_t (*func)(void *drcontext, void *tag, instrlist_t *bb,
                                              bool for_trace, bool translating))

core/lib/instrument.h

@@ -1,5 +1,5 @@
 /* **********************************************************
- * Copyright (c) 2010-2021 Google, Inc.  All rights reserved.
+ * Copyright (c) 2010-2022 Google, Inc.  All rights reserved.
  * Copyright (c) 2002-2010 VMware, Inc.  All rights reserved.
  * **********************************************************/
 
@@ -68,6 +68,10 @@ instrument_init(void);
 void
 instrument_exit_event(void);
 void
+instrument_post_attach_event(void);
+void
+instrument_pre_detach_event(void);
+void
 instrument_exit(void);
 bool
 is_in_client_lib(app_pc addr);

core/synch.c

@@ -1874,6 +1874,8 @@ send_all_other_threads_native(void)
     wait_for_outstanding_nudges();
 #endif
 
+    instrument_pre_detach_event();
+
     /* Suspend all threads except those trying to synch with us */
     if (!synch_with_all_threads(desired_state, &threads, &num_threads,
                                 THREAD_SYNCH_NO_LOCKS_NO_XFER,
@@ -1996,6 +1998,8 @@ detach_on_permanent_stack(bool internal, bool do_cleanup, dr_stats_t *drstats)
     if (!atomic_compare_exchange(&dynamo_detaching_flag, LOCK_FREE_STATE, LOCK_SET_STATE))
         return;
 
+    instrument_pre_detach_event();
+
     /* Unprotect .data for exit cleanup.
      * XXX: more secure to not do this until we've synched, but then need
      * alternative prot for started_detach and init_apc_go_native*

core/unix/os.c

@@ -752,6 +752,7 @@ our_init(int argc, char **argv, char **envp)
     }
 #endif
     if (takeover) {
+        dynamo_control_via_attach = true;
         if (dynamorio_app_init() == 0 /* success */) {
             dynamorio_app_take_over();
         }

suite/tests/api/startstop.c

@@ -1,5 +1,5 @@
 /* **********************************************************
- * Copyright (c) 2011-2017 Google, Inc.  All rights reserved.
+ * Copyright (c) 2011-2022 Google, Inc.  All rights reserved.
  * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
  * **********************************************************/
 
@@ -115,6 +115,18 @@ event_bb(void *drcontext, void *tag, instrlist_t *bb, bool for_trace, bool trans
     return DR_EMIT_DEFAULT;
 }
 
+static void
+event_post_attach(void)
+{
+    print("in %s\n", __FUNCTION__);
+}
+
+static void
+event_pre_detach(void)
+{
+    print("in %s\n", __FUNCTION__);
+}
+
 static volatile bool sideline_exit = false;
 static void *sideline_continue;
 static void *go_native;
@@ -203,6 +215,9 @@ main(void)
      * just using it for testing.
      */
     dr_register_bb_event(event_bb);
+    if (!dr_register_post_attach_event(event_post_attach))
+        print("Failed to register post-attach event");
+    dr_register_pre_detach_event(event_pre_detach);
 
     /* Wait for all the threads to be scheduled */
     VPRINT("waiting for ready\n");

suite/tests/api/startstop.expect

@@ -1 +1,25 @@
+in event_post_attach
+in event_pre_detach
+in event_post_attach
+in event_pre_detach
+in event_post_attach
+in event_pre_detach
+in event_post_attach
+in event_pre_detach
+in event_post_attach
+in event_pre_detach
+in event_post_attach
+in event_pre_detach
+in event_post_attach
+in event_pre_detach
+in event_post_attach
+in event_pre_detach
+in event_post_attach
+in event_pre_detach
+in event_post_attach
+in event_pre_detach
+in event_post_attach
+in event_pre_detach
 all done: 10 iters
+in event_post_attach
+in event_pre_detach

suite/tests/api/static_prepop.c

@@ -85,12 +85,27 @@ event_exit(void)
     dr_fprintf(STDERR, "Exit event\n");
 }
 
+static void
+event_post_attach(void)
+{
+    dr_fprintf(STDERR, "in %s\n", __FUNCTION__);
+}
+
+static void
+event_pre_detach(void)
+{
+    dr_fprintf(STDERR, "in %s\n", __FUNCTION__);
+}
+
 DR_EXPORT void
 dr_client_main(client_id_t id, int argc, const char *argv[])
 {
     print("in dr_client_main\n");
     dr_register_bb_event(event_bb);
     dr_register_exit_event(event_exit);
+    if (!dr_register_post_attach_event(event_post_attach))
+        print("Failed to register post-attach event");
+    dr_register_pre_detach_event(event_pre_detach);
 
     /* XXX i#975: add some more thorough tests of different events */
 }

suite/tests/api/static_prepop.expect

@@ -6,7 +6,9 @@ bb asm_label2
 bb asm_label3
 bb asm_return
 pre-DR start
+in event_post_attach
 pre-DR detach
+in event_pre_detach
 Exit event
 pre-DR init
 in dr_client_main
@@ -16,6 +18,8 @@ bb asm_label2
 bb asm_label3
 bb asm_return
 pre-DR start
+in event_post_attach
 pre-DR detach with stats
+in event_pre_detach
 Exit event
 all done

suite/tests/client-interface/events.dll.c

@@ -1,5 +1,5 @@
 /* **********************************************************
- * Copyright (c) 2011-2019 Google, Inc.  All rights reserved.
+ * Copyright (c) 2011-2022 Google, Inc.  All rights reserved.
  * Copyright (c) 2008-2010 VMware, Inc.  All rights reserved.
  * **********************************************************/
 
@@ -737,4 +737,9 @@ dr_init(client_id_t id)
         dr_fprintf(STDERR, "failed to unregister for persist rw events");
     if (!dr_unregister_persist_patch(event_persist_patch))
         dr_fprintf(STDERR, "failed to unregister for persist patch event");
+
+    if (dr_register_post_attach_event(exit_event1))
+        dr_fprintf(STDERR, "should fail to register for post-attach event");
+    if (dr_unregister_post_attach_event(exit_event1))
+        dr_fprintf(STDERR, "should fail to unregister for post-attach event");
 }