i#241,i#1693: add kernel transfer client events (#2719)

Adds a new client API event type: a kernel-mediated control transfer.  This
includes UNIX signal delivery and return, Windows APCs, callbacks,
exceptions, NtContinue, NtSetContextThread, and callback returns.  It is
also raised on client redirects.

The new interface passes the source context (control and integer, no
multimedia) but passes just key target fields directly (new pc, new xsp)
and does not pass in the full target register state to avoid copying costs
(particulary for multimedia).  Adds a new internal os_cxt_ptr_t type to
support pointing at a CONTEXT or sig_full_cxt_t for the new events and only
copying from that state if the client calls dr_get_mcontext().  Adds extra
logic to get the source context for dr_redirect_execution() called from the
exception event.

Includes limited support for calling dr_set_mcontext() and changing the pc
or other state, though there seem to be few use cases of this and given the
difficulty in testing it, it's not clear it's a worthwhile feature.

Adds documentation on the disparity between xbp being in CONTEXT_CONTROL
yet in DR_MC_INTEGER, cautioning clients to use both INTEGER and CONTROL
when they care about xbp.

Includes logic to pass the real Ki pc and not the hook-displaced pc for
dispatchers.

I don't think it's possible to pass the real xsi and not the syscall return
address for a cbret: we live with that.

Adds corresponding routines drmgr_register_kernel_xfer_event() and
drmgr_register_kernel_xfer_event_ex().  Re-implements drmgr's CLS API using
the new kernel xfer event (this is required for proper ordering of CLS vs
other clients using the xfer event).  Removes
DRMGR_PRIORITY_INSERT_CLS_ENTRY, DRMGR_PRIORITY_INSERT_CLS_EXIT,
DRMGR_PRIORITY_NAME_CLS_ENTRY, and DRMGR_PRIORITY_NAME_CLS_EXIT.

Adds tests to client.signal, client.flush, client.events,
client.drmgr-test, and a new test client.winxfer.  Testing of
dr_set_mcontext() on Windows was done manually: automated testing is
challenging to set up and left for future work.

Fixes #241
Fixes #1693

api/docs/API.doxy

@@ -1,5 +1,5 @@
 # **********************************************************
-# Copyright (c) 2012-2015 Google, Inc.  All rights reserved.
+# Copyright (c) 2012-2017 Google, Inc.  All rights reserved.
 # Copyright (c) 2007-2010 VMware, Inc.  All rights reserved.
 # **********************************************************/
 
@@ -83,7 +83,7 @@ GENERATE_TODOLIST      = YES
 GENERATE_TESTLIST      = YES
 GENERATE_BUGLIST       = YES
 GENERATE_DEPRECATEDLIST= YES
-# choices: linux, VMsafe, NYI_kernel_mediated_transfer_events
+# choices: linux, VMsafe
 ENABLED_SECTIONS       =
 MAX_INITIALIZER_LINES  = 30
 SHOW_USED_FILES        = YES

api/docs/intro.dox

@@ -823,15 +823,13 @@ set of event callbacks.  These events include the following:
     (dr_register_fork_init_event())
   - Application library load and unload
     (dr_register_module_load_event(), dr_register_module_unload_event())
-\ifnot NYI_kernel_mediated_transfer_events
   - Application fault or exception (signal on Linux)
     (dr_register_exception_event(), dr_register_signal_event())
-\else
-  - Kernel-mediated control transfers:
-    - Application fault or exception
-    - Application APC (Asynchronous Procedure Call) or callback (Windows)
-    - Application signal (Linux)
-\endif
+  - Kernel-mediated control transfers (dr_register_kernel_xfer_event()):
+    - Application APC (Asynchronous Procedure Call), callback, or exception
+      dispatcher execution (Windows)
+    - Application signal delivery (Linux)
+    - System call that changes the context
   - System call interception: pre-system call, post-system call, and system
     call filtering by number
     (dr_register_pre_syscall_event(), dr_register_post_syscall_event(),

api/docs/release.dox

@@ -145,6 +145,10 @@ following minor compatibility changes:
    such as a static initialiser, will still need to be rewritten.
    DynamoRIO_PAGE_SIZE_COMPATIBILITY will be set automatically if the
    client targets version 6.2 or earlier.
+ - Removed DRMGR_PRIORITY_INSERT_CLS_ENTRY, DRMGR_PRIORITY_INSERT_CLS_EXIT,
+   DRMGR_PRIORITY_NAME_CLS_ENTRY, and DRMGR_PRIORITY_NAME_CLS_EXIT, as
+   the new kernel xfer event (drmgr_register_kernel_xfer_event()) removes the
+   need for them.
 
 Further non-compatibility-affecting changes include:
 
@@ -200,6 +204,9 @@ Further non-compatibility-affecting changes include:
    identifier #DR_WINDOWS_VERSION_10_1703 to distinguish this major update.
  - Added support for Windows 10 1709.  We provide an artificial version
    identifier #DR_WINDOWS_VERSION_10_1709 to distinguish this major update.
+ - Added an event for kernel-mediated control flow via
+   dr_register_kernel_xfer_event() with corresponding routines
+   drmgr_register_kernel_xfer_event() and drmgr_register_kernel_xfer_event_ex().
 
 **************************************************
 <hr>

core/arch/arch.c

@@ -3256,6 +3256,7 @@ dr_mcontext_to_priv_mcontext(priv_mcontext_t *dst, dr_mcontext_t *src)
             dst->xsp = save_xsp;
         }
         if (TEST(DR_MC_CONTROL, src->flags)) {
+            /* XXX i#2710: mc->lr should be under DR_MC_CONTROL */
             dst->xsp = src->xsp;
             dst->xflags = src->xflags;
             dst->pc = src->pc;

core/globals.h

@@ -422,7 +422,9 @@ typedef struct _client_data_t {
     /* flags for dr_get_mcontext (i#117/PR 395156) */
     bool           mcontext_in_dcontext;
     bool           suspended;
+    /* 2 other ways to point at a context for dr_{g,s}et_mcontext() */
     priv_mcontext_t *cur_mc;
+    os_cxt_ptr_t   os_cxt;
 } client_data_t;
 #else
 # define IS_CLIENT_THREAD(dcontext) false

core/lib/globals_shared.h

@@ -1874,19 +1874,26 @@ typedef union _dr_simd_t {
 /** Values for the flags field of dr_mcontext_t */
 typedef enum {
     /**
-     * Selects the xdi, xsi, xbp, xbx, xdx, xcx, xax, and r8-r15 fields (i.e.,
+     * On x86, selects the xdi, xsi, xbp, xbx, xdx, xcx, xax, and r8-r15 fields (i.e.,
      * all of the general-purpose registers excluding xsp, xip, and xflags).
+     * On ARM, selects r0-r12 and r14.
+     * On AArch64, selects r0-r30.
      */
     DR_MC_INTEGER    = 0x01,
+#ifdef AVOID_API_EXPORT
+/* XXX i#2710: The link register should be under DR_MC_CONTROL */
+#endif
     /**
-     * Selects the xsp, xflags, and xip fields.
-     * \note: The xip field is only honored as an input for
+     * On x86, selects the xsp, xflags, and xip fields.
+     * On ARM, selects the sp, pc, and cpsr fields.
+     * On AArch64, selects the sp, pc, and nzcv fields.
+     * \note: The xip/pc field is only honored as an input for
      * dr_redirect_execution(), and as an output for system call
      * events.
      */
     DR_MC_CONTROL    = 0x02,
     /**
-     * Selects the ymm (and xmm) fields.  This flag is ignored unless
+     * Selects the simd fields.  This flag is ignored unless
      * dr_mcontext_xmm_fields_valid() returns true.  If
      * dr_mcontext_xmm_fields_valid() returns false, the application values of
      * the multimedia registers remain in the registers themselves.

core/lib/instrument.c

@@ -214,6 +214,7 @@ static callback_list_t module_unload_callbacks = {0,};
 static callback_list_t filter_syscall_callbacks = {0,};
 static callback_list_t pre_syscall_callbacks = {0,};
 static callback_list_t post_syscall_callbacks = {0,};
+static callback_list_t kernel_xfer_callbacks = {0,};
 #ifdef WINDOWS
 static callback_list_t exception_callbacks = {0,};
 #else
@@ -747,6 +748,7 @@ void free_all_callback_lists()
     free_callback_list(&filter_syscall_callbacks);
     free_callback_list(&pre_syscall_callbacks);
     free_callback_list(&post_syscall_callbacks);
+    free_callback_list(&kernel_xfer_callbacks);
 #ifdef WINDOWS
     free_callback_list(&exception_callbacks);
 #else
@@ -1130,6 +1132,20 @@ dr_unregister_post_syscall_event(void (*func)(void *drcontext, int sysnum))
     return remove_callback(&post_syscall_callbacks, (void (*)(void))func, true);
 }
 
+void
+dr_register_kernel_xfer_event(void (*func)(void *drcontext,
+                                           const dr_kernel_xfer_info_t *info))
+{
+    add_callback(&kernel_xfer_callbacks, (void (*)(void))func, true);
+}
+
+bool
+dr_unregister_kernel_xfer_event(void (*func)(void *drcontext,
+                                             const dr_kernel_xfer_info_t *info))
+{
+    return remove_callback(&kernel_xfer_callbacks, (void (*)(void))func, true);
+}
+
 #ifdef PROGRAM_SHEPHERDING
 void
 dr_register_security_event(void (*func)(void *drcontext, void *source_tag,
@@ -2044,12 +2060,61 @@ instrument_invoke_another_syscall(dcontext_t *dcontext)
     return dcontext->client_data->invoke_another_syscall;
 }
 
+bool
+instrument_kernel_xfer(dcontext_t *dcontext, dr_kernel_xfer_type_t type,
+                       os_cxt_ptr_t source_os_cxt, dr_mcontext_t *source_dmc,
+                       priv_mcontext_t *source_mc,
+                       app_pc target_pc, reg_t target_xsp,
+                       os_cxt_ptr_t target_os_cxt, priv_mcontext_t *target_mc,
+                       int sig)
+{
+    if (kernel_xfer_callbacks.num == 0) {
+        return false;
+    }
+    dr_kernel_xfer_info_t info;
+    info.type = type;
+    info.source_mcontext = NULL;
+    info.target_pc = target_pc;
+    info.target_xsp = target_xsp;
+    info.sig = sig;
+    dr_mcontext_t dr_mcontext;
+    dr_mcontext.size = sizeof(dr_mcontext);
+    dr_mcontext.flags = DR_MC_CONTROL | DR_MC_INTEGER;
+
+    if (source_dmc != NULL)
+        info.source_mcontext = source_dmc;
+    else if (source_mc != NULL) {
+        if (priv_mcontext_to_dr_mcontext(&dr_mcontext, source_mc))
+            info.source_mcontext = &dr_mcontext;
+    } else if (!is_os_cxt_ptr_null(source_os_cxt)) {
+        if (os_context_to_mcontext(&dr_mcontext, NULL, source_os_cxt))
+            info.source_mcontext = &dr_mcontext;
+    }
+    /* Our compromise to reduce context copying is to provide the PC and XSP inline,
+     * and only get more if the user calls dr_get_mcontext(), which we support again
+     * without any copying if not used by taking in a raw os_context_t.
+     */
+    dcontext->client_data->os_cxt = target_os_cxt;
+    dcontext->client_data->cur_mc = target_mc;
+    call_all(kernel_xfer_callbacks, int (*)(void *, const dr_kernel_xfer_info_t *),
+             (void *)dcontext, &info);
+    set_os_cxt_ptr_null(&dcontext->client_data->os_cxt);
+    dcontext->client_data->cur_mc = NULL;
+    return true;
+}
+
 #ifdef WINDOWS
 /* Notify user of exceptions.  Note: not called for RaiseException */
 bool
 instrument_exception(dcontext_t *dcontext, dr_exception_t *exception)
 {
     bool res = true;
+    /* Ensure that dr_get_mcontext() called from instrument_kernel_xfer() from
+     * dr_redirect_execution() will get the source context.
+     * cur_mc will later be clobbered by instrument_kernel_xfer() which is ok:
+     * the redirect ends the callback calling.
+     */
+    dcontext->client_data->cur_mc = dr_mcontext_as_priv_mcontext(exception->mcontext);
     /* We short-circuit if any client wants to "own" the fault and not pass on.
      * This does violate the "priority order" of events where the last one is
      * supposed to have final say b/c it won't even see the event: but only one
@@ -2058,6 +2123,7 @@ instrument_exception(dcontext_t *dcontext, dr_exception_t *exception)
     call_all_ret(res, = res &&, , exception_callbacks,
                  bool (*)(void *, dr_exception_t *),
                  (void *)dcontext, exception);
+    dcontext->client_data->cur_mc = NULL;
     return res;
 }
 #else
@@ -6360,6 +6426,10 @@ dr_get_mcontext_priv(dcontext_t *dcontext, dr_mcontext_t *dmc, priv_mcontext_t *
         return true;
     }
 
+    if (!is_os_cxt_ptr_null(dcontext->client_data->os_cxt)) {
+        return os_context_to_mcontext(dmc, mc, dcontext->client_data->os_cxt);
+    }
+
     if (dcontext->client_data->suspended) {
         /* A thread suspended by dr_suspend_all_other_threads() has its
          * context translated lazily here.
@@ -6467,6 +6537,16 @@ dr_set_mcontext(void *drcontext, dr_mcontext_t *context)
             return false;
         return true;
     }
+    if (dcontext->client_data->cur_mc != NULL) {
+        return dr_mcontext_to_priv_mcontext(dcontext->client_data->cur_mc, context);
+    }
+    if (!is_os_cxt_ptr_null(dcontext->client_data->os_cxt)) {
+        /* It would be nice to fail for #DR_XFER_CALLBACK_RETURN but we'd need to
+         * store yet more state to do so.  The pc will be ignored, and xsi
+         * changes will likely cause crashes.
+         */
+        return mcontext_to_os_context(dcontext->client_data->os_cxt, context, NULL);
+    }
 
     /* copy the machine context to the dstack area created with
      * dr_prepare_for_call().  note that xmm0-5 copied there
@@ -6527,6 +6607,23 @@ dr_redirect_execution(dr_mcontext_t *mcontext)
     dcontext->next_tag = canonicalize_pc_target(dcontext, mcontext->pc);
     dcontext->whereami = WHERE_FCACHE;
     set_last_exit(dcontext, (linkstub_t *)get_client_linkstub());
+#ifdef CLIENT_INTERFACE
+    if (kernel_xfer_callbacks.num > 0) {
+        /* This can only be called from a clean call or an exception event.
+         * For both of those we can get the current mcontext via dr_get_mcontext()
+         * (the latter b/c we explicitly store to cur_mc just for this use case).
+         */
+        dr_mcontext_t src_dmc;
+        src_dmc.size = sizeof(src_dmc);
+        src_dmc.flags = DR_MC_CONTROL | DR_MC_INTEGER;
+        dr_get_mcontext(dcontext, &src_dmc);
+        if (instrument_kernel_xfer(dcontext, DR_XFER_CLIENT_REDIRECT,
+                                   osc_empty, &src_dmc, NULL,
+                                   dcontext->next_tag, mcontext->xsp, osc_empty,
+                                   dr_mcontext_as_priv_mcontext(mcontext), 0))
+            dcontext->next_tag = canonicalize_pc_target(dcontext, mcontext->pc);
+    }
+#endif
     transfer_to_dispatch(dcontext, dr_mcontext_as_priv_mcontext(mcontext),
                          true/*full_DR_state*/);
     /* on success we won't get here */

core/lib/instrument.h

@@ -1,5 +1,5 @@
 /* **********************************************************
- * Copyright (c) 2010-2016 Google, Inc.  All rights reserved.
+ * Copyright (c) 2010-2017 Google, Inc.  All rights reserved.
  * Copyright (c) 2002-2010 VMware, Inc.  All rights reserved.
  * **********************************************************/
 
@@ -102,6 +102,15 @@ bool instrument_filter_syscall(dcontext_t *dcontext, int sysnum);
 bool instrument_pre_syscall(dcontext_t *dcontext, int sysnum);
 void instrument_post_syscall(dcontext_t *dcontext, int sysnum);
 bool instrument_invoke_another_syscall(dcontext_t *dcontext);
+/* returns whether a client event was called which might have changed the context */
+bool instrument_kernel_xfer(dcontext_t *dcontext, dr_kernel_xfer_type_t type,
+                            /* only one of these 3 should be non-NULL */
+                            os_cxt_ptr_t source_os_cxt, dr_mcontext_t *source_dmc,
+                            priv_mcontext_t *source_mc,
+                            app_pc target_pc, reg_t target_xsp,
+                            /* only one of these 2 should be non-NULL */
+                            os_cxt_ptr_t target_os_cxt, priv_mcontext_t *target_mc,
+                            int sig);
 
 void instrument_nudge(dcontext_t *dcontext, client_id_t id, uint64 arg);
 /* post instrument_event() cleanup */