Fix instrumentation regressions for libc-2.29 on ARM (#653)

* Fixes loading pre-instrumentation register values on aarch64
* Fix the handling of trap instruction on ARM for register liveness analysis
* Implement a byte matching heuristic for determining the length of _start
* Implement a spin lock use __atomic_test_and_set on ARM
*   1. Change stop signal from SIGBUS to (SIGRTMIN + 4) in dyninstAPI_RT.
*   2. In dyninstAPI, only do dyninstRT signal check if the signal is indeed (SIGRTMIN + 4)
* Refacotr BPatch and DyninstRT to use breakpoint as the way of communication

dataflowAPI/src/liveness.C

@@ -567,6 +567,9 @@ ReadWriteInfo LivenessAnalyzer::calcRWSets(Instruction curInsn, Block *blk, Addr
       if (curInsn.getOperation().getID() == power_op_svcs) {
 	isSyscall = true;
       }
+      if (curInsn.getOperation().getID() == aarch64_op_svc) {
+          isSyscall = true;
+      }
       if (isInterrupt || isSyscall) {
 	ret.read |= (abi->getSyscallReadRegisters());
 	ret.written |= (abi->getSyscallWrittenRegisters());

dyninstAPI/src/dynProcess.C

@@ -836,6 +836,18 @@ bool PCProcess::loadRTLib() {
    {
 	   startup_printf("%s[%d]: DYNINSTinit not called automatically\n", FILE__, __LINE__);
    }
+
+   // Install a breakpoint in DYNINSTtrapFunction.
+   // This is used as RT signal.
+   Address addr = getRTTrapFuncAddr();
+   if (addr == 0) {
+       startup_printf("%s[%d]: Cannot find DYNINSTtrapFunction. Needed as RT signal\n", FILE__, __LINE__);
+       return false;
+   }
+   if (!setBreakpoint(addr)) {
+       startup_printf("%s[%d]: Cannot set breakpoint in DYNINSTtrapFunction.\n", FILE__, __LINE__);
+       return false;
+   }
    startup_printf("%s[%d]: DYNINSTinit succeeded\n", FILE__, __LINE__);
    return setRTLibInitParams();
 }
@@ -3200,6 +3212,14 @@ Address PCProcess::getRTEventArg3Addr() {
     return sync_event_arg3_addr_;
 }
 
+Address PCProcess::getRTTrapFuncAddr() {
+    if (rt_trap_func_addr_ == 0) {
+        func_instance* func = findOnlyOneFunction("DYNINSTtrapFunction");
+        rt_trap_func_addr_ = func->addr();
+    }
+    return rt_trap_func_addr_;
+}
+
 bool PCProcess::hasPendingEvents() {
    // Go to the muxer as a final arbiter
    return PCEventMuxer::muxer().hasPendingEvents(this);

dyninstAPI/src/dynProcess.h

@@ -354,6 +354,7 @@ class PCProcess : public AddressSpace {
           sync_event_arg2_addr_(0),
           sync_event_arg3_addr_(0),
           sync_event_breakpoint_addr_(0),
+          rt_trap_func_addr_(0),
        thread_hash_tids(0),
        thread_hash_indices(0),
        thread_hash_size(0),
@@ -394,6 +395,7 @@ class PCProcess : public AddressSpace {
           sync_event_arg2_addr_(0),
           sync_event_arg3_addr_(0),
           sync_event_breakpoint_addr_(0),
+          rt_trap_func_addr_(0),
        thread_hash_tids(0),
        thread_hash_indices(0),
        thread_hash_size(0),
@@ -437,6 +439,7 @@ class PCProcess : public AddressSpace {
           sync_event_arg2_addr_(parent->sync_event_arg2_addr_),
           sync_event_arg3_addr_(parent->sync_event_arg3_addr_),
           sync_event_breakpoint_addr_(parent->sync_event_breakpoint_addr_),
+          rt_trap_func_addr_(parent->rt_trap_func_addr_),
        thread_hash_tids(parent->thread_hash_tids),
        thread_hash_indices(parent->thread_hash_indices),
        thread_hash_size(parent->thread_hash_size),
@@ -488,6 +491,7 @@ class PCProcess : public AddressSpace {
     Address getRTEventArg1Addr();
     Address getRTEventArg2Addr();
     Address getRTEventArg3Addr();
+    Address getRTTrapFuncAddr();
 
     // Shared library managment
     void addASharedObject(mapped_object *newObj);
@@ -593,6 +597,7 @@ class PCProcess : public AddressSpace {
     Address sync_event_arg2_addr_;
     Address sync_event_arg3_addr_;
     Address sync_event_breakpoint_addr_;
+    Address rt_trap_func_addr_;
     Address thread_hash_tids;
     Address thread_hash_indices;
     int thread_hash_size;

dyninstAPI/src/emit-aarch64.C

@@ -274,17 +274,17 @@ void EmitterAARCH64::emitLoadOrigRegister(Address register_num, Register destina
    registerSlot *dest = (*gen.rs())[destination];
    assert(dest);
 
-   if (register_num == REG_SP) {
+   if (src->name == "sp") {
       insnCodeGen::generateAddSubImmediate(gen, insnCodeGen::Add, 0,
-             TRAMP_FRAME_SIZE_64, destination, REG_SP, true);
+             TRAMP_FRAME_SIZE_64, REG_SP, destination, true);
 
       return;
    }
 
    if (src->spilledState == registerSlot::unspilled)
    {
       // not on the stack. Directly move the value
-      emitMoveRegToReg((Register) register_num, destination, gen);
+      insnCodeGen::generateMove(gen, destination, (Register) register_num, true);
       return;
    }
 

dyninstAPI/src/pcEventHandler.C

@@ -449,22 +449,6 @@ bool PCEventHandler::handleSignal(EventSignal::const_ptr ev, PCProcess *evProc)
             FILE__, __LINE__, ev->getProcess()->getPid(), ev->getThread()->getLWP(),
             ev->getSignal());
 
-    // Check whether it is a signal from the RT library (note: this will internally
-    // handle any entry/exit to syscalls and make the necessary up calls as appropriate)
-    RTSignalResult result = handleRTSignal(ev, evProc);
-    if( result == ErrorInDecoding ) {
-        proccontrol_printf("%s[%d]: failed to determine whether signal came from RT library\n",
-                FILE__, __LINE__);
-        return false;
-    }
-
-    if( result == IsRTSignal ) {
-        // handleRTSignal internally does all handling for the events in order to keep
-        // related logic in one place
-        proccontrol_printf("%s[%d]: signal came from RT library\n", FILE__, __LINE__);
-        return true;
-    }
-
     // check if windows access violation, defensive mode, and write permissions.
     // unprotect pages if necessary.
     if (evProc->getHybridMode() == BPatch_defensiveMode 
@@ -576,81 +560,61 @@ bool PCEventHandler::handleSignal(EventSignal::const_ptr ev, PCProcess *evProc)
     return true;
 }
 
-PCEventHandler::RTSignalResult
-PCEventHandler::handleRTSignal(EventSignal::const_ptr ev, PCProcess *evProc) const {
+bool PCEventHandler::handleRTBreakpoint(EventBreakpoint::const_ptr ev, PCProcess *evProc) const {
     // Check whether the signal was sent from the RT library by checking variables
     // in the library -- if we cannot be determine whether this signal came from
     // the RT library, assume it did not.
 
-    if( evProc->runtime_lib.size() == 0 ) return NotRTSignal;
+    if( evProc->runtime_lib.size() == 0 ) return false;
 
-    Address sync_event_breakpoint_addr = evProc->getRTEventBreakpointAddr();
+    Address rtTrapFuncAddr = evProc->getRTTrapFuncAddr();
     Address sync_event_id_addr = evProc->getRTEventIdAddr();
     Address sync_event_arg1_addr = evProc->getRTEventArg1Addr();
 
-    int breakpoint = 0;
     int status = 0;
     Address arg1 = 0;
     int zero = 0;
 
     // Check that all addresses could be determined
-    if(    sync_event_breakpoint_addr == 0 
+    if (rtTrapFuncAddr == 0
         || sync_event_id_addr == 0 
         || sync_event_arg1_addr == 0 ) 
     {
-        return NotRTSignal;
-    }
+        proccontrol_printf("%s[%d]: signal is not RT library breakpoint. Some address is 0: rtTrapFuncAddr %lx, sync_even_id_addr %lx, sync_event_arg1_addr %lx\n", 
+                FILE__, __LINE__, rtTrapFuncAddr, sync_event_id_addr, sync_event_arg1_addr);
 
-    // First, check breakpoint...
-    if( !evProc->readDataWord((const void *)sync_event_breakpoint_addr,
-                sizeof(int), &breakpoint, false) ) return NotRTSignal;
-
-    switch(breakpoint) {
-        case NoRTBreakpoint:
-            proccontrol_printf("%s[%d]: signal is not RT library signal\n",
-                    FILE__, __LINE__);
-            return NotRTSignal;
-        case NormalRTBreakpoint:
-        case SoftRTBreakpoint:
-            // More work to do
-            break;
-        default:
-            proccontrol_printf("%s[%d]: invalid value for RT library breakpoint variable\n",
-                    FILE__, __LINE__);
-            return NotRTSignal;
+        return false;
     }
 
-    // Make sure we don't get this event twice....
-    if( !evProc->writeDataWord((void *)sync_event_breakpoint_addr, sizeof(int), &zero) ) {
-        proccontrol_printf("%s[%d]: failed to reset RT library breakpoint variable\n",
-                FILE__, __LINE__);
-        return NotRTSignal;
+    // On x86, the PC is shifted by one byte when hitting the trap instruction
+    if (ev->getAddress() != rtTrapFuncAddr && ev->getAddress() != rtTrapFuncAddr + 1) {
+        proccontrol_printf("%s[%d]: signal is not RT library breakpoint. Breakpoint from address %lx, expected %lx\n",
+                FILE__, __LINE__, ev->getAddress(), rtTrapFuncAddr);
+        return false;
     }
 
     // Get the type of the event
     if( !evProc->readDataWord((const void *)sync_event_id_addr, sizeof(int),
-                &status, false) ) return NotRTSignal;
+                &status, false) ) return false;
 
     if( status == DSE_undefined ) {
-        proccontrol_printf("%s[%d]: signal is not RT library signal\n", FILE__, __LINE__);
-        return NotRTSignal;
+        proccontrol_printf("%s[%d]: signal is not RT library breakpointl\n", FILE__, __LINE__);
+        return false;
     }
 
     // get runtime library arg1 address
     if( !evProc->readDataWord((const void *)sync_event_arg1_addr,
                 evProc->getAddressWidth(), &arg1, false) ) {
         proccontrol_printf("%s[%d]: failed to read RT library arg1 variable\n",
                 FILE__, __LINE__);
-        return NotRTSignal;
+        return false;
     }
 
-    if( !isValidRTSignal(ev->getSignal(), (RTBreakpointVal) breakpoint, arg1, status) ) return NotRTSignal;
-
     BPatch_process *bproc = BPatch::bpatch->getProcessByPid(evProc->getPid());
     if( bproc == NULL ) {
         proccontrol_printf("%s[%d]: no corresponding BPatch_process for process %d\n",
                 FILE__, __LINE__, evProc->getPid());
-        return ErrorInDecoding;
+        return false;
     }
 
     // See pcEventHandler.h (SYSCALL HANDLING) for a description of what
@@ -691,7 +655,7 @@ PCEventHandler::handleRTSignal(EventSignal::const_ptr ev, PCProcess *evProc) con
         // This is not currently used by Dyninst internals for anything
         // We rely on ProcControlAPI for this and it should be impossible
         // to get this via a breakpoint
-        return ErrorInDecoding;
+        return false;
     case DSE_exitEntry:
         proccontrol_printf("%s[%d]: decoded exitEntry, arg = %lx\n",
                       FILE__, __LINE__, arg1);
@@ -715,26 +679,26 @@ PCEventHandler::handleRTSignal(EventSignal::const_ptr ev, PCProcess *evProc) con
         proccontrol_printf("%s[%d]: decoded loadLibrary (error), arg = %lx\n",
                       FILE__, __LINE__, arg1);
         // This is no longer used
-        return ErrorInDecoding;
+        return false;
     case DSE_lwpExit:
         proccontrol_printf("%s[%d]: decoded lwpExit (error), arg = %lx\n",
                       FILE__, __LINE__, arg1);
         // This is not currently used on any platform
-        return ErrorInDecoding;
+        return false;
     case DSE_snippetBreakpoint:
         proccontrol_printf("%s[%d]: decoded snippetBreak, arg = %lx\n",
                       FILE__, __LINE__, arg1);
-        bproc->setLastSignal(ev->getSignal());
+        bproc->setLastSignal(SIGTRAP);
         evProc->setDesiredProcessState(PCProcess::ps_stopped);
         break;
     case DSE_stopThread:
         proccontrol_printf("%s[%d]: decoded stopThread, arg = %lx\n",
                       FILE__, __LINE__, arg1);
-        bproc->setLastSignal(ev->getSignal());
+        bproc->setLastSignal(SIGTRAP);
         if( !handleStopThread(evProc, arg1) ) {
             proccontrol_printf("%s[%d]: failed to handle stopped thread event\n",
                     FILE__, __LINE__);
-            return ErrorInDecoding;
+            return false;
         }
         break;
     case DSE_dynFuncCall:
@@ -743,7 +707,7 @@ PCEventHandler::handleRTSignal(EventSignal::const_ptr ev, PCProcess *evProc) con
         if( !handleDynFuncCall(evProc, bproc, arg1) ) {
             proccontrol_printf("%s[%d]: failed to handle dynamic callsite event\n",
                     FILE__, __LINE__);
-            return ErrorInDecoding;
+            return false;
         }
         break;
     case DSE_userMessage:
@@ -752,11 +716,11 @@ PCEventHandler::handleRTSignal(EventSignal::const_ptr ev, PCProcess *evProc) con
         if( !handleUserMessage(evProc, bproc, arg1) ) {
             proccontrol_printf("%s[%d]: failed to handle user message event\n",
                     FILE__, __LINE__);
-            return ErrorInDecoding;
+            return false;
         }
         break;
     default:
-        return NotRTSignal;
+        return false;
     }
 
     // Behavior common to all syscalls
@@ -774,7 +738,7 @@ PCEventHandler::handleRTSignal(EventSignal::const_ptr ev, PCProcess *evProc) con
         ProcControlAPI::mbox()->enqueue(newEvt);
     }
 
-    return IsRTSignal;
+    return true;
 }
 
 bool PCEventHandler::handleUserMessage(PCProcess *evProc, BPatch_process *bpProc, 
@@ -967,6 +931,18 @@ bool PCEventHandler::handleLibrary(EventLibrary::const_ptr ev, PCProcess *evProc
 }
 
 bool PCEventHandler::handleBreakpoint(EventBreakpoint::const_ptr ev, PCProcess *evProc) const {
+    proccontrol_printf("Enter PCEventHandler::handleBreakpoint\n");
+
+    // Check whether it is a breakpoint in the RT library (note: this will internally
+    // handle any entry/exit to syscalls and make the necessary up calls as appropriate)
+    bool result = handleRTBreakpoint(ev, evProc);
+    if (result) {
+        // handleRTBreakpoint internally does all handling for the events in order to keep
+        // related logic in one place
+        proccontrol_printf("%s[%d]: breakpoint came from RT library\n", FILE__, __LINE__);
+        return true;
+    }
+
     if( dyn_debug_proccontrol && evProc->isBootstrapped() ) {
         RegisterPool regs;
         if( !ev->getThread()->getAllRegisters(regs) ) {
@@ -978,7 +954,6 @@ bool PCEventHandler::handleBreakpoint(EventBreakpoint::const_ptr ev, PCProcess *
             }
         }
     }
-
     return true;
 }
 

dyninstAPI/src/pcEventHandler.h

@@ -88,19 +88,13 @@ class PCEventHandler {
     bool handleBreakpoint(ProcControlAPI::EventBreakpoint::const_ptr ev, PCProcess *evProc) const;
     bool handleRPC(ProcControlAPI::EventRPC::const_ptr ev, PCProcess *evProc) const;
 
-    enum RTSignalResult {
-        ErrorInDecoding,
-        NotRTSignal,
-        IsRTSignal
-    };
-
     enum RTBreakpointVal {
         NoRTBreakpoint,
         NormalRTBreakpoint,
         SoftRTBreakpoint
     };
 
-    RTSignalResult handleRTSignal(ProcControlAPI::EventSignal::const_ptr ev, PCProcess *evProc) const;
+    bool handleRTBreakpoint(ProcControlAPI::EventBreakpoint::const_ptr ev, PCProcess *evProc) const;
     bool handleStopThread(PCProcess *evProc, Address rt_arg) const;
     bool handleUserMessage(PCProcess *evProc, BPatch_process *bpProc, Address rt_arg) const;
     bool handleDynFuncCall(PCProcess *evProc, BPatch_process *bpProc, Address rt_arg) const;

dyninstAPI_RT/h/dyninstAPI_RT.h

@@ -51,7 +51,7 @@
 #define SYN_INST_BUF_SIZE (1024*1024*1)
 #endif
 
-#define DYNINST_BREAKPOINT_SIGNUM SIGBUS
+#define DYNINST_BREAKPOINT_SIGNUM (SIGRTMIN+4)
 
 #include <stdio.h>
 #include "dyninstRTExport.h"

dyninstAPI_RT/src/RTcommon.c

@@ -145,10 +145,6 @@ DLLEXPORT void DYNINST_unlock_tramp_guard()
   DYNINST_tls_tramp_guard = 1;
 }
 
-#if defined(os_linux)
-void DYNINSTlinuxBreakPoint();
-#endif
-
 DECLARE_DYNINST_LOCK(DYNINST_trace_lock);
 
 /**
@@ -299,11 +295,7 @@ DLLEXPORT void DYNINST_instExecEntry(void *arg1) {
    DYNINST_synch_event_id = DSE_execEntry;
    DYNINST_synch_event_arg1 = arg1;
    /* Stop ourselves */
-#if defined(os_linux)
-   DYNINSTlinuxBreakPoint();
-#else
    DYNINSTbreakPoint();
-#endif
    /* Once the stop completes, clean up */
    DYNINST_synch_event_id = DSE_undefined;
    DYNINST_synch_event_arg1 = NULL;
@@ -746,3 +738,10 @@ void* dyninstTrapTranslate(void *source,
    return target;
 }
 
+DLLEXPORT void DYNINSTtrapFunction(){
+   __asm__ __volatile__(
+           "nop\n"
+           :::);
+}
+
+

dyninstAPI_RT/src/RTcommon.h

@@ -35,6 +35,7 @@
 #include "RTthread.h"
 #include <stdarg.h>
 
+void DYNINSTtrapFunction();
 void DYNINSTbreakPoint();
 /* Use a signal that is safe if we're not attached. */
 void DYNINSTsafeBreakPoint();