i#1684 xarch-IR: Support separate host and target arch (#4325)

Adds support for building with different host and target
architectures.  The goal is to support running drdisas, drdecode,
drraw2trace, and drcachesim trace analyzers on one host machine while
targeting recorded bytes or traces from a different type of machine:
e.g., processing aarch64 traces on an x86_64 machine.  The goal is
*not* to turn DR into a cross-ISA binary translator: we only support
standalone mode here, not full DR managed mode.  Long-term it would be
nicer to split out the DR standalone mode interfaces into separate
libraries but that is beyond the scope of the current work.

Adds a top-level CMake option variable "TARGET_ARCH" which defaults to
CMAKE_SYSTEM_PROCESSOR but can be set to a different value.  The
regular arch variables/defines X86, AARCH64, and ARM are set from
TARGET_ARCH, while new values DR_HOST_{X86,AARCH64,ARM} are set for
the host.

Uses a target-centric approach, where the bulk of the code is built
targeting the TARGET_ARCH, with only assembly code and other limited code
using the DR_HOST_* arch.  This limits the scope of the code changes as
compared to the opposite host-centric approach.  A new define
DR_HOST_NOT_TARGET is used to simply disable cases where the host and
target are intertwined and difficult to easily separate, such as TLS
handling and assembly code used for application execution (such as
icache_op_isb_asm).

The key code changed to use DR_HOST_* includes:
+ *.asm files
+ arch_exports.h inline asm for atomics and other key utilities
+ system call defines (we don't want to have our raw syscall wrappers running
  incorrect syscall numbers: we'll end up with fork bombs or other madness)
+ injection code
+ data section setup
+ drsyms libraries

Code using built-in __*__ arch defines is changed to use our defines:
+ sigcontext.h, which is also augmented for some missing AArch64 types
+ x86_code_test.c

Code reading ELF headers is relaxed to allow *any* arch of the same
bitwdith (i#1345 prevents different bitwidth), since we need the host for
standalone init code but the target for raw2trace module mapping.

Nearly all tests are disabled not just from running but also from
building for simplicity, to reduce code changes here.  Some api.* decoder
tests, drdisas, and the altbin test for drcachesim post-processing and
analysis are enbabled in host!=target builds.
Tests are disabled by using a macro to set up the tests and shifting all
the setup code prior to the tests themselves, allowing a return() to skip
over the bulk of the tests to avoid a giant if().

Only Linux host-x86_64-target-aarch64 is explicitly tested and enabled
here.  Adding the inverse, and adding host-i386-target-arm32, should not be
much work, mostly with compiler flags, but will be done separately.  Adding
Windows support is also separated out as it requires a number of changes
for compiler problems with aarch64 code.

Adds a new host-x86_64-target-aarch64 build to runsuite.cmake, allocated to
a new Travis job to avoid slowing down the current jobs.

Issue: #1684, #4318, #1345

.travis.yml

@@ -107,6 +107,11 @@ jobs:
       # gcc on Travis claims to not be CMAKE_COMPILER_IS_GNUCC so we only run clang.
       compiler: clang
       env: DYNAMORIO_CROSS_AARCHXX_LINUX_ONLY=no DEPLOY=no EXTRA_ARGS=64_only
+    # AArch64 drdecode and drmemtrace on x86:
+    - if: type != cron AND env(TRAVIS_EVENT_TYPE) != cron
+      os: linux
+      compiler: gcc
+      env: DYNAMORIO_A64_ON_X86_ONLY=yes DEPLOY=no
 
     #######################################################################
     # Package jobs

CMakeLists.txt

@@ -208,12 +208,34 @@ endif ()
 
 # The target architecture.
 # For cross-compilation this should still work as you're supposed to set this var.
-if (CMAKE_SYSTEM_PROCESSOR MATCHES "^arm")
+# X64 mean 64-bit generically, whether AMD64 or AARCH64.
+set(TARGET_ARCH "${CMAKE_SYSTEM_PROCESSOR}" CACHE STRING "Target architecture")
+if (TARGET_ARCH MATCHES "^arm")
   set(ARM 1) # This means AArch32.
-elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "^aarch64")
+  set(X64 OFF)
+  message(STATUS "Building for ARM")
+elseif (TARGET_ARCH MATCHES "^aarch64")
   set(AARCH64 1)
+  set(X64 1)
+  message(STATUS "Building for AArch64")
 else ()
   set(X86 1) # This means IA-32 or AMD64
+  message(STATUS "Building for x86")
+  # Whether 64-bit is expected to be selected by user setting up compiler
+  # prior to invoking CMake: it has to be that way for Windows, and for
+  # Linux the user should set CFLAGS to -m32 or -m64
+  # to override gcc's default.  To simplify matters we only look at
+  # CMAKE_C_SIZEOF_DATA_PTR, controlled by CFLAGS, so the user doesn't
+  # have to also set CXXFLAGS (CMAKE_SIZEOF_VOID_P happens
+  # to come from CXXFLAGS).  (CMAKE_C_SIZEOF_DATA_PTR can be relied on
+  # to be set in all CMake versions we support.)
+  # For target!=host, we still need the host compiler to match the target,
+  # so we do not support differing bitwidths.
+  if (CMAKE_C_SIZEOF_DATA_PTR EQUAL 8)
+    set(X64 ON)
+  else ()
+    set(X64 OFF)
+  endif ()
 endif ()
 
 if (X86)
@@ -233,20 +255,45 @@ else ()
   set(ARCH_NAME_SHARED ${ARCH_NAME})
 endif ()
 
-# The plan is to use X64 to mean 64-bit generically, whether AMD64 or AARCH64.
-# Whether 64-bit is expected to be selected by user setting up compiler
-# prior to invoking CMake: it has to be that way for Windows, and for
-# Linux the user should set CFLAGS to -m32 or -m64
-# to override gcc's default.  To simplify matters we only look at
-# CMAKE_C_SIZEOF_DATA_PTR, controlled by CFLAGS, so the user doesn't
-# have to also set CXXFLAGS (CMAKE_SIZEOF_VOID_P happens
-# to come from CXXFLAGS).  (CMAKE_C_SIZEOF_DATA_PTR can be relied on
-# to be set in all CMake versions we support.)
-if (CMAKE_C_SIZEOF_DATA_PTR EQUAL 8)
-  set(X64 ON)
-else(CMAKE_C_SIZEOF_DATA_PTR EQUAL 8)
-  set(X64 OFF)
-endif (CMAKE_C_SIZEOF_DATA_PTR EQUAL 8)
+# The execution architecture, which might differ from the target for building
+# an AArch64 decoder to execute on x86 machines (i#1684).
+if (CMAKE_SYSTEM_PROCESSOR MATCHES "^arm")
+  set(DR_HOST_ARM 1)
+  set(DR_HOST_ARCH_NAME "arm")
+  set(DR_HOST_AARCHXX 1)
+  set(DR_HOST_ARCH_NAME_SHARED aarchxx)
+elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "^aarch64")
+  set(DR_HOST_AARCH64 1)
+  set(DR_HOST_ARCH_NAME "aarch64")
+  set(DR_HOST_AARCHXX 1)
+  set(DR_HOST_ARCH_NAME_SHARED aarchxx)
+  set(DR_HOST_X64 1)
+elseif (CMAKE_C_SIZEOF_DATA_PTR EQUAL 8)
+  set(DR_HOST_X86 1)
+  set(DR_HOST_ARCH_NAME "x86")
+  set(DR_HOST_ARCH_NAME_SHARED ${DR_HOST_ARCH_NAME})
+  set(DR_HOST_X64 1)
+else ()
+  set(DR_HOST_X86 1)
+  set(DR_HOST_ARCH_NAME "x86")
+  set(DR_HOST_ARCH_NAME_SHARED ${DR_HOST_ARCH_NAME})
+endif ()
+if (NOT "${TARGET_ARCH}" STREQUAL "${CMAKE_SYSTEM_PROCESSOR}")
+  set(DR_HOST_NOT_TARGET 1)
+  if (WIN32)
+    # TODO i#1684: Fix Windows compiler warnings for AArch64 on x86.
+    message(FATAL_ERROR "Host != target is not yet supported on Windows")
+  endif ()
+  if ((DR_HOST_X64 AND NOT X64) OR (NOT DR_HOST_X64 AND X64))
+    # XXX i#1345: We need to resolve the hardcoded ELF bitwidth types.
+    # We likely also have general C type issues if the host is 32-bit.
+    message(FATAL_ERROR "Different-bitwidth host-vs-target not supported: i#1345.")
+  endif ()
+  if (ARM)
+    # TODO i#1684: We don't have full support for targeting arm in i386 yet.
+    message(FATAL_ERROR "Targeting ARM on i386 is not yet supported: i#1684.")
+  endif ()
+endif ()
 
 option(VMKERNEL "target VMkernel (not officially supported yet)")
 

clients/drcachesim/tests/burst_traceopts.cpp

@@ -232,6 +232,10 @@ main(int argc, const char *argv[])
 #    undef ARM_32
 #    undef ARM_64
 #    undef DR_APP_EXPORTS
+#    undef DR_HOST_X64
+#    undef DR_HOST_X86
+#    undef DR_HOST_ARM
+#    undef DR_HOST_AARCH64
 #    include "asm_defines.asm"
 /* clang-format off */
 START_FILE

core/CMakeLists.txt

@@ -70,10 +70,10 @@ endif ()
 set(asm_deps
   "${PROJECT_SOURCE_DIR}/core/arch/asm_defines.asm"
   "${PROJECT_BINARY_DIR}/configure.h")
-add_asm_target(arch/${ARCH_NAME}/${ARCH_NAME}.asm arch_core_asm_src arch_core_asm_tgt
-  "_core" "" "${asm_deps}")
-if (NOT "${ARCH_NAME}" STREQUAL "${ARCH_NAME_SHARED}")
-  add_asm_target(arch/${ARCH_NAME_SHARED}/${ARCH_NAME_SHARED}.asm
+add_asm_target(arch/${DR_HOST_ARCH_NAME}/${DR_HOST_ARCH_NAME}.asm
+  arch_core_asm_src arch_core_asm_tgt "_core" "" "${asm_deps}")
+if (NOT "${DR_HOST_ARCH_NAME}" STREQUAL "${DR_HOST_ARCH_NAME_SHARED}")
+  add_asm_target(arch/${DR_HOST_ARCH_NAME_SHARED}/${DR_HOST_ARCH_NAME_SHARED}.asm
     archshared_core_asm_src archshared_core_asm_tgt
     "_core" "" "${asm_deps}")
 endif ()
@@ -86,7 +86,7 @@ if (UNIX)
   # our auxiliary tools (i#1504), but we do *not* want our own memcpy and
   # memset for static-lib DR core.  Thus we separate them out.  The i#1504
   # glibc versioning is only an issue on x86.
-  add_asm_target(arch/${ARCH_NAME}/memfuncs.asm memfuncs_asm_src memfuncs_asm_tgt
+  add_asm_target(arch/${DR_HOST_ARCH_NAME}/memfuncs.asm memfuncs_asm_src memfuncs_asm_tgt
     "_memfuncs" "" "${asm_deps}")
   add_library(drmemfuncs STATIC ${memfuncs_asm_src} lib/memmove.c)
   add_gen_events_deps(drmemfuncs)
@@ -98,7 +98,7 @@ endif ()
 # i#1409: to share core libc-ish code with non-core, we use the "drlibc" library.
 add_asm_target(drlibc/drlibc_xarch.asm drlibc_xarch_asm_src drlibc_xarch_asm_tgt
   "_core" "" "${asm_deps}")
-add_asm_target(drlibc/drlibc_${ARCH_NAME}.asm drlibc_arch_asm_src
+add_asm_target(drlibc/drlibc_${DR_HOST_ARCH_NAME}.asm drlibc_arch_asm_src
   drlibc_arch_asm_tgt
   "" "" "${asm_deps}")
 set(DRLIBC_SRCS
@@ -1050,7 +1050,10 @@ endif ()
 ###########################################################################
 # Unit tests
 
-if (BUILD_TESTS)
+# We can't run core unit tests when the target is not the host, just like we can't
+# run DR itself managing an app: DR is a same-arch system for code cache operation
+# and only supports target!=host for decoding and drmemtrace analysis (i#1684).
+if (BUILD_TESTS AND NOT DR_HOST_NOT_TARGET)
   add_executable(unit_tests unit_tests.c
     # These unit tests have been moved from the x86_code module into a new x86_code
     # test module that gets its own clang/gcc options for testing (-mno-vzeroupper).
@@ -1124,7 +1127,7 @@ if (BUILD_TESTS)
     set_tests_properties(unit_tests PROPERTIES LABELS OSX)
   endif ()
   copy_target_to_device(unit_tests "${location_suffix}")
-endif (BUILD_TESTS)
+endif ()
 
 ###########################################################################
 

core/arch/aarchxx/mangle.c

@@ -2853,8 +2853,15 @@ mangle_icache_op(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr,
                 opnd_create_base_disp(dr_reg_stolen, DR_REG_NULL, 0, 16, OPSZ_16),
                 opnd_create_reg(xt == dr_reg_stolen ? DR_REG_X0 : xt),
                 opnd_create_reg(DR_REG_X30)));
+#    ifdef DR_HOST_NOT_TARGET
+        /* We built all our asm code for the host, but here we need it for the target.
+         * We have to ifdef it out to separate.  Xref i#1684.
+         */
+        ASSERT_NOT_REACHED();
+#    else
         insert_mov_immed_arch(dcontext, NULL, NULL, (ptr_int_t)icache_op_ic_ivau_asm,
                               opnd_create_reg(DR_REG_X30), ilist, instr, NULL, NULL);
+#    endif
         PRE(ilist, instr, /* mov x0, x28 */
             XINST_CREATE_move(dcontext, opnd_create_reg(DR_REG_X0),
                               opnd_create_reg(dr_reg_stolen)));
@@ -2887,8 +2894,15 @@ mangle_icache_op(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr,
                 dcontext,
                 opnd_create_base_disp(dr_reg_stolen, DR_REG_NULL, 0, 8, OPSZ_16),
                 opnd_create_reg(DR_REG_X1), opnd_create_reg(DR_REG_X2)));
+#    ifdef DR_HOST_NOT_TARGET
+        /* We built all our asm code for the host, but here we need it for the target.
+         * We have to ifdef it out to separate.  Xref i#1684.
+         */
+        ASSERT_NOT_REACHED();
+#    else
         insert_mov_immed_arch(dcontext, NULL, NULL, (ptr_int_t)icache_op_isb_asm,
                               opnd_create_reg(DR_REG_X2), ilist, instr, NULL, NULL);
+#    endif
         insert_mov_immed_arch(dcontext, NULL, NULL, (ptr_int_t)pc,
                               opnd_create_reg(DR_REG_X1), ilist, instr, NULL, NULL);
         PRE(ilist, instr, /* mov x0, x28 */

core/arch/arch_exports.h

@@ -468,7 +468,7 @@ atomic_add_exchange_int64(volatile int64 *var, int64 value)
 #    define atomic_add_exchange atomic_add_exchange_int
 
 #else /* UNIX */
-#    ifdef X86
+#    ifdef DR_HOST_X86
 /* IA-32 vol 3 7.1.4: processor will internally suppress the bus lock
  * if target is within cache line.
  */
@@ -632,7 +632,7 @@ atomic_add_exchange_int64(volatile int64 *var, int64 value)
 #        define SET_IF_NOT_ZERO(flag) SET_FLAG(nz, flag)
 #        define SET_IF_NOT_LESS(flag) SET_FLAG(nl, flag)
 
-#    elif defined(AARCH64)
+#    elif defined(DR_HOST_AARCH64)
 
 #        define ATOMIC_1BYTE_WRITE(target, value, hot_patch)   \
             do {                                               \
@@ -817,7 +817,7 @@ atomic_dec_becomes_zero(volatile int *var)
     return atomic_add_exchange_int(var, -1) == 0;
 }
 
-#    elif defined(ARM)
+#    elif defined(DR_HOST_ARM)
 
 #        define ATOMIC_1BYTE_WRITE(target, value, hot_patch)   \
             do {                                               \
@@ -979,7 +979,7 @@ proc_get_timestamp(void);
 #        define ATOMIC_COMPARE_EXCHANGE_PTR ATOMIC_COMPARE_EXCHANGE
 #    endif
 
-#    ifndef AARCH64
+#    ifndef DR_HOST_AARCH64
 /* Atomically increments *var by 1
  * Returns true if the resulting value is zero, otherwise returns false
  */
@@ -1646,7 +1646,7 @@ use_addr_prefix_on_short_disp(void)
 /** Specifies which processor mode to use when decoding or encoding. */
 typedef enum _dr_isa_mode_t {
     DR_ISA_IA32,              /**< IA-32 (Intel/AMD 32-bit mode). */
-    DR_ISA_X86 = DR_ISA_IA32, /**< Alis for DR_ISA_IA32. */
+    DR_ISA_X86 = DR_ISA_IA32, /**< Alias for DR_ISA_IA32. */
     DR_ISA_AMD64,             /**< AMD64 (Intel/AMD 64-bit mode). */
     DR_ISA_ARM_A32,           /**< ARM A32 (AArch32 ARM). */
     DR_ISA_ARM_THUMB,         /**< Thumb (ARM T32). */

core/arch/asm_defines.asm

@@ -41,24 +41,41 @@
 
 #include "configure.h"
 
-#if (defined(X86_64) || defined(ARM_64)) && !defined(X64)
-# define X64
-#endif
+#ifdef DR_HOST_NOT_TARGET
+#   undef X86
+#   undef AARCH64
+#   undef AARCHXX
+#   undef X64
+#   ifdef DR_HOST_X86
+#       define X86
+#   elif defined(DR_HOST_AARCH64)
+#       define AARCH64
+#   elif defined(DR_HOST_ARM)
+#       define ARM
+#   endif
+#   if defined(DR_HOST_X64)
+#       define X64
+#   endif
+#else
+#   if (defined(X86_64) || defined(ARM_64)) && !defined(X64)
+#     define X64
+#   endif
 
-#if (defined(X86_64) || defined(X86_32)) && !defined(X86)
-# define X86
-#endif
+#   if (defined(X86_64) || defined(X86_32)) && !defined(X86)
+#     define X86
+#   endif
 
-#if defined(ARM_64) && !defined(AARCH64)
-# define AARCH64
-#endif
+#   if defined(ARM_64) && !defined(AARCH64)
+#     define AARCH64
+#   endif
 
-#if defined(ARM_32) && !defined(ARM)
-# define ARM
-#endif
+#   if defined(ARM_32) && !defined(ARM)
+#     define ARM
+#   endif
 
-#if (defined(ARM_32) || defined(ARM_64)) && !defined(AARCHXX)
-# define AARCHXX
+#   if (defined(ARM_32) || defined(ARM_64)) && !defined(AARCHXX)
+#     define AARCHXX
+#   endif
 #endif
 
 #if (defined(ARM) && defined(X64)) || (defined(AARCH64) && !defined(X64))

core/arch/x86_code_test.c

@@ -1,5 +1,5 @@
 /* **********************************************************
- * Copyright (c) 2013-2019 Google, Inc.  All rights reserved.
+ * Copyright (c) 2013-2020 Google, Inc.  All rights reserved.
  * Copyright (c) 2001-2010 VMware, Inc.  All rights reserved.
  * **********************************************************/
 
@@ -105,7 +105,7 @@ test_cpuid()
 #    endif
 }
 
-#    ifdef __AVX__
+#    if !defined(DR_HOST_NOT_TARGET) && defined(__AVX__)
 
 static void
 unit_test_get_ymm_caller_saved()
@@ -393,7 +393,7 @@ unit_test_asm(dcontext_t *dc)
     print_file(STDERR, "testing asm\n");
     test_call_switch_stack(dc);
     test_cpuid();
-#    ifdef UNIX
+#    if defined(UNIX) && !defined(DR_HOST_NOT_TARGET)
 #        ifdef __AVX__
     unit_test_get_ymm_caller_saved();
 #        endif

core/drlibc/drlibc.c

@@ -1,5 +1,5 @@
 /* **********************************************************
- * Copyright (c) 2015-2019 Google, Inc.  All rights reserved.
+ * Copyright (c) 2015-2020 Google, Inc.  All rights reserved.
  * **********************************************************/
 
 /*
@@ -117,6 +117,9 @@ kernel_is_64bit(void)
 void
 clear_icache(void *beg, void *end)
 {
+#    ifdef DR_HOST_NOT_TARGET
+    ASSERT_NOT_REACHED();
+#    else
     static size_t cache_info = 0;
     size_t dcache_line_size;
     size_t icache_line_size;
@@ -159,6 +162,7 @@ clear_icache(void *beg, void *end)
 
     /* Instruction Synchronization Barrier */
     __asm__ __volatile__("isb" : : : "memory");
+#    endif
 }
 #endif
 

core/drlibc/drlibc_module_elf.c

@@ -105,7 +105,7 @@ is_elf_so_header_common(app_pc base, size_t size, bool memory)
          * dr_map_executable_file, but we just don't support that yet until we
          * remove our hardcoded type defines in module_elf.h.
          *
-         * i#1648: We do allow mixing arches of the same bitwidth to better support
+         * i#1684: We do allow mixing arches of the same bitwidth to better support
          * drdecode tools.  We have no standalone_library var access here to limit
          * this relaxation to tools; we assume DR managed code will hit other
          * problems later for the wrong arch and that recognizing an other-arch

core/lib/globals_shared.h

@@ -796,6 +796,17 @@ typedef struct _instr_t instr_t;
 #endif
 /* DR_API EXPORT END */
 
+#ifdef DR_HOST_X86
+#    define IF_HOST_X86_ELSE(x, y) x
+#else
+#    define IF_HOST_X86_ELSE(x, y) y
+#endif
+#ifdef DR_HOST_X64
+#    define IF_HOST_X64_ELSE(x, y) x
+#else
+#    define IF_HOST_X64_ELSE(x, y) y
+#endif
+
 typedef enum {
     SYSLOG_INFORMATION = 0x1,
     SYSLOG_WARNING = 0x2,