Enable AOT linux perf support (bytecodealliance#2930)

And refactor the original perf support
- use WAMR_BUILD_LINUX_PERF as the cmake compilation control
- use WASM_ENABLE_LINUX_PERF as the compiler macro
- use `wamrc --enable-linux-perf` to generate aot file which contains fp operations
- use `iwasm --enable-linux-perf` to create perf map for `perf record`

build-scripts/config_common.cmake

@@ -147,13 +147,20 @@ elseif (WAMR_BUILD_SANITIZER STREQUAL "asan")
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -fno-omit-frame-pointer -fsanitize=address -fno-sanitize-recover=all" )
     set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address")
   endif()
-elseif (WAMR_BUILD_SANITIZER STREQUAL "tsan") 
+elseif (WAMR_BUILD_SANITIZER STREQUAL "tsan")
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -fno-omit-frame-pointer -fsanitize=thread -fno-sanitize-recover=all" )
   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=thread")
 elseif (NOT (WAMR_BUILD_SANITIZER STREQUAL "") )
   message(SEND_ERROR "Unsupported sanitizer: ${WAMR_BUILD_SANITIZER}")
 endif()
 
+if (WAMR_BUILD_LINUX_PERF EQUAL 1)
+  if (NOT WAMR_BUILD_JIT AND NOT WAMR_BUILD_AOT)
+    message(WARNING "only support perf in aot and llvm-jit")
+    set(WAMR_BUILD_LINUX_PERF 0)
+  endif ()
+endif ()
+
 ########################################
 
 message ("-- Build Configurations:")
@@ -440,3 +447,7 @@ if (WAMR_CONFIGUABLE_BOUNDS_CHECKS EQUAL 1)
   add_definitions (-DWASM_CONFIGURABLE_BOUNDS_CHECKS=1)
   message ("     Configurable bounds checks enabled")
 endif ()
+if (WAMR_BUILD_LINUX_PERF EQUAL 1)
+  add_definitions (-DWASM_ENABLE_LINUX_PERF=1)
+  message ("     Enable linux perf support")
+endif ()

core/config.h

@@ -490,4 +490,9 @@
 #define WASM_MAX_INSTANCE_CONTEXTS 8
 #endif
 
+/* linux perf support */
+#ifndef WASM_ENABLE_LINUX_PERF
+#define WASM_ENABLE_LINUX_PERF 0
+#endif
+
 #endif /* end of _CONFIG_H_ */

core/iwasm/aot/aot_loader.c

@@ -2764,6 +2764,104 @@ load_relocation_section(const uint8 *buf, const uint8 *buf_end,
     return ret;
 }
 
+#if WASM_ENABLE_LINUX_PERF != 0
+struct func_info {
+    uint32 idx;
+    void *ptr;
+};
+
+static uint32
+get_func_size(const AOTModule *module, struct func_info *sorted_func_ptrs,
+              uint32 idx)
+{
+    uint32 func_sz;
+
+    if (idx == module->func_count - 1)
+        func_sz = (uintptr_t)module->code + module->code_size
+                  - (uintptr_t)(sorted_func_ptrs[idx].ptr);
+    else
+        func_sz = (uintptr_t)(sorted_func_ptrs[idx + 1].ptr)
+                  - (uintptr_t)(sorted_func_ptrs[idx].ptr);
+
+    return func_sz;
+}
+
+static int
+compare_func_ptrs(const void *f1, const void *f2)
+{
+    return (intptr_t)((struct func_info *)f1)->ptr
+           - (intptr_t)((struct func_info *)f2)->ptr;
+}
+
+static struct func_info *
+sort_func_ptrs(const AOTModule *module, char *error_buf, uint32 error_buf_size)
+{
+    uint64 content_len;
+    struct func_info *sorted_func_ptrs;
+    unsigned i;
+
+    content_len = (uint64)sizeof(struct func_info) * module->func_count;
+    sorted_func_ptrs = loader_malloc(content_len, error_buf, error_buf_size);
+    if (!sorted_func_ptrs)
+        return NULL;
+
+    for (i = 0; i < module->func_count; i++) {
+        sorted_func_ptrs[i].idx = i;
+        sorted_func_ptrs[i].ptr = module->func_ptrs[i];
+    }
+
+    qsort(sorted_func_ptrs, module->func_count, sizeof(struct func_info),
+          compare_func_ptrs);
+
+    return sorted_func_ptrs;
+}
+
+static bool
+create_perf_map(const AOTModule *module, char *error_buf, uint32 error_buf_size)
+{
+    struct func_info *sorted_func_ptrs = NULL;
+    char perf_map_info[128] = { 0 };
+    FILE *perf_map = NULL;
+    uint32 i;
+    pid_t pid = getpid();
+    bool ret = false;
+
+    sorted_func_ptrs = sort_func_ptrs(module, error_buf, error_buf_size);
+    if (!sorted_func_ptrs)
+        goto quit;
+
+    snprintf(perf_map_info, 128, "/tmp/perf-%d.map", pid);
+    perf_map = fopen(perf_map_info, "w");
+    if (!perf_map) {
+        LOG_WARNING("warning: can't create /tmp/perf-%d.map, because %s", pid,
+                    strerror(errno));
+        goto quit;
+    }
+
+    for (i = 0; i < module->func_count; i++) {
+        memset(perf_map_info, 0, 128);
+        snprintf(perf_map_info, 128, "%lx  %x  aot_func#%u\n",
+                 (uintptr_t)sorted_func_ptrs[i].ptr,
+                 get_func_size(module, sorted_func_ptrs, i),
+                 sorted_func_ptrs[i].idx);
+
+        fwrite(perf_map_info, 1, strlen(perf_map_info), perf_map);
+    }
+
+    LOG_VERBOSE("generate /tmp/perf-%d.map", pid);
+    ret = true;
+
+quit:
+    if (sorted_func_ptrs)
+        free(sorted_func_ptrs);
+
+    if (perf_map)
+        fclose(perf_map);
+
+    return ret;
+}
+#endif /* WASM_ENABLE_LINUX_PERF != 0*/
+
 static bool
 load_from_sections(AOTModule *module, AOTSection *sections,
                    bool is_load_from_file_buf, char *error_buf,
@@ -3224,6 +3322,12 @@ load(const uint8 *buf, uint32 size, AOTModule *module, char *error_buf,
     }
 #endif
 
+#if WASM_ENABLE_LINUX_PERF != 0
+    if (wasm_runtime_get_linux_perf())
+        if (!create_perf_map(module, error_buf, error_buf_size))
+            goto fail;
+#endif
+
     return ret;
 fail:
     return false;

core/iwasm/common/wasm_c_api.c

@@ -330,7 +330,7 @@ wasm_config_set_linux_perf_opt(wasm_config_t *config, bool enable)
     if (!config)
         return NULL;
 
-    config->linux_perf_support = enable;
+    config->enable_linux_perf = enable;
     return config;
 }
 
@@ -380,7 +380,8 @@ wasm_engine_new_internal(wasm_config_t *config)
     init_args.mem_alloc_type = config->mem_alloc_type;
     memcpy(&init_args.mem_alloc_option, &config->mem_alloc_option,
            sizeof(MemAllocOption));
-    init_args.linux_perf_support = config->linux_perf_support;
+
+    init_args.enable_linux_perf = config->enable_linux_perf;
 
     if (!wasm_runtime_full_init(&init_args)) {
         LOG_DEBUG("wasm_runtime_full_init failed");

core/iwasm/common/wasm_runtime_common.c

@@ -158,7 +158,7 @@ static JitCompOptions jit_options = { 0 };
 #endif
 
 #if WASM_ENABLE_JIT != 0
-static LLVMJITOptions llvm_jit_options = { 3, 3, 0, false };
+static LLVMJITOptions llvm_jit_options = { 3, 3, 0 };
 #endif
 
 static RunningMode runtime_running_mode = Mode_Default;
@@ -662,14 +662,17 @@ wasm_runtime_full_init(RuntimeInitArgs *init_args)
 #endif
 
 #if WASM_ENABLE_JIT != 0
-    LOG_DEBUG("Start LLVM_JIT, opt_sz=%u, opt_lvl=%u, segue=%s, linux_perf=%s",
-              init_args->llvm_jit_size_level, init_args->llvm_jit_opt_level,
-              init_args->segue_flags ? "Yes" : "No",
-              init_args->linux_perf_support ? "Yes" : "No");
     llvm_jit_options.size_level = init_args->llvm_jit_size_level;
     llvm_jit_options.opt_level = init_args->llvm_jit_opt_level;
     llvm_jit_options.segue_flags = init_args->segue_flags;
-    llvm_jit_options.linux_perf_support = init_args->linux_perf_support;
+#endif
+
+#if WASM_ENABLE_LINUX_PERF != 0
+    wasm_runtime_set_linux_perf(init_args->enable_linux_perf);
+#else
+    if (init_args->enable_linux_perf)
+        LOG_WARNING("warning: to enable linux perf support, please recompile "
+                    "with -DWAMR_BUILD_LINUX_PERF=1");
 #endif
 
     if (!wasm_runtime_env_init()) {
@@ -6146,3 +6149,19 @@ wasm_runtime_get_context(WASMModuleInstanceCommon *inst, void *key)
     return wasm_native_get_context(inst, key);
 }
 #endif /* WASM_ENABLE_MODULE_INST_CONTEXT != 0 */
+
+#if WASM_ENABLE_LINUX_PERF != 0
+static bool enable_linux_perf = false;
+
+bool
+wasm_runtime_get_linux_perf(void)
+{
+    return enable_linux_perf;
+}
+
+void
+wasm_runtime_set_linux_perf(bool flag)
+{
+    enable_linux_perf = flag;
+}
+#endif

core/iwasm/common/wasm_runtime_common.h

@@ -443,7 +443,6 @@ typedef struct LLVMJITOptions {
     uint32 opt_level;
     uint32 size_level;
     uint32 segue_flags;
-    bool linux_perf_support;
 } LLVMJITOptions;
 #endif
 
@@ -1105,6 +1104,14 @@ wasm_runtime_end_blocking_op(WASMExecEnv *exec_env);
 void
 wasm_runtime_interrupt_blocking_op(WASMExecEnv *exec_env);
 
+#if WASM_ENABLE_LINUX_PERF != 0
+bool
+wasm_runtime_get_linux_perf(void);
+
+void
+wasm_runtime_set_linux_perf(bool flag);
+#endif
+
 #ifdef __cplusplus
 }
 #endif

core/iwasm/compilation/aot_llvm.c

@@ -2174,7 +2174,7 @@ jit_stack_size_callback(void *user_data, const char *name, size_t namelen,
 }
 
 static bool
-orc_jit_create(AOTCompContext *comp_ctx, bool linux_perf_support)
+orc_jit_create(AOTCompContext *comp_ctx)
 {
     LLVMErrorRef err;
     LLVMOrcLLLazyJITRef orc_jit = NULL;
@@ -2214,13 +2214,15 @@ orc_jit_create(AOTCompContext *comp_ctx, bool linux_perf_support)
     /* Ownership transfer: LLVMOrcLLJITBuilderRef -> LLVMOrcLLJITRef */
     builder = NULL;
 
-    if (linux_perf_support) {
-        LOG_DEBUG("Enable linux perf support");
+#if WASM_ENABLE_LINUX_PERF != 0
+    if (wasm_runtime_get_linux_perf()) {
+        LOG_DEBUG("Enable linux perf support in JIT");
         LLVMOrcObjectLayerRef obj_linking_layer =
             (LLVMOrcObjectLayerRef)LLVMOrcLLLazyJITGetObjLinkingLayer(orc_jit);
         LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(
             obj_linking_layer, LLVMCreatePerfJITEventListener());
     }
+#endif
 
     /* Ownership transfer: local -> AOTCompContext */
     comp_ctx->orc_jit = orc_jit;
@@ -2320,7 +2322,8 @@ aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option)
         goto fail;
     }
 
-    if (option->linux_perf_support) {
+#if WASM_ENABLE_LINUX_PERF != 0
+    if (wasm_runtime_get_linux_perf()) {
         /* FramePointerKind.All */
         LLVMMetadataRef val =
             LLVMValueAsMetadata(LLVMConstInt(LLVMInt32Type(), 2, false));
@@ -2330,6 +2333,7 @@ aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option)
 
         comp_ctx->emit_frame_pointer = true;
     }
+#endif
 
     if (BH_LIST_ERROR == bh_list_init(&comp_ctx->native_symbols)) {
         goto fail;
@@ -2434,7 +2438,7 @@ aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option)
             goto fail;
 
         /* Create LLJIT Instance */
-        if (!orc_jit_create(comp_ctx, option->linux_perf_support))
+        if (!orc_jit_create(comp_ctx))
             goto fail;
     }
     else {

core/iwasm/compilation/aot_llvm.h

@@ -461,7 +461,6 @@ typedef struct AOTCompOption {
     uint32 bounds_checks;
     uint32 stack_bounds_checks;
     uint32 segue_flags;
-    bool linux_perf_support;
     char **custom_sections;
     uint32 custom_sections_count;
     const char *stack_usage_file;

core/iwasm/fast-jit/jit_compiler.h

@@ -70,7 +70,6 @@ typedef struct JitInterpSwitchInfo {
 typedef struct JitCompOptions {
     uint32 code_cache_size;
     uint32 opt_level;
-    bool linux_perf_support;
 } JitCompOptions;
 
 bool

core/iwasm/include/aot_export.h

@@ -65,7 +65,6 @@ typedef struct AOTCompOption {
     uint32_t bounds_checks;
     uint32_t stack_bounds_checks;
     uint32_t segue_flags;
-    bool linux_perf_support;
     char **custom_sections;
     uint32_t custom_sections_count;
     const char *stack_usage_file;

core/iwasm/include/wasm_c_api.h

@@ -181,15 +181,15 @@ typedef union MemAllocOption {
 struct wasm_config_t {
     mem_alloc_type_t mem_alloc_type;
     MemAllocOption mem_alloc_option;
-    bool linux_perf_support;
+    bool enable_linux_perf;
     /*TODO: wasi args*/
 };
 
 /*
  * by default:
  * - mem_alloc_type is Alloc_With_System_Allocator
  * - mem_alloc_option is all 0
- * - linux_perf_support is false
+ * - enable_linux_perf is false
  */
 WASM_API_EXTERN own wasm_config_t* wasm_config_new(void);
 

core/iwasm/include/wasm_export.h

@@ -172,12 +172,12 @@ typedef struct RuntimeInitArgs {
     /**
      * If enabled
      * - llvm-jit will output a jitdump file for `perf inject`
-     * - aot. TBD
+     * - aot will output a perf-${pid}.map for `perf record`
      * - fast-jit. TBD
      * - multi-tier-jit. TBD
      * - interpreter. TBD
      */
-    bool linux_perf_support;
+    bool enable_linux_perf;
 } RuntimeInitArgs;
 
 #ifndef WASM_VALKIND_T_DEFINED
@@ -243,8 +243,8 @@ WASM_RUNTIME_API_EXTERN bool
 wasm_runtime_full_init(RuntimeInitArgs *init_args);
 
 /**
- * Set the log level. To be called after the runtime is initialized. 
- * 
+ * Set the log level. To be called after the runtime is initialized.
+ *
  * @param level the log level to set
  */
 WASM_RUNTIME_API_EXTERN void

core/iwasm/interpreter/wasm_loader.c

@@ -2892,7 +2892,6 @@ init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf,
     option.opt_level = llvm_jit_options.opt_level;
     option.size_level = llvm_jit_options.size_level;
     option.segue_flags = llvm_jit_options.segue_flags;
-    option.linux_perf_support = llvm_jit_options.linux_perf_support;
 
 #if WASM_ENABLE_BULK_MEMORY != 0
     option.enable_bulk_memory = true;

core/iwasm/interpreter/wasm_mini_loader.c

@@ -1877,7 +1877,6 @@ init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf,
     option.opt_level = llvm_jit_options.opt_level;
     option.size_level = llvm_jit_options.size_level;
     option.segue_flags = llvm_jit_options.segue_flags;
-    option.linux_perf_support = llvm_jit_options.linux_perf_support;
 
 #if WASM_ENABLE_BULK_MEMORY != 0
     option.enable_bulk_memory = true;