From 0e12ff3017d470676e94e561cd0de4ca22230532 Mon Sep 17 00:00:00 2001 From: Rodrigo Kumpera Date: Thu, 16 Apr 2015 19:49:02 -0400 Subject: [PATCH] [jit] Rework the managed code suspend polling and add COOP_GC support. This change enables managed polling to work with both NaCL and coop as they are exactly the same except for the variables used. We cleanup how safepoints are injects to be done on a separate pass instead of having in spread around the JIT. We now inject safepoints at EH handler entry as those did not previously have them. The generated AOT code was greatly optimized by having the polling variable checked before doing the icall. This more than doubles the performance of the generated code. --- mono/mini/aot-compiler.c | 2 ++ mono/mini/aot-runtime.c | 2 ++ mono/mini/cpu-amd64.md | 2 +- mono/mini/cpu-x86.md | 2 +- mono/mini/method-to-ir.c | 5 --- mono/mini/mini-amd64.c | 31 ++++++++++-------- mono/mini/mini-ops.h | 4 +-- mono/mini/mini-runtime.c | 12 +++++++ mono/mini/mini-x86.c | 32 +++++++++++------- mono/mini/mini.c | 71 ++++++++++++++++++++++++++++++++++++++++ mono/mini/mini.h | 2 +- mono/mini/patch-info.h | 1 + 12 files changed, 131 insertions(+), 35 deletions(-) diff --git a/mono/mini/aot-compiler.c b/mono/mini/aot-compiler.c index c777fda7db3c5..4591832e254e6 100644 --- a/mono/mini/aot-compiler.c +++ b/mono/mini/aot-compiler.c @@ -5291,6 +5291,8 @@ encode_patch (MonoAotCompile *acfg, MonoJumpInfo *patch_info, guint8 *buf, guint encode_klass_ref (acfg, patch_info->data.virt_method->klass, p, &p); encode_method_ref (acfg, patch_info->data.virt_method->method, p, &p); break; + case MONO_PATCH_INFO_GC_SAFE_POINT_FLAG: + break; default: g_warning ("unable to handle jump info %d", patch_info->type); g_assert_not_reached (); diff --git a/mono/mini/aot-runtime.c b/mono/mini/aot-runtime.c index d5c0f217bc336..996729a9bfc40 100644 --- a/mono/mini/aot-runtime.c +++ b/mono/mini/aot-runtime.c @@ -3460,6 +3460,8 @@ decode_patch (MonoAotModule *aot_module, MonoMemPool *mp, MonoJumpInfo *ji, guin ji->data.target = info; break; } + case MONO_PATCH_INFO_GC_SAFE_POINT_FLAG: + break; default: g_warning ("unhandled type %d", ji->type); g_assert_not_reached (); diff --git a/mono/mini/cpu-amd64.md b/mono/mini/cpu-amd64.md index 1b330f2de7b66..4fd06a36d5d88 100755 --- a/mono/mini/cpu-amd64.md +++ b/mono/mini/cpu-amd64.md @@ -295,7 +295,7 @@ move_f_to_i8: dest:i src1:f len:5 move_i8_to_f: dest:f src1:i len:5 call_handler: len:14 clob:c nacl:52 aot_const: dest:i len:10 -nacl_gc_safe_point: clob:c +gc_safe_point: clob:c src1:i len:40 x86_test_null: src1:i len:5 x86_compare_membase_reg: src1:b src2:i len:9 x86_compare_membase_imm: src1:b len:13 diff --git a/mono/mini/cpu-x86.md b/mono/mini/cpu-x86.md index b31850b6e2ac8..2bd196cc76c0f 100644 --- a/mono/mini/cpu-x86.md +++ b/mono/mini/cpu-x86.md @@ -251,7 +251,7 @@ call_handler: len:11 clob:c aot_const: dest:i len:5 load_gotaddr: dest:i len:64 got_entry: dest:i src1:b len:7 -nacl_gc_safe_point: clob:c +gc_safe_point: clob:c src1:i len:20 x86_test_null: src1:i len:2 x86_compare_membase_reg: src1:b src2:i len:7 x86_compare_membase_imm: src1:b len:11 diff --git a/mono/mini/method-to-ir.c b/mono/mini/method-to-ir.c index bfff6b96a9d4e..018898013211a 100755 --- a/mono/mini/method-to-ir.c +++ b/mono/mini/method-to-ir.c @@ -7919,11 +7919,6 @@ mono_method_to_ir (MonoCompile *cfg, MonoMethod *method, MonoBasicBlock *start_b cfg->bb_entry = start_bblock; start_bblock->cil_code = NULL; start_bblock->cil_length = 0; -#if defined(__native_client_codegen__) - MONO_INST_NEW (cfg, ins, OP_NACL_GC_SAFE_POINT); - ins->dreg = alloc_dreg (cfg, STACK_I4); - MONO_ADD_INS (start_bblock, ins); -#endif /* EXIT BLOCK */ NEW_BBLOCK (cfg, end_bblock); diff --git a/mono/mini/mini-amd64.c b/mono/mini/mini-amd64.c index e3a711c694a99..f7f25d175b0d1 100755 --- a/mono/mini/mini-amd64.c +++ b/mono/mini/mini-amd64.c @@ -32,6 +32,7 @@ #include #include #include +#include #include "trace.h" #include "ir-emit.h" @@ -6555,22 +6556,26 @@ mono_arch_output_basic_block (MonoCompile *cfg, MonoBasicBlock *bb) MONO_VARINFO (cfg, ins->inst_c0)->live_range_end = code - cfg->native_code; break; } - case OP_NACL_GC_SAFE_POINT: { -#if defined(__native_client_codegen__) && defined(__native_client_gc__) - if (cfg->compile_aot) - code = emit_call (cfg, code, MONO_PATCH_INFO_ABS, (gpointer)mono_nacl_gc, TRUE); - else { - guint8 *br [1]; - - amd64_mov_reg_imm_size (code, AMD64_R11, (gpointer)&__nacl_thread_suspension_needed, 4); - amd64_test_membase_imm_size (code, AMD64_R11, 0, 0xFFFFFFFF, 4); - br[0] = code; x86_branch8 (code, X86_CC_EQ, 0, FALSE); - code = emit_call (cfg, code, MONO_PATCH_INFO_ABS, (gpointer)mono_nacl_gc, TRUE); - amd64_patch (br[0], code); - } + case OP_GC_SAFE_POINT: { + gpointer polling_func = NULL; + int compare_val; +#if defined (USE_COOP_GC) + polling_func = (gpointer)mono_threads_state_poll; + compare_val = 1; +#elif defined(__native_client_codegen__) && defined(__native_client_gc__) + polling_func = (gpointer)mono_nacl_gc; + compare_val = 0xFFFFFFFF; #endif + if (!polling_func) + break; + + amd64_test_membase_imm_size (code, ins->sreg1, 0, compare_val, 4); + br[0] = code; x86_branch8 (code, X86_CC_EQ, 0, FALSE); + code = emit_call (cfg, code, MONO_PATCH_INFO_ABS, polling_func, TRUE); + amd64_patch (br[0], code); break; } + case OP_GC_LIVENESS_DEF: case OP_GC_LIVENESS_USE: case OP_GC_PARAM_SLOT_LIVENESS_DEF: diff --git a/mono/mini/mini-ops.h b/mono/mini/mini-ops.h index daa8cf5d9a0db..2484e6cc9beb0 100644 --- a/mono/mini/mini-ops.h +++ b/mono/mini/mini-ops.h @@ -1071,8 +1071,8 @@ MINI_OP(OP_GC_PARAM_SLOT_LIVENESS_DEF, "gc_param_slot_liveness_def", NONE, NONE, /* #if defined(__native_client_codegen__) || defined(__native_client__) */ /* We have to define these in terms of the TARGET defines, not NaCl defines */ /* because genmdesc.pl doesn't have multiple defines per platform. */ -#if defined(TARGET_AMD64) || defined(TARGET_X86) || defined(TARGET_ARM) -MINI_OP(OP_NACL_GC_SAFE_POINT, "nacl_gc_safe_point", IREG, NONE, NONE) +#if defined(TARGET_AMD64) || defined(TARGET_X86) +MINI_OP(OP_GC_SAFE_POINT, "gc_safe_point", NONE, IREG, NONE) #endif #if defined(TARGET_X86) || defined(TARGET_AMD64) diff --git a/mono/mini/mini-runtime.c b/mono/mini/mini-runtime.c index 1635d210407ad..78fc5b7054778 100755 --- a/mono/mini/mini-runtime.c +++ b/mono/mini/mini-runtime.c @@ -1175,6 +1175,7 @@ mono_patch_info_hash (gconstpointer data) case MONO_PATCH_INFO_MONITOR_ENTER_V4: case MONO_PATCH_INFO_MONITOR_EXIT: case MONO_PATCH_INFO_GOT_OFFSET: + case MONO_PATCH_INFO_GC_SAFE_POINT_FLAG: return (ji->type << 8); case MONO_PATCH_INFO_CASTCLASS_CACHE: return (ji->type << 8) | (ji->data.index); @@ -1355,6 +1356,13 @@ mono_resolve_patch_target (MonoMethod *method, MonoDomain *domain, guint8 *code, target = code_slot; break; } + case MONO_PATCH_INFO_GC_SAFE_POINT_FLAG: +#if defined(__native_client_codegen__) + target = (gpointer)&__nacl_thread_suspension_needed; +#else + target = (gpointer)&mono_polling_required; +#endif + break; case MONO_PATCH_INFO_SWITCH: { gpointer *jump_table; int i; @@ -3205,6 +3213,10 @@ register_icalls (void) #if defined(__native_client__) || defined(__native_client_codegen__) register_icall (mono_nacl_gc, "mono_nacl_gc", "void", TRUE); #endif +#if defined(USE_COOP_GC) + register_icall (mono_threads_state_poll, "mono_threads_state_poll", "void", TRUE); +#endif + #ifndef MONO_ARCH_NO_EMULATE_LONG_MUL_OPTS register_opcode_emulation (OP_LMUL, "__emul_lmul", "long long long", mono_llmult, "mono_llmult", TRUE); register_opcode_emulation (OP_LDIV, "__emul_ldiv", "long long long", mono_lldiv, "mono_lldiv", FALSE); diff --git a/mono/mini/mini-x86.c b/mono/mini/mini-x86.c index 8328084702c68..78a4fc05628e7 100644 --- a/mono/mini/mini-x86.c +++ b/mono/mini/mini-x86.c @@ -29,6 +29,7 @@ #include #include #include +#include #include "trace.h" #include "mini-x86.h" @@ -5023,19 +5024,26 @@ mono_arch_output_basic_block (MonoCompile *cfg, MonoBasicBlock *bb) MONO_VARINFO (cfg, ins->inst_c0)->live_range_end = code - cfg->native_code; break; } - case OP_NACL_GC_SAFE_POINT: { -#if defined(__native_client_codegen__) && defined(__native_client_gc__) - if (cfg->compile_aot) - code = emit_call (cfg, code, MONO_PATCH_INFO_ABS, (gpointer)mono_nacl_gc); - else { - guint8 *br [1]; - - x86_test_mem_imm8 (code, (gpointer)&__nacl_thread_suspension_needed, 0xFFFFFFFF); - br[0] = code; x86_branch8 (code, X86_CC_EQ, 0, FALSE); - code = emit_call (cfg, code, MONO_PATCH_INFO_ABS, (gpointer)mono_nacl_gc); - x86_patch (br[0], code); - } + case OP_GC_SAFE_POINT: { + gpointer polling_func = NULL; + int compare_val; + guint8 *br [1]; + +#if defined (USE_COOP_GC) + polling_func = (gpointer)mono_threads_state_poll; + compare_val = 1; +#elif defined(__native_client_codegen__) && defined(__native_client_gc__) + polling_func = (gpointer)mono_nacl_gc; + compare_val = 0xFFFFFFFF; #endif + if (!polling_func) + break; + + x86_test_membase_imm (code, ins->sreg1, 0, compare_val); + br[0] = code; x86_branch8 (code, X86_CC_EQ, 0, FALSE); + code = emit_call (cfg, code, MONO_PATCH_INFO_ABS, polling_func); + x86_patch (br [0], code); + break; } case OP_GC_LIVENESS_DEF: diff --git a/mono/mini/mini.c b/mono/mini/mini.c index 3ad251c10c50b..774bcc3b5797c 100755 --- a/mono/mini/mini.c +++ b/mono/mini/mini.c @@ -3005,6 +3005,71 @@ is_open_method (MonoMethod *method) } #ifndef DISABLE_JIT + +static void +mono_create_gc_safepoint (MonoCompile *cfg, MonoBasicBlock *bblock) +{ +#if defined(__native_client_codegen__) || USE_COOP_GC + + MonoInst *poll_addr, *ins; + if (cfg->verbose_level) + printf ("ADDING SAFE POINT TO BB %d\n", bblock->block_num); + +#if defined(__native_client_codegen__) + NEW_AOTCONST (cfg, poll_addr, MONO_PATCH_INFO_GC_SAFE_POINT_FLAG, (gpointer)&__nacl_thread_suspension_needed); +#else + NEW_AOTCONST (cfg, poll_addr, MONO_PATCH_INFO_GC_SAFE_POINT_FLAG, (gpointer)&mono_polling_required); +#endif + + MONO_INST_NEW (cfg, ins, OP_GC_SAFE_POINT); + ins->sreg1 = poll_addr->dreg; + + if (bblock->flags & BB_EXCEPTION_HANDLER) { + MonoInst *eh_op = bblock->code; + + // we only skip the ops that start EH blocks. + if (eh_op && eh_op->opcode != OP_START_HANDLER && eh_op->opcode != OP_GET_EX_OBJ) + eh_op = NULL; + + mono_bblock_insert_after_ins (bblock, eh_op, poll_addr); + mono_bblock_insert_after_ins (bblock, poll_addr, ins); + } else if (bblock == cfg->bb_entry) { + mono_bblock_insert_after_ins (bblock, bblock->last_ins, poll_addr); + mono_bblock_insert_after_ins (bblock, poll_addr, ins); + + } else { + mono_bblock_insert_before_ins (bblock, NULL, poll_addr); + mono_bblock_insert_after_ins (bblock, poll_addr, ins); + } + +#endif +} + +/* +This code inserts safepoints into managed code at important code paths. +Those are: + +-the first basic block +-landing BB for exception handlers +-loop body starts. + +*/ +static void +mono_insert_safepoints (MonoCompile *cfg) +{ +#if defined(__native_client_codegen__) || defined(USE_COOP_GC) + MonoBasicBlock *bb; + + if (cfg->verbose_level) + printf ("INSERTING SAFEPOINTS\n"); + + for (bb = cfg->bb_entry; bb; bb = bb->next_bb) { + if (bb->loop_body_start || bb == cfg->bb_entry || bb->flags & BB_EXCEPTION_HANDLER) + mono_create_gc_safepoint (cfg, bb); + } +#endif +} + /* * mini_method_compile: * @method: the method to compile @@ -3126,6 +3191,10 @@ mini_method_compile (MonoMethod *method, guint32 opts, MonoDomain *domain, JitFl cfg->gen_sdb_seq_points = FALSE; } #endif + /* coop / nacl requires loop detection to happen */ +#if defined(__native_client_codegen__) || defined(USE_COOP_GC) + cfg->opt |= MONO_OPT_LOOP; +#endif cfg->explicit_null_checks = debug_options.explicit_null_checks; cfg->soft_breakpoints = debug_options.soft_breakpoints; @@ -3550,6 +3619,8 @@ mini_method_compile (MonoMethod *method, guint32 opts, MonoDomain *domain, JitFl mono_compute_natural_loops (cfg); } + mono_insert_safepoints (cfg); + /* after method_to_ir */ if (parts == 1) { if (MONO_METHOD_COMPILE_END_ENABLED ()) diff --git a/mono/mini/mini.h b/mono/mini/mini.h index baafaf82ffc23..1b97399f8554a 100644 --- a/mono/mini/mini.h +++ b/mono/mini/mini.h @@ -105,7 +105,7 @@ #endif /* Version number of the AOT file format */ -#define MONO_AOT_FILE_VERSION 114 +#define MONO_AOT_FILE_VERSION 115 //TODO: This is x86/amd64 specific. #define mono_simd_shuffle_mask(a,b,c,d) ((a) | ((b) << 2) | ((c) << 4) | ((d) << 6)) diff --git a/mono/mini/patch-info.h b/mono/mini/patch-info.h index f2cd3c7d4f7c4..4f31fd5bd3606 100644 --- a/mono/mini/patch-info.h +++ b/mono/mini/patch-info.h @@ -55,4 +55,5 @@ PATCH_INFO(METHOD_CODE_SLOT, "method_code_slot") PATCH_INFO(LDSTR_LIT, "ldstr_lit") PATCH_INFO(GC_NURSERY_START, "gc_nursery_start") PATCH_INFO(VIRT_METHOD, "virt_method") +PATCH_INFO(GC_SAFE_POINT_FLAG, "gc_safe_point_flag") PATCH_INFO(NONE, "none")