From 834ad77efed0462cf45d1bc13c553a9b088f70ea Mon Sep 17 00:00:00 2001 From: Timo Paulssen Date: Wed, 16 Aug 2017 18:43:20 +0200 Subject: [PATCH] Put inlined blocks between their caller and its succ Previously inlined callees were added to the end of the basic block list. We now put the inlined blocks into the list at the position of the invoke op. However we cannot yet get rid of the goto ops entering and exiting the inlined code as that would lead to odd bugs possibly related to the annotations on these ops. --- src/spesh/codegen.c | 12 ++++- src/spesh/inline.c | 105 +++++++++++++++++++++-------------------- src/spesh/manipulate.c | 1 + src/spesh/optimize.c | 32 +++++++++++-- 4 files changed, 94 insertions(+), 56 deletions(-) diff --git a/src/spesh/codegen.c b/src/spesh/codegen.c index 25138b8f76..7c826f2a65 100644 --- a/src/spesh/codegen.c +++ b/src/spesh/codegen.c @@ -299,7 +299,9 @@ MVMSpeshCode * MVM_spesh_codegen(MVMThreadContext *tc, MVMSpeshGraph *g) { else if (ws->handlers[i].start_offset == -1 || ws->handlers[i].end_offset == -1 || ws->handlers[i].goto_offset == -1) { - MVM_oops(tc, "Spesh: failed to fix up handlers (%d, %d, %d)", + MVM_oops(tc, "Spesh: failed to fix up handler %d in %s (%d, %d, %d)", + i, + MVM_string_utf8_maybe_encode_C_string(tc, g->sf->body.name), (int)ws->handlers[i].start_offset, (int)ws->handlers[i].end_offset, (int)ws->handlers[i].goto_offset); @@ -314,7 +316,13 @@ MVMSpeshCode * MVM_spesh_codegen(MVMThreadContext *tc, MVMSpeshGraph *g) { } else { if (g->inlines[i].start == -1 || g->inlines[i].end == -1) - MVM_oops(tc, "Spesh: failed to fix up inline %d", i); + MVM_oops(tc, "Spesh: failed to fix up inline %d %p (%s) %d %d", + i, + g->inlines[i].g, + MVM_string_utf8_maybe_encode_C_string(tc, g->inlines[i].sf->body.name), + g->inlines[i].start, + g->inlines[i].end + ); } } diff --git a/src/spesh/inline.c b/src/spesh/inline.c index 952be37870..e486e2d4e8 100644 --- a/src/spesh/inline.c +++ b/src/spesh/inline.c @@ -223,7 +223,7 @@ static void fix_wval(MVMThreadContext *tc, MVMSpeshGraph *inliner, } else { MVM_oops(tc, - "Spesh inline: invalid SC index found"); + "Spesh inline: invalid SC index %d found", dep); } } @@ -260,9 +260,10 @@ static void rewrite_outer_lookup(MVMThreadContext *tc, MVMSpeshGraph *g, } /* Merges the inlinee's spesh graph into the inliner. */ -static void merge_graph(MVMThreadContext *tc, MVMSpeshGraph *inliner, +MVMSpeshBB * merge_graph(MVMThreadContext *tc, MVMSpeshGraph *inliner, MVMSpeshGraph *inlinee, MVMStaticFrame *inlinee_sf, - MVMSpeshIns *invoke_ins, MVMSpeshOperand code_ref_reg, + MVMSpeshBB *invoke_bb, MVMSpeshIns *invoke_ins, + MVMSpeshOperand code_ref_reg, MVMuint32 *inline_boundary_handler) { MVMSpeshFacts **merged_facts; MVMuint16 *merged_fact_counts; @@ -368,19 +369,11 @@ static void merge_graph(MVMThreadContext *tc, MVMSpeshGraph *inliner, bb = bb->linear_next; } - /* Incorporate the basic blocks by concatening them onto the end of the - * linear_next chain of the inliner; skip the inlinee's fake entry BB. */ - bb = inliner->entry; - while (bb) { - if (!bb->linear_next) { - /* Found the end; insert and we're done. */ - bb->linear_next = inlinee_first_bb = inlinee->entry->linear_next; - bb = NULL; - } - else { - bb = bb->linear_next; - } - } + bb = invoke_bb->linear_next; + invoke_bb->linear_next = inlinee_first_bb = inlinee->entry->linear_next; + inlinee_last_bb->linear_next = bb; + + bb = NULL; /* Make all of the inlinee's entry block's successors (except the linear * next) also be successors of the inliner's entry block; this keeps any @@ -420,7 +413,7 @@ static void merge_graph(MVMThreadContext *tc, MVMSpeshGraph *inliner, * some extra spesh slots. */ if (!same_comp_unit) { bb = inlinee->entry; - while (bb) { + while (1) { MVMSpeshIns *ins = bb->first_ins; while (ins) { MVMuint16 opcode = ins->info->opcode; @@ -428,6 +421,7 @@ static void merge_graph(MVMThreadContext *tc, MVMSpeshGraph *inliner, fix_wval(tc, inliner, inlinee, ins); ins = ins->next; } + if (bb == inlinee_last_bb) break; bb = bb->linear_next; } } @@ -572,8 +566,16 @@ static void merge_graph(MVMThreadContext *tc, MVMSpeshGraph *inliner, orig_handlers * sizeof(MVMSpeshIns *)); MVMint32 found_invoke = 0; bb = inliner->entry; - while (bb && !bb->inlined) { + while (bb) { MVMSpeshIns *ins = bb->first_ins; + /* Inlinees are self-contained with regards to handlers, so we + * don't have to look into them. */ + while (bb && bb != inlinee_last_bb) { + bb = bb->linear_next; + } + if (!bb) { + break; + } while (ins) { MVMSpeshAnn *ann = ins->annotations; while (ann) { @@ -654,6 +656,8 @@ static void merge_graph(MVMThreadContext *tc, MVMSpeshGraph *inliner, inliner->num_locals += inlinee->num_locals; inliner->num_lexicals += inlinee->num_lexicals; inliner->num_handlers += inlinee->num_handlers + 1 + active_handlers_at_invoke; + + return inlinee_last_bb; } /* Tweak the successor of a BB, also updating the target BBs pred. */ @@ -713,7 +717,7 @@ static void return_to_box(MVMThreadContext *tc, MVMSpeshGraph *g, MVMSpeshBB *re box_operands[2] = target; MVM_spesh_manipulate_insert_ins(tc, return_bb, return_ins, box_ins); - /* Now turn return instruction node into lookup of appropraite box + /* Now turn return instruction node into lookup of appropriate box * type. */ return_ins->info = MVM_op_get_op(box_type_op); return_ins->operands[0] = target; @@ -796,7 +800,7 @@ static void rewrite_obj_return(MVMThreadContext *tc, MVMSpeshGraph *g, static void rewrite_returns(MVMThreadContext *tc, MVMSpeshGraph *inliner, MVMSpeshGraph *inlinee, MVMSpeshBB *invoke_bb, - MVMSpeshIns *invoke_ins) { + MVMSpeshIns *invoke_ins, MVMSpeshBB *inlinee_last_bb) { /* Locate return instructions. */ MVMSpeshBB *bb = inlinee->entry; while (bb) { @@ -842,6 +846,7 @@ static void rewrite_returns(MVMThreadContext *tc, MVMSpeshGraph *inliner, } ins = ins->next; } + if (bb == inlinee_last_bb) break; bb = bb->linear_next; } } @@ -850,7 +855,7 @@ static void rewrite_returns(MVMThreadContext *tc, MVMSpeshGraph *inliner, * register set operations. */ static void rewrite_args(MVMThreadContext *tc, MVMSpeshGraph *inliner, MVMSpeshGraph *inlinee, MVMSpeshBB *invoke_bb, - MVMSpeshCallInfo *call_info) { + MVMSpeshCallInfo *call_info, MVMSpeshBB *inlinee_last_bb) { /* Look for param-taking instructions. Track what arg instructions we * use in the process. */ MVMSpeshBB *bb = inlinee->entry; @@ -907,6 +912,7 @@ static void rewrite_args(MVMThreadContext *tc, MVMSpeshGraph *inliner, } ins = next; } + if (bb == inlinee_last_bb) break; bb = bb->linear_next; } @@ -950,9 +956,11 @@ static void rewrite_args(MVMThreadContext *tc, MVMSpeshGraph *inliner, * and end inline annotations. */ static void annotate_inline_start_end(MVMThreadContext *tc, MVMSpeshGraph *inliner, MVMSpeshGraph *inlinee, MVMint32 idx, + MVMSpeshBB *inlinee_last_bb, MVMuint32 inline_boundary_handler) { /* Annotate first instruction as an inline start. */ MVMSpeshAnn *start_ann = MVM_spesh_alloc(tc, inliner, sizeof(MVMSpeshAnn)); + MVMSpeshAnn *end_ann = MVM_spesh_alloc(tc, inliner, sizeof(MVMSpeshAnn)); MVMSpeshBB *bb = inlinee->entry->succ[0]; start_ann->next = bb->first_ins->annotations; start_ann->type = MVM_SPESH_ANN_INLINE_START; @@ -967,31 +975,26 @@ static void annotate_inline_start_end(MVMThreadContext *tc, MVMSpeshGraph *inlin bb->first_ins->annotations = start_ann; /* Now look for last instruction and annotate it. */ - while (bb) { - if (!bb->linear_next) { - /* Annotate it as the inline end. */ - MVMSpeshAnn *end_ann = MVM_spesh_alloc(tc, inliner, sizeof(MVMSpeshAnn)); - end_ann->next = bb->last_ins->annotations; - end_ann->type = MVM_SPESH_ANN_INLINE_END; - end_ann->data.inline_idx = idx; - bb->last_ins->annotations = end_ann; - - /* Insert annotation for handler boundary fixup; we add the end - * one that is needed and also a dummy goto one to keep things - * that want all three happy. */ - end_ann = MVM_spesh_alloc(tc, inliner, sizeof(MVMSpeshAnn)); - end_ann->next = bb->last_ins->annotations; - end_ann->type = MVM_SPESH_ANN_FH_END; - end_ann->data.frame_handler_index = inline_boundary_handler; - bb->last_ins->annotations = end_ann; - end_ann = MVM_spesh_alloc(tc, inliner, sizeof(MVMSpeshAnn)); - end_ann->next = bb->last_ins->annotations; - end_ann->type = MVM_SPESH_ANN_FH_GOTO; - end_ann->data.frame_handler_index = inline_boundary_handler; - bb->last_ins->annotations = end_ann; - } - bb = bb->linear_next; - } + end_ann->next = inlinee_last_bb->last_ins->annotations; + end_ann->type = MVM_SPESH_ANN_INLINE_END; + end_ann->data.inline_idx = idx; + inlinee_last_bb->last_ins->annotations = end_ann; + + /* Insert annotation for handler boundary fixup; we add the end + * one that is needed and also a dummy goto one to keep things + * that want all three happy. */ + end_ann = MVM_spesh_alloc(tc, inliner, sizeof(MVMSpeshAnn)); + end_ann->next = inlinee_last_bb->last_ins->annotations; + end_ann->type = MVM_SPESH_ANN_FH_END; + end_ann->data.frame_handler_index = inline_boundary_handler; + inlinee_last_bb->last_ins->annotations = end_ann; + end_ann = MVM_spesh_alloc(tc, inliner, sizeof(MVMSpeshAnn)); + end_ann->next = inlinee_last_bb->last_ins->annotations; + end_ann->type = MVM_SPESH_ANN_FH_GOTO; + end_ann->data.frame_handler_index = inline_boundary_handler; + inlinee_last_bb->last_ins->annotations = end_ann; + + return; } /* Drives the overall inlining process. */ @@ -1001,8 +1004,8 @@ void MVM_spesh_inline(MVMThreadContext *tc, MVMSpeshGraph *inliner, MVMStaticFrame *inlinee_sf, MVMSpeshOperand code_ref_reg) { /* Merge inlinee's graph into the inliner. */ MVMuint32 inline_boundary_handler; - merge_graph(tc, inliner, inlinee, inlinee_sf, invoke_ins, code_ref_reg, - &inline_boundary_handler); + MVMSpeshBB *inlinee_last_bb = merge_graph(tc, inliner, inlinee, inlinee_sf, + invoke_bb, invoke_ins, code_ref_reg, &inline_boundary_handler); /* If we're profiling, note it's an inline. */ if (inlinee->entry->linear_next->first_ins->info->opcode == MVM_OP_prof_enterspesh) { @@ -1014,17 +1017,17 @@ void MVM_spesh_inline(MVMThreadContext *tc, MVMSpeshGraph *inliner, } /* Re-write returns to a set and goto. */ - rewrite_returns(tc, inliner, inlinee, invoke_bb, invoke_ins); + rewrite_returns(tc, inliner, inlinee, invoke_bb, invoke_ins, inlinee_last_bb); /* Re-write the argument passing instructions to poke values into the * appropriate slots. */ - rewrite_args(tc, inliner, inlinee, invoke_bb, call_info); + rewrite_args(tc, inliner, inlinee, invoke_bb, call_info, inlinee_last_bb); /* Annotate first and last instruction with inline table annotations; also * add annotations for fixing up the handlers table inline boundary * indicators. */ annotate_inline_start_end(tc, inliner, inlinee, inliner->num_inlines - 1, - inline_boundary_handler); + inlinee_last_bb, inline_boundary_handler); /* Finally, turn the invoke instruction into a goto. */ invoke_ins->info = MVM_op_get_op(MVM_OP_goto); diff --git a/src/spesh/manipulate.c b/src/spesh/manipulate.c index d10d03d815..1a6e1e5f20 100644 --- a/src/spesh/manipulate.c +++ b/src/spesh/manipulate.c @@ -46,6 +46,7 @@ void MVM_spesh_manipulate_delete_ins(MVMThreadContext *tc, MVMSpeshGraph *g, next->annotations = ann; } break; + case MVM_SPESH_ANN_INLINE_END: case MVM_SPESH_ANN_FH_END: /* This moves to the previous instruction. */ if (!prev) { diff --git a/src/spesh/optimize.c b/src/spesh/optimize.c index 8b0bbfd3d3..6dddc45fea 100644 --- a/src/spesh/optimize.c +++ b/src/spesh/optimize.c @@ -2391,14 +2391,40 @@ static void eliminate_unused_log_guards(MVMThreadContext *tc, MVMSpeshGraph *g) /* Sometimes - almost always due to other optmimizations having done their * work - we end up with an unconditional goto at the end of a basic block * that points right to the very next basic block. Delete these. */ +static int any_deopt_annotations(MVMSpeshAnn *ann) { + while (ann) { + switch (ann->type) { + case MVM_SPESH_ANN_DEOPT_ALL_INS: + case MVM_SPESH_ANN_DEOPT_INLINE: + return 1; + } + ann = ann->next; + } + return 0; +} +static int any_inline_end_annotations(MVMSpeshAnn *ann) { + while (ann) { + switch (ann->type) { + case MVM_SPESH_ANN_INLINE_END: + return 1; + } + ann = ann->next; + } + return 0; +} static void eliminate_pointless_gotos(MVMThreadContext *tc, MVMSpeshGraph *g) { MVMSpeshBB *cur_bb = g->entry; while (cur_bb) { if (!cur_bb->jumplist) { MVMSpeshIns *last_ins = cur_bb->last_ins; - if (last_ins && last_ins->info->opcode == MVM_OP_goto) - if (last_ins->operands[0].ins_bb == cur_bb->linear_next) - MVM_spesh_manipulate_delete_ins(tc, g, cur_bb, last_ins); + if ( + last_ins + && last_ins->info->opcode == MVM_OP_goto + && last_ins->operands[0].ins_bb == cur_bb->linear_next + && ! any_deopt_annotations(last_ins->annotations) + && (! any_inline_end_annotations(last_ins->annotations)) + ) + MVM_spesh_manipulate_delete_ins(tc, g, cur_bb, last_ins); } cur_bb = cur_bb->linear_next; }