Update IR

IR commit: fe4ba285bc576d83bea4a8099fb7315b8bc8c7fb

ext/opcache/jit/ir/ir.h

@@ -541,6 +541,7 @@ void ir_strtab_free(ir_strtab *strtab);
 # define IR_DEBUG_GCM_SPLIT    (1<<28)
 # define IR_DEBUG_SCHEDULE     (1<<29)
 # define IR_DEBUG_RA           (1<<30)
+# define IR_DEBUG_BB_SCHEDULE  (1U<<31)
 #endif
 
 typedef struct _ir_ctx           ir_ctx;

ext/opcache/jit/ir/ir_aarch64.dasc

@@ -1010,7 +1010,7 @@ binop_fp:
 				return IR_RETURN_FP;
 			}
 		case IR_IF:
-			if (ir_in_same_block(ctx, insn->op2) && ctx->use_lists[insn->op2].count == 1) {
+			if (!IR_IS_CONST_REF(insn->op2) && ctx->use_lists[insn->op2].count == 1) {
 				op2_insn = &ctx->ir_base[insn->op2];
 				if (op2_insn->op >= IR_EQ && op2_insn->op <= IR_UGT) {
 					if (IR_IS_TYPE_INT(ctx->ir_base[op2_insn->op1].type)) {
@@ -1020,7 +1020,7 @@ binop_fp:
 						ctx->rules[insn->op2] = IR_FUSED | IR_CMP_FP;
 						return IR_CMP_AND_BRANCH_FP;
 					}
-				} else if (op2_insn->op == IR_OVERFLOW) {
+				} else if (op2_insn->op == IR_OVERFLOW && ir_in_same_block(ctx, insn->op2)) {
 					ctx->rules[insn->op2] = IR_FUSED | IR_SIMPLE | IR_OVERFLOW;
 					return IR_OVERFLOW_AND_BRANCH;
 				}
@@ -1033,7 +1033,7 @@ binop_fp:
 			}
 		case IR_GUARD:
 		case IR_GUARD_NOT:
-			if (ir_in_same_block(ctx, insn->op2) && ctx->use_lists[insn->op2].count == 1) {
+			if (!IR_IS_CONST_REF(insn->op2) && ctx->use_lists[insn->op2].count == 1) {
 				op2_insn = &ctx->ir_base[insn->op2];
 				if (op2_insn->op >= IR_EQ && op2_insn->op <= IR_UGT
 					// TODO: register allocator may clobber operands of CMP before they are used in the GUARD_CMP
@@ -1047,7 +1047,7 @@ binop_fp:
 						ctx->rules[insn->op2] = IR_FUSED | IR_CMP_FP;
 						return IR_GUARD_CMP_FP;
 					}
-				} else if (op2_insn->op == IR_OVERFLOW) {
+				} else if (op2_insn->op == IR_OVERFLOW && ir_in_same_block(ctx, insn->op2)) {
 					ctx->rules[insn->op2] = IR_FUSED | IR_SIMPLE | IR_OVERFLOW;
 					return IR_GUARD_OVERFLOW;
 				}

ext/opcache/jit/ir/ir_cfg.c

@@ -1151,13 +1151,11 @@ static void ir_insert_chain_before(ir_chain *chains, uint32_t c, uint32_t before
 }
 
 #ifndef IR_DEBUG_BB_SCHEDULE_GRAPH
-# define IR_DEBUG_BB_SCHEDULE_GRAPH 0
-#endif
-#ifndef IR_DEBUG_BB_SCHEDULE_EDGES
-# define IR_DEBUG_BB_SCHEDULE_EDGES 0
-#endif
-#ifndef IR_DEBUG_BB_SCHEDULE_CHAINS
-# define IR_DEBUG_BB_SCHEDULE_CHAINS 0
+# ifdef IR_DEBUG
+#  define IR_DEBUG_BB_SCHEDULE_GRAPH 1
+# else
+#  define IR_DEBUG_BB_SCHEDULE_GRAPH 0
+# endif
 #endif
 
 #if IR_DEBUG_BB_SCHEDULE_GRAPH
@@ -1210,20 +1208,17 @@ static void ir_dump_cfg_freq_graph(ir_ctx *ctx, float *bb_freq, uint32_t edges_c
 }
 #endif
 
-#if IR_DEBUG_BB_SCHEDULE_EDGES
+#ifdef IR_DEBUG
 static void ir_dump_edges(ir_ctx *ctx, uint32_t edges_count, ir_edge_info *edges)
 {
 	uint32_t i;
 
 	fprintf(stderr, "Edges:\n");
 	for (i = 0; i < edges_count; i++) {
-		fprintf(stderr, "\tBB%d -> BB%d [label=\"%0.3f\"]\n", edges[i].from, edges[i].to, edges[i].freq);
+		fprintf(stderr, "\tBB%d -> BB%d %0.3f\n", edges[i].from, edges[i].to, edges[i].freq);
 	}
-	fprintf(stderr, "}\n");
 }
-#endif
 
-#if IR_DEBUG_BB_SCHEDULE_CHAINS
 static void ir_dump_chains(ir_ctx *ctx, ir_chain *chains)
 {
 	uint32_t b, tail, i;
@@ -1507,8 +1502,10 @@ static int ir_schedule_blocks_bottom_up(ir_ctx *ctx)
 	/* 2. Sort EDGEs according to their frequencies */
 	qsort(edges, edges_count, sizeof(ir_edge_info), ir_edge_info_cmp);
 
-#if IR_DEBUG_BB_SCHEDULE_EDGES
-	ir_dump_edges(ctx, edges_count, edges);
+#ifdef IR_DEBUG
+	if (ctx->flags & IR_DEBUG_BB_SCHEDULE) {
+		ir_dump_edges(ctx, edges_count, edges);
+	}
 #endif
 
 	/* 3. Process EDGEs in the decreasing frequency order and join the connected chains */
@@ -1555,13 +1552,17 @@ static int ir_schedule_blocks_bottom_up(ir_ctx *ctx)
 	}
 
 #if IR_DEBUG_BB_SCHEDULE_GRAPH
-	ir_dump_cfg_freq_graph(ctx, bb_freq, edges_count, edges, chains);
+	if (ctx->flags & IR_DEBUG_BB_SCHEDULE) {
+		ir_dump_cfg_freq_graph(ctx, bb_freq, edges_count, edges, chains);
+	}
 #endif
 
 	ir_mem_free(bb_freq);
 
-#if IR_DEBUG_BB_SCHEDULE_CHAINS
-	ir_dump_chains(ctx, chains);
+#ifdef IR_DEBUG
+	if (ctx->flags & IR_DEBUG_BB_SCHEDULE) {
+		ir_dump_chains(ctx, chains);
+	}
 #endif
 
 	/* 4. Merge empty entry blocks */
@@ -1585,8 +1586,10 @@ static int ir_schedule_blocks_bottom_up(ir_ctx *ctx)
 			}
 		}
 
-#if IR_DEBUG_BB_SCHEDULE_CHAINS
-		ir_dump_chains(ctx, chains);
+#ifdef IR_DEBUG
+		if (ctx->flags & IR_DEBUG_BB_SCHEDULE) {
+			ir_dump_chains(ctx, chains);
+		}
 #endif
 	}
 
@@ -1619,8 +1622,10 @@ static int ir_schedule_blocks_bottom_up(ir_ctx *ctx)
 		}
 	}
 
-#if IR_DEBUG_BB_SCHEDULE_CHAINS
-	ir_dump_chains(ctx, chains);
+#ifdef IR_DEBUG
+	if (ctx->flags & IR_DEBUG_BB_SCHEDULE) {
+		ir_dump_chains(ctx, chains);
+	}
 #endif
 
 	/* 7. Form a final BB order */

ext/opcache/jit/ir/ir_gcm.c

@@ -84,6 +84,7 @@ static uint32_t ir_gcm_select_best_block(ir_ctx *ctx, ir_ref ref, uint32_t lca)
 		return lca;
 	}
 
+#if 0 /* This is not necessary anymore. Conditions may be fused with IF across BBs. */
 	if (ctx->ir_base[ref].op >= IR_EQ && ctx->ir_base[ref].op <= IR_UGT) {
 		ir_use_list *use_list = &ctx->use_lists[ref];
 
@@ -96,6 +97,7 @@ static uint32_t ir_gcm_select_best_block(ir_ctx *ctx, ir_ref ref, uint32_t lca)
 			}
 		}
 	}
+#endif
 
 	flags = (bb->flags & IR_BB_LOOP_HEADER) ? bb->flags : ctx->cfg_blocks[bb->loop_header].flags;
 	if ((flags & IR_BB_LOOP_WITH_ENTRY)
@@ -487,9 +489,19 @@ static void ir_gcm_schedule_late(ir_ctx *ctx, ir_ref ref, uint32_t b)
 		b = ir_gcm_select_best_block(ctx, ref, lca);
 
 		ctx->cfg_map[ref] = b;
-		if (ctx->ir_base[ref + 1].op == IR_OVERFLOW) {
-			/* OVERFLOW is a projection and must be scheduled together with previous ADD/SUB/MUL_OV */
-			ctx->cfg_map[ref + 1] = b;
+
+		/* OVERFLOW is a projection of ADD/SUB/MUL_OV and must be scheduled into the same block */
+		if (ctx->ir_base[ref].op >= IR_ADD_OV && ctx->ir_base[ref].op <= IR_MUL_OV) {
+			ir_use_list *use_list = &ctx->use_lists[ref];
+			ir_ref n, *p, use;
+
+			for (n = use_list->count, p = &ctx->use_edges[use_list->refs]; n < 0; p++, n--) {
+				use = *p;
+				if (ctx->ir_base[use].op == IR_OVERFLOW) {
+					ctx->cfg_map[use] = b;
+					break;
+				}
+			}
 		}
 	}
 }

ext/opcache/jit/ir/ir_private.h

@@ -582,6 +582,17 @@ IR_ALWAYS_INLINE void ir_bitqueue_init(ir_bitqueue *q, uint32_t n)
 	q->set = ir_bitset_malloc(n);
 }
 
+IR_ALWAYS_INLINE void ir_bitqueue_grow(ir_bitqueue *q, uint32_t n)
+{
+	uint32_t len = ir_bitset_len(n);
+	IR_ASSERT(len >= q->len);
+	if (len > q->len) {
+		q->set = ir_mem_realloc(q->set, len * (IR_BITSET_BITS / 8));
+		memset(q->set + q->len, 0, (len - q->len) * (IR_BITSET_BITS / 8));
+		q->len = len;
+	}
+}
+
 IR_ALWAYS_INLINE void ir_bitqueue_free(ir_bitqueue *q)
 {
 	ir_mem_free(q->set);

ext/opcache/jit/ir/ir_ra.c

@@ -2115,7 +2115,7 @@ int ir_gen_dessa_moves(ir_ctx *ctx, uint32_t b, emit_copy_t emit_copy)
 	ir_insn *insn;
 	uint32_t len;
 	ir_bitset todo, ready;
-	bool have_constants = 0;
+	bool have_constants_or_addresses = 0;
 
 	bb = &ctx->cfg_blocks[b];
 	if (!(bb->flags & IR_BB_DESSA_MOVES)) {
@@ -2141,8 +2141,8 @@ int ir_gen_dessa_moves(ir_ctx *ctx, uint32_t b, emit_copy_t emit_copy)
 		insn = &ctx->ir_base[ref];
 		if (insn->op == IR_PHI) {
 			input = ir_insn_op(insn, k);
-			if (IR_IS_CONST_REF(input)) {
-				have_constants = 1;
+			if (IR_IS_CONST_REF(input) || !ctx->vregs[input]) {
+				have_constants_or_addresses = 1;
 			} else if (ctx->vregs[input] != ctx->vregs[ref]) {
 				s = ctx->vregs[input];
 				d = ctx->vregs[ref];
@@ -2204,13 +2204,13 @@ int ir_gen_dessa_moves(ir_ctx *ctx, uint32_t b, emit_copy_t emit_copy)
 	ir_mem_free(todo);
 	ir_mem_free(loc);
 
-	if (have_constants) {
+	if (have_constants_or_addresses) {
 		for (i = 0, p = &ctx->use_edges[use_list->refs]; i < use_list->count; i++, p++) {
 			ref = *p;
 			insn = &ctx->ir_base[ref];
 			if (insn->op == IR_PHI) {
 				input = ir_insn_op(insn, k);
-				if (IR_IS_CONST_REF(input)) {
+				if (IR_IS_CONST_REF(input) || !ctx->vregs[input]) {
 					emit_copy(ctx, insn->type, input, ref);
 				}
 			}

ext/opcache/jit/ir/ir_sccp.c

@@ -347,7 +347,8 @@ static void ir_sccp_remove_insn(ir_ctx *ctx, ir_insn *_values, ir_ref ref, ir_bi
 	for (j = 1, p = insn->ops + j; j <= n; j++, p++) {
 		ir_ref input = *p;
 		*p = IR_UNUSED;
-		if (input > 0 && _values[input].op == IR_BOTTOM) {
+		/* we may skip nodes that are going to be removed by SCCP (TOP, CONST and COPY) */
+		if (input > 0 && _values[input].op > IR_COPY) {
 			ir_use_list_remove_all(ctx, input, ref);
 			if (ir_is_dead(ctx, input)) {
 				/* schedule DCE */
@@ -396,13 +397,12 @@ static void ir_sccp_replace_insn(ir_ctx *ctx, ir_insn *_values, ir_ref ref, ir_r
 	for (j = 1, p = insn->ops + 1; j <= n; j++, p++) {
 		ir_ref input = *p;
 		*p = IR_UNUSED;
-		if (input > 0) {
+		/* we may skip nodes that are going to be removed by SCCP (TOP, CONST and COPY) */
+		if (input > 0 && _values[input].op > IR_COPY) {
 			ir_use_list_remove_all(ctx, input, ref);
-			if (_values[input].op == IR_BOTTOM) {
-				if (ir_is_dead(ctx, input)) {
-					/* schedule DCE */
-					ir_bitqueue_add(worklist, input);
-				}
+			if (ir_is_dead(ctx, input)) {
+				/* schedule DCE */
+				ir_bitqueue_add(worklist, input);
 			}
 		}
 	}
@@ -429,8 +429,9 @@ static void ir_sccp_replace_insn(ir_ctx *ctx, ir_insn *_values, ir_ref ref, ir_r
 				}
 			}
 #endif
-			/* schedule folding */
-			if (worklist && _values[use].op == IR_BOTTOM) {
+			/* we may skip nodes that are going to be removed by SCCP (TOP, CONST and COPY) */
+			if (worklist && _values[use].op > IR_COPY) {
+				/* schedule folding */
 				ir_bitqueue_add(worklist, use);
 			}
 		}
@@ -1067,7 +1068,7 @@ static ir_ref ir_ext_const(ir_ctx *ctx, ir_insn *val_insn, ir_op op, ir_type typ
 	return ir_const(ctx, new_val, type);
 }
 
-static ir_ref ir_ext_ref(ir_ctx *ctx, ir_ref var_ref, ir_ref src_ref, ir_op op, ir_type type)
+static ir_ref ir_ext_ref(ir_ctx *ctx, ir_ref var_ref, ir_ref src_ref, ir_op op, ir_type type, ir_bitqueue *worklist)
 {
 	uint32_t optx = IR_OPTX(op, type, 1);
 	ir_ref ref;
@@ -1079,6 +1080,7 @@ static ir_ref ir_ext_ref(ir_ctx *ctx, ir_ref var_ref, ir_ref src_ref, ir_op op,
 			if (!IR_IS_CONST_REF(src_ref)) {
 				ir_use_list_remove_one(ctx, src_ref, var_ref);
 			}
+			ir_bitqueue_add(worklist, ref);
 			return ref;
 		}
 	}
@@ -1091,6 +1093,8 @@ static ir_ref ir_ext_ref(ir_ctx *ctx, ir_ref var_ref, ir_ref src_ref, ir_op op,
 	if (!IR_IS_CONST_REF(src_ref)) {
 		ir_use_list_replace_one(ctx, src_ref, var_ref, ref);
 	}
+	ir_bitqueue_grow(worklist, ref + 1);
+	ir_bitqueue_add(worklist, ref);
 	return ref;
 }
 
@@ -1162,17 +1166,15 @@ static bool ir_try_promote_ext(ir_ctx *ctx, ir_ref ext_ref, ir_insn *insn, ir_bi
 				 && !IR_IS_SYM_CONST(ctx->ir_base[use_insn->op1].op)) {
 					ctx->ir_base[use].op1 = ir_ext_const(ctx, &ctx->ir_base[use_insn->op1], op, type);
 				} else {
-					ctx->ir_base[use].op1 = ir_ext_ref(ctx, use, use_insn->op1, op, type);
-					ir_bitqueue_add(worklist, ctx->ir_base[use].op1);
+					ctx->ir_base[use].op1 = ir_ext_ref(ctx, use, use_insn->op1, op, type, worklist);
 				}
 			}
 			if (use_insn->op2 != ref) {
 				if (IR_IS_CONST_REF(use_insn->op2)
 				 && !IR_IS_SYM_CONST(ctx->ir_base[use_insn->op2].op)) {
 					ctx->ir_base[use].op2 = ir_ext_const(ctx, &ctx->ir_base[use_insn->op2], op, type);
 				} else {
-					ctx->ir_base[use].op2 = ir_ext_ref(ctx, use, use_insn->op2, op, type);
-					ir_bitqueue_add(worklist, ctx->ir_base[use].op2);
+					ctx->ir_base[use].op2 = ir_ext_ref(ctx, use, use_insn->op2, op, type, worklist);
 				}
 			}
 		}
@@ -1185,8 +1187,7 @@ static bool ir_try_promote_ext(ir_ctx *ctx, ir_ref ext_ref, ir_insn *insn, ir_bi
 	 && !IR_IS_SYM_CONST(ctx->ir_base[phi_insn->op2].op)) {
 		ctx->ir_base[ref].op2 = ir_ext_const(ctx, &ctx->ir_base[phi_insn->op2], op, type);
 	} else {
-		ctx->ir_base[ref].op2 = ir_ext_ref(ctx, ref, phi_insn->op2, op, type);
-		ir_bitqueue_add(worklist, ctx->ir_base[ref].op2);
+		ctx->ir_base[ref].op2 = ir_ext_ref(ctx, ref, phi_insn->op2, op, type, worklist);
 	}
 
 	return 1;