Skip to content

Commit

Permalink
JIT: Improve property access (Avoid unnecessary property address load…
Browse files Browse the repository at this point in the history 
…ing and exception check).
  • Loading branch information
@dstogov
dstogov committed Sep 27, 2021
1 parent dbdef59 commit 35ff71f
Show file tree
Hide file tree
Showing 4 changed files with 295 additions and 94 deletions.
6 changes: 2 additions & 4 deletions ext/opcache/jit/zend_jit.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3180,8 +3180,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op
}
if (!zend_jit_incdec_obj(&dasm_state, opline, op_array, ssa, ssa_op,
op1_info, op1_addr,
0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN,
zend_may_throw(opline, ssa_op, op_array, ssa))) {
0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN)) {
goto jit_failure;
}
goto done;
Expand Down Expand Up @@ -3229,8 +3228,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op
}
if (!zend_jit_assign_obj_op(&dasm_state, opline, op_array, ssa, ssa_op,
op1_info, op1_addr, OP1_DATA_INFO(), OP1_DATA_RANGE(),
0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN,
zend_may_throw(opline, ssa_op, op_array, ssa))) {
0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN)) {
goto jit_failure;
}
goto done;
Expand Down
188 changes: 145 additions & 43 deletions ext/opcache/jit/zend_jit_arm64.dasc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4396,20 +4396,37 @@ static int zend_jit_math_helper(dasm_State **Dst,
|.cold_code
}
|6:
if (Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
| LOAD_ZVAL_ADDR FCARG1x, real_addr
} else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, res_addr
}
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var);
if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) {
return 0;
if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) {
if (Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
| LOAD_ZVAL_ADDR FCARG1x, real_addr
} else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, res_addr
}
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var);
if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) {
return 0;
}
op1_addr = real_addr;
}
| LOAD_ZVAL_ADDR FCARG2x, op1_addr
} else {
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var);
if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) {
return 0;
}
op1_addr = real_addr;
}
| LOAD_ZVAL_ADDR FCARG2x, op1_addr
if (Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
| LOAD_ZVAL_ADDR FCARG1x, real_addr
} else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, res_addr
}
op1_addr = real_addr;
}
| LOAD_ZVAL_ADDR FCARG2x, op1_addr
if (Z_MODE(op2_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var);
if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) {
Expand Down Expand Up @@ -4720,20 +4737,37 @@ static int zend_jit_long_math_helper(dasm_State **Dst,
|.cold_code
}
|6:
if (Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
| LOAD_ZVAL_ADDR FCARG1x, real_addr
} else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, res_addr
}
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var);
if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) {
return 0;
if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) {
if (Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
| LOAD_ZVAL_ADDR FCARG1x, real_addr
} else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, res_addr
}
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var);
if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) {
return 0;
}
op1_addr = real_addr;
}
| LOAD_ZVAL_ADDR FCARG2x, op1_addr
} else {
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var);
if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) {
return 0;
}
op1_addr = real_addr;
}
| LOAD_ZVAL_ADDR FCARG2x, op1_addr
if (Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
| LOAD_ZVAL_ADDR FCARG1x, real_addr
} else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, res_addr
}
op1_addr = real_addr;
}
| LOAD_ZVAL_ADDR FCARG2x, op1_addr
if (Z_MODE(op2_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var);
if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) {
Expand Down Expand Up @@ -4850,10 +4884,17 @@ static int zend_jit_concat_helper(dasm_State **Dst,
|.cold_code
|6:
}
if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, res_addr
if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) {
if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, res_addr
}
| LOAD_ZVAL_ADDR FCARG2x, op1_addr
} else {
| LOAD_ZVAL_ADDR FCARG2x, op1_addr
if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, res_addr
}
}
| LOAD_ZVAL_ADDR FCARG2x, op1_addr
| LOAD_ZVAL_ADDR CARG3, op2_addr
| SET_EX_OPLINE opline, REG0
| EXT_CALL concat_function, REG0
Expand Down Expand Up @@ -12186,16 +12227,14 @@ static int zend_jit_fetch_obj(dasm_State **Dst,
int32_t exit_point;
const void *exit_addr;
uint32_t type;
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0);
zend_jit_addr val_addr = prop_addr;

if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
&& !delayed_fetch_this
&& !op1_avoid_refcounting) {
flags = ZEND_JIT_EXIT_FREE_OP1;
}

| LOAD_ZVAL_ADDR REG0, prop_addr

if ((opline->result_type & (IS_VAR|IS_TMP_VAR))
&& !(flags & ZEND_JIT_EXIT_FREE_OP1)
&& (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))
Expand All @@ -12214,6 +12253,8 @@ static int zend_jit_fetch_obj(dasm_State **Dst,
exit_point = zend_jit_trace_get_exit_point(opline, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
} else {
val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0);
| LOAD_ZVAL_ADDR REG0, prop_addr
if (op1_avoid_refcounting) {
SET_STACK_REG(JIT_G(current_frame)->stack,
EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
Expand Down Expand Up @@ -12398,8 +12439,7 @@ static int zend_jit_incdec_obj(dasm_State **Dst,
bool on_this,
bool delayed_fetch_this,
zend_class_entry *trace_ce,
uint8_t prop_type,
int may_throw)
uint8_t prop_type)
{
zval *member;
zend_string *name;
Expand All @@ -12409,6 +12449,7 @@ static int zend_jit_incdec_obj(dasm_State **Dst,
zend_jit_addr prop_addr;
bool needs_slow_path = 0;
bool use_prop_guard = 0;
bool may_throw = 0;

ZEND_ASSERT(opline->op2_type == IS_CONST);
ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
Expand Down Expand Up @@ -12544,6 +12585,7 @@ static int zend_jit_incdec_obj(dasm_State **Dst,
}
}
if (ZEND_TYPE_IS_SET(prop_info->type)) {
may_throw = 1;
| SET_EX_OPLINE opline, REG0
if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
| LOAD_ADDR FCARG2x, prop_info
Expand Down Expand Up @@ -12593,11 +12635,7 @@ static int zend_jit_incdec_obj(dasm_State **Dst,

if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) {
uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN;
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);

if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, prop_addr
}
zend_jit_addr var_addr = prop_addr;

if (use_prop_guard) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
Expand All @@ -12608,6 +12646,11 @@ static int zend_jit_incdec_obj(dasm_State **Dst,
}

if (var_info & MAY_BE_REF) {
may_throw = 1;
var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, prop_addr
}
| IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2, ZREG_TMP1
| GET_ZVAL_PTR FCARG1x, var_addr, TMP1
| ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)]
Expand Down Expand Up @@ -12648,6 +12691,10 @@ static int zend_jit_incdec_obj(dasm_State **Dst,
if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) {
| IF_NOT_ZVAL_TYPE var_addr, IS_LONG, >2, ZREG_TMP1
}
if (Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) {

This comment has been minimized.

Sign in to view

Copy link
@shqking

shqking Sep 28, 2021

Contributor

Hi. I think these lines should be put after line 12722.
See the corresponding update in JIT/x86 35ff71f#diff-c0fa9f6cbf84b02388699bafb28a11d7f69780bbf2b0e1bceea4cb99763c1328R13428

This comment has been minimized.

Sign in to view

Copy link
@dstogov

dstogov Sep 28, 2021

Author Member

You are right. Thanks. Fixed.
var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
| LOAD_ZVAL_ADDR FCARG1x, prop_addr
}
if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) {
if (opline->result_type != IS_UNUSED) {
| ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
Expand All @@ -12667,6 +12714,9 @@ static int zend_jit_incdec_obj(dasm_State **Dst,
|.cold_code
}
if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) {
if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
may_throw = 1;
}
if (var_info & MAY_BE_LONG) {
|2:
}
Expand Down Expand Up @@ -12718,6 +12768,7 @@ static int zend_jit_incdec_obj(dasm_State **Dst,
}

if (needs_slow_path) {
may_throw = 1;
|.cold_code
|7:
| SET_EX_OPLINE opline, REG0
Expand Down Expand Up @@ -12754,6 +12805,9 @@ static int zend_jit_incdec_obj(dasm_State **Dst,

|9:
if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) {
if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) {
may_throw = 1;
}
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2
}

Expand Down Expand Up @@ -12781,8 +12835,7 @@ static int zend_jit_assign_obj_op(dasm_State **Dst,
bool on_this,
bool delayed_fetch_this,
zend_class_entry *trace_ce,
uint8_t prop_type,
int may_throw)
uint8_t prop_type)
{
zval *member;
zend_string *name;
Expand All @@ -12792,6 +12845,7 @@ static int zend_jit_assign_obj_op(dasm_State **Dst,
zend_jit_addr prop_addr;
bool needs_slow_path = 0;
bool use_prop_guard = 0;
bool may_throw = 0;
binary_op_type binary_op = get_binary_op(opline->extended_value);

ZEND_ASSERT(opline->op2_type == IS_CONST);
Expand Down Expand Up @@ -12847,6 +12901,7 @@ static int zend_jit_assign_obj_op(dasm_State **Dst,
}
if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR))
&& (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
may_throw = 1;
| b >8
} else {
| b ->exception_handler
Expand Down Expand Up @@ -12936,6 +12991,8 @@ static int zend_jit_assign_obj_op(dasm_State **Dst,
if (ZEND_TYPE_IS_SET(prop_info->type)) {
uint32_t info = val_info;

may_throw = 1;

if (opline) {
| SET_EX_OPLINE opline, REG0
}
Expand Down Expand Up @@ -12983,9 +13040,6 @@ static int zend_jit_assign_obj_op(dasm_State **Dst,
uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN;
uint32_t var_def_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN;

var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0);
| LOAD_ZVAL_ADDR REG0, prop_addr

if (use_prop_guard) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
Expand All @@ -12995,6 +13049,9 @@ static int zend_jit_assign_obj_op(dasm_State **Dst,
}

if (var_info & MAY_BE_REF) {
may_throw = 1;
var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0);
| LOAD_ZVAL_ADDR REG0, prop_addr
| IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2, ZREG_TMP1
| GET_ZVAL_PTR FCARG1x, var_addr, TMP1
| ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)]
Expand All @@ -13019,7 +13076,14 @@ static int zend_jit_assign_obj_op(dasm_State **Dst,
case ZEND_ADD:
case ZEND_SUB:
case ZEND_MUL:
case ZEND_DIV:
if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if (opline->extended_value != ZEND_ADD ||
(var_info & MAY_BE_ANY) != MAY_BE_ARRAY ||
(val_info & MAY_BE_ANY) == MAY_BE_ARRAY) {
may_throw = 1;
}
}
if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, 0, var_addr, var_def_info, var_info,
1 /* may overflow */, 0)) {
return 0;
Expand All @@ -13028,9 +13092,42 @@ static int zend_jit_assign_obj_op(dasm_State **Dst,
case ZEND_BW_OR:
case ZEND_BW_AND:
case ZEND_BW_XOR:
may_throw = 1;
if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
(val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if ((var_info & MAY_BE_ANY) != MAY_BE_STRING ||
(val_info & MAY_BE_ANY) != MAY_BE_STRING) {
may_throw = 1;
}
}
goto long_math;
case ZEND_SL:
case ZEND_SR:
if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
(val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
may_throw = 1;
}
if ((opline+1)->op1_type != IS_CONST ||
Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG ||
Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) < 0) {
may_throw = 1;
}
goto long_math;
case ZEND_MOD:
if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if (opline->extended_value != ZEND_ADD ||
(var_info & MAY_BE_ANY) != MAY_BE_ARRAY ||
(val_info & MAY_BE_ANY) == MAY_BE_ARRAY) {
may_throw = 1;
}
}
if ((opline+1)->op1_type != IS_CONST ||
Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG ||
Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) == 0) {
may_throw = 1;
}
long_math:
if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value,
IS_CV, opline->op1, var_addr, var_info, NULL,
(opline+1)->op1_type, (opline+1)->op1, val_addr, val_info,
Expand All @@ -13040,6 +13137,7 @@ static int zend_jit_assign_obj_op(dasm_State **Dst,
}
break;
case ZEND_CONCAT:
may_throw = 1;
if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, var_addr,
0)) {
return 0;
Expand All @@ -13051,6 +13149,7 @@ static int zend_jit_assign_obj_op(dasm_State **Dst,
}

if (needs_slow_path) {
may_throw = 1;
|.cold_code
|7:
| SET_EX_OPLINE opline, REG0
Expand All @@ -13075,6 +13174,9 @@ static int zend_jit_assign_obj_op(dasm_State **Dst,

|9:
if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) {
if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) {
may_throw = 1;
}
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2
}

Expand Down

0 comments on commit 35ff71f

Please sign in to comment.