Skip to content

Commit

Permalink
Properly patch jmp tables
Browse files Browse the repository at this point in the history
  • Loading branch information
@dstogov
dstogov committed Aug 17, 2020
1 parent 4522cbb commit 8202b4e
Show file tree
Hide file tree
Showing 4 changed files with 40 additions and 12 deletions.
2 changes: 1 addition & 1 deletion ext/opcache/jit/zend_jit.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2925,7 +2925,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op
case ZEND_SWITCH_LONG:
case ZEND_SWITCH_STRING:
case ZEND_MATCH:
if (!zend_jit_switch(&dasm_state, opline, op_array, ssa, NULL)) {
if (!zend_jit_switch(&dasm_state, opline, op_array, ssa, NULL, NULL)) {
goto jit_failure;
}
goto done;
Expand Down
1 change: 1 addition & 0 deletions ext/opcache/jit/zend_jit_internal.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,7 @@ typedef struct _zend_jit_trace_info {
uint32_t stack_map_size;
uint32_t flags; /* See ZEND_JIT_TRACE_... defines above */
uint32_t polymorphism; /* Counter of polymorphic calls */
uint32_t jmp_table_size;/* number of jmp_table slots */
const zend_op *opline; /* first opline */
const void *code_start; /* address of native code */
zend_jit_trace_exit_info *exit_info; /* info about side exits */
Expand Down
6 changes: 5 additions & 1 deletion ext/opcache/jit/zend_jit_trace.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4163,7 +4163,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
case ZEND_SWITCH_LONG:
case ZEND_SWITCH_STRING:
case ZEND_MATCH:
if (!zend_jit_switch(&dasm_state, opline, op_array, op_array_ssa, p+1)) {
if (!zend_jit_switch(&dasm_state, opline, op_array, op_array_ssa, p+1, &zend_jit_traces[ZEND_JIT_TRACE_NUM])) {
goto jit_failure;
}
goto done;
Expand Down Expand Up @@ -4827,6 +4827,7 @@ static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace
t->stack_map_size = 0;
t->flags = 0;
t->polymorphism = 0;
t->jmp_table_size = 0;
t->opline = trace_buffer[1].opline;
t->exit_info = exit_info;
t->stack_map = NULL;
Expand Down Expand Up @@ -5353,6 +5354,7 @@ static void zend_jit_blacklist_trace_exit(uint32_t trace_num, uint32_t exit_num)
zend_jit_link_side_trace(
zend_jit_traces[trace_num].code_start,
zend_jit_traces[trace_num].code_size,
zend_jit_traces[trace_num].jmp_table_size,
exit_num,
handler);
}
Expand Down Expand Up @@ -5421,6 +5423,7 @@ static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace
t->stack_map_size = 0;
t->flags = 0;
t->polymorphism = polymorphism;
t->jmp_table_size = 0;
t->opline = NULL;
t->exit_info = exit_info;
t->stack_map = NULL;
Expand Down Expand Up @@ -5464,6 +5467,7 @@ static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace
zend_jit_link_side_trace(
zend_jit_traces[parent_num].code_start,
zend_jit_traces[parent_num].code_size,
zend_jit_traces[parent_num].jmp_table_size,
exit_num,
handler);

Expand Down
43 changes: 33 additions & 10 deletions ext/opcache/jit/zend_jit_x86.dasc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ static size_t tsrm_tls_offset;
|.globals zend_lb
static void* dasm_labels[zend_lb_MAX];

|.section code, cold_code
|.section code, cold_code, jmp_table

#define IS_32BIT(addr) (((uintptr_t)(addr)) <= 0xffffffff)

Expand Down Expand Up @@ -3093,12 +3093,26 @@ mrm:
typedef ZEND_SET_ALIGNED(1, uint16_t unaligned_uint16_t);
typedef ZEND_SET_ALIGNED(1, int32_t unaligned_int32_t);

static int zend_jit_patch(const void *code, size_t size, const void *from_addr, const void *to_addr)
static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size, const void *from_addr, const void *to_addr)
{
int ret = 0;
uint8_t *p = (uint8_t*)code;
uint8_t *end = p + size - 5;
uint8_t *p, *end;

if (jmp_table_size) {
const void **jmp_slot = (const void **)((char*)code + size);

size -= jmp_table_size * sizeof(void*);
do {
jmp_slot--;
if (*jmp_slot == from_addr) {
*jmp_slot = to_addr;
ret++;
}
} while (--jmp_table_size);
}

p = (uint8_t*)code;
end = p + size - 5;
while (p < end) {
if ((*(unaligned_uint16_t*)p & 0xf0ff) == 0x800f && p + *(unaligned_int32_t*)(p+2) == (uint8_t*)from_addr - 6) {
*(unaligned_int32_t*)(p+2) = ((uint8_t*)to_addr - (p + 6));
Expand All @@ -3115,9 +3129,9 @@ static int zend_jit_patch(const void *code, size_t size, const void *from_addr,
return ret;
}

static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t exit_num, const void *addr)
static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t jmp_table_size, uint32_t exit_num, const void *addr)
{
return zend_jit_patch(code, size, zend_jit_trace_get_exit_addr(exit_num), addr);
return zend_jit_patch(code, size, jmp_table_size, zend_jit_trace_get_exit_addr(exit_num), addr);
}

static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t)
Expand Down Expand Up @@ -11782,7 +11796,7 @@ static int zend_jit_fetch_this(dasm_State **Dst, const zend_op *opline, const ze
return 1;
}

static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, zend_jit_trace_rec *trace)
static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, zend_jit_trace_rec *trace, zend_jit_trace_info *trace_info)
{
HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2));
const zend_op *next_opline = NULL;
Expand Down Expand Up @@ -11920,9 +11934,12 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o
|.else
| jmp aword [FCARG2a * 4 + >4]
|.endif
|.cold_code
|.jmp_table
|.align aword
|4:
if (trace_info) {
trace_info->jmp_table_size += count;
}
p = jumptable->arData;
do {
if (Z_TYPE(p->val) == IS_UNDEF) {
Expand Down Expand Up @@ -11981,9 +11998,12 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o
|.else
| jmp aword [r0 + >4]
|.endif
|.cold_code
|.jmp_table
|.align aword
|4:
if (trace_info) {
trace_info->jmp_table_size += zend_hash_num_elements(jumptable);
}
ZEND_HASH_FOREACH_VAL(jumptable, val) {
target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(val));
if (!next_opline) {
Expand Down Expand Up @@ -12063,9 +12083,12 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o
|.else
| jmp aword [r0 + >4]
|.endif
|.cold_code
|.jmp_table
|.align aword
|4:
if (trace_info) {
trace_info->jmp_table_size += zend_hash_num_elements(jumptable);
}
ZEND_HASH_FOREACH_VAL(jumptable, val) {
target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(val));
if (!next_opline) {
Expand Down

0 comments on commit 8202b4e

Please sign in to comment.