Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
tcg/aarch64: Introduce prepare_host_addr 
Merge tcg_out_tlb_load, add_qemu_ldst_label, tcg_out_test_alignment,
and some code that lived in both tcg_out_qemu_ld and tcg_out_qemu_st
into one function that returns HostAddress and TCGLabelQemuLdst structures.

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
  • Loading branch information
@rth7680
rth7680 committed May 11, 2023
1 parent 2227037 commit 4bcc264
Showing 1 changed file with 100 additions and 147 deletions.
247 changes: 100 additions & 147 deletions tcg/aarch64/tcg-target.c.inc
View file
Expand Up @@ -1667,50 +1667,71 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
tcg_out_goto(s, lb->raddr);
return true;
}

static void add_qemu_ldst_label(TCGContext *s, bool is_ld, MemOpIdx oi,
TCGType ext, TCGReg data_reg, TCGReg addr_reg,
tcg_insn_unit *raddr, tcg_insn_unit *label_ptr)
#else
static bool tcg_out_fail_alignment(TCGContext *s, TCGLabelQemuLdst *l)
{
TCGLabelQemuLdst *label = new_ldst_label(s);
if (!reloc_pc19(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) {
return false;
}

tcg_out_mov(s, TCG_TYPE_TL, TCG_REG_X1, l->addrlo_reg);
tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_X0, TCG_AREG0);

label->is_ld = is_ld;
label->oi = oi;
label->type = ext;
label->datalo_reg = data_reg;
label->addrlo_reg = addr_reg;
label->raddr = tcg_splitwx_to_rx(raddr);
label->label_ptr[0] = label_ptr;
/* "Tail call" to the helper, with the return address back inline. */
tcg_out_adr(s, TCG_REG_LR, l->raddr);
tcg_out_goto_long(s, (const void *)(l->is_ld ? helper_unaligned_ld
: helper_unaligned_st));
return true;
}

/* We expect to use a 7-bit scaled negative offset from ENV. */
QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0);
QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -512);
static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
{
return tcg_out_fail_alignment(s, l);
}

/* These offsets are built into the LDP below. */
QEMU_BUILD_BUG_ON(offsetof(CPUTLBDescFast, mask) != 0);
QEMU_BUILD_BUG_ON(offsetof(CPUTLBDescFast, table) != 8);
static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
{
return tcg_out_fail_alignment(s, l);
}
#endif /* CONFIG_SOFTMMU */

/* Load and compare a TLB entry, emitting the conditional jump to the
slow path for the failure case, which will be patched later when finalizing
the slow path. Generated code returns the host addend in X1,
clobbers X0,X2,X3,TMP. */
static void tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg, MemOp opc,
tcg_insn_unit **label_ptr, int mem_index,
bool is_read)
/*
* For softmmu, perform the TLB load and compare.
* For useronly, perform any required alignment tests.
* In both cases, return a TCGLabelQemuLdst structure if the slow path
* is required and fill in @h with the host address for the fast path.
*/
static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h,
TCGReg addr_reg, MemOpIdx oi,
bool is_ld)
{
TCGType addr_type = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32;
TCGLabelQemuLdst *ldst = NULL;
MemOp opc = get_memop(oi);
unsigned a_bits = get_alignment_bits(opc);
unsigned s_bits = opc & MO_SIZE;
unsigned a_mask = (1u << a_bits) - 1;

#ifdef CONFIG_SOFTMMU
unsigned s_bits = opc & MO_SIZE;
unsigned s_mask = (1u << s_bits) - 1;
unsigned mem_index = get_mmuidx(oi);
TCGReg x3;
TCGType mask_type;
uint64_t compare_mask;

ldst = new_ldst_label(s);
ldst->is_ld = is_ld;
ldst->oi = oi;
ldst->addrlo_reg = addr_reg;

mask_type = (TARGET_PAGE_BITS + CPU_TLB_DYN_MAX_BITS > 32
? TCG_TYPE_I64 : TCG_TYPE_I32);

/* Load env_tlb(env)->f[mmu_idx].{mask,table} into {x0,x1}. */
QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0);
QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -512);
QEMU_BUILD_BUG_ON(offsetof(CPUTLBDescFast, mask) != 0);
QEMU_BUILD_BUG_ON(offsetof(CPUTLBDescFast, table) != 8);
tcg_out_insn(s, 3314, LDP, TCG_REG_X0, TCG_REG_X1, TCG_AREG0,
TLB_MASK_TABLE_OFS(mem_index), 1, 0);

Expand All @@ -1723,15 +1744,17 @@ static void tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg, MemOp opc,
tcg_out_insn(s, 3502, ADD, 1, TCG_REG_X1, TCG_REG_X1, TCG_REG_X0);

/* Load the tlb comparator into X0, and the fast path addend into X1. */
tcg_out_ld(s, TCG_TYPE_TL, TCG_REG_X0, TCG_REG_X1, is_read
? offsetof(CPUTLBEntry, addr_read)
: offsetof(CPUTLBEntry, addr_write));
tcg_out_ld(s, TCG_TYPE_TL, TCG_REG_X0, TCG_REG_X1,
is_ld ? offsetof(CPUTLBEntry, addr_read)
: offsetof(CPUTLBEntry, addr_write));
tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_X1, TCG_REG_X1,
offsetof(CPUTLBEntry, addend));

/* For aligned accesses, we check the first byte and include the alignment
bits within the address. For unaligned access, we check that we don't
cross pages using the address of the last byte of the access. */
/*
* For aligned accesses, we check the first byte and include the alignment
* bits within the address. For unaligned access, we check that we don't
* cross pages using the address of the last byte of the access.
*/
if (a_bits >= s_bits) {
x3 = addr_reg;
} else {
Expand All @@ -1749,57 +1772,47 @@ static void tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg, MemOp opc,
tcg_out_cmp(s, TARGET_LONG_BITS == 64, TCG_REG_X0, TCG_REG_X3, 0);

/* If not equal, we jump to the slow path. */
*label_ptr = s->code_ptr;
ldst->label_ptr[0] = s->code_ptr;
tcg_out_insn(s, 3202, B_C, TCG_COND_NE, 0);
}

*h = (HostAddress){
.base = TCG_REG_X1,
.index = addr_reg,
.index_ext = addr_type
};
#else
static void tcg_out_test_alignment(TCGContext *s, bool is_ld, TCGReg addr_reg,
unsigned a_bits)
{
unsigned a_mask = (1 << a_bits) - 1;
TCGLabelQemuLdst *label = new_ldst_label(s);

label->is_ld = is_ld;
label->addrlo_reg = addr_reg;
if (a_mask) {
ldst = new_ldst_label(s);

/* tst addr, #mask */
tcg_out_logicali(s, I3404_ANDSI, 0, TCG_REG_XZR, addr_reg, a_mask);
ldst->is_ld = is_ld;
ldst->oi = oi;
ldst->addrlo_reg = addr_reg;

label->label_ptr[0] = s->code_ptr;

/* b.ne slow_path */
tcg_out_insn(s, 3202, B_C, TCG_COND_NE, 0);
/* tst addr, #mask */
tcg_out_logicali(s, I3404_ANDSI, 0, TCG_REG_XZR, addr_reg, a_mask);

label->raddr = tcg_splitwx_to_rx(s->code_ptr);
}

static bool tcg_out_fail_alignment(TCGContext *s, TCGLabelQemuLdst *l)
{
if (!reloc_pc19(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) {
return false;
/* b.ne slow_path */
ldst->label_ptr[0] = s->code_ptr;
tcg_out_insn(s, 3202, B_C, TCG_COND_NE, 0);
}

tcg_out_mov(s, TCG_TYPE_TL, TCG_REG_X1, l->addrlo_reg);
tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_X0, TCG_AREG0);

/* "Tail call" to the helper, with the return address back inline. */
tcg_out_adr(s, TCG_REG_LR, l->raddr);
tcg_out_goto_long(s, (const void *)(l->is_ld ? helper_unaligned_ld
: helper_unaligned_st));
return true;
}

static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
{
return tcg_out_fail_alignment(s, l);
}
if (USE_GUEST_BASE) {
*h = (HostAddress){
.base = TCG_REG_GUEST_BASE,
.index = addr_reg,
.index_ext = addr_type
};
} else {
*h = (HostAddress){
.base = addr_reg,
.index = TCG_REG_XZR,
.index_ext = TCG_TYPE_I64
};
}
#endif

static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
{
return tcg_out_fail_alignment(s, l);
return ldst;
}
#endif /* CONFIG_SOFTMMU */

static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp memop, TCGType ext,
TCGReg data_r, HostAddress h)
Expand Down Expand Up @@ -1857,93 +1870,33 @@ static void tcg_out_qemu_st_direct(TCGContext *s, MemOp memop,
static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg,
MemOpIdx oi, TCGType data_type)
{
MemOp memop = get_memop(oi);
TCGType addr_type = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32;
TCGLabelQemuLdst *ldst;
HostAddress h;

/* Byte swapping is left to middle-end expansion. */
tcg_debug_assert((memop & MO_BSWAP) == 0);
ldst = prepare_host_addr(s, &h, addr_reg, oi, true);
tcg_out_qemu_ld_direct(s, get_memop(oi), data_type, data_reg, h);

#ifdef CONFIG_SOFTMMU
tcg_insn_unit *label_ptr;

tcg_out_tlb_read(s, addr_reg, memop, &label_ptr, get_mmuidx(oi), 1);

h = (HostAddress){
.base = TCG_REG_X1,
.index = addr_reg,
.index_ext = addr_type
};
tcg_out_qemu_ld_direct(s, memop, data_type, data_reg, h);

add_qemu_ldst_label(s, true, oi, data_type, data_reg, addr_reg,
s->code_ptr, label_ptr);
#else /* !CONFIG_SOFTMMU */
unsigned a_bits = get_alignment_bits(memop);
if (a_bits) {
tcg_out_test_alignment(s, true, addr_reg, a_bits);
}
if (USE_GUEST_BASE) {
h = (HostAddress){
.base = TCG_REG_GUEST_BASE,
.index = addr_reg,
.index_ext = addr_type
};
} else {
h = (HostAddress){
.base = addr_reg,
.index = TCG_REG_XZR,
.index_ext = TCG_TYPE_I64
};
if (ldst) {
ldst->type = data_type;
ldst->datalo_reg = data_reg;
ldst->raddr = tcg_splitwx_to_rx(s->code_ptr);
}
tcg_out_qemu_ld_direct(s, memop, data_type, data_reg, h);
#endif /* CONFIG_SOFTMMU */
}

static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg,
MemOpIdx oi, TCGType data_type)
{
MemOp memop = get_memop(oi);
TCGType addr_type = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32;
TCGLabelQemuLdst *ldst;
HostAddress h;

/* Byte swapping is left to middle-end expansion. */
tcg_debug_assert((memop & MO_BSWAP) == 0);

#ifdef CONFIG_SOFTMMU
tcg_insn_unit *label_ptr;

tcg_out_tlb_read(s, addr_reg, memop, &label_ptr, get_mmuidx(oi), 0);

h = (HostAddress){
.base = TCG_REG_X1,
.index = addr_reg,
.index_ext = addr_type
};
tcg_out_qemu_st_direct(s, memop, data_reg, h);
ldst = prepare_host_addr(s, &h, addr_reg, oi, false);
tcg_out_qemu_st_direct(s, get_memop(oi), data_reg, h);

add_qemu_ldst_label(s, false, oi, data_type, data_reg, addr_reg,
s->code_ptr, label_ptr);
#else /* !CONFIG_SOFTMMU */
unsigned a_bits = get_alignment_bits(memop);
if (a_bits) {
tcg_out_test_alignment(s, false, addr_reg, a_bits);
if (ldst) {
ldst->type = data_type;
ldst->datalo_reg = data_reg;
ldst->raddr = tcg_splitwx_to_rx(s->code_ptr);
}
if (USE_GUEST_BASE) {
h = (HostAddress){
.base = TCG_REG_GUEST_BASE,
.index = addr_reg,
.index_ext = addr_type
};
} else {
h = (HostAddress){
.base = addr_reg,
.index = TCG_REG_XZR,
.index_ext = TCG_TYPE_I64
};
}
tcg_out_qemu_st_direct(s, memop, data_reg, h);
#endif /* CONFIG_SOFTMMU */
}

static const tcg_insn_unit *tb_ret_addr;
Expand Down

0 comments on commit 4bcc264

Please sign in to comment.