Skip to content

Commit

Permalink
accel/tcg: Add fast path for translator_ld*
Browse files Browse the repository at this point in the history 
Cache the translation from guest to host address, so we may
use direct loads when we hit on the primary translation page.

Look up the second translation page only once, during translation.
This obviates another lookup of the second page within tb_gen_code
after translation.

Fixes a bug in that plugin_insn_append should be passed the bytes
in the original memory order, not bswapped by pieces.

Acked-by: Ilya Leoshkevich <iii@linux.ibm.com>
Tested-by: Ilya Leoshkevich <iii@linux.ibm.com>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
  • Loading branch information
@rth7680
rth7680 committed Sep 1, 2022
1 parent 603fd5a commit 281db73
Show file tree
Hide file tree
Showing 3 changed files with 141 additions and 71 deletions.
23 changes: 8 additions & 15 deletions accel/tcg/translate-all.c
View file
Expand Up @@ -1385,8 +1385,7 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
{
CPUArchState *env = cpu->env_ptr;
TranslationBlock *tb, *existing_tb;
tb_page_addr_t phys_pc, phys_page2;
target_ulong virt_page2;
tb_page_addr_t phys_pc;
tcg_insn_unit *gen_code_buf;
int gen_code_size, search_size, max_insns;
#ifdef CONFIG_PROFILER
Expand Down Expand Up @@ -1429,6 +1428,8 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
tb->flags = flags;
tb->cflags = cflags;
tb->trace_vcpu_dstate = *cpu->trace_dstate;
tb->page_addr[0] = phys_pc;
tb->page_addr[1] = -1;
tcg_ctx->tb_cflags = cflags;
tb_overflow:

Expand Down Expand Up @@ -1622,13 +1623,11 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
}

/*
* If the TB is not associated with a physical RAM page then
* it must be a temporary one-insn TB, and we have nothing to do
* except fill in the page_addr[] fields. Return early before
* attempting to link to other TBs or add to the lookup table.
* If the TB is not associated with a physical RAM page then it must be
* a temporary one-insn TB, and we have nothing left to do. Return early
* before attempting to link to other TBs or add to the lookup table.
*/
if (phys_pc == -1) {
tb->page_addr[0] = tb->page_addr[1] = -1;
if (tb->page_addr[0] == -1) {
return tb;
}

Expand All @@ -1639,17 +1638,11 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
*/
tcg_tb_insert(tb);

/* check next page if needed */
virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
phys_page2 = -1;
if ((pc & TARGET_PAGE_MASK) != virt_page2) {
phys_page2 = get_page_addr_code(env, virt_page2);
}
/*
* No explicit memory barrier is required -- tb_link_page() makes the
* TB visible in a consistent state.
*/
existing_tb = tb_link_page(tb, phys_pc, phys_page2);
existing_tb = tb_link_page(tb, tb->page_addr[0], tb->page_addr[1]);
/* if the TB already exists, discard what we just translated */
if (unlikely(existing_tb != tb)) {
uintptr_t orig_aligned = (uintptr_t)gen_code_buf;
Expand Down
126 changes: 97 additions & 29 deletions accel/tcg/translator.c
View file
Expand Up @@ -42,15 +42,6 @@ bool translator_use_goto_tb(DisasContextBase *db, target_ulong dest)
return ((db->pc_first ^ dest) & TARGET_PAGE_MASK) == 0;
}

static inline void translator_page_protect(DisasContextBase *dcbase,
target_ulong pc)
{
#ifdef CONFIG_USER_ONLY
dcbase->page_protect_end = pc | ~TARGET_PAGE_MASK;
page_protect(pc);
#endif
}

void translator_loop(CPUState *cpu, TranslationBlock *tb, int max_insns,
target_ulong pc, void *host_pc,
const TranslatorOps *ops, DisasContextBase *db)
Expand All @@ -66,7 +57,12 @@ void translator_loop(CPUState *cpu, TranslationBlock *tb, int max_insns,
db->num_insns = 0;
db->max_insns = max_insns;
db->singlestep_enabled = cflags & CF_SINGLE_STEP;
translator_page_protect(db, db->pc_next);
db->host_addr[0] = host_pc;
db->host_addr[1] = NULL;

#ifdef CONFIG_USER_ONLY
page_protect(pc);
#endif

ops->init_disas_context(db, cpu);
tcg_debug_assert(db->is_jmp == DISAS_NEXT); /* no early exit */
Expand Down Expand Up @@ -151,31 +147,103 @@ void translator_loop(CPUState *cpu, TranslationBlock *tb, int max_insns,
#endif
}

static inline void translator_maybe_page_protect(DisasContextBase *dcbase,
target_ulong pc, size_t len)
static void *translator_access(CPUArchState *env, DisasContextBase *db,
target_ulong pc, size_t len)
{
#ifdef CONFIG_USER_ONLY
target_ulong end = pc + len - 1;
void *host;
target_ulong base, end;
TranslationBlock *tb;

tb = db->tb;

if (end > dcbase->page_protect_end) {
translator_page_protect(dcbase, end);
/* Use slow path if first page is MMIO. */
if (unlikely(tb->page_addr[0] == -1)) {
return NULL;
}

end = pc + len - 1;
if (likely(is_same_page(db, end))) {
host = db->host_addr[0];
base = db->pc_first;
} else {
host = db->host_addr[1];
base = TARGET_PAGE_ALIGN(db->pc_first);
if (host == NULL) {
tb->page_addr[1] =
get_page_addr_code_hostp(env, base, &db->host_addr[1]);
#ifdef CONFIG_USER_ONLY
page_protect(end);
#endif
/* We cannot handle MMIO as second page. */
assert(tb->page_addr[1] != -1);
host = db->host_addr[1];
}

/* Use slow path when crossing pages. */
if (is_same_page(db, pc)) {
return NULL;
}
}

tcg_debug_assert(pc >= base);
return host + (pc - base);
}

#define GEN_TRANSLATOR_LD(fullname, type, load_fn, swap_fn) \
type fullname ## _swap(CPUArchState *env, DisasContextBase *dcbase, \
abi_ptr pc, bool do_swap) \
{ \
translator_maybe_page_protect(dcbase, pc, sizeof(type)); \
type ret = load_fn(env, pc); \
if (do_swap) { \
ret = swap_fn(ret); \
} \
plugin_insn_append(pc, &ret, sizeof(ret)); \
return ret; \
uint8_t translator_ldub(CPUArchState *env, DisasContextBase *db, abi_ptr pc)
{
uint8_t ret;
void *p = translator_access(env, db, pc, sizeof(ret));

if (p) {
plugin_insn_append(pc, p, sizeof(ret));
return ldub_p(p);
}
ret = cpu_ldub_code(env, pc);
plugin_insn_append(pc, &ret, sizeof(ret));
return ret;
}

FOR_EACH_TRANSLATOR_LD(GEN_TRANSLATOR_LD)
uint16_t translator_lduw(CPUArchState *env, DisasContextBase *db, abi_ptr pc)
{
uint16_t ret, plug;
void *p = translator_access(env, db, pc, sizeof(ret));

#undef GEN_TRANSLATOR_LD
if (p) {
plugin_insn_append(pc, p, sizeof(ret));
return lduw_p(p);
}
ret = cpu_lduw_code(env, pc);
plug = tswap16(ret);
plugin_insn_append(pc, &plug, sizeof(ret));
return ret;
}

uint32_t translator_ldl(CPUArchState *env, DisasContextBase *db, abi_ptr pc)
{
uint32_t ret, plug;
void *p = translator_access(env, db, pc, sizeof(ret));

if (p) {
plugin_insn_append(pc, p, sizeof(ret));
return ldl_p(p);
}
ret = cpu_ldl_code(env, pc);
plug = tswap32(ret);
plugin_insn_append(pc, &plug, sizeof(ret));
return ret;
}

uint64_t translator_ldq(CPUArchState *env, DisasContextBase *db, abi_ptr pc)
{
uint64_t ret, plug;
void *p = translator_access(env, db, pc, sizeof(ret));

if (p) {
plugin_insn_append(pc, p, sizeof(ret));
return ldq_p(p);
}
ret = cpu_ldq_code(env, pc);
plug = tswap64(ret);
plugin_insn_append(pc, &plug, sizeof(ret));
return ret;
}
63 changes: 36 additions & 27 deletions include/exec/translator.h
View file
Expand Up @@ -81,24 +81,14 @@ typedef enum DisasJumpType {
* Architecture-agnostic disassembly context.
*/
typedef struct DisasContextBase {
const TranslationBlock *tb;
TranslationBlock *tb;
target_ulong pc_first;
target_ulong pc_next;
DisasJumpType is_jmp;
int num_insns;
int max_insns;
bool singlestep_enabled;
#ifdef CONFIG_USER_ONLY
/*
* Guest address of the last byte of the last protected page.
*
* Pages containing the translated instructions are made non-writable in
* order to achieve consistency in case another thread is modifying the
* code while translate_insn() fetches the instruction bytes piecemeal.
* Such writer threads are blocked on mmap_lock() in page_unprotect().
*/
target_ulong page_protect_end;
#endif
void *host_addr[2];
} DisasContextBase;

/**
Expand Down Expand Up @@ -183,24 +173,43 @@ bool translator_use_goto_tb(DisasContextBase *db, target_ulong dest);
* the relevant information at translation time.
*/

#define GEN_TRANSLATOR_LD(fullname, type, load_fn, swap_fn) \
type fullname ## _swap(CPUArchState *env, DisasContextBase *dcbase, \
abi_ptr pc, bool do_swap); \
static inline type fullname(CPUArchState *env, \
DisasContextBase *dcbase, abi_ptr pc) \
{ \
return fullname ## _swap(env, dcbase, pc, false); \
}
uint8_t translator_ldub(CPUArchState *env, DisasContextBase *db, abi_ptr pc);
uint16_t translator_lduw(CPUArchState *env, DisasContextBase *db, abi_ptr pc);
uint32_t translator_ldl(CPUArchState *env, DisasContextBase *db, abi_ptr pc);
uint64_t translator_ldq(CPUArchState *env, DisasContextBase *db, abi_ptr pc);

#define FOR_EACH_TRANSLATOR_LD(F) \
F(translator_ldub, uint8_t, cpu_ldub_code, /* no swap */) \
F(translator_lduw, uint16_t, cpu_lduw_code, bswap16) \
F(translator_ldl, uint32_t, cpu_ldl_code, bswap32) \
F(translator_ldq, uint64_t, cpu_ldq_code, bswap64)
static inline uint16_t
translator_lduw_swap(CPUArchState *env, DisasContextBase *db,
abi_ptr pc, bool do_swap)
{
uint16_t ret = translator_lduw(env, db, pc);
if (do_swap) {
ret = bswap16(ret);
}
return ret;
}

FOR_EACH_TRANSLATOR_LD(GEN_TRANSLATOR_LD)
static inline uint32_t
translator_ldl_swap(CPUArchState *env, DisasContextBase *db,
abi_ptr pc, bool do_swap)
{
uint32_t ret = translator_ldl(env, db, pc);
if (do_swap) {
ret = bswap32(ret);
}
return ret;
}

#undef GEN_TRANSLATOR_LD
static inline uint64_t
translator_ldq_swap(CPUArchState *env, DisasContextBase *db,
abi_ptr pc, bool do_swap)
{
uint64_t ret = translator_ldq_swap(env, db, pc, false);
if (do_swap) {
ret = bswap64(ret);
}
return ret;
}

/*
* Return whether addr is on the same page as where disassembly started.
Expand Down

0 comments on commit 281db73

Please sign in to comment.