Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
accel/tcg: Honor atomicity of loads 
Create ldst_atomicity.c.inc.

Not required for user-only code loads, because we've ensured that
the page is read-only before beginning to translate code.

Reviewed-by: Peter Maydell <peter.maydell@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
  • Loading branch information
@rth7680
rth7680 committed May 15, 2023
1 parent d385d08 commit 2a37ccd
Show file tree
Hide file tree
Showing 3 changed files with 716 additions and 51 deletions.
175 changes: 136 additions & 39 deletions accel/tcg/cputlb.c
View file
Expand Up @@ -1668,6 +1668,9 @@ tb_page_addr_t get_page_addr_code_hostp(CPUArchState *env, target_ulong addr,
return qemu_ram_addr_from_host_nofail(p);
}

/* Load/store with atomicity primitives. */
#include "ldst_atomicity.c.inc"

#ifdef CONFIG_PLUGIN
/*
* Perform a TLB lookup and populate the qemu_plugin_hwaddr structure.
Expand Down Expand Up @@ -2035,35 +2038,7 @@ static void validate_memop(MemOpIdx oi, MemOp expected)
* specifically for reading instructions from system memory. It is
* called by the translation loop and in some helpers where the code
* is disassembled. It shouldn't be called directly by guest code.
*/

typedef uint64_t FullLoadHelper(CPUArchState *env, target_ulong addr,
MemOpIdx oi, uintptr_t retaddr);

static inline uint64_t QEMU_ALWAYS_INLINE
load_memop(const void *haddr, MemOp op)
{
switch (op) {
case MO_UB:
return ldub_p(haddr);
case MO_BEUW:
return lduw_be_p(haddr);
case MO_LEUW:
return lduw_le_p(haddr);
case MO_BEUL:
return (uint32_t)ldl_be_p(haddr);
case MO_LEUL:
return (uint32_t)ldl_le_p(haddr);
case MO_BEUQ:
return ldq_be_p(haddr);
case MO_LEUQ:
return ldq_le_p(haddr);
default:
qemu_build_not_reached();
}
}

/*
*
* For the benefit of TCG generated code, we want to avoid the
* complication of ABI-specific return type promotion and always
* return a value extended to the register size of the host. This is
Expand Down Expand Up @@ -2119,17 +2094,139 @@ static uint64_t do_ld_bytes_beN(MMULookupPageData *p, uint64_t ret_be)
return ret_be;
}

/**
* do_ld_parts_beN
* @p: translation parameters
* @ret_be: accumulated data
*
* As do_ld_bytes_beN, but atomically on each aligned part.
*/
static uint64_t do_ld_parts_beN(MMULookupPageData *p, uint64_t ret_be)
{
void *haddr = p->haddr;
int size = p->size;

do {
uint64_t x;
int n;

/*
* Find minimum of alignment and size.
* This is slightly stronger than required by MO_ATOM_SUBALIGN, which
* would have only checked the low bits of addr|size once at the start,
* but is just as easy.
*/
switch (((uintptr_t)haddr | size) & 7) {
case 4:
x = cpu_to_be32(load_atomic4(haddr));
ret_be = (ret_be << 32) | x;
n = 4;
break;
case 2:
case 6:
x = cpu_to_be16(load_atomic2(haddr));
ret_be = (ret_be << 16) | x;
n = 2;
break;
default:
x = *(uint8_t *)haddr;
ret_be = (ret_be << 8) | x;
n = 1;
break;
case 0:
g_assert_not_reached();
}
haddr += n;
size -= n;
} while (size != 0);
return ret_be;
}

/**
* do_ld_parts_be4
* @p: translation parameters
* @ret_be: accumulated data
*
* As do_ld_bytes_beN, but with one atomic load.
* Four aligned bytes are guaranteed to cover the load.
*/
static uint64_t do_ld_whole_be4(MMULookupPageData *p, uint64_t ret_be)
{
int o = p->addr & 3;
uint32_t x = load_atomic4(p->haddr - o);

x = cpu_to_be32(x);
x <<= o * 8;
x >>= (4 - p->size) * 8;
return (ret_be << (p->size * 8)) | x;
}

/**
* do_ld_parts_be8
* @p: translation parameters
* @ret_be: accumulated data
*
* As do_ld_bytes_beN, but with one atomic load.
* Eight aligned bytes are guaranteed to cover the load.
*/
static uint64_t do_ld_whole_be8(CPUArchState *env, uintptr_t ra,
MMULookupPageData *p, uint64_t ret_be)
{
int o = p->addr & 7;
uint64_t x = load_atomic8_or_exit(env, ra, p->haddr - o);

x = cpu_to_be64(x);
x <<= o * 8;
x >>= (8 - p->size) * 8;
return (ret_be << (p->size * 8)) | x;
}

/*
* Wrapper for the above.
*/
static uint64_t do_ld_beN(CPUArchState *env, MMULookupPageData *p,
uint64_t ret_be, int mmu_idx,
MMUAccessType type, uintptr_t ra)
uint64_t ret_be, int mmu_idx, MMUAccessType type,
MemOp mop, uintptr_t ra)
{
MemOp atom;
unsigned tmp, half_size;

if (unlikely(p->flags & TLB_MMIO)) {
return do_ld_mmio_beN(env, p, ret_be, mmu_idx, type, ra);
} else {
}

/*
* It is a given that we cross a page and therefore there is no
* atomicity for the load as a whole, but subobjects may need attention.
*/
atom = mop & MO_ATOM_MASK;
switch (atom) {
case MO_ATOM_SUBALIGN:
return do_ld_parts_beN(p, ret_be);

case MO_ATOM_IFALIGN_PAIR:
case MO_ATOM_WITHIN16_PAIR:
tmp = mop & MO_SIZE;
tmp = tmp ? tmp - 1 : 0;
half_size = 1 << tmp;
if (atom == MO_ATOM_IFALIGN_PAIR
? p->size == half_size
: p->size >= half_size) {
if (!HAVE_al8_fast && p->size < 4) {
return do_ld_whole_be4(p, ret_be);
} else {
return do_ld_whole_be8(env, ra, p, ret_be);
}
}
/* fall through */

case MO_ATOM_IFALIGN:
case MO_ATOM_WITHIN16:
case MO_ATOM_NONE:
return do_ld_bytes_beN(p, ret_be);

default:
g_assert_not_reached();
}
}

Expand All @@ -2153,7 +2250,7 @@ static uint16_t do_ld_2(CPUArchState *env, MMULookupPageData *p, int mmu_idx,
}

/* Perform the load host endian, then swap if necessary. */
ret = load_memop(p->haddr, MO_UW);
ret = load_atom_2(env, ra, p->haddr, memop);
if (memop & MO_BSWAP) {
ret = bswap16(ret);
}
Expand All @@ -2170,7 +2267,7 @@ static uint32_t do_ld_4(CPUArchState *env, MMULookupPageData *p, int mmu_idx,
}

/* Perform the load host endian. */
ret = load_memop(p->haddr, MO_UL);
ret = load_atom_4(env, ra, p->haddr, memop);
if (memop & MO_BSWAP) {
ret = bswap32(ret);
}
Expand All @@ -2187,7 +2284,7 @@ static uint64_t do_ld_8(CPUArchState *env, MMULookupPageData *p, int mmu_idx,
}

/* Perform the load host endian. */
ret = load_memop(p->haddr, MO_UQ);
ret = load_atom_8(env, ra, p->haddr, memop);
if (memop & MO_BSWAP) {
ret = bswap64(ret);
}
Expand Down Expand Up @@ -2263,8 +2360,8 @@ static uint32_t do_ld4_mmu(CPUArchState *env, target_ulong addr, MemOpIdx oi,
return do_ld_4(env, &l.page[0], l.mmu_idx, access_type, l.memop, ra);
}

ret = do_ld_beN(env, &l.page[0], 0, l.mmu_idx, access_type, ra);
ret = do_ld_beN(env, &l.page[1], ret, l.mmu_idx, access_type, ra);
ret = do_ld_beN(env, &l.page[0], 0, l.mmu_idx, access_type, l.memop, ra);
ret = do_ld_beN(env, &l.page[1], ret, l.mmu_idx, access_type, l.memop, ra);
if ((l.memop & MO_BSWAP) == MO_LE) {
ret = bswap32(ret);
}
Expand Down Expand Up @@ -2297,8 +2394,8 @@ static uint64_t do_ld8_mmu(CPUArchState *env, target_ulong addr, MemOpIdx oi,
return do_ld_8(env, &l.page[0], l.mmu_idx, access_type, l.memop, ra);
}

ret = do_ld_beN(env, &l.page[0], 0, l.mmu_idx, access_type, ra);
ret = do_ld_beN(env, &l.page[1], ret, l.mmu_idx, access_type, ra);
ret = do_ld_beN(env, &l.page[0], 0, l.mmu_idx, access_type, l.memop, ra);
ret = do_ld_beN(env, &l.page[1], ret, l.mmu_idx, access_type, l.memop, ra);
if ((l.memop & MO_BSWAP) == MO_LE) {
ret = bswap64(ret);
}
Expand Down

0 comments on commit 2a37ccd

Please sign in to comment.