Skip to content

Commit

Permalink
target/arm: Use tcg_gen_gvec_5_ptr for sve FMLA/FCMLA
Browse files Browse the repository at this point in the history 
Now that we can pass 7 parameters, do not encode register
operands within simd_data.

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Reviewed-by: Taylor Simpson <tsimpson@quicinc.com>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
Message-id: 20200507172352.15418-2-richard.henderson@linaro.org
Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
  • Loading branch information
@rth7680 @pm215
rth7680 authored and pm215 committed May 11, 2020
1 parent 4758567 commit 08975da
Show file tree
Hide file tree
Showing 3 changed files with 118 additions and 162 deletions.
53 changes: 34 additions & 19 deletions target/arm/helper-sve.h
View file
Expand Up @@ -1099,25 +1099,40 @@ DEF_HELPER_FLAGS_6(sve_fcadd_s, TCG_CALL_NO_RWG,
DEF_HELPER_FLAGS_6(sve_fcadd_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)

DEF_HELPER_FLAGS_3(sve_fmla_zpzzz_h, TCG_CALL_NO_RWG, void, env, ptr, i32)
DEF_HELPER_FLAGS_3(sve_fmla_zpzzz_s, TCG_CALL_NO_RWG, void, env, ptr, i32)
DEF_HELPER_FLAGS_3(sve_fmla_zpzzz_d, TCG_CALL_NO_RWG, void, env, ptr, i32)

DEF_HELPER_FLAGS_3(sve_fmls_zpzzz_h, TCG_CALL_NO_RWG, void, env, ptr, i32)
DEF_HELPER_FLAGS_3(sve_fmls_zpzzz_s, TCG_CALL_NO_RWG, void, env, ptr, i32)
DEF_HELPER_FLAGS_3(sve_fmls_zpzzz_d, TCG_CALL_NO_RWG, void, env, ptr, i32)

DEF_HELPER_FLAGS_3(sve_fnmla_zpzzz_h, TCG_CALL_NO_RWG, void, env, ptr, i32)
DEF_HELPER_FLAGS_3(sve_fnmla_zpzzz_s, TCG_CALL_NO_RWG, void, env, ptr, i32)
DEF_HELPER_FLAGS_3(sve_fnmla_zpzzz_d, TCG_CALL_NO_RWG, void, env, ptr, i32)

DEF_HELPER_FLAGS_3(sve_fnmls_zpzzz_h, TCG_CALL_NO_RWG, void, env, ptr, i32)
DEF_HELPER_FLAGS_3(sve_fnmls_zpzzz_s, TCG_CALL_NO_RWG, void, env, ptr, i32)
DEF_HELPER_FLAGS_3(sve_fnmls_zpzzz_d, TCG_CALL_NO_RWG, void, env, ptr, i32)

DEF_HELPER_FLAGS_3(sve_fcmla_zpzzz_h, TCG_CALL_NO_RWG, void, env, ptr, i32)
DEF_HELPER_FLAGS_3(sve_fcmla_zpzzz_s, TCG_CALL_NO_RWG, void, env, ptr, i32)
DEF_HELPER_FLAGS_3(sve_fcmla_zpzzz_d, TCG_CALL_NO_RWG, void, env, ptr, i32)
DEF_HELPER_FLAGS_7(sve_fmla_zpzzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_7(sve_fmla_zpzzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_7(sve_fmla_zpzzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, ptr, i32)

DEF_HELPER_FLAGS_7(sve_fmls_zpzzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_7(sve_fmls_zpzzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_7(sve_fmls_zpzzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, ptr, i32)

DEF_HELPER_FLAGS_7(sve_fnmla_zpzzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_7(sve_fnmla_zpzzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_7(sve_fnmla_zpzzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, ptr, i32)

DEF_HELPER_FLAGS_7(sve_fnmls_zpzzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_7(sve_fnmls_zpzzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_7(sve_fnmls_zpzzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, ptr, i32)

DEF_HELPER_FLAGS_7(sve_fcmla_zpzzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_7(sve_fcmla_zpzzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_7(sve_fcmla_zpzzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, ptr, i32)

DEF_HELPER_FLAGS_5(sve_ftmad_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve_ftmad_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
Expand Down
157 changes: 60 additions & 97 deletions target/arm/sve_helper.c
View file
Expand Up @@ -3372,23 +3372,11 @@ DO_ZPZ_FP(sve_ucvt_dd, uint64_t, , uint64_to_float64)

#undef DO_ZPZ_FP

/* 4-operand predicated multiply-add. This requires 7 operands to pass
* "properly", so we need to encode some of the registers into DESC.
*/
QEMU_BUILD_BUG_ON(SIMD_DATA_SHIFT + 20 > 32);

static void do_fmla_zpzzz_h(CPUARMState *env, void *vg, uint32_t desc,
static void do_fmla_zpzzz_h(void *vd, void *vn, void *vm, void *va, void *vg,
float_status *status, uint32_t desc,
uint16_t neg1, uint16_t neg3)
{
intptr_t i = simd_oprsz(desc);
unsigned rd = extract32(desc, SIMD_DATA_SHIFT, 5);
unsigned rn = extract32(desc, SIMD_DATA_SHIFT + 5, 5);
unsigned rm = extract32(desc, SIMD_DATA_SHIFT + 10, 5);
unsigned ra = extract32(desc, SIMD_DATA_SHIFT + 15, 5);
void *vd = &env->vfp.zregs[rd];
void *vn = &env->vfp.zregs[rn];
void *vm = &env->vfp.zregs[rm];
void *va = &env->vfp.zregs[ra];
uint64_t *g = vg;

do {
Expand All @@ -3401,45 +3389,42 @@ static void do_fmla_zpzzz_h(CPUARMState *env, void *vg, uint32_t desc,
e1 = *(uint16_t *)(vn + H1_2(i)) ^ neg1;
e2 = *(uint16_t *)(vm + H1_2(i));
e3 = *(uint16_t *)(va + H1_2(i)) ^ neg3;
r = float16_muladd(e1, e2, e3, 0, &env->vfp.fp_status_f16);
r = float16_muladd(e1, e2, e3, 0, status);
*(uint16_t *)(vd + H1_2(i)) = r;
}
} while (i & 63);
} while (i != 0);
}

void HELPER(sve_fmla_zpzzz_h)(CPUARMState *env, void *vg, uint32_t desc)
void HELPER(sve_fmla_zpzzz_h)(void *vd, void *vn, void *vm, void *va,
void *vg, void *status, uint32_t desc)
{
do_fmla_zpzzz_h(env, vg, desc, 0, 0);
do_fmla_zpzzz_h(vd, vn, vm, va, vg, status, desc, 0, 0);
}

void HELPER(sve_fmls_zpzzz_h)(CPUARMState *env, void *vg, uint32_t desc)
void HELPER(sve_fmls_zpzzz_h)(void *vd, void *vn, void *vm, void *va,
void *vg, void *status, uint32_t desc)
{
do_fmla_zpzzz_h(env, vg, desc, 0x8000, 0);
do_fmla_zpzzz_h(vd, vn, vm, va, vg, status, desc, 0x8000, 0);
}

void HELPER(sve_fnmla_zpzzz_h)(CPUARMState *env, void *vg, uint32_t desc)
void HELPER(sve_fnmla_zpzzz_h)(void *vd, void *vn, void *vm, void *va,
void *vg, void *status, uint32_t desc)
{
do_fmla_zpzzz_h(env, vg, desc, 0x8000, 0x8000);
do_fmla_zpzzz_h(vd, vn, vm, va, vg, status, desc, 0x8000, 0x8000);
}

void HELPER(sve_fnmls_zpzzz_h)(CPUARMState *env, void *vg, uint32_t desc)
void HELPER(sve_fnmls_zpzzz_h)(void *vd, void *vn, void *vm, void *va,
void *vg, void *status, uint32_t desc)
{
do_fmla_zpzzz_h(env, vg, desc, 0, 0x8000);
do_fmla_zpzzz_h(vd, vn, vm, va, vg, status, desc, 0, 0x8000);
}

static void do_fmla_zpzzz_s(CPUARMState *env, void *vg, uint32_t desc,
static void do_fmla_zpzzz_s(void *vd, void *vn, void *vm, void *va, void *vg,
float_status *status, uint32_t desc,
uint32_t neg1, uint32_t neg3)
{
intptr_t i = simd_oprsz(desc);
unsigned rd = extract32(desc, SIMD_DATA_SHIFT, 5);
unsigned rn = extract32(desc, SIMD_DATA_SHIFT + 5, 5);
unsigned rm = extract32(desc, SIMD_DATA_SHIFT + 10, 5);
unsigned ra = extract32(desc, SIMD_DATA_SHIFT + 15, 5);
void *vd = &env->vfp.zregs[rd];
void *vn = &env->vfp.zregs[rn];
void *vm = &env->vfp.zregs[rm];
void *va = &env->vfp.zregs[ra];
uint64_t *g = vg;

do {
Expand All @@ -3452,45 +3437,42 @@ static void do_fmla_zpzzz_s(CPUARMState *env, void *vg, uint32_t desc,
e1 = *(uint32_t *)(vn + H1_4(i)) ^ neg1;
e2 = *(uint32_t *)(vm + H1_4(i));
e3 = *(uint32_t *)(va + H1_4(i)) ^ neg3;
r = float32_muladd(e1, e2, e3, 0, &env->vfp.fp_status);
r = float32_muladd(e1, e2, e3, 0, status);
*(uint32_t *)(vd + H1_4(i)) = r;
}
} while (i & 63);
} while (i != 0);
}

void HELPER(sve_fmla_zpzzz_s)(CPUARMState *env, void *vg, uint32_t desc)
void HELPER(sve_fmla_zpzzz_s)(void *vd, void *vn, void *vm, void *va,
void *vg, void *status, uint32_t desc)
{
do_fmla_zpzzz_s(env, vg, desc, 0, 0);
do_fmla_zpzzz_s(vd, vn, vm, va, vg, status, desc, 0, 0);
}

void HELPER(sve_fmls_zpzzz_s)(CPUARMState *env, void *vg, uint32_t desc)
void HELPER(sve_fmls_zpzzz_s)(void *vd, void *vn, void *vm, void *va,
void *vg, void *status, uint32_t desc)
{
do_fmla_zpzzz_s(env, vg, desc, 0x80000000, 0);
do_fmla_zpzzz_s(vd, vn, vm, va, vg, status, desc, 0x80000000, 0);
}

void HELPER(sve_fnmla_zpzzz_s)(CPUARMState *env, void *vg, uint32_t desc)
void HELPER(sve_fnmla_zpzzz_s)(void *vd, void *vn, void *vm, void *va,
void *vg, void *status, uint32_t desc)
{
do_fmla_zpzzz_s(env, vg, desc, 0x80000000, 0x80000000);
do_fmla_zpzzz_s(vd, vn, vm, va, vg, status, desc, 0x80000000, 0x80000000);
}

void HELPER(sve_fnmls_zpzzz_s)(CPUARMState *env, void *vg, uint32_t desc)
void HELPER(sve_fnmls_zpzzz_s)(void *vd, void *vn, void *vm, void *va,
void *vg, void *status, uint32_t desc)
{
do_fmla_zpzzz_s(env, vg, desc, 0, 0x80000000);
do_fmla_zpzzz_s(vd, vn, vm, va, vg, status, desc, 0, 0x80000000);
}

static void do_fmla_zpzzz_d(CPUARMState *env, void *vg, uint32_t desc,
static void do_fmla_zpzzz_d(void *vd, void *vn, void *vm, void *va, void *vg,
float_status *status, uint32_t desc,
uint64_t neg1, uint64_t neg3)
{
intptr_t i = simd_oprsz(desc);
unsigned rd = extract32(desc, SIMD_DATA_SHIFT, 5);
unsigned rn = extract32(desc, SIMD_DATA_SHIFT + 5, 5);
unsigned rm = extract32(desc, SIMD_DATA_SHIFT + 10, 5);
unsigned ra = extract32(desc, SIMD_DATA_SHIFT + 15, 5);
void *vd = &env->vfp.zregs[rd];
void *vn = &env->vfp.zregs[rn];
void *vm = &env->vfp.zregs[rm];
void *va = &env->vfp.zregs[ra];
uint64_t *g = vg;

do {
Expand All @@ -3503,31 +3485,35 @@ static void do_fmla_zpzzz_d(CPUARMState *env, void *vg, uint32_t desc,
e1 = *(uint64_t *)(vn + i) ^ neg1;
e2 = *(uint64_t *)(vm + i);
e3 = *(uint64_t *)(va + i) ^ neg3;
r = float64_muladd(e1, e2, e3, 0, &env->vfp.fp_status);
r = float64_muladd(e1, e2, e3, 0, status);
*(uint64_t *)(vd + i) = r;
}
} while (i & 63);
} while (i != 0);
}

void HELPER(sve_fmla_zpzzz_d)(CPUARMState *env, void *vg, uint32_t desc)
void HELPER(sve_fmla_zpzzz_d)(void *vd, void *vn, void *vm, void *va,
void *vg, void *status, uint32_t desc)
{
do_fmla_zpzzz_d(env, vg, desc, 0, 0);
do_fmla_zpzzz_d(vd, vn, vm, va, vg, status, desc, 0, 0);
}

void HELPER(sve_fmls_zpzzz_d)(CPUARMState *env, void *vg, uint32_t desc)
void HELPER(sve_fmls_zpzzz_d)(void *vd, void *vn, void *vm, void *va,
void *vg, void *status, uint32_t desc)
{
do_fmla_zpzzz_d(env, vg, desc, INT64_MIN, 0);
do_fmla_zpzzz_d(vd, vn, vm, va, vg, status, desc, INT64_MIN, 0);
}

void HELPER(sve_fnmla_zpzzz_d)(CPUARMState *env, void *vg, uint32_t desc)
void HELPER(sve_fnmla_zpzzz_d)(void *vd, void *vn, void *vm, void *va,
void *vg, void *status, uint32_t desc)
{
do_fmla_zpzzz_d(env, vg, desc, INT64_MIN, INT64_MIN);
do_fmla_zpzzz_d(vd, vn, vm, va, vg, status, desc, INT64_MIN, INT64_MIN);
}

void HELPER(sve_fnmls_zpzzz_d)(CPUARMState *env, void *vg, uint32_t desc)
void HELPER(sve_fnmls_zpzzz_d)(void *vd, void *vn, void *vm, void *va,
void *vg, void *status, uint32_t desc)
{
do_fmla_zpzzz_d(env, vg, desc, 0, INT64_MIN);
do_fmla_zpzzz_d(vd, vn, vm, va, vg, status, desc, 0, INT64_MIN);
}

/* Two operand floating-point comparison controlled by a predicate.
Expand Down Expand Up @@ -3809,22 +3795,13 @@ void HELPER(sve_fcadd_d)(void *vd, void *vn, void *vm, void *vg,
* FP Complex Multiply
*/

QEMU_BUILD_BUG_ON(SIMD_DATA_SHIFT + 22 > 32);

void HELPER(sve_fcmla_zpzzz_h)(CPUARMState *env, void *vg, uint32_t desc)
void HELPER(sve_fcmla_zpzzz_h)(void *vd, void *vn, void *vm, void *va,
void *vg, void *status, uint32_t desc)
{
intptr_t j, i = simd_oprsz(desc);
unsigned rd = extract32(desc, SIMD_DATA_SHIFT, 5);
unsigned rn = extract32(desc, SIMD_DATA_SHIFT + 5, 5);
unsigned rm = extract32(desc, SIMD_DATA_SHIFT + 10, 5);
unsigned ra = extract32(desc, SIMD_DATA_SHIFT + 15, 5);
unsigned rot = extract32(desc, SIMD_DATA_SHIFT + 20, 2);
unsigned rot = simd_data(desc);
bool flip = rot & 1;
float16 neg_imag, neg_real;
void *vd = &env->vfp.zregs[rd];
void *vn = &env->vfp.zregs[rn];
void *vm = &env->vfp.zregs[rm];
void *va = &env->vfp.zregs[ra];
uint64_t *g = vg;

neg_imag = float16_set_sign(0, (rot & 2) != 0);
Expand All @@ -3851,32 +3828,25 @@ void HELPER(sve_fcmla_zpzzz_h)(CPUARMState *env, void *vg, uint32_t desc)

if (likely((pg >> (i & 63)) & 1)) {
d = *(float16 *)(va + H1_2(i));
d = float16_muladd(e2, e1, d, 0, &env->vfp.fp_status_f16);
d = float16_muladd(e2, e1, d, 0, status);
*(float16 *)(vd + H1_2(i)) = d;
}
if (likely((pg >> (j & 63)) & 1)) {
d = *(float16 *)(va + H1_2(j));
d = float16_muladd(e4, e3, d, 0, &env->vfp.fp_status_f16);
d = float16_muladd(e4, e3, d, 0, status);
*(float16 *)(vd + H1_2(j)) = d;
}
} while (i & 63);
} while (i != 0);
}

void HELPER(sve_fcmla_zpzzz_s)(CPUARMState *env, void *vg, uint32_t desc)
void HELPER(sve_fcmla_zpzzz_s)(void *vd, void *vn, void *vm, void *va,
void *vg, void *status, uint32_t desc)
{
intptr_t j, i = simd_oprsz(desc);
unsigned rd = extract32(desc, SIMD_DATA_SHIFT, 5);
unsigned rn = extract32(desc, SIMD_DATA_SHIFT + 5, 5);
unsigned rm = extract32(desc, SIMD_DATA_SHIFT + 10, 5);
unsigned ra = extract32(desc, SIMD_DATA_SHIFT + 15, 5);
unsigned rot = extract32(desc, SIMD_DATA_SHIFT + 20, 2);
unsigned rot = simd_data(desc);
bool flip = rot & 1;
float32 neg_imag, neg_real;
void *vd = &env->vfp.zregs[rd];
void *vn = &env->vfp.zregs[rn];
void *vm = &env->vfp.zregs[rm];
void *va = &env->vfp.zregs[ra];
uint64_t *g = vg;

neg_imag = float32_set_sign(0, (rot & 2) != 0);
Expand All @@ -3903,32 +3873,25 @@ void HELPER(sve_fcmla_zpzzz_s)(CPUARMState *env, void *vg, uint32_t desc)

if (likely((pg >> (i & 63)) & 1)) {
d = *(float32 *)(va + H1_2(i));
d = float32_muladd(e2, e1, d, 0, &env->vfp.fp_status);
d = float32_muladd(e2, e1, d, 0, status);
*(float32 *)(vd + H1_2(i)) = d;
}
if (likely((pg >> (j & 63)) & 1)) {
d = *(float32 *)(va + H1_2(j));
d = float32_muladd(e4, e3, d, 0, &env->vfp.fp_status);
d = float32_muladd(e4, e3, d, 0, status);
*(float32 *)(vd + H1_2(j)) = d;
}
} while (i & 63);
} while (i != 0);
}

void HELPER(sve_fcmla_zpzzz_d)(CPUARMState *env, void *vg, uint32_t desc)
void HELPER(sve_fcmla_zpzzz_d)(void *vd, void *vn, void *vm, void *va,
void *vg, void *status, uint32_t desc)
{
intptr_t j, i = simd_oprsz(desc);
unsigned rd = extract32(desc, SIMD_DATA_SHIFT, 5);
unsigned rn = extract32(desc, SIMD_DATA_SHIFT + 5, 5);
unsigned rm = extract32(desc, SIMD_DATA_SHIFT + 10, 5);
unsigned ra = extract32(desc, SIMD_DATA_SHIFT + 15, 5);
unsigned rot = extract32(desc, SIMD_DATA_SHIFT + 20, 2);
unsigned rot = simd_data(desc);
bool flip = rot & 1;
float64 neg_imag, neg_real;
void *vd = &env->vfp.zregs[rd];
void *vn = &env->vfp.zregs[rn];
void *vm = &env->vfp.zregs[rm];
void *va = &env->vfp.zregs[ra];
uint64_t *g = vg;

neg_imag = float64_set_sign(0, (rot & 2) != 0);
Expand All @@ -3955,12 +3918,12 @@ void HELPER(sve_fcmla_zpzzz_d)(CPUARMState *env, void *vg, uint32_t desc)

if (likely((pg >> (i & 63)) & 1)) {
d = *(float64 *)(va + H1_2(i));
d = float64_muladd(e2, e1, d, 0, &env->vfp.fp_status);
d = float64_muladd(e2, e1, d, 0, status);
*(float64 *)(vd + H1_2(i)) = d;
}
if (likely((pg >> (j & 63)) & 1)) {
d = *(float64 *)(va + H1_2(j));
d = float64_muladd(e4, e3, d, 0, &env->vfp.fp_status);
d = float64_muladd(e4, e3, d, 0, status);
*(float64 *)(vd + H1_2(j)) = d;
}
} while (i & 63);
Expand Down

0 comments on commit 08975da

Please sign in to comment.