Skip to content

Commit

Permalink
target/arm: Implement 128-bit ZIP, UZP, TRN
Browse files Browse the repository at this point in the history 
Reviewed-by: Peter Maydell <peter.maydell@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
Message-id: 20210525010358.152808-80-richard.henderson@linaro.org
Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
  • Loading branch information
@rth7680 @pm215
rth7680 authored and pm215 committed May 25, 2021
1 parent 12c563f commit 74b64b2
Show file tree
Hide file tree
Showing 4 changed files with 90 additions and 8 deletions.
3 changes: 3 additions & 0 deletions target/arm/helper-sve.h
View file
Expand Up @@ -689,16 +689,19 @@ DEF_HELPER_FLAGS_4(sve_zip_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_zip_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_zip_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_zip_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_zip_q, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)

DEF_HELPER_FLAGS_4(sve_uzp_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_uzp_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_uzp_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_uzp_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_uzp_q, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)

DEF_HELPER_FLAGS_4(sve_trn_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_trn_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_trn_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_trn_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_trn_q, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)

DEF_HELPER_FLAGS_4(sve_compact_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_compact_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
Expand Down
8 changes: 8 additions & 0 deletions target/arm/sve.decode
View file
Expand Up @@ -590,6 +590,14 @@ UZP2_z 00000101 .. 1 ..... 011 011 ..... ..... @rd_rn_rm
TRN1_z 00000101 .. 1 ..... 011 100 ..... ..... @rd_rn_rm
TRN2_z 00000101 .. 1 ..... 011 101 ..... ..... @rd_rn_rm

# SVE2 permute vector segments
ZIP1_q 00000101 10 1 ..... 000 000 ..... ..... @rd_rn_rm_e0
ZIP2_q 00000101 10 1 ..... 000 001 ..... ..... @rd_rn_rm_e0
UZP1_q 00000101 10 1 ..... 000 010 ..... ..... @rd_rn_rm_e0
UZP2_q 00000101 10 1 ..... 000 011 ..... ..... @rd_rn_rm_e0
TRN1_q 00000101 10 1 ..... 000 110 ..... ..... @rd_rn_rm_e0
TRN2_q 00000101 10 1 ..... 000 111 ..... ..... @rd_rn_rm_e0

### SVE Permute - Predicated Group

# SVE compress active elements
Expand Down
29 changes: 21 additions & 8 deletions target/arm/sve_helper.c
View file
Expand Up @@ -3492,36 +3492,45 @@ void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
*(TYPE *)(vd + H(2 * i + 0)) = *(TYPE *)(vn + H(i)); \
*(TYPE *)(vd + H(2 * i + sizeof(TYPE))) = *(TYPE *)(vm + H(i)); \
} \
if (sizeof(TYPE) == 16 && unlikely(oprsz & 16)) { \
memset(vd + oprsz - 16, 0, 16); \
} \
}

DO_ZIP(sve_zip_b, uint8_t, H1)
DO_ZIP(sve_zip_h, uint16_t, H1_2)
DO_ZIP(sve_zip_s, uint32_t, H1_4)
DO_ZIP(sve_zip_d, uint64_t, )
DO_ZIP(sve2_zip_q, Int128, )

#define DO_UZP(NAME, TYPE, H) \
void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
{ \
intptr_t oprsz = simd_oprsz(desc); \
intptr_t oprsz_2 = oprsz / 2; \
intptr_t odd_ofs = simd_data(desc); \
intptr_t i; \
intptr_t i, p; \
ARMVectorReg tmp_m; \
if (unlikely((vm - vd) < (uintptr_t)oprsz)) { \
vm = memcpy(&tmp_m, vm, oprsz); \
} \
for (i = 0; i < oprsz_2; i += sizeof(TYPE)) { \
*(TYPE *)(vd + H(i)) = *(TYPE *)(vn + H(2 * i + odd_ofs)); \
} \
for (i = 0; i < oprsz_2; i += sizeof(TYPE)) { \
*(TYPE *)(vd + H(oprsz_2 + i)) = *(TYPE *)(vm + H(2 * i + odd_ofs)); \
} \
i = 0, p = odd_ofs; \
do { \
*(TYPE *)(vd + H(i)) = *(TYPE *)(vn + H(p)); \
i += sizeof(TYPE), p += 2 * sizeof(TYPE); \
} while (p < oprsz); \
p -= oprsz; \
do { \
*(TYPE *)(vd + H(i)) = *(TYPE *)(vm + H(p)); \
i += sizeof(TYPE), p += 2 * sizeof(TYPE); \
} while (p < oprsz); \
tcg_debug_assert(i == oprsz); \
}

DO_UZP(sve_uzp_b, uint8_t, H1)
DO_UZP(sve_uzp_h, uint16_t, H1_2)
DO_UZP(sve_uzp_s, uint32_t, H1_4)
DO_UZP(sve_uzp_d, uint64_t, )
DO_UZP(sve2_uzp_q, Int128, )

#define DO_TRN(NAME, TYPE, H) \
void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
Expand All @@ -3535,12 +3544,16 @@ void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
*(TYPE *)(vd + H(i + 0)) = ae; \
*(TYPE *)(vd + H(i + sizeof(TYPE))) = be; \
} \
if (sizeof(TYPE) == 16 && unlikely(oprsz & 16)) { \
memset(vd + oprsz - 16, 0, 16); \
} \
}

DO_TRN(sve_trn_b, uint8_t, H1)
DO_TRN(sve_trn_h, uint16_t, H1_2)
DO_TRN(sve_trn_s, uint32_t, H1_4)
DO_TRN(sve_trn_d, uint64_t, )
DO_TRN(sve2_trn_q, Int128, )

#undef DO_ZIP
#undef DO_UZP
Expand Down
58 changes: 58 additions & 0 deletions target/arm/translate-sve.c
View file
Expand Up @@ -2624,6 +2624,32 @@ static bool trans_ZIP2_z(DisasContext *s, arg_rrr_esz *a)
return do_zip(s, a, true);
}

static bool do_zip_q(DisasContext *s, arg_rrr_esz *a, bool high)
{
if (!dc_isar_feature(aa64_sve_f64mm, s)) {
return false;
}
if (sve_access_check(s)) {
unsigned vsz = vec_full_reg_size(s);
unsigned high_ofs = high ? QEMU_ALIGN_DOWN(vsz, 32) / 2 : 0;
tcg_gen_gvec_3_ool(vec_full_reg_offset(s, a->rd),
vec_full_reg_offset(s, a->rn) + high_ofs,
vec_full_reg_offset(s, a->rm) + high_ofs,
vsz, vsz, 0, gen_helper_sve2_zip_q);
}
return true;
}

static bool trans_ZIP1_q(DisasContext *s, arg_rrr_esz *a)
{
return do_zip_q(s, a, false);
}

static bool trans_ZIP2_q(DisasContext *s, arg_rrr_esz *a)
{
return do_zip_q(s, a, true);
}

static gen_helper_gvec_3 * const uzp_fns[4] = {
gen_helper_sve_uzp_b, gen_helper_sve_uzp_h,
gen_helper_sve_uzp_s, gen_helper_sve_uzp_d,
Expand All @@ -2639,6 +2665,22 @@ static bool trans_UZP2_z(DisasContext *s, arg_rrr_esz *a)
return do_zzz_data_ool(s, a, 1 << a->esz, uzp_fns[a->esz]);
}

static bool trans_UZP1_q(DisasContext *s, arg_rrr_esz *a)
{
if (!dc_isar_feature(aa64_sve_f64mm, s)) {
return false;
}
return do_zzz_data_ool(s, a, 0, gen_helper_sve2_uzp_q);
}

static bool trans_UZP2_q(DisasContext *s, arg_rrr_esz *a)
{
if (!dc_isar_feature(aa64_sve_f64mm, s)) {
return false;
}
return do_zzz_data_ool(s, a, 16, gen_helper_sve2_uzp_q);
}

static gen_helper_gvec_3 * const trn_fns[4] = {
gen_helper_sve_trn_b, gen_helper_sve_trn_h,
gen_helper_sve_trn_s, gen_helper_sve_trn_d,
Expand All @@ -2654,6 +2696,22 @@ static bool trans_TRN2_z(DisasContext *s, arg_rrr_esz *a)
return do_zzz_data_ool(s, a, 1 << a->esz, trn_fns[a->esz]);
}

static bool trans_TRN1_q(DisasContext *s, arg_rrr_esz *a)
{
if (!dc_isar_feature(aa64_sve_f64mm, s)) {
return false;
}
return do_zzz_data_ool(s, a, 0, gen_helper_sve2_trn_q);
}

static bool trans_TRN2_q(DisasContext *s, arg_rrr_esz *a)
{
if (!dc_isar_feature(aa64_sve_f64mm, s)) {
return false;
}
return do_zzz_data_ool(s, a, 16, gen_helper_sve2_trn_q);
}

/*
*** SVE Permute Vector - Predicated Group
*/
Expand Down

0 comments on commit 74b64b2

Please sign in to comment.