Skip to content

Commit

Permalink
target/arm: Fix 32-bit SMOPA
Browse files Browse the repository at this point in the history
While the 8-bit input elements are sequential in the input vector,
the 32-bit output elements are not sequential in the output matrix.
Do not attempt to compute 2 32-bit outputs at the same time.

Cc: qemu-stable@nongnu.org
Fixes: 23a5e38 ("target/arm: Implement SME integer outer product")
Resolves: https://gitlab.com/qemu-project/qemu/-/issues/2083
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Message-id: 20240305163931.242795-1-richard.henderson@linaro.org
Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
(cherry picked from commit d572bcb)
Signed-off-by: Michael Tokarev <mjt@tls.msk.ru>
  • Loading branch information
rth7680 authored and Michael Tokarev committed Mar 9, 2024
1 parent dc65c42 commit 43b8258
Show file tree
Hide file tree
Showing 4 changed files with 148 additions and 34 deletions.
79 changes: 46 additions & 33 deletions target/arm/sme_helper.c
Original file line number Diff line number Diff line change
Expand Up @@ -1116,11 +1116,32 @@ void HELPER(sme_bfmopa)(void *vza, void *vzn, void *vzm, void *vpn,
}
}

typedef uint64_t IMOPFn(uint64_t, uint64_t, uint64_t, uint8_t, bool);
typedef uint32_t IMOPFn32(uint32_t, uint32_t, uint32_t, uint8_t, bool);
static inline void do_imopa_s(uint32_t *za, uint32_t *zn, uint32_t *zm,
uint8_t *pn, uint8_t *pm,
uint32_t desc, IMOPFn32 *fn)
{
intptr_t row, col, oprsz = simd_oprsz(desc) / 4;
bool neg = simd_data(desc);

for (row = 0; row < oprsz; ++row) {
uint8_t pa = (pn[H1(row >> 1)] >> ((row & 1) * 4)) & 0xf;
uint32_t *za_row = &za[tile_vslice_index(row)];
uint32_t n = zn[H4(row)];

static inline void do_imopa(uint64_t *za, uint64_t *zn, uint64_t *zm,
uint8_t *pn, uint8_t *pm,
uint32_t desc, IMOPFn *fn)
for (col = 0; col < oprsz; ++col) {
uint8_t pb = pm[H1(col >> 1)] >> ((col & 1) * 4);
uint32_t *a = &za_row[H4(col)];

*a = fn(n, zm[H4(col)], *a, pa & pb, neg);
}
}
}

typedef uint64_t IMOPFn64(uint64_t, uint64_t, uint64_t, uint8_t, bool);
static inline void do_imopa_d(uint64_t *za, uint64_t *zn, uint64_t *zm,
uint8_t *pn, uint8_t *pm,
uint32_t desc, IMOPFn64 *fn)
{
intptr_t row, col, oprsz = simd_oprsz(desc) / 8;
bool neg = simd_data(desc);
Expand All @@ -1140,25 +1161,16 @@ static inline void do_imopa(uint64_t *za, uint64_t *zn, uint64_t *zm,
}

#define DEF_IMOP_32(NAME, NTYPE, MTYPE) \
static uint64_t NAME(uint64_t n, uint64_t m, uint64_t a, uint8_t p, bool neg) \
static uint32_t NAME(uint32_t n, uint32_t m, uint32_t a, uint8_t p, bool neg) \
{ \
uint32_t sum0 = 0, sum1 = 0; \
uint32_t sum = 0; \
/* Apply P to N as a mask, making the inactive elements 0. */ \
n &= expand_pred_b(p); \
sum0 += (NTYPE)(n >> 0) * (MTYPE)(m >> 0); \
sum0 += (NTYPE)(n >> 8) * (MTYPE)(m >> 8); \
sum0 += (NTYPE)(n >> 16) * (MTYPE)(m >> 16); \
sum0 += (NTYPE)(n >> 24) * (MTYPE)(m >> 24); \
sum1 += (NTYPE)(n >> 32) * (MTYPE)(m >> 32); \
sum1 += (NTYPE)(n >> 40) * (MTYPE)(m >> 40); \
sum1 += (NTYPE)(n >> 48) * (MTYPE)(m >> 48); \
sum1 += (NTYPE)(n >> 56) * (MTYPE)(m >> 56); \
if (neg) { \
sum0 = (uint32_t)a - sum0, sum1 = (uint32_t)(a >> 32) - sum1; \
} else { \
sum0 = (uint32_t)a + sum0, sum1 = (uint32_t)(a >> 32) + sum1; \
} \
return ((uint64_t)sum1 << 32) | sum0; \
sum += (NTYPE)(n >> 0) * (MTYPE)(m >> 0); \
sum += (NTYPE)(n >> 8) * (MTYPE)(m >> 8); \
sum += (NTYPE)(n >> 16) * (MTYPE)(m >> 16); \
sum += (NTYPE)(n >> 24) * (MTYPE)(m >> 24); \
return neg ? a - sum : a + sum; \
}

#define DEF_IMOP_64(NAME, NTYPE, MTYPE) \
Expand All @@ -1184,16 +1196,17 @@ DEF_IMOP_64(umopa_d, uint16_t, uint16_t)
DEF_IMOP_64(sumopa_d, int16_t, uint16_t)
DEF_IMOP_64(usmopa_d, uint16_t, int16_t)

#define DEF_IMOPH(NAME) \
void HELPER(sme_##NAME)(void *vza, void *vzn, void *vzm, void *vpn, \
void *vpm, uint32_t desc) \
{ do_imopa(vza, vzn, vzm, vpn, vpm, desc, NAME); }

DEF_IMOPH(smopa_s)
DEF_IMOPH(umopa_s)
DEF_IMOPH(sumopa_s)
DEF_IMOPH(usmopa_s)
DEF_IMOPH(smopa_d)
DEF_IMOPH(umopa_d)
DEF_IMOPH(sumopa_d)
DEF_IMOPH(usmopa_d)
#define DEF_IMOPH(NAME, S) \
void HELPER(sme_##NAME##_##S)(void *vza, void *vzn, void *vzm, \
void *vpn, void *vpm, uint32_t desc) \
{ do_imopa_##S(vza, vzn, vzm, vpn, vpm, desc, NAME##_##S); }

DEF_IMOPH(smopa, s)
DEF_IMOPH(umopa, s)
DEF_IMOPH(sumopa, s)
DEF_IMOPH(usmopa, s)

DEF_IMOPH(smopa, d)
DEF_IMOPH(umopa, d)
DEF_IMOPH(sumopa, d)
DEF_IMOPH(usmopa, d)
2 changes: 1 addition & 1 deletion tests/tcg/aarch64/Makefile.target
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ endif

# SME Tests
ifneq ($(CROSS_AS_HAS_ARMV9_SME),)
AARCH64_TESTS += sme-outprod1
AARCH64_TESTS += sme-outprod1 sme-smopa-1 sme-smopa-2
endif

# System Registers Tests
Expand Down
47 changes: 47 additions & 0 deletions tests/tcg/aarch64/sme-smopa-1.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
#include <stdio.h>
#include <string.h>

int main()
{
static const int cmp[4][4] = {
{ 110, 134, 158, 182 },
{ 390, 478, 566, 654 },
{ 670, 822, 974, 1126 },
{ 950, 1166, 1382, 1598 }
};
int dst[4][4];
int *tmp = &dst[0][0];

asm volatile(
".arch armv8-r+sme\n\t"
"smstart\n\t"
"index z0.b, #0, #1\n\t"
"movprfx z1, z0\n\t"
"add z1.b, z1.b, #16\n\t"
"ptrue p0.b\n\t"
"smopa za0.s, p0/m, p0/m, z0.b, z1.b\n\t"
"ptrue p0.s, vl4\n\t"
"mov w12, #0\n\t"
"st1w { za0h.s[w12, #0] }, p0, [%0]\n\t"
"add %0, %0, #16\n\t"
"st1w { za0h.s[w12, #1] }, p0, [%0]\n\t"
"add %0, %0, #16\n\t"
"st1w { za0h.s[w12, #2] }, p0, [%0]\n\t"
"add %0, %0, #16\n\t"
"st1w { za0h.s[w12, #3] }, p0, [%0]\n\t"
"smstop"
: "+r"(tmp) : : "memory");

if (memcmp(cmp, dst, sizeof(dst)) == 0) {
return 0;
}

/* See above for correct results. */
for (int i = 0; i < 4; ++i) {
for (int j = 0; j < 4; ++j) {
printf("%6d", dst[i][j]);
}
printf("\n");
}
return 1;
}
54 changes: 54 additions & 0 deletions tests/tcg/aarch64/sme-smopa-2.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
#include <stdio.h>
#include <string.h>

int main()
{
static const long cmp[4][4] = {
{ 110, 134, 158, 182 },
{ 390, 478, 566, 654 },
{ 670, 822, 974, 1126 },
{ 950, 1166, 1382, 1598 }
};
long dst[4][4];
long *tmp = &dst[0][0];
long svl;

/* Validate that we have a wide enough vector for 4 elements. */
asm(".arch armv8-r+sme-i64\n\trdsvl %0, #1" : "=r"(svl));
if (svl < 32) {
return 0;
}

asm volatile(
"smstart\n\t"
"index z0.h, #0, #1\n\t"
"movprfx z1, z0\n\t"
"add z1.h, z1.h, #16\n\t"
"ptrue p0.b\n\t"
"smopa za0.d, p0/m, p0/m, z0.h, z1.h\n\t"
"ptrue p0.d, vl4\n\t"
"mov w12, #0\n\t"
"st1d { za0h.d[w12, #0] }, p0, [%0]\n\t"
"add %0, %0, #32\n\t"
"st1d { za0h.d[w12, #1] }, p0, [%0]\n\t"
"mov w12, #2\n\t"
"add %0, %0, #32\n\t"
"st1d { za0h.d[w12, #0] }, p0, [%0]\n\t"
"add %0, %0, #32\n\t"
"st1d { za0h.d[w12, #1] }, p0, [%0]\n\t"
"smstop"
: "+r"(tmp) : : "memory");

if (memcmp(cmp, dst, sizeof(dst)) == 0) {
return 0;
}

/* See above for correct results. */
for (int i = 0; i < 4; ++i) {
for (int j = 0; j < 4; ++j) {
printf("%6ld", dst[i][j]);
}
printf("\n");
}
return 1;
}

0 comments on commit 43b8258

Please sign in to comment.