Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
host/include/aarch64: Implement aes-round.h 
Detect AES in cpuinfo; implement the accel hooks.

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
  • Loading branch information
@rth7680
rth7680 committed Jul 8, 2023
1 parent d6a2443 commit 8d97f28
Show file tree
Hide file tree
Showing 4 changed files with 217 additions and 0 deletions.
1 change: 1 addition & 0 deletions host/include/aarch64/host/cpuinfo.h
View file
Expand Up @@ -9,6 +9,7 @@
#define CPUINFO_ALWAYS (1u << 0) /* so cpuinfo is nonzero */
#define CPUINFO_LSE (1u << 1)
#define CPUINFO_LSE2 (1u << 2)
#define CPUINFO_AES (1u << 3)

/* Initialized with a constructor. */
extern unsigned cpuinfo;
Expand Down
205 changes: 205 additions & 0 deletions host/include/aarch64/host/crypto/aes-round.h
View file
@@ -0,0 +1,205 @@
/*
* AArch64 specific aes acceleration.
* SPDX-License-Identifier: GPL-2.0-or-later
*/

#ifndef AARCH64_HOST_CRYPTO_AES_ROUND_H
#define AARCH64_HOST_CRYPTO_AES_ROUND_H

#include "host/cpuinfo.h"
#include <arm_neon.h>

#ifdef __ARM_FEATURE_AES
# define HAVE_AES_ACCEL true
#else
# define HAVE_AES_ACCEL likely(cpuinfo & CPUINFO_AES)
#endif
#if !defined(__ARM_FEATURE_AES) && defined(CONFIG_ARM_AES_BUILTIN)
# define ATTR_AES_ACCEL __attribute__((target("+crypto")))
#else
# define ATTR_AES_ACCEL
#endif

static inline uint8x16_t aes_accel_bswap(uint8x16_t x)
{
return vqtbl1q_u8(x, (uint8x16_t){ 15, 14, 13, 12, 11, 10, 9, 8,
7, 6, 5, 4, 3, 2, 1, 0, });
}

#ifdef CONFIG_ARM_AES_BUILTIN
# define aes_accel_aesd vaesdq_u8
# define aes_accel_aese vaeseq_u8
# define aes_accel_aesmc vaesmcq_u8
# define aes_accel_aesimc vaesimcq_u8
# define aes_accel_aesd_imc(S, K) vaesimcq_u8(vaesdq_u8(S, K))
# define aes_accel_aese_mc(S, K) vaesmcq_u8(vaeseq_u8(S, K))
#else
static inline uint8x16_t aes_accel_aesd(uint8x16_t d, uint8x16_t k)
{
asm(".arch_extension aes\n\t"
"aesd %0.16b, %1.16b" : "+w"(d) : "w"(k));
return d;
}

static inline uint8x16_t aes_accel_aese(uint8x16_t d, uint8x16_t k)
{
asm(".arch_extension aes\n\t"
"aese %0.16b, %1.16b" : "+w"(d) : "w"(k));
return d;
}

static inline uint8x16_t aes_accel_aesmc(uint8x16_t d)
{
asm(".arch_extension aes\n\t"
"aesmc %0.16b, %1.16b" : "=w"(d) : "w"(d));
return d;
}

static inline uint8x16_t aes_accel_aesimc(uint8x16_t d)
{
asm(".arch_extension aes\n\t"
"aesimc %0.16b, %1.16b" : "=w"(d) : "w"(d));
return d;
}

/* Most CPUs fuse AESD+AESIMC in the execution pipeline. */
static inline uint8x16_t aes_accel_aesd_imc(uint8x16_t d, uint8x16_t k)
{
asm(".arch_extension aes\n\t"
"aesd %0.16b, %1.16b\n\t"
"aesimc %0.16b, %0.16b" : "+w"(d) : "w"(k));
return d;
}

/* Most CPUs fuse AESE+AESMC in the execution pipeline. */
static inline uint8x16_t aes_accel_aese_mc(uint8x16_t d, uint8x16_t k)
{
asm(".arch_extension aes\n\t"
"aese %0.16b, %1.16b\n\t"
"aesmc %0.16b, %0.16b" : "+w"(d) : "w"(k));
return d;
}
#endif /* CONFIG_ARM_AES_BUILTIN */

static inline void ATTR_AES_ACCEL
aesenc_MC_accel(AESState *ret, const AESState *st, bool be)
{
uint8x16_t t = (uint8x16_t)st->v;

if (be) {
t = aes_accel_bswap(t);
t = aes_accel_aesmc(t);
t = aes_accel_bswap(t);
} else {
t = aes_accel_aesmc(t);
}
ret->v = (AESStateVec)t;
}

static inline void ATTR_AES_ACCEL
aesenc_SB_SR_AK_accel(AESState *ret, const AESState *st,
const AESState *rk, bool be)
{
uint8x16_t t = (uint8x16_t)st->v;
uint8x16_t z = { };

if (be) {
t = aes_accel_bswap(t);
t = aes_accel_aese(t, z);
t = aes_accel_bswap(t);
} else {
t = aes_accel_aese(t, z);
}
ret->v = (AESStateVec)t ^ rk->v;
}

static inline void ATTR_AES_ACCEL
aesenc_SB_SR_MC_AK_accel(AESState *ret, const AESState *st,
const AESState *rk, bool be)
{
uint8x16_t t = (uint8x16_t)st->v;
uint8x16_t z = { };

if (be) {
t = aes_accel_bswap(t);
t = aes_accel_aese_mc(t, z);
t = aes_accel_bswap(t);
} else {
t = aes_accel_aese_mc(t, z);
}
ret->v = (AESStateVec)t ^ rk->v;
}

static inline void ATTR_AES_ACCEL
aesdec_IMC_accel(AESState *ret, const AESState *st, bool be)
{
uint8x16_t t = (uint8x16_t)st->v;

if (be) {
t = aes_accel_bswap(t);
t = aes_accel_aesimc(t);
t = aes_accel_bswap(t);
} else {
t = aes_accel_aesimc(t);
}
ret->v = (AESStateVec)t;
}

static inline void ATTR_AES_ACCEL
aesdec_ISB_ISR_AK_accel(AESState *ret, const AESState *st,
const AESState *rk, bool be)
{
uint8x16_t t = (uint8x16_t)st->v;
uint8x16_t z = { };

if (be) {
t = aes_accel_bswap(t);
t = aes_accel_aesd(t, z);
t = aes_accel_bswap(t);
} else {
t = aes_accel_aesd(t, z);
}
ret->v = (AESStateVec)t ^ rk->v;
}

static inline void ATTR_AES_ACCEL
aesdec_ISB_ISR_AK_IMC_accel(AESState *ret, const AESState *st,
const AESState *rk, bool be)
{
uint8x16_t t = (uint8x16_t)st->v;
uint8x16_t k = (uint8x16_t)rk->v;
uint8x16_t z = { };

if (be) {
t = aes_accel_bswap(t);
k = aes_accel_bswap(k);
t = aes_accel_aesd(t, z);
t ^= k;
t = aes_accel_aesimc(t);
t = aes_accel_bswap(t);
} else {
t = aes_accel_aesd(t, z);
t ^= k;
t = aes_accel_aesimc(t);
}
ret->v = (AESStateVec)t;
}

static inline void ATTR_AES_ACCEL
aesdec_ISB_ISR_IMC_AK_accel(AESState *ret, const AESState *st,
const AESState *rk, bool be)
{
uint8x16_t t = (uint8x16_t)st->v;
uint8x16_t z = { };

if (be) {
t = aes_accel_bswap(t);
t = aes_accel_aesd_imc(t, z);
t = aes_accel_bswap(t);
} else {
t = aes_accel_aesd_imc(t, z);
}
ret->v = (AESStateVec)t ^ rk->v;
}

#endif /* AARCH64_HOST_CRYPTO_AES_ROUND_H */
9 changes: 9 additions & 0 deletions meson.build
View file
Expand Up @@ -2665,6 +2665,15 @@ config_host_data.set('CONFIG_AVX512BW_OPT', get_option('avx512bw') \
int main(int argc, char *argv[]) { return bar(argv[0]); }
'''), error_message: 'AVX512BW not available').allowed())

# For both AArch64 and AArch32, detect if builtins are available.
config_host_data.set('CONFIG_ARM_AES_BUILTIN', cc.compiles('''
#include <arm_neon.h>
#ifndef __ARM_FEATURE_AES
__attribute__((target("+crypto")))
#endif
void foo(uint8x16_t *p) { *p = vaesmcq_u8(*p); }
'''))

have_pvrdma = get_option('pvrdma') \
.require(rdma.found(), error_message: 'PVRDMA requires OpenFabrics libraries') \
.require(cc.compiles(gnu_source_prefix + '''
Expand Down
2 changes: 2 additions & 0 deletions util/cpuinfo-aarch64.c
View file
Expand Up @@ -56,10 +56,12 @@ unsigned __attribute__((constructor)) cpuinfo_init(void)
unsigned long hwcap = qemu_getauxval(AT_HWCAP);
info |= (hwcap & HWCAP_ATOMICS ? CPUINFO_LSE : 0);
info |= (hwcap & HWCAP_USCAT ? CPUINFO_LSE2 : 0);
info |= (hwcap & HWCAP_AES ? CPUINFO_AES: 0);
#endif
#ifdef CONFIG_DARWIN
info |= sysctl_for_bool("hw.optional.arm.FEAT_LSE") * CPUINFO_LSE;
info |= sysctl_for_bool("hw.optional.arm.FEAT_LSE2") * CPUINFO_LSE2;
info |= sysctl_for_bool("hw.optional.arm.FEAT_AES") * CPUINFO_AES;
#endif

cpuinfo = info;
Expand Down

0 comments on commit 8d97f28

Please sign in to comment.