Skip to content

Commit

Permalink
Provide semi-optimized _mm_aesenc_si128 for ARMv7-A
Browse files Browse the repository at this point in the history 
For the sake of orthogonality, every supported intrinsic shall be
implemented for both ARMv7-A+NEON and ARMv8-A with diverse extensions.
This patch provides a semi-optimized _mm_aesenc_si128 for ARMv7-A with
NEON. There is room for performance improvements. See
  "Higher-Order Masking in Practice: A Vector Implementation of Masked AES
   for ARM NEON"
  https://link.springer.com/chapter/10.1007/978-3-319-16715-2_10

And corresponding source code:
  https://github.com/Ko-/aes-masked-neon
  • Loading branch information
@jserv
jserv committed Jul 15, 2020
1 parent bb5e2b1 commit 3e384f7
Show file tree
Hide file tree
Showing 3 changed files with 102 additions and 36 deletions.
130 changes: 102 additions & 28 deletions sse2neon.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3963,7 +3963,50 @@ FORCE_INLINE __m128i _mm_clmulepi64_si128(__m128i _a, __m128i _b, const int imm)
}
}

#if !defined(__ARM_FEATURE_CRYPTO) && defined(__aarch64__)
#if !defined(__ARM_FEATURE_CRYPTO)
/* clang-format off */
#define SSE2NEON_AES_DATA(w) \
{ \
w(0x63), w(0x7c), w(0x77), w(0x7b), w(0xf2), w(0x6b), w(0x6f), \
w(0xc5), w(0x30), w(0x01), w(0x67), w(0x2b), w(0xfe), w(0xd7), \
w(0xab), w(0x76), w(0xca), w(0x82), w(0xc9), w(0x7d), w(0xfa), \
w(0x59), w(0x47), w(0xf0), w(0xad), w(0xd4), w(0xa2), w(0xaf), \
w(0x9c), w(0xa4), w(0x72), w(0xc0), w(0xb7), w(0xfd), w(0x93), \
w(0x26), w(0x36), w(0x3f), w(0xf7), w(0xcc), w(0x34), w(0xa5), \
w(0xe5), w(0xf1), w(0x71), w(0xd8), w(0x31), w(0x15), w(0x04), \
w(0xc7), w(0x23), w(0xc3), w(0x18), w(0x96), w(0x05), w(0x9a), \
w(0x07), w(0x12), w(0x80), w(0xe2), w(0xeb), w(0x27), w(0xb2), \
w(0x75), w(0x09), w(0x83), w(0x2c), w(0x1a), w(0x1b), w(0x6e), \
w(0x5a), w(0xa0), w(0x52), w(0x3b), w(0xd6), w(0xb3), w(0x29), \
w(0xe3), w(0x2f), w(0x84), w(0x53), w(0xd1), w(0x00), w(0xed), \
w(0x20), w(0xfc), w(0xb1), w(0x5b), w(0x6a), w(0xcb), w(0xbe), \
w(0x39), w(0x4a), w(0x4c), w(0x58), w(0xcf), w(0xd0), w(0xef), \
w(0xaa), w(0xfb), w(0x43), w(0x4d), w(0x33), w(0x85), w(0x45), \
w(0xf9), w(0x02), w(0x7f), w(0x50), w(0x3c), w(0x9f), w(0xa8), \
w(0x51), w(0xa3), w(0x40), w(0x8f), w(0x92), w(0x9d), w(0x38), \
w(0xf5), w(0xbc), w(0xb6), w(0xda), w(0x21), w(0x10), w(0xff), \
w(0xf3), w(0xd2), w(0xcd), w(0x0c), w(0x13), w(0xec), w(0x5f), \
w(0x97), w(0x44), w(0x17), w(0xc4), w(0xa7), w(0x7e), w(0x3d), \
w(0x64), w(0x5d), w(0x19), w(0x73), w(0x60), w(0x81), w(0x4f), \
w(0xdc), w(0x22), w(0x2a), w(0x90), w(0x88), w(0x46), w(0xee), \
w(0xb8), w(0x14), w(0xde), w(0x5e), w(0x0b), w(0xdb), w(0xe0), \
w(0x32), w(0x3a), w(0x0a), w(0x49), w(0x06), w(0x24), w(0x5c), \
w(0xc2), w(0xd3), w(0xac), w(0x62), w(0x91), w(0x95), w(0xe4), \
w(0x79), w(0xe7), w(0xc8), w(0x37), w(0x6d), w(0x8d), w(0xd5), \
w(0x4e), w(0xa9), w(0x6c), w(0x56), w(0xf4), w(0xea), w(0x65), \
w(0x7a), w(0xae), w(0x08), w(0xba), w(0x78), w(0x25), w(0x2e), \
w(0x1c), w(0xa6), w(0xb4), w(0xc6), w(0xe8), w(0xdd), w(0x74), \
w(0x1f), w(0x4b), w(0xbd), w(0x8b), w(0x8a), w(0x70), w(0x3e), \
w(0xb5), w(0x66), w(0x48), w(0x03), w(0xf6), w(0x0e), w(0x61), \
w(0x35), w(0x57), w(0xb9), w(0x86), w(0xc1), w(0x1d), w(0x9e), \
w(0xe1), w(0xf8), w(0x98), w(0x11), w(0x69), w(0xd9), w(0x8e), \
w(0x94), w(0x9b), w(0x1e), w(0x87), w(0xe9), w(0xce), w(0x55), \
w(0x28), w(0xdf), w(0x8c), w(0xa1), w(0x89), w(0x0d), w(0xbf), \
w(0xe6), w(0x42), w(0x68), w(0x41), w(0x99), w(0x2d), w(0x0f), \
w(0xb0), w(0x54), w(0xbb), w(0x16) \
}
/* clang-format on */

// In the absence of crypto extensions, implement aesenc using regular neon
// intrinsics instead. See:
// https://www.workofard.com/2017/01/accelerated-aes-for-the-arm64-linux-kernel/
Expand All @@ -3972,29 +4015,12 @@ FORCE_INLINE __m128i _mm_clmulepi64_si128(__m128i _a, __m128i _b, const int imm)
// for more information Reproduced with permission of the author.
FORCE_INLINE __m128i _mm_aesenc_si128(__m128i EncBlock, __m128i RoundKey)
{
static const uint8_t crypto_aes_sbox[256] = {
0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b,
0xfe, 0xd7, 0xab, 0x76, 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0,
0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, 0xb7, 0xfd, 0x93, 0x26,
0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15,
0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2,
0xeb, 0x27, 0xb2, 0x75, 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0,
0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, 0x53, 0xd1, 0x00, 0xed,
0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf,
0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f,
0x50, 0x3c, 0x9f, 0xa8, 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5,
0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, 0xcd, 0x0c, 0x13, 0xec,
0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73,
0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14,
0xde, 0x5e, 0x0b, 0xdb, 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c,
0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, 0xe7, 0xc8, 0x37, 0x6d,
0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08,
0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f,
0x4b, 0xbd, 0x8b, 0x8a, 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e,
0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, 0xe1, 0xf8, 0x98, 0x11,
0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf,
0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f,
0xb0, 0x54, 0xbb, 0x16};
#if defined(__aarch64__)
/* X Macro trick. See https://en.wikipedia.org/wiki/X_Macro */
#define SSE2NEON_AES_H0(x) (x)
static const uint8_t crypto_aes_sbox[256] =
SSE2NEON_AES_DATA(SSE2NEON_AES_H0);
#undef SSE2NEON_AES_H0
static const uint8_t shift_rows[] = {0x0, 0x5, 0xa, 0xf, 0x4, 0x9,
0xe, 0x3, 0x8, 0xd, 0x2, 0x7,
0xc, 0x1, 0x6, 0xb};
Expand All @@ -4020,15 +4046,63 @@ FORCE_INLINE __m128i _mm_aesenc_si128(__m128i EncBlock, __m128i RoundKey)

// add round key
return vreinterpretq_m128i_u8(w) ^ RoundKey;

#else /* ARMv7-A NEON implementation */
#define SSE2NEON_AES_B2W(b0, b1, b2, b3) \
(((uint32_t)(b3) << 24) | ((uint32_t)(b2) << 16) | ((uint32_t)(b1) << 8) | \
(b0))
#define SSE2NEON_AES_F2(x) ((x << 1) ^ (((x >> 7) & 1) * 0x011b /* WPOLY */))
#define SSE2NEON_AES_F3(x) (SSE2NEON_AES_F2(x) ^ x)
#define SSE2NEON_AES_U0(p) \
SSE2NEON_AES_B2W(SSE2NEON_AES_F2(p), p, p, SSE2NEON_AES_F3(p))
#define SSE2NEON_AES_U1(p) \
SSE2NEON_AES_B2W(SSE2NEON_AES_F3(p), SSE2NEON_AES_F2(p), p, p)
#define SSE2NEON_AES_U2(p) \
SSE2NEON_AES_B2W(p, SSE2NEON_AES_F3(p), SSE2NEON_AES_F2(p), p)
#define SSE2NEON_AES_U3(p) \
SSE2NEON_AES_B2W(p, p, SSE2NEON_AES_F3(p), SSE2NEON_AES_F2(p))
static const uint32_t ALIGN_STRUCT(16) aes_table[4][256] = {
SSE2NEON_AES_DATA(SSE2NEON_AES_U0),
SSE2NEON_AES_DATA(SSE2NEON_AES_U1),
SSE2NEON_AES_DATA(SSE2NEON_AES_U2),
SSE2NEON_AES_DATA(SSE2NEON_AES_U3),
};
#undef SSE2NEON_AES_B2W
#undef SSE2NEON_AES_F2
#undef SSE2NEON_AES_F3
#undef SSE2NEON_AES_U0
#undef SSE2NEON_AES_U1
#undef SSE2NEON_AES_U2
#undef SSE2NEON_AES_U3

uint32_t x0 = _mm_cvtsi128_si32(EncBlock);
uint32_t x1 = _mm_cvtsi128_si32(_mm_shuffle_epi32(EncBlock, 0x55));
uint32_t x2 = _mm_cvtsi128_si32(_mm_shuffle_epi32(EncBlock, 0xAA));
uint32_t x3 = _mm_cvtsi128_si32(_mm_shuffle_epi32(EncBlock, 0xFF));

__m128i out = _mm_set_epi32(
(aes_table[0][x3 & 0xff] ^ aes_table[1][(x0 >> 8) & 0xff] ^
aes_table[2][(x1 >> 16) & 0xff] ^ aes_table[3][x2 >> 24]),
(aes_table[0][x2 & 0xff] ^ aes_table[1][(x3 >> 8) & 0xff] ^
aes_table[2][(x0 >> 16) & 0xff] ^ aes_table[3][x1 >> 24]),
(aes_table[0][x1 & 0xff] ^ aes_table[1][(x2 >> 8) & 0xff] ^
aes_table[2][(x3 >> 16) & 0xff] ^ aes_table[3][x0 >> 24]),
(aes_table[0][x0 & 0xff] ^ aes_table[1][(x1 >> 8) & 0xff] ^
aes_table[2][(x2 >> 16) & 0xff] ^ aes_table[3][x3 >> 24]));

return _mm_xor_si128(out, RoundKey);
#endif
}
#elif defined(__ARM_FEATURE_CRYPTO)
#undef SSE2NEON_AES_DATA

#else /* __ARM_FEATURE_CRYPTO */
// Implements equivalent of 'aesenc' by combining AESE (with an empty key) and
// AESMC and then manually applying the real key as an xor operation This
// AESMC and then manually applying the real key as an xor operation. This
// unfortunately means an additional xor op; the compiler should be able to
// optimise this away for repeated calls however See
// optimize this away for repeated calls however. See
// https://blog.michaelbrase.com/2018/05/08/emulating-x86-aes-intrinsics-on-armv8-a
// for more details.
inline __m128i _mm_aesenc_si128(__m128i a, __m128i b)
FORCE_INLINE __m128i _mm_aesenc_si128(__m128i a, __m128i b)
{
return vreinterpretq_m128i_u8(
vaesmcq_u8(vaeseq_u8(vreinterpretq_u8_m128i(a), vdupq_n_u8(0))) ^
Expand Down
6 changes: 0 additions & 6 deletions tests/impl.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -541,11 +541,9 @@ const char *SSE2NEONTest::getInstructionTestString(InstructionTest test)
case IT_MM_POPCNT_U64:
ret = "MM_POPCNT_U64";
break;
#if !defined(__arm__) && __ARM_ARCH != 7
case IT_MM_AESENC_SI128:
ret = "IT_MM_AESENC_SI128";
break;
#endif
case IT_MM_CLMULEPI64_SI128:
ret = "IT_MM_CLMULEPI64_SI128";
break;
Expand Down Expand Up @@ -2841,7 +2839,6 @@ inline __m128i aesenc_128_reference(__m128i a, __m128i b)
return a;
}

#if !defined(__arm__) && __ARM_ARCH != 7
bool test_mm_aesenc_si128(const int32_t *a, const int32_t *b)
{
__m128i data = _mm_loadu_si128((const __m128i *) a);
Expand All @@ -2852,7 +2849,6 @@ bool test_mm_aesenc_si128(const int32_t *a, const int32_t *b)

return validate128(resultReference, resultIntrinsic);
}
#endif

bool test_mm_malloc(const size_t *a, const size_t *b)
{
Expand Down Expand Up @@ -3466,11 +3462,9 @@ class SSE2NEONTestImpl : public SSE2NEONTest
case IT_MM_POPCNT_U64:
ret = test_mm_popcnt_u64((const uint64_t *) mTestIntPointer1);
break;
#if !defined(__arm__) && __ARM_ARCH != 7
case IT_MM_AESENC_SI128:
ret = test_mm_aesenc_si128(mTestIntPointer1, mTestIntPointer2);
break;
#endif
case IT_MM_CLMULEPI64_SI128:
ret = test_mm_clmulepi64_si128((const uint64_t *) mTestIntPointer1,
(const uint64_t *) mTestIntPointer2);
Expand Down
2 changes: 0 additions & 2 deletions tests/impl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,9 +159,7 @@ enum InstructionTest {
IT_MM_AVG_EPU16,
IT_MM_POPCNT_U32,
IT_MM_POPCNT_U64,
#if !defined(__arm__) && __ARM_ARCH != 7
IT_MM_AESENC_SI128,
#endif
IT_MM_CLMULEPI64_SI128,
IT_MM_MALLOC,
IT_MM_CRC32_U8, // Unit test implemented
Expand Down

0 comments on commit 3e384f7

Please sign in to comment.